diff --git a/.github/workflows/area-labels.yml b/.github/workflows/area-labels.yml index fb0dbe9b8..e64aaef4b 100644 --- a/.github/workflows/area-labels.yml +++ b/.github/workflows/area-labels.yml @@ -23,6 +23,8 @@ jobs: gh pr list -R "$GITHUB_REPOSITORY" --base "${{ github.event.repository.default_branch }}" -L 200 --json number --jq '.[].number' echo EOF } >> "$GITHUB_OUTPUT" + - if: steps.prs.outputs.numbers != '' + uses: actions/checkout@v7 - if: steps.prs.outputs.numbers != '' uses: actions/labeler@v7 with: diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index c067651f3..e3e9c238d 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -18,7 +18,7 @@ permissions: concurrency: group: build-${{ github.event_name }}-${{ github.event.pull_request.number || inputs.pr || github.ref }} - cancel-in-progress: true + cancel-in-progress: ${{ github.event_name != 'push' }} env: CONFIGURE: >- @@ -36,6 +36,8 @@ jobs: pull-requests: read outputs: sha: ${{ steps.gate.outputs.sha }} + pr: ${{ steps.gate.outputs.pr }} + head: ${{ steps.gate.outputs.head }} relinker: ${{ steps.gate.outputs.relinker }} full: ${{ steps.gate.outputs.full }} steps: @@ -50,6 +52,8 @@ jobs: throw new Error('Automatic full builds only run on main'); } core.setOutput('sha', context.sha); + core.setOutput('pr', ''); + core.setOutput('head', ''); core.setOutput('relinker', true); core.setOutput('full', true); await core.summary.addRaw(`Building main at ${context.sha}: all builds and tests enabled`).write(); @@ -83,7 +87,7 @@ jobs: if (!manual && pr.head.sha !== context.payload.pull_request.head.sha) { throw new Error('The pull request has newer commits; use the latest run'); } - if (pr.mergeable !== true || !pr.merge_commit_sha) { + if (pr.mergeable === false) { throw new Error('The pull request does not merge cleanly into main; rebase it first'); } let relinker = manual; @@ -93,11 +97,15 @@ jobs: }); relinker = commits.length > 0; } - core.setOutput('sha', pr.merge_commit_sha); + const { data: main } = await github.rest.repos.getBranch({ owner, repo, branch: 'main' }); + core.setOutput('sha', main.commit.sha); + core.setOutput('pr', String(pull_number)); + core.setOutput('head', pr.head.sha); core.setOutput('relinker', relinker); core.setOutput('full', manual); + core.info(`PR #${pull_number} at ${pr.head.sha} merged into main at ${main.commit.sha}`); await core.summary.addRaw( - `PR #${pull_number} at ${pr.head.sha} merged into main as ${pr.merge_commit_sha}: full builds ${manual ? 'enabled' : 'disabled'}, ` + + `PR #${pull_number} at ${pr.head.sha} merged into main at ${main.commit.sha}: full builds ${manual ? 'enabled' : 'disabled'}, ` + `relinker ${relinker ? 'enabled' : 'skipped because the author has no commit in main'}` ).write(); @@ -127,6 +135,16 @@ jobs: with: ref: ${{ needs.eligibility.outputs.sha }} persist-credentials: false + - name: Merge the pull request into main + if: ${{ needs.eligibility.outputs.head != '' }} + shell: bash + env: + PR_NUMBER: ${{ needs.eligibility.outputs.pr }} + PR_HEAD: ${{ needs.eligibility.outputs.head }} + run: | + git fetch --no-tags --unshallow origin "refs/pull/$PR_NUMBER/head" + if [ "$(git rev-parse FETCH_HEAD)" != "$PR_HEAD" ]; then echo "::error::The pull request has newer commits; start a new build"; exit 1; fi + if ! git -c user.name=ci -c user.email=ci@localhost merge --no-ff --no-edit FETCH_HEAD; then echo "::error::The pull request does not merge cleanly into main; rebase it first"; exit 1; fi - if: runner.os == 'Windows' id: toolchain uses: actions/cache@v6 @@ -143,6 +161,33 @@ jobs: - run: cmake --build build-relinker --parallel 3 - run: ctest --test-dir build-relinker --output-on-failure --timeout 120 + register-context: + strategy: + fail-fast: false + matrix: + os: [ubuntu-latest, macos-15-intel] + configuration: [Debug, Release] + runs-on: ${{ matrix.os }} + timeout-minutes: 5 + steps: + - uses: actions/checkout@v7 + - run: cmake -S core/libs/tests/register_context -B build/register-context -DCMAKE_BUILD_TYPE=${{ matrix.configuration }} + - run: cmake --build build/register-context --parallel + - run: ctest --test-dir build/register-context --output-on-failure --repeat until-fail:3 --no-tests=error + + write-watch: + strategy: + fail-fast: false + matrix: + os: [macos-15-intel, macos-15] + runs-on: ${{ matrix.os }} + timeout-minutes: 5 + steps: + - uses: actions/checkout@v7 + - run: cmake -S core/libs/tests/write_watch -B build/write-watch -DCMAKE_BUILD_TYPE=Release + - run: cmake --build build/write-watch --parallel + - run: ctest --test-dir build/write-watch --output-on-failure --repeat until-fail:3 --no-tests=error + linux: needs: eligibility if: ${{ (github.event_name == 'push' || github.event_name == 'workflow_dispatch') && needs.eligibility.outputs.full == 'true' }} @@ -154,10 +199,21 @@ jobs: ref: ${{ needs.eligibility.outputs.sha }} persist-credentials: false submodules: true + - name: Merge the pull request into main + if: ${{ needs.eligibility.outputs.head != '' }} + shell: bash + env: + PR_NUMBER: ${{ needs.eligibility.outputs.pr }} + PR_HEAD: ${{ needs.eligibility.outputs.head }} + run: | + git fetch --no-tags --unshallow origin "refs/pull/$PR_NUMBER/head" + if [ "$(git rev-parse FETCH_HEAD)" != "$PR_HEAD" ]; then echo "::error::The pull request has newer commits; start a new build"; exit 1; fi + if ! git -c user.name=ci -c user.email=ci@localhost merge --no-ff --no-edit FETCH_HEAD; then echo "::error::The pull request does not merge cleanly into main; rebase it first"; exit 1; fi + git submodule update --init --recursive - uses: hendrikmuhs/ccache-action@v1.2.24 with: key: linux - save: false + save: ${{ github.event_name == 'push' && github.ref == 'refs/heads/main' }} evict-old-files: job - name: Install lavapipe for the GPU execution tests run: sudo apt-get update && sudo apt-get install -y --no-install-recommends libvulkan1 mesa-vulkan-drivers @@ -166,10 +222,11 @@ jobs: run: python3 tools/check_test_registration.py build - run: cmake --build build - run: cmake --build build --target libs - - run: ctest --test-dir build --output-on-failure --timeout 120 --parallel 4 --output-log ctest.log + - run: ctest --test-dir build --output-on-failure --timeout 120 --parallel 4 --output-log ctest.log --output-junit ctest.xml env: ANYPS5_REQUIRE_VULKAN: 1 GALLIUM_OVERRIDE_CPU_CAPS: avx + - run: python3 tools/check_skips.py build/ctest.xml - name: Report tests slower than 30 s if: ${{ !cancelled() }} shell: bash @@ -196,6 +253,17 @@ jobs: ref: ${{ needs.eligibility.outputs.sha }} persist-credentials: false submodules: true + - name: Merge the pull request into main + if: ${{ needs.eligibility.outputs.head != '' }} + shell: bash + env: + PR_NUMBER: ${{ needs.eligibility.outputs.pr }} + PR_HEAD: ${{ needs.eligibility.outputs.head }} + run: | + git fetch --no-tags --unshallow origin "refs/pull/$PR_NUMBER/head" + if [ "$(git rev-parse FETCH_HEAD)" != "$PR_HEAD" ]; then echo "::error::The pull request has newer commits; start a new build"; exit 1; fi + if ! git -c user.name=ci -c user.email=ci@localhost merge --no-ff --no-edit FETCH_HEAD; then echo "::error::The pull request does not merge cleanly into main; rebase it first"; exit 1; fi + git submodule update --init --recursive - id: toolchain uses: actions/cache@v6 with: @@ -209,7 +277,7 @@ jobs: - uses: hendrikmuhs/ccache-action@v1.2.24 with: key: windows - save: false + save: ${{ github.event_name == 'push' && github.ref == 'refs/heads/main' }} evict-old-files: job - shell: cmd run: cmake -S . -B build %CONFIGURE% @@ -221,7 +289,8 @@ jobs: - shell: cmd run: cmake --build build --target libs - shell: cmd - run: ctest --test-dir build --output-on-failure --timeout 120 --parallel 4 --output-log ctest.log + run: ctest --test-dir build --output-on-failure --timeout 120 --parallel 4 --output-log ctest.log --output-junit ctest.xml + - run: python tools/check_skips.py build/ctest.xml --job windows - name: Report tests slower than 30 s if: ${{ !cancelled() }} shell: bash diff --git a/.github/workflows/labels.yml b/.github/workflows/labels.yml index 21fac27ac..703f7ecee 100644 --- a/.github/workflows/labels.yml +++ b/.github/workflows/labels.yml @@ -19,11 +19,12 @@ jobs: GH_REPO: ${{ github.repository }} steps: - run: | - label() { gh label create "$1" --color "$2" --description "$3"; } + existing=$(gh label list --limit 1000 --json name --jq '.[].name') + label() { grep -qixF -- "$1" <<< "$existing" || gh label create "$1" --color "$2" --description "$3"; } force_label() { gh label create "$1" --color "$2" --description "$3" --force; } force_label 'needs rebase' b60205 'Conflicts with main, rebase needed' - force_label 'intake paused' d93f0b 'Closed while PR intake is limited to authors with at least 20 PRs merged into main' + force_label 'intake paused' d93f0b 'Closed while PR intake is limited to authors with at least 10 PRs merged into main' force_label ci 5319e7 '.github' force_label agc 1d76db 'libSceAgc*' force_label shader 1d76db 'core/shader' diff --git a/.github/workflows/pr-intake.yml b/.github/workflows/pr-intake.yml index a632bd61c..c45ffe04c 100644 --- a/.github/workflows/pr-intake.yml +++ b/.github/workflows/pr-intake.yml @@ -27,7 +27,7 @@ jobs: if (pr.state !== 'open' || pr.base.ref !== 'main') { return; } - const minimumMergedPRs = 20; + const minimumMergedPRs = 10; const { data: mergedPRs } = await github.rest.search.issuesAndPullRequests({ q: `repo:${owner}/${repo} is:pr is:merged base:main author:${pr.user.login}`, per_page: 1 @@ -48,7 +48,7 @@ jobs: try { await github.rest.issues.createLabel({ owner, repo, name, color: 'd93f0b', - description: 'Closed while PR intake is limited to authors with at least 20 PRs merged into main' + description: 'Closed while PR intake is limited to authors with at least 10 PRs merged into main' }); } catch (createError) { if (createError.status !== 422) { diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 522ada2fd..544be3ea4 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -49,6 +49,8 @@ jobs: path: C:\winlibs key: ${{ env.WINLIBS_ZIP }} - if: matrix.platform == 'windows' && steps.toolchain.outputs.cache-hit != 'true' + env: + SSL_CERT_FILE: 'C:\Program Files\Git\mingw64\etc\ssl\certs\ca-bundle.crt' run: | curl -sSfL -o "$RUNNER_TEMP/winlibs.zip" "https://github.com/brechtsanders/winlibs_mingw/releases/download/$WINLIBS_RELEASE/$WINLIBS_ZIP" 7z x "$RUNNER_TEMP/winlibs.zip" -oC:/winlibs @@ -58,12 +60,17 @@ jobs: with: key: ${{ matrix.platform }} save: false + - name: Install lavapipe for the GPU execution tests + if: matrix.platform == 'linux' + run: sudo apt-get update && sudo apt-get install -y --no-install-recommends libvulkan1 mesa-vulkan-drivers - name: Configure Linux if: matrix.platform == 'linux' run: cmake -S . -B build $CONFIGURE - name: Configure Windows if: matrix.platform == 'windows' shell: cmd + env: + SSL_CERT_FILE: 'C:\Program Files\Git\mingw64\etc\ssl\certs\ca-bundle.crt' run: cmake -S . -B build %CONFIGURE% - name: Build project and tests (Linux) if: matrix.platform == 'linux' @@ -75,6 +82,9 @@ jobs: - name: Run tests (Linux) if: matrix.platform == 'linux' run: ctest --test-dir build --output-on-failure --timeout 120 --stop-on-failure --no-tests=error + env: + ANYPS5_REQUIRE_VULKAN: 1 + GALLIUM_OVERRIDE_CPU_CAPS: avx - name: Run tests (Windows) if: matrix.platform == 'windows' shell: cmd diff --git a/CMakeLists.txt b/CMakeLists.txt index 4c6923e7e..f71c2a4de 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -39,12 +39,24 @@ if(BUILD_TESTING) if(Python3_Interpreter_FOUND) add_test(NAME pr_overlap_encoding COMMAND ${Python3_EXECUTABLE} ${CMAKE_CURRENT_SOURCE_DIR}/tools/tests/test_pr_overlap_encoding.py) + add_test(NAME pr_overlap_tools COMMAND ${Python3_EXECUTABLE} + ${CMAKE_CURRENT_SOURCE_DIR}/tools/tests/test_pr_overlap.py) add_test(NAME progress_report COMMAND ${Python3_EXECUTABLE} ${CMAKE_CURRENT_SOURCE_DIR}/tools/tests/test_progress.py) + add_test(NAME progress_encoding COMMAND ${Python3_EXECUTABLE} + ${CMAKE_CURRENT_SOURCE_DIR}/tools/tests/test_progress_encoding.py) add_test(NAME import_audit COMMAND ${Python3_EXECUTABLE} ${CMAKE_CURRENT_SOURCE_DIR}/tools/tests/test_import_audit.py) + add_test(NAME conventions_function_bodies COMMAND ${Python3_EXECUTABLE} + ${CMAKE_CURRENT_SOURCE_DIR}/tools/tests/test_check_conventions_comments.py) + add_test(NAME test_registration COMMAND ${Python3_EXECUTABLE} + ${CMAKE_CURRENT_SOURCE_DIR}/tools/tests/test_test_registration.py) add_test(NAME hardware_oracle COMMAND ${Python3_EXECUTABLE} ${CMAKE_CURRENT_SOURCE_DIR}/tools/hw-oracle/test_hw_oracle.py) + add_test(NAME pr_overlap_exports COMMAND ${Python3_EXECUTABLE} + ${CMAKE_CURRENT_SOURCE_DIR}/tools/tests/test_pr_overlap_exports.py) + add_test(NAME nid_database_cache COMMAND ${Python3_EXECUTABLE} + ${CMAKE_CURRENT_SOURCE_DIR}/tools/tests/test_nid_names.py) endif() endif() @@ -88,6 +100,11 @@ set(FT_DISABLE_BROTLI ON CACHE BOOL "" FORCE) set(SKIP_INSTALL_ALL ON CACHE BOOL "" FORCE) add_subdirectory(3rdparty/freetype EXCLUDE_FROM_ALL) set_target_properties(freetype PROPERTIES POSITION_INDEPENDENT_CODE ON) +if(WIN32) + get_target_property(freetypeSources freetype SOURCES) + list(FILTER freetypeSources EXCLUDE REGEX "ftver\\.rc$") + set_target_properties(freetype PROPERTIES SOURCES "${freetypeSources}") +endif() add_subdirectory(core/shader/recompiler) add_subdirectory(core/Decoder/Jpeg) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index fc1d7a685..8fb682627 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -31,7 +31,13 @@ Python 3 is optional; without it some relinker tests are not registered. Every push to `main`, including a merged pull request, automatically runs full Linux and Windows builds and tests and the relinker matrix on Linux, Windows and macOS, using the pushed commit without contributor checks. -For pull requests, full Linux and Windows builds and tests run only when a collaborator with write access selects Actions > Build > Run workflow on `main` and enters an open pull request number targeting `main`. This manual run also builds and tests the relinker on Linux, Windows and macOS, regardless of the author's previous contributions or the changed files. The relinker matrix also runs automatically when a pull request targeting `main` is opened, updated or reopened, if its author already has a commit in `main`; full builds never run automatically for pull requests. All build jobs build the selected pull request merged into the current `main` (GitHub's merge commit), so a pull request that no longer merges cleanly must be rebased first. New commits require another manual run for full builds. +For pull requests, full Linux and Windows builds and tests run only when a collaborator with write access selects Actions > Build > Run workflow on `main` and enters an open pull request number targeting `main`. A collaborator can start the same manual run from the command line, replacing `1234` with the pull request number: + +``` +gh workflow run build.yml --repo boykopovar/AnyPS5 --ref main --field pr=1234 +``` + +This manual run also builds and tests the relinker on Linux, Windows and macOS, regardless of the author's previous contributions or the changed files. The relinker matrix also runs automatically when a pull request targeting `main` is opened, updated or reopened, if its author already has a commit in `main`; full builds never run automatically for pull requests. All build jobs build the selected pull request merged into the current `main` (GitHub's merge commit), so a pull request that no longer merges cleanly must be rebased first. New commits require another manual run for full builds. The Conventions check runs automatically for every contributor, including first-time contributors. It uses the base branch checker and reads pull request Git objects without executing pull request code. The Conventions check runs on every pull request and fails when a rule on this page is broken. It accepts code comments only when the pull request also changes [TechnicalDebt](docs/dev/TechnicalDebt.md), and only UTF-8 text files. Run it locally before pushing: diff --git a/core/Decoder/Png/src/Png.cpp b/core/Decoder/Png/src/Png.cpp index d137df164..cb04f25bd 100644 --- a/core/Decoder/Png/src/Png.cpp +++ b/core/Decoder/Png/src/Png.cpp @@ -138,9 +138,9 @@ std::optional
ParseHeader(std::span png) { while (png.size() - offset >= CHUNK_OVERHEAD) { const std::uint32_t length = readBigEndian32(&png[offset]); const std::uint8_t* type = &png[offset + 4]; + if (length > png.size() - offset - CHUNK_OVERHEAD) break; if (isChunkType(type, "tRNS")) header.hasTransparency = true; if (isChunkType(type, "tRNS") || isChunkType(type, "IDAT") || isChunkType(type, "IEND")) break; - if (length > png.size() - offset - CHUNK_OVERHEAD) break; offset += CHUNK_OVERHEAD + length; } return header; diff --git a/core/Decoder/Png/tests/Png.cpp b/core/Decoder/Png/tests/Png.cpp index 7287ed5b3..0c1ac81d1 100644 --- a/core/Decoder/Png/tests/Png.cpp +++ b/core/Decoder/Png/tests/Png.cpp @@ -153,6 +153,12 @@ int main() { Require(paletteHeader.has_value()); Require(paletteHeader->width == 2 && paletteHeader->height == 2 && paletteHeader->bitDepth == 2); Require(paletteHeader->colorType == Decoder::Png::ColorType::Palette && paletteHeader->hasTransparency); + for (const std::size_t size : {std::size_t{69}, std::size_t{71}}) { + const auto truncated = Decoder::Png::ParseHeader(std::span(PALETTE_TRNS, size)); + Require(truncated.has_value() && !truncated->hasTransparency); + } + const auto completeTransparency = Decoder::Png::ParseHeader(std::span(PALETTE_TRNS, 73)); + Require(completeTransparency.has_value() && completeTransparency->hasTransparency); const auto palette = Decoder::Png::Decode(PALETTE_TRNS); Require(palette.has_value()); Require(palette->pixels == std::vector({255, 0, 0, 255, 0, 255, 0, 0, 0, 0, 255, 255, 255, 255, 0, 128})); diff --git a/core/libs/CMakeLists.txt b/core/libs/CMakeLists.txt index 5569c3a1b..32eb46fa5 100644 --- a/core/libs/CMakeLists.txt +++ b/core/libs/CMakeLists.txt @@ -10,6 +10,7 @@ add_executable(nid_patcher ${CMAKE_CURRENT_SOURCE_DIR}/nid/src/ExportExclusions.cpp ${CMAKE_CURRENT_SOURCE_DIR}/nid/src/ElfNidPatcher.cpp ${CMAKE_CURRENT_SOURCE_DIR}/nid/src/PeNidPatcher.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/nid/src/MachONidPatcher.cpp ${CMAKE_CURRENT_SOURCE_DIR}/nid/src/BinaryPatcherFactory.cpp ${CMAKE_CURRENT_SOURCE_DIR}/nid/src/main.cpp ) @@ -131,6 +132,11 @@ endforeach() add_custom_target(libs DEPENDS ${PATCHED_OUTPUTS}) if(BUILD_TESTING) + find_package(Python3 QUIET COMPONENTS Interpreter) + if(Python3_Interpreter_FOUND) + add_test(NAME nid_patcher_macho COMMAND ${Python3_EXECUTABLE} + ${CMAKE_CURRENT_SOURCE_DIR}/nid/tests/test_macho_nids.py $) + endif() add_test_executable(patched_libraries_tests tests/PatchedLibraries.cpp) add_dependencies(patched_libraries_tests libs) if(MINGW) @@ -138,6 +144,19 @@ if(BUILD_TESTING) target_link_options(patched_libraries_tests PRIVATE -static -static-libgcc -static-libstdc++) endif() add_test(NAME patched_libraries COMMAND patched_libraries_tests ${PATCHED_DIR}) + add_test_executable(guest_kernel_sys_tests tests/GuestKernelSys.cpp) + target_include_directories(guest_kernel_sys_tests PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}) + target_link_libraries(guest_kernel_sys_tests PRIVATE libkernel_sys libkernel libc) + configure_windows_unwind(guest_kernel_sys_tests) + add_test(NAME guest_kernel_sys COMMAND guest_kernel_sys_tests) + if(WIN32) + set_tests_properties(guest_kernel_sys PROPERTIES ENVIRONMENT_MODIFICATION + "PATH=path_list_prepend:$") + endif() + if(APPLE) + add_subdirectory(tests/write_watch) + endif() + add_test_executable(guest_offline_imports_tests tests/GuestOfflineImports.cpp) target_include_directories(guest_offline_imports_tests PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}) target_link_libraries(guest_offline_imports_tests PRIVATE libSceHttp libSceNpEntitlementAccess libSceRudp libc) @@ -147,6 +166,11 @@ if(BUILD_TESTING) set_tests_properties(guest_offline_imports PROPERTIES ENVIRONMENT_MODIFICATION "PATH=path_list_prepend:$") endif() + + if(UNIX AND (CMAKE_SYSTEM_PROCESSOR MATCHES "^(x86_64|AMD64)$" OR CMAKE_OSX_ARCHITECTURES STREQUAL "x86_64")) + add_subdirectory(tests/register_context) + endif() + add_test_executable(savedata_memory_metadata_tests tests/SaveDataMemoryMetadata.cpp) target_include_directories(savedata_memory_metadata_tests PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}) target_link_libraries(savedata_memory_metadata_tests PRIVATE libSceSaveData.native) diff --git a/core/libs/SceTypes.hpp b/core/libs/SceTypes.hpp index 887b13d57..99211616a 100644 --- a/core/libs/SceTypes.hpp +++ b/core/libs/SceTypes.hpp @@ -205,6 +205,19 @@ struct ModuleSegmentInfo { std::int32_t prot; }; +struct ModuleInfo { + std::uint64_t st_size; + char name[256]; + ModuleSegmentInfo segments[4]; + std::uint32_t segment_count; + std::uint8_t fingerprint[20]; +}; +static_assert(offsetof(ModuleInfo, name) == 0x8); +static_assert(offsetof(ModuleInfo, segments) == 0x108); +static_assert(offsetof(ModuleInfo, segment_count) == 0x148); +static_assert(offsetof(ModuleInfo, fingerprint) == 0x14C); +static_assert(sizeof(ModuleInfo) == 0x160); + struct ModuleInfoEx { std::uint64_t st_size; char name[256]; @@ -534,6 +547,13 @@ struct Audio3dOpenParameters { std::uint32_t num_beds; }; +struct Audio3dAttribute { + std::uint32_t attribute_id; + std::uint32_t pad; + const void* value; + std::uint64_t value_size; +}; + using AudioPropagationHandle = std::uint64_t; struct AudioPropagationStructDescriptor { @@ -1827,6 +1847,19 @@ struct VideoOutOutputStatus { std::uint64_t reserved[3] = {}; }; +struct VideoOutResolutionStatus { + std::uint32_t fullWidth = 0; + std::uint32_t fullHeight = 0; + std::uint32_t paneWidth = 0; + std::uint32_t paneHeight = 0; + std::uint64_t refreshRate = 0; + float screenSizeInInch = 0.0f; + std::uint16_t flags = 0; + std::uint16_t reserved0 = 0; + std::uint32_t reserved1[3] = {}; +}; +static_assert(sizeof(VideoOutResolutionStatus) == 48 && offsetof(VideoOutResolutionStatus, refreshRate) == 16 && offsetof(VideoOutResolutionStatus, screenSizeInInch) == 24); + struct VideoOutOutputOptions { std::uint32_t internalData[16] = {}; }; struct VideoOutColorSettings { diff --git a/core/libs/nid/include/nid/MachONidPatcher.hpp b/core/libs/nid/include/nid/MachONidPatcher.hpp new file mode 100644 index 000000000..f49732cd8 --- /dev/null +++ b/core/libs/nid/include/nid/MachONidPatcher.hpp @@ -0,0 +1,30 @@ +#ifndef CORE_LIBS_NID_MACHONIDPATCHER_HPP +#define CORE_LIBS_NID_MACHONIDPATCHER_HPP + +#include +#include +#include +#include + +namespace Nid { + +constexpr std::uint32_t kMachOMagic64 = 0xfeedfacfu; + +struct MachOExport { + std::string Name; + std::uint64_t Flags = 0; + std::uint64_t Address = 0; + std::uint64_t Other = 0; + std::string ImportName; +}; + +std::vector ReadMachOExports(const std::vector& binary); + +class MachONidPatcher final : public IBinaryPatcher { +public: + void PatchNids(std::vector& binary, const std::string& libraryName, const std::unordered_set& excludedExports) const override; +}; + +} + +#endif diff --git a/core/libs/nid/src/BinaryPatcherFactory.cpp b/core/libs/nid/src/BinaryPatcherFactory.cpp index c11cd304f..a8ff6f007 100644 --- a/core/libs/nid/src/BinaryPatcherFactory.cpp +++ b/core/libs/nid/src/BinaryPatcherFactory.cpp @@ -1,5 +1,6 @@ #include #include +#include #include #include @@ -10,6 +11,8 @@ std::unique_ptr MakePatcher(const std::vector& bin return std::make_unique(); if (binary.size() >= 2 && binary[0] == 'M' && binary[1] == 'Z') return std::make_unique(); + if (binary.size() >= 4 && binary[0] == 0xcf && binary[1] == 0xfa && binary[2] == 0xed && binary[3] == 0xfe) + return std::make_unique(); throw std::runtime_error("unrecognized binary format"); } diff --git a/core/libs/nid/src/ExportExclusions.cpp b/core/libs/nid/src/ExportExclusions.cpp index 64d8fa747..9c00d6b3e 100644 --- a/core/libs/nid/src/ExportExclusions.cpp +++ b/core/libs/nid/src/ExportExclusions.cpp @@ -1,6 +1,7 @@ #include #include #include +#include #include #include #include @@ -121,6 +122,12 @@ std::unordered_set ReadExportExclusions(const std::string& path) { exports = ReadPeExports(binary); } else if (binary.size() >= 4 && binary[0] == 0x7f && binary[1] == 'E' && binary[2] == 'L' && binary[3] == 'F') { exports = ReadElfExports(binary); + } else if (binary.size() >= 4 && binary[0] == 0xcf && binary[1] == 0xfa && binary[2] == 0xed && binary[3] == 0xfe) { + for (const auto& entry : ReadMachOExports(binary)) { + if (entry.Name.size() < 2 || entry.Name[0] != '_') throw std::runtime_error("invalid reference Mach-O export: " + entry.Name); + if (!exports.insert(entry.Name.substr(1)).second) throw std::runtime_error("duplicate reference export: " + entry.Name); + } + if (exports.empty()) throw std::runtime_error("reference Mach-O has no exports"); } else { throw std::runtime_error("unrecognized export reference format: " + path); } diff --git a/core/libs/nid/src/MachONidPatcher.cpp b/core/libs/nid/src/MachONidPatcher.cpp new file mode 100644 index 000000000..45bdee273 --- /dev/null +++ b/core/libs/nid/src/MachONidPatcher.cpp @@ -0,0 +1,493 @@ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace Nid { + +namespace { + +using Internal::Read; +using Internal::Write; + +constexpr std::uint32_t kLcSegment64 = 0x19; +constexpr std::uint32_t kLcCodeSignature = 0x1d; +constexpr std::uint32_t kLcDyldInfo = 0x22; +constexpr std::uint32_t kLcDyldInfoOnly = 0x80000022u; +constexpr std::uint32_t kLcDyldExportsTrie = 0x80000033u; +constexpr std::uint32_t kLcDyldChainedFixups = 0x80000034u; +constexpr std::uint32_t kLcLoadDylib = 0x0c; +constexpr std::uint32_t kLcLoadWeakDylib = 0x80000018u; +constexpr std::uint32_t kLcReexportDylib = 0x8000001fu; +constexpr std::uint32_t kLcLazyLoadDylib = 0x20; +constexpr std::uint32_t kLcLoadUpwardDylib = 0x80000023u; + +constexpr std::uint64_t kExportWeakDefinition = 0x04; +constexpr std::uint64_t kExportReexport = 0x08; +constexpr std::uint64_t kExportStubAndResolver = 0x10; + +constexpr std::uint32_t kImportFormatPlain = 1; +constexpr std::uint32_t kImportFormatAddend = 2; +constexpr std::uint32_t kImportFormatAddend64 = 3; + +constexpr std::uint64_t kPageSize = 0x1000; + +struct MachHeader64 { + std::uint32_t Magic; + std::uint32_t CpuType; + std::uint32_t CpuSubtype; + std::uint32_t FileType; + std::uint32_t CommandCount; + std::uint32_t CommandBytes; + std::uint32_t Flags; + std::uint32_t Reserved; +}; + +struct LoadCommand { + std::uint32_t Cmd; + std::uint32_t Size; +}; + +struct SegmentCommand64 { + std::uint32_t Cmd; + std::uint32_t Size; + char Name[16]; + std::uint64_t VmAddress; + std::uint64_t VmSize; + std::uint64_t FileOffset; + std::uint64_t FileSize; + std::uint32_t MaxProtection; + std::uint32_t InitProtection; + std::uint32_t SectionCount; + std::uint32_t Flags; +}; + +struct LinkeditDataCommand { + std::uint32_t Cmd; + std::uint32_t Size; + std::uint32_t DataOffset; + std::uint32_t DataSize; +}; + +struct DyldInfoCommand { + std::uint32_t Cmd; + std::uint32_t Size; + std::uint32_t RebaseOffset; + std::uint32_t RebaseSize; + std::uint32_t BindOffset; + std::uint32_t BindSize; + std::uint32_t WeakBindOffset; + std::uint32_t WeakBindSize; + std::uint32_t LazyBindOffset; + std::uint32_t LazyBindSize; + std::uint32_t ExportOffset; + std::uint32_t ExportSize; +}; + +struct ChainedFixupsHeader { + std::uint32_t Version; + std::uint32_t StartsOffset; + std::uint32_t ImportsOffset; + std::uint32_t SymbolsOffset; + std::uint32_t ImportsCount; + std::uint32_t ImportsFormat; + std::uint32_t SymbolsFormat; +}; + +void RequireRange(const std::vector& binary, std::uint64_t offset, std::uint64_t size) { + if (offset > binary.size() || size > binary.size() - offset) throw std::runtime_error("Mach-O range out of bounds"); +} + +std::uint64_t ReadUleb(const std::vector& data, std::size_t& cursor, std::size_t end) { + std::uint64_t value = 0; + unsigned shift = 0; + for (;;) { + if (cursor >= end) throw std::runtime_error("truncated uleb128 in export trie"); + const auto byte = data[cursor++]; + if (shift >= 64 || (shift == 63 && (byte & 0x7e) != 0)) throw std::runtime_error("uleb128 overflow in export trie"); + value |= std::uint64_t{byte & 0x7fu} << shift; + if ((byte & 0x80) == 0) return value; + shift += 7; + } +} + +void WriteUleb(std::vector& out, std::uint64_t value) { + do { + std::uint8_t byte = value & 0x7f; + value >>= 7; + if (value != 0) byte |= 0x80; + out.push_back(byte); + } while (value != 0); +} + +std::size_t UlebSize(std::uint64_t value) { + std::size_t size = 1; + while (value >>= 7) ++size; + return size; +} + +std::string ReadTrieString(const std::vector& data, std::size_t& cursor, std::size_t end) { + const auto start = cursor; + while (cursor < end && data[cursor] != 0) ++cursor; + if (cursor >= end) throw std::runtime_error("unterminated string in export trie"); + std::string text(reinterpret_cast(data.data() + start), cursor - start); + ++cursor; + return text; +} + +struct Image { + MachHeader64 Header {}; + std::vector Commands; +}; + +Image ParseImage(const std::vector& binary) { + Image image; + image.Header = Read(binary, 0); + if (image.Header.Magic != kMachOMagic64) throw std::runtime_error("not a 64-bit little-endian Mach-O image"); + RequireRange(binary, sizeof(MachHeader64), image.Header.CommandBytes); + std::size_t cursor = sizeof(MachHeader64); + const auto end = cursor + image.Header.CommandBytes; + for (std::uint32_t index = 0; index < image.Header.CommandCount; ++index) { + if (cursor + sizeof(LoadCommand) > end) throw std::runtime_error("load command out of bounds"); + const auto command = Read(binary, cursor); + if (command.Size < sizeof(LoadCommand) || command.Size % 8 != 0 || command.Size > end - cursor) throw std::runtime_error("invalid load command size"); + image.Commands.push_back(cursor); + cursor += command.Size; + } + return image; +} + +struct TrieLocation { + std::size_t OffsetField; + std::size_t SizeField; +}; + +std::optional FindTrie(const std::vector& binary, const Image& image) { + std::optional found; + for (const auto offset : image.Commands) { + const auto cmd = Read(binary, offset).Cmd; + std::optional here; + if (cmd == kLcDyldExportsTrie) here = TrieLocation{offset + offsetof(LinkeditDataCommand, DataOffset), offset + offsetof(LinkeditDataCommand, DataSize)}; + else if (cmd == kLcDyldInfo || cmd == kLcDyldInfoOnly) { + if (Read(binary, offset).ExportSize != 0) + here = TrieLocation{offset + offsetof(DyldInfoCommand, ExportOffset), offset + offsetof(DyldInfoCommand, ExportSize)}; + } + if (!here) continue; + if (found) throw std::runtime_error("Mach-O image has more than one export trie"); + found = here; + } + return found; +} + +void ParseTrieNode(const std::vector& data, std::size_t start, std::size_t end, std::size_t node, std::string& prefix, std::set& visited, std::vector& out) { + if (node >= end - start) throw std::runtime_error("export trie node out of bounds"); + if (!visited.insert(node).second) throw std::runtime_error("export trie has a cycle"); + std::size_t cursor = start + node; + const auto terminalSize = ReadUleb(data, cursor, end); + if (terminalSize > end - cursor) throw std::runtime_error("export trie terminal out of bounds"); + const auto children = cursor + terminalSize; + if (terminalSize != 0) { + MachOExport entry; + entry.Name = prefix; + entry.Flags = ReadUleb(data, cursor, children); + if (entry.Flags & kExportReexport) { + entry.Other = ReadUleb(data, cursor, children); + entry.ImportName = ReadTrieString(data, cursor, children); + } else { + entry.Address = ReadUleb(data, cursor, children); + if (entry.Flags & kExportStubAndResolver) entry.Other = ReadUleb(data, cursor, children); + } + out.push_back(std::move(entry)); + } + cursor = children; + if (cursor >= end) throw std::runtime_error("export trie child count out of bounds"); + const auto childCount = data[cursor++]; + for (unsigned index = 0; index < childCount; ++index) { + const auto label = ReadTrieString(data, cursor, end); + const auto child = ReadUleb(data, cursor, end); + const auto length = prefix.size(); + prefix += label; + ParseTrieNode(data, start, end, static_cast(child), prefix, visited, out); + prefix.resize(length); + } +} + +std::vector ParseTrie(const std::vector& binary, std::uint32_t offset, std::uint32_t size) { + RequireRange(binary, offset, size); + std::vector exports; + if (size == 0) return exports; + std::string prefix; + std::set visited; + ParseTrieNode(binary, offset, std::size_t{offset} + size, 0, prefix, visited, exports); + return exports; +} + +class TrieBuilder { +public: + explicit TrieBuilder(const std::vector& exports) { + for (const auto& entry : exports) Insert(entry); + } + + std::vector Build() { + std::vector order; + Collect(&root, order); + for (bool changed = true; changed;) { + changed = false; + std::uint64_t offset = 0; + for (auto* node : order) { + if (node->Offset != offset) { + node->Offset = offset; + changed = true; + } + offset += NodeSize(*node); + } + } + std::vector out; + for (auto* node : order) { + if (out.size() != node->Offset) throw std::runtime_error("export trie layout mismatch"); + const auto terminal = TerminalBytes(*node); + WriteUleb(out, terminal.size()); + out.insert(out.end(), terminal.begin(), terminal.end()); + if (node->Children.size() > 255) throw std::runtime_error("export trie node has too many children"); + out.push_back(static_cast(node->Children.size())); + for (const auto& [label, child] : node->Children) { + out.insert(out.end(), label.begin(), label.end()); + out.push_back(0); + WriteUleb(out, child->Offset); + } + } + while (out.size() % 8 != 0) out.push_back(0); + return out; + } + +private: + struct Node { + std::vector>> Children; + std::optional Entry; + std::uint64_t Offset = 0; + }; + + Node root; + + void Insert(const MachOExport& entry) { + Node* node = &root; + std::string_view rest = entry.Name; + while (!rest.empty()) { + bool descended = false; + for (auto& [label, child] : node->Children) { + std::size_t common = 0; + while (common < label.size() && common < rest.size() && label[common] == rest[common]) ++common; + if (common == 0) continue; + if (common < label.size()) { + auto middle = std::make_unique(); + middle->Children.emplace_back(label.substr(common), std::move(child)); + label.resize(common); + child = std::move(middle); + } + node = child.get(); + rest.remove_prefix(common); + descended = true; + break; + } + if (!descended) { + node->Children.emplace_back(std::string(rest), std::make_unique()); + node = node->Children.back().second.get(); + rest = {}; + } + } + if (node->Entry) throw std::runtime_error("duplicate patched export name: " + entry.Name); + node->Entry = entry; + } + + static void Collect(Node* node, std::vector& order) { + order.push_back(node); + for (auto& [label, child] : node->Children) Collect(child.get(), order); + } + + static std::vector TerminalBytes(const Node& node) { + std::vector bytes; + if (!node.Entry) return bytes; + const auto& entry = *node.Entry; + WriteUleb(bytes, entry.Flags); + if (entry.Flags & kExportReexport) { + WriteUleb(bytes, entry.Other); + bytes.insert(bytes.end(), entry.ImportName.begin(), entry.ImportName.end()); + bytes.push_back(0); + } else { + WriteUleb(bytes, entry.Address); + if (entry.Flags & kExportStubAndResolver) WriteUleb(bytes, entry.Other); + } + return bytes; + } + + static std::uint64_t NodeSize(const Node& node) { + const auto terminal = TerminalBytes(node).size(); + std::uint64_t size = UlebSize(terminal) + terminal + 1; + for (const auto& [label, child] : node.Children) size += label.size() + 1 + UlebSize(child->Offset); + return size; + } +}; + +std::string StripUnderscore(const std::string& name) { + return !name.empty() && name[0] == '_' ? name.substr(1) : name; +} + +bool IsGuestFacingImport(const std::string& name) { + using namespace Internal; + const bool hasNidPostfix = name.size() >= kNidPostfixLen && name.compare(name.size() - kNidPostfixLen, kNidPostfixLen, kNidPostfix) == 0; + const bool hasScePrefix = name.size() >= 3u && + std::tolower(static_cast(name[0])) == 's' && + std::tolower(static_cast(name[1])) == 'c' && + std::tolower(static_cast(name[2])) == 'e'; + return hasNidPostfix || hasScePrefix; +} + +std::vector DylibOrdinalsArePrx(const std::vector& binary, const Image& image) { + std::vector result; + for (const auto offset : image.Commands) { + const auto command = Read(binary, offset); + if (command.Cmd != kLcLoadDylib && command.Cmd != kLcLoadWeakDylib && command.Cmd != kLcReexportDylib && command.Cmd != kLcLazyLoadDylib && command.Cmd != kLcLoadUpwardDylib) continue; + const auto nameOffset = Read(binary, offset + 8); + if (nameOffset >= command.Size) throw std::runtime_error("dylib name out of bounds"); + const auto name = Internal::ReadCStr(binary, offset + nameOffset); + result.push_back(name.size() >= 4 && name.compare(name.size() - 4, 4, ".prx") == 0); + } + return result; +} + +void PatchChainedImports(std::vector& binary, const Image& image) { + std::optional fixups; + for (const auto offset : image.Commands) { + const auto cmd = Read(binary, offset).Cmd; + if (cmd == kLcDyldChainedFixups) fixups = Read(binary, offset); + if ((cmd == kLcDyldInfo || cmd == kLcDyldInfoOnly)) { + const auto info = Read(binary, offset); + if (info.BindSize != 0 || info.LazyBindSize != 0 || info.WeakBindSize != 0) throw std::runtime_error("Mach-O bind opcodes are not supported; link with chained fixups"); + } + } + if (!fixups || fixups->DataSize == 0) return; + RequireRange(binary, fixups->DataOffset, fixups->DataSize); + const auto base = std::size_t{fixups->DataOffset}; + const auto header = Read(binary, base); + if (header.Version != 0) throw std::runtime_error("unsupported chained fixups version"); + if (header.SymbolsFormat != 0) throw std::runtime_error("compressed chained fixup symbols are not supported"); + const std::size_t entrySize = header.ImportsFormat == kImportFormatPlain ? 4 : header.ImportsFormat == kImportFormatAddend ? 8 : header.ImportsFormat == kImportFormatAddend64 ? 16 : 0; + if (entrySize == 0) throw std::runtime_error("unsupported chained import format"); + if (header.ImportsOffset > fixups->DataSize || std::uint64_t{header.ImportsCount} * entrySize > fixups->DataSize - header.ImportsOffset || header.SymbolsOffset > fixups->DataSize) + throw std::runtime_error("chained imports out of bounds"); + const auto symbols = base + header.SymbolsOffset; + const auto symbolsEnd = base + fixups->DataSize; + const auto prx = DylibOrdinalsArePrx(binary, image); + + struct Import { std::uint32_t NameOffset; std::uint32_t Ordinal; }; + std::vector imports; + for (std::uint32_t index = 0; index < header.ImportsCount; ++index) { + const auto at = base + header.ImportsOffset + index * entrySize; + if (header.ImportsFormat == kImportFormatAddend64) { + const auto raw = Read(binary, at); + imports.push_back({static_cast(raw >> 32), static_cast(raw & 0xffff)}); + } else { + const auto raw = Read(binary, at); + imports.push_back({raw >> 9, raw & 0xff}); + } + } + std::set nameOffsets; + for (const auto& item : imports) nameOffsets.insert(item.NameOffset); + std::set rewritten; + for (const auto& item : imports) { + const bool special = header.ImportsFormat == kImportFormatAddend64 ? item.Ordinal >= 0xfff0 : item.Ordinal >= 0xf0; + if (item.Ordinal == 0 || special || item.Ordinal > prx.size() || !prx[item.Ordinal - 1]) continue; + if (rewritten.contains(item.NameOffset)) continue; + const auto at = symbols + item.NameOffset; + if (at >= symbolsEnd) throw std::runtime_error("chained import name out of bounds"); + const auto name = Internal::ReadCStr(binary, at); + if (at + name.size() >= symbolsEnd) throw std::runtime_error("unterminated chained import name"); + const auto bare = StripUnderscore(name); + if (bare == name || !IsGuestFacingImport(bare)) continue; + const auto patched = "_" + ResolveOneName(bare); + if (patched.size() > name.size()) throw std::runtime_error("import NID longer than original name: " + name); + const auto inside = nameOffsets.upper_bound(item.NameOffset); + if (inside != nameOffsets.end() && *inside <= item.NameOffset + name.size()) throw std::runtime_error("chained import names share storage: " + name); + std::memcpy(binary.data() + at, patched.data(), patched.size()); + std::fill(binary.begin() + static_cast(at + patched.size()), binary.begin() + static_cast(at + name.size() + 1), 0); + rewritten.insert(item.NameOffset); + } +} + +void WriteTrie(std::vector& binary, const Image& image, const TrieLocation& location, const std::vector& trie) { + const auto oldOffset = Read(binary, location.OffsetField); + const auto oldSize = Read(binary, location.SizeField); + if (trie.size() <= oldSize) { + std::copy(trie.begin(), trie.end(), binary.begin() + oldOffset); + std::fill(binary.begin() + oldOffset + static_cast(trie.size()), binary.begin() + oldOffset + oldSize, 0); + Write(binary, location.SizeField, static_cast(trie.size())); + return; + } + std::optional linkedit; + for (const auto offset : image.Commands) { + const auto cmd = Read(binary, offset).Cmd; + if (cmd == kLcCodeSignature) throw std::runtime_error("signed Mach-O images cannot grow their export trie"); + if (cmd != kLcSegment64) continue; + const auto segment = Read(binary, offset); + if (std::strncmp(segment.Name, "__LINKEDIT", sizeof(segment.Name)) == 0) linkedit = offset; + } + if (!linkedit) throw std::runtime_error("Mach-O image has no __LINKEDIT segment"); + auto segment = Read(binary, *linkedit); + if (segment.FileOffset + segment.FileSize != binary.size()) throw std::runtime_error("__LINKEDIT does not end the file"); + const auto newOffset = (binary.size() + 7) & ~std::size_t{7}; + if (newOffset + trie.size() > 0xffffffffu) throw std::runtime_error("export trie offset overflow"); + binary.resize(newOffset, 0); + binary.insert(binary.end(), trie.begin(), trie.end()); + std::fill(binary.begin() + oldOffset, binary.begin() + oldOffset + oldSize, 0); + segment.FileSize = binary.size() - segment.FileOffset; + segment.VmSize = std::max(segment.VmSize, (segment.FileSize + kPageSize - 1) & ~(kPageSize - 1)); + Write(binary, *linkedit, segment); + Write(binary, location.OffsetField, static_cast(newOffset)); + Write(binary, location.SizeField, static_cast(trie.size())); +} + +} + +std::vector ReadMachOExports(const std::vector& binary) { + const auto image = ParseImage(binary); + const auto location = FindTrie(binary, image); + if (!location) return {}; + return ParseTrie(binary, Read(binary, location->OffsetField), Read(binary, location->SizeField)); +} + +void MachONidPatcher::PatchNids(std::vector& binary, const std::string& libraryName, const std::unordered_set& excludedExports) const { + const auto image = ParseImage(binary); + const auto location = FindTrie(binary, image); + if (!location) throw std::runtime_error("no export trie"); + auto exports = ParseTrie(binary, Read(binary, location->OffsetField), Read(binary, location->SizeField)); + if (exports.empty()) throw std::runtime_error("no exported names"); + + std::vector names; + names.reserve(exports.size()); + for (const auto& entry : exports) { + if (entry.Name.empty() || entry.Name[0] != '_') throw std::runtime_error("exported name without a C underscore: " + entry.Name); + names.push_back(entry.Name.substr(1)); + } + const auto nidMap = ResolveNids(names, libraryName, excludedExports); + for (std::size_t index = 0; index < exports.size(); ++index) { + const auto& name = names[index]; + const bool marked = name.ends_with(Internal::kNidPostfix) || Internal::IsNidNoPatch(name) || Internal::IsNidNoPatchCut(name); + if ((exports[index].Flags & kExportWeakDefinition) != 0 && !marked) continue; + const auto found = nidMap.find(name); + if (found == nidMap.end()) throw std::runtime_error("symbol not in nid map: " + name); + exports[index].Name = "_" + found->second; + } + WriteTrie(binary, image, *location, TrieBuilder(exports).Build()); + PatchChainedImports(binary, ParseImage(binary)); +} + +} diff --git a/core/libs/nid/tests/test_macho_nids.py b/core/libs/nid/tests/test_macho_nids.py new file mode 100644 index 000000000..1d5c118ce --- /dev/null +++ b/core/libs/nid/tests/test_macho_nids.py @@ -0,0 +1,235 @@ +"""Check nid_patcher on small byte-built Mach-O dylibs: the export trie, the chained imports and what it refuses.""" + +import hashlib +import struct +import subprocess +import sys +import tempfile +from pathlib import Path + +SUFFIX = bytes([0x51, 0x8D, 0x64, 0xA6, 0x35, 0xDE, 0xD8, 0xC1, 0xE6, 0xB0, 0x39, 0xB1, 0xC3, 0xE5, 0x52, 0x30]) +BASE64 = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+-" + +LC_SEGMENT_64 = 0x19 +LC_LOAD_DYLIB = 0x0C +LC_CODE_SIGNATURE = 0x1D +LC_DYLD_INFO_ONLY = 0x80000022 +LC_DYLD_EXPORTS_TRIE = 0x80000033 +LC_DYLD_CHAINED_FIXUPS = 0x80000034 +EXPORT_WEAK = 0x04 +PAGE = 0x1000 + + +def nid(name): + digest = hashlib.sha1(name.encode() + SUFFIX).digest() + reversed_bytes = digest[:8][::-1] + text = "" + for index in (0, 3): + triple = (reversed_bytes[index] << 16) | (reversed_bytes[index + 1] << 8) | reversed_bytes[index + 2] + text += BASE64[(triple >> 18) & 63] + BASE64[(triple >> 12) & 63] + BASE64[(triple >> 6) & 63] + BASE64[triple & 63] + tail = (reversed_bytes[6] << 16) | (reversed_bytes[7] << 8) + return text + BASE64[(tail >> 18) & 63] + BASE64[(tail >> 12) & 63] + BASE64[(tail >> 6) & 63] + + +def uleb(value): + out = bytearray() + while True: + byte = value & 0x7F + value >>= 7 + out.append(byte | (0x80 if value else 0)) + if not value: + return bytes(out) + + +def read_uleb(data, position): + value = shift = 0 + while True: + byte = data[position] + position += 1 + value |= (byte & 0x7F) << shift + if not byte & 0x80: + return value, position + shift += 7 + + +def build_trie(entries): + offsets = [0] * len(entries) + for _ in range(8): + root = bytearray(b"\0") + bytes([len(entries)]) + for (name, _, _), offset in zip(entries, offsets): + root += name.encode() + b"\0" + uleb(offset) + nodes = [] + for name, flags, address in entries: + terminal = uleb(flags) + uleb(address) + nodes.append(uleb(len(terminal)) + terminal + b"\0") + position, expected = len(root), [] + for node in nodes: + expected.append(position) + position += len(node) + if expected == offsets: + break + offsets = expected + trie = bytes(root) + b"".join(nodes) + return trie + bytes(-len(trie) % 8) + + +def parse_trie(data, start, size): + found = {} + + def walk(node, prefix): + position = start + node + terminal, position = read_uleb(data, position) + children = position + terminal + if terminal: + flags, position = read_uleb(data, position) + address, position = read_uleb(data, position) + found[prefix] = (flags, address) + count = data[children] + position = children + 1 + for _ in range(count): + end = data.index(0, position) + label = data[position:end].decode() + child, position = read_uleb(data, end + 1) + walk(child, prefix + label) + + if size: + walk(0, "") + return found + + +def build_fixups(imports): + symbols = bytearray(b"\0") + entries = [] + for name, ordinal in imports: + entries.append(ordinal | (len(symbols) << 9)) + symbols += name.encode() + b"\0" + imports_offset = 32 + header = struct.pack("<7I", 0, 28, imports_offset, imports_offset + 4 * len(imports), len(imports), 1, 0) + blob = header + struct.pack("> 9) + result.append((data[start:data.index(0, start)].decode(), raw & 0xFF)) + return result + + +def patch(patcher, directory, name, image): + path = Path(directory) / name + path.write_bytes(image) + result = subprocess.run([str(patcher), "libtest", str(path)], capture_output=True, text=True, timeout=30) + return result, path + + +def exports_of(data): + offset = command_fields(data)[LC_DYLD_EXPORTS_TRIE][0] + start, size = struct.unpack_from("= filesize and len(data) > len(original), (fileoff, filesize, len(data)) + trie_offset, trie_size = struct.unpack_from("= len(original) and trie_offset + trie_size <= len(data) and trie_offset % 8 == 0, (trie_offset, trie_size) + + +def check_preserved(patcher, directory): + reference = Path(directory) / "reference.dylib" + reference.write_bytes(build_dylib([("_puts", 0, 0x100)], [])) + path = Path(directory) / "preserve.dylib" + path.write_bytes(build_dylib([("_puts_nid_postfix", 0, 0x100), ("_exit_nid_postfix", 0, 0x108)], [])) + result = subprocess.run([str(patcher), "libtest", "--preserve-exports", str(reference), str(path)], capture_output=True, text=True, timeout=30) + assert result.returncode == 0, (result.stdout, result.stderr) + assert exports_of(path.read_bytes()) == {"_puts_nid_postfix": (0, 0x100), "_" + nid("exit"): (0, 0x108)}, exports_of(path.read_bytes()) + + +def check_refusals(patcher, directory): + growing = [("_ab", 0, 0x100), ("_cd", 0, 0x108)] + result, _ = patch(patcher, directory, "signed.dylib", build_dylib(growing, [], signed=True)) + assert result.returncode != 0 and "signed Mach-O images cannot grow their export trie" in result.stdout + result.stderr, (result.stdout, result.stderr) + result, _ = patch(patcher, directory, "opcodes.dylib", build_dylib([("_puts_nid_postfix", 0, 0x100)], [], bind_opcodes=True)) + assert result.returncode != 0 and "bind opcodes are not supported" in result.stdout + result.stderr, (result.stdout, result.stderr) + result, _ = patch(patcher, directory, "long.dylib", build_dylib([("_puts_nid_postfix", 0, 0x100)], [("_sceXy", 1)])) + assert result.returncode != 0 and "import NID longer than original name" in result.stdout + result.stderr, (result.stdout, result.stderr) + + +def main(): + patcher = Path(sys.argv[1]).resolve() + with tempfile.TemporaryDirectory(prefix="anyps5-macho-nid-") as directory: + check_shrinking(patcher, directory) + check_growing(patcher, directory) + check_preserved(patcher, directory) + check_refusals(patcher, directory) + print("NID patcher Mach-O tests passed") + + +if __name__ == "__main__": + main() diff --git a/core/libs/prx/libSceAgc/Shader/include/ShaderConstants.hpp b/core/libs/prx/libSceAgc/Shader/include/ShaderConstants.hpp index 78a080587..14c803b8f 100644 --- a/core/libs/prx/libSceAgc/Shader/include/ShaderConstants.hpp +++ b/core/libs/prx/libSceAgc/Shader/include/ShaderConstants.hpp @@ -78,7 +78,7 @@ enum class PrimitiveType : std::uint32_t { }; constexpr std::uint32_t VGT_SHADER_STAGES_GS_BIT = 0x20u; -constexpr std::uint32_t VGT_SHADER_STAGES_NGG_BIT = 0x04u; +constexpr std::uint32_t VGT_SHADER_STAGES_HS_BIT = 0x04u; enum class AgcDirectResourceType : std::uint32_t { GdsCounterRange = 0, diff --git a/core/libs/prx/libSceAgc/Shader/src/InterpolantMapping.cpp b/core/libs/prx/libSceAgc/Shader/src/InterpolantMapping.cpp index 3e70bd95b..b3b0da8f4 100644 --- a/core/libs/prx/libSceAgc/Shader/src/InterpolantMapping.cpp +++ b/core/libs/prx/libSceAgc/Shader/src/InterpolantMapping.cpp @@ -110,8 +110,7 @@ int CreateInterpolantMapping(const char* fn, ShaderRegister* regs, const Shader* extern "C" { -APS5_EXPORT("HV4j+E0MBHE", sceAgcCreateInterpolantMapping); -int APS5_VABI sceAgcCreateInterpolantMapping(ShaderRegister* regs, const Shader* gs, const Shader* ps) { +int APS5_VABI sceAgcCreateInterpolantMapping_0100(ShaderRegister* regs, const Shader* gs, const Shader* ps) { return CreateInterpolantMapping(__func__, regs, gs, ps, CreateInterpolantValue); } diff --git a/core/libs/prx/libSceAgc/Shader/src/LinkShaders.cpp b/core/libs/prx/libSceAgc/Shader/src/LinkShaders.cpp index b58f19b40..1044b410f 100644 --- a/core/libs/prx/libSceAgc/Shader/src/LinkShaders.cpp +++ b/core/libs/prx/libSceAgc/Shader/src/LinkShaders.cpp @@ -7,7 +7,7 @@ #include extern "C" int APS5_VABI sceAgcCreatePrimState(ShaderRegister*, ShaderRegister*, const Shader*, const Shader*, std::uint32_t); -extern "C" int APS5_VABI sceAgcCreateInterpolantMapping(ShaderRegister*, const Shader*, const Shader*); +extern "C" int APS5_VABI sceAgcCreateInterpolantMapping_0100(ShaderRegister*, const Shader*, const Shader*); extern "C" int APS5_VABI sceAgcLinkShaders(ShaderRegister* context, ShaderRegister* primitive, const void* reserved, const Shader* vertex, const Shader* pixel, std::uint32_t primitiveType) { @@ -26,7 +26,11 @@ extern "C" int APS5_VABI sceAgcLinkShaders(ShaderRegister* context, ShaderRegist std::array contextValues{}; std::array primitiveValues{}; sceAgcCreatePrimState(contextValues.data(), primitiveValues.data(), nullptr, vertex, primitiveType); - sceAgcCreateInterpolantMapping(contextValues.data() + 2, vertex, pixel); + sceAgcCreateInterpolantMapping_0100(contextValues.data() + 2, vertex, pixel); + if (pixel != nullptr) { + const std::array stages{vertex, pixel}; + AgcDriverResolveGraphicsStagesAbi_nid_postfix(stages, contextValues, primitiveValues); + } AgcDriverResolveGraphicsAbi_nid_postfix(vertex, pixel, primitiveType); transaction.Commit(); std::copy(contextValues.begin(), contextValues.end(), context); diff --git a/core/libs/prx/libSceAgc/Shader/src/PrimState.cpp b/core/libs/prx/libSceAgc/Shader/src/PrimState.cpp index 8d3564c76..9963ae2db 100644 --- a/core/libs/prx/libSceAgc/Shader/src/PrimState.cpp +++ b/core/libs/prx/libSceAgc/Shader/src/PrimState.cpp @@ -25,31 +25,38 @@ int APS5_VABI sceAgcCreatePrimState(ShaderRegister* cx_regs, ShaderRegister* uc_ throw std::runtime_error(std::string(__func__) + ": invalid shader type or missing special registers"); } (void)GraphicsPrimTypeToGsOut(prim_type); - const auto valid = [](const Shader* shader) { return shader->specials->vgt_shader_stages_en.offset == ShaderRegs::VGT_SHADER_STAGES_EN && shader->specials->vgt_gs_out_prim_type.offset == ShaderRegs::VGT_GS_OUT_PRIM_TYPE; }; - if (!valid(gs) || (hs != nullptr && !valid(hs))) { + const auto validStages = [](const Shader* shader) { return shader->specials->vgt_shader_stages_en.offset == ShaderRegs::VGT_SHADER_STAGES_EN; }; + if (!validStages(gs) || (hs != nullptr && !validStages(hs))) { throw std::runtime_error(std::string(__func__) + ": invalid context register offsets"); } if (gs->specials->ge_cntl.offset != ShaderRegs::GE_CNTL || gs->specials->ge_user_vgpr_en.offset != ShaderRegs::GE_USER_VGPR_EN || (hs != nullptr && hs->specials->ge_user_vgpr_en.offset != ShaderRegs::GE_USER_VGPR_EN)) { throw std::runtime_error(std::string(__func__) + ": invalid user configuration register offsets"); } std::array contextValues{gs->specials->vgt_shader_stages_en, gs->specials->vgt_gs_out_prim_type}; - if ((contextValues[0].value & ShaderRegs::VGT_SHADER_STAGES_GS_BIT) == 0) { - contextValues[1] = {ShaderRegs::VGT_GS_OUT_PRIM_TYPE, GraphicsPrimTypeToGsOut(prim_type)}; - } if (hs != nullptr) { contextValues[0].value |= hs->specials->vgt_shader_stages_en.value; - if ((contextValues[0].value & ShaderRegs::VGT_SHADER_STAGES_GS_BIT) == 0) contextValues[1] = hs->specials->vgt_gs_out_prim_type; + } + if ((contextValues[0].value & ShaderRegs::VGT_SHADER_STAGES_GS_BIT) != 0) { + if (gs->specials->vgt_gs_out_prim_type.offset != ShaderRegs::VGT_GS_OUT_PRIM_TYPE) throw std::runtime_error(std::string(__func__) + ": invalid context register offsets"); + } else if (hs != nullptr) { + if (hs->specials->vgt_gs_out_prim_type.offset != ShaderRegs::VGT_GS_OUT_PRIM_TYPE) throw std::runtime_error(std::string(__func__) + ": invalid context register offsets"); + contextValues[1] = hs->specials->vgt_gs_out_prim_type; + } else { + contextValues[1] = {ShaderRegs::VGT_GS_OUT_PRIM_TYPE, GraphicsPrimTypeToGsOut(prim_type)}; } std::array primitiveValues{gs->specials->ge_cntl, hs != nullptr ? hs->specials->ge_user_vgpr_en : gs->specials->ge_user_vgpr_en, ShaderRegister{ShaderRegs::VGT_PRIMITIVE_TYPE, prim_type}}; const std::array stages{hs, gs}; - AgcDriverResolveGraphicsStagesAbi_nid_postfix(std::span(stages).subspan(hs == nullptr ? 1u : 0u), contextValues, primitiveValues); + const bool patch = prim_type == static_cast(ShaderRegs::PrimitiveType::Patch); + if (patch == ((contextValues[0].value & ShaderRegs::VGT_SHADER_STAGES_HS_BIT) != 0)) { + AgcDriverResolveGraphicsStagesAbi_nid_postfix(std::span(stages).subspan(hs == nullptr ? 1u : 0u), contextValues, primitiveValues); + } if (cx_regs != nullptr) std::copy(contextValues.begin(), contextValues.end(), cx_regs); if (uc_regs != nullptr) std::copy(primitiveValues.begin(), primitiveValues.end(), uc_regs); return 0; } int APS5_VABI sceAgcUpdatePrimState(ShaderRegister* cx_regs, ShaderRegister* uc_regs, std::uint32_t prim_type) { - if (cx_regs != nullptr && (cx_regs[0].value & (ShaderRegs::VGT_SHADER_STAGES_GS_BIT | ShaderRegs::VGT_SHADER_STAGES_NGG_BIT)) == 0) { + if (cx_regs != nullptr && (cx_regs[0].value & (ShaderRegs::VGT_SHADER_STAGES_GS_BIT | ShaderRegs::VGT_SHADER_STAGES_HS_BIT)) == 0) { cx_regs[1].value &= ~0x7u; cx_regs[1].value |= GraphicsPrimTypeToGsOut(prim_type); } diff --git a/core/libs/prx/libSceAgc/tests/InterpolantMapping.cpp b/core/libs/prx/libSceAgc/tests/InterpolantMapping.cpp index 95e97ae37..1ea1d6ca2 100644 --- a/core/libs/prx/libSceAgc/tests/InterpolantMapping.cpp +++ b/core/libs/prx/libSceAgc/tests/InterpolantMapping.cpp @@ -11,7 +11,7 @@ #include #include -extern "C" int APS5_VABI sceAgcCreateInterpolantMapping(ShaderRegister* regs, const Shader* gs, const Shader* ps); +extern "C" int APS5_VABI sceAgcCreateInterpolantMapping_0100(ShaderRegister* regs, const Shader* gs, const Shader* ps); extern "C" int APS5_VABI sceAgcUnknownCreateInterpolantMapping(ShaderRegister* regs, const Shader* gs, const Shader* ps); namespace { @@ -100,7 +100,7 @@ void testMapping() { checkIdentity(split, static_cast(inputs.size()), "unused interpolants are not the identity"); auto regular = filled(); - check(sceAgcCreateInterpolantMapping(regular.data(), &gs, &ps) == 0, "mapping failed"); + check(sceAgcCreateInterpolantMapping_0100(regular.data(), &gs, &ps) == 0, "mapping failed"); check(std::memcmp(regular.data(), preparedMapping.data(), sizeof(regular)) == 0, "prepared mapping differs from published registers"); check(regular[3].value == split[3].value && regular[4].value == split[4].value, "mappings differ outside the high f16 half mode"); checkIdentity(regular, static_cast(inputs.size()), "unused interpolants are not the identity"); @@ -115,7 +115,7 @@ void testRejections() { std::array regs{}; std::memset(regs.data(), 0xcc, sizeof(regs)); const auto saved = regs; - expectFailure([&] { sceAgcCreateInterpolantMapping(regs.data(), &gs, &ps); }); + expectFailure([&] { sceAgcCreateInterpolantMapping_0100(regs.data(), &gs, &ps); }); expectFailure([&] { sceAgcUnknownCreateInterpolantMapping(regs.data(), &gs, &ps); }); check(std::memcmp(regs.data(), saved.data(), sizeof(regs)) == 0, "rejected mapping wrote registers"); ps.num_input_semantics = 1; diff --git a/core/libs/prx/libSceAgc/tests/ShaderHelpers.cpp b/core/libs/prx/libSceAgc/tests/ShaderHelpers.cpp index ab1541ded..c02780707 100644 --- a/core/libs/prx/libSceAgc/tests/ShaderHelpers.cpp +++ b/core/libs/prx/libSceAgc/tests/ShaderHelpers.cpp @@ -11,12 +11,15 @@ extern "C" int APS5_VABI sceAgcLinkShaders(ShaderRegister*, ShaderRegister*, const void*, const Shader*, const Shader*, std::uint32_t); extern "C" int APS5_VABI sceAgcCreatePrimState(ShaderRegister*, ShaderRegister*, const Shader*, const Shader*, std::uint32_t); -extern "C" int APS5_VABI sceAgcCreateInterpolantMapping(ShaderRegister*, const Shader*, const Shader*); +extern "C" int APS5_VABI sceAgcCreateInterpolantMapping_0100(ShaderRegister*, const Shader*, const Shader*); namespace { std::array preparedContext; std::array preparedPrimitive; std::size_t stageCount = 0; +std::array linkedStages{}; +std::size_t linkedStageCount = 0; +std::size_t linkedContextCount = 0; unsigned preparations = 0; unsigned mappings = 0; unsigned links = 0; @@ -26,13 +29,39 @@ const Shader* mappedPixel = nullptr; void Require(bool condition) { if (!condition) throw std::runtime_error("shader helper preparation mismatch"); } + +void OptionalGeometryOutputRegister(ShaderSpecialRegs& special, const Shader& vertex, const Shader& hull) { + using namespace ShaderRegs; + special.vgt_gs_out_prim_type = {}; + std::array noGeometryContext{}; + Require(sceAgcCreatePrimState(noGeometryContext.data(), nullptr, nullptr, &vertex, 7) == 0); + Require(noGeometryContext[0].offset == VGT_SHADER_STAGES_EN && noGeometryContext[1].offset == VGT_GS_OUT_PRIM_TYPE && noGeometryContext[1].value == static_cast(GsOutputPrimitiveType::Rectangle2D)); + bool rejectedMissingHullOutput = false; + try { static_cast(sceAgcCreatePrimState(noGeometryContext.data(), nullptr, &hull, &vertex, 7)); } + catch (const std::runtime_error&) { rejectedMissingHullOutput = true; } + Require(rejectedMissingHullOutput); + special.vgt_shader_stages_en.value = VGT_SHADER_STAGES_GS_BIT; + bool rejectedMissingGeometryOutput = false; + try { static_cast(sceAgcCreatePrimState(noGeometryContext.data(), nullptr, nullptr, &vertex, 7)); } + catch (const std::runtime_error&) { rejectedMissingGeometryOutput = true; } + Require(rejectedMissingGeometryOutput); + special.vgt_shader_stages_en.value = 0; + special.vgt_gs_out_prim_type = {VGT_GS_OUT_PRIM_TYPE, 2}; +} } extern "C" void AgcDriverResolveGraphicsStagesAbi_nid_postfix(std::span stages, std::span context, std::span primitive) { - Require(context.size() == 2 && primitive.size() == 3); - std::copy(context.begin(), context.end(), preparedContext.begin()); - std::copy(primitive.begin(), primitive.end(), preparedPrimitive.begin()); - stageCount = stages.size(); + Require((context.size() == 2 || context.size() == 34) && primitive.size() == 3); + if (context.size() == 2) { + std::copy(context.begin(), context.end(), preparedContext.begin()); + std::copy(primitive.begin(), primitive.end(), preparedPrimitive.begin()); + stageCount = stages.size(); + } else { + Require(stages.size() == 2); + std::copy(stages.begin(), stages.end(), linkedStages.begin()); + linkedStageCount = stages.size(); + linkedContextCount = context.size(); + } ++preparations; } @@ -79,7 +108,7 @@ int main() { context.back().value = primitive.back().value = 0xdeadbeef; const auto prepare = [&] { Require(sceAgcCreatePrimState(outputs & 1 ? context.data() : nullptr, outputs & 2 ? primitive.data() : nullptr, hs, &vertex, 7) == 0); }; if (primitiveFirst) prepare(); - Require(sceAgcCreateInterpolantMapping(interpolants.data(), &vertex, &pixel) == 0); + Require(sceAgcCreateInterpolantMapping_0100(interpolants.data(), &vertex, &pixel) == 0); if (!primitiveFirst) prepare(); Require(preparations == 1 && mappings == 1 && links == 1 && stageCount == (hs ? 2u : 1u)); Require(preparedContext[0].offset == VGT_SHADER_STAGES_EN && preparedContext[1].offset == VGT_GS_OUT_PRIM_TYPE); @@ -91,6 +120,19 @@ int main() { } } } + preparations = 0; + std::array patchContext{}; + std::array patchPrimitive{}; + Require(sceAgcCreatePrimState(patchContext.data(), patchPrimitive.data(), nullptr, &vertex, 9) == 0); + Require(preparations == 0 && patchContext[0].offset == VGT_SHADER_STAGES_EN && patchContext[0].value == 0 && patchPrimitive[2].offset == VGT_PRIMITIVE_TYPE && patchPrimitive[2].value == 9); + ShaderSpecialRegs hullSpecial = special; + hullSpecial.vgt_shader_stages_en.value = VGT_SHADER_STAGES_HS_BIT; + hull.specials = &hullSpecial; + Require(sceAgcCreatePrimState(patchContext.data(), patchPrimitive.data(), &hull, &vertex, 4) == 0); + Require(preparations == 0 && patchContext[0].value == VGT_SHADER_STAGES_HS_BIT && patchPrimitive[2].value == 4); + Require(sceAgcCreatePrimState(patchContext.data(), patchPrimitive.data(), &hull, &vertex, 9) == 0); + Require(preparations == 1 && stageCount == 2 && preparedContext[0].value == VGT_SHADER_STAGES_HS_BIT && preparedPrimitive[2].value == 9 && patchPrimitive[2].value == 9); + hull.specials = &special; std::array linkedContext{}; std::array linkedPrimitive{}; for (auto& value : linkedContext) value.value = 0xdeadbeef; @@ -104,23 +146,28 @@ int main() { for (const auto value : linkedContext) Require(value.value == 0xdeadbeef); for (const auto value : linkedPrimitive) Require(value.value == 0xdeadbeef); rejected = false; - try { sceAgcCreateInterpolantMapping(interpolants.data(), &vertex, &pixel); } + try { sceAgcCreateInterpolantMapping_0100(interpolants.data(), &vertex, &pixel); } catch (const std::runtime_error&) { rejected = true; } Require(rejected); for (const auto value : interpolants) Require(value.value == 0xdeadbeef); rejectLink = false; pixel.num_input_semantics = 33; rejected = false; - try { sceAgcCreateInterpolantMapping(interpolants.data(), &vertex, &pixel); } + try { sceAgcCreateInterpolantMapping_0100(interpolants.data(), &vertex, &pixel); } catch (const std::runtime_error&) { rejected = true; } Require(rejected); for (const auto value : interpolants) Require(value.value == 0xdeadbeef); pixel.num_input_semantics = 1; + preparations = mappings = links = 0; + linkedStageCount = linkedContextCount = 0; Require(sceAgcLinkShaders(linkedContext.data(), linkedPrimitive.data(), nullptr, &vertex, &pixel, 7) == 0); + Require(preparations == 2 && linkedStageCount == 2 && linkedContextCount == linkedContext.size()); + Require(linkedStages[0] == &vertex && linkedStages[1] == &pixel); preparations = mappings = links = 0; Require(sceAgcCreatePrimState(nullptr, nullptr, nullptr, nullptr, 7) == 0); - Require(sceAgcCreateInterpolantMapping(interpolants.data(), &vertex, nullptr) == 0); + Require(sceAgcCreateInterpolantMapping_0100(interpolants.data(), &vertex, nullptr) == 0); Require(preparations == 0 && mappings == 0 && links == 0); + OptionalGeometryOutputRegister(special, vertex, hull); std::cout << "shader helper preparation tests passed\n"; } catch (const std::exception& error) { std::cerr << error.what() << '\n'; diff --git a/core/libs/prx/libSceAgcDriver/CMakeLists.txt b/core/libs/prx/libSceAgcDriver/CMakeLists.txt index 1d8e5eacf..d24c90ab0 100644 --- a/core/libs/prx/libSceAgcDriver/CMakeLists.txt +++ b/core/libs/prx/libSceAgcDriver/CMakeLists.txt @@ -45,6 +45,17 @@ add_custom_command( DEPENDS "${agcTextureDetileShaderSource}" glslang-standalone VERBATIM ) +set(agcTextureDetileImageShaderSpv ${agcTextureDetileGeneratedDir}/TextureDetileImage.spv) +set(agcTextureDetileImageShaderHeader ${agcTextureDetileGeneratedDir}/TextureDetileImage_spv.h) +add_custom_command( + OUTPUT "${agcTextureDetileImageShaderHeader}" + COMMAND ${CMAKE_COMMAND} -E make_directory "${agcTextureDetileGeneratedDir}" + COMMAND "$" -V --target-env vulkan1.1 -DLINEAR_IMAGE=1 -o "${agcTextureDetileImageShaderSpv}" "${agcTextureDetileShaderSource}" + COMMAND ${CMAKE_COMMAND} -DINPUT=${agcTextureDetileImageShaderSpv} -DOUTPUT=${agcTextureDetileImageShaderHeader} + -DSYMBOL=TEXTURE_DETILE_IMAGE_SPV -P "${CMAKE_CURRENT_SOURCE_DIR}/embed_spirv.cmake" + DEPENDS "${agcTextureDetileShaderSource}" glslang-standalone + VERBATIM +) set(agcMeshArgumentsShaderSource ${CMAKE_CURRENT_SOURCE_DIR}/Graphics/shaders/MeshArguments.comp) set(agcMeshArgumentsShaderSpv ${agcTextureDetileGeneratedDir}/MeshArguments.spv) @@ -59,6 +70,19 @@ add_custom_command( VERBATIM ) +set(agcCmaskClearShaderSource ${CMAKE_CURRENT_SOURCE_DIR}/Graphics/shaders/CmaskClear.comp) +set(agcCmaskClearShaderSpv ${agcTextureDetileGeneratedDir}/CmaskClear.spv) +set(agcCmaskClearShaderHeader ${agcTextureDetileGeneratedDir}/CmaskClear_spv.h) +add_custom_command( + OUTPUT "${agcCmaskClearShaderHeader}" + COMMAND ${CMAKE_COMMAND} -E make_directory "${agcTextureDetileGeneratedDir}" + COMMAND "$" -V --target-env vulkan1.1 -o "${agcCmaskClearShaderSpv}" "${agcCmaskClearShaderSource}" + COMMAND ${CMAKE_COMMAND} -DINPUT=${agcCmaskClearShaderSpv} -DOUTPUT=${agcCmaskClearShaderHeader} + -DSYMBOL=CMASK_CLEAR_SPV -P "${CMAKE_CURRENT_SOURCE_DIR}/embed_spirv.cmake" + DEPENDS "${agcCmaskClearShaderSource}" glslang-standalone + VERBATIM +) + set(agcSampleCounterShaderSource ${CMAKE_CURRENT_SOURCE_DIR}/Graphics/shaders/SampleCounter.comp) set(agcSampleCounterShaderSpv ${agcTextureDetileGeneratedDir}/SampleCounter.spv) set(agcSampleCounterShaderHeader ${agcTextureDetileGeneratedDir}/SampleCounter_spv.h) @@ -72,6 +96,19 @@ add_custom_command( VERBATIM ) +set(agcSampleDumpsShaderSource ${CMAKE_CURRENT_SOURCE_DIR}/Graphics/shaders/SampleDumps.comp) +set(agcSampleDumpsShaderSpv ${agcTextureDetileGeneratedDir}/SampleDumps.spv) +set(agcSampleDumpsShaderHeader ${agcTextureDetileGeneratedDir}/SampleDumps_spv.h) +add_custom_command( + OUTPUT "${agcSampleDumpsShaderHeader}" + COMMAND ${CMAKE_COMMAND} -E make_directory "${agcTextureDetileGeneratedDir}" + COMMAND "$" -V --target-env vulkan1.1 -o "${agcSampleDumpsShaderSpv}" "${agcSampleDumpsShaderSource}" + COMMAND ${CMAKE_COMMAND} -DINPUT=${agcSampleDumpsShaderSpv} -DOUTPUT=${agcSampleDumpsShaderHeader} + -DSYMBOL=SAMPLE_DUMPS_SPV -P "${CMAKE_CURRENT_SOURCE_DIR}/embed_spirv.cmake" + DEPENDS "${agcSampleDumpsShaderSource}" glslang-standalone + VERBATIM +) + set(agcDriverSources ${CMAKE_CURRENT_SOURCE_DIR}/Unimplemented.cpp ${CMAKE_CURRENT_SOURCE_DIR}/Execution/src/DisplayBuffer.cpp @@ -80,6 +117,7 @@ set(agcDriverSources ${CMAKE_CURRENT_SOURCE_DIR}/Execution/src/Driver/Diagnostics.cpp ${CMAKE_CURRENT_SOURCE_DIR}/Execution/src/Driver/Dispatch/DataHitVerification.cpp ${CMAKE_CURRENT_SOURCE_DIR}/Execution/src/Driver/Dispatch/Dispatch.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/Execution/src/Driver/Dispatch/DispatchAhead.cpp ${CMAKE_CURRENT_SOURCE_DIR}/Execution/src/Driver/Dispatch/DispatchCache.cpp ${CMAKE_CURRENT_SOURCE_DIR}/Execution/src/Driver/Dispatch/DispatchCacheStatistics.cpp ${CMAKE_CURRENT_SOURCE_DIR}/Execution/src/Driver/Dispatch/DispatchInsert.cpp @@ -137,9 +175,11 @@ set(agcDriverSources ${CMAKE_CURRENT_SOURCE_DIR}/Graphics/src/ColorTargetLayout.cpp ${CMAKE_CURRENT_SOURCE_DIR}/Graphics/src/ColorTargetTransfer.cpp ${CMAKE_CURRENT_SOURCE_DIR}/Graphics/src/Pipeline.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/Graphics/src/PipelineLibrary.cpp ${CMAKE_CURRENT_SOURCE_DIR}/Graphics/src/PipelineCache.cpp ${CMAKE_CURRENT_SOURCE_DIR}/Graphics/src/Resources.cpp ${CMAKE_CURRENT_SOURCE_DIR}/Graphics/src/Recorder.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/Graphics/src/ParallelCompare.cpp ${CMAKE_CURRENT_SOURCE_DIR}/Graphics/src/ShaderInputState.cpp ${CMAKE_CURRENT_SOURCE_DIR}/Graphics/src/ShaderResources.cpp ${CMAKE_CURRENT_SOURCE_DIR}/Graphics/src/ShaderAddressResources.cpp @@ -155,11 +195,15 @@ set(agcDriverSources ${CMAKE_CURRENT_SOURCE_DIR}/Graphics/src/TextureTiling.cpp ${CMAKE_CURRENT_SOURCE_DIR}/Graphics/src/TextureDetiler.cpp ${CMAKE_CURRENT_SOURCE_DIR}/Graphics/src/Texture.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/Graphics/src/PassHazards.cpp ${CMAKE_CURRENT_SOURCE_DIR}/Graphics/src/UnitShadow.cpp ${CMAKE_CURRENT_SOURCE_DIR}/Graphics/src/Sampler.cpp ${agcTextureDetileShaderHeader} + ${agcCmaskClearShaderHeader} ${agcMeshArgumentsShaderHeader} + ${agcTextureDetileImageShaderHeader} ${agcSampleCounterShaderHeader} + ${agcSampleDumpsShaderHeader} ${CMAKE_CURRENT_SOURCE_DIR}/Eq/src/Event.cpp ${CMAKE_CURRENT_SOURCE_DIR}/Eq/src/Query.cpp ${CMAKE_CURRENT_SOURCE_DIR}/Resource/src/Lifecycle.cpp @@ -178,6 +222,7 @@ add_library(libSceAgcDriver SHARED EXCLUDE_FROM_ALL target_include_directories(libSceAgcDriver PRIVATE ${LIBS_INCLUDE_DIR} ${agcTextureDetileGeneratedRoot} + ${CMAKE_SOURCE_DIR}/core/shader/recompiler/RdnaDecoder/include ${CMAKE_SOURCE_DIR}/3rdparty/Vulkan-Headers/include ${CMAKE_SOURCE_DIR}/3rdparty/SPIRV-Headers/include ) @@ -264,6 +309,12 @@ target_link_libraries(agc_shader_memory_tests PRIVATE shader_recompiler_agc_driv set_target_properties(agc_shader_memory_tests PROPERTIES CXX_EXTENSIONS OFF) configure_windows_unwind(agc_shader_memory_tests) +add_test_executable(agc_shared_write_tests EXCLUDE_FROM_ALL tests/SharedWrites.cpp) +target_include_directories(agc_shared_write_tests PRIVATE ${LIBS_INCLUDE_DIR}) +target_link_libraries(agc_shared_write_tests PRIVATE libSceAgcDriver libkernel libc) +set_target_properties(agc_shared_write_tests PROPERTIES CXX_EXTENSIONS OFF) +configure_windows_unwind(agc_shared_write_tests) + add_test_executable(agc_write_tracking_tests EXCLUDE_FROM_ALL tests/WriteTracking.cpp Execution/src/GuestMemory.cpp @@ -287,6 +338,15 @@ target_link_libraries(agc_structurizer_tests PRIVATE shader_recompiler_agc_drive set_target_properties(agc_structurizer_tests PROPERTIES CXX_EXTENSIONS OFF) configure_windows_unwind(agc_structurizer_tests) +add_test_executable(agc_fragment_barycentric_tests EXCLUDE_FROM_ALL tests/FragmentBarycentric.cpp) +target_include_directories(agc_fragment_barycentric_tests PRIVATE ${LIBS_INCLUDE_DIR} ${CMAKE_SOURCE_DIR}/3rdparty/Vulkan-Headers/include) +set_target_properties(agc_fragment_barycentric_tests PROPERTIES CXX_EXTENSIONS OFF) +configure_windows_unwind(agc_fragment_barycentric_tests) +if(BUILD_TESTING) + set_target_properties(agc_fragment_barycentric_tests PROPERTIES EXCLUDE_FROM_ALL OFF) + add_test(NAME agc_fragment_barycentric COMMAND agc_fragment_barycentric_tests) +endif() + add_test_executable(agc_subgroup_clock_tests EXCLUDE_FROM_ALL tests/SubgroupClock.cpp Execution/src/SubgroupClock.cpp @@ -299,6 +359,23 @@ if(BUILD_TESTING) add_test(NAME agc_subgroup_clock COMMAND agc_subgroup_clock_tests) endif() +add_test_executable(agc_tessellation_factor_ring_tests EXCLUDE_FROM_ALL + tests/TessellationFactorRing.cpp + State/src/Configuration.cpp +) +target_include_directories(agc_tessellation_factor_ring_tests PRIVATE ${LIBS_INCLUDE_DIR}) +target_link_libraries(agc_tessellation_factor_ring_tests PRIVATE libc) +set_target_properties(agc_tessellation_factor_ring_tests PROPERTIES CXX_EXTENSIONS OFF) +configure_windows_unwind(agc_tessellation_factor_ring_tests) +if(BUILD_TESTING) + set_target_properties(agc_tessellation_factor_ring_tests PROPERTIES EXCLUDE_FROM_ALL OFF) + add_test(NAME agc_tessellation_factor_ring COMMAND agc_tessellation_factor_ring_tests) + if(WIN32) + set_tests_properties(agc_tessellation_factor_ring PROPERTIES ENVIRONMENT_MODIFICATION + "PATH=path_list_prepend:$") + endif() +endif() + add_test_executable(agc_spirv_module_tests EXCLUDE_FROM_ALL tests/SpirvModule.cpp) target_link_libraries(agc_spirv_module_tests PRIVATE shader_recompiler_agc_driver libc) set_target_properties(agc_spirv_module_tests PROPERTIES CXX_EXTENSIONS OFF) @@ -316,6 +393,19 @@ if(ANYPS5_ENABLE_SPIRV_TOOLS) configure_windows_unwind(agc_spirv_optimizer_tests) endif() +add_test_executable(agc_float16_conversion_tests EXCLUDE_FROM_ALL tests/Float16Conversion.cpp) +target_link_libraries(agc_float16_conversion_tests PRIVATE shader_recompiler_agc_driver libc) +set_target_properties(agc_float16_conversion_tests PROPERTIES CXX_EXTENSIONS OFF) +configure_windows_unwind(agc_float16_conversion_tests) +if(BUILD_TESTING) + set_target_properties(agc_float16_conversion_tests PROPERTIES EXCLUDE_FROM_ALL OFF) + add_test(NAME agc_float16_conversion COMMAND agc_float16_conversion_tests) + set_tests_properties(agc_float16_conversion PROPERTIES ENVIRONMENT "ANYPS5_NO_SHADER_CACHE=1") + if(WIN32) + set_tests_properties(agc_float16_conversion PROPERTIES ENVIRONMENT_MODIFICATION "PATH=path_list_prepend:$") + endif() +endif() + add_test_executable(agc_dpp_write_tests EXCLUDE_FROM_ALL tests/DppWrite.cpp) target_link_libraries(agc_dpp_write_tests PRIVATE shader_recompiler_agc_driver libc) set_target_properties(agc_dpp_write_tests PROPERTIES CXX_EXTENSIONS OFF) @@ -326,6 +416,16 @@ target_link_libraries(agc_masked_select_tests PRIVATE shader_recompiler_agc_driv set_target_properties(agc_masked_select_tests PROPERTIES CXX_EXTENSIONS OFF) configure_windows_unwind(agc_masked_select_tests) +add_test_executable(agc_constant_bit_test_tests EXCLUDE_FROM_ALL tests/ConstantBitTest.cpp) +target_link_libraries(agc_constant_bit_test_tests PRIVATE shader_recompiler_agc_driver libc) +set_target_properties(agc_constant_bit_test_tests PROPERTIES CXX_EXTENSIONS OFF) +configure_windows_unwind(agc_constant_bit_test_tests) + +add_test_executable(agc_scalar_select_mask_tests EXCLUDE_FROM_ALL tests/ScalarSelectMask.cpp) +target_link_libraries(agc_scalar_select_mask_tests PRIVATE shader_recompiler_agc_driver libc) +set_target_properties(agc_scalar_select_mask_tests PROPERTIES CXX_EXTENSIONS OFF) +configure_windows_unwind(agc_scalar_select_mask_tests) + add_test_executable(agc_wave_control_decode_tests EXCLUDE_FROM_ALL tests/WaveControlDecode.cpp) target_include_directories(agc_wave_control_decode_tests PRIVATE ${LIBS_INCLUDE_DIR}) target_link_libraries(agc_wave_control_decode_tests PRIVATE shader_recompiler_agc_driver libc) @@ -340,10 +440,41 @@ if(BUILD_TESTING) endif() endif() +add_test_executable(agc_denormal_flush_elimination_tests EXCLUDE_FROM_ALL tests/DenormalFlushElimination.cpp) +target_link_libraries(agc_denormal_flush_elimination_tests PRIVATE shader_recompiler_agc_driver libc) +set_target_properties(agc_denormal_flush_elimination_tests PROPERTIES CXX_EXTENSIONS OFF) +configure_windows_unwind(agc_denormal_flush_elimination_tests) +if(BUILD_TESTING) + set_target_properties(agc_denormal_flush_elimination_tests PROPERTIES EXCLUDE_FROM_ALL OFF) + add_test(NAME agc_denormal_flush_elimination COMMAND agc_denormal_flush_elimination_tests) + if(WIN32) + set_tests_properties(agc_denormal_flush_elimination PROPERTIES ENVIRONMENT_MODIFICATION + "PATH=path_list_prepend:$") + endif() +endif() + add_test_executable(agc_vop1_control_decode_tests EXCLUDE_FROM_ALL tests/Vop1ControlDecode.cpp) target_link_libraries(agc_vop1_control_decode_tests PRIVATE shader_recompiler_agc_driver libc) set_target_properties(agc_vop1_control_decode_tests PROPERTIES CXX_EXTENSIONS OFF) configure_windows_unwind(agc_vop1_control_decode_tests) +add_test_executable(agc_vop2_encoding_decode_tests EXCLUDE_FROM_ALL tests/Vop2EncodingDecode.cpp) +target_link_libraries(agc_vop2_encoding_decode_tests PRIVATE shader_recompiler_agc_driver libc) +set_target_properties(agc_vop2_encoding_decode_tests PROPERTIES CXX_EXTENSIONS OFF) +configure_windows_unwind(agc_vop2_encoding_decode_tests) +if(BUILD_TESTING) + set_target_properties(agc_vop2_encoding_decode_tests PROPERTIES EXCLUDE_FROM_ALL OFF) + add_test(NAME agc_vop2_encoding_decode COMMAND agc_vop2_encoding_decode_tests) + if(WIN32) + set_tests_properties(agc_vop2_encoding_decode PROPERTIES ENVIRONMENT_MODIFICATION + "PATH=path_list_prepend:$") + endif() +endif() + +add_test_executable(agc_pass_hazards_tests EXCLUDE_FROM_ALL tests/PassHazards.cpp Graphics/src/PassHazards.cpp) +target_include_directories(agc_pass_hazards_tests PRIVATE ${LIBS_INCLUDE_DIR} ${CMAKE_SOURCE_DIR}/3rdparty/Vulkan-Headers/include) +target_link_libraries(agc_pass_hazards_tests PRIVATE libc) +set_target_properties(agc_pass_hazards_tests PROPERTIES CXX_EXTENSIONS OFF) +configure_windows_unwind(agc_pass_hazards_tests) add_test_executable(agc_dcc_key_sharing_tests EXCLUDE_FROM_ALL tests/DccKeySharing.cpp) target_include_directories(agc_dcc_key_sharing_tests PRIVATE ${LIBS_INCLUDE_DIR} ${CMAKE_SOURCE_DIR}/core/shader/recompiler ${CMAKE_SOURCE_DIR}/3rdparty/Vulkan-Headers/include) @@ -359,6 +490,20 @@ if(BUILD_TESTING) endif() endif() +add_test_executable(agc_buffer_pool_tests EXCLUDE_FROM_ALL tests/BufferPool.cpp) +target_include_directories(agc_buffer_pool_tests PRIVATE ${LIBS_INCLUDE_DIR} ${CMAKE_SOURCE_DIR}/core/shader/recompiler ${CMAKE_SOURCE_DIR}/3rdparty/Vulkan-Headers/include) +target_link_libraries(agc_buffer_pool_tests PRIVATE libSceAgcDriver libc) +set_target_properties(agc_buffer_pool_tests PROPERTIES CXX_EXTENSIONS OFF) +configure_windows_unwind(agc_buffer_pool_tests) +if(BUILD_TESTING) + set_target_properties(agc_buffer_pool_tests PROPERTIES EXCLUDE_FROM_ALL OFF) + add_test(NAME agc_buffer_pool COMMAND agc_buffer_pool_tests) + if(WIN32) + set_tests_properties(agc_buffer_pool PROPERTIES ENVIRONMENT_MODIFICATION + "PATH=path_list_prepend:$") + endif() +endif() + add_test_executable(agc_display_buffer_dcc_tests EXCLUDE_FROM_ALL tests/DisplayBufferDcc.cpp) target_include_directories(agc_display_buffer_dcc_tests PRIVATE ${LIBS_INCLUDE_DIR} ${CMAKE_SOURCE_DIR}/core/shader/recompiler ${CMAKE_SOURCE_DIR}/3rdparty/Vulkan-Headers/include) target_link_libraries(agc_display_buffer_dcc_tests PRIVATE libSceAgcDriver libc) @@ -378,6 +523,11 @@ target_link_libraries(agc_validate_program_tests PRIVATE shader_recompiler_agc_d set_target_properties(agc_validate_program_tests PROPERTIES CXX_EXTENSIONS OFF) configure_windows_unwind(agc_validate_program_tests) +add_test_executable(agc_formatted_buffer_once_tests EXCLUDE_FROM_ALL tests/FormattedBufferOnce.cpp) +target_link_libraries(agc_formatted_buffer_once_tests PRIVATE shader_recompiler_agc_driver libc) +set_target_properties(agc_formatted_buffer_once_tests PROPERTIES CXX_EXTENSIONS OFF) +configure_windows_unwind(agc_formatted_buffer_once_tests) + add_test_executable(agc_shader_disk_cache_tests EXCLUDE_FROM_ALL tests/ShaderDiskCache.cpp) target_include_directories(agc_shader_disk_cache_tests PRIVATE ${LIBS_INCLUDE_DIR}) target_link_libraries(agc_shader_disk_cache_tests PRIVATE shader_recompiler_agc_driver libc) @@ -398,6 +548,12 @@ target_link_libraries(agc_shader_control_flow_tests PRIVATE shader_recompiler_ag set_target_properties(agc_shader_control_flow_tests PROPERTIES CXX_EXTENSIONS OFF) configure_windows_unwind(agc_shader_control_flow_tests) +add_test_executable(agc_shader_mode_register_tests EXCLUDE_FROM_ALL tests/ShaderModeRegisters.cpp) +target_include_directories(agc_shader_mode_register_tests PRIVATE ${LIBS_INCLUDE_DIR}) +target_link_libraries(agc_shader_mode_register_tests PRIVATE shader_recompiler_agc_driver libc) +set_target_properties(agc_shader_mode_register_tests PROPERTIES CXX_EXTENSIONS OFF) +configure_windows_unwind(agc_shader_mode_register_tests) + add_test_executable(agc_driver_flip_tests EXCLUDE_FROM_ALL tests/Flip.cpp ) @@ -441,11 +597,66 @@ add_custom_command( VERBATIM ) +foreach(agcTriangleStage IN ITEMS vert frag) + set(agcTriangleSource ${CMAKE_CURRENT_SOURCE_DIR}/tests/shaders/Triangle.${agcTriangleStage}) + set(agcTriangleSpv ${agcTestShaderGeneratedRoot}/Triangle_${agcTriangleStage}.spv) + set(agcTriangleHeader ${agcTestShaderGeneratedRoot}/Triangle_${agcTriangleStage}_spv.h) + add_custom_command( + OUTPUT "${agcTriangleHeader}" + COMMAND ${CMAKE_COMMAND} -E make_directory "${agcTestShaderGeneratedRoot}" + COMMAND "$" -V --target-env vulkan1.1 -o "${agcTriangleSpv}" "${agcTriangleSource}" + COMMAND ${CMAKE_COMMAND} -DINPUT=${agcTriangleSpv} -DOUTPUT=${agcTriangleHeader} + -DSYMBOL=TRIANGLE_${agcTriangleStage}_SPV -P "${CMAKE_CURRENT_SOURCE_DIR}/embed_spirv.cmake" + DEPENDS "${agcTriangleSource}" glslang-standalone + VERBATIM + ) + list(APPEND agcTriangleHeaders "${agcTriangleHeader}") +endforeach() + +foreach(agcCoverStage IN ITEMS vert frag) + string(TOUPPER "${agcCoverStage}" agcCoverSymbol) + set(agcCoverShaderSpv ${agcTestShaderGeneratedRoot}/Cover_${agcCoverStage}.spv) + set(agcCoverShaderHeader ${agcTestShaderGeneratedRoot}/Cover_${agcCoverStage}_spv.h) + add_custom_command( + OUTPUT "${agcCoverShaderHeader}" + COMMAND ${CMAKE_COMMAND} -E make_directory "${agcTestShaderGeneratedRoot}" + COMMAND "$" -V --target-env vulkan1.1 -o "${agcCoverShaderSpv}" "${CMAKE_CURRENT_SOURCE_DIR}/tests/shaders/Cover.${agcCoverStage}" + COMMAND ${CMAKE_COMMAND} -DINPUT=${agcCoverShaderSpv} -DOUTPUT=${agcCoverShaderHeader} + -DSYMBOL=COVER_${agcCoverSymbol}_SPV -P "${CMAKE_CURRENT_SOURCE_DIR}/embed_spirv.cmake" + DEPENDS "${CMAKE_CURRENT_SOURCE_DIR}/tests/shaders/Cover.${agcCoverStage}" glslang-standalone + VERBATIM + ) + list(APPEND agcCoverShaderHeaders ${agcCoverShaderHeader}) +endforeach() + +add_test_executable(agc_driver_cmask_fast_clear_tests EXCLUDE_FROM_ALL + tests/CmaskFastClear.cpp + ${agcCoverShaderHeaders} +) + +target_include_directories(agc_driver_cmask_fast_clear_tests PRIVATE + ${LIBS_INCLUDE_DIR} + ${agcTestShaderGeneratedRoot} + ${CMAKE_SOURCE_DIR}/core/shader/recompiler + ${CMAKE_SOURCE_DIR}/3rdparty/Vulkan-Headers/include +) + +target_link_libraries(agc_driver_cmask_fast_clear_tests PRIVATE + libSceAgcDriver + SDL2-static + libc +) + +set_target_properties(agc_driver_cmask_fast_clear_tests PROPERTIES CXX_EXTENSIONS OFF) +configure_windows_unwind(agc_driver_cmask_fast_clear_tests) + add_test_executable(agc_driver_recorder_tests EXCLUDE_FROM_ALL tests/Recorder.cpp tests/ResidentPresent.cpp + ${agcTriangleHeaders} ${agcSampleLodShaderHeader} ${agcSampleArrayShaderHeader} + ${agcCoverShaderHeaders} ) target_include_directories(agc_driver_recorder_tests PRIVATE @@ -456,6 +667,7 @@ target_include_directories(agc_driver_recorder_tests PRIVATE ) target_link_libraries(agc_driver_recorder_tests PRIVATE + libkernel libSceAgcDriver SDL2-static libc @@ -463,6 +675,37 @@ target_link_libraries(agc_driver_recorder_tests PRIVATE set_target_properties(agc_driver_recorder_tests PROPERTIES CXX_EXTENSIONS OFF) configure_windows_unwind(agc_driver_recorder_tests) +if(BUILD_TESTING) + set_target_properties(agc_driver_recorder_tests PROPERTIES EXCLUDE_FROM_ALL OFF) + add_test(NAME agc_pipeline_cache COMMAND agc_driver_recorder_tests --pipeline-cache-only) + add_test(NAME agc_pipeline_library COMMAND agc_driver_recorder_tests --pipeline-library-only) + add_test(NAME agc_pipeline_library_off COMMAND agc_driver_recorder_tests --pipeline-library-only) + set_tests_properties(agc_pipeline_library agc_pipeline_library_off PROPERTIES SKIP_RETURN_CODE 77) + set_tests_properties(agc_pipeline_library_off PROPERTIES ENVIRONMENT "APS5_NO_GPL=1") + add_test(NAME agc_pipeline_library_fast_only COMMAND agc_driver_recorder_tests --pipeline-library-only) + set_tests_properties(agc_pipeline_library_fast_only PROPERTIES ENVIRONMENT "APS5_NO_PIPELINE_LTO=1" SKIP_RETURN_CODE 77) + if(WIN32) + set_tests_properties(agc_pipeline_library agc_pipeline_library_off agc_pipeline_library_fast_only PROPERTIES ENVIRONMENT_MODIFICATION + "PATH=path_list_prepend:$") + endif() + set_tests_properties(agc_pipeline_cache PROPERTIES SKIP_RETURN_CODE 77) + if(WIN32) + set_tests_properties(agc_pipeline_cache PROPERTIES ENVIRONMENT_MODIFICATION + "PATH=path_list_prepend:$") + endif() + add_test(NAME agc_unchanged_cpu_stamps COMMAND agc_driver_recorder_tests --unchanged-cpu-stamps-only) + set_tests_properties(agc_unchanged_cpu_stamps PROPERTIES SKIP_RETURN_CODE 77) + if(WIN32) + set_tests_properties(agc_unchanged_cpu_stamps PROPERTIES ENVIRONMENT_MODIFICATION + "PATH=path_list_prepend:$") + endif() + add_test(NAME agc_host_import_unmap COMMAND agc_driver_recorder_tests --host-import-unmap-only) + set_tests_properties(agc_host_import_unmap PROPERTIES SKIP_RETURN_CODE 77) + if(WIN32) + set_tests_properties(agc_host_import_unmap PROPERTIES ENVIRONMENT_MODIFICATION + "PATH=path_list_prepend:$") + endif() +endif() add_test_executable(agc_recorder_kept_bytes_tests EXCLUDE_FROM_ALL tests/RecorderKeptBytes.cpp) target_include_directories(agc_recorder_kept_bytes_tests PRIVATE ${LIBS_INCLUDE_DIR} ${CMAKE_SOURCE_DIR}/core/shader/recompiler ${CMAKE_SOURCE_DIR}/3rdparty/Vulkan-Headers/include) @@ -506,9 +749,11 @@ add_test_executable(agc_driver_graphics_tests EXCLUDE_FROM_ALL tests/GuestTextureResource.cpp tests/GuestSamplerResource.cpp tests/TextureDetiler.cpp + tests/DepthSurfaceReuse.cpp Graphics/src/Draw.cpp Graphics/src/DepthSurface.cpp Graphics/src/Pipeline.cpp + Graphics/src/PipelineLibrary.cpp Graphics/src/State.cpp Graphics/src/ColorTargetLayout.cpp Graphics/src/ColorTargetTransfer.cpp @@ -526,13 +771,18 @@ add_test_executable(agc_driver_graphics_tests EXCLUDE_FROM_ALL Graphics/src/TextureTiling.cpp Graphics/src/TextureDetiler.cpp Graphics/src/Texture.cpp + Graphics/src/PassHazards.cpp Graphics/src/UnitShadow.cpp Graphics/src/Sampler.cpp Graphics/src/Recorder.cpp + Graphics/src/ParallelCompare.cpp Graphics/src/ShaderInputState.cpp ${agcTextureDetileShaderHeader} + ${agcCmaskClearShaderHeader} ${agcMeshArgumentsShaderHeader} + ${agcTextureDetileImageShaderHeader} ${agcSampleCounterShaderHeader} + ${agcSampleDumpsShaderHeader} Execution/src/GuestMemory.cpp Execution/src/Pm4.cpp ) @@ -609,6 +859,28 @@ foreach(agcExecutionTestSource IN LISTS agcExecutionTestSources) list(APPEND agcExecutionTests ${agcExecutionTest}) endforeach() +target_include_directories(agc_driver_dual_source_blend_tests PRIVATE $) + +target_include_directories(agc_driver_packed_unorm_color_target_tests PRIVATE $) + +foreach(stage IN ITEMS vert frag) + set(agcProvokingShaderSource ${CMAKE_CURRENT_SOURCE_DIR}/tests/shaders/ProvokingVertex.${stage}) + set(agcProvokingShaderSpv ${agcTestShaderGeneratedRoot}/ProvokingVertex_${stage}.spv) + set(agcProvokingShaderHeader ${agcTestShaderGeneratedRoot}/ProvokingVertex_${stage}_spv.h) + string(TOUPPER "PROVOKING_VERTEX_${stage}_SPV" agcProvokingShaderSymbol) + add_custom_command( + OUTPUT "${agcProvokingShaderHeader}" + COMMAND ${CMAKE_COMMAND} -E make_directory "${agcTestShaderGeneratedRoot}" + COMMAND "$" -V --target-env vulkan1.1 -o "${agcProvokingShaderSpv}" "${agcProvokingShaderSource}" + COMMAND ${CMAKE_COMMAND} -DINPUT=${agcProvokingShaderSpv} -DOUTPUT=${agcProvokingShaderHeader} + -DSYMBOL=${agcProvokingShaderSymbol} -P "${CMAKE_CURRENT_SOURCE_DIR}/embed_spirv.cmake" + DEPENDS "${agcProvokingShaderSource}" glslang-standalone + VERBATIM + ) + target_sources(agc_driver_provoking_vertex_tests PRIVATE ${agcProvokingShaderHeader}) +endforeach() +target_include_directories(agc_driver_provoking_vertex_tests PRIVATE ${agcTestShaderGeneratedRoot}) + target_include_directories(agc_driver_prepared_exports_tests PRIVATE $) target_include_directories(agc_driver_prepared_publication_tests PRIVATE $) @@ -621,12 +893,22 @@ target_include_directories(agc_driver_prepared_shaders_tests PRIVATE ) if(BUILD_TESTING) + set_target_properties(agc_shader_mode_register_tests PROPERTIES EXCLUDE_FROM_ALL OFF) + add_test(NAME agc_shader_mode_register COMMAND agc_shader_mode_register_tests) + set_tests_properties(agc_shader_mode_register PROPERTIES ENVIRONMENT "ANYPS5_NO_SHADER_CACHE=1" WORKING_DIRECTORY "${CMAKE_SOURCE_DIR}" TIMEOUT 20) + if(WIN32) + set_tests_properties(agc_shader_mode_register PROPERTIES ENVIRONMENT_MODIFICATION "PATH=path_list_prepend:$") + endif() add_test(NAME agc_shader_bindless COMMAND agc_shader_memory_tests --bindless) set_tests_properties(agc_shader_bindless PROPERTIES ENVIRONMENT "ANYPS5_NO_SHADER_CACHE=1" WORKING_DIRECTORY "${CMAKE_SOURCE_DIR}") + add_test(NAME agc_shader_frontend_pair COMMAND agc_shader_memory_tests --frontend-pair) + add_test(NAME agc_shader_frontend_pair_off COMMAND agc_shader_memory_tests --frontend-pair) + set_tests_properties(agc_shader_frontend_pair PROPERTIES ENVIRONMENT "ANYPS5_NO_SHADER_CACHE=1" WORKING_DIRECTORY "${CMAKE_SOURCE_DIR}") + set_tests_properties(agc_shader_frontend_pair_off PROPERTIES ENVIRONMENT "ANYPS5_NO_SHADER_CACHE=1;APS5_NO_PERF_FRONTEND_PAIR=1" WORKING_DIRECTORY "${CMAKE_SOURCE_DIR}") if(WIN32) - set_tests_properties(agc_shader_bindless PROPERTIES ENVIRONMENT_MODIFICATION "PATH=path_list_prepend:$") + set_tests_properties(agc_shader_bindless agc_shader_frontend_pair agc_shader_frontend_pair_off PROPERTIES ENVIRONMENT_MODIFICATION "PATH=path_list_prepend:$") endif() - foreach(agcTest IN ITEMS agc_driver_tests agc_shader_memory_tests agc_structurizer_tests agc_spirv_module_tests agc_dpp_write_tests agc_masked_select_tests agc_vop1_control_decode_tests agc_validate_program_tests agc_shader_disk_cache_tests agc_shader_device_profile_tests agc_shader_control_flow_tests agc_driver_flip_tests agc_driver_pm4_tests agc_driver_graphics_tests agc_driver_bda_device_tests agc_driver_mesh_tests ${agcExecutionTests}) + foreach(agcTest IN ITEMS agc_shared_write_tests agc_driver_tests agc_shader_memory_tests agc_structurizer_tests agc_spirv_module_tests agc_dpp_write_tests agc_masked_select_tests agc_constant_bit_test_tests agc_scalar_select_mask_tests agc_vop1_control_decode_tests agc_validate_program_tests agc_pass_hazards_tests agc_shader_disk_cache_tests agc_shader_device_profile_tests agc_shader_control_flow_tests agc_driver_flip_tests agc_driver_pm4_tests agc_driver_graphics_tests agc_driver_bda_device_tests agc_driver_mesh_tests agc_driver_cmask_fast_clear_tests agc_formatted_buffer_once_tests ${agcExecutionTests}) if(agcTest STREQUAL "agc_driver_constant_store_tests") continue() endif() @@ -639,7 +921,9 @@ if(BUILD_TESTING) "PATH=path_list_prepend:$") endif() endforeach() + set_tests_properties(agc_driver_provoking_vertex PROPERTIES TIMEOUT 30) set_tests_properties(agc_driver_flat_store PROPERTIES RUN_SERIAL TRUE) + set_tests_properties(agc_driver_storage_cache_budget PROPERTIES TIMEOUT 30) add_test(NAME agc_driver_sampled_cache_budget_memory COMMAND agc_driver_sampled_cache_budget_tests memory) set_tests_properties(agc_driver_sampled_cache_budget_memory PROPERTIES ENVIRONMENT "ANYPS5_NO_SHADER_CACHE=1" SKIP_RETURN_CODE 77) if(WIN32) @@ -686,10 +970,14 @@ if(BUILD_TESTING) endif() endforeach() endforeach() - foreach(preparedMode IN ITEMS indirect uncached debug debug_failure) + foreach(preparedMode IN ITEMS indirect uncached debug debug_failure deferred deferred_undecodable) set(preparedTest "agc_driver_prepared_shaders_${preparedMode}") if(preparedMode STREQUAL "indirect") add_test(NAME ${preparedTest} COMMAND agc_driver_prepared_shaders_tests --indirect) + elseif(preparedMode STREQUAL "deferred") + add_test(NAME ${preparedTest} COMMAND agc_driver_prepared_shaders_tests --deferred) + elseif(preparedMode STREQUAL "deferred_undecodable") + add_test(NAME ${preparedTest} COMMAND agc_driver_prepared_shaders_tests --deferred-undecodable) elseif(preparedMode STREQUAL "debug_failure") add_test(NAME ${preparedTest} COMMAND agc_driver_prepared_shaders_tests --fail-before-registration) set_tests_properties(${preparedTest} PROPERTIES WILL_FAIL TRUE TIMEOUT 30) @@ -706,6 +994,10 @@ if(BUILD_TESTING) set_tests_properties(${preparedTest} PROPERTIES ENVIRONMENT_MODIFICATION "PATH=path_list_prepend:$") endif() endforeach() + set_tests_properties(agc_driver_prepared_shaders + agc_driver_prepared_shaders_indirect agc_driver_prepared_shaders_uncached + agc_driver_prepared_shaders_debug agc_driver_prepared_shaders_debug_failure + PROPERTIES RESOURCE_LOCK shader_failure_capture) endif() option(AGC_BUILD_VISUAL_TEST "Enable the standalone AGC SPIR-V visual test" OFF) @@ -737,37 +1029,7 @@ if(AGC_BUILD_VISUAL_TEST) add_test_executable(agc_driver_visual_test EXCLUDE_FROM_ALL tests/Visual.cpp - Execution/src/VulkanDevice.cpp - Execution/src/DisplayFormat.cpp - Execution/src/AspectFit.cpp - Execution/src/PresentationScaler.cpp - Execution/src/GuestMemory.cpp - Execution/src/BdaFeatures.cpp - Execution/src/SubgroupClock.cpp - Execution/src/ShaderDeviceProfile.cpp - Graphics/src/Draw.cpp - Graphics/src/DepthSurface.cpp - Graphics/src/ColorTargetLayout.cpp - Graphics/src/ColorTargetTransfer.cpp - Graphics/src/Pipeline.cpp - Graphics/src/PipelineCache.cpp - Graphics/src/Resources.cpp - Graphics/src/ShaderResources.cpp - Graphics/src/ShaderAddressResources.cpp - Graphics/src/GuestBufferMemory.cpp - Graphics/src/BdaResources.cpp - Graphics/src/BufferAddress.cpp - Graphics/src/ShaderValidation.cpp - Graphics/src/GuestTextureResource.cpp - Graphics/src/GuestSamplerResource.cpp - Graphics/src/TextureFormat.cpp - Graphics/src/DccMetadata.cpp - Graphics/src/TextureTiling.cpp - Graphics/src/TextureDetiler.cpp - Graphics/src/Texture.cpp - Graphics/src/UnitShadow.cpp - Graphics/src/Sampler.cpp - ${agcTextureDetileShaderHeader} + ${agcDriverSources} ${visualShaders} ) @@ -783,15 +1045,17 @@ if(AGC_BUILD_VISUAL_TEST) ) target_link_libraries(agc_driver_visual_test PRIVATE + shader_recompiler_agc_driver SDL2-static libc + libkernel ) set_target_properties(agc_driver_visual_test PROPERTIES CXX_EXTENSIONS OFF) configure_windows_unwind(agc_driver_visual_test) if(WIN32) - set(visualRuntimeFiles "$") + set(visualRuntimeFiles "$" "$") if(MINGW) get_filename_component(visualCompilerDirectory "${CMAKE_CXX_COMPILER}" DIRECTORY) @@ -815,7 +1079,7 @@ if(AGC_BUILD_VISUAL_TEST) COMMAND "${CMAKE_COMMAND}" -E copy_if_different ${visualRuntimeFiles} "$" - DEPENDS libc + DEPENDS libc libkernel COMMAND_EXPAND_LISTS VERBATIM ) @@ -938,6 +1202,14 @@ if(BUILD_TESTING) set_tests_properties(vector_fused_multiply_add PROPERTIES ENVIRONMENT_MODIFICATION "PATH=path_list_prepend:$") endif() + add_test_executable(vector_constant_factor_tininess_tests ${LIBS_INCLUDE_DIR}/tests/VectorConstantFactorTininess.cpp) + target_link_libraries(vector_constant_factor_tininess_tests PRIVATE shader_recompiler_agc_driver libc) + configure_windows_unwind(vector_constant_factor_tininess_tests) + add_test(NAME vector_constant_factor_tininess COMMAND vector_constant_factor_tininess_tests) + if(WIN32) + set_tests_properties(vector_constant_factor_tininess PROPERTIES ENVIRONMENT_MODIFICATION + "PATH=path_list_prepend:$") + endif() add_test_executable(scalar_saveexec_tests ${LIBS_INCLUDE_DIR}/tests/ScalarSaveexec.cpp) target_link_libraries(scalar_saveexec_tests PRIVATE shader_recompiler_agc_driver libc) configure_windows_unwind(scalar_saveexec_tests) @@ -962,6 +1234,14 @@ if(BUILD_TESTING) set_tests_properties(scalar_bitcount64 PROPERTIES ENVIRONMENT_MODIFICATION "PATH=path_list_prepend:$") endif() + add_test_executable(bitfield_mask_tests ${LIBS_INCLUDE_DIR}/tests/BitfieldMask.cpp) + target_link_libraries(bitfield_mask_tests PRIVATE shader_recompiler_agc_driver libc) + configure_windows_unwind(bitfield_mask_tests) + add_test(NAME bitfield_mask COMMAND bitfield_mask_tests) + if(WIN32) + set_tests_properties(bitfield_mask PROPERTIES ENVIRONMENT_MODIFICATION + "PATH=path_list_prepend:$") + endif() add_test_executable(scalar_scratch_load_tests ${LIBS_INCLUDE_DIR}/tests/ScalarScratchLoad.cpp) target_link_libraries(scalar_scratch_load_tests PRIVATE shader_recompiler_agc_driver libc) configure_windows_unwind(scalar_scratch_load_tests) @@ -1005,6 +1285,19 @@ if(BUILD_TESTING) endif() endif() +add_test_executable(agc_embedded_vertex_fetch_tests EXCLUDE_FROM_ALL tests/EmbeddedVertexFetch.cpp) +target_link_libraries(agc_embedded_vertex_fetch_tests PRIVATE shader_recompiler_agc_driver libc) +set_target_properties(agc_embedded_vertex_fetch_tests PROPERTIES CXX_EXTENSIONS OFF) +configure_windows_unwind(agc_embedded_vertex_fetch_tests) +if(BUILD_TESTING) + set_target_properties(agc_embedded_vertex_fetch_tests PROPERTIES EXCLUDE_FROM_ALL OFF) + add_test(NAME agc_embedded_vertex_fetch COMMAND agc_embedded_vertex_fetch_tests) + if(WIN32) + set_tests_properties(agc_embedded_vertex_fetch PROPERTIES ENVIRONMENT_MODIFICATION + "PATH=path_list_prepend:$") + endif() +endif() + add_test_executable(agc_rdna_truncated_modifier_tests EXCLUDE_FROM_ALL tests/RdnaTruncatedModifier.cpp) target_link_libraries(agc_rdna_truncated_modifier_tests PRIVATE shader_recompiler_agc_driver libc) set_target_properties(agc_rdna_truncated_modifier_tests PROPERTIES CXX_EXTENSIONS OFF) @@ -1018,6 +1311,19 @@ if(BUILD_TESTING) endif() endif() +add_test_executable(agc_mubuf_lds_decode_tests EXCLUDE_FROM_ALL tests/MubufLdsDecode.cpp) +target_link_libraries(agc_mubuf_lds_decode_tests PRIVATE shader_recompiler_agc_driver libc) +set_target_properties(agc_mubuf_lds_decode_tests PROPERTIES CXX_EXTENSIONS OFF) +configure_windows_unwind(agc_mubuf_lds_decode_tests) +if(BUILD_TESTING) + set_target_properties(agc_mubuf_lds_decode_tests PROPERTIES EXCLUDE_FROM_ALL OFF) + add_test(NAME agc_mubuf_lds_decode COMMAND agc_mubuf_lds_decode_tests) + if(WIN32) + set_tests_properties(agc_mubuf_lds_decode PROPERTIES ENVIRONMENT_MODIFICATION + "PATH=path_list_prepend:$") + endif() +endif() + add_test_executable(agc_vop1_sdwa_move_decode_tests EXCLUDE_FROM_ALL tests/Vop1SdwaMoveDecode.cpp) target_link_libraries(agc_vop1_sdwa_move_decode_tests PRIVATE shader_recompiler_agc_driver libc) set_target_properties(agc_vop1_sdwa_move_decode_tests PROPERTIES CXX_EXTENSIONS OFF) @@ -1045,6 +1351,20 @@ if(BUILD_TESTING) endif() endif() +add_test_executable(agc_position_invariance_tests EXCLUDE_FROM_ALL tests/PositionInvariance.cpp) +target_link_libraries(agc_position_invariance_tests PRIVATE shader_recompiler_agc_driver libc) +set_target_properties(agc_position_invariance_tests PROPERTIES CXX_EXTENSIONS OFF) +configure_windows_unwind(agc_position_invariance_tests) +if(BUILD_TESTING) + set_target_properties(agc_position_invariance_tests PROPERTIES EXCLUDE_FROM_ALL OFF) + add_test(NAME agc_position_invariance COMMAND agc_position_invariance_tests) + set_tests_properties(agc_position_invariance PROPERTIES ENVIRONMENT "ANYPS5_NO_SHADER_CACHE=1") + if(WIN32) + set_tests_properties(agc_position_invariance PROPERTIES ENVIRONMENT_MODIFICATION + "PATH=path_list_prepend:$") + endif() +endif() + if(BUILD_TESTING) add_test_executable(image_by_tests ${LIBS_INCLUDE_DIR}/tests/ImageBy.cpp) target_link_libraries(image_by_tests PRIVATE shader_recompiler_agc_driver libc) diff --git a/core/libs/prx/libSceAgcDriver/Execution/include/Driver/Dispatch/DispatchCache.hpp b/core/libs/prx/libSceAgcDriver/Execution/include/Driver/Dispatch/DispatchCache.hpp index 8bbb76d59..1591221fd 100644 --- a/core/libs/prx/libSceAgcDriver/Execution/include/Driver/Dispatch/DispatchCache.hpp +++ b/core/libs/prx/libSceAgcDriver/Execution/include/Driver/Dispatch/DispatchCache.hpp @@ -82,6 +82,8 @@ struct DataMask { enum class EntryOutcome { Equal, EqualData, Differing, Inaccessible, QueuedLabel, FlushingImage, PublishMoved, PendingMoved, ForgetMoved }; +bool CacheableResult(const ShaderRecompiler::RecompileResult& compiled); + } #endif diff --git a/core/libs/prx/libSceAgcDriver/Execution/include/Driver/Dispatch/IndirectDispatch.hpp b/core/libs/prx/libSceAgcDriver/Execution/include/Driver/Dispatch/IndirectDispatch.hpp index af38e7fab..5d8257c47 100644 --- a/core/libs/prx/libSceAgcDriver/Execution/include/Driver/Dispatch/IndirectDispatch.hpp +++ b/core/libs/prx/libSceAgcDriver/Execution/include/Driver/Dispatch/IndirectDispatch.hpp @@ -5,7 +5,7 @@ namespace AgcDriver::DriverDetail { -enum IndirectPath { IndirectGpu = 0, IndirectPendingImage = 1, IndirectCopiedWrite = 2, IndirectNotImported = 3, IndirectThreadDimensions = 4, IndirectFillKernel = 5, IndirectMisaligned = 6, IndirectDisabled = 7, IndirectPaths = 8 }; +enum IndirectPath { IndirectGpu = 0, IndirectPendingImage = 1, IndirectCopiedWrite = 2, IndirectNotImported = 3, IndirectThreadDimensions = 4, IndirectFillKernel = 5, IndirectMisaligned = 6, IndirectDisabled = 7, IndirectWorkgroupMemory = 8, IndirectPaths = 9 }; } diff --git a/core/libs/prx/libSceAgcDriver/Execution/include/Driver/Driver.hpp b/core/libs/prx/libSceAgcDriver/Execution/include/Driver/Driver.hpp index 2f12dae2e..dd91c8cc1 100644 --- a/core/libs/prx/libSceAgcDriver/Execution/include/Driver/Driver.hpp +++ b/core/libs/prx/libSceAgcDriver/Execution/include/Driver/Driver.hpp @@ -60,6 +60,9 @@ public: void ResolveShaderAbi(const Shader* shader, std::span context, std::span primitive); void ResolveGraphicsStagesAbi(std::span stages, std::span context, std::span primitive); void ResolveGraphicsAbi(const Shader* vertex, const Shader* pixel, std::uint32_t primitiveType); + static void FoldDrawOffsets(const ShaderRecompiler::RecompileResult& result, const DrawProgram& program, Pm4::DrawParameters& parameters); + static std::optional ClassifyIndirectDraw(const ShaderRecompiler::RecompileResult& result, const Graphics::State& graphics, const DrawProgram& frontProgram, const std::shared_ptr& localDevice, Pm4::DrawParameters& drawParameters, bool traceIndirect); + static std::string dumpRequest(std::uint64_t address, const ShaderRecompiler::RecompileRequest& request); private: friend class SampledReadScope; @@ -74,6 +77,10 @@ private: static bool copySegment(Submission& submission, const std::uint32_t* guest, std::size_t words, std::size_t& budget); static void readRegisterLists(Submission& submission); void waitForFlipRoom(const Submission& submission); + void holdFlipBehindWorker(const Submission& submission); + bool awaitsTitle(std::uint64_t awaited) const; + void noteAwaitingTitle(std::uint32_t queue, std::uint64_t awaited); + void releaseFlipHold(); void reserveOutputs(Submission& submission); void executeRewindTail(const Submission& stalled); void enqueue(Submission submission); @@ -120,22 +127,20 @@ private: static bool drawPrecheck(); std::optional precheckDraw(const QueueState& queue, const Submission& submission, std::span packet, const Pm4::DrawParameters& drawParameters, std::string& rejected, bool& traceIndirect); static std::uint32_t drawUserWord(const DrawProgram& program, std::int32_t sgpr); - std::optional classifyIndirectDraw(const ShaderRecompiler::RecompileResult& result, const Graphics::State& graphics, const DrawProgram& frontProgram, const std::shared_ptr& localDevice, Pm4::DrawParameters& drawParameters, bool traceIndirect); DrawVerdict draw(QueueState& queue, std::span packet, const Submission& submission, std::string& rejected); void addDriverPhases(DispatchClass which, const std::array& ms, bool hit, bool validated); static PendingDispatchPhases& pendingDispatchPhases(); static std::chrono::steady_clock::time_point& packetStartedAt(); static PendingDrawPhases& pendingDrawPhases(); void addDrawPhases(const std::array& ms, bool drawn, std::uint64_t captures); - void noteLabelStore(std::uint64_t address, std::span bytes, std::uint64_t stamp); - bool storedSince(std::span packet, std::uint64_t address, std::size_t bytes, std::uint64_t received); + void noteLabelStore(std::uint64_t address, std::span bytes, std::uint64_t stamp, std::uint32_t queue); + bool storedSince(std::span packet, std::uint64_t address, std::size_t bytes, std::uint64_t received, std::uint32_t* writer = nullptr); static void traceLabel(std::span packet, std::uint32_t queue); static std::array& writeHistory(); static std::size_t& writeCursor(); static void dumpPackets(std::span commands, const std::uint32_t* guest = nullptr); static void validate(const Submission& submission, const std::uint32_t* guest = nullptr); static void reportSkip(const char* kind, const std::string& what); - static std::string dumpRequest(std::uint64_t address, const ShaderRecompiler::RecompileRequest& request); static bool matchesFillKernel(std::span code, const std::vector& userData, const ShaderRecompiler::ShaderComputeStageInfo& compute); static bool fillClearEnabled(); static bool fillClearExactOnly(); @@ -177,6 +182,7 @@ private: template static void forEachWrittenBuffer(const ShaderRecompiler::RecompileResult& compiled, TVisit&& visit); void noteWrittenBuffers(std::uint64_t program, std::uint32_t queue, const ShaderRecompiler::RecompileResult& compiled); + static void appendWrittenRanges(const ShaderRecompiler::RecompileResult& compiled, std::vector>& ranges); void noteForeignWriter(std::uint64_t begin, std::uint64_t end, std::uint32_t queue); void noteDrawWriters(std::span stages, std::uint32_t queue); std::optional newestWriterLocked(std::uint64_t begin, std::uint64_t end) const; @@ -195,7 +201,41 @@ private: static bool pollReapAll(); static bool pollTryEach(); static bool traceLateLabels(); - void waitMemory(std::span packet, std::uint32_t queue, const PacketHistory& context, std::uint64_t received, bool heldAtSubmit); + void waitMemory(std::span packet, std::uint32_t queue, const PacketHistory& context, std::uint64_t received, bool heldAtSubmit, bool requireMemory = false); + struct PendingWait { + std::array words; + std::size_t count; + std::uint64_t received; + }; + static std::vector& pendingWaits(); + static void notePendingWait(std::span packet, std::uint32_t queue, std::uint64_t received); + void landPendingWaits(std::uint32_t queue); + struct ResolvedDispatch { + std::uint64_t address = 0, key = 0, indirectArguments = 0; + std::array packet{}; + std::shared_ptr registeredShader; + std::shared_ptr compiledResult; + std::shared_ptr keepVariant, attachVariant; + std::shared_ptr shaderMemory; + std::vector captured; + std::vector liveWords; + std::shared_ptr capture; + bool dataHit = false, cached = false, validated = false; + std::chrono::steady_clock::time_point resolvedAt{}; + }; + struct GroupCaptureStats { + std::uint64_t runs = 0, resolved = 0, resolvedHits = 0, adopted = 0, addressMismatches = 0, keyMismatches = 0, failures = 0, stale = 0, labelOverlaps = 0, writtenBefore = 0, deviceRejects = 0; + double resolveMs = 0, resolveMaxMs = 0, ageMs = 0, ageMaxMs = 0; + std::chrono::steady_clock::time_point reported = std::chrono::steady_clock::now(); + }; + static std::map& resolvedAhead(); + static GroupCaptureStats& groupCaptureStats(); + static std::size_t& currentPacketOffset(); + static bool& resolvingAhead(); + static bool& gateOpened(); + void resolveGroupAhead(const Submission& submission, const QueueState& live, std::size_t from); + static void clearResolvedAhead(); + static void reportGroupCapture(std::uint32_t queue); static PollStats& pollStats(); static WaitOutcomes& waitOutcomes(); static Graphics::Recorder::LateStatistics& lateCountsSeen(); @@ -302,6 +342,9 @@ private: std::atomic orderHolders{0}; std::atomic runningWorkers{0}; std::atomic queue0Awaited{0}; + std::atomic flipsAhead{0}; + std::atomic flipHolders{0}; + std::atomic queue0AwaitsTitle{false}; std::atomic evidenceReads{0}; std::atomic evidenceValidations{0}; diff --git a/core/libs/prx/libSceAgcDriver/Execution/include/Driver/Memory/BufferFill.hpp b/core/libs/prx/libSceAgcDriver/Execution/include/Driver/Memory/BufferFill.hpp new file mode 100644 index 000000000..7b27661f9 --- /dev/null +++ b/core/libs/prx/libSceAgcDriver/Execution/include/Driver/Memory/BufferFill.hpp @@ -0,0 +1,54 @@ +#ifndef CORE_LIBS_PRX_LIBSCEAGCDRIVER_EXECUTION_INCLUDE_DRIVER_BUFFERFILL_HPP +#define CORE_LIBS_PRX_LIBSCEAGCDRIVER_EXECUTION_INCLUDE_DRIVER_BUFFERFILL_HPP + +#include "Recompiler.hpp" +#include +#include +#include +#include +#include +#include + +namespace AgcDriver::DriverDetail { + +inline constexpr std::array PatternFillKernel{ + 0xbfa00001u, 0xd7460000u, 0x04010c0cu, 0xf4201a84u, 0xfa000000u, 0xbf8cc07fu, + 0x7da8006au, 0xbf880009u, 0xf4201a84u, 0xfa000004u, 0xbf8cc07fu, 0x3602006au, + 0xe0002000u, 0x80000101u, 0xbf8c3f70u, 0xe0102000u, 0x80010100u, 0xbf810000u +}; + +inline bool MatchesPatternFillKernel(std::span code, std::span userData, const ShaderRecompiler::ShaderComputeStageInfo& compute) { + if (userData.size() != 12 || compute.numThreads != std::array{64, 1, 1} || !compute.groupIdEnable[0] || compute.threadIdComponentCount != 1) return false; + if (code.size() < PatternFillKernel.size() || !std::equal(PatternFillKernel.begin(), PatternFillKernel.end(), code.begin())) return false; + for (const auto first : {0u, 4u, 8u}) { + if ((userData[first + 1] & 0xffff0000u) != (first == 8 ? 0x00100000u : 0x00040000u)) return false; + if (userData[first + 3] != (first == 8 ? 0x0004dfacu : 0x00014004u)) return false; + } + return userData[2] >= 1 && userData[10] >= 1; +} + +struct PatternFillRange { + std::uint64_t source, destination, control, invocations; + std::uint32_t records; + + [[nodiscard]] std::optional UniformBytes(std::uint32_t count, std::uint32_t mask) const { + if (mask != 0) return std::nullopt; + const auto bytes = std::min({invocations, std::uint64_t{count}, std::uint64_t{records}}) * 4u; + if (destination % 16 != 0 || bytes % 16 != 0) return std::nullopt; + if (bytes != 0 && ((source < destination + bytes && destination < source + 4) || (control < destination + bytes && destination < control + 8))) return std::nullopt; + return static_cast(bytes); + } +}; + +inline std::optional DecodePatternFillRange(std::span userData, std::span packet) { + if (userData.size() != 12 || packet.size() < 5 || packet[2] != 1 || packet[3] != 1) return std::nullopt; + const std::uint64_t invocations = (packet[4] & 0x20u) != 0 ? packet[1] : std::uint64_t{packet[1]} * 64u; + if (invocations > (std::uint64_t{1} << 32u)) return std::nullopt; + const auto base = [&](std::size_t word) { return userData[word] | (static_cast(userData[word + 1] & 0xffffu) << 32u); }; + if (base(0) % 4 != 0 || base(8) % 4 != 0) return std::nullopt; + return PatternFillRange{base(0), base(4), base(8), invocations, userData[6]}; +} + +} + +#endif diff --git a/core/libs/prx/libSceAgcDriver/Execution/include/Driver/Memory/DwordPatternFill.hpp b/core/libs/prx/libSceAgcDriver/Execution/include/Driver/Memory/DwordPatternFill.hpp new file mode 100644 index 000000000..79edd8b46 --- /dev/null +++ b/core/libs/prx/libSceAgcDriver/Execution/include/Driver/Memory/DwordPatternFill.hpp @@ -0,0 +1,24 @@ +#ifndef CORE_LIBS_PRX_LIBSCEAGCDRIVER_EXECUTION_INCLUDE_DRIVER_MEMORY_DWORDPATTERNFILL_HPP +#define CORE_LIBS_PRX_LIBSCEAGCDRIVER_EXECUTION_INCLUDE_DRIVER_MEMORY_DWORDPATTERNFILL_HPP + +#include "Recompiler.hpp" +#include +#include +#include +#include +#include +#include + +namespace AgcDriver::DriverDetail { + +struct DwordPatternFill { + std::uint64_t base = 0; + std::size_t bytes = 0; + std::array pattern{}; +}; + +std::optional MatchDwordPatternFill(std::span packet, std::span code, const std::vector& userData, const ShaderRecompiler::ShaderComputeStageInfo& compute); + +} + +#endif diff --git a/core/libs/prx/libSceAgcDriver/Execution/include/Driver/Queues/Submission.hpp b/core/libs/prx/libSceAgcDriver/Execution/include/Driver/Queues/Submission.hpp index 1b15bcfe9..f64ecfaa7 100644 --- a/core/libs/prx/libSceAgcDriver/Execution/include/Driver/Queues/Submission.hpp +++ b/core/libs/prx/libSceAgcDriver/Execution/include/Driver/Queues/Submission.hpp @@ -29,6 +29,7 @@ struct Submission { std::map> registerLists; bool suspend = false; bool waitFree = false; + bool holdsFlip = false; std::uint64_t received = 0; std::vector labelWrites; diff --git a/core/libs/prx/libSceAgcDriver/Execution/include/Driver/Shaders/ShaderRegistry.hpp b/core/libs/prx/libSceAgcDriver/Execution/include/Driver/Shaders/ShaderRegistry.hpp index 846729868..3ce4f820d 100644 --- a/core/libs/prx/libSceAgcDriver/Execution/include/Driver/Shaders/ShaderRegistry.hpp +++ b/core/libs/prx/libSceAgcDriver/Execution/include/Driver/Shaders/ShaderRegistry.hpp @@ -6,14 +6,24 @@ #include #include #include +#include #include #include +#include #include #include #include #include #include +namespace ShaderRecompiler { +class ShaderPreparationContext; +} + +namespace AgcDriver { +class VulkanDevice; +} + namespace AgcDriver::DriverDetail { struct ShaderSnapshot; @@ -46,9 +56,13 @@ struct PreparedShaderState { std::vector rectangleProgress; std::vector> fragments; bool rectangleRequested = false; + bool deferred = false; }; struct PreparedShaders : PreparedShaderState { std::mutex mutex; + std::condition_variable settled; + std::exception_ptr failure; + bool pending = false; }; struct RegisteredShaderState { @@ -70,13 +84,17 @@ struct ShaderSnapshot { std::shared_ptr ReadRawComputeShader(std::uint64_t address); +std::shared_ptr PrepareShaderWithDiagnostics(const ShaderRecompiler::RecompileRequest& request, ShaderRecompiler::ShaderPreparationContext* preparation = nullptr); + std::uint64_t NullPixelProgramAddress(); +ShaderSnapshot PrepareNullPixelProgram(const VulkanDevice& device); std::optional RegisteredFloatMode(const ShaderSnapshot& snapshot); void PublishRegisteredShader(std::shared_ptr& registry, const std::shared_ptr& snapshot); void ResolvePreparedGraphics(const ShaderSnapshot& front, const std::shared_ptr& fragment, std::uint32_t primitiveType, const ShaderRecompiler::SpirvTarget& target); ShaderRecompiler::RectListShaders PreparedRectangle(const ShaderSnapshot& snapshot, std::uint64_t vertexId, std::uint64_t fragmentId); +ShaderRecompiler::RectListShaders DrawRectangle(const ShaderSnapshot& front, const std::shared_ptr& fragment, std::uint64_t vertexId, std::uint64_t fragmentId, const ShaderRecompiler::SpirvTarget& target); std::shared_ptr SourceHandleFor(const ShaderSnapshot& snapshot, std::size_t codeOffset, const ShaderRecompiler::RecompileRequest& request); ShaderRecompiler::PreparedShaderInvocation InvocationFor(const ShaderSnapshot& snapshot, std::size_t codeOffset, const ShaderRecompiler::RecompileRequest& request); diff --git a/core/libs/prx/libSceAgcDriver/Execution/include/Driver/Synchronization/SynchronizationStatistics.hpp b/core/libs/prx/libSceAgcDriver/Execution/include/Driver/Synchronization/SynchronizationStatistics.hpp index 0644622d3..48d832499 100644 --- a/core/libs/prx/libSceAgcDriver/Execution/include/Driver/Synchronization/SynchronizationStatistics.hpp +++ b/core/libs/prx/libSceAgcDriver/Execution/include/Driver/Synchronization/SynchronizationStatistics.hpp @@ -12,7 +12,7 @@ struct SubmissionCosts { std::atomic submissions{0}, validateNs{0}, copyNs{0}, dequeueNs{0}, completeNs{0}, suspendNs{0}, suspends{0}, suspendsSkipped{0}, endSubmits{0}, endSkipped{0}, notifiesSkipped{0}; }; -extern std::atomic gpuLabels, completionLabels, notImportedLabels, noOpLabels, unlockedDrains; +extern std::atomic gpuLabels, completionLabels, noOpLabels, unlockedDrains; extern std::atomic labelFallbacks[5]; diff --git a/core/libs/prx/libSceAgcDriver/Execution/include/Driver/Synchronization/WaitMemory.hpp b/core/libs/prx/libSceAgcDriver/Execution/include/Driver/Synchronization/WaitMemory.hpp index 5ff14c7f7..dc1e3b789 100644 --- a/core/libs/prx/libSceAgcDriver/Execution/include/Driver/Synchronization/WaitMemory.hpp +++ b/core/libs/prx/libSceAgcDriver/Execution/include/Driver/Synchronization/WaitMemory.hpp @@ -18,6 +18,7 @@ inline constexpr std::chrono::microseconds PollTryInterval{1000}; struct LabelStore { std::uint64_t stamp = 0; std::uint32_t value = 0; + std::uint32_t queue = 0xffffffffu; }; struct WriteRecord { diff --git a/core/libs/prx/libSceAgcDriver/Execution/include/FragmentBarycentric.hpp b/core/libs/prx/libSceAgcDriver/Execution/include/FragmentBarycentric.hpp new file mode 100644 index 000000000..361aadb06 --- /dev/null +++ b/core/libs/prx/libSceAgcDriver/Execution/include/FragmentBarycentric.hpp @@ -0,0 +1,14 @@ +#ifndef CORE_LIBS_PRX_LIBSCEAGCDRIVER_EXECUTION_INCLUDE_FRAGMENTBARYCENTRIC_HPP +#define CORE_LIBS_PRX_LIBSCEAGCDRIVER_EXECUTION_INCLUDE_FRAGMENTBARYCENTRIC_HPP + +#include + +namespace AgcDriver { + +inline bool FragmentShaderBarycentricUsable(VkDriverId driver, bool reported) { + return reported && driver != VK_DRIVER_ID_MOLTENVK; +} + +} + +#endif diff --git a/core/libs/prx/libSceAgcDriver/Execution/include/GuestMemory.hpp b/core/libs/prx/libSceAgcDriver/Execution/include/GuestMemory.hpp index d45085da8..ba201ca15 100644 --- a/core/libs/prx/libSceAgcDriver/Execution/include/GuestMemory.hpp +++ b/core/libs/prx/libSceAgcDriver/Execution/include/GuestMemory.hpp @@ -75,6 +75,7 @@ bool UnchangedSinceAll(std::span queries); // UnchangedSince(range, it) holds until the next store over the range. std::uint64_t MarkWritten(std::uint64_t address, std::size_t bytes); std::uint64_t StoreOwnBytes(std::uint64_t address, std::size_t bytes, const std::function& store); +std::uint64_t StoreOwnBytes(std::uint64_t address, std::span source); bool StoredOver(std::uint64_t address, std::size_t bytes, std::uint64_t generation); // Collect epoch: within one epoch a range already collected is not walked again, CollectWrites // returns the current generation instead. A guest write landing between two collects of the same diff --git a/core/libs/prx/libSceAgcDriver/Execution/include/Pm4.hpp b/core/libs/prx/libSceAgcDriver/Execution/include/Pm4.hpp index e6d2cbe13..e1a24ec60 100644 --- a/core/libs/prx/libSceAgcDriver/Execution/include/Pm4.hpp +++ b/core/libs/prx/libSceAgcDriver/Execution/include/Pm4.hpp @@ -81,7 +81,7 @@ inline bool DrawOpcode(std::uint32_t opcode) { return opcode == 0x27 || opcode = std::string Name(std::uint32_t header); // A PM4 type-2 packet is a one-dword filler (command-buffer padding); type 3 and type 0 carry a // dword count in bits 29:16. Type 1 is undefined. -inline bool FillerPacket(std::uint32_t header) { return (header >> 30u) == 2u; } +inline bool FillerPacket(std::uint32_t header) { return (header >> 30u) == 2u || header == 0xffff1000u; } inline std::size_t PacketWords(std::uint32_t header) { return FillerPacket(header) ? 1u : static_cast((header >> 16u) & 0x3fffu) + 2u; } std::string_view UnsupportedReason(std::uint32_t header); void Validate(std::span packet, std::uint32_t queue); diff --git a/core/libs/prx/libSceAgcDriver/Execution/include/ProfileOutput.hpp b/core/libs/prx/libSceAgcDriver/Execution/include/ProfileOutput.hpp index 6daec3ba8..617a1399e 100644 --- a/core/libs/prx/libSceAgcDriver/Execution/include/ProfileOutput.hpp +++ b/core/libs/prx/libSceAgcDriver/Execution/include/ProfileOutput.hpp @@ -57,6 +57,8 @@ public: return true; } + void Print(const char* text) { Print("%s", text); } + template void Print(const char* format, TArgs... args) { std::array local{}; diff --git a/core/libs/prx/libSceAgcDriver/Execution/include/QueueState.hpp b/core/libs/prx/libSceAgcDriver/Execution/include/QueueState.hpp index ff24be825..d05c1881a 100644 --- a/core/libs/prx/libSceAgcDriver/Execution/include/QueueState.hpp +++ b/core/libs/prx/libSceAgcDriver/Execution/include/QueueState.hpp @@ -156,6 +156,7 @@ inline Registers InitialContextRegisters() { {0x318, 0}, {0x31b, 0}, {0x31c, 0}, {0x31d, 0}, {0x390, 0}, {0x3b0, 0}, {0x3b8, 0} }; + for (std::uint32_t i = 0; i < 32; ++i) result.emplace(0x191 + i, 0); for (std::uint32_t i = 0; i < 8; ++i) { result.emplace(0x1e0 + i, 0x20010001); result.emplace(0x31c + 0xfu * i, 0); diff --git a/core/libs/prx/libSceAgcDriver/Execution/include/ShaderMemory.hpp b/core/libs/prx/libSceAgcDriver/Execution/include/ShaderMemory.hpp index 457fd8360..75f597a2d 100644 --- a/core/libs/prx/libSceAgcDriver/Execution/include/ShaderMemory.hpp +++ b/core/libs/prx/libSceAgcDriver/Execution/include/ShaderMemory.hpp @@ -85,6 +85,7 @@ private: }; static bool read(void* context, std::uint64_t address, std::uint32_t* value); + static bool accessible(void* context, std::uint64_t address, std::uint64_t bytes); Page& page(std::uint64_t base); // Regions given at construction (the registered shader's code and header), referenced as given: diff --git a/core/libs/prx/libSceAgcDriver/Execution/include/SubgroupClock.hpp b/core/libs/prx/libSceAgcDriver/Execution/include/SubgroupClock.hpp index 6c1c74209..db97b2d58 100644 --- a/core/libs/prx/libSceAgcDriver/Execution/include/SubgroupClock.hpp +++ b/core/libs/prx/libSceAgcDriver/Execution/include/SubgroupClock.hpp @@ -2,11 +2,12 @@ #define CORE_LIBS_PRX_LIBSCEAGCDRIVER_EXECUTION_INCLUDE_SUBGROUPCLOCK_HPP #include +#include #include namespace AgcDriver { -bool NarrowSubgroupClock(VkDriverId driver, std::string_view deviceName); +bool NarrowSubgroupClock(VkDriverId driver, std::uint32_t deviceId, std::string_view deviceName); } diff --git a/core/libs/prx/libSceAgcDriver/Execution/include/VulkanDevice.hpp b/core/libs/prx/libSceAgcDriver/Execution/include/VulkanDevice.hpp index 696cfbdc1..2038fc32f 100644 --- a/core/libs/prx/libSceAgcDriver/Execution/include/VulkanDevice.hpp +++ b/core/libs/prx/libSceAgcDriver/Execution/include/VulkanDevice.hpp @@ -35,6 +35,7 @@ public: VulkanDevice& operator=(const VulkanDevice&) = delete; std::string DeviceName() const; ShaderRecompiler::SpirvTarget Target() const; + std::optional GeometryLimits() const; ShaderRecompiler::SpirvTarget ComputeTarget(std::uint32_t waveSize) const; // Distinguishes this device from every earlier one in the process (a memo keyed by the device // cannot rely on the pointer, which a replacement may reuse). @@ -66,11 +67,9 @@ public: // once that work completed; the batch is submitted by the queue worker's flush rules (or at once // with APS5_LABEL_SUBMIT_NOW=1). `stamp` is the record-order stamp and `queue` the recording // queue for the recorder's pending-label table. Returns 0 when recorded on the GPU, 5 when kept - // as a completion action behind pending write-backs and 6 when kept as one because the memory is - // not host-imported so the GPU cannot store it (both land when their batch is reaped), else why - // the CPU must write it: + // as a completion action behind pending write-backs, else why the CPU must write it: // 1 nothing recorded (the write is already ordered), 2 write-backs pending and 3 memory not - // imported (both only with APS5_DRAIN_COMPLETION_LABELS=1), 4 unsuitable size or alignment. + // imported, 4 unsuitable size or alignment. // `reapFirst` retires finished batches before the checks; a caller recording a group of labels // under one lock passes it for the first label only. int WriteLabelOnGpu(std::uint64_t address, std::span bytes, std::uint64_t stamp, std::uint32_t queue, bool reapFirst = true); @@ -78,6 +77,7 @@ public: // Pending-label table lookup and open-batch overlap test for WAIT_REG_MEM (see Recorder). std::optional PendingLabel(std::uint64_t address, std::size_t bytes, std::uint64_t afterStamp, Graphics::Recorder::LabelRefusal* refusal = nullptr) const; bool OpenWriteOverlaps(std::uint64_t address, std::size_t bytes) const; + bool RecordedWritesSettled(std::uint64_t address, std::size_t bytes) const; // Fills [address, address + bytes) of host-imported guest memory with a repeating 16-byte pattern, // recorded behind the open batch; false when the range is not imported (the caller stores it). // Recorded GPU stores over the range are ordered before the fill by its barrier; finished @@ -159,9 +159,13 @@ public: void* Window() const; void Resize(std::uint32_t width, std::uint32_t height); bool Presentable() const; + bool DualSrcBlend() const; bool PrimitiveListRestart() const; bool SamplerFilterMinmax() const; bool ConservativeRasterization() const; + VkShaderStageFlags SubgroupStages() const; + bool ProvokingVertexLast() const; + bool GraphicsPipelineLibraries() const; // A presentation is a few steps so the presenter holds GuestMemory::GpuMutex only while it // touches the queue. Presentations are slots (FlipInFlight() + 1, each with its own command // buffer, fence, kept resident image and dump buffer): RetirePresents(keep) (no mutex) retires @@ -216,7 +220,7 @@ public: // records. Null when nothing is prepared: the resource cache may serve the dispatch (its // Revalidate stays under the mutex), or APS5_LOCKED_BUILD=1 keeps the whole build under it as // before. `shader` and `snapshots` must outlive the dispatch. - std::shared_ptr PrepareDispatch(const ShaderRecompiler::RecompileResult& shader, std::span snapshots); + std::shared_ptr PrepareDispatch(const ShaderRecompiler::RecompileResult& shader, std::span snapshots, std::uint64_t dispatchThreads = 0); // APS5_PROFILE_DRAW: the parts of a PrepareDispatch in milliseconds, in the order key, find, // precollect, presync, stage A (the driver's 'prepare:' rows). static std::span PreparePhaseMs(const PreparedDispatch& prepared); @@ -224,16 +228,6 @@ public: // variant (design_cpu_final M4): only when the resource cache served or took the object // (reusable, cacheable) and recipes are on (APS5_NO_DISPATCH_RECIPE=1 builds none); else null. void Dispatch(const ShaderRecompiler::RecompileResult& shader, std::uint32_t x, std::uint32_t y, std::uint32_t z, std::span snapshots = {}, std::uint64_t programAddress = 0, std::shared_ptr prepared = nullptr, std::shared_ptr* recipe = nullptr); - // A dispatch whose group counts are the three dwords at `arguments` in guest memory - // (DISPATCH_INDIRECT): the GPU reads them in place from the host import, ordered after everything - // recorded before, so the CPU never waits for the shader that wrote them. When the GPU could not - // see the current bytes the counts are read on the CPU instead (through the flush hook, as the - // driver resolved every indirect dispatch before) and the dispatch is recorded as a direct one: - // cpuReason 1 storage-image results were pending over them, 2 a recorded dispatch writes them - // through a copied buffer (its CPU write-back lands only when the batch is reaped) or a label - // over those dwords is pending, 3 the memory is not host-imported; 0 when recorded GPU-side. argumentReadMs is - // what that CPU read (its sync) took. The device limit on group counts is not checked GPU-side. - // Debug aid: APS5_NO_GPU_INDIRECT=1 (in the driver) keeps every indirect dispatch on the CPU path. struct IndirectOutcome { int cpuReason; double argumentReadMs; diff --git a/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Dispatch/Dispatch.cpp b/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Dispatch/Dispatch.cpp index b0909f721..cb71f04dd 100644 --- a/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Dispatch/Dispatch.cpp +++ b/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Dispatch/Dispatch.cpp @@ -5,7 +5,9 @@ #include "prx/libSceAgcDriver/Execution/include/Driver/Shaders/ShaderRegistry.hpp" #include "prx/libSceAgcDriver/Execution/include/GuestMemory.hpp" #include "prx/libSceAgcDriver/Execution/include/Pm4.hpp" +#include "prx/libSceAgcDriver/Graphics/include/DepthSurface.hpp" #include "Optimization/ResourceProgram.hpp" +#include #include #include @@ -29,6 +31,7 @@ void Driver::dispatch(QueueState& queue, std::span packet, userData.push_back(readUserData(queue.shader, 0x240 + i)); } auto compute = Graphics::DecodeComputeStageInfo(queue.shader, snapshot.header); + const bool resolving = resolvingAhead(); std::vector memory{{snapshot.codeAddress, std::as_bytes(std::span(snapshot.code))}}; if (!snapshot.header.empty()) memory.push_back({snapshot.headerAddress, snapshot.header}); @@ -42,7 +45,23 @@ void Driver::dispatch(QueueState& queue, std::span packet, } const auto codeOffset = static_cast((address - snapshot.codeAddress) / sizeof(std::uint32_t)); std::array resolved{}; - if (indirectArguments != 0 && matchesFillKernel(std::span(snapshot.code).subspan(codeOffset), userData, compute)) { + ResolvedDispatch ahead; + bool fromAhead = false; + auto& groupStats = groupCaptureStats(); + if (!resolving) { + auto& parked = resolvedAhead(); + if (const auto found = parked.find(currentPacketOffset()); found != parked.end()) { + ahead = std::move(found->second); + parked.erase(found); + fromAhead = ahead.address == address; + if (!fromAhead) ++groupStats.addressMismatches; + } + } + if (fromAhead) { + resolved = ahead.packet; + packet = resolved; + indirectArguments = ahead.indirectArguments; + } else if (indirectArguments != 0 && matchesFillKernel(std::span(snapshot.code).subspan(codeOffset), userData, compute)) { recordQueuedLabelsBeforeRead(submission.queue); const auto readStart = std::chrono::steady_clock::now(); @@ -51,13 +70,19 @@ void Driver::dispatch(QueueState& queue, std::span packet, packet = resolved; indirectArguments = 0; } - if (fillBuffer(queue, submission.queue, packet, std::span(snapshot.code).subspan(codeOffset), userData, compute, localDevice)) { - pendingDispatchPhases().outcome = DispatchOutcome::FillHle; - return; - } - if (indirectArguments == 0 && copyBuffer(queue, submission.queue, packet, std::span(snapshot.code).subspan(codeOffset), userData, compute, localDevice, address)) { - pendingDispatchPhases().outcome = DispatchOutcome::CopyHle; - return; + if (resolving) { + if (matchesFillKernel(std::span(snapshot.code).subspan(codeOffset), userData, compute)) return; + if (indirectArguments == 0 && matchesCopyKernel(std::span(snapshot.code).subspan(codeOffset), userData, compute)) return; + } else { + if (const auto htile = Graphics::HtileDepthClearAddress(std::span(snapshot.code).subspan(codeOffset), userData, compute.numThreads); htile != 0) Graphics::NoteHtileDepthClear(htile); + if (fillBuffer(queue, submission.queue, packet, std::span(snapshot.code).subspan(codeOffset), userData, compute, localDevice)) { + pendingDispatchPhases().outcome = DispatchOutcome::FillHle; + return; + } + if (indirectArguments == 0 && copyBuffer(queue, submission.queue, packet, std::span(snapshot.code).subspan(codeOffset), userData, compute, localDevice, address)) { + pendingDispatchPhases().outcome = DispatchOutcome::CopyHle; + return; + } } if (indirectArguments == 0 && (packet[4] & 0x20u) != 0) { const std::array threads{packet[1], packet[2], packet[3]}; @@ -180,14 +205,36 @@ void Driver::dispatch(QueueState& queue, std::span packet, } mix(reinterpret_cast(registeredShader.get())); + if (fromAhead && key != ahead.key) { + fromAhead = false; + ++groupStats.keyMismatches; + } phaseTiming.Phase(PhaseKey); timing.Mark("decode_key_device_setup"); - lookupDispatch(address, submission, key, noDispatchCache, traceCache, profile, memory, phaseTiming, phaseMs, compiledResult, keepVariant, captured, liveWords, dataHit, cached, validated, missedEntry, missedDiffering); + if (fromAhead) { + compiledResult = std::move(ahead.compiledResult); + keepVariant = std::move(ahead.keepVariant); + attachVariant = std::move(ahead.attachVariant); + shaderMemory = std::move(ahead.shaderMemory); + captured = std::move(ahead.captured); + liveWords = std::move(ahead.liveWords); + capture = std::move(ahead.capture); + dataHit = ahead.dataHit; + cached = ahead.cached; + validated = ahead.validated; + ++groupStats.adopted; + const auto age = std::chrono::duration(std::chrono::steady_clock::now() - ahead.resolvedAt).count(); + groupStats.ageMs += age; + groupStats.ageMaxMs = std::max(groupStats.ageMaxMs, age); + phaseTiming.Phase(PhaseLookup); + } else { + lookupDispatch(address, submission, key, noDispatchCache, traceCache, profile, memory, phaseTiming, phaseMs, compiledResult, keepVariant, captured, liveWords, dataHit, cached, validated, missedEntry, missedDiffering); + } timing.Mark("lookup_cache"); if (cached) { require(keepVariant != nullptr && keepVariant->shader == registeredShader, "dispatch cache belongs to another registered shader"); captureMs += phaseTiming.Elapsed(); - } else { + } else if (!fromAhead) { shaderMemory = std::make_shared(memory, &queryPendingWrite, &observePendingWrite, hookWaitCounter()); std::uint64_t forgetAtCapture = 0; @@ -231,7 +278,30 @@ void Driver::dispatch(QueueState& queue, std::span packet, } recompileMs += phaseTiming.Elapsed(); phaseTiming.Phase(PhaseRecompile); - insertDispatch(address, key, noDispatchCache, profile, registeredShader, forgetAtCapture, memory, shaderMemory, captured, capture, compiledResult, missedEntry, missedDiffering, attachVariant, phaseTiming); + insertDispatch(address, key, noDispatchCache || !CacheableResult(*compiledResult), profile, registeredShader, forgetAtCapture, memory, shaderMemory, captured, capture, compiledResult, missedEntry, missedDiffering, attachVariant, phaseTiming); + } + if (resolving) { + auto& parked = resolvedAhead()[currentPacketOffset()]; + parked = ResolvedDispatch{}; + parked.address = address; + parked.key = key; + parked.indirectArguments = indirectArguments; + std::copy_n(packet.begin(), std::min(packet.size(), parked.packet.size()), parked.packet.begin()); + parked.registeredShader = registeredShader; + parked.compiledResult = compiledResult; + parked.keepVariant = keepVariant; + parked.attachVariant = attachVariant; + parked.shaderMemory = shaderMemory; + parked.captured = captured; + parked.liveWords = liveWords; + parked.capture = capture; + parked.dataHit = dataHit; + parked.cached = cached; + parked.validated = validated; + parked.resolvedAt = std::chrono::steady_clock::now(); + ++groupStats.resolved; + if (cached) ++groupStats.resolvedHits; + return; } if (verifyDataHits() && dataHit) verifyDataHit(snapshot, codeOffset, request, memory, address, *keepVariant, liveWords, *compiledResult); @@ -251,6 +321,7 @@ void Driver::dispatch(QueueState& queue, std::span packet, groups[axis] = (groups[axis] + threads - 1) / threads; } } + const std::uint64_t dispatchThreads = indirectArguments != 0 ? 0 : std::uint64_t{groups[0]} * groups[1] * groups[2] * compute.numThreads[0] * compute.numThreads[1] * compute.numThreads[2]; static const bool traceIo = std::getenv("APS5_TRACE_DISPATCH_IO") != nullptr; if (traceIo) { @@ -266,6 +337,11 @@ void Driver::dispatch(QueueState& queue, std::span packet, } const auto rethrow = [&](const std::exception& error) { + if (fromAhead) { + ++groupStats.deviceRejects; + dispatch(queue, packet, submission, indirectArguments); + return true; + } char where[64]; std::snprintf(where, sizeof(where), "compute shader 0x%llx: ", static_cast(address)); throw std::runtime_error(where + std::string(error.what())); @@ -288,9 +364,9 @@ void Driver::dispatch(QueueState& queue, std::span packet, std::shared_ptr prepared; if (recipeHit == nullptr || VulkanDevice::VerifyRecipes()) { try { - prepared = localDevice->PrepareDispatch(compiled, snapshots); + prepared = localDevice->PrepareDispatch(compiled, snapshots, dispatchThreads); } catch (const std::exception& error) { - rethrow(error); + if (rethrow(error)) return; } if (profile) { const auto now = std::chrono::steady_clock::now(); @@ -338,7 +414,7 @@ void Driver::dispatch(QueueState& queue, std::span packet, localDevice->Dispatch(compiled, groups[0], groups[1], groups[2], snapshots, address, std::move(prepared), attachTo != nullptr ? &builtRecipe : nullptr); } } catch (const std::exception& error) { - rethrow(error); + if (rethrow(error)) return; } if (builtRecipe != nullptr) { if (dataHit) { diff --git a/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Dispatch/DispatchAhead.cpp b/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Dispatch/DispatchAhead.cpp new file mode 100644 index 000000000..4867fef41 --- /dev/null +++ b/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Dispatch/DispatchAhead.cpp @@ -0,0 +1,188 @@ +#include "prx/libSceAgcDriver/Execution/include/ProfileOutput.hpp" +#include "prx/libSceAgcDriver/Execution/include/Driver/Driver.hpp" +#include "prx/libSceAgcDriver/Execution/include/Driver/Synchronization/DeferredLabels.hpp" +#include "prx/libSceAgcDriver/Execution/include/Pm4.hpp" +#include "prx/libSceAgcDriver/Execution/include/VideoOutput.hpp" +#include +#include +#include + +namespace AgcDriver::DriverDetail { + +std::map& Driver::resolvedAhead() { + static thread_local std::map resolved; + return resolved; +} + +Driver::GroupCaptureStats& Driver::groupCaptureStats() { + static thread_local GroupCaptureStats stats; + return stats; +} + +std::size_t& Driver::currentPacketOffset() { + static thread_local std::size_t offset = 0; + return offset; +} + +bool& Driver::resolvingAhead() { + static thread_local bool resolving = false; + return resolving; +} + +bool& Driver::gateOpened() { + static thread_local bool opened = false; + return opened; +} + +void Driver::clearResolvedAhead() { + auto& resolved = resolvedAhead(); + if (resolved.empty()) return; + groupCaptureStats().stale += resolved.size(); + resolved.clear(); +} + +namespace { + +struct RunLabel { + std::uint64_t address; + std::size_t size; + std::uint64_t value; +}; + +bool InCode(const ShaderRecompiler::MemoryRegion& region, const std::shared_ptr& code) { + if (code == nullptr) return false; + if (region.guestAddress >= code->codeAddress && region.guestAddress < code->codeAddress + code->code.size() * sizeof(std::uint32_t)) return true; + return !code->header.empty() && region.guestAddress >= code->headerAddress && region.guestAddress < code->headerAddress + code->header.size(); +} + +bool OverlapsRange(std::span regions, const std::vector>& ranges, const std::shared_ptr& code) { + for (const auto& region : regions) { + if (InCode(region, code)) continue; + for (const auto& [begin, end] : ranges) { + if (begin < region.guestAddress + region.bytes.size() && region.guestAddress < end) return true; + } + } + return false; +} + +bool WrittenByRun(std::span wait, const std::vector& labels) { + const std::uint64_t awaited = wait[2] | (static_cast(wait[3]) << 32u); + const auto bytes = Pm4::WaitAwaitedBytes(wait); + for (auto it = labels.rbegin(); it != labels.rend(); ++it) { + if (it->address != awaited || it->size < bytes) continue; + return Pm4::WaitComparesValue(wait, bytes == 8 ? it->value : (it->value & 0xffffffffu)); + } + return false; +} + +void NoteLabel(std::vector& labels, std::uint64_t address, std::span bytes) { + if (bytes.size() != 4 && bytes.size() != 8) return; + std::uint64_t value = 0; + std::memcpy(&value, bytes.data(), bytes.size()); + labels.push_back({address, bytes.size(), value}); +} + +bool OverlapsLabel(std::span regions, const std::vector& labels) { + for (const auto& label : labels) { + for (const auto& region : regions) { + if (label.address < region.guestAddress + region.bytes.size() && region.guestAddress < label.address + label.size) return true; + } + } + return false; +} + +} + +void Driver::resolveGroupAhead(const Submission& submission, const QueueState& live, std::size_t from) { + static const bool allQueues = std::getenv("APS5_GROUP_CAPTURE_ALL_QUEUES") != nullptr; + if (!allQueues && submission.queue == 0) return; + auto& resolved = resolvedAhead(); + if (resolved.contains(from) || device.Load() == nullptr) return; + auto& stats = groupCaptureStats(); + const auto begin = std::chrono::steady_clock::now(); + ++stats.runs; + QueueState scratch = live; + std::vector labels; + std::vector> writes; + for (const auto& label : deferredLabels().labels) NoteLabel(labels, label.address, std::span(label.bytes).first(label.size)); + struct Resolving { + std::size_t offset; + Resolving() : offset(currentPacketOffset()) { resolvingAhead() = true; } + ~Resolving() { + resolvingAhead() = false; + currentPacketOffset() = offset; + } + } resolving; + const auto& commands = submission.commands; + for (std::size_t cursor = from; cursor < commands.size();) { + const auto header = commands[cursor]; + if (Pm4::FillerPacket(header)) { + ++cursor; + continue; + } + const auto count = Pm4::PacketWords(header); + if (cursor + count > commands.size()) break; + const auto packet = std::span(commands).subspan(cursor, count); + const auto opcode = (header >> 8u) & 0xffu; + const auto next = cursor + count; + if (header == FlipPacketHeader || header == RenderingWaitPacketHeader || opcode == 0x22 || (Pm4::Predicated(header) && scratch.predication.operation != 0)) break; + if (opcode == 0x3c || opcode == 0x93) { + if (!Pm4::WaitSatisfiedUnchecked(packet) && !WrittenByRun(packet, labels)) break; + } else if (opcode == 0x15 || opcode == 0x16) { + if (resolved.contains(cursor)) break; + currentPacketOffset() = cursor; + try { + if (opcode == 0x15) dispatch(scratch, packet, submission); + else dispatchIndirect(scratch, packet, submission); + } catch (const std::exception&) { + ++stats.failures; + break; + } + if (const auto found = resolved.find(cursor); found != resolved.end()) { + if (OverlapsLabel(found->second.captured, labels)) { + resolved.erase(found); + ++stats.labelOverlaps; + break; + } + if (OverlapsRange(found->second.captured, writes, found->second.registeredShader)) { + resolved.erase(found); + ++stats.writtenBefore; + break; + } + if (found->second.compiledResult != nullptr) appendWrittenRanges(*found->second.compiledResult, writes); + } + } else if (opcode == 0x37 || opcode == 0x49) { + const auto label = Pm4::DecodeLabelWrite(packet); + if (!label) break; + NoteLabel(labels, label->address, label->Bytes()); + } else if (opcode != 0x3f && opcode != 0x42 && opcode != 0x46 && opcode != 0x58) { + if (Pm4::AccessesMemory(header)) break; + try { + Pm4::Execute(packet, scratch); + } catch (const std::exception&) { + break; + } + } + cursor = next; + } + const auto ms = std::chrono::duration(std::chrono::steady_clock::now() - begin).count(); + stats.resolveMs += ms; + stats.resolveMaxMs = std::max(stats.resolveMaxMs, ms); + reportGroupCapture(submission.queue); +} + +void Driver::reportGroupCapture(std::uint32_t queue) { + static const bool profile = std::getenv("APS5_PROFILE_DRAW") != nullptr; + if (!profile) return; + auto& stats = groupCaptureStats(); + const auto now = std::chrono::steady_clock::now(); + if (now - stats.reported < std::chrono::seconds(10)) return; + AgcDriver::ProfilePrint_nid_no_patch("[groupcap] queue 0x%x (10 s): %llu runs resolved %llu dispatches (%llu cache hits) in %.1f ms (max %.2f ms); %llu adopted %.2f ms after the resolve on average, max %.2f ms; mismatches: address %llu key %llu; %llu resolve failures, %llu stale, %llu label overlaps, %llu read an earlier dispatch's output, %llu rejected by the device\n", + queue, static_cast(stats.runs), static_cast(stats.resolved), static_cast(stats.resolvedHits), stats.resolveMs, stats.resolveMaxMs, + static_cast(stats.adopted), stats.adopted != 0 ? stats.ageMs / static_cast(stats.adopted) : 0.0, stats.ageMaxMs, static_cast(stats.addressMismatches), static_cast(stats.keyMismatches), + static_cast(stats.failures), static_cast(stats.stale), static_cast(stats.labelOverlaps), static_cast(stats.writtenBefore), static_cast(stats.deviceRejects)); + stats = GroupCaptureStats{}; + stats.reported = now; +} + +} diff --git a/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Dispatch/DispatchInsert.cpp b/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Dispatch/DispatchInsert.cpp index 459a69cf9..8478e84a6 100644 --- a/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Dispatch/DispatchInsert.cpp +++ b/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Dispatch/DispatchInsert.cpp @@ -6,6 +6,10 @@ namespace AgcDriver::DriverDetail { +bool CacheableResult(const ShaderRecompiler::RecompileResult& compiled) { + return compiled.poisonedSrtReads == 0u; +} + void Driver::insertDispatch(std::uint64_t address, std::uint64_t key, bool noDispatchCache, bool profile, const std::shared_ptr& registeredShader, std::uint64_t forgetAtCapture, std::span memory, const std::shared_ptr& shaderMemory, const std::vector& captured, const std::shared_ptr& capture, const std::shared_ptr& compiledResult, const std::shared_ptr& missedEntry, bool missedDiffering, std::shared_ptr& attachVariant, DispatchPhaseTiming& phaseTiming) { if (!noDispatchCache) { auto fresh = std::make_shared(); diff --git a/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Dispatch/IndirectDispatch.cpp b/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Dispatch/IndirectDispatch.cpp index 42efc40f8..9be819957 100644 --- a/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Dispatch/IndirectDispatch.cpp +++ b/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Dispatch/IndirectDispatch.cpp @@ -20,7 +20,7 @@ void Driver::countIndirect(int path, double readMs) { lastReport = now; std::uint64_t cpu = 0; for (int i = 1; i < IndirectPaths; ++i) cpu += counts[i]; - std::fprintf(stderr, "[indirect] gpu-side %llu, cpu-side %llu (thread dimensions %llu, fill kernel %llu, pending image results %llu, pending label or copied write %llu, not imported %llu, misaligned %llu, disabled %llu); CPU argument reads took %.1f s\n", static_cast(counts[IndirectGpu]), static_cast(cpu), static_cast(counts[IndirectThreadDimensions]), static_cast(counts[IndirectFillKernel]), static_cast(counts[IndirectPendingImage]), static_cast(counts[IndirectCopiedWrite]), static_cast(counts[IndirectNotImported]), static_cast(counts[IndirectMisaligned]), static_cast(counts[IndirectDisabled]), readWaitedMs / 1000); + std::fprintf(stderr, "[indirect] gpu-side %llu, cpu-side %llu (thread dimensions %llu, fill kernel %llu, pending image results %llu, pending label or copied write %llu, not imported %llu, misaligned %llu, disabled %llu, device memory LDS %llu); CPU argument reads took %.1f s\n", static_cast(counts[IndirectGpu]), static_cast(cpu), static_cast(counts[IndirectThreadDimensions]), static_cast(counts[IndirectFillKernel]), static_cast(counts[IndirectPendingImage]), static_cast(counts[IndirectCopiedWrite]), static_cast(counts[IndirectNotImported]), static_cast(counts[IndirectMisaligned]), static_cast(counts[IndirectDisabled]), static_cast(counts[IndirectWorkgroupMemory]), readWaitedMs / 1000); } void Driver::dispatchIndirect(QueueState& queue, std::span packet, const Submission& submission) { @@ -32,6 +32,14 @@ void Driver::dispatchIndirect(QueueState& queue, std::span else if ((initiator & 0x20u) != 0) path = IndirectThreadDimensions; else if (arguments % 4 != 0) path = IndirectMisaligned; if (path != IndirectGpu) { + if (!resolvingAhead()) { + const auto& resolved = resolvedAhead(); + if (const auto found = resolved.find(currentPacketOffset()); found != resolved.end() && found->second.indirectArguments == 0 && found->second.packet[0] == 0xc0031500u) { + const auto direct = found->second.packet; + dispatch(queue, direct, submission); + return; + } + } recordQueuedLabelsBeforeRead(submission.queue); const auto readStart = std::chrono::steady_clock::now(); const auto direct = Pm4::ReadDispatchArguments(arguments, initiator); diff --git a/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Draw/Draw.cpp b/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Draw/Draw.cpp index bf6ca03d1..6ab1a5cb5 100644 --- a/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Draw/Draw.cpp +++ b/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Draw/Draw.cpp @@ -144,7 +144,7 @@ DrawVerdict Driver::draw(QueueState& queue, std::span packe ShaderMemory shaderMemory(memory, &queryPendingWrite, &observePendingWrite, hookWaitCounter()); std::vector results; std::vector stages; - results.reserve(programs.size() + (graphics.rectList ? 2u : 0u)); + results.reserve(programs.size() + 2u); stages.reserve(programs.size()); std::uint32_t pushCursorBytes = 0; @@ -179,17 +179,7 @@ DrawVerdict Driver::draw(QueueState& queue, std::span packe phaseTiming.Phase(DrawRowDecode); } - static const bool indxOffsetSkipFold = std::getenv("APS5_INDX_OFFSET_SKIP_FOLD") != nullptr; - static const bool indexedOffsetFold = std::getenv("APS5_NO_INDEXED_OFFSET_FOLD") == nullptr; - const auto fold = [&](const ShaderRecompiler::RecompileResult& main, Pm4::DrawParameters& parameters) { - if (parameters.indexed && !indexedOffsetFold) return; - if (main.vertexOffsetSgpr >= 0 && (parameters.firstVertex == 0 || !indxOffsetSkipFold)) { - const auto offset = drawUserWord(programs.front(), main.vertexOffsetSgpr); - require(offset <= std::numeric_limits::max() - parameters.firstVertex, "draw vertex offset overflow"); - parameters.firstVertex += offset; - } - if (main.instanceOffsetSgpr >= 0) parameters.firstInstance = drawUserWord(programs.front(), main.instanceOffsetSgpr); - }; + const auto fold = [&](const ShaderRecompiler::RecompileResult& main, Pm4::DrawParameters& parameters) { FoldDrawOffsets(main, programs.front(), parameters); }; std::optional indirectCpu; std::vector pushOffsets(programs.size(), 0); @@ -197,6 +187,7 @@ DrawVerdict Driver::draw(QueueState& queue, std::span packe for (std::size_t i = 0; i < programs.size(); ++i) { if (roles[i] == Role::GeometryBack) continue; const auto& program = programs[i]; + pushCursorBytes = Graphics::StagePushOffset(pushCursorBytes, program.binary.stage, localDevice->GraphicsPipelineLibraries()); pushOffsets[i] = pushCursorBytes; if (drawHit) { @@ -210,11 +201,12 @@ DrawVerdict Driver::draw(QueueState& queue, std::span packe } const auto& result = *programResults[i]; if (i == 0 && drawParameters.indirect) { - indirectCpu = classifyIndirectDraw(result, graphics, programs.front(), localDevice, drawParameters, traceIndirect); + indirectCpu = ClassifyIndirectDraw(result, graphics, programs.front(), localDevice, drawParameters, traceIndirect); } else if (i == 0) { fold(result, drawParameters); } - require(result.pushConstants.size() <= Graphics::PipelinePushConstantBytes - pushCursorBytes, "stage push constants exceed the pipeline push constant block"); + const auto slotEnd = (pushCursorBytes / Graphics::PipelinePushSlotBytes + 1u) * Graphics::PipelinePushSlotBytes; + require(result.pushConstants.size() <= slotEnd - pushCursorBytes, "stage push constants exceed the pipeline push constant block"); stages.push_back({program.binary.stage, &result, result.pushConstants.empty() ? 0u : pushCursorBytes}); pushCursorBytes += static_cast(result.pushConstants.size()); } @@ -234,7 +226,7 @@ DrawVerdict Driver::draw(QueueState& queue, std::span packe const auto buildRectList = [&] { phaseTiming.Phase(DrawRowVectors); require(programs.size() == 2 && programResults[0] != nullptr && programResults[1] != nullptr, "rect-list requires vertex and fragment programs"); - auto rectangle = PreparedRectangle(*programs[0].snapshot, programResults[0]->variantId, programResults[1]->variantId); + auto rectangle = DrawRectangle(*programs[0].snapshot, programs[1].snapshot, programResults[0]->variantId, programResults[1]->variantId, localDevice->Target()); if (rectListBuilt) { results[rectIndex] = std::move(rectangle.control); results[rectIndex + 1] = std::move(rectangle.evaluation); @@ -250,6 +242,11 @@ DrawVerdict Driver::draw(QueueState& queue, std::span packe phaseTiming.Phase(DrawRowRectList); }; if (graphics.rectList) buildRectList(); + if (!programs.empty() && programResults.back() != nullptr && programResults.back()->barycentricEmulation.active) { + require(!graphics.rectList && graphics.stages.path == Graphics::ShaderPath::Vertex && programs.size() == 2 && programResults[0] != nullptr && stages.size() == 2, "a pixel shader that reads barycentrics without VK_KHR_fragment_shader_barycentric needs a vertex shader before it"); + results.push_back(ShaderRecompiler::BuildBarycentricGeometryShader(*programResults[0], *programResults[1], localDevice->Target(), localDevice->GeometryLimits())); + stages.insert(stages.begin() + 1, Graphics::CompiledShader{Stage::Geometry, &results.back(), 0}); + } std::vector snapshots; const auto snapshot = [&] { snapshots.clear(); diff --git a/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Draw/DrawCapture.cpp b/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Draw/DrawCapture.cpp index 97f85726f..d6d026478 100644 --- a/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Draw/DrawCapture.cpp +++ b/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Draw/DrawCapture.cpp @@ -20,7 +20,7 @@ ShaderRecompiler::RecompileResult Driver::materializeDrawStage(std::size_t i, st program.binary, {waveSize, program.firstUserSgpr, program.userData, std::nullopt, program.binary.stage == Stage::Fragment ? std::optional(pixel) : std::nullopt, vertexInfos[i], memory, RegisteredFloatMode(*program.snapshot)}, localDevice->Target(), - {0, 0, pushOffset, (graphics.stages.mesh ? ShaderRecompiler::MeshDrawPushOffsetBytes : Graphics::PipelinePushConstantBytes) - pushOffset}, + {0, 0, pushOffset, (graphics.stages.mesh ? ShaderRecompiler::MeshDrawPushOffsetBytes : Graphics::PipelinePushSlotBytes) - pushOffset % Graphics::PipelinePushSlotBytes}, ShaderRecompiler::GraphicsCompileContext{program.firstUserSgpr, linked, graphics.stages.mesh, graphics.stages.tessellation, {drawParameters.indexAddress, drawParameters.indexCount, drawParameters.indexSize, drawParameters.instanceCount}} }; const auto waitedBefore = traceCapSync() || profile ? Graphics::Recorder::ThreadWaitedMs() : 0.0; @@ -88,7 +88,7 @@ void Driver::cacheDrawStages(bool useDrawEntries, bool drawHit, const Pm4::DrawP std::uint64_t unstable = 0, mismatches = 0; for (std::size_t i = 0; i < programs.size(); ++i) { const auto& stageCapture = stageCaptures[i]; - if (stageCapture.compiled == nullptr) continue; + if (stageCapture.compiled == nullptr || !CacheableResult(*stageCapture.compiled)) continue; auto variant = std::make_shared(); variant->compiled = stageCapture.compiled; variant->shader = programs[i].snapshot; diff --git a/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Draw/DrawDecode.cpp b/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Draw/DrawDecode.cpp index fac2d2bdc..a34853b83 100644 --- a/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Draw/DrawDecode.cpp +++ b/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Draw/DrawDecode.cpp @@ -35,8 +35,8 @@ void DecodeGraphicsPrograms(DrawDecode& decoded, const QueueState& queue, const require(address - snapshot.codeAddress < snapshot.code.size() * sizeof(std::uint32_t), "graphics program is outside registered shader code"); require((address - snapshot.codeAddress) % sizeof(std::uint32_t) == 0, "graphics entry point is not dword aligned"); require(snapshot.type == type, "graphics program refers to an incompatible shader binary type"); - Graphics::NoteRegisterRead(Graphics::RegisterBank::Shader, rsrc2); - const auto resources = nullPixel && !queue.shader.contains(rsrc2) ? 0u : ReadGraphicsRegister(queue.shader, rsrc2); + if (!nullPixel) Graphics::NoteRegisterRead(Graphics::RegisterBank::Shader, rsrc2); + const auto resources = nullPixel ? 0u : ReadGraphicsRegister(queue.shader, rsrc2); const auto userCount = ((resources >> 1u) & 0x1fu) | (((resources >> 27u) & 1u) << 5u); require(userCount <= 32, "graphics user SGPR count exceeds the register bank"); const auto codeOffset = static_cast((address - snapshot.codeAddress) / sizeof(std::uint32_t)); @@ -116,6 +116,8 @@ std::shared_ptr Driver::decodeDraw(const QueueState& queue, const Su product->state = Graphics::DecodeState(queue); DecodeGraphicsPrograms(*product, queue, *submission.shaders, false, true); product->pixel = Graphics::DecodePixelStageInfo(queue.context, Graphics::ExportMappings(product->state), Graphics::PixelProgramSkipped(queue)); + product->pixel.targetExportPacking = Graphics::ExportPackings(product->state); + product->pixel.dualSourceBlend = product->state.dualSourceBlend; return product; } diff --git a/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Draw/DrawLookup.cpp b/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Draw/DrawLookup.cpp index 8d42a6ae6..0df4edd20 100644 --- a/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Draw/DrawLookup.cpp +++ b/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Draw/DrawLookup.cpp @@ -66,6 +66,8 @@ void Driver::lookupDraw(const Submission& submission, const std::shared_ptr(value)); + mix(pixel.dualSourceBlend); std::lock_guard cacheLock(drawCacheMutex); ++drawEntryCounters.lookups; const auto found = drawCache.find(drawKey); diff --git a/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Draw/DrawPrecheck.cpp b/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Draw/DrawPrecheck.cpp index e75ed53ee..03a3f4533 100644 --- a/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Draw/DrawPrecheck.cpp +++ b/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Draw/DrawPrecheck.cpp @@ -35,8 +35,6 @@ std::optional Driver::precheckDraw(const QueueState& queue, const S if (!colorWrites && !queue.shader.contains(0x8)) { const auto word = [&](std::uint32_t offset) { const auto it = queue.context.find(offset); return it == queue.context.end() ? 0u : it->second; }; if ((word(0x200) & 3u) == 0 || ((word(0x010) & 3u) == 0 && (word(0x011) & 1u) == 0)) return DrawVerdict::Nothing; - rejected = "AGC graphics: depth/stencil-only draws without a pixel shader are not implemented"; - return DrawVerdict::Rejected; } } if (drawParameters.indexed) { diff --git a/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Draw/DrawRegisterKey.cpp b/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Draw/DrawRegisterKey.cpp index 29284743d..36481c410 100644 --- a/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Draw/DrawRegisterKey.cpp +++ b/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Draw/DrawRegisterKey.cpp @@ -105,6 +105,7 @@ bool Driver::sameDecode(const DrawDecode& a, const DrawDecode& b) { const auto& y = b.programs[i]; if (x.binary.stage != y.binary.stage || x.binary.codeAddress != y.binary.codeAddress || x.userDataBase != y.userDataBase || x.firstUserSgpr != y.firstUserSgpr || x.userData != y.userData || x.snapshot != y.snapshot || x.codeOffset != y.codeOffset) return false; } + if (p.targetExportPacking != q.targetExportPacking || p.dualSourceBlend != q.dualSourceBlend) return false; return true; } diff --git a/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Draw/IndirectDraw.cpp b/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Draw/IndirectDraw.cpp index a9ebfa025..21d38fdea 100644 --- a/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Draw/IndirectDraw.cpp +++ b/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Draw/IndirectDraw.cpp @@ -12,7 +12,16 @@ std::uint32_t Driver::drawUserWord(const DrawProgram& program, std::int32_t sgpr return program.userData[index]; } -std::optional Driver::classifyIndirectDraw(const ShaderRecompiler::RecompileResult& result, const Graphics::State& graphics, const DrawProgram& frontProgram, const std::shared_ptr& localDevice, Pm4::DrawParameters& drawParameters, bool traceIndirect) { +void Driver::FoldDrawOffsets(const ShaderRecompiler::RecompileResult& result, const DrawProgram& program, Pm4::DrawParameters& parameters) { + if (result.vertexOffsetSgpr >= 0) { + const auto offset = drawUserWord(program, result.vertexOffsetSgpr); + require(offset <= std::numeric_limits::max() - parameters.firstVertex, "draw vertex offset overflow"); + parameters.firstVertex += offset; + } + if (result.instanceOffsetSgpr >= 0) parameters.firstInstance = drawUserWord(program, result.instanceOffsetSgpr); +} + +std::optional Driver::ClassifyIndirectDraw(const ShaderRecompiler::RecompileResult& result, const Graphics::State& graphics, const DrawProgram& frontProgram, const std::shared_ptr& localDevice, Pm4::DrawParameters& drawParameters, bool traceIndirect) { std::optional indirectCpu; auto& indirect = *drawParameters.indirect; using Rule = Pm4::DrawParameters::IndirectDraw::Rule; diff --git a/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Memory/BufferFill.cpp b/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Memory/BufferFill.cpp index 2d614b5b4..d0b2593e3 100644 --- a/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Memory/BufferFill.cpp +++ b/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Memory/BufferFill.cpp @@ -1,16 +1,22 @@ #include "prx/libSceAgcDriver/Execution/include/Driver/Driver.hpp" +#include "prx/libSceAgcDriver/Execution/include/Driver/Memory/BufferFill.hpp" #include "ThreadOwned.hpp" #include "prx/libSceAgcDriver/Execution/include/Driver/Diagnostics.hpp" +#include "prx/libSceAgcDriver/Execution/include/Driver/Memory/DwordPatternFill.hpp" #include "prx/libSceAgcDriver/Execution/include/GuestMemory.hpp" +#include #include #include +#include #include namespace AgcDriver::DriverDetail { bool Driver::matchesFillKernel(std::span code, const std::vector& userData, const ShaderRecompiler::ShaderComputeStageInfo& compute) { static const bool enabled = std::getenv("APS5_NO_FILL_HLE") == nullptr; + static const bool patternEnabled = std::getenv("APS5_NO_PATTERN_FILL_HLE") == nullptr; if (!enabled || userData.size() < 8 || compute.numThreads[0] != 64 || compute.numThreads[1] != 1 || compute.numThreads[2] != 1) return false; + if (patternEnabled && MatchesPatternFillKernel(code, userData, compute)) return true; static constexpr std::array fillKernel{0xd7460004u, 0x04010c08u, 0x7e000204u, 0x7e020205u, 0x7e040206u, 0x7e060207u, 0xe01c2000u, 0x80000004u, 0xbf810000u}; if (code.size() < fillKernel.size() || !std::equal(fillKernel.begin(), fillKernel.end(), code.begin())) return false; @@ -23,6 +29,34 @@ bool Driver::matchesFillKernel(std::span code, const std::v return type == 0 && stride == 16 && !swizzled && !addTid && dstSel == 0xfacu && format == 0x4bu; } +std::optional MatchDwordPatternFill(std::span packet, std::span code, const std::vector& userData, const ShaderRecompiler::ShaderComputeStageInfo& compute) { + static const bool enabled = std::getenv("APS5_NO_FILL_HLE") == nullptr; + if (!enabled || userData.size() != 10 || packet.size() < 5 || !compute.groupIdEnable[0] || compute.numThreads[0] != 64 || compute.numThreads[1] != 1 || compute.numThreads[2] != 1) return std::nullopt; + static constexpr std::array kernel{ + 0xbfa00003u, 0xd7460002u, 0x04010c0au, 0x7da80408u, 0xbf88003fu, 0x7e000c09u, 0xbf070980u, 0x858a807eu, + 0x7e005700u, 0x100000ffu, 0x4f800000u, 0x7e060f00u, 0xd5766a00u, 0x02020609u, 0x7d8a0280u, 0x4c020080u, + 0x02000101u, 0xd56a0001u, 0x00020700u, 0x4c000303u, 0x4a020303u, 0x02000101u, 0xd56a0000u, 0x00020500u, + 0xd5690001u, 0x00020009u, 0x4c060302u, 0x7d860609u, 0x7d8c02f9u, 0x06068c02u, 0x87ea6a0cu, 0x50000080u, + 0xd5286a00u, 0x003200c1u, 0xd5010000u, 0x002a00c1u, 0xd5690000u, 0x00020009u, 0x4c000102u, 0x7d0a0080u, + 0xbe88246au, 0xbf880015u, 0x7d0a0081u, 0xbe8a246au, 0xbf88000cu, 0x7d0a0082u, 0xbeea246au, 0xbf880003u, + 0x7e000207u, 0xe0102000u, 0x80000002u, 0x8afe7e6au, 0xbf880003u, 0x7e000206u, 0xe0102000u, 0x80000002u, + 0xbefe046au, 0x8afe7e0au, 0xbf880003u, 0x7e000205u, 0xe0102000u, 0x80000002u, 0xbefe040au, 0x8afe7e08u, + 0xbf880003u, 0x7e000204u, 0xe0102000u, 0x80000002u, 0xbf810000u}; + if (code.size() < kernel.size() || !std::equal(kernel.begin(), kernel.end(), code.begin())) return std::nullopt; + const auto stride = (userData[1] >> 16u) & 0x3fffu; + const bool swizzled = ((userData[1] >> 31u) & 1u) != 0; + const bool addTid = ((userData[3] >> 23u) & 1u) != 0; + const auto type = userData[3] >> 30u; + const auto period = userData[9]; + if (type != 0 || stride != 4 || swizzled || addTid || (period != 1u && period != 2u && period != 4u) || (packet[4] & 0x20u) != 0 || packet[2] != 1 || packet[3] != 1) return std::nullopt; + const auto dwords = std::min({static_cast(packet[1]) * 64u, userData[8], userData[2]}); + const auto base = userData[0] | (static_cast(userData[1] & 0xffffu) << 32u); + if (dwords == 0 || dwords % 4u != 0 || base % 16u != 0) return std::nullopt; + DwordPatternFill fill{base, static_cast(dwords * 4u), {}}; + for (std::uint32_t i = 0; i < 4; ++i) fill.pattern[i] = userData[4 + i % period]; + return fill; +} + bool Driver::fillClearEnabled() { static const bool enabled = std::getenv("APS5_NO_FILL_CLEAR") == nullptr; return enabled; @@ -88,20 +122,49 @@ void Driver::fillClearCount(const Graphics::StorageTexture::FillCoverage& covera } bool Driver::fillBuffer(QueueState& queue, std::uint32_t queueId, std::span packet, std::span code, const std::vector& userData, const ShaderRecompiler::ShaderComputeStageInfo& compute, const std::shared_ptr& localDevice) { - if (!matchesFillKernel(code, userData, compute)) return false; - const auto numRecords = userData[2]; - std::array groups{packet[1], packet[2], packet[3]}; - if ((packet[4] & 0x20u) != 0) { - for (std::uint32_t axis = 0; axis < 3; ++axis) { - const auto threads = std::max(readRegister(queue.shader, 0x207 + axis) & 0xffffu, 1u); - groups[axis] = (groups[axis] + threads - 1) / threads; + const auto dwordFill = MatchDwordPatternFill(packet, code, userData, compute); + if (!dwordFill && !matchesFillKernel(code, userData, compute)) return false; + std::uint64_t base; + std::size_t bytes; + std::array pattern; + std::unique_lock inputLock(GuestMemory::GpuMutex(), std::defer_lock); + if (dwordFill) { + base = dwordFill->base; + bytes = dwordFill->bytes; + pattern = dwordFill->pattern; + } else if (MatchesPatternFillKernel(code, userData, compute)) { + const auto range = DecodePatternFillRange(userData, packet); + if (!range) return false; + if (range->invocations == 0) return true; + GuestMemory::TagGpuLockSite(GuestMemory::GpuLockSite::Fill); + inputLock.lock(); + recordLabelsForPacket(localDevice.get(), queueId); + if (!GuestMemory::Accessible(reinterpret_cast(range->control), 8)) return false; + std::array control; + GuestMemory::Read(range->control, std::as_writable_bytes(std::span(control)), 4); + const auto count = range->UniformBytes(control[0], control[1]); + if (!count) return false; + if (*count == 0) return true; + if (!GuestMemory::Accessible(reinterpret_cast(range->source), 4)) return false; + GuestMemory::Read(range->source, std::as_writable_bytes(std::span(pattern).first(1)), 4); + std::fill(pattern.begin() + 1, pattern.end(), pattern[0]); + base = range->destination; + bytes = *count; + } else { + const auto numRecords = userData[2]; + std::array groups{packet[1], packet[2], packet[3]}; + if ((packet[4] & 0x20u) != 0) { + for (std::uint32_t axis = 0; axis < 3; ++axis) { + const auto threads = std::max(readRegister(queue.shader, 0x207 + axis) & 0xffffu, 1u); + groups[axis] = (groups[axis] + threads - 1) / threads; + } } + if (groups[1] != 1 || groups[2] != 1) return false; + const auto records = std::min(static_cast(groups[0]) * 64u, numRecords); + base = userData[0] | (static_cast(userData[1] & 0xffffu) << 32u); + bytes = static_cast(records * 16u); + pattern = {userData[4], userData[5], userData[6], userData[7]}; } - if (groups[1] != 1 || groups[2] != 1) return false; - const auto records = std::min(static_cast(groups[0]) * 64u, numRecords); - const auto base = userData[0] | (static_cast(userData[1] & 0xffffu) << 32u); - const auto bytes = static_cast(records * 16u); - const std::array pattern{userData[4], userData[5], userData[6], userData[7]}; static const bool profile = std::getenv("APS5_PROFILE_DRAW") != nullptr; static std::atomic fills{0}, filledBytes{0}, cpuFills{0}; diff --git a/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Memory/WrittenBuffers.cpp b/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Memory/WrittenBuffers.cpp index b9b65463f..682b28dca 100644 --- a/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Memory/WrittenBuffers.cpp +++ b/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Memory/WrittenBuffers.cpp @@ -20,6 +20,10 @@ void Driver::forEachWrittenBuffer(const ShaderRecompiler::RecompileResult& compi } } +void Driver::appendWrittenRanges(const ShaderRecompiler::RecompileResult& compiled, std::vector>& ranges) { + forEachWrittenBuffer(compiled, [&](std::uint32_t, std::uint64_t begin, std::uint64_t end, bool) { ranges.emplace_back(begin, end); }); +} + void Driver::noteWrittenBuffers(std::uint64_t program, std::uint32_t queue, const ShaderRecompiler::RecompileResult& compiled) { std::lock_guard lock(writtenBuffersMutex); const auto serial = ++writtenBufferSerial; diff --git a/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Packets/PacketExecution.cpp b/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Packets/PacketExecution.cpp index 3a2b72269..f462b90f8 100644 --- a/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Packets/PacketExecution.cpp +++ b/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Packets/PacketExecution.cpp @@ -31,6 +31,16 @@ void Driver::timed(double WorkerProfile::*bucket, TWork&& work) { void Driver::execute(const Submission& submission) { auto* submissionTiming = includeTimingSubmission(submission, true); + struct FlipHold { + Driver& driver; + bool held; + void Release() { + if (!held) return; + held = false; + driver.releaseFlipHold(); + } + ~FlipHold() { Release(); } + } flipHold{*this, submission.holdsFlip}; if (submission.suspend) { PerformanceContext timingContext(submissionTiming); PerformanceTimer timing("Driver.Suspend"); @@ -44,8 +54,9 @@ void Driver::execute(const Submission& submission) { const auto start = profile ? std::chrono::steady_clock::now() : std::chrono::steady_clock::time_point{}; GuestMemory::TagGpuLockSite(GuestMemory::GpuLockSite::Flush); std::lock_guard gpuLock(GuestMemory::GpuMutex()); - if (const auto localDevice = device.Load()) { - recordDeferredLabels(localDevice.get(), submission.queue); + const auto localDevice = device.Load(); + recordDeferredLabels(localDevice.get(), submission.queue); + if (localDevice != nullptr) { if (suspendDrain) localDevice->WaitIdle(); else localDevice->SubmitRecorded(submission.queue == 0); } @@ -98,6 +109,11 @@ void Driver::execute(const Submission& submission) { ++packetProfile.submissions; bumpEpoch(&EpochBumps::submissions); + clearResolvedAhead(); + gateOpened() = true; + struct ClearAhead { + ~ClearAhead() { clearResolvedAhead(); } + } clearAhead; for (std::size_t cursor = 0; cursor < submission.commands.size();) { if (packetEpoch()) bumpEpoch(&EpochBumps::packets); CheckFailure(); @@ -117,6 +133,7 @@ void Driver::execute(const Submission& submission) { GuestMemory::SetCurrentPacket(header == FlipPacketHeader ? 0xffffu : opcode, submission.queue); CaptureTrace::Log("packet submission=%llu queue=%x offset=%zu header=%08x words=%zu", static_cast(submission.serial), submission.queue, cursor, header, packet.size()); if (Pm4::Predicated(header) && queue.predication.operation != 0) { + landPendingWaits(submission.queue); recordQueuedLabelsBeforeRead(submission.queue); if (!Pm4::PredicationPasses(queue)) { cursor = opcode == 0x3f ? submission.conditionalEnds.at(cursor) : nextCursor; @@ -211,19 +228,24 @@ void Driver::execute(const Submission& submission) { if (APS5_ENABLE_TIMING_LOG) frame->CollectBackground(); frame->NoteFlipBatches(batchesAtFlip, unsignaledAtFlip); CaptureTrace::Log("flip frame=%llu submission=%llu offset=%zu batch=%llu unsignaled=%llu", static_cast(frameSerial), static_cast(submission.serial), cursor, static_cast(batchesAtFlip), static_cast(unsignaledAtFlip)); + flipHold.Release(); submission.flips.at(cursor)->GpuReady(frame); - } else if (opcode == 0x15) { - timed(&WorkerProfile::dispatchMs, [&] { dispatch(queue, packet, submission); }); + } else if (opcode == 0x15 || opcode == 0x16) { + landPendingWaits(submission.queue); + currentPacketOffset() = cursor; + if (gateOpened()) { + gateOpened() = false; + timed(&WorkerProfile::dispatchMs, [&] { resolveGroupAhead(submission, queue, cursor); }); + } + if (opcode == 0x15) timed(&WorkerProfile::dispatchMs, [&] { dispatch(queue, packet, submission); }); + else timed(&WorkerProfile::dispatchMs, [&] { dispatchIndirect(queue, packet, submission); }); Graphics::Recorder::CountRecordedWork(); - finishDispatchPacket(false); - } else if (opcode == 0x16) { - timed(&WorkerProfile::dispatchMs, [&] { dispatchIndirect(queue, packet, submission); }); - Graphics::Recorder::CountRecordedWork(); - finishDispatchPacket(true); + finishDispatchPacket(opcode == 0x16); } else if (opcode == 0x3c || opcode == 0x93) { static const bool traceGpu = std::getenv("APS5_TRACE_GPU") != nullptr; const auto waitStart = std::chrono::steady_clock::now(); timed(&WorkerProfile::waitMs, [&] { waitMemory(packet, submission.queue, recent, submission.received, submission.heldAtSubmit.contains(cursor)); }); + gateOpened() = true; if (traceGpu && std::chrono::steady_clock::now() - waitStart > std::chrono::milliseconds(200)) { for (std::size_t next = cursor + count, shown = 0; next < submission.commands.size() && shown < 48; ++shown) { @@ -237,6 +259,7 @@ void Driver::execute(const Submission& submission) { } } } else if (drawPacket) { + landPendingWaits(submission.queue); bool drawn = false; timed(&WorkerProfile::drawMs, [&] { static const bool traceDraws = std::getenv("APS5_TRACE_DRAWS") != nullptr; @@ -268,6 +291,7 @@ void Driver::execute(const Submission& submission) { }); finishDrawPacket(drawn); } else if (opcode == 0x22) { + landPendingWaits(submission.queue); recordQueuedLabelsBeforeRead(submission.queue); const auto condition = Pm4::ReadCondition(packet); if (condition == 0) nextCursor = submission.conditionalEnds.at(cursor); @@ -279,7 +303,7 @@ void Driver::execute(const Submission& submission) { if (Pm4::IndirectRegisterOpcode(opcode)) Pm4::ExecuteIndirectRegisters(packet, submission.registerLists.at(cursor), queue); else Pm4::Execute(packet, queue); if (opcode == 0x49 || opcode == 0x37) { - if (const auto label = Pm4::DecodeLabelWrite(packet)) noteLabelStore(label->address, label->Bytes(), ++eventSerial); + if (const auto label = Pm4::DecodeLabelWrite(packet)) noteLabelStore(label->address, label->Bytes(), ++eventSerial, submission.queue); } } if (endOfPipeInterrupt && !interruptDeferred) AgcDriverDeliverEopInterrupt(submission.queue); diff --git a/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Packets/PacketMemory.cpp b/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Packets/PacketMemory.cpp index de4b778f5..15273ae87 100644 --- a/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Packets/PacketMemory.cpp +++ b/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Packets/PacketMemory.cpp @@ -18,8 +18,8 @@ bool Driver::preparePacketMemory(const Submission& submission, QueueState& queue bool orderedAlready = false; - const auto interruptSelect = (packet[2] >> 24u) & 7u; - endOfPipeInterrupt = opcode == 0x49 && interruptSelect != 0 && interruptSelect != 3; + const auto interruptSelect = opcode == 0x49 ? (packet[2] >> 24u) & 7u : 0u; + endOfPipeInterrupt = interruptSelect != 0 && interruptSelect != 3; interruptDeferred = false; if (!drainAll && endOfPipeInterrupt) { const auto label = Pm4::DecodeLabelWrite(packet); @@ -37,8 +37,8 @@ bool Driver::preparePacketMemory(const Submission& submission, QueueState& queue const auto stamp = ++eventSerial; reason = localDevice != nullptr ? localDevice->WriteLabelOnGpu(label->address, bytes, stamp, submission.queue) : 4; if (reason == 1) GuestMemory::Write(label->address, bytes, 4); - if (reason == 0 || reason == 1 || reason == 5 || reason == 6) { - noteLabelStore(label->address, bytes, stamp); + if (reason == 0 || reason == 1 || reason == 5) { + noteLabelStore(label->address, bytes, stamp, submission.queue); Graphics::Recorder::CloseLabelGroup(GuestMemory::TrackerGeneration()); } countLabelOutcome(reason); @@ -46,7 +46,7 @@ bool Driver::preparePacketMemory(const Submission& submission, QueueState& queue } else { ++noOpLabels; } - if (reason == 0 || reason == 5 || reason == 6) { + if (reason == 0 || reason == 5) { const auto queueId = submission.queue; interruptDeferred = localDevice->AfterRecordedWork([queueId] { AgcDriverDeliverEopInterrupt(queueId); }, submission.queue == 0); if (interruptDeferred && workOpen) localDevice->SubmitRecorded(submission.queue == 0); @@ -77,13 +77,13 @@ bool Driver::preparePacketMemory(const Submission& submission, QueueState& queue recordDeferredLabels(localDevice.get(), submission.queue); const auto stamp = ++eventSerial; const auto reason = localDevice != nullptr ? localDevice->WriteLabelOnGpu(label->address, bytes, stamp, submission.queue) : 4; - wroteOnGpu = reason == 0 || reason == 5 || reason == 6; + wroteOnGpu = reason == 0 || reason == 5; if (reason == 1) { GuestMemory::Write(label->address, bytes, 4); wroteOnGpu = true; } - if (wroteOnGpu) noteLabelStore(label->address, bytes, stamp); + if (wroteOnGpu) noteLabelStore(label->address, bytes, stamp, submission.queue); Graphics::Recorder::CloseLabelGroup(GuestMemory::TrackerGeneration()); countLabelOutcome(reason); ++immediateLabels; @@ -117,8 +117,8 @@ bool Driver::preparePacketMemory(const Submission& submission, QueueState& queue recordDeferredLabels(localDevice.get(), submission.queue); const auto stamp = ++eventSerial; const auto reason = localDevice != nullptr ? localDevice->WriteLabelOnGpu(store->address, bytes, stamp, submission.queue) : 4; - if (reason == 0 || reason == 1 || reason == 5 || reason == 6) noteLabelStore(store->address, bytes, stamp); - if (reason == 0 || reason == 5 || reason == 6) { + if (reason == 0 || reason == 1 || reason == 5) noteLabelStore(store->address, bytes, stamp, submission.queue); + if (reason == 0 || reason == 5) { if (reason == 0) ++storesOnGpu; else ++storesBehindCompletions; wroteOnGpu = true; diff --git a/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Queues/QueueWorker.cpp b/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Queues/QueueWorker.cpp index 9d9232a0a..9fa87efc4 100644 --- a/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Queues/QueueWorker.cpp +++ b/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Queues/QueueWorker.cpp @@ -165,8 +165,8 @@ void Driver::run(std::uint32_t id) noexcept { submission = Submission{}; } catch (...) { const auto error = std::current_exception(); - ReportFailure(error); for (const auto& [offset, flip] : submission.flips) flip->Fail(error); + ReportFailure(error); } } diff --git a/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Queues/Submission.cpp b/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Queues/Submission.cpp index 3df687bf6..eaa9f96ab 100644 --- a/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Queues/Submission.cpp +++ b/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Queues/Submission.cpp @@ -131,6 +131,34 @@ void Driver::waitForFlipRoom(const Submission& submission) { } } +bool Driver::awaitsTitle(std::uint64_t awaited) const { + return runningWorkers.load(std::memory_order_acquire) == 0 && orderHolders.load(std::memory_order_acquire) == 0 && !completionsPending() && !Graphics::Recorder::SnapshotWriteOverlaps(awaited, 4); +} + +void Driver::noteAwaitingTitle(std::uint32_t queue, std::uint64_t awaited) { + if (queue != 0 || flipHolders.load(std::memory_order_acquire) == 0 || queue0AwaitsTitle.load(std::memory_order_acquire) || !awaitsTitle(awaited)) return; + std::lock_guard lock(mutex); + queue0AwaitsTitle.store(true, std::memory_order_release); + changed.notify_all(); +} + +void Driver::holdFlipBehindWorker(const Submission& submission) { + if (submission.queue != 0 || onWorkerThread()) return; + bool flips = false; + for (std::size_t cursor = 0; cursor < submission.commands.size() && !flips; cursor += Pm4::PacketWords(submission.commands[cursor])) flips = submission.commands[cursor] == FlipPacketHeader; + if (!flips) return; + std::unique_lock lock(mutex); + flipHolders.fetch_add(1, std::memory_order_acq_rel); + changed.wait(lock, [&] { return failure != nullptr || stopping.load(std::memory_order_acquire) || shutdownToken.stop_requested() || flipsAhead.load(std::memory_order_acquire) == 0 || queue0AwaitsTitle.load(std::memory_order_acquire); }); + flipHolders.fetch_sub(1, std::memory_order_acq_rel); +} + +void Driver::releaseFlipHold() { + std::lock_guard lock(mutex); + flipsAhead.fetch_sub(1, std::memory_order_acq_rel); + if (flipHolders.load(std::memory_order_acquire) != 0) changed.notify_all(); +} + void Driver::reserveOutputs(Submission& submission) { for (std::size_t cursor = 0; cursor < submission.commands.size();) { const auto* words = submission.commands.data() + cursor; @@ -167,6 +195,7 @@ void Driver::executeRewindTail(const Submission& stalled) { while ((control.load(std::memory_order_acquire) & 0x80000000u) == 0) { CheckFailure(); checkStopping(); + noteAwaitingTitle(stalled.queue, reinterpret_cast(stalled.rewindTail - 1)); PollSleep(); } } @@ -218,6 +247,7 @@ void Driver::Submit(const Packet* packet, std::uint32_t queue) { readRegisterLists(submission); if (APS5_ENABLE_TIMING_LOG) submission.validatedAt = std::chrono::steady_clock::now(); waitForFlipRoom(submission); + holdFlipBehindWorker(submission); if (APS5_ENABLE_TIMING_LOG) submission.roomReadyAt = std::chrono::steady_clock::now(); static const bool trace = std::getenv("APS5_TRACE_GPU") != nullptr; if (trace) std::fprintf(stderr, "[gpu] %.1f submit queue=0x%x dwords=%zu at %p\n", TraceMs(), queue, submission.commands.size(), static_cast(descriptor.addr)); @@ -240,6 +270,10 @@ void Driver::Submit(const Packet* packet, std::uint32_t queue) { costs.copyNs += static_cast(std::chrono::duration_cast((copied - start) + (now - validated)).count()); } if (APS5_ENABLE_TIMING_LOG) submission.enqueuedAt = std::chrono::steady_clock::now(); + if (queue == 0 && !submission.flips.empty()) { + submission.holdsFlip = true; + flipsAhead.fetch_add(1, std::memory_order_acq_rel); + } enqueue(std::move(submission)); ++accepted; } @@ -299,6 +333,7 @@ void Driver::noteWaitBlocked(std::uint32_t queue, std::uint64_t awaited, bool bl if (blocked) runningWorkers.fetch_sub(1, std::memory_order_acq_rel); else runningWorkers.fetch_add(1, std::memory_order_acq_rel); if (queue == 0) queue0Awaited.store(blocked ? awaited : 0, std::memory_order_release); + if (queue == 0 && !blocked) queue0AwaitsTitle.store(false, std::memory_order_release); if (blocked && orderHolders.load(std::memory_order_acquire) != 0) { std::lock_guard lock(mutex); changed.notify_all(); diff --git a/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Shaders/ShaderRegistry.cpp b/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Shaders/ShaderRegistry.cpp index 109b381a7..52a46d03b 100644 --- a/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Shaders/ShaderRegistry.cpp +++ b/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Shaders/ShaderRegistry.cpp @@ -1,6 +1,5 @@ #include "prx/libc/include/general/LogMacros.hpp" #include "ControlFlow/GraphBuilder.hpp" -#include "ControlFlow/Structurizer.hpp" #include "prx/libSceAgcDriver/Execution/include/PerformanceTimer.hpp" #include "prx/libSceAgcDriver/Execution/include/Driver/Driver.hpp" #include "prx/libSceAgcDriver/Execution/include/Driver/Shaders/ShaderRegistry.hpp" @@ -11,14 +10,33 @@ #include "prx/libSceAgcDriver/Execution/include/ShaderPreparationScope.hpp" #include "CompiledVariant.hpp" #include "Optimization/ResourceProgram.hpp" +#include +#include +#include +#include #include #include +#include +#include #include +#include #include +#include #include "RdnaDecoder/RdnaInstructionDecoder.hpp" namespace AgcDriver::DriverDetail { +std::shared_ptr PrepareShaderWithDiagnostics(const ShaderRecompiler::RecompileRequest& request, ShaderRecompiler::ShaderPreparationContext* preparation) { + try { + return ShaderRecompiler::PrepareShader(request, preparation); + } catch (...) { + try { + if (std::getenv("APS5_DUMP_SHADERS") != nullptr) static_cast(Driver::dumpRequest(request.shader.codeAddress, request)); + } catch (...) {} + throw; + } +} + std::shared_ptr ReadRawComputeShader(std::uint64_t address) { GuestMemory::CheckRange(reinterpret_cast(address), sizeof(std::uint32_t), 256); static std::mutex cacheMutex; @@ -85,11 +103,22 @@ std::shared_ptr ReadRawComputeShader(std::uint64_t address throw std::runtime_error("AGC driver: raw compute program has no reachable end within mapped code or the size limit"); } +namespace { + +void AwaitRegisteredPreparation(PreparedShaders& prepared, std::unique_lock& lock) { + prepared.settled.wait(lock, [&] { return !prepared.pending; }); + if (prepared.failure != nullptr) std::rethrow_exception(prepared.failure); +} + +} struct ShaderPreparationTransaction::State { struct Change { std::shared_ptr destination; PreparedShaderState prepared; + bool edited = false; + std::size_t baseEntries = 0; + std::size_t baseAbis = 0; }; static std::mutex writers; static thread_local State* active; @@ -118,15 +147,23 @@ PreparedShaderState& ShaderPreparationTransaction::Edit(const ShaderSnapshot& sn require(!committed, "shader preparation transaction is already committed"); auto& changes = root->changes; const auto found = changes.find(snapshot.prepared.get()); - if (found != changes.end()) return found->second.prepared; - std::lock_guard lock(snapshot.prepared->mutex); - return changes.emplace(snapshot.prepared.get(), State::Change{snapshot.prepared, *snapshot.prepared}).first->second.prepared; + if (found != changes.end()) { + found->second.edited = true; + return found->second.prepared; + } + std::unique_lock lock(snapshot.prepared->mutex); + AwaitRegisteredPreparation(*snapshot.prepared, lock); + return changes.emplace(snapshot.prepared.get(), State::Change{snapshot.prepared, *snapshot.prepared, true, snapshot.prepared->entries.size(), snapshot.prepared->registeredAbis.size()}).first->second.prepared; } const PreparedShaderState& ShaderPreparationTransaction::Read(const ShaderSnapshot& snapshot) const { require(!committed, "shader preparation transaction is already committed"); - const auto found = root->changes.find(snapshot.prepared.get()); - return found != root->changes.end() ? found->second.prepared : *snapshot.prepared; + auto& changes = root->changes; + const auto found = changes.find(snapshot.prepared.get()); + if (found != changes.end()) return found->second.prepared; + std::unique_lock lock(snapshot.prepared->mutex); + AwaitRegisteredPreparation(*snapshot.prepared, lock); + return changes.emplace(snapshot.prepared.get(), State::Change{snapshot.prepared, *snapshot.prepared, false, snapshot.prepared->entries.size(), snapshot.prepared->registeredAbis.size()}).first->second.prepared; } void ShaderPreparationTransaction::Commit() { @@ -136,7 +173,16 @@ void ShaderPreparationTransaction::Commit() { locks.reserve(state->changes.size()); for (const auto& [key, change] : state->changes) locks.emplace_back(change.destination->mutex); for (auto& [key, change] : state->changes) { + if (!change.edited) continue; auto& destination = *change.destination; + for (auto index = std::min(change.baseEntries, destination.entries.size()); index < destination.entries.size(); ++index) { + const auto& entry = destination.entries[index]; + if (std::ranges::none_of(change.prepared.entries, [&](const auto& existing) { return existing.codeOffset == entry.codeOffset && existing.handle == entry.handle; })) change.prepared.entries.push_back(entry); + } + for (auto index = std::min(change.baseAbis, destination.registeredAbis.size()); index < destination.registeredAbis.size(); ++index) { + const auto& abi = destination.registeredAbis[index]; + if (std::ranges::find(change.prepared.registeredAbis, abi) == change.prepared.registeredAbis.end()) change.prepared.registeredAbis.push_back(abi); + } destination.entries.swap(change.prepared.entries); destination.registeredAbis.swap(change.prepared.registeredAbis); destination.graphicsAbis.swap(change.prepared.graphicsAbis); @@ -149,6 +195,18 @@ void ShaderPreparationTransaction::Commit() { committed = true; } +bool SameHeader(const ShaderSnapshot& snapshot, const Shader* shader) { + if (snapshot.headerAddress == reinterpret_cast(shader)) return true; + if (snapshot.header.size() < sizeof(Shader)) return false; + GuestMemory::CheckRange(shader, sizeof(Shader), 1); + Shader used; + Shader registered; + std::memcpy(&used, static_cast(shader), sizeof(Shader)); + std::memcpy(®istered, snapshot.header.data(), sizeof(Shader)); + used.user_data = registered.user_data; + return std::memcmp(&used, ®istered, offsetof(Shader, num_sh_registers) + sizeof(Shader::num_sh_registers)) == 0; +} + void PublishRegisteredShader(std::shared_ptr& registry, const std::shared_ptr& snapshot) { ShaderPreparationTransaction transaction; if (registry != nullptr) { @@ -167,18 +225,32 @@ void PublishRegisteredShader(std::shared_ptr& registry, const st transaction.Commit(); } +namespace { + +bool PreparedAtUse(const ShaderSnapshot& snapshot, const ShaderRecompiler::RecompileRequest& request) { + if (snapshot.codeAddress == NullPixelProgramAddress() && request.shader.stage == ShaderRecompiler::ShaderStage::Fragment && request.context.waveSize == 32u) { + APS5_LOG_ERR("The null pixel program has no wave%u artifact for this draw; preparing it at draw", request.context.waveSize); + return true; + } + if (!snapshot.header.empty()) return snapshot.prepared->deferred; + if (snapshot.type != 0 || request.shader.stage != ShaderRecompiler::ShaderStage::Compute) throw std::runtime_error("AGC driver: unregistered program is not a compute shader"); + APS5_LOG_ERR("Compute shader 0x%llx was not registered; preparing its artifact at dispatch", static_cast(snapshot.codeAddress)); + return true; +} + +} + std::shared_ptr SourceHandleFor(const ShaderSnapshot& snapshot, std::size_t codeOffset, const ShaderRecompiler::RecompileRequest& request) { struct PreparedKeyStorage {}; auto& key = HostThreadLocal, PreparedKeyStorage>(); ShaderRecompiler::BuildPreparedShaderKey(request, key); - std::lock_guard lock(snapshot.prepared->mutex); + std::unique_lock lock(snapshot.prepared->mutex); + AwaitRegisteredPreparation(*snapshot.prepared, lock); for (const auto& entry : snapshot.prepared->entries) { if (entry.codeOffset == codeOffset && ShaderRecompiler::MatchesPreparedShader(request, *entry.handle, key)) return entry.handle; } - if (snapshot.header.empty()) { - if (snapshot.type != 0 || request.shader.stage != ShaderRecompiler::ShaderStage::Compute) throw std::runtime_error("AGC driver: unregistered program is not a compute shader"); - APS5_LOG_ERR("Compute shader 0x%llx was not registered; preparing its artifact at dispatch", static_cast(snapshot.codeAddress)); - auto handle = ShaderRecompiler::PrepareShader(request); + if (PreparedAtUse(snapshot, request)) { + auto handle = PrepareShaderWithDiagnostics(request); snapshot.prepared->entries.push_back({codeOffset, handle}); return handle; } @@ -202,6 +274,17 @@ ShaderRecompiler::RectListShaders PreparedRectangle(const ShaderSnapshot& snapsh throw std::runtime_error("AGC driver: prepared rectangle artifacts are missing"); } +ShaderRecompiler::RectListShaders DrawRectangle(const ShaderSnapshot& front, const std::shared_ptr& fragment, std::uint64_t vertexId, std::uint64_t fragmentId, const ShaderRecompiler::SpirvTarget& target) { + const auto prepared = [&] { + std::lock_guard lock(front.prepared->mutex); + return std::ranges::any_of(front.prepared->rectangles, [&](const auto& entry) { return entry.vertexId == vertexId && entry.fragmentId == fragmentId; }); + }; + if (prepared()) return PreparedRectangle(front, vertexId, fragmentId); + APS5_LOG_ERR("Rect-list draw of shader 0x%llx has no rectangle for fragment shader 0x%llx; preparing it at draw", static_cast(front.codeAddress), static_cast(fragment->codeAddress)); + ResolvePreparedGraphics(front, fragment, 17, target); + return PreparedRectangle(front, vertexId, fragmentId); +} + ShaderRecompiler::PreparedShaderInvocation InvocationFor(const ShaderSnapshot& snapshot, std::size_t codeOffset, const ShaderRecompiler::RecompileRequest& request) { require(codeOffset < snapshot.code.size(), "prepared shader code offset is outside the snapshot"); const auto code = std::span(snapshot.code).subspan(codeOffset); @@ -210,16 +293,15 @@ ShaderRecompiler::PreparedShaderInvocation InvocationFor(const ShaderSnapshot& s struct PreparedKeyStorage {}; auto& key = HostThreadLocal, PreparedKeyStorage>(); ShaderRecompiler::BuildPreparedShaderKey(request, key); - std::lock_guard lock(snapshot.prepared->mutex); + std::unique_lock lock(snapshot.prepared->mutex); + AwaitRegisteredPreparation(*snapshot.prepared, lock); for (const auto& entry : snapshot.prepared->entries) { if (entry.codeOffset != codeOffset) continue; invocationRequest.shader.code = ShaderRecompiler::GetPreparedCode(*entry.handle); if (auto invocation = ShaderRecompiler::PreparedShaderInvocation::TryCreate(invocationRequest, entry.handle, key)) return std::move(*invocation); } - if (snapshot.header.empty()) { - if (snapshot.type != 0 || request.shader.stage != ShaderRecompiler::ShaderStage::Compute) throw std::runtime_error("AGC driver: unregistered program is not a compute shader"); - APS5_LOG_ERR("Compute shader 0x%llx was not registered; preparing its artifact at dispatch", static_cast(snapshot.codeAddress)); - auto handle = ShaderRecompiler::PrepareShader(request); + if (PreparedAtUse(snapshot, request)) { + auto handle = PrepareShaderWithDiagnostics(request); invocationRequest = request; invocationRequest.shader.code = ShaderRecompiler::GetPreparedCode(*handle); auto invocation = ShaderRecompiler::PreparedShaderInvocation::TryCreate(invocationRequest, handle, key); @@ -284,7 +366,7 @@ RegisteredShaderState DecodeRegisteredState(const ShaderSnapshot& snapshot) { if (header.specials != nullptr) { const auto special = ReadHeaderArray(snapshot, header.specials, 1).front(); state.context[special.vgt_shader_stages_en.offset] = special.vgt_shader_stages_en.value; - state.context[special.vgt_gs_out_prim_type.offset] = special.vgt_gs_out_prim_type.value; + if (special.vgt_gs_out_prim_type.offset != 0) state.context[special.vgt_gs_out_prim_type.offset] = special.vgt_gs_out_prim_type.value; state.userConfig[special.ge_cntl.offset] = special.ge_cntl.value; state.userConfig[special.ge_user_vgpr_en.offset] = special.ge_user_vgpr_en.value; } @@ -307,7 +389,17 @@ void BuildRegisteredAbiKey(const QueueState& state, const VulkanDevice& device, } } -std::vector PrepareRegistered(const ShaderSnapshot& snapshot, const VulkanDevice& device, const QueueState& state, bool registration) { +struct RegisteredPreparation { + std::size_t codeOffset = 0; + ShaderRecompiler::RdnaProgram decoded; + ShaderRecompiler::SwappcInfo swappc{}; + std::vector userData; + std::array memory{}; + std::optional request; + std::vector abiKey; +}; + +std::unique_ptr PlanRegistered(const ShaderSnapshot& snapshot, const VulkanDevice& device, const QueueState& state, bool registration, bool* deferred = nullptr) { using Stage = ShaderRecompiler::ShaderStage; const auto header = ReadHeader(snapshot); std::uint32_t programRegister; @@ -322,15 +414,33 @@ std::vector PrepareRegistered(const ShaderSnapshot& snap case 5: stage = Stage::Local; programRegister = 0x148; resourceRegister = 0x10b; firstUser = 8; break; case 6: stage = Stage::Mesh; programRegister = 0x088; resourceRegister = 0x08b; break; case 7: stage = Stage::TessellationControl; programRegister = 0x108; resourceRegister = 0x10b; break; - default: throw std::runtime_error("AGC driver: unsupported registered shader type"); + default: + if (registration) { + APS5_LOG_ERR("Shader 0x%llx has unsupported registered shader type %u; it is not prepared at registration", static_cast(snapshot.codeAddress), static_cast(snapshot.type)); + return nullptr; + } + throw std::runtime_error("AGC driver: unsupported registered shader type " + std::to_string(snapshot.type)); } + const auto defer = [&](const char* reason) { + APS5_LOG_ERR("Shader 0x%llx could not be prepared at registration (%s); preparing it when it is used", static_cast(snapshot.codeAddress), reason); + *deferred = true; + return std::unique_ptr{}; + }; + if (deferred != nullptr && (!state.shader.contains(programRegister) || !state.shader.contains(programRegister + 1))) return defer("its program address is not in the shader registers"); const auto high = RegisterValue(state.shader, programRegister + 1); if ((high & ~0xffu) != 0) throw std::runtime_error("AGC driver: invalid registered program address"); const auto address = (static_cast(RegisterValue(state.shader, programRegister)) << 8u) | (static_cast(high) << 40u); if (address < snapshot.codeAddress || address - snapshot.codeAddress >= snapshot.code.size() * 4u) throw std::runtime_error("AGC driver: registered entry point is outside shader code"); const auto codeOffset = static_cast((address - snapshot.codeAddress) / 4u); const auto code = std::span(snapshot.code).subspan(codeOffset); - const auto decoded = ShaderRecompiler::RdnaInstructionDecoder{}.Decode(code); + auto plan = std::make_unique(); + plan->codeOffset = codeOffset; + try { + plan->decoded = ShaderRecompiler::RdnaInstructionDecoder{}.Decode(code); + } catch (const ShaderRecompiler::UnsupportedInstructionError& error) { + if (deferred == nullptr) throw; + return defer(error.what()); + } std::optional compute; std::optional pixel; std::optional vertex; @@ -342,18 +452,22 @@ std::vector PrepareRegistered(const ShaderSnapshot& snap if (userCount > 32) throw std::runtime_error("AGC driver: static user SGPR count exceeds the register bank"); if (stage == Stage::Compute) { compute = Graphics::DecodeComputeStageInfo(state.shader, snapshot.header); + if (header.specials == nullptr) { + if (deferred != nullptr) return defer("its header has no specials to give its wave size"); + return {}; + } const auto special = ReadHeaderArray(snapshot, header.specials, 1).front(); wave = (special.dispatch_modifier & 0x8000u) != 0 ? 32u : 64u; } else if (stage == Stage::Fragment) { const auto count = RegisterValue(state.context, 0x1b6) & 0x3fu; - if (registration && count != 0) return {}; + if (registration && count != 0) return nullptr; pixel = Graphics::DecodePixelStageInfo(state.context, {}); wave = pixel->wave32 ? 32u : 64u; } else { const auto routing = RegisterValue(state.context, 0x2d5); wave = (routing & 0x00400000u) != 0 ? 32u : 64u; if ((routing & 0x20u) != 0 || stage == Stage::Mesh || (routing & 4u) != 0) { - if (registration) return {}; + if (registration) return nullptr; auto stageState = state; stageState.context[0x1b6] = 0; const auto stages = Graphics::DecodeShaderStages(stageState); @@ -363,30 +477,133 @@ std::vector PrepareRegistered(const ShaderSnapshot& snap } if (firstUser == 0) userCount += 8; } - std::vector userData(userCount); - if (stage != Stage::Compute && stage != Stage::Fragment && snapshot.type != 6) vertex = Graphics::DecodeVertexStageInfo(snapshot.header, snapshot.headerAddress, userData, nullptr, true); - const ShaderRecompiler::SwappcInfo swappc{vertex.has_value(), firstUser, userCount}; - auto graph = ShaderRecompiler::GraphBuilder{}.Build(decoded, &swappc); - ShaderRecompiler::Structurizer{}.Structurize(graph); - const std::array memory{{{snapshot.codeAddress, std::as_bytes(std::span(snapshot.code))}, {snapshot.headerAddress, snapshot.header}}}; - ShaderRecompiler::RecompileRequest request{{stage, address, code, snapshot.headerAddress, snapshot.header}, {wave, firstUser, userData, compute, pixel, vertex, memory, RegisteredFloatMode(snapshot)}, stage == Stage::Compute ? device.ComputeTarget(wave) : device.Target(), {0, 0, 0, 128}, graphics}; + plan->userData.resize(userCount); + if (stage != Stage::Compute && stage != Stage::Fragment && snapshot.type != 6) vertex = Graphics::DecodeVertexStageInfo(snapshot.header, snapshot.headerAddress, plan->userData, nullptr, true); + plan->swappc = {vertex.has_value(), firstUser, userCount}; + if (deferred != nullptr) { + if (const auto counter = ShaderRecompiler::UnresolvableSwappcTarget(plan->decoded, &plan->swappc)) { + char reason[96]; + std::snprintf(reason, sizeof(reason), "its s_swappc_b64 at program counter 0x%x calls a computed target", *counter); + return defer(reason); + } + } + plan->memory = {{{snapshot.codeAddress, std::as_bytes(std::span(snapshot.code))}, {snapshot.headerAddress, snapshot.header}}}; + auto& request = plan->request.emplace(ShaderRecompiler::RecompileRequest{{stage, address, code, snapshot.headerAddress, snapshot.header}, {wave, firstUser, plan->userData, compute, pixel, vertex, plan->memory, RegisteredFloatMode(snapshot)}, stage == Stage::Compute ? device.ComputeTarget(wave) : device.Target(), {0, 0, 0, 128}, graphics}); if (graphics && graphics->mesh) request.layout.pushConstantSizeBytes = ShaderRecompiler::MeshDrawPushOffsetBytes; + if (pixel && device.GraphicsPipelineLibraries()) request.layout.pushConstantOffsetBytes = Graphics::FixedPushOffset(Stage::Fragment); + return plan; +} + +std::vector PrepareRegistered(RegisteredPreparation& plan) { + auto& request = *plan.request; std::vector entries; + ShaderRecompiler::ShaderPreparationContext preparation; const auto append = [&] { PerformanceTimer timing("Shader.PrepareArtifact"); - entries.push_back({codeOffset, ShaderRecompiler::PrepareShader(request)}); + entries.push_back({plan.codeOffset, PrepareShaderWithDiagnostics(request, &preparation)}); }; append(); - if (compute) { + if (request.context.compute) { request.context.compute->partialThreads = {1, 1, 1}; append(); - } else if (pixel) { + } else if (request.context.pixel) { request.layout.pushConstantSizeBytes = 0; append(); } return entries; } +std::vector PrepareRegistered(const ShaderSnapshot& snapshot, const VulkanDevice& device, const QueueState& state, bool registration, bool* deferred = nullptr) { + const auto plan = PlanRegistered(snapshot, device, state, registration, deferred); + return plan != nullptr ? PrepareRegistered(*plan) : std::vector{}; +} + +class RegistrationPreparePool { +public: + static RegistrationPreparePool& Instance() { + static auto* pool = new RegistrationPreparePool(); + return *pool; + } + + void Submit(std::function task) { + if (threads == 0) { + task(); + return; + } + { + std::lock_guard lock(mutex); + tasks.push_back(std::move(task)); + } + ready.notify_one(); + } + +private: + RegistrationPreparePool() { + const auto count = std::max(2u, std::thread::hardware_concurrency() / 2u); + pthread_attr_t attributes; + pthread_attr_init(&attributes); + pthread_attr_setstacksize(&attributes, 64u << 20u); + for (unsigned index = 0; index < count; ++index) { + pthread_t thread; + if (pthread_create(&thread, &attributes, &RegistrationPreparePool::Main, this) == 0) { + pthread_detach(thread); + ++threads; + } + } + pthread_attr_destroy(&attributes); + } + + static void* Main(void* argument) { + auto& pool = *static_cast(argument); + for (;;) { + std::function task; + { + std::unique_lock lock(pool.mutex); + pool.ready.wait(lock, [&] { return !pool.tasks.empty(); }); + task = std::move(pool.tasks.front()); + pool.tasks.pop_front(); + } + task(); + } + return nullptr; + } + + std::mutex mutex; + std::condition_variable ready; + std::deque> tasks; + unsigned threads = 0; +}; + +bool AsyncRegistrationPrepare() { + static const bool sync = std::getenv("APS5_SYNC_SHADER_PREPARE") != nullptr; + return !sync; +} + +void PrepareRegisteredInBackground(std::shared_ptr snapshot, std::shared_ptr plan) { + RegistrationPreparePool::Instance().Submit([snapshot = std::move(snapshot), plan = std::move(plan)] { + PerformanceContext timingContext(FrameTiming::Preparation()); + std::vector entries; + std::exception_ptr failure; + try { + entries = PrepareRegistered(*plan); + } catch (const std::exception& error) { + APS5_LOG_ERR("Shader 0x%llx could not be prepared after registration: %s", static_cast(snapshot->codeAddress), error.what()); + failure = std::current_exception(); + } catch (...) { + failure = std::current_exception(); + } + auto& prepared = *snapshot->prepared; + { + std::lock_guard lock(prepared.mutex); + prepared.entries.insert(prepared.entries.end(), std::make_move_iterator(entries.begin()), std::make_move_iterator(entries.end())); + if (failure == nullptr && !plan->abiKey.empty()) prepared.registeredAbis.push_back(std::move(plan->abiKey)); + prepared.failure = failure; + prepared.pending = false; + } + prepared.settled.notify_all(); + }); +} + } std::vector PrepareGraphicsStages(const DrawDecode& decoded, const ShaderRecompiler::SpirvTarget& target) { @@ -413,7 +630,8 @@ std::vector PrepareGraphicsStages(const DrawDecode& decod } std::vector prepared; std::uint32_t pushOffset = 0; - const auto capacity = decoded.state.stages.mesh ? ShaderRecompiler::MeshDrawPushOffsetBytes : Graphics::PipelinePushConstantBytes; + const bool fixedSlots = target.fixedPushSlots; + const auto capacity = decoded.state.stages.mesh ? ShaderRecompiler::MeshDrawPushOffsetBytes : Graphics::PipelinePushSlotBytes; for (std::size_t index = 0; index < decoded.programs.size(); ++index) { if (decoded.roles[index] == ShaderRecompiler::ProgramRole::GeometryBack) continue; const auto& program = decoded.programs[index]; @@ -421,7 +639,10 @@ std::vector PrepareGraphicsStages(const DrawDecode& decod const auto wave = fragment ? decoded.state.stages.fragmentWaveSize : decoded.state.stages.vertexWaveSize; std::optional vertex; if (!fragment) vertex = Graphics::DecodeVertexStageInfo(program.binary.header, program.binary.headerAddress, program.userData, nullptr, true); - ShaderRecompiler::RecompileRequest request{program.binary, {wave, program.firstUserSgpr, program.userData, {}, fragment ? std::optional(decoded.pixel) : std::nullopt, vertex, memory, RegisteredFloatMode(*program.snapshot)}, target, {0, 0, pushOffset, capacity - pushOffset}, ShaderRecompiler::GraphicsCompileContext{program.firstUserSgpr, linked, decoded.state.stages.mesh, decoded.state.stages.tessellation, {}}}; + pushOffset = Graphics::StagePushOffset(pushOffset, program.binary.stage, fixedSlots); + const auto offset = pushOffset; + const auto room = (fixedSlots && fragment ? Graphics::PipelinePushSlotBytes : capacity) - pushOffset % Graphics::PipelinePushSlotBytes; + ShaderRecompiler::RecompileRequest request{program.binary, {wave, program.firstUserSgpr, program.userData, {}, fragment ? std::optional(decoded.pixel) : std::nullopt, vertex, memory, RegisteredFloatMode(*program.snapshot)}, target, {0, 0, offset, room}, ShaderRecompiler::GraphicsCompileContext{program.firstUserSgpr, linked, decoded.state.stages.mesh, decoded.state.stages.tessellation, {}}}; std::vector key; ShaderRecompiler::BuildPreparedShaderKey(request, key); std::shared_ptr handle; @@ -433,10 +654,10 @@ std::vector PrepareGraphicsStages(const DrawDecode& decod } if (handle == nullptr) { PerformanceTimer timing("Shader.PrepareArtifact"); - handle = ShaderRecompiler::PrepareShader(request); + handle = PrepareShaderWithDiagnostics(request); } const auto bytes = handle->artifact->bindings.pushConstantSizeBytes; - require(bytes <= capacity - pushOffset, "prepared graphics stages exceed the push constant block"); + require(bytes <= room, "prepared graphics stages exceed the push constant block"); prepared.push_back({program.snapshot, {program.codeOffset, handle}}); pushOffset += bytes; } @@ -462,7 +683,7 @@ void Driver::ResolveGraphicsStagesAbi(std::span stages, std require(registry != nullptr && registry->contains(address), "graphics ABI refers to an unregistered shader"); if (owner == nullptr) owner = registry->at(address); const auto& snapshot = *registry->at(address); - require(snapshot.headerAddress == reinterpret_cast(shader), "graphics ABI refers to a replaced shader header"); + require(SameHeader(snapshot, shader), "graphics ABI refers to a replaced shader header"); require(snapshot.registeredState != nullptr, "registered shader state is missing"); const auto& registered = *snapshot.registeredState; for (const auto& [offset, value] : registered.shader) state.shader.insert_or_assign(offset, value); @@ -527,7 +748,7 @@ void Driver::ResolveShaderAbi(const Shader* shader, std::span(const_cast(shader->code)); require(shaders != nullptr && shaders->contains(address), "static ABI refers to an unregistered shader"); snapshot = shaders->at(address); - require(snapshot->headerAddress == reinterpret_cast(shader), "static ABI refers to a replaced shader header"); + require(SameHeader(*snapshot, shader), "static ABI refers to a replaced shader header"); } require(snapshot->registeredState != nullptr, "registered shader state is missing"); QueueState state{}; @@ -589,7 +810,7 @@ void Driver::ResolveGraphicsAbi(const Shader* vertex, const Shader* pixel, std:: const auto address = reinterpret_cast(const_cast(shader->code)); require(shaders != nullptr && shaders->contains(address), "rectangle ABI refers to an unregistered shader"); const auto snapshot = shaders->at(address); - require(snapshot->headerAddress == reinterpret_cast(shader), "rectangle ABI refers to a replaced shader header"); + require(SameHeader(*snapshot, shader), "rectangle ABI refers to a replaced shader header"); return snapshot; }; front = lookup(vertex); @@ -659,6 +880,26 @@ void ResolvePreparedGraphics(const ShaderSnapshot& front, const std::shared_ptr< transaction.Commit(); } +ShaderSnapshot PrepareNullPixelProgram(const VulkanDevice& device) { + ShaderSnapshot null{NullPixelProgramAddress(), reinterpret_cast(&NullPixelShader), NullPixelShader.type, {}, {}}; + null.code.assign(std::begin(NullPixelCode), std::end(NullPixelCode)); + null.header.resize(sizeof(Shader)); + std::memcpy(null.header.data(), &NullPixelShader, sizeof(Shader)); + auto nullRegisteredState = DecodeRegisteredState(null); + nullRegisteredState.shader.insert_or_assign(0x008u, static_cast(null.codeAddress >> 8u)); + nullRegisteredState.shader.insert_or_assign(0x009u, static_cast(null.codeAddress >> 40u)); + nullRegisteredState.shader.insert_or_assign(0x00bu, 0u); + nullRegisteredState.context.insert_or_assign(0x1b3u, 0x2u); + nullRegisteredState.context.insert_or_assign(0x1b4u, 0x2u); + null.registeredState = std::make_shared(std::move(nullRegisteredState)); + QueueState nullState{}; + nullState.shader = null.registeredState->shader; + nullState.context = null.registeredState->context; + nullState.userConfig = null.registeredState->userConfig; + null.prepared->entries = PrepareRegistered(null, device, nullState, true); + return null; +} + void Driver::RegisterShader(const Shader* shader) { PerformanceContext timingContext(FrameTiming::Preparation()); PerformanceTimer timing("Shader.Register"); @@ -705,11 +946,18 @@ void Driver::RegisterShader(const Shader* shader) { registered.shader = snapshot.registeredState->shader; registered.context = snapshot.registeredState->context; registered.userConfig = snapshot.registeredState->userConfig; - snapshot.prepared->entries = PrepareRegistered(snapshot, *localDevice, registered, true); - if ((snapshot.type == 0 || snapshot.type == 1) && !snapshot.prepared->entries.empty()) { - std::vector key; - BuildRegisteredAbiKey(registered, *localDevice, key); - snapshot.prepared->registeredAbis.push_back(std::move(key)); + std::shared_ptr preparation; + if (AsyncRegistrationPrepare()) { + preparation = PlanRegistered(snapshot, *localDevice, registered, true, &snapshot.prepared->deferred); + if (preparation != nullptr && (snapshot.type == 0 || snapshot.type == 1)) BuildRegisteredAbiKey(registered, *localDevice, preparation->abiKey); + snapshot.prepared->pending = preparation != nullptr; + } else { + snapshot.prepared->entries = PrepareRegistered(snapshot, *localDevice, registered, true, &snapshot.prepared->deferred); + if ((snapshot.type == 0 || snapshot.type == 1) && !snapshot.prepared->entries.empty()) { + std::vector key; + BuildRegisteredAbiKey(registered, *localDevice, key); + snapshot.prepared->registeredAbis.push_back(std::move(key)); + } } static const char* traceRegs = std::getenv("APS5_TRACE_SHADER_REGS"); if (traceRegs != nullptr && (std::string(traceRegs) == "all" || std::strtoull(traceRegs, nullptr, 16) == snapshot.codeAddress)) { @@ -726,29 +974,15 @@ void Driver::RegisterShader(const Shader* shader) { print(fields.sh_registers, fields.num_sh_registers); std::fprintf(stderr, "\n"); } + const auto published = std::make_shared(std::move(snapshot)); std::lock_guard lock(mutex); rethrowFailure(); - PublishRegisteredShader(shaders, std::make_shared(std::move(snapshot))); + PublishRegisteredShader(shaders, published); if (shaders->find(NullPixelProgramAddress()) == shaders->end()) { - ShaderSnapshot null{NullPixelProgramAddress(), reinterpret_cast(&NullPixelShader), NullPixelShader.type, {}, {}}; - null.code.assign(std::begin(NullPixelCode), std::end(NullPixelCode)); - null.header.resize(sizeof(Shader)); - std::memcpy(null.header.data(), &NullPixelShader, sizeof(Shader)); - auto nullRegisteredState = DecodeRegisteredState(null); - nullRegisteredState.shader.emplace(0x008u, static_cast(null.codeAddress >> 8u)); - nullRegisteredState.shader.emplace(0x009u, static_cast(null.codeAddress >> 40u)); - nullRegisteredState.shader.emplace(0x00bu, 0u); - nullRegisteredState.context.emplace(0x1b3u, 0x2u); - nullRegisteredState.context.emplace(0x1b4u, 0x2u); - null.registeredState = std::make_shared(std::move(nullRegisteredState)); - QueueState nullState{}; - nullState.shader = null.registeredState->shader; - nullState.context = null.registeredState->context; - nullState.userConfig = null.registeredState->userConfig; - null.prepared->entries = PrepareRegistered(null, *localDevice, nullState, true); - PublishRegisteredShader(shaders, std::make_shared(std::move(null))); + PublishRegisteredShader(shaders, std::make_shared(PrepareNullPixelProgram(*localDevice))); } transaction.Commit(); + if (preparation != nullptr) PrepareRegisteredInBackground(published, std::move(preparation)); } } diff --git a/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Synchronization/DeferredLabels.cpp b/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Synchronization/DeferredLabels.cpp index 395c2ef2e..858cada5e 100644 --- a/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Synchronization/DeferredLabels.cpp +++ b/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Synchronization/DeferredLabels.cpp @@ -117,11 +117,11 @@ void Driver::recordDeferredLabels(VulkanDevice* localDevice, std::uint32_t queue const int reason = localDevice != nullptr ? localDevice->WriteLabelOnGpu(label.address, bytes, stamp, queue, first) : 4; first = false; countLabelOutcome(reason); - if (reason != 0 && reason != 5 && reason != 6) { + if (reason != 0 && reason != 5) { if (reason != 1 && localDevice != nullptr) localDevice->WaitIdle(); GuestMemory::Write(label.address, bytes, 4); } - noteLabelStore(label.address, bytes, stamp); + noteLabelStore(label.address, bytes, stamp, queue); } ++labelGroups; static const bool profile = std::getenv("APS5_PROFILE_DRAW") != nullptr; diff --git a/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Synchronization/LabelHistory.cpp b/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Synchronization/LabelHistory.cpp index f599ec836..696612f79 100644 --- a/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Synchronization/LabelHistory.cpp +++ b/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Synchronization/LabelHistory.cpp @@ -6,18 +6,19 @@ namespace AgcDriver::DriverDetail { -void Driver::noteLabelStore(std::uint64_t address, std::span bytes, std::uint64_t stamp) { +void Driver::noteLabelStore(std::uint64_t address, std::span bytes, std::uint64_t stamp, std::uint32_t queue) { if (bytes.empty() || bytes.size() > 64 || bytes.size() % 4 != 0 || address % 4 != 0) return; std::lock_guard lock(labelStoresMutex); for (std::size_t offset = 0; offset < bytes.size(); offset += 4) { auto& history = labelStores[address + offset]; std::shift_right(history.begin(), history.end(), 1); history[0].stamp = stamp; + history[0].queue = queue; std::memcpy(&history[0].value, bytes.data() + offset, 4); } } -bool Driver::storedSince(std::span packet, std::uint64_t address, std::size_t bytes, std::uint64_t received) { +bool Driver::storedSince(std::span packet, std::uint64_t address, std::size_t bytes, std::uint64_t received, std::uint32_t* writer) { std::lock_guard lock(labelStoresMutex); const auto low = labelStores.find(address); if (low == labelStores.end()) return false; @@ -31,7 +32,10 @@ bool Driver::storedSince(std::span packet, std::uint64_t ad if (upper == high->second.end()) continue; value |= static_cast(upper->value) << 32u; } - if (Pm4::WaitComparesValue(packet, value)) return true; + if (Pm4::WaitComparesValue(packet, value)) { + if (writer != nullptr) *writer = store.queue; + return true; + } } return false; } diff --git a/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Synchronization/SynchronizationStatistics.cpp b/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Synchronization/SynchronizationStatistics.cpp index dfb31f4b7..ca531b0cb 100644 --- a/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Synchronization/SynchronizationStatistics.cpp +++ b/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Synchronization/SynchronizationStatistics.cpp @@ -5,7 +5,7 @@ namespace AgcDriver::DriverDetail { -std::atomic gpuLabels{0}, completionLabels{0}, notImportedLabels{0}, noOpLabels{0}, unlockedDrains{0}; +std::atomic gpuLabels{0}, completionLabels{0}, noOpLabels{0}, unlockedDrains{0}; std::atomic labelFallbacks[5] = {}; @@ -39,13 +39,12 @@ SubmissionCosts& submissionCosts(std::uint32_t queue) { void reportSync() { std::string report; for (const auto& [code, count] : drainCounts) report += " " + (code == 0xffffu ? std::string("flip") : Pm4::Name(code << 8u)) + "=" + std::to_string(count); - AgcDriver::ProfilePrint_nid_no_patch("[sync] %llu device drains by packet:%s (%llu waited without the GPU mutex); %llu labels written on the GPU, %llu deferred behind completions, %llu deferred not imported, %llu no-op, fallbacks: idle %llu, completions %llu, not imported %llu, undecodable %llu; %llu labels queued per worker, recorded in %llu groups, %llu recorded at once; stores: %llu recorded on the GPU, %llu behind completions, %llu on the CPU (idle), %llu synced (drained)\n", static_cast(drainTotal), report.c_str(), static_cast(unlockedDrains.load()), static_cast(gpuLabels.load()), static_cast(completionLabels.load()), static_cast(notImportedLabels.load()), static_cast(noOpLabels.load()), static_cast(labelFallbacks[1].load()), static_cast(labelFallbacks[2].load()), static_cast(labelFallbacks[3].load()), static_cast(labelFallbacks[4].load()), static_cast(queuedLabels.load()), static_cast(labelGroups.load()), static_cast(immediateLabels.load()), static_cast(storesOnGpu.load()), static_cast(storesBehindCompletions.load()), static_cast(storesOnCpu.load()), static_cast(storesDrained.load())); + AgcDriver::ProfilePrint_nid_no_patch("[sync] %llu device drains by packet:%s (%llu waited without the GPU mutex); %llu labels written on the GPU, %llu deferred behind completions, %llu no-op, fallbacks: idle %llu, completions %llu, not imported %llu, undecodable %llu; %llu labels queued per worker, recorded in %llu groups, %llu recorded at once; stores: %llu recorded on the GPU, %llu behind completions, %llu on the CPU (idle), %llu synced (drained)\n", static_cast(drainTotal), report.c_str(), static_cast(unlockedDrains.load()), static_cast(gpuLabels.load()), static_cast(completionLabels.load()), static_cast(noOpLabels.load()), static_cast(labelFallbacks[1].load()), static_cast(labelFallbacks[2].load()), static_cast(labelFallbacks[3].load()), static_cast(labelFallbacks[4].load()), static_cast(queuedLabels.load()), static_cast(labelGroups.load()), static_cast(immediateLabels.load()), static_cast(storesOnGpu.load()), static_cast(storesBehindCompletions.load()), static_cast(storesOnCpu.load()), static_cast(storesDrained.load())); } void countLabelOutcome(int reason) { if (reason == 0) ++gpuLabels; else if (reason == 5) ++completionLabels; - else if (reason == 6) ++notImportedLabels; else ++labelFallbacks[std::min(reason, 4)]; } diff --git a/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Synchronization/WaitMemory.cpp b/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Synchronization/WaitMemory.cpp index 9a6ee823d..25e72727c 100644 --- a/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Synchronization/WaitMemory.cpp +++ b/core/libs/prx/libSceAgcDriver/Execution/src/Driver/Synchronization/WaitMemory.cpp @@ -1,3 +1,4 @@ +#include "prx/libSceAgcDriver/Execution/include/ProfileOutput.hpp" #include "prx/libSceAgcDriver/Execution/include/Driver/Driver.hpp" #include "prx/libSceAgcDriver/Execution/include/Driver/Synchronization/WaitMemory.hpp" #include "prx/libSceAgcDriver/Execution/include/Driver/Diagnostics.hpp" @@ -11,6 +12,9 @@ #include #endif #include +#include +#include +#include namespace AgcDriver::DriverDetail { @@ -49,7 +53,57 @@ bool Driver::traceLateLabels() { return trace; } -void Driver::waitMemory(std::span packet, std::uint32_t queue, const PacketHistory& context, std::uint64_t received, bool heldAtSubmit) { +std::vector& Driver::pendingWaits() { + static thread_local std::vector waits; + return waits; +} + +void Driver::notePendingWait(std::span packet, std::uint32_t queue, std::uint64_t received) { + if (packet.size() > std::tuple_size_v) return; + static const bool allQueues = std::getenv("APS5_LAND_WAITS_ALL_QUEUES") != nullptr; + if (!allQueues && queue == 0) return; + if (Pm4::WaitSatisfiedUnchecked(packet)) return; + auto& waits = pendingWaits(); + for (const auto& wait : waits) { + if (wait.count == packet.size() && std::equal(packet.begin(), packet.end(), wait.words.begin())) return; + } + PendingWait wait{}; + std::copy(packet.begin(), packet.end(), wait.words.begin()); + wait.count = packet.size(); + wait.received = received; + waits.push_back(wait); +} + +void Driver::landPendingWaits(std::uint32_t queue) { + auto& waits = pendingWaits(); + if (waits.empty()) return; + static const bool profile = std::getenv("APS5_PROFILE_DRAW") != nullptr; + struct LandStats { + std::uint64_t lands = 0, waited = 0; + double waitedMs = 0, maxMs = 0; + std::chrono::steady_clock::time_point reported = std::chrono::steady_clock::now(); + }; + static thread_local LandStats stats; + const PacketHistory none{}; + for (const auto& wait : waits) { + const auto packet = std::span(wait.words).first(wait.count); + ++stats.lands; + if (Pm4::WaitSatisfiedUnchecked(packet)) continue; + const auto started = std::chrono::steady_clock::now(); + waitMemory(packet, queue, none, wait.received, false, true); + const auto ms = std::chrono::duration(std::chrono::steady_clock::now() - started).count(); + ++stats.waited; + stats.waitedMs += ms; + stats.maxMs = std::max(stats.maxMs, ms); + } + waits.clear(); + if (profile && std::chrono::steady_clock::now() - stats.reported > std::chrono::seconds(10)) { + AgcDriver::ProfilePrint_nid_no_patch("[landwait] queue 0x%x (10 s): %llu waits landed before a guest read, %llu had to wait for the GPU: %.1f ms in total, max %.2f ms\n", queue, static_cast(stats.lands), static_cast(stats.waited), stats.waitedMs, stats.maxMs); + stats = LandStats{}; + } +} + +void Driver::waitMemory(std::span packet, std::uint32_t queue, const PacketHistory& context, std::uint64_t received, bool heldAtSubmit, bool requireMemory) { auto start = std::chrono::steady_clock::now(); auto lastDone = packetsDone.load(); @@ -82,9 +136,14 @@ void Driver::waitMemory(std::span packet, std::uint32_t que ++outcomes.atEntry; return; } - if (heldAtSubmit || storedSince(packet, awaited, awaitedBytes, received)) return; + if (heldAtSubmit) return; + if (std::uint32_t writer = queue; !requireMemory && storedSince(packet, awaited, awaitedBytes, received, &writer)) { + if (writer != queue) notePendingWait(packet, queue, received); + return; + } static const bool labelShortcut = std::getenv("APS5_NO_LABEL_SHORTCUT") == nullptr; + const bool labelTable = labelShortcut && !requireMemory; static const bool overlapSubmit = std::getenv("APS5_NO_WAIT_OVERLAP_SUBMIT") == nullptr; static const bool waitLock = std::getenv("APS5_WAIT_LOCK") != nullptr; @@ -121,6 +180,8 @@ void Driver::waitMemory(std::span packet, std::uint32_t que if (hit->queue == queue) { ++outcomes.fromRecorderSameQueue; if (!hit->late) epochPoint.bump = false; + } else { + notePendingWait(packet, queue, received); } return true; }; @@ -129,7 +190,7 @@ void Driver::waitMemory(std::span packet, std::uint32_t que const bool unlocked = !waitLock && overlapSubmit; if (unlocked) { - if (labelShortcut) { + if (labelTable) { Graphics::Recorder::LabelRefusal refusal{}; if (takeLabel(Graphics::Recorder::LookupLabel(awaited, awaitedBytes, received, &refusal), refusal, false, true)) return; } @@ -152,7 +213,7 @@ void Driver::waitMemory(std::span packet, std::uint32_t que } if (const auto localDevice = gpuLock.owns_lock() ? device.Load() : std::shared_ptr{}) { - if (labelShortcut) { + if (labelTable) { Graphics::Recorder::LabelRefusal refusal{}; if (takeLabel(localDevice->PendingLabel(awaited, awaitedBytes, received, &refusal), refusal, false, false)) return; } @@ -182,7 +243,7 @@ void Driver::waitMemory(std::span packet, std::uint32_t que if (now - lastTry < PollTryInterval) return false; lastTry = now; } - if (changed && unlocked && labelShortcut) { + if (changed && unlocked && labelTable) { Graphics::Recorder::LabelRefusal refusal{}; if (takeLabel(Graphics::Recorder::LookupLabel(awaited, awaitedBytes, received, &refusal), refusal, true, true)) { @@ -220,7 +281,7 @@ void Driver::waitMemory(std::span packet, std::uint32_t que if (changed) { seenGeneration = generation; recheck = false; - if (labelShortcut) { + if (labelTable) { Graphics::Recorder::LabelRefusal refusal{}; if (takeLabel(localDevice->PendingLabel(awaited, awaitedBytes, received, &refusal), refusal, true, false)) { if (pollProfile) ++poll.tableHits; @@ -268,7 +329,10 @@ void Driver::waitMemory(std::span packet, std::uint32_t que bool spinning = pauseSpin; std::uint32_t polls = 0; while (!Pm4::WaitSatisfiedUnchecked(packet)) { - if (storedSince(packet, awaited, awaitedBytes, received)) return; + if (std::uint32_t writer = queue; !requireMemory && storedSince(packet, awaited, awaitedBytes, received, &writer)) { + if (writer != queue) notePendingWait(packet, queue, received); + return; + } ++polls; if (spinning) { _mm_pause(); @@ -281,7 +345,14 @@ void Driver::waitMemory(std::span packet, std::uint32_t que } CheckFailure(); if (pollService(!spinning)) return; - if (const auto done = packetsDone.load(); done != lastDone || packetsInFlight.load() != 0) { + noteAwaitingTitle(queue, awaited); + if (requireMemory && (polls & 15u) == 0) { + std::unique_lock settleLock(GuestMemory::GpuMutex(), std::try_to_lock); + if (settleLock.owns_lock()) { + if (const auto localDevice = device.Load(); localDevice != nullptr && localDevice->RecordedWritesSettled(awaited, awaitedBytes)) return; + } + } + if (const auto done = packetsDone.load(); done != lastDone || (!requireMemory && packetsInFlight.load() != 0)) { lastDone = done; start = std::chrono::steady_clock::now(); } @@ -301,7 +372,7 @@ void Driver::waitMemory(std::span packet, std::uint32_t que const std::uint64_t reference = wide ? packet[4] | (static_cast(packet[5]) << 32u) : packet[4]; const std::uint64_t mask = wide ? packet[6] | (static_cast(packet[7]) << 32u) : packet[5]; const std::uint64_t current = wide ? *reinterpret_cast(awaited) : *reinterpret_cast(awaited); - std::fprintf(stderr, "[gpu] queue 0x%x WAIT_REG_MEM%s at 0x%llx timed out after %dms (function %u ref 0x%llx mask 0x%llx value 0x%llx)\n", queue, wide ? "_64" : "", + std::fprintf(stderr, "[gpu] queue 0x%x WAIT_REG_MEM%s%s at 0x%llx timed out after %dms (function %u ref 0x%llx mask 0x%llx value 0x%llx)\n", queue, wide ? "_64" : "", requireMemory ? " (landing)" : "", static_cast(awaited), waitTimeoutMs(), packet[1] & 7u, static_cast(reference), static_cast(mask), static_cast(current)); const auto address = packet[2] | (static_cast(packet[3]) << 32u); for (const auto& record : writeHistory()) { diff --git a/core/libs/prx/libSceAgcDriver/Execution/src/GuestMemory.cpp b/core/libs/prx/libSceAgcDriver/Execution/src/GuestMemory.cpp index b944d6438..6f0bc11d2 100644 --- a/core/libs/prx/libSceAgcDriver/Execution/src/GuestMemory.cpp +++ b/core/libs/prx/libSceAgcDriver/Execution/src/GuestMemory.cpp @@ -747,7 +747,7 @@ constexpr std::size_t WriteBlockBytes = 65536; constexpr std::size_t WritePageBytes = 4096; constexpr std::size_t WritePagesPerBlock = WriteBlockBytes / WritePageBytes; -enum class StampKind : std::uint8_t { Cpu, Driver, ImportWindow }; +enum class StampKind : std::uint8_t { Cpu, Driver, ImportWindow, Fresh }; #ifdef _WIN32 void watchPrivateMapping(std::uintptr_t address, std::size_t bytes, std::uint64_t generation); @@ -901,6 +901,25 @@ struct WriteTracker { #endif } + bool unchanged(std::uint64_t first, std::uint64_t last, std::uint64_t since) const { + if (first == last) return stampOf(first) <= since; + const auto newer = [since](std::uint32_t stamp) { return stamp > since; }; +#ifdef _WIN32 + return std::none_of(blocks.begin() + first, blocks.begin() + last + 1, newer); +#else + while (first <= last) { + const auto index = first / LeafBlocks; + const auto end = std::min(last + 1, (index + 1) * LeafBlocks); + if (const auto& leaf = leaves[index]; leaf != nullptr) { + const auto begin = leaf->blocks.begin() + first % LeafBlocks; + if (std::any_of(begin, begin + (end - first), newer)) return false; + } + first = end; + } + return true; +#endif + } + std::uint32_t cpuStampOf(std::uint64_t block) const { #ifdef _WIN32 return cpuBlocks[block]; @@ -920,7 +939,7 @@ struct WriteTracker { } void stamp(std::uint64_t block, std::uint32_t stampGeneration, StampKind kind) { - const bool cpu = kind != StampKind::Driver; + const bool cpu = kind == StampKind::Cpu || kind == StampKind::ImportWindow; const bool written = kind != StampKind::ImportWindow; #ifdef _WIN32 blocks[block] = stampGeneration; @@ -962,6 +981,7 @@ void watchPrivateMapping(std::uintptr_t address, std::size_t bytes, std::uint64_ if (!tracker.watched || bytes == 0 || bytes > tracker.size || address < tracker.base || address - tracker.base > tracker.size - bytes) return; tracker.coverage.Restore(address, bytes, generation); ++tracker.generation; + tracker.noteCpuStore(address, address + bytes, tracker.generation); for (auto block = tracker.blockOf(address); block <= tracker.blockOf(address + bytes - 1); ++block) { tracker.driverPieces.erase(block); tracker.stamp(block, tracker.generation, StampKind::Cpu); @@ -1007,11 +1027,15 @@ bool sharedCollectMemo() { struct StampRuns { WriteTracker& tracker; StampKind kind; + std::size_t stamped = 0; }; void stampWrittenRun(void* context, std::uintptr_t begin, std::uintptr_t end) { - auto& [tracker, kind] = *static_cast(context); + auto& runs = *static_cast(context); + auto& tracker = runs.tracker; + const auto kind = runs.kind; if (end <= begin) return; + ++runs.stamped; const auto generation = tracker.generation.load(std::memory_order_relaxed); if (kind != StampKind::Driver) tracker.noteCpuStore(begin, end, generation); for (auto block = tracker.blockOf(begin); block <= tracker.blockOf(end - 1); ++block) tracker.stamp(block, generation, kind); @@ -1019,9 +1043,10 @@ void stampWrittenRun(void* context, std::uintptr_t begin, std::uintptr_t end) { } #endif -bool walkWrites(WriteTracker& tracker, std::uint64_t first, std::uint64_t stop, StampKind kind) { +bool walkWrites(WriteTracker& tracker, std::uint64_t first, std::uint64_t stop, StampKind kind, bool armedOnly = false) { ++tracker.generation; #ifdef _WIN32 + static_cast(armedOnly); constexpr std::uint64_t page = 4096; // One resetting walk: the kernel reports and clears a page's dirty bit together, a write landing // after the walk passed a page is reported by the next walk, and a clean page is not touched by the @@ -1067,7 +1092,8 @@ bool walkWrites(WriteTracker& tracker, std::uint64_t first, std::uint64_t stop, #else const auto dirtyRuns = collectDirtyRuns.load(std::memory_order_relaxed); StampRuns runs{tracker, kind}; - const bool complete = GuestWriteWatch::GuestWriteWatchCollect_nid_postfix(static_cast(first), static_cast(stop - first), &stampWrittenRun, &runs); + const auto collect = armedOnly ? &GuestWriteWatch::GuestWriteWatchCollectArmed_nid_postfix : &GuestWriteWatch::GuestWriteWatchCollect_nid_postfix; + const bool complete = collect(static_cast(first), static_cast(stop - first), &stampWrittenRun, &runs); if (collectDirtyRuns.load(std::memory_order_relaxed) != dirtyRuns) collectDirty.fetch_add(1, std::memory_order_relaxed); return complete; #endif @@ -1077,8 +1103,8 @@ std::uint64_t collectWrites(std::uint64_t address, std::size_t bytes, bool memoi auto& tracker = Tracker(); // Whole pages, so a page shared with the next range is collected with either. constexpr std::uint64_t page = 4096; - const auto first = address & ~(page - 1); - const auto stop = (address + bytes + page - 1) & ~(page - 1); + auto first = address & ~(page - 1); + auto stop = (address + bytes + page - 1) & ~(page - 1); // One epoch for the lookup and the entry made after the walk: an unbumped thread's fresh epoch // must not differ between them. const auto epoch = currentCollectEpoch(); @@ -1110,6 +1136,16 @@ std::uint64_t collectWrites(std::uint64_t address, std::size_t bytes, bool memoi } } const TimedAccess timed(CounterCollect, bytes); +#ifndef _WIN32 + if (useMemo && threadCollectEpoch != 0 && bytes <= WriteBlockBytes) { + const auto begin = first & ~(WriteBlockBytes - 1); + const auto end = (stop + WriteBlockBytes - 1) & ~(WriteBlockBytes - 1); + if (end > begin && tracker.covers(begin, end - begin)) { + first = begin; + stop = end; + } + } +#endif if (!walkWrites(tracker, first, stop, StampKind::Cpu)) return 0; // Only a completed walk is remembered; a failed one (uncommitted pages) returned 0 above. if (collectMemoEnabled()) { @@ -1177,9 +1213,9 @@ bool ImportWatched(std::uint64_t address, std::size_t bytes, const std::function const auto first = address & ~(page - 1); const auto stop = (address + bytes + page - 1) & ~(page - 1); const bool covered = tracker.covers(first, static_cast(stop - first)); - const bool before = covered && walkWrites(tracker, first, stop, StampKind::Cpu); + const bool before = covered && walkWrites(tracker, first, stop, StampKind::Cpu, true); if (!import()) return false; - if (!before || !walkWrites(tracker, first, stop, StampKind::ImportWindow)) unwatchLocked(tracker, first, static_cast(stop - first)); + if (!before || !walkWrites(tracker, first, stop, StampKind::ImportWindow, true)) unwatchLocked(tracker, first, static_cast(stop - first)); return true; } @@ -1204,10 +1240,7 @@ bool UnchangedSince(std::uint64_t address, std::size_t bytes, std::uint64_t gene if (!tracker.watched || generation == 0 || bytes == 0 || !tracker.covers(address, bytes)) return false; const auto first = tracker.blockOf(address); const auto last = tracker.blockOf(address + bytes - 1); - for (auto block = first; block <= last; ++block) { - if (tracker.stampOf(block) > generation) return false; - } - return true; + return tracker.unchanged(first, last, generation); } bool UnchangedSinceAll(std::span queries) { @@ -1219,9 +1252,7 @@ bool UnchangedSinceAll(std::span queries) { if (generation == 0 || bytes == 0 || !tracker.covers(address, bytes)) return false; const auto first = tracker.blockOf(address); const auto last = tracker.blockOf(address + bytes - 1); - for (auto block = first; block <= last; ++block) { - if (tracker.stampOf(block) > generation) return false; - } + if (!tracker.unchanged(first, last, generation)) return false; } return true; } @@ -1261,6 +1292,15 @@ std::uint64_t StoreOwnBytes(std::uint64_t address, std::size_t bytes, const std: }); } +std::uint64_t StoreOwnBytes(std::uint64_t address, std::span source) { + return StoreOwnBytes(address, source.size(), [&] { +#ifndef _WIN32 + if (GuestWriteWatch::GuestWriteWatchAvailable_nid_postfix() && GuestArena::GuestArenaWriteSharedBacking_nid_no_patch(address, source.data(), source.size())) return; +#endif + std::memcpy(reinterpret_cast(address), source.data(), source.size()); + }); +} + std::uint64_t MarkWritten(std::uint64_t address, std::size_t bytes) { auto& tracker = Tracker(); const auto lock = lockTracker(tracker); @@ -1268,6 +1308,18 @@ std::uint64_t MarkWritten(std::uint64_t address, std::size_t bytes) { if (bytes == 0) return 0; ++tracker.generation; if (!tracker.watched || !tracker.covers(address, bytes)) return 0; +#ifndef _WIN32 + { + const auto blockFirst = address & ~(std::uint64_t{WriteBlockBytes} - 1); + const auto blockStop = (address + bytes + WriteBlockBytes - 1) & ~(std::uint64_t{WriteBlockBytes} - 1); + StampRuns runs{tracker, StampKind::Fresh}; + if (!GuestWriteWatch::GuestWriteWatchCollectFresh_nid_postfix(static_cast(blockFirst), static_cast(blockStop - blockFirst), &stampWrittenRun, &runs)) { + unwatchLocked(tracker, blockFirst, static_cast(blockStop - blockFirst)); + return 0; + } + if (runs.stamped != 0) ++tracker.generation; + } +#endif const auto first = tracker.blockOf(address); const auto last = tracker.blockOf(address + bytes - 1); for (auto block = first; block <= last; ++block) { @@ -1287,15 +1339,15 @@ bool StoredOver(std::uint64_t address, std::size_t bytes, std::uint64_t generati const auto last = tracker.blockOf(end - 1); for (auto block = first; block <= last; ++block) { if (tracker.writtenStampOf(block) <= generation) continue; - if (tracker.cpuStampOf(block) > generation) { + if (const auto found = tracker.cpuPages.find(block); found != tracker.cpuPages.end()) { const auto begin = tracker.blockBegin(block); const auto from = std::max(address, begin) - begin; const auto to = std::min(end, begin + WriteBlockBytes) - begin; - const auto found = tracker.cpuPages.find(block); - if (found == tracker.cpuPages.end()) return true; for (auto page = from / WritePageBytes; page <= (to - 1) / WritePageBytes; ++page) { if (found->second.pages[page] > generation) return true; } + } else if (tracker.cpuStampOf(block) > generation) { + return true; } const auto found = tracker.driverPieces.find(block); if (found == tracker.driverPieces.end()) return true; @@ -1326,8 +1378,18 @@ bool UnchangedSinceCollected(std::uint64_t address, std::size_t bytes, std::uint if (!tracker.watched || generation == 0 || bytes == 0 || !tracker.covers(address, bytes)) return false; const auto first = tracker.blockOf(address); const auto last = tracker.blockOf(address + bytes - 1); + const auto end = address + bytes; for (auto block = first; block <= last; ++block) { - if (tracker.cpuStampOf(block) > generation) return false; + if (const auto found = tracker.cpuPages.find(block); found != tracker.cpuPages.end()) { + const auto begin = tracker.blockBegin(block); + const auto from = std::max(address, begin) - begin; + const auto to = std::min(end, begin + WriteBlockBytes) - begin; + for (auto page = from / WritePageBytes; page <= (to - 1) / WritePageBytes; ++page) { + if (found->second.pages[page] > generation) return false; + } + } else if (tracker.cpuStampOf(block) > generation) { + return false; + } } return true; } @@ -1850,7 +1912,8 @@ void Write(std::uint64_t address, std::span source, std::size_t auto* destination = reinterpret_cast(address); CheckRange(destination, source.size(), alignment, true); // Stamped like a GPU write: a collect memoized for this packet would not see the page fault. - StoreOwnBytes(address, source.size(), [&] { std::memcpy(destination, source.data(), source.size()); }); + if (source.size() >= PageBytes) StoreOwnBytes(address, source); + else StoreOwnBytes(address, source.size(), [&] { std::memcpy(destination, source.data(), source.size()); }); } } diff --git a/core/libs/prx/libSceAgcDriver/Execution/src/Pm4.cpp b/core/libs/prx/libSceAgcDriver/Execution/src/Pm4.cpp index 220e685f1..7520ab41b 100644 --- a/core/libs/prx/libSceAgcDriver/Execution/src/Pm4.cpp +++ b/core/libs/prx/libSceAgcDriver/Execution/src/Pm4.cpp @@ -1,12 +1,14 @@ #include "prx/libSceAgcDriver/Execution/include/Pm4.hpp" #include "prx/libSceAgcDriver/Execution/include/GuestMemory.hpp" #include "prx/libc/include/General.hpp" +#include "prx/libc/include/PreciseWait.hpp" #include #include #include #include #include #include +#include #include #include @@ -113,6 +115,7 @@ std::vector copySource(std::uint64_t source, std::size_t bytes, bool } constexpr std::uint32_t DmaSelectGds = 1; +constexpr std::uint32_t DmaSelectNowhere = 2; bool gdsRange(std::uint64_t offset, std::size_t bytes) { return offset <= GdsBytes && bytes <= GdsBytes - offset; @@ -186,7 +189,7 @@ std::string_view UnsupportedReason(std::uint32_t header) { case 0x11: case 0x12: case 0x13: case 0x15: case 0x16: case 0x20: case 0x22: case 0x26: case 0x27: case 0x2a: case 0x2d: case 0x2f: case 0x35: case 0x37: case 0x40: case 0x42: case 0x45: case 0x46: case 0x50: case 0x58: case 0x63: case 0x64: case 0x69: case 0x76: case 0x79: case 0x7a: - case 0x81: case 0x83: case 0x9f: return {}; + case 0x81: case 0x83: case 0x9f: case 0x28: return {}; case 0x24: case 0x25: case 0x2c: case 0x38: if (IndirectDrawsDisabled()) return "graphics draw, shader stages and guest render-target materialization are not implemented"; return {}; @@ -203,7 +206,7 @@ std::string_view UnsupportedReason(std::uint32_t header) { case 0x43: case 0x47: case 0x48: return "guest cache actions, GPU events and interrupt delivery are not implemented"; case 0x8e: return "GPU LOD statistics are not implemented; synthetic results are forbidden"; - case 0x28: case 0x41: case 0x68: case 0x78: + case 0x41: case 0x68: case 0x78: return "opcode is named but has no handler in the reference dispatch table"; default: return "opcode is not known in the reference"; } @@ -258,7 +261,7 @@ void Validate(std::span packet, std::uint32_t queue) { // neither changes what the packet writes. const auto flags = header & 0xffu; const bool registerWrite = opcode == 0x69 || opcode == 0x76 || opcode == 0x79 || opcode == 0x7a; - auto allowedFlags = opcode == 0x11 ? 2u : registerWrite ? 6u : opcode == 0x3c || opcode == 0x93 ? 2u : 0u; + auto allowedFlags = opcode == 0x11 || opcode == 0x15 || opcode == 0x16 ? 2u : registerWrite ? 6u : opcode == 0x3c || opcode == 0x93 ? 2u : 0u; if (opcode != 0x20 && opcode != 0x22) allowedFlags |= 1u; if ((flags & ~allowedFlags) != 0) throw std::runtime_error("PM4 header flags 0x" + ToHex(flags) + " are not implemented"); switch (opcode) { @@ -320,10 +323,10 @@ void Validate(std::span packet, std::uint32_t queue) { require((packet[8] & 3u) == 0 && packet[8] >= (opcode == 0x2c ? 16u : 20u), "invalid indirect draw stride"); require((packet[9] & ~0x20u) == (opcode == 0x2c ? 2u : 0u), "unsupported indirect draw initiator"); break; - case 0x15: size(5); if ((packet[4] & ~0xa020u) != 0x41u) throw std::runtime_error("dispatch modifiers 0x" + ToHex(packet[4]) + " are not implemented"); break; + case 0x15: size(5); if ((packet[4] & ~0xa024u) != 0x41u) throw std::runtime_error("dispatch modifiers 0x" + ToHex(packet[4]) + " are not implemented"); break; case 0x16: require(packet.size() == 3 || packet.size() == 4, "invalid indirect dispatch size"); - require((packet.back() & ~0xa020u) == 0x41u, "indirect dispatch modifiers are not implemented"); + require((packet.back() & ~0xa024u) == 0x41u, "indirect dispatch modifiers are not implemented"); break; case 0x22: size(5); @@ -333,6 +336,11 @@ void Validate(std::span packet, std::uint32_t queue) { require((packet[4] & ~ConditionalWordsMask) == 0, "COND_EXEC reserved count bits are not implemented"); break; case 0x42: size(2); require(packet[1] == 0, "unsupported PFP_SYNC_ME payload"); break; + case 0x28: + graphics(); + size(3); + require((packet[1] & 0x7fffffffu) == 0 && (packet[2] & 0x7fffffffu) == 0, "CONTEXT_CONTROL register loading and shadowing are not implemented"); + break; case 0x46: { require((packet[1] & ~0x73fu) == 0, "unsupported EVENT_WRITE flags or reserved bits"); const auto eventType = packet[1] & 0x3fu; @@ -348,6 +356,15 @@ void Validate(std::span packet, std::uint32_t queue) { size(2); require(eventIndex == 0 || eventIndex == 7, "invalid cache-flush event index"); break; + case 0x19: case 0x1a: case 0x26: + size(2); + require(eventIndex == 0, "invalid pipeline statistics or SQ event index"); + break; + case 0x24: + graphics(); + size(2); + require(eventIndex == 0, "invalid VGT_FLUSH event index"); + break; case 0x39: graphics(); size(4); @@ -363,11 +380,11 @@ void Validate(std::span packet, std::uint32_t queue) { const auto controlMask = packet.size() == 8 ? 0x86287fc3u : 0xfeecfffbu; require((packet[1] & ~controlMask) == 0, "unsupported ACQUIRE_MEM control flags"); require(queue == 0 || (packet[1] & 0x06287fc3u) == 0, "graphics cache operation in compute queue"); - require(packet[3] == 0 && packet[5] == 0, "ACQUIRE_MEM ranges above 40 bits are not implemented"); + require(packet[3] <= 0xffu && packet[5] <= 0xffu, "invalid ACQUIRE_MEM range high bits"); require(packet[6] <= 0xffffu, "invalid ACQUIRE_MEM poll interval"); - const auto base = static_cast(packet[4]) << 8u; - const auto bytes = static_cast(packet[2]) << 8u; - require(bytes <= (1ull << 40u) - base, "ACQUIRE_MEM range exceeds 40-bit address space"); + const auto base = ((static_cast(packet[5]) << 32u) | packet[4]) << 8u; + const auto bytes = ((static_cast(packet[3]) << 32u) | packet[2]) << 8u; + require(bytes <= (1ull << 48u) - base, "ACQUIRE_MEM range exceeds 48-bit address space"); if (packet.size() == 8) { require((packet[7] & ~0x3ffffu) == 0, "unsupported ACQUIRE_MEM GCR flags"); require((packet[7] & 0x2000u) == 0, "ACQUIRE_MEM cache discard is not implemented"); @@ -377,13 +394,13 @@ void Validate(std::span packet, std::uint32_t queue) { case 0x63: case 0x64: case 0x9f: if (opcode != 0x63) graphics(); size(5); - require((packet[1] & 3u) == 0 && packet[3] == 0x80000000u && packet[4] <= 0x3fffu, "unsupported indirect-register address or control fields"); + require((packet[1] & 3u) == 0 && packet[4] <= 0x3fffu && (packet[3] == 0x80000000u || ((packet[3] & ~0xffffu) == 0 && packet[4] <= 0x10000u - packet[3])), "unsupported indirect-register address or control fields"); break; case 0x69: case 0x76: case 0x79: case 0x7a: { if (IsTagMarker(packet)) break; if (opcode != 0x76) graphics(); require(packet.size() >= 3, "register packet has no values"); - if (opcode == 0x7a) require((packet[1] & 0xf0000000u) == 0 || (packet.size() == 3 && packet[1] == 0x20000243u), "indexed register bank selection is not implemented"); + if (opcode == 0x7a) require((packet[1] & 0xf0000000u) == 0 || (packet.size() == 3 && (packet[1] == 0x20000243u || packet[1] == 0x10000242u)), "indexed register bank selection is not implemented"); const auto offset = registerOffset(packet[1]); require(packet.size() - 2 <= 0x10000u - offset, "register range overflow"); break; @@ -427,7 +444,7 @@ void Validate(std::span packet, std::uint32_t queue) { require(packet.size() >= 5, "WRITE_DATA has no data"); require((packet[1] & ~(0x40110f00u | WriteDataCachePolicy)) == 0, "WRITE_DATA engine or reserved fields are not implemented"); const auto destination = (packet[1] >> 8u) & 0xfu; - require(destination == 1 || destination == 2 || (queue != 0 && destination == 5), "WRITE_DATA register or GDS destination is not implemented"); + require(destination == 1 || destination == 2 || destination == 5, "WRITE_DATA register or GDS destination is not implemented"); require((packet[2] & 3u) == 0, "misaligned WRITE_DATA destination"); break; } @@ -444,7 +461,8 @@ void Validate(std::span packet, std::uint32_t queue) { case 0x50: size(7); require((packet[1] & ~(0xe0300001u | DmaSourceCachePolicy | DmaDestinationCachePolicy)) == 0, "DMA_DATA reserved fields are not implemented"); - require(memorySelector(dmaDestination(packet)) || dmaDestination(packet) == DmaSelectGds, "DMA_DATA register or prefetch destination is not implemented"); + require(memorySelector(dmaDestination(packet)) || dmaDestination(packet) == DmaSelectGds || dmaDestination(packet) == DmaSelectNowhere, "DMA_DATA register destination is not implemented"); + require(dmaDestination(packet) != DmaSelectNowhere || memorySelector(dmaSource(packet)), "DMA_DATA prefetch of a register, GDS or immediate source is not implemented"); require(memorySelector(dmaSource(packet)) || dmaSource(packet) == 2 || dmaSource(packet) == DmaSelectGds, "DMA_DATA register source is not implemented"); require(dmaSource(packet) != DmaSelectGds || (packet[3] == 0 && gdsRange(packet[2], packet[6] & 0x3ffffffu)), "DMA_DATA GDS source range exceeds the GDS"); require(dmaDestination(packet) != DmaSelectGds || (packet[5] == 0 && gdsRange(packet[4], packet[6] & 0x3ffffffu)), "DMA_DATA GDS destination range exceeds the GDS"); @@ -570,6 +588,7 @@ std::optional ResolveStore(std::span packet, co } case 0x50: { if (packet.size() < 7 || dmaDestination(packet) == DmaSelectGds) return std::nullopt; + if (dmaDestination(packet) == DmaSelectNowhere) return StoreWrite{address(packet[2], packet[3]), {}, {}}; const std::size_t bytes = packet[6] & 0x3ffffffu; const auto destination = address(packet[4], packet[5]); if (!fits(destination, bytes)) return std::nullopt; @@ -734,9 +753,20 @@ DrawParameters ResolveDraw(std::span packet, const QueueSta std::vector ReadIndirectRegisters(std::span packet) { require(packet.size() == 5 && IndirectRegisterOpcode((packet[0] >> 8u) & 0xffu), "expected indirect register packet"); - std::vector pairs(static_cast(packet[4]) * 2); // Named for the [hooksync] attribution (the read goes through the flush hook). const GuestMemory::ReadSiteScope site(GuestMemory::ReadSite::Registers); + if (packet[3] != 0x80000000u) { + std::vector values(packet[4]); + GuestMemory::Read(address(packet[1], packet[2]), std::as_writable_bytes(std::span(values)), 4); + std::vector pairs; + pairs.reserve(values.size() * 2); + for (std::size_t i = 0; i < values.size(); ++i) { + pairs.push_back(packet[3] + static_cast(i)); + pairs.push_back(values[i]); + } + return pairs; + } + std::vector pairs(static_cast(packet[4]) * 2); GuestMemory::Read(address(packet[1], packet[2]), std::as_writable_bytes(std::span(pairs)), 4); return pairs; } @@ -760,11 +790,10 @@ void Execute(std::span packet, QueueState& queue) { case 0x10: switch ((packet[0] >> 2u) & 0x3fu) { case 0: return; - case 0x09: queue = QueueState{}; return; + case 0x09: queue = std::move(*std::make_unique()); return; case 0x0b: queue.markers.emplace_back(reinterpret_cast(packet.data() + 1)); return; case 0x0c: - require(!queue.markers.empty(), "marker stack underflow"); - queue.markers.pop_back(); + if (!queue.markers.empty()) queue.markers.pop_back(); return; case 0x1a: if (TraceContextState()) std::fprintf(stderr, "[context] op %u: %zu registers, saved %d, cb0 %x info %x mask %x\n", packet[1], queue.context.size(), queue.savedContext.has_value() ? 1 : 0, queue.context.contains(0x318) ? queue.context.at(0x318) : 0u, queue.context.contains(0x31c) ? queue.context.at(0x31c) : 0u, queue.context.contains(0x8e) ? queue.context.at(0x8e) : 0u); @@ -811,7 +840,7 @@ void Execute(std::span packet, QueueState& queue) { const auto start = std::chrono::steady_clock::now(); bool warned = false; while (!WaitSatisfied(packet)) { - std::this_thread::sleep_for(std::chrono::microseconds(50)); + PreciseSleepUs(50); if (!warned && std::chrono::steady_clock::now() - start > std::chrono::seconds(5)) { warned = true; std::fprintf(stderr, "[gpu] WAIT_REG_MEM at 0x%llx still waiting after 5s (function %u ref 0x%x mask 0x%x)\n", @@ -868,9 +897,10 @@ void Execute(std::span packet, QueueState& queue) { GuestMemory::Write(destination, data); return; } + case 0x28: return; case 0x50: { const std::size_t bytes = packet[6] & 0x3ffffffu; - if (bytes == 0) return; + if (bytes == 0 || dmaDestination(packet) == DmaSelectNowhere) return; auto data = dmaSourceBytes(packet); const auto destination = dmaDestination(packet) == DmaSelectGds ? GdsAddress() + packet[4] : address(packet[4], packet[5]); GuestMemory::CheckRange(reinterpret_cast(destination), bytes, 1, true); diff --git a/core/libs/prx/libSceAgcDriver/Execution/src/ShaderMemory.cpp b/core/libs/prx/libSceAgcDriver/Execution/src/ShaderMemory.cpp index 8c07b7b74..7344a3418 100644 --- a/core/libs/prx/libSceAgcDriver/Execution/src/ShaderMemory.cpp +++ b/core/libs/prx/libSceAgcDriver/Execution/src/ShaderMemory.cpp @@ -201,6 +201,28 @@ bool ShaderMemory::read(void* context, std::uint64_t address, std::uint32_t* val return true; } +bool ShaderMemory::accessible(void* context, std::uint64_t address, std::uint64_t bytes) { + auto& self = *static_cast(context); + if (bytes == 0 || address > std::numeric_limits::max() - bytes) return false; + const auto end = address + bytes; + while (address < end) { + const auto next = self.initial.upper_bound(address); + if (next != self.initial.begin()) { + const auto previous = std::prev(next); + if (address - previous->first < previous->second.size()) { + address = previous->first + previous->second.size(); + continue; + } + } + const auto base = address & ~static_cast(PageBytes - 1); + const auto pageEnd = std::min(base + PageBytes, end); + const auto& page = self.page(base); + if (!page.wordwise && !page.valid.all() && !GuestMemory::Accessible(reinterpret_cast(address), static_cast(pageEnd - address))) return false; + address = pageEnd; + } + return true; +} + ShaderMemory::KnownValueCounts ShaderMemory::KnownValues() { auto& totals = CaptureTotals(); return {totals.wordsKnown.load(std::memory_order_relaxed), totals.wordsKnownVerified.load(std::memory_order_relaxed), totals.wordsKnownMismatches.load(std::memory_order_relaxed)}; @@ -233,6 +255,7 @@ std::shared_ptr ShaderMemory::capture(c runtime.userContext = this; runtime.readMemory = &read; runtime.readSpecializationMemory = &read; + runtime.accessible = &accessible; auto capture = invocation != nullptr ? invocation->Capture(runtime) : handle != nullptr ? ShaderRecompiler::CaptureResources(request, runtime, *handle) : ShaderRecompiler::CaptureResources(request, runtime); if (profile) { totals.captureNanoseconds += NanosecondsSince(started); diff --git a/core/libs/prx/libSceAgcDriver/Execution/src/SubgroupClock.cpp b/core/libs/prx/libSceAgcDriver/Execution/src/SubgroupClock.cpp index ac24b51d0..f586b7f20 100644 --- a/core/libs/prx/libSceAgcDriver/Execution/src/SubgroupClock.cpp +++ b/core/libs/prx/libSceAgcDriver/Execution/src/SubgroupClock.cpp @@ -15,7 +15,8 @@ constexpr std::array NarrowClockFamilies{ } -bool NarrowSubgroupClock(VkDriverId driver, std::string_view deviceName) { +bool NarrowSubgroupClock(VkDriverId driver, std::uint32_t deviceId, std::string_view deviceName) { + if (driver == VK_DRIVER_ID_AMD_PROPRIETARY && (deviceId == 0x1114u || deviceId == 0x744cu || deviceId == 0x747eu)) return true; if (driver != VK_DRIVER_ID_MESA_RADV) return false; constexpr std::string_view prefix = "(RADV "; const auto start = deviceName.rfind(prefix); diff --git a/core/libs/prx/libSceAgcDriver/Execution/src/VulkanDevice.cpp b/core/libs/prx/libSceAgcDriver/Execution/src/VulkanDevice.cpp index 96c366d80..65f70f840 100644 --- a/core/libs/prx/libSceAgcDriver/Execution/src/VulkanDevice.cpp +++ b/core/libs/prx/libSceAgcDriver/Execution/src/VulkanDevice.cpp @@ -6,6 +6,7 @@ #include "prx/libSceAgcDriver/Execution/include/Driver.hpp" #include "prx/libSceAgcDriver/Execution/include/PerformanceTimer.hpp" #include "prx/libSceAgcDriver/Execution/include/BdaFeatures.hpp" +#include "prx/libSceAgcDriver/Execution/include/FragmentBarycentric.hpp" #include "prx/libSceAgcDriver/Execution/include/SubgroupClock.hpp" #include "prx/libSceAgcDriver/Execution/include/ShaderDeviceProfile.hpp" #include "prx/libSceAgcDriver/Execution/include/PresentationScaler.hpp" @@ -160,6 +161,20 @@ struct ComputePipelineObjects { } }; +constexpr std::uint64_t WorkgroupMemoryGranule = 1u << 20u; + +struct WorkgroupMemory { + VkDevice device = VK_NULL_HANDLE; + PFN_vkDestroyDescriptorPool destroyPool = nullptr; + std::unique_ptr buffer; + VkDescriptorPool pool = VK_NULL_HANDLE; + VkDescriptorSet set = VK_NULL_HANDLE; + ~WorkgroupMemory() { + buffer.reset(); + if (pool != VK_NULL_HANDLE) destroyPool(device, pool, nullptr); + } +}; + struct VulkanDevice::State { void* library = nullptr; PFN_vkGetInstanceProcAddr instanceProc = nullptr; @@ -201,16 +216,22 @@ struct VulkanDevice::State { // VK_EXT_descriptor_indexing with non-uniform image array indexing (bindless image tables in // graphics stages, and compute workgroups wider than a wave). bool descriptorIndexing = false; + bool storageBufferUpdateAfterBind = false; + VkPhysicalDeviceDescriptorIndexingPropertiesEXT descriptorIndexingProperties{VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_DESCRIPTOR_INDEXING_PROPERTIES_EXT}; bool imageInt64Atomics = false; + bool bufferInt64Atomics = false; bool primitiveListRestart = false; bool depthClipControl = false; + bool graphicsPipelineLibrary = false; bool imageViewMinLod = false; bool pipelineExecutableInfo = false; bool maintenance8 = false; std::uint32_t srgbDecodeFormats = 0; + bool storageImageWithoutFormat = false; bool depthClamp = false; bool depthBounds = false; bool depthBiasClamp = false; + bool dualSrcBlend = false; bool occlusionQueryPrecise = false; VkDeviceSize hostImportAlignment = 0; bool dmaBufImport = false; @@ -219,8 +240,11 @@ struct VulkanDevice::State { bool samplerAnisotropy = false; bool textureCompressionBC = false; bool samplerFilterMinmax = false; + bool nonSeamlessCubeMap = false; bool fragmentShaderPixelInterlock = false; bool conservativeRasterization = false; + bool provokingVertexLast = false; + bool provokingVertexModePerPipeline = false; // VK_KHR_timeline_semaphore enabled: the recorder's unlocked waits are available. bool timelineSemaphores = false; bool computeWave32 = false; @@ -241,6 +265,8 @@ struct VulkanDevice::State { // Device-local buffers of a repeated 16-byte fill pattern (FillBuffer), most recently used // last; each is filled once by a doubling chain in device memory. std::vector, std::shared_ptr>> patternBuffers; + VkDescriptorSetLayout workgroupMemoryLayout = VK_NULL_HANDLE; + std::shared_ptr workgroupMemory; // Compute pipeline objects by variant (and push-constant use), under their own mutex: the // dispatch's find-or-insert and the verify switch's lookups touch the map, recipes hold weak // references to its objects. @@ -332,10 +358,56 @@ struct VulkanDevice::State { return function; } + void CreateWorkgroupMemoryLayout() { + const VkDescriptorSetLayoutBinding binding{0, VK_DESCRIPTOR_TYPE_STORAGE_BUFFER, 1, VK_SHADER_STAGE_COMPUTE_BIT, nullptr}; + VkDescriptorSetLayoutCreateInfo info{VK_STRUCTURE_TYPE_DESCRIPTOR_SET_LAYOUT_CREATE_INFO}; + info.bindingCount = 1; + info.pBindings = &binding; + check(DeviceFunction("vkCreateDescriptorSetLayout")(device, &info, nullptr, &workgroupMemoryLayout), "vkCreateDescriptorSetLayout workgroup memory"); + } + + std::shared_ptr WorkgroupMemoryFor(const Graphics::Context& context, std::uint32_t x, std::uint32_t y, std::uint32_t z, std::uint32_t dwords) { + const std::uint64_t range = properties.limits.maxStorageBufferRange; + const auto groupBytes = static_cast(dwords) * 4u; + std::uint64_t groups = 1; + for (const std::uint64_t count : {x, y, z}) { + if (count != 0u && groups > range / groupBytes / count) { + throw std::runtime_error("Vulkan dispatch: the LDS of " + std::to_string(x) + "x" + std::to_string(y) + "x" + std::to_string(z) + " workgroups of " + std::to_string(groupBytes) + " bytes each is over maxStorageBufferRange " + std::to_string(range)); + } + groups *= count; + } + const auto bytes = std::max(groups, 1u) * groupBytes; + if (workgroupMemory != nullptr && workgroupMemory->buffer->Size() >= bytes) return workgroupMemory; + const auto size = std::min((bytes + WorkgroupMemoryGranule - 1u) / WorkgroupMemoryGranule * WorkgroupMemoryGranule, range); + auto memory = std::make_shared(); + memory->device = device; + memory->destroyPool = DeviceFunction("vkDestroyDescriptorPool"); + memory->buffer = std::make_unique(context, static_cast(size), VK_BUFFER_USAGE_STORAGE_BUFFER_BIT); + const VkDescriptorPoolSize poolSize{VK_DESCRIPTOR_TYPE_STORAGE_BUFFER, 1}; + VkDescriptorPoolCreateInfo poolInfo{VK_STRUCTURE_TYPE_DESCRIPTOR_POOL_CREATE_INFO}; + poolInfo.maxSets = 1; + poolInfo.poolSizeCount = 1; + poolInfo.pPoolSizes = &poolSize; + check(DeviceFunction("vkCreateDescriptorPool")(device, &poolInfo, nullptr, &memory->pool), "vkCreateDescriptorPool workgroup memory"); + VkDescriptorSetAllocateInfo allocation{VK_STRUCTURE_TYPE_DESCRIPTOR_SET_ALLOCATE_INFO}; + allocation.descriptorPool = memory->pool; + allocation.descriptorSetCount = 1; + allocation.pSetLayouts = &workgroupMemoryLayout; + check(DeviceFunction("vkAllocateDescriptorSets")(device, &allocation, &memory->set), "vkAllocateDescriptorSets workgroup memory"); + const VkDescriptorBufferInfo described{memory->buffer->Handle(), 0, size}; + VkWriteDescriptorSet write{VK_STRUCTURE_TYPE_WRITE_DESCRIPTOR_SET}; + write.dstSet = memory->set; + write.dstBinding = 0; + write.descriptorCount = 1; + write.descriptorType = VK_DESCRIPTOR_TYPE_STORAGE_BUFFER; + write.pBufferInfo = &described; + DeviceFunction("vkUpdateDescriptorSets")(device, 1, &write, 0, nullptr); + workgroupMemory = std::move(memory); + return workgroupMemory; + } + void Upload(std::span pixels) { - APS5_LOG_OUT("Upload pixels=%zu uploadSize=%llu buffer=%p memory=%p mapping=%p", pixels.size(), static_cast(uploadSize), reinterpret_cast(uploadBuffer), reinterpret_cast(uploadMemory), uploadMapping); if (uploadSize < pixels.size()) { - APS5_LOG_OUT("Upload reallocating oldSize=%llu newSize=%zu", static_cast(uploadSize), pixels.size()); if (uploadMapping) DeviceFunction("vkUnmapMemory")(device, uploadMemory); uploadMapping = nullptr; if (uploadBuffer) DeviceFunction("vkDestroyBuffer")(device, uploadBuffer, nullptr); @@ -359,7 +431,6 @@ struct VulkanDevice::State { } } require(memoryType < memoryProperties.memoryTypeCount, "coherent host upload memory is unavailable"); - APS5_LOG_OUT("Upload requirements size=%llu alignment=%llu typeBits=0x%x memoryType=%u", static_cast(requirements.size), static_cast(requirements.alignment), requirements.memoryTypeBits, memoryType); VkMemoryAllocateInfo allocation{VK_STRUCTURE_TYPE_MEMORY_ALLOCATE_INFO}; allocation.allocationSize = requirements.size; allocation.memoryTypeIndex = memoryType; @@ -367,10 +438,8 @@ struct VulkanDevice::State { check(DeviceFunction("vkBindBufferMemory")(device, uploadBuffer, uploadMemory, 0), "vkBindBufferMemory display upload"); check(DeviceFunction("vkMapMemory")(device, uploadMemory, 0, pixels.size(), 0, &uploadMapping), "vkMapMemory display upload"); uploadSize = pixels.size(); - APS5_LOG_OUT("Upload allocation complete buffer=%p memory=%p mapping=%p size=%llu", reinterpret_cast(uploadBuffer), reinterpret_cast(uploadMemory), uploadMapping, static_cast(uploadSize)); } std::memcpy(uploadMapping, pixels.data(), pixels.size()); - APS5_LOG_OUT("Upload memcpy complete bytes=%zu", pixels.size()); } bool SharedSlotInFlight() { @@ -535,6 +604,8 @@ struct VulkanDevice::State { Graphics::ClearImageMirrors(device); Graphics::ClearHostImports(device); patternBuffers.clear(); + workgroupMemory.reset(); + if (workgroupMemoryLayout != VK_NULL_HANDLE) reinterpret_cast(deviceProc(device, "vkDestroyDescriptorSetLayout"))(device, workgroupMemoryLayout, nullptr); descriptorCache.reset(); emptyBuffer.reset(); samplerCache.reset(); @@ -760,6 +831,12 @@ VulkanDevice::VulkanDevice(const PresentationWindow* window) : state(std::make_u VkPhysicalDeviceFeatures2 features{VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_FEATURES_2, &barycentricFeatures}; state->InstanceFunction("vkGetPhysicalDeviceFeatures2")(selected, &features); state->fragmentShaderBarycentric = barycentricFeatures.fragmentShaderBarycentric == VK_TRUE; + if (state->fragmentShaderBarycentric && hasExtension(VK_KHR_DRIVER_PROPERTIES_EXTENSION_NAME)) { + VkPhysicalDeviceDriverProperties driver{VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_DRIVER_PROPERTIES}; + VkPhysicalDeviceProperties2 driverProperties{VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_PROPERTIES_2, &driver}; + state->InstanceFunction("vkGetPhysicalDeviceProperties2")(selected, &driverProperties); + state->fragmentShaderBarycentric = FragmentShaderBarycentricUsable(driver.driverID, state->fragmentShaderBarycentric); + } } VkPhysicalDeviceFragmentShaderInterlockFeaturesEXT interlockFeatures{VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_FRAGMENT_SHADER_INTERLOCK_FEATURES_EXT}; if (hasExtension(VK_EXT_FRAGMENT_SHADER_INTERLOCK_EXTENSION_NAME)) { @@ -779,7 +856,7 @@ VulkanDevice::VulkanDevice(const PresentationWindow* window) : state(std::make_u VkPhysicalDeviceDriverProperties driver{VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_DRIVER_PROPERTIES}; VkPhysicalDeviceProperties2 driverProperties{VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_PROPERTIES_2, &driver}; state->InstanceFunction("vkGetPhysicalDeviceProperties2")(selected, &driverProperties); - state->narrowSubgroupClock = NarrowSubgroupClock(driver.driverID, state->properties.deviceName); + state->narrowSubgroupClock = NarrowSubgroupClock(driver.driverID, state->properties.deviceID, state->properties.deviceName); APS5_LOG_OUT("Shader clock driver=%d narrowSubgroupClock=%d", static_cast(driver.driverID), state->narrowSubgroupClock ? 1 : 0); } std::vector deviceExtensions; @@ -809,6 +886,7 @@ VulkanDevice::VulkanDevice(const PresentationWindow* window) : state(std::make_u atomicInt64Features = {VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_SHADER_ATOMIC_INT64_FEATURES_KHR}; atomicInt64Features.shaderBufferInt64Atomics = VK_TRUE; if (bufferInt64Atomics) deviceExtensions.push_back(VK_KHR_SHADER_ATOMIC_INT64_EXTENSION_NAME); + state->bufferInt64Atomics = bufferInt64Atomics; VkPhysicalDeviceShaderImageAtomicInt64FeaturesEXT imageAtomicInt64Features{VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_SHADER_IMAGE_ATOMIC_INT64_FEATURES_EXT}; if (hasExtension(VK_EXT_SHADER_IMAGE_ATOMIC_INT64_EXTENSION_NAME)) { VkPhysicalDeviceFeatures2 features{VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_FEATURES_2, &imageAtomicInt64Features}; @@ -826,6 +904,21 @@ VulkanDevice::VulkanDevice(const PresentationWindow* window) : state(std::make_u deviceExtensions.push_back(VK_KHR_SHADER_FLOAT_CONTROLS_EXTENSION_NAME); state->capabilities.push_back(spv::CapabilitySignedZeroInfNanPreserve); state->spirvExtensions.push_back("SPV_KHR_float_controls"); + VkPhysicalDeviceShaderFloat16Int8FeaturesKHR float16Features{VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_SHADER_FLOAT16_INT8_FEATURES_KHR}; + if (hasExtension(VK_KHR_SHADER_FLOAT16_INT8_EXTENSION_NAME)) { + VkPhysicalDeviceFeatures2 features{VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_FEATURES_2, &float16Features}; + state->InstanceFunction("vkGetPhysicalDeviceFeatures2")(selected, &features); + } + const bool float16Rte = float16Features.shaderFloat16 == VK_TRUE && floatControls.shaderRoundingModeRTEFloat16 == VK_TRUE && + floatControls.shaderDenormPreserveFloat16 == VK_TRUE && floatControls.shaderSignedZeroInfNanPreserveFloat16 == VK_TRUE; + float16Features = {VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_SHADER_FLOAT16_INT8_FEATURES_KHR}; + float16Features.shaderFloat16 = VK_TRUE; + if (float16Rte) { + deviceExtensions.push_back(VK_KHR_SHADER_FLOAT16_INT8_EXTENSION_NAME); + state->capabilities.push_back(spv::CapabilityFloat16); + state->capabilities.push_back(spv::CapabilityRoundingModeRTE); + state->capabilities.push_back(spv::CapabilityDenormPreserve); + } deviceExtensions.push_back(VK_KHR_BUFFER_DEVICE_ADDRESS_EXTENSION_NAME); deviceExtensions.push_back(VK_KHR_8BIT_STORAGE_EXTENSION_NAME); state->capabilities.push_back(4448); @@ -891,6 +984,21 @@ VulkanDevice::VulkanDevice(const PresentationWindow* window) : state(std::make_u } listRestartFeatures = {VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_PRIMITIVE_TOPOLOGY_LIST_RESTART_FEATURES_EXT}; listRestartFeatures.primitiveTopologyListRestart = state->primitiveListRestart ? VK_TRUE : VK_FALSE; + VkPhysicalDeviceProvokingVertexFeaturesEXT provokingFeatures{VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_PROVOKING_VERTEX_FEATURES_EXT}; + if (hasExtension(VK_EXT_PROVOKING_VERTEX_EXTENSION_NAME)) { + VkPhysicalDeviceFeatures2 features{VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_FEATURES_2, &provokingFeatures}; + state->InstanceFunction("vkGetPhysicalDeviceFeatures2")(selected, &features); + state->provokingVertexLast = provokingFeatures.provokingVertexLast == VK_TRUE; + if (state->provokingVertexLast) { + VkPhysicalDeviceProvokingVertexPropertiesEXT provokingProperties{VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_PROVOKING_VERTEX_PROPERTIES_EXT}; + VkPhysicalDeviceProperties2 properties{VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_PROPERTIES_2, &provokingProperties}; + state->InstanceFunction("vkGetPhysicalDeviceProperties2")(selected, &properties); + state->provokingVertexModePerPipeline = provokingProperties.provokingVertexModePerPipeline == VK_TRUE; + deviceExtensions.push_back(VK_EXT_PROVOKING_VERTEX_EXTENSION_NAME); + } + } + provokingFeatures = {VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_PROVOKING_VERTEX_FEATURES_EXT}; + provokingFeatures.provokingVertexLast = state->provokingVertexLast ? VK_TRUE : VK_FALSE; VkPhysicalDeviceImageViewMinLodFeaturesEXT minLodFeatures{VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_IMAGE_VIEW_MIN_LOD_FEATURES_EXT}; if (hasExtension(VK_EXT_IMAGE_VIEW_MIN_LOD_EXTENSION_NAME)) { VkPhysicalDeviceFeatures2 features{VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_FEATURES_2, &minLodFeatures}; @@ -916,6 +1024,33 @@ VulkanDevice::VulkanDevice(const PresentationWindow* window) : state(std::make_u state->depthClipControl = depthClipFeatures.depthClipControl == VK_TRUE; if (state->depthClipControl) deviceExtensions.push_back(VK_EXT_DEPTH_CLIP_CONTROL_EXTENSION_NAME); } + VkPhysicalDeviceNonSeamlessCubeMapFeaturesEXT nonSeamlessCubeFeatures{VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_NON_SEAMLESS_CUBE_MAP_FEATURES_EXT}; + if (hasExtension(VK_EXT_NON_SEAMLESS_CUBE_MAP_EXTENSION_NAME)) { + VkPhysicalDeviceFeatures2 features{VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_FEATURES_2, &nonSeamlessCubeFeatures}; + state->InstanceFunction("vkGetPhysicalDeviceFeatures2")(selected, &features); + state->nonSeamlessCubeMap = nonSeamlessCubeFeatures.nonSeamlessCubeMap == VK_TRUE; + if (state->nonSeamlessCubeMap) deviceExtensions.push_back(VK_EXT_NON_SEAMLESS_CUBE_MAP_EXTENSION_NAME); + } + nonSeamlessCubeFeatures = {VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_NON_SEAMLESS_CUBE_MAP_FEATURES_EXT}; + nonSeamlessCubeFeatures.nonSeamlessCubeMap = VK_TRUE; + VkPhysicalDeviceGraphicsPipelineLibraryFeaturesEXT libraryFeatures{VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_GRAPHICS_PIPELINE_LIBRARY_FEATURES_EXT}; + VkPhysicalDeviceExtendedDynamicStateFeaturesEXT dynamicStateFeatures{VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_EXTENDED_DYNAMIC_STATE_FEATURES_EXT}; + VkPhysicalDeviceDynamicRenderingFeaturesKHR renderingFeatures{VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_DYNAMIC_RENDERING_FEATURES_KHR}; + constexpr std::array libraryExtensions{VK_KHR_PIPELINE_LIBRARY_EXTENSION_NAME, VK_EXT_GRAPHICS_PIPELINE_LIBRARY_EXTENSION_NAME, VK_EXT_EXTENDED_DYNAMIC_STATE_EXTENSION_NAME, VK_KHR_DYNAMIC_RENDERING_EXTENSION_NAME, VK_KHR_DEPTH_STENCIL_RESOLVE_EXTENSION_NAME, VK_KHR_CREATE_RENDERPASS_2_EXTENSION_NAME}; + if (std::getenv("APS5_NO_GPL") == nullptr && std::all_of(libraryExtensions.begin(), libraryExtensions.end(), hasExtension)) { + dynamicStateFeatures.pNext = &renderingFeatures; + libraryFeatures.pNext = &dynamicStateFeatures; + VkPhysicalDeviceFeatures2 features{VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_FEATURES_2, &libraryFeatures}; + state->InstanceFunction("vkGetPhysicalDeviceFeatures2")(selected, &features); + VkPhysicalDeviceGraphicsPipelineLibraryPropertiesEXT libraryProperties{VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_GRAPHICS_PIPELINE_LIBRARY_PROPERTIES_EXT}; + VkPhysicalDeviceProperties2 properties{VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_PROPERTIES_2, &libraryProperties}; + state->InstanceFunction("vkGetPhysicalDeviceProperties2")(selected, &properties); + state->graphicsPipelineLibrary = libraryFeatures.graphicsPipelineLibrary == VK_TRUE && dynamicStateFeatures.extendedDynamicState == VK_TRUE && renderingFeatures.dynamicRendering == VK_TRUE && libraryProperties.graphicsPipelineLibraryFastLinking == VK_TRUE && properties.properties.limits.maxPushConstantsSize >= Graphics::PipelinePushConstantBytes; + if (state->graphicsPipelineLibrary) deviceExtensions.insert(deviceExtensions.end(), libraryExtensions.begin(), libraryExtensions.end()); + } + renderingFeatures = {VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_DYNAMIC_RENDERING_FEATURES_KHR, nullptr, VK_TRUE}; + libraryFeatures = {VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_GRAPHICS_PIPELINE_LIBRARY_FEATURES_EXT, &dynamicStateFeatures, VK_TRUE}; + dynamicStateFeatures = {VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_EXTENDED_DYNAMIC_STATE_FEATURES_EXT, &renderingFeatures, VK_TRUE}; if (state->meshShader) { deviceExtensions.insert(deviceExtensions.end(), meshExtensions.begin(), meshExtensions.end()); state->capabilities.push_back(5283); @@ -941,7 +1076,7 @@ VulkanDevice::VulkanDevice(const PresentationWindow* window) : state(std::make_u enabled.tessellationShader = available.tessellationShader; state->tessellationShader = enabled.tessellationShader == VK_TRUE; if (state->tessellationShader) state->capabilities.push_back(3); - APS5_LOG_OUT("Vulkan features tessellationAvailable=%u mesh=%u depthClip=%u depthRangeUnrestricted=%u", static_cast(state->tessellationShader), static_cast(state->meshShader), static_cast(state->depthClipControl), static_cast(state->depthRangeUnrestricted)); + APS5_LOG_OUT("Vulkan features tessellationAvailable=%u mesh=%u depthClip=%u depthRangeUnrestricted=%u pipelineLibrary=%u", static_cast(state->tessellationShader), static_cast(state->meshShader), static_cast(state->depthClipControl), static_cast(state->depthRangeUnrestricted), static_cast(state->graphicsPipelineLibrary)); require(available.samplerAnisotropy && available.textureCompressionBC, "device lacks sampler anisotropy or BC texture compression support required for texture sampling"); enabled.samplerAnisotropy = VK_TRUE; enabled.textureCompressionBC = VK_TRUE; @@ -960,6 +1095,8 @@ VulkanDevice::VulkanDevice(const PresentationWindow* window) : state(std::make_u state->depthBounds = enabled.depthBounds == VK_TRUE; enabled.depthBiasClamp = available.depthBiasClamp; state->depthBiasClamp = enabled.depthBiasClamp == VK_TRUE; + enabled.dualSrcBlend = available.dualSrcBlend; + state->dualSrcBlend = enabled.dualSrcBlend == VK_TRUE; enabled.occlusionQueryPrecise = available.occlusionQueryPrecise; state->occlusionQueryPrecise = enabled.occlusionQueryPrecise == VK_TRUE; // Recompiled storage-image access declares no format (the guest descriptor decides it). @@ -981,6 +1118,7 @@ VulkanDevice::VulkanDevice(const PresentationWindow* window) : state(std::make_u enabled.shaderClipDistance = available.shaderClipDistance; if (enabled.shaderStorageImageWriteWithoutFormat) state->capabilities.push_back(spv::CapabilityStorageImageWriteWithoutFormat); if (enabled.shaderStorageImageReadWithoutFormat) state->capabilities.push_back(spv::CapabilityStorageImageReadWithoutFormat); + state->storageImageWithoutFormat = enabled.shaderStorageImageWriteWithoutFormat == VK_TRUE && enabled.shaderStorageImageReadWithoutFormat == VK_TRUE; // Bindless image tables index an image array with a wave-uniform runtime slot. enabled.shaderSampledImageArrayDynamicIndexing = available.shaderSampledImageArrayDynamicIndexing; enabled.shaderStorageImageArrayDynamicIndexing = available.shaderStorageImageArrayDynamicIndexing; @@ -991,17 +1129,23 @@ VulkanDevice::VulkanDevice(const PresentationWindow* window) : state(std::make_u VkPhysicalDeviceFeatures2 features{VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_FEATURES_2, &descriptorIndexingFeatures}; state->InstanceFunction("vkGetPhysicalDeviceFeatures2")(selected, &features); state->descriptorIndexing = descriptorIndexingFeatures.shaderSampledImageArrayNonUniformIndexing == VK_TRUE && descriptorIndexingFeatures.shaderStorageImageArrayNonUniformIndexing == VK_TRUE; + state->storageBufferUpdateAfterBind = descriptorIndexingFeatures.descriptorBindingStorageBufferUpdateAfterBind == VK_TRUE; + if (state->storageBufferUpdateAfterBind) { + VkPhysicalDeviceProperties2 properties{VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_PROPERTIES_2, &state->descriptorIndexingProperties}; + state->InstanceFunction("vkGetPhysicalDeviceProperties2")(selected, &properties); + } } descriptorIndexingFeatures = {VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_DESCRIPTOR_INDEXING_FEATURES_EXT}; + if (state->descriptorIndexing || state->storageBufferUpdateAfterBind) deviceExtensions.push_back(VK_EXT_DESCRIPTOR_INDEXING_EXTENSION_NAME); if (state->descriptorIndexing) { descriptorIndexingFeatures.shaderSampledImageArrayNonUniformIndexing = VK_TRUE; descriptorIndexingFeatures.shaderStorageImageArrayNonUniformIndexing = VK_TRUE; - deviceExtensions.push_back(VK_EXT_DESCRIPTOR_INDEXING_EXTENSION_NAME); state->capabilities.push_back(spv::CapabilityShaderNonUniform); state->capabilities.push_back(spv::CapabilitySampledImageArrayNonUniformIndexing); state->capabilities.push_back(spv::CapabilityStorageImageArrayNonUniformIndexing); state->spirvExtensions.push_back("SPV_EXT_descriptor_indexing"); } + descriptorIndexingFeatures.descriptorBindingStorageBufferUpdateAfterBind = state->storageBufferUpdateAfterBind ? VK_TRUE : VK_FALSE; state->samplerAnisotropy = true; state->textureCompressionBC = true; deviceInfo.pEnabledFeatures = &enabled; @@ -1015,10 +1159,18 @@ VulkanDevice::VulkanDevice(const PresentationWindow* window) : state(std::make_u depthClipFeatures.pNext = const_cast(deviceInfo.pNext); deviceInfo.pNext = &depthClipFeatures; } + if (state->graphicsPipelineLibrary) { + renderingFeatures.pNext = const_cast(deviceInfo.pNext); + deviceInfo.pNext = &libraryFeatures; + } if (state->primitiveListRestart) { listRestartFeatures.pNext = const_cast(deviceInfo.pNext); deviceInfo.pNext = &listRestartFeatures; } + if (state->provokingVertexLast) { + provokingFeatures.pNext = const_cast(deviceInfo.pNext); + deviceInfo.pNext = &provokingFeatures; + } if (state->imageViewMinLod) { minLodFeatures.pNext = const_cast(deviceInfo.pNext); deviceInfo.pNext = &minLodFeatures; @@ -1027,6 +1179,10 @@ VulkanDevice::VulkanDevice(const PresentationWindow* window) : state(std::make_u maintenance8Features.pNext = const_cast(deviceInfo.pNext); deviceInfo.pNext = &maintenance8Features; } + if (state->nonSeamlessCubeMap) { + nonSeamlessCubeFeatures.pNext = const_cast(deviceInfo.pNext); + deviceInfo.pNext = &nonSeamlessCubeFeatures; + } byteFeatures.pNext = const_cast(deviceInfo.pNext); if (state->fragmentShaderBarycentric) { barycentricFeatures.pNext = byteFeatures.pNext; @@ -1040,6 +1196,10 @@ VulkanDevice::VulkanDevice(const PresentationWindow* window) : state(std::make_u clockFeatures.pNext = byteFeatures.pNext; byteFeatures.pNext = &clockFeatures; } + if (float16Rte) { + float16Features.pNext = byteFeatures.pNext; + byteFeatures.pNext = &float16Features; + } if (bufferInt64Atomics) { atomicInt64Features.pNext = byteFeatures.pNext; byteFeatures.pNext = &atomicInt64Features; @@ -1053,7 +1213,7 @@ VulkanDevice::VulkanDevice(const PresentationWindow* window) : state(std::make_u imageRobustnessFeatures.pNext = byteFeatures.pNext; byteFeatures.pNext = &imageRobustnessFeatures; } - if (state->descriptorIndexing) { + if (state->descriptorIndexing || state->storageBufferUpdateAfterBind) { descriptorIndexingFeatures.pNext = byteFeatures.pNext; byteFeatures.pNext = &descriptorIndexingFeatures; } @@ -1149,6 +1309,7 @@ VulkanDevice::VulkanDevice(const PresentationWindow* window) : state(std::make_u state->colorTransfer = std::make_unique(graphicsContext()); state->descriptorCache = std::make_unique(graphicsContext()); state->samplerCache = std::make_unique(); + state->CreateWorkgroupMemoryLayout(); state->recorder = std::make_unique(graphicsContext(), state->timelineSemaphores); state->recorder->Activate(); state->context = buildContext(); @@ -1297,6 +1458,13 @@ bool VulkanDevice::OpenWriteOverlaps(std::uint64_t address, std::size_t bytes) c return state->recorder && state->recorder->OpenWriteOverlaps(address, bytes); } +bool VulkanDevice::RecordedWritesSettled(std::uint64_t address, std::size_t bytes) const { + if (!state->recorder) return true; + const auto& recorder = *state->recorder; + if (!recorder.PendingWriteSettled(address, bytes)) return false; + return Graphics::Recorder::PendingCompletionLabels() == 0 || !recorder.CompletionLabelIn(address, bytes); +} + int VulkanDevice::WriteLabelOnGpu(std::uint64_t address, std::span bytes, std::uint64_t stamp, std::uint32_t queue, bool reapFirst) { if (!state->recorder || bytes.empty() || bytes.size() > 65536 || address % 4 != 0 || bytes.size() % 4 != 0) return 4; auto& recorder = *state->recorder; @@ -1312,13 +1480,7 @@ int VulkanDevice::WriteLabelOnGpu(std::uint64_t address, std::spanaddress == 0) return false; if (Graphics::AnyShadowedOverlaps(address, bytes)) Graphics::PublishShadow(address, bytes, Graphics::PublishScope::PartialUnits, Graphics::PublishReason::Label); - recorder.FlushStoresOverlapping(address, bytes); + if (!recorder.QueuesSampleDumps()) recorder.FlushStoresOverlapping(address, bytes); recorder.FlushKeyStoresOverlapping(address, bytes); - if (!recorder.DumpSamples(import->address + (address - import->base))) return false; + if (!recorder.DumpSamples(import->address + (address - import->base), address)) return false; recorder.NotePendingWrite(address, bytes); GuestMemory::MarkWritten(address, bytes); return true; @@ -2033,6 +2195,8 @@ FrameDumps& Dumps() { } bool VulkanDevice::PresentDisplayBuffer(const DisplayBuffer& buffer) { + static const bool storeAtFlip = Graphics::StorageTexture::StoreAtFlipRequested(std::getenv("APS5_STORE_AT_FLIP")); + if (storeAtFlip) Graphics::StorageTexture::FlushAllPending("flip"); if (buffer.tilingMode == 1) { const auto pixels = ReadDisplayBuffer(buffer); return present(buffer.width, buffer.height, true, pixels); @@ -2461,6 +2625,7 @@ ShaderRecompiler::SpirvTarget VulkanDevice::buildTarget() const { target.fragmentShaderBarycentricEnabled = state->fragmentShaderBarycentric; target.nonConstantImageOffsets = state->maintenance8; target.narrowSubgroupClock = state->narrowSubgroupClock; + target.fixedPushSlots = state->graphicsPipelineLibrary; target.srgbDecodeFormats = state->srgbDecodeFormats; if (state->meshShader) { const auto& mesh = state->meshLimits; @@ -2470,10 +2635,20 @@ ShaderRecompiler::SpirvTarget VulkanDevice::buildTarget() const { return target; } +std::optional VulkanDevice::GeometryLimits() const { + if (!state->geometryShader) return std::nullopt; + const auto& limits = state->properties.limits; + return ShaderRecompiler::GeometryStageLimits{limits.maxGeometryInputComponents, limits.maxGeometryOutputComponents, limits.maxGeometryOutputVertices, limits.maxGeometryTotalOutputComponents, limits.maxFragmentInputComponents}; +} + bool VulkanDevice::PrimitiveListRestart() const { return state->primitiveListRestart; } +bool VulkanDevice::DualSrcBlend() const { + return state->dualSrcBlend; +} + bool VulkanDevice::SamplerFilterMinmax() const { return state->samplerFilterMinmax; } @@ -2482,6 +2657,18 @@ bool VulkanDevice::ConservativeRasterization() const { return state->conservativeRasterization; } +VkShaderStageFlags VulkanDevice::SubgroupStages() const { + return state->subgroup.supportedStages; +} + +bool VulkanDevice::ProvokingVertexLast() const { + return state->provokingVertexLast; +} + +bool VulkanDevice::GraphicsPipelineLibraries() const { + return state->graphicsPipelineLibrary; +} + Graphics::Context VulkanDevice::graphicsContext() const { static const bool noCache = std::getenv("APS5_NO_CONTEXT_CACHE") != nullptr; if (state->contextReady && !noCache) return state->context; @@ -2523,25 +2710,33 @@ Graphics::Context VulkanDevice::buildContext() const { context.samplerCache = state->samplerCache.get(); context.drawIndirectFirstInstance = state->drawIndirectFirstInstance; context.multiDrawIndirect = state->multiDrawIndirect; + context.dualSrcBlend = state->dualSrcBlend; context.depthBounds = state->depthBounds; context.depthBiasClamp = state->depthBiasClamp; context.samplerFilterMinmax = state->samplerFilterMinmax; + context.nonSeamlessCubeMap = state->nonSeamlessCubeMap; context.conservativeRasterization = state->conservativeRasterization; + context.provokingVertexLast = state->provokingVertexLast; + context.graphicsPipelineLibrary = state->graphicsPipelineLibrary; + context.provokingVertexModePerPipeline = state->provokingVertexModePerPipeline; context.drawIndirectCount = state->drawIndirectCount; context.occlusionQueryPrecise = state->occlusionQueryPrecise; context.emptyBuffer = state->emptyBuffer ? state->emptyBuffer->Handle() : VK_NULL_HANDLE; context.copiedWriters = state->copiedWriters.get(); context.functions = state->functionsReady ? &state->deviceFunctions : nullptr; context.descriptorIndexing = state->descriptorIndexing; + context.descriptorIndexingLimits = state->descriptorIndexingProperties; context.imageInt64Atomics = state->imageInt64Atomics; context.geometryShader = state->geometryShader; context.sampleRateShading = state->sampleRateShading; context.nullDescriptors = state->shaderProfile != nullptr && state->shaderProfile->NullDescriptors(); + context.bufferInt64Atomics = state->bufferInt64Atomics; context.primitiveListRestart = state->primitiveListRestart; context.imageViewMinLod = state->imageViewMinLod; context.pipelineExecutableInfo = state->pipelineExecutableInfo; context.srgbDecodeFormats = state->srgbDecodeFormats; context.memoryProperties2 = state->memoryBudget ? state->InstanceFunction("vkGetPhysicalDeviceMemoryProperties2") : nullptr; + context.singlePassStorage = state->storageImageWithoutFormat; return context; } @@ -2628,6 +2823,7 @@ struct PreparedDispatch { // APS5_PROFILE_DRAW: stage A's time, added to the [dispatch] phase totals by the dispatch, and // the prepare's parts for the driver's 'prepare:' rows (PreparePhase order). double prepareMs = 0; + std::uint64_t dispatchThreads = 0; enum PreparePhase : std::size_t { PrepareKey, PrepareFind, PreparePrecollect, PreparePresync, PrepareStageA, PreparePhaseCount }; std::array phaseMs{}; // The stage-A pre-sync (see PrepareDispatch): the newest recorder serial waited for without the @@ -3071,7 +3267,7 @@ std::uint64_t VulkanDevice::presync(std::span VulkanDevice::PrepareDispatch(const ShaderRecompiler::RecompileResult& shader, std::span snapshots) { +std::shared_ptr VulkanDevice::PrepareDispatch(const ShaderRecompiler::RecompileResult& shader, std::span snapshots, std::uint64_t dispatchThreads) { ShaderRecompiler::RuntimeAbi::RequireVersion(shader.runtimeAbiVersion); // APS5_LOCKED_BUILD=1: the whole build under the mutex, as before the split. static const bool lockedBuild = std::getenv("APS5_LOCKED_BUILD") != nullptr; @@ -3081,6 +3277,7 @@ std::shared_ptr VulkanDevice::PrepareDispatch(const ShaderReco const Graphics::CompiledShader compute{ShaderRecompiler::ShaderStage::Compute, &shader, 0}; const auto context = graphicsContext(); auto prepared = std::make_shared(); + prepared->dispatchThreads = dispatchThreads; // A cached build (the map locks itself) is revalidated under the mutex by the dispatch, which // looks it up again by the key made here; the object stays local, only its surfaces matter // below. Anything inserted between here and the dispatch is simply replaced by this build. @@ -3100,7 +3297,7 @@ std::shared_ptr VulkanDevice::PrepareDispatch(const ShaderReco phase(PreparedDispatch::PrepareFind); } if (cached == nullptr) { - prepared->resources = std::make_shared(context, compute, snapshots, true); + prepared->resources = std::make_shared(context, compute, snapshots, true, dispatchThreads); phase(PreparedDispatch::PrepareStageA); } if (cached != nullptr && CachedPrecollect()) { @@ -3208,6 +3405,8 @@ void VulkanDevice::decideIndirect(RecordedDispatch& record, IndirectOutcome& out outcome.cpuReason = 1; Graphics::Recorder::CountSync(2); recorder.Sync(); + } else if (record.shader->program->workgroupMemoryDwords != 0) { + outcome.cpuReason = 8; } else if (labelPending() || std::any_of(state->copiedWriters->begin(), state->copiedWriters->end(), [&](const auto& writer) { return writer->WritesOverlap(arguments, 12); }) || std::any_of(Graphics::DrawCopiedWriters()->begin(), Graphics::DrawCopiedWriters()->end(), [&](const auto& writer) { return writer->WritesOverlap(arguments, 12); })) { outcome.cpuReason = 2; @@ -3331,8 +3530,13 @@ void VulkanDevice::recordDispatch(RecordedDispatch& record) { } context.Resolved(&Graphics::DeviceFunctions::cmdBindPipeline, "vkCmdBindPipeline")(commands, VK_PIPELINE_BIND_POINT_COMPUTE, record.objects->pipeline); resources.Bind(commands, VK_PIPELINE_BIND_POINT_COMPUTE, record.objects->layout); + if (const auto dwords = record.shader->program->workgroupMemoryDwords; dwords != 0) { + auto memory = state->WorkgroupMemoryFor(context, record.x, record.y, record.z, dwords); + context.Resolved(&Graphics::DeviceFunctions::cmdBindDescriptorSets, "vkCmdBindDescriptorSets")(commands, VK_PIPELINE_BIND_POINT_COMPUTE, record.objects->layout, ShaderRecompiler::WorkgroupMemoryDescriptorSet, 1, &memory->set, 0, nullptr); + recorder.Keep(std::move(memory)); + } if (record.pushStages != 0) { - context.Resolved(&Graphics::DeviceFunctions::cmdPushConstants, "vkCmdPushConstants")(commands, record.objects->layout, VK_SHADER_STAGE_COMPUTE_BIT, 0, Graphics::PipelinePushConstantBytes, record.pushBytes->data()); + context.Resolved(&Graphics::DeviceFunctions::cmdPushConstants, "vkCmdPushConstants")(commands, record.objects->layout, VK_SHADER_STAGE_COMPUTE_BIT, 0, Graphics::ComputePushConstantBytes, record.pushBytes->data()); } const auto gpuTiming = recorder.BeginGpuTiming(record.programAddress != 0 ? record.programAddress : record.shader->program->variantId); if (argumentImport != nullptr) context.Resolved(&Graphics::DeviceFunctions::cmdDispatchIndirect, "vkCmdDispatchIndirect")(commands, argumentImport->buffer, arguments - argumentImport->base); @@ -3378,7 +3582,7 @@ VulkanDevice::IndirectOutcome VulkanDevice::dispatch(const ShaderRecompiler::Rec } const std::array shaders{{{ShaderRecompiler::ShaderStage::Compute, &shader, 0}}}; const auto pushStages = Graphics::PushConstantStages(shaders); - if (pushStages != 0 && state->properties.limits.maxPushConstantsSize < Graphics::PipelinePushConstantBytes) { + if (pushStages != 0 && state->properties.limits.maxPushConstantsSize < Graphics::ComputePushConstantBytes) { throw std::runtime_error("Vulkan dispatch: compute push constant range exceeds device limit"); } auto pushBytes = Graphics::AssemblePushConstants(shaders); @@ -3504,7 +3708,7 @@ VulkanDevice::IndirectOutcome VulkanDevice::dispatch(const ShaderRecompiler::Rec } if (resources == nullptr) { const auto buildStart = profile ? std::chrono::steady_clock::now() : std::chrono::steady_clock::time_point{}; - resources = std::make_shared(context, shaders[0], snapshots); + resources = std::make_shared(context, shaders[0], snapshots, false, prepared != nullptr ? prepared->dispatchThreads : 0); if (profile) timer.add(PhaseResourcesFullBuild, std::chrono::duration(std::chrono::steady_clock::now() - buildStart).count()); if (cacheable) { ++d.cacheMisses; @@ -3544,11 +3748,11 @@ VulkanDevice::IndirectOutcome VulkanDevice::dispatch(const ShaderRecompiler::Rec moduleInfo.pCode = shader.spirv.data(); check(state->DeviceFunction("vkCreateShaderModule")(state->device, &moduleInfo, nullptr, &objects->module), "vkCreateShaderModule"); timing.Mark("validate_shader_module"); - const auto setLayout = resources->Layout(); - const VkPushConstantRange push{VK_SHADER_STAGE_COMPUTE_BIT, 0, Graphics::PipelinePushConstantBytes}; + const std::array setLayouts{resources->Layout(), shader.workgroupMemoryDwords != 0 ? state->workgroupMemoryLayout : VK_NULL_HANDLE}; + const VkPushConstantRange push{VK_SHADER_STAGE_COMPUTE_BIT, 0, Graphics::ComputePushConstantBytes}; VkPipelineLayoutCreateInfo layoutInfo{VK_STRUCTURE_TYPE_PIPELINE_LAYOUT_CREATE_INFO}; - layoutInfo.setLayoutCount = 1; - layoutInfo.pSetLayouts = &setLayout; + layoutInfo.setLayoutCount = shader.workgroupMemoryDwords != 0 ? ShaderRecompiler::WorkgroupMemoryDescriptorSet + 1 : 1; + layoutInfo.pSetLayouts = setLayouts.data(); layoutInfo.pushConstantRangeCount = pushStages != 0 ? 1 : 0; layoutInfo.pPushConstantRanges = pushStages != 0 ? &push : nullptr; check(state->DeviceFunction("vkCreatePipelineLayout")(state->device, &layoutInfo, nullptr, &objects->layout), "vkCreatePipelineLayout"); @@ -3566,6 +3770,8 @@ VulkanDevice::IndirectOutcome VulkanDevice::dispatch(const ShaderRecompiler::Rec pipelineInfo.layout = objects->layout; if (profile && shader.spirv.size() > 100000) std::fprintf(stderr, "[dispatch] creating a pipeline for %zu SPIR-V words (program 0x%llx)\n", shader.spirv.size(), static_cast(programAddress)); check(state->DeviceFunction("vkCreateComputePipelines")(state->device, context.pipelineCache, 1, &pipelineInfo, nullptr, &objects->pipeline), "vkCreateComputePipelines"); + objects->destroyModule(objects->device, objects->module, nullptr); + objects->module = VK_NULL_HANDLE; Graphics::LogPipelineStatistics_nid_no_patch(context, objects->pipeline); timing.Mark("pipeline_create"); if (pipelineKey != 0) { diff --git a/core/libs/prx/libSceAgcDriver/Graphics/include/BufferPool.hpp b/core/libs/prx/libSceAgcDriver/Graphics/include/BufferPool.hpp index 8cdc11cb3..454890253 100644 --- a/core/libs/prx/libSceAgcDriver/Graphics/include/BufferPool.hpp +++ b/core/libs/prx/libSceAgcDriver/Graphics/include/BufferPool.hpp @@ -22,6 +22,14 @@ struct BufferAllocation { std::size_t bytes; VkBufferUsageFlags usage; VkMemoryPropertyFlags properties = VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT | VK_MEMORY_PROPERTY_HOST_COHERENT_BIT; + VkDeviceSize offset = 0; + bool slab = false; +}; + +struct SlabSlot { + VkDeviceMemory memory; + VkDeviceSize offset; + void* mapping; }; // Released buffer allocations kept for reuse, since creating, binding and mapping one costs tens of @@ -43,23 +51,39 @@ struct BufferAllocation { // the 256-byte copied-region buffers to fit, and those then missed on every build (81869 misses // and 80850 evictions per run, each a Vulkan create or destroy under the pool mutex, 3.5 s per // run). APS5_BUFFER_POOL_SHARED=1 keeps one tier as before. -// -// Device-local allocations (the detiler's scratch buffers and the staging shadows of written guest -// buffers, see GuestBufferMemory) are retained in a third tier with a budget of their own, video -// memory instead of pinned host memory: APS5_STAGING_POOL_MIB (default 512); 0 keeps them in the -// two host tiers as before. class BufferPool { public: explicit BufferPool(const Context& context); + static VkDeviceSize DeviceBudget(const VkPhysicalDeviceMemoryProperties& memory); ~BufferPool(); BufferPool(const BufferPool&) = delete; BufferPool& operator=(const BufferPool&) = delete; // The size a buffer for `bytes` is created with: its size class, or `bytes` itself when large. - static std::size_t Capacity(std::size_t bytes); + static std::size_t Capacity(std::size_t bytes, VkMemoryPropertyFlags properties); + static VkBufferUsageFlags Usage(VkBufferUsageFlags usage, VkMemoryPropertyFlags properties); std::optional Take(std::size_t bytes, VkBufferUsageFlags usage, VkMemoryPropertyFlags properties = VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT | VK_MEMORY_PROPERTY_HOST_COHERENT_BIT); void Put(const BufferAllocation& allocation) noexcept; + static bool SlabEligible(std::size_t capacity, VkDeviceSize alignment, VkDeviceSize size, VkDeviceSize atom); + static VkDeviceSize SlabBlockBytes(std::size_t capacity); + SlabSlot TakeSlot(const Context& context, std::uint32_t memoryType, std::size_t capacity, bool addressable); + void PutSlot(VkDeviceMemory memory, VkDeviceSize offset) noexcept; + std::size_t SlabBlocks(); private: + struct SlabBlock { + VkDeviceMemory memory = VK_NULL_HANDLE; + std::byte* mapping = nullptr; + VkDeviceSize slotBytes = 0; + std::vector free; + std::uint32_t used = 0; + std::uint64_t slab = 0; + }; + struct Slab { + std::vector available; + std::size_t emptyBlocks = 0; + }; + static std::uint64_t SlabKey(std::uint32_t memoryType, std::size_t capacity, bool addressable); + void freeBlock(SlabBlock& block) noexcept; struct Slot { BufferAllocation allocation; std::uint64_t lastUse; @@ -90,8 +114,9 @@ private: // The tier a buffer of `capacity` and `properties` is retained in (the large one for everything // when shared; the device tier for device-local memory while it has a budget). Tier& tierFor(std::size_t capacity, VkMemoryPropertyFlags properties); - // The device tier's budget (APS5_STAGING_POOL_MIB), read once. - static VkDeviceSize DeviceBudget(); + static bool DeviceTierEnabled(); + static bool DeviceTiered(VkMemoryPropertyFlags properties); + static std::size_t NextCapacity(std::size_t capacity); void destroy(const BufferAllocation& allocation) noexcept; // Moves the tier's least recently used slot (the oldest front of its lists) to `evicted`; the // caller destroys those after releasing the mutex, so builds taking buffers on other threads @@ -104,18 +129,24 @@ private: PFN_vkUnmapMemory unmap; PFN_vkDestroyBuffer destroyBuffer; PFN_vkFreeMemory freeMemory; + PFN_vkAllocateMemory allocateMemory; + PFN_vkMapMemory mapMemory; std::mutex mutex; // Not `small`/`large`: (via ) defines `small` as a macro. Tier smallTier; Tier largeTier; Tier deviceTier; std::uint64_t clock = 0; + std::mutex slabMutex; + std::unordered_map slabs; + std::unordered_map> slabBlocks; static constexpr VkDeviceSize budget = 512ull * 1024 * 1024; // The small tier's own budget (slots of at most half a MiB each): pinned host memory the large // tier's budget does not count. static constexpr VkDeviceSize smallBudget = 64ull * 1024 * 1024; // Requests of this size and more keep their exact size and go to the large tier. static constexpr std::size_t classLimit = std::size_t{1} << 20u; + static constexpr unsigned deviceClassBits = 3; static constexpr std::size_t defaultSlots = 4096; }; diff --git a/core/libs/prx/libSceAgcDriver/Graphics/include/ColorTargetLayout.hpp b/core/libs/prx/libSceAgcDriver/Graphics/include/ColorTargetLayout.hpp index 78112575a..8c492a1a0 100644 --- a/core/libs/prx/libSceAgcDriver/Graphics/include/ColorTargetLayout.hpp +++ b/core/libs/prx/libSceAgcDriver/Graphics/include/ColorTargetLayout.hpp @@ -13,21 +13,31 @@ enum class ColorTileMode : std::uint32_t { Linear = 0, Standard4KB = 5, Standard64KB = 9, + S4KBX = 0x15, + D4KBX = 0x16, RenderTarget = 0x1b }; +constexpr bool ColorTileModeIsXor(ColorTileMode mode) { + return mode == ColorTileMode::S4KBX || mode == ColorTileMode::D4KBX || mode == ColorTileMode::RenderTarget; +} + +constexpr std::uint32_t ColorTileModeBlockBytes(ColorTileMode mode) { + return mode == ColorTileMode::Linear ? 256u : mode == ColorTileMode::Standard4KB || mode == ColorTileMode::S4KBX || mode == ColorTileMode::D4KBX ? 4096u : 65536u; +} + constexpr TextureTileMode ColorTextureTileMode(ColorTileMode mode) { - return mode == ColorTileMode::Linear ? TextureTileMode::kLinear : mode == ColorTileMode::Standard4KB ? TextureTileMode::kStandard4KB : mode == ColorTileMode::Standard64KB ? TextureTileMode::kStandard64KB : TextureTileMode::kR64KBX; + return mode == ColorTileMode::Linear ? TextureTileMode::kLinear : mode == ColorTileMode::Standard4KB ? TextureTileMode::kStandard4KB : mode == ColorTileMode::Standard64KB ? TextureTileMode::kStandard64KB : mode == ColorTileMode::S4KBX ? TextureTileMode::kS4KBX : mode == ColorTileMode::D4KBX ? TextureTileMode::kD4KBX : TextureTileMode::kR64KBX; } ColorTileMode DecodeColorTileMode(std::uint32_t attrib3); class ColorTargetLayout { public: - ColorTargetLayout(std::uint32_t width, std::uint32_t height, ColorTileMode mode, std::uint32_t bytesPerElement = 4); + ColorTargetLayout(std::uint32_t width, std::uint32_t height, ColorTileMode mode, std::uint32_t bytesPerElement = 4, std::uint32_t pipeBankXor = 0); std::size_t Bytes() const { return bytes; } std::size_t LinearBytes() const { return static_cast(width) * height * elementBytes; } - std::size_t Alignment() const { return mode == ColorTileMode::Linear ? 256u : mode == ColorTileMode::Standard4KB ? 4096u : 65536u; } + std::size_t Alignment() const { return ColorTileModeBlockBytes(mode); } std::uint32_t BlocksPerRow() const { return pitch / blockWidth; } std::size_t Offset(std::uint32_t x, std::uint32_t y) const; void Detile(std::span source, std::span destination) const; @@ -43,11 +53,29 @@ private: std::uint32_t elementBytes; std::uint32_t blockWidth = 1; std::uint32_t blockHeight = 1; + std::uint32_t pipeBankXor = 0; // The XOR swizzle is linear over GF(2), so a block offset is xOffsets[x] ^ yOffsets[y]. const std::uint32_t* xOffsets = nullptr; const std::uint32_t* yOffsets = nullptr; }; +class CmaskLayout { +public: + static constexpr std::size_t Alignment = 4096; + CmaskLayout(std::uint32_t width, std::uint32_t height); + std::size_t Bytes() const { return bytes; } + std::uint32_t BlocksPerRow() const { return blocksPerRow; } + std::uint32_t TilesX() const { return (width + 7u) / 8u; } + std::uint32_t TilesY() const { return (height + 7u) / 8u; } + std::size_t Nibble(std::uint32_t tileX, std::uint32_t tileY) const; + +private: + std::uint32_t width; + std::uint32_t height; + std::uint32_t blocksPerRow; + std::size_t bytes; +}; + } #endif diff --git a/core/libs/prx/libSceAgcDriver/Graphics/include/Context.hpp b/core/libs/prx/libSceAgcDriver/Graphics/include/Context.hpp index 26c2b821f..a948e08fb 100644 --- a/core/libs/prx/libSceAgcDriver/Graphics/include/Context.hpp +++ b/core/libs/prx/libSceAgcDriver/Graphics/include/Context.hpp @@ -52,10 +52,23 @@ struct DeviceFunctions { PFN_vkCmdDispatchIndirect cmdDispatchIndirect = nullptr; PFN_vkCmdBeginRenderPass cmdBeginRenderPass = nullptr; PFN_vkCmdEndRenderPass cmdEndRenderPass = nullptr; + PFN_vkCmdBeginRenderingKHR cmdBeginRendering = nullptr; + PFN_vkCmdEndRenderingKHR cmdEndRendering = nullptr; PFN_vkCmdSetViewport cmdSetViewport = nullptr; PFN_vkCmdSetScissor cmdSetScissor = nullptr; PFN_vkCmdSetDepthBounds cmdSetDepthBounds = nullptr; PFN_vkCmdSetDepthBias cmdSetDepthBias = nullptr; + PFN_vkCmdSetCullModeEXT cmdSetCullMode = nullptr; + PFN_vkCmdSetFrontFaceEXT cmdSetFrontFace = nullptr; + PFN_vkCmdSetDepthTestEnableEXT cmdSetDepthTestEnable = nullptr; + PFN_vkCmdSetDepthWriteEnableEXT cmdSetDepthWriteEnable = nullptr; + PFN_vkCmdSetDepthCompareOpEXT cmdSetDepthCompareOp = nullptr; + PFN_vkCmdSetDepthBoundsTestEnableEXT cmdSetDepthBoundsTestEnable = nullptr; + PFN_vkCmdSetStencilTestEnableEXT cmdSetStencilTestEnable = nullptr; + PFN_vkCmdSetStencilOpEXT cmdSetStencilOp = nullptr; + PFN_vkCmdSetStencilCompareMask cmdSetStencilCompareMask = nullptr; + PFN_vkCmdSetStencilWriteMask cmdSetStencilWriteMask = nullptr; + PFN_vkCmdSetStencilReference cmdSetStencilReference = nullptr; PFN_vkCmdBindVertexBuffers cmdBindVertexBuffers = nullptr; PFN_vkCmdBindIndexBuffer cmdBindIndexBuffer = nullptr; PFN_vkCmdDraw cmdDraw = nullptr; @@ -125,7 +138,9 @@ struct Context { bool occlusionQueryPrecise = false; bool depthBounds = false; bool depthBiasClamp = false; + bool dualSrcBlend = false; bool samplerFilterMinmax = false; + bool nonSeamlessCubeMap = false; bool conservativeRasterization = false; VkBuffer emptyBuffer = VK_NULL_HANDLE; // The device's list of recorded dispatches whose copied written buffers await a CPU write-back @@ -137,14 +152,20 @@ struct Context { // VK_EXT_descriptor_indexing with non-uniform sampled/storage image array indexing enabled // (bindless image tables in graphics stages). bool descriptorIndexing = false; + VkPhysicalDeviceDescriptorIndexingPropertiesEXT descriptorIndexingLimits{}; bool imageInt64Atomics = false; bool geometryShader = false; bool sampleRateShading = false; bool nullDescriptors = false; + bool bufferInt64Atomics = false; bool primitiveListRestart = false; bool imageViewMinLod = false; bool pipelineExecutableInfo = false; + bool graphicsPipelineLibrary = false; std::uint32_t srgbDecodeFormats = 0; + bool provokingVertexLast = false; + bool provokingVertexModePerPipeline = false; + bool singlePassStorage = false; template TFunction Function(const char* name) const { @@ -170,6 +191,11 @@ struct Context { } }; +inline void EndRenderPass(const Context& context, VkCommandBuffer commands) { + if (context.graphicsPipelineLibrary) context.Resolved(&DeviceFunctions::cmdEndRendering, "vkCmdEndRenderingKHR")(commands); + else context.Resolved(&DeviceFunctions::cmdEndRenderPass, "vkCmdEndRenderPass")(commands); +} + } #endif diff --git a/core/libs/prx/libSceAgcDriver/Graphics/include/DccMetadata.hpp b/core/libs/prx/libSceAgcDriver/Graphics/include/DccMetadata.hpp index 109a02b39..849923f41 100644 --- a/core/libs/prx/libSceAgcDriver/Graphics/include/DccMetadata.hpp +++ b/core/libs/prx/libSceAgcDriver/Graphics/include/DccMetadata.hpp @@ -83,8 +83,17 @@ struct DccKeyProofCounts { std::uint64_t proved; std::uint64_t scanned; std::uint64_t unstable; + std::uint64_t rangeProved; + std::uint64_t rangeScanned; }; DccKeyProofCounts KeyProofCounts(); +struct DccRangeProof { + std::uint64_t address = 0; + std::uint64_t count = 0; + DccKeys keys = DccKeys::Uncompressed; + std::uint64_t generation = 0; +}; +DccKeys ProvedCurrentDccKeys(std::uint64_t metaAddress, std::uint64_t surfaceBytes, DccRangeProof& proof); // A surface's texels as a read sees them: the guest bytes, or the clear value of fast-cleared keys. void ReadTextureSurface(const GuestTextureResource& resource, DccKeys keys, std::span bytes); void NoteKeysFillOnGpu(std::uint64_t begin, std::size_t count, DccKeys keys); diff --git a/core/libs/prx/libSceAgcDriver/Graphics/include/DepthSurface.hpp b/core/libs/prx/libSceAgcDriver/Graphics/include/DepthSurface.hpp index 2e8ed728c..04fef30c9 100644 --- a/core/libs/prx/libSceAgcDriver/Graphics/include/DepthSurface.hpp +++ b/core/libs/prx/libSceAgcDriver/Graphics/include/DepthSurface.hpp @@ -3,6 +3,7 @@ #include "prx/libSceAgcDriver/Graphics/include/GuestTextureResource.hpp" #include "prx/libSceAgcDriver/Graphics/include/State.hpp" +#include #include #include #include @@ -15,6 +16,8 @@ VkImageView DepthSurfaceView(const Context& context, const DepthTarget& target); std::uint64_t DepthSliceBytes(VkExtent2D extent, std::uint32_t bytesPerTexel); void ClearDepthSurfaces(VkDevice device); bool DepthSurfaceAt(std::uint64_t address); +std::uint64_t HtileDepthClearAddress(std::span code, std::span userData, const std::array& numThreads); +void NoteHtileDepthClear(std::uint64_t htileAddress); std::shared_ptr DepthSurfaceTexture(const Context& context, std::span words, const GuestTextureResource& resource, VkComponentMapping components); } diff --git a/core/libs/prx/libSceAgcDriver/Graphics/include/Draw.hpp b/core/libs/prx/libSceAgcDriver/Graphics/include/Draw.hpp index 078e05bb2..e59172f7c 100644 --- a/core/libs/prx/libSceAgcDriver/Graphics/include/Draw.hpp +++ b/core/libs/prx/libSceAgcDriver/Graphics/include/Draw.hpp @@ -12,11 +12,14 @@ namespace AgcDriver::Graphics { +struct HostImport; + // `recipe`, when given, receives the DrawRecipe a recorded, cacheable, reusable, non-indirect draw // built for its draw-cache entry (design_cpu_final M8; null otherwise, and always under // APS5_NO_DRAW_RECIPE=1). void Draw(const Context& context, const State& state, const Pm4::DrawParameters& draw, std::span shaders, std::span snapshots = {}, std::shared_ptr* recipe = nullptr); std::optional KnownValidationFailure(const Context& context, std::span shaders, const State& state); +std::uint64_t DrawRenderPassKey(const Context& context, const State& state, std::span targetViews); struct DrawInputCopy { std::shared_ptr buffer; @@ -26,7 +29,9 @@ struct DrawInputCopy { std::uint64_t registryGeneration = 0; }; DrawInputCopy CopyDrawInput(const Context& context, Recorder* recorder, std::uint64_t address, std::size_t bytes, std::size_t alignment, Recorder::SnapshotUse use); +DrawInputCopy CopyZeroPaddedDrawInput(const Context& context, std::uint64_t address, std::size_t bytes, std::size_t validBytes); void KeepDrawInput(Recorder* recorder, std::uint64_t address, const DrawInputCopy& copy, Recorder::SnapshotUse use, std::uint32_t derived); +const HostImport* InPlaceDrawInput(const Context& context, std::uint64_t address, std::size_t bytes, std::size_t alignment); std::array MeshIndexBufferDescriptor(const Pm4::DrawParameters& draw); diff --git a/core/libs/prx/libSceAgcDriver/Graphics/include/GuestBufferMemory.hpp b/core/libs/prx/libSceAgcDriver/Graphics/include/GuestBufferMemory.hpp index 556cc619a..7aa407690 100644 --- a/core/libs/prx/libSceAgcDriver/Graphics/include/GuestBufferMemory.hpp +++ b/core/libs/prx/libSceAgcDriver/Graphics/include/GuestBufferMemory.hpp @@ -27,11 +27,12 @@ struct HostImport { VkDeviceMemory memory; VkDeviceAddress address; void* alias = nullptr; - std::weak_ptr range {}; + std::shared_ptr range {}; // Identity for the life of this import (see HostImportSerial); 0 until first asked for. std::uint64_t serial = 0; bool unwatched = false; bool dmaBuf = false; + std::shared_ptr chunk; }; enum class ImportWatch : std::uint8_t { Watch, Unwatch }; @@ -150,6 +151,8 @@ struct MirrorStats { std::uint64_t rebuilds = 0; std::uint64_t blocksCopied = 0; std::uint64_t heapRefills = 0; + std::uint64_t sweeps = 0; + std::uint64_t heapChecks = 0; }; MirrorStats MirrorCounters(); void ClearImageMirrors(VkDevice device); @@ -180,13 +183,14 @@ public: // (DescriptorBinding::bufferWritten): the CPU never has to wait for it. `atomic` marks an // element the shader updates atomically (DescriptorBinding::bufferAtomic), staged in device // memory whatever its size (see AllowDeviceStaging). - void AddWritable(std::uint64_t address, std::size_t bytes, bool atomic = false); + void AddWritable(std::uint64_t address, std::size_t bytes, bool atomic = false, bool swept = false); void AddReadable(std::uint64_t address, std::size_t bytes); // Device-local staging of written and atomic elements inside host imports (see // GuestBufferMemory.cpp): allowed only for a build whose every use records its work and then // calls RecordCopyBacks (a dispatch), since a staged region's results reach guest memory by // that copy alone. Call before Upload. void AllowDeviceStaging() { stagingAllowed = true; } + void AllowAdjustedRegions() { adjustedRegions = true; } // Records the copy-in of every staged region anew for another use of this upload (a resource // cache hit, from ShaderResources::Revalidate, under GuestMemory::GpuMutex, once the imports // were confirmed unchanged): the previous use's copy-back left the shadow behind, and the next @@ -248,6 +252,7 @@ public: // the GPU copies out of an import (gpuCopy) notes its read itself when the copy is recorded. // For the recorder's read tracking (ShaderResources::MarkGpuWrites); nothing once committed. std::vector> InPlaceReads() const; + std::vector> DeviceReads() const; private: struct Region { @@ -290,6 +295,7 @@ private: bool copiedBack = false; // An element the shader updates atomically lies inside (AddWritable's `atomic`). bool atomic = false; + bool swept = false; // The gpuCopy buffer is a device-local staging shadow (see stagingEligible): no host // mapping, so nothing is ever stored from it by the CPU, and the region is taken even when // the import could bind it in place. @@ -323,7 +329,7 @@ private: // The region of a descriptor-bound range (AddWritable/AddReadable), committed pages only. A // range inside a base region of the space adds nothing: the region serves it, as today's merge // of the two did. - void addDescriptorRegion(std::uint64_t address, std::size_t bytes, bool atomic); + void addDescriptorRegion(std::uint64_t address, std::size_t bytes, bool atomic, bool swept); // Gives a region a buffer of its own with its bytes (guest memory for host-backed and writable // ranges, plus the write-back's reference copy for a range a descriptor writes; else its snapshot). void copyRegion(Region& region, bool addressable); @@ -336,12 +342,14 @@ private: // and an atomic element or a size within the written-shadow window. Independent of the import, // so UploadPrepare and UploadFinish decide alike. bool stagingEligible(const Region& region, bool addressable) const; + bool bindableInPlace(std::uint64_t offset, bool addressable) const; // Records the import-to-buffer copies of the given gpuCopy regions into the open batch, with // the barriers that order them after earlier recorded writes and before the shaders reading them. void recordGpuCopies(std::span copies, bool addressable); void takeHeapReferences(); Context context; bool stagingAllowed = false; + bool adjustedRegions = false; GuestAllocations::Lease lease; // The cached address space this build maps through (its lease pins the ranges); `regions` then // holds only the regions outside it (V#s, snapshots, ranges copied per build). diff --git a/core/libs/prx/libSceAgcDriver/Graphics/include/GuestSamplerResource.hpp b/core/libs/prx/libSceAgcDriver/Graphics/include/GuestSamplerResource.hpp index 5312d9b5b..74e86e866 100644 --- a/core/libs/prx/libSceAgcDriver/Graphics/include/GuestSamplerResource.hpp +++ b/core/libs/prx/libSceAgcDriver/Graphics/include/GuestSamplerResource.hpp @@ -6,6 +6,8 @@ #endif #include #include +#include +#include #include namespace AgcDriver::Graphics { @@ -24,12 +26,15 @@ struct GuestSamplerResource { float lodBias; VkBorderColor borderColor; VkSamplerReductionMode reductionMode = VK_SAMPLER_REDUCTION_MODE_WEIGHTED_AVERAGE_EXT; + bool forceDegamma = false; bool compareEnable = false; VkCompareOp compareOp = VK_COMPARE_OP_NEVER; bool unnormalizedCoordinates = false; + bool nonSeamlessCube = false; }; -GuestSamplerResource DecodeSamplerResource(std::span words, bool unnormalizedProven = false); +GuestSamplerResource DecodeSamplerResource(std::span words, bool unnormalizedProven = false, bool forceDegammaPaired = false); +std::optional> SingleLevelSamplerWords(std::span words, bool singleLevelImage, bool mipmappedImage); } diff --git a/core/libs/prx/libSceAgcDriver/Graphics/include/GuestTextureResource.hpp b/core/libs/prx/libSceAgcDriver/Graphics/include/GuestTextureResource.hpp index aa0765d12..4ccd6a4c3 100644 --- a/core/libs/prx/libSceAgcDriver/Graphics/include/GuestTextureResource.hpp +++ b/core/libs/prx/libSceAgcDriver/Graphics/include/GuestTextureResource.hpp @@ -76,6 +76,8 @@ struct GuestTextureResource { std::uint8_t dstSelY; std::uint8_t dstSelZ; std::uint8_t dstSelW; + std::uint32_t pipeBankXor = 0; + std::uint8_t bcSwizzle = 0; // Last mip level the view exposes; the surface itself holds mipCount levels. std::uint32_t lastLevel = 0; // DCC metadata of a compressed surface, or 0 (see DccMetadata.hpp). @@ -89,6 +91,7 @@ struct GuestTextureResource { float EffectiveMinLod(const GuestTextureResource& resource); GuestTextureResource DecodeTextureResource(std::span words); +bool DescriptorSingleLevel(std::span words); bool MatchesGuestDimension(ShaderRecompiler::DescriptorImageShape shape, TextureDimension dimension); } diff --git a/core/libs/prx/libSceAgcDriver/Graphics/include/ParallelCompare.hpp b/core/libs/prx/libSceAgcDriver/Graphics/include/ParallelCompare.hpp new file mode 100644 index 000000000..a996ab230 --- /dev/null +++ b/core/libs/prx/libSceAgcDriver/Graphics/include/ParallelCompare.hpp @@ -0,0 +1,22 @@ +#ifndef CORE_LIBS_PRX_LIBSCEAGCDRIVER_GRAPHICS_INCLUDE_PARALLELCOMPARE_HPP +#define CORE_LIBS_PRX_LIBSCEAGCDRIVER_GRAPHICS_INCLUDE_PARALLELCOMPARE_HPP + +#include +#include +#include + +namespace AgcDriver::Graphics { + +struct CompareSpan { + const void* first; + const void* second; + std::size_t bytes; +}; + +void CompareSpans(std::span spans, std::span equal); +void CompareSpansFrom(std::span spans, std::span equal, std::size_t parallelBytes); +unsigned CompareHelpers(); + +} + +#endif diff --git a/core/libs/prx/libSceAgcDriver/Graphics/include/PassHazards.hpp b/core/libs/prx/libSceAgcDriver/Graphics/include/PassHazards.hpp new file mode 100644 index 000000000..9ce1cb70f --- /dev/null +++ b/core/libs/prx/libSceAgcDriver/Graphics/include/PassHazards.hpp @@ -0,0 +1,67 @@ +#ifndef CORE_LIBS_PRX_LIBSCEAGCDRIVER_GRAPHICS_INCLUDE_PASSHAZARDS_HPP +#define CORE_LIBS_PRX_LIBSCEAGCDRIVER_GRAPHICS_INCLUDE_PASSHAZARDS_HPP + +#ifndef VK_NO_PROTOTYPES +#define VK_NO_PROTOTYPES +#endif +#include +#include +#include +#include +#include +#include + +namespace AgcDriver::Graphics { + +using GuestRangeList = std::span>; + +struct PassAccess { + GuestRangeList reads; + GuestRangeList writes; + std::span> images; + std::span> attachments; + bool anyReads = false; + bool anyWrites = false; +}; + +enum class PassHazard : std::uint8_t { None, ReadAfterWrite, WriteAfterWrite, WriteAfterRead, Image, UnknownRead, UnknownWrite }; + +class GuestRangeSet { +public: + void Insert(std::uint64_t begin, std::uint64_t end); + bool Overlaps(std::uint64_t begin, std::uint64_t end) const; + bool Empty() const { return ranges.empty(); } + std::size_t Size() const { return ranges.size(); } + void Clear() { ranges.clear(); } + +private: + std::vector> ranges; +}; + +class PassHazards { +public: + PassHazard Check(const PassAccess& access, bool samePass) const; + void Add(const PassAccess& access); + void BeginPass() { ++pass; } + void Clear(); + bool Empty() const; + +private: + struct Attachment { + VkImage image; + bool written; + std::uint64_t pass; + }; + bool overlapsAny(const GuestRangeSet& set, GuestRangeList ranges) const; + GuestRangeSet reads; + GuestRangeSet writes; + std::vector> images; + std::vector attachments; + std::uint64_t pass = 0; + bool anyReads = false; + bool anyWrites = false; +}; + +} + +#endif diff --git a/core/libs/prx/libSceAgcDriver/Graphics/include/Pipeline.hpp b/core/libs/prx/libSceAgcDriver/Graphics/include/Pipeline.hpp index 91ac2ae60..d053fd79f 100644 --- a/core/libs/prx/libSceAgcDriver/Graphics/include/Pipeline.hpp +++ b/core/libs/prx/libSceAgcDriver/Graphics/include/Pipeline.hpp @@ -2,6 +2,7 @@ #define CORE_LIBS_PRX_LIBSCEAGCDRIVER_GRAPHICS_INCLUDE_PIPELINE_HPP #include "prx/libSceAgcDriver/Graphics/include/ShaderResources.hpp" +#include "prx/libSceAgcDriver/Graphics/include/PipelineLibrary.hpp" #include namespace AgcDriver::Graphics { @@ -19,12 +20,14 @@ public: Framebuffer(const Framebuffer&) = delete; Framebuffer& operator=(const Framebuffer&) = delete; VkFramebuffer Handle() const { return framebuffer; } + std::span Views() const { return views; } // Forgets the handle without destroying it (the device it belongs to is already gone). void Abandon() noexcept; private: Context context; VkFramebuffer framebuffer = VK_NULL_HANDLE; + std::vector views; }; // The shader modules, layout, render pass and VkPipeline of one draw configuration. Viewport, @@ -48,6 +51,9 @@ public: // The same inside a render pass another pipeline of the same attachments began (compatible by // construction: the attachment formats alone decide). void Continue(VkCommandBuffer commands, const State& state) const; + bool SplitsFaces() const { return backFaces != VK_NULL_HANDLE; } + bool Optimized() const { return optimized != nullptr && optimized->handle.load() != VK_NULL_HANDLE; } + void ContinueBackFaces(VkCommandBuffer commands, const State& state) const; void PushConstants(VkCommandBuffer commands, VkShaderStageFlags stages, std::span bytes) const; // Forgets the Vulkan objects without destroying them: for entries of a device that is already gone. void Abandon() noexcept; @@ -60,15 +66,23 @@ private: std::shared_ptr framebuffer; }; void release() noexcept; + void setLibraryState(VkCommandBuffer commands, const State& state) const; Context context; std::vector _modules; VkPipelineLayout layout = VK_NULL_HANDLE; VkRenderPass renderPass = VK_NULL_HANDLE; VkPipeline pipeline = VK_NULL_HANDLE; + VkPipeline backFaces = VK_NULL_HANDLE; std::size_t attachments = 0; std::size_t colorAttachments = 0; bool depthBounds = false; bool depthBias = false; + bool libraries = false; + std::shared_ptr optimized; + bool dynamicRendering = false; + VkFormat depthFormat = VK_FORMAT_UNDEFINED; + VkImageLayout attachmentLayout = VK_IMAGE_LAYOUT_UNDEFINED; + std::vector colorSlots; std::vector framebuffers; }; @@ -83,13 +97,15 @@ std::shared_ptr CachedPipeline(const Context& context, const State& st // entries of a gone device are recognised by their buffer pool (made and reset with the device, so // it tells device instances apart when the loader reuses a VkDevice handle) and forgotten unused. void ClearCachedPipelines(VkDevice device); +bool HasStencil(VkFormat format); // Device limit checks of the viewport, which is dynamic state and so no longer checked by Pipeline. void ValidateViewport(const Context& context, const VkViewport& viewport); void ValidateDepthBounds(const Context& context, const State& state); +void ValidateProvokingVertex(const Context& context, const State& state, std::span shaders = {}); void ValidateShaderPair(const ShaderRecompiler::RecompileResult& vertex, const ShaderRecompiler::RecompileResult& fragment); // Returns the color attachment locations the pixel shader writes. -std::set ValidateShaders(std::span shaders, const State& state, const VkPhysicalDeviceSubgroupProperties& subgroup, bool fragmentShaderBarycentric, bool descriptorIndexing = false, bool imageInt64Atomics = false, bool geometryShader = false, bool sampleRateShading = false); +std::set ValidateShaders(std::span shaders, const State& state, const VkPhysicalDeviceSubgroupProperties& subgroup, bool fragmentShaderBarycentric, bool descriptorIndexing = false, bool imageInt64Atomics = false, bool geometryShader = false, bool sampleRateShading = false, bool bufferInt64Atomics = false); } diff --git a/core/libs/prx/libSceAgcDriver/Graphics/include/PipelineLibrary.hpp b/core/libs/prx/libSceAgcDriver/Graphics/include/PipelineLibrary.hpp new file mode 100644 index 000000000..62c7e9359 --- /dev/null +++ b/core/libs/prx/libSceAgcDriver/Graphics/include/PipelineLibrary.hpp @@ -0,0 +1,50 @@ +#ifndef CORE_LIBS_PRX_LIBSCEAGCDRIVER_GRAPHICS_INCLUDE_PIPELINELIBRARY_HPP +#define CORE_LIBS_PRX_LIBSCEAGCDRIVER_GRAPHICS_INCLUDE_PIPELINELIBRARY_HPP + +#include "prx/libSceAgcDriver/Graphics/include/Context.hpp" +#include +#include +#include +#include +#include +#include +#include + +namespace AgcDriver::Graphics { + +struct PipelineLibraryKeys { + std::vector vertexInput; + std::vector preRasterization; + std::vector fragmentShader; + std::vector fragmentOutput; + std::vector renderPass; + std::vector layout; +}; + +std::span PipelineLibraryDynamicStates(); + +struct OptimizedPipeline { + VkDevice device = VK_NULL_HANDLE; + PFN_vkDestroyPipeline destroy = nullptr; + std::atomic handle{VK_NULL_HANDLE}; + std::atomic released{false}; + void Release() noexcept; +}; + +VkPipeline LinkPipelineFromLibraries(const Context& context, const VkGraphicsPipelineCreateInfo& info, const VkPipelineRenderingCreateInfoKHR& rendering, const VkPipelineLayoutCreateInfo& layout, const PipelineLibraryKeys& keys, std::shared_ptr* optimized = nullptr); + +void WaitForOptimizedPipelines(VkDevice device); + +void ClearPipelineLibraries(VkDevice device); + +struct PipelineLibraryCounters { + std::array built{}; + std::uint64_t linked = 0; + std::uint64_t optimized = 0; +}; + +PipelineLibraryCounters PipelineLibraryCountersOf(VkDevice device); + +} + +#endif diff --git a/core/libs/prx/libSceAgcDriver/Graphics/include/Recorder.hpp b/core/libs/prx/libSceAgcDriver/Graphics/include/Recorder.hpp index cea65de45..d8888787d 100644 --- a/core/libs/prx/libSceAgcDriver/Graphics/include/Recorder.hpp +++ b/core/libs/prx/libSceAgcDriver/Graphics/include/Recorder.hpp @@ -2,6 +2,7 @@ #define CORE_LIBS_PRX_LIBSCEAGCDRIVER_GRAPHICS_INCLUDE_RECORDER_HPP #include "prx/libSceAgcDriver/Graphics/include/Context.hpp" +#include "prx/libSceAgcDriver/Graphics/include/PassHazards.hpp" #include #include #include @@ -58,16 +59,13 @@ public: // destination access mask. void MarkCovered(VkAccessFlags access); void MarkShaderReadsCovered() { MarkCovered(VK_ACCESS_SHADER_READ_BIT | VK_ACCESS_SHADER_WRITE_BIT); } - // A recorded draw's render pass (Draw.cpp) is left open after the draw: the next draw of the - // same attachments (`key`: the views and the extent) continues it when nothing was recorded in - // between and the earlier draw allowed it (`continuable`: it wrote nothing but its - // attachments, so no barrier is owed inside the pass), and anything else recorded first ends - // it (vkCmdEndRenderPass, one trailing barrier for the pass, the end of the draw class range - // `timing`): Commands(), RecordStore and Submit end it; CommandsInRenderPass hands a - // continuing draw the command buffer without ending it. - bool ContinuesRenderPass(std::uint64_t key) const; - VkCommandBuffer CommandsInRenderPass(); - void LeaveRenderPassOpen(std::uint64_t key, std::uint32_t timing, bool continuable, std::function afterPass = {}); + struct DrawPassStart { + VkCommandBuffer commands = VK_NULL_HANDLE; + bool continued = false; + bool barrier = true; + }; + DrawPassStart StartDrawPass(std::uint64_t key, bool forced, bool fenced, const PassAccess& access); + void LeaveRenderPassOpen(std::uint64_t key, std::uint32_t timing, bool fenced, const PassAccess& access, std::function afterPass = {}); // DCC "uncompressed" key stores (DccMetadata.cpp StoreUncompressedOnGpu): queued on the open // batch and recorded as one run (one barrier pair for every queued fill) at Submit, before a // label store (RecordStore), or before a command that writes or reads a queued range (the @@ -90,12 +88,12 @@ public: static constexpr std::size_t KeptBytesBudget = std::size_t{512} << 20u; void BoundKeptBytes(); std::size_t InFlightKeptBytes() const { return inFlightKeptBytes; } - enum class SnapshotUse : std::uint8_t { Storage, Vertex, Index16, Index32 }; - static constexpr std::size_t DrawSnapshotBudget = std::size_t{256} << 20u; + enum class SnapshotUse : std::uint8_t { Storage, Vertex, Index16, Index32, Index16Restart, Index32Restart }; + static constexpr std::size_t DrawSnapshotBudget = std::size_t{1024} << 20u; static constexpr std::size_t DrawSnapshotEntries = 1024; static constexpr std::size_t DrawInputBudget = std::size_t{1024} << 20u; static constexpr std::size_t DrawInputEntries = 16384; - std::shared_ptr ReusableDrawSnapshot(std::uint64_t address, std::size_t bytes, SnapshotUse use = SnapshotUse::Storage, std::uint32_t* derived = nullptr); + std::shared_ptr ReusableDrawSnapshot(std::uint64_t address, std::size_t bytes, SnapshotUse use = SnapshotUse::Storage, std::uint32_t* derived = nullptr, std::uint64_t generation = 0); void KeepDrawSnapshot(std::uint64_t address, std::size_t bytes, std::uint64_t generation, std::uint64_t registryGeneration, std::shared_ptr buffer, SnapshotUse use = SnapshotUse::Storage, std::uint32_t derived = 0); void OnComplete(std::function action); void NotePendingWrite(std::uint64_t address, std::size_t bytes); @@ -124,7 +122,7 @@ public: // on the open batch, no snapshot: the only reader holds the mutex. `kind` names the reader for // the [recorder] hit counters. APS5_COPY_READ_TRACKING=0 notes nothing (ReadTracking() is then // false and the CPU copy falls back to Idle()). - enum class ReadKind : std::uint8_t { DispatchElement = 0, GpuCopy, AddressBased, Indirect, StorageUpload, CopySource, Count }; + enum class ReadKind : std::uint8_t { DispatchElement = 0, GpuCopy, AddressBased, Indirect, StorageUpload, CopySource, DrawInput, Count }; static bool ReadTracking(); void NotePendingRead(std::uint64_t address, std::size_t bytes, ReadKind kind); void NotePendingReads(std::span> ranges, ReadKind kind); @@ -184,8 +182,25 @@ public: void CountSamples(); bool RecordMeshArguments(VkCommandBuffer commands, VkDeviceAddress record, VkDeviceAddress arguments, std::span rules); std::uint64_t SamplesTotal(); - bool DumpSamples(VkDeviceAddress target); - void NoteSampledDraw(); + bool DumpSamples(VkDeviceAddress target, std::uint64_t address = 0); + void NoteSampledDraw(VkCommandBuffer commands); + bool QueuesSampleDumps() const; + void PrepareSampleSlot(); + void EndPassSamples(); + struct SampleDumpStatistics { + std::uint64_t queued; + std::uint64_t queuedInPass; + std::uint64_t recordedAtOnce; + std::uint64_t segments; + std::uint64_t folds; + }; + static SampleDumpStatistics SampleDumpCounts(); + struct QueuedWrite { + std::uint64_t begin; + std::uint64_t end; + bool computed; + }; + static std::vector QueuedWriteGroups(std::span writes); // Waits only for the batches up to the newest one that writes the range (submitting the open // batch when it is that one); later batches stay in flight. Fences of one queue signal in // submission order, so completions still run in order. Debug aid: APS5_NO_SYNC_THROUGH=1 syncs all. @@ -519,6 +534,12 @@ private: std::shared_ptr samplePool; bool sampleActive = false; bool samplesDrawn = false; + VkQueryPool segments = VK_NULL_HANDLE; + std::shared_ptr segmentPool; + std::uint32_t segmentNext = 0; + std::uint32_t segmentSlot = 0; + bool segmentActive = false; + std::uint32_t queuedDumps = 0; std::vector timedKeys; std::vector timedBytes; // The whole-batch timed range (BatchTimingKey) and its stamps once read (see Completed). @@ -560,6 +581,9 @@ private: VkDeviceSize offset; std::vector bytes; std::uint64_t address; + VkDeviceAddress dumpTarget = 0; + std::uint32_t dumpSegments = 0; + std::uint64_t End() const; }; std::vector queued; } run; @@ -581,14 +605,14 @@ private: }; std::vector images; } tracker; - // The render pass a recorded draw left open (see ContinuesRenderPass). struct RenderPass { bool open = false; - bool continuable = false; + bool fenced = false; std::uint64_t key = 0; std::uint32_t timing = NoTiming; std::function afterPass; } renderPass; + PassHazards passHazards; // A pass ended in this batch: Submit records the host-read barrier its draws left out. bool hostReadOwed = false; // Queued DCC key stores (see QueueKeyStore). @@ -622,9 +646,21 @@ private: bool gpuSampleCounter(); void endSamples(Batch& batch); void foldSamples(Batch& batch, VkDeviceAddress target); + bool segmentMode() const; + bool segmentSlotReady(const Batch& batch) const; + void takeSegmentPool(Batch& batch); + void beginSegment(Batch& batch); + void endSegment(Batch& batch); + void copySegments(VkCommandBuffer commands, std::uint32_t count); + void retireSegments(std::uint32_t count); + void recordQueuedDumps(VkCommandBuffer commands, std::span writes); + void dispatchDumps(VkCommandBuffer commands, std::uint32_t firstDump, std::uint32_t dumps, std::uint32_t firstSegment, std::uint32_t lastSegment); + void foldSegments(); + void foldQueuedDumps(VkCommandBuffer commands, std::span writes, std::size_t first, std::size_t end, std::uint32_t& dumpIndex, std::uint32_t& consumed); struct SampleSegment { std::shared_ptr pool; VkQueryPool handle; + std::uint32_t slot = 0; }; std::vector pendingSamples; bool countingSamples = false; @@ -636,6 +672,9 @@ private: VkPipelineLayout sampleLayout = VK_NULL_HANDLE; VkPipeline samplePipeline = VK_NULL_HANDLE; std::shared_ptr samplePools; + VkPipelineLayout dumpLayout = VK_NULL_HANDLE; + VkPipeline dumpPipeline = VK_NULL_HANDLE; + std::shared_ptr segmentPools; // BeginGpuTiming on the open batch without Commands() (RecordStore times its own run, which // Commands() would close). std::uint32_t beginTiming(std::uint64_t key); @@ -673,7 +712,7 @@ private: // form, toward the host only); returns whether the batch's host-read barrier was included. bool closeStoreRun(bool atSubmit = false); // Ends the render pass a draw left open (vkCmdEndRenderPass, the pass's trailing barrier). - void endOpenRenderPass(); + void endOpenRenderPass(bool barrier = true); // Records the queued key stores as one run (`forWriter`: before a command writing, reading or // labelling over one, not at Submit). void recordKeyStores(bool forWriter); @@ -763,6 +802,7 @@ private: std::map drawSnapshots; std::array drawSnapshotPools; void eraseDrawSnapshot(std::map::iterator entry); + bool refreshDrawSnapshot(DrawSnapshot& entry, std::uint64_t address, std::size_t bytes); }; } diff --git a/core/libs/prx/libSceAgcDriver/Graphics/include/Resources.hpp b/core/libs/prx/libSceAgcDriver/Graphics/include/Resources.hpp index ba4352098..3c288acfb 100644 --- a/core/libs/prx/libSceAgcDriver/Graphics/include/Resources.hpp +++ b/core/libs/prx/libSceAgcDriver/Graphics/include/Resources.hpp @@ -35,6 +35,8 @@ private: // the handles a failed construction left. bool ready = false; VkDeviceSize allocationBytes = 0; + VkDeviceSize offset = 0; + bool slab = false; VkBufferUsageFlags usage; VkMemoryPropertyFlags properties; std::shared_ptr cache; diff --git a/core/libs/prx/libSceAgcDriver/Graphics/include/Sampler.hpp b/core/libs/prx/libSceAgcDriver/Graphics/include/Sampler.hpp index 564679119..a1f894545 100644 --- a/core/libs/prx/libSceAgcDriver/Graphics/include/Sampler.hpp +++ b/core/libs/prx/libSceAgcDriver/Graphics/include/Sampler.hpp @@ -19,15 +19,21 @@ public: Sampler(const Sampler&) = delete; Sampler& operator=(const Sampler&) = delete; + bool ForcesDegamma() const; VkSampler Handle() const; bool RequiresFilterMinmax() const; + bool ReadsOpaqueBlackBorder() const; + bool RequiresNonSeamlessCube() const; private: void release() noexcept; Context context; + bool forcesDegamma = false; VkSampler sampler = VK_NULL_HANDLE; bool requiresFilterMinmax = false; + bool opaqueBlackBorder = false; + bool requiresNonSeamlessCube = false; }; // One VkSampler per distinct S# (its 4 words plus the shader's depth-compare use and the @@ -41,7 +47,7 @@ public: explicit SamplerCache(std::size_t capacity = 1024); SamplerCache(const SamplerCache&) = delete; SamplerCache& operator=(const SamplerCache&) = delete; - std::shared_ptr Get(const Context& context, std::span words, bool compareEnable, bool unnormalizedProven = false); + std::shared_ptr Get(const Context& context, std::span words, bool compareEnable, bool unnormalizedProven = false, bool forceDegammaPaired = false); // APS5_PROFILE_DRAW counters: lookups served by an existing sampler, and samplers created. std::uint64_t Hits() const { return hits; } std::uint64_t Misses() const { return misses; } @@ -60,6 +66,9 @@ private: }; void RequireFilterMinmax(const Context& context, VkFormat format, std::uint32_t samplerMask, std::span> samplers); +void RequireDegammaFormat(std::uint32_t guestFormat, std::uint32_t samplerMask, std::span> samplers); +void RequireBorderSwizzle(std::uint32_t bcSwizzle, std::uint32_t samplerMask, std::span> samplers); +void RequireNonSeamlessCube(bool cube, std::uint32_t samplerMask, std::span> samplers); } diff --git a/core/libs/prx/libSceAgcDriver/Graphics/include/ShaderResources.hpp b/core/libs/prx/libSceAgcDriver/Graphics/include/ShaderResources.hpp index c4f06ee68..53b5650c6 100644 --- a/core/libs/prx/libSceAgcDriver/Graphics/include/ShaderResources.hpp +++ b/core/libs/prx/libSceAgcDriver/Graphics/include/ShaderResources.hpp @@ -37,6 +37,7 @@ struct TextureCacheUse { TextureCacheUse TextureCacheUsage(); std::uint64_t SampledTextureCacheBudget(const Context& context); std::shared_ptr CachedSampledTexture(const Context& context, std::span words); +bool SampledTexturesShareEntry(std::span first, std::span second); // The cached storage image of a surface (render targets use it as their resident image); brought up // to date with guest memory before it is returned. @@ -68,12 +69,12 @@ public: // The layout for `key` (binding, type, count, stage flags per binding, as ShaderResources builds // it from `bindings`), created on first use. - VkDescriptorSetLayout Layout(std::span key, std::span bindings); + VkDescriptorSetLayout Layout(std::span key, std::span bindings, bool updateAfterBind = false); struct SetAllocation { VkDescriptorSet set = VK_NULL_HANDLE; VkDescriptorPool pool = VK_NULL_HANDLE; }; - SetAllocation Allocate(VkDescriptorSetLayout layout, std::span sizes); + SetAllocation Allocate(VkDescriptorSetLayout layout, std::span sizes, bool updateAfterBind = false); void Free(const SetAllocation& allocation) noexcept; // APS5_PROFILE_DRAW counters: layouts served from the map / created, sets allocated, pools opened. struct Stats { @@ -93,6 +94,7 @@ private: mutable std::mutex mutex; std::map, VkDescriptorSetLayout> layouts; std::vector pools; + std::vector updateAfterBindPools; std::vector dedicated; Stats stats; }; @@ -100,14 +102,14 @@ private: class ShaderResources { public: ShaderResources(const Context& context, const ShaderRecompiler::RecompileResult& vertex, const ShaderRecompiler::RecompileResult& fragment, const ColorTarget& target, std::uint64_t indexAddress, std::size_t indexBytes); - ShaderResources(const Context& context, std::span shaders, const ColorTarget& target, std::uint64_t indexAddress, std::size_t indexBytes, std::span snapshots = {}); + ShaderResources(const Context& context, std::span shaders, const ColorTarget& target, std::uint32_t colorAttachments, std::uint64_t indexAddress, std::size_t indexBytes, std::span snapshots = {}); ShaderResources(const Context& context, const CompiledShader& compute, std::span snapshots = {}); // Two-stage build for dispatches (see build): with `deferred` the constructor runs stage A only, // which needs no device lock, and Complete() runs stage B under GuestMemory::GpuMutex; until // then no other member may be used. `compute` and `snapshots` must outlive Complete(). An // address-based shader (BDA tables) builds entirely in Complete(): its lease acquisition // reconciles imports and refreshes mirrors, which needs the lock. - ShaderResources(const Context& context, const CompiledShader& compute, std::span snapshots, bool deferred); + ShaderResources(const Context& context, const CompiledShader& compute, std::span snapshots, bool deferred, std::uint64_t dispatchThreads = 0); void Complete(); bool Completed() const { return completed; } ~ShaderResources(); @@ -155,11 +157,7 @@ public: const std::vector>& GpuWrites() const { return guestMemory.Writes(); } std::vector> InPlaceReads() const { return guestMemory.InPlaceReads(); } std::vector> StorageImages() const; - // Whether a use writes guest memory beyond a draw's attachments (storage images, written or - // copied buffers, an address-based build's unknown writes), or reads `image` (a view of it - // sampled, or the image itself bound): a recorded draw's render pass may only be continued by - // a draw for which neither holds (Draw.cpp). - bool WritesMemory() const; + std::vector> DeviceReads() const { return guestMemory.DeviceReads(); } bool ReadsImage(const StorageTexture* image) const; const std::vector& LayoutKey() const { return layoutKey; } // Debug aid: each bound guest resource with the fraction of sampled bytes that are nonzero. @@ -186,6 +184,8 @@ public: // ranges are fixed by the build), so a Revalidate's collects on the same worker are memo hits. void PrecollectSurfaces() const; bool Reusable() const { return reusable; } + static bool NeverReusable(std::span shaders); + const std::vector>& SampledTextures() const { return textures; } // `shaders` are the stages the object was built from, in build order (a recorded draw's vertex // and fragment stages, or one compute stage): their bindings are walked like the build did. // How a Revalidate proved (or refused) the object, for the [recipe] line: the proof path taken @@ -294,15 +294,15 @@ private: // GuestMemory::GpuMutex): the texture and storage image lookups (they refresh, upload and flush // through the recorder), the rest of the upload, the BDA objects, the descriptor writes and the // reusability record. build runs both. - void build(std::span shaders, const ColorTarget* target, std::uint64_t indexAddress, std::size_t indexBytes); - void buildPrepare(std::span shaders, const ColorTarget* target, std::uint64_t indexAddress, std::size_t indexBytes); + void build(std::span shaders, const ColorTarget* target, std::uint32_t colorAttachments, std::uint64_t indexAddress, std::size_t indexBytes); + void buildPrepare(std::span shaders, const ColorTarget* target, std::uint32_t colorAttachments, std::uint64_t indexAddress, std::size_t indexBytes); void buildComplete(); // APS5_PROFILE_DRAW: closes the current sub-phase of the build into the [resources] totals. double phase(BuildPhase which); // `written` is the element's DescriptorBinding::bufferWritten: a read-only element binds the // same way but is left out of the write set (no write-back, no pending-write note). `atomic` // is its bufferAtomic (see GuestBufferMemory::AddWritable). - std::size_t addGuestBuffer(std::span words, const ColorTarget* target, std::uint64_t indexAddress, std::size_t indexBytes, bool written, bool atomic); + std::size_t addGuestBuffer(std::span words, const ColorTarget* target, std::uint64_t indexAddress, std::size_t indexBytes, bool written, bool atomic, bool read); std::size_t addDataBuffer(std::span words); // Stage A: the layout entry of an image binding (samplers are taken at once, the sampler cache // locks itself); stage B looks the sampled textures and storage images up (resolveImageBinding). @@ -335,12 +335,19 @@ private: std::uint64_t entryGeneration = 0; }; std::vector imageRecords; + struct PreviousSampled { + std::array words{}; + bool depthCompare = false; + std::shared_ptr texture; + }; + PreviousSampled previousSampled; // The next record resolveImageBinding consumes (records follow the deferredImages order). std::size_t nextImageRecord = 0; // Stage B: the record's texture when the fastRevalidate predicate proves it current under the // lock and the cache still holds it; null sends the element to cachedTexture. std::shared_ptr fastTexture(const ImageRecord& record); void resolveImageBinding(const ShaderRecompiler::DescriptorBinding& binding, Binding& item, std::span> shaderSamplers); + void applyAnisoOverride(); void forgetDeferredInputs(); void release() noexcept; void prepareAddressBindings(std::span shaders, std::span snapshots); @@ -414,9 +421,12 @@ private: // The cache pool the set was allocated from, freed back to it on release. VkDescriptorPool cachePool = VK_NULL_HANDLE; std::vector allocations; + std::vector allocationDescriptors; // Guest buffer elements bound read-only: each use of this object skips that many pending-write // notes (counted in MarkGpuWrites for the [buffers] line). std::size_t readOnlyBuffers = 0; + bool drawBuild = false; + std::uint64_t dispatchThreads = 0; std::vector> textures; std::vector textureFirstLayer; std::vector> storageTextures; @@ -428,6 +438,14 @@ private: std::vector storageAtomic64; std::vector> samplers; std::shared_ptr paddingSampler; + struct SamplerSource { + std::array words{}; + bool compareEnable = false; + bool unnormalized = false; + bool singleLevelImage = false; + bool mipmappedImage = false; + }; + std::vector samplerSources; bool reusable = false; std::vector directRegions; std::vector validatedTextures; @@ -457,6 +475,7 @@ private: }; std::vector deferredImages; std::uint64_t storageBuffers = 0; + bool updateAfterBind = false; std::uint32_t plannedSampledImages = 0; std::uint32_t plannedStorageImages = 0; // The compute constructor's shader and captured regions: the caller's objects, valid only until diff --git a/core/libs/prx/libSceAgcDriver/Graphics/include/Shaders.hpp b/core/libs/prx/libSceAgcDriver/Graphics/include/Shaders.hpp index 307dbde3c..a7e8bd9ec 100644 --- a/core/libs/prx/libSceAgcDriver/Graphics/include/Shaders.hpp +++ b/core/libs/prx/libSceAgcDriver/Graphics/include/Shaders.hpp @@ -9,7 +9,29 @@ namespace AgcDriver::Graphics { -inline constexpr std::uint32_t PipelinePushConstantBytes = 128; +inline constexpr std::uint32_t PipelinePushConstantBytes = 256; +inline constexpr std::uint32_t PipelinePushSlotBytes = 128; +inline constexpr std::uint32_t ComputePushConstantBytes = 128; + +inline std::uint32_t PushBlockBytes(bool libraries) { + return libraries ? PipelinePushConstantBytes : PipelinePushSlotBytes; +} + +inline VkShaderStageFlags PreRasterizationPushStages(bool meshShader, bool tessellation, bool geometry) { + VkShaderStageFlags stages = VK_SHADER_STAGE_VERTEX_BIT; + if (tessellation) stages |= VK_SHADER_STAGE_TESSELLATION_CONTROL_BIT | VK_SHADER_STAGE_TESSELLATION_EVALUATION_BIT; + if (geometry) stages |= VK_SHADER_STAGE_GEOMETRY_BIT; + if (meshShader) stages |= VK_SHADER_STAGE_MESH_BIT_EXT; + return stages; +} + +inline std::uint32_t FixedPushOffset(ShaderRecompiler::ShaderStage stage) { + return stage == ShaderRecompiler::ShaderStage::Fragment ? PipelinePushSlotBytes : 0u; +} + +inline std::uint32_t StagePushOffset(std::uint32_t cursor, ShaderRecompiler::ShaderStage stage, bool fixedSlots) { + return fixedSlots && stage == ShaderRecompiler::ShaderStage::Fragment ? PipelinePushSlotBytes : cursor; +} struct CompiledShader { ShaderRecompiler::ShaderStage stage; @@ -23,6 +45,7 @@ inline VkShaderStageFlagBits VulkanStage(ShaderRecompiler::ShaderStage stage) { case ShaderRecompiler::ShaderStage::Local: return VK_SHADER_STAGE_VERTEX_BIT; case ShaderRecompiler::ShaderStage::TessellationControl: return VK_SHADER_STAGE_TESSELLATION_CONTROL_BIT; case ShaderRecompiler::ShaderStage::TessellationEvaluation: return VK_SHADER_STAGE_TESSELLATION_EVALUATION_BIT; + case ShaderRecompiler::ShaderStage::Geometry: return VK_SHADER_STAGE_GEOMETRY_BIT; case ShaderRecompiler::ShaderStage::Mesh: return VK_SHADER_STAGE_MESH_BIT_EXT; case ShaderRecompiler::ShaderStage::Fragment: return VK_SHADER_STAGE_FRAGMENT_BIT; case ShaderRecompiler::ShaderStage::Compute: return VK_SHADER_STAGE_COMPUTE_BIT; @@ -37,6 +60,7 @@ inline VkPipelineStageFlags PipelineStages(std::span shade case VK_SHADER_STAGE_VERTEX_BIT: result |= VK_PIPELINE_STAGE_VERTEX_SHADER_BIT; break; case VK_SHADER_STAGE_TESSELLATION_CONTROL_BIT: result |= VK_PIPELINE_STAGE_TESSELLATION_CONTROL_SHADER_BIT; break; case VK_SHADER_STAGE_TESSELLATION_EVALUATION_BIT: result |= VK_PIPELINE_STAGE_TESSELLATION_EVALUATION_SHADER_BIT; break; + case VK_SHADER_STAGE_GEOMETRY_BIT: result |= VK_PIPELINE_STAGE_GEOMETRY_SHADER_BIT; break; case VK_SHADER_STAGE_MESH_BIT_EXT: result |= VK_PIPELINE_STAGE_MESH_SHADER_BIT_EXT; break; case VK_SHADER_STAGE_FRAGMENT_BIT: result |= VK_PIPELINE_STAGE_FRAGMENT_SHADER_BIT; break; default: throw std::runtime_error("AGC graphics: invalid graphics pipeline stage"); @@ -63,6 +87,7 @@ inline std::array AssemblePushConstants(st if (bytes.empty()) continue; Require(bytes.size() % 4 == 0 && shader.pushConstantOffset % 4 == 0, "shader push constant range is not DWORD aligned"); Require(shader.pushConstantOffset < PipelinePushConstantBytes && bytes.size() <= PipelinePushConstantBytes - shader.pushConstantOffset, "shader push constant range lies outside the pipeline push constant block"); + Require(bytes.size() <= PipelinePushSlotBytes - shader.pushConstantOffset % PipelinePushSlotBytes, "shader push constant range crosses a push constant slot"); for (std::size_t i = 0; i < bytes.size(); ++i) { const auto position = shader.pushConstantOffset + i; Require(!occupied[position], "shader push constant ranges of different stages overlap"); diff --git a/core/libs/prx/libSceAgcDriver/Graphics/include/State.hpp b/core/libs/prx/libSceAgcDriver/Graphics/include/State.hpp index 123dbe52a..05cb27663 100644 --- a/core/libs/prx/libSceAgcDriver/Graphics/include/State.hpp +++ b/core/libs/prx/libSceAgcDriver/Graphics/include/State.hpp @@ -38,6 +38,7 @@ struct ColorTarget { std::uint8_t componentMapping; ColorTileMode tileMode = ColorTileMode::Linear; std::uint32_t elementBytes = 4; + ShaderRecompiler::ColorExportPacking packing = ShaderRecompiler::ColorExportPacking::None; // DCC metadata of a compressed target (CB_COLOR_INFO DCC_ENABLE), or 0 (see DccMetadata.hpp). std::uint64_t dccAddress = 0; bool dccAlphaOnMsb = false; @@ -54,6 +55,8 @@ struct ColorTarget { std::uint32_t depth = 1; std::uint32_t depthSlice = 0; std::uint32_t exportIndex = 0; + bool uintExport = false; + std::uint32_t pipeBankXor = 0; }; struct DepthTarget { @@ -63,6 +66,7 @@ struct DepthTarget { VkFormat format; float clearDepth; std::uint8_t clearStencil; + std::uint64_t htileAddress = 0; }; struct State { @@ -78,6 +82,9 @@ struct State { float depthBiasConstant = 0.0f; float depthBiasSlope = 0.0f; float depthBiasClamp = 0.0f; + bool depthBiasPerFace = false; + float backDepthBiasConstant = 0.0f; + float backDepthBiasSlope = 0.0f; bool stencilTest = false; VkStencilOpState stencilFront{}; VkStencilOpState stencilBack{}; @@ -86,6 +93,7 @@ struct State { std::vector blends; bool hasColorTarget; bool rectList = false; + bool dualSourceBlend = false; VkExtent2D renderExtent; VkPrimitiveTopology topology; bool primitiveRestart = false; @@ -96,6 +104,7 @@ struct State { VkRect2D scissor; VkCullModeFlags cullMode; VkFrontFace frontFace; + VkProvokingVertexModeEXT provokingVertexMode = VK_PROVOKING_VERTEX_MODE_FIRST_VERTEX_EXT; VkPipelineColorBlendAttachmentState blend; std::array blendConstants; }; @@ -103,6 +112,7 @@ struct State { ShaderStages DecodeShaderStages(const QueueState& queue); State DecodeState(const QueueState& queue); std::array ExportMappings(const State& state); +std::array ExportPackings(const State& state); ColorTarget DecodeColorBuffer(const Registers& context, std::uint32_t slot); std::size_t CmaskBytes(std::uint32_t width, std::uint32_t height); std::uint32_t ColorWriteMask(const Registers& context); @@ -151,8 +161,8 @@ struct DrawKeyRange { std::uint32_t first; std::uint32_t count; }; -inline constexpr std::array DrawKeyRegisters{{ - {RegisterBank::Context, 0x000, 1}, {RegisterBank::Context, 0x002, 1}, {RegisterBank::Context, 0x007, 7}, {RegisterBank::Context, 0x010, 6}, {RegisterBank::Context, 0x01a, 4}, +inline constexpr std::array DrawKeyRegisters{{ + {RegisterBank::Context, 0x000, 1}, {RegisterBank::Context, 0x002, 1}, {RegisterBank::Context, 0x005, 1}, {RegisterBank::Context, 0x007, 7}, {RegisterBank::Context, 0x010, 6}, {RegisterBank::Context, 0x01a, 5}, {RegisterBank::Context, 0x080, 4}, {RegisterBank::Context, 0x08c, 4}, {RegisterBank::Context, 0x090, 2}, {RegisterBank::Context, 0x094, 2}, {RegisterBank::Context, 0x0b4, 2}, {RegisterBank::Context, 0x105, 4}, {RegisterBank::Context, 0x10b, 3}, {RegisterBank::Context, 0x10f, 6}, // SPI_PS_INPUT_CNTL_0..31, SPI_PS_INPUT_ENA/ADDR, SPI_PS_IN_CONTROL, SPI_SHADER_POS/Z/COL_FORMAT, // CB_BLEND0..7_CONTROL, GE_MAX_OUTPUT_PER_SUBGROUP. @@ -161,7 +171,7 @@ inline constexpr std::array DrawKeyRegisters{{ // PA_SU_VTX_CNTL, the sample masks, PA_SC_CONSERVATIVE_RASTERIZATION_CNTL. {RegisterBank::Context, 0x200, 8}, {RegisterBank::Context, 0x292, 2}, {RegisterBank::Context, 0x29b, 1}, {RegisterBank::Context, 0x2ab, 1}, {RegisterBank::Context, 0x2ce, 1}, {RegisterBank::Context, 0x2d5, 2}, {RegisterBank::Context, 0x2db, 2}, {RegisterBank::Context, 0x2de, 6}, {RegisterBank::Context, 0x2f8, 2}, {RegisterBank::Context, 0x30e, 2}, {RegisterBank::Context, 0x313, 1}, // CB_COLOR0..7_BASE .. DCC_BASE (15 words a slot), CB_COLOR0..7_BASE_EXT, DCC_BASE_EXT, ATTRIB2, ATTRIB3. - {RegisterBank::Context, 0x318, 0x78}, {RegisterBank::Context, 0x390, 8}, {RegisterBank::Context, 0x3a8, 0x18}, + {RegisterBank::Context, 0x318, 0x78}, {RegisterBank::Context, 0x390, 8}, {RegisterBank::Context, 0x398, 8}, {RegisterBank::Context, 0x3a8, 0x18}, // The pixel program address, RSRC2 and user words; the geometry-back user pointer and program // address; the vertex/geometry-front RSRC1/RSRC2 and user words; the vertex program address; // the hull user pointer, program address, RSRC2 and user words; the local program address. diff --git a/core/libs/prx/libSceAgcDriver/Graphics/include/Texture.hpp b/core/libs/prx/libSceAgcDriver/Graphics/include/Texture.hpp index fb2bb627b..0003ac1e1 100644 --- a/core/libs/prx/libSceAgcDriver/Graphics/include/Texture.hpp +++ b/core/libs/prx/libSceAgcDriver/Graphics/include/Texture.hpp @@ -35,6 +35,7 @@ bool StorageFormatAvailable(const Context& context, std::uint32_t guestFormat); // StorageTexture::upload); false for integer formats and non-clear keys. bool StorageClearAvailable(const Context& context, std::uint32_t guestFormat, DccKeys keys); std::uint64_t SampledTextureMemory(); +bool ClearKeepsDenormals(const Context& context, VkFormat format); // A sampled texture's own VkImage with its memory, shared with the recorder while a recorded upload // still writes it (see the snapshot constructor), so the texture may go before the batch completes. @@ -129,6 +130,7 @@ public: VkImageView View(std::uint32_t mip); VkImageView FirstLayerView(std::uint32_t mip); VkImageView StorageView(std::uint32_t mip, bool firstLayer); + VkImageView ElementView(); VkImageView AtomicView(std::uint32_t mip, bool firstLayer); VkImageView Atomic64View(std::uint32_t mip, bool firstLayer); // Render targets live in the same images: draws attach mip 0 through a view of the color @@ -151,6 +153,7 @@ public: // overlapping pending image, as before). static bool FlushPending(std::uint64_t address, std::size_t bytes, const StorageTexture* except = nullptr, const char* reason = "memory access", PublishScope scope = PublishScope::Whole, bool* published = nullptr); static void FlushAllPending(const char* reason); + static bool StoreAtFlipRequested(const char* value); // See PendingSerial: a change of a surface's source outside the registry (a unit shadow // retile) moves it too. static void BumpPendingSerial(); @@ -228,8 +231,9 @@ public: // `image` and `layer` naming it, whatever other images lie over the range (`others`, of which // `inside` wholly inside it: stale images of earlier uses of the memory, which the title's // transient allocator hands out again); otherwise part of one image (the fill inside the - // surface, the surface inside the fill, or straddling), several, or none but exactly one - // surface's DCC metadata. Images the cache let go (see Flush) do not count. + // surface, the surface inside the fill, or straddling), several, or a fill of one surface's DCC + // keys (Keys: at its dccAddress and at least one key long, and while other images overlap it no + // longer than its key extent; see keysFillMatches). Images the cache let go (see Flush) do not count. enum class FillCover { None, Exact, Inside, Around, Straddle, Several, Keys, Layer }; struct FillCoverage { FillCover cover = FillCover::None; @@ -285,12 +289,15 @@ public: DccKeys UploadedKeys() const { return uploadedKeys; } DccKeys FilledKeys() const { return filledKeys; } DccKeyProof& KeyProof() const { return keyProof; } + DccRangeProof& TargetKeyProof() const { return targetKeyProof; } DccKeys ProvedKeys() const; bool ServesKeysAt(std::uint64_t dccAddress) const; // Brings the image up to date with guest memory before another use; returns whether its content // was still current (nothing uploaded). // Keeps the image current with guest memory (see GuestMemory::CollectWrites). bool Refresh(); + static std::uint64_t SinglePassMoves(); + static std::uint64_t RefreshesProved(); std::uint64_t GuestBytes() const; VkDeviceSize AllocationBytes() const { return memoryBytes; } @@ -309,6 +316,7 @@ private: // APS5_BLOCK_WRITEBACK_EACH=1 stores the touched units only). void writeBack(std::uint64_t address, std::size_t bytes); void writeBackLayers(const std::vector& layers); + bool unchangedSinceBaseline(std::uint64_t from, std::uint64_t to) const; // Tracked units as 64 KiB write-stamp blocks (`blockUnits`: a thin tiled surface at a 64 KiB // aligned base; APS5_NO_BLOCK_TRACKING=1 tracks array layers as above instead): a fill of one // layer, a CPU write or another image's store then costs the blocks it touched, moved through @@ -368,6 +376,9 @@ private: std::uint64_t layerBegin(std::uint32_t layer) const { return descriptor.baseAddress + static_cast(layer) * trackedLayerBytes; } bool anyLayerPending() const; void refreshGeneration(); + bool refreshProved(); + void takeRefreshProof(bool aliased); + bool otherPendingOverlaps() const; // Marks `count` tracked layers from `first` pending and registers the image (MarkDirty's // registration; APS5_EAGER_WRITEBACK=1 stores at once instead). void markLayersPending(std::uint32_t first, std::uint32_t count); @@ -417,8 +428,18 @@ private: void forgetBorrowed(std::uint32_t first, std::uint32_t count); bool clearByKeysFill(DccKeys keys, std::uint8_t key); bool overlaps(std::uint64_t address, std::size_t bytes) const; + // Whether the image is live (not released) and overlaps the fill: the test that gives ClassifyFill + // its overlapping images, and that NoteKeysFill and ClearByKeysFill use to bound the key match. + bool overlapsLive(std::uint64_t address, std::size_t bytes) const; + // Whether a fill of [address, address + bytes) is a fill of this image's DCC keys, `overlapped` + // saying whether any live image overlaps the fill (see Texture.cpp). + bool keysFillMatches(std::uint64_t address, std::size_t bytes, bool overlapped) const; bool pendingUnitInside(std::uint64_t address, std::size_t bytes) const; VkImageView createView(std::uint32_t mip, bool firstLayer, VkFormat format) const; + bool singlePass(); + VkImageView elementLayerView(std::uint32_t level, std::uint32_t layer); + void recordDirectUploadBarrier(VkCommandBuffer commands, bool discard); + void recordDirectUploadDone(VkCommandBuffer commands); void release() noexcept; Context context; @@ -431,10 +452,19 @@ private: SurfaceGeometry geometry; std::vector original; mutable std::array comparedGuestBytes{}; + std::vector generationBaseline; // DCC keys the image content was uploaded under: a fast-cleared surface starts as its clear value. DccKeys uploadedKeys = DccKeys::Uncompressed; mutable DccKeys filledKeys = DccKeys::Uncompressed; mutable DccKeyProof keyProof; + mutable DccRangeProof targetKeyProof; + struct RefreshProof { + std::uint64_t generation = 0; + std::uint64_t pendingSerial = 0; + std::uint64_t keyGeneration = 0; + DccKeys keys = DccKeys::Uncompressed; + }; + RefreshProof refreshProof; struct ForeignKeyProof { std::uint64_t dccAddress = 0; DccKeyProof proof; @@ -470,6 +500,9 @@ private: std::map firstLayerViews; std::map, VkImageView> atomicViews; std::map, VkImageView> uintViews; + VkImageView elementView = VK_NULL_HANDLE; + std::map elementLayerViews; + std::int8_t singlePassState = -1; bool attachable = false; std::map, VkImageView> attachmentViews; VkImage proxyImage = VK_NULL_HANDLE; @@ -509,7 +542,7 @@ private: // element (a fast miss is followed by a full lookup); the refresh, upload, DCC scan and pending // flush rows lie inside the storage and sampled rows. struct LookupOutcomes { - enum Kind : std::size_t { SampledFast, SampledFastMiss, SampledHitView, SampledHitClearedView, SampledHitSnapshot, SampledMadeView, SampledMadeSnapshot, StorageHit, StorageMade, RefreshUnchanged, RefreshCompared, UploadDirect, UploadCpu, UploadClear, DccScan, PendingFlush, Count }; + enum Kind : std::size_t { SampledFast, SampledFastMiss, SampledHitView, SampledHitClearedView, SampledHitSnapshot, SampledMadeView, SampledMadeSnapshot, StorageHit, StorageMade, RefreshUnchanged, RefreshCompared, RefreshProved, UploadDirect, UploadCpu, UploadClear, DccScan, PendingFlush, Count }; static const char* Name(Kind kind); static bool Profiled(); // Charges the time since `start` to `kind` on this thread and returns now. diff --git a/core/libs/prx/libSceAgcDriver/Graphics/include/TextureDetiler.hpp b/core/libs/prx/libSceAgcDriver/Graphics/include/TextureDetiler.hpp index 7ffb40425..2efc3bc46 100644 --- a/core/libs/prx/libSceAgcDriver/Graphics/include/TextureDetiler.hpp +++ b/core/libs/prx/libSceAgcDriver/Graphics/include/TextureDetiler.hpp @@ -5,12 +5,16 @@ #include "prx/libSceAgcDriver/Graphics/include/Context.hpp" #include "prx/libSceAgcDriver/Graphics/include/GuestTextureResource.hpp" #include "prx/libSceAgcDriver/Graphics/include/TextureTiling.hpp" +#include #include +#include #include #include namespace AgcDriver::Graphics { + class Buffer; + // The part of a mip one Dispatch moves: only elements whose tiled offset (relative to the mip's // tiled base; the linear offset of a linear surface) lies in [rangeBegin, rangeEnd), with the // tiled buffer holding the mip from tiledBase and the linear buffer holding `linearBytes` of @@ -29,6 +33,7 @@ namespace AgcDriver::Graphics { std::uint32_t columnEnd = 0; std::uint32_t rowBegin = 0; std::uint32_t rowEnd = 0; + std::uint32_t pipeBankXor = 0; }; class TextureDetiler { @@ -43,13 +48,18 @@ namespace AgcDriver::Graphics { // restricts the move to part of the mip (a partial upload or write-back, StorageTexture), // leaving the other elements of both buffers alone. void Dispatch(VkCommandBuffer commands, TextureTileMode tileMode, std::uint32_t elementBytes, VkBuffer source, std::uint64_t sourceOffset, VkBuffer destination, std::uint64_t destinationOffset, const TileMipLayout& layout, bool retile = false, std::uint32_t slice = 0, bool thick = false, const DetileWindow& window = {}); + void DispatchImage(VkCommandBuffer commands, TextureTileMode tileMode, std::uint32_t elementBytes, VkBuffer tiled, std::uint64_t tiledOffset, VkImageView view, const TileMipLayout& layout, bool retile = false, std::uint32_t slice = 0, const DetileWindow& window = {}); + static VkFormat ImageElementFormat(std::uint32_t elementBytes); // Recycles the descriptor sets of the previous batch; call before recording a new command batch. void BeginBatch(); + void DispatchCmaskClear(VkCommandBuffer commands, VkBuffer cmask, std::uint64_t cmaskOffset, std::size_t cmaskBytes, VkImageView view, std::uint32_t width, std::uint32_t height, std::uint32_t elementBytes, const std::array& clearWords, bool write); + std::uint32_t CmaskErrors(); private: - VkPipeline pipeline(TextureTileMode tileMode, std::uint32_t elementBytes, bool retile, bool thick); + VkPipeline pipeline(TextureTileMode tileMode, std::uint32_t elementBytes, bool retile, bool thick, bool image = false); void release() noexcept; - VkDescriptorSet allocateSet(); + VkDescriptorSet allocateSet(bool image = false); + void checkWindow(const TileMipLayout& layout, const DetileWindow& window, std::uint32_t& columnEnd, std::uint32_t& rowEnd) const; const Context context; VkDescriptorSetLayout descriptorLayout = VK_NULL_HANDLE; @@ -58,6 +68,17 @@ namespace AgcDriver::Graphics { std::vector> pipelines; std::vector descriptorPools; std::size_t allocatedSets = 0; + VkDescriptorSetLayout cmaskDescriptorLayout = VK_NULL_HANDLE; + VkPipelineLayout cmaskPipelineLayout = VK_NULL_HANDLE; + VkPipeline cmaskPipeline = VK_NULL_HANDLE; + std::vector cmaskDescriptorPools; + std::size_t allocatedCmaskSets = 0; + std::unique_ptr cmaskErrors; + VkDescriptorSetLayout imageDescriptorLayout = VK_NULL_HANDLE; + VkPipelineLayout imagePipelineLayout = VK_NULL_HANDLE; + VkShaderModule imageModule = VK_NULL_HANDLE; + std::vector imageDescriptorPools; + std::size_t allocatedImageSets = 0; }; } diff --git a/core/libs/prx/libSceAgcDriver/Graphics/include/TextureFormat.hpp b/core/libs/prx/libSceAgcDriver/Graphics/include/TextureFormat.hpp index 8820c85bc..47552e4dc 100644 --- a/core/libs/prx/libSceAgcDriver/Graphics/include/TextureFormat.hpp +++ b/core/libs/prx/libSceAgcDriver/Graphics/include/TextureFormat.hpp @@ -15,6 +15,7 @@ struct Context; VkFormat ResolveTextureFormat(std::uint32_t guestFormat); std::uint32_t SrgbDecodeFormats(PFN_vkGetPhysicalDeviceFormatProperties formatProperties, VkPhysicalDevice physical); VkFormat SampledTextureFormat(const Context& context, std::uint32_t guestFormat); +bool IsSrgbTextureFormat(std::uint32_t guestFormat); std::optional FindGuestTextureFormat(VkFormat format, std::uint32_t elementBytes); std::optional FindGuestColorTargetFormat(VkFormat format, std::uint32_t elementBytes); std::uint32_t BytesPerElement(std::uint32_t guestFormat); diff --git a/core/libs/prx/libSceAgcDriver/Graphics/include/TextureSwizzleEquations.hpp b/core/libs/prx/libSceAgcDriver/Graphics/include/TextureSwizzleEquations.hpp index 3f1b18e3f..b77267d50 100644 --- a/core/libs/prx/libSceAgcDriver/Graphics/include/TextureSwizzleEquations.hpp +++ b/core/libs/prx/libSceAgcDriver/Graphics/include/TextureSwizzleEquations.hpp @@ -89,6 +89,11 @@ inline constexpr TextureSwizzleEquation kTextureSwizzleEquations[] = { {0x119u, 4u, {0x00000000u, 0x00000000u, 0x00000001u, 0x00001000u, 0x01000000u, 0x00002000u, 0x00000002u, 0x02000000u, 0x20004020u, 0x10020004u, 0x04010010u, 0x08008008u, 0x00000008u, 0x08000000u, 0x00010000u, 0x00000010u}}, {0x119u, 8u, {0x00000000u, 0x00000000u, 0x00000000u, 0x00000001u, 0x01000000u, 0x00001000u, 0x00000002u, 0x02000000u, 0x20002020u, 0x10010004u, 0x04008010u, 0x08004008u, 0x00000008u, 0x08000000u, 0x00008000u, 0x00000010u}}, {0x119u, 16u, {0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x01000000u, 0x00001000u, 0x00000001u, 0x02000000u, 0x20002010u, 0x10010002u, 0x04008008u, 0x08004004u, 0x00000004u, 0x08000000u, 0x00008000u, 0x00000008u}}, + {0x11au, 1u, {0x00000001u, 0x00000002u, 0x01000000u, 0x00001000u, 0x00002000u, 0x02000000u, 0x00000004u, 0x04000000u, 0x00008008u, 0x00010010u, 0x00024040u, 0x00040020u, 0x08000000u, 0x00008000u, 0x00000010u, 0x10000000u}}, + {0x11au, 2u, {0x00000000u, 0x00000001u, 0x01000000u, 0x00001000u, 0x00000002u, 0x02000000u, 0x00002000u, 0x04000000u, 0x00008008u, 0x00010010u, 0x08020040u, 0x00044020u, 0x00000004u, 0x10000000u, 0x00008000u, 0x00000010u}}, + {0x11au, 4u, {0x00000000u, 0x00000000u, 0x00000001u, 0x00001000u, 0x00000002u, 0x01000000u, 0x00002000u, 0x02000000u, 0x00008008u, 0x00010010u, 0x04020040u, 0x00044020u, 0x00000004u, 0x08000000u, 0x00008000u, 0x00000010u}}, + {0x11au, 8u, {0x00000000u, 0x00000000u, 0x00000000u, 0x00000001u, 0x00001000u, 0x01000000u, 0x00000002u, 0x02000000u, 0x00008008u, 0x00010010u, 0x04020040u, 0x00042020u, 0x00000004u, 0x08000000u, 0x00004000u, 0x00000008u}}, + {0x11au, 16u, {0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000001u, 0x01000000u, 0x00001000u, 0x02000000u, 0x00008008u, 0x00010012u, 0x04020040u, 0x00042020u, 0x00000004u, 0x08000000u, 0x00004000u, 0x00000008u}}, }; inline const TextureSwizzleEquation* FindTextureSwizzleEquation(std::uint32_t swizzleMode, std::uint32_t elementBytes) { diff --git a/core/libs/prx/libSceAgcDriver/Graphics/include/UnitShadow.hpp b/core/libs/prx/libSceAgcDriver/Graphics/include/UnitShadow.hpp index 7748a0e9d..4c8124f51 100644 --- a/core/libs/prx/libSceAgcDriver/Graphics/include/UnitShadow.hpp +++ b/core/libs/prx/libSceAgcDriver/Graphics/include/UnitShadow.hpp @@ -141,7 +141,7 @@ void MarkShadowed(const HostImport& import, std::span range // The import retires because its registration vanished or changed size, so the rest of its memory // belongs to the title again and is dropped (counted `dropped on retire`); `lost on retire` counts // the units nothing could record. -void RetireShadow(const Context& context, const HostImport& import, const std::function& registered); +void RetireShadow(VkDevice device, const HostImport& import, const std::function& registered); // Teardown: every shadow of the context's device is published; DestroyShadows then drops them. void PublishAllShadows(const Context& context, PublishReason reason); void DestroyShadows(VkDevice device); diff --git a/core/libs/prx/libSceAgcDriver/Graphics/include/VertexInput.hpp b/core/libs/prx/libSceAgcDriver/Graphics/include/VertexInput.hpp index 1964f46be..d3b182c7a 100644 --- a/core/libs/prx/libSceAgcDriver/Graphics/include/VertexInput.hpp +++ b/core/libs/prx/libSceAgcDriver/Graphics/include/VertexInput.hpp @@ -3,7 +3,9 @@ #include "prx/libSceAgcDriver/Graphics/include/Shaders.hpp" #include +#include #include +#include #include #include #include @@ -18,8 +20,17 @@ struct VertexFormat { const char* scalar; }; +inline bool NullVertexDescriptor(const ShaderRecompiler::VertexAttribute& attribute) { + const auto& fields = attribute.resource.fields; + return fields[0] == 0 && fields[1] == 0 && fields[2] == 0 && fields[3] == 0; +} + inline VertexFormat DecodeVertexFormat(const ShaderRecompiler::VertexAttribute& attribute) { Require(attribute.components >= 1 && attribute.components <= 4, "invalid vertex attribute component count"); + if (NullVertexDescriptor(attribute)) { + const std::array formats{VK_FORMAT_R32_SFLOAT, VK_FORMAT_R32G32_SFLOAT, VK_FORMAT_R32G32B32_SFLOAT, VK_FORMAT_R32G32B32A32_SFLOAT}; + return {formats[attribute.components - 1], attribute.components * 4u, 4u, "f32"}; + } const auto components = attribute.formatComponents == 0u ? attribute.components : attribute.formatComponents; Require(components >= 1u && components <= 4u, "invalid vertex format component count"); const auto format = (attribute.resource.fields[3] >> 12u) & 0x7fu; @@ -108,7 +119,7 @@ inline VertexInputLayout BuildVertexInputLayout(const Context& context, std::spa Require((fields[1] & 0x80000000u) == 0 && (fields[3] & 0x00800000u) == 0 && (fields[3] >> 30u) == 0, "unsupported vertex buffer descriptor flags"); const auto stride = (fields[1] >> 16u) & 0x3fffu; const auto address = fields[0] | (static_cast(fields[1] & 0xffffu) << 32u); - Require(address != 0 && address % format.alignment == 0 && stride % format.alignment == 0, "unaligned vertex buffer"); + Require(NullVertexDescriptor(attribute) || (address != 0 && address % format.alignment == 0 && stride % format.alignment == 0), "unaligned vertex buffer"); Require(stride <= context.limits.maxVertexInputBindingStride, "vertex stride exceeds device limits"); Require(context.formatProperties != nullptr, "missing vertex format property query"); VkFormatProperties properties{}; @@ -123,11 +134,28 @@ inline VertexInputLayout BuildVertexInputLayout(const Context& context, std::spa // The whole byte range a vertex buffer descriptor covers (records * stride, or records when the // stride is 0): what an indirect draw, whose counts only the GPU knows, copies for the fetch. +inline std::uint32_t VertexBufferOutOfBoundsSelect(const ShaderRecompiler::VertexAttribute& attribute) { + return (attribute.resource.fields[3] >> 28u) & 3u; +} + +inline bool VertexFetchOutOfRange(const ShaderRecompiler::VertexAttribute& attribute) { + const auto stride = (attribute.resource.fields[1] >> 16u) & 0x3fffu; + const auto select = VertexBufferOutOfBoundsSelect(attribute); + return stride == 0 && (select == 2u || select == 3u) && attribute.resource.fields[2] == 0; +} + +inline std::uint64_t ZeroStrideAvailableBytes(const ShaderRecompiler::VertexAttribute& attribute) { + const auto records = attribute.resource.fields[2]; + if (VertexBufferOutOfBoundsSelect(attribute) == 2u && records != 0) return std::max(records, DecodeVertexFormat(attribute).bytes); + return records; +} + inline std::size_t VertexBufferExtent(const ShaderRecompiler::VertexAttribute& attribute) { + if (NullVertexDescriptor(attribute)) return DecodeVertexFormat(attribute).bytes; const auto stride = (attribute.resource.fields[1] >> 16u) & 0x3fffu; const auto records = attribute.resource.fields[2]; Require(attribute.fetchIndex <= 1, "unsupported vertex fetch index"); - const auto bytes = stride == 0 ? static_cast(records) : static_cast(records) * stride; + const auto bytes = stride == 0 ? ZeroStrideAvailableBytes(attribute) : static_cast(records) * stride; Require(bytes != 0 && bytes <= std::numeric_limits::max(), "empty or oversized vertex buffer descriptor"); const auto address = attribute.resource.fields[0] | (static_cast(attribute.resource.fields[1] & 0xffffu) << 32u); Require(address != 0 && bytes <= std::numeric_limits::max() - address, "invalid vertex buffer address range"); @@ -137,15 +165,16 @@ inline std::size_t VertexBufferExtent(const ShaderRecompiler::VertexAttribute& a inline std::size_t VertexBufferReadSize(const ShaderRecompiler::VertexAttribute& attribute, std::uint32_t maxIndex, std::uint32_t instances, std::uint32_t firstInstance = 0) { Require(instances != 0, "vertex input requires nonzero instance count"); Require(firstInstance <= std::numeric_limits::max() - (instances - 1u), "vertex input instance range overflow"); + if (NullVertexDescriptor(attribute)) return DecodeVertexFormat(attribute).bytes; const auto stride = (attribute.resource.fields[1] >> 16u) & 0x3fffu; const auto records = attribute.resource.fields[2]; Require(attribute.fetchIndex <= 1, "unsupported vertex fetch index"); const auto index = attribute.fetchIndex == 0 ? maxIndex : firstInstance + instances - 1u; const auto bytes = DecodeVertexFormat(attribute).bytes; - Require(stride == 0 || index < records, "vertex fetch exceeds descriptor record count"); - const auto available = stride == 0 ? static_cast(records) : static_cast(records) * stride; + Require(stride == 0 || VertexBufferOutOfBoundsSelect(attribute) <= 1u || index < records, "vertex fetch exceeds descriptor record count"); const auto required = static_cast(stride) * index + bytes; - Require(required <= available && required <= std::numeric_limits::max(), "vertex fetch exceeds descriptor byte range"); + Require(stride != 0 || required <= ZeroStrideAvailableBytes(attribute), "vertex fetch exceeds descriptor byte range"); + Require(required <= std::numeric_limits::max(), "vertex fetch exceeds addressable byte range"); const auto address = attribute.resource.fields[0] | (static_cast(attribute.resource.fields[1] & 0xffffu) << 32u); Require(address != 0 && required <= std::numeric_limits::max() - address, "invalid vertex buffer address range"); return static_cast(required); @@ -163,6 +192,7 @@ struct VertexCopyPlan { std::vector> copies; std::vector copyOf; std::vector offsets; + std::vector alignments; }; inline VertexCopyPlan PlanVertexCopies(std::span fetches) { @@ -186,7 +216,9 @@ inline VertexCopyPlan PlanVertexCopies(std::span fetches) { if (!joins) { lead = i; plan.copies.emplace_back(fetch.begin, fetch.end); + plan.alignments.push_back(fetch.alignment); } + plan.alignments.back() = std::max(plan.alignments.back(), fetch.alignment); auto& copy = plan.copies.back(); copy.second = std::max(copy.second, fetch.end); plan.copyOf[i] = plan.copies.size() - 1; @@ -195,6 +227,34 @@ inline VertexCopyPlan PlanVertexCopies(std::span fetches) { return plan; } +inline std::optional HighestDrawIndex(std::span indices, std::uint32_t indexSize, bool skipRestart) { + Require(indexSize == 2 || indexSize == 4, "unsupported index size"); + const auto restartIndex = indexSize == 2 ? 0xffffu : 0xffffffffu; + std::optional highest; + for (std::size_t offset = 0; offset + indexSize <= indices.size(); offset += indexSize) { + std::uint32_t index = 0; + if (indexSize == 2) { + std::uint16_t value = 0; + std::memcpy(&value, indices.data() + offset, sizeof(value)); + index = value; + } else { + std::memcpy(&index, indices.data() + offset, sizeof(index)); + } + if (skipRestart && index == restartIndex) continue; + highest = std::max(highest.value_or(0u), index); + } + return highest; +} + +inline std::vector SoloZeroPaddedFetchIndices(const std::vector& fetches, const std::vector& fetchValid) { + Require(fetchValid.size() == fetches.size(), "fetch validity does not match the fetch count"); + std::vector solo; + for (std::size_t i = 0; i < fetches.size(); ++i) { + if (fetchValid[i] < fetches[i].end - fetches[i].begin) solo.push_back(i); + } + return solo; +} + } #endif diff --git a/core/libs/prx/libSceAgcDriver/Graphics/shaders/CmaskClear.comp b/core/libs/prx/libSceAgcDriver/Graphics/shaders/CmaskClear.comp new file mode 100644 index 000000000..0b2dde1f1 --- /dev/null +++ b/core/libs/prx/libSceAgcDriver/Graphics/shaders/CmaskClear.comp @@ -0,0 +1,38 @@ +#version 450 + +layout(local_size_x = 8, local_size_y = 8) in; + +layout(set = 0, binding = 0, std430) readonly buffer Cmask { uint words[]; } cmask; +layout(set = 0, binding = 1, std430) buffer Errors { uint codes; } errors; +layout(set = 0, binding = 2) writeonly uniform uimage2D target; + +layout(push_constant) uniform Push { + uint base; + uint width; + uint height; + uint blocksPerRow; + uint clear0; + uint clear1; + uint write; +} push; + +const uint xMasks[13] = uint[](0x008u, 0x000u, 0x040u, 0x000u, 0x080u, 0x000u, 0x100u, 0x000u, 0x200u, 0x008u, 0x010u, 0x040u, 0x020u); +const uint yMasks[13] = uint[](0x000u, 0x010u, 0x000u, 0x040u, 0x000u, 0x080u, 0x000u, 0x100u, 0x000u, 0x008u, 0x010u, 0x020u, 0x040u); + +void main() { + uvec2 pixel = gl_GlobalInvocationID.xy; + if (pixel.x >= push.width || pixel.y >= push.height) return; + uint x = pixel.x & ~7u; + uint y = pixel.y & ~7u; + uint offset = 0u; + for (uint bit = 0u; bit < 13u; ++bit) offset |= (uint(bitCount((x & xMasks[bit]) ^ (y & yMasks[bit]))) & 1u) << bit; + uint nibble = ((y / 512u) * push.blocksPerRow + x / 1024u) * 8192u + offset; + uint byteAddress = push.base + nibble / 2u; + uint code = (cmask.words[byteAddress / 4u] >> ((byteAddress % 4u) * 8u + (nibble % 2u) * 4u)) & 0xfu; + if (code == 0xfu) return; + if (code == 0u && push.write != 0u) { + imageStore(target, ivec2(pixel), uvec4(push.clear0, push.clear1, 0u, 0u)); + return; + } + if ((pixel.x & 7u) == 0u && (pixel.y & 7u) == 0u) atomicOr(errors.codes, 1u << code); +} diff --git a/core/libs/prx/libSceAgcDriver/Graphics/shaders/SampleDumps.comp b/core/libs/prx/libSceAgcDriver/Graphics/shaders/SampleDumps.comp new file mode 100644 index 000000000..d8e62f91b --- /dev/null +++ b/core/libs/prx/libSceAgcDriver/Graphics/shaders/SampleDumps.comp @@ -0,0 +1,40 @@ +#version 450 +#extension GL_EXT_buffer_reference : require +#extension GL_EXT_buffer_reference_uvec2 : require + +layout(local_size_x = 1) in; +layout(buffer_reference, std430, buffer_reference_align = 8) buffer Words { uint words[]; }; +layout(push_constant) uniform Parameters { + Words counter; + Words dumps; + uint firstDump; + uint dumpCount; + uint firstSegment; + uint lastSegment; +} parameters; + +void main() { + uint low = parameters.counter.words[0]; + uint high = parameters.counter.words[1]; + uint segment = parameters.firstSegment; + for (uint dump = parameters.firstDump; dump < parameters.firstDump + parameters.dumpCount; ++dump) { + uint end = parameters.dumps.words[dump * 4u + 2u]; + for (; segment < end; ++segment) { + uint carry; + low = uaddCarry(low, parameters.counter.words[4u + 2u * segment], carry); + high = high + parameters.counter.words[5u + 2u * segment] + carry; + } + Words target = Words(uvec2(parameters.dumps.words[dump * 4u], parameters.dumps.words[dump * 4u + 1u])); + for (uint db = 0u; db < 16u; ++db) { + target.words[db * 4u] = db == 0u ? low : 0u; + target.words[db * 4u + 1u] = (db == 0u ? high : 0u) | 0x80000000u; + } + } + for (; segment < parameters.lastSegment; ++segment) { + uint carry; + low = uaddCarry(low, parameters.counter.words[4u + 2u * segment], carry); + high = high + parameters.counter.words[5u + 2u * segment] + carry; + } + parameters.counter.words[0] = low; + parameters.counter.words[1] = high; +} diff --git a/core/libs/prx/libSceAgcDriver/Graphics/shaders/TextureDetile.comp b/core/libs/prx/libSceAgcDriver/Graphics/shaders/TextureDetile.comp index 38b17aeae..fcff04497 100644 --- a/core/libs/prx/libSceAgcDriver/Graphics/shaders/TextureDetile.comp +++ b/core/libs/prx/libSceAgcDriver/Graphics/shaders/TextureDetile.comp @@ -1,4 +1,7 @@ #version 450 +#ifdef LINEAR_IMAGE +#extension GL_EXT_shader_image_load_formatted : require +#endif layout(constant_id = 0) const uint ELEMENT_BYTES = 4u; layout(constant_id = 1) const uint BLOCK_BYTES = 256u; @@ -29,8 +32,13 @@ layout(constant_id = 21) const uint BLOCK_HEIGHT = 0u; layout(local_size_x = 8, local_size_y = 8) in; +#ifdef LINEAR_IMAGE +layout(set = 0, binding = 0, std430) buffer Tiled { uint data[]; } tiledBuffer; +layout(set = 0, binding = 1) uniform uimage2D linearImage; +#else layout(set = 0, binding = 0, std430) readonly buffer Input { uint data[]; } inputBuffer; layout(set = 0, binding = 1, std430) buffer Output { uint data[]; } outputBuffer; +#endif layout(push_constant) uniform Push { uint srcBase; @@ -55,6 +63,7 @@ layout(push_constant) uniform Push { // whole mip). uint columnBegin; uint rowBegin; + uint pipeBankXor; } params; uint equationBit(uint mask, uvec2 p) { @@ -117,7 +126,7 @@ uint standard64Extra(uint x, uint y) { uint blockOffset(uvec2 p) { if (TILE_FAMILY == 2u) { - return equationOffset(p); + return equationOffset(p) ^ params.pipeBankXor; } uint offset = standardOffset(p.x, p.y); if (BLOCK_BYTES > 4096u) { @@ -141,6 +150,43 @@ uvec2 blockExtent() { return uvec2(width, BLOCK_BYTES / (width * ELEMENT_BYTES)); } +#ifdef LINEAR_IMAGE +void storeElement(uint src, uvec2 p) { + uvec4 value = uvec4(0u); + uint word = src >> 2; + if (ELEMENT_BYTES == 1u) { + value.x = (tiledBuffer.data[word] >> ((src & 3u) * 8u)) & 0xffu; + } else if (ELEMENT_BYTES == 2u) { + value.x = (tiledBuffer.data[word] >> ((src & 3u) * 8u)) & 0xffffu; + } else { + value.x = tiledBuffer.data[word]; + if (ELEMENT_BYTES >= 8u) value.y = tiledBuffer.data[word + 1u]; + if (ELEMENT_BYTES >= 16u) { + value.z = tiledBuffer.data[word + 2u]; + value.w = tiledBuffer.data[word + 3u]; + } + } + imageStore(linearImage, ivec2(p), value); +} + +void loadElement(uvec2 p, uint dst) { + uvec4 value = imageLoad(linearImage, ivec2(p)); + uint word = dst >> 2; + if (ELEMENT_BYTES >= 4u) { + tiledBuffer.data[word] = value.x; + if (ELEMENT_BYTES >= 8u) tiledBuffer.data[word + 1u] = value.y; + if (ELEMENT_BYTES >= 16u) { + tiledBuffer.data[word + 2u] = value.z; + tiledBuffer.data[word + 3u] = value.w; + } + } else { + uint mask = ELEMENT_BYTES == 1u ? 0xffu : 0xffffu; + uint shift = (dst & 3u) * 8u; + atomicAnd(tiledBuffer.data[word], ~(mask << shift)); + atomicOr(tiledBuffer.data[word], (value.x & mask) << shift); + } +} +#else void copyElement(uint src, uint dst) { if (ELEMENT_BYTES >= 4u) { for (uint i = 0u; i < ELEMENT_BYTES; i += 4u) { @@ -154,6 +200,7 @@ void copyElement(uint src, uint dst) { atomicOr(outputBuffer.data[dst >> 2], value << shift); } } +#endif void main() { uvec2 p = gl_GlobalInvocationID.xy + uvec2(params.columnBegin, params.rowBegin); @@ -169,11 +216,19 @@ void main() { if (linearOffset < params.rangeBegin || linearOffset >= params.rangeEnd) { return; } +#ifdef LINEAR_IMAGE + if (RETILE != 0u) { + loadElement(p, params.dstBase + linearOffset - params.tiledBase); + } else { + storeElement(params.srcBase + linearOffset - params.tiledBase, p); + } +#else if (RETILE != 0u) { copyElement(params.srcBase + linearOffset - params.linearBase, params.dstBase + linearOffset - params.tiledBase); } else { copyElement(params.srcBase + linearOffset - params.tiledBase, params.dstBase + linearOffset - params.linearBase); } +#endif return; } @@ -193,10 +248,18 @@ void main() { tiledOffset -= params.tiledBase; linearOffset -= params.linearBase; +#ifdef LINEAR_IMAGE + if (RETILE != 0u) { + loadElement(p, params.dstBase + tiledOffset); + } else { + storeElement(params.srcBase + tiledOffset, p); + } +#else if (RETILE != 0u) { copyElement(params.srcBase + linearOffset, params.dstBase + tiledOffset); } else { copyElement(params.srcBase + tiledOffset, params.dstBase + linearOffset); } +#endif } diff --git a/core/libs/prx/libSceAgcDriver/Graphics/src/BufferPool.cpp b/core/libs/prx/libSceAgcDriver/Graphics/src/BufferPool.cpp index bdee4b50f..2c0c39987 100644 --- a/core/libs/prx/libSceAgcDriver/Graphics/src/BufferPool.cpp +++ b/core/libs/prx/libSceAgcDriver/Graphics/src/BufferPool.cpp @@ -5,6 +5,7 @@ #include #include #include +#include namespace AgcDriver::Graphics { @@ -16,12 +17,19 @@ bool sharedTiers() { return shared; } +bool exactSizes() { + static const bool exact = std::getenv("APS5_NO_BUFFER_CLASSES") != nullptr; + return exact; } -BufferPool::BufferPool(const Context& context) : device(context.device), unmap(context.Function("vkUnmapMemory")), destroyBuffer(context.Function("vkDestroyBuffer")), freeMemory(context.Function("vkFreeMemory")), maxSlots(std::max(1, std::min(MaxSlots(), context.limits.maxMemoryAllocationCount / 6u))) { +constexpr VkBufferUsageFlags deviceUsage = VK_BUFFER_USAGE_STORAGE_BUFFER_BIT | VK_BUFFER_USAGE_TRANSFER_SRC_BIT | VK_BUFFER_USAGE_TRANSFER_DST_BIT; + +} + +BufferPool::BufferPool(const Context& context) : device(context.device), unmap(context.Function("vkUnmapMemory")), destroyBuffer(context.Function("vkDestroyBuffer")), freeMemory(context.Function("vkFreeMemory")), allocateMemory(context.Function("vkAllocateMemory")), mapMemory(context.Function("vkMapMemory")), maxSlots(std::max(1, std::min(MaxSlots(), context.limits.maxMemoryAllocationCount / 6u))) { smallTier.budget = smallBudget; largeTier.budget = budget; - deviceTier.budget = DeviceBudget(); + deviceTier.budget = DeviceTierEnabled() ? DeviceBudget(context.memory) : 0; } BufferPool::~BufferPool() { @@ -30,38 +38,159 @@ BufferPool::~BufferPool() { for (const auto& slot : slots) destroy(slot.allocation); } } + for (auto& [memory, block] : slabBlocks) freeBlock(*block); } -VkDeviceSize BufferPool::DeviceBudget() { - static const VkDeviceSize deviceBudget = [] { +bool BufferPool::SlabEligible(std::size_t capacity, VkDeviceSize alignment, VkDeviceSize size, VkDeviceSize atom) { + if (capacity >= classLimit || size > capacity || !std::has_single_bit(capacity)) return false; + const auto alignmentOk = alignment == 0 || (std::has_single_bit(alignment) && alignment <= capacity); + const auto atomOk = atom <= 1 || (std::has_single_bit(atom) && atom <= capacity); + return alignmentOk && atomOk; +} + +VkDeviceSize BufferPool::SlabBlockBytes(std::size_t capacity) { + return std::max(VkDeviceSize{2} << 20u, static_cast(capacity) * 8u); +} + +std::uint64_t BufferPool::SlabKey(std::uint32_t memoryType, std::size_t capacity, bool addressable) { + return (static_cast(memoryType) << 40u) | (static_cast(addressable ? 1u : 0u) << 39u) | static_cast(capacity); +} + +SlabSlot BufferPool::TakeSlot(const Context& context, std::uint32_t memoryType, std::size_t capacity, bool addressable) { + const auto key = SlabKey(memoryType, capacity, addressable); + const auto takeFrom = [&](SlabBlock& block, Slab& slab) { + const auto index = block.free.back(); + block.free.pop_back(); + if (block.used++ == 0) --slab.emptyBlocks; + if (block.free.empty()) slab.available.erase(std::find(slab.available.begin(), slab.available.end(), &block)); + const auto offset = static_cast(index) * block.slotBytes; + return SlabSlot{block.memory, offset, block.mapping != nullptr ? block.mapping + offset : nullptr}; + }; + { + std::lock_guard lock(slabMutex); + auto& slab = slabs[key]; + if (!slab.available.empty()) return takeFrom(*slab.available.back(), slab); + } + const auto blockBytes = SlabBlockBytes(capacity); + auto block = std::make_unique(); + block->slotBytes = capacity; + block->slab = key; + VkMemoryAllocateInfo allocation{VK_STRUCTURE_TYPE_MEMORY_ALLOCATE_INFO}; + const VkMemoryAllocateFlagsInfo flags{VK_STRUCTURE_TYPE_MEMORY_ALLOCATE_FLAGS_INFO, nullptr, VK_MEMORY_ALLOCATE_DEVICE_ADDRESS_BIT, 0}; + if (addressable) allocation.pNext = &flags; + allocation.allocationSize = blockBytes; + allocation.memoryTypeIndex = memoryType; + Check(allocateMemory(device, &allocation, nullptr, &block->memory), "vkAllocateMemory buffer slab"); + if ((context.memory.memoryTypes[memoryType].propertyFlags & VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT) != 0) { + void* mapping = nullptr; + const auto mapped = mapMemory(device, block->memory, 0, VK_WHOLE_SIZE, 0, &mapping); + if (mapped != VK_SUCCESS) freeMemory(device, block->memory, nullptr); + Check(mapped, "vkMapMemory buffer slab"); + block->mapping = static_cast(mapping); + } + const auto slots = static_cast(blockBytes / capacity); + block->free.reserve(slots); + for (std::uint32_t index = slots; index-- != 0;) block->free.push_back(index); + std::lock_guard lock(slabMutex); + auto& slab = slabs[key]; + auto& inserted = *block; + slabBlocks.emplace(inserted.memory, std::move(block)); + slab.available.push_back(&inserted); + ++slab.emptyBlocks; + return takeFrom(*slab.available.back(), slab); +} + +void BufferPool::freeBlock(SlabBlock& block) noexcept { + if (block.mapping != nullptr) unmap(device, block.memory); + freeMemory(device, block.memory, nullptr); +} + +void BufferPool::PutSlot(VkDeviceMemory memory, VkDeviceSize offset) noexcept { + std::unique_ptr released; + { + std::lock_guard lock(slabMutex); + const auto found = slabBlocks.find(memory); + if (found == slabBlocks.end()) return; + auto& block = *found->second; + auto& slab = slabs[block.slab]; + if (block.free.empty()) slab.available.push_back(&block); + block.free.push_back(static_cast(offset / block.slotBytes)); + if (--block.used != 0) return; + if (slab.emptyBlocks == 0) { + ++slab.emptyBlocks; + return; + } + slab.available.erase(std::find(slab.available.begin(), slab.available.end(), &block)); + released = std::move(found->second); + slabBlocks.erase(found); + } + freeBlock(*released); +} + +std::size_t BufferPool::SlabBlocks() { + std::lock_guard lock(slabMutex); + return slabBlocks.size(); +} + +bool BufferPool::DeviceTierEnabled() { + static const bool enabled = [] { const char* value = std::getenv("APS5_STAGING_POOL_MIB"); - return (value != nullptr ? std::strtoull(value, nullptr, 10) : 512ull) << 20u; + return value == nullptr || std::strtoull(value, nullptr, 10) != 0; }(); - return deviceBudget; + return enabled; +} + +VkDeviceSize BufferPool::DeviceBudget(const VkPhysicalDeviceMemoryProperties& memory) { + static const char* value = std::getenv("APS5_STAGING_POOL_MIB"); + if (value != nullptr) return static_cast(std::strtoull(value, nullptr, 10)) << 20u; + VkDeviceSize largest = 0; + for (std::uint32_t heap = 0; heap < memory.memoryHeapCount && heap < VK_MAX_MEMORY_HEAPS; ++heap) { + if ((memory.memoryHeaps[heap].flags & VK_MEMORY_HEAP_DEVICE_LOCAL_BIT) != 0) largest = std::max(largest, memory.memoryHeaps[heap].size); + } + return std::max(VkDeviceSize{512} << 20u, largest / 8u); } void BufferPool::destroy(const BufferAllocation& allocation) noexcept { + if (allocation.slab) { + destroyBuffer(device, allocation.buffer, nullptr); + PutSlot(allocation.memory, allocation.offset); + return; + } // Device-local allocations (see DeviceBuffer) are never mapped. if (allocation.mapping != nullptr) unmap(device, allocation.memory); destroyBuffer(device, allocation.buffer, nullptr); freeMemory(device, allocation.memory, nullptr); } -std::size_t BufferPool::Capacity(std::size_t bytes) { - static const bool exact = std::getenv("APS5_NO_BUFFER_CLASSES") != nullptr; +bool BufferPool::DeviceTiered(VkMemoryPropertyFlags properties) { + return (properties & VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT) != 0 && (properties & VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT) == 0 && DeviceTierEnabled(); +} + +std::size_t BufferPool::Capacity(std::size_t bytes, VkMemoryPropertyFlags properties) { constexpr std::size_t smallest = 256; - if (exact || bytes >= classLimit) return bytes; - return std::max(smallest, std::bit_ceil(bytes)); + if (exactSizes()) return bytes; + if (bytes < classLimit) return std::max(smallest, std::bit_ceil(bytes)); + if (!DeviceTiered(properties)) return bytes; + const auto step = std::bit_floor(bytes) >> deviceClassBits; + return (bytes + step - 1) / step * step; +} + +std::size_t BufferPool::NextCapacity(std::size_t capacity) { + return capacity < classLimit ? capacity * 2 : capacity + (std::bit_floor(capacity) >> deviceClassBits); +} + +VkBufferUsageFlags BufferPool::Usage(VkBufferUsageFlags usage, VkMemoryPropertyFlags properties) { + return DeviceTiered(properties) ? usage | deviceUsage : usage; } BufferPool::Tier& BufferPool::tierFor(std::size_t capacity, VkMemoryPropertyFlags properties) { - if ((properties & VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT) != 0 && (properties & VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT) == 0 && DeviceBudget() != 0) return deviceTier; + if (DeviceTiered(properties)) return deviceTier; return !sharedTiers() && capacity < classLimit ? smallTier : largeTier; } std::optional BufferPool::Take(std::size_t bytes, VkBufferUsageFlags usage, VkMemoryPropertyFlags properties) { static const bool profile = std::getenv("APS5_PROFILE_DRAW") != nullptr; - const auto capacity = Capacity(bytes); + const auto capacity = Capacity(bytes, properties); std::lock_guard lock(mutex); ++clock; if (profile) { @@ -73,7 +202,9 @@ std::optional BufferPool::Take(std::size_t bytes, VkBufferUsag } } auto& tier = tierFor(capacity, properties); - const auto found = tier.free.find({capacity, usage, properties}); + const auto largest = &tier == &deviceTier && !exactSizes() && bytes <= std::numeric_limits::max() / 2 ? std::max(capacity, bytes * 2) : capacity; + auto found = tier.free.end(); + for (auto size = capacity; size <= largest && found == tier.free.end(); size = NextCapacity(size)) found = tier.free.find({size, usage, properties}); if (found == tier.free.end() || found->second.empty()) { ++tier.misses; return std::nullopt; diff --git a/core/libs/prx/libSceAgcDriver/Graphics/src/ColorTargetLayout.cpp b/core/libs/prx/libSceAgcDriver/Graphics/src/ColorTargetLayout.cpp index 95cbf8470..b4deb5dda 100644 --- a/core/libs/prx/libSceAgcDriver/Graphics/src/ColorTargetLayout.cpp +++ b/core/libs/prx/libSceAgcDriver/Graphics/src/ColorTargetLayout.cpp @@ -1,10 +1,12 @@ #include "prx/libSceAgcDriver/Graphics/include/ColorTargetLayout.hpp" #include "prx/libSceAgcDriver/Graphics/include/TextureSwizzleEquations.hpp" +#include #include #include #include #include #include +#include namespace AgcDriver::Graphics { namespace { @@ -41,11 +43,12 @@ std::uint32_t standard64Extra(std::uint32_t x, std::uint32_t y, std::uint32_t el ColorTileMode DecodeColorTileMode(std::uint32_t attrib3) { const auto resourceType = (attrib3 >> 24u) & 3u; - require((attrib3 & 0x80002000u) == 0 && ((resourceType == 1 && (attrib3 & 0x1fffu) == 0) || resourceType == 2) && ((attrib3 >> 27u) & 7u) == 1, "AGC graphics: unsupported color depth, dimension, resource level or metadata mode"); + require((attrib3 & 0x80002000u) == 0 && ((resourceType == 0 && (attrib3 & 0x1fffu) == 0) || resourceType == 1 || resourceType == 2) && ((attrib3 >> 27u) & 7u) == 1, "AGC graphics: unsupported color depth, dimension, resource level or metadata mode"); const auto mode = (attrib3 >> 14u) & 0x1fu; const auto fmaskMode = (attrib3 >> 19u) & 0x1fu; - require(fmaskMode == 0 || fmaskMode == 0x18, "AGC graphics: unsupported color FMASK swizzle mode"); - require(mode == 0 || mode == 5 || mode == 9 || mode == 0x1b, "AGC graphics: unsupported color tile mode"); + require(mode == 0 || mode == 5 || mode == 9 || mode == 0x15 || mode == 0x16 || mode == 0x1b, "AGC graphics: unsupported color tile mode"); + require(fmaskMode == 0 || fmaskMode == 0x18 || fmaskMode == (mode & ~3u), "AGC graphics: unsupported color FMASK swizzle mode"); + require(resourceType != 0 || mode == 0 || mode == 0x1b, "AGC graphics: 1D color targets with a standard swizzle mode are invalid"); return static_cast(mode); } @@ -56,14 +59,15 @@ struct SwizzleTables { std::vector y; }; -const SwizzleTables& renderTargetTables(std::uint32_t bytesPerElement, std::uint32_t blockWidth, std::uint32_t blockHeight) { - static std::once_flag once[5]; - static SwizzleTables tables[5]; +const SwizzleTables& equationTables(ColorTileMode mode, std::uint32_t bytesPerElement, std::uint32_t blockWidth, std::uint32_t blockHeight) { + static std::once_flag once[3][5]; + static SwizzleTables tables[3][5]; + const auto modeIndex = mode == ColorTileMode::S4KBX ? 0u : mode == ColorTileMode::D4KBX ? 1u : 2u; const auto index = static_cast(std::countr_zero(bytesPerElement)); - std::call_once(once[index], [&] { - const auto* equation = FindTextureSwizzleEquation(27u, bytesPerElement); - require(equation != nullptr, "AGC graphics: no SW_64KB_R_X equation for the color element size"); - auto& table = tables[index]; + std::call_once(once[modeIndex][index], [&] { + const auto* equation = FindTextureSwizzleEquation(static_cast(mode), bytesPerElement); + require(equation != nullptr, "AGC graphics: no XOR swizzle equation for the color tile mode and element size"); + auto& table = tables[modeIndex][index]; table.x.resize(blockWidth); table.y.resize(blockHeight); for (std::uint32_t x = 0; x < blockWidth; ++x) { @@ -77,7 +81,7 @@ const SwizzleTables& renderTargetTables(std::uint32_t bytesPerElement, std::uint table.y[y] = offset; } }); - return tables[index]; + return tables[modeIndex][index]; } const SwizzleTables& standardTables(std::uint32_t bytesPerElement, std::uint32_t blockWidth, std::uint32_t blockHeight, bool block64KB) { @@ -96,9 +100,10 @@ const SwizzleTables& standardTables(std::uint32_t bytesPerElement, std::uint32_t } -ColorTargetLayout::ColorTargetLayout(std::uint32_t width, std::uint32_t height, ColorTileMode mode, std::uint32_t bytesPerElement) : width(width), height(height), pitch(width), mode(mode), bytes(0), elementBytes(bytesPerElement) { +ColorTargetLayout::ColorTargetLayout(std::uint32_t width, std::uint32_t height, ColorTileMode mode, std::uint32_t bytesPerElement, std::uint32_t pipeBankXor) : width(width), height(height), pitch(width), mode(mode), bytes(0), elementBytes(bytesPerElement), pipeBankXor(pipeBankXor) { require(width != 0 && height != 0 && width <= 16384 && height <= 16384, "AGC graphics: invalid color surface extent"); require(std::has_single_bit(bytesPerElement) && bytesPerElement <= 16u, "AGC graphics: unsupported color element size"); + require(pipeBankXor == 0 || (ColorTileModeIsXor(mode) && pipeBankXor < ColorTileModeBlockBytes(mode) && pipeBankXor % 256u == 0), "AGC graphics: a color pipe/bank XOR applies only to whole 256-byte units of XOR swizzle blocks"); std::uint32_t paddedHeight = height; switch (mode) { case ColorTileMode::Linear: { @@ -106,14 +111,16 @@ ColorTargetLayout::ColorTargetLayout(std::uint32_t width, std::uint32_t height, pitch = (width + pitchAlignment - 1u) / pitchAlignment * pitchAlignment; break; } + case ColorTileMode::S4KBX: + case ColorTileMode::D4KBX: case ColorTileMode::RenderTarget: { // SW_64KB_R_X: 64 KiB blocks of 2^(16 - log2(bpe)) elements, wider than tall for odd powers. - const auto log2Elements = 16u - static_cast(std::countr_zero(bytesPerElement)); + const auto log2Elements = (mode == ColorTileMode::RenderTarget ? 16u : 12u) - static_cast(std::countr_zero(bytesPerElement)); blockWidth = 1u << ((log2Elements + 1u) / 2u); blockHeight = 1u << (log2Elements / 2u); pitch = (width + blockWidth - 1u) / blockWidth * blockWidth; paddedHeight = (height + blockHeight - 1u) / blockHeight * blockHeight; - const auto& tables = renderTargetTables(bytesPerElement, blockWidth, blockHeight); + const auto& tables = equationTables(mode, bytesPerElement, blockWidth, blockHeight); xOffsets = tables.x.data(); yOffsets = tables.y.data(); break; @@ -142,7 +149,7 @@ ColorTargetLayout::ColorTargetLayout(std::uint32_t width, std::uint32_t height, std::size_t ColorTargetLayout::offset(std::uint32_t x, std::uint32_t y) const { if (mode == ColorTileMode::Linear) return (static_cast(y) * pitch + x) * elementBytes; const auto block = static_cast(y / blockHeight) * (pitch / blockWidth) + x / blockWidth; - return block * Alignment() + (xOffsets[x % blockWidth] ^ yOffsets[y % blockHeight]); + return block * Alignment() + (xOffsets[x % blockWidth] ^ yOffsets[y % blockHeight] ^ pipeBankXor); } std::size_t ColorTargetLayout::Offset(std::uint32_t x, std::uint32_t y) const { @@ -166,4 +173,23 @@ void ColorTargetLayout::Tile(std::span source, std::span(blocksPerRow) * ((height + 511u) / 512u) * Alignment; +} + +std::size_t CmaskLayout::Nibble(std::uint32_t tileX, std::uint32_t tileY) const { + require(tileX < TilesX() && tileY < TilesY(), "AGC graphics: CMASK tile out of range"); + static constexpr std::array, 13> equation{{ + {0x008u, 0x000u}, {0x000u, 0x010u}, {0x040u, 0x000u}, {0x000u, 0x040u}, {0x080u, 0x000u}, {0x000u, 0x080u}, {0x100u, 0x000u}, + {0x000u, 0x100u}, {0x200u, 0x000u}, {0x008u, 0x008u}, {0x010u, 0x010u}, {0x040u, 0x020u}, {0x020u, 0x040u}, + }}; + const auto x = tileX * 8u; + const auto y = tileY * 8u; + std::size_t offset = 0; + for (std::size_t bit = 0; bit < equation.size(); ++bit) offset |= static_cast(parity((x & equation[bit].first) ^ (y & equation[bit].second))) << bit; + const auto block = static_cast(y / 512u) * blocksPerRow + x / 1024u; + return block * Alignment * 2u + offset; +} + } diff --git a/core/libs/prx/libSceAgcDriver/Graphics/src/ColorTargetTransfer.cpp b/core/libs/prx/libSceAgcDriver/Graphics/src/ColorTargetTransfer.cpp index 36690adc4..763964967 100644 --- a/core/libs/prx/libSceAgcDriver/Graphics/src/ColorTargetTransfer.cpp +++ b/core/libs/prx/libSceAgcDriver/Graphics/src/ColorTargetTransfer.cpp @@ -7,7 +7,7 @@ namespace AgcDriver::Graphics { void ReadColorTarget(const ColorTarget& target, std::span destination) { PerformanceTimer timing("ColorTarget.Read"); - const ColorTargetLayout layout(target.extent.width, target.extent.height, target.tileMode, target.elementBytes); + const ColorTargetLayout layout(target.extent.width, target.extent.height, target.tileMode, target.elementBytes, target.pipeBankXor); Require(target.bytes == layout.Bytes() && destination.size() == layout.LinearBytes(), "color target transfer size mismatch"); if (target.tileMode == ColorTileMode::Linear && layout.Bytes() == layout.LinearBytes()) { GuestMemory::Read(target.address, destination, layout.Alignment()); @@ -23,7 +23,7 @@ void ReadColorTarget(const ColorTarget& target, std::span destination void WriteColorTarget(const ColorTarget& target, std::span source) { PerformanceTimer timing("ColorTarget.Write"); - const ColorTargetLayout layout(target.extent.width, target.extent.height, target.tileMode, target.elementBytes); + const ColorTargetLayout layout(target.extent.width, target.extent.height, target.tileMode, target.elementBytes, target.pipeBankXor); Require(target.bytes == layout.Bytes() && source.size() == layout.LinearBytes(), "color target transfer size mismatch"); if (target.tileMode == ColorTileMode::Linear && layout.Bytes() == layout.LinearBytes()) { GuestMemory::Write(target.address, source, layout.Alignment()); diff --git a/core/libs/prx/libSceAgcDriver/Graphics/src/DccMetadata.cpp b/core/libs/prx/libSceAgcDriver/Graphics/src/DccMetadata.cpp index 5e4729b1f..69b14ec8f 100644 --- a/core/libs/prx/libSceAgcDriver/Graphics/src/DccMetadata.cpp +++ b/core/libs/prx/libSceAgcDriver/Graphics/src/DccMetadata.cpp @@ -82,6 +82,8 @@ struct ProofCounters { std::atomic proved{0}; std::atomic scanned{0}; std::atomic unstable{0}; + std::atomic rangeProved{0}; + std::atomic rangeScanned{0}; }; ProofCounters& Proofs() { @@ -458,15 +460,18 @@ DccKeys textureClearKeys(const GuestTextureResource& resource, std::uint64_t gue return keys; } -DccKeys currentDccKeys(std::uint64_t metaAddress, std::size_t count) { +DccKeys currentDccKeys(std::uint64_t metaAddress, std::size_t count, bool& memoized) { + memoized = false; if (metaAddress != 0 && count != 0 && GuestMemory::GpuMutex().HeldByThisThread()) { if (auto* recorder = Recorder::Active(); recorder != nullptr && recorder->PendingWriteOverlaps(metaAddress, count)) { - if (const auto keys = PendingStoreKeys(*recorder, metaAddress, count)) return *keys; + if (const auto keys = PendingStoreKeys(*recorder, metaAddress, count)) { + memoized = true; + return *keys; + } Recorder::CountSync(2); recorder->SyncThrough(metaAddress, count); } } - bool memoized = false; return readDccKeys(metaAddress, count, memoized); } @@ -489,12 +494,14 @@ DccKeys ReadDccKeys(std::uint64_t metaAddress, std::uint64_t surfaceBytes) { } DccKeys CurrentDccKeys(std::uint64_t metaAddress, std::uint64_t surfaceBytes) { - return currentDccKeys(metaAddress, DccKeyBytes(surfaceBytes)); + bool memoized = false; + return currentDccKeys(metaAddress, DccKeyBytes(surfaceBytes), memoized); } DccKeys CurrentDccKeys(std::uint64_t metaAddress, std::uint64_t surfaceBytes, std::size_t keyCount) { Require(keyCount >= DccKeyBytes(surfaceBytes), "a DCC key range covers fewer keys than one per 256 surface bytes"); - return currentDccKeys(metaAddress, keyCount); + bool memoized = false; + return currentDccKeys(metaAddress, keyCount, memoized); } bool IsDccClear(DccKeys keys) { @@ -618,7 +625,30 @@ DccKeys ProvedClearKeys(const GuestTextureResource& resource, std::uint64_t gues DccKeyProofCounts KeyProofCounts() { const auto& counters = Proofs(); - return {counters.proved.load(std::memory_order_relaxed), counters.scanned.load(std::memory_order_relaxed), counters.unstable.load(std::memory_order_relaxed)}; + return {counters.proved.load(std::memory_order_relaxed), counters.scanned.load(std::memory_order_relaxed), counters.unstable.load(std::memory_order_relaxed), counters.rangeProved.load(std::memory_order_relaxed), counters.rangeScanned.load(std::memory_order_relaxed)}; +} + +DccKeys ProvedCurrentDccKeys(std::uint64_t metaAddress, std::uint64_t surfaceBytes, DccRangeProof& proof) { + const auto count = DccKeyBytes(surfaceBytes); + if (metaAddress == 0 || count == 0 || !KeyFastPath() || !GuestMemory::GpuMutex().HeldByThisThread()) return CurrentDccKeys(metaAddress, surfaceBytes); + auto& counters = Proofs(); + const auto collected = GuestMemory::CollectWrites(metaAddress, count); + if (proof.generation != 0 && proof.address == metaAddress && proof.count == count && collected != 0 && GuestMemory::UnchangedSince(metaAddress, count, proof.generation)) { + counters.rangeProved.fetch_add(1, std::memory_order_relaxed); + return proof.keys; + } + bool memoized = false; + const auto keys = currentDccKeys(metaAddress, count, memoized); + bool stable = collected != 0 && !memoized; + if (stable) { + if (auto* recorder = Recorder::Active(); recorder != nullptr) { + const auto info = recorder->DescribePendingWrite(metaAddress, count); + stable = !info.has_value() || info->signaled; + } + } + proof = stable ? DccRangeProof{metaAddress, count, keys, collected} : DccRangeProof{}; + counters.rangeScanned.fetch_add(1, std::memory_order_relaxed); + return keys; } void ReadTextureSurface(const GuestTextureResource& resource, DccKeys keys, std::span bytes) { diff --git a/core/libs/prx/libSceAgcDriver/Graphics/src/DepthSurface.cpp b/core/libs/prx/libSceAgcDriver/Graphics/src/DepthSurface.cpp index 33194fdb2..f7ace9d03 100644 --- a/core/libs/prx/libSceAgcDriver/Graphics/src/DepthSurface.cpp +++ b/core/libs/prx/libSceAgcDriver/Graphics/src/DepthSurface.cpp @@ -10,6 +10,7 @@ #include #include #include +#include #include #include @@ -76,6 +77,20 @@ public: throw; } } + + void ClearDepth(float value) { + auto* recorder = Recorder::Active(); + std::unique_ptr batch; + if (recorder == nullptr) batch = std::make_unique(context); + const auto commands = recorder != nullptr ? recorder->Commands() : batch->Handle(); + RecordMemoryBarrier(context, commands, VK_PIPELINE_STAGE_ALL_COMMANDS_BIT, VK_PIPELINE_STAGE_TRANSFER_BIT, VK_ACCESS_DEPTH_STENCIL_ATTACHMENT_READ_BIT | VK_ACCESS_DEPTH_STENCIL_ATTACHMENT_WRITE_BIT | VK_ACCESS_SHADER_READ_BIT, VK_ACCESS_TRANSFER_WRITE_BIT); + const VkClearDepthStencilValue clear{value, 0}; + const VkImageSubresourceRange range{VK_IMAGE_ASPECT_DEPTH_BIT, 0, 1, 0, 1}; + context.Function("vkCmdClearDepthStencilImage")(commands, image, VK_IMAGE_LAYOUT_GENERAL, &clear, 1, &range); + RecordMemoryBarrier(context, commands, VK_PIPELINE_STAGE_TRANSFER_BIT, VK_PIPELINE_STAGE_EARLY_FRAGMENT_TESTS_BIT | VK_PIPELINE_STAGE_LATE_FRAGMENT_TESTS_BIT | VK_PIPELINE_STAGE_FRAGMENT_SHADER_BIT | VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT, VK_ACCESS_TRANSFER_WRITE_BIT, VK_ACCESS_DEPTH_STENCIL_ATTACHMENT_READ_BIT | VK_ACCESS_DEPTH_STENCIL_ATTACHMENT_WRITE_BIT | VK_ACCESS_SHADER_READ_BIT); + if (batch) batch->SubmitAndWait(); + else Recorder::CountBarriers(Recorder::CommandClass::Draw, 2); + } ~DepthSurface() { release(); } DepthSurface(const DepthSurface&) = delete; DepthSurface& operator=(const DepthSurface&) = delete; @@ -139,6 +154,11 @@ std::vector>& surfaces() { return *list; } +std::set& clearedHtiles() { + static auto* cleared = new std::set(); + return *cleared; +} + } std::uint64_t DepthSliceBytes(VkExtent2D extent, std::uint32_t bytesPerTexel) { @@ -151,13 +171,29 @@ std::uint64_t DepthSliceBytes(VkExtent2D extent, std::uint32_t bytesPerTexel) { VkImageView DepthSurfaceView(const Context& context, const DepthTarget& target) { std::lock_guard lock(surfacesMutex()); + const bool cleared = target.htileAddress != 0 && clearedHtiles().erase(target.htileAddress) != 0; for (const auto& surface : surfaces()) { - if (surface->context.device == context.device && sameSurface(surface->target, target)) return surface->view; + if (surface->context.device != context.device || !sameSurface(surface->target, target)) continue; + if (cleared) surface->ClearDepth(target.clearDepth); + return surface->view; } surfaces().push_back(std::make_unique(context, target)); return surfaces().back()->view; } +std::uint64_t HtileDepthClearAddress(std::span code, std::span userData, const std::array& numThreads) { + static constexpr std::array htileMaskKernel{0xd7460000u, 0x04010c06u, 0x34000084u, 0xdc388000u, 0x04020000u, 0xbf8c3f70u, 0xd7710008u, 0x00120805u, 0xd7710009u, 0x00120a05u, 0xd771000au, 0x00120c05u, 0xd771000bu, 0x00120e05u, 0xdc788000u, 0x00020800u, 0xbf810000u}; + if (userData.size() < 6 || numThreads[0] != 64 || numThreads[1] != 1 || numThreads[2] != 1) return 0; + if (code.size() < htileMaskKernel.size() || !std::equal(htileMaskKernel.begin(), htileMaskKernel.end(), code.begin())) return 0; + if ((userData[5] & 0xfu) != 0 || (userData[4] & 0xfu) != 0) return 0; + return userData[2] | (static_cast(userData[3]) << 32u); +} + +void NoteHtileDepthClear(std::uint64_t htileAddress) { + std::lock_guard lock(surfacesMutex()); + clearedHtiles().insert(htileAddress); +} + void ClearDepthSurfaces(VkDevice device) { std::lock_guard lock(surfacesMutex()); std::erase_if(surfaces(), [&](const auto& surface) { return surface->context.device == device; }); @@ -169,7 +205,16 @@ std::shared_ptr DepthSurfaceTexture(const Context& context, std::spancontext.device == context.device && (surface->target.address == resource.baseAddress || (surface->target.stencilAddress != 0 && surface->target.stencilAddress == resource.baseAddress)); }); - return found == list.rend() ? nullptr : (*found)->Sampled(words, resource, components); + if (found == list.rend()) return nullptr; + const auto& target = (*found)->target; + if (resource.width != target.extent.width || resource.height != target.extent.height) return nullptr; + const bool stencil = target.stencilAddress != 0 && resource.baseAddress == target.stencilAddress; + if (!stencil) { + const bool d16 = target.format == VK_FORMAT_D16_UNORM || target.format == VK_FORMAT_D16_UNORM_S8_UINT; + const bool depthBits = words.size() >= 4 && ShaderRecompiler::DepthBitsTextureWidth(words[1], words[3]) == (d16 ? 16u : 32u); + if (ResolveTextureFormat(resource.format) != (d16 ? VK_FORMAT_R16_UNORM : VK_FORMAT_R32_SFLOAT) && !depthBits) return nullptr; + } + return (*found)->Sampled(words, resource, components); } bool DepthSurfaceAt(std::uint64_t address) { diff --git a/core/libs/prx/libSceAgcDriver/Graphics/src/Draw.cpp b/core/libs/prx/libSceAgcDriver/Graphics/src/Draw.cpp index 7e94a8675..fb6a8680e 100644 --- a/core/libs/prx/libSceAgcDriver/Graphics/src/Draw.cpp +++ b/core/libs/prx/libSceAgcDriver/Graphics/src/Draw.cpp @@ -8,6 +8,8 @@ #include "prx/libSceAgcDriver/Graphics/include/ShaderResources.hpp" #include "prx/libSceAgcDriver/Graphics/include/Recorder.hpp" #include "prx/libSceAgcDriver/Graphics/include/TextureFormat.hpp" +#include "prx/libSceAgcDriver/Graphics/include/UnitShadow.hpp" +#include "prx/libSceAgcDriver/Graphics/include/GuestBufferMemory.hpp" #include "prx/libSceAgcDriver/Execution/include/GuestMemory.hpp" #include "prx/libSceAgcDriver/Execution/include/CaptureTrace.hpp" #include "prx/libSceAgcDriver/Execution/include/PerformanceTimer.hpp" @@ -29,10 +31,28 @@ #include #include #include +#include #include namespace AgcDriver::Graphics { +std::uint64_t DrawRenderPassKey(const Context& context, const State& state, std::span targetViews) { + std::uint64_t key = 14695981039346656037ull; + const auto mix = [&](std::uint64_t value) { + key ^= value; + key *= 1099511628211ull; + }; + for (const auto view : targetViews) mix(reinterpret_cast(view)); + if (state.blends.size() != state.colors.size()) { + mix(state.blends.size()); + for (const auto& color : state.colors) mix(color.exportIndex); + } + mix(state.renderExtent.width); + mix(state.renderExtent.height); + if (!context.provokingVertexModePerPipeline) mix(state.provokingVertexMode); + return key; +} + namespace { std::uint32_t GuestFormatFor(VkFormat format, std::uint32_t elementBytes) { @@ -46,6 +66,7 @@ GuestTextureResource SurfaceForTarget(const ColorTarget& color) { const bool chain = color.mipCount > 1; GuestTextureResource surface{}; surface.baseAddress = chain ? color.surfaceAddress : color.address; + surface.pipeBankXor = color.pipeBankXor; surface.width = chain ? color.surfaceExtent.width : color.extent.width; surface.height = chain ? color.surfaceExtent.height : color.extent.height; surface.depthOrLastArray = color.depth - 1u; @@ -113,7 +134,7 @@ void storeClearTexels(const Context& context, const ColorTarget& color, const st void materializeRegisterClear(const Context& context, const ColorTarget& color, StorageTexture& resident) { if (color.dccAddress == 0 || resident.Descriptor().dccAddress != color.dccAddress) return; - if (CurrentDccKeys(color.dccAddress, color.bytes) != DccKeys::ClearRegister) return; + if (ProvedCurrentDccKeys(color.dccAddress, color.bytes, resident.TargetKeyProof()) != DccKeys::ClearRegister) return; const auto texel = clearTexel(color, DccKeys::ClearRegister); const char* refusal = nullptr; bool cleared = clearToTexel(resident, texel, color.elementBytes, refusal); @@ -134,22 +155,156 @@ void materializeRegisterClear(const Context& context, const ColorTarget& color, resident.Refresh(); } -bool materializeCmaskClear(const Context& context, const ColorTarget& color, StorageTexture* resident) { +enum class CmaskState { Expanded, Cleared, Partial }; + +struct CmaskProof { + std::size_t bytes = 0; + std::uint64_t generation = 0; +}; + +std::mutex& cmaskProofMutex() { + static std::mutex mutex; + return mutex; +} + +std::unordered_map& cmaskProofs() { + static std::unordered_map proofs; + return proofs; +} + +bool cmaskProved(std::uint64_t address, std::size_t bytes) { + if (GuestMemory::CollectWrites(address, bytes) == 0) return false; + std::lock_guard lock(cmaskProofMutex()); + const auto found = cmaskProofs().find(address); + return found != cmaskProofs().end() && found->second.bytes == bytes && !GuestMemory::StoredOver(address, bytes, found->second.generation); +} + +void proveCmask(std::uint64_t address, std::size_t bytes, std::uint64_t generation) { + std::lock_guard lock(cmaskProofMutex()); + if (generation == 0) cmaskProofs().erase(address); + else cmaskProofs()[address] = {bytes, generation}; +} + +std::string hexAddress(std::uint64_t address) { + char text[24]; + std::snprintf(text, sizeof(text), "0x%llx", static_cast(address)); + return text; +} + +void requireCmaskCodes(std::uint32_t codes) { + if (codes == 0) return; + Require((codes & 1u) == 0, "a CMASK fast clear of a DCC color target, whose clear belongs in its DCC keys, was recorded"); + std::string listed; + for (std::uint32_t code = 1; code < 15; ++code) { + if ((codes >> code) & 1u) listed += (listed.empty() ? "" : ", ") + std::to_string(code); + } + Require(false, "CMASK codes " + listed + " were recorded, which are not a single-sample fast-clear code (0 cleared, 15 expanded)"); +} + +CmaskState readCmask(const ColorTarget& color, const CmaskLayout& layout, std::vector& cleared) { + const auto address = color.cmaskAddress; + const auto bytes = layout.Bytes(); + const bool locked = GuestMemory::GpuMutex().HeldByThisThread(); + auto* recorder = Recorder::Active(); + if (locked && recorder != nullptr && recorder->PendingWriteOverlaps(address, bytes)) { + Recorder::CountSync(2); + recorder->SyncThrough(address, bytes); + } + if (AnyShadowedOverlaps(address, bytes)) { + Require(locked, "a CMASK under a unit shadow must be read under the device lock"); + PublishShadow(address, bytes, PublishScope::Whole, PublishReason::Keys); + if (recorder != nullptr) { + Recorder::CountSync(2); + recorder->Sync(); + } + } + std::vector mask(bytes); + GuestMemory::Read(address, mask, CmaskLayout::Alignment); + const auto uniform = [&](std::byte value) { return std::all_of(mask.begin(), mask.end(), [&](std::byte entry) { return entry == value; }); }; + if (uniform(std::byte{0xff})) return CmaskState::Expanded; + if (uniform(std::byte{0})) return CmaskState::Cleared; + for (std::uint32_t tileY = 0; tileY < layout.TilesY(); ++tileY) { + for (std::uint32_t tileX = 0; tileX < layout.TilesX(); ++tileX) { + const auto nibble = layout.Nibble(tileX, tileY); + const auto code = (std::to_integer(mask[nibble / 2u]) >> ((nibble % 2u) * 4u)) & 0xfu; + if (code == 0u) cleared.push_back(tileY * layout.TilesX() + tileX); + else Require(code == 0xfu, "CMASK code " + std::to_string(code) + " of color target " + hexAddress(color.address) + " is not a single-sample fast-clear code (0 cleared, 15 expanded)"); + } + } + if (cleared.empty()) return CmaskState::Expanded; + if (cleared.size() == static_cast(layout.TilesX()) * layout.TilesY()) return CmaskState::Cleared; + return CmaskState::Partial; +} + +void storeCmaskClearTexels(const ColorTarget& color, const CmaskLayout& layout, const std::vector* tiles, const std::array& texel) { + StorageTexture::FlushPending(color.address, color.bytes, nullptr, "fast-clear materialization"); + const ColorTargetLayout surface(color.extent.width, color.extent.height, color.tileMode, color.elementBytes); + Require(surface.Bytes() == color.bytes, "CMASK fast clear over a color target whose layout differs from its surface"); + std::vector texels(surface.Bytes()); + GuestMemory::Read(color.address, texels, surface.Alignment()); + const auto fill = [&](std::uint32_t tile) { + const auto x0 = (tile % layout.TilesX()) * 8u; + const auto y0 = (tile / layout.TilesX()) * 8u; + for (auto y = y0; y < std::min(y0 + 8u, color.extent.height); ++y) { + for (auto x = x0; x < std::min(x0 + 8u, color.extent.width); ++x) std::memcpy(texels.data() + surface.Offset(x, y), texel.data(), color.elementBytes); + } + }; + if (tiles != nullptr) { + for (const auto tile : *tiles) fill(tile); + } else { + for (std::uint32_t tile = 0; tile < layout.TilesX() * layout.TilesY(); ++tile) fill(tile); + } + GuestMemory::Write(color.address, texels, surface.Alignment()); +} + +void recordCmaskClear(const Context& context, Recorder& recorder, const HostImport& import, const ColorTarget& color, const CmaskLayout& layout, const std::shared_ptr& resident, VkImageView view) { + const auto address = color.cmaskAddress; + const auto bytes = layout.Bytes(); + StorageTexture::FlushPending(address, bytes, nullptr, "fast-clear materialization"); + if (AnyShadowedOverlaps(address, bytes)) PublishShadow(address, bytes, PublishScope::Whole, PublishReason::Keys); + recorder.FlushKeyStoresOverlapping(address, bytes); + const auto commands = recorder.Commands(); + recorder.Keep(resident); + context.detiler->DispatchCmaskClear(commands, import.buffer, address - import.base, bytes, view, color.extent.width, color.extent.height, color.elementBytes, color.clearWords, color.dccAddress == 0); + recorder.NotePendingWrite(address, bytes); + proveCmask(address, bytes, GuestMemory::MarkWritten(address, bytes)); +} + +bool materializeCmaskClear(const Context& context, const ColorTarget& color, const std::shared_ptr& resident) { if (color.cmaskAddress == 0) return false; - const auto metadataBytes = static_cast(color.cmaskBytes) * 256u; - const auto state = CurrentDccKeys(color.cmaskAddress, metadataBytes); - if (state == DccKeys::Uncompressed) return false; - Require(color.dccAddress == 0, std::string("CMASK of a DCC color target that is not all expanded is not modeled (") + DccKeysName(state) + ")"); - Require(state == DccKeys::Clear0000, std::string("CMASK whose tiles are not all fast-cleared or all expanded is not modeled (") + DccKeysName(state) + ")"); + if (context.detiler != nullptr) requireCmaskCodes(context.detiler->CmaskErrors()); + const CmaskLayout layout(color.extent.width, color.extent.height); + const auto bytes = layout.Bytes(); + Require(color.cmaskBytes == bytes, "CMASK size differs from the color target's CMASK layout"); + if (cmaskProved(color.cmaskAddress, bytes)) return false; + auto* recorder = Recorder::Active(); + if (resident != nullptr && context.detiler != nullptr && recorder != nullptr && GuestMemory::GpuMutex().HeldByThisThread()) { + const auto* import = HostImportFor(context, color.cmaskAddress, bytes); + const auto view = import != nullptr ? resident->ElementView() : VK_NULL_HANDLE; + if (view != VK_NULL_HANDLE) { + recordCmaskClear(context, *recorder, *import, color, layout, resident, view); + return true; + } + } + const auto collected = GuestMemory::CollectWrites(color.cmaskAddress, bytes); + std::vector cleared; + const auto state = readCmask(color, layout, cleared); + if (state == CmaskState::Expanded) { + proveCmask(color.cmaskAddress, bytes, collected); + return false; + } + Require(color.dccAddress == 0, "CMASK fast clear of a DCC color target, whose clear belongs in its DCC keys"); const auto texel = clearTexel(color, DccKeys::ClearRegister); const char* refusal = nullptr; - if (resident == nullptr || !clearToTexel(*resident, texel, color.elementBytes, refusal)) { - StorageTexture::FlushPending(color.address, color.bytes, nullptr, "fast-clear materialization"); - writeTexels(color, texel); + const bool filled = state == CmaskState::Cleared && resident != nullptr && clearToTexel(*resident, texel, color.elementBytes, refusal); + if (!filled) { + storeCmaskClearTexels(color, layout, state == CmaskState::Cleared ? nullptr : &cleared, texel); if (resident != nullptr) resident->Refresh(); } - MarkDccUncompressed(context, color.cmaskAddress, metadataBytes); - return true; + const std::vector expanded(bytes, std::byte{0xff}); + GuestMemory::Write(color.cmaskAddress, expanded, CmaskLayout::Alignment); + proveCmask(color.cmaskAddress, bytes, GuestMemory::CollectWrites(color.cmaskAddress, bytes)); + return filled; } void imageBarrier(const Context& context, VkCommandBuffer commands, VkImage image, VkImageLayout oldLayout, VkImageLayout newLayout, VkPipelineStageFlags sourceStage, VkPipelineStageFlags destinationStage, VkAccessFlags sourceAccess, VkAccessFlags destinationAccess) { @@ -464,7 +619,7 @@ bool ValidationKey(const Context& context, std::span shade for (const auto& shader : shaders) { Require(shader.program != nullptr, "missing compiled shader"); const auto& program = *shader.program; - const bool generated = state.rectList && (shader.stage == Stage::TessellationControl || shader.stage == Stage::TessellationEvaluation); + const bool generated = (state.rectList && (shader.stage == Stage::TessellationControl || shader.stage == Stage::TessellationEvaluation)) || shader.stage == Stage::Geometry; if (!generated && program.PipelineVariantId() == 0) { keyed = false; break; @@ -546,7 +701,7 @@ std::set CachedFragmentOutputs(const Context& context, std::span< } std::set outputs; try { - outputs = ValidateShaders(shaders, state, context.subgroup, context.fragmentShaderBarycentric, context.descriptorIndexing, context.imageInt64Atomics, context.geometryShader, context.sampleRateShading); + outputs = ValidateShaders(shaders, state, context.subgroup, context.fragmentShaderBarycentric, context.descriptorIndexing, context.imageInt64Atomics, context.geometryShader, context.sampleRateShading, context.bufferInt64Atomics); } catch (const std::exception& error) { if (keyed) { std::lock_guard lock(validationMutex()); @@ -581,7 +736,7 @@ bool MovableBuffers() { return enabled; } -ResourceCache::Key DrawResourceKey(const Context& context, std::span shaders, const ColorTarget& target, std::uint64_t indexAddress, std::uint64_t indexBytes, bool ranges) { +ResourceCache::Key DrawResourceKey(const Context& context, std::span shaders, const ColorTarget& target, std::uint32_t colorAttachments, std::uint64_t indexAddress, std::uint64_t indexBytes, bool ranges) { ResourceCache::Key key{0xffffffffu}; const auto append64 = [&](std::uint64_t value) { key.push_back(static_cast(value)); @@ -594,8 +749,9 @@ ResourceCache::Key DrawResourceKey(const Context& context, std::span(part.size())); key.insert(key.end(), part.begin(), part.end()); } + key.push_back(colorAttachments); if (ranges) { - append64(target.address); + append64(target.address | target.pipeBankXor); append64(target.bytes); append64(indexAddress); append64(indexBytes); @@ -753,11 +909,37 @@ DrawInputCopy CopyDrawInput(const Context& context, Recorder* recorder, std::uin return copy; } +DrawInputCopy CopyZeroPaddedDrawInput(const Context& context, std::uint64_t address, std::size_t bytes, std::size_t validBytes) { + Require(validBytes <= bytes, "the valid bytes of a vertex fetch exceed the fetch"); + GuestMemory::FlushGpuWrites(address, bytes); + DrawInputCopy copy; + copy.buffer = std::make_shared(context, bytes, VK_BUFFER_USAGE_VERTEX_BUFFER_BIT); + const auto span = copy.buffer->Bytes(); + GuestMemory::Read(address, span.subspan(0, validBytes), 1); + std::fill(span.begin() + static_cast(validBytes), span.end(), std::byte{0}); + return copy; +} + void KeepDrawInput(Recorder* recorder, std::uint64_t address, const DrawInputCopy& copy, Recorder::SnapshotUse use, std::uint32_t derived) { if (recorder == nullptr || copy.reused || copy.generation == 0 || copy.buffer == nullptr) return; recorder->KeepDrawSnapshot(address, copy.buffer->Bytes().size(), copy.generation, copy.registryGeneration, copy.buffer, use, derived); } +const HostImport* InPlaceDrawInput(const Context& context, std::uint64_t address, std::size_t bytes, std::size_t alignment) { + auto* recorder = context.recorder; + if (recorder == nullptr || bytes == 0) return nullptr; + GuestMemory::FlushGpuWrites(address, bytes); + if (recorder->PendingWriteOverlaps(address, bytes) || recorder->PendingLabelIn(address, bytes) || recorder->QueuedStoreOverlaps(address, bytes)) return nullptr; + const auto overlaps = [&](const auto& writer) { return writer->WritesOverlap(address, bytes); }; + if (context.copiedWriters != nullptr && std::any_of(context.copiedWriters->begin(), context.copiedWriters->end(), overlaps)) return nullptr; + const auto drawWriters = DrawCopiedWriters(); + if (std::any_of(drawWriters->begin(), drawWriters->end(), overlaps)) return nullptr; + const auto* import = HostImportFor(context, address, bytes); + if (import == nullptr || (address - import->base) % alignment != 0) return nullptr; + recorder->NotePendingRead(address, bytes, Recorder::ReadKind::DrawInput); + return import; +} + namespace { // The draw's inputs before its resources (prepareDrawInputs): the validated parameters, the index @@ -768,16 +950,25 @@ struct DrawInputs { bool nothing = false; std::uint64_t indexBytes = 0; std::shared_ptr indices; + VkBuffer indexHandle = VK_NULL_HANDLE; + VkDeviceSize indexOffset = 0; std::uint32_t maxIndex = 0; VertexInputLayout vertexInput; std::vector> vertexBuffers; std::vector vertexHandles; std::vector vertexOffsets; + std::vector> inPlaceReads; std::set fragmentOutputs; VkPipelineStageFlags shaderStages = 0; std::uint32_t meshGroups = 0; }; +std::shared_ptr zeroVertexBuffer(const Context& context, const ShaderRecompiler::VertexAttribute& attribute) { + auto zero = std::make_shared(context, DecodeVertexFormat(attribute).bytes, VK_BUFFER_USAGE_VERTEX_BUFFER_BIT); + std::fill(zero->Bytes().begin(), zero->Bytes().end(), std::byte{0}); + return zero; +} + // Draw's validation, index and vertex phases. With `recipe` the fragment outputs, the pipeline // stages and the vertex input layout are the recipe's (derived from the same compiled stages) // instead of computed. @@ -846,29 +1037,39 @@ DrawInputs prepareDrawInputs(const Context& context, const State& state, const P timer.phase(PhaseValidate); inputs.maxIndex = draw.indexed ? 0u : draw.firstVertex + draw.indexCount - 1u; if (draw.indexed) { - const auto use = draw.indexSize == 2 ? Recorder::SnapshotUse::Index16 : Recorder::SnapshotUse::Index32; - auto copy = CopyDrawInput(context, context.recorder, draw.indexAddress, static_cast(indexBytes), draw.indexSize, use); - std::uint32_t highest = copy.derived; + const bool listTopology = state.topology == VK_PRIMITIVE_TOPOLOGY_POINT_LIST || state.topology == VK_PRIMITIVE_TOPOLOGY_LINE_LIST || state.topology == VK_PRIMITIVE_TOPOLOGY_TRIANGLE_LIST; + const bool fanGeometry = state.stages.mesh && state.stages.mesh->inputPrimitive == 5; + const bool skipRestart = state.primitiveRestart && (!listTopology || context.primitiveListRestart) && !fanGeometry; + const auto use = draw.indexSize == 2 ? (skipRestart ? Recorder::SnapshotUse::Index16Restart : Recorder::SnapshotUse::Index16) : (skipRestart ? Recorder::SnapshotUse::Index32Restart : Recorder::SnapshotUse::Index32); + const auto* import = InPlaceDrawInput(context, draw.indexAddress, static_cast(indexBytes), draw.indexSize); + DrawInputCopy copy; + if (import == nullptr) copy = CopyDrawInput(context, context.recorder, draw.indexAddress, static_cast(indexBytes), draw.indexSize, use); + else GuestMemory::CheckRange(reinterpret_cast(draw.indexAddress), static_cast(indexBytes), draw.indexSize); + std::optional highest; if (!copy.reused) { - highest = 0; - const auto bytes = copy.buffer->Bytes(); - for (std::size_t offset = 0; offset < indexBytes; offset += draw.indexSize) { - std::uint32_t index = 0; - if (draw.indexSize == 2) { - std::uint16_t value = 0; - std::memcpy(&value, bytes.data() + offset, sizeof(value)); - index = value; - } else { - std::memcpy(&index, bytes.data() + offset, sizeof(index)); - } - highest = std::max(highest, index); - } - KeepDrawInput(context.recorder, draw.indexAddress, copy, use, highest); + const auto bytes = import != nullptr ? std::span(reinterpret_cast(draw.indexAddress), static_cast(indexBytes)) : copy.buffer->Bytes().first(static_cast(indexBytes)); + highest = HighestDrawIndex(bytes, draw.indexSize, skipRestart); + if (import == nullptr) KeepDrawInput(context.recorder, draw.indexAddress, copy, use, skipRestart ? (highest ? *highest + 1u : 0u) : highest.value_or(0u)); + } else if (!skipRestart) { + highest = copy.derived; + } else if (copy.derived != 0) { + highest = copy.derived - 1u; + } + if (!highest) { + inputs.nothing = true; + return inputs; + } + Require(*highest <= context.limits.maxDrawIndexedIndexValue, "index exceeds the device's indexed draw limit"); + Require(!fanGeometry || !state.primitiveRestart || *highest != (draw.indexSize == 2 ? 0xffffu : 0xffffffffu), "primitive restart in a triangle fan geometry draw is unsupported"); + inputs.maxIndex = *highest; + if (import != nullptr) { + inputs.indexHandle = import->buffer; + inputs.indexOffset = draw.indexAddress - import->base; + inputs.inPlaceReads.emplace_back(draw.indexAddress, draw.indexAddress + indexBytes); + } else { + inputs.indices = std::move(copy.buffer); + inputs.indexHandle = inputs.indices->Handle(); } - Require(highest <= context.limits.maxDrawIndexedIndexValue, "index exceeds the device's indexed draw limit"); - Require(!state.stages.mesh || state.stages.mesh->inputPrimitive != 5 || !state.primitiveRestart || highest != (draw.indexSize == 2 ? 0xffffu : 0xffffffffu), "primitive restart in a triangle fan geometry draw is unsupported"); - inputs.maxIndex = highest; - inputs.indices = std::move(copy.buffer); } APS5_LOG_CHARS_OUT_DEBUG("Index validation OK"); const auto& attributes = shaders.front().program->vertexAttributes; @@ -882,26 +1083,84 @@ DrawInputs prepareDrawInputs(const Context& context, const State& state, const P inputs.maxIndex += draw.firstVertex; } std::vector fetches; + std::vector fetchValid; fetches.reserve(attributes.size()); - for (const auto& attribute : attributes) { + fetchValid.reserve(attributes.size()); + std::vector fetchOf(attributes.size(), 0); + std::vector> zeroed(attributes.size()); + for (std::size_t i = 0; i < attributes.size(); ++i) { + const auto& attribute = attributes[i]; + if (NullVertexDescriptor(attribute) || VertexFetchOutOfRange(attribute)) { + zeroed[i] = zeroVertexBuffer(context, attribute); + continue; + } // An indirect draw's counts are unknown here: the descriptor's whole range is copied. const auto bytes = args != nullptr ? VertexBufferExtent(attribute) : VertexBufferReadSize(attribute, inputs.maxIndex, draw.instanceCount, draw.firstInstance); const auto& fields = attribute.resource.fields; const auto address = fields[0] | (static_cast(fields[1] & 0xffffu) << 32u); Require(!state.hasColorTarget || address + bytes <= state.color.address || state.color.address + state.color.bytes <= address, "vertex buffer aliases the render target"); - fetches.push_back({address, address + bytes, (fields[1] >> 16u) & 0x3fffu, attribute.fetchIndex, DecodeVertexFormat(attribute).alignment}); + const auto stride = (fields[1] >> 16u) & 0x3fffu; + fetchOf[i] = fetches.size(); + fetches.push_back({address, address + bytes, stride, attribute.fetchIndex, DecodeVertexFormat(attribute).alignment}); + const auto recordBytes = static_cast(fields[2]) * stride; + fetchValid.push_back(args == nullptr && stride != 0 ? static_cast(std::min(recordBytes, static_cast(bytes))) : bytes); } - const auto plan = PlanVertexCopies(fetches); - for (const auto& [begin, end] : plan.copies) { + const auto soloFetches = SoloZeroPaddedFetchIndices(fetches, fetchValid); + std::vector isSolo(fetches.size(), false); + for (const auto f : soloFetches) isSolo[f] = true; + std::vector plannedFetches; + plannedFetches.reserve(fetches.size()); + std::vector plannedOf(fetches.size(), 0); + for (std::size_t f = 0; f < fetches.size(); ++f) { + if (isSolo[f]) continue; + plannedOf[f] = plannedFetches.size(); + plannedFetches.push_back(fetches[f]); + } + const auto plan = PlanVertexCopies(plannedFetches); + std::vector rangeHandles; + std::vector rangeOffsets; + rangeHandles.reserve(plan.copies.size()); + rangeOffsets.reserve(plan.copies.size()); + for (std::size_t range = 0; range < plan.copies.size(); ++range) { + const auto [begin, end] = plan.copies[range]; const auto bytes = static_cast(end - begin); GuestMemory::CheckRange(reinterpret_cast(begin), bytes, 1); + if (const auto* import = InPlaceDrawInput(context, begin, bytes, plan.alignments[range])) { + rangeHandles.push_back(import->buffer); + rangeOffsets.push_back(begin - import->base); + inputs.inPlaceReads.emplace_back(begin, end); + continue; + } auto copy = CopyDrawInput(context, context.recorder, begin, bytes, 1, Recorder::SnapshotUse::Vertex); KeepDrawInput(context.recorder, begin, copy, Recorder::SnapshotUse::Vertex, 0); + rangeHandles.push_back(copy.buffer->Handle()); + rangeOffsets.push_back(0); + inputs.vertexBuffers.push_back(std::move(copy.buffer)); + } + std::vector soloHandles(fetches.size(), VK_NULL_HANDLE); + for (const auto f : soloFetches) { + const auto begin = fetches[f].begin; + const auto bytes = static_cast(fetches[f].end - begin); + auto copy = CopyZeroPaddedDrawInput(context, begin, bytes, fetchValid[f]); + soloHandles[f] = copy.buffer->Handle(); inputs.vertexBuffers.push_back(std::move(copy.buffer)); } for (std::size_t i = 0; i < attributes.size(); ++i) { - inputs.vertexHandles.push_back(inputs.vertexBuffers[plan.copyOf[i]]->Handle()); - inputs.vertexOffsets[i] = plan.offsets[i]; + if (zeroed[i] != nullptr) { + inputs.vertexHandles.push_back(zeroed[i]->Handle()); + inputs.vertexOffsets[i] = 0; + inputs.vertexBuffers.push_back(std::move(zeroed[i])); + continue; + } + const auto f = fetchOf[i]; + if (isSolo[f]) { + inputs.vertexHandles.push_back(soloHandles[f]); + inputs.vertexOffsets[i] = 0; + continue; + } + const auto j = plannedOf[f]; + inputs.vertexHandles.push_back(rangeHandles[plan.copyOf[j]]); + inputs.vertexOffsets[i] = rangeOffsets[plan.copyOf[j]] + plan.offsets[j]; } timer.phase(PhaseVertex); return inputs; @@ -924,7 +1183,7 @@ std::shared_ptr refreshResidentTarget(const Context& context, co std::lock_guard lock(reportMutex); if (reported.insert(color.address).second) std::fprintf(stderr, "[gpu] color target 0x%llx stays non-resident: %s\n", static_cast(color.address), error.what()); } - if (resident != nullptr) materializeCmaskClear(context, color, resident.get()); + if (resident != nullptr) materializeCmaskClear(context, color, resident); if (profile) { const auto lookupUs = std::chrono::duration(std::chrono::steady_clock::now() - lookupStart).count(); ++outcome.targetLookups; @@ -964,9 +1223,9 @@ ResolvedResources resolveDrawResources(const Context& context, const State& stat // The render target and index ranges stay out of the key (see DrawResourceKey); a hit repeats // the alias checks instead. Debug aid: APS5_NO_DRAW_KEY_TRIM=1 keys them as before. static const bool trimKey = std::getenv("APS5_NO_DRAW_KEY_TRIM") == nullptr; - resolved.cacheable = recordable && !noDrawResourceCache && !noTextureCache && std::all_of(shaders.begin(), shaders.end(), [](const CompiledShader& shader) { return shader.program != nullptr && shader.program->PipelineVariantId() != 0; }); + resolved.cacheable = recordable && !noDrawResourceCache && !noTextureCache && std::all_of(shaders.begin(), shaders.end(), [](const CompiledShader& shader) { return shader.program != nullptr && shader.program->PipelineVariantId() != 0; }) && !ShaderResources::NeverReusable(shaders); if (resolved.cacheable) { - resolved.contentKey = DrawResourceKey(context, shaders, state.color, draw.indexAddress, indexBytes, !trimKey); + resolved.contentKey = DrawResourceKey(context, shaders, state.color, static_cast(state.colors.size()), draw.indexAddress, indexBytes, !trimKey); if (auto cached = SharedResourceCache().Find(resolved.contentKey)) { const bool valid = cached->Revalidate(shaders); auto* recorder = Recorder::Active(); @@ -990,7 +1249,7 @@ ResolvedResources resolveDrawResources(const Context& context, const State& stat } timer.phase(PhaseLookup); if (resolved.resources == nullptr) { - resolved.resources = std::make_shared(context, shaders, state.color, draw.indexAddress, static_cast(indexBytes), snapshots); + resolved.resources = std::make_shared(context, shaders, state.color, static_cast(state.colors.size()), draw.indexAddress, static_cast(indexBytes), snapshots); resolved.built = &resolved.resources->Timing(); outcome.addressBased = resolved.resources->HoldsLease(); outcome.kind = outcome.addressBased ? KindBda : KindBuild; @@ -1016,7 +1275,7 @@ struct IndirectRecord { // The draw commands of one draw: the vertex and index buffer binds, then the direct draw, the // GPU-side indirect draw from `argumentBuffer` or the CPU-read records with the driver's rules. void recordDrawCommands(const Context& context, VkCommandBuffer commands, const State& state, const Pm4::DrawParameters& draw, const DrawInputs& inputs, const IndirectRecord* indirect, VkBuffer argumentBuffer, VkDeviceSize argumentOffset) { - if (context.recorder != nullptr) context.recorder->NoteSampledDraw(); + if (context.recorder != nullptr) context.recorder->NoteSampledDraw(commands); const auto* args = indirect != nullptr ? indirect->args : nullptr; if (state.stages.mesh && args != nullptr) { context.Function("vkCmdDrawMeshTasksIndirectEXT")(commands, argumentBuffer, argumentOffset, 1, sizeof(VkDrawMeshTasksIndirectCommandEXT)); @@ -1028,7 +1287,7 @@ void recordDrawCommands(const Context& context, VkCommandBuffer commands, const return; } if (!inputs.vertexHandles.empty()) context.Resolved(&DeviceFunctions::cmdBindVertexBuffers, "vkCmdBindVertexBuffers")(commands, 0, static_cast(inputs.vertexHandles.size()), inputs.vertexHandles.data(), inputs.vertexOffsets.data()); - if (draw.indexed) context.Resolved(&DeviceFunctions::cmdBindIndexBuffer, "vkCmdBindIndexBuffer")(commands, inputs.indices->Handle(), 0, draw.indexSize == 2 ? VK_INDEX_TYPE_UINT16 : VK_INDEX_TYPE_UINT32); + if (draw.indexed) context.Resolved(&DeviceFunctions::cmdBindIndexBuffer, "vkCmdBindIndexBuffer")(commands, inputs.indexHandle, inputs.indexOffset, draw.indexSize == 2 ? VK_INDEX_TYPE_UINT16 : VK_INDEX_TYPE_UINT32); if (args == nullptr) { if (draw.indexed) context.Resolved(&DeviceFunctions::cmdDrawIndexed, "vkCmdDrawIndexed")(commands, draw.indexCount, draw.instanceCount, 0, static_cast(draw.firstVertex), draw.firstInstance); else context.Resolved(&DeviceFunctions::cmdDraw, "vkCmdDraw")(commands, draw.indexCount, draw.instanceCount, draw.firstVertex, draw.firstInstance); @@ -1235,7 +1494,7 @@ void pushDrawConstants(const Pipeline& pipeline, VkCommandBuffer commands, const const auto firstVertex = draw.indirect ? draw.indirect->vertexConstant : draw.firstVertex; const auto firstInstance = draw.indirect ? draw.indirect->instanceConstant : draw.firstInstance; const std::array words{draw.indexCount, firstVertex, firstInstance, draw.indexed ? draw.indexSize : 0u, static_cast(meshArguments), static_cast(meshArguments >> 32u)}; - static_assert(ShaderRecompiler::MeshDrawPushOffsetBytes + ShaderRecompiler::MeshDrawPushBytes == PipelinePushConstantBytes); + static_assert(ShaderRecompiler::MeshDrawPushOffsetBytes + ShaderRecompiler::MeshDrawPushBytes == PipelinePushSlotBytes); std::memcpy(block.data() + ShaderRecompiler::MeshDrawPushOffsetBytes, words.data(), sizeof(words)); pipeline.PushConstants(commands, stages | VK_SHADER_STAGE_MESH_BIT_EXT, block); } @@ -1335,18 +1594,7 @@ void recordDraw(const Context& context, const State& state, const Pm4::DrawParam // within the open batch) and the extent. The previous draw's pass is continued only when this // draw neither reads its attachments (a barrier would be owed, which no pass allows) nor // records anything outside a pass (an indirect draw's argument barrier and scratch copies). - std::uint64_t passKey = 14695981039346656037ull; - const auto mix = [&](std::uint64_t value) { - passKey ^= value; - passKey *= 1099511628211ull; - }; - for (const auto view : record.targetViews) mix(reinterpret_cast(view)); - if (state.blends.size() != state.colors.size()) { - mix(state.blends.size()); - for (const auto& color : state.colors) mix(color.exportIndex); - } - mix(state.renderExtent.width); - mix(state.renderExtent.height); + const auto passKey = DrawRenderPassKey(context, state, record.targetViews); const bool readsTarget = std::any_of(record.targets.begin(), record.targets.end(), [&](const std::shared_ptr& target) { return resources.ReadsImage(target.get()); }); // A queued DCC key store over memory the draw writes or reads in place (unknown for an // address-based build), or over its GPU-side records, lands before it, as before a dispatch @@ -1363,16 +1611,33 @@ void recordDraw(const Context& context, const State& state, const Pm4::DrawParam const auto drawBindings = resources.PrepareDrawBindings(*recorder, record.moved); const bool capture = CaptureInputsEnabled(); const bool meshIndirect = state.stages.mesh && args != nullptr; - const bool continued = !capture && !readsTarget && !gpuIndirect && !meshIndirect && recorder->ContinuesRenderPass(passKey); + const bool addressBased = resources.HoldsLease(); + auto passReads = addressBased ? std::vector>{} : resources.DeviceReads(); + if (gpuIndirect) { + passReads.emplace_back(args->arguments, args->arguments + args->RangeBytes()); + if (args->countIndirect) passReads.emplace_back(args->countAddress, args->countAddress + 4); + } + const auto passImages = resources.StorageImages(); + std::vector> passAttachments; + passAttachments.reserve(record.targets.size()); + for (const auto& target : record.targets) { + if (target != nullptr) passAttachments.emplace_back(target->Image(), true); + } + const PassAccess passAccess{passReads, resources.GpuWrites(), passImages, passAttachments, addressBased || resources.NeedsCompletion(), addressBased || resources.NeedsCompletion()}; + const bool fenced = state.depth.has_value() || !record.proxies.empty(); + const auto start = recorder->StartDrawPass(passKey, capture || readsTarget || gpuIndirect || meshIndirect, fenced, passAccess); + const bool continued = start.continued; outcome.passContinued = continued; outcome.passBegun = !continued; - const auto commands = continued ? recorder->CommandsInRenderPass() : recorder->Commands(); + const auto commands = start.commands; + if (!continued) recorder->PrepareSampleSlot(); if (capture) captureInputs(context, *recorder, commands, resources, drawBindings, record.targets.empty() || record.targets.front() == nullptr ? 0 : record.targets.front()->Descriptor().baseAddress); // The draw's [gputime] class range: from its first barrier to the pass's trailing barrier (a // continued draw lies inside its pass's range). const auto drawTiming = !continued ? recorder->BeginGpuTiming(CommandClass::Draw) : Recorder::NoTiming; if (!continued && Recorder::BarrierValidate()) { auto reads = resources.InPlaceReads(); + reads.insert(reads.end(), inputs.inPlaceReads.begin(), inputs.inPlaceReads.end()); if (gpuIndirect) { reads.emplace_back(args->arguments, args->arguments + args->RangeBytes()); if (args->countIndirect) reads.emplace_back(args->countAddress, args->countAddress + 4); @@ -1393,9 +1658,11 @@ void recordDraw(const Context& context, const State& state, const Pm4::DrawParam proxy->RecordAttachmentProxyLoad(commands, VK_IMAGE_LAYOUT_GENERAL); countBarrier(2); } - const VkMemoryBarrier before{VK_STRUCTURE_TYPE_MEMORY_BARRIER, nullptr, VK_ACCESS_SHADER_WRITE_BIT | VK_ACCESS_TRANSFER_WRITE_BIT | VK_ACCESS_COLOR_ATTACHMENT_WRITE_BIT | VK_ACCESS_DEPTH_STENCIL_ATTACHMENT_WRITE_BIT, VK_ACCESS_COLOR_ATTACHMENT_READ_BIT | VK_ACCESS_COLOR_ATTACHMENT_WRITE_BIT | VK_ACCESS_DEPTH_STENCIL_ATTACHMENT_READ_BIT | VK_ACCESS_DEPTH_STENCIL_ATTACHMENT_WRITE_BIT | VK_ACCESS_INDEX_READ_BIT | VK_ACCESS_VERTEX_ATTRIBUTE_READ_BIT | VK_ACCESS_UNIFORM_READ_BIT | VK_ACCESS_SHADER_READ_BIT | VK_ACCESS_SHADER_WRITE_BIT}; - context.Resolved(&DeviceFunctions::cmdPipelineBarrier, "vkCmdPipelineBarrier")(commands, VK_PIPELINE_STAGE_ALL_COMMANDS_BIT, VK_PIPELINE_STAGE_COLOR_ATTACHMENT_OUTPUT_BIT | VK_PIPELINE_STAGE_EARLY_FRAGMENT_TESTS_BIT | VK_PIPELINE_STAGE_LATE_FRAGMENT_TESTS_BIT | VK_PIPELINE_STAGE_VERTEX_INPUT_BIT | inputs.shaderStages, 0, 1, &before, 0, nullptr, 0, nullptr); - countBarrier(1); + if (start.barrier) { + const VkMemoryBarrier before{VK_STRUCTURE_TYPE_MEMORY_BARRIER, nullptr, VK_ACCESS_SHADER_WRITE_BIT | VK_ACCESS_TRANSFER_WRITE_BIT | VK_ACCESS_COLOR_ATTACHMENT_WRITE_BIT | VK_ACCESS_DEPTH_STENCIL_ATTACHMENT_WRITE_BIT, VK_ACCESS_COLOR_ATTACHMENT_READ_BIT | VK_ACCESS_COLOR_ATTACHMENT_WRITE_BIT | VK_ACCESS_DEPTH_STENCIL_ATTACHMENT_READ_BIT | VK_ACCESS_DEPTH_STENCIL_ATTACHMENT_WRITE_BIT | VK_ACCESS_INDEX_READ_BIT | VK_ACCESS_VERTEX_ATTRIBUTE_READ_BIT | VK_ACCESS_UNIFORM_READ_BIT | VK_ACCESS_SHADER_READ_BIT | VK_ACCESS_SHADER_WRITE_BIT}; + context.Resolved(&DeviceFunctions::cmdPipelineBarrier, "vkCmdPipelineBarrier")(commands, VK_PIPELINE_STAGE_ALL_COMMANDS_BIT, VK_PIPELINE_STAGE_ALL_GRAPHICS_BIT, 0, 1, &before, 0, nullptr, 0, nullptr); + countBarrier(1); + } APS5_LOG_CHARS_OUT_DEBUG("Upload barrier recorded"); if (meshIndirect) { meshArguments = recordMeshArguments(context, commands, recorder, true, state, draw, *record.indirect, countBarrier); @@ -1419,13 +1686,14 @@ void recordDraw(const Context& context, const State& state, const Pm4::DrawParam pushDrawConstants(*record.pipeline, commands, state, draw, shaders, resources, record.pushBytes, record.pushStages, meshArguments != nullptr ? meshArguments->DeviceAddress() : 0); APS5_LOG_CHARS_OUT_DEBUG("Push constants recorded"); recordDrawCommands(context, commands, state, draw, inputs, record.indirect, argumentBuffer, argumentOffset); + if (record.pipeline->SplitsFaces()) { + record.pipeline->ContinueBackFaces(commands, state); + recordDrawCommands(context, commands, state, draw, inputs, record.indirect, argumentBuffer, argumentOffset); + } if (meshArguments != nullptr) recorder->Keep(meshArguments); if (args != nullptr) CountIndirectDraw(record.indirect->path, record.indirect->readMs, rewritten); auto checkRecords = indirectRecordCheck(record.indirect); APS5_LOG_CHARS_OUT_DEBUG("Draw recorded"); - // The pass stays open for the next draw of these attachments; the recorder ends it (and the - // draw class range) before anything else is recorded. A draw that wrote memory owes the next - // one a barrier, so its pass cannot be continued. std::function storeProxies; if (!continued && !record.proxies.empty()) { storeProxies = [proxies = record.proxies](VkCommandBuffer passCommands) { @@ -1435,7 +1703,7 @@ void recordDraw(const Context& context, const State& state, const Pm4::DrawParam } }; } - recorder->LeaveRenderPassOpen(passKey, drawTiming, !resources.WritesMemory(), std::move(storeProxies)); + recorder->LeaveRenderPassOpen(passKey, drawTiming, fenced, passAccess, std::move(storeProxies)); timer.phase(PhaseRecord); keepRecordedDraw(*recorder, record.resources, std::move(record.pipeline), std::move(record.framebuffer), inputs, std::move(record.targets), std::move(scratch), std::move(checkRecords), record.listed, record.completion, outcome); timer.phase(PhaseKeep); @@ -1724,7 +1992,7 @@ void Draw(const Context& context, const State& state, const Pm4::DrawParameters& static const bool drawTransitions = std::getenv("APS5_DRAW_TRANSITIONS") != nullptr; const bool lean = recorded && !drawTransitions; if (!lean && !resolved.moved.empty()) { - resolved.resources = std::make_shared(context, shaders, state.color, draw.indexAddress, static_cast(indexBytes), snapshots); + resolved.resources = std::make_shared(context, shaders, state.color, static_cast(state.colors.size()), draw.indexAddress, static_cast(indexBytes), snapshots); resolved.moved.clear(); } APS5_LOG_CHARS_OUT_DEBUG("Creating Pipeline"); @@ -1777,11 +2045,7 @@ void Draw(const Context& context, const State& state, const Pm4::DrawParameters& recipe->framebuffer = framebuffer; for (const auto& owner : owners) recipe->targets.emplace_back(owner); recipe->targetViews = targetViews; - std::uint64_t passKey = 14695981039346656037ull; - for (const auto view : targetViews) passKey = (passKey ^ reinterpret_cast(view)) * 1099511628211ull; - passKey = (passKey ^ state.renderExtent.width) * 1099511628211ull; - passKey = (passKey ^ state.renderExtent.height) * 1099511628211ull; - recipe->passKey = passKey; + recipe->passKey = DrawRenderPassKey(context, state, targetViews); recipe->vertexInput = inputs.vertexInput; recipe->pushStages = PushConstantStages(shaders); if (recipe->pushStages != 0) { @@ -1820,11 +2084,13 @@ void Draw(const Context& context, const State& state, const Pm4::DrawParameters& // A queued label store over such memory likewise (Recorder::RecordStore). if (recorded && recorder->HasQueuedStores() && (resources->HoldsLease() || recorder->AnyQueuedStore(touches))) recorder->FlushStores(); const auto commands = recorded ? recorder->Commands() : batch->Handle(); + if (recorded) recorder->PrepareSampleSlot(); APS5_LOG_CHARS_OUT_DEBUG("CommandBatch created"); // The draw's [gputime] class range: from its first barrier to the download barrier. const auto drawTiming = recorded ? recorder->BeginGpuTiming(CommandClass::Draw) : Recorder::NoTiming; if (recorded && Recorder::BarrierValidate()) { auto reads = resources->InPlaceReads(); + reads.insert(reads.end(), inputs.inPlaceReads.begin(), inputs.inPlaceReads.end()); if (gpuIndirect) { reads.emplace_back(args->arguments, args->arguments + args->RangeBytes()); if (args->countIndirect) reads.emplace_back(args->countAddress, args->countAddress + 4); @@ -1875,7 +2141,7 @@ void Draw(const Context& context, const State& state, const Pm4::DrawParameters& if (binding.gpuTiling) { CopyBuffer(context, commands, binding.tiled->Handle(), 0, binding.tiledDevice->Handle(), 0, binding.original.size()); memoryBarrier(context, commands, VK_PIPELINE_STAGE_TRANSFER_BIT, VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT, VK_ACCESS_TRANSFER_WRITE_BIT, VK_ACCESS_SHADER_READ_BIT | VK_ACCESS_SHADER_WRITE_BIT); - context.detiler->Dispatch(commands, TextureTileMode::kR64KBX, binding.color.elementBytes, binding.tiledDevice->Handle(), 0, binding.linearDevice->Handle(), 0, binding.mip); + context.detiler->Dispatch(commands, TextureTileMode::kR64KBX, binding.color.elementBytes, binding.tiledDevice->Handle(), 0, binding.linearDevice->Handle(), 0, binding.mip, false, 0, false, {.pipeBankXor = binding.color.pipeBankXor}); memoryBarrier(context, commands, VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT, VK_PIPELINE_STAGE_TRANSFER_BIT, VK_ACCESS_SHADER_WRITE_BIT, VK_ACCESS_TRANSFER_READ_BIT); } imageBarrier(context, commands, binding.target->Image(), VK_IMAGE_LAYOUT_UNDEFINED, VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL, VK_PIPELINE_STAGE_TOP_OF_PIPE_BIT, VK_PIPELINE_STAGE_TRANSFER_BIT, 0, VK_ACCESS_TRANSFER_WRITE_BIT); @@ -1890,11 +2156,16 @@ void Draw(const Context& context, const State& state, const Pm4::DrawParameters& pushDrawConstants(*pipeline, commands, state, draw, shaders, *resources, nullptr, 0, meshArguments != nullptr ? meshArguments->DeviceAddress() : 0); APS5_LOG_CHARS_OUT_DEBUG("Push constants recorded"); recordDrawCommands(context, commands, state, draw, inputs, args != nullptr ? &indirect : nullptr, argumentBuffer, argumentOffset); + if (pipeline->SplitsFaces()) { + pipeline->ContinueBackFaces(commands, state); + recordDrawCommands(context, commands, state, draw, inputs, args != nullptr ? &indirect : nullptr, argumentBuffer, argumentOffset); + } if (meshArguments != nullptr && recorded) recorder->Keep(meshArguments); if (args != nullptr) CountIndirectDraw(indirect.path, indirect.readMs, rewritten); auto checkRecords = indirectRecordCheck(args != nullptr ? &indirect : nullptr); APS5_LOG_CHARS_OUT_DEBUG("Draw recorded"); - context.Resolved(&DeviceFunctions::cmdEndRenderPass, "vkCmdEndRenderPass")(commands); + if (recorded) recorder->EndPassSamples(); + EndRenderPass(context, commands); APS5_LOG_CHARS_OUT_DEBUG("Render pass ended"); for (auto& binding : targets) { if (binding.proxied) { @@ -1931,7 +2202,7 @@ void Draw(const Context& context, const State& state, const Pm4::DrawParameters& CopyBuffer(context, commands, binding.linearDevice->Handle(), 0, binding.dump->Handle(), 0, binding.linearDevice->Size()); } } - context.detiler->Dispatch(commands, TextureTileMode::kR64KBX, binding.color.elementBytes, binding.linearDevice->Handle(), 0, binding.tiledDevice->Handle(), 0, binding.mip, true); + context.detiler->Dispatch(commands, TextureTileMode::kR64KBX, binding.color.elementBytes, binding.linearDevice->Handle(), 0, binding.tiledDevice->Handle(), 0, binding.mip, true, 0, false, {.pipeBankXor = binding.color.pipeBankXor}); memoryBarrier(context, commands, VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT, VK_PIPELINE_STAGE_TRANSFER_BIT, VK_ACCESS_SHADER_WRITE_BIT, VK_ACCESS_TRANSFER_READ_BIT); CopyBuffer(context, commands, binding.tiledDevice->Handle(), 0, binding.tiled->Handle(), 0, binding.original.size()); } @@ -2154,7 +2425,7 @@ void RunColorMetadataPass(const Context& context, const ColorMetadataPass& pass) for (const auto& color : pass.targets) { if (color.cmaskAddress != 0) { const auto resident = metadataPassResident(context, color); - if (materializeCmaskClear(context, color, resident.get()) && resident != nullptr) resident->MarkDirty(); + if (materializeCmaskClear(context, color, resident) && resident != nullptr) resident->MarkDirty(); } if (color.dccAddress == 0) continue; auto keys = CurrentDccKeys(color.dccAddress, color.bytes); diff --git a/core/libs/prx/libSceAgcDriver/Graphics/src/GuestBufferMemory.cpp b/core/libs/prx/libSceAgcDriver/Graphics/src/GuestBufferMemory.cpp index 92b900e7d..8b206eb5d 100644 --- a/core/libs/prx/libSceAgcDriver/Graphics/src/GuestBufferMemory.cpp +++ b/core/libs/prx/libSceAgcDriver/Graphics/src/GuestBufferMemory.cpp @@ -17,9 +17,11 @@ #include #include #include +#include #include #endif #include +#include #include #include #include @@ -29,6 +31,7 @@ #include #include #include +#include #include #include #include @@ -36,6 +39,8 @@ #include #include #include +#include +#include namespace AgcDriver::Graphics { @@ -83,6 +88,17 @@ struct GuestBufferMemory::AddressSpace { std::uint64_t serial = 0; GuestAllocations::Lease lease; std::vector base; + struct HeapRun { + std::uint64_t begin; + std::uint64_t bytes; + std::size_t first; + std::size_t last; + }; + bool mirrored = false; + std::vector writableMirrors; + std::vector heapMirrors; + std::vector heapRuns; + std::unique_ptr[]> verified; std::vector copied; // The BDA table entries of `base`, in its order. std::vector ranges; @@ -154,11 +170,20 @@ void destroyImport(const Context& context, const HostImport& entry) { // Frees a dropped import's Vulkan objects when it is released. Never copied: a copy would destroy the // same handles twice (and a temporary would destroy them at once). struct RetiredImport { - RetiredImport(const Context& context, const HostImport& entry) : context(context), entry(entry) {} + RetiredImport(VkDevice device, PFN_vkDestroyBuffer destroyBuffer, PFN_vkFreeMemory freeMemory, const HostImport& entry) + : device(device), destroyBuffer(destroyBuffer), freeMemory(freeMemory), entry(entry) {} RetiredImport(const RetiredImport&) = delete; RetiredImport& operator=(const RetiredImport&) = delete; - ~RetiredImport() { destroyImport(context, entry); } - Context context; + ~RetiredImport() { + destroyBuffer(device, entry.buffer, nullptr); + freeMemory(device, entry.memory, nullptr); +#ifdef _WIN32 + GuestArena::GuestArenaUnmapAlias_nid_postfix(entry.alias); +#endif + } + VkDevice device; + PFN_vkDestroyBuffer destroyBuffer; + PFN_vkFreeMemory freeMemory; HostImport entry; }; @@ -167,12 +192,12 @@ struct RetiredImport { // used it was synchronous and it is destroyed at once. const GuestAllocations::Range* containingRange(const GuestAllocations::Lease& lease, std::uint64_t begin, std::uint64_t end); -void retireImport(const Context& context, HostImports& state, std::map::iterator it, const GuestAllocations::Lease& lease) { +void retireImport(VkDevice device, HostImports& state, std::map::iterator it, const GuestAllocations::Lease& lease) { // Results shadowed for the import reach its (old) buffer first where the memory is still a // readable registered range (the import retires because its registration vanished or changed // size); the holder below outlives the batch that copies them. - RetireShadow(context, it->second, [&lease](std::uint64_t begin, std::uint64_t end) { return containingRange(lease, begin, end) != nullptr; }); - auto holder = std::make_shared(context, it->second); + RetireShadow(device, it->second, [&lease](std::uint64_t begin, std::uint64_t end) { return containingRange(lease, begin, end) != nullptr; }); + auto holder = std::make_shared(state.device, state.destroyBuffer, state.freeMemory, it->second); if (auto* recorder = Recorder::Active(); recorder != nullptr && !recorder->Idle()) recorder->Keep(std::move(holder)); ++state.epoch; state.imports.erase(it); @@ -192,7 +217,7 @@ const char* bindImport(const Context& context, HostImport& entry, VkExternalMemo VkBufferCreateInfo info{VK_STRUCTURE_TYPE_BUFFER_CREATE_INFO, &external}; info.size = entry.bytes; // INDIRECT_BUFFER: DISPATCH_INDIRECT group counts are read in place (VulkanDevice::DispatchIndirect). - info.usage = VK_BUFFER_USAGE_STORAGE_BUFFER_BIT | VK_BUFFER_USAGE_SHADER_DEVICE_ADDRESS_BIT | VK_BUFFER_USAGE_TRANSFER_SRC_BIT | VK_BUFFER_USAGE_TRANSFER_DST_BIT | VK_BUFFER_USAGE_INDIRECT_BUFFER_BIT; + info.usage = VK_BUFFER_USAGE_STORAGE_BUFFER_BIT | VK_BUFFER_USAGE_SHADER_DEVICE_ADDRESS_BIT | VK_BUFFER_USAGE_TRANSFER_SRC_BIT | VK_BUFFER_USAGE_TRANSFER_DST_BIT | VK_BUFFER_USAGE_INDIRECT_BUFFER_BIT | VK_BUFFER_USAGE_VERTEX_BUFFER_BIT | VK_BUFFER_USAGE_INDEX_BUFFER_BIT; info.sharingMode = VK_SHARING_MODE_EXCLUSIVE; if (const auto result = context.Function("vkCreateBuffer")(context.device, &info, nullptr, &entry.buffer); result != VK_SUCCESS) return failed("vkCreateBuffer", result); VkMemoryRequirements requirements{}; @@ -226,7 +251,15 @@ const char* createHostPointerImport(const Context& context, HostImport& entry, V #ifndef _WIN32 int udmabufDevice() { - static const int device = open("/dev/udmabuf", O_RDWR | O_CLOEXEC); + static const int device = [] { + const int result = open("/dev/udmabuf", O_RDWR | O_CLOEXEC); + if (result < 0) { + const int error = errno; + std::fprintf(stderr, "[gpu] open /dev/udmabuf: %s; shared direct memory is copied instead of imported, and GPU stores through FLAT/GLOBAL addresses do not reach it%s\n", + std::strerror(error), error == EACCES ? " (give the user read-write access to /dev/udmabuf, for example through the kvm group)" : ""); + } + return result; + }(); return device; } @@ -258,12 +291,165 @@ const char* createDmaBufImport(const Context& context, HostImport& entry, int fi } #endif +#ifndef _WIN32 +constexpr std::uint64_t ImportChunkBytes = std::uint64_t{64} << 20u; + +struct ImportChunk { + VkDevice device = VK_NULL_HANDLE; + PFN_vkFreeMemory freeMemory = nullptr; + VkDeviceMemory memory = VK_NULL_HANDLE; + std::uint32_t memoryType = 0; + ImportChunk() = default; + ImportChunk(const ImportChunk&) = delete; + ImportChunk& operator=(const ImportChunk&) = delete; + ~ImportChunk() { + if (memory != VK_NULL_HANDLE) freeMemory(device, memory, nullptr); + } +}; + +struct ImportChunks { + std::mutex mutex; + VkDevice device = VK_NULL_HANDLE; + std::map, std::weak_ptr> chunks; +}; + +ImportChunks& Chunks() { + static ImportChunks chunks; + return chunks; +} + +std::shared_ptr importChunk(const Context& context, int file, std::uint64_t index, std::uint32_t bufferTypes, const char*& step, VkResult& failure) { + struct stat info {}; + if (fstat(file, &info) != 0) { + failure = VK_ERROR_INVALID_EXTERNAL_HANDLE; + step = "fstat of the shared backing"; + return nullptr; + } + auto& chunks = Chunks(); + std::lock_guard lock(chunks.mutex); + if (chunks.device != context.device) { + chunks.chunks.clear(); + chunks.device = context.device; + } + const auto key = std::make_tuple(info.st_dev, info.st_ino, index); + if (const auto found = chunks.chunks.find(key); found != chunks.chunks.end()) { + if (auto chunk = found->second.lock()) return chunk; + chunks.chunks.erase(found); + } + if (static_cast(info.st_size) < (index + 1) * ImportChunkBytes) { + failure = VK_ERROR_INVALID_EXTERNAL_HANDLE; + step = "a chunk past the end of the shared backing"; + return nullptr; + } + udmabuf_create request{}; + request.memfd = static_cast(file); + request.flags = UDMABUF_FLAGS_CLOEXEC; + request.offset = index * ImportChunkBytes; + request.size = ImportChunkBytes; + const int device = udmabufDevice(); + const int buffer = device < 0 ? -1 : ioctl(device, UDMABUF_CREATE, &request); + if (buffer < 0) { + failure = VK_ERROR_INVALID_EXTERNAL_HANDLE; + step = device < 0 ? "open /dev/udmabuf" : "UDMABUF_CREATE"; + return nullptr; + } + VkMemoryFdPropertiesKHR properties{VK_STRUCTURE_TYPE_MEMORY_FD_PROPERTIES_KHR}; + if (const auto result = context.Function("vkGetMemoryFdPropertiesKHR")(context.device, VK_EXTERNAL_MEMORY_HANDLE_TYPE_DMA_BUF_BIT_EXT, buffer, &properties); result != VK_SUCCESS) { + close(buffer); + failure = result; + step = "vkGetMemoryFdPropertiesKHR"; + return nullptr; + } + const auto types = properties.memoryTypeBits & bufferTypes; + if (types == 0) { + close(buffer); + failure = VK_ERROR_FORMAT_NOT_SUPPORTED; + step = "memory type selection"; + return nullptr; + } + const VkImportMemoryFdInfoKHR import{VK_STRUCTURE_TYPE_IMPORT_MEMORY_FD_INFO_KHR, nullptr, VK_EXTERNAL_MEMORY_HANDLE_TYPE_DMA_BUF_BIT_EXT, buffer}; + const VkMemoryAllocateFlagsInfo flags{VK_STRUCTURE_TYPE_MEMORY_ALLOCATE_FLAGS_INFO, &import, VK_MEMORY_ALLOCATE_DEVICE_ADDRESS_BIT, 0}; + VkMemoryAllocateInfo allocation{VK_STRUCTURE_TYPE_MEMORY_ALLOCATE_INFO, &flags}; + allocation.allocationSize = ImportChunkBytes; + allocation.memoryTypeIndex = static_cast(std::countr_zero(types)); + auto chunk = std::make_shared(); + if (const auto result = context.Function("vkAllocateMemory")(context.device, &allocation, nullptr, &chunk->memory); result != VK_SUCCESS) { + close(buffer); + failure = result; + step = "vkAllocateMemory"; + return nullptr; + } + chunk->device = context.device; + chunk->freeMemory = context.Function("vkFreeMemory"); + chunk->memoryType = allocation.memoryTypeIndex; + for (auto it = chunks.chunks.begin(); it != chunks.chunks.end();) it = it->second.expired() ? chunks.chunks.erase(it) : std::next(it); + chunks.chunks[key] = chunk; + return chunk; +} + +const char* createChunkedDmaBufImport(const Context& context, HostImport& entry, int file, std::uint64_t offset, bool& fits, VkResult& failure) { + const auto index = offset / ImportChunkBytes; + const auto within = offset - index * ImportChunkBytes; + fits = entry.bytes != 0 && within + entry.bytes <= ImportChunkBytes; + if (!fits) return nullptr; + const VkExternalMemoryBufferCreateInfo external{VK_STRUCTURE_TYPE_EXTERNAL_MEMORY_BUFFER_CREATE_INFO, nullptr, VK_EXTERNAL_MEMORY_HANDLE_TYPE_DMA_BUF_BIT_EXT}; + VkBufferCreateInfo info{VK_STRUCTURE_TYPE_BUFFER_CREATE_INFO, &external}; + info.size = entry.bytes; + info.usage = VK_BUFFER_USAGE_STORAGE_BUFFER_BIT | VK_BUFFER_USAGE_SHADER_DEVICE_ADDRESS_BIT | VK_BUFFER_USAGE_TRANSFER_SRC_BIT | VK_BUFFER_USAGE_TRANSFER_DST_BIT | VK_BUFFER_USAGE_INDIRECT_BUFFER_BIT; + info.sharingMode = VK_SHARING_MODE_EXCLUSIVE; + VkBuffer buffer = VK_NULL_HANDLE; + if (const auto result = context.Function("vkCreateBuffer")(context.device, &info, nullptr, &buffer); result != VK_SUCCESS) { + failure = result; + return "vkCreateBuffer"; + } + const auto destroy = [&] { context.Function("vkDestroyBuffer")(context.device, buffer, nullptr); }; + VkMemoryRequirements requirements{}; + context.Function("vkGetBufferMemoryRequirements")(context.device, buffer, &requirements); + if (requirements.alignment == 0 || within % requirements.alignment != 0 || within + requirements.size > ImportChunkBytes) { + destroy(); + fits = false; + return nullptr; + } + const char* step = nullptr; + auto chunk = importChunk(context, file, index, requirements.memoryTypeBits, step, failure); + if (chunk == nullptr) { + destroy(); + return step; + } + if (((requirements.memoryTypeBits >> chunk->memoryType) & 1u) == 0) { + destroy(); + fits = false; + return nullptr; + } + if (const auto result = context.Function("vkBindBufferMemory")(context.device, buffer, chunk->memory, within); result != VK_SUCCESS) { + destroy(); + failure = result; + return "vkBindBufferMemory"; + } + const VkBufferDeviceAddressInfo addressInfo{VK_STRUCTURE_TYPE_BUFFER_DEVICE_ADDRESS_INFO, nullptr, buffer}; + entry.buffer = buffer; + entry.memory = VK_NULL_HANDLE; + entry.address = context.Function("vkGetBufferDeviceAddressKHR")(context.device, &addressInfo); + entry.chunk = std::move(chunk); + entry.dmaBuf = true; + return nullptr; +} +#endif + const char* createImport(const Context& context, HostImport& entry, VkResult& failure) { const char* step = createHostPointerImport(context, entry, failure); #ifndef _WIN32 int file = -1; std::uint64_t offset = 0; - if (step != nullptr && context.dmaBufImport && GuestArena::GuestArenaSharedBacking_nid_postfix(static_cast(entry.base), static_cast(entry.bytes), &file, &offset)) step = createDmaBufImport(context, entry, file, offset, failure); + if (step != nullptr && context.dmaBufImport && GuestArena::GuestArenaSharedBacking_nid_postfix(static_cast(entry.base), static_cast(entry.bytes), &file, &offset)) { + bool fits = false; + VkResult chunkFailure = VK_SUCCESS; + if (createChunkedDmaBufImport(context, entry, file, offset, fits, chunkFailure) == nullptr && fits) { + close(file); + return nullptr; + } + step = createDmaBufImport(context, entry, file, offset, failure); + } #endif return step; } @@ -340,7 +526,7 @@ bool sameRange(const HostImport& entry, const GuestAllocations::Lease& lease) { const HostImport* importAllocation(const Context& context, HostImports& state, std::uint64_t base, std::uint64_t bytes, const GuestAllocations::Lease& lease) { if (const auto found = state.imports.find(base); found != state.imports.end()) { if (found->second.bytes == bytes && sameRange(found->second, lease)) return &found->second; - retireImport(context, state, found, lease); + retireImport(context.device, state, found, lease); } const auto alignment = context.hostImportAlignment; if (alignment == 0 || base % alignment != 0 || bytes % alignment != 0 || state.failed.contains(base)) return nullptr; @@ -490,6 +676,8 @@ void refreshImports(const Context& context, HostImports& state, const GuestAlloc state.refreshedGeneration = 0; ++state.epoch; } + if (state.destroyBuffer == nullptr) state.destroyBuffer = context.Function("vkDestroyBuffer"); + if (state.freeMemory == nullptr) state.freeMemory = context.Function("vkFreeMemory"); const auto generation = GuestAllocations::GuestAllocationsGeneration_nid_postfix(); if (generation == state.refreshedGeneration) return; state.refreshedGeneration = generation; @@ -503,7 +691,7 @@ void refreshImports(const Context& context, HostImports& state, const GuestAlloc static const bool trace = std::getenv("APS5_TRACE_HOST_IMPORT") != nullptr; if (trace && range != nullptr && range->bytes == it->second.bytes) std::fprintf(stderr, "[gpu] host import of 0x%llx+0x%llx retired: the range was mapped again\n", static_cast(it->first), static_cast(it->second.bytes)); const auto next = std::next(it); - retireImport(context, state, it, lease); + retireImport(context.device, state, it, lease); it = next; } for (auto it = state.failed.begin(); it != state.failed.end();) it = leasedRangeAt(lease, *it) != nullptr ? std::next(it) : state.failed.erase(it); @@ -536,6 +724,8 @@ struct ImageMirrors { std::uint64_t builds = 0; std::uint64_t heapRefills = 0; std::uint64_t heapUnwatched = 0; + std::uint64_t heapImported = 0; + std::uint64_t heapImportedBytes = 0; std::uint64_t subranges = 0; std::uint64_t rebuilds = 0; std::uint64_t refreshes = 0; @@ -544,6 +734,11 @@ struct ImageMirrors { // Refreshes that had to wait for recorded work writing the range. std::uint64_t syncs = 0; std::uint64_t serials = 0; + std::uint64_t version = 0; + std::uint64_t pinnedSpace = 0; + std::uint64_t pinnedVersion = 0; + std::uint64_t sweeps = 0; + std::uint64_t heapChecks = 0; std::chrono::steady_clock::time_point lastReport = std::chrono::steady_clock::now(); }; @@ -597,12 +792,26 @@ std::uint64_t heapMirrorBudget() { return bytes; } -[[noreturn]] void heapMirrorFatal(const GuestAllocations::Range& range, std::uint64_t held, const char* reason) { - std::fprintf(stderr, "FATAL: heap mirror of 0x%llx+0x%llx: %s; heap mirrors hold %llu MiB of %llu MiB (APS5_HEAP_MIRROR_MIB)\n", static_cast(range.address), static_cast(range.bytes), reason, static_cast(held >> 20u), static_cast(heapMirrorBudget() >> 20u)); - std::fflush(stderr); - std::abort(); +std::string heapMirrorError(const GuestAllocations::Range& range, std::uint64_t held, std::string_view reason) { + constexpr std::string_view prefix = "AGC graphics: "; + if (reason.starts_with(prefix)) reason.remove_prefix(prefix.size()); + char where[64]; + std::snprintf(where, sizeof(where), "heap mirror of 0x%llx+0x%llx: ", static_cast(range.address), static_cast(range.bytes)); + char holding[96]; + std::snprintf(holding, sizeof(holding), "; heap mirrors hold %llu MiB of %llu MiB (APS5_HEAP_MIRROR_MIB)", static_cast(held >> 20u), static_cast(heapMirrorBudget() >> 20u)); + return std::string(prefix) + where + std::string(reason) + holding; } +std::uint64_t heapMirrorsBeside(const GuestAllocations::Range& range) { + auto& state = Mirrors(); + std::lock_guard lock(state.mutex); + std::uint64_t held = state.heapBytes; + if (const auto found = state.entries.find(range.address); found != state.entries.end() && found->second->heap) held -= found->second->bytes; + return held; +} + +void sweepMirrors(); + bool sameRange(const std::weak_ptr& mirrored, const std::shared_ptr& range) { return !mirrored.owner_before(range) && !range.owner_before(mirrored); } @@ -778,11 +987,30 @@ void refreshMirror(ImageMirror& mirror, std::uint64_t address, std::uint64_t byt compareBlocks(blocks); } -void refreshHeapMirrors(std::vector& mirrors, std::vector& blocks) { +void appendChangedBlocks(std::span mirrors, std::uint64_t generation, std::vector& blocks) { constexpr std::uint64_t block = 65536; - std::sort(mirrors.begin(), mirrors.end(), [](const ImageMirror* left, const ImageMirror* right) { return left->base < right->base; }); thread_local std::vector* changedSlot = nullptr; auto& changed = ShaderRecompiler::ThreadOwned(changedSlot); + Mirrors().heapChecks += mirrors.size(); + for (auto* const pointer : mirrors) { + auto& mirror = *pointer; + changed.assign(mirror.generations.size(), 0); + GuestMemory::ChangedBlocks(mirror.base, static_cast(mirror.bytes), mirror.generations, changed); + const auto aligned = mirror.base / block * block; + const auto end = mirror.base + mirror.bytes; + bool refilled = false; + for (std::size_t at = 0; at < changed.size(); ++at) { + if (changed[at] == 0) continue; + const auto from = std::max(mirror.base, aligned + at * block); + blocks.push_back({&mirror, from, static_cast(std::min(end, aligned + (at + 1) * block) - from), generation}); + refilled = true; + } + if (refilled) ++Mirrors().heapRefills; + } +} + +void refreshHeapMirrors(std::vector& mirrors, std::vector& blocks) { + std::sort(mirrors.begin(), mirrors.end(), [](const ImageMirror* left, const ImageMirror* right) { return left->base < right->base; }); for (std::size_t first = 0; first < mirrors.size();) { auto last = first + 1; while (last < mirrors.size() && mirrors[last]->base == mirrors[last - 1]->base + mirrors[last - 1]->bytes) ++last; @@ -791,25 +1019,45 @@ void refreshHeapMirrors(std::vector& mirrors, std::vector(bytes)); Require(generation != 0, "a heap mirror's range is no longer write-watched"); - for (auto index = first; index < last; ++index) { - auto& mirror = *mirrors[index]; - changed.assign(mirror.generations.size(), 0); - GuestMemory::ChangedBlocks(mirror.base, static_cast(mirror.bytes), mirror.generations, changed); - const auto aligned = mirror.base / block * block; - const auto end = mirror.base + mirror.bytes; - bool refilled = false; - for (std::size_t at = 0; at < changed.size(); ++at) { - if (changed[at] == 0) continue; - const auto from = std::max(mirror.base, aligned + at * block); - blocks.push_back({&mirror, from, static_cast(std::min(end, aligned + (at + 1) * block) - from), generation}); - refilled = true; - } - if (refilled) ++Mirrors().heapRefills; - } + appendChangedBlocks(std::span(mirrors).subspan(first, last - first), generation, blocks); first = last; } } +void planSpaceMirrors(GuestBufferMemory::AddressSpace& space) { + for (std::size_t index = 0; index < space.base.size(); ++index) { + const auto& region = space.base[index]; + if (region.mirror == nullptr) continue; + space.mirrored = true; + if (!region.mirror->heap) { + if (region.mirror->writable) space.writableMirrors.push_back(index); + continue; + } + auto* mirror = region.mirror.get(); + if (space.heapRuns.empty() || mirror->base != space.heapMirrors.back()->base + space.heapMirrors.back()->bytes) space.heapRuns.push_back({mirror->base, 0, space.heapMirrors.size(), 0}); + space.heapMirrors.push_back(mirror); + auto& run = space.heapRuns.back(); + run.bytes = mirror->base + mirror->bytes - run.begin; + run.last = space.heapMirrors.size(); + } + space.verified = std::make_unique[]>(space.heapRuns.size()); +} + +std::vector> refreshHeapRuns(const GuestBufferMemory::AddressSpace& space, std::vector& blocks) { + std::vector> checked; + checked.reserve(space.heapRuns.size()); + for (std::size_t index = 0; index < space.heapRuns.size(); ++index) { + const auto& run = space.heapRuns[index]; + prepareRange(run.begin, run.bytes); + const auto generation = GuestMemory::CollectWrites(run.begin, static_cast(run.bytes)); + Require(generation != 0, "a heap mirror's range is no longer write-watched"); + const auto since = space.verified[index].load(std::memory_order_relaxed); + if (since == 0 || !GuestMemory::UnchangedSince(run.begin, static_cast(run.bytes), since)) appendChangedBlocks(std::span(space.heapMirrors).subspan(run.first, run.last - run.first), generation, blocks); + checked.emplace_back(index, generation); + } + return checked; +} + // The mirror for a leased image range: the existing one, or a new one when none exists or the // registered Range object changed (a protection change). Null when the range cannot be mirrored. An // existing writable mirror's blocks are appended to `blocks` for the caller's one compare of every @@ -823,6 +1071,7 @@ std::shared_ptr acquireMirror(const Context& context, const std::sh state.failed.clear(); state.heapBytes = 0; state.device = context.device; + ++state.version; } if (state.failed.contains(range->address)) return nullptr; const auto found = state.entries.find(range->address); @@ -844,19 +1093,20 @@ std::shared_ptr acquireMirror(const Context& context, const std::sh const auto generation = GuestMemory::CollectWrites(range->address, range->bytes); constexpr std::uint64_t block = 65536; mirror->generations.assign(static_cast(((range->address + range->bytes + block - 1) / block) - range->address / block), generation); - std::lock_guard lock(state.mutex); if (generation == 0) { + std::lock_guard lock(state.mutex); ++state.heapUnwatched; return nullptr; } - std::uint64_t held = state.heapBytes; - if (const auto found = state.entries.find(range->address); found != state.entries.end() && found->second->heap) held -= found->second->bytes; - if (held + range->bytes > heapMirrorBudget()) heapMirrorFatal(*range, held, "past the budget"); + if (heapMirrorsBeside(*range) + range->bytes > heapMirrorBudget()) { + sweepMirrors(); + if (const auto held = heapMirrorsBeside(*range); held + range->bytes > heapMirrorBudget()) throw std::runtime_error(heapMirrorError(*range, held, "past the budget")); + } } try { mirror->buffer = std::make_shared(context, range->bytes, VK_BUFFER_USAGE_STORAGE_BUFFER_BIT | VK_BUFFER_USAGE_SHADER_DEVICE_ADDRESS_BIT); } catch (const std::runtime_error& error) { - if (heap) heapMirrorFatal(*range, state.heapBytes, error.what()); + if (heap) throw std::runtime_error(heapMirrorError(*range, heapMirrorsBeside(*range), error.what())); std::fprintf(stderr, "[gpu] image mirror of 0x%llx+0x%llx failed: %s; falling back to copies\n", static_cast(range->address), static_cast(range->bytes), error.what()); std::lock_guard lock(state.mutex); state.failed.insert(range->address); @@ -876,6 +1126,7 @@ std::shared_ptr acquireMirror(const Context& context, const std::sh if (const auto found = state.entries.find(range->address); found != state.entries.end() && found->second->heap) state.heapBytes -= found->second->bytes; if (heap) state.heapBytes += mirror->bytes; state.entries[range->address] = mirror; + ++state.version; return mirror; } @@ -893,15 +1144,44 @@ std::shared_ptr findMirror(const Context& context, std::uint64_t be // Drops mirrors whose registered range is gone (a later build made a new one for the replacement). void sweepMirrors() { + const auto space = Spaces().current.load(); auto& state = Mirrors(); std::lock_guard lock(state.mutex); + if (space != nullptr && state.pinnedSpace == space->serial && state.pinnedVersion == state.version) return; + ++state.sweeps; + bool pinned = space != nullptr; + std::size_t leased = 0; for (auto it = state.entries.begin(); it != state.entries.end();) { if (!it->second->range.expired()) { + if (pinned) { + while (leased < space->lease.size() && space->lease[leased]->address < it->first) ++leased; + pinned = leased < space->lease.size() && sameRange(it->second->range, space->lease[leased]); + } ++it; continue; } if (it->second->heap) state.heapBytes -= it->second->bytes; it = state.entries.erase(it); + ++state.version; + } + state.pinnedSpace = pinned ? space->serial : 0; + state.pinnedVersion = state.version; +} + +void releaseImportedMirrors(const Context& context, const std::vector& imported) { + if (imported.empty()) return; + std::vector> released; + auto& state = Mirrors(); + std::lock_guard lock(state.mutex); + if (state.device != context.device || state.entries.empty()) return; + for (const auto base : imported) { + const auto found = state.entries.find(base); + if (found == state.entries.end() || !found->second->heap) continue; + state.heapBytes -= found->second->bytes; + ++state.heapImported; + state.heapImportedBytes += found->second->bytes; + released.push_back(std::move(found->second)); + state.entries.erase(found); } } @@ -927,7 +1207,7 @@ void reportMirrors() { } heapBytes = state.heapBytes; } - AgcDriver::ProfilePrint_nid_no_patch("[buffers] image mirrors: %zu ranges (%.1f MiB, %zu writable, %zu heap %.1f MiB), %llu address-based builds served, %llu descriptor sub-ranges bound, %llu rebuilds, %llu refreshes: %llu blocks compared, %llu copied, %llu refresh syncs; heap refills %llu, unwatched heaps %llu\n", count, bytes / 1048576.0, writable, heaps, heapBytes / 1048576.0, static_cast(state.builds), static_cast(state.subranges), static_cast(state.rebuilds), static_cast(state.refreshes), static_cast(state.blocksCompared), static_cast(state.blocksCopied), static_cast(state.syncs), static_cast(state.heapRefills), static_cast(state.heapUnwatched)); + AgcDriver::ProfilePrint_nid_no_patch("[buffers] image mirrors: %zu ranges (%.1f MiB, %zu writable, %zu heap %.1f MiB), %llu address-based builds served, %llu descriptor sub-ranges bound, %llu rebuilds, %llu refreshes: %llu blocks compared, %llu copied, %llu refresh syncs; heap refills %llu, unwatched heaps %llu, %llu heap mirrors released for imports (%.1f MiB)\n", count, bytes / 1048576.0, writable, heaps, heapBytes / 1048576.0, static_cast(state.builds), static_cast(state.subranges), static_cast(state.rebuilds), static_cast(state.refreshes), static_cast(state.blocksCompared), static_cast(state.blocksCopied), static_cast(state.syncs), static_cast(state.heapRefills), static_cast(state.heapUnwatched), static_cast(state.heapImported), state.heapImportedBytes / 1048576.0); } // Deferred lease release. The lease an address-based build takes (AcquireRegistered) is dropped by its @@ -973,10 +1253,11 @@ SnapshotStats& Snapshots() { return stats; } -bool WaitForLeases() noexcept { +bool WaitForLeasesAndImports(std::uintptr_t address, std::size_t bytes) noexcept { const auto start = std::chrono::steady_clock::now(); bool synced = false; bool drained = false; + bool retiredImport = false; // A range pinned only by the cached address space is released by dropping the cache's // reference: no GPU wait, no device lock (the mutating thread may be a driver thread holding // it). The registry rescans; a range still pinned by a build or a batch holding the space comes @@ -993,7 +1274,40 @@ bool WaitForLeases() noexcept { return true; } if (GuestMemory::GpuMutex().HeldByThisThread()) { - std::this_thread::yield(); + try { + auto* recorder = Recorder::Active(); + if (recorder != nullptr && !recorder->Idle()) { + Recorder::CountSync(4); + recorder->Sync(); + synced = true; + drained = true; + } + const auto end = static_cast(address) + bytes; + auto& imports = Imports(); + const auto lease = GuestAllocations::GuestAllocationsAcquire_nid_postfix(); + { + std::lock_guard importsLock(imports.mutex); + for (auto it = imports.imports.begin(); it != imports.imports.end();) { + const auto base = it->first; + if (base >= end || address >= base + it->second.bytes) { + ++it; + continue; + } + const auto next = std::next(it); + retireImport(imports.device, imports, it, lease); + it = next; + retiredImport = true; + } + } + if (retiredImport && recorder != nullptr && !recorder->Idle()) { + Recorder::CountSync(4); + recorder->Sync(); + synced = true; + drained = true; + } + } catch (const std::exception& error) { + std::fprintf(stderr, "[gpu] lease wait failed: %s\n", error.what()); + } } else { try { std::lock_guard lock(GuestMemory::GpuMutex()); @@ -1017,6 +1331,48 @@ bool WaitForLeases() noexcept { // yet to record (a test's, or a build whose worker gave up the lock); give it time. std::this_thread::yield(); } + + const auto end = static_cast(address) + bytes; + auto& imports = Imports(); + bool overlapsImport = false; + { + std::lock_guard importsLock(imports.mutex); + for (const auto& [base, entry] : imports.imports) { + if (base < end && address < base + entry.bytes) { + overlapsImport = true; + break; + } + } + } + if (overlapsImport) { + if (recorder != nullptr && !recorder->Idle()) { + Recorder::CountSync(4); + recorder->Sync(); + synced = true; + drained = true; + } + const auto lease = GuestAllocations::GuestAllocationsAcquire_nid_postfix(); + { + std::lock_guard importsLock(imports.mutex); + for (auto it = imports.imports.begin(); it != imports.imports.end();) { + const auto base = it->first; + if (base >= end || address >= base + it->second.bytes) { + ++it; + continue; + } + const auto next = std::next(it); + retireImport(imports.device, imports, it, lease); + it = next; + retiredImport = true; + } + } + if (retiredImport && recorder != nullptr && !recorder->Idle()) { + Recorder::CountSync(4); + recorder->Sync(); + synced = true; + drained = true; + } + } } catch (const std::exception& error) { std::fprintf(stderr, "[gpu] lease wait failed: %s\n", error.what()); } @@ -1027,12 +1383,12 @@ bool WaitForLeases() noexcept { if (synced) ++state.stats.contentionSyncs; if (drained) ++state.stats.contentionDrains; state.stats.contentionMs += std::chrono::duration(std::chrono::steady_clock::now() - start).count(); - return synced; + return synced || retiredImport; } void ensurePinWaiter() { static const bool registered = [] { - GuestAllocations::GuestAllocationsSetPinWaiter_nid_postfix(&WaitForLeases); + GuestAllocations::GuestAllocationsSetPinWaiter_nid_postfix(&WaitForLeasesAndImports); return true; }(); static_cast(registered); @@ -1105,7 +1461,7 @@ std::string AddressCopyOverflow(std::vector copies, std::uint64_t l MirrorStats MirrorCounters() { auto& state = Mirrors(); std::lock_guard lock(state.mutex); - MirrorStats stats{0, state.heapBytes, state.rebuilds, state.blocksCopied, state.heapRefills}; + MirrorStats stats{0, state.heapBytes, state.rebuilds, state.blocksCopied, state.heapRefills, state.sweeps, state.heapChecks}; for (const auto& [base, mirror] : state.entries) stats.heapMirrors += mirror->heap ? 1 : 0; return stats; } @@ -1313,6 +1669,7 @@ void SetImportWatch(const Context& context, ImportWatch watch) { const HostImport* HostImportFor(const Context& context, std::uint64_t address, std::size_t bytes) { if (context.hostImportAlignment == 0 || bytes == 0 || bytes > std::numeric_limits::max() - address) return nullptr; + ensurePinWaiter(); auto& state = Imports(); std::lock_guard lock(state.mutex); // A hit is only valid while the registry has not changed since the imports were reconciled. @@ -1394,6 +1751,7 @@ void GuestBufferMemory::AcquireRegistered() { std::vector blocks; std::vector heaps; bool mirrored = false; + bool served = false; auto& spaces = Spaces(); // Read before the lease below is acquired: a mutation ending in between leaves the space // stale by its generation, never newer than it (see AddressSpace). @@ -1421,11 +1779,11 @@ void GuestBufferMemory::AcquireRegistered() { // an existing mirror. The preparation may wait for recorded work, whose completions can // retire an import the space's regions point at: UploadFinish re-checks the epoch // before any of them is dereferenced. - for (const auto& region : space->base) { - if (region.mirror == nullptr) continue; - mirrored = true; - if (region.mirror->heap) heaps.push_back(region.mirror.get()); - else if (region.mirror->writable && prepareRefresh(*region.mirror, region.begin, region.end - region.begin)) appendBlocks(blocks, *region.mirror, region.begin, region.end - region.begin); + served = true; + mirrored = space->mirrored; + for (const auto index : space->writableMirrors) { + const auto& region = space->base[index]; + if (prepareRefresh(*region.mirror, region.begin, region.end - region.begin)) appendBlocks(blocks, *region.mirror, region.begin, region.end - region.begin); } lap(timing.mirrorsUs, at); } @@ -1450,8 +1808,9 @@ void GuestBufferMemory::AcquireRegistered() { // must not happen under it. regions.reserve(lease.size()); std::vector copies; + std::vector imported; for (const auto& range : lease) { - if (!range->readable) continue; + if (!range->readable || !range->gpu) continue; validate(range->address, range->bytes); Region region{range->address, range->address + range->bytes, range->writable, {}, nullptr}; const HostImport* entry = nullptr; @@ -1466,6 +1825,7 @@ void GuestBufferMemory::AcquireRegistered() { // Reused by Upload while no import was dropped since (see importsEpoch). region.direct = entry; regions.push_back(std::move(region)); + imported.push_back(range->address); continue; } if (mirrorsEnabled()) { @@ -1488,6 +1848,7 @@ void GuestBufferMemory::AcquireRegistered() { const char* reason = !mirrorsEnabled() ? "mirrors disabled by APS5_NO_LEASE_MIRROR" : copied.sparse ? "unreadable pages" : !range->releasable ? "image mirror refused" : "outside the write-watched arena"; copies.push_back({copied.begin, copied.end, committed, reason}); } + releaseImportedMirrors(context, imported); regionsSorted = sortedLookup; static const std::uint64_t copyLimit = [] { const char* value = std::getenv("APS5_ADDRESS_COPY_MAX_MIB"); @@ -1537,16 +1898,20 @@ void GuestBufferMemory::AcquireRegistered() { } regions = std::move(extras); lease.clear(); + planSpaceMirrors(*built); space = std::move(built); spaces.droppedByWaiter.store(false, std::memory_order_relaxed); spaces.current.store(space); } } } - refreshHeapMirrors(heaps, blocks); + std::vector> checked; + if (served) checked = refreshHeapRuns(*space, blocks); + else refreshHeapMirrors(heaps, blocks); lap(timing.mirrorsUs, at); const auto copiedBefore = Mirrors().blocksCopied; compareBlocks(blocks); + for (const auto& [run, collected] : checked) space->verified[run].store(collected, std::memory_order_relaxed); lap(timing.compareUs, at); if (profile) { timing.blocksCompared += blocks.size(); @@ -1632,7 +1997,7 @@ const GuestBufferMemory::Region* GuestBufferMemory::owner(std::uint64_t address) return own; } -void GuestBufferMemory::addDescriptorRegion(std::uint64_t address, std::size_t bytes, bool atomic) { +void GuestBufferMemory::addDescriptorRegion(std::uint64_t address, std::size_t bytes, bool atomic, bool swept) { validate(address, bytes); const auto begin = address & ~std::uint64_t{3}; const auto end = begin == address ? address + bytes : (address + bytes + 3) & ~std::uint64_t{3}; @@ -1650,6 +2015,7 @@ void GuestBufferMemory::addDescriptorRegion(std::uint64_t address, std::size_t b // the shader stores to them; what is written back is decided by Writes() alone. Region region{begin, end, true, {}, nullptr}; region.atomic = atomic; + region.swept = swept; auto committed = GuestMemory::DescribeCommitted(begin, static_cast(end - begin)); if (!committed.whole) { // A GPU heap bound whole while the guest commits its pages on demand, or a descriptor left @@ -1660,8 +2026,8 @@ void GuestBufferMemory::addDescriptorRegion(std::uint64_t address, std::size_t b regionsSorted = false; } -void GuestBufferMemory::AddWritable(std::uint64_t address, std::size_t bytes, bool atomic) { - addDescriptorRegion(address, bytes, atomic); +void GuestBufferMemory::AddWritable(std::uint64_t address, std::size_t bytes, bool atomic, bool swept) { + addDescriptorRegion(address, bytes, atomic, swept); writes.emplace_back(address, address + bytes); } @@ -1669,7 +2035,7 @@ void GuestBufferMemory::AddReadable(std::uint64_t address, std::size_t bytes) { // Not in `writes`: no reference copy (copyRegion), no write-back, no pending-write note, no // direct-write mark, and UploadPrepare copies it without the device lock. A written descriptor // overlapping the range still covers it through its own Writes() entry. - addDescriptorRegion(address, bytes, false); + addDescriptorRegion(address, bytes, false, false); } void GuestBufferMemory::AddSnapshot(const GuestMemorySnapshot& snapshot) { @@ -1792,6 +2158,8 @@ VkBufferUsageFlags gpuCopyUsage(bool addressable) { // APS5_WRITTEN_SHADOW_MIN_KIB / APS5_WRITTEN_SHADOW_MAX_KIB (16 / 2048) bound the written window // (a kernel streaming once through a large buffer, the engine's memcpy kernel over 4-8 MiB video // frames, would only gain the two copies), APS5_ATOMIC_STAGE_MAX_KIB (1024) the atomic one. +constexpr std::uint64_t SweptShadowMax = std::uint64_t{16} << 20u; + bool writtenShadowEnabled() { static const bool disabled = std::getenv("APS5_NO_WRITTEN_SHADOW") != nullptr; return !disabled; @@ -1922,12 +2290,17 @@ bool GuestBufferMemory::gpuCopyEligible(const Region& region) const { return region.end - region.begin <= gpuCopyLimit(); } +bool GuestBufferMemory::bindableInPlace(std::uint64_t offset, bool addressable) const { + if (adjustedRegions && !addressable) return offset % 4 == 0; + return offset % context.limits.minStorageBufferOffsetAlignment == 0; +} + bool GuestBufferMemory::stagingEligible(const Region& region, bool addressable) const { if (!stagingAllowed || addressable || region.unstaged || !gpuCopiesEnabled() || region.sparse || region.mirror != nullptr) return false; const auto bytes = region.end - region.begin; if (!WritesOverlap(region.begin, static_cast(bytes))) return false; if (region.atomic && atomicStagingEnabled() && bytes <= atomicStageMax()) return true; - return writtenShadowEnabled() && bytes >= writtenShadowMin() && bytes <= writtenShadowMax(); + return writtenShadowEnabled() && bytes >= writtenShadowMin() && bytes <= (region.swept ? std::max(writtenShadowMax(), SweptShadowMax) : writtenShadowMax()); } void GuestBufferMemory::UploadPrepare(bool addressable) { @@ -1966,7 +2339,9 @@ void GuestBufferMemory::UploadPrepare(bool addressable) { previous.hostBacked = true; } mergeBacked(previous, region); + if (region.end > previous.end) previous.direct = nullptr; previous.atomic = previous.atomic || region.atomic; + previous.swept = previous.swept || region.swept; // A range starting before the mirror (only possible after the swap) keeps its prefix; the // earlier merged region ends at or before it, so the merged list stays sorted. previous.begin = std::min(previous.begin, region.begin); @@ -2040,7 +2415,7 @@ void GuestBufferMemory::UploadPrepare(bool addressable) { // device-local buffer that the CPU fallback replaces (none without a recorder to // record the copies: UploadFinish then binds the region in place). bool staged = Recorder::Active() != nullptr && stagingEligible(region, addressable); - const bool misaligned = (region.begin - entry->base) % context.limits.minStorageBufferOffsetAlignment != 0; + const bool misaligned = !bindableInPlace(region.begin - entry->base, addressable); if (staged) { const auto allocateStart = std::chrono::steady_clock::now(); region.buffer = stagingBuffer(context, static_cast(region.end - region.begin), gpuCopyUsage(addressable)); @@ -2174,7 +2549,7 @@ void GuestBufferMemory::UploadFinish(bool addressable) { // A staged region (see stagingEligible) is copied out of the import even when aligned; // without a recorder to record the copies it binds in place like any other. const bool staged = recorder != nullptr && stagingEligible(region, addressable); - if (entry != nullptr && !staged && (region.begin - entry->base) % context.limits.minStorageBufferOffsetAlignment == 0) { + if (entry != nullptr && !staged && bindableInPlace(region.begin - entry->base, addressable)) { region.direct = entry; region.snapshot.clear(); // A buffer UploadPrepare made for a GPU copy is not needed: the import serves the @@ -2789,6 +3164,20 @@ std::vector> GuestBufferMemory::InPlaceR return result; } +std::vector> GuestBufferMemory::DeviceReads() const { + std::vector> result; + if (!uploaded || committed) return result; + if (space != nullptr) { + for (const auto& region : space->base) { + if (region.direct != nullptr || region.mirror != nullptr) result.emplace_back(region.begin, region.end); + } + } + for (const auto& region : regions) { + if (region.direct != nullptr || region.mirror != nullptr) result.emplace_back(region.begin, region.end); + } + return result; +} + void GuestBufferMemory::RecordStagingCopies(Recorder& recorder) { if (!uploaded || committed) return; static const bool profile = std::getenv("APS5_PROFILE_DRAW") != nullptr; diff --git a/core/libs/prx/libSceAgcDriver/Graphics/src/GuestSamplerResource.cpp b/core/libs/prx/libSceAgcDriver/Graphics/src/GuestSamplerResource.cpp index e70e48018..86713d404 100644 --- a/core/libs/prx/libSceAgcDriver/Graphics/src/GuestSamplerResource.cpp +++ b/core/libs/prx/libSceAgcDriver/Graphics/src/GuestSamplerResource.cpp @@ -54,7 +54,7 @@ float toSignedLodBias(std::uint32_t raw) { } -GuestSamplerResource DecodeSamplerResource(std::span words, bool unnormalizedProven) { +GuestSamplerResource DecodeSamplerResource(std::span words, bool unnormalizedProven, bool forceDegammaPaired) { Require(words.size() == 4, "guest sampler descriptor must contain 4 dwords"); const auto clampX = (words[0] >> 0u) & 0x7u; @@ -99,7 +99,7 @@ GuestSamplerResource DecodeSamplerResource(std::span words, } else { Require(!unnormalizedProven, "guest sampler descriptor is bound as unnormalized without FORCE_UNNORMALIZED"); } - Require(!forceSrgb, "guest sampler descriptor forces sRGB decoding which is not implemented"); + Require(!forceSrgb || forceDegammaPaired, "guest sampler descriptor forces sRGB decoding which is not implemented"); // TRUNC_COORD picks point-sampled texels by truncation instead of rounding, and the perf fields // trade mip/depth precision for speed; Vulkan's nearest filtering already floors, so these only // move texel selection by half a texel at most and are accepted as is. ANISO_THRESHOLD and @@ -107,12 +107,11 @@ GuestSamplerResource DecodeSamplerResource(std::span words, static_cast(truncCoord); static_cast(anisoThreshold); static_cast(anisoBias); - Require(!disableCubeWrap, "guest sampler descriptor disables seamless cube filtering which is not implemented"); const auto reductionMode = toVkReductionMode(filterMode); Require(!disableDegamma, "guest sampler descriptor disables degamma which is not implemented"); Require(lodBiasSec == 0, "guest sampler descriptor uses a secondary LOD bias which is not implemented"); Require(!pointPreclamp, "guest sampler descriptor uses point preclamping which is not implemented"); - Require(!anisoOverride, "guest sampler descriptor uses an anisotropy override which is not implemented"); + static_cast(anisoOverride); Require(!blendZeroPrt, "guest sampler descriptor uses PRT blend-zero which is not implemented"); if (mipFilter > 2u) Require(false, "guest sampler descriptor uses an unknown mip filter " + std::to_string(mipFilter)); Require(mipFilter != 2u || reductionMode == VK_SAMPLER_REDUCTION_MODE_WEIGHTED_AVERAGE_EXT, "guest sampler descriptor combines a min or max reduction with a linear mip filter, which is not implemented"); @@ -161,6 +160,8 @@ GuestSamplerResource DecodeSamplerResource(std::span words, result.reductionMode = reductionMode; const std::array compareOps{VK_COMPARE_OP_NEVER, VK_COMPARE_OP_LESS, VK_COMPARE_OP_EQUAL, VK_COMPARE_OP_LESS_OR_EQUAL, VK_COMPARE_OP_GREATER, VK_COMPARE_OP_NOT_EQUAL, VK_COMPARE_OP_GREATER_OR_EQUAL, VK_COMPARE_OP_ALWAYS}; result.compareOp = compareOps.at(depthCompareFunc); + result.forceDegamma = forceSrgb; + result.nonSeamlessCube = disableCubeWrap; if (forceUnormCoords) { result.unnormalizedCoordinates = true; result.mipmapMode = VK_SAMPLER_MIPMAP_MODE_NEAREST; @@ -173,4 +174,11 @@ GuestSamplerResource DecodeSamplerResource(std::span words, return result; } +std::optional> SingleLevelSamplerWords(std::span words, bool singleLevelImage, bool mipmappedImage) { + const auto filters = words[2]; + if (((filters >> 29u) & 1u) == 0 || !singleLevelImage || (!isAnisoFilter((filters >> 20u) & 3u) && !isAnisoFilter((filters >> 22u) & 3u))) return std::nullopt; + Require(!mipmappedImage, "guest sampler with ANISO_OVERRIDE is paired with both single-level and mipmapped images in one draw, which is not implemented"); + return std::array{words[0], words[1], filters & ~((2u << 20u) | (2u << 22u)), words[3]}; +} + } diff --git a/core/libs/prx/libSceAgcDriver/Graphics/src/GuestTextureResource.cpp b/core/libs/prx/libSceAgcDriver/Graphics/src/GuestTextureResource.cpp index f97d82c06..8fea5aec0 100644 --- a/core/libs/prx/libSceAgcDriver/Graphics/src/GuestTextureResource.cpp +++ b/core/libs/prx/libSceAgcDriver/Graphics/src/GuestTextureResource.cpp @@ -53,7 +53,7 @@ GuestTextureResource DecodeTextureResource(std::span words) Require(words.size() == 8, "guest texture descriptor must contain 8 dwords"); const auto base40 = (static_cast(words[0]) | (static_cast(words[1]) << 32u)) & 0xffffffffffull; - const auto baseAddress = base40 << 8u; + auto baseAddress = base40 << 8u; Require(baseAddress != 0, "guest texture descriptor has a null base address"); const auto minLod = (words[1] >> 8u) & 0xfffu; @@ -123,7 +123,7 @@ GuestTextureResource DecodeTextureResource(std::span words) std::fprintf(stderr, "[gpu] texture DCC color transform is ignored (word6=0x%08x word7=0x%08x)\n", words[6], words[7]); } } - Require(bcSwizzle == 0, "guest texture descriptor uses a BC swizzle which is not implemented"); + Require(bcSwizzle <= 5u, "guest texture descriptor uses a reserved BC swizzle"); Require(baseLevel <= lastLevel, "guest texture descriptor has a base mip level past its last mip level"); @@ -140,8 +140,12 @@ GuestTextureResource DecodeTextureResource(std::span words) // Views may name levels past MAX_MIP (a 512x512 view through 1x1 over a 9-level surface); the // hardware never addresses them, so the view ends at the surface's last level. if (baseLevel <= maxMip) lastLevel = std::min(lastLevel, maxMip); - // XOR swizzles fold a pipe/bank XOR into the low address bits; only unmodified 64 KiB bases are modeled. - Require(XorSwizzleMode(tileMode) == 0 || (baseAddress & (tileMode == TextureTileMode::kS4KBX || tileMode == TextureTileMode::kD4KBX ? 0xfffu : 0xffffu)) == 0, "guest texture descriptor combines an XOR swizzle with a pipe/bank XOR base which is not implemented"); + std::uint32_t pipeBankXor = 0; + if (XorSwizzleMode(tileMode) != 0) { + pipeBankXor = static_cast(baseAddress & (tileMode == TextureTileMode::kS4KBX || tileMode == TextureTileMode::kD4KBX ? 0xfffu : 0xffffu)); + baseAddress -= pipeBankXor; + Require(baseAddress != 0, "guest texture descriptor has a null base address"); + } switch (dimension) { case TextureDimension::k1D: @@ -175,6 +179,7 @@ GuestTextureResource DecodeTextureResource(std::span words) GuestTextureResource result{}; result.baseAddress = baseAddress; + result.pipeBankXor = pipeBankXor; result.width = width; result.height = height; result.depthOrLastArray = depth; @@ -190,6 +195,7 @@ GuestTextureResource DecodeTextureResource(std::span words) result.dstSelY = static_cast(dstSelY); result.dstSelZ = static_cast(dstSelZ); result.dstSelW = static_cast(dstSelW); + result.bcSwizzle = static_cast(bcSwizzle); result.dccAddress = metaCompress ? metaAddr << 8u : 0u; result.dccAlphaOnMsb = dccAlphaPos; result.dccPipeAligned = metaPipeAligned; @@ -206,6 +212,10 @@ float EffectiveMinLod(const GuestTextureResource& resource) { return std::min(static_cast(resource.minLod) / 256.0f, static_cast(resource.lastLevel)); } +bool DescriptorSingleLevel(std::span words) { + return ((words[3] >> 12u) & 0xfu) == ((words[3] >> 16u) & 0xfu); +} + bool MatchesGuestDimension(ShaderRecompiler::DescriptorImageShape shape, TextureDimension dimension) { switch (shape) { case ShaderRecompiler::DescriptorImageShape::Image1D: return dimension == TextureDimension::k1D; diff --git a/core/libs/prx/libSceAgcDriver/Graphics/src/ParallelCompare.cpp b/core/libs/prx/libSceAgcDriver/Graphics/src/ParallelCompare.cpp new file mode 100644 index 000000000..b4fce7184 --- /dev/null +++ b/core/libs/prx/libSceAgcDriver/Graphics/src/ParallelCompare.cpp @@ -0,0 +1,104 @@ +#include "prx/libSceAgcDriver/Graphics/include/ParallelCompare.hpp" +#include +#include +#include +#include +#include +#include + +namespace AgcDriver::Graphics { +namespace { + +class ComparePool { +public: + explicit ComparePool(unsigned count) { + for (unsigned i = 0; i < count; ++i) std::thread([this] { work(); }).detach(); + } + + bool Run(std::span spans, std::span equal) { + std::unique_lock caller(callers, std::try_to_lock); + if (!caller.owns_lock()) return false; + { + std::unique_lock lock(mutex); + finished.wait(lock, [&] { return working == 0; }); + jobSpans = spans; + jobEqual = equal; + next.store(0, std::memory_order_relaxed); + done.store(0, std::memory_order_relaxed); + ++generation; + } + wake.notify_all(); + drain(spans, equal); + std::unique_lock lock(mutex); + finished.wait(lock, [&] { return working == 0 && done.load(std::memory_order_acquire) == spans.size(); }); + jobSpans = {}; + jobEqual = {}; + return true; + } + +private: + void drain(std::span spans, std::span equal) { + std::size_t compared = 0; + for (auto index = next.fetch_add(1, std::memory_order_relaxed); index < spans.size(); index = next.fetch_add(1, std::memory_order_relaxed)) { + const auto& span = spans[index]; + equal[index] = std::memcmp(span.first, span.second, span.bytes) == 0 ? 1 : 0; + ++compared; + } + if (compared != 0) done.fetch_add(compared, std::memory_order_acq_rel); + } + + void work() { + std::uint64_t seen = 0; + std::unique_lock lock(mutex); + for (;;) { + wake.wait(lock, [&] { return generation != seen; }); + seen = generation; + const auto spans = jobSpans; + const auto equal = jobEqual; + ++working; + lock.unlock(); + drain(spans, equal); + lock.lock(); + --working; + if (working == 0) finished.notify_all(); + } + } + + std::mutex callers; + std::mutex mutex; + std::condition_variable wake; + std::condition_variable finished; + std::uint64_t generation = 0; + unsigned working = 0; + std::span jobSpans; + std::span jobEqual; + std::atomic next{0}; + std::atomic done{0}; +}; + +ComparePool* sharedPool() { + static ComparePool* const pool = new ComparePool(CompareHelpers()); + return pool; +} + +} + +unsigned CompareHelpers() { + static const unsigned helpers = std::clamp(std::thread::hardware_concurrency() / 8u, 1u, 4u); + return helpers; +} + +void CompareSpansFrom(std::span spans, std::span equal, std::size_t parallelBytes) { + std::size_t total = 0; + for (const auto& span : spans) total += span.bytes; + if (spans.size() > 1 && total >= parallelBytes) { + if (sharedPool()->Run(spans, equal)) return; + } + for (std::size_t index = 0; index < spans.size(); ++index) equal[index] = std::memcmp(spans[index].first, spans[index].second, spans[index].bytes) == 0 ? 1 : 0; +} + +void CompareSpans(std::span spans, std::span equal) { + CompareSpansFrom(spans, equal, std::size_t{1} << 20u); +} + +} diff --git a/core/libs/prx/libSceAgcDriver/Graphics/src/PassHazards.cpp b/core/libs/prx/libSceAgcDriver/Graphics/src/PassHazards.cpp new file mode 100644 index 000000000..a3a4c9738 --- /dev/null +++ b/core/libs/prx/libSceAgcDriver/Graphics/src/PassHazards.cpp @@ -0,0 +1,91 @@ +#include "prx/libSceAgcDriver/Graphics/include/PassHazards.hpp" +#include + +namespace AgcDriver::Graphics { + +void GuestRangeSet::Insert(std::uint64_t begin, std::uint64_t end) { + if (begin >= end) return; + auto first = std::lower_bound(ranges.begin(), ranges.end(), begin, [](const auto& range, std::uint64_t value) { return range.second < value; }); + auto last = first; + while (last != ranges.end() && last->first <= end) { + begin = std::min(begin, last->first); + end = std::max(end, last->second); + ++last; + } + if (first == last) { + ranges.insert(first, {begin, end}); + return; + } + *first = {begin, end}; + ranges.erase(first + 1, last); +} + +bool GuestRangeSet::Overlaps(std::uint64_t begin, std::uint64_t end) const { + if (begin >= end) return false; + const auto it = std::upper_bound(ranges.begin(), ranges.end(), begin, [](std::uint64_t value, const auto& range) { return value < range.second; }); + return it != ranges.end() && it->first < end; +} + +bool PassHazards::overlapsAny(const GuestRangeSet& set, GuestRangeList ranges) const { + if (set.Empty()) return false; + return std::any_of(ranges.begin(), ranges.end(), [&](const auto& range) { return set.Overlaps(range.first, range.second); }); +} + +PassHazard PassHazards::Check(const PassAccess& access, bool samePass) const { + const auto touchesMemory = [](const PassAccess& use) { + const auto nonEmpty = [](GuestRangeList ranges) { return std::any_of(ranges.begin(), ranges.end(), [](const auto& range) { return range.first < range.second; }); }; + return use.anyReads || use.anyWrites || nonEmpty(use.reads) || nonEmpty(use.writes); + }; + const bool earlierTouches = anyReads || anyWrites || !reads.Empty() || !writes.Empty(); + if ((access.anyWrites && earlierTouches) || (anyWrites && touchesMemory(access))) return PassHazard::UnknownWrite; + const bool writesRanges = std::any_of(access.writes.begin(), access.writes.end(), [](const auto& range) { return range.first < range.second; }); + if ((access.anyReads && !writes.Empty()) || (anyReads && writesRanges)) return PassHazard::UnknownRead; + if (overlapsAny(writes, access.reads)) return PassHazard::ReadAfterWrite; + if (overlapsAny(writes, access.writes)) return PassHazard::WriteAfterWrite; + if (overlapsAny(reads, access.writes)) return PassHazard::WriteAfterRead; + const auto conflicts = [](bool earlier, bool later) { return earlier || later; }; + for (const auto& [image, written] : access.images) { + if (image == VK_NULL_HANDLE) continue; + if (std::any_of(images.begin(), images.end(), [&](const auto& seen) { return seen.first == image && conflicts(seen.second, written); })) return PassHazard::Image; + if (std::any_of(attachments.begin(), attachments.end(), [&](const Attachment& seen) { return seen.image == image && conflicts(seen.written, written); })) return PassHazard::Image; + } + for (const auto& [image, written] : access.attachments) { + if (image == VK_NULL_HANDLE) continue; + if (std::any_of(images.begin(), images.end(), [&](const auto& seen) { return seen.first == image && conflicts(seen.second, written); })) return PassHazard::Image; + if (std::any_of(attachments.begin(), attachments.end(), [&](const Attachment& seen) { return seen.image == image && !(samePass && seen.pass == pass) && conflicts(seen.written, written); })) return PassHazard::Image; + } + return PassHazard::None; +} + +void PassHazards::Add(const PassAccess& access) { + for (const auto& [begin, end] : access.reads) reads.Insert(begin, end); + for (const auto& [begin, end] : access.writes) writes.Insert(begin, end); + for (const auto& [image, written] : access.images) { + if (image == VK_NULL_HANDLE) continue; + const auto it = std::find_if(images.begin(), images.end(), [&](const auto& seen) { return seen.first == image; }); + if (it == images.end()) images.emplace_back(image, written); + else it->second = it->second || written; + } + for (const auto& [image, written] : access.attachments) { + if (image == VK_NULL_HANDLE) continue; + const auto it = std::find_if(attachments.begin(), attachments.end(), [&](const Attachment& seen) { return seen.image == image && seen.pass == pass; }); + if (it == attachments.end()) attachments.push_back({image, written, pass}); + else it->written = it->written || written; + } + anyReads = anyReads || access.anyReads; + anyWrites = anyWrites || access.anyWrites; +} + +void PassHazards::Clear() { + reads.Clear(); + writes.Clear(); + images.clear(); + attachments.clear(); + anyReads = anyWrites = false; +} + +bool PassHazards::Empty() const { + return reads.Empty() && writes.Empty() && images.empty() && attachments.empty() && !anyReads && !anyWrites; +} + +} diff --git a/core/libs/prx/libSceAgcDriver/Graphics/src/Pipeline.cpp b/core/libs/prx/libSceAgcDriver/Graphics/src/Pipeline.cpp index 3f932eeaa..2ab89e7d4 100644 --- a/core/libs/prx/libSceAgcDriver/Graphics/src/Pipeline.cpp +++ b/core/libs/prx/libSceAgcDriver/Graphics/src/Pipeline.cpp @@ -1,6 +1,7 @@ #include "prx/libSceAgcDriver/Execution/include/ProfileOutput.hpp" #include "prx/libSceAgcDriver/Execution/include/PerformanceTimer.hpp" #include "prx/libSceAgcDriver/Graphics/include/Pipeline.hpp" +#include "prx/libSceAgcDriver/Graphics/include/PipelineLibrary.hpp" #include "prx/libSceAgcDriver/Graphics/include/PipelineSpecialization.hpp" #include "prx/libSceAgcDriver/Graphics/include/VertexInput.hpp" #include @@ -11,6 +12,7 @@ #include #include #include +#include #include #include #include @@ -62,10 +64,16 @@ void LogPipelineStatistics_nid_no_patch(const Context& context, VkPipeline pipel } } +bool HasStencil(VkFormat format) { + return format == VK_FORMAT_D16_UNORM_S8_UINT || format == VK_FORMAT_D24_UNORM_S8_UINT || format == VK_FORMAT_D32_SFLOAT_S8_UINT || format == VK_FORMAT_S8_UINT; +} + Framebuffer::Framebuffer(const Context& context, VkRenderPass renderPass, std::span targets, VkExtent2D extent) : context(context) { // Cached objects outlive their device's teardown; they must not keep its buffer pool alive past it. this->context.bufferPool.reset(); Require(extent.width != 0 && extent.height != 0 && extent.width <= context.limits.maxFramebufferWidth && extent.height <= context.limits.maxFramebufferHeight, "framebuffer extent exceeds device limits"); + views.assign(targets.begin(), targets.end()); + if (renderPass == VK_NULL_HANDLE) return; VkFramebufferCreateInfo framebufferInfo{VK_STRUCTURE_TYPE_FRAMEBUFFER_CREATE_INFO}; framebufferInfo.renderPass = renderPass; framebufferInfo.attachmentCount = static_cast(targets.size()); @@ -84,6 +92,20 @@ void ValidateDepthBounds(const Context& context, const State& state) { Require(!state.depthBoundsTest || context.depthRangeUnrestricted || (state.minDepthBounds >= 0.0f && state.minDepthBounds <= 1.0f && state.maxDepthBounds >= 0.0f && state.maxDepthBounds <= 1.0f), "depth bounds outside [0, 1] require VK_EXT_depth_range_unrestricted"); } +void ValidateProvokingVertex(const Context& context, const State& state, std::span shaders) { + Require(state.provokingVertexMode == VK_PROVOKING_VERTEX_MODE_FIRST_VERTEX_EXT || state.provokingVertexMode == VK_PROVOKING_VERTEX_MODE_LAST_VERTEX_EXT, "invalid provoking vertex mode"); + if (state.provokingVertexMode == VK_PROVOKING_VERTEX_MODE_FIRST_VERTEX_EXT) return; + Require(context.provokingVertexLast, "last provoking vertex requires VK_EXT_provoking_vertex with provokingVertexLast enabled"); + Require(!state.rectList && state.stages.path == ShaderPath::Vertex && !state.stages.mesh && !state.stages.tessellation, "last provoking vertex is unsupported for generated primitive pipelines"); + Require(state.topology == VK_PRIMITIVE_TOPOLOGY_POINT_LIST || state.topology == VK_PRIMITIVE_TOPOLOGY_LINE_LIST || state.topology == VK_PRIMITIVE_TOPOLOGY_LINE_STRIP || state.topology == VK_PRIMITIVE_TOPOLOGY_TRIANGLE_LIST || state.topology == VK_PRIMITIVE_TOPOLOGY_TRIANGLE_STRIP || state.topology == VK_PRIMITIVE_TOPOLOGY_TRIANGLE_FAN, "last provoking vertex is unsupported for this primitive topology"); + for (const auto& shader : shaders) { + if (shader.stage != ShaderRecompiler::ShaderStage::Fragment) continue; + Require(shader.program != nullptr, "missing compiled fragment shader"); + Require(std::none_of(shader.program->fragmentParameters.begin(), shader.program->fragmentParameters.end(), [](const ShaderRecompiler::FragmentParameter& parameter) { return parameter.flat && parameter.perVertex; }), "last provoking vertex with explicit per-vertex flat interpolation is unsupported"); + Require(state.topology != VK_PRIMITIVE_TOPOLOGY_TRIANGLE_STRIP || std::none_of(shader.program->fragmentParameters.begin(), shader.program->fragmentParameters.end(), [](const ShaderRecompiler::FragmentParameter& parameter) { return parameter.perVertex; }), "last provoking vertex with explicit per-vertex triangle-strip interpolation is unsupported"); + } +} + void ValidateViewport(const Context& context, const VkViewport& viewport) { Require(std::isfinite(viewport.minDepth) && std::isfinite(viewport.maxDepth), "non-finite viewport depth range"); Require(context.depthRangeUnrestricted || (viewport.minDepth >= 0 && viewport.minDepth <= 1 && viewport.maxDepth >= 0 && viewport.maxDepth <= 1), "viewport depth outside [0, 1] requires VK_EXT_depth_range_unrestricted"); @@ -93,7 +115,68 @@ void ValidateViewport(const Context& context, const VkViewport& viewport) { Require(std::min(viewport.y, viewport.y + viewport.height) >= context.limits.viewportBoundsRange[0] && std::max(viewport.y, viewport.y + viewport.height) <= context.limits.viewportBoundsRange[1], "viewport Y exceeds device bounds"); } -Pipeline::Pipeline(const Context& context, const State& state, const VertexInputLayout& vertexInput, const ShaderResources& resources, std::span shaders, VkImageLayout attachmentLayout) : context(context), _modules(shaders.size()), attachments(state.colors.size() + (state.depth ? 1u : 0u)), colorAttachments(state.colors.size()), depthBounds(state.depth.has_value() && state.depthBoundsTest), depthBias(state.depth.has_value() && state.depthBias) { +namespace { + +VkShaderStageFlags pushStagesOf(const Context& context) { + return PreRasterizationPushStages(context.meshShader, context.tessellationShader, context.geometryShader); +} + +template +void appendKey(std::vector& key, const TValue& value) { + static_assert(std::is_trivially_copyable_v); + const auto bytes = std::as_bytes(std::span(&value, 1)); + key.insert(key.end(), bytes.begin(), bytes.end()); +} + +bool libraryKeys(PipelineLibraryKeys& keys, const Context& context, const State& state, const VertexInputLayout& input, const ShaderResources& resources, std::span shaders) { + using Stage = ShaderRecompiler::ShaderStage; + if (!context.graphicsPipelineLibrary || context.pipelineExecutableInfo || state.depthBiasPerFace || state.rectList) return false; + keys = {}; + for (const auto& shader : shaders) { + if (shader.stage == Stage::Geometry || shader.program->PipelineVariantId() == 0) return false; + auto& key = shader.stage == Stage::Fragment ? keys.fragmentShader : keys.preRasterization; + appendKey(key, shader.stage); + appendKey(key, shader.program->PipelineVariantId()); + appendKey(key, shader.pushConstantOffset); + } + for (const auto& binding : input.bindings) { + appendKey(keys.vertexInput, binding.binding); + appendKey(keys.vertexInput, binding.stride); + appendKey(keys.vertexInput, binding.inputRate); + } + appendKey(keys.vertexInput, input.attributes.size()); + for (const auto& attribute : input.attributes) { + appendKey(keys.vertexInput, attribute.location); + appendKey(keys.vertexInput, attribute.binding); + appendKey(keys.vertexInput, attribute.format); + appendKey(keys.vertexInput, attribute.offset); + } + appendKey(keys.vertexInput, state.topology); + appendKey(keys.vertexInput, state.primitiveRestart); + appendKey(keys.preRasterization, state.provokingVertexMode); + appendKey(keys.preRasterization, state.negativeOneToOne); + appendKey(keys.preRasterization, state.depthClamp && context.depthClamp); + appendKey(keys.preRasterization, state.conservativeRasterization); + appendKey(keys.preRasterization, state.depth.has_value() && state.depthBias); + appendKey(keys.preRasterization, state.stages.tessellation.has_value()); + if (state.stages.tessellation) appendKey(keys.preRasterization, state.stages.tessellation->inputControlPoints); + for (const auto& blend : state.blends) appendKey(keys.fragmentOutput, blend); + for (const auto value : state.blendConstants) appendKey(keys.fragmentOutput, value); + appendKey(keys.renderPass, state.blends.size()); + for (const auto& color : state.colors) { + appendKey(keys.renderPass, color.format); + appendKey(keys.renderPass, color.exportIndex); + } + appendKey(keys.renderPass, state.depth.has_value()); + if (state.depth) appendKey(keys.renderPass, state.depth->format); + appendKey(keys.layout, pushStagesOf(context)); + for (const auto word : resources.LayoutKey()) appendKey(keys.layout, word); + return true; +} + +} + +Pipeline::Pipeline(const Context& context, const State& state, const VertexInputLayout& vertexInput, const ShaderResources& resources, std::span shaders, VkImageLayout attachmentLayout) : context(context), _modules(shaders.size()), attachments(state.colors.size() + (state.depth ? 1u : 0u)), colorAttachments(state.colors.size()), depthBounds(state.depth.has_value() && state.depthBoundsTest), depthBias(state.depth.has_value() && state.depthBias), dynamicRendering(context.graphicsPipelineLibrary) { PerformanceTimer timing("Vulkan.GraphicsPipeline"); // A cached pipeline may outlive its device's teardown (see ClearCachedPipelines); it must not keep // the buffer pool, which is reset with the device, alive past it. @@ -103,9 +186,12 @@ Pipeline::Pipeline(const Context& context, const State& state, const VertexInput Require(state.hasColorTarget || (context.limits.framebufferNoAttachmentsSampleCounts & VK_SAMPLE_COUNT_1_BIT) != 0, "device does not support single-sample rendering without attachments"); Require(!depthBounds || context.depthBounds, "device does not support the depth bounds test"); Require(!depthBias || state.depthBiasClamp == 0.0f || context.depthBiasClamp, "device does not support depth bias clamping"); + Require(!state.dualSourceBlend || (context.dualSrcBlend && state.blends.size() <= context.limits.maxFragmentDualSrcAttachments), "device does not support dual-source blending into this many attachments"); Require(!state.negativeOneToOne || context.depthClipControl, "negative-one-to-one depth clipping requires VK_EXT_depth_clip_control with depthClipControl enabled"); Require(state.conservativeRasterization == VK_CONSERVATIVE_RASTERIZATION_MODE_DISABLED_EXT || context.conservativeRasterization, "conservative rasterization requires VK_EXT_conservative_rasterization with at most 1/256 pixel of overestimation and degenerate triangles rasterized"); + ValidateProvokingVertex(context, state, shaders); if (state.rectList) Require(context.tessellationShader && context.limits.maxTessellationPatchSize >= 4, "rect-list requires tessellation with four output control points"); + if (std::any_of(shaders.begin(), shaders.end(), [](const CompiledShader& shader) { return shader.stage == ShaderRecompiler::ShaderStage::Geometry; })) Require(context.geometryShader, "device does not support geometry shaders"); if (state.stages.tessellation) { Require(context.tessellationShader, "device does not support tessellation shaders"); Require(state.stages.tessellation->inputControlPoints <= context.limits.maxTessellationPatchSize && state.stages.tessellation->outputControlPoints <= context.limits.maxTessellationPatchSize, "tessellation patch exceeds device limits"); @@ -118,7 +204,10 @@ Pipeline::Pipeline(const Context& context, const State& state, const VertexInput Require(mesh.maxVertices <= context.meshLimits.maxMeshOutputVertices && mesh.maxPrimitives <= context.meshLimits.maxMeshOutputPrimitives && static_cast(mesh.ldsSizeDwords) * 4 <= context.meshLimits.maxMeshSharedMemorySize, "mesh output or LDS exceeds device limits"); } const auto pushStages = PushConstantStages(shaders); - Require(pushStages == 0 || context.limits.maxPushConstantsSize >= PipelinePushConstantBytes, "graphics push constant range exceeds device limit"); + Require(pushStages == 0 || context.limits.maxPushConstantsSize >= PushBlockBytes(dynamicRendering), "graphics push constant range exceeds device limit"); + if (dynamicRendering) { + for (const auto& shader : shaders) Require(shader.program->pushConstants.empty() || (shader.pushConstantOffset >= PipelinePushSlotBytes) == (shader.stage == ShaderRecompiler::ShaderStage::Fragment), "a stage's push constants lie outside its push constant slot"); + } try { std::vector stages(shaders.size()); std::vector specializations; @@ -140,12 +229,13 @@ Pipeline::Pipeline(const Context& context, const State& state, const VertexInput // A descriptor set layout with the same bindings as this one is compatible with the pipeline // layout, so later draws bind their own ShaderResources' set under it. const auto setLayout = resources.Layout(); - const VkPushConstantRange push{pushStages, 0, PipelinePushConstantBytes}; + const VkPushConstantRange push{pushStages, 0, PipelinePushSlotBytes}; + const std::array slots{{{pushStagesOf(context), 0, PipelinePushSlotBytes}, {VK_SHADER_STAGE_FRAGMENT_BIT, PipelinePushSlotBytes, PipelinePushSlotBytes}}}; VkPipelineLayoutCreateInfo layoutInfo{VK_STRUCTURE_TYPE_PIPELINE_LAYOUT_CREATE_INFO}; layoutInfo.setLayoutCount = 1; layoutInfo.pSetLayouts = &setLayout; - layoutInfo.pushConstantRangeCount = pushStages != 0 ? 1 : 0; - layoutInfo.pPushConstantRanges = pushStages != 0 ? &push : nullptr; + layoutInfo.pushConstantRangeCount = dynamicRendering ? static_cast(slots.size()) : pushStages != 0 ? 1 : 0; + layoutInfo.pPushConstantRanges = dynamicRendering ? slots.data() : pushStages != 0 ? &push : nullptr; Check(context.Function("vkCreatePipelineLayout")(context.device, &layoutInfo, nullptr, &layout), "vkCreatePipelineLayout graphics"); std::vector colors; std::vector references(state.blends.size(), VkAttachmentReference{VK_ATTACHMENT_UNUSED, attachmentLayout}); @@ -180,12 +270,27 @@ Pipeline::Pipeline(const Context& context, const State& state, const VertexInput colors.push_back(depth); subpass.pDepthStencilAttachment = &depthReference; } + std::vector renderingColors(state.blends.size(), VK_FORMAT_UNDEFINED); + for (const auto& color : state.colors) renderingColors.at(color.exportIndex) = color.format; + VkPipelineRenderingCreateInfoKHR rendering{VK_STRUCTURE_TYPE_PIPELINE_RENDERING_CREATE_INFO_KHR}; + rendering.colorAttachmentCount = static_cast(renderingColors.size()); + rendering.pColorAttachmentFormats = renderingColors.empty() ? nullptr : renderingColors.data(); + if (state.depth) { + rendering.depthAttachmentFormat = state.depth->format; + if (HasStencil(state.depth->format)) rendering.stencilAttachmentFormat = state.depth->format; + } + if (dynamicRendering) { + depthFormat = state.depth ? state.depth->format : VK_FORMAT_UNDEFINED; + colorSlots.assign(state.blends.size(), ~0u); + for (std::uint32_t index = 0; index < state.colors.size(); ++index) colorSlots.at(state.colors[index].exportIndex) = index; + this->attachmentLayout = attachmentLayout; + } VkRenderPassCreateInfo passInfo{VK_STRUCTURE_TYPE_RENDER_PASS_CREATE_INFO}; passInfo.attachmentCount = static_cast(colors.size()); passInfo.pAttachments = colors.empty() ? nullptr : colors.data(); passInfo.subpassCount = 1; passInfo.pSubpasses = &subpass; - Check(context.Function("vkCreateRenderPass")(context.device, &passInfo, nullptr, &renderPass), "vkCreateRenderPass"); + if (!dynamicRendering) Check(context.Function("vkCreateRenderPass")(context.device, &passInfo, nullptr, &renderPass), "vkCreateRenderPass"); VkPipelineVertexInputStateCreateInfo input{VK_STRUCTURE_TYPE_PIPELINE_VERTEX_INPUT_STATE_CREATE_INFO}; input.vertexBindingDescriptionCount = static_cast(vertexInput.bindings.size()); input.pVertexBindingDescriptions = vertexInput.bindings.data(); @@ -203,22 +308,32 @@ Pipeline::Pipeline(const Context& context, const State& state, const VertexInput if (state.negativeOneToOne) viewports.pNext = &depthClip; viewports.viewportCount = 1; viewports.scissorCount = 1; + PipelineLibraryKeys keys; + libraries = libraryKeys(keys, context, state, vertexInput, resources, shaders); std::vector dynamicStates{VK_DYNAMIC_STATE_VIEWPORT, VK_DYNAMIC_STATE_SCISSOR}; if (depthBounds) dynamicStates.push_back(VK_DYNAMIC_STATE_DEPTH_BOUNDS); if (depthBias) dynamicStates.push_back(VK_DYNAMIC_STATE_DEPTH_BIAS); + if (libraries) dynamicStates.assign(PipelineLibraryDynamicStates().begin(), PipelineLibraryDynamicStates().end()); VkPipelineDynamicStateCreateInfo dynamic{VK_STRUCTURE_TYPE_PIPELINE_DYNAMIC_STATE_CREATE_INFO}; dynamic.dynamicStateCount = static_cast(dynamicStates.size()); dynamic.pDynamicStates = dynamicStates.data(); VkPipelineRasterizationStateCreateInfo raster{VK_STRUCTURE_TYPE_PIPELINE_RASTERIZATION_STATE_CREATE_INFO}; raster.polygonMode = VK_POLYGON_MODE_FILL; raster.depthClampEnable = state.depthClamp && context.depthClamp ? VK_TRUE : VK_FALSE; - raster.cullMode = state.cullMode; + Require(!state.depthBiasPerFace || (depthBias && state.cullMode == VK_CULL_MODE_NONE), "per-face depth bias needs both faces rasterized"); + raster.cullMode = state.depthBiasPerFace ? VK_CULL_MODE_BACK_BIT : state.cullMode; raster.frontFace = state.frontFace; raster.depthBiasEnable = depthBias; raster.lineWidth = 1; VkPipelineRasterizationConservativeStateCreateInfoEXT conservative{VK_STRUCTURE_TYPE_PIPELINE_RASTERIZATION_CONSERVATIVE_STATE_CREATE_INFO_EXT}; conservative.conservativeRasterizationMode = state.conservativeRasterization; if (state.conservativeRasterization != VK_CONSERVATIVE_RASTERIZATION_MODE_DISABLED_EXT) raster.pNext = &conservative; + VkPipelineRasterizationProvokingVertexStateCreateInfoEXT provoking{VK_STRUCTURE_TYPE_PIPELINE_RASTERIZATION_PROVOKING_VERTEX_STATE_CREATE_INFO_EXT}; + provoking.provokingVertexMode = state.provokingVertexMode; + if (context.provokingVertexLast) { + provoking.pNext = raster.pNext; + raster.pNext = &provoking; + } VkPipelineMultisampleStateCreateInfo samples{VK_STRUCTURE_TYPE_PIPELINE_MULTISAMPLE_STATE_CREATE_INFO}; samples.rasterizationSamples = VK_SAMPLE_COUNT_1_BIT; VkPipelineDepthStencilStateCreateInfo depthStencil{VK_STRUCTURE_TYPE_PIPELINE_DEPTH_STENCIL_STATE_CREATE_INFO}; @@ -252,8 +367,16 @@ Pipeline::Pipeline(const Context& context, const State& state, const VertexInput pipelineInfo.pDynamicState = &dynamic; pipelineInfo.layout = layout; pipelineInfo.renderPass = renderPass; + if (dynamicRendering) pipelineInfo.pNext = &rendering; timing.Mark("modules_and_state"); - Check(context.Function("vkCreateGraphicsPipelines")(context.device, context.pipelineCache, 1, &pipelineInfo, nullptr, &pipeline), "vkCreateGraphicsPipelines"); + if (libraries) pipeline = LinkPipelineFromLibraries(context, pipelineInfo, rendering, layoutInfo, keys, &optimized); + else Check(context.Function("vkCreateGraphicsPipelines")(context.device, context.pipelineCache, 1, &pipelineInfo, nullptr, &pipeline), "vkCreateGraphicsPipelines"); + if (state.depthBiasPerFace) { + raster.cullMode = VK_CULL_MODE_FRONT_BIT; + Check(context.Function("vkCreateGraphicsPipelines")(context.device, context.pipelineCache, 1, &pipelineInfo, nullptr, &backFaces), "vkCreateGraphicsPipelines"); + } + for (const auto module : _modules) context.Function("vkDestroyShaderModule")(context.device, module, nullptr); + _modules.clear(); timing.Mark("create"); LogPipelineStatistics_nid_no_patch(context, pipeline); } catch (...) { @@ -268,22 +391,29 @@ Pipeline::~Pipeline() { void Pipeline::release() noexcept { framebuffers.clear(); + if (optimized != nullptr) optimized->Release(); + optimized.reset(); if (pipeline) context.Function("vkDestroyPipeline")(context.device, pipeline, nullptr); + if (backFaces) context.Function("vkDestroyPipeline")(context.device, backFaces, nullptr); if (renderPass) context.Function("vkDestroyRenderPass")(context.device, renderPass, nullptr); if (layout) context.Function("vkDestroyPipelineLayout")(context.device, layout, nullptr); for (auto module : _modules) { if (module) context.Function("vkDestroyShaderModule")(context.device, module, nullptr); } pipeline = VK_NULL_HANDLE; + backFaces = VK_NULL_HANDLE; renderPass = VK_NULL_HANDLE; layout = VK_NULL_HANDLE; _modules.clear(); } void Pipeline::Abandon() noexcept { + if (optimized != nullptr) optimized->released.store(true); + optimized.reset(); for (auto& entry : framebuffers) entry.framebuffer->Abandon(); framebuffers.clear(); pipeline = VK_NULL_HANDLE; + backFaces = VK_NULL_HANDLE; renderPass = VK_NULL_HANDLE; layout = VK_NULL_HANDLE; _modules.clear(); @@ -336,6 +466,35 @@ std::shared_ptr Pipeline::AcquireFramebuffer(std::span colors(colorSlots.size(), VkRenderingAttachmentInfoKHR{VK_STRUCTURE_TYPE_RENDERING_ATTACHMENT_INFO_KHR}); + for (std::size_t slot = 0; slot < colorSlots.size(); ++slot) { + if (colorSlots[slot] == ~0u) continue; + colors[slot].imageView = views[colorSlots[slot]]; + colors[slot].imageLayout = attachmentLayout; + colors[slot].loadOp = VK_ATTACHMENT_LOAD_OP_LOAD; + colors[slot].storeOp = VK_ATTACHMENT_STORE_OP_STORE; + } + VkRenderingAttachmentInfoKHR depth{VK_STRUCTURE_TYPE_RENDERING_ATTACHMENT_INFO_KHR}; + if (depthFormat != VK_FORMAT_UNDEFINED) { + depth.imageView = views.back(); + depth.imageLayout = VK_IMAGE_LAYOUT_GENERAL; + depth.loadOp = VK_ATTACHMENT_LOAD_OP_LOAD; + depth.storeOp = VK_ATTACHMENT_STORE_OP_STORE; + } + VkRenderingInfoKHR rendering{VK_STRUCTURE_TYPE_RENDERING_INFO_KHR}; + rendering.renderArea = {{0, 0}, extent}; + rendering.layerCount = 1; + rendering.colorAttachmentCount = static_cast(colors.size()); + rendering.pColorAttachments = colors.empty() ? nullptr : colors.data(); + if (depthFormat != VK_FORMAT_UNDEFINED) rendering.pDepthAttachment = &depth; + if (depthFormat != VK_FORMAT_UNDEFINED && HasStencil(depthFormat)) rendering.pStencilAttachment = &depth; + context.Resolved(&DeviceFunctions::cmdBeginRendering, "vkCmdBeginRenderingKHR")(commands, &rendering); + Continue(commands, state); + return; + } VkRenderPassBeginInfo begin{VK_STRUCTURE_TYPE_RENDER_PASS_BEGIN_INFO}; begin.renderPass = renderPass; begin.framebuffer = framebuffer.Handle(); @@ -345,17 +504,57 @@ void Pipeline::Begin(VkCommandBuffer commands, const Framebuffer& framebuffer, V } void Pipeline::Continue(VkCommandBuffer commands, const State& state) const { - context.Resolved(&DeviceFunctions::cmdBindPipeline, "vkCmdBindPipeline")(commands, VK_PIPELINE_BIND_POINT_GRAPHICS, pipeline); + const auto best = optimized != nullptr ? optimized->handle.load() : VK_NULL_HANDLE; + context.Resolved(&DeviceFunctions::cmdBindPipeline, "vkCmdBindPipeline")(commands, VK_PIPELINE_BIND_POINT_GRAPHICS, best != VK_NULL_HANDLE ? best : pipeline); context.Resolved(&DeviceFunctions::cmdSetViewport, "vkCmdSetViewport")(commands, 0, 1, &state.viewport); context.Resolved(&DeviceFunctions::cmdSetScissor, "vkCmdSetScissor")(commands, 0, 1, &state.scissor); + if (libraries) { + setLibraryState(commands, state); + return; + } if (depthBias) context.Resolved(&DeviceFunctions::cmdSetDepthBias, "vkCmdSetDepthBias")(commands, state.depthBiasConstant, state.depthBiasClamp, state.depthBiasSlope); if (!depthBounds) return; context.Resolved(&DeviceFunctions::cmdSetDepthBounds, "vkCmdSetDepthBounds")(commands, state.minDepthBounds, state.maxDepthBounds); } +void Pipeline::setLibraryState(VkCommandBuffer commands, const State& state) const { + const bool depth = state.depth.has_value(); + context.Resolved(&DeviceFunctions::cmdSetCullMode, "vkCmdSetCullModeEXT")(commands, state.cullMode); + context.Resolved(&DeviceFunctions::cmdSetFrontFace, "vkCmdSetFrontFaceEXT")(commands, state.frontFace); + context.Resolved(&DeviceFunctions::cmdSetDepthBias, "vkCmdSetDepthBias")(commands, depthBias ? state.depthBiasConstant : 0.0f, depthBias ? state.depthBiasClamp : 0.0f, depthBias ? state.depthBiasSlope : 0.0f); + context.Resolved(&DeviceFunctions::cmdSetDepthBounds, "vkCmdSetDepthBounds")(commands, depthBounds ? state.minDepthBounds : 0.0f, depthBounds ? state.maxDepthBounds : 1.0f); + context.Resolved(&DeviceFunctions::cmdSetDepthTestEnable, "vkCmdSetDepthTestEnableEXT")(commands, depth && state.depthTest); + context.Resolved(&DeviceFunctions::cmdSetDepthWriteEnable, "vkCmdSetDepthWriteEnableEXT")(commands, depth && state.depthWrite); + context.Resolved(&DeviceFunctions::cmdSetDepthCompareOp, "vkCmdSetDepthCompareOpEXT")(commands, depth ? state.depthCompare : VK_COMPARE_OP_ALWAYS); + context.Resolved(&DeviceFunctions::cmdSetDepthBoundsTestEnable, "vkCmdSetDepthBoundsTestEnableEXT")(commands, depthBounds); + context.Resolved(&DeviceFunctions::cmdSetStencilTestEnable, "vkCmdSetStencilTestEnableEXT")(commands, depth && state.stencilTest); + const auto stencil = [&](VkStencilFaceFlags face, const VkStencilOpState& op) { + context.Resolved(&DeviceFunctions::cmdSetStencilOp, "vkCmdSetStencilOpEXT")(commands, face, op.failOp, op.passOp, op.depthFailOp, op.compareOp); + context.Resolved(&DeviceFunctions::cmdSetStencilCompareMask, "vkCmdSetStencilCompareMask")(commands, face, op.compareMask); + context.Resolved(&DeviceFunctions::cmdSetStencilWriteMask, "vkCmdSetStencilWriteMask")(commands, face, op.writeMask); + context.Resolved(&DeviceFunctions::cmdSetStencilReference, "vkCmdSetStencilReference")(commands, face, op.reference); + }; + stencil(VK_STENCIL_FACE_FRONT_BIT, depth ? state.stencilFront : VkStencilOpState{}); + stencil(VK_STENCIL_FACE_BACK_BIT, depth ? state.stencilBack : VkStencilOpState{}); +} + +void Pipeline::ContinueBackFaces(VkCommandBuffer commands, const State& state) const { + Require(backFaces != VK_NULL_HANDLE, "the pipeline draws both faces in one pass"); + context.Resolved(&DeviceFunctions::cmdBindPipeline, "vkCmdBindPipeline")(commands, VK_PIPELINE_BIND_POINT_GRAPHICS, backFaces); + context.Resolved(&DeviceFunctions::cmdSetViewport, "vkCmdSetViewport")(commands, 0, 1, &state.viewport); + context.Resolved(&DeviceFunctions::cmdSetScissor, "vkCmdSetScissor")(commands, 0, 1, &state.scissor); + context.Resolved(&DeviceFunctions::cmdSetDepthBias, "vkCmdSetDepthBias")(commands, state.backDepthBiasConstant, state.depthBiasClamp, state.backDepthBiasSlope); + if (depthBounds) context.Resolved(&DeviceFunctions::cmdSetDepthBounds, "vkCmdSetDepthBounds")(commands, state.minDepthBounds, state.maxDepthBounds); +} + void Pipeline::PushConstants(VkCommandBuffer commands, VkShaderStageFlags stages, std::span bytes) const { if (stages == 0) return; - context.Resolved(&DeviceFunctions::cmdPushConstants, "vkCmdPushConstants")(commands, layout, stages, 0, PipelinePushConstantBytes, bytes.data()); + if (dynamicRendering) { + context.Resolved(&DeviceFunctions::cmdPushConstants, "vkCmdPushConstants")(commands, layout, pushStagesOf(context), 0, PipelinePushSlotBytes, bytes.data()); + context.Resolved(&DeviceFunctions::cmdPushConstants, "vkCmdPushConstants")(commands, layout, VK_SHADER_STAGE_FRAGMENT_BIT, PipelinePushSlotBytes, PipelinePushSlotBytes, bytes.data() + PipelinePushSlotBytes); + return; + } + context.Resolved(&DeviceFunctions::cmdPushConstants, "vkCmdPushConstants")(commands, layout, stages, 0, PipelinePushSlotBytes, bytes.data()); } namespace { @@ -371,16 +570,19 @@ void append(std::vector& key, const TValue& value) { // (the recompiler could not identify it, so nothing else may share its pipeline). The rect-list // control and evaluation stages are generated from the vertex and fragment results, which the key // already names, so they carry no id of their own. -std::vector pipelineKey(const Context& context, const State& state, const VertexInputLayout& input, const ShaderResources& resources, std::span shaders, VkImageLayout attachmentLayout) { +void pipelineKey(std::vector& key, const Context& context, const State& state, const VertexInputLayout& input, const ShaderResources& resources, std::span shaders, VkImageLayout attachmentLayout) { using Stage = ShaderRecompiler::ShaderStage; - std::vector key; + key.clear(); append(key, context.device); append(key, attachmentLayout); append(key, shaders.size()); for (const auto& shader : shaders) { Require(shader.program != nullptr, "missing compiled shader"); - const bool generated = state.rectList && (shader.stage == Stage::TessellationControl || shader.stage == Stage::TessellationEvaluation); - if (!generated && shader.program->PipelineVariantId() == 0) return {}; + const bool generated = (state.rectList && (shader.stage == Stage::TessellationControl || shader.stage == Stage::TessellationEvaluation)) || shader.stage == Stage::Geometry; + if (!generated && shader.program->PipelineVariantId() == 0) { + key.clear(); + return; + } append(key, shader.stage); append(key, generated ? std::uint64_t{0} : shader.program->PipelineVariantId()); // Where the stage's push constants sit in the block (AssemblePushConstants). @@ -408,6 +610,7 @@ std::vector pipelineKey(const Context& context, const State& state, c append(key, state.primitiveRestart); append(key, state.cullMode); append(key, state.frontFace); + append(key, state.provokingVertexMode); append(key, state.negativeOneToOne); append(key, state.depthClamp && context.depthClamp); append(key, state.conservativeRasterization); @@ -427,6 +630,7 @@ std::vector pipelineKey(const Context& context, const State& state, c append(key, state.depthCompare); append(key, state.depthBoundsTest); append(key, state.depthBias); + append(key, state.depthBiasPerFace); append(key, state.stencilTest); append(key, state.stencilFront); append(key, state.stencilBack); @@ -452,16 +656,10 @@ std::vector pipelineKey(const Context& context, const State& state, c append(key, tessellation.partitioning); append(key, tessellation.outputTopology); } - return key; } std::uint64_t hashKey(const std::vector& key) { - std::uint64_t hash = 14695981039346656037ull; - for (const auto byte : key) { - hash ^= static_cast(byte); - hash *= 1099511628211ull; - } - return hash; + return std::hash{}(std::string_view(reinterpret_cast(key.data()), key.size())); } struct PipelineStore { @@ -475,6 +673,7 @@ struct PipelineStore { std::shared_ptr pipeline; }; std::mutex mutex; + std::vector scratchKey; // Least recently used first. std::list entries; std::unordered_map::iterator> index; @@ -526,7 +725,8 @@ std::shared_ptr CachedPipeline(const Context& context, const State& st auto& store = Pipelines(); std::lock_guard lock(store.mutex); reportPipelines(store); - const auto key = pipelineKey(context, state, vertexInput, resources, shaders, attachmentLayout); + auto& key = store.scratchKey; + pipelineKey(key, context, state, vertexInput, resources, shaders, attachmentLayout); if (key.empty()) { ++store.uncached; return std::make_shared(context, state, vertexInput, resources, shaders, attachmentLayout); @@ -558,7 +758,7 @@ std::shared_ptr CachedPipeline(const Context& context, const State& st auto pipeline = std::make_shared(context, state, vertexInput, resources, shaders, attachmentLayout); store.entries.push_back({context.device, context.bufferPool, hash, key, pipeline}); store.index[hash] = std::prev(store.entries.end()); - constexpr std::size_t bound = 256; + constexpr std::size_t bound = 1024; while (store.entries.size() > bound) { // Only an entry no recorded draw still holds may go (Kept keeps its shared_ptr until the fence). const auto victim = std::find_if(store.entries.begin(), store.entries.end(), [](const PipelineStore::Entry& entry) { return entry.pipeline.use_count() == 1; }); @@ -581,6 +781,7 @@ void ClearCachedPipelines(VkDevice device) { store.index.erase(it->hash); it = store.entries.erase(it); } + ClearPipelineLibraries(device); } } diff --git a/core/libs/prx/libSceAgcDriver/Graphics/src/PipelineLibrary.cpp b/core/libs/prx/libSceAgcDriver/Graphics/src/PipelineLibrary.cpp new file mode 100644 index 000000000..e1db0d226 --- /dev/null +++ b/core/libs/prx/libSceAgcDriver/Graphics/src/PipelineLibrary.cpp @@ -0,0 +1,256 @@ +#include "prx/libSceAgcDriver/Graphics/include/PipelineLibrary.hpp" +#include "prx/libSceAgcDriver/Execution/include/PerformanceTimer.hpp" +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace AgcDriver::Graphics { + +namespace { + +constexpr std::array dynamicStates{ + VK_DYNAMIC_STATE_VIEWPORT, + VK_DYNAMIC_STATE_SCISSOR, + VK_DYNAMIC_STATE_DEPTH_BIAS, + VK_DYNAMIC_STATE_DEPTH_BOUNDS, + VK_DYNAMIC_STATE_STENCIL_COMPARE_MASK, + VK_DYNAMIC_STATE_STENCIL_WRITE_MASK, + VK_DYNAMIC_STATE_STENCIL_REFERENCE, + VK_DYNAMIC_STATE_CULL_MODE_EXT, + VK_DYNAMIC_STATE_FRONT_FACE_EXT, + VK_DYNAMIC_STATE_DEPTH_TEST_ENABLE_EXT, + VK_DYNAMIC_STATE_DEPTH_WRITE_ENABLE_EXT, + VK_DYNAMIC_STATE_DEPTH_COMPARE_OP_EXT, + VK_DYNAMIC_STATE_DEPTH_BOUNDS_TEST_ENABLE_EXT, + VK_DYNAMIC_STATE_STENCIL_TEST_ENABLE_EXT, + VK_DYNAMIC_STATE_STENCIL_OP_EXT, +}; + +using Key = std::vector; + +struct DeviceLibraries { + PFN_vkDestroyPipeline destroyPipeline = nullptr; + PFN_vkDestroyPipelineLayout destroyLayout = nullptr; + std::map layouts; + std::array, 4> parts; + PipelineLibraryCounters counters; +}; + +struct LibraryStore { + std::mutex mutex; + std::map devices; + std::condition_variable idle; + std::condition_variable work; + std::deque>> jobs; + std::map pending; + bool started = false; +}; + +LibraryStore& Libraries() { + static auto* store = new LibraryStore(); + return *store; +} + +void runJobs(LibraryStore& store) { + std::unique_lock lock(store.mutex); + for (;;) { + store.work.wait(lock, [&] { return !store.jobs.empty(); }); + auto [device, job] = std::move(store.jobs.front()); + store.jobs.pop_front(); + lock.unlock(); + job(); + lock.lock(); + if (--store.pending[device] == 0) store.idle.notify_all(); + } +} + +void enqueue(LibraryStore& store, VkDevice device, std::function job) { + if (!store.started) { + std::thread(runJobs, std::ref(store)).detach(); + store.started = true; + } + ++store.pending[device]; + store.jobs.emplace_back(device, std::move(job)); + store.work.notify_one(); +} + +Key Combined(const Key& part, const Key& context) { + Key key; + const auto append = [&](const Key& value) { + const auto size = value.size(); + const auto bytes = std::as_bytes(std::span(&size, 1)); + key.insert(key.end(), bytes.begin(), bytes.end()); + key.insert(key.end(), value.begin(), value.end()); + }; + append(part); + append(context); + return key; +} + +} + +std::span PipelineLibraryDynamicStates() { + return dynamicStates; +} + +VkPipeline LinkPipelineFromLibraries(const Context& context, const VkGraphicsPipelineCreateInfo& info, const VkPipelineRenderingCreateInfoKHR& rendering, const VkPipelineLayoutCreateInfo& layout, const PipelineLibraryKeys& keys, std::shared_ptr* optimized) { + Require(info.renderPass == VK_NULL_HANDLE && info.pDynamicState != nullptr, "pipeline library parts need dynamic rendering and the library dynamic states"); + auto& store = Libraries(); + std::lock_guard lock(store.mutex); + auto& device = store.devices[context.device]; + if (device.destroyPipeline == nullptr) { + device.destroyPipeline = context.Function("vkDestroyPipeline"); + device.destroyLayout = context.Function("vkDestroyPipelineLayout"); + } + auto pipelineLayout = device.layouts.find(keys.layout); + if (pipelineLayout == device.layouts.end()) { + VkPipelineLayout handle = VK_NULL_HANDLE; + Check(context.Function("vkCreatePipelineLayout")(context.device, &layout, nullptr, &handle), "vkCreatePipelineLayout library"); + pipelineLayout = device.layouts.emplace(keys.layout, handle).first; + } + std::vector preRasterization; + const VkPipelineShaderStageCreateInfo* fragment = nullptr; + bool mesh = false; + for (std::uint32_t index = 0; index < info.stageCount; ++index) { + const auto& stage = info.pStages[index]; + if (stage.stage == VK_SHADER_STAGE_FRAGMENT_BIT) fragment = &stage; + else preRasterization.push_back(stage); + mesh |= stage.stage == VK_SHADER_STAGE_MESH_BIT_EXT; + } + Require(fragment != nullptr && !preRasterization.empty(), "a pipeline built from libraries needs a fragment and a pre-rasterization stage"); + constexpr std::array flags{ + VK_GRAPHICS_PIPELINE_LIBRARY_VERTEX_INPUT_INTERFACE_BIT_EXT, + VK_GRAPHICS_PIPELINE_LIBRARY_PRE_RASTERIZATION_SHADERS_BIT_EXT, + VK_GRAPHICS_PIPELINE_LIBRARY_FRAGMENT_SHADER_BIT_EXT, + VK_GRAPHICS_PIPELINE_LIBRARY_FRAGMENT_OUTPUT_INTERFACE_BIT_EXT, + }; + const std::array partKeys{&keys.vertexInput, &keys.preRasterization, &keys.fragmentShader, &keys.fragmentOutput}; + const Key none; + const std::array contextKeys{&none, &keys.layout, &keys.layout, &keys.renderPass}; + const VkPipelineRenderingCreateInfoKHR shaderRendering{VK_STRUCTURE_TYPE_PIPELINE_RENDERING_CREATE_INFO_KHR}; + VkPipelineDepthStencilStateCreateInfo dynamicDepth{VK_STRUCTURE_TYPE_PIPELINE_DEPTH_STENCIL_STATE_CREATE_INFO}; + std::vector libraries; + for (std::size_t part = 0; part < flags.size(); ++part) { + if (part == 0 && mesh) continue; + const auto key = Combined(*partKeys[part], *contextKeys[part]); + auto& cache = device.parts[part]; + if (const auto found = cache.find(key); found != cache.end()) { + libraries.push_back(found->second); + continue; + } + PerformanceTimer timing("Vulkan.GraphicsPipelineLibrary"); + VkGraphicsPipelineLibraryCreateInfoEXT library{VK_STRUCTURE_TYPE_GRAPHICS_PIPELINE_LIBRARY_CREATE_INFO_EXT, flags[part] == VK_GRAPHICS_PIPELINE_LIBRARY_FRAGMENT_OUTPUT_INTERFACE_BIT_EXT ? &rendering : &shaderRendering}; + library.flags = flags[part]; + VkGraphicsPipelineCreateInfo create{VK_STRUCTURE_TYPE_GRAPHICS_PIPELINE_CREATE_INFO, &library}; + create.flags = VK_PIPELINE_CREATE_LIBRARY_BIT_KHR | VK_PIPELINE_CREATE_RETAIN_LINK_TIME_OPTIMIZATION_INFO_BIT_EXT; + create.pDynamicState = info.pDynamicState; + create.layout = pipelineLayout->second; + switch (flags[part]) { + case VK_GRAPHICS_PIPELINE_LIBRARY_VERTEX_INPUT_INTERFACE_BIT_EXT: + create.pVertexInputState = info.pVertexInputState; + create.pInputAssemblyState = info.pInputAssemblyState; + create.layout = VK_NULL_HANDLE; + break; + case VK_GRAPHICS_PIPELINE_LIBRARY_PRE_RASTERIZATION_SHADERS_BIT_EXT: + create.stageCount = static_cast(preRasterization.size()); + create.pStages = preRasterization.data(); + create.pViewportState = info.pViewportState; + create.pRasterizationState = info.pRasterizationState; + create.pTessellationState = info.pTessellationState; + break; + case VK_GRAPHICS_PIPELINE_LIBRARY_FRAGMENT_SHADER_BIT_EXT: + create.stageCount = 1; + create.pStages = fragment; + create.pMultisampleState = info.pMultisampleState; + create.pDepthStencilState = info.pDepthStencilState != nullptr ? info.pDepthStencilState : &dynamicDepth; + break; + default: + create.pMultisampleState = info.pMultisampleState; + create.pColorBlendState = info.pColorBlendState; + create.layout = VK_NULL_HANDLE; + break; + } + VkPipeline handle = VK_NULL_HANDLE; + Check(context.Function("vkCreateGraphicsPipelines")(context.device, context.pipelineCache, 1, &create, nullptr, &handle), "vkCreateGraphicsPipelines library"); + cache.emplace(key, handle); + ++device.counters.built[part]; + libraries.push_back(handle); + } + PerformanceTimer timing("Vulkan.GraphicsPipelineLink"); + VkPipelineLibraryCreateInfoKHR linked{VK_STRUCTURE_TYPE_PIPELINE_LIBRARY_CREATE_INFO_KHR}; + linked.libraryCount = static_cast(libraries.size()); + linked.pLibraries = libraries.data(); + VkGraphicsPipelineCreateInfo create{VK_STRUCTURE_TYPE_GRAPHICS_PIPELINE_CREATE_INFO, &linked}; + create.layout = info.layout; + VkPipeline pipeline = VK_NULL_HANDLE; + Check(context.Function("vkCreateGraphicsPipelines")(context.device, context.pipelineCache, 1, &create, nullptr, &pipeline), "vkCreateGraphicsPipelines link"); + ++device.counters.linked; + static const bool optimize = std::getenv("APS5_NO_PIPELINE_LTO") == nullptr; + if (optimize && optimized != nullptr) { + auto slot = std::make_shared(); + slot->device = context.device; + slot->destroy = device.destroyPipeline; + *optimized = slot; + const auto createPipelines = context.Function("vkCreateGraphicsPipelines"); + const auto cache = context.pipelineCache; + const auto pipelineLayout = info.layout; + enqueue(store, context.device, [slot, createPipelines, cache, pipelineLayout, libraries, &store, devicePointer = &device] { + if (slot->released.load()) return; + VkPipelineLibraryCreateInfoKHR linked{VK_STRUCTURE_TYPE_PIPELINE_LIBRARY_CREATE_INFO_KHR}; + linked.libraryCount = static_cast(libraries.size()); + linked.pLibraries = libraries.data(); + VkGraphicsPipelineCreateInfo create{VK_STRUCTURE_TYPE_GRAPHICS_PIPELINE_CREATE_INFO, &linked}; + create.flags = VK_PIPELINE_CREATE_LINK_TIME_OPTIMIZATION_BIT_EXT; + create.layout = pipelineLayout; + VkPipeline result = VK_NULL_HANDLE; + if (createPipelines(slot->device, cache, 1, &create, nullptr, &result) != VK_SUCCESS) return; + slot->handle.store(result); + if (slot->released.load()) { + if (const auto handle = slot->handle.exchange(VK_NULL_HANDLE)) slot->destroy(slot->device, handle, nullptr); + } + std::lock_guard lock(store.mutex); + ++devicePointer->counters.optimized; + }); + } + return pipeline; +} + +void OptimizedPipeline::Release() noexcept { + released.store(true); + if (const auto pipeline = handle.exchange(VK_NULL_HANDLE)) destroy(device, pipeline, nullptr); +} + +void WaitForOptimizedPipelines(VkDevice device) { + auto& store = Libraries(); + std::unique_lock lock(store.mutex); + store.idle.wait(lock, [&] { return store.pending[device] == 0; }); +} + +void ClearPipelineLibraries(VkDevice device) { + auto& store = Libraries(); + std::unique_lock lock(store.mutex); + store.idle.wait(lock, [&] { return store.pending[device] == 0; }); + const auto found = store.devices.find(device); + if (found == store.devices.end()) return; + auto& libraries = found->second; + for (auto& part : libraries.parts) { + for (const auto& [key, handle] : part) libraries.destroyPipeline(device, handle, nullptr); + } + for (const auto& [key, handle] : libraries.layouts) libraries.destroyLayout(device, handle, nullptr); + store.devices.erase(found); +} + +PipelineLibraryCounters PipelineLibraryCountersOf(VkDevice device) { + auto& store = Libraries(); + std::lock_guard lock(store.mutex); + const auto found = store.devices.find(device); + return found == store.devices.end() ? PipelineLibraryCounters{} : found->second.counters; +} + +} diff --git a/core/libs/prx/libSceAgcDriver/Graphics/src/Recorder.cpp b/core/libs/prx/libSceAgcDriver/Graphics/src/Recorder.cpp index 73aa04140..5297bd42f 100644 --- a/core/libs/prx/libSceAgcDriver/Graphics/src/Recorder.cpp +++ b/core/libs/prx/libSceAgcDriver/Graphics/src/Recorder.cpp @@ -3,9 +3,11 @@ #include "prx/libc/include/HostThreadLocal.hpp" #include "prx/libSceAgcDriver/Graphics/include/Recorder.hpp" #include "prx/libSceAgcDriver/Graphics/include/Resources.hpp" +#include "prx/libSceAgcDriver/Graphics/include/ParallelCompare.hpp" #include "prx/libSceAgcDriver/Graphics/include/Texture.hpp" #include "prx/libSceAgcDriver/Graphics/shaders/MeshArguments_spv.h" #include "prx/libSceAgcDriver/Graphics/shaders/SampleCounter_spv.h" +#include "prx/libSceAgcDriver/Graphics/shaders/SampleDumps_spv.h" #include "prx/libSceAgcDriver/Execution/include/GuestMemory.hpp" #include "prx/libSceAgcDriver/Execution/include/PerformanceTimer.hpp" #include "prx/libSceAgcDriver/Execution/include/Pm4Opcodes.hpp" @@ -174,6 +176,13 @@ std::atomic nextRecorderId{1}; std::atomic samplesPassed{0}; constexpr std::uint32_t SampleFoldCapacity = 64; constexpr std::uint32_t SampleCounterPushBytes = 24; +constexpr std::uint32_t SegmentSlots = 128; +constexpr std::uint32_t SegmentFoldCapacity = 1024; +constexpr std::uint32_t DumpFoldCapacity = 1024; +constexpr std::uint32_t SampleDumpBytes = 15 * 16 + 8; +constexpr VkDeviceSize DumpListOffset = 16 + 8 * SegmentFoldCapacity; +constexpr std::uint32_t SampleDumpPushBytes = 32; +std::atomic dumpsQueued{0}, dumpsQueuedInPass{0}, dumpsAtOnce{0}, segmentsClosed{0}, dumpFolds{0}; struct SamplePoolCache { VkDevice device; @@ -1150,8 +1159,11 @@ Recorder::~Recorder() { if (sampleCounterState > 0) { samplesPassed.store(SamplesTotal(), std::memory_order_release); if (samplePools != nullptr) std::static_pointer_cast(samplePools)->close(); + if (segmentPools != nullptr) std::static_pointer_cast(segmentPools)->close(); context.Function("vkDestroyPipeline")(context.device, samplePipeline, nullptr); context.Function("vkDestroyPipelineLayout")(context.device, sampleLayout, nullptr); + context.Function("vkDestroyPipeline")(context.device, dumpPipeline, nullptr); + context.Function("vkDestroyPipelineLayout")(context.device, dumpLayout, nullptr); sampleCounter.reset(); } for (auto& [commands, fence] : spare) { @@ -1360,16 +1372,27 @@ void Recorder::MarkCovered(VkAccessFlags access) { if (open != nullptr) open->coveredAccess = access; } -bool Recorder::ContinuesRenderPass(std::uint64_t key) const { - return open != nullptr && open->renderPass.open && open->renderPass.continuable && open->renderPass.key == key; +Recorder::DrawPassStart Recorder::StartDrawPass(std::uint64_t key, bool forced, bool fenced, const PassAccess& access) { + ensureOpen(); + auto& pass = open->renderPass; + if (!pass.open || forced || (countingSamples && segmentMode() && !segmentSlotReady(*open))) { + const auto commands = Commands(); + open->passHazards.Clear(); + open->passHazards.BeginPass(); + return {commands, false, true}; + } + const auto hazard = open->passHazards.Check(access, pass.key == key); + if (hazard == PassHazard::None && pass.key == key) return {open->commands, true, false}; + const bool merge = MergeBarriers() && !fenced && !pass.fenced && !pass.afterPass; + const bool elide = merge && hazard == PassHazard::None; + endOpenRenderPass(!merge); + const auto commands = Commands(); + if (!elide) open->passHazards.Clear(); + open->passHazards.BeginPass(); + return {commands, false, !elide}; } -VkCommandBuffer Recorder::CommandsInRenderPass() { - Require(open != nullptr && open->renderPass.open, "no render pass is open in the recorder"); - return open->commands; -} - -void Recorder::LeaveRenderPassOpen(std::uint64_t key, std::uint32_t timing, bool continuable, std::function afterPass) { +void Recorder::LeaveRenderPassOpen(std::uint64_t key, std::uint32_t timing, bool fenced, const PassAccess& access, std::function afterPass) { Require(open != nullptr, "no batch is open for the render pass"); auto& pass = open->renderPass; if (!pass.open) { @@ -1378,19 +1401,23 @@ void Recorder::LeaveRenderPassOpen(std::uint64_t key, std::uint32_t timing, bool } pass.open = true; pass.key = key; - pass.continuable = continuable; + pass.fenced = pass.fenced || fenced; + open->passHazards.Add(access); } -void Recorder::endOpenRenderPass() { +void Recorder::endOpenRenderPass(bool barrier) { auto& pass = open->renderPass; - context.Resolved(&DeviceFunctions::cmdEndRenderPass, "vkCmdEndRenderPass")(open->commands); + if (open->segmentActive) endSegment(*open); + EndRenderPass(context, open->commands); // The pass's attachment and shader writes are visible to everything recorded after it (the // host sees them at the batch's fence). - recordBarrier(open->commands, VK_PIPELINE_STAGE_ALL_COMMANDS_BIT, VK_PIPELINE_STAGE_ALL_COMMANDS_BIT, VK_ACCESS_SHADER_WRITE_BIT | VK_ACCESS_COLOR_ATTACHMENT_WRITE_BIT | VK_ACCESS_DEPTH_STENCIL_ATTACHMENT_WRITE_BIT | VK_ACCESS_TRANSFER_WRITE_BIT, VK_ACCESS_SHADER_READ_BIT | VK_ACCESS_SHADER_WRITE_BIT | VK_ACCESS_TRANSFER_READ_BIT | VK_ACCESS_TRANSFER_WRITE_BIT); - CountBarriers(CommandClass::Draw); + if (barrier) { + recordBarrier(open->commands, VK_PIPELINE_STAGE_ALL_COMMANDS_BIT, VK_PIPELINE_STAGE_ALL_COMMANDS_BIT, VK_ACCESS_SHADER_WRITE_BIT | VK_ACCESS_COLOR_ATTACHMENT_WRITE_BIT | VK_ACCESS_DEPTH_STENCIL_ATTACHMENT_WRITE_BIT | VK_ACCESS_TRANSFER_WRITE_BIT, VK_ACCESS_SHADER_READ_BIT | VK_ACCESS_SHADER_WRITE_BIT | VK_ACCESS_TRANSFER_READ_BIT | VK_ACCESS_TRANSFER_WRITE_BIT); + CountBarriers(CommandClass::Draw); + } if (pass.afterPass) pass.afterPass(open->commands); EndGpuTiming(pass.timing); - open->coveredAccess = VK_ACCESS_SHADER_READ_BIT | VK_ACCESS_SHADER_WRITE_BIT | VK_ACCESS_TRANSFER_READ_BIT | VK_ACCESS_TRANSFER_WRITE_BIT; + open->coveredAccess = barrier ? VK_ACCESS_SHADER_READ_BIT | VK_ACCESS_SHADER_WRITE_BIT | VK_ACCESS_TRANSFER_READ_BIT | VK_ACCESS_TRANSFER_WRITE_BIT : 0; open->hostReadOwed = true; pass = {}; } @@ -1488,7 +1515,7 @@ void Recorder::ensureOpen() { throw; } batch->queue = GuestMemory::GpuLockThreadTag(); - if (countingSamples) beginSamples(*batch); + if (countingSamples && !segmentMode()) beginSamples(*batch); open = std::move(batch); // The whole-batch range: its start stamp waits for the previous batches like any // bottom-of-pipe stamp, so it marks when this batch's execution began. @@ -1593,6 +1620,7 @@ bool Recorder::closeStoreRun(bool atSubmit) { if (LabelRunsPerBatch()) { auto stores = std::move(run.queued); run.queued.clear(); + open->queuedDumps = 0; if (stores.empty()) return false; // A label says the work before it is done: the queued key stores (results of that work) // land ahead of every store of the run, and the pass a draw left open ends. @@ -1602,7 +1630,7 @@ bool Recorder::closeStoreRun(bool atSubmit) { run.timing = beginTiming(ClassKey(CommandClass::LabelRun)); if (BarrierValidate()) { std::vector> ranges; - for (const auto& store : stores) ranges.emplace_back(store.address, store.address + store.bytes.size()); + for (const auto& store : stores) ranges.emplace_back(store.address, store.End()); NoteAccess(CommandClass::LabelRun, Access{{}, ranges, {}, VK_PIPELINE_STAGE_TRANSFER_BIT}); } // At Submit every result of the batch reaches the host before the stores (the title's @@ -1612,24 +1640,49 @@ bool Recorder::closeStoreRun(bool atSubmit) { std::uint32_t barriers = 2; std::uint64_t storedBytes = 0; const auto update = function(cmdUpdateBuffer, "vkCmdUpdateBuffer"); - for (const auto& store : stores) { - const VkDeviceSize end = store.offset + store.bytes.size(); - // Two transfers to the same bytes have no order of their own within the run. - if (std::any_of(run.recorded.begin(), run.recorded.end(), [&](const auto& earlier) { return std::get<0>(earlier) == store.buffer && store.offset < std::get<2>(earlier) && std::get<1>(earlier) < end; })) { - recordBarrier(commands, VK_PIPELINE_STAGE_TRANSFER_BIT, VK_PIPELINE_STAGE_TRANSFER_BIT, VK_ACCESS_TRANSFER_WRITE_BIT, VK_ACCESS_TRANSFER_WRITE_BIT); + std::vector writes; + writes.reserve(stores.size()); + for (const auto& store : stores) writes.push_back({store.address, store.End(), store.dumpTarget != 0}); + const bool dumps = std::any_of(writes.begin(), writes.end(), [](const QueuedWrite& write) { return write.computed; }); + if (dumps) recordQueuedDumps(commands, stores); + std::size_t first = 0; + std::uint32_t dumpIndex = 0, consumed = 0; + for (const auto end : QueuedWriteGroups(writes)) { + if (first != 0) { + recordBarrier(commands, VK_PIPELINE_STAGE_TRANSFER_BIT | VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT, VK_PIPELINE_STAGE_TRANSFER_BIT | VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT, VK_ACCESS_TRANSFER_WRITE_BIT | VK_ACCESS_SHADER_WRITE_BIT, VK_ACCESS_TRANSFER_WRITE_BIT | VK_ACCESS_SHADER_READ_BIT | VK_ACCESS_SHADER_WRITE_BIT); ++barriers; - storeWawBarriers.fetch_add(1, std::memory_order_relaxed); + run.recorded.clear(); } - update(commands, store.buffer, store.offset, store.bytes.size(), store.bytes.data()); - run.recorded.emplace_back(store.buffer, store.offset, end); - storedBytes += store.bytes.size(); + for (std::size_t i = first; i < end; ++i) { + const auto& store = stores[i]; + if (store.dumpTarget != 0) continue; + const VkDeviceSize storeEnd = store.offset + store.bytes.size(); + if (std::any_of(run.recorded.begin(), run.recorded.end(), [&](const auto& earlier) { return std::get<0>(earlier) == store.buffer && store.offset < std::get<2>(earlier) && std::get<1>(earlier) < storeEnd; })) { + recordBarrier(commands, VK_PIPELINE_STAGE_TRANSFER_BIT, VK_PIPELINE_STAGE_TRANSFER_BIT, VK_ACCESS_TRANSFER_WRITE_BIT, VK_ACCESS_TRANSFER_WRITE_BIT); + ++barriers; + storeWawBarriers.fetch_add(1, std::memory_order_relaxed); + } + update(commands, store.buffer, store.offset, store.bytes.size(), store.bytes.data()); + run.recorded.emplace_back(store.buffer, store.offset, storeEnd); + storedBytes += store.bytes.size(); + } + const auto before = dumpIndex; + foldQueuedDumps(commands, stores, first, end, dumpIndex, consumed); + if (dumpIndex != before) ++barriers; + first = end; } + if (dumps) { + retireSegments(consumed); + dumpFolds.fetch_add(1, std::memory_order_relaxed); + } + const VkPipelineStageFlags written = VK_PIPELINE_STAGE_TRANSFER_BIT | (dumps ? VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT : 0u); + const VkAccessFlags writtenAccess = VK_ACCESS_TRANSFER_WRITE_BIT | (dumps ? VK_ACCESS_SHADER_WRITE_BIT : 0u); if (atSubmit) { - recordBarrier(commands, VK_PIPELINE_STAGE_TRANSFER_BIT, VK_PIPELINE_STAGE_HOST_BIT, VK_ACCESS_TRANSFER_WRITE_BIT, VK_ACCESS_HOST_READ_BIT); + recordBarrier(commands, written, VK_PIPELINE_STAGE_HOST_BIT, writtenAccess, VK_ACCESS_HOST_READ_BIT); open->coveredAccess = 0; storeRunsAtSubmit.fetch_add(1, std::memory_order_relaxed); } else { - recordBarrier(commands, VK_PIPELINE_STAGE_TRANSFER_BIT, VK_PIPELINE_STAGE_HOST_BIT | VK_PIPELINE_STAGE_ALL_COMMANDS_BIT, VK_ACCESS_TRANSFER_WRITE_BIT, VK_ACCESS_HOST_READ_BIT | VK_ACCESS_SHADER_READ_BIT | VK_ACCESS_SHADER_WRITE_BIT | VK_ACCESS_TRANSFER_READ_BIT | VK_ACCESS_TRANSFER_WRITE_BIT); + recordBarrier(commands, written, VK_PIPELINE_STAGE_HOST_BIT | VK_PIPELINE_STAGE_ALL_COMMANDS_BIT, writtenAccess, VK_ACCESS_HOST_READ_BIT | VK_ACCESS_SHADER_READ_BIT | VK_ACCESS_SHADER_WRITE_BIT | VK_ACCESS_TRANSFER_READ_BIT | VK_ACCESS_TRANSFER_WRITE_BIT); open->coveredAccess = VK_ACCESS_HOST_READ_BIT | VK_ACCESS_SHADER_READ_BIT | VK_ACCESS_SHADER_WRITE_BIT | VK_ACCESS_TRANSFER_READ_BIT | VK_ACCESS_TRANSFER_WRITE_BIT; storeRunsForced.fetch_add(1, std::memory_order_relaxed); } @@ -1655,12 +1708,12 @@ bool Recorder::closeStoreRun(bool atSubmit) { bool Recorder::QueuedStoreOverlaps(std::uint64_t address, std::size_t bytes) const { if (open == nullptr || bytes == 0) return false; const auto end = address + bytes; - return std::any_of(open->run.queued.begin(), open->run.queued.end(), [&](const Batch::StoreRun::Queued& store) { return address < store.address + store.bytes.size() && store.address < end; }); + return std::any_of(open->run.queued.begin(), open->run.queued.end(), [&](const Batch::StoreRun::Queued& store) { return address < store.End() && store.address < end; }); } bool Recorder::AnyQueuedStore(const std::function& overlaps) const { if (open == nullptr) return false; - return std::any_of(open->run.queued.begin(), open->run.queued.end(), [&](const Batch::StoreRun::Queued& store) { return overlaps(store.address, store.address + store.bytes.size()); }); + return std::any_of(open->run.queued.begin(), open->run.queued.end(), [&](const Batch::StoreRun::Queued& store) { return overlaps(store.address, store.End()); }); } void Recorder::FlushStores() { @@ -1744,6 +1797,7 @@ void reportBarriers() { AgcDriver::ProfilePrint_nid_no_patch("; validate: would emit %llu (%s) + %llu batch ends, would skip %llu; emitted/skipped by class:%s", static_cast(emitted), kinds.c_str() + 1, static_cast(validateBatchEnds.exchange(0, std::memory_order_relaxed)), static_cast(skipped), classes.c_str()); } AgcDriver::ProfilePrint_nid_no_patch("\n"); + AgcDriver::ProfilePrint_nid_no_patch("[samples] occlusion dumps (cumulative): %llu queued (%llu inside an open pass), %llu recorded at once; %llu segments closed; %llu folds of queued dumps\n", static_cast(dumpsQueued.load(std::memory_order_relaxed)), static_cast(dumpsQueuedInPass.load(std::memory_order_relaxed)), static_cast(dumpsAtOnce.load(std::memory_order_relaxed)), static_cast(segmentsClosed.load(std::memory_order_relaxed)), static_cast(dumpFolds.load(std::memory_order_relaxed))); } } @@ -1887,18 +1941,19 @@ void Recorder::CountSamples() { GuestMemory::AssertGpuLockHeld("Recorder::CountSamples"); gpuSampleCounter(); countingSamples = true; - if (open == nullptr || open->samples != VK_NULL_HANDLE) return; + if (segmentMode() || open == nullptr || open->samples != VK_NULL_HANDLE) return; if (open->renderPass.open) endOpenRenderPass(); beginSamples(*open); } std::uint64_t Recorder::SamplesTotal() { if (sampleCounterState <= 0) return samplesPassed.load(std::memory_order_acquire); + if (segmentMode() && !Idle()) Sync(); std::uint64_t total = 0; std::memcpy(&total, sampleCounter->Bytes().data(), sizeof(total)); for (const auto& segment : pendingSamples) { std::uint64_t samples = 0; - Check(context.Function("vkGetQueryPoolResults")(context.device, segment.handle, 0, 1, sizeof(samples), &samples, sizeof(samples), VK_QUERY_RESULT_64_BIT | VK_QUERY_RESULT_WAIT_BIT), "vkGetQueryPoolResults occlusion"); + Check(context.Function("vkGetQueryPoolResults")(context.device, segment.handle, segment.slot, 1, sizeof(samples), &samples, sizeof(samples), VK_QUERY_RESULT_64_BIT | VK_QUERY_RESULT_WAIT_BIT), "vkGetQueryPoolResults occlusion"); total += samples; } pendingSamples.clear(); @@ -1906,8 +1961,13 @@ std::uint64_t Recorder::SamplesTotal() { return total; } -void Recorder::NoteSampledDraw() { - if (open != nullptr && open->sampleActive) open->samplesDrawn = true; +void Recorder::NoteSampledDraw(VkCommandBuffer commands) { + if (open == nullptr) return; + if (segmentMode()) { + if (countingSamples && commands == open->commands && !open->segmentActive) beginSegment(*open); + return; + } + if (open->sampleActive) open->samplesDrawn = true; } bool Recorder::gpuSampleCounter() { @@ -1916,7 +1976,7 @@ bool Recorder::gpuSampleCounter() { if (!context.bufferDeviceAddress) return false; VkShaderModule module = VK_NULL_HANDLE; try { - sampleCounter = std::make_unique(context, 16 + 8 * SampleFoldCapacity, VK_BUFFER_USAGE_STORAGE_BUFFER_BIT | VK_BUFFER_USAGE_TRANSFER_DST_BIT | VK_BUFFER_USAGE_SHADER_DEVICE_ADDRESS_BIT); + sampleCounter = std::make_unique(context, DumpListOffset + 16 * DumpFoldCapacity, VK_BUFFER_USAGE_STORAGE_BUFFER_BIT | VK_BUFFER_USAGE_TRANSFER_DST_BIT | VK_BUFFER_USAGE_SHADER_DEVICE_ADDRESS_BIT); std::memset(sampleCounter->Bytes().data(), 0, sampleCounter->Bytes().size()); const auto initial = samplesPassed.load(std::memory_order_acquire); std::memcpy(sampleCounter->Bytes().data(), &initial, sizeof(initial)); @@ -1937,9 +1997,25 @@ bool Recorder::gpuSampleCounter() { pipelineInfo.layout = sampleLayout; Check(context.Function("vkCreateComputePipelines")(context.device, context.pipelineCache, 1, &pipelineInfo, nullptr, &samplePipeline), "vkCreateComputePipelines sample counter"); context.Function("vkDestroyShaderModule")(context.device, module, nullptr); + module = VK_NULL_HANDLE; + const VkPushConstantRange dumpPush{VK_SHADER_STAGE_COMPUTE_BIT, 0, SampleDumpPushBytes}; + layoutInfo.pPushConstantRanges = &dumpPush; + Check(context.Function("vkCreatePipelineLayout")(context.device, &layoutInfo, nullptr, &dumpLayout), "vkCreatePipelineLayout sample dumps"); + moduleInfo.codeSize = sizeof(SAMPLE_DUMPS_SPV); + moduleInfo.pCode = SAMPLE_DUMPS_SPV; + Check(context.Function("vkCreateShaderModule")(context.device, &moduleInfo, nullptr, &module), "vkCreateShaderModule sample dumps"); + pipelineInfo.stage.module = module; + pipelineInfo.layout = dumpLayout; + Check(context.Function("vkCreateComputePipelines")(context.device, context.pipelineCache, 1, &pipelineInfo, nullptr, &dumpPipeline), "vkCreateComputePipelines sample dumps"); + context.Function("vkDestroyShaderModule")(context.device, module, nullptr); + module = VK_NULL_HANDLE; } catch (const std::exception& error) { if (module != VK_NULL_HANDLE) context.Function("vkDestroyShaderModule")(context.device, module, nullptr); + if (samplePipeline != VK_NULL_HANDLE) context.Function("vkDestroyPipeline")(context.device, samplePipeline, nullptr); + if (dumpLayout != VK_NULL_HANDLE) context.Function("vkDestroyPipelineLayout")(context.device, dumpLayout, nullptr); if (sampleLayout != VK_NULL_HANDLE) context.Function("vkDestroyPipelineLayout")(context.device, sampleLayout, nullptr); + samplePipeline = VK_NULL_HANDLE; + dumpLayout = VK_NULL_HANDLE; sampleLayout = VK_NULL_HANDLE; sampleCounter.reset(); std::fprintf(stderr, "[gpu] occlusion counter dumps drain the device: %s\n", error.what()); @@ -1949,6 +2025,167 @@ bool Recorder::gpuSampleCounter() { return true; } +std::uint64_t Recorder::Batch::StoreRun::Queued::End() const { + return address + (dumpTarget != 0 ? SampleDumpBytes : bytes.size()); +} + +std::vector Recorder::QueuedWriteGroups(std::span writes) { + std::vector ends; + std::vector> computed; + for (std::size_t i = 0; i < writes.size(); ++i) { + const auto& write = writes[i]; + if (!write.computed && std::any_of(computed.begin(), computed.end(), [&](const auto& range) { return write.begin < range.second && range.first < write.end; })) { + ends.push_back(i); + computed.clear(); + } + if (write.computed) computed.emplace_back(write.begin, write.end); + } + if (!writes.empty()) ends.push_back(writes.size()); + return ends; +} + +bool Recorder::segmentMode() const { + return sampleCounterState > 0; +} + +bool Recorder::QueuesSampleDumps() const { + return segmentMode() && LabelRunsPerBatch(); +} + +Recorder::SampleDumpStatistics Recorder::SampleDumpCounts() { + return SampleDumpStatistics{dumpsQueued.load(std::memory_order_relaxed), dumpsQueuedInPass.load(std::memory_order_relaxed), dumpsAtOnce.load(std::memory_order_relaxed), segmentsClosed.load(std::memory_order_relaxed), dumpFolds.load(std::memory_order_relaxed)}; +} + +bool Recorder::segmentSlotReady(const Batch& batch) const { + return batch.segmentActive || (batch.segments != VK_NULL_HANDLE && batch.segmentNext < SegmentSlots); +} + +void Recorder::PrepareSampleSlot() { + GuestMemory::AssertGpuLockHeld("Recorder::PrepareSampleSlot"); + if (!countingSamples || !segmentMode()) return; + ensureOpen(); + Require(!open->renderPass.open, "a sample slot is prepared inside a render pass"); + if (!segmentSlotReady(*open)) takeSegmentPool(*open); +} + +void Recorder::EndPassSamples() { + if (open != nullptr && open->segmentActive) endSegment(*open); +} + +void Recorder::takeSegmentPool(Batch& batch) { + auto cache = std::static_pointer_cast(segmentPools); + if (cache == nullptr) { + cache = std::make_shared(); + cache->device = context.device; + cache->destroy = context.Function("vkDestroyQueryPool"); + segmentPools = cache; + } + if (batch.segmentPool != nullptr) batch.kept.push_back(std::move(batch.segmentPool)); + batch.segmentPool.reset(); + batch.segments = cache->take(); + if (batch.segments == VK_NULL_HANDLE) { + VkQueryPoolCreateInfo info{VK_STRUCTURE_TYPE_QUERY_POOL_CREATE_INFO}; + info.queryType = VK_QUERY_TYPE_OCCLUSION; + info.queryCount = SegmentSlots; + Check(context.Function("vkCreateQueryPool")(context.device, &info, nullptr, &batch.segments), "vkCreateQueryPool occlusion segments"); + } + try { + batch.segmentPool = std::make_shared(cache, batch.segments); + } catch (...) { + cache->put(batch.segments); + batch.segments = VK_NULL_HANDLE; + throw; + } + batch.segmentNext = 0; + function(cmdResetQueryPool, "vkCmdResetQueryPool")(batch.commands, batch.segments, 0, SegmentSlots); +} + +void Recorder::beginSegment(Batch& batch) { + Require(batch.segments != VK_NULL_HANDLE && batch.segmentNext < SegmentSlots, "an occlusion segment begins without a reset query slot"); + batch.segmentSlot = batch.segmentNext++; + function(cmdBeginQuery, "vkCmdBeginQuery")(batch.commands, batch.segments, batch.segmentSlot, context.occlusionQueryPrecise ? VK_QUERY_CONTROL_PRECISE_BIT : 0u); + batch.segmentActive = true; +} + +void Recorder::endSegment(Batch& batch) { + function(cmdEndQuery, "vkCmdEndQuery")(batch.commands, batch.segments, batch.segmentSlot); + batch.segmentActive = false; + pendingSamples.push_back({batch.segmentPool, batch.segments, batch.segmentSlot}); + segmentsClosed.fetch_add(1, std::memory_order_relaxed); +} + +void Recorder::copySegments(VkCommandBuffer commands, std::uint32_t count) { + Require(count <= pendingSamples.size() && count <= SegmentFoldCapacity, "occlusion segments exceed the fold capacity"); + const auto copy = function(cmdCopyQueryPoolResults, "vkCmdCopyQueryPoolResults"); + for (std::uint32_t i = 0; i < count;) { + const auto& first = pendingSamples[i]; + std::uint32_t run = 1; + while (i + run < count && pendingSamples[i + run].handle == first.handle && pendingSamples[i + run].slot == first.slot + run) ++run; + copy(commands, first.handle, first.slot, run, sampleCounter->Handle(), 16 + 8 * static_cast(i), 8, VK_QUERY_RESULT_64_BIT | VK_QUERY_RESULT_WAIT_BIT); + i += run; + } +} + +void Recorder::retireSegments(std::uint32_t count) { + for (std::uint32_t i = 0; i < count; ++i) { + if (pendingSamples[i].pool != nullptr) open->kept.push_back(std::move(pendingSamples[i].pool)); + } + pendingSamples.erase(pendingSamples.begin(), pendingSamples.begin() + count); +} + +void Recorder::recordQueuedDumps(VkCommandBuffer commands, std::span writes) { + std::uint32_t segments = 0; + std::vector list; + for (const auto& write : writes) { + if (write.dumpTarget == 0) continue; + segments = write.dumpSegments; + list.insert(list.end(), {static_cast(write.dumpTarget), static_cast(write.dumpTarget >> 32u), write.dumpSegments, 0u}); + } + Require(list.size() <= 4u * DumpFoldCapacity, "queued occlusion dumps exceed the fold capacity"); + copySegments(commands, segments); + function(cmdUpdateBuffer, "vkCmdUpdateBuffer")(commands, sampleCounter->Handle(), DumpListOffset, list.size() * sizeof(std::uint32_t), list.data()); +} + +void Recorder::dispatchDumps(VkCommandBuffer commands, std::uint32_t firstDump, std::uint32_t dumps, std::uint32_t firstSegment, std::uint32_t lastSegment) { + recordBarrier(commands, VK_PIPELINE_STAGE_ALL_COMMANDS_BIT, VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT, VK_ACCESS_SHADER_WRITE_BIT | VK_ACCESS_TRANSFER_WRITE_BIT | VK_ACCESS_COLOR_ATTACHMENT_WRITE_BIT, VK_ACCESS_SHADER_READ_BIT | VK_ACCESS_SHADER_WRITE_BIT); + function(cmdBindPipeline, "vkCmdBindPipeline")(commands, VK_PIPELINE_BIND_POINT_COMPUTE, dumpPipeline); + struct { + VkDeviceAddress counter; + VkDeviceAddress list; + std::uint32_t firstDump; + std::uint32_t dumps; + std::uint32_t firstSegment; + std::uint32_t lastSegment; + } parameters{sampleCounter->DeviceAddress(), sampleCounter->DeviceAddress() + DumpListOffset, firstDump, dumps, firstSegment, lastSegment}; + static_assert(sizeof(parameters) == SampleDumpPushBytes); + function(cmdPushConstants, "vkCmdPushConstants")(commands, dumpLayout, VK_SHADER_STAGE_COMPUTE_BIT, 0, SampleDumpPushBytes, ¶meters); + function(cmdDispatch, "vkCmdDispatch")(commands, 1, 1, 1); +} + +void Recorder::foldQueuedDumps(VkCommandBuffer commands, std::span writes, std::size_t first, std::size_t end, std::uint32_t& dumpIndex, std::uint32_t& consumed) { + std::uint32_t count = 0, segments = consumed; + for (std::size_t i = first; i < end; ++i) { + if (writes[i].dumpTarget == 0) continue; + ++count; + segments = writes[i].dumpSegments; + } + if (count == 0) return; + dispatchDumps(commands, dumpIndex, count, consumed, segments); + dumpIndex += count; + consumed = segments; +} + +void Recorder::foldSegments() { + const auto commands = open->commands; + const auto count = static_cast(std::min(pendingSamples.size(), SegmentFoldCapacity)); + recordBarrier(commands, VK_PIPELINE_STAGE_ALL_COMMANDS_BIT, VK_PIPELINE_STAGE_TRANSFER_BIT, VK_ACCESS_SHADER_WRITE_BIT | VK_ACCESS_TRANSFER_WRITE_BIT, VK_ACCESS_TRANSFER_WRITE_BIT); + copySegments(commands, count); + dispatchDumps(commands, 0, 0, 0, count); + recordBarrier(commands, VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT, VK_PIPELINE_STAGE_ALL_COMMANDS_BIT | VK_PIPELINE_STAGE_HOST_BIT, VK_ACCESS_SHADER_WRITE_BIT, VK_ACCESS_SHADER_READ_BIT | VK_ACCESS_SHADER_WRITE_BIT | VK_ACCESS_TRANSFER_READ_BIT | VK_ACCESS_TRANSFER_WRITE_BIT | VK_ACCESS_HOST_READ_BIT); + CountBarriers(CommandClass::Copy, 3); + retireSegments(count); +} + void Recorder::endSamples(Batch& batch) { if (!batch.sampleActive) return; function(cmdEndQuery, "vkCmdEndQuery")(batch.commands, batch.samples, 0); @@ -1969,7 +2206,7 @@ void Recorder::foldSamples(Batch& batch, VkDeviceAddress target) { if (done != 0) recordBarrier(commands, VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT, VK_PIPELINE_STAGE_TRANSFER_BIT, VK_ACCESS_SHADER_READ_BIT | VK_ACCESS_SHADER_WRITE_BIT, VK_ACCESS_TRANSFER_WRITE_BIT); for (std::uint32_t i = 0; i < count; ++i) { const auto& segment = pendingSamples[done + i]; - copy(commands, segment.handle, 0, 1, sampleCounter->Handle(), 16 + 8 * i, 8, VK_QUERY_RESULT_64_BIT | VK_QUERY_RESULT_WAIT_BIT); + copy(commands, segment.handle, segment.slot, 1, sampleCounter->Handle(), 16 + 8 * i, 8, VK_QUERY_RESULT_64_BIT | VK_QUERY_RESULT_WAIT_BIT); if (segment.pool != nullptr) batch.kept.push_back(segment.pool); } done += count; @@ -1991,10 +2228,30 @@ void Recorder::foldSamples(Batch& batch, VkDeviceAddress target) { pendingSamples.clear(); } -bool Recorder::DumpSamples(VkDeviceAddress target) { +bool Recorder::DumpSamples(VkDeviceAddress target, std::uint64_t address) { GuestMemory::AssertGpuLockHeld("Recorder::DumpSamples"); if (target == 0 || !gpuSampleCounter()) return false; countingSamples = true; + if (segmentMode()) { + ensureOpen(); + if (address != 0 && LabelRunsPerBatch() && pendingSamples.size() < SegmentFoldCapacity && open->queuedDumps < DumpFoldCapacity) { + if (open->segmentActive) endSegment(*open); + if (open->renderPass.open) dumpsQueuedInPass.fetch_add(1, std::memory_order_relaxed); + if (open->run.queued.empty()) storeRuns.fetch_add(1, std::memory_order_relaxed); + auto& queued = open->run.queued.emplace_back(); + queued.address = address; + queued.dumpTarget = target; + queued.dumpSegments = static_cast(pendingSamples.size()); + ++open->queuedDumps; + dumpsQueued.fetch_add(1, std::memory_order_relaxed); + return true; + } + FlushStores(); + Commands(); + foldSamples(*open, target); + dumpsAtOnce.fetch_add(1, std::memory_order_relaxed); + return true; + } Commands(); endSamples(*open); foldSamples(*open, target); @@ -2219,12 +2476,16 @@ void Recorder::eraseDrawSnapshot(std::map::iterat drawSnapshots.erase(entry); } -std::shared_ptr Recorder::ReusableDrawSnapshot(std::uint64_t address, std::size_t bytes, SnapshotUse use, std::uint32_t* derived) { +std::shared_ptr Recorder::ReusableDrawSnapshot(std::uint64_t address, std::size_t bytes, SnapshotUse use, std::uint32_t* derived, std::uint64_t generation) { auto found = use == SnapshotUse::Vertex ? drawSnapshots.lower_bound({address, use, bytes}) : drawSnapshots.find({address, use, bytes}); if (found == drawSnapshots.end() || std::get<0>(found->first) != address || std::get<1>(found->first) != use) return {}; - if (found->second.registryGeneration != GuestAllocations::GuestAllocationsGeneration_nid_postfix() || !GuestMemory::UnchangedSince(address, bytes, found->second.generation)) { - eraseDrawSnapshot(found); - return {}; + const bool sameRegistry = found->second.registryGeneration == GuestAllocations::GuestAllocationsGeneration_nid_postfix(); + if (!sameRegistry || !GuestMemory::UnchangedSince(address, bytes, found->second.generation)) { + if (!sameRegistry || generation == 0 || use != SnapshotUse::Storage || !refreshDrawSnapshot(found->second, address, bytes)) { + eraseDrawSnapshot(found); + return {}; + } + found->second.generation = generation; } auto& recency = drawSnapshotPools[SnapshotPool(use)].recency; recency.splice(recency.end(), recency, found->second.recent); @@ -2232,6 +2493,45 @@ std::shared_ptr Recorder::ReusableDrawSnapshot(std::uint64_t address, st return found->second.buffer; } +bool Recorder::refreshDrawSnapshot(DrawSnapshot& entry, std::uint64_t address, std::size_t bytes) { + const auto held = entry.buffer->Bytes(); + if (held.size() != bytes || bytes == 0) return false; + constexpr std::uint64_t block = 65536; + const auto aligned = address / block * block; + const auto end = address + bytes; + const auto count = static_cast((end - 1) / block - address / block + 1); + thread_local std::vector generations; + thread_local std::vector changed; + thread_local std::vector> differing; + thread_local std::vector spans; + thread_local std::vector equal; + generations.assign(count, entry.generation); + changed.assign(count, GuestMemory::BlockWritten); + differing.clear(); + spans.clear(); + const bool sole = entry.buffer.use_count() == 1; + GuestMemory::ChangedBlocks(address, bytes, generations, changed); + for (std::size_t k = 0; k < count; ++k) { + if (changed[k] == GuestMemory::BlockUnchanged) continue; + const auto from = std::max(address, aligned + k * block); + const auto to = std::min(end, aligned + (k + 1) * block); + const auto offset = static_cast(from - address); + spans.push_back({held.data() + offset, reinterpret_cast(from), static_cast(to - from)}); + } + equal.assign(spans.size(), 0); + CompareSpans(spans, equal); + for (std::size_t k = 0; k < spans.size(); ++k) { + if (equal[k] != 0) continue; + const auto offset = static_cast(static_cast(spans[k].first) - held.data()); + const auto length = spans[k].bytes; + if (!sole) return false; + if (!differing.empty() && differing.back().first + differing.back().second == offset) differing.back().second += length; + else differing.emplace_back(offset, length); + } + for (const auto& [offset, length] : differing) std::memcpy(held.data() + offset, reinterpret_cast(address + offset), length); + return true; +} + void Recorder::KeepDrawSnapshot(std::uint64_t address, std::size_t bytes, std::uint64_t generation, std::uint64_t registryGeneration, std::shared_ptr buffer, SnapshotUse use, std::uint32_t derived) { const bool storage = use == SnapshotUse::Storage; const auto budget = storage ? DrawSnapshotBudget : DrawInputBudget; @@ -2687,7 +2987,7 @@ void Recorder::AfterCompletions(std::uint64_t address, std::span(address), copy.size(), 4, true); - GuestMemory::StoreOwnBytes(address, copy.size(), [&] { std::memcpy(reinterpret_cast(address), copy.data(), copy.size()); }); + GuestMemory::StoreOwnBytes(address, copy); }); // Counted per batch and counted down when the batch finishes, on every exit path of finish() // (a label whose range became unmapped is reported there, and a lost device throws before the @@ -2772,9 +3072,13 @@ void Recorder::Submit() { GuestMemory::AssertGpuLockHeld("Recorder::Submit"); if (!open->keyStores.empty()) recordKeyStores(false); if (open->renderPass.open) endOpenRenderPass(); - endSamples(*open); - if (pendingSamples.size() >= SampleFoldCapacity) foldSamples(*open, 0); + const bool segments = segmentMode(); + if (!segments) { + endSamples(*open); + if (pendingSamples.size() >= SampleFoldCapacity) foldSamples(*open, 0); + } const bool hostReadCovered = (open->run.open || !open->run.queued.empty()) && closeStoreRun(true); + if (segments && pendingSamples.size() >= SampleFoldCapacity) foldSegments(); if (BarrierValidate()) validateBatchEnds.fetch_add(1, std::memory_order_relaxed); if (open->hostReadOwed && !hostReadCovered) { // A fence makes nothing visible to the host: a lean draw's writes (no download barrier of @@ -3250,7 +3554,7 @@ void Recorder::finish(std::unique_ptr batch, bool wait, int source, bool AgcDriver::ProfilePrint_nid_no_patch("[recorder] completion label stores: %llu run, %llu skipped (no CPU write-back overlapped them); %llu counted pending at a write-back; %llu write-backs over a tracked label; %llu write-back completions pending\n", static_cast(completionStoresRun.load()), static_cast(completionStoresSkipped.load()), static_cast(completionLabelsCountedLate.load()), static_cast(writeBacksOverLabels.load()), static_cast(writeBackCompletions.load())); AgcDriver::ProfilePrint_nid_no_patch("[recorder] submits %llu, vkQueueSubmit mean %.1f us, max %.1f us\n", static_cast(submitCount), submitCount != 0 ? submitUs / static_cast(submitCount) : 0.0, submitMaxUs); const auto reads = Recorder::ReadCounts(); - AgcDriver::ProfilePrint_nid_no_patch("[recorder] in-place reads: %llu noted, %llu queries, hits by reader: dispatch element %llu, gpu copy %llu, address-based %llu, indirect %llu, storage upload %llu, copy source %llu; %llu hits on signaled batches ignored\n", static_cast(reads.noted), static_cast(reads.queries), static_cast(reads.hits[0]), static_cast(reads.hits[1]), static_cast(reads.hits[2]), static_cast(reads.hits[3]), static_cast(reads.hits[4]), static_cast(reads.hits[5]), static_cast(reads.staleIgnored)); + AgcDriver::ProfilePrint_nid_no_patch("[recorder] in-place reads: %llu noted, %llu queries, hits by reader: dispatch element %llu, gpu copy %llu, address-based %llu, indirect %llu, storage upload %llu, copy source %llu, draw input %llu; %llu hits on signaled batches ignored\n", static_cast(reads.noted), static_cast(reads.queries), static_cast(reads.hits[0]), static_cast(reads.hits[1]), static_cast(reads.hits[2]), static_cast(reads.hits[3]), static_cast(reads.hits[4]), static_cast(reads.hits[5]), static_cast(reads.hits[6]), static_cast(reads.staleIgnored)); } } } report{profile, source, waitStart, *this, signaledAtStart}; @@ -3361,6 +3665,8 @@ void Recorder::release(Batch& batch) noexcept { if (batch.samples != VK_NULL_HANDLE && batch.samplePool == nullptr) context.Function("vkDestroyQueryPool")(context.device, batch.samples, nullptr); batch.samples = VK_NULL_HANDLE; batch.samplePool.reset(); + batch.segments = VK_NULL_HANDLE; + batch.segmentPool.reset(); // A completed (or never submitted) batch's objects are kept for reuse: the fence is signaled or // untouched, so resetting it cannot block, and the command buffer is no longer pending. if (batch.commands != VK_NULL_HANDLE && batch.fence != VK_NULL_HANDLE && spare.size() < 64 && function(resetFences, "vkResetFences")(context.device, 1, &batch.fence) == VK_SUCCESS) { diff --git a/core/libs/prx/libSceAgcDriver/Graphics/src/Resources.cpp b/core/libs/prx/libSceAgcDriver/Graphics/src/Resources.cpp index d4849f846..361f9d540 100644 --- a/core/libs/prx/libSceAgcDriver/Graphics/src/Resources.cpp +++ b/core/libs/prx/libSceAgcDriver/Graphics/src/Resources.cpp @@ -7,24 +7,27 @@ namespace AgcDriver::Graphics { -Buffer::Buffer(const Context& context, std::size_t size, VkBufferUsageFlags usage, VkMemoryPropertyFlags properties) : context(context), size(size), capacity(BufferPool::Capacity(size)), usage(usage), properties(properties) { +Buffer::Buffer(const Context& context, std::size_t size, VkBufferUsageFlags usage, VkMemoryPropertyFlags properties) : context(context), size(size), capacity(BufferPool::Capacity(size, properties)), usage(BufferPool::Usage(usage, properties)), properties(properties) { Require(size != 0, "zero-sized GPU buffer"); const bool addressable = (usage & VK_BUFFER_USAGE_SHADER_DEVICE_ADDRESS_BIT) != 0; Require(!addressable || context.bufferDeviceAddress, "buffer device address is not enabled"); cache = GetBufferPool(context); - if (const auto allocation = cache->Take(size, usage, properties)) { + if (const auto allocation = cache->Take(size, this->usage, properties)) { buffer = allocation->buffer; memory = allocation->memory; mapping = allocation->mapping; deviceAddress = allocation->address; allocationBytes = allocation->allocationBytes; + capacity = allocation->bytes; + offset = allocation->offset; + slab = allocation->slab; ready = true; return; } try { VkBufferCreateInfo info{VK_STRUCTURE_TYPE_BUFFER_CREATE_INFO}; info.size = capacity; - info.usage = usage; + info.usage = this->usage; info.sharingMode = VK_SHARING_MODE_EXCLUSIVE; Check(context.Function("vkCreateBuffer")(context.device, &info, nullptr, &buffer), "vkCreateBuffer"); VkMemoryRequirements requirements{}; @@ -46,6 +49,18 @@ Buffer::Buffer(const Context& context, std::size_t size, VkBufferUsageFlags usag } else { allocation.memoryTypeIndex = context.MemoryType(requirements.memoryTypeBits, properties); } + if ((properties & VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT) != 0 && BufferPool::SlabEligible(capacity, requirements.alignment, requirements.size, context.limits.nonCoherentAtomSize)) { + const auto slot = cache->TakeSlot(context, allocation.memoryTypeIndex, capacity, addressable); + memory = slot.memory; + offset = slot.offset; + slab = true; + allocationBytes = capacity; + Check(context.Function("vkBindBufferMemory")(context.device, buffer, memory, offset), "vkBindBufferMemory slab"); + initializeAddress(usage); + mapping = slot.mapping; + ready = true; + return; + } Check(context.Function("vkAllocateMemory")(context.device, &allocation, nullptr, &memory), "vkAllocateMemory buffer"); Check(context.Function("vkBindBufferMemory")(context.device, buffer, memory, 0), "vkBindBufferMemory"); initializeAddress(usage); @@ -63,7 +78,12 @@ Buffer::~Buffer() { void Buffer::release() noexcept { if (ready && cache) { - cache->Put({buffer, memory, mapping, deviceAddress, allocationBytes, capacity, usage, properties}); + cache->Put({buffer, memory, mapping, deviceAddress, allocationBytes, capacity, usage, properties, offset, slab}); + return; + } + if (slab) { + if (buffer) context.Function("vkDestroyBuffer")(context.device, buffer, nullptr); + cache->PutSlot(memory, offset); return; } if (mapping) context.Function("vkUnmapMemory")(context.device, memory); @@ -84,23 +104,25 @@ void Buffer::Invalidate() { Require(mapping != nullptr, "cannot invalidate an unmapped GPU buffer"); VkMappedMemoryRange range{VK_STRUCTURE_TYPE_MAPPED_MEMORY_RANGE}; range.memory = memory; - range.size = VK_WHOLE_SIZE; + range.offset = offset; + range.size = slab ? static_cast(capacity) : VK_WHOLE_SIZE; Check(context.Function("vkInvalidateMappedMemoryRanges")(context.device, 1, &range), "vkInvalidateMappedMemoryRanges"); } -DeviceBuffer::DeviceBuffer(const Context& context, std::size_t size, VkBufferUsageFlags usage) : context(context), size(size), capacity(BufferPool::Capacity(size)), usage(usage) { +DeviceBuffer::DeviceBuffer(const Context& context, std::size_t size, VkBufferUsageFlags usage) : context(context), size(size), capacity(BufferPool::Capacity(size, VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT)), usage(BufferPool::Usage(usage, VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT)) { Require(size != 0, "zero-sized device buffer"); cache = GetBufferPool(context); - if (const auto allocation = cache->Take(size, usage, VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT)) { + if (const auto allocation = cache->Take(size, this->usage, VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT)) { buffer = allocation->buffer; memory = allocation->memory; allocationBytes = allocation->allocationBytes; + capacity = allocation->bytes; return; } try { VkBufferCreateInfo info{VK_STRUCTURE_TYPE_BUFFER_CREATE_INFO}; info.size = capacity; - info.usage = usage; + info.usage = this->usage; info.sharingMode = VK_SHARING_MODE_EXCLUSIVE; Check(context.Function("vkCreateBuffer")(context.device, &info, nullptr, &buffer), "vkCreateBuffer device"); VkMemoryRequirements requirements{}; @@ -164,6 +186,21 @@ void FillDeviceFunctions(const Context& context, DeviceFunctions& functions) { functions.cmdSetScissor = context.Function("vkCmdSetScissor"); functions.cmdSetDepthBounds = context.Function("vkCmdSetDepthBounds"); functions.cmdSetDepthBias = context.Function("vkCmdSetDepthBias"); + if (context.graphicsPipelineLibrary) { + functions.cmdBeginRendering = context.Function("vkCmdBeginRenderingKHR"); + functions.cmdEndRendering = context.Function("vkCmdEndRenderingKHR"); + functions.cmdSetCullMode = context.Function("vkCmdSetCullModeEXT"); + functions.cmdSetFrontFace = context.Function("vkCmdSetFrontFaceEXT"); + functions.cmdSetDepthTestEnable = context.Function("vkCmdSetDepthTestEnableEXT"); + functions.cmdSetDepthWriteEnable = context.Function("vkCmdSetDepthWriteEnableEXT"); + functions.cmdSetDepthCompareOp = context.Function("vkCmdSetDepthCompareOpEXT"); + functions.cmdSetDepthBoundsTestEnable = context.Function("vkCmdSetDepthBoundsTestEnableEXT"); + functions.cmdSetStencilTestEnable = context.Function("vkCmdSetStencilTestEnableEXT"); + functions.cmdSetStencilOp = context.Function("vkCmdSetStencilOpEXT"); + functions.cmdSetStencilCompareMask = context.Function("vkCmdSetStencilCompareMask"); + functions.cmdSetStencilWriteMask = context.Function("vkCmdSetStencilWriteMask"); + functions.cmdSetStencilReference = context.Function("vkCmdSetStencilReference"); + } functions.cmdBindVertexBuffers = context.Function("vkCmdBindVertexBuffers"); functions.cmdBindIndexBuffer = context.Function("vkCmdBindIndexBuffer"); functions.cmdDraw = context.Function("vkCmdDraw"); diff --git a/core/libs/prx/libSceAgcDriver/Graphics/src/Sampler.cpp b/core/libs/prx/libSceAgcDriver/Graphics/src/Sampler.cpp index 851ae3bf7..1e8dcd657 100644 --- a/core/libs/prx/libSceAgcDriver/Graphics/src/Sampler.cpp +++ b/core/libs/prx/libSceAgcDriver/Graphics/src/Sampler.cpp @@ -1,4 +1,5 @@ #include "prx/libSceAgcDriver/Graphics/include/Sampler.hpp" +#include "prx/libSceAgcDriver/Graphics/include/TextureFormat.hpp" #include #include #include @@ -10,6 +11,7 @@ namespace AgcDriver::Graphics { Require(descriptor.maxAnisotropy <= context.limits.maxSamplerAnisotropy, "guest sampler descriptor requests an anisotropy ratio beyond the device limit"); Require(descriptor.lodBias >= -context.limits.maxSamplerLodBias && descriptor.lodBias <= context.limits.maxSamplerLodBias, "guest sampler descriptor requests a LOD bias beyond the device limit"); Require(!(descriptor.unnormalizedCoordinates && descriptor.compareEnable), "guest sampler descriptor with unnormalized coordinates enables depth comparison, which is not implemented"); + forcesDegamma = descriptor.forceDegamma; VkSamplerCreateInfo info{VK_STRUCTURE_TYPE_SAMPLER_CREATE_INFO}; VkSamplerReductionModeCreateInfoEXT reduction{VK_STRUCTURE_TYPE_SAMPLER_REDUCTION_MODE_CREATE_INFO_EXT, nullptr, descriptor.reductionMode}; @@ -33,7 +35,13 @@ namespace AgcDriver::Graphics { info.minLod = descriptor.minLod; info.maxLod = descriptor.maxLod; info.borderColor = descriptor.borderColor; + const bool clampsToBorder = descriptor.addressModeU == VK_SAMPLER_ADDRESS_MODE_CLAMP_TO_BORDER || descriptor.addressModeV == VK_SAMPLER_ADDRESS_MODE_CLAMP_TO_BORDER || descriptor.addressModeW == VK_SAMPLER_ADDRESS_MODE_CLAMP_TO_BORDER; + opaqueBlackBorder = clampsToBorder && (descriptor.borderColor == VK_BORDER_COLOR_INT_OPAQUE_BLACK || descriptor.borderColor == VK_BORDER_COLOR_FLOAT_OPAQUE_BLACK); info.unnormalizedCoordinates = descriptor.unnormalizedCoordinates ? VK_TRUE : VK_FALSE; + if (descriptor.nonSeamlessCube) { + if (context.nonSeamlessCubeMap) info.flags |= VK_SAMPLER_CREATE_NON_SEAMLESS_CUBE_MAP_BIT_EXT; + else requiresNonSeamlessCube = true; + } Check(context.Function("vkCreateSampler")(context.device, &info, nullptr, &sampler), "vkCreateSampler"); } @@ -53,20 +61,25 @@ namespace AgcDriver::Graphics { return requiresFilterMinmax; } + bool Sampler::ReadsOpaqueBlackBorder() const { + return opaqueBlackBorder; + } + SamplerCache::SamplerCache(std::size_t capacity) : capacity(std::max(capacity, 1)) {} - std::shared_ptr SamplerCache::Get(const Context& context, std::span words, bool compareEnable, bool unnormalizedProven) { + std::shared_ptr SamplerCache::Get(const Context& context, std::span words, bool compareEnable, bool unnormalizedProven, bool forceDegammaPaired) { Require(words.size() == 4, "guest sampler descriptor must contain 4 dwords"); const std::array key{words[0], words[1], words[2], words[3], (compareEnable ? 1u : 0u) | (unnormalizedProven ? 2u : 0u)}; std::lock_guard lock(mutex); ++clock; if (const auto found = entries.find(key); found != entries.end()) { + Require(forceDegammaPaired || !found->second.sampler->ForcesDegamma(), "guest sampler descriptor forces sRGB decoding which is not implemented"); ++hits; found->second.lastUse = clock; return found->second.sampler; } ++misses; - auto resource = DecodeSamplerResource(words, unnormalizedProven); + auto resource = DecodeSamplerResource(words, unnormalizedProven, forceDegammaPaired); resource.compareEnable = compareEnable; auto sampler = std::make_shared(context, resource); // The cap keeps live samplers well below the device's limit (NVIDIA: ~4000); a set in flight @@ -92,4 +105,36 @@ namespace AgcDriver::Graphics { if ((properties.optimalTilingFeatures & VK_FORMAT_FEATURE_SAMPLED_IMAGE_FILTER_MINMAX_BIT_EXT) == 0) Require(false, "a sampled texture whose format " + std::to_string(format) + " does not support min/max filtering is sampled through a min or max reduction sampler with linear filtering"); } + bool Sampler::ForcesDegamma() const { + return forcesDegamma; + } + + bool Sampler::RequiresNonSeamlessCube() const { + return requiresNonSeamlessCube; + } + + void RequireDegammaFormat(std::uint32_t guestFormat, std::uint32_t samplerMask, std::span> samplers) { + for (std::uint32_t element = 0; element < 32u && element < samplers.size(); ++element) { + if (((samplerMask >> element) & 1u) != 0u && samplers[element]->ForcesDegamma() && !IsSrgbTextureFormat(guestFormat)) Require(false, "a texture of guest format " + std::to_string(guestFormat) + ", which is not sRGB, is sampled through a sampler that forces sRGB decoding, which is not implemented"); + } + } + + void RequireBorderSwizzle(std::uint32_t bcSwizzle, std::uint32_t samplerMask, std::span> samplers) { + constexpr std::uint32_t BcSwizzleXyzw = 0u; + constexpr std::uint32_t BcSwizzleZyxw = 4u; + if (bcSwizzle == BcSwizzleXyzw || bcSwizzle == BcSwizzleZyxw) return; + for (std::uint32_t element = 0; element < 32u; ++element) { + if (((samplerMask >> element) & 1u) == 0u) continue; + if (element >= samplers.size()) Require(false, "a sampled texture is paired with sampler element " + std::to_string(element) + ", which its shader does not bind"); + if (samplers[element]->ReadsOpaqueBlackBorder()) Require(false, "a sampled texture whose BC swizzle " + std::to_string(bcSwizzle) + " moves the alpha channel is sampled through an opaque black border, which is not implemented"); + } + } + + void RequireNonSeamlessCube(bool cube, std::uint32_t samplerMask, std::span> samplers) { + if (!cube) return; + for (std::uint32_t element = 0; element < 32u && element < samplers.size(); ++element) { + if (((samplerMask >> element) & 1u) != 0u && samplers[element]->RequiresNonSeamlessCube()) Require(false, "a cube texture is sampled through a sampler that disables seamless cube filtering, which needs VK_EXT_non_seamless_cube_map"); + } + } + } diff --git a/core/libs/prx/libSceAgcDriver/Graphics/src/ShaderAddressResources.cpp b/core/libs/prx/libSceAgcDriver/Graphics/src/ShaderAddressResources.cpp index 895fa359d..04601fb8e 100644 --- a/core/libs/prx/libSceAgcDriver/Graphics/src/ShaderAddressResources.cpp +++ b/core/libs/prx/libSceAgcDriver/Graphics/src/ShaderAddressResources.cpp @@ -15,8 +15,8 @@ void ShaderResources::prepareAddressBindings(std::span sha if (binding.role == ShaderRecompiler::DescriptorRole::BdaPagetable) ++tables; else ++faults; } - const bool rectListFault = shader.stage == ShaderRecompiler::ShaderStage::TessellationControl && tables == 0 && faults == 1; - Require((tables == faults || rectListFault) && tables <= 1 && faults <= 1, "invalid BDA table and fault descriptors"); + const bool faultOnly = tables == 0 && faults == 1 && shader.program->bdaAbiVersion == ShaderRecompiler::BdaAbi::Version; + Require((tables == faults || faultOnly) && tables <= 1 && faults <= 1, "invalid BDA table and fault descriptors"); Require(shader.program->bdaAbiVersion == (faults == 0 ? 0u : ShaderRecompiler::BdaAbi::Version), "incompatible BDA ABI version"); // Rect-list validation needs a fault buffer, but never accesses guest addresses. usesBda = usesBda || tables != 0; diff --git a/core/libs/prx/libSceAgcDriver/Graphics/src/ShaderInputState.cpp b/core/libs/prx/libSceAgcDriver/Graphics/src/ShaderInputState.cpp index 2d7b65e3c..1939acc1e 100644 --- a/core/libs/prx/libSceAgcDriver/Graphics/src/ShaderInputState.cpp +++ b/core/libs/prx/libSceAgcDriver/Graphics/src/ShaderInputState.cpp @@ -5,6 +5,7 @@ #include "prx/libSceAgcDriver/Execution/include/GuestMemory.hpp" #include #include +#include #include #include #include @@ -38,14 +39,14 @@ std::uint32_t read(const Registers& registers, std::uint32_t offset, RegisterBan return it->second; } -template T _readHeaderPod(std::span header, std::uint64_t headerAddress, const void* pointer) { +template T _readHeaderPod(std::span header, std::uint64_t headerAddress, const void* pointer, std::size_t bytes = sizeof(T)) { if (pointer == nullptr) throw std::runtime_error("AGC graphics: null AGC header pointer"); const auto address = reinterpret_cast(pointer); if (address < headerAddress) throw std::runtime_error("AGC graphics: AGC header pointer precedes the shader header"); const auto offset = address - headerAddress; - if (offset + sizeof(T) > header.size()) throw std::runtime_error("AGC graphics: AGC header pointer is outside the registered shader header"); - T value; - std::memcpy(&value, header.data() + offset, sizeof(T)); + if (offset > header.size() || bytes > header.size() - offset) throw std::runtime_error("AGC graphics: AGC header pointer is outside the registered shader header"); + T value{}; + std::memcpy(&value, header.data() + offset, bytes); return value; } @@ -106,7 +107,7 @@ ShaderRecompiler::ShaderPixelStageInfo DecodePixelStageInfo(const Registers& con constexpr std::uint32_t knownMask = PixelInputBit(PixelInput::PerspectiveSample) | PixelInputBit(PixelInput::PerspectiveCenter) | PixelInputBit(PixelInput::PerspectiveCentroid) | PixelInputBit(PixelInput::LinearSample) | PixelInputBit(PixelInput::LinearCenter) | PixelInputBit(PixelInput::LinearCentroid) | PixelInputBit(PixelInput::PositionX) | PixelInputBit(PixelInput::PositionY) | PixelInputBit(PixelInput::PositionZ) | PixelInputBit(PixelInput::PositionW) | - PixelInputBit(PixelInput::FrontFace) | PixelInputBit(PixelInput::Ancillary); + PixelInputBit(PixelInput::FrontFace) | PixelInputBit(PixelInput::Ancillary) | PixelInputBit(PixelInput::LineStipple) | PixelInputBit(PixelInput::PositionFixedPoint); if ((activeInputs & ~knownMask) != 0) { char message[128]; std::snprintf(message, sizeof(message), "AGC graphics: unsupported SPI_PS_INPUT_ENA/ADDR bit combination (ena 0x%x addr 0x%x)", ena, addr); @@ -157,7 +158,7 @@ ShaderRecompiler::ShaderPixelStageInfo DecodePixelStageInfo(const Registers& con .conservativeZExport = static_cast(conservativeZExport), .orderedPixelShader = ((shaderControl >> 16u) & 0x1u) != 0, .targetOutputMode = targetOutputMode, - .targetExportMapping = exportMappings + .targetExportMapping = nullProgram ? std::array{} : exportMappings }; } @@ -167,7 +168,8 @@ ShaderRecompiler::ShaderVertexStageInfo DecodeVertexStageInfo(std::span(header, headerAddress, shader.user_data); + constexpr auto userDataBytes = offsetof(ShaderUserData, sharp_resource_count) + sizeof(ShaderUserData{}.sharp_resource_count); + const auto userDataHeader = _readHeaderPod(header, headerAddress, shader.user_data, userDataBytes); if (userDataHeader.direct_resource_count > ShaderRegs::AGC_DIRECT_RESOURCE_TYPE_COUNT) throw std::runtime_error("AGC graphics: AGC direct-resource count exceeds the known resource domain"); std::array directOffsets{}; directOffsets.fill(ShaderRegs::AGC_ILLEGAL_DIRECT_OFFSET); diff --git a/core/libs/prx/libSceAgcDriver/Graphics/src/ShaderResources.cpp b/core/libs/prx/libSceAgcDriver/Graphics/src/ShaderResources.cpp index d79c2e918..8afcbb6c1 100644 --- a/core/libs/prx/libSceAgcDriver/Graphics/src/ShaderResources.cpp +++ b/core/libs/prx/libSceAgcDriver/Graphics/src/ShaderResources.cpp @@ -29,6 +29,8 @@ #include #include #include +#include +#include #include "prx/libc/include/GuestAllocations.hpp" namespace AgcDriver::Graphics { @@ -83,6 +85,8 @@ TextureKey MakeTextureKey(VkDevice device, std::span words, TextureKey key{device, {}, {static_cast(components.r), static_cast(components.g), static_cast(components.b), static_cast(components.a)}}; key.depthCompare = depthCompare; std::copy(words.begin(), words.end(), key.words.begin()); + key.words[5] &= ~((0xfffu << 8u) | (1u << 25u)); + key.words[6] &= ~0xffu; return key; } @@ -100,6 +104,7 @@ struct CachedTexture { std::shared_ptr source; std::uint64_t sourceVersion = 0; std::uint64_t accounted = 0; + std::uint64_t present = 0; }; // Entries in use order (front = most recent) with a hash index by key: a lookup is O(1) and the @@ -111,11 +116,24 @@ struct TextureCache { std::unordered_map::iterator, TextureKeyHash> index; std::unordered_multimap::iterator> views; std::uint64_t bytes = 0; + std::size_t passed = 0; + std::chrono::steady_clock::time_point passedSince{}; + std::uint64_t firstUsesPresent = 0; + std::unordered_set firstUses; + struct Counts { + std::uint64_t inserted = 0; + std::uint64_t uncached = 0; + std::uint64_t evicted = 0; + std::uint64_t passed = 0; + std::uint64_t stale = 0; + } counts; + std::chrono::steady_clock::time_point countsReported = std::chrono::steady_clock::now(); VkPhysicalDevice budgetDevice = VK_NULL_HANDLE; std::chrono::steady_clock::time_point budgetRead{}; std::uint64_t budget = 0; std::uint64_t reportedBudget = 0; std::chrono::steady_clock::time_point reportedAt{}; + std::uint64_t erasures = 0; }; TextureCache& Textures() { @@ -141,6 +159,11 @@ void noteUncached(const std::shared_ptr& image) { uncached.any.store(true, std::memory_order_release); } +bool SamplerCacheDisabled() { + static const bool disabled = std::getenv("APS5_NO_SAMPLER_CACHE") != nullptr; + return disabled; +} + bool TextureHashEnabled() { static const bool disabled = std::getenv("APS5_NO_TEXTURE_HASH") != nullptr; return !disabled; @@ -169,6 +192,7 @@ void eraseTexture(TextureCache& cache, std::list::iterator it) { } cache.index.erase(it->key); cache.entries.erase(it); + ++cache.erasures; } void dropUncachedViews(TextureCache& cache) { @@ -190,6 +214,99 @@ void dropUncachedViews(TextureCache& cache) { // Moves an entry to the front (most recently used). void touchTexture(TextureCache& cache, std::list::iterator it) { cache.entries.splice(cache.entries.begin(), cache.entries, it); + it->present = Recorder::Presents(); +} + +bool admitTexture(TextureCache& cache, const TextureKey& key, std::uint64_t incoming, std::uint64_t budget) { + if (cache.bytes + incoming <= budget) return true; + if (const auto now = std::chrono::steady_clock::now(); now - cache.passedSince >= std::chrono::seconds(1)) { + cache.passed = 0; + cache.passedSince = now; + } + const auto present = Recorder::Presents(); + if (cache.firstUsesPresent != present) { + cache.firstUses.clear(); + cache.firstUsesPresent = present; + } + const bool firstUse = incoming != 0 && present != 0 && !cache.firstUses.contains(key); + const auto ceiling = budget + budget / 4u; + while (!cache.entries.empty() && cache.bytes + incoming > budget) { + const auto back = std::prev(cache.entries.end()); + if (firstUse && back->present + 1u >= present) { + cache.firstUses.insert(key); + ++cache.counts.uncached; + return false; + } + const bool over = cache.bytes + incoming > ceiling; + const bool keep = back->accounted == 0 || (!over && back->texture.use_count() > 1); + if (keep && cache.passed < cache.entries.size()) { + cache.entries.splice(cache.entries.begin(), cache.entries, back); + ++cache.passed; + ++cache.counts.passed; + } else if (keep && !over) { + break; + } else { + eraseTexture(cache, back); + ++cache.counts.evicted; + } + } + return true; +} + +void reportCacheCounts(TextureCache& cache, std::uint64_t budget) { + static const bool profile = std::getenv("APS5_PROFILE_DRAW") != nullptr; + if (!profile) return; + const auto now = std::chrono::steady_clock::now(); + if (now - cache.countsReported < std::chrono::seconds(10)) return; + cache.countsReported = now; + const auto counts = std::exchange(cache.counts, {}); + AgcDriver::ProfilePrint_nid_no_patch("[texture-cache] (10 s): %llu textures cached, %llu used uncached (first use this frame, the least recently used entry used this frame or the last), %llu evicted, %llu held ones passed over, %llu found changed; %llu MiB in %zu entries, budget %llu MiB, frame %llu\n", static_cast(counts.inserted), static_cast(counts.uncached), static_cast(counts.evicted), static_cast(counts.passed), static_cast(counts.stale), static_cast(cache.bytes >> 20u), cache.entries.size(), static_cast(budget >> 20u), static_cast(Recorder::Presents())); +} + +struct ImageMemoEntry { + VkDevice device = VK_NULL_HANDLE; + std::array words{}; + GuestTextureResource resource{}; + std::uint64_t guestBytes = 0; + VkComponentMapping components{}; + bool hasEntry = false; + std::list::iterator entry{}; + std::uint64_t erasures = 0; +}; + +constexpr std::size_t ImageMemoSlots = 1024; +thread_local std::unique_ptr> imageMemo; + +ImageMemoEntry& imageMemoSlot(std::span words) { + if (imageMemo == nullptr) imageMemo = std::make_unique>(); + const auto hash = hashWords(14695981039346656037ull, words.data(), words.size()); + return (*imageMemo)[(hash ^ (hash >> 29u)) % ImageMemoSlots]; +} + +const ImageMemoEntry& decodedImage(VkDevice device, std::span words) { + auto& slot = imageMemoSlot(words); + if (slot.device == device && std::equal(words.begin(), words.end(), slot.words.begin())) return slot; + slot.device = VK_NULL_HANDLE; + slot.resource = DecodeTextureResource(words); + slot.guestBytes = DescribeSurface(slot.resource).guestBytes; + slot.components = ViewComponents(slot.resource); + std::copy(words.begin(), words.end(), slot.words.begin()); + slot.hasEntry = false; + slot.device = device; + return slot; +} + +std::list::iterator findSampledEntry(TextureCache& cache, VkDevice device, const std::array& words, VkComponentMapping components) { + auto& slot = imageMemoSlot(words); + const bool same = slot.device == device && slot.words == words && slot.components.r == components.r && slot.components.g == components.g && slot.components.b == components.b && slot.components.a == components.a; + if (same && slot.hasEntry && slot.erasures == cache.erasures) return slot.entry; + const auto it = findTexture(cache, MakeTextureKey(device, words, components)); + if (same) { + slot.hasEntry = it != cache.entries.end(); + slot.entry = it; + slot.erasures = cache.erasures; + } + return it; } // APS5_PROFILE_DRAW: what the sampled-texture and storage-image lookups did, printed as [textures] @@ -522,6 +639,7 @@ std::shared_ptr cachedTexture(const Context& context, std::span(source != nullptr ? 0u : bytes), nullptr, *keys, generation}; if (source != nullptr) { @@ -545,13 +663,22 @@ std::shared_ptr cachedTexture(const Context& context, std::span budget) eraseTexture(cache, std::prev(cache.entries.end())); - cache.bytes += entry.accounted; auto texture = entry.texture; + if (!admitTexture(cache, key, entry.accounted, budget)) { + logLookup({texture.get(), resource, guestBytes, *keys, generation, source.get()}); + reportCacheCounts(cache, budget); + reportTextureCounters(); + if (profile) LookupOutcomes::Add(source != nullptr ? LookupOutcomes::SampledMadeView : LookupOutcomes::SampledMadeSnapshot, start); + return texture; + } + cache.bytes += entry.accounted; + entry.present = Recorder::Presents(); + ++cache.counts.inserted; logLookup({texture.get(), resource, guestBytes, *keys, generation, source.get()}); cache.entries.push_front(std::move(entry)); cache.index[key] = cache.entries.begin(); if (source != nullptr) cache.views.emplace(source.get(), cache.entries.begin()); + reportCacheCounts(cache, budget); reportTextureCounters(); if (profile) LookupOutcomes::Add(source != nullptr ? LookupOutcomes::SampledMadeView : LookupOutcomes::SampledMadeSnapshot, start); return texture; @@ -633,7 +760,7 @@ void evictStorage(StorageTextureCache& cache, std::list::i // mode): the image holds the whole mip chain, and render targets in the same memory attach to it. std::array SurfaceKey(const Context& context, const GuestTextureResource& resource) { // Guest formats that store in the same Vulkan format share the image (views carry the difference). - return {static_cast(resource.baseAddress), static_cast(resource.baseAddress >> 32u), resource.width, resource.height, (resource.depthOrLastArray << 16u) | (resource.mipCount & 0xffffu), (static_cast(resource.tileMode) << 12u) | (static_cast(resource.dimension) << 20u), resource.baseArray, static_cast(StorageFormatForGuest(context, resource.format))}; + return {static_cast(resource.baseAddress | resource.pipeBankXor), static_cast(resource.baseAddress >> 32u), resource.width, resource.height, (resource.depthOrLastArray << 16u) | (resource.mipCount & 0xffffu), (static_cast(resource.tileMode) << 12u) | (static_cast(resource.dimension) << 20u), resource.baseArray, static_cast(StorageFormatForGuest(context, resource.format))}; } struct ExtendedSurfaces { @@ -778,6 +905,14 @@ std::shared_ptr CachedSampledTexture(const Context& context, std::span< return cachedTexture(context, words, resource, ViewComponents(resource)); } +bool SampledTexturesShareEntry(std::span first, std::span second) { + Require(first.size() == 8 && second.size() == 8, "sampled texture keys take eight descriptor words"); + const VkComponentMapping components{VK_COMPONENT_SWIZZLE_R, VK_COMPONENT_SWIZZLE_G, VK_COMPONENT_SWIZZLE_B, VK_COMPONENT_SWIZZLE_A}; + const auto firstKey = MakeTextureKey(VK_NULL_HANDLE, first, components); + const auto secondKey = MakeTextureKey(VK_NULL_HANDLE, second, components); + return firstKey == secondKey && TextureKeyHash{}(firstKey) == TextureKeyHash{}(secondKey); +} + void FlushCachedTextures(VkDevice device) { Require(device != VK_NULL_HANDLE, "cannot flush textures without a Vulkan device"); GuestMemory::AssertGpuLockHeld("FlushCachedTextures"); @@ -896,11 +1031,12 @@ bool SameAsPreviousStorageElement(const ShaderRecompiler::DescriptorBinding& bin } -ShaderResources::ShaderResources(const Context& context, const ShaderRecompiler::RecompileResult& vertex, const ShaderRecompiler::RecompileResult& fragment, const ColorTarget& target, std::uint64_t indexAddress, std::size_t indexBytes) : ShaderResources(context, std::array{{{ShaderRecompiler::ShaderStage::Vertex, &vertex, 0}, {ShaderRecompiler::ShaderStage::Fragment, &fragment, static_cast(vertex.pushConstants.size())}}}, target, indexAddress, indexBytes) {} +ShaderResources::ShaderResources(const Context& context, const ShaderRecompiler::RecompileResult& vertex, const ShaderRecompiler::RecompileResult& fragment, const ColorTarget& target, std::uint64_t indexAddress, std::size_t indexBytes) : ShaderResources(context, std::array{{{ShaderRecompiler::ShaderStage::Vertex, &vertex, 0}, {ShaderRecompiler::ShaderStage::Fragment, &fragment, static_cast(vertex.pushConstants.size())}}}, target, 1, indexAddress, indexBytes) {} -ShaderResources::ShaderResources(const Context& context, std::span shaders, const ColorTarget& target, std::uint64_t indexAddress, std::size_t indexBytes, std::span snapshots) : context(context), guestMemory(context) { +ShaderResources::ShaderResources(const Context& context, std::span shaders, const ColorTarget& target, std::uint32_t colorAttachments, std::uint64_t indexAddress, std::size_t indexBytes, std::span snapshots) : context(context), guestMemory(context) { + drawBuild = true; prepareAddressBindings(shaders, snapshots); - build(shaders, &target, indexAddress, indexBytes); + build(shaders, &target, colorAttachments, indexAddress, indexBytes); } ShaderResources::ShaderResources(const Context& context, const CompiledShader& compute, std::span snapshots) : ShaderResources(context, compute, snapshots, false) {} @@ -915,7 +1051,7 @@ bool UsesAddressTables(const CompiledShader& compute) { } -ShaderResources::ShaderResources(const Context& context, const CompiledShader& compute, std::span snapshots, bool deferred) : context(context), guestMemory(context), deferredCompute(compute), deferredSnapshots(snapshots) { +ShaderResources::ShaderResources(const Context& context, const CompiledShader& compute, std::span snapshots, bool deferred, std::uint64_t dispatchThreads) : context(context), guestMemory(context), dispatchThreads(dispatchThreads), deferredCompute(compute), deferredSnapshots(snapshots) { Require(compute.stage == ShaderRecompiler::ShaderStage::Compute, "compute resources require a compute shader"); // Every use of a compute build is a recorded dispatch that calls MarkGpuWrites, which staged // buffers need (a synchronous draw's use would not). @@ -923,7 +1059,7 @@ ShaderResources::ShaderResources(const Context& context, const CompiledShader& c const std::span shaders(&deferredCompute, 1); if (!deferred) { prepareAddressBindings(shaders, snapshots); - build(shaders, nullptr, 0, 0); + build(shaders, nullptr, 0, 0, 0); forgetDeferredInputs(); return; } @@ -935,7 +1071,7 @@ ShaderResources::ShaderResources(const Context& context, const CompiledShader& c if (lockedBuild) return; unlockedPrepare = true; prepareAddressBindings(shaders, snapshots); - buildPrepare(shaders, nullptr, 0, 0); + buildPrepare(shaders, nullptr, 0, 0, 0); } void ShaderResources::Complete() { @@ -943,7 +1079,7 @@ void ShaderResources::Complete() { const std::span shaders(&deferredCompute, 1); if (lockedBuild) { prepareAddressBindings(shaders, deferredSnapshots); - buildPrepare(shaders, nullptr, 0, 0); + buildPrepare(shaders, nullptr, 0, 0, 0); } buildComplete(); forgetDeferredInputs(); @@ -957,13 +1093,40 @@ void ShaderResources::forgetDeferredInputs() { deferredSnapshots = {}; } -void ShaderResources::build(std::span shaders, const ColorTarget* target, std::uint64_t indexAddress, std::size_t indexBytes) { - buildPrepare(shaders, target, indexAddress, indexBytes); +void ShaderResources::build(std::span shaders, const ColorTarget* target, std::uint32_t colorAttachments, std::uint64_t indexAddress, std::size_t indexBytes) { + buildPrepare(shaders, target, colorAttachments, indexAddress, indexBytes); buildComplete(); } namespace { +VkDescriptorSetLayout CreateSetLayout(const Context& context, std::span bindings, bool updateAfterBind) { + std::vector flags; + for (const auto& binding : bindings) flags.push_back(updateAfterBind && binding.descriptorType == VK_DESCRIPTOR_TYPE_STORAGE_BUFFER ? VK_DESCRIPTOR_BINDING_UPDATE_AFTER_BIND_BIT : 0); + VkDescriptorSetLayoutBindingFlagsCreateInfo bindingFlags{VK_STRUCTURE_TYPE_DESCRIPTOR_SET_LAYOUT_BINDING_FLAGS_CREATE_INFO}; + bindingFlags.bindingCount = static_cast(flags.size()); + bindingFlags.pBindingFlags = flags.data(); + VkDescriptorSetLayoutCreateInfo info{VK_STRUCTURE_TYPE_DESCRIPTOR_SET_LAYOUT_CREATE_INFO}; + info.pNext = updateAfterBind ? &bindingFlags : nullptr; + info.flags = updateAfterBind ? VK_DESCRIPTOR_SET_LAYOUT_CREATE_UPDATE_AFTER_BIND_POOL_BIT : 0; + info.bindingCount = static_cast(bindings.size()); + info.pBindings = bindings.data(); + VkDescriptorSetLayout layout = VK_NULL_HANDLE; + Check(context.Function("vkCreateDescriptorSetLayout")(context.device, &info, nullptr, &layout), "vkCreateDescriptorSetLayout"); + return layout; +} + +VkDescriptorPool CreateSetPool(const Context& context, std::span sizes, bool updateAfterBind) { + VkDescriptorPoolCreateInfo info{VK_STRUCTURE_TYPE_DESCRIPTOR_POOL_CREATE_INFO}; + info.flags = updateAfterBind ? VK_DESCRIPTOR_POOL_CREATE_UPDATE_AFTER_BIND_BIT : 0; + info.maxSets = 1; + info.poolSizeCount = static_cast(sizes.size()); + info.pPoolSizes = sizes.data(); + VkDescriptorPool pool = VK_NULL_HANDLE; + Check(context.Function("vkCreateDescriptorPool")(context.device, &info, nullptr, &pool), "vkCreateDescriptorPool"); + return pool; +} + // APS5_PROFILE_DRAW: the [resources] phase totals. Each thread accumulates its builds' phases in // arrays of its own and merges them into the shared totals every 1000 of its builds (and when it // ends), so no build takes the shared mutex per phase; the totals lag by up to 999 builds per @@ -1044,7 +1207,8 @@ double ShaderResources::phase(BuildPhase which) { namespace { -void ValidateDescriptorLayout(const Context& context, std::span bindings) { +void ValidateDescriptorLayout(const Context& context, std::span bindings, bool updateAfterBind) { + const auto maxStageResources = updateAfterBind ? context.descriptorIndexingLimits.maxPerStageUpdateAfterBindResources : context.limits.maxPerStageResources; std::map heapTotals; std::map, std::uint64_t> stageTotals; std::map stageResources; @@ -1055,7 +1219,7 @@ void ValidateDescriptorLayout(const Context& context, std::span shaders, const ColorTarget* target, std::uint64_t indexAddress, std::size_t indexBytes) { +void ShaderResources::buildPrepare(std::span shaders, const ColorTarget* target, std::uint32_t colorAttachments, std::uint64_t indexAddress, std::size_t indexBytes) { PerformanceTimer frameTiming("Resources.Prepare"); const auto stageStart = std::chrono::steady_clock::now(); phaseStart = stageStart; @@ -1112,11 +1276,18 @@ void ShaderResources::buildPrepare(std::span shaders, cons ShaderRecompiler::RuntimeAbi::RequireVersion(shader.program->runtimeAbiVersion); const VkShaderStageFlags flags = VulkanStage(shader.stage); std::uint64_t stageStorageBuffers = 0; - std::uint64_t stageResources = 0; + std::uint64_t stageResources = shader.stage == ShaderRecompiler::ShaderStage::Fragment ? colorAttachments : 0; + Require(stageResources <= context.limits.maxPerStageResources, "shader descriptors exceed per-stage limits"); std::vector offsetsInData; std::int64_t shaderData = -1; const auto firstSampler = samplers.size(); const auto firstDeferred = deferredImages.size(); + const auto requireStageLimits = [&] { + if (stageStorageBuffers <= context.limits.maxPerStageDescriptorStorageBuffers && stageResources <= context.limits.maxPerStageResources) return; + const auto& limits = context.descriptorIndexingLimits; + Require(stageStorageBuffers <= limits.maxPerStageDescriptorUpdateAfterBindStorageBuffers && stageResources <= limits.maxPerStageUpdateAfterBindResources, "shader descriptors exceed per-stage limits"); + updateAfterBind = true; + }; for (const auto& binding : shader.program->bindings) { Require(binding.descriptorSet == 0, "unexpected descriptor set: every shader resource must use descriptor set zero"); Require(occupied.insert(binding.binding).second, "duplicate shader binding"); @@ -1125,7 +1296,7 @@ void ShaderResources::buildPrepare(std::span shaders, cons const bool imageRole = binding.role == ShaderRecompiler::DescriptorRole::GuestImages || binding.role == ShaderRecompiler::DescriptorRole::GuestSamplers; if (imageRole) { if (binding.kind == ShaderRecompiler::DescriptorKind::SampledImage || binding.kind == ShaderRecompiler::DescriptorKind::StorageImage) stageResources += binding.count; - Require(stageResources <= context.limits.maxPerStageResources, "shader descriptors exceed per-stage limits"); + requireStageLimits(); addImageBinding(binding, flags); continue; } @@ -1136,7 +1307,7 @@ void ShaderResources::buildPrepare(std::span shaders, cons stageStorageBuffers += binding.count; stageResources += binding.count; storageBuffers += binding.count; - Require(stageStorageBuffers <= context.limits.maxPerStageDescriptorStorageBuffers && stageResources <= context.limits.maxPerStageResources, "shader descriptors exceed per-stage limits"); + requireStageLimits(); Binding item{{binding.binding, VK_DESCRIPTOR_TYPE_STORAGE_BUFFER, binding.count, flags, nullptr}, {}}; if (binding.role == ShaderRecompiler::DescriptorRole::GuestBuffers) { Require(binding.guestDescriptor.size() == static_cast(binding.count) * 4, "guest buffer descriptor must contain four DWORDs per array element"); @@ -1145,7 +1316,8 @@ void ShaderResources::buildPrepare(std::span shaders, cons // vector) counts as written, like imageWritten below. const bool written = element >= binding.bufferWritten.size() || binding.bufferWritten[element]; const bool atomic = element < binding.bufferAtomic.size() && binding.bufferAtomic[element]; - const auto index = addGuestBuffer(std::span(binding.guestDescriptor).subspan(static_cast(element) * 4, 4), target, indexAddress, indexBytes, written, atomic); + const bool read = element < binding.bufferRead.size() && binding.bufferRead[element]; + const auto index = addGuestBuffer(std::span(binding.guestDescriptor).subspan(static_cast(element) * 4, 4), target, indexAddress, indexBytes, written, atomic, read); const auto& push = shader.program->pushConstants; if (!push.empty()) { const auto position = shader.program->memoryOffsetDword * 4u + element; @@ -1183,11 +1355,15 @@ void ShaderResources::buildPrepare(std::span shaders, cons deferredImages[index].samplerCount = samplers.size() - firstSampler; } } - Require(storageBuffers <= context.limits.maxDescriptorSetStorageBuffers, "pipeline descriptors exceed device limits"); + if (storageBuffers > context.limits.maxDescriptorSetStorageBuffers) { + Require(storageBuffers <= context.descriptorIndexingLimits.maxDescriptorSetUpdateAfterBindStorageBuffers, "pipeline descriptors exceed device limits"); + updateAfterBind = true; + } timing.bindingsMs = phase(BuildPhase::Bindings); // For every build, locked ones included: their stage B then takes the fast path too, and the // collects cost the same wherever they run. if (precollectImages()) phase(BuildPhase::Precollect); + if (drawBuild && !usesBda && std::all_of(allocations.begin(), allocations.end(), [&](const Allocation& allocation) { return !allocation.guest || allocation.pushByte >= 0 || (allocation.dataAllocation >= 0 && allocation.dataByte < allocations[static_cast(allocation.dataAllocation)].size); })) guestMemory.AllowAdjustedRegions(); guestMemory.UploadPrepare(usesBda); timing.uploadMs = phase(BuildPhase::Upload); std::vector description; @@ -1195,16 +1371,14 @@ void ShaderResources::buildPrepare(std::span shaders, cons description.push_back(binding.layout); layoutKey.insert(layoutKey.end(), {binding.layout.binding, static_cast(binding.layout.descriptorType), binding.layout.descriptorCount, binding.layout.stageFlags}); } + if (updateAfterBind) layoutKey.push_back(VK_DESCRIPTOR_SET_LAYOUT_CREATE_UPDATE_AFTER_BIND_POOL_BIT); static const bool noLayoutCache = std::getenv("APS5_NO_LAYOUT_CACHE") != nullptr; static const bool noPoolCache = std::getenv("APS5_NO_POOL_CACHE") != nullptr; if (context.descriptorCache != nullptr && !noLayoutCache) { - _layout = context.descriptorCache->Layout(layoutKey, description); + _layout = context.descriptorCache->Layout(layoutKey, description, updateAfterBind); } else { - ValidateDescriptorLayout(context, description); - VkDescriptorSetLayoutCreateInfo info{VK_STRUCTURE_TYPE_DESCRIPTOR_SET_LAYOUT_CREATE_INFO}; - info.bindingCount = static_cast(description.size()); - info.pBindings = description.data(); - Check(context.Function("vkCreateDescriptorSetLayout")(context.device, &info, nullptr, &_layout), "vkCreateDescriptorSetLayout"); + ValidateDescriptorLayout(context, description, updateAfterBind); + _layout = CreateSetLayout(context, description, updateAfterBind); ownsLayout = true; } if (!bindings.empty()) { @@ -1216,16 +1390,12 @@ void ShaderResources::buildPrepare(std::span shaders, cons if (plannedStorageImages != 0) sizes.push_back({VK_DESCRIPTOR_TYPE_STORAGE_IMAGE, plannedStorageImages}); if (!samplers.empty()) sizes.push_back({VK_DESCRIPTOR_TYPE_SAMPLER, static_cast(samplers.size())}); if (context.descriptorCache != nullptr && !noPoolCache) { - const auto allocated = context.descriptorCache->Allocate(_layout, sizes); + const auto allocated = context.descriptorCache->Allocate(_layout, sizes, updateAfterBind); _set = allocated.set; cachePool = allocated.pool; } if (_set == VK_NULL_HANDLE) { - VkDescriptorPoolCreateInfo poolInfo{VK_STRUCTURE_TYPE_DESCRIPTOR_POOL_CREATE_INFO}; - poolInfo.maxSets = 1; - poolInfo.poolSizeCount = static_cast(sizes.size()); - poolInfo.pPoolSizes = sizes.data(); - Check(context.Function("vkCreateDescriptorPool")(context.device, &poolInfo, nullptr, &pool), "vkCreateDescriptorPool"); + pool = CreateSetPool(context, sizes, updateAfterBind); VkDescriptorSetAllocateInfo allocation{VK_STRUCTURE_TYPE_DESCRIPTOR_SET_ALLOCATE_INFO}; allocation.descriptorPool = pool; allocation.descriptorSetCount = 1; @@ -1252,8 +1422,12 @@ void ShaderResources::buildComplete() { try { // The lookups run in plan order: Revalidate walks the bindings the same way, and consecutive // storage elements of one mip chain share the previous element's image. + previousSampled = {}; + for (auto& source : samplerSources) source.singleLevelImage = source.mipmappedImage = false; for (const auto& deferred : deferredImages) resolveImageBinding(*deferred.binding, bindings[deferred.index], std::span>(samplers).subspan(deferred.firstSampler, deferred.samplerCount)); + previousSampled = {}; deferredImages.clear(); + applyAnisoOverride(); // The records served their purpose: each holds the cache entry's objects as of stage A, // which would otherwise keep a replaced texture or an evicted storage image (and its device // memory) alive, outside the caches' budgets, for as long as this object is cached. @@ -1276,6 +1450,7 @@ void ShaderResources::buildComplete() { bufferCount += binding.allocations.size(); if (binding.layout.descriptorType != VK_DESCRIPTOR_TYPE_STORAGE_BUFFER) imageCount += binding.layout.descriptorCount; } + allocationDescriptors.resize(allocations.size()); std::vector buffers; std::vector images; buffers.reserve(bufferCount); @@ -1291,7 +1466,10 @@ void ShaderResources::buildComplete() { switch (binding.layout.descriptorType) { case VK_DESCRIPTOR_TYPE_STORAGE_BUFFER: write.pBufferInfo = buffers.data() + buffers.size(); - for (const auto index : binding.allocations) buffers.push_back(descriptor(allocations[index])); + for (const auto index : binding.allocations) { + allocationDescriptors[index] = descriptor(allocations[index]); + buffers.push_back(allocationDescriptors[index]); + } break; case VK_DESCRIPTOR_TYPE_SAMPLED_IMAGE: write.pImageInfo = images.data() + images.size(); @@ -1402,6 +1580,12 @@ void ShaderResources::noteReusable() { reusable = true; } +bool ShaderResources::NeverReusable(std::span shaders) { + return std::any_of(shaders.begin(), shaders.end(), [](const CompiledShader& shader) { + return shader.program != nullptr && std::any_of(shader.program->bindings.begin(), shader.program->bindings.end(), [](const ShaderRecompiler::DescriptorBinding& binding) { return binding.role == ShaderRecompiler::DescriptorRole::BdaPagetable || binding.role == ShaderRecompiler::DescriptorRole::FaultBuffer; }); + }); +} + // APS5_PROFILE_DRAW: the per-device descriptor caches' counters, every 10 s. void ShaderResources::reportDescriptorCaches() const { static std::mutex reportMutex; @@ -2354,24 +2538,21 @@ DescriptorCache::DescriptorCache(const Context& context) : context(context), des DescriptorCache::~DescriptorCache() { for (const auto pool : pools) destroyPool(context.device, pool, nullptr); + for (const auto pool : updateAfterBindPools) destroyPool(context.device, pool, nullptr); for (const auto pool : dedicated) destroyPool(context.device, pool, nullptr); for (const auto& [key, layout] : layouts) destroyLayout(context.device, layout, nullptr); } -VkDescriptorSetLayout DescriptorCache::Layout(std::span key, std::span bindings) { +VkDescriptorSetLayout DescriptorCache::Layout(std::span key, std::span bindings, bool updateAfterBind) { std::lock_guard lock(mutex); std::vector keyCopy(key.begin(), key.end()); if (const auto found = layouts.find(keyCopy); found != layouts.end()) { ++stats.layoutHits; return found->second; } - ValidateDescriptorLayout(context, bindings); + ValidateDescriptorLayout(context, bindings, updateAfterBind); ++stats.layoutMisses; - VkDescriptorSetLayoutCreateInfo info{VK_STRUCTURE_TYPE_DESCRIPTOR_SET_LAYOUT_CREATE_INFO}; - info.bindingCount = static_cast(bindings.size()); - info.pBindings = bindings.data(); - VkDescriptorSetLayout layout = VK_NULL_HANDLE; - Check(context.Function("vkCreateDescriptorSetLayout")(context.device, &info, nullptr, &layout), "vkCreateDescriptorSetLayout"); + const auto layout = CreateSetLayout(context, bindings, updateAfterBind); layouts.emplace(std::move(keyCopy), layout); return layout; } @@ -2393,10 +2574,11 @@ constexpr std::array ChainPoolSizes{{{VK_DESCRIPTOR_TYP constexpr std::array ChainPoolCreateSizes{{{VK_DESCRIPTOR_TYPE_STORAGE_BUFFER, 4096 + 2}, {VK_DESCRIPTOR_TYPE_SAMPLED_IMAGE, 1024}, {VK_DESCRIPTOR_TYPE_STORAGE_IMAGE, 1024}, {VK_DESCRIPTOR_TYPE_SAMPLER, 512}}}; } -DescriptorCache::SetAllocation DescriptorCache::Allocate(VkDescriptorSetLayout layout, std::span sizes) { +DescriptorCache::SetAllocation DescriptorCache::Allocate(VkDescriptorSetLayout layout, std::span sizes, bool updateAfterBind) { bool fits = true; for (const auto& size : sizes) { - Require(size.descriptorCount <= SetDescriptorLimit(context.limits, size.type), "descriptor set exceeds the device's per-set descriptor limit"); + const auto limit = updateAfterBind && size.type == VK_DESCRIPTOR_TYPE_STORAGE_BUFFER ? context.descriptorIndexingLimits.maxDescriptorSetUpdateAfterBindStorageBuffers : SetDescriptorLimit(context.limits, size.type); + Require(size.descriptorCount <= limit, "descriptor set exceeds the device's per-set descriptor limit"); const auto capacity = std::find_if(ChainPoolSizes.begin(), ChainPoolSizes.end(), [&](const auto& item) { return item.type == size.type; }); if (capacity == ChainPoolSizes.end() || size.descriptorCount > capacity->descriptorCount) fits = false; } @@ -2407,12 +2589,7 @@ DescriptorCache::SetAllocation DescriptorCache::Allocate(VkDescriptorSetLayout l allocation.pSetLayouts = &layout; if (!fits) { dedicated.reserve(dedicated.size() + 1); - VkDescriptorPoolCreateInfo poolInfo{VK_STRUCTURE_TYPE_DESCRIPTOR_POOL_CREATE_INFO}; - poolInfo.maxSets = 1; - poolInfo.poolSizeCount = static_cast(sizes.size()); - poolInfo.pPoolSizes = sizes.data(); - VkDescriptorPool pool = VK_NULL_HANDLE; - Check(context.Function("vkCreateDescriptorPool")(context.device, &poolInfo, nullptr, &pool), "vkCreateDescriptorPool"); + const auto pool = CreateSetPool(context, sizes, updateAfterBind); allocation.descriptorPool = pool; VkDescriptorSet set = VK_NULL_HANDLE; const auto result = allocate(context.device, &allocation, &set); @@ -2427,7 +2604,8 @@ DescriptorCache::SetAllocation DescriptorCache::Allocate(VkDescriptorSetLayout l } // Newest pool first: it has the most room; a full or fragmented pool is left for its sets to // drain and tried again later. - for (auto it = pools.rbegin(); it != pools.rend(); ++it) { + auto& chain = updateAfterBind ? updateAfterBindPools : pools; + for (auto it = chain.rbegin(); it != chain.rend(); ++it) { allocation.descriptorPool = *it; VkDescriptorSet set = VK_NULL_HANDLE; const auto result = allocate(context.device, &allocation, &set); @@ -2438,13 +2616,13 @@ DescriptorCache::SetAllocation DescriptorCache::Allocate(VkDescriptorSetLayout l if (result != VK_ERROR_OUT_OF_POOL_MEMORY && result != VK_ERROR_FRAGMENTED_POOL) Check(result, "vkAllocateDescriptorSets"); } VkDescriptorPoolCreateInfo poolInfo{VK_STRUCTURE_TYPE_DESCRIPTOR_POOL_CREATE_INFO}; - poolInfo.flags = VK_DESCRIPTOR_POOL_CREATE_FREE_DESCRIPTOR_SET_BIT; + poolInfo.flags = VK_DESCRIPTOR_POOL_CREATE_FREE_DESCRIPTOR_SET_BIT | (updateAfterBind ? VK_DESCRIPTOR_POOL_CREATE_UPDATE_AFTER_BIND_BIT : 0); poolInfo.maxSets = ChainPoolSets; poolInfo.poolSizeCount = static_cast(ChainPoolCreateSizes.size()); poolInfo.pPoolSizes = ChainPoolCreateSizes.data(); VkDescriptorPool pool = VK_NULL_HANDLE; Check(context.Function("vkCreateDescriptorPool")(context.device, &poolInfo, nullptr, &pool), "vkCreateDescriptorPool"); - pools.push_back(pool); + chain.push_back(pool); ++stats.pools; allocation.descriptorPool = pool; VkDescriptorSet set = VK_NULL_HANDLE; @@ -2469,7 +2647,7 @@ DescriptorCache::Stats DescriptorCache::Counters() const { return stats; } -std::size_t ShaderResources::addGuestBuffer(std::span words, const ColorTarget* target, std::uint64_t indexAddress, std::size_t indexBytes, bool written, bool atomic) { +std::size_t ShaderResources::addGuestBuffer(std::span words, const ColorTarget* target, std::uint64_t indexAddress, std::size_t indexBytes, bool written, bool atomic, bool read) { Require(words.size() == 4, "buffer descriptor must contain four DWORDs"); Require((words[1] & 0x40000000u) == 0, "buffer descriptor has reserved bits set"); const ShaderRecompiler::ShaderBufferResource descriptor{{words[0], words[1], words[2], words[3]}}; @@ -2488,7 +2666,8 @@ std::size_t ShaderResources::addGuestBuffer(std::span words static const bool allWritten = std::getenv("APS5_ALL_BUFFERS_WRITTEN") != nullptr; Require(!written || !overlap(address, size, indexAddress, indexBytes), "writable shader buffer aliases the index buffer"); written = written || allWritten; - if (written) guestMemory.AddWritable(address, size, atomic); + const bool swept = read && dispatchThreads != 0 && descriptor.Stride() != 0 && dispatchThreads * descriptor.Stride() * 2u >= byteSize; + if (written) guestMemory.AddWritable(address, size, atomic, swept); else { guestMemory.AddReadable(address, size); ++readOnlyBuffers; @@ -2707,14 +2886,14 @@ void ShaderResources::addImageBinding(const ShaderRecompiler::DescriptorBinding& const auto words = std::span(binding.guestDescriptor).subspan(static_cast(element) * elementWords, elementWords); const bool compareEnable = binding.samplerDepthCompare.at(element); const bool unnormalized = element < binding.samplerUnnormalized.size() && binding.samplerUnnormalized[element]; - static const bool noSamplerCache = std::getenv("APS5_NO_SAMPLER_CACHE") != nullptr; - if (context.samplerCache != nullptr && !noSamplerCache) { - samplers.push_back(context.samplerCache->Get(context, words, compareEnable, unnormalized)); + if (context.samplerCache != nullptr && !SamplerCacheDisabled()) { + samplers.push_back(context.samplerCache->Get(context, words, compareEnable, unnormalized, true)); } else { - auto resource = DecodeSamplerResource(words, unnormalized); + auto resource = DecodeSamplerResource(words, unnormalized, true); resource.compareEnable = compareEnable; samplers.push_back(std::make_shared(context, resource)); } + samplerSources.push_back({{words[0], words[1], words[2], words[3]}, compareEnable, unnormalized}); item.imageAllocations.push_back(samplers.size() - 1); } Require(samplers.size() <= context.limits.maxDescriptorSetSamplers, "pipeline sampler descriptors exceed device limits"); @@ -2765,20 +2944,36 @@ bool ShaderResources::precollectImages() { const auto elementWords = binding.guestDescriptor.size() / binding.count; for (std::uint32_t element = 0; element < binding.count; ++element) { const auto words = std::span(binding.guestDescriptor).subspan(static_cast(element) * elementWords, elementWords); + const bool sampled = binding.kind == ShaderRecompiler::DescriptorKind::SampledImage; + if (sampled && !imageRecords.empty()) { + const auto& previous = imageRecords.back(); + if (previous.sampled && previous.decoded && words.size() == 8 && std::equal(words.begin(), words.end(), previous.words.begin()) && (binding.imageDepthCompare.empty() || !binding.imageDepthCompare.at(element))) { + ImageRecord repeated; + repeated.sampled = true; + repeated.decoded = previous.decoded; + repeated.words = previous.words; + repeated.resource = previous.resource; + repeated.guestBytes = previous.guestBytes; + repeated.components = previous.components; + imageRecords.push_back(repeated); + continue; + } + } ImageRecord record; - record.sampled = binding.kind == ShaderRecompiler::DescriptorKind::SampledImage; + record.sampled = sampled; if (words[0] != 0u || (words[1] & 0xffu) != 0u) { - record.resource = DecodeTextureResource(words); - record.guestBytes = DescribeSurface(record.resource).guestBytes; + const auto* memo = words.size() == 8 ? &decodedImage(context.device, words) : nullptr; + record.resource = memo != nullptr ? memo->resource : DecodeTextureResource(words); + record.guestBytes = memo != nullptr ? memo->guestBytes : DescribeSurface(record.resource).guestBytes; record.generation = GuestMemory::CollectWrites(record.resource.baseAddress, static_cast(record.guestBytes)); record.decoded = true; if (record.sampled && !noRecords && words.size() == 8 && (binding.imageDepthCompare.empty() || !binding.imageDepthCompare.at(element))) { std::copy(words.begin(), words.end(), record.words.begin()); - record.components = ViewComponents(record.resource); + record.components = memo->components; record.keys = TextureClearKeys(record.resource, record.guestBytes); auto& cache = Textures(); std::lock_guard lock(cache.mutex); - if (const auto it = findTexture(cache, MakeTextureKey(context.device, words, record.components)); it != cache.entries.end()) { + if (const auto it = findSampledEntry(cache, context.device, record.words, record.components); it != cache.entries.end()) { record.texture = it->texture; record.source = it->source; record.entryKeys = it->keys; @@ -2813,6 +3008,7 @@ std::shared_ptr ShaderResources::fastTexture(const ImageRecord& record) // between the stages marks them uncompressed). const auto address = record.resource.baseAddress; const auto bytes = static_cast(record.guestBytes); + if (DepthSurfaceAt(address)) return nullptr; if (GuestMemory::CollectWrites(address, bytes) == 0) return nullptr; if (PendingStorageOverlaps(address, bytes, record.source.get())) return nullptr; auto keys = record.keys; @@ -2839,7 +3035,7 @@ std::shared_ptr ShaderResources::fastTexture(const ImageRecord& record) // next lookup would make another), and it takes the stage-A generation like a hit would. auto& cache = Textures(); std::lock_guard lock(cache.mutex); - const auto it = findTexture(cache, MakeTextureKey(context.device, record.words, record.components)); + const auto it = findSampledEntry(cache, context.device, record.words, record.components); if (it == cache.entries.end() || it->texture != record.texture) return nullptr; if (it->source == nullptr) it->generation = record.generation; touchTexture(cache, it); @@ -2848,6 +3044,22 @@ std::shared_ptr ShaderResources::fastTexture(const ImageRecord& record) return record.texture; } +void ShaderResources::applyAnisoOverride() { + for (std::size_t index = 0; index < samplerSources.size(); ++index) { + const auto& source = samplerSources[index]; + const auto variant = SingleLevelSamplerWords(source.words, source.singleLevelImage, source.mipmappedImage); + if (!variant.has_value()) continue; + const auto& words = *variant; + if (context.samplerCache != nullptr && !SamplerCacheDisabled()) { + samplers[index] = context.samplerCache->Get(context, words, source.compareEnable, source.unnormalized); + } else { + auto resource = DecodeSamplerResource(words, source.unnormalized); + resource.compareEnable = source.compareEnable; + samplers[index] = std::make_shared(context, resource); + } + } +} + void ShaderResources::resolveImageBinding(const ShaderRecompiler::DescriptorBinding& binding, Binding& item, std::span> shaderSamplers) { auto& counters = TextureCounts(); // The element's stage-A record, when the pass ran (records follow the plan order exactly). @@ -2864,22 +3076,41 @@ void ShaderResources::resolveImageBinding(const ShaderRecompiler::DescriptorBind continue; } const auto resource = record != nullptr && record->decoded ? record->resource : DecodeTextureResource(words); + RequireDegammaFormat(resource.format, binding.imageSamplers[element], shaderSamplers); + RequireNonSeamlessCube(resource.dimension == TextureDimension::kCube, binding.imageSamplers[element], shaderSamplers); const bool firstLayer = binding.imageShape == ShaderRecompiler::DescriptorImageShape::Image2D && resource.dimension == TextureDimension::k2DArray; if (!firstLayer && !MatchesGuestDimension(*binding.imageShape, resource.dimension)) throw std::runtime_error("AGC graphics: guest texture dimension disagrees with the shader's declared image shape (shape " + std::to_string(static_cast(*binding.imageShape)) + ", dimension " + std::to_string(static_cast(resource.dimension)) + ")"); const VkComponentMapping components = ViewComponents(resource); const auto guestBytes = record != nullptr && record->decoded ? record->guestBytes : DescribeSurface(resource).guestBytes; std::shared_ptr texture; - if (record != nullptr && record->texture != nullptr) { + const bool depthCompare = !binding.imageDepthCompare.empty() && binding.imageDepthCompare.at(element); + if (previousSampled.texture != nullptr && previousSampled.depthCompare == depthCompare && words.size() == 8 && std::equal(words.begin(), words.end(), previousSampled.words.begin())) texture = previousSampled.texture; + if (texture == nullptr && record != nullptr && record->texture != nullptr) { texture = fastTexture(*record); (texture != nullptr ? counters.fastHits : counters.fastMisses).fetch_add(1, std::memory_order_relaxed); } - if (texture == nullptr) texture = cachedTexture(context, words, resource, components, guestBytes, !binding.imageDepthCompare.empty() && binding.imageDepthCompare.at(element)); + if (texture == nullptr) texture = cachedTexture(context, words, resource, components, guestBytes, depthCompare); if (element < binding.imageUnnormalized.size() && binding.imageUnnormalized[element]) { const auto range = texture->SampledViewRange(firstLayer); const bool singleLevel = range.levels == 1u && range.layers == 1u && resource.baseLevel == 0u && EffectiveMinLod(resource) == 0.0f; if (!singleLevel || (range.type != VK_IMAGE_VIEW_TYPE_1D && range.type != VK_IMAGE_VIEW_TYPE_2D)) throw std::runtime_error("AGC graphics: guest texture sampled with unnormalized coordinates is not a single-level, single-layer 1D or 2D view starting at mip 0, which is not implemented (base level " + std::to_string(resource.baseLevel) + ", levels " + std::to_string(range.levels) + ", layers " + std::to_string(range.layers) + ", view type " + std::to_string(static_cast(range.type)) + ")"); } RequireFilterMinmax(context, texture->ViewFormat(), binding.imageSamplers[element], shaderSamplers); + RequireBorderSwizzle(resource.bcSwizzle, binding.imageSamplers[element], shaderSamplers); + if (!shaderSamplers.empty()) { + const bool singleLevel = DescriptorSingleLevel(words); + const auto firstSampler = static_cast(shaderSamplers.data() - samplers.data()); + for (std::uint32_t sampler = 0; sampler < shaderSamplers.size() && sampler < 32u; ++sampler) { + if (((binding.imageSamplers[element] >> sampler) & 1u) == 0) continue; + auto& source = samplerSources.at(firstSampler + sampler); + (singleLevel ? source.singleLevelImage : source.mipmappedImage) = true; + } + } + if (words.size() == 8) { + std::copy(words.begin(), words.end(), previousSampled.words.begin()); + previousSampled.depthCompare = depthCompare; + previousSampled.texture = texture; + } textures.push_back(std::move(texture)); textureFirstLayer.push_back(firstLayer); describedRanges.push_back({"texture", resource.baseAddress, guestBytes, resource.width, resource.height, resource.format, static_cast(resource.tileMode), resource.dccAddress}); @@ -2889,6 +3120,7 @@ void ShaderResources::resolveImageBinding(const ShaderRecompiler::DescriptorBind reportTextureCounters(); return; } + previousSampled = {}; // Consecutive identical storage descriptors address successive mips of one texture (dynamic-mip // storage writes). std::uint32_t mipOffset = 0; @@ -3076,7 +3308,7 @@ std::shared_ptr ShaderResources::PrepareDrawBindi const auto bytes = static_cast(GuestBufferMemory::ViewBytes(size, item.adjustment)); const auto registryGeneration = GuestAllocations::GuestAllocationsGeneration_nid_postfix(); const auto generation = GuestMemory::CollectWrites(begin, bytes); - auto buffer = recorder.ReusableDrawSnapshot(begin, bytes); + auto buffer = recorder.ReusableDrawSnapshot(begin, bytes, Recorder::SnapshotUse::Storage, nullptr, generation); if (buffer == nullptr) { buffer = std::make_shared(context, bytes, VK_BUFFER_USAGE_STORAGE_BUFFER_BIT | VK_BUFFER_USAGE_TRANSFER_SRC_BIT); std::memcpy(buffer->Bytes().data(), reinterpret_cast(begin), bytes); @@ -3107,23 +3339,33 @@ std::shared_ptr ShaderResources::PrepareDrawBindi } const auto update = context.Resolved(&DeviceFunctions::updateDescriptorSets, "vkUpdateDescriptorSets"); update(context.device, 0, nullptr, static_cast(copies.size()), copies.data()); + std::vector snapshotIndices(allocations.size(), selected.size()); + for (std::size_t index = 0; index < selected.size(); ++index) snapshotIndices[selected[index]] = index; + std::size_t infoCount = 0; + for (const auto& binding : bindings) infoCount += binding.allocations.size(); std::vector infos; - infos.reserve(selected.size()); - for (const auto& snapshot : result->snapshots) infos.push_back({snapshot.buffer->Handle(), 0, snapshot.buffer->Bytes().size()}); + infos.reserve(infoCount); std::vector writes; for (const auto& binding : bindings) { - for (std::size_t element = 0; element < binding.allocations.size(); ++element) { - const auto found = std::find(selected.begin(), selected.end(), binding.allocations[element]); - if (found == selected.end()) continue; - VkWriteDescriptorSet write{VK_STRUCTURE_TYPE_WRITE_DESCRIPTOR_SET}; - write.dstSet = result->allocation.set; - write.dstBinding = binding.layout.binding; - write.dstArrayElement = static_cast(element); - write.descriptorCount = 1; - write.descriptorType = binding.layout.descriptorType; - write.pBufferInfo = &infos[static_cast(found - selected.begin())]; - writes.push_back(write); + if (std::none_of(binding.allocations.begin(), binding.allocations.end(), + [&](std::size_t index) { return snapshotIndices[index] != selected.size(); })) continue; + const auto first = infos.size(); + for (const auto index : binding.allocations) { + const auto snapshotIndex = snapshotIndices[index]; + if (snapshotIndex == selected.size()) infos.push_back(allocationDescriptors[index]); + else { + const auto& snapshot = result->snapshots[snapshotIndex]; + infos.push_back({snapshot.buffer->Handle(), 0, snapshot.buffer->Bytes().size()}); + } } + VkWriteDescriptorSet write{VK_STRUCTURE_TYPE_WRITE_DESCRIPTOR_SET}; + write.dstSet = result->allocation.set; + write.dstBinding = binding.layout.binding; + write.dstArrayElement = 0; + write.descriptorCount = static_cast(binding.allocations.size()); + write.descriptorType = binding.layout.descriptorType; + write.pBufferInfo = infos.data() + first; + writes.push_back(write); } update(context.device, static_cast(writes.size()), writes.data(), 0, nullptr); recorder.Keep(result); @@ -3187,10 +3429,6 @@ void ShaderResources::WriteBackBuffers() { guestMemory.WriteBack(); } -bool ShaderResources::WritesMemory() const { - return HoldsLease() || NeedsCompletion() || !guestMemory.Writes().empty() || std::any_of(storageWritten.begin(), storageWritten.end(), [](bool written) { return written; }); -} - bool ShaderResources::ReadsOverlap(std::uint64_t address, std::size_t bytes) const { const auto reads = guestMemory.InPlaceReads(); return std::any_of(reads.begin(), reads.end(), [&](const auto& range) { return address < range.second && range.first < address + bytes; }); diff --git a/core/libs/prx/libSceAgcDriver/Graphics/src/ShaderValidation.cpp b/core/libs/prx/libSceAgcDriver/Graphics/src/ShaderValidation.cpp index 527ef1087..e880bddaf 100644 --- a/core/libs/prx/libSceAgcDriver/Graphics/src/ShaderValidation.cpp +++ b/core/libs/prx/libSceAgcDriver/Graphics/src/ShaderValidation.cpp @@ -19,6 +19,7 @@ struct Decoration { std::optional set; std::optional binding; std::optional stride; + std::optional index; bool block = false; bool patch = false; bool perPrimitive = false; @@ -40,6 +41,7 @@ struct Module { std::set interface; std::map inputs; std::map outputs; + std::optional secondSource; bool position = false; bool primitiveIndices = false; bool fragmentBarycentric = false; @@ -102,6 +104,11 @@ struct Module { Require(input == (stage == Stage::TessellationEvaluation), "invalid tessellation level direction"); return; } + if (stage == Stage::Geometry) { + Require(value == spv::BuiltInPosition && Signature(type) == "f32x4" && (input || !position), "unsupported geometry built-in"); + if (!input) position = true; + return; + } if (stage == Stage::TessellationControl || stage == Stage::TessellationEvaluation || stage == Stage::Mesh) { const auto signature = Signature(type); if (!input && value == spv::BuiltInPosition) { @@ -129,7 +136,7 @@ struct Module { } }; -Module Inspect(const CompiledShader& compiled, const State& state, const VkPhysicalDeviceSubgroupProperties& subgroup, bool fragmentShaderBarycentric, bool descriptorIndexing, bool imageInt64Atomics, bool geometryShader, bool sampleRateShading) { +Module Inspect(const CompiledShader& compiled, const State& state, const VkPhysicalDeviceSubgroupProperties& subgroup, bool fragmentShaderBarycentric, bool descriptorIndexing, bool imageInt64Atomics, bool geometryShader, bool sampleRateShading, bool bufferInt64Atomics) { using Stage = ShaderRecompiler::ShaderStage; Require(compiled.program != nullptr, "missing compiled shader"); const auto& shader = *compiled.program; @@ -140,6 +147,7 @@ Module Inspect(const CompiledShader& compiled, const State& state, const VkPhysi const bool mesh = stage == Stage::Mesh; const bool control = stage == Stage::TessellationControl; const bool evaluation = stage == Stage::TessellationEvaluation; + const bool geometry = stage == Stage::Geometry; const bool rectListControl = state.rectList && control; if (rectListControl) { Require(shader.bdaAbiVersion == ShaderRecompiler::BdaAbi::Version, "incompatible rect-list fault ABI version"); @@ -147,7 +155,7 @@ Module Inspect(const CompiledShader& compiled, const State& state, const VkPhysi const auto& binding = shader.bindings.front(); Require(binding.role == ShaderRecompiler::DescriptorRole::FaultBuffer && binding.kind == ShaderRecompiler::DescriptorKind::StorageBuffer && binding.descriptorSet == 0 && binding.count == 1 && binding.guestDescriptor.empty() && !binding.readOnly, "invalid rect-list fault buffer contract"); } - const auto model = mesh ? spv::ExecutionModelMeshEXT : control ? spv::ExecutionModelTessellationControl : evaluation ? spv::ExecutionModelTessellationEvaluation : vertex ? spv::ExecutionModelVertex : spv::ExecutionModelFragment; + const auto model = mesh ? spv::ExecutionModelMeshEXT : control ? spv::ExecutionModelTessellationControl : evaluation ? spv::ExecutionModelTessellationEvaluation : geometry ? spv::ExecutionModelGeometry : vertex ? spv::ExecutionModelVertex : spv::ExecutionModelFragment; const auto& words = shader.spirv; Require(words.size() >= 5, "SPIR-V header is truncated: " + std::to_string(words.size()) + " words"); @@ -210,6 +218,10 @@ Module Inspect(const CompiledShader& compiled, const State& state, const VkPhysi mesh && capability == spv::CapabilityMeshShadingEXT; + const bool isGeometryCapability = + geometry && + capability == spv::CapabilityGeometry; + const bool isInterlockCapability = fragment && capability == spv::CapabilityFragmentShaderPixelInterlockEXT; @@ -229,6 +241,8 @@ Module Inspect(const CompiledShader& compiled, const State& state, const VkPhysi capability == spv::CapabilityInt64 || capability == spv::CapabilityInt16 || capability == spv::CapabilityFloat16 || + capability == spv::CapabilityRoundingModeRTE || + capability == spv::CapabilityDenormPreserve || capability == spv::CapabilityFloat64 || capability == spv::CapabilityStorageBuffer8BitAccess || capability == spv::CapabilityPhysicalStorageBufferAddresses || @@ -250,10 +264,12 @@ Module Inspect(const CompiledShader& compiled, const State& state, const VkPhysi isBdaCapability || isTessellationCapability || isMeshCapability || + isGeometryCapability || isInterlockCapability || isFeatureCapability || isDescriptorIndexingCapability || - (imageInt64Atomics && (capability == spv::CapabilityInt64Atomics || capability == spv::CapabilityInt64ImageEXT)), + ((imageInt64Atomics || bufferInt64Atomics) && capability == spv::CapabilityInt64Atomics) || + (imageInt64Atomics && capability == spv::CapabilityInt64ImageEXT), std::string("SPIR-V requires unsupported device capability ") + std::to_string(static_cast(capability))); @@ -302,8 +318,8 @@ Module Inspect(const CompiledShader& compiled, const State& state, const VkPhysi for (std::size_t i = 5; i < count; ++i) Require(module.interface.insert(instruction[i]).second, "duplicate SPIR-V interface ID"); break; case spv::OpExecutionMode: - if (count == 4 && instruction[2] == spv::ExecutionModeSignedZeroInfNanPreserve) { - Require(instruction[3] == 32u || instruction[3] == 64u, "SignedZeroInfNanPreserve requires Float32 or Float64"); + if (count == 4 && (instruction[2] == spv::ExecutionModeSignedZeroInfNanPreserve || instruction[2] == spv::ExecutionModeRoundingModeRTE || instruction[2] == spv::ExecutionModeDenormPreserve)) { + Require(instruction[3] == 16u || instruction[3] == 32u || instruction[3] == 64u, "float controls execution modes require Float16, Float32 or Float64"); executionModeTargets.insert(instruction[1]); break; } @@ -317,13 +333,13 @@ Module Inspect(const CompiledShader& compiled, const State& state, const VkPhysi Require(count >= 3, "malformed SPIR-V decoration"); auto& decoration = module.decorations[instruction[1]]; const auto kind = static_cast(instruction[2]); - if (kind == spv::DecorationLocation || kind == spv::DecorationBuiltIn || kind == spv::DecorationDescriptorSet || kind == spv::DecorationBinding || kind == spv::DecorationArrayStride) { + if (kind == spv::DecorationLocation || kind == spv::DecorationBuiltIn || kind == spv::DecorationDescriptorSet || kind == spv::DecorationBinding || kind == spv::DecorationArrayStride || kind == spv::DecorationIndex) { Require(count == 4, "malformed SPIR-V literal decoration"); - auto* field = kind == spv::DecorationLocation ? &decoration.location : kind == spv::DecorationBuiltIn ? &decoration.builtin : kind == spv::DecorationDescriptorSet ? &decoration.set : kind == spv::DecorationBinding ? &decoration.binding : &decoration.stride; + auto* field = kind == spv::DecorationLocation ? &decoration.location : kind == spv::DecorationBuiltIn ? &decoration.builtin : kind == spv::DecorationDescriptorSet ? &decoration.set : kind == spv::DecorationBinding ? &decoration.binding : kind == spv::DecorationIndex ? &decoration.index : &decoration.stride; Require(!field->has_value(), "duplicate SPIR-V decoration"); *field = instruction[3]; } - Require(kind != spv::DecorationComponent && kind != spv::DecorationIndex && kind != spv::DecorationStream && kind != spv::DecorationXfbBuffer && kind != spv::DecorationXfbStride, "unsupported shader interface packing or transform feedback"); + Require(kind != spv::DecorationComponent && kind != spv::DecorationStream && kind != spv::DecorationXfbBuffer && kind != spv::DecorationXfbStride, "unsupported shader interface packing or transform feedback"); if (kind == spv::DecorationPatch) decoration.patch = true; if (kind == spv::DecorationPerPrimitiveEXT) decoration.perPrimitive = true; if (kind == spv::DecorationPerVertexKHR) { @@ -390,6 +406,12 @@ Module Inspect(const CompiledShader& compiled, const State& state, const VkPhysi mode(state.rectList ? spv::ExecutionModeSpacingEqual : spv::ExecutionModeSpacingFractionalOdd, {}); mode(spv::ExecutionModeVertexOrderCw, {}); Require(module.modes.size() == 3, "unsupported tessellation-evaluation execution mode"); + } else if (geometry) { + mode(spv::ExecutionModeTriangles, {}); + mode(spv::ExecutionModeInvocations, {1u}); + mode(spv::ExecutionModeOutputTriangleStrip, {}); + mode(spv::ExecutionModeOutputVertices, {3u}); + Require(module.modes.size() == 4, "unsupported geometry execution mode"); } else if (fragment) { for (const auto& [name, operands] : module.modes) Require(operands.empty() && (name == spv::ExecutionModeOriginUpperLeft || name == spv::ExecutionModeEarlyFragmentTests || name == spv::ExecutionModeDepthReplacing || name == spv::ExecutionModeDepthLess || name == spv::ExecutionModeDepthGreater || name == spv::ExecutionModePixelInterlockOrderedEXT), "unsupported fragment execution mode"); } else Require(module.modes.empty(), "unsupported vertex execution mode"); @@ -404,7 +426,7 @@ Module Inspect(const CompiledShader& compiled, const State& state, const VkPhysi const bool input = variable.storage == spv::StorageClassInput; const bool output = variable.storage == spv::StorageClassOutput; const bool tessellationLevels = decoration.builtin && (*decoration.builtin == spv::BuiltInTessLevelOuter || *decoration.builtin == spv::BuiltInTessLevelInner); - const bool vertexArray = !tessellationLevels && ((control && (input || output)) || (evaluation && input) || (mesh && output)) && !decoration.patch; + const bool vertexArray = !tessellationLevels && ((control && (input || output)) || (evaluation && input) || (geometry && input) || (mesh && output)) && !decoration.patch; const auto& outer = module.Type(typeId); if (decoration.perVertex) { Require(fragment && input && module.fragmentBarycentric && !decoration.patch && !decoration.perPrimitive && !decoration.builtin && decoration.location.has_value(), "PerVertexKHR requires a fragment location input with FragmentBarycentricKHR"); @@ -420,6 +442,8 @@ Module Inspect(const CompiledShader& compiled, const State& state, const VkPhysi if (mesh) { const bool primitive = decoration.perPrimitive || (decoration.builtin && (*decoration.builtin == spv::BuiltInPrimitiveTriangleIndicesEXT || *decoration.builtin == spv::BuiltInCullPrimitiveEXT)); Require(length->second == (primitive ? state.stages.mesh->maxPrimitives : state.stages.mesh->maxVertices), "mesh interface array disagrees with output limits"); + } else if (geometry) { + Require(length->second == 3u, "geometry interface array must hold the three vertices of a triangle"); } else { const auto expected = state.rectList ? (control && input ? 3u : 4u) : control && input ? state.stages.tessellation->inputControlPoints : state.stages.tessellation->outputControlPoints; Require(length->second == expected, "tessellation interface array disagrees with control-point count"); @@ -427,9 +451,13 @@ Module Inspect(const CompiledShader& compiled, const State& state, const VkPhysi typeId = outer[2]; } const auto& type = module.Type(typeId); + Require(!decoration.index || (fragment && output && state.dualSourceBlend && decoration.location == 0u && *decoration.index == 1u), "only the second dual-source blend color may use the Index decoration, at location 0 index 1"); if (variable.storage == spv::StorageClassInput || variable.storage == spv::StorageClassOutput) { Require(module.interface.contains(id), "SPIR-V input or output is absent from the entry point interface"); - if (decoration.location) { + if (decoration.index) { + Require(!module.secondSource, "duplicate second dual-source blend color"); + module.secondSource = module.Signature(typeId); + } else if (decoration.location) { Require(!vertexArray || (outer[0] & 0xffffu) == spv::OpTypeArray, "per-vertex interface lacks a control-point or mesh-output dimension"); Require(!decoration.perPrimitive, "per-primitive user outputs are unsupported"); Require(!decoration.builtin, "shader input cannot have both location and built-in decorations"); @@ -450,11 +478,11 @@ Module Inspect(const CompiledShader& compiled, const State& state, const VkPhysi push = true; Require(type.size() == 3 && module.decorations[typeId].block, "push constant variable must be a Block struct with exactly one member"); const auto offset = module.offsets.find({typeId, 0}); - Require(offset != module.offsets.end() && offset->second == 0, "push constant member must have offset zero"); + Require(offset != module.offsets.end() && offset->second % PipelinePushSlotBytes == 0 && offset->second < PipelinePushConstantBytes, "push constant member must start at a stage slot"); const auto& array = module.Type(type[2]); Require(array.size() == 4 && (array[0] & 0xffffu) == spv::OpTypeArray && module.Signature(array[2]) == "u32", "push constant member must be an array of u32"); const auto length = module.constants.find(array[3]); - Require(length != module.constants.end() && length->second == PipelinePushConstantBytes / 4, "push constant array must contain 32 elements"); + Require(length != module.constants.end() && length->second == PipelinePushSlotBytes / 4, "push constant array must contain 32 elements"); const auto stride = module.decorations[type[2]].stride; Require(stride.has_value() && *stride == 4, "push constant array must have an ArrayStride of 4"); } else if (variable.storage == spv::StorageClassWorkgroup) { @@ -513,29 +541,34 @@ Module Inspect(const CompiledShader& compiled, const State& state, const VkPhysi Require(push == (!shader.pushConstants.empty() || mesh), "recompiler push constant metadata disagrees with SPIR-V"); for (const auto id : module.interface) Require(module.variables.contains(id), "entry point interface contains an unknown variable"); if (mesh) Require(module.primitiveIndices, "mesh shader does not export primitive indices"); - if (stage == Stage::Vertex || mesh || evaluation) Require(module.position, "vertex shader does not export position"); + if (stage == Stage::Vertex || mesh || evaluation || geometry) Require(module.position, "vertex shader does not export position"); return module; } } -std::set ValidateShaders(std::span shaders, const State& state, const VkPhysicalDeviceSubgroupProperties& subgroup, bool fragmentShaderBarycentric, bool descriptorIndexing, bool imageInt64Atomics, bool geometryShader, bool sampleRateShading) { +std::set ValidateShaders(std::span shaders, const State& state, const VkPhysicalDeviceSubgroupProperties& subgroup, bool fragmentShaderBarycentric, bool descriptorIndexing, bool imageInt64Atomics, bool geometryShader, bool sampleRateShading, bool bufferInt64Atomics) { using Stage = ShaderRecompiler::ShaderStage; const bool tessellation = state.stages.path == ShaderPath::Tessellation; const bool mesh = state.stages.path == ShaderPath::Geometry; Require(state.stages.path == ShaderPath::Vertex || tessellation || mesh, "unsupported graphics shader path"); Require(state.stages.mesh.has_value() == mesh && state.stages.tessellation.has_value() == tessellation, "graphics stage configuration disagrees with its path"); Require(!state.rectList || (state.stages.path == ShaderPath::Vertex && state.topology == VK_PRIMITIVE_TOPOLOGY_PATCH_LIST && state.cullMode == VK_CULL_MODE_NONE), "invalid rect-list pipeline state"); - Require(shaders.size() == (tessellation || state.rectList ? 4u : 2u), "incorrect graphics stage count"); + const bool geometry = state.stages.path == ShaderPath::Vertex && !state.rectList && shaders.size() == 3u; + Require(geometry || shaders.size() == (tessellation || state.rectList ? 4u : 2u), "incorrect graphics stage count"); + if (geometry) { + Require(shaders.back().program != nullptr && shaders.back().program->barycentricEmulation.active, "a geometry stage is generated only for emulated barycentrics"); + Require(state.topology == VK_PRIMITIVE_TOPOLOGY_TRIANGLE_LIST || state.topology == VK_PRIMITIVE_TOPOLOGY_TRIANGLE_STRIP || state.topology == VK_PRIMITIVE_TOPOLOGY_TRIANGLE_FAN, "emulated barycentrics require triangle primitives"); + } const std::array tessStages{Stage::Local, Stage::TessellationControl, Stage::TessellationEvaluation, Stage::Fragment}; static_cast(AssemblePushConstants(shaders)); Module previous; for (std::size_t i = 0; i < shaders.size(); ++i) { - const auto expected = state.rectList ? (i == 0 ? Stage::Vertex : tessStages[i]) : tessellation ? tessStages[i] : i == 1 ? Stage::Fragment : state.stages.path == ShaderPath::Geometry ? Stage::Mesh : Stage::Vertex; + const auto expected = state.rectList ? (i == 0 ? Stage::Vertex : tessStages[i]) : tessellation ? tessStages[i] : geometry && i == 1 ? Stage::Geometry : i == shaders.size() - 1 ? Stage::Fragment : state.stages.path == ShaderPath::Geometry ? Stage::Mesh : Stage::Vertex; Require(shaders[i].program != nullptr, "missing compiled shader"); Require(shaders[i].stage == expected, "graphics stage order disagrees"); for (const auto& binding : shaders[i].program->bindings) Require(binding.descriptorSet == 0, "graphics resource uses a descriptor set other than zero"); - const auto current = Inspect(shaders[i], state, subgroup, fragmentShaderBarycentric, descriptorIndexing, imageInt64Atomics, geometryShader, sampleRateShading); + const auto current = Inspect(shaders[i], state, subgroup, fragmentShaderBarycentric, descriptorIndexing, imageInt64Atomics, geometryShader, sampleRateShading, bufferInt64Atomics); if (i != 0) { for (const auto& [location, signature] : current.inputs) { const auto output = previous.outputs.find(location); @@ -553,9 +586,13 @@ std::set ValidateShaders(std::span shaders, std::set locations; for (const auto& [location, signature] : previous.outputs) { if (location >= attachments) continue; - Require(signature == "vertex:f32x4", "fragment shader must export float4 colors to its attachments"); + const auto color = std::find_if(state.colors.begin(), state.colors.end(), [&](const ColorTarget& target) { return target.exportIndex == location; }); + const bool uintExport = color != state.colors.end() && color->uintExport; + const bool packed = color != state.colors.end() && color->packing != ShaderRecompiler::ColorExportPacking::None; + Require(signature == (uintExport || packed ? "vertex:u32x4" : "vertex:f32x4"), packed ? "fragment shader must export uint4 words to its packed 10_11_11 unorm attachments" : uintExport ? "fragment shader must export uint4 colors to its unsigned integer attachments" : "fragment shader must export float4 colors to its attachments"); locations.insert(location); } + Require(!state.dualSourceBlend || previous.secondSource == "f32x4", "dual-source blending needs a float4 second color at location 0 index 1"); return locations; } diff --git a/core/libs/prx/libSceAgcDriver/Graphics/src/State.cpp b/core/libs/prx/libSceAgcDriver/Graphics/src/State.cpp index dc0ab6fb9..60d0c2dad 100644 --- a/core/libs/prx/libSceAgcDriver/Graphics/src/State.cpp +++ b/core/libs/prx/libSceAgcDriver/Graphics/src/State.cpp @@ -58,7 +58,7 @@ std::string vteMessage(std::uint32_t viewportControl) { std::string conservativeMessage(std::uint32_t control) { std::ostringstream message; - message << "AGC graphics: PA_SC_CONSERVATIVE_RASTERIZATION_CNTL=0x" << std::hex << control << ": only 0x6000 (off) and 0x6001 (overestimation) are supported"; + message << "AGC graphics: PA_SC_CONSERVATIVE_RASTERIZATION_CNTL=0x" << std::hex << control << ": only off (over- and underestimation disabled, with at most the inner-to-normal overrides, NULL_SQUAD_AA_MASK_ENABLE and the uncertainty-region mode and edge rules set) and 0x6001 (overestimation) are supported"; return message.str(); } @@ -72,7 +72,7 @@ void requireConservativeTriangles(bool triangles, const char* primitive, const c VkConservativeRasterizationModeEXT decodeConservativeRasterization(const QueueState& queue) { const auto& cx = queue.context; const auto control = read(cx, 0x313); - if (control == 0x6000u) return VK_CONSERVATIVE_RASTERIZATION_MODE_DISABLED_EXT; + if ((control & ~0x001f6000u) == 0) return VK_CONSERVATIVE_RASTERIZATION_MODE_DISABLED_EXT; if (control != 0x6001u) throw std::runtime_error(conservativeMessage(control)); const auto stages = read(cx, 0x2d5); if ((stages & 0x20u) != 0) { @@ -105,7 +105,7 @@ constexpr std::uint32_t ShaderControlMask = ~(0x0000f870u | 0x00020600u | 0x0001 constexpr std::uint32_t PixelStageRunsMask = 0x00020747u; constexpr std::uint32_t AlphaToCoverageMask = ~0x0001ff00u; constexpr std::uint32_t ScanModeMask = ~0x22u; -constexpr std::uint32_t ScanControlMask = ~0x06023fffu; +constexpr std::uint32_t ScanControlMask = ~0x76023fffu; constexpr std::uint32_t ScreenOffsetMask = ~0x01ff01ffu; // Bits 26/27 (ZCLIP_NEAR/FAR_DISABLE) become depth clamping; bit 19 selects the [0, 1] clip space. constexpr std::uint32_t ClipControlMask = ~(0x80000u | 0x01000000u | 0x0c000000u); @@ -193,7 +193,9 @@ VkStencilOpState stencilFace(std::uint32_t compare, std::uint32_t ops, std::uint } void decodeDepth(const Registers& cx, std::uint32_t depthControl, State& result) { - zero(cx, 0x000, 0x00001f9fu, "depth/stencil clear, copy, resummarize or decompress draws (DB_RENDER_CONTROL)"); + zero(cx, 0x000, 0x00001f9cu, "depth copy, resummarize or decompress draws (DB_RENDER_CONTROL)"); + const bool depthClear = (read(cx, 0x000) & 1u) != 0; + const bool stencilClear = (read(cx, 0x000) & 2u) != 0; const auto view = read(cx, 0x002); zero(cx, 0x002, 0x3c000000u, "depth mips (DB_DEPTH_VIEW MIP_LEVEL)"); zero(cx, 0x010, 0x000f100cu, "multisampled, partially resident or mipmapped depth (DB_Z_INFO)"); @@ -209,6 +211,8 @@ void decodeDepth(const Registers& cx, std::uint32_t depthControl, State& result) }; const bool depthReadOnly = (view & 0x01000000u) != 0; const bool stencilReadOnly = (view & 0x02000000u) != 0; + Require(!stencilClear || (stencil && !stencilReadOnly), "stencil clear requires a writable stencil plane"); + Require(!depthClear || (zFormat != 0 && !depthReadOnly), "depth clear requires a writable depth plane"); DepthTarget depth{}; depth.address = zFormat != 0 ? base(0x012, 0x01a) : 0; depth.stencilAddress = stencil ? base(0x013, 0x01b) : 0; @@ -222,6 +226,7 @@ void decodeDepth(const Registers& cx, std::uint32_t depthControl, State& result) } depth.format = zFormat == 1 ? (stencil ? VK_FORMAT_D16_UNORM_S8_UINT : VK_FORMAT_D16_UNORM) : (stencil ? VK_FORMAT_D32_SFLOAT_S8_UINT : VK_FORMAT_D32_SFLOAT); depth.clearDepth = readFloat(cx, 0x00b); + if (zFormat != 0 && (read(cx, 0x010) & 0x20000000u) != 0 && find(cx, 0x005) != cx.end()) depth.htileAddress = base(0x005, 0x01e); depth.clearStencil = static_cast(read(cx, 0x00a) & 0xffu); result.depth = depth; result.depthTest = (depthControl & 2u) != 0; @@ -233,8 +238,26 @@ void decodeDepth(const Registers& cx, std::uint32_t depthControl, State& result) result.minDepthBounds = readFloat(cx, 0x008); result.maxDepthBounds = readFloat(cx, 0x009); } + if (depthClear) { + result.depthTest = true; + result.depthWrite = true; + result.depthCompare = VK_COMPARE_OP_ALWAYS; + result.depthBoundsTest = false; + } result.stencilTest = (depthControl & 1u) != 0; - if (result.stencilTest) { + if (stencilClear) { + VkStencilOpState clear{}; + clear.failOp = VK_STENCIL_OP_REPLACE; + clear.passOp = VK_STENCIL_OP_REPLACE; + clear.depthFailOp = VK_STENCIL_OP_REPLACE; + clear.compareOp = VK_COMPARE_OP_ALWAYS; + clear.compareMask = 0xffu; + clear.writeMask = 0xffu; + clear.reference = depth.clearStencil; + result.stencilTest = true; + result.stencilFront = clear; + result.stencilBack = clear; + } else if (result.stencilTest) { const auto ops = read(cx, 0x10b); result.stencilFront = stencilFace((depthControl >> 8u) & 7u, ops, read(cx, 0x10c), stencilReadOnly); result.stencilBack = (depthControl & 0x80u) != 0 ? stencilFace((depthControl >> 20u) & 7u, ops >> 12u, read(cx, 0x10d), stencilReadOnly) : result.stencilFront; @@ -247,15 +270,19 @@ void decodeDepthBias(const Registers& cx, std::uint32_t raster, State& result) { const bool frontBias = (raster & 0x800u) != 0; const bool backBias = (raster & 0x1000u) != 0; if (!(front && frontBias) && !(back && backBias)) return; - if (front && back && (frontBias != backBias || read(cx, 0x2e0) != read(cx, 0x2e2) || read(cx, 0x2e1) != read(cx, 0x2e3))) throw std::runtime_error("AGC graphics: " + zeroMessage(0x205, raster, "depth bias differing between front and back faces")); + const bool perFace = front && back && (frontBias != backBias || read(cx, 0x2e0) != read(cx, 0x2e2) || read(cx, 0x2e1) != read(cx, 0x2e3)); const bool d16 = result.depth->format == VK_FORMAT_D16_UNORM || result.depth->format == VK_FORMAT_D16_UNORM_S8_UINT; const auto format = find(cx, 0x2de) == cx.end() ? (d16 ? 0xf0u : 0x1e9u) : read(cx, 0x2de); if (format != (d16 ? 0xf0u : 0x1e9u)) throw std::runtime_error("AGC graphics: " + zeroMessage(0x2de, format, "depth bias in units other than the depth format")); const auto scale = front && frontBias ? 0x2e0u : 0x2e2u; result.depthBias = true; - result.depthBiasSlope = readFloat(cx, scale) / 16.0f; - result.depthBiasConstant = readFloat(cx, scale + 1u); + result.depthBiasSlope = frontBias || !perFace ? readFloat(cx, scale) / 16.0f : 0.0f; + result.depthBiasConstant = frontBias || !perFace ? readFloat(cx, scale + 1u) : 0.0f; result.depthBiasClamp = readFloat(cx, 0x2df); + if (!perFace) return; + result.depthBiasPerFace = true; + result.backDepthBiasSlope = backBias ? readFloat(cx, 0x2e2) / 16.0f : 0.0f; + result.backDepthBiasConstant = backBias ? readFloat(cx, 0x2e3) : 0.0f; } bool depthPassThrough(std::uint32_t depthControl) { @@ -298,12 +325,37 @@ VkBlendFactor blendFactor(std::uint32_t value) { case 10: return VK_BLEND_FACTOR_SRC_ALPHA_SATURATE; case 13: return VK_BLEND_FACTOR_CONSTANT_COLOR; case 14: return VK_BLEND_FACTOR_ONE_MINUS_CONSTANT_COLOR; + case 15: return VK_BLEND_FACTOR_SRC1_COLOR; + case 16: return VK_BLEND_FACTOR_ONE_MINUS_SRC1_COLOR; + case 17: return VK_BLEND_FACTOR_SRC1_ALPHA; + case 18: return VK_BLEND_FACTOR_ONE_MINUS_SRC1_ALPHA; case 19: return VK_BLEND_FACTOR_CONSTANT_ALPHA; case 20: return VK_BLEND_FACTOR_ONE_MINUS_CONSTANT_ALPHA; default: throw std::runtime_error("AGC graphics: unsupported blend factor " + std::to_string(value)); } } +VkBlendFactor colorFactorWithOpaqueDestination(VkBlendFactor factor) { + switch (factor) { + case VK_BLEND_FACTOR_DST_ALPHA: return VK_BLEND_FACTOR_ONE; + case VK_BLEND_FACTOR_ONE_MINUS_DST_ALPHA: return VK_BLEND_FACTOR_ZERO; + case VK_BLEND_FACTOR_SRC_ALPHA_SATURATE: return VK_BLEND_FACTOR_ZERO; + default: return factor; + } +} + +VkBlendFactor alphaFactorWithOpaqueDestination(VkBlendFactor factor) { + switch (factor) { + case VK_BLEND_FACTOR_DST_ALPHA: case VK_BLEND_FACTOR_DST_COLOR: return VK_BLEND_FACTOR_ONE; + case VK_BLEND_FACTOR_ONE_MINUS_DST_ALPHA: case VK_BLEND_FACTOR_ONE_MINUS_DST_COLOR: return VK_BLEND_FACTOR_ZERO; + default: return factor; + } +} + +bool secondSource(VkBlendFactor factor) { + return factor == VK_BLEND_FACTOR_SRC1_COLOR || factor == VK_BLEND_FACTOR_ONE_MINUS_SRC1_COLOR || factor == VK_BLEND_FACTOR_SRC1_ALPHA || factor == VK_BLEND_FACTOR_ONE_MINUS_SRC1_ALPHA; +} + VkBlendOp blendOp(std::uint32_t value) { switch (value) { case 0: return VK_BLEND_OP_ADD; @@ -319,6 +371,7 @@ struct DecodedColorFormat { VkFormat format; std::uint32_t elementBytes; std::uint8_t componentMapping = 0xe4u; + ShaderRecompiler::ColorExportPacking packing = ShaderRecompiler::ColorExportPacking::None; }; // CB_COLOR_INFO FORMAT / NUMBER_TYPE / COMP_SWAP to a Vulkan attachment format. AMD formats list @@ -364,7 +417,8 @@ DecodedColorFormat DecodeColorFormat(std::uint32_t format, std::uint32_t number, return fail(); case 6: // COLOR_10_11_11: red in the low 11 bits, the Vulkan B10G11R11 packing. - if (swap != 0 || number != floating) return fail(); + if (swap != 0 || (number != floating && number != unorm)) return fail(); + if (number == unorm) return {VK_FORMAT_R32_UINT, 4, 0xe4u, ShaderRecompiler::ColorExportPacking::Unorm10_11_11}; return {VK_FORMAT_B10G11R11_UFLOAT_PACK32, 4}; case 9: // COLOR_2_10_10_10 keeps red in the low bits, the Vulkan A2B10G10R10 packing. @@ -375,6 +429,7 @@ DecodedColorFormat DecodeColorFormat(std::uint32_t format, std::uint32_t number, if (number == unorm) return {alternate ? VK_FORMAT_B8G8R8A8_UNORM : VK_FORMAT_R8G8B8A8_UNORM, 4, reversed}; if (number == snorm) return {alternate ? VK_FORMAT_B8G8R8A8_SNORM : VK_FORMAT_R8G8B8A8_SNORM, 4, reversed}; if (number == srgb) return {alternate ? VK_FORMAT_B8G8R8A8_SRGB : VK_FORMAT_R8G8B8A8_SRGB, 4, reversed}; + if (number == uint && !alternate) return {VK_FORMAT_R8G8B8A8_UINT, 4, reversed}; return fail(); } case 11: @@ -476,6 +531,9 @@ ShaderStages DecodeShaderStages(const QueueState& queue) { std::string DepthMaintenanceRejection(const QueueState& queue) { const auto control = find(queue.context, 0x000); if (control == queue.context.end() || (control->second & ~0x2063u) == 0) return {}; + const auto depthControl = find(queue.context, 0x200); + const bool colorMasks = find(queue.context, 0x8e) != queue.context.end() && find(queue.context, 0x8f) != queue.context.end(); + if ((control->second & ~0x2063u) == 0x10u && depthControl != queue.context.end() && (depthControl->second & 3u) == 0 && colorMasks && ColorWriteMask(queue.context) == 0) return {}; return zeroMessage(0x000, control->second, "DB_RENDER_CONTROL depth copy, resummarize or decompress"); } @@ -514,9 +572,11 @@ State DecodeState(const QueueState& queue) { zero(cx, 0x207, ~LayerExports, "clip distances, layer, viewport or auxiliary vertex exports"); { const auto depthControl = read(cx, 0x200); - if ((depthControl & 0xbu) != 0 && depthSurfaceBound(cx)) { + const auto renderControl = find(cx, 0x000); + const bool clear = renderControl != cx.end() && (renderControl->second & 3u) != 0; + if (clear || ((depthControl & 0xbu) != 0 && depthSurfaceBound(cx))) { decodeDepth(cx, depthControl, result); - } else if (depthPassThrough(depthControl) || ((depthControl & 3u) != 0 && depthPlanesAbsent(cx))) { + } else if (depthPassThrough(depthControl) || ((depthControl & 0xbu) != 0 && depthPlanesAbsent(cx))) { static bool reported = false; if (!reported) { reported = true; @@ -531,7 +591,7 @@ State DecodeState(const QueueState& queue) { } else { if ((effectiveDepthControl(depthControl) & DepthControlMask) != 0) zero(cx, 0x200, DepthControlMask, "depth, stencil or conditional color writes"); } - Require((depthControl & 8u) == 0 || result.depth.has_value(), "depth bounds without a depth surface"); + Require((depthControl & 8u) == 0 || result.depth.has_value() || depthPlanesAbsent(cx), "depth bounds without a depth surface"); Require((depthControl & 0xc0000000u) == 0, "depth-conditional color writes are unsupported"); } zero(cx, 0x203, shaderControlMask(read(cx, 0x1c4)), "depth export, shader coverage or ordered fragment execution"); @@ -553,12 +613,15 @@ State DecodeState(const QueueState& queue) { const auto raster = read(cx, 0x205); // Bits 5-10 give the front/back polygon type (2 = filled triangles), which POLY_MODE (bit 3) turns on // explicitly; KEEP_TOGETHER_ENABLE (bit 24) only affects primitive distribution across the chip. - const auto rasterMode = raster & ~0x7u & ~(1u << 24u) & ~0x1800u; - if (rasterMode != 0 && rasterMode != 0x240u && rasterMode != 0x248u) throw std::runtime_error("AGC graphics: " + zeroMessage(0x205, raster, "polygon mode, depth bias, provoking vertex or nonstandard rasterization")); + const bool pointsOrLines = result.topology == VK_PRIMITIVE_TOPOLOGY_POINT_LIST || result.topology == VK_PRIMITIVE_TOPOLOGY_LINE_LIST; + const auto lineOffset = (raster & 0x1800u) != 0 && !pointsOrLines ? 0x2000u : 0u; + const auto rasterMode = raster & ~0x7u & ~(1u << 24u) & ~0x1800u & ~lineOffset & ~(1u << 19u); + if (rasterMode != 0 && rasterMode != 0x240u && rasterMode != 0x248u) throw std::runtime_error("AGC graphics: " + zeroMessage(0x205, raster, "polygon mode, depth bias or nonstandard rasterization")); result.cullMode = ((raster & 1u) != 0 ? VK_CULL_MODE_FRONT_BIT : 0u) | ((raster & 2u) != 0 ? VK_CULL_MODE_BACK_BIT : 0u); if (result.rectList) result.cullMode = VK_CULL_MODE_NONE; if ((raster & 0x1800u) != 0 && result.depth) decodeDepthBias(cx, raster, result); result.frontFace = (raster & 4u) != 0 ? VK_FRONT_FACE_CLOCKWISE : VK_FRONT_FACE_COUNTER_CLOCKWISE; + result.provokingVertexMode = (raster & (1u << 19u)) != 0 ? VK_PROVOKING_VERTEX_MODE_LAST_VERTEX_EXT : VK_PROVOKING_VERTEX_MODE_FIRST_VERTEX_EXT; APS5_LOG_OUT_DEBUG("Raster=0x%x cullMode=0x%x frontFace=%u negativeOneToOne=%u", raster, static_cast(result.cullMode), static_cast(result.frontFace), result.negativeOneToOne ? 1u : 0u); const auto shaderMask = read(cx, 0x8f); // Channels of targets the pixel shader does not export are never written, so the target mask only @@ -591,9 +654,11 @@ State DecodeState(const QueueState& queue) { if (!written(slot)) continue; // Export formats only matter for the targets the draw writes. const auto slotExport = (exportFormat >> (4u * index)) & 0xfu; - if (slotExport == 0 || slotExport == 7 || slotExport == 8 || slotExport > 9) throw std::runtime_error("AGC graphics: color export format " + std::to_string(slotExport) + " is unsupported"); + if (slotExport == 0 || slotExport == 8 || slotExport > 9) throw std::runtime_error("AGC graphics: color export format " + std::to_string(slotExport) + " is unsupported"); auto color = DecodeColorBuffer(cx, slot); + if (slotExport == 7 && ((read(cx, 0x31c + slot * 0xfu) >> 8u) & 7u) != 4u) throw std::runtime_error("AGC graphics: color export format 7 (UINT16_ABGR) into a target that is not unsigned integer is unsupported"); color.exportIndex = index; + color.uintExport = slotExport == 7; APS5_LOG_OUT_DEBUG("Color %u address=0x%llx extent=%ux%u bytes=%llu VkFormat=%u", slot, static_cast(color.address), color.extent.width, color.extent.height, static_cast(color.bytes), static_cast(color.format)); if (result.colors.empty()) { result.renderExtent = color.extent; @@ -623,19 +688,27 @@ State DecodeState(const QueueState& queue) { const auto minDepth = result.negativeOneToOne ? zo - zs : zo; const auto maxDepth = zo + zs; APS5_LOG_OUT_DEBUG("Viewport transform scale=(%f,%f,%f) offset=(%f,%f,%f) depth=(%f,%f)", xs, ys, zs, xo, yo, zo, minDepth, maxDepth); - if (!(xs > 0 && ys != 0 && std::isfinite(minDepth) && std::isfinite(maxDepth))) { + const bool areaFree = result.stages.path == ShaderPath::Vertex && (result.rectList || result.topology == VK_PRIMITIVE_TOPOLOGY_TRIANGLE_LIST || result.topology == VK_PRIMITIVE_TOPOLOGY_TRIANGLE_FAN || result.topology == VK_PRIMITIVE_TOPOLOGY_TRIANGLE_STRIP); + const bool collapsed = (xs == 0 || ys == 0) && xs >= 0 && areaFree && std::isfinite(xo) && std::isfinite(yo) && std::isfinite(minDepth) && std::isfinite(maxDepth); + if (!collapsed && !(xs > 0 && ys != 0 && std::isfinite(minDepth) && std::isfinite(maxDepth))) { std::ostringstream message; message << "AGC graphics: unsupported viewport transform: scale=(" << xs << ", " << ys << ", " << zs << "), offset=(" << xo << ", " << yo << ", " << zo << "), depth=(" << minDepth << ", " << maxDepth << "), negativeOneToOne=" << result.negativeOneToOne; throw std::runtime_error(message.str()); } Require(readFloat(cx, 0xb4) <= readFloat(cx, 0xb5), "inverted viewport depth clamp bounds"); - result.viewport = {xo - xs, yo - ys, 2 * xs, 2 * ys, minDepth, maxDepth}; + result.viewport = collapsed ? VkViewport{xo, yo, 1, 1, minDepth, maxDepth} : VkViewport{xo - xs, yo - ys, 2 * xs, 2 * ys, minDepth, maxDepth}; + if (const auto renderControl = find(cx, 0x000); result.depth && renderControl != cx.end() && (renderControl->second & 1u) != 0) { + result.viewport.minDepth = result.depth->clearDepth; + result.viewport.maxDepth = result.depth->clearDepth; + result.depthBias = false; + } APS5_LOG_OUT_DEBUG("Viewport x=%f y=%f w=%f h=%f minDepth=%f maxDepth=%f", result.viewport.x, result.viewport.y, result.viewport.width, result.viewport.height, result.viewport.minDepth, result.viewport.maxDepth); result.scissor = {{0, 0}, result.renderExtent}; intersect(result.scissor, cx, 0xc, true); intersect(result.scissor, cx, 0x81, false); intersect(result.scissor, cx, 0x90, false); if ((read(cx, 0x292) & 2u) != 0) intersect(result.scissor, cx, 0x94, false); + if (collapsed) result.scissor.extent = {0, 0}; APS5_LOG_OUT_DEBUG("Scissor offset=(%d,%d) extent=%ux%u", result.scissor.offset.x, result.scissor.offset.y, result.scissor.extent.width, result.scissor.extent.height); result.blends.assign(exportCount, VkPipelineColorBlendAttachmentState{}); for (const auto& color : result.colors) { @@ -651,6 +724,7 @@ State DecodeState(const QueueState& queue) { } state.blendEnable = (blend >> 30u) & 1u; if (state.blendEnable && (mapping == 0x1bu || mapping == 0x93u)) throw std::runtime_error("AGC graphics: blending into a color target with a reversed component order is not implemented"); + Require(!state.blendEnable || !color.uintExport, "blending into an unsigned integer target is unsupported"); if (state.blendEnable) { Require((read(cx, 0x31c + slot * 0xfu) & 0x10000u) == 0, "blend bypass conflicts with enabled blending"); state.srcColorBlendFactor = blendFactor(blend & 0x1fu); @@ -665,15 +739,46 @@ State DecodeState(const QueueState& queue) { state.dstColorBlendFactor = state.dstAlphaBlendFactor; state.colorBlendOp = state.alphaBlendOp; } + if ((read(cx, 0x31d + slot * 0xfu) & 0x20000u) != 0) { + state.srcColorBlendFactor = colorFactorWithOpaqueDestination(state.srcColorBlendFactor); + state.dstColorBlendFactor = colorFactorWithOpaqueDestination(state.dstColorBlendFactor); + state.srcAlphaBlendFactor = alphaFactorWithOpaqueDestination(state.srcAlphaBlendFactor); + state.dstAlphaBlendFactor = alphaFactorWithOpaqueDestination(state.dstAlphaBlendFactor); + } for (std::uint32_t i = 0; i < 4; ++i) result.blendConstants[i] = readFloat(cx, 0x105 + i); } + if (color.packing != ShaderRecompiler::ColorExportPacking::None) { + const auto slotExport = (exportFormat >> (4u * color.exportIndex)) & 0xfu; + if (slotExport == 5 || slotExport == 6) throw std::runtime_error("AGC graphics: color export format " + std::to_string(slotExport) + " into a 10_11_11 unorm target is unsupported"); + Require(!state.blendEnable, "blending into a 10_11_11 unorm color target is unsupported"); + Require((exportedMask & 7u) == 7u, "partial writes of a 10_11_11 unorm color target are unsupported"); + } + if (state.blendEnable && (secondSource(state.srcColorBlendFactor) || secondSource(state.dstColorBlendFactor) || secondSource(state.srcAlphaBlendFactor) || secondSource(state.dstAlphaBlendFactor))) { + const auto addition = [](VkBlendOp op) { return op == VK_BLEND_OP_ADD || op == VK_BLEND_OP_SUBTRACT || op == VK_BLEND_OP_REVERSE_SUBTRACT; }; + Require(addition(state.colorBlendOp) && addition(state.alphaBlendOp), "dual-source blending with a MIN or MAX operation is unsupported"); + result.dualSourceBlend = true; + } result.blends[color.exportIndex] = state; } if (!result.colors.empty()) result.blend = result.blends[result.colors.front().exportIndex]; + if (result.dualSourceBlend) { + Require(exportSlots.size() == 2 && exportSlots[0] == 0 && exportSlots[1] == 1 && exportCount == 1, "dual-source blending needs pixel exports to MRT slots 0 and 1 and color writes to slot 0 alone"); + Require(result.colors.front().componentMapping == 0xe4u, "dual-source blending into a component-swapped color target is unsupported"); + Require(((exportFormat >> 4u) & 0xfu) == (exportFormat & 0xfu), "dual-source blending needs the MRT1 export in the MRT0 export format"); + Require(((shaderMask >> 4u) & 0xfu) == (shaderMask & 0xfu), "dual-source blending needs the MRT1 export with the MRT0 shader mask"); + } APS5_LOG_OUT_DEBUG("DecodeState done colorTarget=%u render=%ux%u topology=%u", result.hasColorTarget ? 1u : 0u, result.renderExtent.width, result.renderExtent.height, static_cast(result.topology)); return result; } +std::array ExportPackings(const State& state) { + std::array packings{}; + for (const auto& color : state.colors) { + if (color.exportIndex < packings.size()) packings[color.exportIndex] = color.packing; + } + return packings; +} + std::array ExportMappings(const State& state) { std::array mappings{}; mappings.fill(0xe4u); @@ -684,10 +789,7 @@ std::array ExportMappings(const State& state) { } std::size_t CmaskBytes(std::uint32_t width, std::uint32_t height) { - constexpr std::size_t metablockWidth = 1024; - constexpr std::size_t metablockHeight = 512; - constexpr std::size_t metablockBytes = 4096; - return ((width + metablockWidth - 1) / metablockWidth) * ((height + metablockHeight - 1) / metablockHeight) * metablockBytes; + return CmaskLayout(width, height).Bytes(); } std::uint32_t ColorWriteMask(const Registers& context) { @@ -719,18 +821,21 @@ ColorTarget DecodeColorBuffer(const Registers& cx, std::uint32_t slot) { const auto slice = view & 0x1fffu; Require(slice == ((view >> 13u) & 0x1fffu), "color views of several array slices are unsupported"); const auto viewMip = (view >> 26u) & 0xfu; - zero(cx, 0x31d + stride, ~0u, "color samples, fragments or destination alpha override"); + zero(cx, 0x31d + stride, ~0x20000u, "color samples, fragments or destination alpha override"); const auto attrib2 = read(cx, 0x3b0 + slot); const auto maxMip = attrib2 >> 28u; Require(viewMip <= maxMip, "color view mip exceeds the surface"); const auto attrib3 = read(cx, 0x3b8 + slot); color.tileMode = DecodeColorTileMode(attrib3); const bool volume = ((attrib3 >> 24u) & 3u) == 2u; + Require(((attrib3 >> 24u) & 3u) != 0u || (attrib2 & 0x3fffu) == 0u, "1D color targets taller than one row are unsupported"); if (volume) { color.depth = (attrib3 & 0x1fffu) + 1u; Require(maxMip == 0 && (info & 0x10000000u) == 0, "mipmapped or DCC 3D color targets are unsupported"); Require(slice < color.depth, "the color view slice is beyond the 3D surface"); color.depthSlice = slice; + } else if ((attrib3 & 0x1fffu) != 0) { + Require(slice <= (attrib3 & 0x1fffu), "the color view slice is beyond the array surface"); } color.extent = {((attrib2 >> 14u) & 0x3fffu) + 1u, (attrib2 & 0x3fffu) + 1u}; color.elementBytes = decoded.elementBytes; @@ -750,22 +855,32 @@ ColorTarget DecodeColorBuffer(const Registers& cx, std::uint32_t slot) { const auto high = read(cx, 0x390 + slot); Require((high & ~0xffu) == 0, "invalid color address extension"); color.surfaceAddress = (static_cast(high) << 40u) | (static_cast(read(cx, 0x318 + stride)) << 8u); + if (ColorTileModeIsXor(color.tileMode)) { + color.pipeBankXor = static_cast(color.surfaceAddress & (ColorTileModeBlockBytes(color.tileMode) - 1u)); + color.surfaceAddress -= color.pipeBankXor; + } if (slice != 0 && !volume) color.surfaceAddress += slice * ComputeSurfaceSize(ComputeElementMipLayout(ColorTextureTileMode(color.tileMode), color.elementBytes, color.surfaceExtent.width, color.surfaceExtent.height, color.mipCount), 1); color.address = color.surfaceAddress + mipOffset; color.bytes = colorLayout.Bytes(); GuestMemory::CheckRange(reinterpret_cast(color.address), color.bytes, colorLayout.Alignment(), true); color.format = decoded.format; color.componentMapping = decoded.componentMapping; + color.packing = decoded.packing; + Require(color.packing == ShaderRecompiler::ColorExportPacking::None || (info & 0x10000000u) == 0, "DCC-compressed 10_11_11 unorm color targets are unsupported"); + Require(color.packing == ShaderRecompiler::ColorExportPacking::None || (info & 0x00040000u) == 0, "truncating (ROUND_MODE) 10_11_11 unorm color targets are unsupported"); for (std::uint32_t word = 0; word < 2; ++word) { const auto clear = find(cx, 0x323 + word + stride); color.clearWords[word] = clear == cx.end() ? 0u : clear->second; } if ((info & 0x2000u) != 0) { Require(maxMip == 0 && !volume && slice == 0, "CMASK fast clears of a mipmapped, 3D or array color target are unsupported"); + Require(((attrib3 >> 19u) & 0x1fu) == 0x18u && (attrib3 & 0x4000000u) != 0, "CMASK fast clears need pipe-aligned SW_64KB_Z_X metadata (CB_COLOR_ATTRIB3 FMASK_SW_MODE 24, CMASK_PIPE_ALIGNED)"); + Require(color.elementBytes <= 8, "CMASK fast clears of texels over 64 bits are unsupported"); const auto cmaskHigh = find(cx, 0x398 + slot); color.cmaskAddress = ((cmaskHigh == cx.end() ? 0ull : static_cast(cmaskHigh->second & 0xffu)) << 40u) | (static_cast(read(cx, 0x31f + stride)) << 8u); Require(color.cmaskAddress != 0, "CMASK fast clears without a CMASK address are unsupported"); color.cmaskBytes = CmaskBytes(color.extent.width, color.extent.height); + GuestMemory::CheckRange(reinterpret_cast(color.cmaskAddress), color.cmaskBytes, CmaskLayout::Alignment, true); } if ((info & 0x10000000u) != 0) { if (maxMip == 0) { @@ -846,9 +961,20 @@ std::string DrawRejection(const QueueState& queue, bool indexed) { } if (auto reason = nonzero(cx, 0x207, ~LayerExports, "clip distances, layer, viewport or auxiliary vertex exports"); !reason.empty()) return reason; if (value(cx, 0x200, word)) { - const bool surface = (word & 0xbu) != 0 && depthSurfaceBound(cx); - if (!surface && !((word & 3u) != 0 && depthPlanesAbsent(cx)) && !depthPassThrough(word) && !IgnoreDepthTest() && (effectiveDepthControl(word) & DepthControlMask) != 0) return zeroMessage(0x200, word, "depth, stencil or conditional color writes"); - if (auto reason = require((word & 8u) == 0 || surface, "depth bounds without a depth surface"); !reason.empty()) return reason; + const auto renderControl = find(cx, 0x000); + const bool depthClear = renderControl != cx.end() && (renderControl->second & 1u) != 0; + const bool stencilClear = renderControl != cx.end() && (renderControl->second & 2u) != 0; + if (stencilClear) { + std::uint32_t stencil = 0, view = 0; + if ((value(cx, 0x011, stencil) && (stencil & 1u) == 0) || (value(cx, 0x002, view) && (view & 0x02000000u) != 0)) return require(false, "stencil clear requires a writable stencil plane"); + } + if (depthClear) { + std::uint32_t z = 0, view = 0; + if ((value(cx, 0x010, z) && (z & 3u) == 0) || (value(cx, 0x002, view) && (view & 0x01000000u) != 0)) return require(false, "depth clear requires a writable depth plane"); + } + const bool surface = ((word & 0xbu) != 0 || depthClear || stencilClear) && depthSurfaceBound(cx); + if (!surface && !((word & 0xbu) != 0 && depthPlanesAbsent(cx)) && !depthPassThrough(word) && !IgnoreDepthTest() && (effectiveDepthControl(word) & DepthControlMask) != 0) return zeroMessage(0x200, word, "depth, stencil or conditional color writes"); + if (auto reason = require((word & 8u) == 0 || surface || depthPlanesAbsent(cx), "depth bounds without a depth surface"); !reason.empty()) return reason; if (auto reason = require((word & 0xc0000000u) == 0, "depth-conditional color writes are unsupported"); !reason.empty()) return reason; } std::uint32_t zFormat = 0; @@ -863,7 +989,7 @@ std::string DrawRejection(const QueueState& queue, bool indexed) { if (value(cx, 0x83, word) && word != 0xffffu) return require(false, "clip rectangles are unsupported"); if (value(cx, 0x8c, word) && (word & 0xfu) != 0xau) return require(false, "nonstandard triangle edge rules are unsupported"); if (value(cx, 0x2f9, word) && word != 0x2du) return require(false, "nonstandard pixel center or vertex quantization is unsupported"); - if (value(cx, 0x313, word) && word != 0x6000u && word != 0x6001u) return conservativeMessage(word); + if (value(cx, 0x313, word) && (word & ~0x001f6000u) != 0 && word != 0x6001u) return conservativeMessage(word); std::uint32_t other = 0; if (value(cx, 0x30e, word) && value(cx, 0x30f, other) && (word != 0xffffffffu || other != 0xffffffffu)) return require(false, "sample masks are unsupported"); if (value(cx, 0x206, word) && word != 0x43fu) return vteMessage(word); @@ -884,7 +1010,7 @@ std::string DrawRejection(const QueueState& queue, bool indexed) { bool PixelProgramSkipped(const QueueState& queue) { const auto low = find(queue.shader, 0x008, RegisterBank::Shader); const auto high = find(queue.shader, 0x009, RegisterBank::Shader); - if (low != queue.shader.end() && high != queue.shader.end() && low->second == 0 && high->second == 0) return true; + if ((low == queue.shader.end() || low->second == 0) && (high == queue.shader.end() || high->second == 0)) return true; const auto& cx = queue.context; const auto targetMask = find(cx, 0x8e); const auto shaderMask = find(cx, 0x8f); diff --git a/core/libs/prx/libSceAgcDriver/Graphics/src/Texture.cpp b/core/libs/prx/libSceAgcDriver/Graphics/src/Texture.cpp index 312921acd..dc6f3e0f7 100644 --- a/core/libs/prx/libSceAgcDriver/Graphics/src/Texture.cpp +++ b/core/libs/prx/libSceAgcDriver/Graphics/src/Texture.cpp @@ -77,6 +77,7 @@ thread_local const char* uploadReason = "first"; // selections widened to every pending unit (the hysteresis in writeBack), units dropped as dead // (DiscardPendingInside) and units an alias's own results superseded (markLayersPending); relaxed, // reported only. +std::atomic singlePassMoves{0}; std::atomic partialUploads{0}, partialUploadBytes{0}, partialWriteBacks{0}, partialWriteBackBytes{0}, emptyWriteBacks{0}, coalescedWriteBacks{0}, unitsDropped{0}, unitsSuperseded{0}; // FlushPending listings skipped because no pending unit of the image lay inside the range, images // whose selected pending units were dropped because their memory is no longer registered (see @@ -86,6 +87,7 @@ std::atomic pretestSkipped{0}, unregisteredDropped{0}, keyFlipKep // Images stored by a FlushPending after a hook skip of theirs (AccessKeptByCpu), of which by the // hook for the read site of the last skip; images evictStale dropped (the [hooksync] and [storage] lines). std::atomic flushedAfterSkip{0}, flushedAfterSkipSameSite{0}, staleEvicted{0}; +std::atomic refreshesProved{0}; struct StorageTraffic { std::mutex mutex; @@ -340,7 +342,7 @@ Texture::Texture(const Context& context, TextureDetiler& detiler, const GuestTex for (std::uint32_t level = 0; level < mips.size(); ++level) { if (!geometry.HasLayer(level, layer)) continue; const auto& mip = mips[level]; - detiler.Dispatch(commands, descriptor.tileMode, elementBytes, tiled->Handle(), guestLayerOffset + mip.tiledOffset, linear->Handle(), linearLayerOffset + mip.linearOffset, mip, false, layer, geometry.thick); + detiler.Dispatch(commands, descriptor.tileMode, elementBytes, tiled->Handle(), guestLayerOffset + mip.tiledOffset, linear->Handle(), linearLayerOffset + mip.linearOffset, mip, false, layer, geometry.thick, {.pipeBankXor = descriptor.pipeBankXor}); } } @@ -460,7 +462,7 @@ bool Texture::CanCopyFrom(const StorageTexture& source, const GuestTextureResour if (IsBlockCompressed(descriptor.format) || IsBlockCompressed(from.format)) return false; // Same memory, same layout, same texel size: the GPU copy reinterprets the texels exactly as a // guest read through the sampled descriptor would. - return descriptor.baseAddress == from.baseAddress && descriptor.width == from.width && descriptor.height == from.height && descriptor.dimension == from.dimension && descriptor.tileMode == from.tileMode && descriptor.mipCount <= from.mipCount && descriptor.depthOrLastArray == from.depthOrLastArray && BytesPerElement(descriptor.format) == BytesPerElement(from.format) && BlockWidth(descriptor.format) == BlockWidth(from.format); + return descriptor.baseAddress == from.baseAddress && descriptor.pipeBankXor == from.pipeBankXor && descriptor.width == from.width && descriptor.height == from.height && descriptor.dimension == from.dimension && descriptor.tileMode == from.tileMode && descriptor.mipCount <= from.mipCount && descriptor.depthOrLastArray == from.depthOrLastArray && BytesPerElement(descriptor.format) == BytesPerElement(from.format) && BlockWidth(descriptor.format) == BlockWidth(from.format); } Texture::Texture(const Context& context, const std::shared_ptr& source, const GuestTextureResource& descriptor, VkComponentMapping components) : context(context), storageSource(source) { @@ -902,7 +904,7 @@ bool ClearLayoutFor(VkFormat format, ClearLayout& layout) { // A float of `bits` (16: IEEE half; 11 and 10: the unsigned 5-bit-exponent floats of // B10G11R11) as a float32, or nullopt for a NaN, a denormal or (unsigned) an infinity. -std::optional SmallFloat(std::uint32_t code, std::uint32_t bits) { +std::optional SmallFloat(std::uint32_t code, std::uint32_t bits, bool denormals = false) { const std::uint32_t mantissaBits = bits == 16 ? 10u : bits - 5u; const bool negative = bits == 16 && (code >> 15u) != 0; const std::uint32_t exponent = (code >> mantissaBits) & 0x1fu; @@ -912,8 +914,9 @@ std::optional SmallFloat(std::uint32_t code, std::uint32_t bits) { return negative ? -std::numeric_limits::infinity() : std::numeric_limits::infinity(); } if (exponent == 0) { - if (mantissa != 0) return std::nullopt; - return negative ? -0.0f : 0.0f; + if (mantissa != 0 && !denormals) return std::nullopt; + const float value = std::ldexp(static_cast(mantissa) / static_cast(1u << mantissaBits), -14); + return negative ? -value : value; } const float value = std::ldexp(1.0f + static_cast(mantissa) / static_cast(1u << mantissaBits), static_cast(exponent) - 15); return negative ? -value : value; @@ -925,7 +928,7 @@ std::optional SmallFloat(std::uint32_t code, std::uint32_t bits) { // to the same code), float ones the decoded value. Codes a clear might store differently are // refused: NaNs and denormals (an implementation may canonicalize or flush them) and the most // negative SNORM code (stored as its neighbour, the same -1.0). -bool FillClearColor(VkFormat format, std::uint32_t elementBytes, std::span pattern, VkClearColorValue& clear) { +bool FillClearColor(VkFormat format, std::uint32_t elementBytes, std::span pattern, VkClearColorValue& clear, bool denormals = false) { ClearLayout layout{}; if (elementBytes == 0 || elementBytes > 16 || 16 % elementBytes != 0 || !ClearLayoutFor(format, layout)) return false; std::array bytes{}; @@ -961,13 +964,13 @@ bool FillClearColor(VkFormat format, std::uint32_t elementBytes, std::span(code); } else { - const auto value = SmallFloat(code, channel.bits); + const auto value = SmallFloat(code, channel.bits, denormals); if (!value) return false; clear.float32[channel.component] = *value; } break; case ClearKind::UFloat: { - const auto value = SmallFloat(code, channel.bits); + const auto value = SmallFloat(code, channel.bits, denormals); if (!value) return false; clear.float32[channel.component] = *value; break; @@ -1000,6 +1003,93 @@ bool AdjacentGenerationEnabled() { } +bool ClearKeepsDenormals(const Context& context, VkFormat format) { + static std::mutex mutex; + static std::map, bool> known; + std::lock_guard lock(mutex); + if (const auto found = known.find({context.device, format}); found != known.end()) return found->second; + ClearLayout layout{}; + bool smallFloats = ClearLayoutFor(format, layout) && (layout.kind == ClearKind::Float || layout.kind == ClearKind::UFloat); + std::array pattern{}; + VkClearColorValue clear{}; + std::uint32_t elementBits = 0; + constexpr std::array halfCodes{0x0001u, 0x83ffu, 0x0200u, 0x8001u}; + for (std::uint32_t i = 0; smallFloats && i < layout.channels; ++i) { + const auto& channel = layout.channel[i]; + if (channel.bits == 32) { + smallFloats = false; + break; + } + const auto code = channel.bits == 16 ? halfCodes[i] : 1u; + pattern[channel.offset / 32u] |= code << (channel.offset % 32u); + clear.float32[channel.component] = *SmallFloat(code, channel.bits, true); + elementBits = std::max(elementBits, channel.offset + channel.bits); + } + bool keeps = false; + if (smallFloats) { + const auto elementBytes = static_cast((elementBits + 7u) / 8u); + VkImage image = VK_NULL_HANDLE; + VkDeviceMemory memory = VK_NULL_HANDLE; + struct Release { + const Context& context; + VkImage& image; + VkDeviceMemory& memory; + ~Release() { + if (image) context.Function("vkDestroyImage")(context.device, image, nullptr); + if (memory) context.Function("vkFreeMemory")(context.device, memory, nullptr); + } + } release{context, image, memory}; + VkImageCreateInfo info{VK_STRUCTURE_TYPE_IMAGE_CREATE_INFO}; + info.imageType = VK_IMAGE_TYPE_2D; + info.format = format; + info.extent = {1, 1, 1}; + info.mipLevels = 1; + info.arrayLayers = 1; + info.samples = VK_SAMPLE_COUNT_1_BIT; + info.tiling = VK_IMAGE_TILING_OPTIMAL; + info.usage = VK_IMAGE_USAGE_TRANSFER_SRC_BIT | VK_IMAGE_USAGE_TRANSFER_DST_BIT; + info.sharingMode = VK_SHARING_MODE_EXCLUSIVE; + info.initialLayout = VK_IMAGE_LAYOUT_UNDEFINED; + Check(context.Function("vkCreateImage")(context.device, &info, nullptr, &image), "vkCreateImage denormal clear probe"); + VkMemoryRequirements requirements{}; + context.Function("vkGetImageMemoryRequirements")(context.device, image, &requirements); + VkMemoryAllocateInfo allocation{VK_STRUCTURE_TYPE_MEMORY_ALLOCATE_INFO}; + allocation.allocationSize = requirements.size; + allocation.memoryTypeIndex = context.MemoryType(requirements.memoryTypeBits, VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT); + Check(context.Function("vkAllocateMemory")(context.device, &allocation, nullptr, &memory), "vkAllocateMemory denormal clear probe"); + Check(context.Function("vkBindImageMemory")(context.device, image, memory, 0), "vkBindImageMemory denormal clear probe"); + Buffer readback(context, elementBytes, VK_BUFFER_USAGE_TRANSFER_DST_BIT); + CommandBatch batch(context); + const auto commands = batch.Handle(); + const VkImageSubresourceRange range{VK_IMAGE_ASPECT_COLOR_BIT, 0, 1, 0, 1}; + VkImageMemoryBarrier toClear{VK_STRUCTURE_TYPE_IMAGE_MEMORY_BARRIER}; + toClear.dstAccessMask = VK_ACCESS_TRANSFER_WRITE_BIT; + toClear.oldLayout = VK_IMAGE_LAYOUT_UNDEFINED; + toClear.newLayout = VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL; + toClear.srcQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED; + toClear.dstQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED; + toClear.image = image; + toClear.subresourceRange = range; + context.Function("vkCmdPipelineBarrier")(commands, VK_PIPELINE_STAGE_TOP_OF_PIPE_BIT, VK_PIPELINE_STAGE_TRANSFER_BIT, 0, 0, nullptr, 0, nullptr, 1, &toClear); + context.Function("vkCmdClearColorImage")(commands, image, VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL, &clear, 1, &range); + VkImageMemoryBarrier toCopy = toClear; + toCopy.srcAccessMask = VK_ACCESS_TRANSFER_WRITE_BIT; + toCopy.dstAccessMask = VK_ACCESS_TRANSFER_READ_BIT; + toCopy.oldLayout = VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL; + toCopy.newLayout = VK_IMAGE_LAYOUT_TRANSFER_SRC_OPTIMAL; + context.Function("vkCmdPipelineBarrier")(commands, VK_PIPELINE_STAGE_TRANSFER_BIT, VK_PIPELINE_STAGE_TRANSFER_BIT, 0, 0, nullptr, 0, nullptr, 1, &toCopy); + VkBufferImageCopy region{}; + region.imageSubresource = {VK_IMAGE_ASPECT_COLOR_BIT, 0, 0, 1}; + region.imageExtent = {1, 1, 1}; + context.Function("vkCmdCopyImageToBuffer")(commands, image, VK_IMAGE_LAYOUT_TRANSFER_SRC_OPTIMAL, readback.Handle(), 1, ®ion); + RecordMemoryBarrier(context, commands, VK_PIPELINE_STAGE_TRANSFER_BIT, VK_PIPELINE_STAGE_HOST_BIT, VK_ACCESS_TRANSFER_WRITE_BIT, VK_ACCESS_HOST_READ_BIT); + batch.SubmitAndWait(); + keeps = std::memcmp(readback.Bytes().data(), pattern.data(), elementBytes) == 0; + } + known.emplace(std::pair{context.device, format}, keeps); + return keeps; +} + VkImageView StorageTexture::createView(std::uint32_t mip, bool firstLayer, VkFormat format) const { Require(mip < descriptor.mipCount, "storage texture mip level is outside the texture"); Require(!firstLayer || descriptor.dimension == TextureDimension::k2DArray, "a first-layer storage view needs a 2D array surface"); @@ -1019,6 +1109,39 @@ VkImageView StorageTexture::createView(std::uint32_t mip, bool firstLayer, VkFor return created; } +bool StorageTexture::singlePass() { + if (singlePassState < 0) { + const auto elementBytes = BytesPerElement(descriptor.format); + const auto format = TextureDetiler::ImageElementFormat(elementBytes); + bool eligible = context.singlePassStorage && format != VK_FORMAT_UNDEFINED && BlockWidth(descriptor.format) == 1 && !geometry.thick && geometry.imageDepth == 1 && (descriptor.dimension == TextureDimension::k2D || descriptor.dimension == TextureDimension::k2DArray || descriptor.dimension == TextureDimension::kCube); + if (eligible) { + VkFormatProperties properties{}; + context.formatProperties(context.physical, format, &properties); + eligible = (properties.optimalTilingFeatures & VK_FORMAT_FEATURE_STORAGE_IMAGE_BIT) != 0; + } + singlePassState = eligible ? 1 : 0; + } + return singlePassState == 1; +} + +VkImageView StorageTexture::elementLayerView(std::uint32_t level, std::uint32_t layer) { + const auto key = (level << 16) | layer; + if (const auto found = elementLayerViews.find(key); found != elementLayerViews.end()) return found->second; + VkImageViewUsageCreateInfo usage{VK_STRUCTURE_TYPE_IMAGE_VIEW_USAGE_CREATE_INFO}; + usage.usage = VK_IMAGE_USAGE_STORAGE_BIT; + VkImageViewCreateInfo viewInfo{VK_STRUCTURE_TYPE_IMAGE_VIEW_CREATE_INFO}; + viewInfo.pNext = &usage; + viewInfo.image = image; + viewInfo.viewType = VK_IMAGE_VIEW_TYPE_2D; + viewInfo.format = TextureDetiler::ImageElementFormat(BytesPerElement(descriptor.format)); + viewInfo.components = {VK_COMPONENT_SWIZZLE_IDENTITY, VK_COMPONENT_SWIZZLE_IDENTITY, VK_COMPONENT_SWIZZLE_IDENTITY, VK_COMPONENT_SWIZZLE_IDENTITY}; + viewInfo.subresourceRange = {VK_IMAGE_ASPECT_COLOR_BIT, level, 1u, geometry.CopyLayer(layer), 1u}; + VkImageView created = VK_NULL_HANDLE; + Check(context.Function("vkCreateImageView")(context.device, &viewInfo, nullptr, &created), "vkCreateImageView storage element"); + elementLayerViews.emplace(key, created); + return created; +} + VkImageView StorageTexture::AttachmentView(VkFormat format, std::uint32_t mip, std::uint32_t depthSlice) { Require(attachable, "storage image cannot be a color attachment"); Require(mip < descriptor.mipCount, "attachment mip exceeds the storage image"); @@ -1159,6 +1282,16 @@ VkImageView StorageTexture::StorageView(std::uint32_t mip, bool firstLayer) { return created; } +VkImageView StorageTexture::ElementView() { + if (elementView != VK_NULL_HANDLE) return elementView; + if (descriptor.dimension != TextureDimension::k2D || geometry.imageLayers != 1) return VK_NULL_HANDLE; + const auto bytes = BytesPerElement(descriptor.format); + const auto format = bytes == 1 ? VK_FORMAT_R8_UINT : bytes == 2 ? VK_FORMAT_R16_UINT : bytes == 4 ? VK_FORMAT_R32_UINT : bytes == 8 ? VK_FORMAT_R32G32_UINT : VK_FORMAT_UNDEFINED; + if (format == VK_FORMAT_UNDEFINED || StorageFormatOrUndefined(context, format) != format) return VK_NULL_HANDLE; + elementView = createView(0, false, format); + return elementView; +} + VkImageView StorageTexture::AtomicView(std::uint32_t mip, bool firstLayer) { if (storageFormat == VK_FORMAT_R32_UINT) return firstLayer ? FirstLayerView(mip) : View(mip); Require(storageFormat == VK_FORMAT_R32_SINT || storageFormat == VK_FORMAT_R32_SFLOAT, "storage image atomics need a surface of one 32-bit component"); @@ -1180,7 +1313,7 @@ VkImageView StorageTexture::Atomic64View(std::uint32_t mip, bool firstLayer) { } const char* LookupOutcomes::Name(Kind kind) { - static constexpr const char* names[Count] = {"sampled fast hit", "sampled fast miss", "sampled hit view", "sampled hit cleared view", "sampled hit snapshot", "sampled made view", "sampled made snapshot", "storage hit", "storage made", "refresh unchanged", "refresh compared", "upload direct", "upload cpu", "upload clear", "dcc scan", "pending flush"}; + static constexpr const char* names[Count] = {"sampled fast hit", "sampled fast miss", "sampled hit view", "sampled hit cleared view", "sampled hit snapshot", "sampled made view", "sampled made snapshot", "storage hit", "storage made", "refresh unchanged", "refresh compared", "refresh proved", "upload direct", "upload cpu", "upload clear", "dcc scan", "pending flush"}; return kind < Count ? names[kind] : "?"; } @@ -1205,6 +1338,13 @@ LookupOutcomes& ThreadLookupOutcomes() { bool StorageTexture::Refresh() { const bool profile = LookupOutcomes::Profiled(); auto start = profile ? std::chrono::steady_clock::now() : std::chrono::steady_clock::time_point{}; + if (refreshProved()) { + NoteProved(); + refreshesProved.fetch_add(1, std::memory_order_relaxed); + if (profile) LookupOutcomes::Add(LookupOutcomes::RefreshProved, start); + return true; + } + refreshProof = {}; struct Exempt { const StorageTexture* previous; ~Exempt() { refreshing = previous; } @@ -1357,6 +1497,7 @@ bool StorageTexture::Refresh() { ++Profile().storageReused; layerGeneration.assign(trackedLayers, current); refreshGeneration(); + takeRefreshProof(alias != nullptr); if (profile) LookupOutcomes::Add(stamped ? LookupOutcomes::RefreshUnchanged : LookupOutcomes::RefreshCompared, start); return true; } @@ -1468,6 +1609,53 @@ DccKeys StorageTexture::ProvedKeys() const { return *keys; } +std::uint64_t StorageTexture::SinglePassMoves() { + return singlePassMoves.load(std::memory_order_relaxed); +} + +bool StorageTexture::otherPendingOverlaps() const { + auto& pending = Pending(); + std::lock_guard lock(pending.mutex); + const auto other = [&](const StorageTexture* texture) { return texture != this && texture->overlaps(descriptor.baseAddress, static_cast(guestBytes)); }; + return std::any_of(pending.textures.begin(), pending.textures.end(), other) || std::any_of(pending.flushing.begin(), pending.flushing.end(), other); +} + +bool StorageTexture::refreshProved() { + if (refreshProof.generation == 0) return false; + if (generation != refreshProof.generation || uploadedKeys != refreshProof.keys || !GuestMemory::GpuMutex().HeldByThisThread()) return false; + const auto bytes = static_cast(guestBytes); + std::array queries{}; + std::size_t used = 0; + if (GuestMemory::CollectWrites(descriptor.baseAddress, bytes) == 0) return false; + queries[used++] = {descriptor.baseAddress, bytes, generation}; + if (descriptor.dccAddress != 0) { + const auto keyCount = static_cast(guestBytes / 256); + if (keyCount == 0 || keyProof.generation != refreshProof.keyGeneration || keyProof.keys != uploadedKeys) return false; + if (GuestMemory::CollectWrites(descriptor.dccAddress, keyCount) == 0) return false; + queries[used++] = {descriptor.dccAddress, keyCount, keyProof.generation}; + } + if (!GuestMemory::UnchangedSinceAll(std::span(queries.data(), used))) return false; + const auto serial = PendingSerial(); + if (serial != refreshProof.pendingSerial) { + if (otherPendingOverlaps()) return false; + refreshProof.pendingSerial = serial; + } + return true; +} + +void StorageTexture::takeRefreshProof(bool aliased) { + refreshProof = {}; + if (aliased || generation == 0 || !GuestMemory::GpuMutex().HeldByThisThread()) return; + if (descriptor.dccAddress != 0 && (keyProof.generation == 0 || keyProof.keys != uploadedKeys)) return; + const auto serial = PendingSerial(); + if (otherPendingOverlaps()) return; + refreshProof = {generation, serial, keyProof.generation, uploadedKeys}; +} + +std::uint64_t StorageTexture::RefreshesProved() { + return refreshesProved.load(std::memory_order_relaxed); +} + bool StorageTexture::ServesKeysAt(std::uint64_t dccAddress) const { const bool locked = GuestMemory::GpuMutex().HeldByThisThread(); const auto readFollowed = [&] { @@ -1551,6 +1739,8 @@ bool StorageTexture::compareUntracked(std::uint64_t address, std::size_t bytes, return true; } +constexpr std::uint64_t GenerationBaselineBytes = 1ull << 20; + void StorageTexture::upload(const std::vector* layers) { CaptureTrace::Log("upload image=%llx bytes=%llu generation=%llu reason=%s partial=%d", static_cast(descriptor.baseAddress), static_cast(guestBytes), static_cast(generation), uploadReason, layers != nullptr); const bool profile = LookupOutcomes::Profiled(); @@ -1563,8 +1753,18 @@ void StorageTexture::upload(const std::vector* layers) { if (layers != nullptr && ((!blockUnits && trackedLayers != arrayLayers) || std::all_of(layers->begin(), layers->end(), [](bool selected) { return selected; }))) layers = nullptr; const auto now = GuestMemory::CollectWrites(descriptor.baseAddress, static_cast(guestBytes)); const auto stampLayers = [&](bool selectedOnly) { + const bool whole = !selectedOnly || layers == nullptr; + if (guestBytes > GenerationBaselineBytes) generationBaseline.clear(); + else if (whole) { + generationBaseline.resize(static_cast(guestBytes)); + GuestMemory::ReadCommitted(descriptor.baseAddress, generationBaseline); + } for (std::uint32_t layer = 0; layer < trackedLayers; ++layer) { - if (!selectedOnly || layers == nullptr || (*layers)[layer]) layerGeneration[layer] = now; + if (!whole && !(*layers)[layer]) continue; + layerGeneration[layer] = now; + if (whole || generationBaseline.size() != guestBytes) continue; + const auto offset = static_cast(layerBegin(layer) - descriptor.baseAddress); + GuestMemory::ReadCommitted(layerBegin(layer), std::span(generationBaseline).subspan(offset, static_cast(layerBytes(layer)))); } refreshGeneration(); }; @@ -1661,7 +1861,8 @@ void StorageTexture::upload(const std::vector* layers) { if ((*layers)[layer]) uploadedBytes += trackedLayerBytes; } } - auto linear = std::make_shared(context, static_cast(linearBytes), VK_BUFFER_USAGE_STORAGE_BUFFER_BIT | VK_BUFFER_USAGE_TRANSFER_SRC_BIT); + const bool direct = singlePass(); + auto linear = direct ? nullptr : std::make_shared(context, static_cast(linearBytes), VK_BUFFER_USAGE_STORAGE_BUFFER_BIT | VK_BUFFER_USAGE_TRANSFER_SRC_BIT); detiler.BeginBatch(); auto* recorder = Recorder::Active(); std::unique_ptr batch; @@ -1672,12 +1873,12 @@ void StorageTexture::upload(const std::vector* layers) { recorder->FlushStoresOverlapping(descriptor.baseAddress, static_cast(guestBytes)); commands = recorder->Commands(); timing = recorder->BeginGpuTiming(Recorder::CommandClass::StorageUpload); - recorder->Keep(linear, linear->Size()); + if (linear) recorder->Keep(linear, linear->Size()); // The image itself must outlive the recorded copy: the cache may evict it right after. if (auto self = weak_from_this().lock()) recorder->Keep(std::move(self)); // The detile reads the tiled bytes from the import when the batch runs. recorder->NotePendingRead(descriptor.baseAddress, static_cast(guestBytes), Recorder::ReadKind::StorageUpload); - Recorder::CountBarriers(Recorder::CommandClass::StorageUpload, 3); + Recorder::CountBarriers(Recorder::CommandClass::StorageUpload, direct ? 2 : 3); if (Recorder::BarrierValidate()) { const std::pair read{descriptor.baseAddress, descriptor.baseAddress + guestBytes}; const std::pair written{image, true}; @@ -1688,13 +1889,33 @@ void StorageTexture::upload(const std::vector* layers) { commands = batch->Handle(); } const auto importOffset = descriptor.baseAddress - import->base; + if (direct) { + recordDirectUploadBarrier(commands, layers == nullptr); + for (std::uint32_t layer = 0; layer < arrayLayers; ++layer) { + if (layers != nullptr && !(*layers)[layer]) continue; + for (std::uint32_t level = 0; level < mips.size(); ++level) { + if (!geometry.HasLayer(level, layer)) continue; + const auto& mip = mips[level]; + detiler.DispatchImage(commands, descriptor.tileMode, elementBytes, import->buffer, importOffset + geometry.GuestLayerOffset(layer) + mip.tiledOffset, elementLayerView(level, layer), mip, false, layer, {.pipeBankXor = descriptor.pipeBankXor}); + } + } + recordDirectUploadDone(commands); + singlePassMoves.fetch_add(1, std::memory_order_relaxed); + if (batch) batch->SubmitAndWait(); + else recorder->EndGpuTiming(timing, uploadedBytes); + countStorageUpload(1, uploadedBytes); + ++Profile().storageDirectUploads; + ++version; + if (profile) LookupOutcomes::Add(LookupOutcomes::UploadDirect, start); + return; + } RecordMemoryBarrier(context, commands, VK_PIPELINE_STAGE_ALL_COMMANDS_BIT | VK_PIPELINE_STAGE_HOST_BIT, VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT, VK_ACCESS_SHADER_WRITE_BIT | VK_ACCESS_TRANSFER_WRITE_BIT | VK_ACCESS_HOST_WRITE_BIT, VK_ACCESS_SHADER_READ_BIT | VK_ACCESS_SHADER_WRITE_BIT); for (std::uint32_t layer = 0; layer < arrayLayers; ++layer) { if (layers != nullptr && !(*layers)[layer]) continue; for (std::uint32_t level = 0; level < mips.size(); ++level) { if (!geometry.HasLayer(level, layer)) continue; const auto& mip = mips[level]; - detiler.Dispatch(commands, descriptor.tileMode, elementBytes, import->buffer, importOffset + geometry.GuestLayerOffset(layer) + mip.tiledOffset, linear->Handle(), geometry.LinearLayerOffset(layer) + mip.linearOffset, mip, false, layer, geometry.thick); + detiler.Dispatch(commands, descriptor.tileMode, elementBytes, import->buffer, importOffset + geometry.GuestLayerOffset(layer) + mip.tiledOffset, linear->Handle(), geometry.LinearLayerOffset(layer) + mip.linearOffset, mip, false, layer, geometry.thick, {.pipeBankXor = descriptor.pipeBankXor}); } } const auto linearRead = WholeBufferBarrier(linear->Handle(), VK_ACCESS_SHADER_WRITE_BIT, VK_ACCESS_TRANSFER_READ_BIT); @@ -1745,7 +1966,7 @@ void StorageTexture::upload(const std::vector* layers) { for (std::uint32_t level = 0; level < mips.size(); ++level) { if (!geometry.HasLayer(level, layer)) continue; const auto& mip = mips[level]; - detiler.Dispatch(commands, descriptor.tileMode, elementBytes, tiled.Handle(), geometry.GuestLayerOffset(layer) + mip.tiledOffset, linear.Handle(), geometry.LinearLayerOffset(layer) + mip.linearOffset, mip, false, layer, geometry.thick); + detiler.Dispatch(commands, descriptor.tileMode, elementBytes, tiled.Handle(), geometry.GuestLayerOffset(layer) + mip.tiledOffset, linear.Handle(), geometry.LinearLayerOffset(layer) + mip.linearOffset, mip, false, layer, geometry.thick, {.pipeBankXor = descriptor.pipeBankXor}); } } const auto linearRead = WholeBufferBarrier(linear.Handle(), VK_ACCESS_SHADER_WRITE_BIT, VK_ACCESS_TRANSFER_READ_BIT); @@ -1945,6 +2166,33 @@ StorageTexture::PaddingSeeds StorageTexture::paddingSeeds(const HostImport& impo return seeds; } +void StorageTexture::recordDirectUploadBarrier(VkCommandBuffer commands, bool discard) { + const VkMemoryBarrier sourceReady{VK_STRUCTURE_TYPE_MEMORY_BARRIER, nullptr, VK_ACCESS_SHADER_WRITE_BIT | VK_ACCESS_TRANSFER_WRITE_BIT | VK_ACCESS_HOST_WRITE_BIT, VK_ACCESS_SHADER_READ_BIT | VK_ACCESS_SHADER_WRITE_BIT}; + VkImageMemoryBarrier toWrite{VK_STRUCTURE_TYPE_IMAGE_MEMORY_BARRIER}; + toWrite.srcAccessMask = discard ? 0u : VK_ACCESS_SHADER_WRITE_BIT | VK_ACCESS_SHADER_READ_BIT | VK_ACCESS_TRANSFER_WRITE_BIT | VK_ACCESS_COLOR_ATTACHMENT_WRITE_BIT; + toWrite.dstAccessMask = VK_ACCESS_SHADER_WRITE_BIT; + toWrite.oldLayout = discard ? VK_IMAGE_LAYOUT_UNDEFINED : VK_IMAGE_LAYOUT_GENERAL; + toWrite.newLayout = VK_IMAGE_LAYOUT_GENERAL; + toWrite.srcQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED; + toWrite.dstQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED; + toWrite.image = image; + toWrite.subresourceRange = {VK_IMAGE_ASPECT_COLOR_BIT, 0, descriptor.mipCount, 0, geometry.imageLayers}; + context.Function("vkCmdPipelineBarrier")(commands, VK_PIPELINE_STAGE_ALL_COMMANDS_BIT | VK_PIPELINE_STAGE_HOST_BIT, VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT, 0, 1, &sourceReady, 0, nullptr, 1, &toWrite); +} + +void StorageTexture::recordDirectUploadDone(VkCommandBuffer commands) { + VkImageMemoryBarrier written{VK_STRUCTURE_TYPE_IMAGE_MEMORY_BARRIER}; + written.srcAccessMask = VK_ACCESS_SHADER_WRITE_BIT; + written.dstAccessMask = VK_ACCESS_SHADER_READ_BIT | VK_ACCESS_SHADER_WRITE_BIT | VK_ACCESS_TRANSFER_READ_BIT | VK_ACCESS_TRANSFER_WRITE_BIT | VK_ACCESS_COLOR_ATTACHMENT_READ_BIT | VK_ACCESS_COLOR_ATTACHMENT_WRITE_BIT; + written.oldLayout = VK_IMAGE_LAYOUT_GENERAL; + written.newLayout = VK_IMAGE_LAYOUT_GENERAL; + written.srcQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED; + written.dstQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED; + written.image = image; + written.subresourceRange = {VK_IMAGE_ASPECT_COLOR_BIT, 0, descriptor.mipCount, 0, geometry.imageLayers}; + context.Function("vkCmdPipelineBarrier")(commands, VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT, VK_PIPELINE_STAGE_ALL_COMMANDS_BIT, 0, 0, nullptr, 0, nullptr, 1, &written); +} + std::uint64_t StorageTexture::uploadWindows(const HostImport& import, std::span> runs, bool discard) { const auto elementBytes = BytesPerElement(descriptor.format); // Each run's pieces by source (a unit shadow's slab while fresh, else the import), the tail @@ -1984,7 +2232,8 @@ std::uint64_t StorageTexture::uploadWindows(const HostImport& import, std::span< linearTotal += (window.linearBytes + 255) & ~std::uint64_t{255}; } for (const auto& [begin, end] : runs) uploadedBytes += end - begin; - auto linear = std::make_shared(context, static_cast(linearTotal), VK_BUFFER_USAGE_STORAGE_BUFFER_BIT | VK_BUFFER_USAGE_TRANSFER_SRC_BIT); + const bool direct = singlePass(); + auto linear = direct ? nullptr : std::make_shared(context, static_cast(linearTotal), VK_BUFFER_USAGE_STORAGE_BUFFER_BIT | VK_BUFFER_USAGE_TRANSFER_SRC_BIT); detiler.BeginBatch(); auto* recorder = Recorder::Active(); std::unique_ptr batch; @@ -2001,13 +2250,13 @@ std::uint64_t StorageTexture::uploadWindows(const HostImport& import, std::span< for (const auto& [begin, end] : reads) recorder->FlushStoresOverlapping(begin, static_cast(end - begin)); commands = recorder->Commands(); timing = recorder->BeginGpuTiming(Recorder::CommandClass::StorageUpload); - recorder->Keep(linear, linear->Size()); + if (linear) recorder->Keep(linear, linear->Size()); for (const auto& source : sources) { if (source.slab != nullptr) recorder->Keep(source.slab); } if (auto self = weak_from_this().lock()) recorder->Keep(std::move(self)); if (!reads.empty()) recorder->NotePendingReads(reads, Recorder::ReadKind::StorageUpload); - Recorder::CountBarriers(Recorder::CommandClass::StorageUpload, 3); + Recorder::CountBarriers(Recorder::CommandClass::StorageUpload, direct ? 2 : 3); if (Recorder::BarrierValidate()) { const std::pair written{image, true}; recorder->NoteAccess(Recorder::CommandClass::StorageUpload, Recorder::Access{reads, {}, std::span(&written, 1), VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT | VK_PIPELINE_STAGE_TRANSFER_BIT}); @@ -2017,6 +2266,27 @@ std::uint64_t StorageTexture::uploadWindows(const HostImport& import, std::span< commands = batch->Handle(); } const auto importOffset = descriptor.baseAddress - import.base; + if (direct) { + recordDirectUploadBarrier(commands, discard); + for (const auto& window : windows) { + const auto& mip = mips[window.level]; + const auto& source = sourceOf(window); + auto detile = window.window; + detile.pipeBankXor = descriptor.pipeBankXor; + const auto view = elementLayerView(window.level, window.layer); + if (source.shadow) { + detiler.DispatchImage(commands, descriptor.tileMode, elementBytes, source.buffer, source.offset + (window.tiledBegin - source.begin), view, mip, false, window.layer, detile); + } else { + detile.tiledBase = 0; + detiler.DispatchImage(commands, descriptor.tileMode, elementBytes, import.buffer, importOffset + geometry.GuestLayerOffset(window.layer) + mip.tiledOffset, view, mip, false, window.layer, detile); + } + } + recordDirectUploadDone(commands); + singlePassMoves.fetch_add(1, std::memory_order_relaxed); + if (batch) batch->SubmitAndWait(); + else recorder->EndGpuTiming(timing, uploadedBytes); + return uploadedBytes; + } RecordMemoryBarrier(context, commands, VK_PIPELINE_STAGE_ALL_COMMANDS_BIT | VK_PIPELINE_STAGE_HOST_BIT, VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT, VK_ACCESS_SHADER_WRITE_BIT | VK_ACCESS_TRANSFER_WRITE_BIT | VK_ACCESS_HOST_WRITE_BIT, VK_ACCESS_SHADER_READ_BIT | VK_ACCESS_SHADER_WRITE_BIT); std::vector regions; for (std::size_t i = 0; i < windows.size(); ++i) { @@ -2024,6 +2294,7 @@ std::uint64_t StorageTexture::uploadWindows(const HostImport& import, std::span< const auto& mip = mips[window.level]; const auto& source = sourceOf(window); auto detile = window.window; + detile.pipeBankXor = descriptor.pipeBankXor; if (source.shadow) { // The slab holds the piece from the window's first tiled byte: the window's tiled base // is its range begin (sliceWindows builds it so), read from the piece's slab offset. @@ -2154,7 +2425,8 @@ std::uint64_t StorageTexture::writeBackWindows(const HostImport& import, std::sp flushBegin = std::min(flushBegin, seed.begin); flushEnd = std::max(flushEnd, seed.end); } - auto linear = std::make_shared(context, static_cast(linearTotal), VK_BUFFER_USAGE_STORAGE_BUFFER_BIT | VK_BUFFER_USAGE_TRANSFER_DST_BIT); + const bool direct = singlePass(); + auto linear = direct ? nullptr : std::make_shared(context, static_cast(linearTotal), VK_BUFFER_USAGE_STORAGE_BUFFER_BIT | VK_BUFFER_USAGE_TRANSFER_DST_BIT); auto tiledScratch = std::make_shared(context, static_cast(scratchTotal), VK_BUFFER_USAGE_STORAGE_BUFFER_BIT | VK_BUFFER_USAGE_TRANSFER_SRC_BIT | VK_BUFFER_USAGE_TRANSFER_DST_BIT); detiler.BeginBatch(); auto* recorder = Recorder::Active(); @@ -2167,12 +2439,12 @@ std::uint64_t StorageTexture::writeBackWindows(const HostImport& import, std::sp recorder->FlushStoresOverlapping(flushBegin, static_cast(flushEnd - flushBegin)); commands = recorder->Commands(&covered); timing = recorder->BeginGpuTiming(Recorder::CommandClass::StorageWriteBack); - recorder->Keep(linear, linear->Size()); + if (linear) recorder->Keep(linear, linear->Size()); recorder->Keep(tiledScratch, tiledScratch->Size()); for (const auto& pieces : slabPieces) recorder->Keep(pieces.slab); for (const auto& slab : padding.slabs) recorder->Keep(slab); if (auto self = weak_from_this().lock()) recorder->Keep(std::move(self)); - Recorder::CountBarriers(Recorder::CommandClass::StorageWriteBack, 4); + Recorder::CountBarriers(Recorder::CommandClass::StorageWriteBack, direct ? 3 : 4); if (Recorder::BarrierValidate()) { const std::pair read{image, false}; std::vector> seedReads; @@ -2193,7 +2465,7 @@ std::uint64_t StorageTexture::writeBackWindows(const HostImport& import, std::sp toSource.dstQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED; toSource.image = image; toSource.subresourceRange = {VK_IMAGE_ASPECT_COLOR_BIT, 0, descriptor.mipCount, 0, geometry.imageLayers}; - context.Function("vkCmdPipelineBarrier")(commands, VK_PIPELINE_STAGE_ALL_COMMANDS_BIT, VK_PIPELINE_STAGE_TRANSFER_BIT, 0, 0, nullptr, 0, nullptr, 1, &toSource); + if (!direct) context.Function("vkCmdPipelineBarrier")(commands, VK_PIPELINE_STAGE_ALL_COMMANDS_BIT, VK_PIPELINE_STAGE_TRANSFER_BIT, 0, 0, nullptr, 0, nullptr, 1, &toSource); if (!seeds.empty() || !padding.copies.empty()) { // The seeded units' import bytes (every earlier writer of them, host stores included) // precede the seed copies; the previous command's trailing barrier may have covered that. @@ -2214,20 +2486,37 @@ std::uint64_t StorageTexture::writeBackWindows(const HostImport& import, std::sp // orders the scratch writes). for (const auto& [source, copies] : padding.copies) context.Resolved(&DeviceFunctions::cmdCopyBuffer, "vkCmdCopyBuffer")(commands, source, tiledScratch->Handle(), static_cast(copies.size()), copies.data()); } - std::vector regions; - for (std::size_t i = 0; i < windows.size(); ++i) { - for (auto region : windows[i].regions) { - region.bufferOffset += linearPositions[i]; - regions.push_back(region); - } - } - context.Function("vkCmdCopyImageToBuffer")(commands, image, VK_IMAGE_LAYOUT_TRANSFER_SRC_OPTIMAL, linear->Handle(), static_cast(regions.size()), regions.data()); - const auto linearRead = WholeBufferBarrier(linear->Handle(), VK_ACCESS_TRANSFER_WRITE_BIT, VK_ACCESS_SHADER_READ_BIT); const VkMemoryBarrier importReady{VK_STRUCTURE_TYPE_MEMORY_BARRIER, nullptr, VK_ACCESS_SHADER_READ_BIT | VK_ACCESS_SHADER_WRITE_BIT | VK_ACCESS_TRANSFER_READ_BIT | VK_ACCESS_TRANSFER_WRITE_BIT | VK_ACCESS_HOST_WRITE_BIT, VK_ACCESS_SHADER_WRITE_BIT | VK_ACCESS_SHADER_READ_BIT}; - context.Function("vkCmdPipelineBarrier")(commands, VK_PIPELINE_STAGE_ALL_COMMANDS_BIT | VK_PIPELINE_STAGE_HOST_BIT, VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT, 0, 1, &importReady, 1, &linearRead, 0, nullptr); - for (std::size_t i = 0; i < windows.size(); ++i) { - const auto& window = windows[i]; - detiler.Dispatch(commands, descriptor.tileMode, elementBytes, linear->Handle(), linearPositions[i], tiledScratch->Handle(), scratchPositions[i], mips[window.level], true, window.layer, false, window.window); + if (direct) { + VkImageMemoryBarrier toRead = toSource; + toRead.srcAccessMask = VK_ACCESS_SHADER_WRITE_BIT | VK_ACCESS_COLOR_ATTACHMENT_WRITE_BIT | VK_ACCESS_TRANSFER_WRITE_BIT; + toRead.dstAccessMask = VK_ACCESS_SHADER_READ_BIT; + toRead.newLayout = VK_IMAGE_LAYOUT_GENERAL; + context.Function("vkCmdPipelineBarrier")(commands, VK_PIPELINE_STAGE_ALL_COMMANDS_BIT | VK_PIPELINE_STAGE_HOST_BIT, VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT, 0, 1, &importReady, 0, nullptr, 1, &toRead); + for (std::size_t i = 0; i < windows.size(); ++i) { + const auto& window = windows[i]; + auto retile = window.window; + retile.pipeBankXor = descriptor.pipeBankXor; + detiler.DispatchImage(commands, descriptor.tileMode, elementBytes, tiledScratch->Handle(), scratchPositions[i], elementLayerView(window.level, window.layer), mips[window.level], true, window.layer, retile); + } + singlePassMoves.fetch_add(1, std::memory_order_relaxed); + } else { + std::vector regions; + for (std::size_t i = 0; i < windows.size(); ++i) { + for (auto region : windows[i].regions) { + region.bufferOffset += linearPositions[i]; + regions.push_back(region); + } + } + context.Function("vkCmdCopyImageToBuffer")(commands, image, VK_IMAGE_LAYOUT_TRANSFER_SRC_OPTIMAL, linear->Handle(), static_cast(regions.size()), regions.data()); + const auto linearRead = WholeBufferBarrier(linear->Handle(), VK_ACCESS_TRANSFER_WRITE_BIT, VK_ACCESS_SHADER_READ_BIT); + context.Function("vkCmdPipelineBarrier")(commands, VK_PIPELINE_STAGE_ALL_COMMANDS_BIT | VK_PIPELINE_STAGE_HOST_BIT, VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT, 0, 1, &importReady, 1, &linearRead, 0, nullptr); + for (std::size_t i = 0; i < windows.size(); ++i) { + const auto& window = windows[i]; + auto retile = window.window; + retile.pipeBankXor = descriptor.pipeBankXor; + detiler.Dispatch(commands, descriptor.tileMode, elementBytes, linear->Handle(), linearPositions[i], tiledScratch->Handle(), scratchPositions[i], mips[window.level], true, window.layer, false, retile); + } } { // The retiled scratch (and a seed's slab bytes, which the scratch copies overwrite in @@ -2247,7 +2536,7 @@ std::uint64_t StorageTexture::writeBackWindows(const HostImport& import, std::sp // merges its leading barrier. constexpr VkAccessFlags storedAccess = VK_ACCESS_SHADER_READ_BIT | VK_ACCESS_SHADER_WRITE_BIT | VK_ACCESS_TRANSFER_READ_BIT | VK_ACCESS_TRANSFER_WRITE_BIT | VK_ACCESS_HOST_READ_BIT; const VkMemoryBarrier stored{VK_STRUCTURE_TYPE_MEMORY_BARRIER, nullptr, VK_ACCESS_TRANSFER_WRITE_BIT | VK_ACCESS_SHADER_WRITE_BIT, storedAccess}; - context.Function("vkCmdPipelineBarrier")(commands, VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT | VK_PIPELINE_STAGE_TRANSFER_BIT, VK_PIPELINE_STAGE_ALL_COMMANDS_BIT | VK_PIPELINE_STAGE_HOST_BIT, 0, 1, &stored, 0, nullptr, 1, &backToGeneral); + context.Function("vkCmdPipelineBarrier")(commands, VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT | VK_PIPELINE_STAGE_TRANSFER_BIT, VK_PIPELINE_STAGE_ALL_COMMANDS_BIT | VK_PIPELINE_STAGE_HOST_BIT, 0, 1, &stored, 0, nullptr, direct ? 0u : 1u, &backToGeneral); if (batch) { batch->SubmitAndWait(); } else { @@ -2285,6 +2574,22 @@ bool StorageTexture::overlaps(std::uint64_t address, std::size_t bytes) const { return address < descriptor.baseAddress + guestBytes && descriptor.baseAddress < address + bytes; } +bool StorageTexture::overlapsLive(std::uint64_t address, std::size_t bytes) const { + return !released && guestBytes != 0 && overlaps(address, bytes); +} + +// A fill of this image's DCC keys starts at the dccAddress and is at least one key per 256 guest +// bytes. While any live image overlaps the fill (`overlapped`), it must also stay inside the key extent +// (DccKeyCount, the pipe-aligned count, at least guestBytes / 256): the bytes past the keys may be +// memory the overlapping image owns, so a longer fill is not a key fill. The bound is inferred, not +// measured (docs/dev/TechnicalDebt.md). With nothing overlapping, the length rule alone decides, as +// it did before the bound. +bool StorageTexture::keysFillMatches(std::uint64_t address, std::size_t bytes, bool overlapped) const { + constexpr std::uint64_t keyBytes = 256; + if (released || descriptor.dccAddress != address || guestBytes / keyBytes == 0 || bytes < guestBytes / keyBytes) return false; + return !overlapped || bytes <= DccKeyCount(descriptor, guestBytes); +} + void StorageTexture::MarkDirty() { if (descriptor.dccAddress != 0 && IsDccClear(uploadedKeys) && !IsDccClear(filledKeys)) { traceKeyStore("first write", descriptor, guestBytes); @@ -2479,15 +2784,24 @@ void StorageTexture::FlushAllPending(const char* reason) { static_cast(FlushPending(0, std::numeric_limits::max(), nullptr, reason)); } +bool StorageTexture::StoreAtFlipRequested(const char* value) { + if (value == nullptr) return false; + if (std::strcmp(value, "1") == 0) return true; + throw std::runtime_error(std::string("APS5_STORE_AT_FLIP=") + value + ": expected 1"); +} + std::shared_ptr StorageTexture::FindPending(std::uint64_t address, std::uint64_t bytes) { auto& pending = Pending(); std::lock_guard lock(pending.mutex); + StorageTexture* containing = nullptr; for (auto* texture : pending.textures) { + if (texture->descriptor.baseAddress != address || texture->guestBytes < bytes) continue; + if (texture->guestBytes == bytes) return texture->weak_from_this().lock(); // Containment, not equality: a descriptor of a chain's first mips (its own guestBytes are // shorter) is served by the chain's image; CanCopyFrom then checks the geometry. - if (texture->descriptor.baseAddress == address && texture->guestBytes >= bytes) return texture->weak_from_this().lock(); + if (containing == nullptr) containing = texture; } - return nullptr; + return containing != nullptr ? containing->weak_from_this().lock() : nullptr; } bool PendingStorageOverlaps(std::uint64_t address, std::size_t bytes, const StorageTexture* except) { @@ -2762,7 +3076,7 @@ StorageTexture::FillCoverage StorageTexture::ClassifyFill(std::uint64_t address, std::lock_guard lock(live.mutex); std::vector overlapping; for (auto* texture : live.textures) { - if (!texture->released && texture->guestBytes != 0 && texture->overlaps(address, bytes)) overlapping.push_back(texture); + if (texture->overlapsLive(address, bytes)) overlapping.push_back(texture); } const auto end = address + bytes; StorageTexture* covered = nullptr; @@ -2803,14 +3117,11 @@ StorageTexture::FillCoverage StorageTexture::ClassifyFill(std::uint64_t address, if (texture->descriptor.baseAddress >= address && texture->descriptor.baseAddress + texture->guestBytes <= end) ++coverage.inside; } } - } else if (overlapping.empty()) { - constexpr std::uint64_t keyBytes = 256; - for (const auto* texture : live.textures) { - if (!texture->released && texture->descriptor.dccAddress == address && texture->guestBytes / keyBytes != 0 && bytes >= texture->guestBytes / keyBytes) coverage.cover = FillCover::Keys; - } + } else if (std::any_of(live.textures.begin(), live.textures.end(), [&](const StorageTexture* texture) { return texture->keysFillMatches(address, bytes, !overlapping.empty()); })) { + coverage.cover = FillCover::Keys; } else if (overlapping.size() > 1) { coverage.cover = FillCover::Several; - } else { + } else if (overlapping.size() == 1) { auto* single = overlapping.front(); const auto begin = single->descriptor.baseAddress; const auto stop = begin + single->guestBytes; @@ -2848,13 +3159,13 @@ std::size_t StorageTexture::NoteKeysFill(std::uint64_t address, std::size_t byte case 0xff: keys = DccKeys::Uncompressed; break; default: return 0; } - constexpr std::uint64_t keyBytes = 256; static const bool traceKeys = std::getenv("APS5_TRACE_DCC_KEYS") != nullptr; auto& live = Live(); std::lock_guard lock(live.mutex); + const bool overlapped = std::any_of(live.textures.begin(), live.textures.end(), [&](const StorageTexture* texture) { return texture->overlapsLive(address, bytes); }); std::size_t covered = 0; for (auto* texture : live.textures) { - if (texture->released || texture->descriptor.dccAddress != address || texture->guestBytes / keyBytes == 0 || bytes < texture->guestBytes / keyBytes) continue; + if (!texture->keysFillMatches(address, bytes, overlapped)) continue; texture->filledKeys = keys; ++covered; if (traceKeys && IsDccClear(keys)) std::fprintf(stderr, "[dcc-keys] %s key fill over 0x%llx+0x%llx (uploaded %s, dirty %d)\n", DccKeysName(keys), static_cast(texture->descriptor.baseAddress), static_cast(texture->guestBytes), DccKeysName(texture->uploadedKeys), texture->dirty ? 1 : 0); @@ -2876,13 +3187,13 @@ std::size_t StorageTexture::ClearByKeysFill(std::uint64_t address, std::size_t b case 0x20: keys = DccKeys::ClearRegister; break; default: return 0; } - constexpr std::uint64_t keyBytes = 256; std::vector> covered; { auto& live = Live(); std::lock_guard lock(live.mutex); + const bool overlapped = std::any_of(live.textures.begin(), live.textures.end(), [&](const StorageTexture* texture) { return texture->overlapsLive(address, bytes); }); for (auto* texture : live.textures) { - if (texture->released || texture->descriptor.dccAddress != address || texture->guestBytes / keyBytes == 0 || bytes < texture->guestBytes / keyBytes) continue; + if (!texture->keysFillMatches(address, bytes, overlapped)) continue; if (auto shared = texture->weak_from_this().lock()) covered.push_back(std::move(shared)); } } @@ -2972,7 +3283,8 @@ bool StorageTexture::FillClear(std::span pattern, std::u } } traceRefusal{*this, pattern, layer, refusal}; VkClearColorValue clearValue{}; - if (!FillClearColor(storageFormat, BytesPerElement(descriptor.format), pattern, clearValue)) { + const auto elementBytes = BytesPerElement(descriptor.format); + if (!FillClearColor(storageFormat, elementBytes, pattern, clearValue) && !(ClearKeepsDenormals(context, storageFormat) && FillClearColor(storageFormat, elementBytes, pattern, clearValue, true))) { refusal = "pattern"; return false; } @@ -3433,6 +3745,13 @@ void StorageTexture::writeBack(std::uint64_t address, std::size_t bytes) { writeBackLayers(layers); } +bool StorageTexture::unchangedSinceBaseline(std::uint64_t from, std::uint64_t to) const { + const auto& baseline = originalValid ? original : generationBaseline; + if (baseline.size() != guestBytes) return false; + const auto offset = static_cast(from - descriptor.baseAddress); + return GuestMemory::EqualsCommittedUnsynced(from, std::span(baseline).subspan(offset, static_cast(to - from))); +} + void StorageTexture::writeBackLayers(const std::vector& layers) { CaptureTrace::Log("writeback image=%llx generation=%llu reason=%s units=%zu", static_cast(descriptor.baseAddress), static_cast(generation), flushReason, static_cast(std::count(layers.begin(), layers.end(), true))); static const bool profile = std::getenv("APS5_PROFILE_DRAW") != nullptr; @@ -3494,7 +3813,7 @@ void StorageTexture::writeBackLayers(const std::vector& layers) { const auto to = std::min(at + block, end); if (from >= to) continue; const bool edge = from != at || to != at + block; - if (changedBlocks[static_cast((at - spanBegin) / block)] == GuestMemory::BlockWritten && (compared || !edge || GuestMemory::StoredOver(from, static_cast(to - from), layerGeneration[layer]))) { + if (changedBlocks[static_cast((at - spanBegin) / block)] == GuestMemory::BlockWritten && (compared || !edge || GuestMemory::StoredOver(from, static_cast(to - from), layerGeneration[layer])) && !unchangedSinceBaseline(from, to)) { skippedAny = true; skippedLayer[layer] = true; ++skipped; @@ -3590,7 +3909,7 @@ void StorageTexture::writeBackLayers(const std::vector& layers) { recorder->FlushStoresOverlapping(firstStored, static_cast(lastStored - firstStored)); commands = recorder->Commands(); timing = recorder->BeginGpuTiming(Recorder::CommandClass::StorageWriteBack); - recorder->Keep(linear, linear->Size()); + if (linear) recorder->Keep(linear, linear->Size()); recorder->Keep(tiledScratch, tiledScratch->Size()); for (const auto& slab : padding.slabs) recorder->Keep(slab); // The image itself must outlive the recorded retile: the cache may evict it right after. @@ -3635,7 +3954,7 @@ void StorageTexture::writeBackLayers(const std::vector& layers) { for (std::uint32_t level = 0; level < mips.size(); ++level) { if (!geometry.HasLayer(level, layer)) continue; const auto& mip = mips[level]; - detiler.Dispatch(commands, descriptor.tileMode, elementBytes, linear->Handle(), geometry.LinearLayerOffset(layer) + mip.linearOffset, tiledScratch->Handle(), geometry.GuestLayerOffset(layer) + mip.tiledOffset, mip, true, layer, geometry.thick); + detiler.Dispatch(commands, descriptor.tileMode, elementBytes, linear->Handle(), geometry.LinearLayerOffset(layer) + mip.linearOffset, tiledScratch->Handle(), geometry.GuestLayerOffset(layer) + mip.tiledOffset, mip, true, layer, geometry.thick, {.pipeBankXor = descriptor.pipeBankXor}); } } { @@ -3728,7 +4047,7 @@ void StorageTexture::writeBackLayers(const std::vector& layers) { for (std::uint32_t level = 0; level < mips.size(); ++level) { if (!geometry.HasLayer(level, layer)) continue; const auto& mip = mips[level]; - detiler.Dispatch(commands, descriptor.tileMode, elementBytes, linear.Handle(), geometry.LinearLayerOffset(layer) + mip.linearOffset, tiled.Handle(), geometry.GuestLayerOffset(layer) + mip.tiledOffset, mip, true, layer, geometry.thick); + detiler.Dispatch(commands, descriptor.tileMode, elementBytes, linear.Handle(), geometry.LinearLayerOffset(layer) + mip.linearOffset, tiled.Handle(), geometry.GuestLayerOffset(layer) + mip.tiledOffset, mip, true, layer, geometry.thick, {.pipeBankXor = descriptor.pipeBankXor}); } } // Debug aid: APS5_DUMP_STORAGE= saves that storage image's first mip after each of @@ -3820,6 +4139,10 @@ void StorageTexture::release() noexcept { atomicViews.clear(); for (const auto& [key, uint] : uintViews) context.Function("vkDestroyImageView")(context.device, uint, nullptr); uintViews.clear(); + if (elementView) context.Function("vkDestroyImageView")(context.device, elementView, nullptr); + elementView = VK_NULL_HANDLE; + for (const auto& [key, element] : elementLayerViews) context.Function("vkDestroyImageView")(context.device, element, nullptr); + elementLayerViews.clear(); for (const auto& [format, attachment] : attachmentViews) context.Function("vkDestroyImageView")(context.device, attachment, nullptr); attachmentViews.clear(); if (proxyView) context.Function("vkDestroyImageView")(context.device, proxyView, nullptr); diff --git a/core/libs/prx/libSceAgcDriver/Graphics/src/TextureDetiler.cpp b/core/libs/prx/libSceAgcDriver/Graphics/src/TextureDetiler.cpp index b065119d1..bed858e7a 100644 --- a/core/libs/prx/libSceAgcDriver/Graphics/src/TextureDetiler.cpp +++ b/core/libs/prx/libSceAgcDriver/Graphics/src/TextureDetiler.cpp @@ -1,11 +1,15 @@ #include "prx/libSceAgcDriver/Graphics/include/TextureDetiler.hpp" +#include "prx/libSceAgcDriver/Graphics/include/Resources.hpp" #include "prx/libSceAgcDriver/Graphics/include/TextureSwizzleEquations.hpp" #include "prx/libSceAgcDriver/Graphics/include/TextureTiling.hpp" #include "prx/libSceAgcDriver/Graphics/shaders/TextureDetile_spv.h" +#include "prx/libSceAgcDriver/Graphics/shaders/CmaskClear_spv.h" +#include "prx/libSceAgcDriver/Graphics/shaders/TextureDetileImage_spv.h" #include #include #include #include +#include #include #include @@ -31,6 +35,7 @@ struct Push { std::uint32_t linearBase; std::uint32_t columnBegin; std::uint32_t rowBegin; + std::uint32_t pipeBankXor; }; std::uint32_t BlockBytesFor(TextureTileMode tileMode) { @@ -54,8 +59,8 @@ std::uint32_t BlockBytesFor(TextureTileMode tileMode) { throw std::runtime_error("AGC graphics: TextureDetiler encountered an unknown tile mode"); } -std::uint32_t PipelineKey(TextureTileMode tileMode, std::uint32_t elementBytes, bool retile, bool thick) { - return (thick ? 1u << 17 : 0u) | (retile ? 1u << 16 : 0u) | (static_cast(tileMode) << 8) | elementBytes; +std::uint32_t PipelineKey(TextureTileMode tileMode, std::uint32_t elementBytes, bool retile, bool thick, bool image) { + return (image ? 1u << 18 : 0u) | (thick ? 1u << 17 : 0u) | (retile ? 1u << 16 : 0u) | (static_cast(tileMode) << 8) | elementBytes; } } @@ -87,6 +92,10 @@ TextureDetiler::TextureDetiler(const Context& context) : context(context) { moduleInfo.codeSize = sizeof(TEXTURE_DETILE_SPV); moduleInfo.pCode = TEXTURE_DETILE_SPV; Check(context.Function("vkCreateShaderModule")(context.device, &moduleInfo, nullptr, &module), "vkCreateShaderModule"); + bindings[1].descriptorType = VK_DESCRIPTOR_TYPE_STORAGE_IMAGE; + Check(context.Function("vkCreateDescriptorSetLayout")(context.device, &layoutInfo, nullptr, &imageDescriptorLayout), "vkCreateDescriptorSetLayout"); + pipelineLayoutInfo.pSetLayouts = &imageDescriptorLayout; + Check(context.Function("vkCreatePipelineLayout")(context.device, &pipelineLayoutInfo, nullptr, &imagePipelineLayout), "vkCreatePipelineLayout"); } catch (...) { release(); throw; @@ -98,16 +107,36 @@ TextureDetiler::~TextureDetiler() { } void TextureDetiler::release() noexcept { + for (const auto pool : cmaskDescriptorPools) context.Function("vkDestroyDescriptorPool")(context.device, pool, nullptr); + if (cmaskPipeline) context.Function("vkDestroyPipeline")(context.device, cmaskPipeline, nullptr); + if (cmaskPipelineLayout) context.Function("vkDestroyPipelineLayout")(context.device, cmaskPipelineLayout, nullptr); + if (cmaskDescriptorLayout) context.Function("vkDestroyDescriptorSetLayout")(context.device, cmaskDescriptorLayout, nullptr); + cmaskErrors.reset(); for (const auto pool : descriptorPools) context.Function("vkDestroyDescriptorPool")(context.device, pool, nullptr); + for (const auto pool : imageDescriptorPools) context.Function("vkDestroyDescriptorPool")(context.device, pool, nullptr); + if (imageModule) context.Function("vkDestroyShaderModule")(context.device, imageModule, nullptr); + if (imagePipelineLayout) context.Function("vkDestroyPipelineLayout")(context.device, imagePipelineLayout, nullptr); + if (imageDescriptorLayout) context.Function("vkDestroyDescriptorSetLayout")(context.device, imageDescriptorLayout, nullptr); for (const auto& entry : pipelines) context.Function("vkDestroyPipeline")(context.device, entry.second, nullptr); if (module) context.Function("vkDestroyShaderModule")(context.device, module, nullptr); if (pipelineLayout) context.Function("vkDestroyPipelineLayout")(context.device, pipelineLayout, nullptr); if (descriptorLayout) context.Function("vkDestroyDescriptorSetLayout")(context.device, descriptorLayout, nullptr); } -VkPipeline TextureDetiler::pipeline(TextureTileMode tileMode, std::uint32_t elementBytes, bool retile, bool thick) { +VkFormat TextureDetiler::ImageElementFormat(std::uint32_t elementBytes) { + switch (elementBytes) { + case 1: return VK_FORMAT_R8_UINT; + case 2: return VK_FORMAT_R16_UINT; + case 4: return VK_FORMAT_R32_UINT; + case 8: return VK_FORMAT_R32G32_UINT; + case 16: return VK_FORMAT_R32G32B32A32_UINT; + default: return VK_FORMAT_UNDEFINED; + } +} + +VkPipeline TextureDetiler::pipeline(TextureTileMode tileMode, std::uint32_t elementBytes, bool retile, bool thick, bool image) { Require(std::has_single_bit(elementBytes) && elementBytes <= 16u, "unsupported element size for texture detiling"); - const auto key = PipelineKey(tileMode, elementBytes, retile, thick); + const auto key = PipelineKey(tileMode, elementBytes, retile, thick, image); for (const auto& entry : pipelines) { if (entry.first == key) return entry.second; } @@ -115,8 +144,8 @@ VkPipeline TextureDetiler::pipeline(TextureTileMode tileMode, std::uint32_t elem // per-bit XOR equation for the equation family (2); 20-21 override the block extent in elements. std::array values{elementBytes, BlockBytesFor(tileMode), tileMode == TextureTileMode::kLinear ? 0u : 1u, retile ? 1u : 0u}; if (thick) { - const auto thickMode = tileMode == TextureTileMode::kStandard4KB ? 0x105u : tileMode == TextureTileMode::kStandard64KB ? 0x109u : tileMode == TextureTileMode::kS64KBX ? 0x119u : 0u; - Require(thickMode != 0, "3D textures are only detiled thick from SW_4KB_S, SW_64KB_S or SW_64KB_S_X"); + const auto thickMode = tileMode == TextureTileMode::kStandard4KB ? 0x105u : tileMode == TextureTileMode::kStandard64KB ? 0x109u : tileMode == TextureTileMode::kS64KBX ? 0x119u : tileMode == TextureTileMode::kD64KBX ? 0x11au : 0u; + Require(thickMode != 0, "3D textures are only detiled thick from SW_4KB_S, SW_64KB_S, SW_64KB_S_X or SW_64KB_D_X"); const auto* equation = FindTextureSwizzleEquation(thickMode, elementBytes); Require(equation != nullptr, "no thick swizzle equation for the element size"); values[2] = 2u; @@ -130,6 +159,12 @@ VkPipeline TextureDetiler::pipeline(TextureTileMode tileMode, std::uint32_t elem values[2] = 2u; std::copy(equation->bits.begin(), equation->bits.end(), values.begin() + 4); } + if (image && imageModule == VK_NULL_HANDLE) { + VkShaderModuleCreateInfo moduleInfo{VK_STRUCTURE_TYPE_SHADER_MODULE_CREATE_INFO}; + moduleInfo.codeSize = sizeof(TEXTURE_DETILE_IMAGE_SPV); + moduleInfo.pCode = TEXTURE_DETILE_IMAGE_SPV; + Check(context.Function("vkCreateShaderModule")(context.device, &moduleInfo, nullptr, &imageModule), "vkCreateShaderModule"); + } std::array entries{}; for (std::uint32_t index = 0; index < entries.size(); ++index) entries[index] = {index, index * 4u, 4}; VkSpecializationInfo specialization{}; @@ -139,27 +174,32 @@ VkPipeline TextureDetiler::pipeline(TextureTileMode tileMode, std::uint32_t elem specialization.pData = values.data(); VkPipelineShaderStageCreateInfo stage{VK_STRUCTURE_TYPE_PIPELINE_SHADER_STAGE_CREATE_INFO}; stage.stage = VK_SHADER_STAGE_COMPUTE_BIT; - stage.module = module; + stage.module = image ? imageModule : module; stage.pName = "main"; stage.pSpecializationInfo = &specialization; VkComputePipelineCreateInfo createInfo{VK_STRUCTURE_TYPE_COMPUTE_PIPELINE_CREATE_INFO}; createInfo.stage = stage; - createInfo.layout = pipelineLayout; + createInfo.layout = image ? imagePipelineLayout : pipelineLayout; VkPipeline result = VK_NULL_HANDLE; Check(context.Function("vkCreateComputePipelines")(context.device, context.pipelineCache, 1, &createInfo, nullptr, &result), "vkCreateComputePipelines"); pipelines.emplace_back(key, result); return result; } +void TextureDetiler::checkWindow(const TileMipLayout& layout, const DetileWindow& window, std::uint32_t& columnEnd, std::uint32_t& rowEnd) const { + Require(layout.width != 0 && layout.height != 0, "texture detiling requires a non-empty mip layout"); + Require(layout.tiledSize != 0 && layout.linearSize != 0, "texture detiling requires a non-empty mip layout"); + columnEnd = window.columnEnd != 0 ? std::min(window.columnEnd, layout.width) : layout.width; + rowEnd = window.rowEnd != 0 ? std::min(window.rowEnd, layout.height) : layout.height; + if (!(window.columnBegin < columnEnd && window.rowBegin < rowEnd)) Require(false, "texture detiling window lies outside the mip: columns " + std::to_string(window.columnBegin) + ".." + std::to_string(columnEnd) + ", rows " + std::to_string(window.rowBegin) + ".." + std::to_string(rowEnd) + " of " + std::to_string(layout.width) + "x" + std::to_string(layout.height)); +} + void TextureDetiler::Dispatch(VkCommandBuffer commands, TextureTileMode tileMode, std::uint32_t elementBytes, VkBuffer source, std::uint64_t sourceOffset, VkBuffer destination, std::uint64_t destinationOffset, const TileMipLayout& layout, bool retile, std::uint32_t slice, bool thick, const DetileWindow& window) { Require(commands != VK_NULL_HANDLE, "texture detiling requires an active command buffer"); Require(window.rangeBegin < window.rangeEnd && window.tiledBase <= window.rangeBegin, "texture detiling window is empty"); Require(source != VK_NULL_HANDLE && destination != VK_NULL_HANDLE, "texture detiling requires source and destination buffers"); - Require(layout.width != 0 && layout.height != 0, "texture detiling requires a non-empty mip layout"); - Require(layout.tiledSize != 0 && layout.linearSize != 0, "texture detiling requires a non-empty mip layout"); - const auto columnEnd = window.columnEnd != 0 ? std::min(window.columnEnd, layout.width) : layout.width; - const auto rowEnd = window.rowEnd != 0 ? std::min(window.rowEnd, layout.height) : layout.height; - if (!(window.columnBegin < columnEnd && window.rowBegin < rowEnd)) Require(false, "texture detiling window lies outside the mip: columns " + std::to_string(window.columnBegin) + ".." + std::to_string(columnEnd) + ", rows " + std::to_string(window.rowBegin) + ".." + std::to_string(rowEnd) + " of " + std::to_string(layout.width) + "x" + std::to_string(layout.height)); + std::uint32_t columnEnd = 0, rowEnd = 0; + checkWindow(layout, window, columnEnd, rowEnd); const auto target = pipeline(tileMode, elementBytes, retile, thick); const auto alignment = std::max(context.limits.minStorageBufferOffsetAlignment, 4); const auto sourceDescriptorOffset = sourceOffset - sourceOffset % alignment; @@ -211,10 +251,175 @@ void TextureDetiler::Dispatch(VkCommandBuffer commands, TextureTileMode tileMode push.linearBase = window.linearBase; push.columnBegin = window.columnBegin; push.rowBegin = window.rowBegin; + push.pipeBankXor = window.pipeBankXor; context.Function("vkCmdPushConstants")(commands, pipelineLayout, VK_SHADER_STAGE_COMPUTE_BIT, 0, sizeof(Push), &push); const auto groupsX = (columnEnd - window.columnBegin + 7u) / 8u; const auto groupsY = (rowEnd - window.rowBegin + 7u) / 8u; context.Function("vkCmdDispatch")(commands, groupsX, groupsY, 1); } +std::uint32_t TextureDetiler::CmaskErrors() { + if (cmaskErrors == nullptr) return 0; + std::uint32_t codes = 0; + std::memcpy(&codes, cmaskErrors->Bytes().data(), sizeof(codes)); + return codes; +} + +void TextureDetiler::DispatchCmaskClear(VkCommandBuffer commands, VkBuffer cmask, std::uint64_t cmaskOffset, std::size_t cmaskBytes, VkImageView view, std::uint32_t width, std::uint32_t height, std::uint32_t elementBytes, const std::array& clearWords, bool write) { + Require(commands != VK_NULL_HANDLE && cmask != VK_NULL_HANDLE && view != VK_NULL_HANDLE, "a CMASK clear needs a command buffer, the CMASK buffer and the target view"); + Require(elementBytes == 1 || elementBytes == 2 || elementBytes == 4 || elementBytes == 8, "a CMASK clear of this element size is not modeled"); + Require(cmaskOffset % 4 == 0 && cmaskBytes % 4 == 0 && cmaskBytes != 0, "a CMASK clear needs a word-aligned CMASK"); + if (cmaskPipeline == VK_NULL_HANDLE) { + std::array bindings{}; + for (std::uint32_t index = 0; index < bindings.size(); ++index) { + bindings[index].binding = index; + bindings[index].descriptorType = index == 2 ? VK_DESCRIPTOR_TYPE_STORAGE_IMAGE : VK_DESCRIPTOR_TYPE_STORAGE_BUFFER; + bindings[index].descriptorCount = 1; + bindings[index].stageFlags = VK_SHADER_STAGE_COMPUTE_BIT; + } + VkDescriptorSetLayoutCreateInfo layoutInfo{VK_STRUCTURE_TYPE_DESCRIPTOR_SET_LAYOUT_CREATE_INFO}; + layoutInfo.bindingCount = static_cast(bindings.size()); + layoutInfo.pBindings = bindings.data(); + Check(context.Function("vkCreateDescriptorSetLayout")(context.device, &layoutInfo, nullptr, &cmaskDescriptorLayout), "vkCreateDescriptorSetLayout CMASK clear"); + const VkPushConstantRange pushRange{VK_SHADER_STAGE_COMPUTE_BIT, 0, 7 * sizeof(std::uint32_t)}; + VkPipelineLayoutCreateInfo pipelineLayoutInfo{VK_STRUCTURE_TYPE_PIPELINE_LAYOUT_CREATE_INFO}; + pipelineLayoutInfo.setLayoutCount = 1; + pipelineLayoutInfo.pSetLayouts = &cmaskDescriptorLayout; + pipelineLayoutInfo.pushConstantRangeCount = 1; + pipelineLayoutInfo.pPushConstantRanges = &pushRange; + Check(context.Function("vkCreatePipelineLayout")(context.device, &pipelineLayoutInfo, nullptr, &cmaskPipelineLayout), "vkCreatePipelineLayout CMASK clear"); + VkShaderModuleCreateInfo moduleInfo{VK_STRUCTURE_TYPE_SHADER_MODULE_CREATE_INFO}; + moduleInfo.codeSize = sizeof(CMASK_CLEAR_SPV); + moduleInfo.pCode = CMASK_CLEAR_SPV; + VkShaderModule cmaskModule = VK_NULL_HANDLE; + Check(context.Function("vkCreateShaderModule")(context.device, &moduleInfo, nullptr, &cmaskModule), "vkCreateShaderModule CMASK clear"); + VkComputePipelineCreateInfo pipelineInfo{VK_STRUCTURE_TYPE_COMPUTE_PIPELINE_CREATE_INFO}; + pipelineInfo.stage = {VK_STRUCTURE_TYPE_PIPELINE_SHADER_STAGE_CREATE_INFO}; + pipelineInfo.stage.stage = VK_SHADER_STAGE_COMPUTE_BIT; + pipelineInfo.stage.module = cmaskModule; + pipelineInfo.stage.pName = "main"; + pipelineInfo.layout = cmaskPipelineLayout; + const auto created = context.Function("vkCreateComputePipelines")(context.device, context.pipelineCache, 1, &pipelineInfo, nullptr, &cmaskPipeline); + context.Function("vkDestroyShaderModule")(context.device, cmaskModule, nullptr); + Check(created, "vkCreateComputePipelines CMASK clear"); + cmaskErrors = std::make_unique(context, 4, VK_BUFFER_USAGE_STORAGE_BUFFER_BIT); + std::memset(cmaskErrors->Bytes().data(), 0, 4); + } + constexpr std::uint32_t setsPerPool = 64; + const auto poolIndex = allocatedCmaskSets / setsPerPool; + if (poolIndex == cmaskDescriptorPools.size()) { + const std::array sizes{{{VK_DESCRIPTOR_TYPE_STORAGE_BUFFER, setsPerPool * 2}, {VK_DESCRIPTOR_TYPE_STORAGE_IMAGE, setsPerPool}}}; + VkDescriptorPoolCreateInfo info{VK_STRUCTURE_TYPE_DESCRIPTOR_POOL_CREATE_INFO}; + info.maxSets = setsPerPool; + info.poolSizeCount = static_cast(sizes.size()); + info.pPoolSizes = sizes.data(); + cmaskDescriptorPools.reserve(cmaskDescriptorPools.size() + 1); + VkDescriptorPool pool = VK_NULL_HANDLE; + Check(context.Function("vkCreateDescriptorPool")(context.device, &info, nullptr, &pool), "vkCreateDescriptorPool CMASK clear"); + cmaskDescriptorPools.push_back(pool); + } + VkDescriptorSetAllocateInfo allocation{VK_STRUCTURE_TYPE_DESCRIPTOR_SET_ALLOCATE_INFO}; + allocation.descriptorPool = cmaskDescriptorPools.at(poolIndex); + allocation.descriptorSetCount = 1; + allocation.pSetLayouts = &cmaskDescriptorLayout; + VkDescriptorSet set = VK_NULL_HANDLE; + Check(context.Function("vkAllocateDescriptorSets")(context.device, &allocation, &set), "vkAllocateDescriptorSets CMASK clear"); + ++allocatedCmaskSets; + const auto alignment = std::max(context.limits.minStorageBufferOffsetAlignment, 4); + const auto descriptorOffset = cmaskOffset - cmaskOffset % alignment; + const auto base = cmaskOffset - descriptorOffset; + const VkDescriptorBufferInfo cmaskInfo{cmask, descriptorOffset, base + cmaskBytes}; + const VkDescriptorBufferInfo errorInfo{cmaskErrors->Handle(), 0, 4}; + const VkDescriptorImageInfo imageInfo{VK_NULL_HANDLE, view, VK_IMAGE_LAYOUT_GENERAL}; + std::array writes{}; + for (std::uint32_t index = 0; index < writes.size(); ++index) { + writes[index].sType = VK_STRUCTURE_TYPE_WRITE_DESCRIPTOR_SET; + writes[index].dstSet = set; + writes[index].dstBinding = index; + writes[index].descriptorCount = 1; + writes[index].descriptorType = index == 2 ? VK_DESCRIPTOR_TYPE_STORAGE_IMAGE : VK_DESCRIPTOR_TYPE_STORAGE_BUFFER; + } + writes[0].pBufferInfo = &cmaskInfo; + writes[1].pBufferInfo = &errorInfo; + writes[2].pImageInfo = &imageInfo; + context.Function("vkUpdateDescriptorSets")(context.device, static_cast(writes.size()), writes.data(), 0, nullptr); + const auto barrier = [&](VkPipelineStageFlags source, VkPipelineStageFlags destination, VkAccessFlags sourceAccess, VkAccessFlags destinationAccess) { + VkMemoryBarrier memory{VK_STRUCTURE_TYPE_MEMORY_BARRIER}; + memory.srcAccessMask = sourceAccess; + memory.dstAccessMask = destinationAccess; + context.Function("vkCmdPipelineBarrier")(commands, source, destination, 0, 1, &memory, 0, nullptr, 0, nullptr); + }; + constexpr VkAccessFlags anyWrite = VK_ACCESS_SHADER_WRITE_BIT | VK_ACCESS_TRANSFER_WRITE_BIT | VK_ACCESS_COLOR_ATTACHMENT_WRITE_BIT | VK_ACCESS_HOST_WRITE_BIT; + constexpr VkAccessFlags anyAccess = anyWrite | VK_ACCESS_SHADER_READ_BIT | VK_ACCESS_TRANSFER_READ_BIT | VK_ACCESS_COLOR_ATTACHMENT_READ_BIT | VK_ACCESS_HOST_READ_BIT | VK_ACCESS_INDIRECT_COMMAND_READ_BIT; + barrier(VK_PIPELINE_STAGE_ALL_COMMANDS_BIT, VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT, anyWrite, VK_ACCESS_SHADER_READ_BIT | VK_ACCESS_SHADER_WRITE_BIT); + context.Function("vkCmdBindPipeline")(commands, VK_PIPELINE_BIND_POINT_COMPUTE, cmaskPipeline); + context.Function("vkCmdBindDescriptorSets")(commands, VK_PIPELINE_BIND_POINT_COMPUTE, cmaskPipelineLayout, 0, 1, &set, 0, nullptr); + const std::uint32_t mask = elementBytes >= 4 ? 0xffffffffu : (1u << (elementBytes * 8u)) - 1u; + const std::array push{static_cast(base), width, height, (width + 1023u) / 1024u, clearWords[0] & mask, elementBytes == 8 ? clearWords[1] : 0u, write ? 1u : 0u}; + context.Function("vkCmdPushConstants")(commands, cmaskPipelineLayout, VK_SHADER_STAGE_COMPUTE_BIT, 0, sizeof(push), push.data()); + context.Function("vkCmdDispatch")(commands, (width + 7u) / 8u, (height + 7u) / 8u, 1); + barrier(VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT, VK_PIPELINE_STAGE_ALL_COMMANDS_BIT, VK_ACCESS_SHADER_WRITE_BIT | VK_ACCESS_SHADER_READ_BIT, anyAccess); + context.Function("vkCmdFillBuffer")(commands, cmask, cmaskOffset, cmaskBytes, 0xffffffffu); + barrier(VK_PIPELINE_STAGE_TRANSFER_BIT, VK_PIPELINE_STAGE_ALL_COMMANDS_BIT | VK_PIPELINE_STAGE_HOST_BIT, VK_ACCESS_TRANSFER_WRITE_BIT, anyAccess); +} + +void TextureDetiler::DispatchImage(VkCommandBuffer commands, TextureTileMode tileMode, std::uint32_t elementBytes, VkBuffer tiled, std::uint64_t tiledOffset, VkImageView view, const TileMipLayout& layout, bool retile, std::uint32_t slice, const DetileWindow& window) { + Require(commands != VK_NULL_HANDLE, "texture detiling requires an active command buffer"); + Require(window.rangeBegin < window.rangeEnd && window.tiledBase <= window.rangeBegin, "texture detiling window is empty"); + Require(tiled != VK_NULL_HANDLE && view != VK_NULL_HANDLE, "texture detiling requires a tiled buffer and an image view"); + Require(ImageElementFormat(elementBytes) != VK_FORMAT_UNDEFINED, "unsupported element size for texture detiling through an image"); + std::uint32_t columnEnd = 0, rowEnd = 0; + checkWindow(layout, window, columnEnd, rowEnd); + const auto target = pipeline(tileMode, elementBytes, retile, false, true); + const auto alignment = std::max(context.limits.minStorageBufferOffsetAlignment, 4); + const auto descriptorOffset = tiledOffset - tiledOffset % alignment; + const auto base = tiledOffset - descriptorOffset; + Require(base <= UINT32_MAX, "texture detiling buffer offset exceeds addressable range"); + const std::uint64_t tiledBytes = window.rangeEnd == 0xffffffffu ? layout.tiledSize : std::min(layout.tiledSize, window.rangeEnd) - window.tiledBase; + const auto range = (base + tiledBytes + 3) / 4 * 4; + Require(range <= context.limits.maxStorageBufferRange, "texture detiling buffer range exceeds device limits"); + const auto set = allocateSet(true); + const VkDescriptorBufferInfo bufferInfo{tiled, descriptorOffset, range}; + const VkDescriptorImageInfo imageInfo{VK_NULL_HANDLE, view, VK_IMAGE_LAYOUT_GENERAL}; + std::array writes{}; + writes[0].sType = VK_STRUCTURE_TYPE_WRITE_DESCRIPTOR_SET; + writes[0].dstSet = set; + writes[0].dstBinding = 0; + writes[0].descriptorCount = 1; + writes[0].descriptorType = VK_DESCRIPTOR_TYPE_STORAGE_BUFFER; + writes[0].pBufferInfo = &bufferInfo; + writes[1].sType = VK_STRUCTURE_TYPE_WRITE_DESCRIPTOR_SET; + writes[1].dstSet = set; + writes[1].dstBinding = 1; + writes[1].descriptorCount = 1; + writes[1].descriptorType = VK_DESCRIPTOR_TYPE_STORAGE_IMAGE; + writes[1].pImageInfo = &imageInfo; + context.Function("vkUpdateDescriptorSets")(context.device, static_cast(writes.size()), writes.data(), 0, nullptr); + context.Function("vkCmdBindPipeline")(commands, VK_PIPELINE_BIND_POINT_COMPUTE, target); + context.Function("vkCmdBindDescriptorSets")(commands, VK_PIPELINE_BIND_POINT_COMPUTE, imagePipelineLayout, 0, 1, &set, 0, nullptr); + Push push{}; + push.srcBase = static_cast(retile ? 0 : base); + push.dstBase = static_cast(retile ? base : 0); + push.width = layout.width; + push.height = layout.height; + push.pitchBytes = layout.pitchBytes; + push.blocksPerRow = layout.blocksPerRow; + push.tail = layout.tail ? 1u : 0u; + push.tailX = layout.tailX; + push.tailY = layout.tailY; + push.elementBytes = elementBytes; + push.slice = slice; + push.rangeBegin = window.rangeBegin; + push.rangeEnd = window.rangeEnd; + push.tiledBase = window.tiledBase; + push.linearBase = 0; + push.columnBegin = window.columnBegin; + push.rowBegin = window.rowBegin; + push.pipeBankXor = window.pipeBankXor; + context.Function("vkCmdPushConstants")(commands, imagePipelineLayout, VK_SHADER_STAGE_COMPUTE_BIT, 0, sizeof(Push), &push); + const auto groupsX = (columnEnd - window.columnBegin + 7u) / 8u; + const auto groupsY = (rowEnd - window.rowBegin + 7u) / 8u; + context.Function("vkCmdDispatch")(commands, groupsX, groupsY, 1); +} + } diff --git a/core/libs/prx/libSceAgcDriver/Graphics/src/TextureDetilerDescriptors.cpp b/core/libs/prx/libSceAgcDriver/Graphics/src/TextureDetilerDescriptors.cpp index ed3b905e4..4d482528c 100644 --- a/core/libs/prx/libSceAgcDriver/Graphics/src/TextureDetilerDescriptors.cpp +++ b/core/libs/prx/libSceAgcDriver/Graphics/src/TextureDetilerDescriptors.cpp @@ -1,6 +1,7 @@ #include "prx/libSceAgcDriver/Graphics/include/Recorder.hpp" #include "prx/libSceAgcDriver/Graphics/include/TextureDetiler.hpp" #include "prx/libSceAgcDriver/Execution/include/GuestMemory.hpp" +#include namespace AgcDriver::Graphics { @@ -13,29 +14,41 @@ void TextureDetiler::BeginBatch() { Check(context.Function("vkResetDescriptorPool")(context.device, pool, 0), "vkResetDescriptorPool texture detiler"); } allocatedSets = 0; + for (const auto pool : cmaskDescriptorPools) { + Check(context.Function("vkResetDescriptorPool")(context.device, pool, 0), "vkResetDescriptorPool CMASK clear"); + } + allocatedCmaskSets = 0; + for (const auto pool : imageDescriptorPools) { + Check(context.Function("vkResetDescriptorPool")(context.device, pool, 0), "vkResetDescriptorPool texture detiler"); + } + allocatedImageSets = 0; } -VkDescriptorSet TextureDetiler::allocateSet() { +VkDescriptorSet TextureDetiler::allocateSet(bool image) { constexpr std::uint32_t setsPerPool = 64; - const auto poolIndex = allocatedSets / setsPerPool; - if (poolIndex == descriptorPools.size()) { - const VkDescriptorPoolSize size{VK_DESCRIPTOR_TYPE_STORAGE_BUFFER, setsPerPool * 2}; + auto& pools = image ? imageDescriptorPools : descriptorPools; + auto& allocated = image ? allocatedImageSets : allocatedSets; + const auto poolIndex = allocated / setsPerPool; + if (poolIndex == pools.size()) { + std::array sizes{}; + sizes[0] = {VK_DESCRIPTOR_TYPE_STORAGE_BUFFER, image ? setsPerPool : setsPerPool * 2}; + sizes[1] = {VK_DESCRIPTOR_TYPE_STORAGE_IMAGE, setsPerPool}; VkDescriptorPoolCreateInfo info{VK_STRUCTURE_TYPE_DESCRIPTOR_POOL_CREATE_INFO}; info.maxSets = setsPerPool; - info.poolSizeCount = 1; - info.pPoolSizes = &size; - descriptorPools.reserve(descriptorPools.size() + 1); + info.poolSizeCount = image ? 2u : 1u; + info.pPoolSizes = sizes.data(); + pools.reserve(pools.size() + 1); VkDescriptorPool pool = VK_NULL_HANDLE; Check(context.Function("vkCreateDescriptorPool")(context.device, &info, nullptr, &pool), "vkCreateDescriptorPool texture detiler"); - descriptorPools.push_back(pool); + pools.push_back(pool); } VkDescriptorSetAllocateInfo allocation{VK_STRUCTURE_TYPE_DESCRIPTOR_SET_ALLOCATE_INFO}; - allocation.descriptorPool = descriptorPools.at(poolIndex); + allocation.descriptorPool = pools.at(poolIndex); allocation.descriptorSetCount = 1; - allocation.pSetLayouts = &descriptorLayout; + allocation.pSetLayouts = image ? &imageDescriptorLayout : &descriptorLayout; VkDescriptorSet set = VK_NULL_HANDLE; Check(context.Function("vkAllocateDescriptorSets")(context.device, &allocation, &set), "vkAllocateDescriptorSets texture detiler"); - ++allocatedSets; + ++allocated; return set; } diff --git a/core/libs/prx/libSceAgcDriver/Graphics/src/TextureFormat.cpp b/core/libs/prx/libSceAgcDriver/Graphics/src/TextureFormat.cpp index d5978cf76..96eeaf9c2 100644 --- a/core/libs/prx/libSceAgcDriver/Graphics/src/TextureFormat.cpp +++ b/core/libs/prx/libSceAgcDriver/Graphics/src/TextureFormat.cpp @@ -20,6 +20,7 @@ struct FormatEntry { constexpr FormatEntry kFormatLookup[] = { {1, VK_FORMAT_R8_UNORM, 1, false}, + {2, VK_FORMAT_R8_SNORM, 1, false}, {5, VK_FORMAT_R8_UINT, 1, false}, {6, VK_FORMAT_R8_SINT, 1, false}, {7, VK_FORMAT_R16_UNORM, 2, false}, @@ -133,6 +134,19 @@ VkFormat SampledTextureFormat(const Context& context, std::uint32_t guestFormat) return ResolveTextureFormat(static_cast(ShaderRecompiler::SrgbUnormFormat(format))); } +bool IsSrgbTextureFormat(std::uint32_t guestFormat) { + switch (findFormatEntry(guestFormat).vkFormat) { + case VK_FORMAT_R8_SRGB: + case VK_FORMAT_R8G8_SRGB: + case VK_FORMAT_R8G8B8A8_SRGB: + case VK_FORMAT_BC1_RGBA_SRGB_BLOCK: + case VK_FORMAT_BC2_SRGB_BLOCK: + case VK_FORMAT_BC3_SRGB_BLOCK: + case VK_FORMAT_BC7_SRGB_BLOCK: return true; + default: return false; + } +} + std::optional FindGuestTextureFormat(VkFormat format, std::uint32_t elementBytes) { for (const auto& entry : kFormatLookup) { if (!entry.blockCompressed && entry.vkFormat == format && entry.bytesPerElement == elementBytes) return entry.guestFormat; diff --git a/core/libs/prx/libSceAgcDriver/Graphics/src/TextureTiling.cpp b/core/libs/prx/libSceAgcDriver/Graphics/src/TextureTiling.cpp index 2a2beb77f..6eca2ae65 100644 --- a/core/libs/prx/libSceAgcDriver/Graphics/src/TextureTiling.cpp +++ b/core/libs/prx/libSceAgcDriver/Graphics/src/TextureTiling.cpp @@ -116,6 +116,10 @@ bool GetMipTailLayout(TextureTileMode tileMode, const BlockLayout& block, std::u case TextureTileMode::kD64KBX: case TextureTileMode::kR64KBX: out = MakeMipTailLayout(kMipTailThin64KB[index], block.blockWidth >> 1u, block.blockHeight); + if (tileMode == TextureTileMode::kZ64KBX && bytesPerElement <= 2u) { + out.widthLimit >>= 1u; + if (bytesPerElement == 1u) out.heightLimit >>= 1u; + } return true; } throw std::runtime_error("AGC graphics: GetMipTailLayout encountered an unknown tile mode"); @@ -260,8 +264,9 @@ std::array ThickBlockExtent(TextureTileMode tileMode, std::uin switch (tileMode) { case TextureTileMode::kStandard4KB: return {1u << thick4KB[index][0], 1u << thick4KB[index][1], 1u << thick4KB[index][2]}; case TextureTileMode::kStandard64KB: - case TextureTileMode::kS64KBX: return {1u << thick64KB[index][0], 1u << thick64KB[index][1], 1u << thick64KB[index][2]}; - default: throw std::runtime_error("AGC graphics: thick 3D textures are only supported in SW_4KB_S, SW_64KB_S and SW_64KB_S_X, not in swizzle mode " + std::to_string(EquationSwizzleMode(tileMode)) + " at " + std::to_string(bytesPerElement) + " bytes per element"); + case TextureTileMode::kS64KBX: + case TextureTileMode::kD64KBX: return {1u << thick64KB[index][0], 1u << thick64KB[index][1], 1u << thick64KB[index][2]}; + default: throw std::runtime_error("AGC graphics: thick 3D textures are only supported in SW_4KB_S, SW_64KB_S, SW_64KB_S_X and SW_64KB_D_X, not in swizzle mode " + std::to_string(EquationSwizzleMode(tileMode)) + " at " + std::to_string(bytesPerElement) + " bytes per element"); } } @@ -335,8 +340,8 @@ ThickLayout ComputeThickLayout(TextureTileMode tileMode, std::uint32_t format, s mip.tailX = 0; mip.tailY = 0; const bool tiled = tileMode != TextureTileMode::kLinear; - const auto levelWidth = tiled ? std::max(ShiftCeil(elementsWidth0, level), 1u) : mip.width; - const auto levelHeight = tiled ? std::max(ShiftCeil(elementsHeight0, level), 1u) : mip.height; + const auto levelWidth = std::max(ShiftCeil(elementsWidth0, level), 1u); + const auto levelHeight = std::max(ShiftCeil(elementsHeight0, level), 1u); const auto paddedWidth = AlignUp(levelWidth, block[0]); mip.pitchBytes = paddedWidth * bytesPerElement; if (mip.tail) { @@ -356,7 +361,7 @@ ThickLayout ComputeThickLayout(TextureTileMode tileMode, std::uint32_t format, s mip.tiledSize = blockBytes; } else if (!tiled) { mip.blocksPerRow = paddedWidth; - mip.tiledSize = static_cast(mip.pitchBytes) * mip.height; + mip.tiledSize = static_cast(mip.pitchBytes) * levelHeight; } else { mip.blocksPerRow = paddedWidth / block[0]; mip.tiledSize = static_cast(mip.blocksPerRow) * (AlignUp(levelHeight, block[1]) / block[1]) * blockBytes; @@ -377,7 +382,7 @@ ThickLayout ComputeThickLayout(TextureTileMode tileMode, std::uint32_t format, s SurfaceGeometry DescribeSurface(const GuestTextureResource& descriptor) { SurfaceGeometry geometry; - if (descriptor.dimension == TextureDimension::k3D && (descriptor.tileMode == TextureTileMode::kZ64KBX || descriptor.tileMode == TextureTileMode::kD64KBX || descriptor.tileMode == TextureTileMode::kR64KBX)) { + if (descriptor.dimension == TextureDimension::k3D && (descriptor.tileMode == TextureTileMode::kZ64KBX || descriptor.tileMode == TextureTileMode::kR64KBX)) { const auto depth = descriptor.depthOrLastArray + 1u; geometry.mips = ComputeMipLayout(descriptor.tileMode, descriptor.format, descriptor.width, descriptor.height, descriptor.mipCount); geometry.layers = depth; diff --git a/core/libs/prx/libSceAgcDriver/Graphics/src/UnitShadow.cpp b/core/libs/prx/libSceAgcDriver/Graphics/src/UnitShadow.cpp index 10b10065e..a6a8c6af1 100644 --- a/core/libs/prx/libSceAgcDriver/Graphics/src/UnitShadow.cpp +++ b/core/libs/prx/libSceAgcDriver/Graphics/src/UnitShadow.cpp @@ -116,15 +116,24 @@ VkDeviceSize SlabBytes(const ShadowSlab& slab) { } // The shadows overlapping [address, end): imports never overlap, so at most a few. -std::vector> overlapping(const Shadows& registry, std::uint64_t address, std::uint64_t end) { - std::vector> result; - if (end <= address || registry.byBase.empty()) return result; +template +bool anyOverlapping(const Shadows& registry, std::uint64_t address, std::uint64_t end, TVisit&& visit) { + if (end <= address || registry.byBase.empty()) return false; auto it = registry.byBase.upper_bound(address); if (it != registry.byBase.begin()) --it; for (; it != registry.byBase.end() && it->first < end; ++it) { const auto& shadow = it->second; - if (address < shadow->ImportEnd() && shadow->importBase < end) result.push_back(shadow); + if (address < shadow->ImportEnd() && shadow->importBase < end && visit(shadow)) return true; } + return false; +} + +std::vector> overlapping(const Shadows& registry, std::uint64_t address, std::uint64_t end) { + std::vector> result; + anyOverlapping(registry, address, end, [&](const std::shared_ptr& shadow) { + result.push_back(shadow); + return false; + }); return result; } @@ -485,15 +494,15 @@ bool AnyShadowedOverlaps(std::uint64_t address, std::size_t bytes) { const auto end = bytes > std::numeric_limits::max() - address ? std::numeric_limits::max() : address + bytes; auto& registry = Registry(); std::lock_guard lock(registry.mutex); - for (const auto& shadow : overlapping(registry, address, end)) { - if (shadow->liveUnits == 0) continue; + return anyOverlapping(registry, address, end, [&](const std::shared_ptr& shadow) { + if (shadow->liveUnits == 0) return false; const auto begin = std::max(address, shadow->importBase); const auto stop = std::min(end, shadow->ImportEnd()); for (auto unit = shadow->UnitOf(begin); unit <= shadow->UnitOf(stop - 1); ++unit) { if (shadow->Live(unit)) return true; } - } - return false; + return false; + }); } bool AnyShadowedOverlaps(std::span> ranges) { @@ -727,7 +736,7 @@ void MarkShadowed(const HostImport& import, std::span range StorageTexture::BumpPendingSerial(); } -void RetireShadow(const Context& context, const HostImport& import, const std::function& registered) { +void RetireShadow(VkDevice device, const HostImport& import, const std::function& registered) { if (!UnitShadowEnabled()) return; auto& registry = Registry(); std::shared_ptr shadow; @@ -757,7 +766,7 @@ void RetireShadow(const Context& context, const HostImport& import, const std::f if (dropped != 0) Stats().droppedOnRetire.fetch_add(dropped, std::memory_order_relaxed); std::size_t published = 0; if (any) { - if (GuestMemory::GpuMutex().HeldByThisThread() && shadow->context.device == context.device) { + if (GuestMemory::GpuMutex().HeldByThisThread() && shadow->context.device == device) { published = publishUnits(shadow, 0, shadow->Units() - 1, selected, PublishReason::Retire); } else { Stats().lostOnRetire.fetch_add(shadow->liveUnits, std::memory_order_relaxed); diff --git a/core/libs/prx/libSceAgcDriver/Resource/src/Registration.cpp b/core/libs/prx/libSceAgcDriver/Resource/src/Registration.cpp index ff798fca2..9fe7f3639 100644 --- a/core/libs/prx/libSceAgcDriver/Resource/src/Registration.cpp +++ b/core/libs/prx/libSceAgcDriver/Resource/src/Registration.cpp @@ -28,6 +28,14 @@ int APS5_VABI sceAgcDriverRegisterResource(uint32_t* resource_handle, uint32_t o return SCE_AGC_ERROR_RESOURCE_REGISTRATION_UNAVAILABLE; } +int APS5_VABI sceAgcDriverRegisterMultipleResources(uint32_t* resourceHandle, uint32_t ownerHandle, const void* resources, uint32_t resourceCount) { + (void)resourceHandle; + (void)ownerHandle; + (void)resources; + (void)resourceCount; + return SCE_AGC_ERROR_RESOURCE_REGISTRATION_UNAVAILABLE; +} + int APS5_VABI sceAgcDriverRegisterWorkloadStream(uint32_t stream_id, const void* stream) { (void)stream_id; (void)stream; diff --git a/core/libs/prx/libSceAgcDriver/State/src/Configuration.cpp b/core/libs/prx/libSceAgcDriver/State/src/Configuration.cpp index 134e81a64..468f92e98 100644 --- a/core/libs/prx/libSceAgcDriver/State/src/Configuration.cpp +++ b/core/libs/prx/libSceAgcDriver/State/src/Configuration.cpp @@ -2,10 +2,23 @@ #include #include +#include #include #include "SceTypes.hpp" #include "prx/libc/include/General.hpp" +namespace { + +struct TessellationFactorRing { + uintptr_t base = 0xff0000000; + uint32_t size = 0x20000; +}; + +std::mutex tessellationFactorRingMutex; +TessellationFactorRing tessellationFactorRing; + +} + extern "C" { int APS5_VABI sceAgcDriverSetHsOffchipParam(uint64_t value0, uint64_t value1, uint64_t value2) { @@ -17,6 +30,16 @@ int APS5_VABI sceAgcDriverSetHsOffchipParam(uint64_t value0, uint64_t value1, ui int APS5_VABI sceAgcDriverSetTFRing(const volatile void* base, uint32_t size) { if (base == nullptr || size == 0) throw std::invalid_argument("sceAgcDriverSetTFRing: empty ring"); + std::lock_guard lock(tessellationFactorRingMutex); + tessellationFactorRing = {reinterpret_cast(base), size}; + return 0; +} + +int APS5_VABI sceAgcDriverGetTFRing(uintptr_t* base, uint32_t* size) { + if (base == nullptr || size == nullptr) throw std::invalid_argument("sceAgcDriverGetTFRing: null output"); + std::lock_guard lock(tessellationFactorRingMutex); + *base = tessellationFactorRing.base; + *size = tessellationFactorRing.size; return 0; } diff --git a/core/libs/prx/libSceAgcDriver/Unimplemented.cpp b/core/libs/prx/libSceAgcDriver/Unimplemented.cpp index 934422871..7b2d31ea6 100644 --- a/core/libs/prx/libSceAgcDriver/Unimplemented.cpp +++ b/core/libs/prx/libSceAgcDriver/Unimplemented.cpp @@ -10,11 +10,6 @@ int APS5_VABI sceAgcDriverGetShaderDebuggingStatus() { return 0; } -int APS5_VABI sceAgcDriverRegisterMultipleResources() { - NotImplemented_nid_no_patch(__func__); - return 0; -} - int APS5_VABI sceAgcDriverRegisterDefaultOwner(uint32_t* owner_handle) { (void)owner_handle; return static_cast(0x8A6C9018); diff --git a/core/libs/prx/libSceAgcDriver/tests/BdaExecution.cpp b/core/libs/prx/libSceAgcDriver/tests/BdaExecution.cpp index faf2f911c..7c4c2adc2 100644 --- a/core/libs/prx/libSceAgcDriver/tests/BdaExecution.cpp +++ b/core/libs/prx/libSceAgcDriver/tests/BdaExecution.cpp @@ -1,9 +1,12 @@ #include "BdaShader.hpp" #include "BdaAbi.hpp" #include "prx/libSceAgcDriver/Graphics/include/Resources.hpp" +#include #include #include #include +#include +#include namespace { @@ -87,6 +90,72 @@ private: } +static void RunBdaDwordWriteTests(const Context& context) { + namespace Abi = ShaderRecompiler::BdaAbi; + constexpr std::uint64_t guest = 0x7fff12340f80ULL; + constexpr std::size_t firstBytes = 160; + Buffer first(context, firstBytes, VK_BUFFER_USAGE_STORAGE_BUFFER_BIT | VK_BUFFER_USAGE_SHADER_DEVICE_ADDRESS_BIT); + Buffer second(context, 16, VK_BUFFER_USAGE_STORAGE_BUFFER_BIT | VK_BUFFER_USAGE_SHADER_DEVICE_ADDRESS_BIT); + Buffer readOnly(context, 16, VK_BUFFER_USAGE_STORAGE_BUFFER_BIT | VK_BUFFER_USAGE_SHADER_DEVICE_ADDRESS_BIT); + const std::array ranges{{ + {guest, guest + firstBytes, first.DeviceAddress(), Abi::Read | Abi::Write, 0}, + {guest + firstBytes, guest + firstBytes + 16, second.DeviceAddress(), Abi::Read | Abi::Write, 0}, + {guest + firstBytes + 16, guest + firstBytes + 32, readOnly.DeviceAddress(), Abi::Read, 0}, + }}; + Buffer table(context, sizeof(Abi::Header) + sizeof(ranges), VK_BUFFER_USAGE_STORAGE_BUFFER_BIT); + Buffer fault(context, Abi::FaultBufferBytes, VK_BUFFER_USAGE_STORAGE_BUFFER_BIT); + Buffer output(context, 4, VK_BUFFER_USAGE_STORAGE_BUFFER_BIT); + const std::array values{0x11223344u, 0x55667788u, 0x99aabbccu, 0xddeeff00u}; + const auto bytesAt = [&](std::uint64_t address) -> std::byte* { + for (std::size_t index = 0; index < ranges.size(); ++index) { + if (address < ranges[index].begin || address >= ranges[index].end) continue; + auto& buffer = index == 0 ? first : index == 1 ? second : readOnly; + return buffer.Bytes().data() + (address - ranges[index].begin); + } + return nullptr; + }; + const auto noted = [&](std::uint64_t address) { + const auto page = static_cast(address >> Abi::WrittenPageShift) + 1u; + std::vector slots(Abi::WrittenPageSlots); + std::memcpy(slots.data(), fault.Bytes().data() + Abi::WrittenSlotsWord * sizeof(std::uint32_t), slots.size() * sizeof(std::uint32_t)); + return std::find(slots.begin(), slots.end(), page) != slots.end(); + }; + const auto write = [&](std::uint64_t address, std::uint32_t dwords, std::uint32_t storedBytes, Abi::FaultReason reason) { + const Abi::Header header{Abi::Version, static_cast(ranges.size()), sizeof(Abi::Range), 0}; + std::memcpy(table.Bytes().data(), &header, sizeof(header)); + std::memcpy(table.Bytes().data() + sizeof(header), ranges.data(), sizeof(ranges)); + std::memset(fault.Bytes().data(), 0, fault.Bytes().size()); + for (auto* buffer : {&first, &second, &readOnly}) std::memset(buffer->Bytes().data(), 0xa5, buffer->Bytes().size()); + Pipeline pipeline(context, MakeBdaDwordWriteTestShader(address, dwords, values.data()), {&table, &fault, &output}); + pipeline.Run(1); + Abi::Fault report{}; + std::memcpy(&report, fault.Bytes().data(), sizeof(report)); + for (std::uint32_t byte = 0; byte < dwords * 4u; ++byte) { + const auto* stored = bytesAt(address + byte); + const auto expected = byte < storedBytes ? static_cast(values[byte / 4u] >> ((byte % 4u) * 8u)) : std::byte{0xa5}; + Require(stored != nullptr && *stored == expected, "BDA dword write stored incorrect bytes at offset " + std::to_string(byte) + " of guest + " + std::to_string(address - guest)); + } + Require(*bytesAt(address - 1) == std::byte{0xa5}, "BDA dword write touched the byte before it"); + if (address + dwords * 4u < guest + firstBytes + 32) Require(*bytesAt(address + dwords * 4u) == std::byte{0xa5}, "BDA dword write touched the byte after it"); + if (static_cast(reason) == 0) { + Require(report.state == Abi::FaultState::Empty, "a mapped BDA dword write published a fault"); + } else { + Require(report.state == Abi::FaultState::Ready && report.reason == reason && report.instruction == 0x1234, "a BDA dword write did not publish its fault"); + } + if (storedBytes != 0) Require(noted(address) && noted(address + storedBytes - 1u), "a BDA dword write did not note its written pages"); + }; + write(guest + 16, 4, 16, static_cast(0)); + write(guest + 20, 4, 16, static_cast(0)); + write(guest + 8, 2, 8, static_cast(0)); + write(guest + 12, 2, 8, static_cast(0)); + write(guest + 32, 3, 12, static_cast(0)); + write(guest + 0x78, 4, 16, static_cast(0)); + write(guest + 0x7c, 2, 8, static_cast(0)); + write(guest + 41, 4, 16, static_cast(0)); + write(guest + firstBytes - 8, 4, 16, static_cast(0)); + write(guest + firstBytes + 12, 4, 4, Abi::FaultReason::Permission); +} + void RunBdaExecutionTests(const Context& context) { namespace Abi = ShaderRecompiler::BdaAbi; Buffer first(context, 3, VK_BUFFER_USAGE_STORAGE_BUFFER_BIT | VK_BUFFER_USAGE_SHADER_DEVICE_ADDRESS_BIT); @@ -163,6 +232,48 @@ void RunBdaExecutionTests(const Context& context) { read(guest + 1, 4, coherent, stops, 0x55443322, true); } } + Buffer wide(context, 32, VK_BUFFER_USAGE_STORAGE_BUFFER_BIT | VK_BUFFER_USAGE_SHADER_DEVICE_ADDRESS_BIT); + for (std::uint32_t dword = 0; dword < 8u; ++dword) { + const std::uint32_t value = 0xa0b0c000u + dword; + std::memcpy(wide.Bytes().data() + dword * 4u, &value, sizeof(value)); + } + constexpr std::uint64_t spanGuest = 0x7fff56780000ULL; + const std::array spanRanges{{ranges[0], ranges[1], {spanGuest, spanGuest + 32, wide.DeviceAddress(), Abi::Read, 0}}}; + Buffer spanTable(context, sizeof(Abi::Header) + sizeof(spanRanges), VK_BUFFER_USAGE_STORAGE_BUFFER_BIT); + const auto span = [&](std::uint64_t address, std::int32_t offset, std::uint32_t extracted, bool coherent, bool stops, std::array values, Abi::FaultReason reason = {}, std::uint64_t faultAddress = 0, std::uint32_t faultBytes = 0) { + const Abi::Header header{Abi::Version, 3, sizeof(Abi::Range), 0}; + std::memcpy(spanTable.Bytes().data(), &header, sizeof(header)); + std::memcpy(spanTable.Bytes().data() + sizeof(header), spanRanges.data(), sizeof(spanRanges)); + std::memset(fault.Bytes().data(), 0, fault.Bytes().size()); + const std::array sentinel{0xdeadbeef, 0xdeadbeef, 0xdeadbeef, 0xdeadbeef}; + std::memcpy(words.Bytes().data(), sentinel.data(), sizeof(sentinel)); + Pipeline pipeline(context, MakeBdaSpanReadTestShader(address, static_cast(offset), extracted, coherent, stops), {&spanTable, &fault, &words}); + pipeline.Run(1); + Abi::Fault report{}; + std::array result{}; + std::memcpy(&report, fault.Bytes().data(), sizeof(report)); + std::memcpy(result.data(), words.Bytes().data(), sizeof(result)); + const bool faults = static_cast(reason) != 0u; + if (faults) { + Require(report.state == Abi::FaultState::Ready && report.reason == reason && report.address == faultAddress && report.bytes == faultBytes && report.instruction == 0x1234, "BDA span read did not publish its first fault"); + } else { + Require(report.state == Abi::FaultState::Empty, "mapped BDA span read published a fault"); + } + Require(result == (faults && stops ? sentinel : values), faults && stops ? "faulting BDA span read continued to output a substitute value" : "BDA span read produced incorrect data"); + }; + const auto at = [&](std::uint32_t dword) { return 0xa0b0c000u + dword; }; + for (const bool stops : {true, false}) { + for (const bool coherent : {false, true}) { + span(spanGuest + 16, -12, 0b1010u, coherent, stops, {0, at(2), 0, at(4)}); + span(spanGuest, 4, 0b0101u, coherent, stops, {at(1), 0, at(3), 0}); + span(spanGuest + 24, 0, 0b0011u, coherent, stops, {at(6), at(7), 0, 0}); + span(spanGuest + 20, 8, 0b0001u, coherent, stops, {at(7), 0, 0, 0}); + span(spanGuest + 28, 0, 0b0101u, coherent, stops, {at(7), 0, 0, 0}, Abi::FaultReason::Unmapped, spanGuest + 36, 1); + span(guest, 1, 0b1001u, coherent, stops, {0x55443322, 0, 0, 0}, Abi::FaultReason::Unmapped, guest + 13, 1); + span(4, -8, 0b0001u, coherent, stops, {0, 0, 0, 0}, Abi::FaultReason::Overflow, 4, 0); + } + } ranges[0].permissions = 0; run(guest, 8, 0, Abi::FaultReason::Permission); + RunBdaDwordWriteTests(context); } diff --git a/core/libs/prx/libSceAgcDriver/tests/BdaResources.cpp b/core/libs/prx/libSceAgcDriver/tests/BdaResources.cpp index f803903fc..fdf13a5c5 100644 --- a/core/libs/prx/libSceAgcDriver/tests/BdaResources.cpp +++ b/core/libs/prx/libSceAgcDriver/tests/BdaResources.cpp @@ -3,6 +3,8 @@ #include "prx/libc/include/GuestAllocations.hpp" #include "prx/libc/include/GuestHeap.hpp" #include "prx/libc/include/GuestArena.hpp" +#include "prx/libSceAgcDriver/Execution/include/GuestMemory.hpp" +#include "prx/libc/include/GuestWriteWatch.hpp" #ifdef _WIN32 #ifndef NOMINMAX #define NOMINMAX @@ -11,12 +13,18 @@ #endif #include #include +#include #include #include +#include #include #include +#include #include +#ifndef _WIN32 +#include +#endif namespace { using namespace AgcDriver::Graphics; @@ -32,10 +40,230 @@ void reject(TAction action, const char* reason) { throw std::runtime_error(std::string("expected BDA rejection: ") + reason); } +void importCrossingTests(const Context& context, const BdaTestAccess& access) { + constexpr std::size_t half = 65536; + void* block = ::operator new(2 * half, std::align_val_t{half}); + auto* guest = static_cast(block); + std::memset(guest, 0x11, half); + std::memset(guest + half, 0x22, half); + const auto first = reinterpret_cast(block); + const auto second = first + half; + auto importing = context; + importing.hostImportAlignment = half; + const auto registry = [&](bool add) { + auto* mutation = GuestAllocations::GuestAllocationsBegin_nid_postfix(); + for (auto* range : {guest, guest + half}) { + if (add) GuestAllocations::GuestAllocationsAdd_nid_postfix(mutation, range, half, true, false, true); + else GuestAllocations::GuestAllocationsRemove_nid_postfix(mutation, range); + } + GuestAllocations::GuestAllocationsEnd_nid_postfix(mutation); + }; + registry(true); + { + GuestBufferMemory leased(importing); + leased.AcquireRegistered(); + Require(HostImportCovers(importing, first, half) && HostImportCovers(importing, second, half), "the registered ranges were not imported"); + const auto secondImport = *HostImportFor(importing, second, half); + leased.AddReadable(second - 16, 32); + leased.AddReadable(second + 64, 16); + leased.Upload(true); + std::uint32_t adjustment = 0; + const auto crossing = leased.Descriptor(second - 16, 32, adjustment); + const auto crossingBytes = access.bytes(crossing.buffer); + Require(crossing.offset + crossing.range <= crossingBytes.size(), "a view crossing into the next registered range is bound past its host import"); + Require(crossingBytes[crossing.offset + adjustment + 15] == std::byte{0x11} && crossingBytes[crossing.offset + adjustment + 16] == std::byte{0x22}, "a view crossing into the next registered range misses its bytes"); + const auto inside = leased.Descriptor(second + 64, 16, adjustment); + const auto insideBytes = access.bytes(inside.buffer); + Require(inside.offset + inside.range <= insideBytes.size() && (inside.buffer == secondImport.buffer ? inside.offset + adjustment == 64 : insideBytes[inside.offset + adjustment] == std::byte{0x22}), "a view in the second registered range is bound past its buffer or misses its bytes"); + for (const auto& range : leased.AddressRanges()) { + if (range.end <= first || range.begin >= second + half) continue; + const auto mapped = access.addressBytes(range.deviceAddress); + Require(mapped.size() >= range.end - range.begin, "a BDA range runs past the end of its host import"); + if (range.begin <= second && second < range.end && range.deviceAddress != secondImport.address) Require(mapped[second - range.begin] == std::byte{0x22}, "the BDA range over the second registered range misses its bytes"); + } + leased.WriteBack(); + } + registry(false); + Require(HostImportFor(importing, first, half) == nullptr && !HostImportCovers(importing, second, half), "the host imports outlived their ranges"); + ::operator delete(block, std::align_val_t{half}); +} + ShaderRecompiler::DescriptorBinding binding(Role role, std::uint32_t slot) { return {ShaderRecompiler::DescriptorKind::StorageBuffer, role, 0, slot, 1, {}, false}; } +void importedHeapMirrorTests(const Context& context, const BdaTestAccess& access) { + if (!GuestArena::GuestArenaAvailable_nid_postfix() || !GuestArena::GuestArenaWriteWatched_nid_postfix()) { + std::cout << "guest arena unavailable or not write-watched: heap mirrors of imported ranges not tested\n"; + return; + } + constexpr std::size_t bytes = 1u << 20u; + void* block = GuestArena::GuestArenaAllocate_nid_postfix(bytes, bytes); +#ifdef _WIN32 + GuestArena::GuestArenaCommit_nid_postfix(block, bytes, PAGE_READWRITE, bytes); +#endif + std::memset(block, 0x11, bytes); + const auto address = reinterpret_cast(block); + auto importing = context; + importing.hostImportAlignment = bytes; + const auto registry = [&](bool add) { + auto* mutation = GuestAllocations::GuestAllocationsBegin_nid_postfix(); + if (add) GuestAllocations::GuestAllocationsAdd_nid_postfix(mutation, block, bytes, true, false, true); + else GuestAllocations::GuestAllocationsRemove_nid_postfix(mutation, block); + GuestAllocations::GuestAllocationsEnd_nid_postfix(mutation); + }; + const auto build = [&](const Context& with) { + GuestAllocations::GuestAllocationsEnd_nid_postfix(GuestAllocations::GuestAllocationsBegin_nid_postfix()); + GuestBufferMemory leased(with); + leased.AcquireRegistered(); + leased.Upload(true); + const auto ranges = leased.AddressRanges(); + const auto found = std::find_if(ranges.begin(), ranges.end(), [&](const auto& range) { return range.begin == address && range.end == address + bytes; }); + const auto device = found != ranges.end() ? found->deviceAddress : 0; + leased.WriteBack(); + return device; + }; + const auto before = MirrorCounters(); + registry(true); + Require(build(context) != 0, "the heap range is missing from the BDA table of a build without host imports"); + const auto mirrored = MirrorCounters(); + Require(mirrored.heapMirrors == before.heapMirrors + 1 && mirrored.heapBytes == before.heapBytes + bytes, "a range built without host imports was not heap mirrored"); + Require(build(importing) != 0 && HostImportCovers(importing, address, bytes), "the heap mirrored range was not imported once imports were available"); + const auto imported = MirrorCounters(); + Require(imported.heapMirrors == before.heapMirrors && imported.heapBytes == before.heapBytes, "a heap mirror outlived the host import that serves its range"); + Require(build(importing) != 0 && MirrorCounters().heapMirrors == before.heapMirrors && MirrorCounters().rebuilds == imported.rebuilds, "an imported range was heap mirrored again"); + static_cast(block)[bytes / 2] = 0x22; + const auto device = build(context); + const auto again = MirrorCounters(); + Require(device != 0 && again.heapMirrors == before.heapMirrors + 1 && again.heapBytes == before.heapBytes + bytes && again.rebuilds == imported.rebuilds + 1, "a range no import serves was not heap mirrored again"); + Require(access.addressBytes(device)[bytes / 2] == std::byte{0x22} && access.addressBytes(device)[bytes / 2 + 1] == std::byte{0x11}, "a heap mirror made after its import missed the guest bytes"); + registry(false); + Require(HostImportFor(importing, address, bytes) == nullptr && !HostImportCovers(importing, address, bytes), "the import outlived its range"); + Require(build(context) == 0, "the unregistered heap range is still in the BDA table"); + Require(MirrorCounters().heapMirrors == before.heapMirrors && MirrorCounters().heapBytes == before.heapBytes, "the heap mirror outlived its range"); +#ifdef _WIN32 + GuestArena::GuestArenaReset_nid_postfix(block, bytes); +#endif + GuestArena::GuestArenaRelease_nid_postfix(block, bytes); +} + +void importedFreshTests(const Context& context) { +#ifndef _WIN32 + if (!GuestWriteWatch::GuestWriteWatchAvailable_nid_postfix()) { + std::cout << "write watch unavailable: imported never-collected pages not tested\n"; + return; + } + constexpr std::size_t blockBytes = 65536; + void* area = mmap(nullptr, 2 * blockBytes, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + Require(area != MAP_FAILED, "cannot map the imported block"); + const auto base = (reinterpret_cast(area) + blockBytes - 1) & ~(blockBytes - 1); + std::memset(reinterpret_cast(base), 0x55, blockBytes); + GuestWriteWatch::GuestWriteWatchRegister_nid_postfix(reinterpret_cast(base), blockBytes); + auto importing = context; + importing.hostImportAlignment = blockBytes; + const auto registry = [&](bool add) { + auto* mutation = GuestAllocations::GuestAllocationsBegin_nid_postfix(); + if (add) GuestAllocations::GuestAllocationsAdd_nid_postfix(mutation, reinterpret_cast(base), blockBytes, true, true, true); + else GuestAllocations::GuestAllocationsRemove_nid_postfix(mutation, reinterpret_cast(base)); + GuestAllocations::GuestAllocationsEnd_nid_postfix(mutation); + }; + registry(true); + if (HostImportFor(importing, base, blockBytes) == nullptr) { + std::cout << "the never-collected block was not imported: imported never-collected pages not tested\n"; + } else { + const auto gpuWrite = AgcDriver::GuestMemory::MarkWritten(base, 4); + Require(gpuWrite != 0, "a GPU write into an imported never-collected block was not stamped"); + reinterpret_cast(base)[5 * 4096 + 8] = 0x66; + static_cast(AgcDriver::GuestMemory::CollectWritesUncached(base, blockBytes)); + Require(AgcDriver::GuestMemory::WrittenSince(base + 5 * 4096, 4096, gpuWrite), "a CPU write to an imported page protected after the import was not seen"); + } + registry(false); + Require(HostImportFor(importing, base, blockBytes) == nullptr, "the import outlived its range"); + GuestWriteWatch::GuestWriteWatchUnregister_nid_postfix(reinterpret_cast(base), blockBytes); + munmap(area, 2 * blockBytes); +#else + static_cast(context); +#endif +} + +void importWatchTests() { +#ifndef _WIN32 + if (!GuestWriteWatch::GuestWriteWatchAvailable_nid_postfix()) { + std::cout << "write watch unavailable: import watch not tested\n"; + return; + } + constexpr std::size_t bytes = 4 * 65536; + void* block = mmap(nullptr, bytes, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + Require(block != MAP_FAILED, "cannot map the import watch block"); + auto* guest = static_cast(block); + const auto address = reinterpret_cast(block); + GuestWriteWatch::GuestWriteWatchRegister_nid_postfix(block, bytes); + std::memset(block, 0x22, bytes); + const auto written = [&] { + std::size_t reported = 0; + Require(GuestWriteWatch::GuestWriteWatchCollect_nid_postfix(address, bytes, [](void* context, std::uintptr_t begin, std::uintptr_t end) { *static_cast(context) += end - begin; }, &reported), "the write watch did not collect the block"); + return reported; + }; + Require(AgcDriver::GuestMemory::ImportWatched(address, bytes, [] { return true; }), "importing a block nothing had collected failed"); + Require(written() == bytes, "an import write-protected pages nothing had collected"); + guest[3 * 4096] = 0x33; + Require(AgcDriver::GuestMemory::ImportWatched(address, bytes, [] { return true; }), "importing a collected block failed"); + Require(written() == 0, "an import did not take the writes to pages a collect had protected"); + GuestWriteWatch::GuestWriteWatchUnregister_nid_postfix(block, bytes); + munmap(block, bytes); + constexpr std::size_t blockBytes = 65536; + void* area = mmap(nullptr, bytes + 2 * blockBytes, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + Require(area != MAP_FAILED, "cannot map the GPU write block"); + const auto aligned = (reinterpret_cast(area) + blockBytes - 1) & ~(blockBytes - 1); + const auto imported = aligned + 4096; + GuestWriteWatch::GuestWriteWatchRegister_nid_postfix(reinterpret_cast(imported), bytes); + Require(AgcDriver::GuestMemory::ImportWatched(imported, bytes, [] { return true; }), "importing a fresh range failed"); + const auto gpuWrite = AgcDriver::GuestMemory::MarkWritten(imported + 4096, 4096); + Require(gpuWrite != 0, "a GPU write into the imported range was not stamped"); + static_cast(AgcDriver::GuestMemory::CollectWritesUncached(imported, bytes)); + Require(!AgcDriver::GuestMemory::WrittenSince(imported + 4096, 4096, gpuWrite), "a collect after a GPU write into an uncollected import read as a CPU write over the GPU's results"); + reinterpret_cast(imported)[4096 + 100] = 0x44; + static_cast(AgcDriver::GuestMemory::CollectWritesUncached(imported, bytes)); + Require(AgcDriver::GuestMemory::WrittenSince(imported + 4096, 4096, gpuWrite), "a CPU write after a GPU write into an import was not seen"); + GuestWriteWatch::GuestWriteWatchUnregister_nid_postfix(reinterpret_cast(imported), bytes); + munmap(area, bytes + 2 * blockBytes); + + void* labels = mmap(nullptr, 2 * blockBytes, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + Require(labels != MAP_FAILED, "cannot map the label block"); + const auto label = (reinterpret_cast(labels) + blockBytes - 1) & ~(blockBytes - 1); + auto* words = reinterpret_cast(label); + GuestWriteWatch::GuestWriteWatchRegister_nid_postfix(reinterpret_cast(label), blockBytes); + words[0] = 1; + static_cast(AgcDriver::GuestMemory::CollectWritesUncached(label, 4)); + const auto before = AgcDriver::GuestMemory::TrackerGeneration(); + Require(AgcDriver::GuestMemory::MarkWritten(label, 4) != 0, "a GPU write next to never-collected pages was not stamped"); + Require(AgcDriver::GuestMemory::UnchangedSinceCollected(label, 4, before), "a GPU write next to never-collected pages read as a CPU write of its block"); + Require(AgcDriver::GuestMemory::StoredOver(label + 8 * 4096, 4096, before), "a never-collected page a GPU write collected is not reported as possibly stored"); + const std::array since{before}; + std::array changed{}; + std::array cpu{}; + Require(AgcDriver::GuestMemory::ChangedBlocks(label, blockBytes, since, changed, cpu) && changed[0] == AgcDriver::GuestMemory::BlockWritten && cpu[0] == 0, "collecting never-collected pages at a GPU write stamped their block as written by the CPU"); + const auto now = AgcDriver::GuestMemory::TrackerGeneration(); + Require(AgcDriver::GuestMemory::MarkWritten(label, 4) == now + 1, "a GPU write into a block with nothing left to collect advanced the generation more than once"); + words[1] = 2; + Require(!AgcDriver::GuestMemory::UnchangedSinceCollected(label, 4, before), "a CPU write next to a GPU write was not seen"); + void* awaitedArea = mmap(nullptr, 2 * blockBytes, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + Require(awaitedArea != MAP_FAILED, "cannot map the awaited block"); + const auto awaitedBlock = (reinterpret_cast(awaitedArea) + blockBytes - 1) & ~(blockBytes - 1); + const auto awaited = awaitedBlock + 5 * 4096; + GuestWriteWatch::GuestWriteWatchRegister_nid_postfix(reinterpret_cast(awaitedBlock), blockBytes); + static_cast(AgcDriver::GuestMemory::CollectWritesUncached(awaitedBlock, 4)); + const auto closed = AgcDriver::GuestMemory::TrackerGeneration(); + *reinterpret_cast(awaited) = 7; + Require(AgcDriver::GuestMemory::MarkWritten(awaitedBlock, 4) != 0, "a GPU write into the awaited block was not stamped"); + Require(!AgcDriver::GuestMemory::UnchangedSinceCollected(awaited, 4, closed), "a CPU store to a never-collected page that a GPU write elsewhere in its block collected was not seen"); + GuestWriteWatch::GuestWriteWatchUnregister_nid_postfix(reinterpret_cast(awaitedBlock), blockBytes); + munmap(awaitedArea, 2 * blockBytes); + GuestWriteWatch::GuestWriteWatchUnregister_nid_postfix(reinterpret_cast(label), blockBytes); + munmap(labels, 2 * blockBytes); +#endif +} + void heapMirrorTests(const Context& context, const BdaTestAccess& access) { if (!GuestArena::GuestArenaAvailable_nid_postfix() || !GuestArena::GuestArenaWriteWatched_nid_postfix()) { std::cout << "guest arena unavailable or not write-watched: heap mirrors not tested\n"; @@ -51,7 +279,7 @@ void heapMirrorTests(const Context& context, const BdaTestAccess& access) { const auto address = reinterpret_cast(block); const auto registry = [&](bool add, bool writable) { auto* mutation = GuestAllocations::GuestAllocationsBegin_nid_postfix(); - if (add) GuestAllocations::GuestAllocationsAdd_nid_postfix(mutation, block, bytes, true, writable); + if (add) GuestAllocations::GuestAllocationsAdd_nid_postfix(mutation, block, bytes, true, writable, true); else GuestAllocations::GuestAllocationsRemove_nid_postfix(mutation, block); GuestAllocations::GuestAllocationsEnd_nid_postfix(mutation); }; @@ -102,6 +330,10 @@ void heapMirrorTests(const Context& context, const BdaTestAccess& access) { guest[65536 + 7] = 0x44; Require(build(0, {}) == first && MirrorCounters().blocksCopied == one.blocksCopied + 2, "two written heap blocks were not read again"); Require(access.addressBytes(first)[5] == std::byte{0x33} && access.addressBytes(first)[65536 + 7] == std::byte{0x44}, "the heap mirror missed the CPU writes"); + const auto quiet = MirrorCounters(); + Require(build(0, {}) == first && MirrorCounters().heapChecks == quiet.heapChecks && MirrorCounters().sweeps == quiet.sweeps && MirrorCounters().blocksCopied == quiet.blocksCopied, "an unchanged heap range was compared block by block or the mirrors were swept again"); + guest[65536 + 9] = 0x66; + Require(build(0, {}) == first && MirrorCounters().heapChecks > quiet.heapChecks && MirrorCounters().blocksCopied == quiet.blocksCopied + 1 && access.addressBytes(first)[65536 + 9] == std::byte{0x66}, "a CPU write after an unchanged build was missed"); registry(false, false); sweep(); const auto swept = MirrorCounters(); @@ -116,6 +348,13 @@ void heapMirrorTests(const Context& context, const BdaTestAccess& access) { }); Require(guest[16] == 0x77 && guest[16 + 8] == 0x55, "a writable heap mirror's write-back lost the GPU's store or rolled back the CPU's"); Require(build(0, {}) == device && access.addressBytes(device)[16] == std::byte{0x77} && access.addressBytes(device)[16 + 8] == std::byte{0x55}, "the writable heap mirror missed the stores"); + Require(build(address + 40, [&](GuestBufferMemory& leased) { + std::uint32_t adjustment = 0; + const auto view = leased.Descriptor(address + 40, 8, adjustment); + access.bytes(view.buffer)[view.offset + adjustment] = std::byte{0x78}; + }) == device && guest[40] == 0x78, "a writable heap mirror's second write-back lost the GPU's store"); + guest[65536 + 1] = 0x79; + Require(build(0, {}) == device && access.addressBytes(device)[40] == std::byte{0x78} && access.addressBytes(device)[65536 + 1] == std::byte{0x79}, "the writable heap mirror missed a store after a write-back"); registry(false, true); sweep(); Require(MirrorCounters().heapMirrors == before.heapMirrors, "the writable heap mirror outlived its range"); @@ -131,7 +370,7 @@ void heapMirrorTests(const Context& context, const BdaTestAccess& access) { const auto registerPair = [&](bool add) { auto* mutation = GuestAllocations::GuestAllocationsBegin_nid_postfix(); for (auto* range : {first, second}) { - if (add) GuestAllocations::GuestAllocationsAdd_nid_postfix(mutation, range, half, true, false); + if (add) GuestAllocations::GuestAllocationsAdd_nid_postfix(mutation, range, half, true, false, true); else GuestAllocations::GuestAllocationsRemove_nid_postfix(mutation, range); } GuestAllocations::GuestAllocationsEnd_nid_postfix(mutation); @@ -184,7 +423,7 @@ void heapMirrorTests(const Context& context, const BdaTestAccess& access) { GuestAllocations::GuestAllocationsInvalidate_nid_postfix(page, 4096); const auto registerRange = [&](bool add) { auto* mutation = GuestAllocations::GuestAllocationsBegin_nid_postfix(); - if (add) GuestAllocations::GuestAllocationsAdd_nid_postfix(mutation, raw, size, true, true); + if (add) GuestAllocations::GuestAllocationsAdd_nid_postfix(mutation, raw, size, true, true, true); else GuestAllocations::GuestAllocationsRemove_nid_postfix(mutation, raw); GuestAllocations::GuestAllocationsEnd_nid_postfix(mutation); }; @@ -229,6 +468,71 @@ void heapMirrorTests(const Context& context, const BdaTestAccess& access) { GuestArena::GuestArenaRelease_nid_postfix(raw, size); } #endif + { + constexpr std::size_t large = 36 * 65536; + constexpr std::size_t small = 17 * 65536; + constexpr std::size_t total = 2 * large + small; + void* raw = GuestArena::GuestArenaAllocate_nid_postfix(total, 65536); +#ifdef _WIN32 + GuestArena::GuestArenaCommit_nid_postfix(raw, total, PAGE_READWRITE, total); +#endif + auto* const heap = static_cast(raw); + const std::array heaps{heap, heap + large, heap + 2 * large}; + const std::array sizes{large, large, small}; + for (std::size_t index = 0; index < heaps.size(); ++index) std::memset(heaps[index], 0x21 + static_cast(index), sizes[index]); + const auto change = [&](std::initializer_list added, std::initializer_list removed) { + auto* mutation = GuestAllocations::GuestAllocationsBegin_nid_postfix(); + for (const auto index : removed) GuestAllocations::GuestAllocationsRemove_nid_postfix(mutation, heaps[index]); + for (const auto index : added) GuestAllocations::GuestAllocationsAdd_nid_postfix(mutation, heaps[index], sizes[index], true, false, true); + GuestAllocations::GuestAllocationsEnd_nid_postfix(mutation); + }; + const auto mirrored = [&](std::size_t index) { + GuestBufferMemory leased(context); + leased.AcquireRegistered(); + leased.Upload(true); + const auto ranges = leased.AddressRanges(); + const auto begin = reinterpret_cast(heaps[index]); + const auto found = std::find_if(ranges.begin(), ranges.end(), [&](const auto& entry) { return entry.begin == begin && entry.end == begin + sizes[index]; }); + Require(found != ranges.end(), "a mirrored heap range is missing from the BDA table"); + const auto last = access.addressBytes(found->deviceAddress)[sizes[index] - 1]; + leased.WriteBack(); + return last; + }; + sweep(); + const auto before = MirrorCounters(); + change({0}, {}); + Require(mirrored(0) == std::byte{0x21}, "the first heap range was not mirrored"); + const auto one = MirrorCounters(); + access.limitMemory(large - 1); + change({1}, {0}); + Require(mirrored(1) == std::byte{0x22}, "a heap mirror past APS5_HEAP_MIRROR_MIB was not made in the memory of the expired mirror"); + const auto swept = MirrorCounters(); + Require(swept.heapMirrors == one.heapMirrors && swept.heapBytes == one.heapBytes, "the expired heap mirror was not swept before the allocation"); + access.limitMemory(small - 1); + change({2}, {}); + char expected[64]; + std::snprintf(expected, sizeof(expected), "heap mirror of 0x%llx+0x%llx: ", static_cast(reinterpret_cast(heaps[2])), static_cast(small)); + const auto attempts = access.allocationAttempts(); + bool refused = false; + try { + mirrored(2); + } catch (const std::runtime_error& error) { + const std::string what = error.what(); + refused = what.find(expected) != std::string::npos && what.find("Vulkan result -2") != std::string::npos; + } + Require(refused, "a heap mirror the memory cannot hold did not fail its build with the Vulkan result"); + Require(access.allocationAttempts() == attempts + 1, "a refused heap mirror was allocated again"); + Require(MirrorCounters().heapMirrors == swept.heapMirrors && MirrorCounters().heapBytes == swept.heapBytes, "a refused heap mirror was registered"); + access.limitMemory(std::nullopt); + Require(mirrored(2) == std::byte{0x23} && MirrorCounters().heapMirrors == swept.heapMirrors + 1, "a heap mirror was not made once the memory was free again"); + change({}, {1, 2}); + sweep(); + Require(MirrorCounters().heapMirrors == before.heapMirrors && MirrorCounters().heapBytes == before.heapBytes, "the heap mirrors outlived their ranges"); +#ifdef _WIN32 + GuestArena::GuestArenaReset_nid_postfix(raw, total); +#endif + GuestArena::GuestArenaRelease_nid_postfix(raw, total); + } #ifdef _WIN32 GuestArena::GuestArenaReset_nid_postfix(block, bytes); #endif @@ -237,6 +541,34 @@ void heapMirrorTests(const Context& context, const BdaTestAccess& access) { } +void gpuMappingTests(const Context& context) { + constexpr std::size_t bytes = 65536; + auto* cpu = static_cast(::operator new(bytes, std::align_val_t{bytes})); + auto* gpu = static_cast(::operator new(bytes, std::align_val_t{bytes})); + { + GuestAllocations::Mutation mutation; + mutation.Add(cpu, bytes, true, true, false); + mutation.Add(gpu, bytes, true, true, true); + } + { + GuestBufferMemory leased(context); + leased.AcquireRegistered(); + leased.Upload(true); + const auto ranges = leased.AddressRanges(); + const auto has = [&](const std::byte* block) { return std::any_of(ranges.begin(), ranges.end(), [&](const auto& range) { return range.begin == reinterpret_cast(block); }); }; + Require(has(gpu), "a GPU-mapped range is missing from the BDA table"); + Require(!has(cpu), "a range mapped without GPU access is in the BDA table"); + leased.WriteBack(); + } + { + GuestAllocations::Mutation mutation; + mutation.Remove(cpu); + mutation.Remove(gpu); + } + ::operator delete(cpu, std::align_val_t{bytes}); + ::operator delete(gpu, std::align_val_t{bytes}); +} + void RunBdaResourceTests(const Context& context, const BdaTestAccess& access) { alignas(64) std::array guest{}; guest[0] = 123; @@ -282,7 +614,8 @@ void RunBdaResourceTests(const Context& context, const BdaTestAccess& access) { control.bindings.push_back(writable); const std::array stages{{{ShaderRecompiler::ShaderStage::TessellationControl, &control, 0}}}; const std::array unusedSnapshots{{{0, source}}}; - ShaderResources resources(disabled, stages, ColorTarget{}, 0, 0, unusedSnapshots); + ShaderResources resources(disabled, stages, ColorTarget{}, 0, 0, 0, unusedSnapshots); + Require(ShaderResources::NeverReusable(stages) && !resources.Reusable(), "a fault-buffer build is reusable"); const auto fault = access.bytes(access.descriptor(5).buffer); for (const auto byte : fault) Require(byte == std::byte{}, "rect-list fault buffer was not initialized"); const ShaderRecompiler::BdaAbi::Fault report{ShaderRecompiler::BdaAbi::FaultState::Ready, ShaderRecompiler::BdaAbi::FaultReason::InvalidRectangle, 0, 0, 0, 0, 0}; @@ -296,6 +629,11 @@ void RunBdaResourceTests(const Context& context, const BdaTestAccess& access) { } { ShaderResources resources(context, compiled, snapshots); + Require(ShaderResources::NeverReusable(std::span(&compiled, 1)) && !resources.Reusable(), "an address-based build is reusable"); + ShaderRecompiler::RecompileResult plain; + plain.bindings = {binding(Role::GuestBuffers, 6)}; + const CompiledShader plainStage{ShaderRecompiler::ShaderStage::Compute, &plain, 0}; + Require(!ShaderResources::NeverReusable(std::span(&plainStage, 1)), "a build without address tables or a fault buffer is refused the resource cache"); const auto table = access.bytes(access.descriptor(4).buffer); ShaderRecompiler::BdaAbi::Header header{}; ShaderRecompiler::BdaAbi::Range range{}; @@ -385,7 +723,12 @@ void RunBdaResourceTests(const Context& context, const BdaTestAccess& access) { const auto dropped = AddressSpaceCounters(); if (dropped.enabled) Require(dropped.waiterDrops == after.waiterDrops + 1 && LeaseCounters().cacheDrops == dropped.waiterDrops, "the free did not drop the cached address space"); } + importCrossingTests(context, access); heapMirrorTests(context, access); + importedHeapMirrorTests(context, access); + importWatchTests(); + importedFreshTests(context); + gpuMappingTests(context); Require(AddressCopyOverflow({{0x1000, 0x3000, 0x2000, "uncommitted pages"}}, 0x2000).empty(), "copies within the limit were refused"); const auto copies = AddressCopyOverflow({{0x1000, 0x2000, 0x1000, "not mirrored"}, {0x10000, 0x30000, 0x18000, "uncommitted pages"}}, 0x2000); Require(!copies.empty() && copies.find("0x10000+0x20000 (0.1 MiB committed, uncommitted pages)") < copies.find("0x1000+0x1000"), "the copy limit does not name the largest copy first"); diff --git a/core/libs/prx/libSceAgcDriver/tests/BdaShader.cpp b/core/libs/prx/libSceAgcDriver/tests/BdaShader.cpp index 9c4dd099d..efc43683d 100644 --- a/core/libs/prx/libSceAgcDriver/tests/BdaShader.cpp +++ b/core/libs/prx/libSceAgcDriver/tests/BdaShader.cpp @@ -2,15 +2,17 @@ #include "SpirvBackend/SpirvBda.hpp" #include "SpirvBackend/SpirvMemory/SpirvTypes.hpp" #include "SpirvBackend/SpirvMemory/SpirvConstants.hpp" +#include "IntermediateRepresentation/IrBuilder.hpp" namespace { template -std::vector MakeShader(std::uint64_t address, bool coherent, bool stops, const TEmit& emit) { +std::vector MakeShader(std::uint64_t address, bool coherent, bool stops, const TEmit& emit, std::uint32_t extracted = 0, bool writes = false) { using namespace ShaderRecompiler; IrProgram program; program.Resources().stage = IrShaderStage::Compute; program.Info().usesDma = true; + program.Info().bdaWrites = writes; if (coherent) { MemoryInfo memory; memory.coherent = true; @@ -38,7 +40,14 @@ std::vector MakeShader(std::uint64_t address, bool coherent, bool state.module.AddFunction(spv::OpFunction, TypeVoid(state), main, spv::FunctionControlMaskNone, TypeFunction(state)); EmitLabel(state, state.module.AllocateId()); SpirvValueEmitContext ctx(state); - auto& instruction = program.CreateValue(IrOpcode::LoadAddressU32, IrType::U32); + auto& instruction = extracted != 0u ? program.CreateValue(IrOpcode::LoadAddressU32x4, IrType::U32x4) : program.CreateValue(IrOpcode::LoadAddressU32, IrType::U32); + IrBuilder ir(program); + for (std::uint32_t dword = 0; dword < 4u; ++dword) { + if ((extracted & (1u << dword)) == 0u) continue; + auto& extract = program.CreateValue(IrOpcode::CompositeExtractU32x4, IrType::U32); + extract.AddArgument(&instruction); + extract.AddArgument(&ir.Constant(dword)); + } MemoryFlags flags{}; flags.pc = 0x1234; instruction.SetFlags(flags); @@ -67,6 +76,29 @@ std::vector MakeBdaTestShader(std::uint64_t address, std::uint32_ }); } +std::vector MakeBdaSpanReadTestShader(std::uint64_t address, std::uint32_t offset, std::uint32_t extracted, bool coherent, bool stops) { + using namespace ShaderRecompiler; + return MakeShader(address, coherent, stops, [&](SpirvValueEmitContext& ctx, const IrValue& instruction, std::uint32_t base) { + const auto values = EmitBdaDwordReads(ctx, instruction, base, offset, 4u); + return std::vector(values.begin(), values.end()); + }, extracted); +} + +std::vector MakeBdaDwordWriteTestShader(std::uint64_t address, std::uint32_t dwords, const std::uint32_t* values) { + using namespace ShaderRecompiler; + return MakeShader(address, false, true, [&](SpirvValueEmitContext& ctx, const IrValue& instruction, std::uint32_t base) { + auto& state = ctx.state; + std::vector parts; + for (std::uint32_t dword = 0; dword < dwords; ++dword) parts.push_back(ConstantU32(state, values[dword])); + const auto composite = state.module.AllocateId(); + if (dwords == 2u) state.module.AddFunction(spv::OpCompositeConstruct, TypeU32Vector(state, 2u), composite, parts[0], parts[1]); + else if (dwords == 3u) state.module.AddFunction(spv::OpCompositeConstruct, TypeU32Vector(state, 3u), composite, parts[0], parts[1], parts[2]); + else state.module.AddFunction(spv::OpCompositeConstruct, TypeU32Vector(state, 4u), composite, parts[0], parts[1], parts[2], parts[3]); + EmitBdaDwordWrites(ctx, instruction, base, 0u, dwords, composite); + return std::vector{ConstantU32(state, 1u)}; + }, 0u, true); +} + std::vector MakeBdaDwordReadTestShader(std::uint64_t address, std::uint32_t dwords, bool coherent, bool stops) { using namespace ShaderRecompiler; return MakeShader(address, coherent, stops, [&](SpirvValueEmitContext& ctx, const IrValue& instruction, std::uint32_t base) { diff --git a/core/libs/prx/libSceAgcDriver/tests/BdaShader.hpp b/core/libs/prx/libSceAgcDriver/tests/BdaShader.hpp index 2baa4468a..27c99edba 100644 --- a/core/libs/prx/libSceAgcDriver/tests/BdaShader.hpp +++ b/core/libs/prx/libSceAgcDriver/tests/BdaShader.hpp @@ -7,6 +7,8 @@ std::vector MakeBdaTestShader(std::uint64_t address, std::uint32_t bits, std::int64_t offset = 0); std::vector MakeBdaDwordReadTestShader(std::uint64_t address, std::uint32_t dwords, bool coherent, bool stops); +std::vector MakeBdaSpanReadTestShader(std::uint64_t address, std::uint32_t offset, std::uint32_t extracted, bool coherent, bool stops); +std::vector MakeBdaDwordWriteTestShader(std::uint64_t address, std::uint32_t dwords, const std::uint32_t* values); void RunBdaExecutionTests(const AgcDriver::Graphics::Context& context); void RunBdaContractTests(); diff --git a/core/libs/prx/libSceAgcDriver/tests/BdaTests.hpp b/core/libs/prx/libSceAgcDriver/tests/BdaTests.hpp index c18bd2d52..0d90f1c58 100644 --- a/core/libs/prx/libSceAgcDriver/tests/BdaTests.hpp +++ b/core/libs/prx/libSceAgcDriver/tests/BdaTests.hpp @@ -3,12 +3,15 @@ #include "prx/libSceAgcDriver/Graphics/include/Context.hpp" #include +#include #include struct BdaTestAccess { std::function(VkBuffer)> bytes; std::function descriptor; std::function(VkDeviceAddress)> addressBytes; + std::function)> limitMemory; + std::function allocationAttempts; }; void RunBdaResourceTests(const AgcDriver::Graphics::Context& context, const BdaTestAccess& access); diff --git a/core/libs/prx/libSceAgcDriver/tests/BufferPool.cpp b/core/libs/prx/libSceAgcDriver/tests/BufferPool.cpp new file mode 100644 index 000000000..8c8cb0ec7 --- /dev/null +++ b/core/libs/prx/libSceAgcDriver/tests/BufferPool.cpp @@ -0,0 +1,330 @@ +#include "prx/libSceAgcDriver/Graphics/include/BufferPool.hpp" +#include "prx/libSceAgcDriver/Graphics/include/Resources.hpp" +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +using namespace AgcDriver::Graphics; + +namespace { + +int failures = 0; + +void Expect(bool condition, const std::string& what) { + if (condition) return; + std::fprintf(stderr, "FAIL: %s\n", what.c_str()); + ++failures; +} + +constexpr std::size_t MiB = std::size_t{1} << 20u; +constexpr VkMemoryPropertyFlags HostVisible = VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT | VK_MEMORY_PROPERTY_HOST_COHERENT_BIT; + +struct MockDevice { + std::uint64_t next = 1; + std::map sizes; + std::map usages; + std::map> hostMemory; + std::uint64_t allocations = 0; + std::uint64_t frees = 0; + std::uint64_t destroyedBuffers = 0; + VkDeviceSize liveBytes = 0; + std::map memoryBytes; + std::map> bound; +}; + +MockDevice mock; + +VKAPI_ATTR VkResult VKAPI_CALL mockCreateBuffer(VkDevice, const VkBufferCreateInfo* info, const VkAllocationCallbacks*, VkBuffer* buffer) { + *buffer = reinterpret_cast(static_cast(mock.next++)); + mock.sizes[*buffer] = info->size; + mock.usages[*buffer] = info->usage; + return VK_SUCCESS; +} + +VKAPI_ATTR void VKAPI_CALL mockGetBufferMemoryRequirements(VkDevice, VkBuffer buffer, VkMemoryRequirements* requirements) { + *requirements = {mock.sizes.at(buffer), 256, 3}; +} + +VKAPI_ATTR VkResult VKAPI_CALL mockAllocateMemory(VkDevice, const VkMemoryAllocateInfo* info, const VkAllocationCallbacks*, VkDeviceMemory* memory) { + *memory = reinterpret_cast(static_cast(mock.next++)); + if (info->memoryTypeIndex == 1) mock.hostMemory[*memory].resize(info->allocationSize); + mock.memoryBytes[*memory] = info->allocationSize; + mock.liveBytes += info->allocationSize; + ++mock.allocations; + return VK_SUCCESS; +} + +VKAPI_ATTR VkResult VKAPI_CALL mockBindBufferMemory(VkDevice, VkBuffer buffer, VkDeviceMemory memory, VkDeviceSize offset) { + if (offset + mock.sizes.at(buffer) > mock.memoryBytes.at(memory)) return VK_ERROR_OUT_OF_DEVICE_MEMORY; + mock.bound[buffer] = {memory, offset}; + return VK_SUCCESS; +} + +VKAPI_ATTR VkResult VKAPI_CALL mockMapMemory(VkDevice, VkDeviceMemory memory, VkDeviceSize, VkDeviceSize, VkMemoryMapFlags, void** data) { + *data = mock.hostMemory.at(memory).data(); + return VK_SUCCESS; +} + +VKAPI_ATTR void VKAPI_CALL mockUnmapMemory(VkDevice, VkDeviceMemory) {} + +VKAPI_ATTR void VKAPI_CALL mockDestroyBuffer(VkDevice, VkBuffer, const VkAllocationCallbacks*) { + ++mock.destroyedBuffers; +} + +VKAPI_ATTR void VKAPI_CALL mockFreeMemory(VkDevice, VkDeviceMemory memory, const VkAllocationCallbacks*) { + mock.liveBytes -= mock.memoryBytes.at(memory); + mock.memoryBytes.erase(memory); + mock.hostMemory.erase(memory); + ++mock.frees; +} + +VKAPI_ATTR VkDeviceAddress VKAPI_CALL mockGetBufferDeviceAddress(VkDevice, const VkBufferDeviceAddressInfo* info) { + return 0x100000000000ULL + reinterpret_cast(info->buffer) * 0x10000; +} + +PFN_vkVoidFunction VKAPI_CALL mockProc(VkDevice, const char* name) { + static const std::map table{ + {"vkCreateBuffer", reinterpret_cast(mockCreateBuffer)}, + {"vkGetBufferMemoryRequirements", reinterpret_cast(mockGetBufferMemoryRequirements)}, + {"vkAllocateMemory", reinterpret_cast(mockAllocateMemory)}, + {"vkBindBufferMemory", reinterpret_cast(mockBindBufferMemory)}, + {"vkMapMemory", reinterpret_cast(mockMapMemory)}, + {"vkUnmapMemory", reinterpret_cast(mockUnmapMemory)}, + {"vkDestroyBuffer", reinterpret_cast(mockDestroyBuffer)}, + {"vkFreeMemory", reinterpret_cast(mockFreeMemory)}, + {"vkGetBufferDeviceAddressKHR", reinterpret_cast(mockGetBufferDeviceAddress)}, + }; + const auto it = table.find(name); + return it == table.end() ? nullptr : it->second; +} + +Context mockContext() { + Context context{}; + context.deviceProc = mockProc; + context.memory.memoryTypeCount = 2; + context.memory.memoryTypes[0].propertyFlags = VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT; + context.memory.memoryTypes[1].propertyFlags = HostVisible | VK_MEMORY_PROPERTY_HOST_CACHED_BIT; + context.limits.maxMemoryAllocationCount = 1u << 20u; + context.bufferDeviceAddress = true; + return context; +} + +void SizeClassesAndDirections() { + mock = MockDevice{}; + auto context = mockContext(); + VkBuffer first = VK_NULL_HANDLE; + { + DeviceBuffer upload(context, 10 * MiB + 4096, VK_BUFFER_USAGE_STORAGE_BUFFER_BIT | VK_BUFFER_USAGE_TRANSFER_SRC_BIT); + first = upload.Handle(); + Expect(mock.sizes.at(first) == 11 * MiB, "a 10 MiB + 4 KiB device buffer was created with " + std::to_string(mock.sizes.at(first)) + " bytes, not its 11 MiB class"); + constexpr VkBufferUsageFlags all = VK_BUFFER_USAGE_STORAGE_BUFFER_BIT | VK_BUFFER_USAGE_TRANSFER_SRC_BIT | VK_BUFFER_USAGE_TRANSFER_DST_BIT; + Expect((mock.usages.at(first) & all) == all, "a device buffer was created without storage and both transfer directions"); + Expect(upload.Size() == 10 * MiB + 4096, "a device buffer reports its class instead of the requested size"); + } + const auto made = mock.allocations; + DeviceBuffer writeBack(context, 10 * MiB + 512 * 1024, VK_BUFFER_USAGE_STORAGE_BUFFER_BIT | VK_BUFFER_USAGE_TRANSFER_DST_BIT); + Expect(writeBack.Handle() == first && mock.allocations == made, "a retained 11 MiB device buffer did not serve a write-back of another size and direction in its class"); + Expect(writeBack.Size() == 10 * MiB + 512 * 1024, "a reused device buffer reports the retained size"); +} + +void LargerClass() { + mock = MockDevice{}; + auto context = mockContext(); + constexpr auto usage = VK_BUFFER_USAGE_STORAGE_BUFFER_BIT | VK_BUFFER_USAGE_TRANSFER_DST_BIT; + VkBuffer twelve = VK_NULL_HANDLE; + { + DeviceBuffer large(context, 12 * MiB, usage); + twelve = large.Handle(); + } + { + DeviceBuffer smaller(context, 7 * MiB, usage); + Expect(smaller.Handle() == twelve, "a 7 MiB request did not take the retained 12 MiB buffer"); + } + const auto made = mock.allocations; + { + DeviceBuffer tooSmall(context, 5 * MiB, usage); + Expect(tooSmall.Handle() != twelve && mock.allocations == made + 1, "a 5 MiB request took a retained buffer of more than twice its size"); + Expect(mock.sizes.at(tooSmall.Handle()) == 5 * MiB, "a 5 MiB device buffer was not created at its own class"); + } + DeviceBuffer again(context, 12 * MiB, usage); + Expect(again.Handle() == twelve, "a buffer served to a smaller request did not return to its own 12 MiB class"); + DeviceBuffer best(context, 4 * MiB + 1, usage); + Expect(best.Handle() != twelve && mock.sizes.at(best.Handle()) == 5 * MiB, "a 4 MiB + 1 request did not take the smallest retained class that fits (5 MiB)"); +} + +void KeptUntilFence() { + mock = MockDevice{}; + auto context = mockContext(); + constexpr auto usage = VK_BUFFER_USAGE_STORAGE_BUFFER_BIT | VK_BUFFER_USAGE_TRANSFER_SRC_BIT; + std::vector> batch; + for (int i = 0; i < 3; ++i) batch.push_back(std::make_shared(context, 6 * MiB + static_cast(i + 1) * 4096, usage)); + for (std::size_t i = 0; i < batch.size(); ++i) { + for (std::size_t j = i + 1; j < batch.size(); ++j) Expect(batch[i]->Handle() != batch[j]->Handle(), "two buffers of one unfinished batch share a VkBuffer"); + } + const auto kept = batch.front()->Handle(); + const auto made = mock.allocations; + { + DeviceBuffer during(context, 6 * MiB, usage); + Expect(during.Handle() != kept && mock.allocations == made + 1, "a buffer kept by an unfinished batch was handed out again"); + } + batch.clear(); + std::vector> after; + for (int i = 0; i < 3; ++i) after.push_back(std::make_unique(context, 6 * MiB + 100, usage)); + after.push_back(std::make_unique(context, 6 * MiB, usage)); + Expect(mock.allocations == made + 1, "the buffers released after the batch fence were not reused (" + std::to_string(mock.allocations - made - 1) + " new allocations)"); +} + +void Budget() { + mock = MockDevice{}; + auto context = mockContext(); + constexpr auto usage = VK_BUFFER_USAGE_STORAGE_BUFFER_BIT | VK_BUFFER_USAGE_TRANSFER_DST_BIT; + std::vector> live; + for (int i = 0; i < 12; ++i) live.push_back(std::make_unique(context, (48 + static_cast(i) * 3) * MiB, usage)); + const auto first = live.front()->Handle(); + VkDeviceSize madeBytes = 0; + for (const auto& buffer : live) madeBytes += mock.sizes.at(buffer->Handle()); + Expect(madeBytes > 512 * MiB, "the budget case does not exceed the budget"); + for (auto& buffer : live) buffer.reset(); + Expect(mock.frees != 0 && mock.liveBytes <= 512 * MiB, "the device tier retains " + std::to_string(mock.liveBytes / MiB) + " MiB after " + std::to_string(mock.frees) + " evictions, over its 512 MiB budget"); + DeviceBuffer probe(context, 48 * MiB, usage); + Expect(probe.Handle() != first, "the least recently used device buffer survived the eviction"); +} + +void AddressAndHostUnchanged() { + mock = MockDevice{}; + auto context = mockContext(); + VkBuffer scratch = VK_NULL_HANDLE; + { + DeviceBuffer buffer(context, 2 * MiB, VK_BUFFER_USAGE_STORAGE_BUFFER_BIT | VK_BUFFER_USAGE_TRANSFER_SRC_BIT); + scratch = buffer.Handle(); + } + Buffer addressed(context, 2 * MiB, VK_BUFFER_USAGE_STORAGE_BUFFER_BIT | VK_BUFFER_USAGE_TRANSFER_SRC_BIT | VK_BUFFER_USAGE_TRANSFER_DST_BIT | VK_BUFFER_USAGE_SHADER_DEVICE_ADDRESS_BIT, VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT); + Expect(addressed.Handle() != scratch && addressed.DeviceAddress() != 0, "an addressable device-local buffer took a scratch buffer without a device address"); + Buffer shadow(context, 2 * MiB - 256, VK_BUFFER_USAGE_STORAGE_BUFFER_BIT | VK_BUFFER_USAGE_TRANSFER_SRC_BIT | VK_BUFFER_USAGE_TRANSFER_DST_BIT, VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT); + Expect(shadow.Handle() == scratch, "a device-local staging buffer did not share the scratch buffers' class"); + VkBuffer host = VK_NULL_HANDLE; + { + Buffer copy(context, 10 * MiB + 4096, VK_BUFFER_USAGE_TRANSFER_SRC_BIT, HostVisible); + host = copy.Handle(); + Expect(mock.sizes.at(host) == 10 * MiB + 4096, "a large host buffer was rounded to a class"); + Expect(mock.usages.at(host) == VK_BUFFER_USAGE_TRANSFER_SRC_BIT, "a host buffer was created with more usages than asked"); + } + Buffer other(context, 10 * MiB + 8192, VK_BUFFER_USAGE_TRANSFER_SRC_BIT, HostVisible); + Expect(other.Handle() != host, "a large host buffer of another exact size was reused"); + Buffer same(context, 10 * MiB + 4096, VK_BUFFER_USAGE_TRANSFER_SRC_BIT, HostVisible); + Expect(same.Handle() == host, "a large host buffer of the same exact size was not reused"); +} + +void SmallDeviceClasses() { + mock = MockDevice{}; + auto context = mockContext(); + constexpr auto usage = VK_BUFFER_USAGE_STORAGE_BUFFER_BIT | VK_BUFFER_USAGE_TRANSFER_DST_BIT; + VkBuffer mebibyte = VK_NULL_HANDLE; + VkBuffer half = VK_NULL_HANDLE; + { + DeviceBuffer one(context, MiB, usage); + mebibyte = one.Handle(); + } + { + DeviceBuffer smaller(context, 300 * 1024, usage); + Expect(smaller.Handle() != mebibyte && mock.sizes.at(smaller.Handle()) == 512 * 1024, "a 300 KiB request took a 1 MiB buffer or skipped its power-of-two class"); + half = smaller.Handle(); + } + DeviceBuffer again(context, 260 * 1024, usage); + Expect(again.Handle() == half, "a 260 KiB request did not take the retained 512 KiB buffer"); + constexpr auto indirect = VK_BUFFER_USAGE_INDIRECT_BUFFER_BIT | VK_BUFFER_USAGE_TRANSFER_DST_BIT; + VkBuffer tiny = VK_NULL_HANDLE; + { + DeviceBuffer args(context, 20, indirect); + tiny = args.Handle(); + } + const auto made = mock.allocations; + DeviceBuffer nextArgs(context, 20, indirect); + Expect(nextArgs.Handle() == tiny && mock.allocations == made, "a 20-byte device request did not reuse the retained 256-byte buffer of its class"); +} + +void Slabs() { + Expect(BufferPool::SlabEligible(512, 256, 512, 64), "a size class with a smaller alignment and atom was refused a slab"); + Expect(!BufferPool::SlabEligible(512, 1024, 512, 64), "a size class below its alignment was given a slab"); + Expect(!BufferPool::SlabEligible(512, 256, 640, 64), "a buffer needing more than its class was given a slab"); + Expect(!BufferPool::SlabEligible(512, 256, 512, 1024), "a size class below the non-coherent atom was given a slab"); + Expect(!BufferPool::SlabEligible(MiB, 256, MiB, 64), "an exact-size buffer was given a slab"); + mock = MockDevice{}; + { + auto context = mockContext(); + context.limits.nonCoherentAtomSize = 64; + constexpr VkBufferUsageFlags storage = VK_BUFFER_USAGE_STORAGE_BUFFER_BIT; + { + Buffer a(context, 300, storage); + Buffer b(context, 300, storage); + Buffer c(context, 260, storage); + Expect(mock.allocations == 1, "buffers of one size class did not share one memory block"); + const auto block = mock.bound.at(a.Handle()).first; + std::set offsets; + for (auto* buffer : {&a, &b, &c}) { + const auto [memory, offset] = mock.bound.at(buffer->Handle()); + Expect(memory == block, "buffers of one size class were bound to different blocks"); + Expect(offset % 512 == 0, "a slab slot is not aligned to its size class"); + Expect(offsets.insert(offset).second, "two live buffers share a slab slot"); + Expect(buffer->Bytes().data() == mock.hostMemory.at(memory).data() + offset, "a slab buffer's mapping is not its slot"); + } + std::memset(a.Bytes().data(), 0x11, a.Bytes().size()); + std::memset(b.Bytes().data(), 0x22, b.Bytes().size()); + std::memset(c.Bytes().data(), 0x33, c.Bytes().size()); + Expect(std::all_of(a.Bytes().begin(), a.Bytes().end(), [](std::byte value) { return value == std::byte{0x11}; }), "a neighbouring slab buffer overwrote another"); + Expect(std::all_of(b.Bytes().begin(), b.Bytes().end(), [](std::byte value) { return value == std::byte{0x22}; }), "a neighbouring slab buffer overwrote another"); + Buffer larger(context, 600, storage); + Expect(mock.allocations == 2 && mock.bound.at(larger.Handle()).first != block, "another size class did not get a block of its own"); + } + Expect(mock.frees == 0, "releasing slab buffers freed device memory"); + Buffer again(context, 300, storage); + Expect(mock.allocations == 2, "a released slab buffer was not reused"); + } + Expect(mock.liveBytes == 0, "slab blocks outlived their pool"); + mock = MockDevice{}; + { + auto context = mockContext(); + BufferPool pool(context); + constexpr std::size_t slot = 512 * 1024; + const auto perBlock = static_cast(BufferPool::SlabBlockBytes(slot) / slot); + std::vector slots; + for (std::size_t i = 0; i <= perBlock; ++i) slots.push_back(pool.TakeSlot(context, 1, slot, false)); + Expect(mock.allocations == 2 && pool.SlabBlocks() == 2, "a full block did not open a second one"); + std::set> distinct; + for (const auto& taken : slots) distinct.insert({taken.memory, taken.offset}); + Expect(distinct.size() == slots.size(), "a slab handed out one slot twice"); + for (const auto& taken : slots) pool.PutSlot(taken.memory, taken.offset); + Expect(mock.frees == 1 && pool.SlabBlocks() == 1, "emptied blocks were not freed down to one spare"); + const auto reused = pool.TakeSlot(context, 1, slot, false); + Expect(mock.allocations == 2, "the spare block was not reused"); + pool.PutSlot(reused.memory, reused.offset); + } + Expect(mock.liveBytes == 0, "slab blocks outlived their pool"); +} + +} + +int main() { + try { + SizeClassesAndDirections(); + LargerClass(); + KeptUntilFence(); + Budget(); + AddressAndHostUnchanged(); + SmallDeviceClasses(); + Slabs(); + } catch (const std::exception& error) { + std::fprintf(stderr, "FAIL: %s\n", error.what()); + return 1; + } + if (failures != 0) return 1; + std::puts("buffer pool tests passed"); + return 0; +} diff --git a/core/libs/prx/libSceAgcDriver/tests/CmaskFastClear.cpp b/core/libs/prx/libSceAgcDriver/tests/CmaskFastClear.cpp new file mode 100644 index 000000000..ffb977d6e --- /dev/null +++ b/core/libs/prx/libSceAgcDriver/tests/CmaskFastClear.cpp @@ -0,0 +1,199 @@ +#include "prx/libSceAgcDriver/Execution/include/GuestMemory.hpp" +#include "prx/libSceAgcDriver/Execution/include/VulkanDevice.hpp" +#include "prx/libSceAgcDriver/Graphics/include/ColorTargetLayout.hpp" +#include "prx/libSceAgcDriver/Graphics/include/Texture.hpp" +#include "prx/libc/include/GuestAllocations.hpp" +#include "Cover_vert_spv.h" +#include "Cover_frag_spv.h" +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace { + +using namespace AgcDriver::Graphics; + +constexpr int VulkanTestSkipped = 77; +constexpr std::uint32_t Width = 61; +constexpr std::uint32_t Height = 13; +constexpr std::uint32_t Stale = 0x5a5a5a5au; +constexpr std::uint32_t Clear = 0x11223344u; +constexpr std::uint32_t Drawn = 0xffffffffu; + +alignas(4096) std::array SurfaceStorage{}; +const std::span Surface{reinterpret_cast((reinterpret_cast(SurfaceStorage.data()) + 65535u) & ~std::uintptr_t{65535u}), 65536}; +alignas(4096) std::array Cmask{}; + +ColorTarget target(ColorTileMode mode) { + const ColorTargetLayout layout(Width, Height, mode, 4); + ColorTarget color{}; + color.address = reinterpret_cast(Surface.data()); + color.extent = {Width, Height}; + color.format = VK_FORMAT_R8G8B8A8_UNORM; + color.bytes = layout.Bytes(); + color.componentMapping = 0xe4u; + color.tileMode = mode; + color.elementBytes = 4; + color.clearWords = {Clear, 0}; + color.surfaceAddress = color.address; + color.surfaceExtent = color.extent; + color.cmaskAddress = reinterpret_cast(Cmask.data()); + color.cmaskBytes = CmaskLayout(Width, Height).Bytes(); + return color; +} + +void prepare(const ColorTarget& color, const std::function& cleared, std::uint8_t code = 0) { + std::lock_guard lock(AgcDriver::GuestMemory::GpuMutex()); + StorageTexture::FlushPending(color.address, Surface.size(), nullptr, "test"); + std::vector stale(Surface.size()); + for (std::size_t offset = 0; offset < stale.size(); offset += 4) std::memcpy(stale.data() + offset, &Stale, 4); + AgcDriver::GuestMemory::Write(color.address, stale); + const CmaskLayout layout(Width, Height); + std::vector mask(Cmask.size(), std::byte{0xff}); + for (std::uint32_t tileY = 0; tileY < layout.TilesY(); ++tileY) { + for (std::uint32_t tileX = 0; tileX < layout.TilesX(); ++tileX) { + if (!cleared(tileX, tileY)) continue; + const auto nibble = layout.Nibble(tileX, tileY); + mask[nibble / 2u] &= std::byte{static_cast((nibble % 2u) == 0 ? 0xf0u : 0x0fu)}; + mask[nibble / 2u] |= std::byte{static_cast((nibble % 2u) == 0 ? code : code << 4u)}; + } + } + AgcDriver::GuestMemory::Write(color.cmaskAddress, mask); +} + +void draw(AgcDriver::VulkanDevice& device, const ColorTarget& color, VkRect2D scissor) { + ShaderRecompiler::RecompileResult vertex; + vertex.spirv = std::vector(std::begin(COVER_VERT_SPV), std::end(COVER_VERT_SPV)); + ShaderRecompiler::RecompileResult fragment; + fragment.spirv = std::vector(std::begin(COVER_FRAG_SPV), std::end(COVER_FRAG_SPV)); + const std::array shaders{{ + {ShaderRecompiler::ShaderStage::Vertex, &vertex, 0}, + {ShaderRecompiler::ShaderStage::Fragment, &fragment, 0} + }}; + State state{}; + state.stages = {ShaderPath::Vertex, 0u, 64u, 64u, std::nullopt, std::nullopt}; + state.color = color; + state.colors = {color}; + state.hasColorTarget = true; + state.renderExtent = {Width, Height}; + state.topology = VK_PRIMITIVE_TOPOLOGY_TRIANGLE_LIST; + state.viewport = {0, 0, static_cast(Width), static_cast(Height), 0, 1}; + state.negativeOneToOne = false; + state.scissor = scissor; + state.cullMode = VK_CULL_MODE_NONE; + state.frontFace = VK_FRONT_FACE_COUNTER_CLOCKWISE; + state.blend.colorWriteMask = 15; + state.blends = {state.blend}; + state.blendConstants = {}; + std::lock_guard lock(AgcDriver::GuestMemory::GpuMutex()); + device.Draw(state, {0, 3, 0, 1, 0, false}, shaders); + device.WaitIdle(); +} + +void expect(AgcDriver::VulkanDevice& device, const ColorTarget& color, const std::function& texel, const std::string& what) { + std::lock_guard lock(AgcDriver::GuestMemory::GpuMutex()); + StorageTexture::FlushPending(color.address, color.bytes, nullptr, "test"); + device.WaitIdle(); + const ColorTargetLayout layout(Width, Height, color.tileMode, 4); + for (std::uint32_t y = 0; y < Height; ++y) { + for (std::uint32_t x = 0; x < Width; ++x) { + std::uint32_t stored = 0; + std::memcpy(&stored, Surface.data() + layout.Offset(x, y), 4); + Require(stored == texel(x, y), what + ": texel (" + std::to_string(x) + ", " + std::to_string(y) + ") holds " + std::to_string(stored)); + } + } + Require(std::all_of(Cmask.begin(), Cmask.end(), [](std::uint8_t value) { return value == 0xffu; }), what + ": the CMASK was not left expanded"); +} + +void eliminate(AgcDriver::VulkanDevice& device, const ColorTarget& color) { + std::lock_guard lock(AgcDriver::GuestMemory::GpuMutex()); + device.ColorMetadataPass({ColorMetadataPass::Mode::EliminateFastClear, {color}}); +} + +void run(AgcDriver::VulkanDevice& device, ColorTileMode mode, const std::string& name) { + const auto color = target(mode); + prepare(color, [](std::uint32_t, std::uint32_t) { return true; }); + draw(device, color, {{0, 0}, {20, Height}}); + expect(device, color, [](std::uint32_t x, std::uint32_t) { return x < 20 ? Drawn : Clear; }, name + " fast-cleared target drawn over its left part"); + draw(device, color, {{40, 0}, {Width - 40, Height}}); + expect(device, color, [](std::uint32_t x, std::uint32_t) { return x < 20 || x >= 40 ? Drawn : Clear; }, name + " fast-cleared target drawn again over its right part"); + prepare(color, [](std::uint32_t tileX, std::uint32_t tileY) { return (tileX == 0 && tileY == 0) || (tileX == 7 && tileY == 1); }); + draw(device, color, {{0, 0}, {20, Height}}); + expect(device, color, [](std::uint32_t x, std::uint32_t y) { return x < 20 ? Drawn : x >= 56 && y >= 8 ? Clear : Stale; }, name + " partly fast-cleared target drawn over its left part"); + prepare(color, [](std::uint32_t, std::uint32_t) { return true; }); + eliminate(device, color); + expect(device, color, [](std::uint32_t, std::uint32_t) { return Clear; }, name + " fast-cleared target eliminated without a draw"); + prepare(color, [](std::uint32_t tileX, std::uint32_t tileY) { return tileX == 7 && tileY == 1; }); + eliminate(device, color); + expect(device, color, [](std::uint32_t x, std::uint32_t y) { return x >= 56 && y >= 8 ? Clear : Stale; }, name + " partly fast-cleared edge tile eliminated"); + prepare(color, [](std::uint32_t tileX, std::uint32_t tileY) { return (tileX + tileY) % 2u == 0; }); + eliminate(device, color); + expect(device, color, [](std::uint32_t x, std::uint32_t y) { return (x / 8u + y / 8u) % 2u == 0 ? Clear : Stale; }, name + " checkerboard of fast-cleared tiles eliminated"); + prepare(color, [](std::uint32_t tileX, std::uint32_t tileY) { return tileX >= 3 && tileY == 1; }); + draw(device, color, {{0, 0}, {Width, 4}}); + expect(device, color, [](std::uint32_t x, std::uint32_t y) { return y < 4 ? Drawn : y >= 8 && x >= 24 ? Clear : Stale; }, name + " fast-cleared bottom edge tiles under a draw of the top rows"); +} + +void invalidCode(AgcDriver::VulkanDevice& device) { + const auto color = target(ColorTileMode::RenderTarget); + prepare(color, [](std::uint32_t tileX, std::uint32_t tileY) { return tileX == 2 && tileY == 0; }, 3); + eliminate(device, color); + { + std::lock_guard lock(AgcDriver::GuestMemory::GpuMutex()); + device.WaitIdle(); + } + prepare(color, [](std::uint32_t, std::uint32_t) { return true; }); + try { + eliminate(device, color); + } catch (const std::runtime_error& error) { + Require(std::string(error.what()).find("were recorded, which are not a single-sample fast-clear code") != std::string::npos, std::string("unexpected failure: ") + error.what()); + return; + } + throw std::runtime_error("a CMASK code 3 read by the GPU pass did not throw"); +} + +} + +int main() { + try { + std::unique_ptr device; + try { + device = std::make_unique(); + } catch (const std::exception& error) { + if (std::getenv("ANYPS5_REQUIRE_VULKAN") != nullptr) throw; + std::printf("skipped, no usable Vulkan device: %s\n", error.what()); + return VulkanTestSkipped; + } + { + GuestAllocations::Mutation mutation; + mutation.Add(SurfaceStorage.data(), SurfaceStorage.size(), true, true, true); + mutation.Add(Cmask.data(), Cmask.size(), true, true, true); + } + run(*device, ColorTileMode::Linear, "linear"); + run(*device, ColorTileMode::RenderTarget, "SW_64KB_R_X"); + invalidCode(*device); + device->WaitIdle(); + device.reset(); + { + GuestAllocations::Mutation mutation; + mutation.Remove(SurfaceStorage.data()); + mutation.Remove(Cmask.data()); + } + std::cout << "CMASK fast clear draw and eliminate tests passed\n"; + return 0; + } catch (const std::exception& error) { + std::cerr << error.what() << '\n'; + return 1; + } +} diff --git a/core/libs/prx/libSceAgcDriver/tests/ColorTargetLayout.cpp b/core/libs/prx/libSceAgcDriver/tests/ColorTargetLayout.cpp index 40b69b7a4..d414e2976 100644 --- a/core/libs/prx/libSceAgcDriver/tests/ColorTargetLayout.cpp +++ b/core/libs/prx/libSceAgcDriver/tests/ColorTargetLayout.cpp @@ -1,11 +1,14 @@ #include "BdaTests.hpp" #include "prx/libSceAgcDriver/Graphics/include/ColorTargetTransfer.hpp" #include "prx/libSceAgcDriver/Graphics/include/TextureTiling.hpp" +#include "prx/libSceAgcDriver/Graphics/include/TextureSwizzleEquations.hpp" #include #include +#include #include #include #include +#include #include namespace { @@ -24,10 +27,20 @@ void reject(TAction action) { void RunColorTargetLayoutTests() { Require(DecodeColorTileMode(0x4dc6c000) == ColorTileMode::RenderTarget, "logged color descriptor was rejected"); Require(DecodeColorTileMode(0x09000000) == ColorTileMode::Linear, "linear descriptor changed"); - reject([] { DecodeColorTileMode(0x4dc6c001); }); + Require(DecodeColorTileMode(0x4dc6c001) == ColorTileMode::RenderTarget, "2D array color descriptor with MIP0_DEPTH 1 was rejected"); reject([] { DecodeColorTileMode(0x4dc6e000); }); reject([] { DecodeColorTileMode(0xcdc6c000); }); reject([] { DecodeColorTileMode(0x09004000); }); + Require(DecodeColorTileMode(0x4cc6c000) == ColorTileMode::RenderTarget, "1D color descriptor was rejected"); + reject([] { DecodeColorTileMode(0x4cc6c001); }); + reject([] { DecodeColorTileMode(0x4fc6c000); }); + Require(DecodeColorTileMode(0x08000000) == ColorTileMode::Linear, "linear 1D color descriptor was rejected"); + reject([] { DecodeColorTileMode(0x08014000); }); + reject([] { DecodeColorTileMode(0x08024000); }); + Require(DecodeColorTileMode(0x0da58000) == ColorTileMode::D4KBX, "a SW_4KB_D_X color descriptor with its Z-mode FMASK swizzle was rejected"); + Require(DecodeColorTileMode(0x0da54000) == ColorTileMode::S4KBX, "a SW_4KB_S_X color descriptor with its Z-mode FMASK swizzle was rejected"); + Require(DecodeColorTileMode(0x09058000) == ColorTileMode::D4KBX, "a SW_4KB_D_X color descriptor without an FMASK swizzle was rejected"); + reject([] { DecodeColorTileMode(0x0d458000); }); reject([] { ColorTargetLayout(0, 1, ColorTileMode::RenderTarget); }); const ColorTargetLayout padded(63, 2, ColorTileMode::Linear); Require(padded.Bytes() == 512 && padded.LinearBytes() == 504 && padded.Offset(0, 1) == 256, "linear rows are not padded to 256 bytes"); @@ -71,6 +84,33 @@ void RunColorTargetLayoutTests() { } Require(DecodeColorTileMode(0x4dc24000) == ColorTileMode::Standard64KB, "64 KiB standard color descriptor was rejected"); reject([&] { layout.Offset(257, 0); }); + Require(CmaskLayout(1, 1).Bytes() == 4096 && CmaskLayout(1024, 512).Bytes() == 4096 && CmaskLayout(1025, 513).Bytes() == 16384 && CmaskLayout(1920, 1080).Bytes() == 24576 && CmaskLayout(2432, 1368).Bytes() == 36864 && CmaskLayout(3328, 1872).Bytes() == 65536 && CmaskLayout(3840, 2160).Bytes() == 81920, "CMASK size differs from addrlib's pipe-aligned SW_64KB_Z_X CMASK"); + struct CmaskReference { + std::uint32_t width; + std::uint32_t height; + std::uint32_t tileX; + std::uint32_t tileY; + std::size_t nibble; + }; + constexpr std::array cmaskReferences{{ + {20, 12, 2, 1, 0x600u}, {61, 13, 7, 1, 0x1401u}, {61, 13, 5, 0, 0x1201u}, {1920, 1080, 239, 134, 0xb347u}, {1920, 1080, 128, 64, 0x6000u}, + {1920, 1080, 1, 1, 0x1u}, {3840, 2160, 479, 269, 0x2651du}, {3840, 2160, 200, 77, 0xb30cu}, {2432, 1368, 303, 170, 0x10acfu}, {1025, 513, 128, 64, 0x6000u}, + }}; + for (const auto& reference : cmaskReferences) Require(CmaskLayout(reference.width, reference.height).Nibble(reference.tileX, reference.tileY) == reference.nibble, "CMASK nibble address differs from addrlib's"); + for (const auto [width, height] : {std::pair{61u, 13u}, std::pair{1920u, 1080u}}) { + const CmaskLayout cmask(width, height); + std::vector seen(cmask.Bytes() * 2u); + for (std::uint32_t tileY = 0; tileY < cmask.TilesY(); ++tileY) { + for (std::uint32_t tileX = 0; tileX < cmask.TilesX(); ++tileX) { + const auto nibble = cmask.Nibble(tileX, tileY); + Require(nibble < seen.size() && !seen[nibble], "CMASK tiles alias or leave the CMASK"); + seen[nibble] = true; + } + } + } + reject([] { CmaskLayout(0, 1); }); + reject([] { CmaskLayout(61, 13).Nibble(8, 0); }); + reject([] { CmaskLayout(61, 13).Nibble(0, 2); }); std::vector tiled(layout.Bytes(), std::byte{0x5a}); std::vector linear(layout.LinearBytes()); std::vector restored(linear.size()); @@ -111,4 +151,54 @@ void RunColorTargetLayoutTests() { std::array linearReadback{}; ReadColorTarget(target, linearReadback); Require(linearReadback == linearPixels && guest[512] == std::byte{0x6b}, "linear guest color transfer changed"); + + const ColorTargetLayout plainLayout(256, 70, ColorTileMode::RenderTarget, 4); + const ColorTargetLayout xorLayout(256, 70, ColorTileMode::RenderTarget, 4, 0x5600); + for (const auto& [x, y] : {std::pair{0u, 0u}, std::pair{1u, 0u}, std::pair{0u, 1u}, std::pair{127u, 69u}, std::pair{128u, 0u}, std::pair{255u, 69u}, std::pair{200u, 33u}}) { + const auto plain = plainLayout.Offset(x, y); + Require(xorLayout.Offset(x, y) == (plain & ~std::size_t{0xffff}) + ((plain & 0xffffu) ^ 0x5600u), "a pipe/bank XOR did not apply to the offset inside the SW_64KB_R_X block"); + } + std::vector xorLinear(xorLayout.LinearBytes()); + for (std::size_t i = 0; i < xorLinear.size(); ++i) xorLinear[i] = static_cast((i * 7u) & 255u); + std::vector xorTiled(xorLayout.Bytes(), std::byte{0}); + std::vector xorRestored(xorLinear.size()); + xorLayout.Tile(xorLinear, xorTiled); + xorLayout.Detile(xorTiled, xorRestored); + Require(xorRestored == xorLinear, "color tiling round trip with a pipe/bank XOR lost pixels"); + reject([] { static_cast(ColorTargetLayout(64, 64, ColorTileMode::Standard64KB, 4, 0x100)); }); + reject([] { static_cast(ColorTargetLayout(64, 64, ColorTileMode::Linear, 4, 0x100)); }); + reject([] { static_cast(ColorTargetLayout(64, 64, ColorTileMode::RenderTarget, 4, 0x80)); }); + reject([] { static_cast(ColorTargetLayout(64, 64, ColorTileMode::RenderTarget, 4, 0x10000)); }); + + std::fill(guest.begin(), guest.end(), std::byte{0x6b}); + target = {reinterpret_cast(guest.data()), {2, 2}, VK_FORMAT_R8G8B8A8_UNORM, guest.size(), 0xe4, ColorTileMode::RenderTarget}; + target.pipeBankXor = 0x3200; + WriteColorTarget(target, pixels); + Require(guest[0x3200] == std::byte{0x32} && guest[0x3210] == std::byte{0x32} && guest[0] == std::byte{0x6b} && guest[16] == std::byte{0x6b}, "a color target with a pipe/bank XOR was written at the unswizzled offsets"); + readback.fill(std::byte{0}); + ReadColorTarget(target, readback); + Require(readback == pixels, "guest color transfer round trip with a pipe/bank XOR failed"); + + for (const auto mode : {ColorTileMode::S4KBX, ColorTileMode::D4KBX}) { + const auto* equation = FindTextureSwizzleEquation(static_cast(mode), 4u); + Require(equation != nullptr, "the 4 KiB XOR equation is missing"); + const auto expected = [&](std::uint32_t x, std::uint32_t y) { + std::size_t offset = 0; + for (std::uint32_t bit = 0; bit < 16u; ++bit) offset |= static_cast(std::popcount((x & equation->bits[bit] & 0xfffu) ^ ((y << 12u) & equation->bits[bit] & 0xfff000u)) & 1) << bit; + return offset; + }; + const ColorTargetLayout xor4Kb(40, 20, mode, 4, 0xa00); + Require(xor4Kb.Alignment() == 4096u && xor4Kb.Bytes() == 64u * 32u * 4u && xor4Kb.BlocksPerRow() == 2u, "the 4 KiB XOR color layout has the wrong block geometry"); + for (const auto& [x, y] : {std::pair{0u, 0u}, std::pair{1u, 0u}, std::pair{0u, 1u}, std::pair{31u, 19u}, std::pair{32u, 0u}, std::pair{39u, 19u}}) { + Require(xor4Kb.Offset(x, y) == (x / 32u) * 4096u + (expected(x % 32u, y) ^ 0xa00u), "a 4 KiB XOR color offset does not follow its swizzle equation and pipe/bank XOR"); + } + std::vector source(xor4Kb.LinearBytes()); + for (std::size_t i = 0; i < source.size(); ++i) source[i] = static_cast((i * 13u) & 255u); + std::vector tiled4Kb(xor4Kb.Bytes(), std::byte{0}); + std::vector back(source.size()); + xor4Kb.Tile(source, tiled4Kb); + xor4Kb.Detile(tiled4Kb, back); + Require(back == source, "4 KiB XOR color tiling round trip lost pixels"); + reject([mode] { static_cast(ColorTargetLayout(40, 20, mode, 4, 0x1000)); }); + } } diff --git a/core/libs/prx/libSceAgcDriver/tests/ConstantBitTest.cpp b/core/libs/prx/libSceAgcDriver/tests/ConstantBitTest.cpp new file mode 100644 index 000000000..92786750e --- /dev/null +++ b/core/libs/prx/libSceAgcDriver/tests/ConstantBitTest.cpp @@ -0,0 +1,50 @@ +#include "IntermediateRepresentation/IrBuilder.hpp" +#include "Optimization/ConstantFolder.hpp" +#include +#include +#include + +using namespace ShaderRecompiler; + +namespace { + +bool foldsTo(std::uint32_t word, bool expected, bool maskedShift) { + IrProgram program; + auto& block = program.CreateBlock(); + program.SetEntryBlock(block); + program.BlockOrder().push_back(&block); + IrBuilder builder(program); + builder.SetInsertionPoint(block); + auto& lane = builder.Emit(IrOpcode::LaneId, IrType::U32, {}); + auto& shift = maskedShift ? builder.Emit(IrOpcode::BitwiseAnd32, IrType::U32, {&lane, &builder.Constant(31u)}) : lane; + auto& shifted = builder.Emit(IrOpcode::ShiftRightLogical32, IrType::U32, {&builder.Constant(word), &shift}); + auto& bit = builder.Emit(IrOpcode::BitwiseAnd32, IrType::U32, {&shifted, &builder.Constant(1u)}); + auto& test = builder.Emit(IrOpcode::INotEqual32, IrType::Bool, {&bit, &builder.Constant(0u)}); + auto& user = builder.Emit(IrOpcode::Reference, IrType::Void, {&test}); + static_cast(builder.Emit(IrOpcode::Return, IrType::Void, {})); + ConstantFolder().Fold(program); + const IrValue* result = user.Argument(0)->Resolve(); + if (!maskedShift || (word != 0u && word != 0xffffffffu)) return result->Opcode() == IrOpcode::INotEqual32; + return result->HasImmediate() && result->ImmediateBool() == expected; +} + +bool run(const char* name, bool passed) { + if (!passed) std::fprintf(stderr, "%s\n", name); + return passed; +} + +} + +int main() { + try { + bool passed = true; + passed &= run("an all-zero word must fold the lane bit test to false", foldsTo(0u, false, true)); + passed &= run("an all-one word must fold the lane bit test to true", foldsTo(0xffffffffu, true, true)); + passed &= run("a mixed word must keep the lane bit test", foldsTo(0x0000ffffu, false, true)); + passed &= run("an unmasked lane number must keep the lane bit test", foldsTo(0xffffffffu, true, false)); + return passed ? 0 : 1; + } catch (const std::exception& error) { + std::fprintf(stderr, "%s\n", error.what()); + return 1; + } +} diff --git a/core/libs/prx/libSceAgcDriver/tests/DenormalFlushElimination.cpp b/core/libs/prx/libSceAgcDriver/tests/DenormalFlushElimination.cpp new file mode 100644 index 000000000..47020acb0 --- /dev/null +++ b/core/libs/prx/libSceAgcDriver/tests/DenormalFlushElimination.cpp @@ -0,0 +1,196 @@ +#include "IntermediateRepresentation/IrProgram.hpp" +#include "Optimization/DenormalFlushEliminator.hpp" +#include +#include +#include +#include +#include +#include + +using namespace ShaderRecompiler; + +namespace { + +struct Builder { + IrProgram program; + IrBlock* block = nullptr; + + Builder() { + block = &newBlock(); + program.SetEntryBlock(*block); + } + + IrBlock& newBlock() { + auto& created = program.CreateBlock(); + program.BlockOrder().push_back(&created); + return created; + } + + IrValue& constant(std::uint32_t value) { + auto& created = program.CreateValue(IrOpcode::Void, IrType::U32); + created.SetImmediateU32(value); + return created; + } + + IrValue& emit(IrOpcode opcode, IrType type, std::initializer_list arguments) { + auto& created = program.CreateValue(opcode, type); + for (auto* argument : arguments) created.AddArgument(argument); + block->AppendInstruction(&created); + return created; + } + + IrValue& unknown() { + return emit(IrOpcode::LaneId, IrType::U32, {}); + } + + IrValue& flush(IrValue& bits) { + auto& exponent = emit(IrOpcode::BitwiseAnd32, IrType::U32, {&bits, &constant(0x7f800000u)}); + auto& denormal = emit(IrOpcode::IEqual32, IrType::Bool, {&exponent, &constant(0u)}); + auto& sign = emit(IrOpcode::BitwiseAnd32, IrType::U32, {&bits, &constant(0x80000000u)}); + return emit(IrOpcode::SelectU32, IrType::U32, {&denormal, &sign, &bits}); + } + + IrValue& phi(IrBlock& block, std::initializer_list> incoming) { + auto& created = program.CreateValue(IrOpcode::Phi, IrType::U32); + block.AppendInstruction(&created); + for (const auto& [predecessor, value] : incoming) created.AddPhiOperand(predecessor, value); + return created; + } + + void keep(IrValue& value) { + (void)emit(IrOpcode::ReferenceU32, IrType::Void, {&value}); + } + + std::uint32_t eliminate() { + return DenormalFlushEliminator{}.Eliminate(program).removedFlushes; + } +}; + +bool expect(const char* name, bool condition) { + if (!condition) std::fprintf(stderr, "%s\n", name); + return condition; +} + +bool run(const char* name, const std::function& test) { + try { + return expect(name, test()); + } catch (const std::exception& error) { + std::fprintf(stderr, "%s: %s\n", name, error.what()); + return false; + } +} + +} + +int main() { + bool passed = true; + + passed &= run("a flush of a flushed value is removed and the first flush stays", [] { + Builder b; + auto& first = b.flush(b.unknown()); + auto& second = b.flush(first); + b.keep(second); + return b.eliminate() == 1u && !second.HasUses() && first.HasUses(); + }); + + passed &= run("a flush of an unknown value stays", [] { + Builder b; + auto& flushed = b.flush(b.unknown()); + b.keep(flushed); + return b.eliminate() == 0u && flushed.HasUses(); + }); + + passed &= run("a sign change or absolute value of a flushed value needs no flush", [] { + Builder b; + auto& first = b.flush(b.unknown()); + auto& negated = b.flush(b.emit(IrOpcode::BitwiseXor32, IrType::U32, {&first, &b.constant(0x80000000u)})); + auto& absolute = b.flush(b.emit(IrOpcode::BitwiseAnd32, IrType::U32, {&b.constant(0x7fffffffu), &first})); + b.keep(negated); + b.keep(absolute); + return b.eliminate() == 2u && !negated.HasUses() && !absolute.HasUses(); + }); + + passed &= run("a select of flushed values needs no flush, one with an unknown arm does", [] { + Builder b; + auto& condition = b.emit(IrOpcode::ULessThan32, IrType::Bool, {&b.unknown(), &b.constant(16u)}); + auto& both = b.flush(b.emit(IrOpcode::SelectU32, IrType::U32, {&condition, &b.flush(b.unknown()), &b.flush(b.unknown())})); + auto& mixed = b.flush(b.emit(IrOpcode::SelectU32, IrType::U32, {&condition, &b.flush(b.unknown()), &b.unknown()})); + b.keep(both); + b.keep(mixed); + return b.eliminate() == 1u && !both.HasUses() && mixed.HasUses(); + }); + + passed &= run("normal and zero constants need no flush, a denormal constant does", [] { + Builder b; + auto& one = b.flush(b.constant(0x3f800000u)); + auto& negativeZero = b.flush(b.constant(0x80000000u)); + auto& denormal = b.flush(b.constant(0x00000001u)); + b.keep(one); + b.keep(negativeZero); + b.keep(denormal); + return b.eliminate() == 2u && !one.HasUses() && !negativeZero.HasUses() && denormal.HasUses(); + }); + + passed &= run("an integer or f16 converted to f32 needs no flush", [] { + Builder b; + auto& integer = b.flush(b.emit(IrOpcode::BitCastU32F32, IrType::U32, {&b.emit(IrOpcode::ConvertF32U32, IrType::F32, {&b.unknown()})})); + auto& half = b.flush(b.emit(IrOpcode::BitCastU32F32, IrType::U32, {&b.emit(IrOpcode::ConvertF32F16, IrType::F32, {&b.emit(IrOpcode::BitCastF16U16, IrType::F16, {&b.emit(IrOpcode::ConvertU16U32, IrType::U16, {&b.unknown()})})})})); + b.keep(integer); + b.keep(half); + return b.eliminate() == 2u && !integer.HasUses() && !half.HasUses(); + }); + + passed &= run("a clamped flushed value needs no flush, an arithmetic result does", [] { + Builder b; + auto& first = b.emit(IrOpcode::BitCastF32U32, IrType::F32, {&b.flush(b.unknown())}); + auto& clamped = b.flush(b.emit(IrOpcode::BitCastU32F32, IrType::U32, {&b.emit(IrOpcode::FPSaturate32, IrType::F32, {&first})})); + auto& product = b.flush(b.emit(IrOpcode::BitCastU32F32, IrType::U32, {&b.emit(IrOpcode::FPMul32, IrType::F32, {&first, &first})})); + b.keep(clamped); + b.keep(product); + return b.eliminate() == 1u && !clamped.HasUses() && product.HasUses(); + }); + + passed &= run("a phi of flushed values needs no flush", [] { + Builder b; + IrBlock& head = *b.block; + IrBlock& body = b.newBlock(); + IrBlock& merge = b.newBlock(); + head.AddBranch(&body); + head.AddBranch(&merge); + body.AddBranch(&merge); + auto& fromHead = b.flush(b.unknown()); + b.block = &body; + auto& fromBody = b.flush(b.unknown()); + b.block = &merge; + auto& merged = b.flush(b.phi(merge, {{&head, &fromHead}, {&body, &fromBody}})); + b.keep(merged); + return b.eliminate() == 1u && !merged.HasUses(); + }); + + passed &= run("a loop phi needs no flush when every value it takes is flushed, and keeps it otherwise", [] { + const auto loop = [](bool flushedBackEdge) { + Builder b; + IrBlock& head = *b.block; + IrBlock& body = b.newBlock(); + IrBlock& exit = b.newBlock(); + head.AddBranch(&body); + body.AddBranch(&body); + body.AddBranch(&exit); + auto& initial = b.flush(b.unknown()); + b.block = &body; + auto& carried = b.program.CreateValue(IrOpcode::Phi, IrType::U32); + body.AppendInstruction(&carried); + auto& next = b.flush(carried); + auto& backEdge = flushedBackEdge ? next : b.emit(IrOpcode::BitwiseXor32, IrType::U32, {&b.unknown(), &next}); + carried.AddPhiOperand(&head, &initial); + carried.AddPhiOperand(&body, &backEdge); + b.block = &exit; + b.keep(next); + b.eliminate(); + return !next.HasUses(); + }; + return loop(true) && !loop(false); + }); + + return passed ? 0 : 1; +} diff --git a/core/libs/prx/libSceAgcDriver/tests/DepthSurfaceReuse.cpp b/core/libs/prx/libSceAgcDriver/tests/DepthSurfaceReuse.cpp new file mode 100644 index 000000000..b91361cea --- /dev/null +++ b/core/libs/prx/libSceAgcDriver/tests/DepthSurfaceReuse.cpp @@ -0,0 +1,181 @@ +#include "GraphicsTests.hpp" +#include "prx/libSceAgcDriver/Graphics/include/DepthSurface.hpp" +#include "prx/libSceAgcDriver/Graphics/include/GuestTextureResource.hpp" +#include "prx/libSceAgcDriver/Graphics/include/Texture.hpp" +#include +#include +#include +#include +#include + +namespace { + +using namespace AgcDriver::Graphics; + +constexpr std::uint32_t FormatR16Unorm = 7; +constexpr std::uint32_t FormatR16Uint = 11; +constexpr std::uint32_t FormatR32Uint = 20; +constexpr std::uint32_t FormatR32Float = 22; +constexpr std::uint32_t FormatR11G11B10Float = 36; +constexpr std::uint32_t TileDepth64KB = 0x18; +constexpr std::uint32_t TileRenderTarget64KB = 0x1b; +constexpr std::uint32_t Type2D = 9; +constexpr std::uint32_t Type2DArray = 13; + +std::uintptr_t nextHandle = 0x1000; + +template +THandle newHandle() { + return reinterpret_cast(nextHandle += 0x10); +} + +VKAPI_ATTR VkResult VKAPI_CALL createImage(VkDevice, const VkImageCreateInfo*, const VkAllocationCallbacks*, VkImage* image) { + *image = newHandle(); + return VK_SUCCESS; +} + +VKAPI_ATTR void VKAPI_CALL imageMemoryRequirements(VkDevice, VkImage, VkMemoryRequirements* requirements) { + *requirements = {65536, 65536, 1}; +} + +VKAPI_ATTR VkResult VKAPI_CALL allocateMemory(VkDevice, const VkMemoryAllocateInfo*, const VkAllocationCallbacks*, VkDeviceMemory* memory) { + *memory = newHandle(); + return VK_SUCCESS; +} + +VKAPI_ATTR VkResult VKAPI_CALL bindImageMemory(VkDevice, VkImage, VkDeviceMemory, VkDeviceSize) { + return VK_SUCCESS; +} + +VKAPI_ATTR VkResult VKAPI_CALL createImageView(VkDevice, const VkImageViewCreateInfo*, const VkAllocationCallbacks*, VkImageView* view) { + *view = newHandle(); + return VK_SUCCESS; +} + +VKAPI_ATTR VkResult VKAPI_CALL allocateCommandBuffers(VkDevice, const VkCommandBufferAllocateInfo*, VkCommandBuffer* commands) { + *commands = newHandle(); + return VK_SUCCESS; +} + +VKAPI_ATTR VkResult VKAPI_CALL createFence(VkDevice, const VkFenceCreateInfo*, const VkAllocationCallbacks*, VkFence* fence) { + *fence = newHandle(); + return VK_SUCCESS; +} + +VKAPI_ATTR VkResult VKAPI_CALL beginCommandBuffer(VkCommandBuffer, const VkCommandBufferBeginInfo*) { + return VK_SUCCESS; +} + +VKAPI_ATTR VkResult VKAPI_CALL endCommandBuffer(VkCommandBuffer) { + return VK_SUCCESS; +} + +VKAPI_ATTR VkResult VKAPI_CALL queueSubmit(VkQueue, std::uint32_t, const VkSubmitInfo*, VkFence) { + return VK_SUCCESS; +} + +VKAPI_ATTR VkResult VKAPI_CALL waitForFences(VkDevice, std::uint32_t, const VkFence*, VkBool32, std::uint64_t) { + return VK_SUCCESS; +} + +VKAPI_ATTR VkResult VKAPI_CALL getFenceStatus(VkDevice, VkFence) { + return VK_SUCCESS; +} + +VKAPI_ATTR void VKAPI_CALL freeCommandBuffers(VkDevice, VkCommandPool, std::uint32_t, const VkCommandBuffer*) {} +VKAPI_ATTR void VKAPI_CALL destroyFence(VkDevice, VkFence, const VkAllocationCallbacks*) {} +VKAPI_ATTR void VKAPI_CALL destroyImageView(VkDevice, VkImageView, const VkAllocationCallbacks*) {} +VKAPI_ATTR void VKAPI_CALL destroyImage(VkDevice, VkImage, const VkAllocationCallbacks*) {} +VKAPI_ATTR void VKAPI_CALL freeMemory(VkDevice, VkDeviceMemory, const VkAllocationCallbacks*) {} +VKAPI_ATTR void VKAPI_CALL cmdPipelineBarrier(VkCommandBuffer, VkPipelineStageFlags, VkPipelineStageFlags, VkDependencyFlags, std::uint32_t, const VkMemoryBarrier*, std::uint32_t, const VkBufferMemoryBarrier*, std::uint32_t, const VkImageMemoryBarrier*) {} +VKAPI_ATTR void VKAPI_CALL cmdClearDepthStencilImage(VkCommandBuffer, VkImage, VkImageLayout, const VkClearDepthStencilValue*, std::uint32_t, const VkImageSubresourceRange*) {} + +VKAPI_ATTR void VKAPI_CALL formatProperties(VkPhysicalDevice, VkFormat, VkFormatProperties* properties) { + *properties = {}; + properties->optimalTilingFeatures = VK_FORMAT_FEATURE_DEPTH_STENCIL_ATTACHMENT_BIT | VK_FORMAT_FEATURE_SAMPLED_IMAGE_BIT; +} + +PFN_vkVoidFunction VKAPI_CALL deviceProc(VkDevice, const char* name) { + static const std::map table{ + {"vkCreateImage", reinterpret_cast(createImage)}, + {"vkGetImageMemoryRequirements", reinterpret_cast(imageMemoryRequirements)}, + {"vkAllocateMemory", reinterpret_cast(allocateMemory)}, + {"vkBindImageMemory", reinterpret_cast(bindImageMemory)}, + {"vkCreateImageView", reinterpret_cast(createImageView)}, + {"vkAllocateCommandBuffers", reinterpret_cast(allocateCommandBuffers)}, + {"vkCreateFence", reinterpret_cast(createFence)}, + {"vkBeginCommandBuffer", reinterpret_cast(beginCommandBuffer)}, + {"vkEndCommandBuffer", reinterpret_cast(endCommandBuffer)}, + {"vkQueueSubmit", reinterpret_cast(queueSubmit)}, + {"vkWaitForFences", reinterpret_cast(waitForFences)}, + {"vkGetFenceStatus", reinterpret_cast(getFenceStatus)}, + {"vkFreeCommandBuffers", reinterpret_cast(freeCommandBuffers)}, + {"vkDestroyFence", reinterpret_cast(destroyFence)}, + {"vkDestroyImageView", reinterpret_cast(destroyImageView)}, + {"vkDestroyImage", reinterpret_cast(destroyImage)}, + {"vkFreeMemory", reinterpret_cast(freeMemory)}, + {"vkCmdPipelineBarrier", reinterpret_cast(cmdPipelineBarrier)}, + {"vkCmdClearDepthStencilImage", reinterpret_cast(cmdClearDepthStencilImage)} + }; + const auto it = table.find(name); + return it == table.end() ? nullptr : it->second; +} + +struct View { + std::uint64_t address; + std::uint32_t format; + std::uint32_t width; + std::uint32_t height; + std::uint32_t tileMode = TileDepth64KB; + std::uint32_t type = Type2D; + std::uint32_t lastArray = 0; +}; + +std::array tsharp(const View& view) { + std::array words{}; + const auto base = view.address >> 8u; + const auto widthMinus1 = view.width - 1u; + const auto heightMinus1 = view.height - 1u; + words[0] = static_cast(base); + words[1] = static_cast((base >> 32u) & 0xffu) | ((view.format & 0x1ffu) << 20u) | ((widthMinus1 & 0x3u) << 30u); + words[2] = ((widthMinus1 >> 2u) & 0xfffu) | ((heightMinus1 & 0x3fffu) << 14u); + words[3] = 4u | (5u << 3u) | (6u << 6u) | (7u << 9u) | ((view.tileMode & 0x1fu) << 20u) | ((view.type & 0xfu) << 28u); + words[4] = view.lastArray & 0x1fffu; + return words; +} + +std::shared_ptr lookup(const Context& context, const View& view) { + const auto words = tsharp(view); + return DepthSurfaceTexture(context, words, DecodeTextureResource(words), VkComponentMapping{}); +} + +} + +void RunDepthSurfaceReuseTests() { + static int deviceTag = 0; + Context context{}; + context.device = reinterpret_cast(&deviceTag); + context.deviceProc = deviceProc; + context.formatProperties = formatProperties; + context.memory.memoryTypeCount = 1; + context.memory.memoryTypes[0].propertyFlags = VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT; + context.limits.maxFramebufferWidth = 16384; + context.limits.maxFramebufferHeight = 16384; + constexpr std::uint64_t depth32 = 0x40000000; + constexpr std::uint64_t depth16 = 0x50000000; + DepthSurfaceView(context, {depth32, 0, {384, 384}, VK_FORMAT_D32_SFLOAT, 1.0f, 0}); + DepthSurfaceView(context, {depth16, 0, {256, 256}, VK_FORMAT_D16_UNORM, 1.0f, 0}); + + Require(lookup(context, {depth32, FormatR32Float, 384, 384}) != nullptr, "an R32F view of a D32 surface's own extent must sample its depth plane"); + Require(lookup(context, {depth16, FormatR16Unorm, 256, 256}) != nullptr, "an R16 view of a D16 surface's own extent must sample its depth plane"); + Require(lookup(context, {depth32, FormatR32Uint, 384, 384}) != nullptr, "a 32-bit raw depth bits view of a D32 surface must sample its depth plane"); + Require(lookup(context, {depth16, FormatR16Uint, 256, 256}) != nullptr, "a 16-bit raw depth bits view of a D16 surface must sample its depth plane"); + + Require(lookup(context, {depth32, FormatR32Float, 192, 192}) == nullptr, "a view of another extent over a depth surface must be read as reused memory"); + Require(lookup(context, {depth32, FormatR11G11B10Float, 384, 384}) == nullptr, "an R11G11B10 view of a D32 surface's own extent must be read as reused memory"); + Require(lookup(context, {depth32, FormatR32Float, 512, 512, TileDepth64KB, Type2DArray, 5}) == nullptr, "a 2D array view of another extent over a depth surface must be read as reused memory"); + Require(lookup(context, {depth32, FormatR32Uint, 384, 384, TileRenderTarget64KB}) == nullptr, "an R32 uint view without a depth layout must be read as reused memory"); + Require(lookup(context, {depth16, FormatR32Float, 256, 256}) == nullptr, "an R32F view of a D16 surface's own extent must be read as reused memory"); + + ClearDepthSurfaces(context.device); +} diff --git a/core/libs/prx/libSceAgcDriver/tests/EmbeddedVertexFetch.cpp b/core/libs/prx/libSceAgcDriver/tests/EmbeddedVertexFetch.cpp new file mode 100644 index 000000000..46d1e6a89 --- /dev/null +++ b/core/libs/prx/libSceAgcDriver/tests/EmbeddedVertexFetch.cpp @@ -0,0 +1,47 @@ +#include "RdnaDecoder/RdnaInstructionDecoder.hpp" +#include "Translation/EmbeddedVertexFetch.hpp" +#include +#include +#include +#include + +using namespace ShaderRecompiler; + +namespace { + +constexpr std::uint32_t AttributeTable = 0; +constexpr std::uint32_t BufferTable = 2; + +std::int32_t FetchedAttribute(std::uint32_t source, std::uint32_t field) { + const std::array code{ + 0xbe9403ffu, source, + 0xbe9503ffu, field, + 0x93961514u, + 0xf4080405u, 0x2c000000u, + 0x02140b08u, + 0xe00c2000u, 0x80040b0au, + 0xbf810000u, + }; + const auto program = RdnaInstructionDecoder{}.Decode(code); + const auto plan = EmbeddedVertexFetchAnalyzer{}.Analyze(program, AttributeTable, BufferTable, 0, 16, 64); + return plan.loads.size() == 1 ? plan.loads.front().attributeId : -1; +} + +int Expect(const char* name, std::uint32_t source, std::uint32_t field, std::uint32_t offset) { + const auto expected = static_cast(offset / 16u); + const auto actual = FetchedAttribute(source, field); + if (actual == expected) return 0; + std::fprintf(stderr, "%s: s_bfe_u32 0x%08x, 0x%08x gives buffer slot %d, expected %d\n", name, source, field, actual, expected); + return 1; +} + +} + +int main() { + int failures = 0; + failures += Expect("width 30 at offset 0", 0x9e000000u, 0x001e0000u, 0x1e000000u); + failures += Expect("width 8 at offset 4", 0xfff00320u, 0x00080004u, 0x32u); + failures += Expect("width 0", 0x00000320u, 0x00000000u, 0u); + failures += Expect("width above 32", 0x00000320u, 0x007f0000u, 0x320u); + return failures == 0 ? 0 : 1; +} diff --git a/core/libs/prx/libSceAgcDriver/tests/Flip.cpp b/core/libs/prx/libSceAgcDriver/tests/Flip.cpp index da4fb199c..4ce4faa60 100644 --- a/core/libs/prx/libSceAgcDriver/tests/Flip.cpp +++ b/core/libs/prx/libSceAgcDriver/tests/Flip.cpp @@ -35,6 +35,8 @@ struct State { bool entered = false; bool fail = false; bool checkSelfWait = false; + bool gate = false; + bool waiting = false; std::atomic alive = 0; std::atomic ready = 0; std::atomic failed = 0; @@ -80,10 +82,24 @@ private: std::shared_ptr state; }; +class RenderingGate final : public AgcDriver::IRenderingWait { +public: + explicit RenderingGate(std::shared_ptr value) : state(std::move(value)) {} + void Wait() override { + std::unique_lock lock(state->mutex); + state->waiting = true; + state->changed.notify_all(); + state->changed.wait(lock, [&] { return !state->gate; }); + } +private: + std::shared_ptr state; +}; + class Output final : public AgcDriver::IVideoOutput { public: void Fail(std::exception_ptr error) noexcept override { if (!error) std::terminate(); } std::shared_ptr state = std::make_shared(); + std::shared_ptr CaptureRenderingWait(std::uint32_t) override { return std::make_shared(state); } std::shared_ptr Reserve(const AgcDriver::FlipInfo& info) override { std::lock_guard lock(state->mutex); state->last = info; @@ -165,6 +181,90 @@ void testFlipAndBoundary() { AgcDriverUnregisterVideoOutput_nid_postfix(7, replacement); } +void testFlipHold() { + auto output = std::make_shared(); + AgcDriverRegisterVideoOutput_nid_postfix(7, output); + { + std::lock_guard lock(output->state->mutex); + output->state->gate = true; + } + struct GateRelease { + std::shared_ptr state; + ~GateRelease() { + { + std::lock_guard lock(state->mutex); + state->gate = false; + } + state->changed.notify_all(); + } + } gateRelease{output->state}; + std::array gated{0xc0021018, 7, 0, 0, 0xc004105c, 7, 0xfffffffeu, 1, 0, 0}; + Packet gatedFrame{gated.data(), static_cast(gated.size()), 0, {}}; + check(sceAgcDriverSubmitDcb(&gatedFrame) == 0, "gated flip submission failed"); + { + std::unique_lock lock(output->state->mutex); + check(output->state->changed.wait_for(lock, std::chrono::seconds(5), [&] { return output->state->waiting; }), "worker did not reach the rendering wait"); + } + auto held = std::async(std::launch::async, [] { submitFlip(); }); + check(held.wait_for(std::chrono::milliseconds(200)) == std::future_status::timeout, "a flip was accepted before the worker reached the previous one"); + check(output->state->ready == 0, "a flip passed a closed rendering wait"); + { + std::lock_guard lock(output->state->mutex); + output->state->gate = false; + } + output->state->changed.notify_all(); + check(held.wait_for(std::chrono::seconds(5)) == std::future_status::ready, "a flip stayed held after the worker reached the previous one"); + held.get(); + AgcDriverWaitIdle_nid_postfix(); + check(output->state->ready == 2, "held flips were lost"); + alignas(4) static std::atomic label = 0; + struct LabelRelease { + ~LabelRelease() { label.store(1, std::memory_order_release); } + } labelRelease; + const auto labelAddress = reinterpret_cast(&label); + std::array waiting{0xc0053c00, 0x13, static_cast(labelAddress), static_cast(static_cast(labelAddress) >> 32u), 1, 0xffffffffu, 0x19, 0xc004105c, 7, 0xfffffffeu, 1, 0, 0}; + Packet waitingFrame{waiting.data(), static_cast(waiting.size()), 0, {}}; + check(sceAgcDriverSubmitDcb(&waitingFrame) == 0, "waiting flip submission failed"); + auto released = std::async(std::launch::async, [] { submitFlip(); }); + check(released.wait_for(std::chrono::seconds(5)) == std::future_status::ready, "a flip was held while the worker waited for a label only the title stores"); + released.get(); + check(output->state->ready == 2, "a flip passed an unsatisfied wait"); + label.store(1, std::memory_order_release); + AgcDriverWaitIdle_nid_postfix(); + check(output->state->ready == 4, "flips behind a title-stored label were lost"); + { + std::lock_guard lock(output->state->mutex); + output->state->gate = true; + output->state->waiting = false; + } + alignas(4) static std::atomic later = 0; + struct LaterRelease { + ~LaterRelease() { later.store(1, std::memory_order_release); } + } laterRelease; + const auto laterAddress = reinterpret_cast(&later); + std::array gatedWait{0xc0021018, 7, 0, 0, 0xc0053c00, 0x13, static_cast(laterAddress), static_cast(static_cast(laterAddress) >> 32u), 1, 0xffffffffu, 0x19, 0xc004105c, 7, 0xfffffffeu, 1, 0, 0}; + Packet gatedWaitFrame{gatedWait.data(), static_cast(gatedWait.size()), 0, {}}; + check(sceAgcDriverSubmitDcb(&gatedWaitFrame) == 0, "gated waiting flip submission failed"); + { + std::unique_lock lock(output->state->mutex); + check(output->state->changed.wait_for(lock, std::chrono::seconds(5), [&] { return output->state->waiting; }), "worker did not reach the second rendering wait"); + } + auto progressed = std::async(std::launch::async, [] { submitFlip(); }); + check(progressed.wait_for(std::chrono::milliseconds(200)) == std::future_status::timeout, "a flip was accepted while the worker was held before a label wait"); + { + std::lock_guard lock(output->state->mutex); + output->state->gate = false; + } + output->state->changed.notify_all(); + check(progressed.wait_for(std::chrono::seconds(5)) == std::future_status::ready, "a flip stayed held after the worker went on to wait for a label only the title stores"); + progressed.get(); + check(output->state->ready == 4, "a flip passed an unsatisfied wait after the rendering wait"); + later.store(1, std::memory_order_release); + AgcDriverWaitIdle_nid_postfix(); + check(output->state->ready == 6, "flips behind a later title-stored label were lost"); + AgcDriverUnregisterVideoOutput_nid_postfix(7, output); +} + void testFailure() { auto output = std::make_shared(); output->state->fail = true; @@ -201,7 +301,7 @@ void testReset(bool compute) { int main(int argc, char** argv) { try { if (argc == 2) testReset(std::string(argv[1]) == "compute"); - else { testFlipAndBoundary(); testFailure(); } + else { testFlipAndBoundary(); testFlipHold(); testFailure(); } const auto shutdown = expectFailure([] { LibcRunShutdown_nid_postfix(); }); check(shutdown.find(argc == 2 ? (std::string(argv[1]) == "compute" ? "not readable" : "required shader register") : "intentional flip failure") != std::string::npos, "shutdown lost worker failure"); std::puts("AGC flip and suspend tests passed"); diff --git a/core/libs/prx/libSceAgcDriver/tests/Float16Conversion.cpp b/core/libs/prx/libSceAgcDriver/tests/Float16Conversion.cpp new file mode 100644 index 000000000..4035a5849 --- /dev/null +++ b/core/libs/prx/libSceAgcDriver/tests/Float16Conversion.cpp @@ -0,0 +1,97 @@ +#include "Recompiler.hpp" +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace { + +using namespace ShaderRecompiler; + +void require(bool condition, const std::string& message) { + if (!condition) throw std::runtime_error(message); +} + +constexpr std::array CvtF16F32Store{0x7e021500u, 0xe0700000u, 0x80000100u, 0xbf810000u}; + +std::vector recompile(std::span capabilities) { + const std::array userData{0x10000000u, 0x00100000u, 0x40u, 0x00027facu}; + const std::array extensions{"SPV_KHR_storage_buffer_storage_class"}; + RecompileRequest request{}; + request.shader = {ShaderStage::Compute, 0x20000u, CvtF16F32Store, 0, {}}; + request.context.waveSize = 64; + request.context.userDataBaseRegister = 0; + request.context.userData = userData; + request.context.compute = ShaderComputeStageInfo{{64u, 1u, 1u}, 0u, {false, false, false}, false, 1u}; + request.target.vulkanVersion = 0x00403000u; + request.target.spirvVersion = 0x00010600u; + request.target.subgroupSize = 64; + request.target.bdaAbiVersion = 1; + request.target.supportedCapabilities = capabilities; + request.target.supportedExtensions = extensions; + request.target.maxWorkgroupSize = {1024u, 1024u, 64u}; + request.target.maxWorkgroupInvocations = 1024; + request.target.maxWorkgroupSharedMemoryBytes = 49152; + request.layout = {0, 0, 0, 128}; + request.useCache = false; + auto result = Recompile(request); + require(!result.spirv.empty(), "v_cvt_f16_f32 did not recompile"); + return std::move(result.spirv); +} + +struct Usage { + std::size_t floatConverts = 0; + bool float16Capability = false; + bool roundingModeRte16 = false; + bool denormPreserve16 = false; +}; + +Usage scan(std::span words) { + Usage usage; + for (std::size_t cursor = 5; cursor < words.size();) { + const auto count = words[cursor] >> 16u; + const auto op = static_cast(words[cursor] & 0xffffu); + require(count != 0 && cursor + count <= words.size(), "truncated SPIR-V instruction"); + if (op == spv::OpFConvert) ++usage.floatConverts; + if (op == spv::OpCapability && words[cursor + 1] == spv::CapabilityFloat16) usage.float16Capability = true; + if (op == spv::OpExecutionMode && count == 4 && words[cursor + 3] == 16u) { + if (words[cursor + 2] == spv::ExecutionModeRoundingModeRTE) usage.roundingModeRte16 = true; + if (words[cursor + 2] == spv::ExecutionModeDenormPreserve) usage.denormPreserve16 = true; + } + cursor += count; + } + return usage; +} + +void verifyNative() { + const std::array capabilities{spv::CapabilityShader, spv::CapabilityGroupNonUniform, spv::CapabilityFloat16, spv::CapabilityRoundingModeRTE, spv::CapabilityDenormPreserve}; + const auto usage = scan(recompile(capabilities)); + require(usage.floatConverts != 0 && usage.float16Capability, "a device that rounds 16-bit floats to nearest even did not get OpFConvert to f16"); + require(usage.roundingModeRte16 && usage.denormPreserve16, "native f16 conversion without the 16-bit RoundingModeRTE and DenormPreserve execution modes"); +} + +void verifyEmulated() { + const std::array withoutRte{spv::CapabilityShader, spv::CapabilityGroupNonUniform, spv::CapabilityFloat16}; + const auto usage = scan(recompile(withoutRte)); + require(usage.floatConverts == 0 && !usage.float16Capability && !usage.roundingModeRte16, "a device without RoundingModeRTE for 16-bit floats got native f16 conversion"); +} + +} + +int main() { + try { + verifyNative(); + verifyEmulated(); + } catch (const std::exception& error) { + std::fprintf(stderr, "Float16Conversion: %s\n", error.what()); + return 1; + } + std::puts("Float16Conversion: native and emulated f16 conversion selected by device capabilities"); + return 0; +} diff --git a/core/libs/prx/libSceAgcDriver/tests/FormattedBufferOnce.cpp b/core/libs/prx/libSceAgcDriver/tests/FormattedBufferOnce.cpp new file mode 100644 index 000000000..d4ae978ef --- /dev/null +++ b/core/libs/prx/libSceAgcDriver/tests/FormattedBufferOnce.cpp @@ -0,0 +1,108 @@ +#include "BdaAbi.hpp" +#include "Recompiler.hpp" +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace { + +using namespace ShaderRecompiler; + +void Check(bool condition, const std::string& message) { + if (!condition) throw std::runtime_error(message); +} + +SpirvTarget Target() { + static constexpr std::array capabilities{ + spv::CapabilityInt64, spv::CapabilityPhysicalStorageBufferAddresses, spv::CapabilityStorageBuffer8BitAccess, + }; + static constexpr std::array extensions{ + "SPV_EXT_descriptor_indexing", "SPV_KHR_8bit_storage", "SPV_KHR_float_controls", "SPV_KHR_physical_storage_buffer", + }; + return {0x00401000u, 0x00010300u, 32, BdaAbi::Version, capabilities, extensions, false, + {1024, 1024, 64}, 1024, 32768, {}, {}}; +} + +struct Counts { + std::size_t words = 0; + std::size_t switches = 0; + std::size_t calls = 0; + std::size_t functions = 0; + std::size_t notInlined = 0; +}; + +Counts Count(std::span words) { + Check(words.size() >= 5 && words[0] == spv::MagicNumber, "formatted buffer once: missing SPIR-V module"); + Counts counts{words.size()}; + for (std::size_t offset = 5; offset < words.size();) { + const auto count = words[offset] >> spv::WordCountShift; + Check(count != 0 && count <= words.size() - offset, "formatted buffer once: malformed SPIR-V instruction"); + const auto opcode = words[offset] & spv::OpCodeMask; + if (opcode == spv::OpSwitch) ++counts.switches; + if (opcode == spv::OpFunctionCall) ++counts.calls; + if (opcode == spv::OpFunction) { + ++counts.functions; + if (count > 3 && (words[offset + 3] & spv::FunctionControlDontInlineMask) != 0) ++counts.notInlined; + } + offset += count; + } + return counts; +} + +Counts Compile(std::size_t loads, bool formattedStores = false, bool gpu = false) { + std::vector code; + if (gpu) code.push_back(0x7e160500u); + for (std::uint32_t load = 0; load < loads; ++load) code.insert(code.end(), {0xe0000000u | (load * 4u), 0x80020000u | ((10u + load) << 8u)}); + code.push_back(0xbf8c3f70u); + for (std::uint32_t load = 0; load < loads; ++load) code.insert(code.end(), {(formattedStores ? 0xe0100000u : 0xe0700000u) | (load * 4u), 0x80010000u | ((10u + load) << 8u)}); + code.push_back(0xbf810000u); + const std::span words(code); + const std::array memory{{{reinterpret_cast(code.data()), std::as_bytes(words)}}}; + const std::array userData{0u, 0u, 0u, 0u, 0x10000000u, 0u, 0x1000u, 0x31016facu, 0x20000000u, 0u, 0x1000u, 0x31038facu}; + RecompileRequest request{}; + request.shader = {ShaderStage::Compute, reinterpret_cast(code.data()), words, 0, {}}; + request.target = Target(); + request.layout = {0, 0, 0, 128}; + request.context.waveSize = 32; + request.context.userDataBaseRegister = 0; + request.context.userData = userData; + request.context.compute = ShaderComputeStageInfo{{32u, 1u, 1u}, 0u, {false, false, false}, false, 1u}; + request.context.memory = memory; + request.useCache = false; + return Count(Recompile(request).spirv.Words()); +} + +} + +int main() { + try { + const auto none = Compile(0); + const auto one = Compile(1); + const auto three = Compile(3); + Check(one.calls == none.calls + 1 && one.functions == none.functions + 1, "formatted buffer once: one formatted load emitted " + std::to_string(one.calls - none.calls) + " calls and " + std::to_string(one.functions - none.functions) + " functions more than none, expected one call to one shared function"); + Check(three.calls == one.calls + 2 && three.functions == one.functions, "formatted buffer once: two more formatted loads from the same descriptor emitted " + std::to_string(three.calls - one.calls) + " calls and " + std::to_string(three.functions - one.functions) + " functions more, expected two calls and no function"); + Check(three.switches == one.switches, "formatted buffer once: two more formatted loads from the same descriptor emitted " + std::to_string(three.switches - one.switches) + " more format switches"); + Check(one.notInlined == none.notInlined + 1, "formatted buffer once: the shared formatted access function may be inlined back into its callers"); + const auto storeOne = Compile(1, true); + const auto storeThree = Compile(3, true); + Check(storeOne.functions == none.functions + 2 && storeOne.calls == none.calls + 2, "formatted buffer once: a formatted load and a formatted store to another resource emitted " + std::to_string(storeOne.functions - none.functions) + " functions and " + std::to_string(storeOne.calls - none.calls) + " calls more than none, expected one function and one call each"); + Check(storeThree.functions == storeOne.functions && storeThree.calls == storeOne.calls + 4 && storeThree.switches == storeOne.switches, "formatted buffer once: two more formatted loads and stores emitted " + std::to_string(storeThree.functions - storeOne.functions) + " functions, " + std::to_string(storeThree.calls - storeOne.calls) + " calls and " + std::to_string(storeThree.switches - storeOne.switches) + " switches more, expected four calls only"); + Check(three.words - one.words < one.words / 10u, "formatted buffer once: 2 more formatted loads added " + std::to_string(three.words - one.words) + " SPIR-V words to a " + std::to_string(one.words) + "-word module"); + const auto gpuOne = Compile(1, false, true); + const auto gpuThree = Compile(3, false, true); + Check(gpuThree.functions == gpuOne.functions && gpuThree.calls > gpuOne.calls, "formatted buffer once: two more formatted loads from a GPU-selected descriptor emitted " + std::to_string(gpuThree.functions - gpuOne.functions) + " more functions, expected calls to the shared one"); + Check(gpuThree.switches == gpuOne.switches, "formatted buffer once: two more formatted loads from a GPU-selected descriptor emitted " + std::to_string(gpuThree.switches - gpuOne.switches) + " more format switches"); + Check(gpuThree.words - gpuOne.words < gpuOne.words / 10u, "formatted buffer once: 2 more formatted loads from a GPU-selected descriptor added " + std::to_string(gpuThree.words - gpuOne.words) + " SPIR-V words to a " + std::to_string(gpuOne.words) + "-word module"); + std::printf("formatted buffer once tests passed (%zu words for 1 load, %zu for 3; GPU-selected %zu and %zu)\n", one.words, three.words, gpuOne.words, gpuThree.words); + return 0; + } catch (const std::exception& error) { + std::fprintf(stderr, "%s\n", error.what()); + return 1; + } +} diff --git a/core/libs/prx/libSceAgcDriver/tests/FragmentBarycentric.cpp b/core/libs/prx/libSceAgcDriver/tests/FragmentBarycentric.cpp new file mode 100644 index 000000000..6b67d8629 --- /dev/null +++ b/core/libs/prx/libSceAgcDriver/tests/FragmentBarycentric.cpp @@ -0,0 +1,33 @@ +#include "prx/libSceAgcDriver/Execution/include/FragmentBarycentric.hpp" +#include + +namespace { + +struct Case { + VkDriverId driver; + bool reported; + bool usable; +}; + +constexpr Case Cases[]{ + {VK_DRIVER_ID_MOLTENVK, true, false}, + {VK_DRIVER_ID_MOLTENVK, false, false}, + {VK_DRIVER_ID_MESA_RADV, true, true}, + {VK_DRIVER_ID_MESA_RADV, false, false}, + {VK_DRIVER_ID_NVIDIA_PROPRIETARY, true, true}, + {VK_DRIVER_ID_AMD_PROPRIETARY, true, true}, + {VK_DRIVER_ID_INTEL_PROPRIETARY_WINDOWS, false, false}, +}; + +} + +int main() { + int failures = 0; + for (const auto& item : Cases) { + if (AgcDriver::FragmentShaderBarycentricUsable(item.driver, item.reported) != item.usable) { + std::fprintf(stderr, "driver %d reported %d: expected %s fragment barycentric\n", static_cast(item.driver), item.reported ? 1 : 0, item.usable ? "usable" : "unusable"); + ++failures; + } + } + return failures == 0 ? 0 : 1; +} diff --git a/core/libs/prx/libSceAgcDriver/tests/Graphics.cpp b/core/libs/prx/libSceAgcDriver/tests/Graphics.cpp index a035d3bf6..ee6645aad 100644 --- a/core/libs/prx/libSceAgcDriver/tests/Graphics.cpp +++ b/core/libs/prx/libSceAgcDriver/tests/Graphics.cpp @@ -1,20 +1,27 @@ #include "BdaTests.hpp" #include "GraphicsTests.hpp" +#include "prx/libSceAgcDriver/Execution/include/GuestMemory.hpp" +#include "prx/libSceAgcDriver/Graphics/include/Recorder.hpp" #include "prx/libSceAgcDriver/Graphics/include/Pipeline.hpp" #include "prx/libSceAgcDriver/Graphics/include/TextureDetiler.hpp" #include "prx/libSceAgcDriver/Graphics/include/VertexInput.hpp" #include "prx/libSceAgcDriver/Graphics/include/Draw.hpp" +#include "prx/libSceAgcDriver/Graphics/include/BufferPool.hpp" #include "prx/libSceAgcDriver/Graphics/include/ShaderInputState.hpp" #include "prx/libSceAgcDriver/Graphics/include/DccMetadata.hpp" +#include "prx/libSceAgcDriver/Graphics/include/ParallelCompare.hpp" +#include "prx/libSceAgcDriver/Graphics/include/DepthSurface.hpp" #include "prx/libSceAgcDriver/Graphics/include/Texture.hpp" #include "SceShaders.hpp" #include "ControlFlow/RequestSerializer.hpp" #include "CacheKey.hpp" +#include "Optimization/ResourceProgram.hpp" #include "BdaAbi.hpp" #include #include #include #include +#include #include #include #include @@ -27,6 +34,7 @@ #include #include #include +#include #include namespace { @@ -66,6 +74,8 @@ AgcDriver::QueueState makeState() { queue.context[0x114] = 0; queue.context[0xb4] = 0; queue.context[0xb5] = std::bit_cast(1.0f); + queue.shader[0x008] = 0x100; + queue.shader[0x009] = 0; return queue; } @@ -123,9 +133,26 @@ void stateTests() { queue.context[0x293] = 0x06020000u; (void)AgcDriver::Graphics::DecodeState(queue); Require(AgcDriver::Graphics::DrawRejection(queue, false).find("sample iteration") == std::string::npos, "per-engine primitive discard was rejected"); + queue.context[0x293] = 0x760201bcu; + (void)AgcDriver::Graphics::DecodeState(queue); + Require(AgcDriver::Graphics::DrawRejection(queue, false).find("sample iteration") == std::string::npos, "an out-of-order watermark without out-of-order rasterization was rejected"); + queue.context[0x293] = 0x7e0201bcu; + Require(AgcDriver::Graphics::DrawRejection(queue, false).find("sample iteration") != std::string::npos, "out-of-order rasterization was accepted"); queue.context[0x293] = 0x06030000u; Require(AgcDriver::Graphics::DrawRejection(queue, false).find("sample iteration") != std::string::npos, "per-sample shading was accepted"); expectFailure([&] { AgcDriver::Graphics::DecodeState(queue); }, "sample iteration"); + queue.context[0x293] = 0; + for (const auto disabled : {0x6000u, 0x00100000u, 0u, 0x000e6000u}) { + queue.context[0x313] = disabled; + Require(AgcDriver::Graphics::DecodeState(queue).conservativeRasterization == VK_CONSERVATIVE_RASTERIZATION_MODE_DISABLED_EXT, "disabled conservative rasterization decoded as enabled"); + Require(AgcDriver::Graphics::DrawRejection(queue, false).find("PA_SC_CONSERVATIVE") == std::string::npos, "disabled conservative rasterization was rejected"); + } + for (const auto enabled : {0x00e00001u, 0x01e00022u, 0x000e6001u, 0x000e6020u}) { + queue.context[0x313] = enabled; + Require(AgcDriver::Graphics::DrawRejection(queue, false).find("PA_SC_CONSERVATIVE") != std::string::npos, "conservative rasterization was accepted"); + expectFailure([&] { AgcDriver::Graphics::DecodeState(queue); }, "PA_SC_CONSERVATIVE"); + } + queue.context[0x313] = 0x6000u; queue = makeState(); queue.userConfig.erase(0x24b); queue.context[0x2a5] = 0; @@ -178,6 +205,13 @@ void stateTests() { state = AgcDriver::Graphics::DecodeState(queue); Require(!state.depthTest && !state.stencilTest, "tests on absent depth and stencil planes were kept"); Require(AgcDriver::Graphics::DrawRejection(queue, false).empty(), "tests on absent depth and stencil planes were rejected"); + queue.context[0x200] = 0x007007bbu; + state = AgcDriver::Graphics::DecodeState(queue); + Require(!state.depth.has_value() && !state.depthTest && !state.depthBoundsTest && !state.stencilTest, "a depth bounds test on absent depth and stencil planes was kept"); + Require(AgcDriver::Graphics::DrawRejection(queue, false).empty(), "a depth bounds test on absent depth and stencil planes was rejected"); + queue.context[0x200] = 8u; + Require(!AgcDriver::Graphics::DecodeState(queue).depthBoundsTest && AgcDriver::Graphics::DrawRejection(queue, false).empty(), "a depth bounds test alone on absent depth and stencil planes was kept or rejected"); + queue.context[0x200] = 0x007007b3; queue.context[0x011] = 0x20000181; queue.context[0x012] = 0x00001000; queue.context[0x013] = 0x00002000; @@ -192,6 +226,8 @@ void stateTests() { queue.context[0x10d] = 0x01ffff00; state =AgcDriver::Graphics::DecodeState(queue); Require(!state.depthTest && state.stencilTest, "a depth test on an absent depth plane was kept beside a stencil plane"); + queue.context[0x200] = 0x007007bbu; + expectFailure([&] { AgcDriver::Graphics::DecodeState(queue); }, "depth bounds without a depth plane"); queue = makeState(); queue.context[0x10f] = 0x7fc00000; expectFailure([&] { AgcDriver::Graphics::DecodeState(queue); }, "non-finite"); @@ -221,10 +257,26 @@ void stateTests() { Require(!AgcDriver::Graphics::PixelProgramSkipped(queue), "a pixel program address was read as unset"); Require(AgcDriver::Graphics::DrawRejection(queue, true).find("missing register at DWORD 0x1b3") != std::string::npos, "a real pixel program without SPI_PS_INPUT_ENA was accepted"); expectFailure([&] { AgcDriver::Graphics::DecodePixelStageInfo(queue.context, std::array{0xe4u, 0xe4u, 0xe4u, 0xe4u, 0xe4u, 0xe4u, 0xe4u, 0xe4u}); }, "missing register"); + AgcDriver::QueueState cleared; + cleared.context[0x1b3] = 2; + cleared.context[0x1b4] = 2; + cleared.context[0x1b6] = 2; + cleared.context[0x192] = 7; + const auto clearedPixel = AgcDriver::Graphics::DecodePixelStageInfo(cleared.context, std::array{0xe4u, 0xe4u, 0xe4u, 0xe4u, 0xe4u, 0xe4u, 0xe4u, 0xe4u}); + Require(clearedPixel.interpolatorSettings[0] == 0 && clearedPixel.interpolatorSettings[1] == 7, "an unwritten SPI_PS_INPUT_CNTL_0 did not read as its clear-state value"); + cleared.ClearContext(); + for (std::uint32_t i = 0; i < 32; ++i) Require(cleared.context.at(0x191 + i) == 0, "CLEAR_STATE did not reset SPI_PS_INPUT_CNTL"); queue.shader[0x008] = 0; Require(AgcDriver::Graphics::PixelProgramSkipped(queue), "a zero pixel program address was not read as unset"); Require(AgcDriver::Graphics::DrawRejection(queue, true).find("writes color") != std::string::npos, "a draw without a pixel program that writes color was accepted"); + queue.shader.erase(0x008); + queue.shader.erase(0x009); + Require(AgcDriver::Graphics::PixelProgramSkipped(queue), "an unwritten pixel program address was not read as unset"); + Require(AgcDriver::Graphics::DrawRejection(queue, true).find("writes color") != std::string::npos, "a draw with an unwritten pixel program that writes color was accepted"); queue.context[0x8e] = 0; + Require(AgcDriver::Graphics::DrawRejection(queue, true).empty(), "a depth-only draw with an unwritten pixel program was rejected"); + queue.shader[0x008] = 0; + queue.shader[0x009] = 0; Require(AgcDriver::Graphics::DrawRejection(queue, true).empty(), "a depth-only draw without a pixel program was rejected"); state = AgcDriver::Graphics::DecodeState(queue); Require(!state.hasColorTarget, "a depth-only draw without a pixel program decoded a color target"); @@ -522,6 +574,45 @@ void ComputeScratchTests() { Require(back.request.context.compute->scratchDwords == 24u, "the compute scratch size did not survive serialization"); } +void shaderUserDataTailPaddingTests() { + constexpr std::size_t userDataOffset = 288; + constexpr auto userDataBytes = offsetof(ShaderUserData, sharp_resource_count) + sizeof(ShaderUserData{}.sharp_resource_count); + static_assert(userDataBytes == 54); + std::vector header(userDataOffset + userDataBytes); + Shader shader{}; + shader.user_data = reinterpret_cast(header.data() + userDataOffset); + std::memcpy(header.data(), &shader, sizeof(shader)); + const auto headerAddress = reinterpret_cast(header.data()); + const auto info = AgcDriver::Graphics::DecodeVertexStageInfo(header, headerAddress, {}, nullptr, true); + Require(!info.fetchEmbedded, "a ShaderUserData block without trailing struct padding was rejected"); +} + +void ColorPipeBankXorTests() { + constexpr std::uint32_t side = 128; + constexpr std::size_t blockBytes = 65536; + static std::vector storage(2 * blockBytes + side * side * 4); + const auto block = (reinterpret_cast(storage.data()) + blockBytes - 1) / blockBytes * blockBytes; + auto queue = makeState(); + queue.context[0x3b8] |= static_cast(AgcDriver::Graphics::ColorTileMode::RenderTarget) << 14u; + queue.context[0x3b0] = ((side - 1u) << 14u) | (side - 1u); + for (const auto offset : {0xdu, 0x82u, 0x91u, 0x95u}) queue.context[offset] = (side << 16u) | side; + for (const std::uint32_t pipeBankXor : {0u, 0x5600u, 0xff00u}) { + const auto address = block + pipeBankXor; + queue.context[0x318] = static_cast(address >> 8u); + queue.context[0x390] = static_cast(address >> 40u); + const auto color = AgcDriver::Graphics::DecodeState(queue).color; + Require(color.address == block && color.surfaceAddress == block && color.pipeBankXor == pipeBankXor, "a SW_64KB_R_X color base did not split into its block base and pipe/bank XOR " + std::to_string(pipeBankXor)); + } + queue.context[0x3b8] = 0x9000000u | (0x16u << 14u) | (0x14u << 19u); + for (const std::uint32_t pipeBankXor : {0u, 0xa00u, 0xf00u}) { + const auto address = block + 0x3000u + pipeBankXor; + queue.context[0x318] = static_cast(address >> 8u); + queue.context[0x390] = static_cast(address >> 40u); + const auto color = AgcDriver::Graphics::DecodeState(queue).color; + Require(color.tileMode == AgcDriver::Graphics::ColorTileMode::D4KBX && color.address == block + 0x3000u && color.pipeBankXor == pipeBankXor, "a SW_4KB_D_X color base did not split into its 4 KiB block base and pipe/bank XOR " + std::to_string(pipeBankXor)); + } +} + void PixelInputLayoutTests() { using ShaderRecompiler::PixelInput; using ShaderRecompiler::PixelInputVgpr; @@ -538,9 +629,15 @@ void PixelInputLayoutTests() { Require(pixel.linearCentroid && !pixel.noPerspective && PixelInputVgpr(pixel.inputAddr, PixelInput::LinearCentroid) == 4u && PixelInputVgpr(pixel.inputAddr, PixelInput::PositionX) == 6u && PixelInputVgpr(pixel.inputAddr, PixelInput::FrontFace) == 7u, "the linear centroid layout moved the inputs"); pixel = decode(0x506u, 0x7afu); Require(pixel.inputAddr == 0x7afu && !pixel.posY && pixel.posZ && PixelInputVgpr(pixel.inputAddr, PixelInput::PerspectiveCentroid) == 4u && PixelInputVgpr(pixel.inputAddr, PixelInput::PositionX) == 12u && PixelInputVgpr(pixel.inputAddr, PixelInput::PositionZ) == 14u, "ADDR-only inputs did not reserve their VGPRs"); - for (const auto bit : {0x8u, 0x80u, 0x4000u, 0x8000u}) { + for (const auto bit : {0x8u, 0x4000u}) { expectFailure([&] { static_cast(decode(0x2u | bit, 0x2u | bit)); }, "unsupported SPI_PS_INPUT_ENA/ADDR"); } + pixel = decode(0x80u, 0x80u); + Require(pixel.inputAddr == 0x80u && PixelInputVgpr(pixel.inputAddr, PixelInput::LineStipple) == 0u, "a pixel shader with only LINE_STIPPLE_TEX enabled was rejected"); + pixel = decode(0x8082u, 0x8082u); + Require(PixelInputVgpr(pixel.inputAddr, PixelInput::LineStipple) == 2u && PixelInputVgpr(pixel.inputAddr, PixelInput::PositionFixedPoint) == 3u, "LINE_STIPPLE_TEX and POS_FIXED_PT did not take their VGPRs after the perspective center"); + Require(readsBuiltin(pixelBuiltinsRead(0x8082u, 0x8082u, 3u), spv::BuiltInFragCoord), "POS_FIXED_PT is not read from the fragment coordinate"); + Require(!readsBuiltin(pixelBuiltinsRead(0x82u, 0x82u, 2u), spv::BuiltInFragCoord), "LINE_STIPPLE_TEX read the fragment coordinate"); pixel = decode(0x546u, 0x7c7u); { ShaderRecompiler::RecompileRequest request{}; @@ -572,6 +669,25 @@ void PixelInputLayoutTests() { Require(!readsBuiltin(read, spv::BuiltInFragCoord) && !readsBuiltin(read, spv::BuiltInBaryCoordKHR), "an ADDR-only centroid pair was loaded"); } +void OpaqueDestinationAlphaTests() { + auto queue = makeState(); + for (std::uint32_t constant = 0x105; constant < 0x109; ++constant) queue.context[constant] = 0; + queue.context[0x1e0] = (1u << 30u) | (1u << 29u) | 6u | (7u << 8u) | (8u << 16u) | (10u << 24u); + auto blend = AgcDriver::Graphics::DecodeState(queue).blend; + Require(blend.srcColorBlendFactor == VK_BLEND_FACTOR_DST_ALPHA && blend.dstColorBlendFactor == VK_BLEND_FACTOR_ONE_MINUS_DST_ALPHA, "color blend factors changed"); + Require(blend.srcAlphaBlendFactor == VK_BLEND_FACTOR_DST_COLOR && blend.dstAlphaBlendFactor == VK_BLEND_FACTOR_SRC_ALPHA_SATURATE, "alpha blend factors changed"); + queue.context[0x31d] = 0x20000u; + blend = AgcDriver::Graphics::DecodeState(queue).blend; + Require(blend.srcColorBlendFactor == VK_BLEND_FACTOR_ONE && blend.dstColorBlendFactor == VK_BLEND_FACTOR_ZERO, "an opaque destination left a destination-alpha color factor"); + Require(blend.srcAlphaBlendFactor == VK_BLEND_FACTOR_ONE && blend.dstAlphaBlendFactor == VK_BLEND_FACTOR_SRC_ALPHA_SATURATE, "an opaque destination left a destination alpha factor"); + queue.context[0x1e0] = (1u << 30u) | 10u | (9u << 8u); + blend = AgcDriver::Graphics::DecodeState(queue).blend; + Require(blend.srcColorBlendFactor == VK_BLEND_FACTOR_ZERO && blend.dstColorBlendFactor == VK_BLEND_FACTOR_ONE_MINUS_DST_COLOR, "alpha saturation with an opaque destination"); + Require(blend.dstAlphaBlendFactor == VK_BLEND_FACTOR_ZERO, "an opaque destination left a destination-color alpha factor"); + queue.context[0x31d] = 0x40000u; + expectFailure([&] { AgcDriver::Graphics::DecodeState(queue); }, "destination alpha override"); +} + void DisabledColorTests() { auto queue = makeState(); queue.context[0x8e] = 0; @@ -724,6 +840,19 @@ void DepthStencilTests() { Require(state.depth && state.depth->address == 0x10000 && state.depth->stencilAddress == 0x20000 && state.depth->format == VK_FORMAT_D32_SFLOAT_S8_UINT && state.depth->clearStencil == 7, "depth surface decode changed"); Require(state.renderExtent.width == 4 && state.renderExtent.height == 2, "render extent ignores the depth surface"); Require(!state.depthTest && !state.depthWrite && state.stencilTest, "depth/stencil enables changed"); + Require(state.depth->htileAddress == 0, "an HTILE surface without DB_HTILE_DATA_BASE decoded an HTILE address"); + queue.context[0x005] = 0x300; + queue.context[0x01e] = 0x01; + log.clear(); + AgcDriver::Graphics::RegisterReadLog() = &log; + state = AgcDriver::Graphics::DecodeState(queue); + AgcDriver::Graphics::RegisterReadLog() = nullptr; + for (const auto read : log) Require(AgcDriver::Graphics::DrawKeyCovers(read), "DrawKeyRegisters lacks an HTILE register the decoder reads: " + std::to_string(read.offset)); + Require(state.depth->htileAddress == (0x30000ull | (1ull << 40u)), "the HTILE address was not decoded from DB_HTILE_DATA_BASE and its high byte"); + queue.context[0x010] &= ~0x20000000u; + state = AgcDriver::Graphics::DecodeState(queue); + Require(state.depth->htileAddress == 0, "a surface without TILE_SURFACE_ENABLE decoded an HTILE address"); + queue.context[0x010] |= 0x20000000u; const auto& front = state.stencilFront; Require(front.compareOp == VK_COMPARE_OP_ALWAYS && front.passOp == VK_STENCIL_OP_INCREMENT_AND_CLAMP && front.failOp == VK_STENCIL_OP_KEEP && front.reference == 1 && front.writeMask == 0xff, "stencil mask pass changed"); Require(std::memcmp(&state.stencilBack, &front, sizeof(front)) == 0, "back faces without BACKFACE_ENABLE must use the front state"); @@ -749,7 +878,68 @@ void DepthStencilTests() { expectFailure([&] { AgcDriver::Graphics::DecodeState(queue); }, "stencil add/subtract"); queue.context[0x10b] = 0; queue.context[0x000] = 1; - expectFailure([&] { AgcDriver::Graphics::DecodeState(queue); }, "DB_RENDER_CONTROL"); + queue.context[0x00b] = std::bit_cast(0.25f); + state = AgcDriver::Graphics::DecodeState(queue); + Require(state.depth && state.depthTest && state.depthWrite && state.depthCompare == VK_COMPARE_OP_ALWAYS && !state.depthBoundsTest && !state.depthBias && state.viewport.minDepth == 0.25f && state.viewport.maxDepth == 0.25f, "a DEPTH_CLEAR_ENABLE draw does not store DB_DEPTH_CLEAR everywhere it covers"); + Require(AgcDriver::Graphics::DrawRejection(queue, false).empty(), "precheck rejected a depth clear"); + queue.context[0x002] = 0x01000000; + expectFailure([&] { AgcDriver::Graphics::DecodeState(queue); }, "writable depth plane"); + Require(AgcDriver::Graphics::DrawRejection(queue, false).find("writable depth plane") != std::string::npos, "precheck accepted read-only depth clear"); + queue.context[0x002] = 0; + queue.context[0x00b] = 0; + queue.context[0x000] = 0x22; + state = AgcDriver::Graphics::DecodeState(queue); + const auto clears = [](const VkStencilOpState& face) { + return face.compareOp == VK_COMPARE_OP_ALWAYS && face.passOp == VK_STENCIL_OP_REPLACE && face.failOp == VK_STENCIL_OP_REPLACE && face.depthFailOp == VK_STENCIL_OP_REPLACE && face.writeMask == 0xff && face.reference == 7; + }; + Require(state.stencilTest && clears(state.stencilFront) && clears(state.stencilBack), "a STENCIL_CLEAR_ENABLE draw does not store DB_STENCIL_CLEAR"); + queue.context[0x200] = 0; + state = AgcDriver::Graphics::DecodeState(queue); + Require(state.stencilTest && clears(state.stencilFront), "a STENCIL_CLEAR_ENABLE draw without a stencil test does not store DB_STENCIL_CLEAR"); + queue.context[0x002] = 0x02000000; + expectFailure([&] { AgcDriver::Graphics::DecodeState(queue); }, "writable stencil plane"); + Require(AgcDriver::Graphics::DrawRejection(queue, false).find("writable stencil plane") != std::string::npos, "precheck accepted read-only stencil clear"); + queue.context[0x002] = 0; + for (const auto offset : {0x31bu, 0x31cu, 0x31du}) queue.context[offset + 0xfu] = queue.context.at(offset); + for (const auto offset : {0x3b0u, 0x3b8u}) queue.context[offset + 1u] = queue.context.at(offset); + const auto secondColor = reinterpret_cast(sliceMemory.data()); + queue.context[0x327] = static_cast(secondColor >> 8u); + queue.context[0x391] = static_cast(secondColor >> 40u); + queue.context[0x1e1] = 0; + queue.context[0x8e] = 0xf3; + queue.context[0x8f] = 0xff; + queue.context[0x1c5] = 0x99; + queue.context[0x90] = 0x80000001; + queue.context[0x91] = 0x00020003; + queue.context[0x10b] = 0x00050050; + queue.context[0x10c] = 0x05ffff02; + queue.context[0x10d] = 0x090000ff; + for (const auto control : {0u, 1u, 0x81u}) { + queue.context[0x200] = control; + state = AgcDriver::Graphics::DecodeState(queue); + Require(state.stencilTest && clears(state.stencilFront) && clears(state.stencilBack), "stencil clear used the overridden stencil operations"); + Require(state.depth && !state.depthTest && !state.depthWrite, "stencil clear changed depth state"); + Require(state.colors.size() == 2 && state.blends.size() == 2 && state.colors[1].address == secondColor && state.blends[0].colorWriteMask == 3 && state.blends[1].colorWriteMask == 0xf, "stencil clear lost MRT color writes"); + Require(state.scissor.offset.x == 1 && state.scissor.offset.y == 0 && state.scissor.extent.width == 2 && state.scissor.extent.height == 2, "stencil clear changed scissor coverage"); + Require(AgcDriver::Graphics::DrawRejection(queue, false).empty(), "precheck rejected combined color/stencil clear"); + } + queue.context[0x002] = 0x01000000; + state = AgcDriver::Graphics::DecodeState(queue); + Require(!state.depthWrite && clears(state.stencilFront), "read-only depth prevented stencil clear"); + queue.context[0x002] = 0; + queue.context[0x000] = 0x23; + state = AgcDriver::Graphics::DecodeState(queue); + Require(state.depthWrite && state.depthCompare == VK_COMPARE_OP_ALWAYS && clears(state.stencilFront), "a combined depth and stencil clear draw does not clear both planes"); + for (const auto control : {4u, 8u}) { + queue.context[0x000] = 0x22u | control; + expectFailure([&] { AgcDriver::Graphics::DecodeState(queue); }, "DB_RENDER_CONTROL"); + } + queue.context[0x000] = 0x22; + queue.context[0x011] = 0; + expectFailure([&] { AgcDriver::Graphics::DecodeState(queue); }, "writable stencil plane"); + Require(AgcDriver::Graphics::DrawRejection(queue, false).find("writable stencil plane") != std::string::npos, "precheck accepted clear without a stencil plane"); + queue.context[0x010] = 0; + expectFailure([&] { AgcDriver::Graphics::DecodeState(queue); }, "writable stencil plane"); queue = makeState(); queue.context[0x31b] = 1u << 26u; expectFailure([&] { AgcDriver::Graphics::DecodeState(queue); }, "mip exceeds"); @@ -761,6 +951,11 @@ void DepthStencilTests() { queue.context[0x31b] = 1u | (1u << 13u); const auto slice = AgcDriver::Graphics::DecodeState(queue); Require(slice.color.address == sliced + 1024u && slice.color.bytes == 1024u, "a color view of one slice did not move the target by one slice"); + queue.context[0x3b8] = 0x09000001; + const auto arraySlice = AgcDriver::Graphics::DecodeState(queue); + Require(arraySlice.color.address == sliced + 1024u && arraySlice.color.bytes == 1024u && arraySlice.color.depth == 1u, "a color view of one slice of a 2D array target did not move the target by one slice"); + queue.context[0x31b] = 2u | (2u << 13u); + expectFailure([&] { AgcDriver::Graphics::DecodeState(queue); }, "beyond the array surface"); queue.context[0x3b8] = 0x0a000003; queue.context[0x31b] = 2u | (2u << 13u); const auto volume = AgcDriver::Graphics::DecodeState(queue); @@ -772,8 +967,33 @@ void DepthStencilTests() { expectFailure([&] { AgcDriver::Graphics::DecodeState(queue); }, "DCC 3D color targets"); } +void OneDimensionalColorTests() { + auto queue = makeState(); + queue.context[0x3b8] = 0x08000000; + expectFailure([&] { AgcDriver::Graphics::DecodeState(queue); }, "taller than one row"); + queue.context[0x3b0] = 63u << 14u; + const auto line = AgcDriver::Graphics::DecodeState(queue); + Require(line.color.extent.width == 64u && line.color.extent.height == 1u && line.color.address == reinterpret_cast(colorMemory.data()) && line.color.depth == 1u, "a 1D color target did not decode as one row"); +} + +void htileDepthClearTests() { + static constexpr std::array kernel{0xd7460000u, 0x04010c06u, 0x34000084u, 0xdc388000u, 0x04020000u, 0xbf8c3f70u, 0xd7710008u, 0x00120805u, 0xd7710009u, 0x00120a05u, 0xd771000au, 0x00120c05u, 0xd771000bu, 0x00120e05u, 0xdc788000u, 0x00020800u, 0xbf810000u}; + const std::array wave{64, 1, 1}; + const auto address = [&](std::span code, std::array userData, std::array threads) { + return AgcDriver::Graphics::HtileDepthClearAddress(code, userData, threads); + }; + Require(address(kernel, {0, 0, 0x0dc50000u, 0x40u, 0u, 0x00000ff0u}, wave) == 0x400dc50000ull, "RADV's HTILE depth fast clear (ZMASK to 0) was not recognized"); + Require(address(kernel, {0, 0, 0x0dc50000u, 0x40u, 0xfffff00fu, 0x00000ff0u}, wave) == 0, "an HTILE initialization to expanded (ZMASK 0xf) was taken for a clear"); + Require(address(kernel, {0, 0, 0x0dc50000u, 0x40u, 0x000003f0u, 0xfffffc0fu}, wave) == 0, "a stencil-only HTILE update that keeps ZMASK was taken for a depth clear"); + Require(address(kernel, {0, 0, 0x0dc50000u, 0x40u, 0u, 0x00000ff0u}, {32, 1, 1}) == 0, "the HTILE kernel matched with another group size"); + auto other = kernel; + other[6] ^= 1u; + Require(address(other, {0, 0, 0x0dc50000u, 0x40u, 0u, 0x00000ff0u}, wave) == 0, "a different kernel was taken for RADV's HTILE update"); + Require(address(std::span(kernel).first(16), {0, 0, 0x0dc50000u, 0x40u, 0u, 0x00000ff0u}, wave) == 0, "a truncated kernel was taken for RADV's HTILE update"); +} + void depthMaintenanceTests() { - for (const auto mode : {0x4u, 0x8u, 0x10u, 0x80u, 0x100u, 0x1000u, 0x4000u}) { + for (const auto mode : {0x4u, 0x8u, 0x80u, 0x100u, 0x1000u, 0x4000u}) { for (const auto clear : {0u, 1u, 2u, 3u}) { auto queue = makeState(); queue.context[0x000] = mode | clear; @@ -792,6 +1012,20 @@ void depthMaintenanceTests() { queue.context[0x000] = control; Require(AgcDriver::Graphics::DepthMaintenanceRejection(queue).empty(), "ordinary depth controls were mistaken for maintenance"); } + for (const auto clear : {0u, 1u, 2u, 3u}) { + auto queue = makeState(); + queue.context[0x000] = 0x10u | clear; + queue.context[0x200] = 0; + queue.context[0x8e] = 0; + queue.context[0x8f] = 0; + Require(AgcDriver::Graphics::DepthMaintenanceRejection(queue).empty(), "a resummarize draw without depth, stencil or color work was rejected"); + queue.context[0x200] = 2; + Require(AgcDriver::Graphics::DepthMaintenanceRejection(queue).find("DB_RENDER_CONTROL") != std::string::npos, "a resummarize draw with a depth test passed"); + queue.context[0x200] = 0; + queue.context[0x8e] = 0xf; + queue.context[0x8f] = 0xf; + Require(AgcDriver::Graphics::DepthMaintenanceRejection(queue).find("DB_RENDER_CONTROL") != std::string::npos, "a resummarize draw with color writes passed"); + } auto absent = makeState(); absent.context.erase(0x000); Require(AgcDriver::Graphics::DepthMaintenanceRejection(absent).empty(), "an absent depth control produced a maintenance verdict"); @@ -856,11 +1090,31 @@ void DepthBoundsBiasTests() { for (const auto offset : {0x2e1u, 0x2e3u}) queue.context[offset] = bits(4.0f); state = AgcDriver::Graphics::DecodeState(queue); Require(state.depthBias && state.depthBiasSlope == 2.0f && state.depthBiasConstant == 4.0f && state.depthBiasClamp == 0.5f, "depth bias decode changed"); + Require(!state.depthBiasPerFace, "equal front and back depth bias must draw both faces in one pass"); + queue.context[0x2e2] = bits(-64.0f); queue.context[0x2e3] = bits(8.0f); - expectFailure([&] { AgcDriver::Graphics::DecodeState(queue); }, "differing between front and back"); + state = AgcDriver::Graphics::DecodeState(queue); + Require(state.depthBias && state.depthBiasPerFace && state.cullMode == VK_CULL_MODE_NONE, "differing front and back depth bias must draw each face with its own bias"); + Require(state.depthBiasSlope == 2.0f && state.depthBiasConstant == 4.0f && state.backDepthBiasSlope == -4.0f && state.backDepthBiasConstant == 8.0f && state.depthBiasClamp == 0.5f, "per-face depth bias decode changed"); + queue.context[0x205] = 0x00000a48u; + state = AgcDriver::Graphics::DecodeState(queue); + Require(state.depthBias && state.depthBiasPerFace && state.depthBiasConstant == 4.0f && state.backDepthBiasSlope == 0.0f && state.backDepthBiasConstant == 0.0f, "back faces without depth bias must draw unbiased"); + queue.context[0x205] = 0x00001248u; + state = AgcDriver::Graphics::DecodeState(queue); + Require(state.depthBias && state.depthBiasPerFace && state.depthBiasSlope == 0.0f && state.depthBiasConstant == 0.0f && state.backDepthBiasConstant == 8.0f, "front faces without depth bias must draw unbiased"); queue.context[0x205] = 0x00001a4au; state = AgcDriver::Graphics::DecodeState(queue); - Require(state.depthBias && state.depthBiasConstant == 4.0f, "culled back faces must not constrain the front depth bias"); + Require(state.depthBias && !state.depthBiasPerFace && state.depthBiasConstant == 4.0f, "culled back faces must not constrain the front depth bias"); + queue.context[0x205] = 0x00001a49u; + state = AgcDriver::Graphics::DecodeState(queue); + Require(state.depthBias && !state.depthBiasPerFace && state.depthBiasSlope == -4.0f && state.depthBiasConstant == 8.0f, "culled front faces must draw with the back depth bias"); + queue.context[0x205] = 0x00003a46u; + state = AgcDriver::Graphics::DecodeState(queue); + Require(state.depthBias && state.depthBiasConstant == 4.0f && state.cullMode == VK_CULL_MODE_BACK_BIT, "a triangle draw with the point and line offset enable lost its depth bias"); + queue.userConfig[0x242] = 2; + expectFailure([&] { AgcDriver::Graphics::DecodeState(queue); }, "polygon mode, depth bias or nonstandard rasterization"); + queue.userConfig[0x242] = 4; + queue.context[0x205] = 0x00001a4au; queue.context[0x2de] = 0x1f0u; expectFailure([&] { AgcDriver::Graphics::DecodeState(queue); }, "units other than the depth format"); queue.context[0x2de] = 0x1e9u; @@ -1023,7 +1277,7 @@ void metadataPassTests() { } } -alignas(256) std::array cmaskMemory{}; +alignas(4096) std::array cmaskMemory{}; void cmaskTests() { using AgcDriver::Graphics::DecodeColorBuffer; @@ -1034,8 +1288,15 @@ void cmaskTests() { queue.context[0x398] = static_cast(cmaskAddress >> 40u); queue.context[0x323] = 0x11223344; queue.context[0x324] = 0; + queue.context[0x3b8] = 0x0dc00000u; const auto target = DecodeColorBuffer(queue.context, 0); Require(target.cmaskAddress == cmaskAddress && target.cmaskBytes == cmaskMemory.size(), "the fast-clear target lost its CMASK"); + std::vector log; + AgcDriver::Graphics::RegisterReadLog() = &log; + static_cast(DecodeColorBuffer(queue.context, 0)); + AgcDriver::Graphics::RegisterReadLog() = nullptr; + Require(!log.empty(), "the register facade recorded nothing for a fast-clear target"); + for (const auto read : log) Require(AgcDriver::Graphics::DrawKeyCovers(read), "DrawKeyRegisters lacks a register the fast-clear target decode reads: " + std::to_string(read.offset)); Require(DecodeColorBuffer(makeState().context, 0).cmaskAddress == 0, "a target without FAST_CLEAR got a CMASK"); using AgcDriver::Graphics::CmaskBytes; Require(CmaskBytes(1920, 1080) == 0x6000 && CmaskBytes(960, 544) == 0x2000 && CmaskBytes(800, 450) == 0x1000 && CmaskBytes(1024, 512) == 0x1000 && CmaskBytes(3840, 2160) == 0x14000, "a CMASK does not span whole 1024x512 metablocks"); @@ -1046,6 +1307,16 @@ void cmaskTests() { unaddressed.context[0x31f] = 0; unaddressed.context[0x398] = 0; expectFailure([&] { DecodeColorBuffer(unaddressed.context, 0); }, "without a CMASK address"); + auto unaligned = queue; + unaligned.context[0x3b8] = 0x09c00000u; + expectFailure([&] { DecodeColorBuffer(unaligned.context, 0); }, "pipe-aligned SW_64KB_Z_X"); + unaligned.context[0x3b8] = 0x0d000000u; + expectFailure([&] { DecodeColorBuffer(unaligned.context, 0); }, "pipe-aligned SW_64KB_Z_X"); + unaligned = queue; + unaligned.context[0x31f] += 1u; + expectFailure([&] { DecodeColorBuffer(unaligned.context, 0); }, "misaligned"); + using AgcDriver::Graphics::CmaskLayout; + Require(CmaskBytes(61, 13) == CmaskLayout(61, 13).Bytes() && CmaskBytes(3840, 2160) == CmaskLayout(3840, 2160).Bytes(), "the CMASK size and layout disagree"); queue.context[0x0] = 0; queue.context[0x202] = 0xcc0020; @@ -1068,9 +1339,42 @@ void cmaskTests() { std::memset(colorMemory.data(), 0x5a, colorMemory.size()); AgcDriver::Graphics::RunColorMetadataPass(context, *pass); Require(texels(0x5a5a5a5au) && cmaskIs(0xff), "a pass over an expanded CMASK changed the texels"); + const CmaskLayout layout(64, 4); + const auto setTile = [&](std::uint32_t tileX, std::uint8_t code) { + const auto nibble = layout.Nibble(tileX, 0); + auto& entry = cmaskMemory[nibble / 2u]; + const auto shift = (nibble % 2u) * 4u; + entry = static_cast((entry & ~(0xfu << shift)) | (code << shift)); + }; + const auto tiles = [&](const auto& cleared) { + for (std::size_t offset = 0; offset < colorMemory.size(); offset += 4) { + std::uint32_t texel = 0; + std::memcpy(&texel, colorMemory.data() + offset, 4); + if (texel != (cleared((offset / 4u) % 64u / 8u) ? 0x11223344u : 0x5a5a5a5au)) return false; + } + return true; + }; + std::memset(colorMemory.data(), 0x5a, colorMemory.size()); cmaskMemory.fill(0); cmaskMemory[cmaskMemory.size() - 1] = 0xff; - expectFailure([&] { AgcDriver::Graphics::RunColorMetadataPass(context, *pass); }, "not all fast-cleared or all expanded"); + AgcDriver::Graphics::RunColorMetadataPass(context, *pass); + Require(texels(0x11223344) && cmaskIs(0xff), "CMASK nibbles past the target's tiles changed what its tiles hold"); + std::memset(colorMemory.data(), 0x5a, colorMemory.size()); + cmaskMemory.fill(0xff); + setTile(1, 0); + setTile(7, 0); + AgcDriver::Graphics::RunColorMetadataPass(context, *pass); + Require(tiles([](std::uint32_t tile) { return tile == 1 || tile == 7; }) && cmaskIs(0xff), "a partly fast-cleared CMASK did not clear exactly its cleared tiles"); + std::memset(colorMemory.data(), 0x5a, colorMemory.size()); + cmaskMemory.fill(0xff); + setTile(4, 0); + AgcDriver::Graphics::RunColorMetadataPass(context, {AgcDriver::Graphics::ColorMetadataPass::Mode::DccDecompress, pass->targets}); + Require(tiles([](std::uint32_t tile) { return tile == 4; }) && cmaskIs(0xff), "a DCC decompress pass did not eliminate the CMASK fast clear"); + std::memset(colorMemory.data(), 0x5a, colorMemory.size()); + cmaskMemory.fill(0xff); + setTile(2, 0); + setTile(3, 3); + expectFailure([&] { AgcDriver::Graphics::RunColorMetadataPass(context, *pass); }, "not a single-sample fast-clear code"); Require(texels(0x5a5a5a5au), "a refused pass changed the texels"); queue.context[0x31c] |= 0x10000000; @@ -1086,10 +1390,39 @@ void cmaskTests() { Require(texels(0x11223344) && cmaskIs(0xff) && std::ranges::all_of(dccKeys, [](std::uint8_t key) { return key == 0xff; }), "a DCC fast clear under an expanded CMASK was not eliminated through its keys"); std::memset(colorMemory.data(), 0x5a, colorMemory.size()); cmaskMemory.fill(0); - expectFailure([&] { AgcDriver::Graphics::RunColorMetadataPass(context, *pass); }, "DCC color target that is not all expanded"); + expectFailure([&] { AgcDriver::Graphics::RunColorMetadataPass(context, *pass); }, "DCC color target"); Require(texels(0x5a5a5a5au), "a refused pass changed the texels of a DCC target"); } +void uint8x4TargetTests() { + auto queue = makeState(); + queue.context[0x1c5] = 7; + queue.context[0x31c] = (queue.context[0x31c] & ~0x1ffcu) | (10u << 2u) | (4u << 8u); + const auto state = AgcDriver::Graphics::DecodeState(queue); + Require(state.color.format == VK_FORMAT_R8G8B8A8_UINT && state.color.elementBytes == 4 && state.color.uintExport, "an 8_8_8_8 UINT color target was rejected"); + for (const std::uint32_t swap : {2u, 3u}) { + queue.context[0x31c] = (queue.context[0x31c] & ~0x1800u) | (swap << 11u); + Require(AgcDriver::Graphics::DecodeState(queue).color.format == VK_FORMAT_R8G8B8A8_UINT, "a reversed 8_8_8_8 UINT color target was rejected"); + } + queue.context[0x31c] = (queue.context[0x31c] & ~0x1800u) | (1u << 11u); + expectFailure([&] { AgcDriver::Graphics::DecodeState(queue); }, "unsupported color format 10 number type 4 component swap 1"); +} + +void uint16ExportTests() { + auto queue = makeState(); + queue.context[0x1c5] = 7; + expectFailure([&] { AgcDriver::Graphics::DecodeState(queue); }, "color export format 7"); + queue.context[0x31c] = (queue.context[0x31c] & ~0x77cu) | 0x404u; + const auto state = AgcDriver::Graphics::DecodeState(queue); + Require(state.hasColorTarget && state.color.format == VK_FORMAT_R8_UINT && state.color.uintExport, "a UINT16_ABGR export into an unsigned integer target did not decode"); + Require(!AgcDriver::Graphics::DecodeState(makeState()).color.uintExport, "a float export was marked unsigned integer"); + auto blended = queue; + blended.context[0x1e0] = 1u << 30u; + expectFailure([&] { AgcDriver::Graphics::DecodeState(blended); }, "blending into an unsigned integer target"); + queue.context[0x1c5] = 8; + expectFailure([&] { AgcDriver::Graphics::DecodeState(queue); }, "color export format 8"); +} + void DepthClipTests() { auto queue = makeState(); const auto direct = AgcDriver::Graphics::DecodeState(queue); @@ -1117,6 +1450,21 @@ void DepthClipTests() { queue.context[0xb4] = std::bit_cast(2.0f); expectFailure([&] { AgcDriver::Graphics::DecodeState(queue); }, "inverted viewport depth clamp"); queue.context[0xb4] = 0; + auto collapsedQueue = makeState(); + collapsedQueue.context[0x10f] = 0; + collapsedQueue.context[0x110] = 0; + collapsedQueue.context[0x111] = std::bit_cast(-0.0f); + collapsedQueue.context[0x112] = 0; + const auto collapsed = AgcDriver::Graphics::DecodeState(collapsedQueue); + Require(collapsed.scissor.extent.width == 0 && collapsed.scissor.extent.height == 0, "a triangle draw through a zero-scale viewport rasterizes"); + Require(collapsed.viewport.width > 0 && collapsed.viewport.height > 0 && collapsed.viewport.minDepth == 0 && collapsed.viewport.maxDepth == 1, "a zero-scale viewport did not become a valid Vulkan viewport"); + collapsedQueue.userConfig[0x242] = 2; + expectFailure([&] { AgcDriver::Graphics::DecodeState(collapsedQueue); }, "unsupported viewport transform"); + collapsedQueue.userConfig[0x242] = 4; + collapsedQueue.context[0x10f] = std::bit_cast(-0.0f); + Require(AgcDriver::Graphics::DecodeState(collapsedQueue).scissor.extent.width == 0, "a negative-zero viewport scale was not collapsed"); + collapsedQueue.context[0x10f] = std::bit_cast(-1.0f); + expectFailure([&] { AgcDriver::Graphics::DecodeState(collapsedQueue); }, "unsupported viewport transform"); for (std::uint32_t bit = 0; bit < 32; ++bit) { if (bit == 19 || bit == 24 || bit == 26 || bit == 27) continue; for (const auto linearBit : {0u, 0x01000000u}) { @@ -1165,6 +1513,7 @@ void InitialContextTests() { struct MockDescriptorWrite { std::uint32_t binding; + std::uint32_t arrayElement; std::uint32_t count; VkDescriptorType type; std::vector buffers; @@ -1177,13 +1526,22 @@ struct MockVulkan { std::map bufferUsage; std::map allocationFlags; std::map bufferMemory; + std::map bufferOffset; std::map> memories; + std::map allocationSizes; + VkDeviceSize allocatedBytes = 0; + std::optional memoryLimit; + std::uint64_t allocationAttempts = 0; std::vector layoutBindings; + VkDescriptorSetLayoutCreateFlags layoutFlags = 0; + std::vector layoutBindingFlags; std::vector poolSizes; std::uint32_t poolMaxSets = 0; VkDescriptorPoolCreateFlags poolFlags = 0; std::uint32_t freedSets = 0; std::vector writes; + std::vector copies; + bool allowCopies = false; std::uint32_t boundSets = 0; std::uint32_t boundFirst = 0; VkPipelineBindPoint boundPoint = VK_PIPELINE_BIND_POINT_MAX_ENUM; @@ -1217,8 +1575,12 @@ VKAPI_ATTR void VKAPI_CALL mockGetBufferMemoryRequirements(VkDevice, VkBuffer bu } VKAPI_ATTR VkResult VKAPI_CALL mockAllocateMemory(VkDevice, const VkMemoryAllocateInfo* info, const VkAllocationCallbacks*, VkDeviceMemory* memory) { + ++mock.allocationAttempts; + if (mock.memoryLimit.has_value() && mock.allocatedBytes + info->allocationSize > *mock.memoryLimit) return VK_ERROR_OUT_OF_DEVICE_MEMORY; *memory = makeHandle(); mock.memories[*memory] = std::vector(info->allocationSize); + mock.allocationSizes[*memory] = info->allocationSize; + mock.allocatedBytes += info->allocationSize; if (info->pNext != nullptr) { const auto* flags = static_cast(info->pNext); mock.allocationFlags[*memory] = flags->flags; @@ -1229,11 +1591,17 @@ VKAPI_ATTR VkResult VKAPI_CALL mockAllocateMemory(VkDevice, const VkMemoryAlloca } VKAPI_ATTR VkResult VKAPI_CALL mockBindBufferMemory(VkDevice, VkBuffer buffer, VkDeviceMemory memory, VkDeviceSize offset) { - Require(offset == 0, "mock buffer memory must be bound at offset zero"); + Require(offset + mock.bufferSizes.at(buffer) <= mock.memories.at(memory).size(), "mock buffer memory is bound past its allocation"); mock.bufferMemory[buffer] = memory; + mock.bufferOffset[buffer] = offset; return VK_SUCCESS; } +std::span mockBufferMemory(VkBuffer buffer) { + auto& memory = mock.memories.at(mock.bufferMemory.at(buffer)); + return std::span(memory).subspan(static_cast(mock.bufferOffset.at(buffer)), static_cast(mock.bufferSizes.at(buffer))); +} + VKAPI_ATTR VkResult VKAPI_CALL mockMapMemory(VkDevice, VkDeviceMemory memory, VkDeviceSize offset, VkDeviceSize, VkMemoryMapFlags, void** data) { Require(offset == 0, "mock memory must be mapped from offset zero"); *data = mock.memories.at(memory).data(); @@ -1246,13 +1614,21 @@ VKAPI_ATTR void VKAPI_CALL mockDestroyBuffer(VkDevice, VkBuffer, const VkAllocat --mock.live; } -VKAPI_ATTR void VKAPI_CALL mockFreeMemory(VkDevice, VkDeviceMemory, const VkAllocationCallbacks*) { +VKAPI_ATTR void VKAPI_CALL mockFreeMemory(VkDevice, VkDeviceMemory memory, const VkAllocationCallbacks*) { + if (const auto found = mock.allocationSizes.find(memory); found != mock.allocationSizes.end()) { + mock.allocatedBytes -= found->second; + mock.allocationSizes.erase(found); + } --mock.live; } VKAPI_ATTR VkResult VKAPI_CALL mockCreateDescriptorSetLayout(VkDevice, const VkDescriptorSetLayoutCreateInfo* info, const VkAllocationCallbacks*, VkDescriptorSetLayout* layout) { *layout = makeHandle(); mock.layoutBindings.assign(info->pBindings, info->pBindings + info->bindingCount); + mock.layoutFlags = info->flags; + const auto* bindingFlags = static_cast(info->pNext); + if (bindingFlags != nullptr) mock.layoutBindingFlags.assign(bindingFlags->pBindingFlags, bindingFlags->pBindingFlags + bindingFlags->bindingCount); + else mock.layoutBindingFlags.clear(); ++mock.live; return VK_SUCCESS; } @@ -1285,10 +1661,11 @@ VKAPI_ATTR VkResult VKAPI_CALL mockFreeDescriptorSets(VkDevice, VkDescriptorPool return VK_SUCCESS; } -VKAPI_ATTR void VKAPI_CALL mockUpdateDescriptorSets(VkDevice, std::uint32_t count, const VkWriteDescriptorSet* writes, std::uint32_t copyCount, const VkCopyDescriptorSet*) { - Require(copyCount == 0, "descriptor copies are not expected"); +VKAPI_ATTR void VKAPI_CALL mockUpdateDescriptorSets(VkDevice, std::uint32_t count, const VkWriteDescriptorSet* writes, std::uint32_t copyCount, const VkCopyDescriptorSet* copies) { + Require(copyCount == 0 || mock.allowCopies, "descriptor copies are not expected"); + if (copyCount != 0) mock.copies.insert(mock.copies.end(), copies, copies + copyCount); for (std::uint32_t i = 0; i < count; ++i) { - MockDescriptorWrite write{writes[i].dstBinding, writes[i].descriptorCount, writes[i].descriptorType, {}}; + MockDescriptorWrite write{writes[i].dstBinding, writes[i].dstArrayElement, writes[i].descriptorCount, writes[i].descriptorType, {}}; write.buffers.assign(writes[i].pBufferInfo, writes[i].pBufferInfo + writes[i].descriptorCount); mock.writes.push_back(write); } @@ -1300,6 +1677,12 @@ VKAPI_ATTR void VKAPI_CALL mockCmdBindDescriptorSets(VkCommandBuffer, VkPipeline mock.boundSets = count; } +VKAPI_ATTR VkResult VKAPI_CALL mockGetMemoryHostPointerProperties(VkDevice, VkExternalMemoryHandleTypeFlagBits type, const void* pointer, VkMemoryHostPointerPropertiesEXT* properties) { + Require(type == VK_EXTERNAL_MEMORY_HANDLE_TYPE_HOST_ALLOCATION_BIT_EXT && pointer != nullptr, "invalid host pointer import query"); + properties->memoryTypeBits = 1; + return VK_SUCCESS; +} + VKAPI_ATTR VkDeviceAddress VKAPI_CALL mockGetBufferDeviceAddress(VkDevice, const VkBufferDeviceAddressInfo* info) { Require(mock.bufferMemory.contains(info->buffer), "BDA buffer was not bound"); Require((mock.bufferUsage.at(info->buffer) & VK_BUFFER_USAGE_SHADER_DEVICE_ADDRESS_BIT) != 0, "BDA buffer usage is missing"); @@ -1361,14 +1744,52 @@ VKAPI_ATTR void VKAPI_CALL mockCmdDispatch(VkCommandBuffer, std::uint32_t x, std } VKAPI_ATTR void VKAPI_CALL mockCmdUpdateBuffer(VkCommandBuffer, VkBuffer buffer, VkDeviceSize offset, VkDeviceSize size, const void* data) { - auto& memory = mock.memories.at(mock.bufferMemory.at(buffer)); + auto memory = mockBufferMemory(buffer); Require(offset + size <= memory.size(), "a buffer update exceeds its buffer"); std::memcpy(memory.data() + offset, data, static_cast(size)); } +VKAPI_ATTR VkResult VKAPI_CALL mockAllocateCommandBuffers(VkDevice, const VkCommandBufferAllocateInfo* info, VkCommandBuffer* commands) { + for (std::uint32_t index = 0; index < info->commandBufferCount; ++index) commands[index] = makeHandle(); + mock.live += info->commandBufferCount; + return VK_SUCCESS; +} + +VKAPI_ATTR void VKAPI_CALL mockFreeCommandBuffers(VkDevice, VkCommandPool, std::uint32_t count, const VkCommandBuffer*) { mock.live -= count; } +VKAPI_ATTR VkResult VKAPI_CALL mockCreateFence(VkDevice, const VkFenceCreateInfo*, const VkAllocationCallbacks*, VkFence* fence) { *fence = makeHandle(); ++mock.live; return VK_SUCCESS; } +VKAPI_ATTR void VKAPI_CALL mockDestroyFence(VkDevice, VkFence, const VkAllocationCallbacks*) { --mock.live; } +VKAPI_ATTR VkResult VKAPI_CALL mockGetFenceStatus(VkDevice, VkFence) { return VK_SUCCESS; } +VKAPI_ATTR VkResult VKAPI_CALL mockResetFences(VkDevice, std::uint32_t, const VkFence*) { return VK_SUCCESS; } +VKAPI_ATTR VkResult VKAPI_CALL mockWaitForFences(VkDevice, std::uint32_t, const VkFence*, VkBool32, std::uint64_t) { return VK_SUCCESS; } +VKAPI_ATTR VkResult VKAPI_CALL mockBeginCommandBuffer(VkCommandBuffer, const VkCommandBufferBeginInfo*) { return VK_SUCCESS; } +VKAPI_ATTR VkResult VKAPI_CALL mockEndCommandBuffer(VkCommandBuffer) { return VK_SUCCESS; } +VKAPI_ATTR VkResult VKAPI_CALL mockQueueSubmit(VkQueue, std::uint32_t, const VkSubmitInfo*, VkFence) { return VK_SUCCESS; } +VKAPI_ATTR void VKAPI_CALL mockCmdPipelineBarrier(VkCommandBuffer, VkPipelineStageFlags, VkPipelineStageFlags, VkDependencyFlags, std::uint32_t, const VkMemoryBarrier*, std::uint32_t, const VkBufferMemoryBarrier*, std::uint32_t, const VkImageMemoryBarrier*) {} +VKAPI_ATTR void VKAPI_CALL mockCmdBeginQuery(VkCommandBuffer, VkQueryPool, std::uint32_t, VkQueryControlFlags) {} +VKAPI_ATTR void VKAPI_CALL mockCmdEndQuery(VkCommandBuffer, VkQueryPool, std::uint32_t) {} +VKAPI_ATTR void VKAPI_CALL mockCmdResetQueryPool(VkCommandBuffer, VkQueryPool, std::uint32_t, std::uint32_t) {} +VKAPI_ATTR void VKAPI_CALL mockCmdCopyQueryPoolResults(VkCommandBuffer, VkQueryPool, std::uint32_t, std::uint32_t, VkBuffer, VkDeviceSize, VkDeviceSize, VkQueryResultFlags) {} + PFN_vkVoidFunction VKAPI_CALL mockProc(VkDevice, const char* name) { static const std::map table{ + {"vkAllocateCommandBuffers", reinterpret_cast(mockAllocateCommandBuffers)}, + {"vkFreeCommandBuffers", reinterpret_cast(mockFreeCommandBuffers)}, + {"vkCreateFence", reinterpret_cast(mockCreateFence)}, + {"vkDestroyFence", reinterpret_cast(mockDestroyFence)}, + {"vkGetFenceStatus", reinterpret_cast(mockGetFenceStatus)}, + {"vkResetFences", reinterpret_cast(mockResetFences)}, + {"vkWaitForFences", reinterpret_cast(mockWaitForFences)}, + {"vkBeginCommandBuffer", reinterpret_cast(mockBeginCommandBuffer)}, + {"vkEndCommandBuffer", reinterpret_cast(mockEndCommandBuffer)}, + {"vkQueueSubmit", reinterpret_cast(mockQueueSubmit)}, + {"vkCmdPipelineBarrier", reinterpret_cast(mockCmdPipelineBarrier)}, + {"vkCmdBeginQuery", reinterpret_cast(mockCmdBeginQuery)}, + {"vkCmdEndQuery", reinterpret_cast(mockCmdEndQuery)}, + {"vkCmdResetQueryPool", reinterpret_cast(mockCmdResetQueryPool)}, + {"vkCmdCopyQueryPoolResults", reinterpret_cast(mockCmdCopyQueryPoolResults)}, + {"vkGetBufferDeviceAddressKHR", reinterpret_cast(mockGetBufferDeviceAddress)}, + {"vkGetMemoryHostPointerPropertiesEXT", reinterpret_cast(mockGetMemoryHostPointerProperties)}, {"vkCreateBuffer", reinterpret_cast(mockCreateBuffer)}, {"vkGetBufferMemoryRequirements", reinterpret_cast(mockGetBufferMemoryRequirements)}, {"vkAllocateMemory", reinterpret_cast(mockAllocateMemory)}, @@ -1450,7 +1871,7 @@ std::vector ShaderDataWords(std::initializer_list return words; } -bool sameBytes(const std::vector& memory, const void* expected, std::size_t bytes) { +bool sameBytes(std::span memory, const void* expected, std::size_t bytes) { return memory.size() >= bytes && std::memcmp(memory.data(), expected, bytes) == 0; } @@ -1468,8 +1889,8 @@ const VkDescriptorSetLayoutBinding& findLayoutBinding(std::uint32_t binding) { throw std::runtime_error("expected descriptor set layout binding is missing for binding " + std::to_string(binding)); } -const std::vector& bufferBytes(VkBuffer buffer) { - return mock.memories.at(mock.bufferMemory.at(buffer)); +std::span bufferBytes(VkBuffer buffer) { + return mockBufferMemory(buffer); } void expectResourceFailure(const ShaderRecompiler::RecompileResult& vertex, const ShaderRecompiler::RecompileResult& fragment, std::string_view reason) { @@ -1506,11 +1927,16 @@ void pushConstantTests() { std::array shaders{{{ShaderRecompiler::ShaderStage::Vertex, &vertex, 0}, {ShaderRecompiler::ShaderStage::Fragment, &fragment, 8}}}; Require(AgcDriver::Graphics::PushConstantStages(shaders) == (VK_SHADER_STAGE_VERTEX_BIT | VK_SHADER_STAGE_FRAGMENT_BIT), "push constant stage union changed"); const auto bytes = AgcDriver::Graphics::AssemblePushConstants(shaders); - Require(bytes.size() == 128 && bytes[0] == std::byte{1} && bytes[7] == std::byte{1} && bytes[8] == std::byte{2} && bytes[19] == std::byte{2} && bytes[20] == std::byte{0} && bytes[127] == std::byte{0}, "assembled push constants are misplaced"); + Require(bytes.size() == AgcDriver::Graphics::PipelinePushConstantBytes && bytes[0] == std::byte{1} && bytes[7] == std::byte{1} && bytes[8] == std::byte{2} && bytes[19] == std::byte{2} && bytes[20] == std::byte{0} && bytes[127] == std::byte{0} && bytes.back() == std::byte{0}, "assembled push constants are misplaced"); shaders[1].pushConstantOffset = 4; expectFailure([&] { AgcDriver::Graphics::AssemblePushConstants(shaders); }, "overlap"); shaders[1].pushConstantOffset = 120; + expectFailure([&] { AgcDriver::Graphics::AssemblePushConstants(shaders); }, "crosses a push constant slot"); + shaders[1].pushConstantOffset = AgcDriver::Graphics::PipelinePushConstantBytes - 4; expectFailure([&] { AgcDriver::Graphics::AssemblePushConstants(shaders); }, "outside the pipeline push constant block"); + shaders[1].pushConstantOffset = AgcDriver::Graphics::PipelinePushSlotBytes; + const auto slotted = AgcDriver::Graphics::AssemblePushConstants(shaders); + Require(slotted[128] == std::byte{2} && slotted[139] == std::byte{2} && slotted[8] == std::byte{0}, "a pixel shader in its push slot was misplaced"); shaders[1].pushConstantOffset = 2; expectFailure([&] { AgcDriver::Graphics::AssemblePushConstants(shaders); }, "DWORD aligned"); shaders[1].pushConstantOffset = 8; @@ -1524,6 +1950,58 @@ void pushConstantTests() { expectFailure([&] { AgcDriver::Graphics::AssemblePushConstants(shaders); }, "missing compiled shader"); } +void descriptorSnapshotTests() { + using AgcDriver::Graphics::DescriptorCache; + using AgcDriver::Graphics::Recorder; + using AgcDriver::Graphics::ShaderResources; + for (const unsigned selected : {1u, 2u, 3u}) { + mock = MockVulkan{}; + mock.allowCopies = true; + auto context = mockContext(); + { + DescriptorCache cache(context); + context.descriptorCache = &cache; + std::lock_guard lock(AgcDriver::GuestMemory::GpuMutex()); + Recorder recorder(context); + ShaderRecompiler::RecompileResult vertex, fragment; + auto binding = makeBinding(Role::GuestBuffers, 0, 2, join(vsharp(guestFirst.data(), 16), vsharp(guestSecond.data(), 32))); + binding.bufferWritten = {false, false}; + vertex.bindings.push_back(binding); + fragment.bindings.push_back(makeBinding(Role::GuestBuffers, 4, 1, vsharp(guestThird.data(), 8))); + const auto state = AgcDriver::Graphics::DecodeState(makeState()); + ShaderResources resources(context, vertex, fragment, state.color, 0, 0); + const auto original = findWrite(0).buffers; + std::vector moved; + for (std::size_t index = 0; index < 2; ++index) { + if ((selected & (1u << index)) == 0) continue; + const std::size_t bytes = index == 0 ? 24 : 12; + moved.push_back({index, 0, bytes, std::vector(bytes / 4, static_cast(0x12340000u + index))}); + } + mock.writes.clear(); + const auto snapshots = resources.PrepareDrawBindings(recorder, moved); + Require(snapshots != nullptr && snapshots->snapshots.size() == moved.size(), "draw snapshots are missing"); + Require(mock.copies.size() == 2 && mock.copies[0].descriptorCount == 2 && mock.copies[1].descriptorCount == 1, "snapshot descriptor set did not copy every original binding"); + Require(mock.writes.size() == 1, "snapshot updates must rewrite one complete affected array"); + const auto& write = mock.writes.front(); + Require(write.binding == 0 && write.arrayElement == 0 && write.count == 2, "snapshot update did not start from array element zero"); + std::size_t snapshot = 0; + for (std::size_t index = 0; index < 2; ++index) { + const auto& descriptor = write.buffers[index]; + if ((selected & (1u << index)) == 0) { + Require(descriptor.buffer == original[index].buffer && descriptor.offset == original[index].offset && descriptor.range == original[index].range, "snapshot update changed an untouched buffer descriptor"); + } else { + const auto& item = snapshots->snapshots[snapshot]; + Require(descriptor.buffer == item.buffer->Handle() && descriptor.offset == 0 && descriptor.range == moved[snapshot].size, "snapshot descriptor used another element's buffer or range"); + Require(sameBytes(bufferBytes(descriptor.buffer), moved[snapshot].words.data(), moved[snapshot].size), "snapshot descriptor lost its buffer data"); + ++snapshot; + } + } + recorder.Sync(); + } + Require(mock.live == 0, "snapshot descriptor resources leaked Vulkan objects"); + } +} + void resourceTests() { mock = MockVulkan{}; const auto context = mockContext(); @@ -1556,9 +2034,9 @@ void resourceTests() { Require(findWrite(44).buffers.size() == 1 && findWrite(44).buffers[0].range == 8 && sameBytes(bufferBytes(findWrite(44).buffers[0].buffer), guestThird.data(), 8), "fragment guest buffer is incorrect"); resources.Bind(commands, VK_PIPELINE_BIND_POINT_GRAPHICS, VK_NULL_HANDLE); Require(mock.boundPoint == VK_PIPELINE_BIND_POINT_GRAPHICS && mock.boundFirst == 0 && mock.boundSets == 1, "exactly one descriptor set must be bound at set zero"); - auto& first = mock.memories.at(mock.bufferMemory.at(array.buffers[0].buffer)); + auto first = mockBufferMemory(array.buffers[0].buffer); std::memset(first.data(), 0xab, 16); - auto& shaderData = mock.memories.at(mock.bufferMemory.at(findWrite(5).buffers[0].buffer)); + auto shaderData = mockBufferMemory(findWrite(5).buffers[0].buffer); std::memset(shaderData.data(), 0xcd, 12); resources.WriteBack(); Require(guestFirst[0] == 0xabababab && guestFirst[3] == 0xabababab, "guest buffer was not written back"); @@ -1586,7 +2064,7 @@ void resourceTests() { resources.Bind(commands, VK_PIPELINE_BIND_POINT_COMPUTE, VK_NULL_HANDLE); Require(mock.boundPoint == VK_PIPELINE_BIND_POINT_COMPUTE && mock.boundSets == 1, "compute descriptors were bound to the wrong bind point"); guestThird = {0xaaaaaaaa, 0xbbbbbbbb}; - std::memset(mock.memories.at(mock.bufferMemory.at(findWrite(3).buffers[0].buffer)).data(), 0x5a, 8); + std::memset(mockBufferMemory(findWrite(3).buffers[0].buffer).data(), 0x5a, 8); resources.WriteBack(); Require(guestThird[0] == 0x5a5a5a5a && guestThird[1] == 0x5a5a5a5a, "compute buffer was not written back"); guestThird = {0xaaaaaaaa, 0xbbbbbbbb}; @@ -1605,7 +2083,7 @@ void resourceTests() { Require(buffer.range == sizeof(guestSecond), "strided buffer range does not cover every record"); Require(sameBytes(bufferBytes(buffer.buffer), guestSecond.data(), sizeof(guestSecond)), "strided buffer contents were not uploaded"); const std::uint32_t changed = 0x12345678u; - auto& bytes = mock.memories.at(mock.bufferMemory.at(buffer.buffer)); + auto bytes = mockBufferMemory(buffer.buffer); std::memcpy(bytes.data() + 16, &changed, sizeof(changed)); resources.WriteBack(); Require(guestSecond[4] == changed && guestSecond[0] == 1 && guestSecond[7] == 8, "strided buffer write back changed the wrong record"); @@ -1652,6 +2130,32 @@ void resourceTests() { expectSingleAccepted(changed([](auto& binding) { binding.count = 3; binding.guestDescriptor = join(join(vsharp(guestFirst.data(), 16), vsharp(guestSecond.data(), 32)), vsharp(reinterpret_cast(0x1000), 8)); }), "an unmapped V# element"); expectSingleFailure(changed([&](auto& binding) { binding.count = 2; binding.guestDescriptor = join(vsharp(guestFirst.data(), 16), vsharp(reinterpret_cast(state.color.address), 64)); }), "aliases the render target"); expectSingleFailure(changed([](auto& binding) { binding.count = 17; binding.guestDescriptor.assign(68, 0); }), "per-stage limits"); + { + ShaderRecompiler::RecompileResult vertex; + ShaderRecompiler::RecompileResult fragment; + std::vector words; + for (int index = 0; index < 17; ++index) words = join(words, vsharp(reinterpret_cast(0x1000), 8)); + vertex.bindings.push_back(makeBinding(Role::GuestBuffers, 0, 17, words)); + auto updateAfterBind = mockContext(); + updateAfterBind.descriptorIndexingLimits.maxPerStageDescriptorUpdateAfterBindStorageBuffers = 17; + updateAfterBind.descriptorIndexingLimits.maxPerStageUpdateAfterBindResources = 128; + updateAfterBind.descriptorIndexingLimits.maxDescriptorSetUpdateAfterBindStorageBuffers = 32; + mock = MockVulkan{}; + { + AgcDriver::Graphics::ShaderResources resources(updateAfterBind, vertex, fragment, state.color, 0, 0); + Require(mock.layoutFlags == VK_DESCRIPTOR_SET_LAYOUT_CREATE_UPDATE_AFTER_BIND_POOL_BIT, "buffers above the per-stage limit did not get an update-after-bind layout"); + Require(mock.layoutBindingFlags.size() == 1 && mock.layoutBindingFlags[0] == VK_DESCRIPTOR_BINDING_UPDATE_AFTER_BIND_BIT, "the guest buffers binding is not update-after-bind"); + Require(mock.poolFlags == VK_DESCRIPTOR_POOL_CREATE_UPDATE_AFTER_BIND_BIT, "the update-after-bind set did not come from an update-after-bind pool"); + } + Require(mock.live == 0, "update-after-bind shader resources leaked Vulkan objects"); + updateAfterBind.descriptorIndexingLimits.maxPerStageDescriptorUpdateAfterBindStorageBuffers = 16; + expectFailure([&] { AgcDriver::Graphics::ShaderResources resources(updateAfterBind, vertex, fragment, state.color, 0, 0); }, "shader descriptors exceed per-stage limits"); + Require(mock.live == 0, "failed update-after-bind shader resources leaked Vulkan objects"); + mock = MockVulkan{}; + vertex.bindings.front() = makeBinding(Role::GuestBuffers, 0, 1, vsharp(reinterpret_cast(0x1000), 8)); + { AgcDriver::Graphics::ShaderResources resources(updateAfterBind, vertex, fragment, state.color, 0, 0); } + Require(mock.layoutFlags == 0 && mock.layoutBindingFlags.empty() && mock.poolFlags == 0, "buffers within the per-stage limit took the update-after-bind path"); + } { ShaderRecompiler::RecompileResult vertex; ShaderRecompiler::RecompileResult fragment; @@ -1674,6 +2178,28 @@ void resourceTests() { expectStageResources(Role::GuestImages, Kind::StorageImage, 8, 3, "detiler is unavailable"); expectStageResources(Role::GuestSamplers, Kind::Sampler, 4, 2, "shader sampler descriptors exceed per-stage limits"); } + { + ShaderRecompiler::RecompileResult vertex; + ShaderRecompiler::RecompileResult fragment; + fragment.bindings.push_back(makeBinding(Role::GuestBuffers, 0, 2, join(vsharp(guestFirst.data(), 16), vsharp(guestSecond.data(), 32)))); + const std::array shaders{{{ShaderRecompiler::ShaderStage::Vertex, &vertex, 0}, {ShaderRecompiler::ShaderStage::Fragment, &fragment, 0}}}; + auto limited = mockContext(); + limited.limits.maxPerStageResources = 2; + const auto expectAttachments = [&](std::uint32_t attachments, bool accepted, std::string_view what) { + mock = MockVulkan{}; + if (accepted) { + AgcDriver::Graphics::ShaderResources resources(limited, shaders, state.color, attachments, 0, 0); + } else { + expectFailure([&] { AgcDriver::Graphics::ShaderResources resources(limited, shaders, state.color, attachments, 0, 0); }, "shader descriptors exceed per-stage limits"); + } + Require(mock.live == 0, std::string(what) + " leaked Vulkan objects"); + }; + expectAttachments(0, true, "a pixel stage at the per-stage limit"); + expectAttachments(1, false, "a pixel stage over the per-stage limit with its colour attachment"); + std::swap(vertex.bindings, fragment.bindings); + expectAttachments(1, true, "a vertex stage at the per-stage limit"); + expectAttachments(3, false, "colour attachments over the per-stage limit"); + } { ShaderRecompiler::RecompileResult vertex; ShaderRecompiler::RecompileResult fragment; @@ -1836,6 +2362,7 @@ struct ModuleShape { bool layer = false; bool fragDepth = false; std::uint32_t sampleMaskLength = 0; + std::uint32_t floatControlsWidth = 0; }; void emit(std::vector& out, spv::Op op, std::initializer_list operands) { @@ -2022,12 +2549,121 @@ std::vector makeModule(const ModuleShape& shape) { if (shape.fragmentMode) emit(words, spv::OpExecutionMode, {main, *shape.fragmentMode}); if (shape.fragment) emit(words, spv::OpExecutionMode, {main, spv::ExecutionModeOriginUpperLeft}); if (shape.fragDepth) emit(words, spv::OpExecutionMode, {main, spv::ExecutionModeDepthReplacing}); + if (shape.floatControlsWidth != 0) { + for (const auto mode : {spv::ExecutionModeRoundingModeRTE, spv::ExecutionModeDenormPreserve, spv::ExecutionModeSignedZeroInfNanPreserve}) emit(words, spv::OpExecutionMode, {main, mode, shape.floatControlsWidth}); + } words.insert(words.end(), annotations.begin(), annotations.end()); words.insert(words.end(), declarations.begin(), declarations.end()); words.insert(words.end(), function.begin(), function.end()); return words; } +void barycentricEmulationTests() { + using namespace ShaderRecompiler; + using namespace AgcDriver::Graphics; + const std::array parameters{{{0u, 0u, false, false}, {1u, 1u, false, true}, {2u, 2u, true, false}}}; + const auto layout = LayoutBarycentricEmulation(parameters, {true, true, true}); + Require(layout.smoothLocation == 1u && layout.linearLocation == 3u && layout.perVertexLocations == std::vector>{{1u, 4u}}, "emulated barycentrics did not take the free locations in order"); + Require(LayoutBarycentricEmulation(parameters, {}).perVertexLocations.empty() && LayoutBarycentricEmulation(parameters, {}).smoothLocation == BarycentricEmulationLayout::NoLocation, "an inactive emulation was given locations"); + std::vector crowded; + for (std::uint32_t location = 0; location < 11u; ++location) crowded.push_back({location, location, false, true}); + expectFailure([&] { static_cast(LayoutBarycentricEmulation(crowded, {true, false, false})); }, "exceed 32 locations"); + + const auto compile = [&](std::span code, bool barycentric, std::uint32_t secondInput = 0xffffffffu) { + auto queue = makeState(); + queue.context[0x1b3] = 0x2u; + queue.context[0x1b4] = 0x2u; + queue.context[0x1b6] = secondInput == 0xffffffffu ? 1u : 2u; + queue.context[0x191] = 0u; + if (secondInput != 0xffffffffu) queue.context[0x192] = secondInput; + RecompileRequest request{}; + request.shader = {ShaderStage::Fragment, 0x30000u, code, 0, {}}; + request.context.waveSize = 64; + request.context.pixel = DecodePixelStageInfo(queue.context, IdentityExports); + request.target.vulkanVersion = 0x00401000u; + request.target.spirvVersion = 0x00010300u; + request.target.subgroupSize = 64; + request.target.fragmentShaderBarycentricEnabled = barycentric; + request.layout.pushConstantSizeBytes = 128; + request.useCache = false; + return Recompile(request); + }; + const auto decorations = [](const RecompileResult& result, spv::Decoration decoration) { + const auto& words = result.spirv.Words(); + std::set inputs; + for (std::size_t at = 5; at < words.size() && (words[at] >> 16u) != 0; at += words[at] >> 16u) { + if (static_cast(words[at] & 0xffffu) == spv::OpVariable && words[at + 3] == spv::StorageClassInput) inputs.insert(words[at + 2]); + } + std::vector values; + for (std::size_t at = 5; at < words.size() && (words[at] >> 16u) != 0; at += words[at] >> 16u) { + if (static_cast(words[at] & 0xffffu) == spv::OpDecorate && words[at + 2] == decoration && inputs.contains(words[at + 1])) values.push_back((words[at] >> 16u) > 3u ? words[at + 3] : words[at + 1]); + } + return values; + }; + const auto barycentricCapability = [](const RecompileResult& result) { + const auto& words = result.spirv.Words(); + for (std::size_t at = 5; at < words.size() && (words[at] >> 16u) != 0; at += words[at] >> 16u) { + if (static_cast(words[at] & 0xffffu) == spv::OpCapability && words[at + 1] == spv::CapabilityFragmentBarycentricKHR) return true; + } + return false; + }; + static constexpr std::array readsIj{0xf800180fu, 0x01000100u, 0xbf810000u}; + static constexpr std::array interpolates{0xc8080000u, 0xc8090001u, 0xf800180fu, 0x02020202u, 0xbf810000u}; + static constexpr std::array readsVertex{0xc80a0000u, 0xf800180fu, 0x02020202u, 0xbf810000u}; + static constexpr std::array shiftsIj{0x060000f0u, 0xc8080000u, 0xc8090001u, 0xf800180fu, 0x02020202u, 0xbf810000u}; + auto fragment = compile(readsIj, false); + Require(fragment.barycentricEmulation.active && fragment.barycentricEmulation.smooth && !fragment.barycentricEmulation.linear, "reading the I/J VGPRs without the extension did not emulate barycentrics"); + const auto builtins = decorations(fragment, spv::DecorationBuiltIn); + Require(std::find(builtins.begin(), builtins.end(), static_cast(spv::BuiltInBaryCoordKHR)) == builtins.end() && !barycentricCapability(fragment), "emulated barycentrics still need VK_KHR_fragment_shader_barycentric"); + Require(decorations(fragment, spv::DecorationLocation) == std::vector{0u}, "the emulated barycentrics were not read from location 0"); + Require(!compile(readsIj, true).barycentricEmulation.active, "a device with the extension emulated barycentrics"); + Require(!compile(interpolates, false).barycentricEmulation.active, "plain v_interp_p1/p2 lost host interpolation"); + Require(compile(shiftsIj, false).barycentricEmulation.active, "v_interp_p1/p2 through a changed I lost exact barycentrics"); + static constexpr std::array overwritesI{0xc8080000u, 0xc8090001u, 0x7e0002f2u, 0xf800180fu, 0x02020002u, 0xbf810000u}; + static constexpr std::array accumulatesInI{0xc8000000u, 0xc8010001u, 0xf800180fu, 0x00000000u, 0xbf810000u}; + for (const auto& code : {std::span(overwritesI), std::span(accumulatesInI)}) { + const auto reused = compile(code, false); + const auto reusedBuiltins = decorations(reused, spv::DecorationBuiltIn); + Require(!reused.barycentricEmulation.active && !barycentricCapability(reused) && std::find(reusedBuiltins.begin(), reusedBuiltins.end(), static_cast(spv::BuiltInBaryCoordKHR)) == reusedBuiltins.end(), "reusing the I/J VGPRs after v_interp_p1/p2 lost host interpolation"); + } + static constexpr std::array widensExec{0xbefe0381u, 0xc8080000u, 0xbefe03c1u, 0xc8090001u, 0xf800180fu, 0x02020202u, 0xbf810000u}; + Require(compile(widensExec, false).barycentricEmulation.active, "v_interp_p2_f32 under a wider EXEC than its v_interp_p1_f32 lost exact barycentrics"); + static constexpr std::array sharedSlot{0xc8080000u, 0xc8090001u, 0xc80e0402u, 0xf800180fu, 0x03030202u, 0xbf810000u}; + const auto shared = compile(sharedSlot, false, 0x400u); + Require(shared.barycentricEmulation.active && !barycentricCapability(shared), "inputs sharing a slot with different interpolation did not emulate per-vertex parameters"); + fragment = compile(readsVertex, false); + Require(fragment.barycentricEmulation.active && !fragment.barycentricEmulation.smooth && fragment.fragmentParameters.size() == 1u && fragment.fragmentParameters[0].perVertex, "v_interp_mov of P10 did not emulate per-vertex parameters"); + Require(decorations(fragment, spv::DecorationLocation) == std::vector{0u} && decorations(fragment, spv::DecorationPerVertexKHR).empty() && !decorations(fragment, spv::DecorationFlat).empty() && !barycentricCapability(fragment), "the emulated per-vertex parameter was not a flat array at the first free location"); + + RecompileResult vertex; + vertex.spirv = makeModule({.parameterOutput = true}); + vertex.parameterExports = {0u}; + const std::array capabilities{spv::CapabilityShader, spv::CapabilityGeometry}; + SpirvTarget target{}; + target.vulkanVersion = VK_API_VERSION_1_1; + target.spirvVersion = 0x00010300u; + target.supportedCapabilities = capabilities; + const GeometryStageLimits limits{64u, 128u, 256u, 1024u, 128u}; + auto geometry = BuildBarycentricGeometryShader(vertex, fragment, target, limits); + const std::array shaders{{{ShaderStage::Vertex, &vertex, 0}, {ShaderStage::Geometry, &geometry, 0}, {ShaderStage::Fragment, &fragment, 0}}}; + const VkPhysicalDeviceSubgroupProperties subgroup{VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_SUBGROUP_PROPERTIES}; + auto state = DecodeState(makeState()); + Require(state.topology == VK_PRIMITIVE_TOPOLOGY_TRIANGLE_LIST, "the reference state is not a triangle list"); + ValidateShaders(shaders, state, subgroup, false, false, false, true); + state.topology = VK_PRIMITIVE_TOPOLOGY_POINT_LIST; + expectFailure([&] { ValidateShaders(shaders, state, subgroup, false, false, false, true); }, "triangle primitives"); + state.topology = VK_PRIMITIVE_TOPOLOGY_TRIANGLE_STRIP; + ValidateShaders(shaders, state, subgroup, false, false, false, true); + const auto plain = compile(interpolates, false); + const std::array plainShaders{{{ShaderStage::Vertex, &vertex, 0}, {ShaderStage::Geometry, &geometry, 0}, {ShaderStage::Fragment, &plain, 0}}}; + expectFailure([&] { ValidateShaders(plainShaders, state, subgroup, false, false, false, true); }, "only for emulated barycentrics"); + expectFailure([&] { static_cast(BuildBarycentricGeometryShader(vertex, plain, target, limits)); }, "does not emulate barycentrics"); + expectFailure([&] { static_cast(BuildBarycentricGeometryShader(vertex, fragment, target, std::nullopt)); }, "geometry shaders are unavailable"); + expectFailure([&] { static_cast(BuildBarycentricGeometryShader(vertex, fragment, target, GeometryStageLimits{64u, 12u, 256u, 1024u, 128u})); }, "exceeds device limits"); + vertex.parameterExports.clear(); + expectFailure([&] { static_cast(BuildBarycentricGeometryShader(vertex, fragment, target, limits)); }, "not exported by the vertex shader"); +} + void rectListTests() { using namespace ShaderRecompiler; using namespace AgcDriver::Graphics; @@ -2041,6 +2677,10 @@ void rectListTests() { target.spirvVersion = 0x00010300u; target.supportedCapabilities = capabilities; target.tessellation = TessellationTargetLimits{32, 128, 128, 120, 4096, 128, 128}; + vertex.bindings = {{DescriptorKind::StorageBuffer, DescriptorRole::GuestBuffers, 0, 0, 1, {}}}; + const auto faultAuxiliary = BuildRectListShaders(vertex, fragment, target); + Require(faultAuxiliary.control.bindings.size() == 1 && faultAuxiliary.control.bindings[0].role == DescriptorRole::FaultBuffer && faultAuxiliary.control.bindings[0].binding == RuntimeAbi::BindingNumber(RuntimeAbi::Stage::TessellationControl, RuntimeAbi::Binding::FaultBuffer), "the rect-list control shader's fault buffer is not in the tessellation control binding group"); + vertex.bindings.clear(); for (const auto version : {0x00010300u, 0x00010400u}) { target.spirvVersion = version; auto auxiliary = BuildRectListShaders(vertex, fragment, target); @@ -2348,6 +2988,128 @@ void orderedPixelShaderTests() { expectFailure([&] { static_cast(ShaderRecompiler::Recompile(orderedPixelRequest(0x30600u, plain))); }, "fragmentShaderPixelInterlock"); } +void ProvokingVertexTests() { + using namespace AgcDriver::Graphics; + constexpr std::uint32_t lastVertex = 1u << 19u; + auto queue = makeState(); + queue.context[0x1b3] = 2; + queue.context[0x1b4] = 2; + const auto first = DecodeState(queue); + Require(first.provokingVertexMode == VK_PROVOKING_VERTEX_MODE_FIRST_VERTEX_EXT, "the default provoking vertex is not first"); + for (const auto primitive : {2u, 4u, 5u, 6u}) { + queue.userConfig[0x242] = primitive; + for (const auto last : {false, true, false}) { + queue.context[0x205] = 0x240u | (last ? lastVertex : 0u); + const auto state = DecodeState(queue); + Require(state.provokingVertexMode == (last ? VK_PROVOKING_VERTEX_MODE_LAST_VERTEX_EXT : VK_PROVOKING_VERTEX_MODE_FIRST_VERTEX_EXT), "PROVOKING_VTX_LAST did not select the requested vertex for primitive " + std::to_string(primitive)); + const auto rejection = DrawRejection(queue, false); + Require(rejection.empty(), "the draw precheck rejected provoking vertex mode for primitive " + std::to_string(primitive) + ": " + rejection); + Require(state.cullMode == first.cullMode && state.frontFace == first.frontFace && state.depthBias == first.depthBias, "PROVOKING_VTX_LAST changed another rasterization field"); + } + } + queue.context[0x205] |= lastVertex; + queue.ClearContext(); + Require((queue.context.at(0x205) & lastVertex) == 0, "context reset kept last provoking vertex mode"); + Require(DrawKeyCovers({RegisterBank::Context, 0x205}), "the draw cache key does not cover provoking vertex mode"); + const Context unsupported{}; + ValidateProvokingVertex(unsupported, first); + auto last = first; + last.provokingVertexMode = VK_PROVOKING_VERTEX_MODE_LAST_VERTEX_EXT; + expectFailure([&] { ValidateProvokingVertex(unsupported, last); }, "last provoking vertex requires VK_EXT_provoking_vertex with provokingVertexLast enabled"); + auto supported = unsupported; + supported.provokingVertexLast = true; + ValidateProvokingVertex(supported, last); + for (const auto topology : {VK_PRIMITIVE_TOPOLOGY_POINT_LIST, VK_PRIMITIVE_TOPOLOGY_LINE_LIST, VK_PRIMITIVE_TOPOLOGY_LINE_STRIP, VK_PRIMITIVE_TOPOLOGY_TRIANGLE_LIST, VK_PRIMITIVE_TOPOLOGY_TRIANGLE_STRIP, VK_PRIMITIVE_TOPOLOGY_TRIANGLE_FAN}) { + auto state = last; + state.topology = topology; + ValidateProvokingVertex(supported, state); + } + for (const auto topology : {VK_PRIMITIVE_TOPOLOGY_PATCH_LIST, VK_PRIMITIVE_TOPOLOGY_LINE_LIST_WITH_ADJACENCY, VK_PRIMITIVE_TOPOLOGY_LINE_STRIP_WITH_ADJACENCY, VK_PRIMITIVE_TOPOLOGY_TRIANGLE_LIST_WITH_ADJACENCY, VK_PRIMITIVE_TOPOLOGY_TRIANGLE_STRIP_WITH_ADJACENCY, VK_PRIMITIVE_TOPOLOGY_MAX_ENUM}) { + auto state = last; + state.topology = topology; + expectFailure([&] { ValidateProvokingVertex(supported, state); }, "last provoking vertex is unsupported for this primitive topology"); + } + const std::array flatColor{0xc8120002u, 0xc8160102u, 0xc81a0202u, 0xc81e0302u, 0xf800180fu, 0x07060504u, 0xbf810000u}; + const auto flatPixel = recompilePixel({0x400u}, flatColor); + const auto inputs = locatedInputs(flatPixel.spirv.Words()); + Require(inputs.size() == 1 && inputs[0].location == 0 && inputs[0].flat && !inputs[0].perVertex, "flat v_interp_mov P0 did not compile to a flat input"); + Require(flatPixel.fragmentParameters.size() == 1 && flatPixel.fragmentParameters[0].flat && !flatPixel.fragmentParameters[0].perVertex, "flat v_interp_mov P0 has explicit per-vertex metadata"); + const std::array flatShaders{{{ShaderRecompiler::ShaderStage::Fragment, &flatPixel, 0}}}; + ValidateProvokingVertex(supported, last, flatShaders); + ShaderRecompiler::RecompileResult fragment{}; + const std::array shaders{{{ShaderRecompiler::ShaderStage::Fragment, &fragment, 0}}}; + for (const auto topology : {VK_PRIMITIVE_TOPOLOGY_TRIANGLE_LIST, VK_PRIMITIVE_TOPOLOGY_TRIANGLE_STRIP, VK_PRIMITIVE_TOPOLOGY_TRIANGLE_FAN}) { + auto state = last; + state.topology = topology; + for (const auto flat : {false, true}) { + for (const auto perVertex : {false, true}) { + fragment.fragmentParameters = {{0, 0, flat, perVertex}}; + auto firstState = state; + firstState.provokingVertexMode = VK_PROVOKING_VERTEX_MODE_FIRST_VERTEX_EXT; + ValidateProvokingVertex(unsupported, firstState, shaders); + if (flat && perVertex) { + expectFailure([&] { ValidateProvokingVertex(supported, state, shaders); }, "last provoking vertex with explicit per-vertex flat interpolation is unsupported"); + } else if (perVertex && topology == VK_PRIMITIVE_TOPOLOGY_TRIANGLE_STRIP) { + expectFailure([&] { ValidateProvokingVertex(supported, state, shaders); }, "last provoking vertex with explicit per-vertex triangle-strip interpolation is unsupported"); + } else { + ValidateProvokingVertex(supported, state, shaders); + } + } + } + } + for (const auto path : {ShaderPath::Geometry, ShaderPath::Tessellation, ShaderPath::TessellationGeometry}) { + auto generated = last; + generated.stages.path = path; + expectFailure([&] { ValidateProvokingVertex(supported, generated); }, "last provoking vertex is unsupported for generated primitive pipelines"); + generated.provokingVertexMode = VK_PROVOKING_VERTEX_MODE_FIRST_VERTEX_EXT; + ValidateProvokingVertex(unsupported, generated); + } + for (std::uint32_t generated = 0; generated < 3; ++generated) { + auto state = last; + if (generated == 0) state.rectList = true; + if (generated == 1) state.stages.mesh.emplace(); + if (generated == 2) state.stages.tessellation.emplace(); + expectFailure([&] { ValidateProvokingVertex(supported, state); }, "last provoking vertex is unsupported for generated primitive pipelines"); + } + last.provokingVertexMode = VK_PROVOKING_VERTEX_MODE_MAX_ENUM_EXT; + expectFailure([&] { ValidateProvokingVertex(supported, last); }, "invalid provoking vertex mode"); +} + +void ProvokingVertexRenderPassTests() { + using namespace AgcDriver::Graphics; + const auto first = DecodeState(makeState()); + const std::array views{reinterpret_cast(std::uintptr_t{1})}; + const std::array modes{ + VK_PROVOKING_VERTEX_MODE_FIRST_VERTEX_EXT, VK_PROVOKING_VERTEX_MODE_FIRST_VERTEX_EXT, + VK_PROVOKING_VERTEX_MODE_LAST_VERTEX_EXT, VK_PROVOKING_VERTEX_MODE_LAST_VERTEX_EXT, + VK_PROVOKING_VERTEX_MODE_FIRST_VERTEX_EXT + }; + for (const bool perPipeline : {false, true}) { + Context context{}; + context.provokingVertexLast = true; + context.provokingVertexModePerPipeline = perPipeline; + auto state = first; + const auto initialKey = DrawRenderPassKey(context, state, views); + auto previousKey = initialKey; + auto previousMode = modes.front(); + for (const auto mode : modes) { + state.provokingVertexMode = mode; + const auto key = DrawRenderPassKey(context, state, views); + const bool reusable = key == previousKey; + Require(reusable == (perPipeline || mode == previousMode), "render-pass reuse does not respect provokingVertexModePerPipeline"); + previousKey = key; + previousMode = mode; + } + Require(previousKey == initialKey, "returning to first provoking vertex changed the render-pass identity"); + state.viewport.x += 1.0f; + Require(DrawRenderPassKey(context, state, views) == initialKey, "dynamic viewport changes prevent render-pass reuse"); + const std::array differentViews{reinterpret_cast(std::uintptr_t{2})}; + Require(DrawRenderPassKey(context, state, differentViews) != initialKey, "different attachments share a render-pass identity"); + ++state.renderExtent.width; + Require(DrawRenderPassKey(context, state, views) != initialKey, "different render extents share a render-pass identity"); + } +} + void ConservativeRasterizationTests() { auto queue = makeState(); queue.context[0x1b3] = 2; @@ -2409,6 +3171,23 @@ void ConservativeRasterizationTests() { } } +void floatControlsModeTests() { + AgcDriver::Graphics::State state{}; + state.stages.path = AgcDriver::Graphics::ShaderPath::Vertex; + ShaderRecompiler::RecompileResult vertex; + ShaderRecompiler::RecompileResult pixel; + const std::array shaders{{{ShaderRecompiler::ShaderStage::Vertex, &vertex, 0}, {ShaderRecompiler::ShaderStage::Fragment, &pixel, 0}}}; + const VkPhysicalDeviceSubgroupProperties subgroup{VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_SUBGROUP_PROPERTIES}; + for (const auto width : {16u, 32u, 64u}) { + vertex.spirv = makeModule({.floatControlsWidth = width}); + pixel.spirv = makeModule({.fragment = true, .floatControlsWidth = width}); + AgcDriver::Graphics::ValidateShaders(shaders, state, subgroup, false); + } + vertex.spirv = makeModule({.floatControlsWidth = 8u}); + pixel.spirv = makeModule({.fragment = true}); + expectFailure([&] { AgcDriver::Graphics::ValidateShaders(shaders, state, subgroup, false); }, "float controls execution modes"); +} + void pixelParameterSlotTests() { using AgcDriver::Graphics::CompiledShader; const VkPhysicalDeviceSubgroupProperties subgroup{VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_SUBGROUP_PROPERTIES}; @@ -2451,6 +3230,232 @@ void pixelParameterSlotTests() { expectFailure([&] { recompilePixel({0x423u, 0x3u}, shared); }, "passes its vertices through unchanged"); } +ShaderRecompiler::RecompileRequest packedExportRequest(const AgcDriver::QueueState& queue, const AgcDriver::Graphics::State& state, std::span code) { + ShaderRecompiler::RecompileRequest request{}; + request.shader = {ShaderRecompiler::ShaderStage::Fragment, 0x30000u, code, 0, {}}; + request.context.waveSize = 64; + request.context.pixel = AgcDriver::Graphics::DecodePixelStageInfo(queue.context, AgcDriver::Graphics::ExportMappings(state)); + request.context.pixel->targetExportPacking = AgcDriver::Graphics::ExportPackings(state); + request.target.vulkanVersion = 0x00401000u; + request.target.spirvVersion = 0x00010300u; + request.target.subgroupSize = 64; + request.layout.pushConstantSizeBytes = 128; + request.useCache = true; + return request; +} + +void unorm10_11_11TargetTests() { + using ShaderRecompiler::ColorExportPacking; + auto queue = makeState(); + queue.context[0x31c] = (queue.context.at(0x31c) & ~0x7cu) | (6u << 2u); + queue.context[0x1b3] = 2; + queue.context[0x1b4] = 2; + const auto state = AgcDriver::Graphics::DecodeState(queue); + Require(state.color.format == VK_FORMAT_R32_UINT && state.color.elementBytes == 4 && state.color.bytes == colorMemory.size() && state.color.packing == ColorExportPacking::Unorm10_11_11, "a 10_11_11 unorm target did not decode as a packed R32_UINT attachment"); + const auto packings = AgcDriver::Graphics::ExportPackings(state); + Require(packings[0] == ColorExportPacking::Unorm10_11_11 && std::all_of(packings.begin() + 1, packings.end(), [](ColorExportPacking packing) { return packing == ColorExportPacking::None; }), "export 0 did not carry the 10_11_11 unorm packing alone"); + auto changed = queue; + changed.context[0x31c] |= 7u << 8u; + const auto floating = AgcDriver::Graphics::DecodeState(changed); + Require(floating.color.format == VK_FORMAT_B10G11R11_UFLOAT_PACK32 && floating.color.packing == ColorExportPacking::None, "a 10_11_11 float target changed"); + changed = queue; + changed.context[0x31c] |= 1u << 8u; + expectFailure([&] { AgcDriver::Graphics::DecodeState(changed); }, "unsupported color format 6 number type 1"); + changed = queue; + changed.context[0x1e0] = 0x40010001u; + for (std::uint32_t i = 0; i < 4; ++i) changed.context[0x105 + i] = 0; + expectFailure([&] { AgcDriver::Graphics::DecodeState(changed); }, "blending into a 10_11_11 unorm color target"); + changed = queue; + changed.context[0x8e] = 3; + expectFailure([&] { AgcDriver::Graphics::DecodeState(changed); }, "partial writes of a 10_11_11 unorm color target"); + changed.context[0x8e] = 7; + Require(AgcDriver::Graphics::DecodeState(changed).color.packing == ColorExportPacking::Unorm10_11_11, "an RGB write of a 10_11_11 unorm target was refused"); + for (const auto format : {5u, 6u}) { + changed = queue; + changed.context[0x1c5] = format; + expectFailure([&] { AgcDriver::Graphics::DecodeState(changed); }, "color export format " + std::to_string(format) + " into a 10_11_11 unorm target"); + } + changed = queue; + changed.context[0x1c5] = 4; + Require(AgcDriver::Graphics::DecodeState(changed).color.packing == ColorExportPacking::Unorm10_11_11, "an FP16_ABGR export into a 10_11_11 unorm target was refused"); + changed = queue; + changed.context[0x31c] |= 0x10000000u; + changed.context[0x325] = 0x1234; + expectFailure([&] { AgcDriver::Graphics::DecodeState(changed); }, "DCC-compressed 10_11_11 unorm color targets"); + changed = queue; + changed.context[0x31c] |= 0x40000u; + expectFailure([&] { AgcDriver::Graphics::DecodeState(changed); }, "truncating (ROUND_MODE) 10_11_11 unorm color targets"); + + constexpr std::array code{0xf800180fu, 0x07060504u, 0xbf810000u}; + const auto request = packedExportRequest(queue, state, code); + auto plain = request; + plain.context.pixel->targetExportPacking.fill(ColorExportPacking::None); + const ShaderRecompiler::RequestSerializer serializer; + Require(serializer.Deserialize(serializer.Serialize(request)).request.context.pixel->targetExportPacking == request.context.pixel->targetExportPacking, "the 10_11_11 unorm packing was lost in serialization"); + std::vector key; + std::vector plainKey; + ShaderRecompiler::BuildPreparedShaderKey(request, key); + ShaderRecompiler::BuildPreparedShaderKey(plain, plainKey); + Require(key == plainKey, "the export packing entered the static shader ABI"); + const auto prepared = ShaderRecompiler::PrepareShader(plain); + Require(ShaderRecompiler::MatchesPreparedShader(request, *prepared), "a packed draw did not match the registered pixel shader"); + const auto packed = ShaderRecompiler::Recompile(request); + const auto unpacked = ShaderRecompiler::Recompile(plain); + Require(packed.variantId == unpacked.variantId && packed.variantId == ShaderRecompiler::GetPreparedArtifact(*prepared).variantId, "the packed draw did not reuse the registered artifact"); + Require(packed.PipelineVariantId() != unpacked.PipelineVariantId() && packed.spirv.Words() != unpacked.spirv.Words(), "the export packing did not specialize the module"); + Require(ShaderRecompiler::Recompile(request).PipelineVariantId() == packed.PipelineVariantId(), "the packed specialization was not reused"); + ShaderRecompiler::RecompileResult vertex; + vertex.spirv = makeModule({}); + const VkPhysicalDeviceSubgroupProperties subgroup{VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_SUBGROUP_PROPERTIES}; + const auto validate = [&](const ShaderRecompiler::RecompileResult& pixel, const AgcDriver::Graphics::State& target) { + const std::array shaders{{{ShaderRecompiler::ShaderStage::Vertex, &vertex, 0}, {ShaderRecompiler::ShaderStage::Fragment, &pixel, 0}}}; + return AgcDriver::Graphics::ValidateShaders(shaders, target, subgroup, false); + }; + Require(validate(packed, state) == std::set{0u}, "the packed export did not reach attachment 0"); + expectFailure([&] { validate(unpacked, state); }, "uint4 words to its packed 10_11_11 unorm attachments"); + auto plainState = state; + plainState.colors[0].packing = ColorExportPacking::None; + expectFailure([&] { validate(packed, plainState); }, "float4 colors"); + auto uintExport = request; + uintExport.context.pixel->targetOutputMode[0] = 7; + expectFailure([&] { ShaderRecompiler::Recompile(uintExport); }, "fragment output 0 has no 10_11_11 unorm packing path"); +} + +AgcDriver::QueueState dualSourceQueue() { + auto queue = makeState(); + queue.context[0x8f] = 0xffu; + queue.context[0x1c5] = 0x44u; + queue.context[0x1e0] = 0x40010f01u; + for (std::uint32_t i = 0; i < 4; ++i) queue.context[0x105 + i] = 0; + queue.context[0x1b3] = 2; + queue.context[0x1b4] = 2; + return queue; +} + +struct OutputDecorations { + std::map locations; + std::map indices; +}; + +OutputDecorations outputDecorations(std::span words) { + OutputDecorations result; + std::set outputs; + for (std::size_t at = 5; at < words.size() && (words[at] >> 16u) != 0; at += words[at] >> 16u) { + const auto op = static_cast(words[at] & 0xffffu); + if (op == spv::OpVariable && words[at + 3] == spv::StorageClassOutput) outputs.insert(words[at + 2]); + if (op != spv::OpDecorate) continue; + if (words[at + 2] == spv::DecorationLocation) result.locations[words[at + 1]] = words[at + 3]; + if (words[at + 2] == spv::DecorationIndex) result.indices[words[at + 1]] = words[at + 3]; + } + std::erase_if(result.locations, [&](const auto& entry) { return !outputs.contains(entry.first); }); + return result; +} + +void DualSourceBlendTests() { + auto queue = dualSourceQueue(); + const auto state = AgcDriver::Graphics::DecodeState(queue); + Require(state.dualSourceBlend && state.colors.size() == 1 && state.blends.size() == 1, "an ONE + SRC1_COLOR blend did not decode as dual-source blending into one target"); + Require(state.blends[0].srcColorBlendFactor == VK_BLEND_FACTOR_ONE && state.blends[0].dstColorBlendFactor == VK_BLEND_FACTOR_SRC1_COLOR && state.blends[0].srcAlphaBlendFactor == VK_BLEND_FACTOR_ONE && state.blends[0].dstAlphaBlendFactor == VK_BLEND_FACTOR_SRC1_COLOR, "BLEND_SRC1_COLOR did not become VK_BLEND_FACTOR_SRC1_COLOR"); + const std::array, 4> factors{{{15u, VK_BLEND_FACTOR_SRC1_COLOR}, {16u, VK_BLEND_FACTOR_ONE_MINUS_SRC1_COLOR}, {17u, VK_BLEND_FACTOR_SRC1_ALPHA}, {18u, VK_BLEND_FACTOR_ONE_MINUS_SRC1_ALPHA}}}; + for (const auto& [value, factor] : factors) { + auto changed = queue; + changed.context[0x1e0] = 0x60000000u | (value << 24u) | (1u << 16u) | (value << 8u) | value; + const auto decoded = AgcDriver::Graphics::DecodeState(changed); + Require(decoded.dualSourceBlend && decoded.blends[0].srcColorBlendFactor == factor && decoded.blends[0].dstColorBlendFactor == factor && decoded.blends[0].srcAlphaBlendFactor == VK_BLEND_FACTOR_ONE && decoded.blends[0].dstAlphaBlendFactor == factor, "blend factor " + std::to_string(value) + " decoded wrongly"); + } + auto changed = queue; + changed.context[0x1e0] = 0x00010f01u; + Require(!AgcDriver::Graphics::DecodeState(changed).dualSourceBlend, "SRC1 factors of a disabled blend enabled dual-source blending"); + changed = queue; + changed.context[0x1e0] = 0x40010001u; + Require(!AgcDriver::Graphics::DecodeState(changed).dualSourceBlend, "a blend without SRC1 factors enabled dual-source blending"); + for (const auto operation : {2u, 3u}) { + changed.context[0x1e0] = 0x40010f01u | (operation << 5u); + expectFailure([&] { AgcDriver::Graphics::DecodeState(changed); }, "MIN or MAX operation"); + } + changed.context[0x1e0] = 0x40010f01u | (4u << 5u); + Require(AgcDriver::Graphics::DecodeState(changed).blends[0].colorBlendOp == VK_BLEND_OP_REVERSE_SUBTRACT, "a reverse-subtract dual-source blend was refused"); + changed = queue; + changed.context[0x8f] = 0xfu; + expectFailure([&] { AgcDriver::Graphics::DecodeState(changed); }, "MRT slots 0 and 1"); + changed.context[0x8f] = 0xfffu; + changed.context[0x1c5] = 0x444u; + expectFailure([&] { AgcDriver::Graphics::DecodeState(changed); }, "MRT slots 0 and 1"); + changed.context[0x8f] = 0xf000fu; + changed.context[0x1c5] = 0x44u; + expectFailure([&] { AgcDriver::Graphics::DecodeState(changed); }, "MRT slots 0 and 1"); + alignas(256) static std::array slotOneMemory{}; + changed = queue; + for (const auto offset : {0x31bu, 0x31cu, 0x31du}) changed.context[offset + 0xfu] = changed.context.at(offset); + for (const auto offset : {0x3b0u, 0x3b8u}) changed.context[offset + 1u] = changed.context.at(offset); + changed.context[0x318 + 0xfu] = static_cast(reinterpret_cast(slotOneMemory.data()) >> 8u); + changed.context[0x390 + 1u] = static_cast(reinterpret_cast(slotOneMemory.data()) >> 40u); + changed.context[0x1e1] = 0; + changed.context[0x8e] = 0xffu; + expectFailure([&] { AgcDriver::Graphics::DecodeState(changed); }, "color writes to slot 0 alone"); + for (const auto format : {0x04u, 0x14u, 0x74u, 0x94u}) { + changed = queue; + changed.context[0x1c5] = format; + expectFailure([&] { AgcDriver::Graphics::DecodeState(changed); }, "MRT1 export in the MRT0 export format"); + } + changed = queue; + changed.context[0x1c5] = 0x99u; + Require(AgcDriver::Graphics::DecodeState(changed).dualSourceBlend, "a 32_ABGR MRT0 and MRT1 export pair was refused"); + for (const auto mask : {0x7fu, 0xf7u}) { + changed = queue; + changed.context[0x8f] = mask; + expectFailure([&] { AgcDriver::Graphics::DecodeState(changed); }, "MRT1 export with the MRT0 shader mask"); + } + changed = queue; + changed.context[0x31c] = 0x8804u; + expectFailure([&] { AgcDriver::Graphics::DecodeState(changed); }, "component-swapped color target"); + + constexpr std::array code{0xf800000fu, 0x07060504u, 0xf800181fu, 0x0b0a0908u, 0xbf810000u}; + ShaderRecompiler::RecompileRequest request{}; + request.shader = {ShaderRecompiler::ShaderStage::Fragment, 0x30000u, code, 0, {}}; + request.context.waveSize = 64; + request.context.pixel = AgcDriver::Graphics::DecodePixelStageInfo(queue.context, AgcDriver::Graphics::ExportMappings(state)); + request.context.pixel->dualSourceBlend = state.dualSourceBlend; + request.target.vulkanVersion = 0x00401000u; + request.target.spirvVersion = 0x00010300u; + request.target.subgroupSize = 64; + request.layout.pushConstantSizeBytes = 128; + request.useCache = true; + auto plain = request; + plain.context.pixel->dualSourceBlend = false; + const ShaderRecompiler::RequestSerializer serializer; + Require(serializer.Deserialize(serializer.Serialize(request)).request.context.pixel->dualSourceBlend, "dual-source blending was lost in serialization"); + std::vector key; + std::vector plainKey; + ShaderRecompiler::BuildPreparedShaderKey(request, key); + ShaderRecompiler::BuildPreparedShaderKey(plain, plainKey); + Require(key == plainKey, "dual-source blending entered the static shader ABI"); + const auto prepared = ShaderRecompiler::PrepareShader(plain); + Require(ShaderRecompiler::MatchesPreparedShader(request, *prepared), "a dual-source draw did not match the registered pixel shader"); + const auto dual = ShaderRecompiler::Recompile(request); + const auto single = ShaderRecompiler::Recompile(plain); + Require(dual.variantId == single.variantId && dual.variantId == ShaderRecompiler::GetPreparedArtifact(*prepared).variantId, "the dual-source draw did not reuse the registered artifact"); + Require(dual.PipelineVariantId() != single.PipelineVariantId() && ShaderRecompiler::Recompile(request).PipelineVariantId() == dual.PipelineVariantId(), "dual-source blending did not key its own specialization"); + const auto dualOutputs = outputDecorations(dual.spirv.Words()); + const auto singleOutputs = outputDecorations(single.spirv.Words()); + Require(dualOutputs.locations.size() == 2 && std::all_of(dualOutputs.locations.begin(), dualOutputs.locations.end(), [](const auto& entry) { return entry.second == 0u; }) && dualOutputs.indices.size() == 1 && dualOutputs.indices.begin()->second == 1u, "the MRT1 export of a dual-source draw was not the location 0 index 1 output"); + std::set singleLocations; + for (const auto& entry : singleOutputs.locations) singleLocations.insert(entry.second); + Require(singleOutputs.indices.empty() && singleLocations == std::set{0u, 1u}, "MRT0 and MRT1 without dual-source blending did not keep locations 0 and 1"); + ShaderRecompiler::RecompileResult vertex; + vertex.spirv = makeModule({}); + const VkPhysicalDeviceSubgroupProperties subgroup{VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_SUBGROUP_PROPERTIES}; + const auto validate = [&](const ShaderRecompiler::RecompileResult& pixel, const AgcDriver::Graphics::State& target) { + const std::array shaders{{{ShaderRecompiler::ShaderStage::Vertex, &vertex, 0}, {ShaderRecompiler::ShaderStage::Fragment, &pixel, 0}}}; + return AgcDriver::Graphics::ValidateShaders(shaders, target, subgroup, false); + }; + Require(validate(dual, state) == std::set{0u}, "the dual-source shader did not validate against its draw"); + expectFailure([&] { validate(single, state); }, "needs a float4 second color at location 0 index 1"); + auto singleState = state; + singleState.dualSourceBlend = false; + expectFailure([&] { validate(dual, singleState); }, "Index decoration"); +} + void validationTests() { AgcDriver::Graphics::State state{}; state.stages.path = AgcDriver::Graphics::ShaderPath::Vertex; @@ -2507,18 +3512,42 @@ void validationTests() { vertex.vertexAttributes[0].components = 2; expectFailure([&] { AgcDriver::Graphics::ValidateShaders(shaders, state, subgroup, false); }, "metadata disagrees"); Require(AgcDriver::Graphics::VertexBufferReadSize(attribute, 2, 1) == 80, "incorrect strided vertex range"); - expectFailure([&] { AgcDriver::Graphics::VertexBufferReadSize(attribute, 3, 1); }, "record count"); + Require(AgcDriver::Graphics::VertexBufferReadSize(attribute, 3, 1) == 112, "an index past the records still spans its fetch"); attribute.fetchIndex = 1; Require(AgcDriver::Graphics::VertexBufferReadSize(attribute, 100, 2) == 48, "instance attributes used the vertex index"); Require(AgcDriver::Graphics::VertexBufferReadSize(attribute, 100, 2, 1) == 80, "first instance was ignored"); - expectFailure([&] { AgcDriver::Graphics::VertexBufferReadSize(attribute, 0, 2, 2); }, "record count"); + Require(AgcDriver::Graphics::VertexBufferReadSize(attribute, 0, 2, 2) == 112, "a first instance past the records still spans its fetch"); expectFailure([&] { AgcDriver::Graphics::VertexBufferReadSize(attribute, 0, 2, 0xffffffffu); }, "instance range overflow"); - expectFailure([&] { AgcDriver::Graphics::VertexBufferReadSize(attribute, 0, 4); }, "record count"); + Require(AgcDriver::Graphics::VertexBufferReadSize(attribute, 0, 4) == 112, "an instance range past the records still spans its fetch"); + attribute.fetchIndex = 0; + attribute.resource.fields[3] = 0x1004dfacu; + Require(AgcDriver::Graphics::VertexBufferReadSize(attribute, 3, 1) == 112, "oob_select 1 past the records still spans its fetch"); + attribute.resource.fields[3] = 0x2004dfacu; + expectFailure([&] { AgcDriver::Graphics::VertexBufferReadSize(attribute, 3, 1); }, "record count"); + attribute.resource.fields[3] = 0x3004dfacu; + expectFailure([&] { AgcDriver::Graphics::VertexBufferReadSize(attribute, 3, 1); }, "record count"); + attribute.resource.fields[3] = 77u << 12u; + attribute.fetchIndex = 1; attribute.resource.fields[1] = 0; attribute.resource.fields[2] = 16; Require(AgcDriver::Graphics::VertexBufferReadSize(attribute, 100, 2) == 16, "zero stride must repeat one value"); attribute.resource.fields[2] = 8; expectFailure([&] { AgcDriver::Graphics::VertexBufferReadSize(attribute, 0, 1); }, "byte range"); + Require(!AgcDriver::Graphics::VertexFetchOutOfRange(attribute), "a structured zero-stride fetch was treated as out of range"); + attribute.resource.fields[3] = (77u << 12u) | (2u << 28u); + attribute.resource.fields[2] = 1; + Require(AgcDriver::Graphics::VertexBufferReadSize(attribute, 100, 2) == 16, "OOB_SELECT 2 must read a zero-stride fetch whole while NUM_RECORDS is nonzero"); + Require(AgcDriver::Graphics::VertexBufferExtent(attribute) == 16, "OOB_SELECT 2 extent must cover the whole fetch"); + Require(!AgcDriver::Graphics::VertexFetchOutOfRange(attribute), "OOB_SELECT 2 with records was treated as out of range"); + attribute.resource.fields[2] = 0; + Require(AgcDriver::Graphics::VertexFetchOutOfRange(attribute), "OOB_SELECT 2 without records must read zero"); + attribute.resource.fields[3] = (77u << 12u) | (3u << 28u); + Require(AgcDriver::Graphics::VertexFetchOutOfRange(attribute), "OOB_SELECT 3 without records must read zero"); + attribute.resource.fields[2] = 8; + Require(!AgcDriver::Graphics::VertexFetchOutOfRange(attribute), "OOB_SELECT 3 with records was treated as out of range"); + expectFailure([&] { AgcDriver::Graphics::VertexBufferReadSize(attribute, 0, 1); }, "byte range"); + attribute.resource.fields[2] = 16; + Require(AgcDriver::Graphics::VertexBufferReadSize(attribute, 0, 1) == 16, "OOB_SELECT 3 must read a fetch inside the range"); attribute.resource.fields[3] = 113u << 12u; expectFailure([&] { AgcDriver::Graphics::DecodeVertexFormat(attribute); }, "unsupported vertex format"); attribute.resource.fields[3] = 50u << 12u; @@ -2527,6 +3556,17 @@ void validationTests() { Require(AgcDriver::Graphics::DecodeVertexFormat(attribute).format == VK_FORMAT_A2B10G10R10_SINT_PACK32 && std::string_view(AgcDriver::Graphics::DecodeVertexFormat(attribute).scalar) == "i32", "2_10_10_10 sint vertex format was not decoded"); attribute.resource.fields[3] = 36u << 12u; Require(AgcDriver::Graphics::DecodeVertexFormat(attribute).format == VK_FORMAT_B10G11R11_UFLOAT_PACK32, "10_11_11 float vertex format was not decoded"); + for (std::uint32_t format = 1; format < 128u; ++format) { + attribute.resource.fields[3] = format << 12u; + std::string_view scalar; + try { + scalar = AgcDriver::Graphics::DecodeVertexFormat(attribute).scalar; + } catch (const std::exception&) { + continue; + } + const auto numeric = ShaderRecompiler::VertexInputNumericClass(static_cast(format)); + Require((scalar == "u32") == (numeric == ShaderRecompiler::IrTextureNumericClass::Uint) && (scalar == "i32") == (numeric == ShaderRecompiler::IrTextureNumericClass::Sint), "vertex format " + std::to_string(format) + " declares a shader input of another numeric type than its attribute"); + } attribute.resource.fields[3] = 43u << 12u; expectFailure([&] { AgcDriver::Graphics::DecodeVertexFormat(attribute); }, "unsupported vertex format"); attribute.resource.fields = {0x1000, 32u << 16u, 3, 77u << 12u}; @@ -2557,6 +3597,15 @@ void validationTests() { expectFailure([&] { AgcDriver::Graphics::ValidateShaders(shaders, state, subgroup, false); }, "unsupported device capability"); AgcDriver::Graphics::ValidateShaders(shaders, state, subgroup, false, false, true); } + { + ShaderRecompiler::RecompileResult vertex; + vertex.spirv = makeModule({}); + vertex.spirv.insert(vertex.spirv.begin() + 5, {(2u << 16u) | spv::OpCapability, static_cast(spv::CapabilityInt64Atomics)}); + const std::array shaders{{{ShaderRecompiler::ShaderStage::Vertex, &vertex, 0}, {ShaderRecompiler::ShaderStage::Fragment, &fragment, 0}}}; + const VkPhysicalDeviceSubgroupProperties subgroup{VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_SUBGROUP_PROPERTIES}; + expectFailure([&] { AgcDriver::Graphics::ValidateShaders(shaders, state, subgroup, false); }, "unsupported device capability 12"); + AgcDriver::Graphics::ValidateShaders(shaders, state, subgroup, false, false, false, false, false, true); + } for (const auto capability : {spv::CapabilityGroupNonUniform, spv::CapabilityGroupNonUniformBallot, spv::CapabilityGroupNonUniformShuffle}) { ShaderRecompiler::RecompileResult vertex; vertex.spirv = makeModule({}); @@ -2696,6 +3745,19 @@ void meshIndexBufferTests() { expectFailure([] { static_cast(AgcDriver::Graphics::MeshIndexBufferDescriptor(AgcDriver::Pm4::DrawParameters{0, 3, 2, 1, 0, true})); }, "invalid mesh index buffer range"); } +void stagingPoolBudgetTests() { + if (std::getenv("APS5_STAGING_POOL_MIB") != nullptr) return; + VkPhysicalDeviceMemoryProperties memory{}; + memory.memoryHeapCount = 2; + memory.memoryHeaps[0] = {VkDeviceSize{32} << 30u, 0}; + memory.memoryHeaps[1] = {VkDeviceSize{16} << 30u, VK_MEMORY_HEAP_DEVICE_LOCAL_BIT}; + Require(AgcDriver::Graphics::BufferPool::DeviceBudget(memory) == VkDeviceSize{2} << 30u, "the staging pool budget is not an eighth of the device-local heap"); + memory.memoryHeaps[1].size = VkDeviceSize{2} << 30u; + Require(AgcDriver::Graphics::BufferPool::DeviceBudget(memory) == VkDeviceSize{512} << 20u, "the staging pool budget fell below 512 MiB"); + memory.memoryHeapCount = 1; + Require(AgcDriver::Graphics::BufferPool::DeviceBudget(memory) == VkDeviceSize{512} << 20u, "a host heap sized the staging pool budget"); +} + void meshArgumentTests() { using AgcDriver::Graphics::MeshArguments; using AgcDriver::Graphics::ResolveMeshArguments; @@ -2733,6 +3795,44 @@ void debugBranchTests() { for (const auto opcode : {0x17u, 0x18u, 0x19u, 0x1au}) Require(recompilesDebugBranch(opcode), "a conditional debug branch did not recompile"); } +void highestDrawIndexTests() { + using AgcDriver::Graphics::HighestDrawIndex; + const auto bytesOf = [](const auto& values) { return std::as_bytes(std::span(values)); }; + const std::array narrow{0, 7, 0xffff, 3, 0xffff}; + Require(HighestDrawIndex(bytesOf(narrow), 2, true) == 7, "a 16-bit restart index counted as a vertex"); + Require(HighestDrawIndex(bytesOf(narrow), 2, false) == 0xffff, "a 16-bit all-ones index without restart was skipped"); + const std::array wide{9, 0xffffffffu, 0xffff, 2}; + Require(HighestDrawIndex(bytesOf(wide), 4, true) == 0xffff, "a 32-bit restart index counted as a vertex, or 0xffff was taken for it"); + Require(HighestDrawIndex(bytesOf(wide), 4, false) == 0xffffffffu, "a 32-bit all-ones index without restart was skipped"); + const std::array restartOnly{0xffff, 0xffff, 0xffff, 0xffff}; + Require(!HighestDrawIndex(bytesOf(restartOnly), 2, true).has_value(), "a draw of only restart indices reached a vertex"); + Require(HighestDrawIndex(bytesOf(restartOnly), 2, false) == 0xffff, "a 16-bit all-ones index without restart was skipped"); + Require(HighestDrawIndex(bytesOf(narrow).first(4), 2, true) == 7 && HighestDrawIndex(bytesOf(narrow).first(2), 2, true) == 0, "the scan read past its index range"); + expectFailure([&] { HighestDrawIndex(bytesOf(narrow), 1, false); }, "unsupported index size"); +} + +void storeAtFlipTests() { + using AgcDriver::Graphics::StorageTexture; + Require(!StorageTexture::StoreAtFlipRequested(nullptr), "an unset APS5_STORE_AT_FLIP stored at each flip"); + Require(StorageTexture::StoreAtFlipRequested("1"), "APS5_STORE_AT_FLIP=1 did not store at each flip"); + for (const char* value : {"0", "", "true", "11"}) expectFailure([&] { StorageTexture::StoreAtFlipRequested(value); }, "expected 1"); +} + +void nullVertexDescriptorTests() { + using AgcDriver::Graphics::DecodeVertexFormat; + constexpr std::array formats{VK_FORMAT_R32_SFLOAT, VK_FORMAT_R32G32_SFLOAT, VK_FORMAT_R32G32B32_SFLOAT, VK_FORMAT_R32G32B32A32_SFLOAT}; + ShaderRecompiler::VertexAttribute attribute{}; + attribute.formatComponents = 1; + for (std::uint32_t components = 1; components <= 4; ++components) { + attribute.components = components; + const auto format = DecodeVertexFormat(attribute); + Require(format.format == formats[components - 1] && format.bytes == components * 4u && std::string_view(format.scalar) == "f32", "a null V# did not read as zero floats of the attribute's width"); + Require(AgcDriver::Graphics::VertexBufferReadSize(attribute, 100000, 3, 2) == format.bytes && AgcDriver::Graphics::VertexBufferExtent(attribute) == format.bytes, "a null V# was sized as a guest range"); + } + attribute.resource.fields = {0x1000, 0, 0, 0}; + expectFailure([&] { DecodeVertexFormat(attribute); }, "unsupported vertex format"); +} + void vertexCopyTests() { using AgcDriver::Graphics::PlanVertexCopies; using AgcDriver::Graphics::VertexFetch; @@ -2741,6 +3841,12 @@ void vertexCopyTests() { const auto plan = PlanVertexCopies(fetches); Require(plan.copies.size() == 1 && plan.copies[0].first == 0x1000 && plan.copies[0].second == 0x1018 + 32 * 9 + 8, "interleaved attributes were not copied as one union"); Require(plan.copyOf == std::vector{0, 0, 0} && plan.offsets == std::vector{0x18, 0, 0xc}, "interleaved attribute offsets are wrong"); + Require(plan.alignments == std::vector{4}, "the union's alignment is not its attributes'"); + } + { + const std::array fetches{{{0x4000, 0x4100, 16, 0, 2}, {0x4004, 0x4104, 16, 0, 4}}}; + const auto plan = PlanVertexCopies(fetches); + Require(plan.copies.size() == 1 && plan.alignments == std::vector{4}, "a union did not take its strictest attribute alignment"); } { const std::array fetches{{ @@ -2769,11 +3875,77 @@ void vertexCopyTests() { } } +void vertexZeroFillTests() { + using AgcDriver::Graphics::CopyZeroPaddedDrawInput; + using AgcDriver::Graphics::VertexBufferReadSize; + constexpr std::uint32_t stride = 24; + constexpr std::uint32_t records = 3; + const ShaderRecompiler::VertexAttribute attribute{0, 4, {{0x1000u, stride << 16u, records, 77u << 12u}}, 0}; + const auto extent = VertexBufferReadSize(attribute, records, 1); + const auto valid = static_cast(stride) * records; + std::vector guest(extent); + std::fill(guest.begin(), guest.begin() + static_cast(valid), std::byte{0x7a}); + std::fill(guest.begin() + static_cast(valid), guest.end(), std::byte{0x55}); + const auto copy = CopyZeroPaddedDrawInput(mockContext(), reinterpret_cast(guest.data()), extent, valid); + const auto bytes = copy.buffer->Bytes(); + Require(bytes.size() == extent, "a zero-padded vertex fetch changed its size"); + Require(std::all_of(bytes.begin(), bytes.begin() + static_cast(valid), [](std::byte value) { return value == std::byte{0x7a}; }), "a vertex fetch did not copy the bytes within the record count"); + Require(std::all_of(bytes.begin() + static_cast(valid), bytes.end(), [](std::byte value) { return value == std::byte{0}; }), "a vertex fetch past the record count did not read as zeros"); +} + +void vertexZeroFillMergeTests() { + using AgcDriver::Graphics::PlanVertexCopies; + using AgcDriver::Graphics::SoloZeroPaddedFetchIndices; + using AgcDriver::Graphics::VertexFetch; + const std::vector fetches{{0x1000u, 0x1000u + 88u, 24u, 0u, 4u}, {0x1010u, 0x1010u + 80u, 24u, 0u, 4u}}; + Require(PlanVertexCopies(fetches).copies.size() == 1, "interleaved attributes share one vertex copy"); + Require(SoloZeroPaddedFetchIndices(fetches, {88u, 80u}).empty(), "fully valid fetches share the straight copy"); + const auto solo = SoloZeroPaddedFetchIndices(fetches, {88u, 72u}); + Require(solo.size() == 1 && solo[0] == 1, "a fetch past the records is copied alone"); +} + +void parallelCompareTests() { + using AgcDriver::Graphics::CompareSpan; + using AgcDriver::Graphics::CompareSpansFrom; + constexpr std::size_t block = 65536; + constexpr std::size_t count = 48; + std::vector first(block * count + 100, std::byte{7}); + auto second = first; + const std::array changedAt{3 * block + 17, 20 * block, 33 * block + block - 1, 47 * block + 120}; + for (const auto at : changedAt) second[at] = std::byte{9}; + std::vector spans; + for (std::size_t k = 0; k < count; ++k) { + const auto length = k + 1 == count ? block + 100 : block; + spans.push_back({first.data() + k * block, second.data() + k * block, length}); + } + const auto expected = [&](std::size_t k) { + return std::none_of(changedAt.begin(), changedAt.end(), [&](std::size_t at) { return at >= k * block && at < k * block + spans[k].bytes; }); + }; + for (const std::size_t threshold : {std::size_t{0}, std::size_t{1} << 40u}) { + for (int round = 0; round < 20; ++round) { + std::vector equal(count, 2); + CompareSpansFrom(spans, equal, threshold); + for (std::size_t k = 0; k < count; ++k) Require((equal[k] == 1) == expected(k) && equal[k] <= 1, "parallel compare misreported a span"); + } + } + std::vector left(count, 2), right(count, 2); + std::thread other([&] { + for (int round = 0; round < 50; ++round) CompareSpansFrom(spans, right, 0); + }); + for (int round = 0; round < 50; ++round) CompareSpansFrom(spans, left, 0); + other.join(); + for (std::size_t k = 0; k < count; ++k) Require((left[k] == 1) == expected(k) && (right[k] == 1) == expected(k), "concurrent parallel compares misreported a span"); + std::vector none; + CompareSpansFrom({}, none, 0); +} + int main() { #ifdef _WIN32 _putenv_s("APS5_PIN_WAIT_MS", "200"); + _putenv_s("APS5_HEAP_MIRROR_MIB", "4"); #else setenv("APS5_PIN_WAIT_MS", "200", 1); + setenv("APS5_HEAP_MIRROR_MIB", "4", 1); #endif try { { @@ -2813,23 +3985,35 @@ int main() { srgb8TargetTests(); DepthClipTests(); DepthStencilTests(); + OneDimensionalColorTests(); + htileDepthClearTests(); ZExportTests(); + unorm10_11_11TargetTests(); DepthBoundsBiasTests(); conservativeZExportTests(); orderedPixelShaderTests(); + ProvokingVertexTests(); + ProvokingVertexRenderPassTests(); ConservativeRasterizationTests(); DisabledColorTests(); + OpaqueDestinationAlphaTests(); CompactedExportTests(); ReversedComponentOrderTests(); + DualSourceBlendTests(); metadataPassTests(); cmaskTests(); + uint16ExportTests(); + uint8x4TargetTests(); ShaderStageTests(); TuningFieldTests(); PixelInputLayoutTests(); + ColorPipeBankXorTests(); ComputeScratchTests(); + shaderUserDataTailPaddingTests(); InitialContextTests(); pushConstantTests(); resourceTests(); + descriptorSnapshotTests(); descriptorCacheTests(); misalignedShaderDataTests(); debugBranchTests(); @@ -2838,16 +4022,25 @@ int main() { sampledBudgetReportTests(); meshArgumentTests(); meshIndexBufferTests(); + stagingPoolBudgetTests(); validationTests(); vertexCopyTests(); + highestDrawIndexTests(); + storeAtFlipTests(); + nullVertexDescriptorTests(); + vertexZeroFillTests(); + vertexZeroFillMergeTests(); + parallelCompareTests(); pixelParameterSlotTests(); rectListTests(); + floatControlsModeTests(); + barycentricEmulationTests(); mock = MockVulkan{}; auto bdaContext = mockContext(); bdaContext.bufferDeviceAddress = true; bdaContext.limits.maxStorageBufferRange = 1u << 27; RunBdaResourceTests(bdaContext, { - [](VkBuffer buffer) -> std::span { return mock.memories.at(mock.bufferMemory.at(buffer)); }, + [](VkBuffer buffer) -> std::span { return mockBufferMemory(buffer); }, [](std::uint32_t binding) { for (auto it = mock.writes.rbegin(); it != mock.writes.rend(); ++it) { if (it->binding == binding) return it->buffers.at(0); @@ -2857,8 +4050,12 @@ int main() { [](VkDeviceAddress address) { const auto offset = address - 0x100000000000ULL; const auto buffer = reinterpret_cast(offset / 0x10000); - return std::span(mock.memories.at(mock.bufferMemory.at(buffer))).subspan(offset % 0x10000); - } + return mockBufferMemory(buffer).subspan(offset % 0x10000); + }, + [](std::optional headroom) { + mock.memoryLimit = headroom.has_value() ? std::optional(mock.allocatedBytes + *headroom) : std::nullopt; + }, + [] { return mock.allocationAttempts; } }); Require(mock.live == 0, "BDA resources leaked Vulkan objects"); RunGuestAllocationTests(); @@ -2868,6 +4065,7 @@ int main() { RunTextureFormatTests(); RunTextureTilingTests(); RunGuestTextureResourceTests(); + RunDepthSurfaceReuseTests(); RunGuestSamplerResourceTests(); mock = MockVulkan{}; auto textureDetilerContext = mockContext(); diff --git a/core/libs/prx/libSceAgcDriver/tests/GraphicsTests.hpp b/core/libs/prx/libSceAgcDriver/tests/GraphicsTests.hpp index f3a1e91df..3a48ac3e9 100644 --- a/core/libs/prx/libSceAgcDriver/tests/GraphicsTests.hpp +++ b/core/libs/prx/libSceAgcDriver/tests/GraphicsTests.hpp @@ -32,6 +32,7 @@ struct TextureDetilerTestAccess { void RunTextureFormatTests(); void RunTextureTilingTests(); void RunGuestTextureResourceTests(); +void RunDepthSurfaceReuseTests(); void RunGuestSamplerResourceTests(); void RunTextureDetilerTests(const AgcDriver::Graphics::Context& context, const TextureDetilerTestAccess& access); diff --git a/core/libs/prx/libSceAgcDriver/tests/GuestAllocations.cpp b/core/libs/prx/libSceAgcDriver/tests/GuestAllocations.cpp index b016d793e..f4f02db6a 100644 --- a/core/libs/prx/libSceAgcDriver/tests/GuestAllocations.cpp +++ b/core/libs/prx/libSceAgcDriver/tests/GuestAllocations.cpp @@ -34,8 +34,8 @@ void RunGuestLeaseWaitTests() { const auto address = reinterpret_cast(memory.data()); { GuestAllocations::Mutation mutation; - mutation.Add(memory.data(), 64, true, true); - mutation.Add(memory.data() + 64, 64, true, true); + mutation.Add(memory.data(), 64, true, true, true); + mutation.Add(memory.data() + 64, 64, true, true, true); } auto lease = GuestAllocations::GuestAllocationsAcquire_nid_postfix(); std::erase_if(lease, [&](const auto& range) { return range->address != address; }); @@ -83,7 +83,7 @@ void RunGuestAllocationTests() { reject([&] { GuestHeap::GuestHeapReallocate_nid_postfix(pointer, 64); }); GuestAllocations::Mutation mutation; bool applied = false; - reject([&] { mutation.Protect(pointer, 32, true, false, [&] { applied = true; }); }); + reject([&] { mutation.Protect(pointer, 32, true, false, true, [&] { applied = true; }); }); Require(!applied, "pinned guest protection changed"); } pointer = GuestHeap::GuestHeapReallocate_nid_postfix(pointer, 64); @@ -102,9 +102,9 @@ void RunGuestAllocationTests() { std::array mapping{}; { GuestAllocations::Mutation mutation; - mutation.Add(mapping.data(), mapping.size(), true, true); + mutation.Add(mapping.data(), mapping.size(), true, true, true); reject([&] { mutation.RequireAvailable(mapping.data() + 32, 16); }); - mutation.Protect(mapping.data() + 32, 32, true, false, [] {}); + mutation.Protect(mapping.data() + 32, 32, true, false, true, [] {}); } { const auto lease = GuestAllocations::GuestAllocationsAcquire_nid_postfix(); @@ -120,7 +120,7 @@ void RunGuestAllocationTests() { applied = true; }); Require(applied, "partial unmap callback was not called"); - reject([&] { mutation.Protect(mapping.data(), mapping.size(), true, true, [] {}); }); + reject([&] { mutation.Protect(mapping.data(), mapping.size(), true, true, true, [] {}); }); mutation.Unmap(mapping.data(), 32, [&](const void*, std::size_t, const void* allocation, bool last) { Require(allocation == mapping.data() && !last, "first fragment released remaining mapping"); }); @@ -144,10 +144,11 @@ void RunGuestAllocationTests() { imageRangeCount = lease.size(); const auto found = std::find_if(lease.begin(), lease.end(), [&](const auto& range) { return imageAddress >= range->address && imageAddress - range->address < range->bytes; }); Require(found != lease.end() && (*found)->writable && !(*found)->releasable, "main image registration is missing or releasable"); + Require(std::none_of(lease.begin(), lease.end(), [](const auto& range) { return !range->releasable && range->gpu; }), "main image registered as GPU-mapped"); allocationAddress = (*found)->allocationAddress; GuestAllocations::Mutation mutation; bool applied = false; - reject([&] { mutation.Protect(&imageProbe, 1, true, false, [&] { applied = true; }); }); + reject([&] { mutation.Protect(&imageProbe, 1, true, false, true, [&] { applied = true; }); }); Require(!applied, "pinned image protection changed"); } { @@ -157,12 +158,12 @@ void RunGuestAllocationTests() { bool applied = false; reject([&] { mutation.Unmap(&imageProbe, 1, [&](const void*, std::size_t, const void*, bool) { applied = true; }); }); Require(!applied, "image memory was unmapped"); - reject([&] { mutation.Protect(&imageProbe, 1, true, false, [] { throw std::runtime_error("host protection failure"); }); }); + reject([&] { mutation.Protect(&imageProbe, 1, true, false, true, [] { throw std::runtime_error("host protection failure"); }); }); } Require(GuestAllocations::GuestAllocationsAcquire_nid_postfix().size() == imageRangeCount, "failed image protection changed registry ranges"); { GuestAllocations::Mutation mutation; - mutation.Protect(&imageProbe, 1, true, true, [] {}); + mutation.Protect(&imageProbe, 1, true, true, true, [] {}); bool applied = false; reject([&] { mutation.Unmap(&imageProbe, 1, [&](const void*, std::size_t, const void*, bool) { applied = true; }); }); Require(!applied, "split image memory became releasable"); diff --git a/core/libs/prx/libSceAgcDriver/tests/GuestSamplerResource.cpp b/core/libs/prx/libSceAgcDriver/tests/GuestSamplerResource.cpp index 4a423ed0a..b266d39a4 100644 --- a/core/libs/prx/libSceAgcDriver/tests/GuestSamplerResource.cpp +++ b/core/libs/prx/libSceAgcDriver/tests/GuestSamplerResource.cpp @@ -93,6 +93,7 @@ struct SamplerCapture { bool created = false; const void* next = nullptr; VkSamplerReductionMode reductionMode = VK_SAMPLER_REDUCTION_MODE_MAX_ENUM; + VkSamplerCreateFlags flags = 0; }; SamplerCapture samplerCapture; @@ -105,6 +106,7 @@ THandle fakeHandle(std::uint64_t value) { VKAPI_ATTR VkResult VKAPI_CALL captureCreateSampler(VkDevice, const VkSamplerCreateInfo* info, const VkAllocationCallbacks*, VkSampler* sampler) { samplerCapture.created = true; samplerCapture.next = info->pNext; + samplerCapture.flags = info->flags; const auto* reduction = static_cast(info->pNext); if (reduction != nullptr && reduction->sType == VK_STRUCTURE_TYPE_SAMPLER_REDUCTION_MODE_CREATE_INFO_EXT && reduction->pNext == nullptr) samplerCapture.reductionMode = reduction->reductionMode; *sampler = fakeHandle(0x5a); @@ -211,6 +213,20 @@ void RunSamplerReductionTests(const Fields& base) { reject([&] { RequireFilterMinmax(context, VK_FORMAT_R32G32_SFLOAT, 0b011u, samplers); }, "does not support min/max filtering"); reject([&] { RequireFilterMinmax(context, VK_FORMAT_R32_SFLOAT, 0b1000u, samplers); }, "sampler element 3, which its shader does not bind"); + const auto borderSampler = [&](std::uint32_t clamp, std::uint32_t borderColorType) { + Fields fields = base; + fields.clampY = clamp; + fields.borderColorType = borderColorType; + return std::make_shared(context, DecodeSamplerResource(pack(fields))); + }; + const std::array borderSamplers{borderSampler(6u, 0u), borderSampler(6u, 1u), borderSampler(6u, 2u), borderSampler(2u, 1u)}; + Require(!borderSamplers[0]->ReadsOpaqueBlackBorder() && borderSamplers[1]->ReadsOpaqueBlackBorder() && !borderSamplers[2]->ReadsOpaqueBlackBorder() && !borderSamplers[3]->ReadsOpaqueBlackBorder(), "opaque black border detection is wrong"); + for (std::uint32_t swizzle = 0; swizzle <= 5u; ++swizzle) RequireBorderSwizzle(swizzle, 0b1101u, borderSamplers); + RequireBorderSwizzle(0u, 0b0010u, borderSamplers); + RequireBorderSwizzle(4u, 0b0010u, borderSamplers); + for (std::uint32_t swizzle : {1u, 2u, 3u, 5u}) reject([&] { RequireBorderSwizzle(swizzle, 0b0010u, borderSamplers); }, "moves the alpha channel is sampled through an opaque black border"); + reject([&] { RequireBorderSwizzle(1u, 0b10000u, borderSamplers); }, "sampler element 4, which its shader does not bind"); + const auto pointFiltered = [](const std::array& words) { return ShaderRecompiler::PointFilteredSamplerWord(words[0], words[2]); }; Require(pointFiltered(pack(base)) == 0x05000000u, "a point-filtered weighted-average sampler kept its bilinear or linear mip filter"); Require(pointFiltered(pack(minPoint)) == pack(minPoint)[2], "a point-filtered min sampler that already point-samples changed"); @@ -221,6 +237,47 @@ void RunSamplerReductionTests(const Fields& base) { reject([&] { pointFiltered(pack(maxLinearMip)); }, "needs point filtering"); } +void RunNonSeamlessCubeTests(const Fields& base) { + Fields nonSeamless = base; + nonSeamless.disableCubeWrap = true; + const auto decoded = DecodeSamplerResource(pack(nonSeamless)); + Require(decoded.nonSeamlessCube, "DISABLE_CUBE_WRAP was not decoded as a non-seamless cube sampler"); + Require(!DecodeSamplerResource(pack(base)).nonSeamlessCube, "a sampler without DISABLE_CUBE_WRAP decoded as non-seamless"); + + Context context{}; + context.limits.maxSamplerAnisotropy = 1.0f; + context.deviceProc = captureProc; + context.nonSeamlessCubeMap = true; + Require((createSampler(context, decoded).flags & VK_SAMPLER_CREATE_NON_SEAMLESS_CUBE_MAP_BIT_EXT) != 0u, "a non-seamless cube sampler was created without VK_SAMPLER_CREATE_NON_SEAMLESS_CUBE_MAP_BIT_EXT"); + Require(createSampler(context, DecodeSamplerResource(pack(base))).flags == 0u, "a seamless sampler was created with flags"); + const std::array honored{std::make_shared(context, decoded)}; + Require(!honored[0]->RequiresNonSeamlessCube(), "a sampler created with the non-seamless flag still restricts cube textures"); + RequireNonSeamlessCube(true, 0b1u, honored); + + context.nonSeamlessCubeMap = false; + Require(createSampler(context, decoded).flags == 0u, "a sampler set the non-seamless flag without the device feature"); + const std::array unsupported{std::make_shared(context, DecodeSamplerResource(pack(base))), std::make_shared(context, decoded)}; + Require(unsupported[1]->RequiresNonSeamlessCube(), "a non-seamless sampler without the device feature does not restrict cube textures"); + RequireNonSeamlessCube(false, 0b11u, unsupported); + RequireNonSeamlessCube(true, 0b01u, unsupported); + reject([&] { RequireNonSeamlessCube(true, 0b10u, unsupported); }, "disables seamless cube filtering, which needs VK_EXT_non_seamless_cube_map"); +} + +void RunSamplerCacheDegammaTests(const Fields& base) { + Context context{}; + context.limits.maxSamplerAnisotropy = 1.0f; + context.deviceProc = captureProc; + Fields forcedSrgb = base; + forcedSrgb.forceSrgb = true; + const auto words = pack(forcedSrgb); + SamplerCache unpaired; + reject([&] { unpaired.Get(context, words, false); }, "forces sRGB decoding"); + SamplerCache cache; + const auto paired = cache.Get(context, words, false, false, true); + Require(paired->ForcesDegamma() && cache.Get(context, words, false, false, true) == paired && cache.Misses() == 1u, "a paired FORCE_DEGAMMA sampler must be created once and then served from the cache"); + reject([&] { cache.Get(context, words, false); }, "forces sRGB decoding"); +} + } void RunGuestSamplerResourceTests() { @@ -360,9 +417,14 @@ void RunGuestSamplerResourceTests() { reject([&] { DecodeSamplerResource(truncatedBlend, true); }, "unnormalized coordinates with MC_COORD_TRUNC"); rejectUnnormalized(base, "bound as unnormalized without FORCE_UNNORMALIZED"); - Fields badSrgb = base; - badSrgb.forceSrgb = true; - rejectFields(badSrgb, "forces sRGB decoding"); + Fields forcedSrgb = base; + forcedSrgb.forceSrgb = true; + rejectFields(forcedSrgb, "forces sRGB decoding"); + const auto forced = DecodeSamplerResource(pack(forcedSrgb), false, true); + const auto plain = DecodeSamplerResource(pack(base)); + Require(forced.forceDegamma && !plain.forceDegamma, "FORCE_DEGAMMA must be decoded into forceDegamma"); + Require(forced.magFilter == plain.magFilter && forced.minFilter == plain.minFilter && forced.mipmapMode == plain.mipmapMode && forced.maxLod == plain.maxLod, "FORCE_DEGAMMA must not change the host sampler state"); + RunSamplerCacheDegammaTests(base); Fields truncated = base; truncated.truncCoord = true; @@ -372,10 +434,7 @@ void RunGuestSamplerResourceTests() { truncated.anisoBias = 1; static_cast(DecodeSamplerResource(pack(truncated))); - Fields badCubeWrap = base; - badCubeWrap.disableCubeWrap = true; - rejectFields(badCubeWrap, "seamless cube filtering"); - + RunNonSeamlessCubeTests(base); RunSamplerReductionTests(base); Fields badDegamma = base; @@ -390,9 +449,31 @@ void RunGuestSamplerResourceTests() { badPreclamp.pointPreclamp = true; rejectFields(badPreclamp, "point preclamping"); - Fields badAnisoOverride = base; - badAnisoOverride.anisoOverride = true; - rejectFields(badAnisoOverride, "anisotropy override"); + Fields anisoOverride = base; + anisoOverride.xyMagFilter = 2; + anisoOverride.xyMinFilter = 2; + anisoOverride.zFilter = 2; + anisoOverride.maxAnisoRatio = 3; + const auto withoutOverride = DecodeSamplerResource(pack(anisoOverride)); + anisoOverride.anisoOverride = true; + const auto withOverride = DecodeSamplerResource(pack(anisoOverride)); + Require(withOverride.anisotropyEnable == withoutOverride.anisotropyEnable && withOverride.maxAnisotropy == withoutOverride.maxAnisotropy && withOverride.magFilter == withoutOverride.magFilter && withOverride.minFilter == withoutOverride.minFilter, "ANISO_OVERRIDE changed the decoded sampler"); + const auto overrideWords = pack(anisoOverride); + Require(!SingleLevelSamplerWords(overrideWords, false, true).has_value(), "ANISO_OVERRIDE changed a sampler of mipmapped images"); + Require(!SingleLevelSamplerWords(overrideWords, false, false).has_value(), "ANISO_OVERRIDE changed a sampler with no paired image"); + const auto singleLevel = SingleLevelSamplerWords(overrideWords, true, false); + Require(singleLevel.has_value() && (*singleLevel)[0] == overrideWords[0] && (*singleLevel)[1] == overrideWords[1] && (*singleLevel)[3] == overrideWords[3], "ANISO_OVERRIDE did not keep the sampler's other words"); + const auto singleLevelSampler = DecodeSamplerResource(*singleLevel); + Require(!singleLevelSampler.anisotropyEnable && singleLevelSampler.maxAnisotropy == 1.0f && singleLevelSampler.magFilter == withOverride.magFilter && singleLevelSampler.minFilter == withOverride.minFilter && singleLevelSampler.mipmapMode == withOverride.mipmapMode, "ANISO_OVERRIDE left anisotropy on a single-level image's sampler"); + reject([&] { SingleLevelSamplerWords(overrideWords, true, true); }, "both single-level and mipmapped"); + Require(!SingleLevelSamplerWords(pack(base), true, false).has_value(), "a sampler without ANISO_OVERRIDE changed"); + Fields linearOverride = base; + linearOverride.anisoOverride = true; + Require(!SingleLevelSamplerWords(pack(linearOverride), true, true).has_value(), "ANISO_OVERRIDE changed a sampler without anisotropy"); + anisoOverride.xyMagFilter = 3; + anisoOverride.xyMinFilter = 3; + const auto linearPlain = DecodeSamplerResource(*SingleLevelSamplerWords(pack(anisoOverride), true, false)); + Require(!linearPlain.anisotropyEnable && linearPlain.magFilter == VK_FILTER_LINEAR && linearPlain.minFilter == VK_FILTER_LINEAR, "ANISO_OVERRIDE did not keep linear filtering"); Fields badBlendZero = base; badBlendZero.blendZeroPrt = true; diff --git a/core/libs/prx/libSceAgcDriver/tests/GuestTextureResource.cpp b/core/libs/prx/libSceAgcDriver/tests/GuestTextureResource.cpp index 44b30348e..0539db1b3 100644 --- a/core/libs/prx/libSceAgcDriver/tests/GuestTextureResource.cpp +++ b/core/libs/prx/libSceAgcDriver/tests/GuestTextureResource.cpp @@ -1,6 +1,7 @@ #include "GraphicsTests.hpp" #include "prx/libSceAgcDriver/Graphics/include/DccMetadata.hpp" #include "prx/libSceAgcDriver/Graphics/include/GuestTextureResource.hpp" +#include "prx/libSceAgcDriver/Graphics/include/ShaderResources.hpp" #include "prx/libSceAgcDriver/Graphics/include/TextureTiling.hpp" #include #include @@ -119,8 +120,10 @@ void RunGuestTextureResourceTests() { tileModes.tileModeRaw = 0x09; Require(DecodeTextureResource(pack(tileModes)).tileMode == TextureTileMode::kStandard64KB, "tile mode 0x09 must decode to standard 64KB"); tileModes.tileModeRaw = 0x1b; - rejectFields(tileModes, "pipe/bank XOR base"); + const auto xorBase = DecodeTextureResource(pack(tileModes)); + Require(xorBase.baseAddress == 0x12340000ull && xorBase.pipeBankXor == 0x5600u, "a 64 KiB XOR swizzle did not split its base into the block base and its pipe/bank XOR"); tileModes.base40 = 0x120000ull; + Require(DecodeTextureResource(pack(tileModes)).pipeBankXor == 0u, "an aligned 64 KiB XOR base gained a pipe/bank XOR"); Require(DecodeTextureResource(pack(tileModes)).tileMode == TextureTileMode::RenderTarget64KB, "tile mode 0x1b must decode to render target 64KB"); constexpr std::array, 7> added{{{0x02, TextureTileMode::kD256B}, {0x06, TextureTileMode::kD4KB}, {0x0a, TextureTileMode::kD64KB}, {0x11, TextureTileMode::kS64KBT}, {0x12, TextureTileMode::kD64KBT}, {0x15, TextureTileMode::kS4KBX}, {0x16, TextureTileMode::kD4KBX}}}; for (const auto& [raw, mode] : added) { @@ -131,10 +134,18 @@ void RunGuestTextureResourceTests() { tileModes.base40 = 0x120010ull; Require(DecodeTextureResource(pack(tileModes)).tileMode == TextureTileMode::kD4KBX, "a 4 KiB XOR swizzle must accept a 4 KiB aligned base"); tileModes.base40 = 0x120011ull; - rejectFields(tileModes, "pipe/bank XOR base"); + const auto xor4Kb = DecodeTextureResource(pack(tileModes)); + Require(xor4Kb.baseAddress == 0x12001000ull && xor4Kb.pipeBankXor == 0x100u, "a 4 KiB XOR swizzle did not split its base into the block base and its pipe/bank XOR"); tileModes.tileModeRaw = 0x12; tileModes.base40 = 0x120010ull; - rejectFields(tileModes, "pipe/bank XOR base"); + const auto xorT = DecodeTextureResource(pack(tileModes)); + Require(xorT.baseAddress == 0x12000000ull && xorT.pipeBankXor == 0x1000u, "a 64 KiB T swizzle did not split its base into the block base and its pipe/bank XOR"); + tileModes.tileModeRaw = 0x09; + const auto standard = DecodeTextureResource(pack(tileModes)); + Require(standard.baseAddress == 0x12001000ull && standard.pipeBankXor == 0u, "a swizzle without XOR addressing split its base"); + tileModes.tileModeRaw = 0x1b; + tileModes.base40 = 0x56ull; + rejectFields(tileModes, "null base address"); tileModes.base40 = base.base40; tileModes.tileModeRaw = 0x03; rejectFields(tileModes, "unsupported tile mode"); @@ -326,9 +337,16 @@ void RunGuestTextureResourceTests() { Require(image.dccPipeAligned == pipeAligned && stored == expected && std::all_of(keys, keys + expected, [](std::uint8_t key) { return key == 0xff; }), std::string("a 1920x1080 SW_64KB_R_X storage image with ") + (pipeAligned ? "pipe-aligned" : "unaligned") + " DCC stored " + std::to_string(stored) + " uncompressed keys, expected " + std::to_string(expected)); } - Fields badSwizzle = base; - badSwizzle.bcSwizzle = 1; - rejectFields(badSwizzle, "BC swizzle"); + for (std::uint32_t swizzle = 0; swizzle <= 5u; ++swizzle) { + Fields borderSwizzle = base; + borderSwizzle.bcSwizzle = swizzle; + Require(DecodeTextureResource(pack(borderSwizzle)).bcSwizzle == swizzle, "a texture descriptor lost its BC swizzle " + std::to_string(swizzle)); + } + for (std::uint32_t swizzle : {6u, 7u}) { + Fields badSwizzle = base; + badSwizzle.bcSwizzle = swizzle; + rejectFields(badSwizzle, "reserved BC swizzle"); + } Fields badLevels = base; badLevels.baseLevel = 2; @@ -408,4 +426,24 @@ void RunGuestTextureResourceTests() { Require(!MatchesGuestDimension(Shape::Image3D, TextureDimension::k2D), "3D shape must never match a guest dimension"); Require(!MatchesGuestDimension(Shape::Image3D, TextureDimension::k2DArray), "3D shape must never match a guest dimension"); Require(!MatchesGuestDimension(Shape::Image3D, TextureDimension::kCube), "3D shape must never match a guest dimension"); + + Fields streamed = base; + streamed.minLodWarn = 0xabc; + streamed.mipStatsCntEn = true; + streamed.mipStatsCntId = 0x5a; + Require(SampledTexturesShareEntry(pack(base), pack(streamed)), "T#s differing only in MIN_LOD_WARN, MIP_STATS_COUNTER_EN and MIP_STATS_COUNTER_ID must share a sampled texture entry"); + for (const auto& field : {&Fields::minLodWarn, &Fields::mipStatsCntId}) { + Fields one = base; + one.*field = 1; + Require(SampledTexturesShareEntry(pack(base), pack(one)), "a T# differing in one streaming-feedback field must share a sampled texture entry"); + } + Fields moved = streamed; + moved.base40 = base.base40 + 1; + Require(!SampledTexturesShareEntry(pack(base), pack(moved)), "T#s with different base addresses must not share a sampled texture entry"); + Fields reformatted = streamed; + reformatted.format = 57; + Require(!SampledTexturesShareEntry(pack(base), pack(reformatted)), "T#s with different formats must not share a sampled texture entry"); + Fields cornered = base; + cornered.cornerSample = true; + Require(!SampledTexturesShareEntry(pack(base), pack(cornered)), "T#s differing in a field next to the feedback fields must not share a sampled texture entry"); } diff --git a/core/libs/prx/libSceAgcDriver/tests/Mesh.cpp b/core/libs/prx/libSceAgcDriver/tests/Mesh.cpp index d4ce2560d..9cc2ceaa9 100644 --- a/core/libs/prx/libSceAgcDriver/tests/Mesh.cpp +++ b/core/libs/prx/libSceAgcDriver/tests/Mesh.cpp @@ -38,6 +38,13 @@ alignas(256) constexpr std::array GeometryCode{ 0xdbfc0410, 0x3000002b, 0xbf8cc07f, 0xf80008cf, 0x2f2e2d2c, 0xf800020f, 0x33323130, 0xbf810000, }; +alignas(256) constexpr auto Wave32GeometryCode = [] { + auto code = GeometryCode; + code[10] = 0x04190a6bu; + code[28] = 0x04250a02u; + return code; +}(); + alignas(256) constexpr std::array PixelCode{ 0xc8020002, 0xc8060102, 0xc80a0202, 0xc80e0302, 0xf800180f, 0x03020100, 0xbf810000, }; @@ -119,6 +126,7 @@ struct MeshDraw { std::array vertexBuffer; std::uint32_t geometryPushBytes = ShaderRecompiler::MeshDrawPushOffsetBytes; bool restart = false; + std::uint32_t waveSize = 64u; }; void DrawMesh(AgcDriver::VulkanDevice& device, const MeshDraw& setup) { @@ -127,10 +135,11 @@ void DrawMesh(AgcDriver::VulkanDevice& device, const MeshDraw& setup) { const auto index = AgcDriver::Graphics::MeshIndexBufferDescriptor(setup.draw); std::copy(index.begin(), index.end(), userData.begin() + ShaderRecompiler::MeshIndexBufferUserWord); std::copy(setup.vertexBuffer.begin(), setup.vertexBuffer.end(), userData.begin() + 8); - const std::array geometryMemory{{{reinterpret_cast(GeometryCode.data()), std::as_bytes(std::span(GeometryCode))}}}; + const auto& code = setup.waveSize == 32u ? Wave32GeometryCode : GeometryCode; + const std::array geometryMemory{{{reinterpret_cast(code.data()), std::as_bytes(std::span(code))}}}; ShaderRecompiler::RecompileRequest geometry{ - {ShaderStage::Mesh, reinterpret_cast(GeometryCode.data()), GeometryCode, 0, {}}, - {64, 0, userData, std::nullopt, std::nullopt, ShaderRecompiler::ShaderVertexStageInfo{}, geometryMemory}, + {ShaderStage::Mesh, reinterpret_cast(code.data()), code, 0, {}}, + {setup.waveSize, 0, userData, std::nullopt, std::nullopt, ShaderRecompiler::ShaderVertexStageInfo{}, geometryMemory}, target, {0, 0, 0, setup.geometryPushBytes}, ShaderRecompiler::GraphicsCompileContext{0, {}, setup.mesh, std::nullopt, {setup.draw.indexAddress, setup.draw.indexCount, setup.draw.indexSize, setup.draw.instanceCount}} @@ -159,7 +168,7 @@ void DrawMesh(AgcDriver::VulkanDevice& device, const MeshDraw& setup) { }}; AgcDriver::Graphics::State state{}; - state.stages = {AgcDriver::Graphics::ShaderPath::Geometry, 0x20u, 64, 64, setup.mesh, std::nullopt}; + state.stages = {AgcDriver::Graphics::ShaderPath::Geometry, 0x20u, setup.waveSize, 64, setup.mesh, std::nullopt}; state.color = {reinterpret_cast(Pixels.data()), {Width, Height}, VK_FORMAT_R8G8B8A8_UNORM, Pixels.size(), 0xe4u}; state.colors = {state.color}; state.hasColorTarget = true; @@ -264,11 +273,22 @@ int main() { CheckTriangles((std::string("non-indexed triangle list, ") + name).c_str()); } + auto wave32Subgroup = SmallSubgroup; + wave32Subgroup.threadsPerGroup = 32u; + for (const auto& subgroup : {wave32Subgroup, SmallSubgroup, WideSubgroup}) { + ClearPixels(); + DrawMesh(device, {subgroup, {reinterpret_cast(Indices.data()), static_cast(Indices.size()), 2, 1, 0, true}, VertexBufferDescriptor(Scrambled.data(), static_cast(Scrambled.size())), ShaderRecompiler::MeshDrawPushOffsetBytes, false, 32u}); + CheckTriangles("indexed wave32 triangle list"); + ClearPixels(); + DrawMesh(device, {subgroup, {0, static_cast(Ordered.size()), 0, 1, 0, false}, VertexBufferDescriptor(Ordered.data(), static_cast(Ordered.size())), ShaderRecompiler::MeshDrawPushOffsetBytes, false, 32u}); + CheckTriangles("non-indexed wave32 triangle list"); + } + constexpr std::size_t recordBlockBytes = 65536; auto* record = static_cast(::operator new(recordBlockBytes, std::align_val_t{65536})); { GuestAllocations::Mutation mutation; - mutation.Add(record, recordBlockBytes, true, true); + mutation.Add(record, recordBlockBytes, true, true, true); } const auto drawIndirect = [&](std::uint32_t count, std::uint32_t instances, std::uint32_t first) { const std::array words{count, instances, first, 0u, 0u}; diff --git a/core/libs/prx/libSceAgcDriver/tests/MubufLdsDecode.cpp b/core/libs/prx/libSceAgcDriver/tests/MubufLdsDecode.cpp new file mode 100644 index 000000000..ac573d987 --- /dev/null +++ b/core/libs/prx/libSceAgcDriver/tests/MubufLdsDecode.cpp @@ -0,0 +1,34 @@ +#include "RdnaDecoder/RdnaInstructionDecoder.hpp" +#include +#include +#include +#include +#include +#include +using namespace ShaderRecompiler; +int main() { + int failures = 0; + const std::array load{0xe0300000u, 0x80010100u, 0xbf810000u}; + try { + const auto program = RdnaInstructionDecoder{}.Decode(std::span(load)); + if (program.instructions.empty() || program.instructions[0].op != RdnaOpcode::BufferLoadDword) { + std::fprintf(stderr, "buffer_load_dword v1, off, s[4:7], 0 does not decode as BufferLoadDword\n"); + ++failures; + } + } catch (const std::exception& error) { + std::fprintf(stderr, "buffer_load_dword v1, off, s[4:7], 0: %s\n", error.what()); + ++failures; + } + const std::array ldsLoad{0xe0310000u, 0x80010100u, 0xbf810000u}; + try { + static_cast(RdnaInstructionDecoder{}.Decode(std::span(ldsLoad))); + std::fprintf(stderr, "buffer_load_dword off, s[4:7], 0 lds decodes as a VGPR load\n"); + ++failures; + } catch (const std::runtime_error& error) { + if (std::string(error.what()).find("unsupported MUBUF lds modifier") == std::string::npos) { + std::fprintf(stderr, "buffer_load_dword off, s[4:7], 0 lds: %s\n", error.what()); + ++failures; + } + } + return failures == 0 ? 0 : 1; +} diff --git a/core/libs/prx/libSceAgcDriver/tests/PassHazards.cpp b/core/libs/prx/libSceAgcDriver/tests/PassHazards.cpp new file mode 100644 index 000000000..7689e5d90 --- /dev/null +++ b/core/libs/prx/libSceAgcDriver/tests/PassHazards.cpp @@ -0,0 +1,199 @@ +#include "prx/libSceAgcDriver/Graphics/include/PassHazards.hpp" +#include +#include +#include +#include +#include + +using namespace AgcDriver::Graphics; + +namespace { + +int failures = 0; + +void Expect(bool condition, const char* what) { + if (condition) return; + std::fprintf(stderr, "FAIL: %s\n", what); + ++failures; +} + +using Ranges = std::vector>; +using Images = std::vector>; + +VkImage image(std::uint64_t id) { + static_assert(sizeof(VkImage) == sizeof(std::uint64_t)); + VkImage handle = VK_NULL_HANDLE; + std::memcpy(&handle, &id, sizeof(handle)); + return handle; +} + +struct Use { + Ranges reads; + Ranges writes; + Images images; + Images attachments; + bool anyReads = false; + bool anyWrites = false; + + PassAccess View() const { return {reads, writes, images, attachments, anyReads, anyWrites}; } +}; + +void testRangeSet() { + GuestRangeSet set; + Expect(set.Empty() && !set.Overlaps(0, 100), "an empty set overlaps nothing"); + set.Insert(0x1000, 0x2000); + set.Insert(0x3000, 0x4000); + Expect(set.Size() == 2, "disjoint ranges stay apart"); + Expect(set.Overlaps(0x1fff, 0x2001) && set.Overlaps(0x0, 0x1001) && set.Overlaps(0x3800, 0x3900), "overlaps inside and at the edges"); + Expect(!set.Overlaps(0x2000, 0x3000) && !set.Overlaps(0x0, 0x1000) && !set.Overlaps(0x4000, 0x5000), "touching ranges do not overlap"); + set.Insert(0x2000, 0x3000); + Expect(set.Size() == 1 && set.Overlaps(0x2800, 0x2801), "an adjacent range merges both neighbours"); + set.Insert(0x500, 0x5000); + Expect(set.Size() == 1 && set.Overlaps(0x600, 0x601) && set.Overlaps(0x4fff, 0x5000), "a covering range swallows the set"); + set.Insert(0x9000, 0x9000); + Expect(set.Size() == 1 && !set.Overlaps(0x9000, 0x9001), "an empty range is ignored"); + set.Clear(); + Expect(set.Empty(), "cleared"); +} + +void testRanges() { + PassHazards hazards; + hazards.BeginPass(); + Use reader; + reader.reads = {{0x1000, 0x2000}}; + Expect(hazards.Check(reader.View(), true) == PassHazard::None, "nothing recorded: no hazard"); + hazards.Add(reader.View()); + Expect(hazards.Check(reader.View(), true) == PassHazard::None, "read after read"); + Use writer; + writer.writes = {{0x1800, 0x1900}}; + Expect(hazards.Check(writer.View(), true) == PassHazard::WriteAfterRead, "write after read of the same bytes"); + Use disjoint; + disjoint.writes = {{0x2000, 0x2100}}; + Expect(hazards.Check(disjoint.View(), true) == PassHazard::None, "write next to a read"); + hazards.Add(disjoint.View()); + Use rereader; + rereader.reads = {{0x20f0, 0x2200}}; + Expect(hazards.Check(rereader.View(), true) == PassHazard::ReadAfterWrite, "read after write"); + Use rewriter; + rewriter.writes = {{0x20fc, 0x2100}}; + Expect(hazards.Check(rewriter.View(), true) == PassHazard::WriteAfterWrite, "write after write"); + Use empty; + empty.reads = {{0x2000, 0x2000}}; + empty.writes = {{0x1000, 0x1000}}; + Expect(hazards.Check(empty.View(), true) == PassHazard::None, "empty ranges touch nothing"); + hazards.Clear(); + Expect(hazards.Empty() && hazards.Check(writer.View(), true) == PassHazard::None, "a barrier clears the accesses"); +} + +void testUnknown() { + PassHazards hazards; + hazards.BeginPass(); + Use addressBased; + addressBased.anyReads = true; + Use imageWriter; + imageWriter.images = {{image(7), true}}; + hazards.Add(imageWriter.View()); + Expect(hazards.Check(addressBased.View(), true) == PassHazard::None, "unknown reads after an image write only"); + hazards.Add(addressBased.View()); + Expect(hazards.Check(addressBased.View(), true) == PassHazard::None, "unknown reads after unknown reads"); + Use writer; + writer.writes = {{0x100, 0x104}}; + Expect(hazards.Check(writer.View(), true) == PassHazard::UnknownRead, "a write after unknown reads (WAR guard)"); + Use reader; + reader.reads = {{0x100, 0x104}}; + Expect(hazards.Check(reader.View(), true) == PassHazard::None, "a ranged read after unknown reads"); + hazards.Clear(); + hazards.Add(writer.View()); + Expect(hazards.Check(addressBased.View(), true) == PassHazard::UnknownRead, "unknown reads after a write"); + hazards.Clear(); + Use unknownWriter; + unknownWriter.anyWrites = true; + Expect(hazards.Check(unknownWriter.View(), true) == PassHazard::None, "unknown writes first"); + hazards.Add(imageWriter.View()); + Expect(hazards.Check(unknownWriter.View(), true) == PassHazard::None, "unknown writes after an image access only"); + hazards.Add(reader.View()); + Expect(hazards.Check(unknownWriter.View(), true) == PassHazard::UnknownWrite, "unknown writes after a read"); + hazards.Clear(); + hazards.Add(unknownWriter.View()); + Expect(hazards.Check(reader.View(), true) == PassHazard::UnknownWrite, "a read after unknown writes"); + Expect(hazards.Check(addressBased.View(), true) == PassHazard::UnknownWrite, "unknown reads after unknown writes"); + Expect(hazards.Check(imageWriter.View(), true) == PassHazard::None, "an image-only access after unknown writes"); +} + +void testImages() { + PassHazards hazards; + hazards.BeginPass(); + Use sampler; + sampler.images = {{image(1), false}}; + hazards.Add(sampler.View()); + Expect(hazards.Check(sampler.View(), true) == PassHazard::None, "two reads of an image"); + Use storer; + storer.images = {{image(1), true}}; + Expect(hazards.Check(storer.View(), true) == PassHazard::Image, "a store after a read of the image"); + Use other; + other.images = {{image(2), true}}; + Expect(hazards.Check(other.View(), true) == PassHazard::None, "a store to another image"); + hazards.Add(other.View()); + Use otherReader; + otherReader.images = {{image(2), false}}; + Expect(hazards.Check(otherReader.View(), true) == PassHazard::Image, "a read after a store of the image"); + Use null; + null.images = {{VK_NULL_HANDLE, true}}; + null.attachments = {{VK_NULL_HANDLE, true}}; + Expect(hazards.Check(null.View(), true) == PassHazard::None, "null images are ignored"); +} + +void testAttachments() { + PassHazards hazards; + hazards.BeginPass(); + Use draw; + draw.attachments = {{image(10), true}, {image(11), true}}; + hazards.Add(draw.View()); + Expect(hazards.Check(draw.View(), true) == PassHazard::None, "attachment writes of the same pass are in rasterization order"); + Expect(hazards.Check(draw.View(), false) == PassHazard::Image, "the same attachments in a new pass"); + Use readOnlyDepth; + readOnlyDepth.attachments = {{image(12), false}}; + hazards.Add(readOnlyDepth.View()); + hazards.BeginPass(); + Use otherPass; + otherPass.attachments = {{image(20), true}, {image(12), false}}; + Expect(hazards.Check(otherPass.View(), false) == PassHazard::None, "a new pass of other targets reading a depth image the last pass only read"); + hazards.Add(otherPass.View()); + Expect(hazards.Check(otherPass.View(), true) == PassHazard::None, "continuing the new pass"); + Use sampleEarlier; + sampleEarlier.images = {{image(10), false}}; + Expect(hazards.Check(sampleEarlier.View(), true) == PassHazard::Image, "sampling a target of an earlier pass"); + Use backToFirst; + backToFirst.attachments = {{image(10), true}}; + Expect(hazards.Check(backToFirst.View(), false) == PassHazard::Image, "rendering into a target of an earlier pass"); + hazards.Clear(); + hazards.BeginPass(); + Use sampled; + sampled.images = {{image(30), false}}; + hazards.Add(sampled.View()); + Use renderInto; + renderInto.attachments = {{image(30), true}}; + Expect(hazards.Check(renderInto.View(), true) == PassHazard::Image, "rendering into an image a draw of the pass sampled"); + Use storageWrite; + storageWrite.images = {{image(31), true}}; + hazards.Add(storageWrite.View()); + Use renderIntoStored; + renderIntoStored.attachments = {{image(31), true}}; + Expect(hazards.Check(renderIntoStored.View(), true) == PassHazard::Image, "rendering into an image a draw stored to"); +} + +} + +int main() { + testRangeSet(); + testRanges(); + testUnknown(); + testImages(); + testAttachments(); + if (failures != 0) { + std::fprintf(stderr, "%d pass hazard checks failed\n", failures); + return 1; + } + std::printf("pass hazard tests passed\n"); + return 0; +} diff --git a/core/libs/prx/libSceAgcDriver/tests/Pm4.cpp b/core/libs/prx/libSceAgcDriver/tests/Pm4.cpp index 9ef389177..90d0e7486 100644 --- a/core/libs/prx/libSceAgcDriver/tests/Pm4.cpp +++ b/core/libs/prx/libSceAgcDriver/tests/Pm4.cpp @@ -73,6 +73,18 @@ void testCatalog() { for (const auto& [id, name] : custom) check(AgcDriver::Pm4::Name(makePacket(0x10, {0}, id << 2)[0]) == name, "custom opcode name mismatch"); } +void testNopPad() { + check(AgcDriver::Pm4::PacketWords(0xffff1000u) == 1, "the one-dword NOP is not one dword"); + for (const auto queue : {0u, 0x20u}) AgcDriver::Pm4::Validate(std::vector{0xffff1000u}, queue); + expectFailure([] { AgcDriver::Pm4::Validate(std::vector{0xffff1000u, 0}, 0); }, "filler size"); + for (const auto header : {0xffff1001u, 0xffff1002u}) check(AgcDriver::Pm4::PacketWords(header) == 0x4001, "a NOP with count 0x3fff and header flags is not sized by its count"); + AgcDriver::Pm4::Validate(makePacket(0x10, {0}), 0); + std::vector largest(0x4000, 0); + largest[0] = 0xfffe1000u; + check(AgcDriver::Pm4::PacketWords(largest[0]) == largest.size(), "the largest counted NOP is not sized by its count"); + AgcDriver::Pm4::Validate(largest, 0); +} + void testRegisters() { AgcDriver::QueueState state; execute(state, makePacket(0x79, {0x242, 4})); @@ -90,10 +102,15 @@ void testRegisters() { pairs[4] = 0xffffffffu; expectFailure([&] { execute(state, packet); }, "sentinel"); check(!registers.contains(0x12), "invalid indirect packet partially changed state"); + std::array values{0x11223344, 0x55667788}; + execute(state, makePacket(opcode, {low(values.data()), high(values.data()), 0x0a, 2})); + check(registers.at(0x0a) == 0x11223344 && registers.at(0x0b) == 0x55667788, "consecutive indirect registers lost"); + expectFailure([&] { execute(state, makePacket(opcode, {low(values.data()), high(values.data()), 0xffff, 2})); }, "indirect-register"); + expectFailure([&] { execute(state, makePacket(opcode, {low(values.data()), high(values.data()), 0x4000000a, 2})); }, "indirect-register"); packet[1] = 0x1000; packet[2] = 0; expectFailure([&] { execute(state, packet); }, "guest"); - packet[3] = 0; + packet[3] = 0x10000; expectFailure([&] { AgcDriver::Pm4::Validate(packet, 0); }, "control"); } execute(state, makePacket(0x69, {0x11, 50, 51})); @@ -102,7 +119,17 @@ void testRegisters() { check(state.userConfig.at(0x10) == 60, "uconfig index zero failed"); execute(state, makePacket(0x7a, {0x20000243, 0x441})); check(state.indexType == 1 && state.userConfig.at(0x243) == 0x441, "indexed VGT_INDEX_TYPE write lost state"); + execute(state, makePacket(0x7a, {0x10000242, 0x11})); + check(state.userConfig.at(0x242) == 0x11, "indexed VGT_PRIMITIVE_TYPE write lost state"); expectFailure([&] { execute(state, makePacket(0x7a, {0x10000010, 1})); }, "bank selection"); + expectFailure([&] { execute(state, makePacket(0x7a, {0x10000243, 1})); }, "bank selection"); + expectFailure([&] { execute(state, makePacket(0x7a, {0x20000242, 1})); }, "bank selection"); + execute(state, makePacket(0x28, {0x80000000, 0x80000000})); + execute(state, makePacket(0x28, {0, 0})); + expectFailure([&] { AgcDriver::Pm4::Validate(makePacket(0x28, {0x80000001, 0x80000000}), 0); }, "shadowing"); + expectFailure([&] { AgcDriver::Pm4::Validate(makePacket(0x28, {0x80000000, 0x80000002}), 0); }, "shadowing"); + expectFailure([&] { AgcDriver::Pm4::Validate(makePacket(0x28, {0x80000000, 0x80000000}), 0x20); }, "compute queue"); + expectFailure([&] { AgcDriver::Pm4::Validate(makePacket(0x28, {0x80000000}), 0); }, "packet size"); expectFailure([&] { execute(state, makePacket(0x69, {0xffff, 1, 2})); }, "overflow"); expectFailure([&] { AgcDriver::Pm4::Validate(makePacket(0x9f, {0, 0, 0x80000000, 0}), 0x20); }, "compute"); } @@ -153,6 +180,10 @@ void testContextAndBases() { AgcDriver::Pm4::Validate(makePacket(0x15, {1, 1, 1, 0x2041}), 0); AgcDriver::Pm4::Validate(makePacket(0x16, {0, 0xa041}), 0); expectFailure([&] { AgcDriver::Pm4::Validate(makePacket(0x15, {1, 1, 1, 0x4041}), 0); }, "dispatch modifiers"); + AgcDriver::Pm4::Validate(makePacket(0x15, {0x280, 1, 1, 0x2045}, 2), 0); + AgcDriver::Pm4::Validate(makePacket(0x16, {0, 0x2045}, 2), 0); + expectFailure([&] { AgcDriver::Pm4::Validate(makePacket(0x15, {1, 1, 1, 0x41}, 4), 0); }, "header flags"); + expectFailure([&] { AgcDriver::Pm4::Validate(makePacket(0x15, {1, 1, 1, 0x49}), 0); }, "dispatch modifiers"); execute(state, makePacket(0x13, {32})); execute(state, makePacket(0x26, {0x1000, 1})); execute(state, makePacket(0x2a, {1})); @@ -161,7 +192,8 @@ void testContextAndBases() { execute(state, makePacket(0x10, {0x00636261}, 0x2c)); check(state.markers.back() == "abc", "marker text lost"); execute(state, makePacket(0x10, {0}, 0x30)); - expectFailure([&] { execute(state, makePacket(0x10, {0}, 0x30)); }, "underflow"); + execute(state, makePacket(0x10, {0}, 0x30)); + check(state.markers.empty(), "an unbalanced marker pop changed the marker stack"); execute(state, makePacket(0x10, {0}, 0x24)); check(state.shader.empty() && state.context == AgcDriver::InitialContextRegisters() && state.dispatchIndirectBase == 0 && state.indexBase == 0 && !state.savedContext, "dispatch reset retained state"); } @@ -300,6 +332,8 @@ void testMemory() { check(data[0] == 41 && data[1] == 42, "WRITE_DATA from the PFP failed"); execute(state, makePacket(0x37, {0x04100200, low(data.data()), high(data.data()), 61, 62})); check(data[0] == 61 && data[1] == 62, "WRITE_DATA with a cache policy failed"); + execute(state, makePacket(0x37, {0x40100500, low(data.data()), high(data.data()), 81, 82})); + check(data[0] == 81 && data[1] == 82, "WRITE_DATA to memory from the graphics queue failed"); expectFailure([&] { AgcDriver::Pm4::Validate(makePacket(0x37, {0x08000100, low(data.data()), high(data.data()), 71}), 0); }, "reserved"); expectFailure([&] { AgcDriver::Pm4::Validate(makePacket(0x37, {0x80000100, low(data.data()), high(data.data()), 51}), 0); }, "engine"); execute(state, makePacket(0x81, {4, 31, 32})); @@ -352,7 +386,15 @@ void testCopies() { check(source == destination, "DMA_DATA GDS round trip failed"); expectFailure([&] { execute(state, makePacket(0x50, {0x60100000, low(source.data()), high(source.data()), 0xfffc, 0, 8})); }, "exceeds the GDS"); expectFailure([&] { execute(state, makePacket(0x50, {0x20000000, 0, 1, low(destination.data()), high(destination.data()), 4})); }, "exceeds the GDS"); - expectFailure([&] { execute(state, makePacket(0x50, {0x60200000, low(source.data()), high(source.data()), 0, 0, 4})); }, "destination is not implemented"); + destination = {}; + const auto prefetch = makePacket(0x50, {0x60200000, low(source.data()), high(source.data()), low(source.data()), high(source.data()), 0x80000010}); + const auto prefetchStore = AgcDriver::Pm4::ResolveStore(prefetch, state, 64); + check(prefetchStore.has_value() && prefetchStore->Bytes().empty(), "a DMA_DATA prefetch resolved as a memory store"); + check(!AgcDriver::Pm4::DecodeMemoryCopy(prefetch).has_value(), "a DMA_DATA prefetch decoded as a copy"); + execute(state, prefetch); + check(source[0] == 11 && source[1] == 12 && destination[0] == 0, "a DMA_DATA prefetch wrote memory"); + expectFailure([&] { execute(state, makePacket(0x50, {0x40200000, 1, 0, 0, 0, 4})); }, "prefetch of a register"); + expectFailure([&] { execute(state, makePacket(0x50, {0x60200000, low(source.data()), high(source.data()), 0, 0, 4 | (1u << 27u)})); }, "register destination"); expectFailure([&] { execute(state, makePacket(0x50, {0x60000000 | (1u << 15u), low(source.data()), high(source.data()), low(destination.data()), high(destination.data()), 4})); }, "reserved fields"); expectFailure([&] { execute(state, makePacket(0x37, {0x100, 0x1000, 0, 1})); }, "guest"); #ifdef _WIN32 @@ -531,6 +573,15 @@ void testEventWrite() { expectFailure([] { AgcDriver::Pm4::Validate(makePacket(0x46, {0x410, 0, 0}), 0); }, "packet size"); expectFailure([] { AgcDriver::Pm4::Validate(makePacket(0x46, {0x13a, 0, 0}), 0); }, "event type 58"); expectFailure([] { AgcDriver::Pm4::Validate(makePacket(0x46, {0x0d}), 0); }, "event type 13"); + for (const auto eventType : {0x19u, 0x1au, 0x26u}) { + AgcDriver::Pm4::Validate(makePacket(0x46, {eventType}), 0); + AgcDriver::Pm4::Validate(makePacket(0x46, {eventType}), 0x20); + expectFailure([&] { AgcDriver::Pm4::Validate(makePacket(0x46, {0x100u | eventType}), 0); }, "pipeline statistics or SQ event index"); + } + AgcDriver::Pm4::Validate(makePacket(0x46, {0x24}), 0); + expectFailure([] { AgcDriver::Pm4::Validate(makePacket(0x46, {0x24}), 0x20); }, "compute queue"); + expectFailure([] { AgcDriver::Pm4::Validate(makePacket(0x46, {0x424}), 0); }, "VGT_FLUSH event index"); + expectFailure([] { AgcDriver::Pm4::Validate(makePacket(0x46, {0x1e, 0x1000, 0}), 0); }, "event type 30"); } void testAcquireMem() { @@ -563,12 +614,14 @@ void testAcquireMem() { invalidWord(0, captured[0] | 2u, "header flags"); invalidWord(1, 4, "control flags"); invalidWord(1, 0x00800000, "control flags"); - invalidWord(3, 1, "above 40 bits"); - invalidWord(5, 1, "above 40 bits"); + invalidWord(3, 0x100, "range high bits"); + invalidWord(5, 0x100, "range high bits"); invalidWord(6, 0x10000, "poll interval"); invalidWord(7, 0x40000, "GCR flags"); invalidWord(7, 0x2000, "cache discard"); - expectFailure([] { AgcDriver::Pm4::Validate(makePacket(0x58, {0, 2, 0, 0xffffffff, 0, 0, 0}), 0); }, "range exceeds"); + AgcDriver::Pm4::Validate(makePacket(0x58, {0, 0xffffffff, 0xff, 0, 0, 10, 0xc3b1}), 0); + AgcDriver::Pm4::Validate(makePacket(0x58, {0, 2, 0, 0xffffffff, 0, 0, 0}), 0); + expectFailure([] { AgcDriver::Pm4::Validate(makePacket(0x58, {0, 2, 0, 0xffffffff, 0xff, 0, 0}), 0); }, "range exceeds"); expectFailure([] { AgcDriver::Pm4::Validate(makePacket(0x58, {0, 0, 0, 0, 0}), 0); }, "packet size"); expectFailure([] { AgcDriver::Pm4::Validate(makePacket(0x58, {0, 0, 0, 0, 0, 0, 0, 0}), 0); }, "packet size"); } @@ -703,6 +756,20 @@ void testRegisterListsReadAtSubmission() { check(std::atomic_ref(done).load() == 1, "register list rewritten after submission was read by the worker"); } +void testSuspendPointWritesQueuedLabels() { + alignas(8) static std::uint32_t gate = 0, marker = 0; + gate = 0; + marker = 0; + auto words = joinPackets({makePacket(0x3c, {0x13, low(&gate), high(&gate), 1, 0xffffffffu, 0x190}), + makePacket(0x49, {0x0030c514, 0x20000000, low(&marker), high(&marker), 1, 0, 0})}); + submitWords(words); + AgcDriverSuspendPoint_nid_postfix(); + std::atomic_ref(gate).store(1); + for (int waited = 0; waited < 2000 && std::atomic_ref(marker).load() == 0; ++waited) std::this_thread::sleep_for(std::chrono::milliseconds(1)); + check(std::atomic_ref(marker).load() == 1, "a label queued before a suspend point was never written"); + AgcDriverWaitIdle_nid_postfix(); +} + void testConditionalSubmission() { alignas(8) static std::uint32_t zero = 0, one = 1, condition = 0; static std::array results{}; @@ -771,6 +838,20 @@ void testConditionalSubmission() { check(results[13] == 0, "a rejected conditional submission executed a guarded packet"); } +void testNopPadSubmission() { + alignas(8) static std::uint32_t zero = 0; + static std::array results{}; + results.fill(0); + auto words = joinPackets({{0xffff1000u, 0xc0027904u, 0x342u, 0xce200000u, 0u, 0xc0017904u, 0x342u, 0xcea00000u}, writeWord(results[0], 81), {0xffff1000u}}); + submitWords(words); + AgcDriverWaitIdle_nid_postfix(); + check(results[0] == 81, "a one-dword NOP stopped the packets after it"); + words = joinPackets({conditional(zero, 6), {0xffff1000u}, writeWord(results[1], 82), writeWord(results[2], 83)}); + submitWords(words); + AgcDriverWaitIdle_nid_postfix(); + check(results[1] == 0 && results[2] == 83, "COND_EXEC did not count a one-dword NOP as one dword"); +} + std::vector branch(std::uint32_t mode, std::uint32_t function, const std::vector* first, const std::vector* second) { const auto address = [](const std::vector* target) { return target ? reinterpret_cast(target->data()) : std::uintptr_t{0}; }; const auto size = [](const std::vector* target) { return target ? static_cast(target->size()) : 0u; }; @@ -952,6 +1033,7 @@ int main(int argc, char** argv) { return 0; } testCatalog(); + testNopPad(); testWriteChangedKeepsUntouchedBytes(); testRegisters(); testRegisterFile(); @@ -975,6 +1057,8 @@ int main(int argc, char** argv) { testRegisterListsReadAtSubmission(); testPredicatedSubmission(); testConditionalSubmission(); + testSuspendPointWritesQueuedLabels(); + testNopPadSubmission(); testBranchSubmission(); LibcRunShutdown_nid_postfix(); std::puts("PM4 catalog, registers, state, memory, conditional execution and submission tests passed"); diff --git a/core/libs/prx/libSceAgcDriver/tests/PositionInvariance.cpp b/core/libs/prx/libSceAgcDriver/tests/PositionInvariance.cpp new file mode 100644 index 000000000..2c9b168bb --- /dev/null +++ b/core/libs/prx/libSceAgcDriver/tests/PositionInvariance.cpp @@ -0,0 +1,89 @@ +#include "Recompiler.hpp" +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace { + +void requireInvariantPositions(const ShaderRecompiler::RecompileResult& result, const char* what) { + constexpr std::uint32_t Variable = ~0u; + const auto& words = result.spirv.Words(); + std::set> positions; + std::set> invariant; + for (std::size_t at = 5; at < words.size() && (words[at] >> 16u) != 0; at += words[at] >> 16u) { + const auto op = static_cast(words[at] & 0xffffu); + const auto count = words[at] >> 16u; + if (op == spv::OpMemberDecorate && count >= 4) { + if (words[at + 3] == spv::DecorationBuiltIn && count == 5 && words[at + 4] == spv::BuiltInPosition) positions.emplace(words[at + 1], words[at + 2]); + if (words[at + 3] == spv::DecorationInvariant) invariant.emplace(words[at + 1], words[at + 2]); + } + if (op == spv::OpDecorate && count >= 3) { + if (words[at + 2] == spv::DecorationBuiltIn && count == 4 && words[at + 3] == spv::BuiltInPosition) positions.emplace(words[at + 1], Variable); + if (words[at + 2] == spv::DecorationInvariant) invariant.emplace(words[at + 1], Variable); + } + } + if (positions.empty()) throw std::runtime_error(std::string(what) + ": the module declares no position output"); + for (const auto& position : positions) { + if (!invariant.contains(position)) throw std::runtime_error(std::string(what) + ": the position output is not Invariant"); + } +} + +constexpr std::array PositionExport{0xf80008cfu, 0u, 0xbf810000u}; + +ShaderRecompiler::RecompileRequest baseRequest(ShaderRecompiler::ShaderStage stage) { + using namespace ShaderRecompiler; + RecompileRequest request{}; + request.shader = {stage, 0x10000u, PositionExport, 0, {}}; + request.context.waveSize = 64; + request.context.vertex = ShaderVertexStageInfo{}; + request.target.vulkanVersion = 0x00401000u; + request.target.spirvVersion = 0x00010300u; + request.target.subgroupSize = 64; + request.layout.pushConstantSizeBytes = 128; + request.useCache = false; + return request; +} + +void verifyVertex() { + using namespace ShaderRecompiler; + requireInvariantPositions(Recompile(baseRequest(ShaderStage::Vertex)), "vertex shader"); +} + +void verifyMesh() { + using namespace ShaderRecompiler; + static constexpr std::array capabilities{spv::CapabilityMeshShadingEXT}; + static constexpr std::array extensions{"SPV_EXT_mesh_shader"}; + static constexpr std::array hiddenUserWords{}; + auto request = baseRequest(ShaderStage::Mesh); + request.context.userData = hiddenUserWords; + request.target.spirvVersion = 0x00010400u; + request.target.supportedCapabilities = capabilities; + request.target.supportedExtensions = extensions; + request.target.maxWorkgroupSize = {1024u, 1024u, 64u}; + request.target.maxWorkgroupInvocations = 1024u; + request.target.maxWorkgroupSharedMemoryBytes = 32768u; + request.target.mesh = MeshTargetLimits{{128u, 1u, 1u}, 128u, 32768u, 256u, 256u, 128u, 32768u, 1u, 1u}; + request.graphics = GraphicsCompileContext{0u, {}, MeshConfiguration{4u, 1u, 3u, 3u, 1u, 64u, 128u, 0u, 4u}, std::nullopt, {}}; + requireInvariantPositions(Recompile(request), "mesh shader"); +} + +} // namespace + +int main() { + try { + verifyVertex(); + verifyMesh(); + std::cout << "Vertex and mesh position outputs are Invariant\n"; + return 0; + } catch (const std::exception& error) { + const std::string message(error.what()); + std::cerr << message.substr(0, message.find("RecompileRequest:")) << '\n'; + return 1; + } +} diff --git a/core/libs/prx/libSceAgcDriver/tests/Recorder.cpp b/core/libs/prx/libSceAgcDriver/tests/Recorder.cpp index a8c50a234..33cc36f95 100644 --- a/core/libs/prx/libSceAgcDriver/tests/Recorder.cpp +++ b/core/libs/prx/libSceAgcDriver/tests/Recorder.cpp @@ -1,3 +1,8 @@ +#include "Triangle_frag_spv.h" +#include "Triangle_vert_spv.h" +#include "prx/libSceAgcDriver/Graphics/include/VertexInput.hpp" +#include "prx/libSceAgcDriver/Graphics/include/Pipeline.hpp" +#include "prx/libSceAgcDriver/Graphics/include/PipelineLibrary.hpp" #include "prx/libSceAgcDriver/Graphics/include/Recorder.hpp" #include "prx/libSceAgcDriver/Graphics/include/Draw.hpp" #include "prx/libSceAgcDriver/Graphics/include/DccMetadata.hpp" @@ -6,7 +11,9 @@ #include "prx/libSceAgcDriver/Graphics/include/ShaderResources.hpp" #include "prx/libSceAgcDriver/Graphics/include/Shaders.hpp" #include "prx/libSceAgcDriver/Graphics/include/Texture.hpp" +#include "prx/libSceAgcDriver/Graphics/include/TextureCache.hpp" #include "prx/libSceAgcDriver/Graphics/include/TextureDetiler.hpp" +#include "prx/libSceAgcDriver/Graphics/include/TextureFormat.hpp" #include "prx/libSceAgcDriver/Graphics/include/UnitShadow.hpp" #include "prx/libSceAgcDriver/Execution/include/BdaFeatures.hpp" #include "prx/libSceAgcDriver/Execution/include/GuestMemory.hpp" @@ -14,9 +21,12 @@ #include "prx/libc/include/GuestAllocations.hpp" #include "prx/libc/include/GuestArena.hpp" #include "prx/libc/include/GuestWriteWatch.hpp" +#include "prx/libc/include/general/VabiMacros.hpp" #include "ResidentPresent.hpp" #include "SampleLod_spv.h" #include "SampleArray_spv.h" +#include "Cover_vert_spv.h" +#include "Cover_frag_spv.h" #include #ifdef _WIN32 #ifndef NOMINMAX @@ -30,6 +40,7 @@ #include #include #include +#include #include #include #include @@ -42,6 +53,13 @@ namespace { +extern "C" { +int APS5_VABI sceKernelAllocateDirectMemory(std::int64_t, std::int64_t, std::size_t, std::size_t, int, std::int64_t*); +int APS5_VABI sceKernelMapDirectMemory(void**, std::size_t, int, int, std::int64_t, std::size_t); +int APS5_VABI sceKernelReleaseDirectMemory(std::int64_t, std::size_t); +int APS5_VABI sceKernelMunmap(void*, std::size_t); +} + using namespace AgcDriver::Graphics; using AgcDriver::GuestMemory::GpuMutex; @@ -145,6 +163,14 @@ public: queue.pQueuePriorities = &priority; VkPhysicalDeviceFeatures enabled{}; enabled.shaderInt64 = VK_TRUE; + graphics = (families[family].queueFlags & VK_QUEUE_GRAPHICS_BIT) != 0; + VkPhysicalDeviceFeatures supported{}; + function("vkGetPhysicalDeviceFeatures")(context.physical, &supported); + enabled.occlusionQueryPrecise = supported.occlusionQueryPrecise; + context.occlusionQueryPrecise = supported.occlusionQueryPrecise == VK_TRUE; + enabled.shaderStorageImageReadWithoutFormat = supported.shaderStorageImageReadWithoutFormat; + enabled.shaderStorageImageWriteWithoutFormat = supported.shaderStorageImageWriteWithoutFormat; + context.singlePassStorage = supported.shaderStorageImageReadWithoutFormat == VK_TRUE && supported.shaderStorageImageWriteWithoutFormat == VK_TRUE; address.pNext = &bytes; std::vector extensionsEnabled{VK_KHR_BUFFER_DEVICE_ADDRESS_EXTENSION_NAME, VK_KHR_8BIT_STORAGE_EXTENSION_NAME}; VkPhysicalDeviceDriverProperties driverProperties{VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_DRIVER_PROPERTIES}; @@ -173,6 +199,38 @@ public: minLod.pNext = address.pNext; address.pNext = &minLod; } + VkPhysicalDeviceDynamicRenderingFeaturesKHR rendering{VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_DYNAMIC_RENDERING_FEATURES_KHR}; + VkPhysicalDeviceExtendedDynamicStateFeaturesEXT dynamicState{VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_EXTENDED_DYNAMIC_STATE_FEATURES_EXT, &rendering}; + VkPhysicalDeviceGraphicsPipelineLibraryFeaturesEXT library{VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_GRAPHICS_PIPELINE_LIBRARY_FEATURES_EXT, &dynamicState}; + const std::array libraryExtensions{VK_KHR_PIPELINE_LIBRARY_EXTENSION_NAME, VK_EXT_GRAPHICS_PIPELINE_LIBRARY_EXTENSION_NAME, VK_EXT_EXTENDED_DYNAMIC_STATE_EXTENSION_NAME, VK_KHR_DYNAMIC_RENDERING_EXTENSION_NAME, VK_KHR_DEPTH_STENCIL_RESOLVE_EXTENSION_NAME, VK_KHR_CREATE_RENDERPASS_2_EXTENSION_NAME}; + if (std::getenv("APS5_NO_GPL") == nullptr && std::all_of(libraryExtensions.begin(), libraryExtensions.end(), hasExtension)) { + VkPhysicalDeviceFeatures2 features{VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_FEATURES_2, &library}; + function("vkGetPhysicalDeviceFeatures2")(context.physical, &features); + VkPhysicalDeviceGraphicsPipelineLibraryPropertiesEXT libraryProperties{VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_GRAPHICS_PIPELINE_LIBRARY_PROPERTIES_EXT}; + VkPhysicalDeviceProperties2 properties{VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_PROPERTIES_2, &libraryProperties}; + function("vkGetPhysicalDeviceProperties2")(context.physical, &properties); + context.graphicsPipelineLibrary = library.graphicsPipelineLibrary == VK_TRUE && dynamicState.extendedDynamicState == VK_TRUE && rendering.dynamicRendering == VK_TRUE && libraryProperties.graphicsPipelineLibraryFastLinking == VK_TRUE && properties.properties.limits.maxPushConstantsSize >= PipelinePushConstantBytes; + } + rendering = {VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_DYNAMIC_RENDERING_FEATURES_KHR, address.pNext, VK_TRUE}; + library = {VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_GRAPHICS_PIPELINE_LIBRARY_FEATURES_EXT, &dynamicState, VK_TRUE}; + dynamicState = {VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_EXTENDED_DYNAMIC_STATE_FEATURES_EXT, &rendering, VK_TRUE}; + if (context.graphicsPipelineLibrary) { + extensionsEnabled.insert(extensionsEnabled.end(), libraryExtensions.begin(), libraryExtensions.end()); + address.pNext = &library; + } + VkPhysicalDeviceRobustness2FeaturesEXT robustness{VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_ROBUSTNESS_2_FEATURES_EXT}; + if (hasExtension(VK_EXT_ROBUSTNESS_2_EXTENSION_NAME)) { + VkPhysicalDeviceFeatures2 features{VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_FEATURES_2, &robustness}; + function("vkGetPhysicalDeviceFeatures2")(context.physical, &features); + nullDescriptors = robustness.nullDescriptor == VK_TRUE; + } + robustness = {VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_ROBUSTNESS_2_FEATURES_EXT}; + robustness.nullDescriptor = VK_TRUE; + if (nullDescriptors) { + extensionsEnabled.push_back(VK_EXT_ROBUSTNESS_2_EXTENSION_NAME); + robustness.pNext = address.pNext; + address.pNext = &robustness; + } VkDeviceCreateInfo device{VK_STRUCTURE_TYPE_DEVICE_CREATE_INFO, &address}; device.queueCreateInfoCount = 1; device.pQueueCreateInfos = &queue; @@ -187,6 +245,7 @@ public: context.limits = properties.limits; context.bufferDeviceAddress = true; context.formatProperties = function("vkGetPhysicalDeviceFormatProperties"); + context.imageFormatProperties = function("vkGetPhysicalDeviceImageFormatProperties"); context.Function("vkGetDeviceQueue")(context.device, family, 0, &context.queue); VkCommandPoolCreateInfo pool{VK_STRUCTURE_TYPE_COMMAND_POOL_CREATE_INFO}; pool.flags = VK_COMMAND_POOL_CREATE_RESET_COMMAND_BUFFER_BIT; @@ -199,7 +258,9 @@ public: } ~Device() { release(); } + bool NullDescriptors() const { return nullDescriptors; } const Context& GetContext() const { return context; } + bool Graphics() const { return graphics; } void WaitQueue() const { Check(context.Function("vkQueueWaitIdle")(context.queue), "vkQueueWaitIdle"); } private: @@ -225,6 +286,8 @@ private: PFN_vkGetInstanceProcAddr instanceProc = nullptr; VkInstance instance = VK_NULL_HANDLE; Context context{}; + bool graphics = false; + bool nullDescriptors = false; }; using Kind = Recorder::ReadKind; @@ -325,6 +388,49 @@ void completionCountTests(const Device& device, Recorder& recorder) { Require(memory[0] == 1 && memory[2] == (storeAlways ? 1u : 7u) && memory[16] == 1, "completion stores ran for the wrong labels (overlapped and not-imported ones store, the untouched GPU-stored one skips)"); } +void directCompletionLabelTests(Recorder& recorder) { + using namespace AgcDriver::GuestMemory; + constexpr std::size_t size = 65536; + struct Release { + Recorder& recorder; + void* mapping = nullptr; + std::int64_t physical = -1; + ~Release() { + recorder.Sync(); + if (mapping != nullptr) sceKernelMunmap(mapping, size); + if (physical >= 0) sceKernelReleaseDirectMemory(physical, size); + } + } release{recorder}; + Require(sceKernelAllocateDirectMemory(0, 0x7fffffffffll, size, size, 0, &release.physical) == 0, "allocate completion label backing"); + Require(sceKernelMapDirectMemory(&release.mapping, size, 3, 0, release.physical, size) == 0, "map completion label backing"); + auto* memory = static_cast(release.mapping); + std::memset(memory, 7, size); + const auto address = reinterpret_cast(memory) + 4092; + std::array value; + for (const auto bytes : {4u, 8u, 16u}) { + value.fill(static_cast(bytes)); + const auto before = CollectWritesUncached(address, bytes); + recorder.NotePendingWrite(address, bytes); + recorder.AfterCompletions(address, std::span(value).first(bytes), 100 + bytes, 0, false); + value.fill(std::byte{99}); + bool observed = false; + Require(recorder.AfterRecordedWork([&] { + observed = std::all_of(memory + 4092, memory + 4092 + bytes, [bytes](auto b) { return b == static_cast(bytes); }); + }), "completion observer was not recorded"); + recorder.Sync(); + Require(observed, "a following completion observed incorrect label bytes"); + Require(memory[4091] == std::byte{7} && memory[4108] == std::byte{7}, "completion label changed neighboring bytes"); + Require(recorder.Idle() && Recorder::PendingCompletionLabels() == 0, "completion label remained pending after sync"); + if (before != 0) { + Require(StoredOver(address, bytes, before), "completion label was not stamped"); + Require(UnchangedSinceCollected(address, bytes, before), "completion label dirtied the guest page"); + const auto after = CollectWritesUncached(address, bytes); + memory[4092] = std::byte{81}; + Require(!UnchangedSinceCollected(address, bytes, after), "completion store disabled subsequent guest tracking"); + } + } +} + void afterRecordedWorkTests(const Device& device, Recorder& recorder) { alignas(64) static std::uint32_t memory[16]; const auto base = reinterpret_cast(memory); @@ -599,6 +705,65 @@ void keyProofTests(const Device& device, Recorder& recorder) { Require(ProvedClearKeys(resource, surfaceBytes, proof) == DccKeys::Clear0001 && KeyProofCounts().proved == before.proved + 1, "(9) the proof did not hold after the batch finished"); } +void targetKeyProofTests(const Device& device, Recorder& recorder) { + using namespace AgcDriver::GuestMemory; + constexpr std::size_t bytes = 65536; + void* block = AllocateWatched(bytes, bytes); + if (block == nullptr) { + std::cout << "no write watching: target key proofs not tested\n"; + return; + } + struct Release { + void* block; + ~Release() { ReleaseWatched(block, bytes); } + } release{block}; + constexpr std::size_t keyCount = 1024; + constexpr std::uint64_t surfaceBytes = keyCount * 256; + auto* keys = static_cast(block); + std::memset(keys, 0x20, keyCount); + const auto address = reinterpret_cast(block); + const auto scans = [] { return KeyProofCounts().rangeScanned; }; + const auto proofs = [] { return KeyProofCounts().rangeProved; }; + DccRangeProof proof; + auto scanned = scans(); + auto proved = proofs(); + Require(ProvedCurrentDccKeys(address, surfaceBytes, proof) == DccKeys::ClearRegister, "register clear keys read as another code"); + if (!KeyFastPath()) { + Require(proof.generation == 0 && scans() == scanned && proofs() == proved, "target key proofs were kept while disabled"); + std::cout << "key fast path off: every target key read scans\n"; + return; + } + Require(scans() == scanned + 1 && proofs() == proved && proof.generation != 0 && proof.keys == DccKeys::ClearRegister, "the first read of a settled key range left no proof"); + Require(ProvedCurrentDccKeys(address, surfaceBytes, proof) == DccKeys::ClearRegister && scans() == scanned + 1 && proofs() == proved + 1, "an unchanged key range was scanned again"); + MarkWritten(address, keyCount); + scanned = scans(); + proved = proofs(); + Require(ProvedCurrentDccKeys(address, surfaceBytes, proof) == DccKeys::ClearRegister && scans() == scanned + 1 && proofs() == proved, "a key store over the range was answered from the proof"); + Require(ProvedCurrentDccKeys(address, surfaceBytes, proof) == DccKeys::ClearRegister && proofs() == proved + 1, "the rescan after a key store left no proof"); + std::memset(keys, 0xff, keyCount); + scanned = scans(); + Require(ProvedCurrentDccKeys(address, surfaceBytes, proof) == DccKeys::Uncompressed && scans() == scanned + 1, "a CPU write of the keys was not seen"); + scanned = scans(); + proved = proofs(); + Require(ProvedCurrentDccKeys(address, surfaceBytes / 2, proof) == DccKeys::Uncompressed && scans() == scanned + 1 && proofs() == proved, "the proof of another key range answered"); + Require(ProvedCurrentDccKeys(address + 256, surfaceBytes / 2, proof) == DccKeys::Uncompressed && scans() == scanned + 2 && proofs() == proved, "the proof of a key range at another address answered"); + Require(ProvedCurrentDccKeys(address, surfaceBytes, proof) == DccKeys::Uncompressed && scans() == scanned + 3, "a proof taken over a shorter range answered the whole range"); + recorder.NotePendingWrite(address, keyCount); + MarkWritten(address, keyCount); + NoteKeysFillOnGpu(address, keyCount, DccKeys::Clear0000); + scanned = scans(); + proved = proofs(); + Require(ProvedCurrentDccKeys(address, surfaceBytes, proof) == DccKeys::Clear0000 && proof.generation == 0, "a pending key fill was not the answer, or its answer was kept"); + Require(ProvedCurrentDccKeys(address, surfaceBytes, proof) == DccKeys::Clear0000 && scans() == scanned + 2 && proofs() == proved, "a pending key fill was answered from a proof"); + std::memset(keys, 0x00, keyCount); + recorder.Submit(); + device.WaitQueue(); + recorder.Sync(); + Require(ProvedCurrentDccKeys(address, surfaceBytes, proof) == DccKeys::Clear0000 && proof.generation != 0, "the landed fill was not proved"); + proved = proofs(); + Require(ProvedCurrentDccKeys(address, surfaceBytes, proof) == DccKeys::Clear0000 && proofs() == proved + 1, "the proof after the fill landed did not hold"); +} + // (8) The group close against a collect in progress: a CPU store made after the close must refuse // the entry although another thread's resetting walk of the label's page (over a large range) may // absorb the store and stamp the block with the generation it took before the close. And an entry @@ -675,7 +840,7 @@ void storeRunTests(const Device& device, Recorder& recorder) { const auto address = reinterpret_cast(block); { GuestAllocations::Mutation mutation; - mutation.Add(block, bytes, true, true); + mutation.Add(block, bytes, true, true, true); } const auto* import = HostImportFor(context, address, bytes); if (import == nullptr) { @@ -749,7 +914,7 @@ void remappedImportTests(const Device& device) { const auto address = reinterpret_cast(block); { GuestAllocations::Mutation mutation; - mutation.Add(block, bytes, true, true); + mutation.Add(block, bytes, true, true, true); } if (HostImportFor(context, address, bytes) == nullptr) { std::cout << "host import of the remap test block refused: remapped imports not tested\n"; @@ -760,7 +925,7 @@ void remappedImportTests(const Device& device) { { GuestAllocations::Mutation mutation; mutation.Remove(block); - mutation.Add(block, bytes, true, true); + mutation.Add(block, bytes, true, true, true); } Require(HostImportFor(context, address, bytes) != nullptr, "the remapped range was not imported again"); const auto second = HostImportSerial(context, address, bytes, true); @@ -794,7 +959,7 @@ void movedMetadataTests(const Device& device, Recorder& recorder) { const auto address = reinterpret_cast(block); { GuestAllocations::Mutation mutation; - mutation.Add(block, bytes, true, true); + mutation.Add(block, bytes, true, true, true); } struct Unregister { const Context& context; @@ -887,7 +1052,7 @@ void viewPastLastMipTests(const Device& device, Recorder& recorder) { const auto address = reinterpret_cast(block); { GuestAllocations::Mutation mutation; - mutation.Add(block, bytes, true, true); + mutation.Add(block, bytes, true, true, true); } struct Unregister { const Context& context; @@ -971,7 +1136,7 @@ void resourceReadTests(const Device& device, Recorder& recorder) { const auto address = reinterpret_cast(block); { GuestAllocations::Mutation mutation; - mutation.Add(block, bytes, true, true); + mutation.Add(block, bytes, true, true, true); } if (HostImportFor(context, address, bytes) == nullptr) { std::cout << "host import of the test block refused: resource read notes not tested\n"; @@ -1057,6 +1222,166 @@ void resourceReadTests(const Device& device, Recorder& recorder) { HostImportFor(context, address, bytes); } +void readWrittenStagingTests(const Device& device, Recorder& recorder) { + const auto& context = device.GetContext(); + if (context.hostImportAlignment == 0) { + std::cout << "host imports unavailable: read and written staging not tested\n"; + return; + } + constexpr std::size_t bytes = 8u << 20u; +#ifdef _WIN32 + void* block = VirtualAlloc(nullptr, bytes, MEM_RESERVE | MEM_COMMIT, PAGE_READWRITE); +#else + void* block = std::aligned_alloc(65536, bytes); +#endif + Require(block != nullptr, "cannot allocate the test block"); + std::memset(block, 0x11, bytes); + const auto address = reinterpret_cast(block); + { + GuestAllocations::Mutation mutation; + mutation.Add(block, bytes, true, true, true); + } + if (HostImportFor(context, address, bytes) == nullptr) { + std::cout << "host import of the test block refused: read and written staging not tested\n"; + return; + } + const auto boundInPlace = [&](std::uint64_t element, std::size_t elementBytes, bool read) { + GuestBufferMemory memory(context); + memory.AllowDeviceStaging(); + memory.AddWritable(element, elementBytes, false, read); + memory.Upload(false); + const auto reads = memory.InPlaceReads(); + const bool direct = reads.size() == 1 && reads[0].first == element && reads[0].second == element + elementBytes; + Require(direct || reads.empty(), "an element was bound neither in place nor staged"); + memory.RecordCopyBacks(recorder); + recorder.Sync(); + return direct; + }; + Require(!boundInPlace(address + 4096, 1u << 20u, false), "a written 1 MiB element was not staged"); + Require(boundInPlace(address + 4096, 4u << 20u, false), "a written-only 4 MiB element was staged past the written window"); + Require(!boundInPlace(address + 4096, 4u << 20u, true), "a read and written 4 MiB element was bound in place"); + const auto buildInPlace = [&](std::uint64_t dispatchThreads, bool read) { + const auto element = address + 4096; + constexpr std::uint32_t stride = 16; + constexpr std::uint32_t records = (4u << 20u) / stride; + ShaderRecompiler::RecompileResult program; + ShaderRecompiler::DescriptorBinding binding; + binding.kind = ShaderRecompiler::DescriptorKind::StorageBuffer; + binding.role = ShaderRecompiler::DescriptorRole::GuestBuffers; + binding.descriptorSet = 0; + binding.binding = 0; + binding.count = 1; + binding.guestDescriptor = {static_cast(element), (static_cast(element >> 32u) & 0xffffu) | (stride << 16u), records, 0x31000000u}; + binding.bufferWritten = {true}; + binding.bufferRead = {read}; + program.bindings.push_back(binding); + const CompiledShader compute{ShaderRecompiler::ShaderStage::Compute, &program, 0}; + ShaderResources resources(context, compute, {}, false, dispatchThreads); + const auto reads = resources.InPlaceReads(); + const bool direct = std::any_of(reads.begin(), reads.end(), [&](const auto& range) { return range.first <= element && range.second >= element + records * stride; }); + resources.MarkGpuWrites(recorder); + recorder.Sync(); + return direct; + }; + Require(!buildInPlace((4u << 20u) / 16u, true), "a dispatch sweeping a read and written 4 MiB element bound it in place"); + Require(buildInPlace(1024, true), "a dispatch touching 16 KiB of a read and written 4 MiB element staged all of it"); + Require(buildInPlace(0, true), "a dispatch of unknown size staged a read and written 4 MiB element"); + Require(buildInPlace((4u << 20u) / 16u, false), "a dispatch sweeping a written-only 4 MiB element staged it"); + Require(std::all_of(static_cast(block), static_cast(block) + bytes, [](std::byte value) { return value == std::byte{0x11}; }), "a staged copy-back changed the guest bytes"); + { + GuestAllocations::Mutation mutation; + mutation.Remove(block); + } + HostImportFor(context, address, bytes); +} + +void misalignedRegionTests(const Device& device, Recorder& recorder) { + const auto& context = device.GetContext(); + const auto alignment = context.limits.minStorageBufferOffsetAlignment; + constexpr std::size_t bytes = 65536; + if (alignment < 8 || alignment > 256) { + std::cout << "storage buffer offset alignment " << alignment << ": misaligned draw regions not tested\n"; + return; + } + void* block = context.hostImportAlignment != 0 ? AllocateWatched(bytes, 65536) : nullptr; + if (block == nullptr) { + std::cout << "host imports or write watching unavailable: misaligned draw regions not tested\n"; + return; + } + struct Release { + void* block; + ~Release() { ReleaseWatched(block, bytes); } + } release{block}; + auto* guest = static_cast(block); + for (std::size_t at = 0; at < bytes; ++at) guest[at] = static_cast(at * 5u + 1u); + const auto address = reinterpret_cast(block); + { + GuestAllocations::Mutation mutation; + mutation.Add(block, bytes, true, true, true); + } + struct Unregister { + const Context& context; + void* block; + ~Unregister() { + { + GuestAllocations::Mutation mutation; + mutation.Remove(block); + } + HostImportFor(context, reinterpret_cast(block), bytes); + } + } unregister{context, block}; + const auto* import = HostImportFor(context, address, bytes); + if (import == nullptr) { + std::cout << "host import of the watched block refused: misaligned draw regions not tested\n"; + return; + } + constexpr std::uint32_t offset = 4096 + 4; + constexpr std::size_t elementBytes = 64; + const auto view = address + offset; + Require((view - import->base) % alignment != 0 && (view - import->base) % 4 == 0, "the test view is not a DWORD-aligned view off the storage buffer offset alignment"); + ShaderRecompiler::RecompileResult vertex; + ShaderRecompiler::DescriptorBinding binding; + binding.kind = ShaderRecompiler::DescriptorKind::StorageBuffer; + binding.role = ShaderRecompiler::DescriptorRole::GuestBuffers; + binding.descriptorSet = 0; + binding.binding = 0; + binding.count = 1; + binding.guestDescriptor = {static_cast(view), static_cast(view >> 32u) & 0xffffu, static_cast(elementBytes), 0x31000000u}; + binding.bufferWritten = {false}; + vertex.bindings.push_back(binding); + vertex.pushConstants.resize(16); + vertex.memoryOffsetDword = 0; + const ShaderRecompiler::RecompileResult fragment; + { + auto regionContext = context; + DescriptorCache cache(regionContext); + regionContext.descriptorCache = &cache; + Recorder regionRecorder(regionContext); + regionRecorder.Activate(); + const ColorTarget target{}; + ShaderResources resources(regionContext, vertex, fragment, target, 0, 0); + Require(resources.Reusable(), "a misaligned read-only draw region bound in place is not reusable"); + const auto reads = resources.InPlaceReads(); + Require(reads.size() == 1 && reads.front().first == view && reads.front().second == view + elementBytes, "the misaligned draw region is not read in place"); + std::array push{}; + resources.PatchPushConstants(push); + const auto adjustment = static_cast(push[0]); + Require(adjustment == (view - import->base) % alignment, "the misaligned region's push constant offset is not its distance from the aligned binding offset"); + const auto check = [&] { + const auto bindings = resources.PrepareDrawBindings(regionRecorder); + Require(bindings != nullptr && bindings->snapshots.size() == 1, "the read-only misaligned draw input was not snapshotted"); + const auto contents = bindings->snapshots.front().buffer->Bytes(); + Require(bindings->snapshots.front().address == view - adjustment && contents.size() == adjustment + elementBytes, "the draw snapshot does not start at the aligned offset below the view"); + Require(std::memcmp(contents.data() + adjustment, guest + offset, elementBytes) == 0, "the shader's patched offset into the draw snapshot misses the view's bytes"); + }; + check(); + guest[offset + 8] ^= 0xffu; + check(); + regionRecorder.Sync(); + } + recorder.Activate(); +} + void misalignedSnapshotTests(const Device& device, Recorder& recorder) { const auto& context = device.GetContext(); const auto alignment = context.limits.minStorageBufferOffsetAlignment; @@ -1080,7 +1405,7 @@ void misalignedSnapshotTests(const Device& device, Recorder& recorder) { const auto address = reinterpret_cast(block); { GuestAllocations::Mutation mutation; - mutation.Add(block, bytes, true, true); + mutation.Add(block, bytes, true, true, true); } struct Unregister { const Context& context; @@ -1155,7 +1480,7 @@ void drawSnapshotReuseTests(const Device& device, Recorder& recorder) { const auto address = reinterpret_cast(block); { GuestAllocations::Mutation mutation; - mutation.Add(block, bytes, true, true); + mutation.Add(block, bytes, true, true, true); } struct Unregister { const Context& context; @@ -1214,6 +1539,14 @@ void drawSnapshotReuseTests(const Device& device, Recorder& recorder) { AgcDriver::GuestMemory::MarkWritten(element, 4); const auto afterStore = snapshot(std::byte{0x33}); Require(afterStore != afterCpu, "a draw snapshot outlived a driver store to its range"); + std::memset(reinterpret_cast(element), 0x33, elementBytes); + Require(snapshot(std::byte{0x33}) == afterStore, "a CPU store of the same bytes made the draw input copy again"); + AgcDriver::GuestMemory::MarkWritten(element, 4); + Require(snapshot(std::byte{0x33}) == afterStore, "a driver store of the same bytes made the draw input copy again"); + reinterpret_cast(element)[elementBytes - 1] = 0x34; + const auto tail = resources.PrepareDrawBindings(snapshotRecorder); + Require(tail != nullptr && tail->snapshots.size() == 1 && tail->snapshots[0].buffer != afterStore && tail->snapshots[0].buffer->Bytes()[elementBytes - 1] == std::byte{0x34}, "a draw snapshot outlived a store that changed its last byte"); + std::memset(reinterpret_cast(element), 0x33, elementBytes); { GuestAllocations::Mutation mutation; } @@ -1223,6 +1556,114 @@ void drawSnapshotReuseTests(const Device& device, Recorder& recorder) { recorder.Activate(); } +void drawSnapshotPatchTests(const Device& device) { + const auto& context = device.GetContext(); + constexpr std::size_t bytes = 4 * 65536; + std::unique_lock gpu(GpuMutex()); + void* block = context.hostImportAlignment != 0 ? AllocateWatched(bytes, 65536) : nullptr; + if (block == nullptr) { + std::cout << "host imports or write watching unavailable: snapshot patching not tested\n"; + return; + } + struct Release { + void* block; + ~Release() { ReleaseWatched(block, bytes); } + } release{block}; + auto* guest = static_cast(block); + for (std::size_t at = 0; at < bytes; ++at) guest[at] = static_cast(at * 13u + 5u); + const auto address = reinterpret_cast(block); + { + GuestAllocations::Mutation mutation; + mutation.Add(block, bytes, true, true, true); + } + struct Unregister { + const Context& context; + void* block; + ~Unregister() { + { + GuestAllocations::Mutation mutation; + mutation.Remove(block); + } + HostImportFor(context, reinterpret_cast(block), bytes); + } + } unregister{context, block}; + if (HostImportFor(context, address, bytes) == nullptr) { + std::cout << "host import of the watched block refused: snapshot patching not tested\n"; + return; + } + if (!AgcDriver::GuestMemory::Watched(address, bytes)) { + std::cout << "host imports are compared, not watched: snapshot patching not tested\n"; + return; + } + constexpr std::size_t elementOffset = 4096; + constexpr std::size_t elementBytes = 3 * 65536; + const auto element = address + elementOffset; + ShaderRecompiler::RecompileResult program; + ShaderRecompiler::DescriptorBinding binding; + binding.kind = ShaderRecompiler::DescriptorKind::StorageBuffer; + binding.role = ShaderRecompiler::DescriptorRole::GuestBuffers; + binding.descriptorSet = 0; + binding.binding = 0; + binding.count = 1; + binding.guestDescriptor = {static_cast(element), static_cast(element >> 32u) & 0xffffu, static_cast(elementBytes), 0x31000000u}; + binding.bufferWritten = {false}; + program.bindings.push_back(binding); + const CompiledShader compute{ShaderRecompiler::ShaderStage::Compute, &program, 0}; + auto snapshotContext = context; + DescriptorCache cache(snapshotContext); + snapshotContext.descriptorCache = &cache; + Recorder snapshotRecorder(snapshotContext); + snapshotRecorder.Activate(); + { + ShaderResources resources(snapshotContext, compute); + const auto snapshot = [&] { + auto bindings = resources.PrepareDrawBindings(snapshotRecorder); + Require(bindings != nullptr && bindings->snapshots.size() == 1, "read-only draw input was not snapshotted"); + auto buffer = bindings->snapshots[0].buffer; + const auto contents = buffer->Bytes(); + Require(contents.size() == elementBytes && std::memcmp(contents.data(), reinterpret_cast(element), elementBytes) == 0, "a draw snapshot does not hold the guest bytes of its draw"); + return buffer; + }; + const auto settle = [&](const std::weak_ptr& buffer) { + snapshotRecorder.Sync(); + gpu.unlock(); + const auto deadline = std::chrono::steady_clock::now() + std::chrono::seconds(10); + while (buffer.use_count() > 1 && std::chrono::steady_clock::now() < deadline) std::this_thread::sleep_for(std::chrono::milliseconds(1)); + gpu.lock(); + Require(buffer.use_count() == 1, "a finished draw still holds its snapshot"); + }; + std::weak_ptr first = snapshot(); + settle(first); + guest[elementOffset + 10] ^= 0xffu; + guest[elementOffset + 65536 + 300] ^= 0xffu; + guest[elementOffset + 2 * 65536 + 7] = guest[elementOffset + 2 * 65536 + 7]; + guest[elementOffset + elementBytes - 1] ^= 0x5au; + AgcDriver::GuestMemory::BumpCollectEpoch(); + { + const auto patched = snapshot(); + Require(patched == first.lock(), "a stale snapshot nothing else holds was copied whole instead of patched"); + } + settle(first); + std::memset(guest + elementOffset + 65536, 0x77, 65536); + AgcDriver::GuestMemory::BumpCollectEpoch(); + { + const auto patched = snapshot(); + Require(patched == first.lock(), "a snapshot whose whole middle block changed was not patched"); + } + settle(first); + const auto held = snapshot(); + Require(held == first.lock(), "an unchanged snapshot was not reused"); + const std::vector before(held->Bytes().begin(), held->Bytes().end()); + guest[elementOffset + 2 * 65536 + 100] ^= 0xffu; + AgcDriver::GuestMemory::BumpCollectEpoch(); + const auto fresh = snapshot(); + Require(fresh != held, "a snapshot a recorded draw still holds was patched"); + Require(std::memcmp(held->Bytes().data(), before.data(), before.size()) == 0, "a held snapshot's bytes changed"); + snapshotRecorder.Sync(); + } +} + + void drawSnapshotEvictionTests(const Device& device) { using namespace AgcDriver::GuestMemory; constexpr std::size_t bytes = 65536; @@ -1307,7 +1748,7 @@ void drawInputReuseTests(const Device& device, Recorder& recorder) { alignas(64) static std::byte other[64]; { GuestAllocations::Mutation mutation; - mutation.Add(other, sizeof(other), true, true); + mutation.Add(other, sizeof(other), true, true, true); } { GuestAllocations::Mutation mutation; @@ -1337,6 +1778,63 @@ void drawInputReuseTests(const Device& device, Recorder& recorder) { recorder.Sync(); } +void drawInputInPlaceTests(const Device& device, Recorder& recorder) { + auto context = device.GetContext(); + constexpr std::size_t bytes = 65536; + void* block = context.hostImportAlignment != 0 ? AllocateWatched(bytes, 65536) : nullptr; + if (block == nullptr) { + std::cout << "host imports or write watching unavailable: in-place draw inputs not tested\n"; + return; + } + struct Release { + void* block; + ~Release() { ReleaseWatched(block, bytes); } + } release{block}; + const auto address = reinterpret_cast(block); + { + GuestAllocations::Mutation mutation; + mutation.Add(block, bytes, true, true, true); + } + struct Unregister { + const Context& context; + void* block; + ~Unregister() { + { + GuestAllocations::Mutation mutation; + mutation.Remove(block); + } + HostImportFor(context, reinterpret_cast(block), bytes); + } + } unregister{context, block}; + if (HostImportFor(context, address, bytes) == nullptr) { + std::cout << "host import of the watched block refused: in-place draw inputs not tested\n"; + return; + } + context.recorder = &recorder; + recorder.Sync(); + const auto vertices = address + 4096; + const auto* import = InPlaceDrawInput(context, vertices, 256, 4); + Require(import != nullptr && import->base <= vertices && vertices + 256 <= import->base + import->bytes, "an imported draw input was not bound in place"); + const auto read = recorder.DescribePendingRead(vertices, 256); + Require(read.has_value() && read->kind == Recorder::ReadKind::DrawInput && read->open, "an in-place draw input's read was not noted on the open batch"); + Require(InPlaceDrawInput(context, vertices + 2, 256, 4) == nullptr, "a draw input misaligned for its format was bound in place"); + Require(InPlaceDrawInput(context, vertices + 2, 256, 2) != nullptr, "an aligned 16-bit draw input was not bound in place"); + recorder.NotePendingWrite(address + 8192, 16); + Require(InPlaceDrawInput(context, address + 8192, 256, 4) != nullptr && !recorder.PendingWriteOverlaps(address + 8192, 256), "a draw input under a pending GPU write was bound before the write completed"); + const std::array label{std::byte{1}, std::byte{2}, std::byte{3}, std::byte{4}}; + recorder.RecordStore(import->buffer, address + 12288 - import->base, label, address + 12288); + Require(recorder.QueuedStoreOverlaps(address + 12288, 4), "the label store was not queued"); + Require(InPlaceDrawInput(context, address + 12288, 256, 4) == nullptr, "a draw input under a queued label store was bound in place"); + Require(InPlaceDrawInput(context, address + 16384, 256, 4) != nullptr, "a queued label store refused a disjoint draw input"); + alignas(64) static std::array unregistered{}; + Require(InPlaceDrawInput(context, reinterpret_cast(unregistered.data()), unregistered.size(), 4) == nullptr, "memory without a host import was bound in place"); + auto unrecorded = context; + unrecorded.recorder = nullptr; + Require(InPlaceDrawInput(unrecorded, vertices, 256, 4) == nullptr, "a draw without a recorder bound its input in place"); + recorder.Sync(); + Require(!recorder.PendingReadOverlaps(vertices, 256), "an in-place draw input's read outlived its batch"); +} + // Unit shadows (UnitShadow.hpp) over a host import of write-watched arena memory: a retile piece's // slab destination and its seeds, freshness from the tracker (a publish never stamps, a CPU write // makes the unit stale), the scopes, the slab boundary, and the retire publish. With @@ -1367,17 +1865,22 @@ void unitShadowTests(const Device& device, Recorder& recorder) { const auto address = reinterpret_cast(block); { GuestAllocations::Mutation mutation; - mutation.Add(block, bytes, true, true); + mutation.Add(block, bytes, true, true, true); } struct Unregister { + const Context& context; void* block; + std::uint64_t address; bool armed = true; ~Unregister() { if (!armed) return; - GuestAllocations::Mutation mutation; - mutation.Remove(block); + { + GuestAllocations::Mutation mutation; + mutation.Remove(block); + } + HostImportFor(context, address, bytes); } - } unregister{block}; + } unregister{context, block, address}; const auto* import = HostImportFor(context, address, bytes); if (import == nullptr) { std::cout << "host import of the shadow test block refused: unit shadows not tested\n"; @@ -1516,7 +2019,7 @@ void unitShadowTests(const Device& device, Recorder& recorder) { { GuestAllocations::Mutation mutation; mutation.Remove(block); - mutation.Add(block, static_cast(5 * unit), true, true); + mutation.Add(block, static_cast(5 * unit), true, true, true); } Require(HostImportFor(context, address, bytes) == nullptr, "a shrunk registration still imports the old range"); Require(!AnyShadowedOverlaps(address, bytes), "the retired import's shadow survived"); @@ -1559,7 +2062,7 @@ void storageRefreshTests(const Device& device, Recorder& recorder, bool watched) const auto address = reinterpret_cast(block); { GuestAllocations::Mutation mutation; - mutation.Add(block, bytes, true, true); + mutation.Add(block, bytes, true, true, true); } struct Unregister { const Context& context; @@ -1669,7 +2172,16 @@ void storageRefreshTests(const Device& device, Recorder& recorder, bool watched) draw({{1.0f, 0.0f, 0.0f, 1.0f}}); image->Refresh(); Require(holds({255, 0, 0, 255}), "a refresh under the image's own clear keys dropped the draw's results"); - Require(memoryHolds({255, 0, 0, 255}), "the refresh's store of the results did not reach guest memory"); + if (watched) { + Require(memoryHolds({255, 0, 0, 255}), "the refresh's store of the results did not reach watched guest memory"); + } else { + Require(StorageTexture::FindPending(address, surfaceBytes) == image, "an unchanged untracked refresh discarded the pending image"); + Require(StorageTexture::FlushPending(address, surfaceBytes, nullptr, "storage refresh visibility test"), "the pending untracked image was not flushed"); + recorder.Submit(); + device.WaitQueue(); + recorder.Sync(); + Require(memoryHolds({255, 0, 0, 255}), "an explicit flush did not make pending results visible in untracked guest memory"); + } if (watched) { draw({{0.0f, 0.0f, 1.0f, 1.0f}}); std::memset(texels, 0x33, 65536); @@ -1688,6 +2200,91 @@ void storageRefreshTests(const Device& device, Recorder& recorder, bool watched) recorder.Sync(); } +void hostImportUnmapTests(const Device& device, Recorder& recorder) { + const auto& context = device.GetContext(); + if (context.hostImportAlignment == 0) { + std::cout << "host imports unavailable: import unmap lifetime not tested\n"; + return; + } + const auto alignment = static_cast(std::max(65536u, context.hostImportAlignment)); + constexpr std::size_t RequestedBytes = 17u << 20u; + const auto bytes = ((RequestedBytes + alignment - 1u) / alignment) * alignment; +#ifdef _WIN32 + void* block = VirtualAlloc(nullptr, bytes, MEM_RESERVE | MEM_COMMIT, PAGE_READWRITE); +#else + void* block = std::aligned_alloc(alignment, bytes); +#endif + Require(block != nullptr, "cannot allocate the host import unmap block"); + struct Block { + void* address; + std::size_t bytes; + bool registered = true; + bool released = false; + void Release() { +#ifdef _WIN32 + Require(VirtualFree(address, 0, MEM_RELEASE) != 0, "cannot release the host import unmap block"); +#else + std::free(address); +#endif + released = true; + } + ~Block() { + if (registered) { + try { + GuestAllocations::Mutation mutation; + mutation.Remove(address); + } catch (...) { + return; + } + } + if (!released) { +#ifdef _WIN32 + VirtualFree(address, 0, MEM_RELEASE); +#else + std::free(address); +#endif + } + } + } allocation{block, bytes}; + const auto address = reinterpret_cast(block); + { + GuestAllocations::Mutation mutation; + mutation.Add(block, bytes, true, true, true); + } + const HostImport* imported = nullptr; + { + std::lock_guard gpu(GpuMutex()); + imported = HostImportFor(context, address, bytes); + } + if (imported == nullptr) { + GuestAllocations::Mutation mutation; + mutation.Remove(block); + allocation.registered = false; + allocation.Release(); + std::cout << "host import refused: import unmap lifetime not tested\n"; + return; + } + bool retiredBeforeRelease = false; + { + GuestAllocations::Mutation mutation; + mutation.Unmap(block, bytes, [&](const void*, std::size_t, const void*, bool) { + retiredBeforeRelease = !HostImportCovers(context, address, bytes); + if (retiredBeforeRelease) allocation.Release(); + }); + allocation.registered = false; + } + if (!retiredBeforeRelease) { + { + std::lock_guard gpu(GpuMutex()); + static_cast(HostImportFor(context, address, bytes)); + recorder.Sync(); + } + allocation.Release(); + } + Require(retiredBeforeRelease, "guest unmap released pages before retiring their host import"); + std::cout << "host import unmap lifetime: ok\n"; +} + void importWatchTests(const Device& device) { using namespace AgcDriver::GuestMemory; const auto& context = device.GetContext(); @@ -1726,7 +2323,7 @@ void importWatchTests(const Device& device) { }; const auto registerRange = [](void* block, std::size_t bytes) { GuestAllocations::Mutation mutation; - mutation.Add(block, bytes, true, true); + mutation.Add(block, bytes, true, true, true); }; const auto unregisterRange = [](void* block) { GuestAllocations::Mutation mutation; @@ -1830,7 +2427,7 @@ void staleGenerationTests(const Device& device, Recorder& recorder) { const auto address = reinterpret_cast(block); { GuestAllocations::Mutation mutation; - mutation.Add(block, bytes, true, true); + mutation.Add(block, bytes, true, true, true); } struct Unregister { const Context& context; @@ -1972,7 +2569,7 @@ void writeBackPaddingTests(const Device& device, Recorder& recorder, bool watche const auto address = blockAddress + test.offset; { GuestAllocations::Mutation mutation; - mutation.Add(block, bytes, true, true); + mutation.Add(block, bytes, true, true, true); } struct Unregister { const Context& context; @@ -2036,6 +2633,442 @@ void writeBackPaddingTests(const Device& device, Recorder& recorder, bool watche std::cout << "write-back padding" << (watched ? " (watched)" : "") << ": ok\n"; } +void singlePassTests(const Device& device, Recorder& recorder, bool watched) { + const auto& base = device.GetContext(); + if (base.hostImportAlignment == 0 || !base.singlePassStorage) { + std::cout << "host imports or storage access without a format unavailable: single-pass storage moves not tested\n"; + return; + } + struct Case { + const char* name; + std::uint32_t format; + TextureTileMode tileMode; + std::uint32_t width; + std::uint32_t height; + std::uint32_t mipCount; + std::uint32_t pipeBankXor = 0; + }; + constexpr Case cases[] = { + {"R8 64 KiB R_X 200x150", 1, TextureTileMode::kR64KBX, 200, 150, 1}, + {"R16 64 KiB R_X 200x150, 3 mips", 7, TextureTileMode::kR64KBX, 200, 150, 3}, + {"RGBA8 64 KiB R_X 520x260, 6 mips", 56, TextureTileMode::kR64KBX, 520, 260, 6}, + {"RGBA16F 64 KiB R_X 384x256", 71, TextureTileMode::kR64KBX, 384, 256, 1}, + {"RGBA16F 64 KiB R_X 256x200, 4 mips", 71, TextureTileMode::kR64KBX, 256, 200, 4}, + {"RGBA32F 64 KiB R_X 136x72, 2 mips", 77, TextureTileMode::kR64KBX, 136, 72, 2}, + {"RGBA8 64 KiB S 300x170", 56, TextureTileMode::kStandard64KB, 300, 170, 1}, + {"RGBA16F 4 KiB S 100x70, 2 mips", 71, TextureTileMode::kStandard4KB, 100, 70, 2}, + {"R8 256 B S 90x40", 1, TextureTileMode::kStandard256B, 90, 40, 1}, + {"RGBA8 64 KiB R_X 520x260, 2 mips, pipe/bank XOR 5", 56, TextureTileMode::kR64KBX, 520, 260, 2, 5}, + }; + std::string failures; + std::size_t tested = 0; + for (const auto& test : cases) { + if (!StorageFormatAvailable(base, test.format)) continue; + const std::string what = std::string(watched ? "(watched) " : "") + test.name; + GuestTextureResource resource{}; + resource.width = test.width; + resource.height = test.height; + resource.mipCount = test.mipCount; + resource.tileMode = test.tileMode; + resource.dimension = TextureDimension::k2D; + resource.format = test.format; + resource.pipeBankXor = test.pipeBankXor; + resource.dstSelX = 4; + resource.dstSelY = 5; + resource.dstSelZ = 6; + resource.dstSelW = 7; + const auto geometry = DescribeSurface(resource); + const auto elementBytes = BytesPerElement(test.format); + const auto bytes = (static_cast(geometry.guestBytes) + 65535) / 65536 * 65536 + 65536; + struct Result { + std::vector uploaded, memory, reuploaded; + std::uint64_t moves = 0; + }; + std::array results; + bool skipped = false; + for (int mode = 0; mode < 2 && !skipped; ++mode) { + auto context = base; + context.singlePassStorage = mode == 1; + void* block = nullptr; + if (watched) { + block = AllocateWatched(bytes, 65536); + } else { +#ifdef _WIN32 + block = VirtualAlloc(nullptr, bytes, MEM_RESERVE | MEM_COMMIT, PAGE_READWRITE); +#else + block = std::aligned_alloc(65536, bytes); +#endif + } + if (block == nullptr) { + std::cout << "no write watching: single-pass storage moves in watched memory not tested\n"; + return; + } + auto* memory = static_cast(block); + std::uint32_t seed = 0x13579bdfu; + const auto random = [&] { + seed ^= seed << 13; + seed ^= seed >> 17; + seed ^= seed << 5; + return static_cast(seed >> 9); + }; + for (std::size_t i = 0; i < bytes; ++i) memory[i] = random(); + const auto address = reinterpret_cast(block); + { + GuestAllocations::Mutation mutation; + mutation.Add(block, bytes, true, true, true); + } + struct Unregister { + const Context& context; + void* block; + std::uint64_t address; + std::size_t bytes; + bool watched; + ~Unregister() { + { + GuestAllocations::Mutation mutation; + mutation.Remove(block); + } + HostImportFor(context, address, bytes); + if (watched) ReleaseWatched(block, bytes); +#ifdef _WIN32 + else VirtualFree(block, 0, MEM_RELEASE); +#else + else std::free(block); +#endif + } + } unregister{base, block, address, bytes, watched}; + if (HostImportFor(base, address, bytes) == nullptr) { + std::cout << "host import refused: single-pass storage moves not tested\n"; + return; + } + TextureDetiler detiler(base); + context.detiler = &detiler; + resource.baseAddress = address; + std::size_t linearBytes = 0; + for (const auto& mip : geometry.mips) linearBytes += static_cast(mip.width) * mip.height * elementBytes; + const auto readImage = [&](const StorageTexture& image) { + Buffer readback(context, linearBytes, VK_BUFFER_USAGE_TRANSFER_DST_BIT); + const auto commands = recorder.Commands(); + RecordMemoryBarrier(context, commands, VK_PIPELINE_STAGE_ALL_COMMANDS_BIT, VK_PIPELINE_STAGE_TRANSFER_BIT, VK_ACCESS_SHADER_WRITE_BIT | VK_ACCESS_TRANSFER_WRITE_BIT | VK_ACCESS_COLOR_ATTACHMENT_WRITE_BIT, VK_ACCESS_TRANSFER_READ_BIT); + std::vector copies; + std::size_t at = 0; + for (std::uint32_t level = 0; level < geometry.mips.size(); ++level) { + VkBufferImageCopy copy{}; + copy.bufferOffset = at; + copy.imageSubresource = {VK_IMAGE_ASPECT_COLOR_BIT, level, 0, 1}; + copy.imageExtent = {geometry.mips[level].width, geometry.mips[level].height, 1}; + copies.push_back(copy); + at += static_cast(geometry.mips[level].width) * geometry.mips[level].height * elementBytes; + } + context.Function("vkCmdCopyImageToBuffer")(commands, image.Image(), VK_IMAGE_LAYOUT_GENERAL, readback.Handle(), static_cast(copies.size()), copies.data()); + RecordMemoryBarrier(context, commands, VK_PIPELINE_STAGE_TRANSFER_BIT, VK_PIPELINE_STAGE_ALL_COMMANDS_BIT | VK_PIPELINE_STAGE_HOST_BIT, VK_ACCESS_TRANSFER_READ_BIT | VK_ACCESS_TRANSFER_WRITE_BIT, VK_ACCESS_HOST_READ_BIT | VK_ACCESS_SHADER_READ_BIT | VK_ACCESS_SHADER_WRITE_BIT | VK_ACCESS_TRANSFER_WRITE_BIT); + recorder.Submit(); + device.WaitQueue(); + recorder.Sync(); + const auto read = readback.Bytes(); + return std::vector(reinterpret_cast(read.data()), reinterpret_cast(read.data()) + linearBytes); + }; + std::vector texels(linearBytes); + for (auto& value : texels) value = random(); + const auto movesBefore = StorageTexture::SinglePassMoves(); + { + auto image = std::make_shared(context, detiler, resource, 0); + recorder.Keep(image); + results[mode].uploaded = readImage(*image); + Buffer staging(context, linearBytes, VK_BUFFER_USAGE_TRANSFER_SRC_BIT); + std::memcpy(staging.Bytes().data(), texels.data(), linearBytes); + const auto commands = recorder.Commands(); + RecordMemoryBarrier(context, commands, VK_PIPELINE_STAGE_ALL_COMMANDS_BIT, VK_PIPELINE_STAGE_TRANSFER_BIT, VK_ACCESS_SHADER_WRITE_BIT | VK_ACCESS_TRANSFER_WRITE_BIT | VK_ACCESS_COLOR_ATTACHMENT_WRITE_BIT, VK_ACCESS_TRANSFER_WRITE_BIT); + std::vector copies; + std::size_t at = 0; + for (std::uint32_t level = 0; level < geometry.mips.size(); ++level) { + VkBufferImageCopy copy{}; + copy.bufferOffset = at; + copy.imageSubresource = {VK_IMAGE_ASPECT_COLOR_BIT, level, 0, 1}; + copy.imageExtent = {geometry.mips[level].width, geometry.mips[level].height, 1}; + copies.push_back(copy); + at += static_cast(geometry.mips[level].width) * geometry.mips[level].height * elementBytes; + } + context.Function("vkCmdCopyBufferToImage")(commands, staging.Handle(), image->Image(), VK_IMAGE_LAYOUT_GENERAL, static_cast(copies.size()), copies.data()); + RecordMemoryBarrier(context, commands, VK_PIPELINE_STAGE_TRANSFER_BIT, VK_PIPELINE_STAGE_ALL_COMMANDS_BIT, VK_ACCESS_TRANSFER_WRITE_BIT, VK_ACCESS_SHADER_READ_BIT | VK_ACCESS_SHADER_WRITE_BIT | VK_ACCESS_TRANSFER_READ_BIT | VK_ACCESS_TRANSFER_WRITE_BIT); + image->MarkDirty(); + image->WriteBack(); + recorder.Submit(); + device.WaitQueue(); + recorder.Sync(); + auto again = std::make_shared(context, detiler, resource, 0); + recorder.Keep(again); + results[mode].reuploaded = readImage(*again); + std::vector read(bytes); + if (watched) AgcDriver::GuestMemory::Read(address, read); + else { + StorageTexture::FlushPending(address, bytes, nullptr, "test", PublishScope::Whole); + recorder.Submit(); + device.WaitQueue(); + recorder.Sync(); + std::memcpy(read.data(), memory, bytes); + } + results[mode].memory.assign(reinterpret_cast(read.data()), reinterpret_cast(read.data()) + bytes); + } + recorder.Sync(); + results[mode].moves = StorageTexture::SinglePassMoves() - movesBefore; + if (results[mode].reuploaded != texels) failures += what + (mode == 1 ? " (single pass)" : " (copies)") + ": the second image does not hold the texels written back\n"; + } + if (results[0].moves != 0 || results[1].moves < 3) failures += what + ": single-pass moves " + std::to_string(results[0].moves) + " with the copies and " + std::to_string(results[1].moves) + " in single pass (want 0 and at least 3)\n"; + if (results[0].uploaded != results[1].uploaded) failures += what + ": the first upload differs between the copies and single pass\n"; + if (results[0].memory != results[1].memory) { + std::size_t differ = 0; + for (std::size_t i = 0; i < results[0].memory.size(); ++i) differ += results[0].memory[i] != results[1].memory[i]; + failures += what + ": guest memory after the write-back differs in " + std::to_string(differ) + " bytes between the copies and single pass\n"; + } + ++tested; + } + if (!failures.empty()) throw std::runtime_error("single-pass storage moves:\n" + failures); + std::cout << "single-pass storage moves" << (watched ? " (watched)" : "") << ": ok (" << tested << " surfaces)\n"; +} + +void refreshProofTests(const Device& device, Recorder& recorder) { + const auto& base = device.GetContext(); + if (base.hostImportAlignment == 0) { + std::cout << "host imports unavailable: refresh proofs not tested\n"; + return; + } + constexpr std::uint32_t side = 256; + constexpr std::size_t surfaceBytes = side * side * 4; + constexpr std::size_t keyCount = surfaceBytes / 256; + constexpr std::size_t bytes = surfaceBytes + 65536; + void* block = AllocateWatched(bytes, 65536); + if (block == nullptr) { + std::cout << "no write watching: refresh proofs not tested\n"; + return; + } + auto* texels = static_cast(block); + auto* keys = texels + surfaceBytes; + std::memset(texels, 0x55, surfaceBytes); + std::memset(keys, 0xff, keyCount); + const auto address = reinterpret_cast(block); + { + GuestAllocations::Mutation mutation; + mutation.Add(block, bytes, true, true, true); + } + struct Unregister { + const Context& context; + void* block; + std::uint64_t address; + ~Unregister() { + { + GuestAllocations::Mutation mutation; + mutation.Remove(block); + } + HostImportFor(context, address, bytes); + } + } unregister{base, block, address}; + if (HostImportFor(base, address, bytes) == nullptr || !AgcDriver::GuestMemory::Watched(address, bytes)) { + std::cout << "no watched host import: refresh proofs not tested\n"; + return; + } + TextureDetiler detiler(base); + auto context = base; + context.detiler = &detiler; + GuestTextureResource resource{}; + resource.baseAddress = address; + resource.width = side; + resource.height = side; + resource.mipCount = 1; + resource.tileMode = TextureTileMode::kR64KBX; + resource.dimension = TextureDimension::k2D; + resource.format = 56; + resource.dstSelX = 4; + resource.dstSelY = 5; + resource.dstSelZ = 6; + resource.dstSelW = 7; + resource.dccAddress = address + surfaceBytes; + const auto proved = [] { return StorageTexture::RefreshesProved(); }; + { + auto image = std::make_shared(context, detiler, resource, 0); + const VkImageSubresourceRange range{VK_IMAGE_ASPECT_COLOR_BIT, 0, 1, 0, 1}; + const auto draw = [&](VkClearColorValue value) { + const auto commands = recorder.Commands(); + recorder.Keep(image); + RecordMemoryBarrier(context, commands, VK_PIPELINE_STAGE_ALL_COMMANDS_BIT, VK_PIPELINE_STAGE_TRANSFER_BIT, VK_ACCESS_SHADER_WRITE_BIT | VK_ACCESS_TRANSFER_WRITE_BIT | VK_ACCESS_COLOR_ATTACHMENT_WRITE_BIT, VK_ACCESS_TRANSFER_WRITE_BIT); + context.Function("vkCmdClearColorImage")(commands, image->Image(), VK_IMAGE_LAYOUT_GENERAL, &value, 1, &range); + RecordMemoryBarrier(context, commands, VK_PIPELINE_STAGE_TRANSFER_BIT, VK_PIPELINE_STAGE_ALL_COMMANDS_BIT, VK_ACCESS_TRANSFER_WRITE_BIT, VK_ACCESS_SHADER_READ_BIT | VK_ACCESS_SHADER_WRITE_BIT | VK_ACCESS_TRANSFER_READ_BIT | VK_ACCESS_TRANSFER_WRITE_BIT); + image->MarkDirty(); + }; + const auto settle = [&] { + image->Refresh(); + image->Refresh(); + }; + const auto expectProved = [&](const char* what) { + const auto before = proved(); + Require(image->Refresh(), what); + Require(proved() == before + 1, what); + }; + const auto expectFull = [&](const char* what) { + const auto before = proved(); + image->Refresh(); + Require(proved() == before, what); + }; + settle(); + expectProved("an unchanged surface was not proved current"); + expectProved("a proved surface was not proved again"); + draw({{1.0f, 0.0f, 0.0f, 1.0f}}); + settle(); + expectProved("a target holding its own pending results was not proved current"); + draw({{0.0f, 1.0f, 0.0f, 1.0f}}); + expectProved("a draw into an already pending target broke its proof"); + texels[0] = 0x11; + expectFull("a CPU write of the surface was answered from the proof"); + settle(); + expectProved("the proof was not taken again after a CPU write"); + AgcDriver::GuestMemory::MarkWritten(address + 65536, 4096); + expectFull("a driver store into the surface was answered from the proof"); + settle(); + AgcDriver::GuestMemory::MarkWritten(resource.dccAddress, keyCount); + expectFull("a key store over the surface's keys was answered from the proof"); + settle(); + expectProved("the proof was not taken again after a key store"); + auto other = resource; + other.dccAddress = 0; + auto pending = std::make_shared(context, detiler, other, 0); + recorder.Keep(pending); + pending->MarkDirty(); + expectFull("another image's pending results over the surface were answered from the proof"); + settle(); + expectProved("the proof was not taken again after another image's results were stored"); + pending->MarkDirty(); + expectFull("another image's pending results were answered from the proof"); + Require(!AgcDriver::Graphics::PendingStorageOverlaps(address, surfaceBytes, image.get()), "the full refresh left another image's results pending"); + auto fresh = std::make_shared(context, detiler, resource, 0); + const auto before = proved(); + fresh->Refresh(); + Require(proved() == before, "a new image of the surface was answered from another image's proof"); + std::memset(keys, 0x00, keyCount); + expectFull("a fast clear of the keys was answered from the proof"); + recorder.Sync(); + } + recorder.Sync(); +} + + +void unchangedCpuStampTests(const Device& device, Recorder& recorder) { + const auto& base = device.GetContext(); + constexpr std::uint32_t side = 256; + constexpr std::size_t surfaceBytes = side * side * 4; + constexpr std::uint64_t offset = 0x2000; + constexpr std::size_t bytes = 6 * 65536; + constexpr std::array red{255, 0, 0, 255}; + const auto pattern = [](std::size_t i) { return static_cast(i * 131u + 7u); }; + for (const bool imported : {false, true}) { +#ifdef _WIN32 + if (imported) { + std::cout << "unchanged CPU stamps over a watched host import: Linux write watch only\n"; + continue; + } +#endif + if (imported && base.hostImportAlignment == 0) { + std::cout << "host imports unavailable: unchanged CPU stamps over a watched host import not tested\n"; + continue; + } + const auto decided = PrepareImportWatch(base); + if (imported) SetImportWatch(base, ImportWatch::Watch); + struct Restore { + const Context& context; + ImportWatch decided; + ~Restore() { SetImportWatch(context, decided); } + } restore{base, decided}; + for (const bool changed : {false, true}) { + const std::string what = std::string(changed ? "a CPU store that changed a byte" : "a CPU store that left the bytes as they were") + (imported ? " over a watched host import" : " over watched memory"); + void* block = AllocateWatched(bytes, 65536); + if (block == nullptr) { + std::cout << "no write watching: unchanged CPU stamps not tested\n"; + return; + } + auto* memory = static_cast(block); + for (std::size_t i = 0; i < bytes; ++i) memory[i] = pattern(i); + const auto blockAddress = reinterpret_cast(block); + const auto address = blockAddress + offset; + { + GuestAllocations::Mutation mutation; + mutation.Add(block, bytes, true, true, true); + } + struct Unregister { + const Context& context; + void* block; + std::uint64_t address; + std::size_t bytes; + ~Unregister() { + { + GuestAllocations::Mutation mutation; + mutation.Remove(block); + } + HostImportFor(context, address, bytes); + ReleaseWatched(block, bytes); + } + } unregister{base, block, blockAddress, bytes}; + AgcDriver::GuestMemory::CollectWritesUncached(blockAddress, bytes); + TextureDetiler detiler(base); + auto context = base; + if (!imported) context.hostImportAlignment = 0; + context.detiler = &detiler; + GuestTextureResource resource{}; + resource.baseAddress = address; + resource.width = side; + resource.height = side; + resource.mipCount = 1; + resource.tileMode = TextureTileMode::kLinear; + resource.dimension = TextureDimension::k2D; + resource.format = 56; + resource.dstSelX = 4; + resource.dstSelY = 5; + resource.dstSelZ = 6; + resource.dstSelW = 7; + Require(DescribeSurface(resource).guestBytes == surfaceBytes, "(c) unexpected surface size"); + { + auto image = std::make_shared(context, detiler, resource, 0); + const VkImageSubresourceRange range{VK_IMAGE_ASPECT_COLOR_BIT, 0, 1, 0, 1}; + const auto commands = recorder.Commands(); + recorder.Keep(image); + const VkClearColorValue value{{1.0f, 0.0f, 0.0f, 1.0f}}; + RecordMemoryBarrier(context, commands, VK_PIPELINE_STAGE_ALL_COMMANDS_BIT, VK_PIPELINE_STAGE_TRANSFER_BIT, VK_ACCESS_SHADER_WRITE_BIT | VK_ACCESS_TRANSFER_WRITE_BIT | VK_ACCESS_COLOR_ATTACHMENT_WRITE_BIT, VK_ACCESS_TRANSFER_WRITE_BIT); + context.Function("vkCmdClearColorImage")(commands, image->Image(), VK_IMAGE_LAYOUT_GENERAL, &value, 1, &range); + RecordMemoryBarrier(context, commands, VK_PIPELINE_STAGE_TRANSFER_BIT, VK_PIPELINE_STAGE_ALL_COMMANDS_BIT, VK_ACCESS_TRANSFER_WRITE_BIT, VK_ACCESS_SHADER_READ_BIT | VK_ACCESS_SHADER_WRITE_BIT | VK_ACCESS_TRANSFER_READ_BIT | VK_ACCESS_TRANSFER_WRITE_BIT); + image->MarkDirty(); + for (std::size_t i = 0; i < 4096; ++i) memory[offset + i] = pattern(offset + i); + if (changed) memory[offset + 16] = static_cast(~pattern(offset + 16)); + AgcDriver::GuestMemory::CollectWritesUncached(blockAddress, bytes); + Require(AgcDriver::GuestMemory::Watched(address, surfaceBytes) && (HostImportFor(context, address, surfaceBytes) != nullptr) == imported, "(c) " + what + ": the image is not on the path under test"); + image->WriteBack(); + recorder.Submit(); + device.WaitQueue(); + recorder.Sync(); + std::vector read(bytes); + AgcDriver::GuestMemory::Read(blockAddress, read); + const auto texel = [&](std::size_t at) { + for (std::size_t c = 0; c < 4; ++c) { + if (std::to_integer(read[at + c]) != red[c]) return false; + } + return true; + }; + const auto headEnd = static_cast(65536); + std::size_t stored = 0, total = 0; + for (std::size_t at = offset; at < offset + surfaceBytes; at += 4) { + ++total; + if (texel(at)) ++stored; + } + if (!changed) { + if (stored != total) throw std::runtime_error(std::string("unchanged CPU stamps: ") + what + " dropped " + std::to_string(total - stored) + " of " + std::to_string(total) + " texels the GPU wrote"); + } else { + Require(std::to_integer(read[offset + 16]) == static_cast(~pattern(offset + 16)), "(c) " + what + ": the changed byte was overwritten by the GPU results"); + for (std::size_t at = headEnd; at < offset + surfaceBytes; at += 4) Require(texel(at), "(c) " + what + ": GPU results outside the block the CPU changed were dropped"); + } + } + recorder.Sync(); + } + } + std::cout << "unchanged CPU stamps: ok\n"; +} + void importWindowTests(const Device& device, Recorder& recorder) { using namespace AgcDriver::GuestMemory; const auto& base = device.GetContext(); @@ -2081,7 +3114,7 @@ void importWindowTests(const Device& device, Recorder& recorder) { const auto address = reinterpret_cast(block); { GuestAllocations::Mutation mutation; - mutation.Add(block, bytes, true, true); + mutation.Add(block, bytes, true, true, true); } struct Unregister { const Context& context; @@ -2170,6 +3203,287 @@ void importWindowTests(const Device& device, Recorder& recorder) { #endif } +void queuedWriteGroupTests() { + using Write = Recorder::QueuedWrite; + using Groups = std::vector; + const auto paint = [](const std::vector& writes, const Groups& groups, std::size_t bytes) { + std::vector memory(bytes, 0); + std::size_t first = 0; + for (const auto end : groups) { + for (const bool computed : {false, true}) { + for (auto i = first; i < end; ++i) { + if (writes[i].computed != computed) continue; + for (auto at = writes[i].begin; at < writes[i].end; ++at) memory[at] = static_cast(i + 1); + } + } + first = end; + } + return memory; + }; + const auto programOrder = [](const std::vector& writes, std::size_t bytes) { + std::vector memory(bytes, 0); + for (std::size_t i = 0; i < writes.size(); ++i) { + for (auto at = writes[i].begin; at < writes[i].end; ++at) memory[at] = static_cast(i + 1); + } + return memory; + }; + Require(Recorder::QueuedWriteGroups({}).empty(), "no queued writes made a group"); + const std::vector query{{0x100, 0x200, false}, {0x100, 0x1f8, true}, {0x108, 0x200, true}, {0x200, 0x300, false}, {0x200, 0x2f8, true}, {0x208, 0x300, true}}; + Require(Recorder::QueuedWriteGroups(query) == Groups{6}, "a clear, a begin dump and an end dump per query slot are not one group"); + const std::vector reused{{0x100, 0x1f8, true}, {0x100, 0x200, false}, {0x100, 0x1f8, true}}; + Require(Recorder::QueuedWriteGroups(reused) == Groups{1, 3}, "a store over a computed write of its group did not start a new group"); + const std::vector apart{{0x100, 0x1f8, true}, {0x1f8, 0x200, false}, {0x300, 0x310, false}}; + Require(Recorder::QueuedWriteGroups(apart) == Groups{3}, "stores beside the computed writes left their group"); + const std::vector stores{{0x100, 0x110, false}, {0x100, 0x110, false}}; + Require(Recorder::QueuedWriteGroups(stores) == Groups{2}, "stores alone were split into groups"); + std::uint64_t seed = 0x9e3779b97f4a7c15ull; + const auto next = [&](std::uint64_t bound) { + seed ^= seed << 13u; + seed ^= seed >> 7u; + seed ^= seed << 17u; + return seed % bound; + }; + constexpr std::size_t bytes = 64; + for (int round = 0; round < 20000; ++round) { + std::vector writes; + const auto count = 1 + next(12); + for (std::uint64_t i = 0; i < count; ++i) { + const auto begin = next(bytes - 1); + const auto end = begin + 1 + next(std::min(16, bytes - begin)); + writes.push_back({begin, std::min(end, bytes), next(2) == 0}); + } + const auto groups = Recorder::QueuedWriteGroups(writes); + Require(!groups.empty() && groups.back() == writes.size() && std::is_sorted(groups.begin(), groups.end()), "queued write groups do not end at the last write in order"); + Require(paint(writes, groups, bytes) == programOrder(writes, bytes), "recording each group's stores before its computed writes changed the final bytes of the program order"); + } +} + +constexpr std::uint64_t SampleReady = 1ull << 63u; + +void queuedSampleDumpTests(Recorder& recorder, const HostImport& import, std::uint64_t* words, std::uint64_t address, VkDeviceAddress target) { + constexpr std::uint64_t untouched = 0xaaaaaaaaaaaaaaaaull; + const auto storeZero = [&](std::uint64_t at, std::size_t bytes) { + const std::vector zero(bytes); + recorder.RecordStore(import.buffer, at - import.base, zero, at); + }; + recorder.Sync(); + const auto base = recorder.SamplesTotal(); + std::fill(words, words + 256, untouched); + Require(recorder.DumpSamples(target, address), "a dump was not queued"); + Require(recorder.HasQueuedStores() && recorder.QueuedStoreOverlaps(address + 240, 8) && !recorder.QueuedStoreOverlaps(address + 248, 8), "a queued dump does not cover its 16 counters"); + recorder.FlushStoresOverlapping(address + 256, 64); + Require(recorder.HasQueuedStores(), "a reader of other bytes recorded the queued dump"); + Require(words[0] == untouched, "a queued dump landed before anything recorded it"); + recorder.FlushStoresOverlapping(address + 16, 8); + Require(!recorder.HasQueuedStores(), "a reader of the dumped bytes left the dump queued"); + recorder.Sync(); + Require(words[0] == (base | SampleReady) && words[2] == SampleReady && words[1] == untouched, "the dump a reader recorded stored the wrong counters"); + std::fill(words, words + 256, untouched); + storeZero(address, 256); + Require(recorder.DumpSamples(target, address), "a dump after a clear was not queued"); + storeZero(address + 512, 16); + Require(recorder.DumpSamples(target + 8, address + 8), "the end dump was not queued"); + Require(recorder.DumpSamples(target + 256, address + 256), "a dump of another slot was not queued"); + storeZero(address + 256, 16); + Require(recorder.DumpSamples(target + 256 + 8, address + 256 + 8), "a dump after a store over an earlier dump was not queued"); + recorder.Sync(); + Require(words[0] == (base | SampleReady) && words[1] == (base | SampleReady) && words[2] == SampleReady && words[3] == SampleReady, "a clear and its dumps landed out of order"); + Require(words[32] == 0 && words[34] == SampleReady, "a store after a dump of its bytes did not land last"); + Require(words[33] == (base | SampleReady) && words[35] == SampleReady, "a dump after a store over an earlier dump landed before it"); + Require(words[64] == 0 && words[65] == 0 && words[66] == untouched, "a store between dumps was lost"); +} + +class CoverPass { +public: + explicit CoverPass(const Context& context) : context(context), target(context, ColorTarget{0, {8, 8}, VK_FORMAT_R8G8B8A8_UNORM, 8 * 8 * 4, 0}, false) { + try { + VkAttachmentDescription attachment{}; + attachment.format = VK_FORMAT_R8G8B8A8_UNORM; + attachment.samples = VK_SAMPLE_COUNT_1_BIT; + attachment.loadOp = VK_ATTACHMENT_LOAD_OP_DONT_CARE; + attachment.storeOp = VK_ATTACHMENT_STORE_OP_STORE; + attachment.stencilLoadOp = VK_ATTACHMENT_LOAD_OP_DONT_CARE; + attachment.stencilStoreOp = VK_ATTACHMENT_STORE_OP_DONT_CARE; + attachment.initialLayout = VK_IMAGE_LAYOUT_UNDEFINED; + attachment.finalLayout = VK_IMAGE_LAYOUT_GENERAL; + const VkAttachmentReference color{0, VK_IMAGE_LAYOUT_COLOR_ATTACHMENT_OPTIMAL}; + VkSubpassDescription subpass{}; + subpass.pipelineBindPoint = VK_PIPELINE_BIND_POINT_GRAPHICS; + subpass.colorAttachmentCount = 1; + subpass.pColorAttachments = &color; + VkRenderPassCreateInfo passInfo{VK_STRUCTURE_TYPE_RENDER_PASS_CREATE_INFO}; + passInfo.attachmentCount = 1; + passInfo.pAttachments = &attachment; + passInfo.subpassCount = 1; + passInfo.pSubpasses = &subpass; + Check(context.Function("vkCreateRenderPass")(context.device, &passInfo, nullptr, &renderPass), "vkCreateRenderPass cover"); + const auto view = target.View(); + VkFramebufferCreateInfo framebufferInfo{VK_STRUCTURE_TYPE_FRAMEBUFFER_CREATE_INFO}; + framebufferInfo.renderPass = renderPass; + framebufferInfo.attachmentCount = 1; + framebufferInfo.pAttachments = &view; + framebufferInfo.width = 8; + framebufferInfo.height = 8; + framebufferInfo.layers = 1; + Check(context.Function("vkCreateFramebuffer")(context.device, &framebufferInfo, nullptr, &framebuffer), "vkCreateFramebuffer cover"); + VkPipelineLayoutCreateInfo layoutInfo{VK_STRUCTURE_TYPE_PIPELINE_LAYOUT_CREATE_INFO}; + Check(context.Function("vkCreatePipelineLayout")(context.device, &layoutInfo, nullptr, &layout), "vkCreatePipelineLayout cover"); + const auto module = [&](const std::uint32_t* code, std::size_t bytes) { + VkShaderModuleCreateInfo info{VK_STRUCTURE_TYPE_SHADER_MODULE_CREATE_INFO}; + info.codeSize = bytes; + info.pCode = code; + VkShaderModule result = VK_NULL_HANDLE; + Check(context.Function("vkCreateShaderModule")(context.device, &info, nullptr, &result), "vkCreateShaderModule cover"); + return result; + }; + vertex = module(COVER_VERT_SPV, sizeof(COVER_VERT_SPV)); + fragment = module(COVER_FRAG_SPV, sizeof(COVER_FRAG_SPV)); + std::array stages{}; + stages[0] = {VK_STRUCTURE_TYPE_PIPELINE_SHADER_STAGE_CREATE_INFO, nullptr, 0, VK_SHADER_STAGE_VERTEX_BIT, vertex, "main", nullptr}; + stages[1] = {VK_STRUCTURE_TYPE_PIPELINE_SHADER_STAGE_CREATE_INFO, nullptr, 0, VK_SHADER_STAGE_FRAGMENT_BIT, fragment, "main", nullptr}; + VkPipelineVertexInputStateCreateInfo vertexInput{VK_STRUCTURE_TYPE_PIPELINE_VERTEX_INPUT_STATE_CREATE_INFO}; + VkPipelineInputAssemblyStateCreateInfo assembly{VK_STRUCTURE_TYPE_PIPELINE_INPUT_ASSEMBLY_STATE_CREATE_INFO}; + assembly.topology = VK_PRIMITIVE_TOPOLOGY_TRIANGLE_LIST; + const VkViewport viewport{0, 0, 8, 8, 0, 1}; + const VkRect2D scissor{{0, 0}, {8, 8}}; + VkPipelineViewportStateCreateInfo viewportState{VK_STRUCTURE_TYPE_PIPELINE_VIEWPORT_STATE_CREATE_INFO}; + viewportState.viewportCount = 1; + viewportState.pViewports = &viewport; + viewportState.scissorCount = 1; + viewportState.pScissors = &scissor; + VkPipelineRasterizationStateCreateInfo raster{VK_STRUCTURE_TYPE_PIPELINE_RASTERIZATION_STATE_CREATE_INFO}; + raster.polygonMode = VK_POLYGON_MODE_FILL; + raster.cullMode = VK_CULL_MODE_NONE; + raster.lineWidth = 1; + VkPipelineMultisampleStateCreateInfo multisample{VK_STRUCTURE_TYPE_PIPELINE_MULTISAMPLE_STATE_CREATE_INFO}; + multisample.rasterizationSamples = VK_SAMPLE_COUNT_1_BIT; + VkPipelineColorBlendAttachmentState blend{}; + blend.colorWriteMask = VK_COLOR_COMPONENT_R_BIT | VK_COLOR_COMPONENT_G_BIT | VK_COLOR_COMPONENT_B_BIT | VK_COLOR_COMPONENT_A_BIT; + VkPipelineColorBlendStateCreateInfo blendState{VK_STRUCTURE_TYPE_PIPELINE_COLOR_BLEND_STATE_CREATE_INFO}; + blendState.attachmentCount = 1; + blendState.pAttachments = &blend; + VkGraphicsPipelineCreateInfo pipelineInfo{VK_STRUCTURE_TYPE_GRAPHICS_PIPELINE_CREATE_INFO}; + pipelineInfo.stageCount = static_cast(stages.size()); + pipelineInfo.pStages = stages.data(); + pipelineInfo.pVertexInputState = &vertexInput; + pipelineInfo.pInputAssemblyState = &assembly; + pipelineInfo.pViewportState = &viewportState; + pipelineInfo.pRasterizationState = &raster; + pipelineInfo.pMultisampleState = &multisample; + pipelineInfo.pColorBlendState = &blendState; + pipelineInfo.layout = layout; + pipelineInfo.renderPass = renderPass; + Check(context.Function("vkCreateGraphicsPipelines")(context.device, VK_NULL_HANDLE, 1, &pipelineInfo, nullptr, &pipeline), "vkCreateGraphicsPipelines cover"); + } catch (...) { + release(); + throw; + } + } + ~CoverPass() { release(); } + CoverPass(const CoverPass&) = delete; + CoverPass& operator=(const CoverPass&) = delete; + + bool Draw(Recorder& recorder) const { + const auto start = recorder.StartDrawPass(PassKey, false, false, PassAccess{}); + const bool continued = start.continued; + const auto commands = start.commands; + if (!continued) { + recorder.PrepareSampleSlot(); + VkRenderPassBeginInfo begin{VK_STRUCTURE_TYPE_RENDER_PASS_BEGIN_INFO}; + begin.renderPass = renderPass; + begin.framebuffer = framebuffer; + begin.renderArea = {{0, 0}, {8, 8}}; + context.Function("vkCmdBeginRenderPass")(commands, &begin, VK_SUBPASS_CONTENTS_INLINE); + } + context.Function("vkCmdBindPipeline")(commands, VK_PIPELINE_BIND_POINT_GRAPHICS, pipeline); + recorder.NoteSampledDraw(commands); + context.Function("vkCmdDraw")(commands, 3, 1, 0, 0); + recorder.LeaveRenderPassOpen(PassKey, Recorder::NoTiming, false, PassAccess{}); + return continued; + } + + static constexpr std::uint64_t Samples = 64; + +private: + static constexpr std::uint64_t PassKey = 0xc0de; + void release() noexcept { + const auto device = context.device; + if (pipeline != VK_NULL_HANDLE) context.Function("vkDestroyPipeline")(device, pipeline, nullptr); + if (vertex != VK_NULL_HANDLE) context.Function("vkDestroyShaderModule")(device, vertex, nullptr); + if (fragment != VK_NULL_HANDLE) context.Function("vkDestroyShaderModule")(device, fragment, nullptr); + if (layout != VK_NULL_HANDLE) context.Function("vkDestroyPipelineLayout")(device, layout, nullptr); + if (framebuffer != VK_NULL_HANDLE) context.Function("vkDestroyFramebuffer")(device, framebuffer, nullptr); + if (renderPass != VK_NULL_HANDLE) context.Function("vkDestroyRenderPass")(device, renderPass, nullptr); + pipeline = VK_NULL_HANDLE; + vertex = fragment = VK_NULL_HANDLE; + layout = VK_NULL_HANDLE; + framebuffer = VK_NULL_HANDLE; + renderPass = VK_NULL_HANDLE; + } + const Context& context; + RenderTarget target; + VkRenderPass renderPass = VK_NULL_HANDLE; + VkFramebuffer framebuffer = VK_NULL_HANDLE; + VkPipelineLayout layout = VK_NULL_HANDLE; + VkShaderModule vertex = VK_NULL_HANDLE; + VkShaderModule fragment = VK_NULL_HANDLE; + VkPipeline pipeline = VK_NULL_HANDLE; +}; + +void inPassSampleDumpTests(Recorder& recorder, const Context& context, const CoverPass& cover, std::uint64_t* words, std::uint64_t address, VkDeviceAddress target) { + const auto samples = [&](std::uint64_t word) { return word & ~SampleReady; }; + recorder.Sync(); + auto base = recorder.SamplesTotal(); + std::fill(words, words + 64, 0); + Require(!cover.Draw(recorder), "the first draw of a batch continued a pass"); + Require(recorder.DumpSamples(target, address), "a dump inside a render pass was not made on the GPU"); + Require(cover.Draw(recorder), "an occlusion dump ended the render pass"); + Require(recorder.DumpSamples(target + 8, address + 8), "the second dump inside the pass was not made on the GPU"); + Require(cover.Draw(recorder), "the second occlusion dump ended the render pass"); + Require(words[0] == 0 && words[1] == 0, "a dump inside the pass landed before its batch ran"); + recorder.Sync(); + const auto first = samples(words[0]) - base, second = samples(words[1]) - base, total = recorder.SamplesTotal() - base; + Require((words[0] & SampleReady) != 0 && (words[1] & SampleReady) != 0, "a dump inside a pass lost its ready bit"); + Require(first > 0 && second > first && total > second, "dumps inside one pass do not split its draws' samples in order"); + if (context.occlusionQueryPrecise) Require(first == CoverPass::Samples && second == 2 * CoverPass::Samples && total == 3 * CoverPass::Samples, "dumps inside one pass counted the wrong samples"); + base = recorder.SamplesTotal(); + Require(!cover.Draw(recorder), "the first draw of a new batch continued a pass"); + Require(recorder.DumpSamples(target, address), "a dump before a reader was not made on the GPU"); + recorder.FlushStoresOverlapping(address, 8); + Require(!cover.Draw(recorder), "a reader of a queued dump recorded it inside the render pass"); + recorder.Sync(); + Require(samples(words[0]) > base && samples(words[0]) < recorder.SamplesTotal(), "the dump a reader recorded in the pass counted the wrong draws"); + base = recorder.SamplesTotal(); + constexpr std::size_t slots = 160; + std::fill(words, words + slots * 32, 0); + std::size_t continued = 0; + for (std::size_t i = 0; i < slots; ++i) { + if (cover.Draw(recorder)) ++continued; + Require(recorder.DumpSamples(target + 256 * i, address + 256 * i), "a dump of many in one pass was not made on the GPU"); + } + recorder.Sync(); + Require(continued + 2 >= slots, "draws between dumps did not keep their render pass"); + std::uint64_t previous = base; + for (std::size_t i = 0; i < slots; ++i) { + const auto value = samples(words[32 * i]); + Require((words[32 * i] & SampleReady) != 0 && value > previous, "a dump past one query pool's slots counted the wrong draws"); + if (context.occlusionQueryPrecise) Require(value == base + (i + 1) * CoverPass::Samples, "a dump past one query pool's slots counted the wrong samples"); + previous = value; + } + base = recorder.SamplesTotal(); + constexpr std::size_t passes = 70; + for (std::size_t i = 0; i < passes; ++i) { + Require(!cover.Draw(recorder), "a draw after other work continued a pass"); + recorder.Commands(); + } + recorder.Submit(); + Require(recorder.DumpSamples(target, address), "a dump after many passes was not made on the GPU"); + recorder.Sync(); + const auto folded = samples(words[0]); + Require(folded > base && folded == recorder.SamplesTotal(), "segments folded at Submit without a dump were lost"); + if (context.occlusionQueryPrecise) Require(folded == base + passes * CoverPass::Samples, "segments folded at Submit without a dump counted the wrong samples"); +} + void pendingKeyStoreTests(const Device& device, Recorder& recorder) { const auto& context = device.GetContext(); if (context.hostImportAlignment == 0) { @@ -2189,7 +3503,7 @@ void pendingKeyStoreTests(const Device& device, Recorder& recorder) { const auto address = reinterpret_cast(block); { GuestAllocations::Mutation mutation; - mutation.Add(block, bytes, true, true); + mutation.Add(block, bytes, true, true, true); } struct Unregister { const Context& context; @@ -2256,7 +3570,7 @@ void sampleDumpTests(const Device& device, Recorder& recorder) { const auto address = reinterpret_cast(block); { GuestAllocations::Mutation mutation; - mutation.Add(block, bytes, true, true); + mutation.Add(block, bytes, true, true, true); } struct Unregister { const Context& context; @@ -2278,7 +3592,7 @@ void sampleDumpTests(const Device& device, Recorder& recorder) { const auto target = import->address + (address - import->base); recorder.Sync(); constexpr std::uint64_t ready = 1ull << 63u; - Require(recorder.DumpSamples(target), "the occlusion counters were not dumped on the GPU"); + Require(recorder.DumpSamples(target, address), "the occlusion counters were not dumped on the GPU"); Require(words[0] == untouched && !recorder.Idle(), "the occlusion counter dump waited for the GPU or landed before its batch ran"); recorder.Sync(); const auto begin = recorder.SamplesTotal(); @@ -2286,7 +3600,7 @@ void sampleDumpTests(const Device& device, Recorder& recorder) { Require(words[db * 2] == ((db == 0 ? begin : 0) | ready), "an occlusion counter dump stored the wrong value"); Require(words[db * 2 + 1] == untouched, "an occlusion counter dump stored over the next counter"); } - Require(recorder.DumpSamples(target + 8), "the second occlusion counter dump was not made on the GPU"); + Require(recorder.DumpSamples(target + 8, address + 8), "the second occlusion counter dump was not made on the GPU"); recorder.Submit(); recorder.Sync(); for (std::size_t db = 0; db < 16; ++db) { @@ -2294,9 +3608,17 @@ void sampleDumpTests(const Device& device, Recorder& recorder) { Require(words[db * 2] == ((db == 0 ? begin : 0) | ready), "the second dump stored over the first"); } Require(recorder.SamplesTotal() == begin, "the sample total moved with nothing drawn"); - for (int i = 0; i < 40; ++i) Require(recorder.DumpSamples(target), "a dump past one batch's query slots was not made on the GPU"); + for (int i = 0; i < 40; ++i) Require(recorder.DumpSamples(target, address), "a dump past one batch's query slots was not made on the GPU"); recorder.Sync(); Require(words[0] == (begin | ready) && recorder.SamplesTotal() == begin, "dumps past one batch's query slots moved the total"); + Require(recorder.QueuesSampleDumps(), "occlusion dumps are recorded at once, not queued with the batch's stores"); + queuedSampleDumpTests(recorder, *import, words, address, target); + if (!device.Graphics()) { + std::cout << "the test queue has no graphics: occlusion dumps inside a render pass not tested\n"; + return; + } + CoverPass cover(context); + inPassSampleDumpTests(recorder, context, cover, words, address, target); } void metadataPassTests(const Device& device, Recorder& recorder) { @@ -2320,7 +3642,7 @@ void metadataPassTests(const Device& device, Recorder& recorder) { const auto address = reinterpret_cast(block); { GuestAllocations::Mutation mutation; - mutation.Add(block, bytes, true, true); + mutation.Add(block, bytes, true, true, true); } struct Unregister { const Context& context; @@ -2446,6 +3768,105 @@ void dataWordPositionsTests() { } } +std::array linearTextureWords(std::uint64_t address, std::uint32_t width, std::uint32_t height, std::uint32_t format) { + const auto base40 = address >> 8u; + std::array words{}; + words[0] = static_cast(base40); + words[1] = static_cast((base40 >> 32u) & 0xffu) | (format << 20u) | (((width - 1u) & 3u) << 30u); + words[2] = (((width - 1u) >> 2u) & 0xfffu) | ((height - 1u) << 14u); + words[3] = 4u | (5u << 3u) | (6u << 6u) | (7u << 9u) | (9u << 28u); + return words; +} + +void imageMemoTests(const Device& device, Recorder& recorder) { + const auto& base = device.GetContext(); + TextureDetiler detiler(base); + auto context = base; + context.detiler = &detiler; + TextureCache textureCache(context); + context.textureCache = &textureCache; + if (!device.NullDescriptors()) { + std::cout << "nullDescriptor unavailable: image memo not tested\n"; + return; + } + context.nullDescriptors = true; + constexpr std::uint32_t width = 64; + constexpr std::uint32_t height = 4; + constexpr std::size_t surfaceBytes = width * height * 4; + constexpr std::size_t bytes = 2 * 65536; + void* block = AllocateWatched(bytes, 65536); + const bool watched = block != nullptr; +#ifdef _WIN32 + if (!watched) block = VirtualAlloc(nullptr, bytes, MEM_RESERVE | MEM_COMMIT, PAGE_READWRITE); +#else + if (!watched) block = std::aligned_alloc(65536, bytes); +#endif + Require(block != nullptr, "cannot allocate the image memo block"); + auto* first = static_cast(block); + auto* second = first + 65536; + std::memset(first, 0x11, surfaceBytes); + std::memset(second, 0x22, surfaceBytes); + ShaderRecompiler::RecompileResult program; + ShaderRecompiler::DescriptorBinding binding; + binding.kind = ShaderRecompiler::DescriptorKind::SampledImage; + binding.role = ShaderRecompiler::DescriptorRole::GuestImages; + binding.descriptorSet = 0; + binding.binding = 1; + binding.count = 1; + binding.imageShape = ShaderRecompiler::DescriptorImageShape::Image2D; + binding.imageSamplers = {0u}; + program.bindings.push_back(binding); + const CompiledShader compiled{ShaderRecompiler::ShaderStage::Compute, &program, 0}; + std::vector> kept; + const auto sample = [&](const std::uint8_t* surface) { + const auto words = linearTextureWords(reinterpret_cast(surface), width, height, 56); + program.bindings[0].guestDescriptor.assign(words.begin(), words.end()); + ShaderResources resources(context, compiled); + Require(resources.SampledTextures().size() == 1, "the build has no sampled texture"); + kept.push_back(resources.SampledTextures().front()); + return kept.back(); + }; + const auto made = sample(first); + Require(sample(first) == made, "an unchanged sampled texture was made again"); + const auto other = sample(second); + Require(other != made && sample(first) == made, "a texture of another descriptor answered for the first"); + ClearCachedTextures(context.device); + const auto remade = sample(first); + Require(remade != made && remade != other, "a texture the cache dropped was bound again"); + Require(sample(first) == remade, "the remade texture is not the cache's"); + const auto replaced = sample(second); + Require(replaced != other && sample(second) == replaced && sample(first) == remade, "the cache does not answer with the replacements"); + const auto firstWords = linearTextureWords(reinterpret_cast(first), width, height, 56); + const auto secondWords = linearTextureWords(reinterpret_cast(second), width, height, 56); + const auto bindElements = [&](std::initializer_list> elements, std::vector depthCompare) { + program.bindings[0].count = static_cast(elements.size()); + program.bindings[0].guestDescriptor.clear(); + for (const auto& words : elements) program.bindings[0].guestDescriptor.insert(program.bindings[0].guestDescriptor.end(), words.begin(), words.end()); + program.bindings[0].imageDepthCompare = std::move(depthCompare); + program.bindings[0].imageSamplers.assign(elements.size(), 0u); + ShaderResources resources(context, compiled); + Require(resources.SampledTextures().size() == elements.size(), "the build has the wrong number of sampled textures"); + kept.insert(kept.end(), resources.SampledTextures().begin(), resources.SampledTextures().end()); + return resources.SampledTextures(); + }; + const auto repeated = bindElements({firstWords, firstWords, secondWords, firstWords}, {}); + Require(repeated[0] == remade && repeated[1] == remade && repeated[2] == replaced && repeated[3] == remade, "repeated elements of one build do not bind the cache's textures"); + program.bindings[0].count = 1; + program.bindings[0].imageDepthCompare.clear(); + program.bindings[0].imageSamplers.assign(1, 0u); + recorder.Submit(); + device.WaitQueue(); + recorder.Sync(); + kept.clear(); + ClearCachedTextures(context.device); + if (watched) ReleaseWatched(block, bytes); +#ifdef _WIN32 + else VirtualFree(block, 0, MEM_RELEASE); +#else + else std::free(block); +#endif +} + // A template's data buffers refreshed by words from a patched compiled result (a data-only hit) // and back: DataWordsHash() follows the buffers exactly, so a later recipe hit's hash compare // (RecordedDispatch::DataRefresh::Hash) decides correctly in both directions. @@ -2758,6 +4179,71 @@ void atomicViewTests(const Device& device, Recorder& recorder) { } } +void pipelineCacheTests(const Device& device, bool benchmark) { + std::thread establishThreadedProcess([] {}); + establishThreadedProcess.join(); + std::lock_guard gpu(GpuMutex()); + const auto& context = device.GetContext(); + struct Cleanup { + VkDevice device; + ~Cleanup() { ClearCachedPipelines(device); } + } cleanup{context.device}; + ShaderRecompiler::RecompileResult vertex, fragment; + vertex.variantId = 1; + fragment.variantId = 2; + vertex.spirv = std::vector(std::begin(TRIANGLE_vert_SPV), std::end(TRIANGLE_vert_SPV)); + fragment.spirv = std::vector(std::begin(TRIANGLE_frag_SPV), std::end(TRIANGLE_frag_SPV)); + const std::array shaders{CompiledShader{ShaderRecompiler::ShaderStage::Vertex, &vertex, 0}, CompiledShader{ShaderRecompiler::ShaderStage::Fragment, &fragment, 0}}; + State state{}; + state.topology = VK_PRIMITIVE_TOPOLOGY_TRIANGLE_LIST; + state.hasColorTarget = true; + state.color.format = VK_FORMAT_R8G8B8A8_UNORM; + state.colors.push_back(state.color); + state.blend.colorWriteMask = 15; + state.blends.push_back(state.blend); + ShaderResources resources(context, shaders, state.color, static_cast(state.colors.size()), 0, 0); + const auto lookup = [&](const VertexInputLayout& input) { + return CachedPipeline(context, state, input, resources, shaders, VK_IMAGE_LAYOUT_GENERAL); + }; + VertexInputLayout input; + const auto first = lookup(input); + Require(lookup(input) == first, "pipeline cache did not retain an identical key"); + state.cullMode = VK_CULL_MODE_BACK_BIT; + Require(lookup(input) != first, "pipeline cache ignored changed raster state"); + state.cullMode = 0; + for (const auto attributes : {0u, 8u, 32u}) { + Require(attributes <= context.limits.maxVertexInputBindings && attributes <= context.limits.maxVertexInputAttributes, "pipeline cache fixture exceeds vertex input limits"); + input.bindings.clear(); + input.attributes.clear(); + for (std::uint32_t i = 0; i < attributes; ++i) { + input.bindings.push_back({i, 16, VK_VERTEX_INPUT_RATE_VERTEX}); + input.attributes.push_back({i, i, VK_FORMAT_R32G32B32A32_SFLOAT, 0}); + } + const auto expected = lookup(input); + if (attributes != 0) Require(expected != first, "pipeline cache ignored changed vertex input"); + Require(lookup(input) == expected, "pipeline cache lost the larger key"); + if (!benchmark) continue; + std::array times; + for (unsigned pass = 0; pass <= times.size(); ++pass) { + const auto started = std::chrono::steady_clock::now(); + for (unsigned i = 0; i < 65536; ++i) Require(lookup(input) == expected, "pipeline cache benchmark missed a warmed key"); + if (pass != 0) times[pass - 1] = std::chrono::duration(std::chrono::steady_clock::now() - started).count() / 65536; + } + std::ranges::sort(times); + std::cout << "Pipeline cache " << attributes << " attributes: median " << times[4] << " us/lookup, p95 pass " << times.back() << " us\n"; + } + input = {}; + Require(lookup(input) == first, "pipeline cache retained bytes from a larger key"); + vertex.variantId = 0; + const auto unidentified = lookup(input); + Require(lookup(input) != unidentified, "pipeline cache retained an unidentified shader"); + vertex.variantId = 1; + Require(lookup(input) == first, "an unidentified shader disturbed the retained pipeline"); + ClearCachedPipelines(context.device); + Require(lookup(input) != first, "clearing the pipeline cache retained its old entry"); + std::cout << "Pipeline cache key and retained lifetime tests passed\n"; +} + void keysFillTests(const Device& device, Recorder& recorder) { const auto& base = device.GetContext(); if (base.hostImportAlignment == 0) { @@ -2782,7 +4268,7 @@ void keysFillTests(const Device& device, Recorder& recorder) { const auto keysAddress = address + surfaceBytes; { GuestAllocations::Mutation mutation; - mutation.Add(block, bytes, true, true); + mutation.Add(block, bytes, true, true, true); } struct Unregister { const Context& context; @@ -2874,10 +4360,191 @@ void keysFillTests(const Device& device, Recorder& recorder) { recorder.Sync(); } +void denormalClearTests(const Device& device, Recorder& recorder) { + const auto& base = device.GetContext(); + if (base.hostImportAlignment == 0) { + std::cout << "host imports unavailable: denormal fill clears not tested\n"; + return; + } + constexpr std::uint32_t side = 64; + constexpr std::size_t bytes = 65536; +#ifdef _WIN32 + void* block = VirtualAlloc(nullptr, bytes, MEM_RESERVE | MEM_COMMIT, PAGE_READWRITE); +#else + void* block = std::aligned_alloc(65536, bytes); +#endif + Require(block != nullptr, "cannot allocate the denormal clear block"); + std::memset(block, 0x55, bytes); + const auto address = reinterpret_cast(block); + { + GuestAllocations::Mutation mutation; + mutation.Add(block, bytes, true, true, true); + } + struct Unregister { + const Context& context; + void* block; + std::uint64_t address; + ~Unregister() { + { + GuestAllocations::Mutation mutation; + mutation.Remove(block); + } + HostImportFor(context, address, bytes); + } + } unregister{base, block, address}; + if (HostImportFor(base, address, bytes) == nullptr) { + std::cout << "host import of the denormal clear block refused: denormal fill clears not tested\n"; + return; + } + TextureDetiler detiler(base); + auto context = base; + context.detiler = &detiler; + GuestTextureResource resource{}; + resource.baseAddress = address; + resource.width = side; + resource.height = side; + resource.mipCount = 1; + resource.tileMode = TextureTileMode::kR64KBX; + resource.dimension = TextureDimension::k2D; + resource.format = 29; + resource.dstSelX = 4; + resource.dstSelY = 5; + resource.dstSelZ = 6; + resource.dstSelW = 7; + Require(ResolveTextureFormat(resource.format) == VK_FORMAT_R16G16_SFLOAT, "guest format 29 is not R16G16_SFLOAT"); + const bool keeps = ClearKeepsDenormals(context, VK_FORMAT_R16G16_SFLOAT); + std::cout << "denormal half clears are " << (keeps ? "kept" : "flushed") << " by this device\n"; + Require(ClearKeepsDenormals(context, VK_FORMAT_R16G16_SFLOAT) == keeps, "the denormal clear probe changed its answer"); + Require(!ClearKeepsDenormals(context, VK_FORMAT_R32G32B32A32_SFLOAT), "a format without small floats probed as keeping denormals"); + { + auto image = std::make_shared(context, detiler, resource, 0); + const auto holds = [&](std::uint32_t texel) { + Buffer readback(context, side * side * 4, VK_BUFFER_USAGE_TRANSFER_DST_BIT); + const auto commands = recorder.Commands(); + RecordMemoryBarrier(context, commands, VK_PIPELINE_STAGE_ALL_COMMANDS_BIT, VK_PIPELINE_STAGE_TRANSFER_BIT, VK_ACCESS_SHADER_WRITE_BIT | VK_ACCESS_TRANSFER_WRITE_BIT | VK_ACCESS_COLOR_ATTACHMENT_WRITE_BIT, VK_ACCESS_TRANSFER_READ_BIT); + VkBufferImageCopy copy{}; + copy.imageSubresource = {VK_IMAGE_ASPECT_COLOR_BIT, 0, 0, 1}; + copy.imageExtent = {side, side, 1}; + context.Function("vkCmdCopyImageToBuffer")(commands, image->Image(), VK_IMAGE_LAYOUT_GENERAL, readback.Handle(), 1, ©); + RecordMemoryBarrier(context, commands, VK_PIPELINE_STAGE_TRANSFER_BIT, VK_PIPELINE_STAGE_HOST_BIT, VK_ACCESS_TRANSFER_WRITE_BIT, VK_ACCESS_HOST_READ_BIT); + recorder.Submit(); + device.WaitQueue(); + recorder.Sync(); + const auto pixels = readback.Bytes(); + for (std::size_t at = 0; at + 4 <= pixels.size(); at += 4) { + std::uint32_t value = 0; + std::memcpy(&value, pixels.data() + at, 4); + if (value != texel) return false; + } + return true; + }; + const char* refusal = nullptr; + const std::array normal{0x3c00bc00u, 0x3c00bc00u, 0x3c00bc00u, 0x3c00bc00u}; + Require(image->FillClear(normal, StorageTexture::WholeImage, refusal), "a normal half pattern was refused"); + Require(holds(0x3c00bc00u), "a normal half fill clear stored other bits"); + const std::array denormal{0x83ff0001u, 0x83ff0001u, 0x83ff0001u, 0x83ff0001u}; + refusal = nullptr; + const bool cleared = image->FillClear(denormal, StorageTexture::WholeImage, refusal); + Require(cleared == keeps, keeps ? "a denormal half pattern was refused on a device whose clears keep denormals" : "a denormal half pattern was cleared on a device whose clears flush them"); + if (cleared) Require(holds(0x83ff0001u), "a denormal half fill clear stored other bits"); + const std::array nan{0x7e007e00u, 0x7e007e00u, 0x7e007e00u, 0x7e007e00u}; + refusal = nullptr; + Require(!image->FillClear(nan, StorageTexture::WholeImage, refusal), "a NaN half pattern was cleared"); + } + recorder.Sync(); +} + +void pipelineLibraryTests(const Device& device) { + std::lock_guard gpu(GpuMutex()); + const auto& context = device.GetContext(); + struct Cleanup { + VkDevice device; + ~Cleanup() { ClearCachedPipelines(device); } + } cleanup{context.device}; + ClearCachedPipelines(context.device); + ShaderRecompiler::RecompileResult vertex, fragment; + vertex.variantId = 11; + fragment.variantId = 12; + vertex.spirv = std::vector(std::begin(TRIANGLE_vert_SPV), std::end(TRIANGLE_vert_SPV)); + fragment.spirv = std::vector(std::begin(TRIANGLE_frag_SPV), std::end(TRIANGLE_frag_SPV)); + const std::array shaders{CompiledShader{ShaderRecompiler::ShaderStage::Vertex, &vertex, 0}, CompiledShader{ShaderRecompiler::ShaderStage::Fragment, &fragment, 0}}; + State state{}; + state.topology = VK_PRIMITIVE_TOPOLOGY_TRIANGLE_LIST; + state.hasColorTarget = true; + state.color.format = VK_FORMAT_R8G8B8A8_UNORM; + state.colors.push_back(state.color); + state.blend.colorWriteMask = 15; + state.blends.push_back(state.blend); + ShaderResources resources(context, shaders, state.color, static_cast(state.colors.size()), 0, 0); + VertexInputLayout input; + const auto lookup = [&] { return CachedPipeline(context, state, input, resources, shaders, VK_IMAGE_LAYOUT_GENERAL); }; + const auto counters = [&] { return PipelineLibraryCountersOf(context.device); }; + const auto first = lookup(); + if (!context.graphicsPipelineLibrary) { + Require(counters().linked == 0, "a pipeline was linked from libraries without VK_EXT_graphics_pipeline_library"); + std::cout << "graphics pipeline libraries unavailable or disabled: pipelines are created whole\n"; + return; + } + const auto built = counters(); + Require(built.linked == 1 && built.built == std::array{1, 1, 1, 1}, "the first pipeline was not linked from four new libraries"); + state.blends.front().blendEnable = VK_TRUE; + state.blends.front().srcColorBlendFactor = VK_BLEND_FACTOR_SRC_ALPHA; + state.blends.front().dstColorBlendFactor = VK_BLEND_FACTOR_ONE_MINUS_SRC_ALPHA; + const auto blended = lookup(); + Require(blended != first && counters().built == std::array{1, 1, 1, 2}, "a blend change rebuilt more than the fragment output library"); + state.cullMode = VK_CULL_MODE_BACK_BIT; + state.frontFace = VK_FRONT_FACE_CLOCKWISE; + const auto culled = lookup(); + Require(culled != blended && counters().built == std::array{1, 1, 1, 2}, "a cull mode change rebuilt a library instead of setting dynamic state"); + input.bindings.push_back({0, 16, VK_VERTEX_INPUT_RATE_VERTEX}); + input.attributes.push_back({0, 0, VK_FORMAT_R32G32B32A32_SFLOAT, 0}); + static_cast(lookup()); + Require(counters().built == std::array{2, 1, 1, 2} && counters().linked == 4, "a vertex input change rebuilt more than the vertex input library"); + Require(StagePushOffset(88, ShaderRecompiler::ShaderStage::Fragment, true) == PipelinePushSlotBytes && StagePushOffset(88, ShaderRecompiler::ShaderStage::Fragment, false) == 88 && StagePushOffset(40, ShaderRecompiler::ShaderStage::TessellationEvaluation, true) == 40, "a stage got the wrong push offset"); + const auto beforeSlots = counters().built; + ShaderRecompiler::RecompileResult otherVertex = vertex; + otherVertex.variantId = 14; + std::array slotted{CompiledShader{ShaderRecompiler::ShaderStage::Vertex, &vertex, 0}, CompiledShader{ShaderRecompiler::ShaderStage::Fragment, &fragment, PipelinePushSlotBytes}}; + static_cast(CachedPipeline(context, state, input, resources, slotted, VK_IMAGE_LAYOUT_GENERAL)); + slotted.front().program = &otherVertex; + static_cast(CachedPipeline(context, state, input, resources, slotted, VK_IMAGE_LAYOUT_GENERAL)); + Require(counters().built[2] == beforeSlots[2] + 1u && counters().built[1] == beforeSlots[1] + 1u, "a pixel shader in its fixed slot built its library again for another vertex shader"); + fragment.variantId = 13; + static_cast(lookup()); + Require(counters().built == std::array{2, 2, 3, 2}, "a new pixel shader rebuilt more than the fragment shader library"); + state.colors.front().format = VK_FORMAT_R16G16B16A16_SFLOAT; + static_cast(lookup()); + Require(counters().built == std::array{2, 2, 3, 3}, "another target format rebuilt more than the fragment output library"); + WaitForOptimizedPipelines(context.device); + const auto lto = std::getenv("APS5_NO_PIPELINE_LTO") == nullptr; + Require(counters().optimized == (lto ? counters().linked : 0u), "a linked pipeline was not optimized in the background"); + Require(first->Optimized() == lto && culled->Optimized() == lto, "a pipeline did not switch to its optimized link"); + ClearCachedPipelines(context.device); + Require(counters().linked == 0, "clearing the pipelines kept the device's libraries"); + std::cout << "Pipeline library reuse tests passed\n"; +} + int main(int argc, char** argv) { try { - Device device; - std::lock_guard gpu(GpuMutex()); + std::unique_ptr created; + try { + created = std::make_unique(); + } catch (const std::exception& error) { + if (std::getenv("ANYPS5_REQUIRE_VULKAN") != nullptr) throw; + std::cout << "skipped, no usable Vulkan device: " << error.what() << '\n'; + return 77; + } + Device& device = *created; + if (argc == 2 && std::string_view(argv[1]) == "--pipeline-library-only") { + pipelineLibraryTests(device); + return 0; + } + if (argc == 2 && (std::string_view(argv[1]) == "--benchmark-pipeline-cache" || std::string_view(argv[1]) == "--pipeline-cache-only")) { + pipelineCacheTests(device, std::string_view(argv[1]) == "--benchmark-pipeline-cache"); + return 0; + } + + std::unique_lock gpu(GpuMutex()); std::cout << "host imports " << (PrepareImportWatch(device.GetContext()) == ImportWatch::Unwatch ? "are compared" : "stay watched") << '\n'; Recorder recorder(device.GetContext()); recorder.Activate(); @@ -2886,10 +4553,30 @@ int main(int argc, char** argv) { std::cout << "Single cube snapshot and storage sampling tests passed\n"; return 0; } + if (argc == 2 && std::string_view(argv[1]) == "--unchanged-cpu-stamps-only") { + unchangedCpuStampTests(device, recorder); + return 0; + } + if (argc == 2 && std::string_view(argv[1]) == "--completion-labels-only") { + completionCountTests(device, recorder); + afterRecordedWorkTests(device, recorder); + directCompletionLabelTests(recorder); + labelTests(recorder); + lateLabelTests(recorder); + largeLabelTests(recorder); + std::cout << "Completion label storage and ordering tests passed\n"; + return 0; + } + if (argc == 2 && std::string_view(argv[1]) == "--host-import-unmap-only") { + gpu.unlock(); + hostImportUnmapTests(device, recorder); + return 0; + } readTrackingTests(device, recorder); writeSettledTests(device, recorder); completionCountTests(device, recorder); afterRecordedWorkTests(device, recorder); + directCompletionLabelTests(recorder); batchStampTests(recorder); labelTests(recorder); lateLabelTests(recorder); @@ -2897,27 +4584,38 @@ int main(int argc, char** argv) { unchangedSinceTests(); closeRaceTests(device, recorder); keyProofTests(device, recorder); + targetKeyProofTests(device, recorder); + refreshProofTests(device, recorder); resourceReadTests(device, recorder); + readWrittenStagingTests(device, recorder); misalignedSnapshotTests(device, recorder); + misalignedRegionTests(device, recorder); drawSnapshotReuseTests(device, recorder); drawSnapshotEvictionTests(device); drawInputReuseTests(device, recorder); + drawInputInPlaceTests(device, recorder); RunResidentPresentTests(device.GetContext()); storeRunTests(device, recorder); + queuedWriteGroupTests(); remappedImportTests(device); movedMetadataTests(device, recorder); viewPastLastMipTests(device, recorder); keysFillTests(device, recorder); + denormalClearTests(device, recorder); unitShadowTests(device, recorder); storageRefreshTests(device, recorder, false); storageRefreshTests(device, recorder, true); writeBackPaddingTests(device, recorder, false); writeBackPaddingTests(device, recorder, true); + singlePassTests(device, recorder, false); + singlePassTests(device, recorder, true); + unchangedCpuStampTests(device, recorder); importWatchTests(device); staleGenerationTests(device, recorder); importWindowTests(device, recorder); dataWordPositionsTests(); dataRefreshTests(device, recorder); + imageMemoTests(device, recorder); minLodTests(device, recorder); firstLayerViewTests(device, recorder); singleCubeTests(device, recorder); @@ -2925,6 +4623,11 @@ int main(int argc, char** argv) { metadataPassTests(device, recorder); pendingKeyStoreTests(device, recorder); sampleDumpTests(device, recorder); + gpu.unlock(); + hostImportUnmapTests(device, recorder); + drawSnapshotPatchTests(device); + gpu.lock(); + recorder.Activate(); std::cout << "Recorder read tracking and label tests passed\n"; return 0; } catch (const std::exception& error) { diff --git a/core/libs/prx/libSceAgcDriver/tests/ScalarSelectMask.cpp b/core/libs/prx/libSceAgcDriver/tests/ScalarSelectMask.cpp new file mode 100644 index 000000000..43258d0d9 --- /dev/null +++ b/core/libs/prx/libSceAgcDriver/tests/ScalarSelectMask.cpp @@ -0,0 +1,352 @@ +#include "ControlFlow/GraphBuilder.hpp" +#include "ControlFlow/Structurizer.hpp" +#include "IntermediateRepresentation/IrProgram.hpp" +#include "RdnaDecoder/RdnaInstructionDecoder.hpp" +#include "Translation/InstructionTranslator.hpp" +#include "Translation/ShaderInputInfoBuilder.hpp" +#include +#include +#include +#include +#include +#include +#include +#include +#include + +using namespace ShaderRecompiler; + +namespace { + +constexpr std::uint32_t Lanes = 64; + +using LaneWords = std::array; + +struct Value { + std::array component{}; +}; + +class Evaluator { +public: + Evaluator(const IrProgram& program, std::uint32_t waveSize) : lanes(waveSize), exec(waveSize >= 64u ? ~0ull : (1ull << waveSize) - 1ull) { + for (std::uint32_t lane = 0; lane < Lanes; ++lane) vector[0][lane] = lane; + for (const auto& block : program.Blocks()) { + for (const auto* inst : block->Instructions()) run(*inst); + } + } + + LaneWords output{}; + bool exported = false; + std::map maskTag; + std::map threadBitValue; + +private: + std::unordered_map values; + std::array vector{}; + std::array scalar{}; + std::map threadBit; + std::uint32_t lanes; + std::uint64_t exec; + std::uint64_t vcc = 0; + bool scc = false; + + static std::uint32_t registerIndex(const IrValue& inst) { + return inst.Argument(0)->Register().index; + } + + Value operand(const IrValue* argument) { + argument = argument->Resolve(); + const auto found = values.find(argument); + if (found != values.end()) return found->second; + if (!argument->HasImmediate()) throw std::runtime_error("the evaluator met a value that has not been computed"); + Value constant{}; + const std::uint32_t word = argument->Type() == IrType::Bool ? (argument->ImmediateBool() ? 1u : 0u) : argument->ImmediateU32(); + constant.component[0].fill(word); + return constant; + } + + Value fromBits(std::uint64_t bits) const { + Value result{}; + for (std::uint32_t lane = 0; lane < lanes; ++lane) result.component[0][lane] = static_cast((bits >> lane) & 1u); + return result; + } + + std::uint64_t toBits(const Value& value) const { + std::uint64_t bits = 0; + for (std::uint32_t lane = 0; lane < lanes; ++lane) bits |= static_cast(value.component[0][lane] & 1u) << lane; + return bits; + } + + static Value uniform(std::uint32_t word) { + Value result{}; + result.component[0].fill(word); + return result; + } + + std::uint32_t uniformWord(const Value& value) const { + for (std::uint32_t lane = 1; lane < lanes; ++lane) { + if (value.component[0][lane] != value.component[0][0]) throw std::runtime_error("a scalar register was written with a value that differs between lanes"); + } + return value.component[0][0]; + } + + template + Value lanewise(const IrValue& inst, TOperation operation) { + const Value left = operand(inst.Argument(0)); + const Value right = operand(inst.Argument(1)); + Value result{}; + for (std::uint32_t lane = 0; lane < Lanes; ++lane) result.component[0][lane] = operation(left.component[0][lane], right.component[0][lane]); + return result; + } + + Value select(const IrValue& inst) { + const Value condition = operand(inst.Argument(0)); + const Value active = operand(inst.Argument(1)); + const Value inactive = operand(inst.Argument(2)); + Value result{}; + for (std::uint32_t lane = 0; lane < Lanes; ++lane) { + result.component[0][lane] = (condition.component[0][lane] & 1u) != 0u ? active.component[0][lane] : inactive.component[0][lane]; + } + return result; + } + + void run(const IrValue& inst) { + switch (inst.Opcode()) { + case IrOpcode::SetExec: exec = toBits(operand(inst.Argument(0))); return; + case IrOpcode::SetExecLo: exec = (exec & ~0xffffffffull) | uniformWord(operand(inst.Argument(0))); return; + case IrOpcode::SetExecHi: exec = (exec & 0xffffffffull) | (static_cast(uniformWord(operand(inst.Argument(0)))) << 32); return; + case IrOpcode::GetExec: values[&inst] = fromBits(exec); return; + case IrOpcode::GetExecLo: values[&inst] = uniform(static_cast(exec)); return; + case IrOpcode::GetExecHi: values[&inst] = uniform(static_cast(exec >> 32)); return; + case IrOpcode::SetVcc: vcc = toBits(operand(inst.Argument(0))); return; + case IrOpcode::SetVccLo: vcc = (vcc & ~0xffffffffull) | uniformWord(operand(inst.Argument(0))); return; + case IrOpcode::SetVccHi: vcc = (vcc & 0xffffffffull) | (static_cast(uniformWord(operand(inst.Argument(0)))) << 32); return; + case IrOpcode::GetVcc: values[&inst] = fromBits(vcc); return; + case IrOpcode::GetVccLo: values[&inst] = uniform(static_cast(vcc)); return; + case IrOpcode::GetVccHi: values[&inst] = uniform(static_cast(vcc >> 32)); return; + case IrOpcode::SetScc: scc = (uniformWord(operand(inst.Argument(0))) & 1u) != 0u; return; + case IrOpcode::GetScc: values[&inst] = uniform(scc ? 1u : 0u); return; + case IrOpcode::SetScalarRegister: scalar.at(registerIndex(inst)) = uniformWord(operand(inst.Argument(1))); return; + case IrOpcode::GetScalarRegister: values[&inst] = uniform(scalar.at(registerIndex(inst))); return; + case IrOpcode::SetScalarMaskTag: maskTag[registerIndex(inst)] = (uniformWord(operand(inst.Argument(1))) & 1u) != 0u; return; + case IrOpcode::GetScalarMaskTag: { + const auto found = maskTag.find(registerIndex(inst)); + values[&inst] = uniform(found != maskTag.end() && found->second ? 1u : 0u); + return; + } + case IrOpcode::SetThreadBitScalarRegister: + threadBit[registerIndex(inst)] = operand(inst.Argument(1)).component[0]; + threadBitValue[registerIndex(inst)] = inst.Argument(1)->Resolve(); + return; + case IrOpcode::GetThreadBitScalarRegister: { + Value result{}; + const auto found = threadBit.find(registerIndex(inst)); + if (found != threadBit.end()) result.component[0] = found->second; + values[&inst] = result; + return; + } + case IrOpcode::SetVectorRegister: vector.at(registerIndex(inst)) = operand(inst.Argument(1)).component[0]; return; + case IrOpcode::GetVectorRegister: { + Value result{}; + result.component[0] = vector.at(registerIndex(inst)); + values[&inst] = result; + return; + } + case IrOpcode::GetBuiltin: values[&inst] = uniform(0u); return; + case IrOpcode::LaneId: { + Value result{}; + for (std::uint32_t lane = 0; lane < Lanes; ++lane) result.component[0][lane] = lane; + values[&inst] = result; + return; + } + case IrOpcode::Ballot: { + const std::uint64_t bits = toBits(operand(inst.Argument(0))); + Value result{}; + result.component[0].fill(static_cast(bits)); + result.component[1].fill(static_cast(bits >> 32)); + values[&inst] = result; + return; + } + case IrOpcode::CompositeExtractU32x4: { + const Value source = operand(inst.Argument(0)); + Value result{}; + result.component[0] = source.component.at(operand(inst.Argument(1)).component[0][0]); + values[&inst] = result; + return; + } + case IrOpcode::CompositeConstructU32x4: { + Value result{}; + for (std::uint32_t index = 0; index < 4; ++index) result.component[index] = operand(inst.Argument(index)).component[0]; + values[&inst] = result; + return; + } + case IrOpcode::UGreaterThan32: values[&inst] = lanewise(inst, [](std::uint32_t a, std::uint32_t b) { return a > b ? 1u : 0u; }); return; + case IrOpcode::ULessThan32: values[&inst] = lanewise(inst, [](std::uint32_t a, std::uint32_t b) { return a < b ? 1u : 0u; }); return; + case IrOpcode::IEqual32: values[&inst] = lanewise(inst, [](std::uint32_t a, std::uint32_t b) { return a == b ? 1u : 0u; }); return; + case IrOpcode::INotEqual32: values[&inst] = lanewise(inst, [](std::uint32_t a, std::uint32_t b) { return a != b ? 1u : 0u; }); return; + case IrOpcode::LogicalAnd: values[&inst] = lanewise(inst, [](std::uint32_t a, std::uint32_t b) { return (a & b) & 1u; }); return; + case IrOpcode::BitwiseAnd32: values[&inst] = lanewise(inst, [](std::uint32_t a, std::uint32_t b) { return a & b; }); return; + case IrOpcode::ShiftRightLogical32: values[&inst] = lanewise(inst, [](std::uint32_t a, std::uint32_t b) { return a >> (b & 31u); }); return; + case IrOpcode::SelectU1: + case IrOpcode::SelectU32: values[&inst] = select(inst); return; + case IrOpcode::SetAttribute: + output = operand(inst.Argument(0)).component[0]; + exported = true; + return; + default: throw std::runtime_error("the evaluator does not support IR opcode " + std::to_string(static_cast(inst.Opcode()))); + } + } +}; + +constexpr std::uint32_t Vcc = 0x6au; +constexpr std::uint32_t Exec = 0x7eu; +constexpr std::uint32_t Zero = 0x80u; +constexpr std::uint32_t MinusOne = 0xc1u; +constexpr std::uint32_t S4 = 0x04u; +constexpr std::uint32_t Literal = 0xffu; + +std::uint32_t vcmpLanesBelow(std::uint32_t count) { + return 0x7d880080u + count; +} + +std::uint32_t sCmpScc(bool value) { + return value ? 0xbf068080u : 0xbf068180u; +} + +std::uint32_t sCselectB64(std::uint32_t destination, std::uint32_t trueSource, std::uint32_t falseSource) { + return 0x85800000u | (destination << 16) | (falseSource << 8) | trueSource; +} + +std::uint32_t sMovB64(std::uint32_t destination, std::uint32_t source) { + return 0xbe800400u | (destination << 16) | source; +} + +std::array vCndmaskV1(std::uint32_t mask) { + return {0xd5010001u, 0x80u | (129u << 9) | (mask << 18)}; +} + +struct Outcome { + LaneWords lanes{}; + std::uint32_t waveSize = 64; + bool maskTagS4 = false; + bool constantSources = false; +}; + +Outcome run(const std::vector& body, std::uint32_t mask, std::uint32_t waveSize = 64) { + std::vector code = body; + const auto select = vCndmaskV1(mask); + code.insert(code.end(), {select[0], select[1], 0xf80008cfu, 0x01010101u, 0xbf810000u}); + const auto decoded = RdnaInstructionDecoder{}.Decode(code); + auto cfg = GraphBuilder{}.Build(decoded); + Structurizer{}.Structurize(cfg); + GuestContext context{}; + context.waveSize = waveSize; + context.vertex = ShaderVertexStageInfo{}; + TranslateOptions options{}; + options.stage = ShaderStageKind::Vertex; + options.waveSize = waveSize; + options.userDataCount = 0; + options.inputInfo = BuildShaderStageInputInfo(ShaderStageKind::Vertex, context, waveSize); + const auto program = InstructionTranslator{}.Translate(decoded, cfg, options); + const Evaluator evaluator(program, waveSize); + if (!evaluator.exported) throw std::runtime_error("the program exported nothing"); + Outcome outcome; + outcome.lanes = evaluator.output; + outcome.waveSize = waveSize; + const auto tag = evaluator.maskTag.find(S4); + outcome.maskTagS4 = tag != evaluator.maskTag.end() && tag->second; + const auto bit = evaluator.threadBitValue.find(S4); + if (bit != evaluator.threadBitValue.end() && bit->second->Opcode() == IrOpcode::SelectU1) { + outcome.constantSources = bit->second->Argument(1)->Resolve()->HasImmediate() && bit->second->Argument(2)->Resolve()->HasImmediate(); + } + return outcome; +} + +int failures = 0; + +void expectLanes(const char* name, const Outcome& outcome, std::uint64_t expectedBits) { + for (std::uint32_t lane = 0; lane < outcome.waveSize; ++lane) { + const std::uint32_t expected = static_cast((expectedBits >> lane) & 1u); + if (outcome.lanes[lane] != expected) { + std::fprintf(stderr, "%s: lane %u holds %u, expected %u (expected mask 0x%016llx)\n", name, lane, outcome.lanes[lane], expected, static_cast(expectedBits)); + ++failures; + return; + } + } +} + +void expectMaskPath(const char* name, const Outcome& outcome) { + if (!outcome.maskTagS4) { + std::fprintf(stderr, "%s: the destination pair was not marked as a mask\n", name); + ++failures; + } +} + +void expectConstantSources(const char* name, const Outcome& outcome, bool expected) { + if (outcome.constantSources != expected) { + std::fprintf(stderr, "%s: the sources of the select are %sconstants\n", name, expected ? "not " : ""); + ++failures; + } +} + +constexpr std::uint64_t AllLanes = ~0ull; + +std::uint64_t below(std::uint32_t count) { + return count >= 64u ? AllLanes : (1ull << count) - 1ull; +} + +void selectCase(const char* name, std::vector setup, std::uint32_t trueSource, std::uint32_t falseSource, std::uint64_t whenSccSet, std::uint64_t whenSccClear, bool maskPath, int constantSources = -1) { + for (const bool sccValue : {true, false}) { + auto body = setup; + body.push_back(sCmpScc(sccValue)); + body.push_back(sCselectB64(4u, trueSource, falseSource)); + const auto outcome = run(body, S4); + const std::string label = std::string(name) + (sccValue ? ", scc = 1" : ", scc = 0"); + expectLanes(label.c_str(), outcome, sccValue ? whenSccSet : whenSccClear); + if (maskPath) expectMaskPath(label.c_str(), outcome); + if (constantSources >= 0) expectConstantSources(label.c_str(), outcome, constantSources != 0); + } +} + +} + +int main() { + try { + selectCase("vcc, exec", {vcmpLanesBelow(4)}, Vcc, Exec, below(4), AllLanes, true); + selectCase("exec, vcc", {vcmpLanesBelow(4)}, Exec, Vcc, AllLanes, below(4), true); + selectCase("wave64 halves, vcc, exec", {vcmpLanesBelow(40)}, Vcc, Exec, below(40), AllLanes, true); + selectCase("wave64 halves, exec, vcc", {vcmpLanesBelow(40)}, Exec, Vcc, AllLanes, below(40), true); + selectCase("exec, zero", {}, Exec, Zero, AllLanes, 0ull, true, 0); + selectCase("zero, exec", {}, Zero, Exec, 0ull, AllLanes, true, 0); + selectCase("zero, zero", {}, Zero, Zero, 0ull, 0ull, true, 1); + selectCase("minus one, zero", {}, MinusOne, Zero, AllLanes, 0ull, true, 1); + selectCase("minus one, exec", {}, MinusOne, Exec, AllLanes, AllLanes, true, 0); + selectCase("destination is the true source", {vcmpLanesBelow(40), sMovB64(4u, Vcc)}, S4, Exec, below(40), AllLanes, false); + selectCase("destination is the false source", {vcmpLanesBelow(40), sMovB64(4u, Vcc)}, Exec, S4, AllLanes, below(40), false); + for (const bool sccValue : {true, false}) { + const auto outcome = run({sCmpScc(sccValue), sCselectB64(4u, Literal, Exec), 0x0000ffffu}, S4); + const std::string label = std::string("literal 0xffff, exec") + (sccValue ? ", scc = 1" : ", scc = 0"); + expectLanes(label.c_str(), outcome, sccValue ? below(16) : AllLanes); + } + for (const bool sccValue : {true, false}) { + const auto outcome = run({sCmpScc(sccValue), sCselectB64(4u, Literal, Zero), 0xffffffffu}, S4); + const std::string label = std::string("wave64, literal all ones, zero") + (sccValue ? ", scc = 1" : ", scc = 0"); + expectLanes(label.c_str(), outcome, sccValue ? below(32) : 0ull); + expectConstantSources(label.c_str(), outcome, false); + } + for (const bool sccValue : {true, false}) { + const auto outcome = run({sCmpScc(sccValue), sCselectB64(4u, Literal, Zero), 0xffffffffu}, S4, 32); + const std::string label = std::string("wave32, literal all ones, zero") + (sccValue ? ", scc = 1" : ", scc = 0"); + expectLanes(label.c_str(), outcome, sccValue ? below(32) : 0ull); + expectConstantSources(label.c_str(), outcome, true); + } + for (const bool sccValue : {true, false}) { + const auto outcome = run({vcmpLanesBelow(40), sCmpScc(sccValue), sCselectB64(Vcc, Vcc, Exec)}, Vcc); + expectLanes(sccValue ? "destination vcc, scc = 1" : "destination vcc, scc = 0", outcome, sccValue ? below(40) : AllLanes); + } + } catch (const std::exception& error) { + std::fprintf(stderr, "%s\n", error.what()); + return 1; + } + return failures == 0 ? 0 : 1; +} diff --git a/core/libs/prx/libSceAgcDriver/tests/ShaderDeviceProfile.cpp b/core/libs/prx/libSceAgcDriver/tests/ShaderDeviceProfile.cpp index f8466db76..11e9c1dd1 100644 --- a/core/libs/prx/libSceAgcDriver/tests/ShaderDeviceProfile.cpp +++ b/core/libs/prx/libSceAgcDriver/tests/ShaderDeviceProfile.cpp @@ -1,6 +1,7 @@ #include "prx/libSceAgcDriver/Execution/include/ShaderDeviceProfile.hpp" #include "BdaAbi.hpp" #include "Optimization/BindingAllocator.hpp" +#include "Optimization/ResourceMaterializer.hpp" #include #include #include @@ -135,17 +136,27 @@ void CheckHeaps() { image.numericClass = IrTextureNumericClass::Float; image.dimension = RdnaImageDimension::Dim2D; const auto single = allocate(image, 1u); + const auto sixteenSamplers = allocate(image, 1u, 16u); + Require(BindingAllocator{}.FindBinding(sixteenSamplers.layout, DescriptorBindingKind::Samplers).resources.size() == 32u, "sixteen logical samplers must fit with both descriptor variants"); const auto full = allocate(image, RuntimeAbi::SampledHeapCapacity, RuntimeAbi::SamplerHeapCapacity / 2u); Require(single.layout.ShaderDataDwords() == full.layout.ShaderDataDwords() && single.layout.memoryOffsetDword == full.layout.memoryOffsetDword && !full.layout.UsesPushData(), "runtime layout depends on resource count"); Require(full.layout.memoryOffsetDword == 0u && full.layout.DispatchThreadLimitDword() == 0u && full.layout.ShaderDataDwords() == 0u, "direct image resources allocated runtime metadata"); Reject([&] { allocate(image, RuntimeAbi::SampledHeapCapacity + 1u); }, "heap capacity exceeded"); + Require(BindingAllocator{}.FindBinding(allocate(image, 40u).layout, DescriptorBindingForImage(image)).resources.size() == 40u, "a sampled heap does not hold 40 images of one class"); + Require(BindingAllocator{}.FindBinding(allocate(image, RuntimeAbi::SampledHeapCapacity).layout, DescriptorBindingForImage(image)).resources.size() == RuntimeAbi::SampledHeapCapacity, "a sampled heap does not hold a full class of images"); + Require(ResourceMaterializer::BindlessSlots() == 16u, "bindless image tables changed size with the sampled heap"); Reject([&] { allocate(image, 1u, RuntimeAbi::SamplerHeapCapacity + 1u); }, "metadata capacity"); image.resourceClass = ImageResourceClass::Storage; image.mipMode = ImageMipMode::DynamicStorage; - image.mipCount = RuntimeAbi::StorageHeapCapacity; + image.mipCount = RuntimeAbi::StorageMipSlots; const auto storage = allocate(image, 1u); Require(BindingAllocator{}.FindBinding(storage.layout, DescriptorBindingForImage(image)).resources.size() == image.mipCount, "storage heap did not reserve each mip"); - Reject([&] { allocate(image, 2u); }, "heap capacity exceeded"); + const auto shared = allocate(image, 2u); + Require(BindingAllocator{}.FindBinding(shared.layout, DescriptorBindingForImage(image)).resources.size() == 2u * image.mipCount, "a storage heap did not hold two dynamic-mip images"); + const auto capacity = RuntimeAbi::StorageHeapCapacity / image.mipCount; + const auto filled = allocate(image, capacity); + Require(BindingAllocator{}.FindBinding(filled.layout, DescriptorBindingForImage(image)).resources.size() == RuntimeAbi::StorageHeapCapacity, "a full storage heap was not allocated"); + Reject([&] { allocate(image, capacity + 1u); }, "heap capacity exceeded"); Reject([&] { allocate(image, 0u, RuntimeAbi::SamplerHeapCapacity / 2u + 1u); }, "sampler pairs"); const std::array dimensions{RdnaImageDimension::Dim1D, RdnaImageDimension::Dim1DArray, RdnaImageDimension::Dim2D, RdnaImageDimension::Dim2DArray, RdnaImageDimension::Dim3D, RdnaImageDimension::Dim2DMsaa, RdnaImageDimension::Dim2DMsaaArray}; std::set classes; diff --git a/core/libs/prx/libSceAgcDriver/tests/ShaderDiskCache.cpp b/core/libs/prx/libSceAgcDriver/tests/ShaderDiskCache.cpp index 3facf9ad5..0aeb9425e 100644 --- a/core/libs/prx/libSceAgcDriver/tests/ShaderDiskCache.cpp +++ b/core/libs/prx/libSceAgcDriver/tests/ShaderDiskCache.cpp @@ -41,7 +41,7 @@ void setEnvironment(const char* name, const std::string& value) { } bool sameBinding(const DescriptorBinding& left, const DescriptorBinding& right) { - return left.kind == right.kind && left.role == right.role && left.descriptorSet == right.descriptorSet && left.binding == right.binding && left.count == right.count && left.guestDescriptor == right.guestDescriptor && left.readOnly == right.readOnly && left.imageShape == right.imageShape && left.samplerDepthCompare == right.samplerDepthCompare && left.imageWritten == right.imageWritten && left.imageDepthCompare == right.imageDepthCompare && left.imageAtomic == right.imageAtomic && left.bufferAtomic == right.bufferAtomic && left.bufferWritten == right.bufferWritten && left.samplerUnnormalized == right.samplerUnnormalized && left.imageUnnormalized == right.imageUnnormalized && left.imageSamplers == right.imageSamplers; + return left.kind == right.kind && left.role == right.role && left.descriptorSet == right.descriptorSet && left.binding == right.binding && left.count == right.count && left.guestDescriptor == right.guestDescriptor && left.readOnly == right.readOnly && left.imageShape == right.imageShape && left.samplerDepthCompare == right.samplerDepthCompare && left.imageWritten == right.imageWritten && left.imageDepthCompare == right.imageDepthCompare && left.imageAtomic == right.imageAtomic && left.bufferAtomic == right.bufferAtomic && left.bufferWritten == right.bufferWritten && left.bufferRead == right.bufferRead && left.samplerUnnormalized == right.samplerUnnormalized && left.imageUnnormalized == right.imageUnnormalized && left.imageSamplers == right.imageSamplers; } bool sameBindings(const std::vector& left, const std::vector& right) { @@ -79,12 +79,15 @@ void requireSameResult(const RecompileResult& left, const RecompileResult& right require(sameBindings(left.bindings, right.bindings), prefix + "bindings differ"); require(left.pushConstants == right.pushConstants, prefix + "push constants differ"); require(left.specialization == right.specialization, prefix + "specialization constants differ"); + require(left.workgroupMemoryDwords == right.workgroupMemoryDwords, prefix + "workgroup memory stride differs"); + require(left.poisonedSrtReads == right.poisonedSrtReads, prefix + "poisoned SRT read counts differ"); require(left.vertexAttributes.size() == right.vertexAttributes.size(), prefix + "vertex attribute count differs"); for (std::size_t i = 0; i < left.vertexAttributes.size(); ++i) { const auto& a = left.vertexAttributes[i]; const auto& b = right.vertexAttributes[i]; require(a.location == b.location && a.components == b.components && a.resource.fields == b.resource.fields && a.fetchIndex == b.fetchIndex && a.formatComponents == b.formatComponents, prefix + "vertex attribute differs"); } + require(left.barycentricEmulation.active == right.barycentricEmulation.active && left.barycentricEmulation.smooth == right.barycentricEmulation.smooth && left.barycentricEmulation.linear == right.barycentricEmulation.linear, prefix + "barycentric emulation differs"); } void requireSameVariant(const CompiledVariant& left, const CompiledVariant& right, const char* what) { @@ -118,6 +121,7 @@ DescriptorBinding sampleBinding(std::uint32_t seed) { binding.samplerUnnormalized = {false, true, true}; binding.imageUnnormalized = {true, false, seed % 2 == 0}; binding.imageSamplers = {0x5u, 0u, 0x80000000u}; + binding.bufferRead = {true, false, false, true, true, false}; return binding; } @@ -135,6 +139,7 @@ RecompileResult sampleResult() { result.bdaAbiVersion = 3; result.memoryOffsetDword = 7; result.hostSubgroupSize = 32; + result.workgroupMemoryDwords = 32769u; result.vertexInputs = {{1, 4, 2}, {5, 2, 0}}; result.vertexInputPatches = {{1, 20, {7, 8, 9}}, {5, 40, {10, 11, 12}}}; result.specialization = {{512, 4}, {516, 0x3f800000u}, {517, 0}}; @@ -149,6 +154,8 @@ RecompileResult sampleResult() { result.instanceOffsetConflict = true; result.parameterExports = {0, 3, 7}; result.fragmentParameters = {{0, 1, true, false, true}, {2, 3, false, true}}; + result.poisonedSrtReads = 3; + result.barycentricEmulation = {true, false, true}; result.variantId = 99; return result; } @@ -193,6 +200,8 @@ CompiledVariant sampleVariant() { image.r128 = true; image.fmaskCompatible = false; image.depthBitsCompatible = false; + image.flatVolumeCompatible = false; + image.flatLineCompatible = false; image.byElements = 4; image.byComponents = 1; image.indirectRoot = 0; @@ -220,6 +229,7 @@ CompiledVariant sampleVariant() { info.info.vertexOffsetSgpr = 6; info.info.hasBitwiseXor = true; info.info.usesDma = true; + info.info.usesFaultBuffer = true; info.bindings.pushDataStartDword = 2; info.bindings.memoryOffsetDword = 1; info.bindings.memoryOffsetCount = 5; @@ -746,7 +756,7 @@ void verifyBindingPlanSelection() { BindingAllocationResult full; builder.Populate(full, compiled, plan, 0u, snapshot, {}); require(full.bindings.size() == 4u, "binding plan lost a descriptor before module selection"); - require(full.bindings[0].bufferAtomic == std::vector{true, false} && full.bindings[0].bufferWritten == std::vector{true, false}, "binding plan lost buffer access metadata"); + require(full.bindings[0].bufferAtomic == std::vector{true, false} && full.bindings[0].bufferWritten == std::vector{true, false} && full.bindings[0].bufferRead == std::vector{false, true}, "binding plan lost buffer access metadata"); const std::array liveBindings{full.bindings[0].binding, full.bindings[0].binding, full.bindings[3].binding}; const auto selected = builder.Select(plan, liveBindings); snapshot.userData.clear(); @@ -1033,8 +1043,10 @@ int main(int argc, char** argv) { verifyVertexTypeSpecialization(); verifyBuiltinSpecialization(); verifySpecializationLiveness(); + ShaderRecompiler::ShaderDiskCache::Flush(); std::error_code error; std::filesystem::remove_all(directory, error); + require(!error, "cannot remove the shader cache test directory: " + error.message()); std::cout << "shader disk cache tests passed\n"; return 0; } catch (const std::exception& error) { diff --git a/core/libs/prx/libSceAgcDriver/tests/ShaderMemory.cpp b/core/libs/prx/libSceAgcDriver/tests/ShaderMemory.cpp index 0d8e318a3..bc78344e5 100644 --- a/core/libs/prx/libSceAgcDriver/tests/ShaderMemory.cpp +++ b/core/libs/prx/libSceAgcDriver/tests/ShaderMemory.cpp @@ -5,6 +5,7 @@ #include "Optimization/ResourceMaterializer.hpp" #include "Optimization/ResourceProgram.hpp" #include "Optimization/ShaderStageInputInfo.hpp" +#include "Optimization/SrtWalker/SrtDescriptorEvaluation.hpp" #include "Optimization/SrtWalker/SrtEvaluator.hpp" #include "Optimization/SrtWalker/SrtFlatSlotClasses.hpp" #include "SpirvBackend/SpirvAnalysis.hpp" @@ -16,6 +17,7 @@ #include #include #include +#include #include #include #include @@ -29,6 +31,17 @@ #include #include #include +#ifdef _WIN32 +#ifndef NOMINMAX +#define NOMINMAX +#endif +#ifndef WIN32_LEAN_AND_MEAN +#define WIN32_LEAN_AND_MEAN +#endif +#include +#else +#include +#endif namespace { @@ -68,6 +81,258 @@ void verifyResult(const ShaderRecompiler::RecompileResult& first, const ShaderRe } } +ShaderRecompiler::RecompileRequest FrontendRequest(std::span code, std::span userData) { + using namespace ShaderRecompiler; + RecompileRequest request{}; + request.shader = {ShaderStage::Compute, 0x100000u, code, 0, {}}; + request.context.waveSize = 64; + request.context.userDataBaseRegister = 0; + request.context.userData = userData; + request.context.compute = ShaderComputeStageInfo{{64u, 1u, 1u}, 0u, {false, false, false}, false, 1u}; + request.target = BufferTarget(); + request.target.vulkanVersion = 0x00401000u; + request.target.spirvVersion = 0x00010300u; + request.target.subgroupSize = 32u; + request.target.fragmentShaderBarycentricEnabled = false; + request.layout.pushConstantSizeBytes = 128; + request.useCache = false; + return request; +} + +void verifyFrontendIdentity() { + using namespace ShaderRecompiler; + const bool reuse = std::getenv("APS5_NO_PERF_FRONTEND_PAIR") == nullptr; + const std::vector code{0xbf800000u, 0xbf810000u}; + const std::array userData{}; + const auto request = FrontendRequest(code, userData); + ShaderPreparationContext preparation; + const auto first = preparation.AcquireFrontend(request); + require(first != nullptr, "frontend pair: no frontend was prepared"); + require((preparation.AcquireFrontend(request) == first) == reuse, "frontend pair: repeated acquisition ignored the reuse switch"); + + const auto relocatedCode = code; + auto relocated = request; + relocated.shader.code = relocatedCode; + relocated.shader.codeAddress += 0x1000u; + require((preparation.AcquireFrontend(relocated) == first) == reuse, "frontend pair: identical words at another allocation or address changed reuse"); + auto partial = request; + partial.context.compute->partialThreads = {33u, 1u, 1u}; + require((preparation.AcquireFrontend(partial) == first) == reuse, "frontend pair: partial-thread state changed the frontend identity"); + auto target = request; + target.target.subgroupSize = 64u; + target.context.waveSize = 32u; + target.context.compute->numThreads = {32u, 2u, 1u}; + require((preparation.AcquireFrontend(target) == first) == reuse, "frontend pair: translation-only state changed the frontend identity"); + const std::array changedUserData{1u, 2u}; + auto userValues = request; + userValues.context.userData = changedUserData; + require((preparation.AcquireFrontend(userValues) == first) == reuse, "frontend pair: user-data values changed the frontend identity"); + + const auto verifyMiss = [&](const RecompileRequest& changed, const char* message) { + ShaderPreparationContext isolated; + const auto original = isolated.AcquireFrontend(request); + const auto replacement = isolated.AcquireFrontend(changed); + require(replacement != original, message); + require((isolated.AcquireFrontend(changed) == replacement) == reuse, "frontend pair: a successful miss was not retained"); + require(isolated.AcquireFrontend(request) != original, "frontend pair: replacement retained more than the last frontend"); + }; + auto changed = request; + auto changedCode = code; + changedCode[0] = 0xbf800001u; + changed.shader.code = changedCode; + verifyMiss(changed, "frontend pair: changed code words reused the frontend"); + changedCode = code; + changedCode.push_back(0xbf800000u); + changed.shader.code = changedCode; + verifyMiss(changed, "frontend pair: changed code length reused the frontend"); + changed = request; + changed.context.userDataBaseRegister = 4u; + verifyMiss(changed, "frontend pair: changed user-data base reused the frontend"); + changed = request; + changed.context.userData = std::span(userData).first(1u); + verifyMiss(changed, "frontend pair: changed user-data count reused the frontend"); + changed = request; + changed.shader.stage = ShaderStage::Fragment; + changed.context.compute.reset(); + changed.context.pixel = ShaderPixelStageInfo{}; + verifyMiss(changed, "frontend pair: a different stage reused the frontend"); + changed = request; + changed.shader.stage = ShaderStage::Vertex; + changed.context.compute.reset(); + changed.context.vertex = ShaderVertexStageInfo{}; + verifyMiss(changed, "frontend pair: a fetch-capable stage reused the compute frontend"); + + auto invalid = request; + invalid.shader.code = {}; + bool failed = false; + try { + static_cast(preparation.AcquireFrontend(invalid)); + } catch (const std::out_of_range& error) { + failed = std::string_view(error.what()).find("before s_endpgm") != std::string_view::npos; + } + require(failed, "frontend pair: invalid code did not fail during frontend construction"); + require((preparation.AcquireFrontend(request) == first) == reuse, "frontend pair: a failed miss replaced the prior frontend"); +} + +void verifyFrontendProgram(const ShaderRecompiler::IrProgram& reused, const ShaderRecompiler::IrProgram& fresh) { + using namespace ShaderRecompiler; + require(ProgramToString(reused) == ProgramToString(fresh), "frontend pair: reused frontend changed the IR"); + require(reused.WaveSize() == fresh.WaveSize() && reused.Info() == fresh.Info(), "frontend pair: reused frontend changed shader information"); + const auto& left = reused.Resources(); + const auto& right = fresh.Resources(); + require(left.stage == right.stage && left.shaderHash == right.shaderHash && left.userDataBase == right.userDataBase && left.userDataCount == right.userDataCount && left.srgbDecodeFormats == right.srgbDecodeFormats, "frontend pair: resource identity changed"); + require(left.memoryInfo == right.memoryInfo && left.materializationSources == right.materializationSources && left.guardedSrtSlots == right.guardedSrtSlots && left.srtGuardOffset == right.srtGuardOffset && left.cleanFlatSlots == right.cleanFlatSlots && left.pureFlatSlots == right.pureFlatSlots, "frontend pair: resource planning changed"); + require(left.descriptorSources.size() == right.descriptorSources.size() && left.srtReads.size() == right.srtReads.size(), "frontend pair: resource counts changed"); + for (std::size_t index = 0; index < left.descriptorSources.size(); ++index) { + const auto& a = left.descriptorSources[index]; + const auto& b = right.descriptorSources[index]; + require(a.dwordCount == b.dwordCount && a.indirectImage == b.indirectImage, "frontend pair: descriptor planning changed"); + for (std::size_t word = 0; word < a.dwords.size(); ++word) { + require((a.dwords[word] == nullptr) == (b.dwords[word] == nullptr), "frontend pair: descriptor word presence changed"); + if (a.dwords[word] != nullptr) require(a.dwords[word] != b.dwords[word] && a.dwords[word]->Id() == b.dwords[word]->Id(), "frontend pair: descriptor values were shared or changed"); + } + } + require(reused.BlockOrder().size() == fresh.BlockOrder().size(), "frontend pair: block count changed"); + for (std::size_t block = 0; block < reused.BlockOrder().size(); ++block) { + const auto* a = reused.BlockOrder()[block]; + const auto* b = fresh.BlockOrder()[block]; + require(a != b && a->Instructions().size() == b->Instructions().size(), "frontend pair: mutable IR blocks were shared or changed"); + auto other = b->Instructions().begin(); + for (const auto* x : a->Instructions()) { + const auto* y = *other++; + require(x != y && x->Flags() == y->Flags(), "frontend pair: mutable IR values were shared or their flags changed"); + } + } +} + +void verifyFrontendPair() { + using namespace ShaderRecompiler; + verifyFrontendIdentity(); + const bool reuse = std::getenv("APS5_NO_PERF_FRONTEND_PAIR") == nullptr; + const std::vector code{0xe0700000u, 0x80000000u, 0xbf810000u}; + const std::array buffer{0x10000000u, 0u, 0x40u, 0x00027facu}; + const auto request = FrontendRequest(code, buffer); + auto partial = request; + partial.context.compute->partialThreads = {33u, 1u, 1u}; + const auto freshFull = PrepareResourceProgram(request); + const auto freshPartial = PrepareResourceProgram(partial); + require(ProgramToString(freshFull) != ProgramToString(freshPartial), "frontend pair: fixture does not distinguish full and partial threads"); + const auto storeThreadLimits = [](const IrProgram& program) { + std::vector pending; + for (const auto* block : program.BlockOrder()) { + for (const auto* inst : block->Instructions()) { + if (inst->Opcode() == IrOpcode::StoreBufferU32) pending.push_back(inst->Argument(inst->ArgumentCount() - 1u)); + } + } + require(pending.size() == 1u, "frontend pair: fixture does not retain exactly one buffer store"); + std::vector visited; + std::uint32_t axes = 0u; + while (!pending.empty()) { + const auto* value = pending.back()->Resolve(); + pending.pop_back(); + if (std::find(visited.begin(), visited.end(), value) != visited.end()) continue; + visited.push_back(value); + if (value->Opcode() == IrOpcode::GetBuiltin && value->Argument(0)->Resolve()->ImmediateU32() == static_cast(StageInputKind::DispatchThreadLimit)) { + const auto axis = value->Argument(1)->Resolve()->ImmediateU32(); + require(axis < 3u, "frontend pair: invalid dispatch-limit axis"); + axes |= 1u << axis; + } + for (const auto* argument : value->Arguments()) pending.push_back(argument); + } + return axes; + }; + require(storeThreadLimits(freshFull) == 0u && storeThreadLimits(freshPartial) == 7u, "frontend pair: buffer-store predicate does not retain all partial-thread bounds"); + require(!freshFull.Resources().memoryInfo.empty() && !freshFull.Resources().descriptorSources.empty() && !freshFull.Info().buffers.empty(), "frontend pair: fixture has no buffer resources"); + + for (const bool partialFirst : {false, true}) { + ShaderPreparationContext preparation; + std::shared_ptr retained; + { + auto temporary = code; + auto borrowed = request; + borrowed.shader.code = temporary; + retained = preparation.AcquireFrontend(borrowed); + std::fill(temporary.begin(), temporary.end(), 0xffffffffu); + require((preparation.AcquireFrontend(request) == retained) == reuse, "frontend pair: mutating the caller's code changed the retained frontend"); + } + require((preparation.AcquireFrontend(request) == retained) == reuse, "frontend pair: destroying the caller's code changed the retained frontend"); + const auto first = PrepareResourceProgram(partialFirst ? partial : request, &preparation); + const auto second = PrepareResourceProgram(partialFirst ? request : partial, &preparation); + verifyFrontendProgram(first, partialFirst ? freshPartial : freshFull); + verifyFrontendProgram(second, partialFirst ? freshFull : freshPartial); + require(first.BlockOrder().front() != second.BlockOrder().front(), "frontend pair: full and partial variants share mutable IR"); + const auto repeated = PrepareResourceProgram(partialFirst ? partial : request, &preparation); + verifyFrontendProgram(repeated, first); + auto translated = request; + translated.context.waveSize = 32u; + translated.context.compute->numThreads = {32u, 2u, 1u}; + const auto changed = PrepareResourceProgram(translated, &preparation); + const auto freshChanged = PrepareResourceProgram(translated); + verifyFrontendProgram(changed, freshChanged); + require(changed.WaveSize() != freshFull.WaveSize(), "frontend pair: changed translation inputs were ignored"); + require((preparation.AcquireFrontend(request) == retained) == reuse, "frontend pair: translating a variant replaced the frontend"); + } + + for (const bool partialFirst : {false, true}) { + auto pairCode = code; + pairCode.insert(pairCode.begin(), 0xbf800000u | static_cast(partialFirst)); + auto full = request; + full.shader.code = pairCode; + auto limited = partial; + limited.shader.code = pairCode; + const auto fullReference = Recompile(full); + const auto partialReference = Recompile(limited); + ShaderPreparationContext preparation; + const auto first = PrepareShader(partialFirst ? limited : full, &preparation); + const auto second = PrepareShader(partialFirst ? full : limited, &preparation); + const auto& fullHandle = partialFirst ? second : first; + const auto& partialHandle = partialFirst ? first : second; + const auto& fullArtifact = GetPreparedArtifact(*fullHandle); + const auto& partialArtifact = GetPreparedArtifact(*partialHandle); + require(first->source != second->source, "frontend pair: full and partial fixtures did not exercise separate sources"); + require(fullArtifact.spirv != partialArtifact.spirv, "frontend pair: full and partial threads share the same artifact"); + SrtRuntime runtime{}; + runtime.userData = buffer; + runtime.shaderBase = full.shader.codeAddress; + const auto fullCapture = CaptureResources(full, runtime, *fullHandle); + const auto partialCapture = CaptureResources(limited, runtime, *partialHandle); + const auto fullResult = MaterializeShader(full, *fullCapture, *fullHandle); + const auto partialResult = MaterializeShader(limited, *partialCapture, *partialHandle); + verifyResult(fullReference, *fullResult); + verifyResult(partialReference, *partialResult); + } + + const std::array color{0x3f800000u}; + for (const bool zeroFirst : {false, true}) { + const std::vector pixelCode{0xbf800000u | static_cast(zeroFirst), 0x7e000200u, 0xf800180fu, 0u, 0xbf810000u}; + auto pixel = FrontendRequest(pixelCode, color); + pixel.shader.stage = ShaderStage::Fragment; + pixel.context.compute.reset(); + pixel.context.pixel = ShaderPixelStageInfo{}; + pixel.context.pixel->wave32 = true; + pixel.context.pixel->targetOutputMode[0] = 9u; + pixel.context.waveSize = 32u; + auto zero = pixel; + zero.layout.pushConstantSizeBytes = 0u; + const auto pixelReference = Recompile(pixel); + const auto zeroReference = Recompile(zero); + ShaderPreparationContext preparation; + const auto first = PrepareShader(zeroFirst ? zero : pixel, &preparation); + const auto second = PrepareShader(zeroFirst ? pixel : zero, &preparation); + const auto& pixelHandle = zeroFirst ? second : first; + const auto& zeroHandle = zeroFirst ? first : second; + require(first->source == second->source, "frontend pair: pixel capacities did not exercise the same source"); + require(GetPreparedArtifact(*pixelHandle).spirv != GetPreparedArtifact(*zeroHandle).spirv, "frontend pair: pixel fixture does not distinguish push-constant capacities"); + SrtRuntime runtime{}; + runtime.userData = color; + runtime.shaderBase = pixel.shader.codeAddress; + const auto pixelCapture = CaptureResources(pixel, runtime, *pixelHandle); + const auto zeroCapture = CaptureResources(zero, runtime, *zeroHandle); + verifyResult(pixelReference, *MaterializeShader(pixel, *pixelCapture, *pixelHandle)); + verifyResult(zeroReference, *MaterializeShader(zero, *zeroCapture, *zeroHandle)); + } +} + void verifyRegisterSources() { using namespace ShaderRecompiler; IrResourcePlan plan; @@ -116,6 +381,66 @@ void verifyEvaluatedValues() { require(!table.Find(&absent, found), "evaluated values: a value that was never inserted was found"); } +void verifySignedSrtComparison() { + using namespace ShaderRecompiler; + IrResourcePlan plan; + plan.srtPlanComplete = true; + plan.userDataBase = 8; + std::vector> values; + std::uint32_t id = 0; + const auto make = [&](IrOpcode opcode, IrType type, std::initializer_list arguments = {}) -> IrValue& { + auto value = std::make_unique(opcode, type, id++); + for (auto* argument : arguments) value->AddArgument(argument); + values.push_back(std::move(value)); + return *values.back(); + }; + const auto constant = [&](std::uint32_t bits) -> IrValue& { + auto& value = make(IrOpcode::Void, IrType::U32); + value.SetImmediateU32(bits); + return value; + }; + auto& registerValue = make(IrOpcode::Void, IrType::ScalarReg); + registerValue.SetRegister({RegisterBank::Scalar, 8u}); + auto& userData = make(IrOpcode::GetUserData, IrType::U32, {®isterValue}); + auto& zero = constant(0u); + auto& intMin = constant(0x80000000u); + auto& intMax = constant(0x7fffffffu); + auto& negative = constant(0xau); + auto& nonnegative = constant(0xbu); + auto& greaterThanMin = constant(0xcu); + auto& equalMin = constant(0xdu); + auto& belowMax = constant(0xeu); + auto& equalMax = constant(0xfu); + auto& isNegative = make(IrOpcode::SLessThan32, IrType::Bool, {&userData, &zero}); + auto& aboveMin = make(IrOpcode::SLessThan32, IrType::Bool, {&intMin, &userData}); + auto& belowMaxValue = make(IrOpcode::SLessThan32, IrType::Bool, {&userData, &intMax}); + DescriptorSource source; + source.dwordCount = 3u; + source.dwords[0] = &make(IrOpcode::SelectU32, IrType::U32, {&isNegative, &negative, &nonnegative}); + source.dwords[1] = &make(IrOpcode::SelectU32, IrType::U32, {&aboveMin, &greaterThanMin, &equalMin}); + source.dwords[2] = &make(IrOpcode::SelectU32, IrType::U32, {&belowMaxValue, &belowMax, &equalMax}); + plan.descriptorSources.push_back(source); + const std::array sourceIndex{0u}; + const std::array>, 4> cases{{ + {0x80000000u, {0xau, 0xdu, 0xeu}}, + {0xffffffffu, {0xau, 0xcu, 0xeu}}, + {0u, {0xbu, 0xcu, 0xeu}}, + {0x7fffffffu, {0xbu, 0xcu, 0xfu}}, + }}; + std::vector descriptors; + std::vector flat; + std::vector active; + for (const auto& [input, expected] : cases) { + const std::array userDataWords{input}; + const SrtRuntime runtime{userDataWords}; + require(Detail::EvaluateRuntimeSourcesImpl(plan, sourceIndex, runtime, descriptors, flat, false, {}, active), "signed SRT comparison: descriptor expression could not be evaluated"); + require(descriptors.size() == 1u && descriptors[0].dwordCount == expected.size(), "signed SRT comparison: descriptor shape changed"); + for (std::size_t index = 0; index < expected.size(); ++index) { + require(descriptors[0].dwords[index] == expected[index], "signed SRT comparison: signed boundary selected the wrong descriptor dword"); + } + } +} + // The pure flat slots of a hand-built plan (Detail::ComputePureFlatSlots): a slot is pure unless // a descriptor dword, a condition, a uniform value or another slot's address cone reaches it. void verifyPureFlatSlots() { @@ -419,6 +744,14 @@ void verifyBindlessTable() { require(wholeCapture->snapshot.images[wholeRoot].dwords == heap[0] && wholeCapture->snapshot.images[wholeDirect].dwords == heap[1] && wholeCapture->snapshot.images[wholeDirect + 1u].dwords == heap[2] && wholeCapture->snapshot.images[wholeDirect + 2u].dwords == heap[3], "bindless: mode T slots are wrong"); whole.context.memory = wholeMemory.Regions(); require(!Recompile(whole, *wholeCapture)->spirv.empty(), "bindless: mode T did not compile"); + std::vector latchCode(wholeCode.begin(), wholeCode.begin() + 12); + latchCode.insert(latchCode.end(), {0xbea80380u, 0xf09c0f08u, 0x00450000u, 0x80288128u, 0xbf0a8428u, 0xbf85fffbu, 0xe0700000u, 0x80070000u, 0xbf810000u}); + auto latch = makeRequest(latchCode); + AgcDriver::ShaderMemory latchMemory({}); + const auto latchCapture = latchMemory.Capture(latch); + const auto latchRegions = latchMemory.Regions(); + latch.context.memory = latchRegions; + require(!Recompile(latch, *latchCapture)->spirv.empty(), "bindless: a sample in a loop latch did not compile"); // A table wider than the slots without a material pattern is rejected. fillSrt(100u); @@ -548,6 +881,64 @@ void verifyDescriptorPhis() { auto dynamic = makeRequest(dynamicCode); expectFailure([&] { static_cast(GetResourcePlan(dynamic)); }, "GetSamplerResource dword 0 is not a valid runtime value", "descriptor Phi: an edge without an SRT slot was accepted"); + + const std::vector loopCode{0xf4080500u, 0xfa000020u, 0xf4080600u, 0xfa000040u, 0xf40c0200u, 0xfa000000u, 0xbe910380u, 0xbf8cc07fu, 0xf09c0f08u, 0x00a20000u, 0xbf8c3f70u, 0xe0700000u, 0x80060000u, 0xbf068011u, 0xbf850003u, 0xf40c0200u, 0xfa000000u, 0xbf820002u, 0xf40c0200u, 0xfa000060u, 0xbf8cc07fu, 0x80118111u, 0xbf0a8211u, 0xbf85fff0u, 0xbf810000u}; + const auto loopCapture = compile(loopCode, 2u, 1u); + const auto& loopImages = loopCapture->snapshot.images; + const auto holdsLoopImage = [&](const std::array& words) { + return std::ranges::any_of(loopImages, [&](const DescriptorValue& value) { + return std::equal(words.begin(), words.end(), value.dwords.begin()); + }); + }; + require(loopImages.size() == 2u && holdsLoopImage(first) && holdsLoopImage(second), "descriptor Phi: the loop's chained T# Phis were not split into the two SRT T#s"); + + const auto entryWrites = [&](std::initializer_list words) { + auto code = loopCode; + code.insert(code.begin() + 8, words); + return code; + }; + const std::array, const char*>, 3> invalidEntries{{ + {entryWrites({0xbe8e1f00u}), "descriptor Phi: a T# holding the program counter was accepted"}, + {entryWrites({0xbe8e037eu}), "descriptor Phi: a T# holding EXEC was accepted"}, + {entryWrites({0x7d840080u, 0xbe8e036au}), "descriptor Phi: a T# holding a compare mask was accepted"}, + }}; + for (const auto& [code, message] : invalidEntries) { + auto request = makeRequest(code); + expectFailure([&] { static_cast(GetResourcePlan(request)); }, "GetImageResource dword 0 is not a valid runtime value", message); + } + auto unwritten = loopCode; + unwritten[4] = 0xf4080200u; + auto unwrittenRequest = makeRequest(unwritten); + require(GetResourcePlan(unwrittenRequest)->info.images.size() == 3u, "descriptor Phi: a T# whose unwritten registers read as 0 was not planned as its own image"); + + const std::array prologue{0xf4080500u, 0xfa000020u, 0xf4080600u, 0xfa000040u, 0xf40c0200u, 0xfa000000u, 0xbf8cc07fu}; + const std::array epilogue{0xbf8cc07fu, 0xf09c0f08u, 0x00a20000u, 0xbf8c3f70u, 0xe0700000u, 0x80060000u, 0xbf810000u}; + const auto planImages = [&](const std::vector& body) { + std::vector code(prologue.begin(), prologue.end()); + code.insert(code.end(), body.begin(), body.end()); + code.insert(code.end(), epilogue.begin(), epilogue.end()); + auto request = makeRequest(code); + return GetResourcePlan(request)->info.images.size(); + }; + const auto distinctLoads = [](std::uint32_t loads) { + std::vector body; + for (std::uint32_t load = 0; load < loads; load++) { + body.insert(body.end(), {0xbf068014u, 0xbf850002u, 0xf40c0200u, 0xfa000060u + load * 0x20u}); + } + return body; + }; + require(planImages(distinctLoads(63u)) == 64u, "descriptor Phi: 64 distinct T#s were not given one image each"); + expectFailure([&] { static_cast(planImages(distinctLoads(64u))); }, "GetImageResource dword 0 is not a valid runtime value", "descriptor Phi: more than 64 distinct T#s were accepted"); + + const auto splitReloads = [](std::uint32_t joins) { + std::vector body; + for (std::uint32_t join = 0; join < joins; join++) { + body.insert(body.end(), {0xbf068014u, 0xbf850003u, 0xf40003c0u, join % 2u == 0u ? 0xfa000088u : 0xfa00008cu, 0xbf820006u, 0xf4080200u, 0xfa000060u, 0xf4040300u, 0xfa000070u, 0xf4000380u, 0xfa000078u}); + } + return body; + }; + require(planImages(splitReloads(8u)) == 4u, "descriptor Phi: separate reloads of T# dword 7 and dwords 0-6 were not split into their 4 T#s"); + expectFailure([&] { static_cast(planImages(splitReloads(260u))); }, "GetImageResource dword 0 is not a valid runtime value", "descriptor Phi: a web over more than 512 descriptor tuples was accepted"); } void verifyProgramCounterRelativeData() { @@ -738,6 +1129,45 @@ void verifyHalfWaveReduction() { const auto wide = "half-wave reduction: a wave64 scan reduced by SPIR-V opcode " + std::to_string(reduction.reduce) + " did not read lanes 31 and 63 as reductions of host invocations 0-31 and 32-63 on 64 lanes"; require(halves(pixel(scan(reduction.identity, reduction.vector, reduction.scalar), 64u, true), reduction.reduce, reduction.identity), wide.c_str()); } + const auto vertex = [](std::vector code, std::uint32_t subgroupSize) { + *std::find(code.begin(), code.end(), 0xf800180fu) = 0xf80008cfu; + RecompileRequest request{}; + request.shader = {ShaderStage::Vertex, 0x10000u, code, 0, {}}; + request.context.waveSize = 64; + request.context.vertex = ShaderVertexStageInfo{}; + request.target.vulkanVersion = 0x00401000u; + request.target.spirvVersion = 0x00010300u; + request.target.subgroupSize = subgroupSize; + request.target.supportedCapabilities = capabilities; + request.target.fragmentShaderBarycentricEnabled = false; + request.layout.pushConstantSizeBytes = 128; + request.useCache = false; + return Recompile(request).spirv; + }; + require(reductions(vertex(scan(0xffffffffu, 0x26000000u, 0x83800000u), 32u), spv::OpGroupNonUniformUMin) == 1u, "half-wave reduction: the wave64 vertex UMin scan, whose entry EXEC is every invocation, did not read lane 31 as a subgroup UMin on 32 lanes"); + require(halves(vertex(scan(0x80000000u, 0x24000000u, 0x84000000u), 64u), spv::OpGroupNonUniformSMax, 0x80000000u), "half-wave reduction: the wave64 vertex SMax scan did not read lanes 31 and 63 as reductions of host invocations 0-31 and 32-63 on 64 lanes"); + const auto floatScan = [&scan](std::uint32_t identity, std::uint32_t vector, std::uint32_t join) { + auto code = scan(identity, vector, 0u); + code[23] = join; + code[24] = 0x00000e06u; + return code; + }; + const auto fmax = floatScan(0xff800000u, 0x20000000u, 0xd5100001u); + const auto fmin = floatScan(0x7f800000u, 0x1e000000u, 0xd50f0001u); + for (const auto& code : {fmax, fmin}) { + const auto narrow = pixel(code, 32u, false); + require(!narrow.empty() && reductions(narrow, spv::OpGroupNonUniformFMax) == 0u && reductions(narrow, spv::OpGroupNonUniformFMin) == 0u, "half-wave reduction: a wave64 v_max_f32/v_min_f32 scan did not read lane 63 as its -inf/+inf keys on 32 lanes"); + require(!pixel(code, 64u, false).empty(), "half-wave reduction: a wave64 v_max_f32/v_min_f32 scan did not build on 64 lanes"); + } + auto fmaxUnderExec = fmax; + fmaxUnderExec[3] = 0xbeea047eu; + expectFailure([&] { static_cast(pixel(fmaxUnderExec, 32u, true)); }, "v_readlane_b32 of lane 63 is outside the 32-lane host subgroup", "half-wave reduction: a v_max_f32 scan under the entry EXEC read lane 63"); + auto fmaxJoinedByMin = fmax; + fmaxJoinedByMin[17] = 0x1e000000u | 0x001a190du; + expectFailure([&] { static_cast(pixel(fmaxJoinedByMin, 32u, true)); }, "v_readlane_b32 of lane 63 is outside the 32-lane host subgroup", "half-wave reduction: a v_max_f32 row scan joined by v_min_f32 read lane 63"); + auto fmaxOfZero = fmax; + fmaxOfZero[6] = 0u; + expectFailure([&] { static_cast(pixel(fmaxOfZero, 32u, true)); }, "v_readlane_b32 of lane 63 is outside the 32-lane host subgroup", "half-wave reduction: a v_max_f32 scan of 0 outside the live lanes read lane 63"); const auto umin = scan(0xffffffffu, 0x26000000u, 0x83800000u); const auto patched = [&umin](std::initializer_list> words) { auto code = umin; @@ -746,6 +1176,10 @@ void verifyHalfWaveReduction() { }; require(reductions(pixel(patched({{3, 0xbeea25c1u}}), 32u, true), spv::OpGroupNonUniformUMin) == 1u, "half-wave reduction: the scan under s_or_saveexec_b64 -1 did not read lane 31 as a subgroup UMin"); const char* lane63 = "v_readlane_b32 of lane 63 is outside the 32-lane host subgroup"; + auto divergent = umin; + divergent.insert(divergent.begin() + 3, 0x88fe1a7eu); + divergent.insert(divergent.begin() + 2, {0xbe860480u, 0x7d821688u, 0xbe9a246au, 0xbf880001u}); + require(reductions(pixel(divergent, 32u, true), spv::OpGroupNonUniformUMin) == 1u, "half-wave reduction: keys masked inside a divergent branch did not read lane 31 as a subgroup UMin"); const char* arithmetic = "v_readlane_b32 of lane 31 of a wave64 half-wave reduction scan needs subgroup arithmetic"; expectFailure([&] { static_cast(pixel(umin, 32u, false)); }, arithmetic, "half-wave reduction: a device without subgroup arithmetic read lane 31 on 32 lanes"); expectFailure([&] { static_cast(pixel(umin, 64u, false)); }, arithmetic, "half-wave reduction: a device without subgroup arithmetic read lane 31 on 64 lanes"); @@ -801,6 +1235,20 @@ void verifyMeshConfiguration() { other.graphics = GraphicsCompileContext{0u, {}, otherMesh, std::nullopt, {}}; RecompileCacheKey::Build(other, key); require(key != first && RecompileCacheKey::ContextHash(request) != RecompileCacheKey::ContextHash(other), "the cache keys ignore the mesh configuration"); + const std::array users{}; + request.context.waveSize = 32u; + request.context.userData = users; + request.context.vertex = ShaderVertexStageInfo{}; + request.target.subgroupSize = 32u; + request.graphics->mesh = MeshConfiguration{1u, 1u, 1u, 1u, 1u, 32u, 256u, 0u, 4u}; + for (const std::uint32_t threads : {32u, 64u, 128u, 480u}) { + request.graphics->mesh->threadsPerGroup = threads; + require(PrepareResourceProgram(request).WaveSize() == 32u, "a wave32 mesh program did not preserve its guest wave size"); + } + for (const std::uint32_t threads : {31u, 512u}) { + request.graphics->mesh->threadsPerGroup = threads; + expectFailure([&] { static_cast(PrepareResourceProgram(request)); }, "mesh shader translation configuration is not supported", "mesh translation accepted a partial wave or overflowing wave count"); + } } ShaderRecompiler::ShaderPixelStageInfo twoParameterPixel() { @@ -916,19 +1364,19 @@ void verifyPixelRequestSerialization() { minimal.context.waveSize = 64; minimal.context.pixel = ShaderPixelStageInfo{}; const auto encoded = serializer.Serialize(minimal); - require(requestPrefix(encoded, 8u) == "NVNQQQ0AAAA=", "new requests did not use serialization version 13"); + require(requestPrefix(encoded, 8u) == "NVNQQQ8AAAA=", "new requests did not use serialization version 15"); constexpr std::size_t mappingOffset = 8u + 37u + 18u + 163u; - for (std::size_t bytes = 0; bytes < 8u; ++bytes) { - expectFailure([&] { static_cast(serializer.Deserialize(requestPrefix(encoded, mappingOffset + bytes))); }, "truncated data", "a truncated version-12 pixel mapping was accepted"); + for (std::size_t bytes = 0; bytes < 17u; ++bytes) { + expectFailure([&] { static_cast(serializer.Deserialize(requestPrefix(encoded, mappingOffset + bytes))); }, "truncated data", "a truncated pixel mapping, packing or dual-source flag was accepted"); } - for (const auto unsupported : {"NVNQQQAAAAA=", "NVNQQQ4AAAA="}) { + for (const auto unsupported : {"NVNQQQAAAAA=", "NVNQQRAAAAA="}) { expectFailure([&] { static_cast(serializer.Deserialize(unsupported)); }, "serialization version", "an unsupported request version was accepted"); } } void verifyLegacyPixelRequests() { using namespace ShaderRecompiler; - static constexpr std::array legacyPixelRequests{ + static constexpr std::array legacyPixelRequests{ "NVNQQQEAAAAFAAADAAAAAAABAAAAAAAAAAAAgb8AAAAAAAAAAAAAAAAAAAAAQAAAAAAAAAAAAAAAAAAAAAABAAAAAAAAAAAAAAAA" "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA" "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAABAAAAAAAAAAAAAAAAAAkAAAAAAAAAAAAA" @@ -979,6 +1427,26 @@ void verifyLegacyPixelRequests() { "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAIAAAABAAAAAAAAAAAAAAAAAAAAAgkAAAAAAAAA" "AAAAAAAAAAAAAAAAAAAAAAAAEEAAAAMBAEAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA" "AAsAAAAAAAAAgAAAAAAAAQAAAAA=", + "NVNQQQsAAAAFAAADAAAAAAABAAAAAAAAAAAAgb8AAAAAAAAAAAAAAAAAAAAAQAAAAAAAAAAAAAAAAAAAAAABAAAAAAAAAAAAAAAA" + "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA" + "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAIAAAABAAAAAAAAAAAAAAAAAAAAAgAJAAAAAAAA" + "AAAAAAAAAAAAAAAAAAAAAAAAABBAAAADAQBAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA" + "AAALAAAAAAAAAIAAAAAAAAEAAAAA", + "NVNQQQwAAAAFAAADAAAAAAABAAAAAAAAAAAAgb8AAAAAAAAAAAAAAAAAAAAAQAAAAAAAAAAAAAAAAAAAAAABAAAAAAAAAAAAAAAA" + "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA" + "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAIAAAABAAAAAAAAAAAAAAAAAAAAAgAJAAAAAAAA" + "AAAAAAAAAAAAAAAAAAAAAAAAABBAAAADAQBAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA" + "AAALAAAAAAAAAIAAAAAAAAEAAAAAAA==", + "NVNQQQ0AAAAFAAADAAAAAAABAAAAAAAAAAAAgb8AAAAAAAAAAAAAAAAAAAAAQAAAAAAAAAAAAAAAAAAAAAABAAAAAAAAAAAAAAAA" + "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA" + "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAIAAAABAAAAAAAAAAAAAAAAAAAAAgAJAAAAAAAA" + "AAAAAAAAAAAAAAAAAAAAAAAAAAAQQAAAAwEAQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA" + "AAAACwAAAAAAAACAAAAAAAABAAAAAAA=", + "NVNQQQ4AAAAFAAADAAAAAAABAAAAAAAAAAAAgb8AAAAAAAAAAAAAAAAAAAAAQAAAAAAAAAAAAAAAAAAAAAABAAAAAAAAAAAAAAAA" + "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA" + "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAIAAAABAAAAAAAAAAAAAAAAAAAAAgAJAAAAAAAA" + "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAABBAAAADAQBAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA" + "AAAAAAAAAAAAAAALAAAAAAAAAIAAAAAAAAEAAAAAAA==", }; const RequestSerializer serializer; for (std::size_t index = 0; index < legacyPixelRequests.size(); ++index) { @@ -988,6 +1456,8 @@ void verifyLegacyPixelRequests() { require(request.context.pixel.has_value(), "legacy pixel state was lost"); const auto& pixel = *request.context.pixel; require(pixel.targetExportMapping == std::array{}, "legacy pixel mapping no longer defaults to zero"); + require(pixel.targetExportPacking == std::array{}, "a legacy request packed a color export"); + require(!pixel.dualSourceBlend, "a legacy request enabled dual-source blending"); require(pixel.inputAddr == 2u && pixel.hasPerspectiveCenterVgpr && pixel.targetOutputMode[0] == 9u, "legacy pixel layout was misread"); require(pixel.conservativeZExport == (version >= 7u ? ConservativeZExport::GreaterThanZ : ConservativeZExport::AnyZ), "legacy conservative Z layout was misread"); require(!pixel.orderedPixelShader, "a legacy request became a primitive-ordered pixel shader"); @@ -1452,6 +1922,8 @@ void verifyUnnormalizedSamplers() { for (const std::uint32_t mimg : {0xf0c00f08u, 0xf11c0108u, 0xf09c0f18u}) { static_cast(recompile(program(mimg), imageData(normalized, mimg == 0xf09c0f18u ? TypeCube : Type2D, Format8888UNorm, mimg == 0xf09c0f18u ? 5u : 0u))); } + const std::vector latchSample{0xbe900380u, 0x7e020280u, 0x7e040280u, 0x7e060280u, 0x7e080280u, 0x7e0a0280u, 0x7e0c0280u, 0xf09c0f28u, 0x00400801u, 0x80108110u, 0xbf0a8410u, 0xbf85fffbu, 0xe0700000u, 0x80030800u, 0xbf810000u}; + static_cast(recompile(latchSample, imageData(normalized, Type2DArray, Format8888UNorm, 5u))); const std::array pixelCode{0xf0800f08u, 0x00400801u, 0xf800180fu, 0x0b0a0908u, 0xbf810000u}; const auto pixelData = userData(unnormalized); @@ -1495,6 +1967,11 @@ void verifyUnusedUnnormalizedSampler() { auto selected = info; selected.images[0].indirectRoot = 0u; expectFailure([&] { static_cast(populate(selected)); }, "unnormalized guest sampler samples an image selected at run time, which is not implemented", "unnormalized samplers: an image table root was accepted"); + snapshot.images[0].dwords[3] = 0xa0000041u; + const auto constant = populate(info); + require(constant.size() == 1u && constant[0].samplerUnnormalized == std::vector{true}, "unnormalized samplers: a 3D view whose channels select constants was bound or refused"); + expectFailure([&] { static_cast(populate(selected)); }, "unnormalized guest sampler samples an image selected at run time, which is not implemented", "unnormalized samplers: an image table root whose channels select constants was accepted"); + snapshot.images[0].dwords[3] = 0x90000facu; auto compared = info; compared.samplers[0].depthCompare = true; expectFailure([&] { static_cast(populate(compared)); }, "unnormalized guest sampler is used with depth comparison, which is not implemented", "unnormalized samplers: a depth-compare S# without live uses was accepted"); @@ -1640,9 +2117,12 @@ void verifyBdaReadFallbackFunctions() { std::map names; std::map definitions; const auto reader = std::string("read_bda_dword_bytes") + (barrier ? "" : "_stop") + (coherent ? "_coherent" : ""); + const auto span = std::string("read_bda_span") + (barrier ? "" : "_stop") + (coherent ? "_coherent" : ""); std::string function; std::size_t compareExchanges = 0; std::size_t mainLookups = 0; + std::size_t mainProbes = 0; + std::size_t spanCalls = 0; std::array readerLoads{}; for (std::size_t cursor = 5; cursor < five.size();) { const auto length = five[cursor] >> 16u; @@ -1651,22 +2131,29 @@ void verifyBdaReadFallbackFunctions() { if (op == spv::OpName) names[five[cursor + 1]] = reinterpret_cast(&five[cursor + 2]); if (op == spv::OpFunction) { function = names[five[cursor + 2]]; - if (function == "record_bda_fault" || function.starts_with("read_bda_dword_bytes")) { - require((five[cursor + 3] & spv::FunctionControlDontInlineMask) != 0u, "BDA read functions: a fault or byte read function may be inlined"); + if (function == "record_bda_fault" || function.starts_with("read_bda_dword_bytes") || function.starts_with("read_bda_span")) { + require((five[cursor + 3] & spv::FunctionControlDontInlineMask) != 0u, "BDA read functions: a fault, byte read or span read function may be inlined"); ++definitions[function]; } } if (op == spv::OpAtomicCompareExchange) ++compareExchanges; if (op == spv::OpFunctionCall && function == "main" && names[five[cursor + 3]] == "get_bda_pointer") ++mainLookups; + if (op == spv::OpFunctionCall && function == "main") { + const auto& callee = names[five[cursor + 3]]; + if (callee == "probe_bda_pointer" || callee.starts_with("read_bda_dword_bytes")) ++mainProbes; + if (callee == span) ++spanCalls; + } if (op == spv::OpLoad && function == reader) ++readerLoads[length > 4u && (five[cursor + 4] & spv::MemoryAccessVolatileMask) != 0u]; cursor += length; } - require(definitions["record_bda_fault"] == 1u && definitions[reader] == 1u, "BDA read functions: the fault and byte read functions are not defined"); + require(definitions["record_bda_fault"] == 1u && definitions[reader] == 1u && definitions[span] == 1u, "BDA read functions: the fault, byte read and span read functions are not defined"); for (const auto& [name, count] : definitions) require(count == 1u, "BDA read functions: a function is defined twice"); require(compareExchanges == 1u, "BDA read functions: a fault is recorded outside record_bda_fault"); require(mainLookups == 0u, "BDA read functions: a read site looks up its bytes inline"); require(readerLoads[coherent] == 4u && readerLoads[!coherent] == 0u, "BDA read functions: the byte loads do not keep the access's coherence"); - require(five.size() - one.size() < 4u * 300u, "BDA read functions: a read site takes 300 SPIR-V words or more"); + require(mainProbes == 0u, "BDA read functions: a read site probes or reads bytes outside its span read function"); + require(spanCalls == 5u, "BDA read functions: a read site does not call its span read function once"); + require(five.size() - one.size() < 4u * 150u, "BDA read functions: a read site takes 150 SPIR-V words or more"); } } } @@ -1811,9 +2298,266 @@ void verifyFunctionLdsBound() { require(unsized == FunctionLdsDwordLimit, "function LDS: an access without a known width must keep the full array"); } +struct NestedRequest { + std::vector code; + std::array userData{}; + ShaderRecompiler::RecompileRequest request{}; + + NestedRequest(std::vector words, const void* srt) : code(std::move(words)) { + using namespace ShaderRecompiler; + const auto address = static_cast(reinterpret_cast(srt)); + userData = {static_cast(address), static_cast(address >> 32u)}; + request.shader = {ShaderStage::Compute, 0x60000u + static_cast(code.size()) * 4u, code, 0, {}}; + request.context.waveSize = 64; + request.context.userDataBaseRegister = 0; + request.context.userData = userData; + request.context.compute = ShaderComputeStageInfo{{64u, 1u, 1u}, 0u, {false, false, false}, false, 1u}; + request.target = BufferTarget(); + request.target.vulkanVersion = 0x00401000u; + request.target.spirvVersion = 0x00010300u; + request.target.subgroupSize = 32; + request.target.fragmentShaderBarycentricEnabled = false; + request.layout.pushConstantSizeBytes = 128; + } +}; + +bool RegionsCover(const std::vector& regions, const void* pointer, std::size_t bytes) { + auto address = static_cast(reinterpret_cast(pointer)); + const auto end = address + bytes; + while (address < end) { + const auto region = std::find_if(regions.begin(), regions.end(), [&](const ShaderRecompiler::MemoryRegion& candidate) { return address >= candidate.guestAddress && address < candidate.guestAddress + candidate.bytes.size(); }); + if (region == regions.end()) return false; + address = region->guestAddress + region->bytes.size(); + } + return true; +} + +bool RegionsTouch(const std::vector& regions, std::uint64_t address, std::size_t bytes) { + return std::any_of(regions.begin(), regions.end(), [&](const ShaderRecompiler::MemoryRegion& region) { return region.guestAddress < address + bytes && address < region.guestAddress + region.bytes.size(); }); +} + +bool GuardRecords(const ShaderRecompiler::ResourceCapture& capture) { + const auto& plan = *capture.plan; + const auto& flat = capture.snapshot.flattenedSrt; + const auto& poison = capture.snapshot.srtPoison; + const auto flags = plan.srtReads.size(); + const auto guards = plan.guardedSrtSlots.size(); + if (flat.size() != flags + guards + 3u * poison.size()) return false; + for (std::size_t guard = 0; guard < guards; ++guard) { + const auto record = std::find_if(poison.begin(), poison.end(), [&](const ShaderRecompiler::SrtReadPoison& entry) { return entry.slot == plan.guardedSrtSlots[guard]; }); + const auto expected = record == poison.end() ? 0u : static_cast(record - poison.begin()) + 1u; + if (flat[flags + guard] != expected) return false; + } + for (std::size_t record = 0; record < poison.size(); ++record) { + const auto* words = flat.data() + flags + guards + 3u * record; + if (words[0] != poison[record].pc || words[1] != static_cast(poison[record].address) || words[2] != static_cast(poison[record].address >> 32u) || flat.at(poison[record].slot) != 0u) return false; + } + return true; +} + +void verifyGuardedNullPointers() { + using namespace ShaderRecompiler; + constexpr std::size_t Reserved = 0x10000; + void* reserved = nullptr; +#ifdef _WIN32 + reserved = VirtualAlloc(nullptr, Reserved, MEM_RESERVE, PAGE_NOACCESS); +#else + reserved = mmap(nullptr, Reserved, PROT_NONE, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + if (reserved == MAP_FAILED) reserved = nullptr; +#endif + require(reserved != nullptr, "guarded pointer: cannot reserve an inaccessible range"); + const auto unmapped = static_cast(reinterpret_cast(reserved)); + alignas(256) static std::array output{}; + alignas(256) static std::array root{}; + static std::uint32_t payload = 0x3f800000u; + static std::uint64_t indirect = 0; + const auto outputAddress = static_cast(reinterpret_cast(output.data())); + root = {static_cast(outputAddress), static_cast((outputAddress >> 32u) & 0xffffu), static_cast(sizeof(output)), 0x30027facu, 1u}; + const auto point = [&](std::uint64_t address) { + root[6] = static_cast(address); + root[7] = static_cast(address >> 32u); + }; + const auto payloadAddress = static_cast(reinterpret_cast(&payload)); + const std::vector branchCode{0xf4080100u, 0xfa000000u, 0xf4000200u, 0xfa000010u, 0xf4040280u, 0xfa000018u, 0xbe8c03ffu, 0x11111111u, + 0xbf8cc07fu, 0xbf068008u, 0xbf850003u, 0xf4000305u, 0xfa000000u, 0xbf8cc07fu, 0x7e02020cu, 0x34040082u, 0xe0701000u, 0x80010102u, 0xbf810000u}; + const std::vector chainCode{0xf4080100u, 0xfa000000u, 0xf4000200u, 0xfa000010u, 0xf4040280u, 0xfa000018u, 0xbe8c03ffu, 0x11111111u, + 0xbf8cc07fu, 0xbf068008u, 0xbf850006u, 0xf4040385u, 0xfa000000u, 0xbf8cc07fu, 0xf4000307u, 0xfa000000u, 0xbf8cc07fu, 0x7e02020cu, 0x34040082u, + 0xe0701000u, 0x80010102u, 0xbf810000u}; + const std::vector descriptorCode{0xf4040280u, 0xfa000018u, 0xbf8cc07fu, 0xf4080105u, 0xfa000000u, 0xbf8cc07fu, 0x34040082u, 0xe0701000u, + 0x80010002u, 0xbf810000u}; + const auto slotPc = [](const IrResourcePlan& plan, std::uint32_t slot) { return plan.srtReads.at(slot).value->Flags().pc; }; + const auto faultBinding = [](const RecompileResult& result) { + return std::any_of(result.bindings.begin(), result.bindings.end(), [](const DescriptorBinding& binding) { return binding.role == DescriptorRole::FaultBuffer; }); + }; + const auto samePipeline = [](const RecompileResult& left, const RecompileResult& right) { + return left.variantId == right.variantId && left.specializationId == right.specializationId && left.PipelineVariantId() == right.PipelineVariantId() && left.spirv == right.spirv; + }; + + NestedRequest branch(branchCode, root.data()); + point(payloadAddress); + AgcDriver::ShaderMemory validMemory({}); + const auto valid = validMemory.Capture(branch.request); + require(valid->snapshot.srtPoison.empty() && valid->plan->guardedSrtSlots.size() == 1u && slotPc(*valid->plan, valid->plan->guardedSrtSlots[0]) == 0x2cu && GuardRecords(*valid), "guarded pointer: a mapped nested pointer was poisoned, or its read has no guard"); + require(valid->snapshot.flattenedSrt.at(valid->plan->guardedSrtSlots[0]) == payload, "guarded pointer: a mapped nested read does not hold its word"); + branch.request.context.memory = validMemory.Regions(); + const auto validResult = Recompile(branch.request, *valid); + require(validResult->poisonedSrtReads == 0u && faultBinding(*validResult) && validResult->bdaAbiVersion == BdaAbi::Version, "guarded pointer: a guarded program has no fault buffer"); + + for (const auto pointer : {std::uint64_t{0}, unmapped}) { + for (const auto condition : {1u, 0u}) { + point(pointer); + root[4] = condition; + branch.request.context.memory = {}; + AgcDriver::ShaderMemory memory({}); + const auto capture = memory.Capture(branch.request); + const auto& poison = capture->snapshot.srtPoison; + require(poison.size() == 1u && poison[0].pc == 0x2cu && poison[0].address == pointer && slotPc(*capture->plan, poison[0].slot) == 0x2cu, "guarded pointer: the nested read was not poisoned at its pc and address"); + require(GuardRecords(*capture), "guarded pointer: the poison record is not where the guard reads it"); + const auto regions = memory.Regions(); + require(!RegionsTouch(regions, pointer, 4u) && RegionsCover(regions, root.data() + 6, 8u), "guarded pointer: the captured regions are wrong"); + branch.request.context.memory = regions; + const auto result = Recompile(branch.request, *capture); + require(result->poisonedSrtReads == 1u && faultBinding(*result) && result->bdaAbiVersion == BdaAbi::Version, "guarded pointer: the poisoned capture does not report through the fault buffer"); + require(samePipeline(*result, *validResult), "guarded pointer: poison changed the variant, the specialization or the module"); + const auto replay = Recompile(branch.request); + require(replay.poisonedSrtReads == 1u && samePipeline(replay, *result), "guarded pointer: a replay of the captured regions did not reproduce the poison"); + verifyResult(*result, replay); +#if ANYPS5_ENABLE_SPIRV_TOOLS + static_cast(ValidateAndOptimizeSpirv(result->spirv, branch.request.target.vulkanVersion, branch.request.target.spirvVersion)); +#endif + } + } + root[4] = 1u; + + NestedRequest chain(chainCode, root.data()); + point(unmapped); + AgcDriver::ShaderMemory chainMemory({}); + const auto chained = chainMemory.Capture(chain.request); + const auto& chainPoison = chained->snapshot.srtPoison; + require(chainPoison.size() == 3u && GuardRecords(*chained), "guarded pointer: a pointer read through an unmapped pointer did not poison its two words and the read through it"); + for (const auto& poison : chainPoison) { + const auto own = slotPc(*chained->plan, poison.slot); + const bool pointerWord = own == 0x2cu && (poison.address == unmapped || poison.address == unmapped + 4u); + const bool throughPointer = own == 0x38u && poison.address == unmapped; + require(poison.pc == 0x2cu && (pointerWord || throughPointer), "guarded pointer: a derived poison does not name the inaccessible read"); + } + chain.request.context.memory = chainMemory.Regions(); + const auto chainResult = Recompile(chain.request, *chained); + indirect = 0; + point(static_cast(reinterpret_cast(&indirect))); + AgcDriver::ShaderMemory nullChainMemory({}); + const auto nullChain = nullChainMemory.Capture(chain.request); + require(nullChain->snapshot.srtPoison.size() == 1u && nullChain->snapshot.srtPoison[0].pc == 0x38u && nullChain->snapshot.srtPoison[0].address == 0u && GuardRecords(*nullChain), "guarded pointer: a null pointer read from memory did not poison the read through it"); + chain.request.context.memory = nullChainMemory.Regions(); + const auto nullChainResult = Recompile(chain.request, *nullChain); + indirect = payloadAddress; + AgcDriver::ShaderMemory mappedChainMemory({}); + const auto mappedChain = mappedChainMemory.Capture(chain.request); + require(mappedChain->snapshot.srtPoison.empty() && mappedChain->plan->guardedSrtSlots.size() == 3u && GuardRecords(*mappedChain), "guarded pointer: a mapped pointer chain was poisoned, or its reads have no guards"); + chain.request.context.memory = mappedChainMemory.Regions(); + const auto mappedChainResult = Recompile(chain.request, *mappedChain); + require(chainResult->poisonedSrtReads == 3u && nullChainResult->poisonedSrtReads == 1u && mappedChainResult->poisonedSrtReads == 0u, "guarded pointer: a chain result does not count its poisoned reads"); + require(samePipeline(*chainResult, *mappedChainResult) && samePipeline(*nullChainResult, *mappedChainResult), "guarded pointer: a chain's poison changed the variant, the specialization or the module"); + + const auto poisonedWords = [](const ResourceCapture& capture, std::uint32_t pc, std::uint64_t base) { + std::vector addresses; + for (const auto& entry : capture.snapshot.srtPoison) { + if (entry.pc != pc || capture.snapshot.flattenedSrt.at(entry.slot) != 0u) return false; + addresses.push_back(entry.address); + } + std::sort(addresses.begin(), addresses.end()); + return addresses == std::vector{base, base + 4u, base + 8u, base + 12u} && GuardRecords(capture); + }; + const auto zeroBuffer = [](const ResourceCapture& capture) { + return capture.snapshot.buffers.size() == 1u && std::all_of(capture.snapshot.buffers[0].dwords.begin(), capture.snapshot.buffers[0].dwords.end(), [](std::uint32_t word) { return word == 0u; }); + }; + alignas(256) static std::array zeroTable{}; + NestedRequest descriptor(descriptorCode, root.data()); + point(static_cast(reinterpret_cast(zeroTable.data()))); + AgcDriver::ShaderMemory zeroMemory({}); + const auto zeroCapture = zeroMemory.Capture(descriptor.request); + require(zeroCapture->snapshot.srtPoison.empty() && zeroBuffer(*zeroCapture) && GuardRecords(*zeroCapture), "guarded pointer: a zero V# read from mapped memory was poisoned or not read"); + descriptor.request.context.memory = zeroMemory.Regions(); + const auto zeroResult = Recompile(descriptor.request, *zeroCapture); + for (const auto pointer : {std::uint64_t{0}, unmapped}) { + point(pointer); + descriptor.request.context.memory = {}; + AgcDriver::ShaderMemory memory({}); + const auto capture = memory.Capture(descriptor.request); + require(poisonedWords(*capture, 0xcu, pointer), "guarded pointer: a V# loaded through a null or unmapped nested pointer did not poison its four words at its pc"); + require(zeroBuffer(*capture), "guarded pointer: a V# loaded through an inaccessible pointer is not bound as zero words"); + const auto regions = memory.Regions(); + descriptor.request.context.memory = regions; + const auto result = Recompile(descriptor.request, *capture); + require(result->poisonedSrtReads == 4u && faultBinding(*result) && result->bdaAbiVersion == BdaAbi::Version, "guarded pointer: a capture with a poisoned V# does not report through the fault buffer"); + require(samePipeline(*result, *zeroResult), "guarded pointer: a V# zeroed by poison and a V# read as zeros take different variants, specializations or modules"); + const auto replay = Recompile(descriptor.request); + require(replay.poisonedSrtReads == 4u && samePipeline(replay, *result), "guarded pointer: a replay did not reproduce the poisoned V#"); + } + + alignas(256) static std::array table{}; + alignas(256) static std::array vertexRoot{}; + table = {0u, 0u, 0u, 0u, static_cast(payloadAddress), static_cast((payloadAddress >> 32u) & 0xffffu), 4u, 0x30027facu}; + const std::vector vertexDescriptorCode{0xf4000084u, 0xfa000000u, 0xf4040104u, 0xfa000008u, 0x7e020280u, 0xbf8cc07fu, 0xbf068002u, 0xbf850006u, + 0xf4080302u, 0xfa000010u, 0xbf8cc07fu, 0xe0300000u, 0x80030100u, 0xbf8c3f70u, 0xf80008cfu, 0x01010101u, 0xbf810000u}; + NestedRequest vertex(vertexDescriptorCode, vertexRoot.data()); + vertex.request.shader.stage = ShaderStage::Vertex; + vertex.request.context.compute.reset(); + vertex.request.context.vertex = ShaderVertexStageInfo{}; + vertex.request.context.userDataBaseRegister = 8; + const auto tableAddress = static_cast(reinterpret_cast(table.data())); + vertexRoot[2] = static_cast(tableAddress); + vertexRoot[3] = static_cast(tableAddress >> 32u); + AgcDriver::ShaderMemory mappedVertexMemory({}); + const auto mappedVertex = mappedVertexMemory.Capture(vertex.request); + require(mappedVertex->snapshot.srtPoison.empty() && mappedVertex->snapshot.buffers.size() == 1u && mappedVertex->snapshot.buffers[0].dwords[0] == table[4], "guarded pointer: a vertex V# behind a mapped pointer was poisoned or not read"); + vertexRoot[2] = 0u; + vertexRoot[3] = 0u; + for (const auto guard : {0u, 1u}) { + vertexRoot[0] = guard; + vertex.request.context.memory = {}; + AgcDriver::ShaderMemory memory({}); + const auto capture = memory.Capture(vertex.request); + require(poisonedWords(*capture, 0x20u, 0x10u), "guarded pointer: a vertex V# behind a branch and a null pointer did not poison its four words at its pc"); + const auto regions = memory.Regions(); + vertex.request.context.memory = regions; + const auto result = Recompile(vertex.request, *capture); + require(result->poisonedSrtReads == 4u && faultBinding(*result) && !result->spirv.empty(), "guarded pointer: a vertex capture with a poisoned V# has no fault buffer"); +#if ANYPS5_ENABLE_SPIRV_TOOLS + static_cast(ValidateAndOptimizeSpirv(result->spirv, vertex.request.target.vulkanVersion, vertex.request.target.spirvVersion)); +#endif + } + NestedRequest rootless(branchCode, nullptr); + AgcDriver::ShaderMemory rootlessMemory({}); + expectFailure([&] { static_cast(rootlessMemory.Capture(rootless.request)); }, "null or misaligned address", "guarded pointer: a null user-data pointer was accepted"); + point(0u); +#ifdef _WIN32 + VirtualFree(reserved, 0, MEM_RELEASE); +#else + munmap(reserved, Reserved); +#endif +} + +void verifyCubeDepthCompareMode() { + using namespace ShaderRecompiler; + ImageResource image{}; + image.resourceClass = ImageResourceClass::Sampled; + image.dimension = RdnaImageDimension::Dim2DArray; + image.depthCompare = true; + const DescriptorValue descriptor{{0x02143930u, 0x08200000u, 0u, 0xb0500facu, 5u, 0x00700000u, 0u, 0u}, 8u}; + const auto modes = ResourceMaterializer::RuntimeImageModes(image); + const auto selected = ResourceMaterializer::RuntimeImageMode(image, descriptor, modes); + require(modes[selected].cube && (modes[selected].emulatedCompare & EmulatedCompare::Enabled) != 0u, "cube color comparison did not select its emulated runtime mode"); +} + int main(int argc, char** argv) { try { using namespace ShaderRecompiler; + if (argc == 2 && std::string_view(argv[1]) == "--frontend-pair") { + verifyFrontendPair(); + std::cout << "Prepared frontend reuse preserves full and partial-thread programs\n"; + return 0; + } if (argc == 2 && std::string_view(argv[1]) == "--bindless") { verifyBindlessTable(); std::cout << "Bindless mapping, indexing capabilities and strict validation passed\n"; @@ -1822,6 +2566,8 @@ int main(int argc, char** argv) { require(argc == 1, "unknown shader memory test arguments"); verifyRegisterSources(); verifyEvaluatedValues(); + verifyFrontendPair(); + verifySignedSrtComparison(); verifyPureFlatSlots(); verifyBindlessTable(); verifyDescriptorPhis(); @@ -1844,6 +2590,8 @@ int main(int argc, char** argv) { verifyTwoLaneUniformValues(); verifyBdaReadFallbackFunctions(); verifyFunctionLdsBound(); + verifyGuardedNullPointers(); + verifyCubeDepthCompareMode(); #if ANYPS5_ENABLE_SPIRV_TOOLS const std::vector minimalSpirv{ 0x07230203u, 0x00010000u, 0u, 5u, 0u, @@ -2030,8 +2778,18 @@ int main(int argc, char** argv) { expectFailure([&] { static_cast(PrepareResourceProgram(request)); }, "shader user data exceeds the scalar register bank", "user data overran scalar register bank"); request.context.userDataBaseRegister = 8; request.context.memory = {}; + AgcDriver::ShaderMemory guarded({}); + const auto poisoned = guarded.Capture(request); + require(poisoned->snapshot.srtPoison.size() == 1u && poisoned->snapshot.srtPoison[0].pc == 8u && poisoned->snapshot.srtPoison[0].address == 0u && GuardRecords(*poisoned), "null nested pointer was not poisoned at its read"); + const auto guardedRegions = guarded.Regions(); + request.context.memory = guardedRegions; + const auto poisonedVertex = Recompile(request, *poisoned); + require(poisonedVertex->poisonedSrtReads == 1u && poisonedVertex->bdaAbiVersion == BdaAbi::Version && poisonedVertex->PipelineVariantId() == first.PipelineVariantId() && std::any_of(poisonedVertex->bindings.begin(), poisonedVertex->bindings.end(), [](const DescriptorBinding& binding) { return binding.role == DescriptorRole::FaultBuffer; }), "a vertex shader reading through a null nested pointer has no fault buffer or another variant"); + request.context.memory = {}; + const std::array nullUserData{}; + request.context.userData = nullUserData; AgcDriver::ShaderMemory invalid({}); - expectFailure([&] { invalid.Capture(request); }, "null or misaligned address", "null nested pointer was accepted"); + expectFailure([&] { invalid.Capture(request); }, "null or misaligned address", "null user-data pointer was accepted"); std::cout << "Shader memory capture, strict validation and deterministic replay passed\n"; return 0; } catch (const std::exception& error) { diff --git a/core/libs/prx/libSceAgcDriver/tests/ShaderModeRegisters.cpp b/core/libs/prx/libSceAgcDriver/tests/ShaderModeRegisters.cpp new file mode 100644 index 000000000..632181959 --- /dev/null +++ b/core/libs/prx/libSceAgcDriver/tests/ShaderModeRegisters.cpp @@ -0,0 +1,190 @@ +#include "ControlFlow/GraphBuilder.hpp" +#include "RdnaDecoder/RdnaInstructionDecoder.hpp" +#include "Recompiler.hpp" +#include "Translation/InstructionTranslator.hpp" +#include "BdaAbi.hpp" +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace { + +using namespace ShaderRecompiler; + +void require(bool condition, const std::string& message) { + if (!condition) throw std::runtime_error(message); +} + +IrProgram translate(std::span code, std::optional mode, std::uint32_t waveSize = 32) { + const auto decoded = RdnaInstructionDecoder{}.Decode(code); + const auto graph = GraphBuilder{}.Build(decoded); + const ShaderComputeInputInfo compute{}; + TranslateOptions options; + options.stage = ShaderStageKind::Compute; + options.waveSize = waveSize; + options.floatMode = mode; + options.inputInfo.compute = &compute; + return InstructionTranslator{}.Translate(decoded, graph, options); +} + +void expectRegister(const IrProgram& program, std::uint32_t index, std::uint32_t expected) { + const IrValue* value = nullptr; + for (const auto& block : program.Blocks()) { + for (const auto* instruction : block->Instructions()) { + if (instruction->Opcode() != IrOpcode::SetScalarRegister) continue; + const auto reg = instruction->Argument(0)->Register(); + if (reg.bank == RegisterBank::Scalar && reg.index == index) value = instruction->Argument(1); + } + } + require(value && value->HasImmediate() && value->ImmediateU32() == expected, + "MODE read into s" + std::to_string(index) + " did not produce " + std::to_string(expected)); +} + +void expectRejected(std::span code, std::optional mode, std::string_view diagnostic, std::uint32_t pc = 0) { + try { + static_cast(translate(code, mode)); + } catch (const std::runtime_error& error) { + const std::string message = error.what(); + require(message.find(diagnostic) != std::string::npos && message.find("at pc " + std::to_string(pc) + " ") != std::string::npos, + "unexpected MODE diagnostic: " + message); + return; + } + throw std::runtime_error("unsupported MODE operation was accepted: " + std::string(diagnostic)); +} + +void verifyReads() { + constexpr std::array code{ + 0xb9140001, 0xb9150801, 0xb9161001, 0xb9171801, 0xb9180041, + 0xb9190841, 0xb91a1041, 0xb91b0081, 0xb91c0881, 0xb91d00c1, 0xbf810000 + }; + constexpr std::array offsets{0, 0, 0, 0, 1, 1, 1, 2, 2, 3}; + constexpr std::array masks{1, 3, 7, 15, 1, 3, 7, 1, 3, 1}; + for (const auto waveSize : {32u, 64u}) { + for (std::uint32_t mode = 0; mode < 256; ++mode) { + const auto program = translate(code, ShaderFloatMode{mode, true, true, true}, waveSize); + for (std::uint32_t index = 0; index < offsets.size(); ++index) { + expectRegister(program, 20 + index, (mode >> offsets[index]) & masks[index]); + } + } + const auto legacy = translate(code, std::nullopt, waveSize); + for (std::uint32_t index = 0; index < offsets.size(); ++index) expectRegister(legacy, 20 + index, 0); + } +} + +void verifyDynamicWrites() { + for (const auto instruction : {0xb9800001u, 0xb9801801u, 0xb9800041u, 0xb9801881u, 0xb980f801u}) { + const std::array code{instruction, 0xbf810000}; + expectRejected(code, std::nullopt, "s_setreg_b32"); + expectRejected(code, ShaderFloatMode{0xc0}, "s_setreg_b32"); + } +} + +void verifyImmediateWrites() { + constexpr std::array instructions{0xba801801, 0xba800801, 0xba800881, 0xba800841}; + constexpr std::array masks{15, 3, 12, 6}; + constexpr std::array offsets{0, 0, 2, 1}; + for (std::uint32_t rounding = 0; rounding < 16; ++rounding) { + for (std::uint32_t value = 0; value < 16; ++value) { + for (std::uint32_t field = 0; field < instructions.size(); ++field) { + const std::array code{instructions[field], 0xfffffff0u | value, 0xb9141801, 0xbf810000}; + const ShaderFloatMode mode{0xc0u | rounding}; + if (((value << offsets[field]) & masks[field]) != (rounding & masks[field])) { + expectRejected(code, mode, "s_setreg_imm32_b32"); + } else { + expectRegister(translate(code, mode), 20, rounding); + } + } + } + } + constexpr std::array full{0xba801801, 0, 0xbf810000}; + static_cast(translate(full, std::nullopt)); + constexpr std::array nonzero{0xba801801, 15, 0xbf810000}; + constexpr std::array denorm{0xba801881, 0, 0xbf810000}; + expectRejected(nonzero, ShaderFloatMode{0xc0}, "s_setreg_imm32_b32"); + expectRejected(denorm, ShaderFloatMode{0xc0}, "s_setreg_imm32_b32"); +} + +void verifyRoundInstruction() { + for (std::uint32_t rounding = 0; rounding < 16; ++rounding) { + for (std::uint32_t value = 0; value < 16; ++value) { + for (const auto upper : {0u, 0x8000u}) { + const std::array code{0xbfa40000u | upper | value, 0xb9141801, 0xbf810000}; + const ShaderFloatMode mode{0xc0u | rounding}; + if (value != rounding) expectRejected(code, mode, "s_round_mode"); + else expectRegister(translate(code, mode), 20, rounding); + } + } + } + constexpr std::array code{0xbfa40000, 0xbf810000}; + static_cast(translate(code, std::nullopt)); + constexpr std::array nonzero{0xbfa40003, 0xbf810000}; + expectRejected(nonzero, ShaderFloatMode{0xc0}, "s_round_mode"); +} + +void verifyUnsupportedReads() { + for (const auto instruction : {0xb9140002u, 0xb9140101u, 0xb91408c1u, 0xb914f801u}) { + const std::array code{instruction, 0xbf810000}; + expectRejected(code, ShaderFloatMode{0xc0}, "s_getreg_b32"); + } +} + +void verifyBranch() { + for (std::uint32_t rounding = 0; rounding < 16; ++rounding) { + const ShaderFloatMode mode{0xc0u | rounding}; + std::array code{0xbf840002, 0xba801801, rounding, 0xb9141801, 0xbf810000}; + expectRegister(translate(code, mode), 20, rounding); + code[2] ^= 1u; + expectRejected(code, mode, "s_setreg_imm32_b32", 4); + } +} + +void verifyRecompilePropagation() { + constexpr std::array code{0xbfa40005, 0xbf810000}; + constexpr std::array capabilities{1, 61}; + RecompileRequest request{}; + request.shader = {ShaderStage::Compute, 0x20000, code, 0, {}}; + request.context.waveSize = 64; + request.context.compute = ShaderComputeStageInfo{{1, 1, 1}, 0, {false, false, false}, false, 0}; + request.context.floatMode = ShaderFloatMode{0xc5}; + request.target.vulkanVersion = 0x00403000; + request.target.spirvVersion = 0x00010600; + request.target.subgroupSize = 64; + request.target.bdaAbiVersion = BdaAbi::Version; + request.target.supportedCapabilities = capabilities; + request.target.maxWorkgroupSize = {1024, 1024, 64}; + request.target.maxWorkgroupInvocations = 1024; + request.layout = {0, 0, 0, 128}; + request.useCache = false; + require(!Recompile(request).spirv.empty(), "declared MODE did not reach full recompilation"); + request.context.floatMode.reset(); + try { + static_cast(Recompile(request)); + } catch (const std::runtime_error& error) { + require(std::string(error.what()).find("s_round_mode") != std::string::npos, "unexpected full recompilation failure"); + return; + } + throw std::runtime_error("full recompilation ignored a runtime MODE change"); +} + +} + +int main() { + int failures = 0; + for (const auto check : {verifyReads, verifyDynamicWrites, verifyImmediateWrites, verifyRoundInstruction, verifyUnsupportedReads, verifyBranch, verifyRecompilePropagation}) { + try { + check(); + } catch (const std::exception& error) { + std::fprintf(stderr, "%s\n", error.what()); + ++failures; + } + } + if (failures != 0) return 1; + std::puts("Shader MODE register tests passed"); + return 0; +} diff --git a/core/libs/prx/libSceAgcDriver/tests/SharedWrites.cpp b/core/libs/prx/libSceAgcDriver/tests/SharedWrites.cpp new file mode 100644 index 000000000..895cbc8de --- /dev/null +++ b/core/libs/prx/libSceAgcDriver/tests/SharedWrites.cpp @@ -0,0 +1,227 @@ +#include "prx/libSceAgcDriver/Execution/include/GuestMemory.hpp" +#include "prx/libc/include/GuestWriteWatch.hpp" +#include "prx/libc/include/GuestArena.hpp" +#include "prx/libc/include/general/VabiMacros.hpp" +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#ifndef _WIN32 +#include +#endif + +extern "C" { +int APS5_VABI sceKernelAllocateDirectMemory(std::int64_t, std::int64_t, std::size_t, std::size_t, int, std::int64_t*); +int APS5_VABI sceKernelMapDirectMemory(void**, std::size_t, int, int, std::int64_t, std::size_t); +int APS5_VABI sceKernelReleaseDirectMemory(std::int64_t, std::size_t); +int APS5_VABI sceKernelMunmap(void*, std::size_t); +int APS5_VABI sceKernelMprotect(const void*, std::size_t, int); +} + +namespace { +using namespace AgcDriver::GuestMemory; +void require(bool condition, const char* reason) { if (!condition) throw std::runtime_error(reason); } +struct Mapping { + std::size_t size; + std::int64_t physical = -1; + void* data = nullptr; + explicit Mapping(std::size_t size) : size(size) { + require(sceKernelAllocateDirectMemory(0, 0x10000000000ll, size, 65536, 3, &physical) == 0, "allocate direct memory failed"); + if (sceKernelMapDirectMemory(&data, size, 3, 0, physical, 65536) != 0) { + sceKernelReleaseDirectMemory(physical, size); + throw std::runtime_error("map direct memory failed"); + } + } + Mapping(const Mapping&) = delete; + Mapping& operator=(const Mapping&) = delete; + ~Mapping() { + if (data != nullptr) sceKernelMunmap(data, size); + if (physical >= 0) sceKernelReleaseDirectMemory(physical, size); + } + std::uint64_t Address() const { return reinterpret_cast(data); } +}; + +void benchmark(bool labels = false, bool backing = true) { + require(WriteWatched(), "write tracking unavailable for benchmark"); + std::thread([] {}).join(); + Mapping mapping(2u << 20u); +#ifndef _WIN32 + if (!backing) GuestArena::GuestArenaSetSharedBackingWriter_nid_no_patch(nullptr); +#else + static_cast(backing); +#endif + std::vector source(1u << 20u, std::byte{17}); + const auto sizes = labels ? std::array{4, 8, 16} : std::array{4096, 65536, 1048576}; + for (const auto bytes : sizes) { + std::vector times; + std::uint64_t faults = 0; + for (unsigned pass = 0; pass < 10; ++pass) { +#ifndef _WIN32 + rusage before{}, after{}; + getrusage(RUSAGE_THREAD, &before); +#endif + const auto start = std::chrono::steady_clock::now(); + for (unsigned i = 0; i < 256; ++i) { + source[0] = static_cast(i); + BumpCollectEpoch(); + if (labels) { + CheckRange(mapping.data, bytes, 4, true); + StoreOwnBytes(mapping.Address(), std::span(source).first(bytes)); + } else Write(mapping.Address(), std::span(source).first(bytes), 1); + } + const auto us = std::chrono::duration(std::chrono::steady_clock::now() - start).count() / 256; +#ifndef _WIN32 + getrusage(RUSAGE_THREAD, &after); + if (pass != 0) faults += after.ru_minflt - before.ru_minflt; +#endif + if (pass != 0) times.push_back(us); + require(std::memcmp(mapping.data, source.data(), bytes) == 0, "copied data differs"); + } + std::sort(times.begin(), times.end()); + std::cout << "bytes=" << bytes << " median_us=" << times[4] << " faults=" << faults << " operations=2304\n"; + } +} + +void smallStores() { + Mapping mapping(2u << 16u); + auto* data = static_cast(mapping.data); + std::memset(data, 19, mapping.size); + std::array source; + source.fill(std::byte{83}); + for (const auto offset : {4u, 4092u, 65532u}) for (const auto size : {4u, 8u, 16u}) { + const auto address = mapping.Address() + offset; + const auto before = CollectWritesUncached(address, size); + const auto stored = StoreOwnBytes(address, std::span(source).first(size)); + require(std::memcmp(data + offset, source.data(), size) == 0, "small backing store differs"); + require(data[offset - 1] == std::byte{19} && data[offset + 16] == std::byte{19}, "small store changed neighboring bytes"); + if (before != 0) { + require(stored > before && StoredOver(address, size, before), "small driver store was not stamped"); + require(UnchangedSinceCollected(address, size, before), "driver store was classified as a guest write"); + data[offset] = std::byte{97}; + StoreOwnBytes(address, std::span(source).first(size)); + require(!UnchangedSinceCollected(address, size, stored), "guest write before a driver store was lost"); + const auto after = CollectWritesUncached(address, size); + data[offset] = std::byte{101}; + require(!UnchangedSinceCollected(address, size, after), "guest write after a driver store was lost"); + } + } + std::array ordinary{}; + StoreOwnBytes(reinterpret_cast(ordinary.data()), source); + require(ordinary == source, "small ordinary store fallback differs"); + require(StoreOwnBytes(0, std::span{}) == 0, "empty store produced a version"); +} + +template void fails(TAction&& fn) { + bool failed = false; + try { fn(); } catch (const std::runtime_error&) { failed = true; } + require(failed, "invalid write was accepted"); +} + +#ifndef _WIN32 +std::size_t directMappings() { + std::ifstream maps("/proc/self/maps"); + require(maps.is_open(), "cannot inspect mapping lifetimes"); + std::size_t count = 0; + for (std::string line; std::getline(maps, line);) if (line.find("/memfd:direct memory") != std::string::npos) ++count; + return count; +} +#endif + +void run() { + constexpr std::size_t part = 65536; + Mapping mapping(2u << 20u); + auto* bytes = static_cast(mapping.data); + std::memset(bytes, 19, mapping.size); + std::vector source(part, std::byte{73}); + const auto before = CollectWritesUncached(mapping.Address(), mapping.size); + Write(mapping.Address() + 123, source, 1); + require(std::memcmp(bytes + 123, source.data(), source.size()) == 0, "shared write differs"); + require(bytes[122] == std::byte{19} && bytes[123 + part] == std::byte{19}, "write changed neighboring bytes"); + if (before != 0) { + require(!UnchangedSince(mapping.Address(), mapping.size, before), "driver write did not advance versions"); + require(Watched(mapping.Address(), mapping.size), "host view disabled guest tracking"); + const auto after = CollectWritesUncached(mapping.Address(), mapping.size); + require(UnchangedSince(mapping.Address(), mapping.size, after), "fresh write version is stale"); + bytes[part + 456] = std::byte{91}; + CollectWritesUncached(mapping.Address(), mapping.size); + require(!UnchangedSince(mapping.Address(), mapping.size, after), "subsequent guest write was lost"); +#ifndef _WIN32 + std::size_t dirty = 0; + const auto collect = [&] { + return GuestWriteWatch::GuestWriteWatchCollect_nid_postfix(mapping.Address(), mapping.size, + [](void* p, std::uintptr_t begin, std::uintptr_t end) { *static_cast(p) += end - begin; }, &dirty); + }; + require(collect(), "guest mapping is no longer watched"); + dirty = 0; + require(GuestArena::GuestArenaWriteSharedBacking_nid_no_patch(mapping.Address(), source.data(), source.size()), "shared writer rejected direct backing"); + require(collect() && dirty == 0, "host write dirtied the tracked guest mapping"); + bytes[part * 4] = std::byte{31}; + require(collect() && dirty == 4096, "guest write tracking was disarmed"); +#endif + } + require(sceKernelMprotect(mapping.data, part, 1) == 0, "read-only protection failed"); + fails([&] { Write(mapping.Address(), source, 1); }); + require(sceKernelMprotect(mapping.data, part, 3) == 0, "write protection restore failed"); + std::vector ordinary(part); + Write(reinterpret_cast(ordinary.data()), source, 1); + require(ordinary == source, "ordinary memory fallback failed"); + { + Mapping replacement(part); + void* originalAlias = nullptr; + require(sceKernelMapDirectMemory(&originalAlias, part, 3, 0, mapping.physical + part, part) == 0, "physical alias failed"); + std::memset(originalAlias, 29, part); + void* middle = bytes + part; + require(sceKernelMapDirectMemory(&middle, part, 3, 0x10, replacement.physical, part) == 0, "fixed backing replacement failed"); + std::vector across(part * 3); + for (std::size_t i = 0; i < across.size(); ++i) across[i] = static_cast((i / part) + 41); + Write(mapping.Address(), across, 1); + require(std::memcmp(mapping.data, across.data(), across.size()) == 0, "write across backing boundaries differs"); + require(std::memcmp(replacement.data, across.data() + part, part) == 0, "physical alias did not see the write"); + require(std::all_of(static_cast(originalAlias), static_cast(originalAlias) + part, [](auto x) { return x == std::byte{29}; }), "replaced virtual range wrote the old backing"); +#ifndef _WIN32 + require(sceKernelMunmap(middle, part) == 0, "partial unmap failed"); + const auto saved = bytes[0]; + across[0] = std::byte{99}; + require(!GuestArena::GuestArenaWriteSharedBacking_nid_no_patch(mapping.Address(), across.data(), across.size()), "writer accepted an unmapped gap"); + require(bytes[0] == saved, "failed shared write changed its prefix"); +#endif + require(sceKernelMapDirectMemory(&middle, part, 3, 0x10, mapping.physical + part, part) == 0, "original backing restore failed"); + require(sceKernelMunmap(originalAlias, part) == 0, "alias unmap failed"); + } + std::vector threads; + for (unsigned i = 0; i < 8; ++i) threads.emplace_back([&, i] { + std::vector input(part, static_cast(i + 57)); + for (unsigned repeat = 0; repeat < 32; ++repeat) Write(mapping.Address() + i * part, input, 1); + }); + for (auto& thread : threads) thread.join(); + for (unsigned i = 0; i < 8; ++i) require(std::all_of(bytes + i * part, bytes + (i + 1) * part, [i](auto x) { return x == static_cast(i + 57); }), "concurrent write differs"); +} + +} + +int main(int argc, char** argv) { + try { + if (argc == 2 && std::string_view(argv[1]) == "--benchmark") benchmark(); + else if (argc == 2 && std::string_view(argv[1]) == "--benchmark-labels") benchmark(true); + else if (argc == 2 && std::string_view(argv[1]) == "--benchmark-fallback") benchmark(false, false); + else if (argc == 2 && std::string_view(argv[1]) == "--benchmark-labels-fallback") benchmark(true, false); + else { +#ifndef _WIN32 + const auto before = directMappings(); +#endif + run(); + smallStores(); +#ifndef _WIN32 + require(directMappings() == before, "physical backing write views leaked"); +#endif + std::cout << "Shared backing write tests passed\n"; + } + } catch (const std::exception& error) { std::cerr << error.what() << '\n'; return 1; } +} diff --git a/core/libs/prx/libSceAgcDriver/tests/Structurizer.cpp b/core/libs/prx/libSceAgcDriver/tests/Structurizer.cpp index bd62650a5..d6dfb7861 100644 --- a/core/libs/prx/libSceAgcDriver/tests/Structurizer.cpp +++ b/core/libs/prx/libSceAgcDriver/tests/Structurizer.cpp @@ -2,6 +2,7 @@ #include #include #include +#include #include #include #include @@ -74,12 +75,144 @@ void requireStructuredBranches(const ControlFlowGraph& graph, const char* name) } } +void requireExactPostDominators(const ControlFlowGraph& graph, const char* name) { + const auto count = static_cast(graph.blocks.size()); + const auto reachesExitAvoiding = [&](std::uint32_t from, std::uint32_t avoided) { + std::vector seen(count, false); + std::vector stack{from}; + seen[from] = true; + while (!stack.empty()) { + const auto id = stack.back(); + stack.pop_back(); + if (graph.blocks[id].successors.empty()) return true; + for (const auto successor : graph.blocks[id].successors) { + if (successor == avoided || seen[successor]) continue; + seen[successor] = true; + stack.push_back(successor); + } + } + return false; + }; + for (const auto& block : graph.blocks) { + std::vector expected; + for (std::uint32_t candidate = 0; candidate < count; ++candidate) { + if (candidate == block.id || !reachesExitAvoiding(block.id, candidate)) expected.push_back(candidate); + } + if (block.postDominators != expected) { + throw std::runtime_error(std::string(name) + ": block " + std::to_string(block.id) + " has " + std::to_string(block.postDominators.size()) + " post-dominators, expected " + std::to_string(expected.size())); + } + } +} + +std::vector> originalPostDominators(const ControlFlowGraph& graph) { + std::vector all; + for (std::uint32_t id = 0; id < graph.blocks.size(); ++id) all.push_back(id); + std::vector> result; + for (const auto& block : graph.blocks) result.push_back(block.successors.empty() ? std::vector{block.id} : all); + bool changed = true; + while (changed) { + changed = false; + for (const auto& block : graph.blocks) { + std::vector next; + if (block.successors.empty()) { + next = {block.id}; + } else { + next = result[block.successors.front()]; + for (std::size_t index = 1; index < block.successors.size(); ++index) { + std::vector intersection; + const auto& successor = result[block.successors[index]]; + std::set_intersection(next.begin(), next.end(), successor.begin(), successor.end(), std::back_inserter(intersection)); + next = std::move(intersection); + } + if (std::find(next.begin(), next.end(), block.id) == next.end()) next.push_back(block.id); + std::sort(next.begin(), next.end()); + next.erase(std::unique(next.begin(), next.end()), next.end()); + } + if (next != result[block.id]) { + result[block.id] = std::move(next); + changed = true; + } + } + } + return result; +} + +std::vector> originalDominators(const ControlFlowGraph& graph) { + std::vector all; + for (std::uint32_t id = 0; id < graph.blocks.size(); ++id) all.push_back(id); + std::vector> result; + for (const auto& block : graph.blocks) result.push_back(block.id == graph.entryBlock ? std::vector{block.id} : all); + bool changed = true; + while (changed) { + changed = false; + for (const auto& block : graph.blocks) { + if (block.id == graph.entryBlock) continue; + std::vector next; + if (block.predecessors.empty()) { + next = {block.id}; + } else { + next = result[block.predecessors.front()]; + for (std::size_t index = 1; index < block.predecessors.size(); ++index) { + std::vector intersection; + const auto& predecessor = result[block.predecessors[index]]; + std::set_intersection(next.begin(), next.end(), predecessor.begin(), predecessor.end(), std::back_inserter(intersection)); + next = std::move(intersection); + } + if (std::find(next.begin(), next.end(), block.id) == next.end()) next.push_back(block.id); + std::sort(next.begin(), next.end()); + next.erase(std::unique(next.begin(), next.end()), next.end()); + } + if (next != result[block.id]) { + result[block.id] = std::move(next); + changed = true; + } + } + } + return result; +} + +void requireOriginalDominanceSets(ControlFlowGraph graph, const char* name) { + for (auto& block : graph.blocks) { + block.dominators.clear(); + block.postDominators.clear(); + } + const auto expected = originalPostDominators(graph); + const auto expectedDominators = originalDominators(graph); + const auto original = graph; + const std::string stopAfterAnalyses = "dominance differential fixture: stop before CFG rewrites"; + graph.unsupported = true; + graph.unsupportedReason = stopAfterAnalyses; + bool stopped = false; + try { + Structurizer{}.Structurize(graph); + } catch (const std::runtime_error& error) { + if (error.what() != stopAfterAnalyses) throw; + stopped = true; + } + if (!stopped || graph.blocks.size() != original.blocks.size() || graph.entryBlock != original.entryBlock) { + throw std::runtime_error(std::string(name) + ": differential fixture did not stop before CFG rewrites"); + } + for (std::size_t id = 0; id < graph.blocks.size(); ++id) { + const auto& block = graph.blocks[id]; + if (block.id != original.blocks[id].id || block.successors != original.blocks[id].successors || block.predecessors != original.blocks[id].predecessors) { + throw std::runtime_error(std::string(name) + ": differential fixture changed the CFG"); + } + if (block.postDominators != expected[id]) { + throw std::runtime_error(std::string(name) + ": block " + std::to_string(id) + " differs from the original post-dominator fixed point"); + } + if (block.dominators != expectedDominators[id]) { + throw std::runtime_error(std::string(name) + ": block " + std::to_string(id) + " differs from the original dominator fixed point"); + } + } +} + } int main() { try { auto nested = makeGraph({{1}, {2}, {5, 3}, {5, 4}, {7}, {6, 7}, {}, {1}}); Structurizer{}.Structurize(nested); + requireExactPostDominators(nested, "nested selections"); if (nested.FindBlock(2).terminator.mergeBlock == nested.FindBlock(3).terminator.mergeBlock) { std::fprintf(stderr, "the nested selections share merge block %u\n", nested.FindBlock(2).terminator.mergeBlock); return 1; @@ -87,21 +220,27 @@ int main() { requireStructuredBranches(nested, "nested selections"); auto exitTail = makeGraph({{1}, {2}, {3, 4}, {6}, {6, 5}, {1}, {}}); Structurizer{}.Structurize(exitTail); + requireExactPostDominators(exitTail, "a loop exit through a tail block"); requireStructuredBranches(exitTail, "a loop exit through a tail block"); auto exitTails = makeGraph({{1}, {2}, {3, 4}, {7}, {5, 6}, {7, 1}, {7}, {}}); Structurizer{}.Structurize(exitTails); + requireExactPostDominators(exitTails, "a loop with two exit tails"); requireStructuredBranches(exitTails, "a loop with two exit tails"); auto endingExits = makeGraph({{6, 1}, {2}, {3}, {6, 4}, {2, 5}, {}, {}}); Structurizer{}.Structurize(endingExits); + requireExactPostDominators(endingExits, "a loop whose two exits end the program"); requireStructuredBranches(endingExits, "a loop whose two exits end the program"); auto threeEndingExits = makeGraph({{1}, {2}, {5, 3}, {6, 4}, {1, 7}, {}, {}, {}}); Structurizer{}.Structurize(threeEndingExits); + requireExactPostDominators(threeEndingExits, "a loop whose three exits end the program"); requireStructuredBranches(threeEndingExits, "a loop whose three exits end the program"); auto innerEndingExit = makeGraph({{1}, {2}, {3}, {7, 4}, {2, 5}, {1, 6}, {}, {}}); Structurizer{}.Structurize(innerEndingExit); + requireExactPostDominators(innerEndingExit, "an inner loop exit that ends the program"); requireStructuredBranches(innerEndingExit, "an inner loop exit that ends the program"); auto earlyReturn = makeGraph({{2, 1}, {4, 2}, {3, 5}, {5}, {}, {}}); Structurizer{}.Structurize(earlyReturn); + requireExactPostDominators(earlyReturn, "an early return inside a selection that joins its parent's merge"); requireStructuredBranches(earlyReturn, "an early return inside a selection that joins its parent's merge"); for (const auto& block : earlyReturn.blocks) { const auto merge = block.terminator.mergeBlock; @@ -110,6 +249,33 @@ int main() { return 1; } } + std::vector> diamonds; + for (std::uint32_t diamond = 0; diamond < 50u; ++diamond) { + const auto head = diamond * 3u; + diamonds.push_back({head + 1u, head + 2u}); + diamonds.push_back({head + 3u}); + diamonds.push_back({head + 3u}); + } + diamonds.push_back({}); + auto chain = makeGraph(diamonds); + Structurizer{}.Structurize(chain); + requireExactPostDominators(chain, "a chain of fifty selections"); + auto sharedDiscard = makeGraph({{5, 1}, {4, 2}, {7, 3}, {7, 4}, {5}, {7, 6}, {}, {}}); + const std::vector sharedDiscardWords{444, 1944, 68, 1128, 8, 2084, 148, 28}; + for (auto& block : sharedDiscard.blocks) { + std::sort(block.successors.begin(), block.successors.end()); + block.estimatedSpirvWords = sharedDiscardWords[block.id]; + } + Structurizer{}.Structurize(sharedDiscard); + requireStructuredBranches(sharedDiscard, "early discards that share one ending block"); + requireOriginalDominanceSets(makeGraph(diamonds), "raw diamond chain"); + requireOriginalDominanceSets(makeGraph({{1}, {2, 3}, {1}, {}}), "a loop with an exit"); + requireOriginalDominanceSets(makeGraph({{1}, {2}, {1}}), "a non-exiting SCC"); + requireOriginalDominanceSets(makeGraph({{1, 3}, {2}, {1}, {}}), "an exit-or-spin branch"); + requireOriginalDominanceSets(makeGraph({{1}, {}, {3}, {}, {5}, {4}}), "unreachable exit and non-exiting islands"); + auto reordered = makeGraph({{1}, {}, {2}, {0, 4}, {1}}); + reordered.entryBlock = 3; + requireOriginalDominanceSets(std::move(reordered), "a nonzero entry with permuted block IDs"); } catch (const std::exception& error) { std::fprintf(stderr, "%s\n", error.what()); return 1; diff --git a/core/libs/prx/libSceAgcDriver/tests/SubgroupClock.cpp b/core/libs/prx/libSceAgcDriver/tests/SubgroupClock.cpp index 8adf998a9..0763da9a2 100644 --- a/core/libs/prx/libSceAgcDriver/tests/SubgroupClock.cpp +++ b/core/libs/prx/libSceAgcDriver/tests/SubgroupClock.cpp @@ -6,24 +6,31 @@ namespace { struct Case { VkDriverId driver; + std::uint32_t deviceId; std::string_view name; bool narrow; }; constexpr Case Cases[]{ - {VK_DRIVER_ID_MESA_RADV, "AMD Radeon RX 6750 XT (RADV NAVI22)", true}, - {VK_DRIVER_ID_MESA_RADV, "AMD Ryzen 5 7600X 6-Core Processor (RADV RAPHAEL_MENDOCINO)", true}, - {VK_DRIVER_ID_MESA_RADV, "AMD Radeon RX 7900 XTX (RADV NAVI31)", true}, - {VK_DRIVER_ID_MESA_RADV, "AMD Radeon 890M (RADV STRIX1)", true}, - {VK_DRIVER_ID_MESA_RADV, "AMD Radeon RX 9070 XT (RADV GFX1201)", false}, - {VK_DRIVER_ID_MESA_RADV, "AMD Radeon RX 5700 XT (RADV NAVI10)", false}, - {VK_DRIVER_ID_MESA_RADV, "AMD BC-250 (RADV GFX1013)", false}, - {VK_DRIVER_ID_MESA_RADV, "AMD Radeon RX 6750 XT (RADV NAVI22", false}, - {VK_DRIVER_ID_MESA_RADV, "AMD Radeon RX 6750 XT (RADV NAVI2)", false}, - {VK_DRIVER_ID_MESA_RADV, "", false}, - {VK_DRIVER_ID_AMD_PROPRIETARY, "AMD Radeon RX 6750 XT (RADV NAVI22)", false}, - {VK_DRIVER_ID_AMD_OPEN_SOURCE, "AMD Radeon RX 6750 XT", false}, - {VK_DRIVER_ID_NVIDIA_PROPRIETARY, "NVIDIA GeForce RTX 5070 Ti", false}, + {VK_DRIVER_ID_MESA_RADV, 0, "AMD Radeon RX 6750 XT (RADV NAVI22)", true}, + {VK_DRIVER_ID_MESA_RADV, 0, "AMD Ryzen 5 7600X 6-Core Processor (RADV RAPHAEL_MENDOCINO)", true}, + {VK_DRIVER_ID_MESA_RADV, 0, "AMD Radeon RX 7900 XTX (RADV NAVI31)", true}, + {VK_DRIVER_ID_MESA_RADV, 0, "AMD Radeon 890M (RADV STRIX1)", true}, + {VK_DRIVER_ID_MESA_RADV, 0, "AMD Radeon RX 9070 XT (RADV GFX1201)", false}, + {VK_DRIVER_ID_MESA_RADV, 0, "AMD Radeon RX 5700 XT (RADV NAVI10)", false}, + {VK_DRIVER_ID_MESA_RADV, 0, "AMD BC-250 (RADV GFX1013)", false}, + {VK_DRIVER_ID_MESA_RADV, 0, "AMD Radeon RX 6750 XT (RADV NAVI22", false}, + {VK_DRIVER_ID_MESA_RADV, 0, "AMD Radeon RX 6750 XT (RADV NAVI2)", false}, + {VK_DRIVER_ID_MESA_RADV, 0, "", false}, + {VK_DRIVER_ID_AMD_PROPRIETARY, 0x1114, "AMD Radeon 860M", true}, + {VK_DRIVER_ID_AMD_PROPRIETARY, 0x1115, "AMD Radeon 860M", false}, + {VK_DRIVER_ID_AMD_PROPRIETARY, 0x744c, "AMD Radeon RX 7900 XT", true}, + {VK_DRIVER_ID_AMD_PROPRIETARY, 0x744d, "AMD Radeon RX 7900 XT", false}, + {VK_DRIVER_ID_AMD_PROPRIETARY, 0x747e, "AMD Radeon RX 7800 XT", true}, + {VK_DRIVER_ID_AMD_PROPRIETARY, 0x747f, "AMD Radeon RX 7800 XT", false}, + {VK_DRIVER_ID_AMD_PROPRIETARY, 0, "AMD Radeon RX 6750 XT (RADV NAVI22)", false}, + {VK_DRIVER_ID_AMD_OPEN_SOURCE, 0x1114, "AMD Radeon 860M", false}, + {VK_DRIVER_ID_NVIDIA_PROPRIETARY, 0, "NVIDIA GeForce RTX 5070 Ti", false}, }; } @@ -31,8 +38,8 @@ constexpr Case Cases[]{ int main() { int failures = 0; for (const auto& item : Cases) { - if (AgcDriver::NarrowSubgroupClock(item.driver, item.name) != item.narrow) { - std::fprintf(stderr, "driver %d \"%.*s\": expected %s subgroup clock\n", static_cast(item.driver), static_cast(item.name.size()), item.name.data(), item.narrow ? "a narrow" : "a full"); + if (AgcDriver::NarrowSubgroupClock(item.driver, item.deviceId, item.name) != item.narrow) { + std::fprintf(stderr, "driver %d device 0x%x \"%.*s\": expected %s subgroup clock\n", static_cast(item.driver), item.deviceId, static_cast(item.name.size()), item.name.data(), item.narrow ? "a narrow" : "a full"); ++failures; } } diff --git a/core/libs/prx/libSceAgcDriver/tests/Submit.cpp b/core/libs/prx/libSceAgcDriver/tests/Submit.cpp index e774ae46e..3c9db0b0c 100644 --- a/core/libs/prx/libSceAgcDriver/tests/Submit.cpp +++ b/core/libs/prx/libSceAgcDriver/tests/Submit.cpp @@ -481,6 +481,25 @@ void testShaderHeaderAlignment() { refused(expectFailure([&] { AgcDriverRegisterShader_nid_postfix(at(4, truncated)); }), "smaller than its fixed fields"); } +void testHeaderWithoutProgramAddress() { + alignas(256) static const std::array code{0xbf810000}; + struct Header { + Shader shader{}; + std::array registers{}; + }; + alignas(8) static Header header{}; + header.shader.file_header = 0x34333231; + header.shader.version = 0x18; + header.shader.header_size = sizeof(Header); + header.shader.shader_size = sizeof(code); + header.shader.code = code.data(); + header.shader.type = 4; + header.registers = {{{0x08a, 0x60000002}, {0x08b, 0x00030008}, {0x0ca, 0x03000002}}}; + header.shader.sh_registers = header.registers.data(); + header.shader.num_sh_registers = header.registers.size(); + AgcDriverRegisterShader_nid_postfix(&header.shader); +} + void testRegisteredFloatMode() { using AgcDriver::DriverDetail::RegisteredFloatMode; using AgcDriver::DriverDetail::ShaderSnapshot; @@ -577,6 +596,13 @@ int main() { "raw compute cache eviction lost snapshot lifetime or exceeded its entry limit"); check(!expectFailure([&] { AgcDriver::DriverDetail::ReadRawComputeShader(rawAddress + 4); }).empty(), "raw compute accepted a misaligned entry"); check(!expectFailure([] { AgcDriver::DriverDetail::ReadRawComputeShader(0); }).empty(), "raw compute accepted an unmapped entry"); + alignas(256) std::array straddling{}; + straddling.fill(0xbf800000); + straddling[63] = 0xf4000000; + straddling[64] = 0xfa000000; + straddling[65] = 0xbf810000; + const auto straddled = AgcDriver::DriverDetail::ReadRawComputeShader(reinterpret_cast(straddling.data())); + check(straddled->code.size() == 66 && straddled->code[64] == 0xfa000000, "raw compute stopped at a memory instruction across its first read window"); #ifdef _WIN32 auto* mapping = static_cast(VirtualAlloc(nullptr, 8192, MEM_RESERVE | MEM_COMMIT, PAGE_READWRITE)); check(mapping != nullptr, "cannot allocate raw compute boundary test"); @@ -616,6 +642,7 @@ int main() { testWaitFreeSubmissionTheCpuWaitsFor(); testMultiSubmissions(); testShaderHeaderAlignment(); + testHeaderWithoutProgramAddress(); testRegisteredFloatMode(); testWorkerFailure(); check(expectFailure([] { LibcRunShutdown_nid_postfix(); }).find("required shader register") != std::string::npos, "shutdown lost worker failure"); diff --git a/core/libs/prx/libSceAgcDriver/tests/TessellationFactorRing.cpp b/core/libs/prx/libSceAgcDriver/tests/TessellationFactorRing.cpp new file mode 100644 index 000000000..f302a0d11 --- /dev/null +++ b/core/libs/prx/libSceAgcDriver/tests/TessellationFactorRing.cpp @@ -0,0 +1,56 @@ +#include "SceTypes.hpp" +#include +#include +#include +#include + +extern "C" { +int APS5_VABI sceAgcDriverSetTFRing(const volatile void* base, uint32_t size); +int APS5_VABI sceAgcDriverGetTFRing(uintptr_t* base, uint32_t* size); +} + +namespace { + +constexpr uint32_t Canary = 0xa5a5a5a5; + +struct RingOutput { + uintptr_t base = ~uintptr_t{0}; + uint32_t size = 0; + uint32_t canary = Canary; +}; + +void Require(bool value) { + if (!value) std::abort(); +} + +template +bool Rejects(TAction action) { + try { action(); } + catch (const std::invalid_argument&) { return true; } + return false; +} + +void RequireRing(uintptr_t expectedBase, uint32_t expectedSize) { + RingOutput output; + Require(sceAgcDriverGetTFRing(&output.base, &output.size) == 0); + Require(output.base == expectedBase && output.size == expectedSize && output.canary == Canary); +} + +} + +int main() { + alignas(256) static uint8_t ring[0x20000]; + const auto ringBase = reinterpret_cast(ring); + RequireRing(0xff0000000, 0x20000); + Require(sceAgcDriverSetTFRing(ring, 0x1b000) == 0); + RequireRing(ringBase, 0x1b000); + std::thread([&] { RequireRing(ringBase, 0x1b000); }).join(); + std::thread([] { Require(sceAgcDriverSetTFRing(ring + 0x1000, 0x2000) == 0); }).join(); + RequireRing(ringBase + 0x1000, 0x2000); + Require(Rejects([] { sceAgcDriverSetTFRing(nullptr, 0x2000); })); + Require(Rejects([] { sceAgcDriverSetTFRing(ring, 0); })); + RequireRing(ringBase + 0x1000, 0x2000); + RingOutput output; + Require(Rejects([&] { sceAgcDriverGetTFRing(nullptr, &output.size); })); + Require(Rejects([&] { sceAgcDriverGetTFRing(&output.base, nullptr); })); +} diff --git a/core/libs/prx/libSceAgcDriver/tests/TextureDetiler.cpp b/core/libs/prx/libSceAgcDriver/tests/TextureDetiler.cpp index 02183e982..1b9c4784e 100644 --- a/core/libs/prx/libSceAgcDriver/tests/TextureDetiler.cpp +++ b/core/libs/prx/libSceAgcDriver/tests/TextureDetiler.cpp @@ -26,6 +26,7 @@ struct Push { std::uint32_t linearBase; std::uint32_t columnBegin; std::uint32_t rowBegin; + std::uint32_t pipeBankXor; }; Push decodePush(const std::vector& bytes) { @@ -165,6 +166,9 @@ void RunTextureDetilerTests(const Context& context, const TextureDetilerTestAcce detiler.Dispatch(commands, TextureTileMode::kD4KBX, 4, source, 0, destination, 0, layout, false, 0); const auto equationSpecialization = access.lastSpecialization(); + Require(decodePush(access.lastDispatch().pushConstants).pipeBankXor == 0u, "a dispatch without a window passed a pipe/bank XOR"); + detiler.Dispatch(commands, TextureTileMode::kD4KBX, 4, source, 0, destination, 0, layout, false, 0, false, {.pipeBankXor = 0xa00u}); + Require(decodePush(access.lastDispatch().pushConstants).pipeBankXor == 0xa00u, "the window's pipe/bank XOR did not reach the detiling shader"); Require(equationSpecialization[0] == 4 && equationSpecialization[1] == 4096 && equationSpecialization[2] == 2, "SW_4KB_D_X detiling must select the equation family over 4 KiB blocks"); reject([&] { detiler.Dispatch(VK_NULL_HANDLE, TextureTileMode::kStandard4KB, 4, source, 0, destination, 0, layout, 0); }, "active command buffer"); diff --git a/core/libs/prx/libSceAgcDriver/tests/TextureFormat.cpp b/core/libs/prx/libSceAgcDriver/tests/TextureFormat.cpp index 245766a02..d04cf6500 100644 --- a/core/libs/prx/libSceAgcDriver/tests/TextureFormat.cpp +++ b/core/libs/prx/libSceAgcDriver/tests/TextureFormat.cpp @@ -77,6 +77,7 @@ void convertedDccClearTests() { void RunTextureFormatTests() { srgbDecodeTests(); Require(ResolveTextureFormat(1) == VK_FORMAT_R8_UNORM, "format 1 must resolve to R8_UNORM"); + Require(ResolveTextureFormat(2) == VK_FORMAT_R8_SNORM, "format 2 must resolve to R8_SNORM"); Require(BytesPerElement(1) == 1u, "format 1 must be one byte wide"); Require(!IsBlockCompressed(1), "format 1 must not be block compressed"); Require(BlockWidth(1) == 1u && BlockHeight(1) == 1u, "format 1 must have a one-texel block"); @@ -114,7 +115,9 @@ void RunTextureFormatTests() { reject([] { ResolveTextureFormat(0); }, "unsupported guest texture format"); reject([] { ResolveTextureFormat(183); }, "unsupported guest texture format"); reject([] { ResolveTextureFormat(9999); }, "unsupported guest texture format"); - reject([] { BytesPerElement(2); }, "unsupported guest texture format"); + reject([] { BytesPerElement(3); }, "unsupported guest texture format"); reject([] { IsBlockCompressed(200); }, "unsupported guest texture format"); + for (const std::uint32_t format : {128u, 129u, 130u, 170u, 172u, 174u, 182u}) Require(IsSrgbTextureFormat(format), "guest format " + std::to_string(format) + " must be an sRGB texture format"); + for (const std::uint32_t format : {1u, 14u, 56u, 71u, 169u, 171u, 173u, 181u}) Require(!IsSrgbTextureFormat(format), "guest format " + std::to_string(format) + " must not be an sRGB texture format"); convertedDccClearTests(); } diff --git a/core/libs/prx/libSceAgcDriver/tests/TextureTiling.cpp b/core/libs/prx/libSceAgcDriver/tests/TextureTiling.cpp index 0db9767f1..c72b4f3b0 100644 --- a/core/libs/prx/libSceAgcDriver/tests/TextureTiling.cpp +++ b/core/libs/prx/libSceAgcDriver/tests/TextureTiling.cpp @@ -278,8 +278,20 @@ void RunTextureTilingTests() { requireThickAddresses(thickVolume(TextureTileMode::kS64KBX, 1, 100, 60, 70, 7), 1, 2, 1179648, xorSx8, "SW_64KB_S_X 8 bpp 100x60x70, 7 levels"); constexpr ElementAddress xorSx128[] = {{0, 0, 0, 0, 0x10000}, {0, 8, 4, 2, 0x18c80}, {0, 4, 1, 1, 0x11830}, {0, 3, 4, 0, 0x10a40}, {0, 8, 0, 2, 0x18480}, {1, 0, 0, 0, 0x8400}, {1, 3, 1, 0, 0x8660}, {1, 2, 0, 0, 0x8600}, {1, 1, 1, 0, 0x8460}, {1, 3, 0, 0, 0x8640}, {2, 0, 0, 0, 0x4400}, {2, 1, 0, 0, 0x4440}, {2, 1, 0, 0, 0x4440}, {2, 0, 0, 0, 0x4400}, {2, 1, 0, 0, 0x4440}, {3, 0, 0, 0, 0x1800}, {3, 0, 0, 0, 0x1800}, {3, 0, 0, 0, 0x1800}, {3, 0, 0, 0, 0x1800}, {3, 0, 0, 0, 0x1800}}; requireThickAddresses(thickVolume(TextureTileMode::kS64KBX, 77, 9, 5, 3, 4), 16, 1, 131072, xorSx128, "SW_64KB_S_X 128 bpp 9x5x3, 4 levels"); + { + const auto linear = DescribeSurface(thickVolume(TextureTileMode::kLinear, 1, 100, 60, 8, 4)); + constexpr std::uint64_t offsets[] = {0x3500u, 0x1700u, 0x800u, 0u}; + Require(!linear.thick && linear.layerBytes == 28928u && linear.guestBytes == 231424u, "linear 8 bpp 100x60x8, 4 levels: slice size differs from addrlib"); + for (std::uint32_t level = 0; level < 4u; ++level) Require(linear.mips[level].tiledOffset == offsets[level], "linear 8 bpp 100x60x8, 4 levels: mip " + std::to_string(level) + " offset differs from addrlib"); + } constexpr ElementAddress xorZx64[] = {{0, 0, 0, 0, 0x8400}, {0, 32, 19, 11, 0xbc350}, {0, 16, 6, 6, 0x69040}, {0, 11, 19, 0, 0xe778}, {0, 32, 0, 11, 0xb8100}, {1, 0, 0, 0, 0x400}, {1, 15, 9, 5, 0x52eb8}, {1, 8, 3, 3, 0x32950}, {1, 5, 9, 0, 0x598}, {1, 15, 0, 5, 0x52fa8}, {2, 0, 0, 0, 0x800}, {2, 7, 4, 2, 0x21ca8}, {2, 4, 1, 1, 0x10090}, {2, 2, 4, 0, 0x1820}, {2, 7, 0, 2, 0x20ca8}, {3, 0, 0, 0, 0x4200}, {3, 3, 1, 0, 0x4238}, {3, 2, 0, 0, 0x4220}, {3, 1, 1, 0, 0x4218}, {3, 3, 0, 0, 0x4228}, {4, 0, 0, 0, 0x200}, {4, 1, 0, 0, 0x208}, {4, 1, 0, 0, 0x208}, {4, 0, 0, 0, 0x200}, {4, 1, 0, 0, 0x208}, {5, 0, 0, 0, 0x2000}, {5, 0, 0, 0, 0x2000}, {5, 0, 0, 0, 0x2000}, {5, 0, 0, 0, 0x2000}, {5, 0, 0, 0, 0x2000}}; requireSliceAddresses(thickVolume(TextureTileMode::kZ64KBX, 71, 33, 20, 12, 6), 8, 786432, xorZx64, "thin SW_64KB_Z_X 64 bpp 33x20x12, 6 levels"); + constexpr ElementAddress xorDx32[] = {{0, 0, 0, 0, 0x0}, {0, 63, 63, 15, 0x3f0fc}, {0, 47, 4, 9, 0x13134}, {0, 54, 3, 3, 0x19af8}, {0, 23, 39, 12, 0x2ba5c}, {0, 57, 14, 7, 0x1c6e4}}; + requireThickAddresses(thickVolume(TextureTileMode::kD64KBX, 56, 64, 64, 16, 1), 4, 1, 262144, xorDx32, "SW_64KB_D_X 32 bpp 64x64x16, 1 level"); + constexpr ElementAddress xorDx8[] = {{0, 0, 0, 0, 0x20000}, {0, 99, 59, 69, 0x112b9f}, {0, 15, 52, 41, 0xa1347}, {0, 86, 43, 67, 0x11637e}, {0, 87, 59, 38, 0xb61fb}, {1, 0, 0, 0, 0x10000}, {1, 49, 29, 34, 0x76d29}, {1, 11, 28, 6, 0x126a3}, {1, 8, 27, 2, 0x12238}, {1, 30, 6, 17, 0x1c756}, {2, 0, 0, 0, 0x800}, {2, 24, 14, 16, 0xee10}, {2, 8, 3, 12, 0x1998}, {2, 15, 5, 9, 0x1d4f}, {2, 8, 8, 11, 0x3824}, {3, 0, 0, 0, 0x200}, {3, 11, 6, 7, 0x7b7}, {3, 7, 3, 6, 0x2fb}, {3, 3, 5, 4, 0x68b}, {3, 7, 2, 0, 0x253}, {4, 0, 0, 0, 0x4200}, {4, 5, 2, 3, 0x4275}, {4, 0, 2, 1, 0x4214}, {4, 2, 1, 1, 0x420e}, {4, 0, 2, 0, 0x4210}, {5, 0, 0, 0, 0x2000}, {5, 2, 0, 1, 0x2006}, {5, 1, 0, 0, 0x2001}, {6, 0, 0, 0, 0x2500}}; + requireThickAddresses(thickVolume(TextureTileMode::kD64KBX, 1, 100, 60, 70, 7), 1, 2, 1179648, xorDx8, "SW_64KB_D_X 8 bpp 100x60x70, 7 levels"); + constexpr ElementAddress xorDx128[] = {{0, 0, 0, 0, 0x10000}, {0, 8, 4, 2, 0x1c180}, {0, 0, 2, 1, 0x10820}, {0, 2, 3, 0, 0x10a40}, {0, 8, 4, 1, 0x1c120}, {1, 0, 0, 0, 0x8100}, {1, 3, 1, 0, 0x8350}, {1, 1, 0, 0, 0x8110}, {1, 0, 1, 0, 0x8140}, {1, 2, 0, 0, 0x8300}, {2, 0, 0, 0, 0x100}, {2, 1, 0, 0, 0x110}, {3, 0, 0, 0, 0x1000}}; + requireThickAddresses(thickVolume(TextureTileMode::kD64KBX, 77, 9, 5, 3, 4), 16, 1, 131072, xorDx128, "SW_64KB_D_X 128 bpp 9x5x3, 4 levels"); } { @@ -297,6 +309,10 @@ void RunTextureTilingTests() { requireThinAddresses(TextureTileMode::kS4KBX, 1, 100, 70, 6, 3, 73728, 24576, atS4KBX, "swizzle mode 21, 1 bytes, 100x70, 6 levels, 3 slices"); constexpr ElementAddress atD4KBX[] = {{0, 0, 0, 0, 0x2000}, {0, 32, 19, 0, 0x3030}, {0, 16, 6, 0, 0x2860}, {0, 32, 0, 0, 0x3400}, {0, 0, 19, 0, 0x2430}, {0, 0, 0, 1, 0x6800}, {0, 32, 19, 1, 0x7830}, {0, 16, 6, 1, 0x6060}, {0, 32, 0, 1, 0x7c00}, {0, 0, 19, 1, 0x6c30}, {1, 0, 0, 0, 0x1000}, {1, 15, 9, 0, 0x139c}, {1, 8, 3, 0, 0x1230}, {1, 15, 0, 0, 0x128c}, {1, 0, 9, 0, 0x1110}, {1, 0, 0, 1, 0x5800}, {1, 15, 9, 1, 0x5b9c}, {1, 8, 3, 1, 0x5a30}, {1, 15, 0, 1, 0x5a8c}, {1, 0, 9, 1, 0x5910}, {2, 0, 0, 0, 0x800}, {2, 7, 4, 0, 0x8cc}, {2, 4, 1, 0, 0x890}, {2, 7, 0, 0, 0x88c}, {2, 0, 4, 0, 0x840}, {2, 0, 0, 1, 0x4000}, {2, 7, 4, 1, 0x40cc}, {2, 4, 1, 1, 0x4090}, {2, 7, 0, 1, 0x408c}, {2, 0, 4, 1, 0x4040}, {3, 0, 0, 0, 0x600}, {3, 3, 1, 0, 0x61c}, {3, 2, 0, 0, 0x608}, {3, 3, 0, 0, 0x60c}, {3, 0, 1, 0, 0x610}, {3, 0, 0, 1, 0x4e00}, {3, 3, 1, 1, 0x4e1c}, {3, 2, 0, 1, 0x4e08}, {3, 3, 0, 1, 0x4e0c}, {3, 0, 1, 1, 0x4e10}}; requireThinAddresses(TextureTileMode::kD4KBX, 56, 33, 20, 4, 2, 32768, 16384, atD4KBX, "swizzle mode 22, 4 bytes, 33x20, 4 levels, 2 slices"); + constexpr ElementAddress atZ64KBX8[] = {{0, 0, 0, 0, 0x20000}, {1, 0, 0, 0, 0x10000}}; + requireThinAddresses(TextureTileMode::kZ64KBX, 1, 256, 256, 9, 1, 196608, 196608, atZ64KBX8, "swizzle mode 24, 1 bytes, 256x256, 9 levels, 1 slices"); + constexpr ElementAddress atZ64KBX16[] = {{0, 0, 0, 0, 0x10000}}; + requireThinAddresses(TextureTileMode::kZ64KBX, 7, 128, 128, 8, 1, 131072, 131072, atZ64KBX16, "swizzle mode 24, 2 bytes, 128x128, 8 levels, 1 slices"); } { diff --git a/core/libs/prx/libSceAgcDriver/tests/Visual.cpp b/core/libs/prx/libSceAgcDriver/tests/Visual.cpp index 8e9d5d0ca..4e94aa248 100644 --- a/core/libs/prx/libSceAgcDriver/tests/Visual.cpp +++ b/core/libs/prx/libSceAgcDriver/tests/Visual.cpp @@ -61,7 +61,8 @@ void Run(SDL_Window* window, const std::filesystem::path& directory, bool verify } AgcDriver::Graphics::State state{}; state.stages.path = AgcDriver::Graphics::ShaderPath::Vertex; - state.color = {reinterpret_cast(Pixels.data()), {Width, Height}, VK_FORMAT_R8G8B8A8_UNORM, Pixels.size()}; + state.color = {reinterpret_cast(Pixels.data()), {Width, Height}, VK_FORMAT_R8G8B8A8_UNORM, Pixels.size(), 0xe4u}; + state.colors = {state.color}; state.hasColorTarget = true; state.renderExtent = {Width, Height}; state.topology = VK_PRIMITIVE_TOPOLOGY_TRIANGLE_LIST; @@ -69,9 +70,11 @@ void Run(SDL_Window* window, const std::filesystem::path& directory, bool verify state.scissor = {{0, 0}, {Width, Height}}; state.frontFace = VK_FRONT_FACE_COUNTER_CLOCKWISE; state.blend.colorWriteMask = 15; - const std::array indices{0, 1, 2}; + state.blends = {state.blend}; + static constexpr std::array indices{0, 1, 2}; const AgcDriver::Pm4::DrawParameters draw{reinterpret_cast(indices.data()), 3, 2, 1, 0}; device.Draw(state, draw, shaders); + device.WaitIdle(); const std::vector indexedPixels(Pixels.begin(), Pixels.end()); for (std::size_t i = 0; i < Pixels.size(); i += 4) { Pixels[i] = std::byte{16}; @@ -81,6 +84,7 @@ void Run(SDL_Window* window, const std::filesystem::path& directory, bool verify } const AgcDriver::Pm4::DrawParameters autoDraw{0, 3, 0, 1, 0, false}; device.Draw(state, autoDraw, shaders); + device.WaitIdle(); Require(std::equal(Pixels.begin(), Pixels.end(), indexedPixels.begin()), "GPU readback: auto draw differs from indexed triangle"); const auto center = (Height / 2 * Width + Width / 2) * 4; Require(std::to_integer(Pixels[center]) > 30 && std::to_integer(Pixels[center + 1]) > 30 && std::to_integer(Pixels[center + 2]) > 30, "GPU readback: triangle center was not rendered"); diff --git a/core/libs/prx/libSceAgcDriver/tests/Vop2EncodingDecode.cpp b/core/libs/prx/libSceAgcDriver/tests/Vop2EncodingDecode.cpp new file mode 100644 index 000000000..83f4a3c52 --- /dev/null +++ b/core/libs/prx/libSceAgcDriver/tests/Vop2EncodingDecode.cpp @@ -0,0 +1,82 @@ +#include "RdnaDecoder/RdnaInstructionDecoder.hpp" +#include +#include +#include +#include + +using namespace ShaderRecompiler; + +namespace { + +struct Accepted { + std::vector words; + RdnaOpcode op; + const char* name; +}; + +struct Refused { + std::vector words; + const char* reason; + const char* name; +}; + +} + +int main() { + const std::vector accepted{ + {{0x2c040300u}, RdnaOpcode::VLshrrevB32, "v_lshrrev_b32"}, + {{0x6e040300u, 0x00003c00u}, RdnaOpcode::VFmamkF16, "v_fmamk_f16"}, + {{0x78040300u}, RdnaOpcode::VPkFmacF16, "v_pk_fmac_f16"}, + {{0xd5160002u, 0x00020300u}, RdnaOpcode::VLshrrevB32, "VOP3 v_lshrrev_b32"}, + {{0xd7640002u, 0x00020300u}, RdnaOpcode::VBcntU32B32, "VOP3 v_bcnt_u32_b32"}, + {{0xd7650002u, 0x00020300u}, RdnaOpcode::VMbcntLoU32B32, "VOP3 v_mbcnt_lo_u32_b32"}, + {{0xd7660002u, 0x00020300u}, RdnaOpcode::VMbcntHiU32B32, "VOP3 v_mbcnt_hi_u32_b32"}, + }; + const std::vector refused{ + {{0x2a040300u}, "VOP2 opcode is not implemented", "VOP2 opcode 0x15"}, + {{0x2e040300u}, "VOP2 opcode is not implemented", "VOP2 opcode 0x17"}, + {{0x32040300u}, "VOP2 opcode is not implemented", "VOP2 opcode 0x19"}, + {{0x44040300u}, "VOP2 opcode is not implemented", "VOP2 opcode 0x22"}, + {{0x46040300u}, "VOP2 opcode is not implemented", "VOP2 opcode 0x23"}, + {{0x48040300u}, "VOP2 opcode is not implemented", "VOP2 opcode 0x24"}, + {{0xd5150002u, 0x00020300u}, "VOP3 opcode is not implemented", "VOP3 opcode 0x115"}, + {{0xd5170002u, 0x00020300u}, "VOP3 opcode is not implemented", "VOP3 opcode 0x117"}, + {{0xd5190002u, 0x00020300u}, "VOP3 opcode is not implemented", "VOP3 opcode 0x119"}, + {{0xd5220002u, 0x00020300u}, "VOP3 opcode is not implemented", "VOP3 opcode 0x122"}, + {{0xd5230002u, 0x00020300u}, "VOP3 opcode is not implemented", "VOP3 opcode 0x123"}, + {{0xd5240002u, 0x00020300u}, "VOP3 opcode is not implemented", "VOP3 opcode 0x124"}, + {{0xd5370002u, 0x00020300u}, "VOP3 opcode is not implemented", "VOP3 opcode 0x137"}, + {{0xd5380002u, 0x00020300u}, "VOP3 opcode is not implemented", "VOP3 opcode 0x138"}, + {{0xd53c0002u, 0x00020300u}, "VOP3 opcode is not implemented", "VOP3 opcode 0x13c"}, + }; + int failures = 0; + for (const auto& test : accepted) { + auto code = test.words; + code.push_back(0xbf810000u); + try { + const auto decoded = RdnaInstructionDecoder{}.Decode(code); + if (decoded.instructions.empty() || decoded.instructions[0].op != test.op || decoded.instructions[0].wordCount != test.words.size()) { + std::fprintf(stderr, "%s decodes to the wrong opcode or length\n", test.name); + ++failures; + } + } catch (const std::exception& error) { + std::fprintf(stderr, "%s was refused: %s\n", test.name, error.what()); + ++failures; + } + } + for (const auto& test : refused) { + auto code = test.words; + code.push_back(0xbf810000u); + std::string refusal; + try { + static_cast(RdnaInstructionDecoder{}.Decode(code)); + } catch (const std::exception& error) { + refusal = error.what(); + } + if (refusal.find(test.reason) == std::string::npos) { + std::fprintf(stderr, "%s was not refused\n", test.name); + ++failures; + } + } + return failures == 0 ? 0 : 1; +} diff --git a/core/libs/prx/libSceAgcDriver/tests/WriteTracking.cpp b/core/libs/prx/libSceAgcDriver/tests/WriteTracking.cpp index f72f451e8..b924f0d5c 100644 --- a/core/libs/prx/libSceAgcDriver/tests/WriteTracking.cpp +++ b/core/libs/prx/libSceAgcDriver/tests/WriteTracking.cpp @@ -15,6 +15,8 @@ #include #include #include +#include +#include namespace { @@ -26,17 +28,20 @@ void Require(bool condition, const std::string& message) { if (!condition) throw std::runtime_error(message); } -void* AllocateWatched(std::size_t bytes) { +void* AllocateWatched(std::size_t bytes, bool crossLeaf = false) { #ifdef _WIN32 + static_cast(crossLeaf); void* block = GuestArena::GuestArenaAllocate_nid_postfix(bytes, Block); GuestArena::GuestArenaCommit_nid_postfix(block, bytes, PAGE_READWRITE, bytes); #else - void* raw = mmap(nullptr, bytes + Block, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + const std::size_t alignment = crossLeaf ? std::size_t{1} << 32 : Block; + const auto prefix = crossLeaf ? Block : 0; + void* raw = mmap(nullptr, bytes + alignment, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); if (raw == MAP_FAILED) throw std::runtime_error("cannot map the watched block"); const auto begin = reinterpret_cast(raw); - const auto aligned = (begin + Block - 1) & ~static_cast(Block - 1); + const auto aligned = ((begin + prefix + alignment - 1) & ~static_cast(alignment - 1)) - prefix; if (aligned != begin) munmap(raw, aligned - begin); - if (aligned + bytes != begin + bytes + Block) munmap(reinterpret_cast(aligned + bytes), begin + Block - aligned); + if (aligned + bytes != begin + bytes + alignment) munmap(reinterpret_cast(aligned + bytes), begin + alignment - aligned); void* block = reinterpret_cast(aligned); GuestWriteWatch::GuestWriteWatchRegister_nid_postfix(block, bytes); #endif @@ -180,6 +185,120 @@ void CheckPrivateMappingReuse() { GuestArena::GuestArenaRelease_nid_postfix(replacement, 3 * Block); } #endif + +void CheckVersionRanges() { + constexpr std::size_t count = 80; + constexpr auto bytes = count * Block; + auto* memory = static_cast(AllocateWatched(bytes, true)); + struct Cleanup { + void* memory; + ~Cleanup() { +#ifdef _WIN32 + GuestArena::GuestArenaRelease_nid_postfix(memory, bytes); +#else + GuestWriteWatch::GuestWriteWatchUnregister_nid_postfix(memory, bytes); + munmap(memory, bytes); +#endif + } + } cleanup{memory}; + const auto base = reinterpret_cast(memory); + Require(UnchangedSince(base, bytes, TrackerGeneration()), "a new range has newer version stamps"); + std::memset(memory, 17, bytes); + const auto initial = CollectWritesUncached(base, bytes); + Require(initial != 0, "version ranges could not be collected"); + struct Write { std::size_t block; std::uint64_t generation; }; + std::vector history; + std::vector checkpoints{initial}; + std::uint32_t random = 0x61387a29; + const auto next = [&] { random = random * 1664525u + 1013904223u; return random; }; + const auto expected = [&](const UnchangedQuery& query) { + if (query.generation == 0) return false; + for (const auto& write : history) { + const auto begin = base + write.block * Block; + if (write.generation > query.generation && begin < query.address + query.bytes && query.address < begin + Block) return false; + } + return true; + }; + for (unsigned step = 0; step < 512; ++step) { + const auto block = step < 3 ? step : next() % count; + std::uint64_t generation; + if (step % 3 == 0) { + memory[block * Block + 9] ^= 1; + generation = CollectWritesUncached(base + block * Block, Block); + } else generation = MarkWritten(base + block * Block + 13, 4); + Require(generation > checkpoints.back(), "version range write did not advance the tracker"); + history.push_back({block, generation}); + checkpoints.push_back(generation); + for (unsigned check = 0; check < 16; ++check) { + std::array queries; + bool all = true; + for (auto& query : queries) { + const auto offset = next() % bytes; + query = {base + offset, 1 + next() % (bytes - offset), checkpoints[next() % checkpoints.size()]}; + const auto same = expected(query); + Require(UnchangedSince(query.address, query.bytes, query.generation) == same, "version range disagrees with write history"); + all = all && same; + } + Require(UnchangedSinceAll(queries) == all, "batched version ranges disagree with write history"); + } + } + Require(!UnchangedSince(base, bytes, 0), "an unknown generation was accepted"); + Require(UnchangedSince(base, bytes, checkpoints.back()), "the latest checkpoint was rejected"); +} + +void CheckCollectBoundaries() { + constexpr std::size_t bytes = 3 * Block; + auto* memory = static_cast(AllocateWatched(bytes)); + struct Cleanup { + void* memory; + ~Cleanup() { +#ifdef _WIN32 + GuestArena::GuestArenaRelease_nid_postfix(memory, bytes); +#else + Unwatch(reinterpret_cast(memory), bytes); + munmap(memory, bytes); +#endif + } + } cleanup{memory}; + const auto base = reinterpret_cast(memory); + std::memset(memory, 17, bytes); + bool unboundedObserved = false; + std::thread observer([&] { + const auto before = CollectWrites(base + 7, 1); + memory[7] = 19; + const auto after = CollectWrites(base + 7, 1); + unboundedObserved = before != 0 && after > before && !UnchangedSince(base + 7, 1, before); + }); + observer.join(); + Require(unboundedObserved, "a thread without collect epochs reused a stale observation"); + for (const auto offset : {7u, 4096u + 3u, static_cast(Block - 8), static_cast(Block + 3)}) { + BumpCollectEpoch(); + const auto before = CollectWrites(base + offset, 16); + Require(before != 0, "an epoch lost watched coverage"); + memory[offset] ^= 1; + BumpCollectEpoch(); + Require(CollectWrites(base + offset, 16) > before && !UnchangedSince(base + offset, 16, before), "a new epoch missed a CPU write"); + const auto observed = TrackerGeneration(); + memory[offset + 1] ^= 1; + Require(CollectWritesUncached(base + offset, 16) > observed && !UnchangedSince(base + offset, 16, observed), "an uncached observation reused an epoch result"); + const auto stored = MarkWritten(base + offset + 2, 1); + Require(CollectWrites(base + offset, 16) >= stored && !UnchangedSince(base + offset, 16, observed), "a memoized observation hid a driver write"); + } +#ifndef _WIN32 + BumpCollectEpoch(); + Require(CollectWrites(base, 1) != 0, "initial partial-block observation failed"); + Unwatch(base + 4096, 4096); + Require(CollectWrites(base + 4096, 1) == 0, "an unwatched neighbor retained an epoch observation"); + BumpCollectEpoch(); + const auto before = CollectWrites(base, 16); + Require(before != 0, "an unwatched neighbor prevented collecting a watched prefix"); + memory[0] ^= 1; + BumpCollectEpoch(); + Require(CollectWrites(base, 16) > before && !UnchangedSince(base, 16, before), "partial-block collection missed a CPU write"); + GuestWriteWatch::GuestWriteWatchRegister_nid_postfix(memory + 4096, 4096); + Require(CollectWrites(base + 4096, 1) != 0, "a re-registered neighbor stayed unwatched"); +#endif +} } int main() { @@ -194,6 +313,8 @@ int main() { #ifdef _WIN32 CheckPrivateMappingReuse(); #endif + CheckVersionRanges(); + CheckCollectBoundaries(); } catch (const std::exception& error) { std::cerr << "write tracking test failed: " << error.what() << "\n"; return 1; diff --git a/core/libs/prx/libSceAgcDriver/tests/execution/BdaSpanReads.cpp b/core/libs/prx/libSceAgcDriver/tests/execution/BdaSpanReads.cpp new file mode 100644 index 000000000..bb0fcc4fd --- /dev/null +++ b/core/libs/prx/libSceAgcDriver/tests/execution/BdaSpanReads.cpp @@ -0,0 +1,159 @@ +#include "prx/libSceAgcDriver/Execution/include/VulkanDevice.hpp" +#include "prx/libSceAgcDriver/Graphics/include/Draw.hpp" +#include "prx/libc/include/GuestAllocations.hpp" +#include "Recompiler.hpp" +#include "VulkanTestDevice.hpp" +#ifdef _WIN32 +#ifndef NOMINMAX +#define NOMINMAX +#endif +#include +#endif +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace { + +using AgcDriver::Graphics::Require; +using ShaderRecompiler::ShaderStage; + +constexpr std::uint32_t Threads = 64; +constexpr std::size_t BlockBytes = 65536; +constexpr std::uint32_t InputDwords = 768; +constexpr std::uint32_t Fill = 0xcdcdcdcdu; +constexpr std::uint32_t LoadBase = 0x100; +constexpr std::uint32_t OutputBase = 0x8000; + +alignas(256) std::array, 4> Programs{}; + +std::span Code(bool coherent, bool barrier) { + auto& program = Programs[(coherent ? 2u : 0u) + (barrier ? 1u : 0u)]; + std::vector code{0x34040084, 0x4a0404ff, LoadBase, 0x34080081, 0x36080882, 0x4a040504, 0x34060083, 0x4a0606ff, OutputBase, 0xdc388ff8 | (coherent ? 0x10000u : 0u), 0x04000002, 0xbf8c3f70}; + if (barrier) code.push_back(0xbf8a0000); + code.insert(code.end(), {0xdc708000, 0x00000503, 0xdc708004, 0x00000703, 0xbf810000}); + std::copy(code.begin(), code.end(), program.begin()); + return std::span(program.data(), code.size()); +} + +class GuestBlock { +public: + GuestBlock() { +#ifdef _WIN32 + block = static_cast(VirtualAlloc(nullptr, BlockBytes, MEM_RESERVE | MEM_COMMIT, PAGE_READWRITE)); +#else + block = static_cast(std::aligned_alloc(BlockBytes, BlockBytes)); +#endif + Require(block != nullptr, "BDA span reads: cannot allocate the guest block"); + GuestAllocations::Mutation().Add(block, BlockBytes, true, true, true); + } + + ~GuestBlock() { + GuestAllocations::Mutation().Remove(block); +#ifdef _WIN32 + VirtualFree(block, 0, MEM_RELEASE); +#else + std::free(block); +#endif + } + + GuestBlock(const GuestBlock&) = delete; + GuestBlock& operator=(const GuestBlock&) = delete; + + std::uint8_t* Data() { return block; } + +private: + std::uint8_t* block = nullptr; +}; + +std::string Hex(std::uint32_t value) { + char text[16]; + std::snprintf(text, sizeof(text), "0x%08x", value); + return text; +} + +std::vector Initial() { + std::vector memory(BlockBytes / 4u, Fill); + for (std::uint32_t dword = 0; dword < InputDwords; ++dword) memory[dword] = 0x52000000u + dword * 0x00010203u; + return memory; +} + +std::uint32_t Bytes(const std::vector& memory, std::uint32_t address) { + std::uint32_t value = 0; + for (std::uint32_t byte = 0; byte < 4u; ++byte) { + const std::uint32_t at = address + byte; + value |= ((memory[at / 4u] >> (at % 4u * 8u)) & 0xffu) << (byte * 8u); + } + return value; +} + +std::vector Expected() { + const auto initial = Initial(); + auto memory = initial; + for (std::uint32_t tid = 0; tid < Threads; ++tid) { + const std::uint32_t load = LoadBase + tid * 16u + (tid & 1u) * 2u - 8u; + memory[OutputBase / 4u + tid * 2u] = Bytes(initial, load + 4u); + memory[OutputBase / 4u + tid * 2u + 1u] = Bytes(initial, load + 12u); + } + return memory; +} + +ShaderRecompiler::RecompileResult Recompile(std::span code, std::uint32_t waveSize, std::span userData, const ShaderRecompiler::SpirvTarget& target) { + const std::array regions{{{reinterpret_cast(code.data()), std::as_bytes(code)}}}; + const ShaderRecompiler::ShaderComputeStageInfo compute{{Threads, 1, 1}, 0, {false, false, false}, false, 1}; + ShaderRecompiler::RecompileRequest request{ + {ShaderStage::Compute, reinterpret_cast(code.data()), code, 0, {}}, + {waveSize, 0, userData, compute, std::nullopt, std::nullopt, regions}, + target, + {0, 0, 0, 128} + }; + request.useCache = false; + return ShaderRecompiler::Recompile(request); +} + +void Run(AgcDriver::VulkanDevice& device, GuestBlock& guest, bool coherent, bool barrier, std::uint32_t waveSize, const ShaderRecompiler::SpirvTarget& target, const std::string& name) { + const auto code = Code(coherent, barrier); + auto memory = Initial(); + std::memcpy(guest.Data(), memory.data(), BlockBytes); + const auto address = static_cast(reinterpret_cast(guest.Data())); + const std::vector userData{static_cast(address), static_cast(address >> 32u)}; + const auto result = Recompile(code, waveSize, userData, target); + device.Dispatch(result, 1, 1, 1, {}, reinterpret_cast(code.data())); + device.WaitIdle(); + std::memcpy(memory.data(), guest.Data(), BlockBytes); + const auto expected = Expected(); + for (std::size_t dword = 0; dword < memory.size(); ++dword) { + Require(memory[dword] == expected[dword], "BDA span reads: " + name + (coherent ? " glc" : "") + (barrier ? " barrier" : "") + " byte " + std::to_string(dword * 4u) + " is " + Hex(memory[dword]) + ", expected " + Hex(expected[dword])); + } +} + +} + +int main() { + try { + const auto device = OpenVulkanTestDevice(); + if (!device) return VulkanTestSkipped; + GuestBlock guest; + const auto native = device->Target(); + const auto split = device->ComputeTarget(32); + for (const bool coherent : {false, true}) { + for (const bool barrier : {false, true}) { + Run(*device, guest, coherent, barrier, 32, native, "wave32"); + Run(*device, guest, coherent, barrier, 64, native, "wave64"); + if (split.subgroupSize != native.subgroupSize) Run(*device, guest, coherent, barrier, 64, split, "wave64 split"); + } + } + std::puts("BDA span read tests passed"); + return 0; + } catch (const std::exception& error) { + std::cerr << error.what() << '\n'; + return 1; + } +} diff --git a/core/libs/prx/libSceAgcDriver/tests/execution/BitfieldMask.cpp b/core/libs/prx/libSceAgcDriver/tests/execution/BitfieldMask.cpp new file mode 100644 index 000000000..77d144b5e --- /dev/null +++ b/core/libs/prx/libSceAgcDriver/tests/execution/BitfieldMask.cpp @@ -0,0 +1,96 @@ +#include "prx/libSceAgcDriver/Graphics/include/Draw.hpp" +#include "Recompiler.hpp" +#include "VulkanTestDevice.hpp" +#include +#include +#include +#include +#include +#include + +namespace { + +using AgcDriver::Graphics::Require; +using ShaderRecompiler::ShaderStage; + +constexpr std::uint32_t Threads = 32u; +alignas(256) std::array Input{}; +alignas(256) std::array Output{}; +alignas(256) constexpr std::array Code{ + 0x34020083u, 0x34040084u, 0xe0341000u, 0x80000401u, 0xbf8c3f70u, 0xd7630008u, 0x00020b04u, + 0xbe8c0380u, 0xbe8d0380u, 0xf4240200u, 0x18000000u, 0xbf8cc07fu, 0xbf060d0du, 0x920a0908u, + 0x850b8081u, 0xd7610009u, 0x00001a0au, 0xd761000au, 0x00001a0bu, 0xbf070d0du, 0x920a0908u, + 0x850b8081u, 0xd761000bu, 0x00001a0bu, 0x800c880cu, 0x800d810du, 0xbf0aa00du, 0xbf85ffedu, + 0xe0781000u, 0x80010802u, 0xbf810000u +}; + +std::array bufferDescriptor(const void* data, std::uint32_t bytes) { + const auto address = reinterpret_cast(data); + return {static_cast(address), static_cast((address >> 32u) & 0xffffu), bytes, 0x31016facu}; +} + +std::uint32_t expectedMask(std::uint32_t width, std::uint32_t offset) { + std::uint32_t mask = 0u; + for (std::uint32_t bit = offset; bit < 32u && bit - offset < width; ++bit) mask |= 1u << bit; + return mask; +} + +void run(AgcDriver::VulkanDevice& device) { + std::array userData{}; + const auto input = bufferDescriptor(Input.data(), static_cast(Input.size() * 4u)); + const auto output = bufferDescriptor(Output.data(), static_cast(Output.size() * 4u)); + std::copy(input.begin(), input.end(), userData.begin()); + std::copy(output.begin(), output.end(), userData.begin() + 4); + const std::span code(Code); + const std::array memory{{{reinterpret_cast(code.data()), std::as_bytes(code)}}}; + const ShaderRecompiler::ShaderComputeStageInfo compute{{Threads, 1u, 1u}, 0u, {false, false, false}, false, 1u}; + ShaderRecompiler::RecompileRequest request{ + {ShaderStage::Compute, reinterpret_cast(code.data()), code, 0u, {}}, + {32u, 0u, userData, compute, std::nullopt, std::nullopt, memory}, + device.Target(), + {0u, 0u, 0u, 128u} + }; + request.useCache = false; + const auto result = ShaderRecompiler::Recompile(request); + for (const std::uint32_t upperBits : {0u, 32u, 0x80000000u, 0xffffffe0u}) { + for (std::uint32_t width = 0u; width < 32u; ++width) { + for (std::uint32_t offset = 0u; offset < Threads; ++offset) { + Input[offset * 2u] = width | upperBits; + Input[offset * 2u + 1u] = offset | upperBits; + } + Output.fill(0xdeadbeefu); + device.Dispatch(result, 1u, 1u, 1u, {}, reinterpret_cast(code.data())); + device.WaitIdle(); + for (std::uint32_t offset = 0u; offset < Threads; ++offset) { + const auto expected = expectedMask(width, offset); + const auto description = "bitfield mask width=" + std::to_string(width | upperBits) + + " offset=" + std::to_string(offset | upperBits); + Require(Output[offset * 4u] == expected, description + " vector result " + + std::to_string(Output[offset * 4u]) + ", expected " + std::to_string(expected)); + Require(Output[offset * 4u + 1u] == expected, description + " scalar result " + + std::to_string(Output[offset * 4u + 1u]) + ", expected " + std::to_string(expected)); + Require(Output[offset * 4u + 2u] == 1u, description + " changed SCC=1"); + Require(Output[offset * 4u + 3u] == 0u, description + " changed SCC=0"); + } + } + } +} + +} + +int main() { + try { + const auto device = OpenVulkanTestDevice(); + if (!device) return VulkanTestSkipped; + if (device->Target().subgroupSize < Threads) { + std::printf("skipped, subgroup size %u cannot hold a wave32\n", device->Target().subgroupSize); + return VulkanTestSkipped; + } + run(*device); + std::puts("4096 scalar/vector bitfield mask execution cases passed"); + return 0; + } catch (const std::exception& error) { + std::cerr << error.what() << '\n'; + return 1; + } +} diff --git a/core/libs/prx/libSceAgcDriver/tests/execution/BufferAtomicCsub.cpp b/core/libs/prx/libSceAgcDriver/tests/execution/BufferAtomicCsub.cpp index 5a2a267f4..0b150b2da 100644 --- a/core/libs/prx/libSceAgcDriver/tests/execution/BufferAtomicCsub.cpp +++ b/core/libs/prx/libSceAgcDriver/tests/execution/BufferAtomicCsub.cpp @@ -28,12 +28,12 @@ constexpr std::uint32_t SharedExhausted = 1000u; alignas(256) std::array Input{}; alignas(256) std::array Output{}; -alignas(256) constexpr std::array Code{ +alignas(256) constexpr std::array Code{ 0x34020084, 0x34040085, 0xe0301000, 0x80000401, 0xe0301004, 0x80000501, 0xe0301008, 0x80000601, - 0xbf8c3f70, 0x7e140280, 0x461414c1, 0x481414c1, 0x7e0e0304, 0xe0d05000, 0x80010702, 0x7e100304, - 0xbefe03ff, 0x55555555, 0xe0d05004, 0x80010802, 0xbefe03c1, 0x7e120305, 0xe0d04800, 0x80010900, - 0x7e160306, 0xe0d04804, 0x80010b00, 0xbf8c3f70, 0xe0701008, 0x80010702, 0xe070100c, 0x80010802, - 0xe0701010, 0x80010a02, 0xbf810000, + 0xbf8c3f70, 0x7e140280, 0xd765000a, 0x000214c1, 0xd766000a, 0x000214c1, 0x7e0e0304, 0xe0d05000, + 0x80010702, 0x7e100304, 0xbefe03ff, 0x55555555, 0xe0d05004, 0x80010802, 0xbefe03c1, 0x7e120305, + 0xe0d04800, 0x80010900, 0x7e160306, 0xe0d04804, 0x80010b00, 0xbf8c3f70, 0xe0701008, 0x80010702, + 0xe070100c, 0x80010802, 0xe0701010, 0x80010a02, 0xbf810000, }; alignas(256) constexpr std::array NoReturnCode{ diff --git a/core/libs/prx/libSceAgcDriver/tests/execution/BufferUnalignedBase.cpp b/core/libs/prx/libSceAgcDriver/tests/execution/BufferUnalignedBase.cpp index dd78c229b..bbd38f1f9 100644 --- a/core/libs/prx/libSceAgcDriver/tests/execution/BufferUnalignedBase.cpp +++ b/core/libs/prx/libSceAgcDriver/tests/execution/BufferUnalignedBase.cpp @@ -143,6 +143,12 @@ void Run(AgcDriver::VulkanDevice& device, std::uint32_t viewOffset, std::uint32_ } void RunPair(AgcDriver::VulkanDevice& device, std::uint32_t viewOffset) { + auto code = PairCode; + const bool int64Atomics = TargetHasCapability(device.Target(), spv::CapabilityInt64Atomics); + if (!int64Atomics) { + code[5] = 0xbf800000u; + code[6] = 0xbf800000u; + } for (std::uint32_t index = 0; index < InputBytes; ++index) Input[index] = static_cast(index * 53u + 7u + (index >> 2u)); Output.fill(Fill); const auto* view = Input.data() + viewOffset; @@ -150,9 +156,9 @@ void RunPair(AgcDriver::VulkanDevice& device, std::uint32_t viewOffset) { for (const auto& descriptor : {BufferDescriptor(Input.data(), InputBytes), BufferDescriptor(view, ViewBytes), BufferDescriptor(Output.data(), ResultBytes)}) { userData.insert(userData.end(), descriptor.begin(), descriptor.end()); } - const std::array memory{{{reinterpret_cast(PairCode.data()), std::as_bytes(std::span(PairCode))}}}; - const auto result = ShaderRecompiler::Recompile(Request(PairCode, userData, device.Target(), memory)); - device.Dispatch(result, 1, 1, 1, {}, reinterpret_cast(PairCode.data())); + const std::array memory{{{reinterpret_cast(code.data()), std::as_bytes(std::span(code))}}}; + const auto result = ShaderRecompiler::Recompile(Request(code, userData, device.Target(), memory)); + device.Dispatch(result, 1, 1, 1, {}, reinterpret_cast(code.data())); device.WaitIdle(); const auto run = "buffer unaligned base glc dwordx2, view +" + std::to_string(viewOffset); for (std::uint32_t lane = 0; lane < Lanes; ++lane) { @@ -162,7 +168,7 @@ void RunPair(AgcDriver::VulkanDevice& device, std::uint32_t viewOffset) { Require(actual == expected, run + ": lane " + std::to_string(lane) + " dword " + std::to_string(half) + " is " + Hex(actual) + ", expected " + Hex(expected)); } } - Require(Output[Lanes * 2u] == Fill + Lanes && Output[Lanes * 2u + 1u] == Fill, run + ": the 64-bit atomic is " + Hex(Output[Lanes * 2u + 1u]) + Hex(Output[Lanes * 2u])); + if (int64Atomics) Require(Output[Lanes * 2u] == Fill + Lanes && Output[Lanes * 2u + 1u] == Fill, run + ": the 64-bit atomic is " + Hex(Output[Lanes * 2u + 1u]) + Hex(Output[Lanes * 2u])); } auto RecompileFormatted(AgcDriver::VulkanDevice& device, std::span code, const std::uint8_t* view, std::uint32_t viewBytes) { diff --git a/core/libs/prx/libSceAgcDriver/tests/execution/Bvh64IntersectRay.cpp b/core/libs/prx/libSceAgcDriver/tests/execution/Bvh64IntersectRay.cpp index 788ff56ff..77ced64f4 100644 --- a/core/libs/prx/libSceAgcDriver/tests/execution/Bvh64IntersectRay.cpp +++ b/core/libs/prx/libSceAgcDriver/tests/execution/Bvh64IntersectRay.cpp @@ -152,7 +152,7 @@ public: #endif Require(block != nullptr, "bvh64 intersect ray: cannot allocate the guest block"); std::memset(block, 0, BlockBytes); - GuestAllocations::Mutation().Add(block, BlockBytes, true, false); + GuestAllocations::Mutation().Add(block, BlockBytes, true, false, true); } ~GuestBlock() { diff --git a/core/libs/prx/libSceAgcDriver/tests/execution/ConservativeRasterization.cpp b/core/libs/prx/libSceAgcDriver/tests/execution/ConservativeRasterization.cpp index 4bc982a2b..3a641d905 100644 --- a/core/libs/prx/libSceAgcDriver/tests/execution/ConservativeRasterization.cpp +++ b/core/libs/prx/libSceAgcDriver/tests/execution/ConservativeRasterization.cpp @@ -86,13 +86,13 @@ void Draw(AgcDriver::VulkanDevice& device, VkConservativeRasterizationModeEXT mo {ShaderStage::Fragment, reinterpret_cast(PixelCode.data()), PixelCode, 0, {}}, {WaveSize, 0, pixelUserData, std::nullopt, pixel, std::nullopt, pixelMemory}, target, - {0, 0, vertexPush, 128 - vertexPush} + PixelPushLayout(vertexPush, target) }; fragment.useCache = false; const auto pixelResult = ShaderRecompiler::Recompile(fragment); const std::array shaders{{ {ShaderStage::Vertex, &vertexResult, 0}, - {ShaderStage::Fragment, &pixelResult, vertexPush} + {ShaderStage::Fragment, &pixelResult, PixelPushOffset(vertexPush, target)} }}; AgcDriver::Graphics::State state{}; diff --git a/core/libs/prx/libSceAgcDriver/tests/execution/ConstantStore.cpp b/core/libs/prx/libSceAgcDriver/tests/execution/ConstantStore.cpp index c9b16efb5..8d542c7ff 100644 --- a/core/libs/prx/libSceAgcDriver/tests/execution/ConstantStore.cpp +++ b/core/libs/prx/libSceAgcDriver/tests/execution/ConstantStore.cpp @@ -35,7 +35,7 @@ public: #endif Require(block != nullptr, "constant store: cannot allocate the guest block"); Clear(); - GuestAllocations::Mutation().Add(block, BlockBytes, true, true); + GuestAllocations::Mutation().Add(block, BlockBytes, true, true, true); } ~GuestBlock() { diff --git a/core/libs/prx/libSceAgcDriver/tests/execution/DccKeysFirstWrite.cpp b/core/libs/prx/libSceAgcDriver/tests/execution/DccKeysFirstWrite.cpp index c3c4bd5eb..47547e67b 100644 --- a/core/libs/prx/libSceAgcDriver/tests/execution/DccKeysFirstWrite.cpp +++ b/core/libs/prx/libSceAgcDriver/tests/execution/DccKeysFirstWrite.cpp @@ -104,7 +104,7 @@ public: #endif Require(block != nullptr, "DCC keys first write: cannot allocate the guest block"); Require(!watched || AgcDriver::GuestMemory::Watched(AddressOf(block), BlockBytes), "DCC keys first write: the watched guest block is not write-watched"); - GuestAllocations::Mutation().Add(block, BlockBytes, true, true); + GuestAllocations::Mutation().Add(block, BlockBytes, true, true, true); } ~GuestBlock() { diff --git a/core/libs/prx/libSceAgcDriver/tests/execution/DepthBiasPerFace.cpp b/core/libs/prx/libSceAgcDriver/tests/execution/DepthBiasPerFace.cpp new file mode 100644 index 000000000..d54ab0399 --- /dev/null +++ b/core/libs/prx/libSceAgcDriver/tests/execution/DepthBiasPerFace.cpp @@ -0,0 +1,147 @@ +#include "prx/libSceAgcDriver/Execution/include/VulkanDevice.hpp" +#include "prx/libSceAgcDriver/Graphics/include/Draw.hpp" +#include "Recompiler.hpp" +#include "VulkanTestDevice.hpp" +#include +#include +#include +#include +#include +#include +#include +#include + +namespace { + +using AgcDriver::Graphics::Require; +using ShaderRecompiler::ShaderStage; + +constexpr std::uint32_t Width = 64; +constexpr std::uint32_t Height = 16; +constexpr std::uint32_t WaveSize = 64; +constexpr std::byte Kept{0x40}; +constexpr float SurfaceDepth = 0.5f; +constexpr float Bias = 1048576.0f; +alignas(256) std::array Pixels{}; +alignas(256) std::array DepthMemory{}; + +alignas(256) constexpr std::array VertexCode{ + 0xe0382000, 0x80000005, 0xbf8c3f70, 0xf80008cf, 0x03020100, 0xbf810000, +}; + +alignas(256) constexpr std::array PixelCode{ + 0x7e0e02f2, 0xf800180f, 0x07070707, 0xbf810000, +}; + +constexpr std::array, 6> Corners{{ + {1.0f, 1.0f}, {1.0f, 15.0f}, {15.0f, 1.0f}, + {17.0f, 1.0f}, {31.0f, 1.0f}, {17.0f, 15.0f}, +}}; + +alignas(256) std::array, Corners.size()> Vertices{}; + +std::array BufferDescriptor(const void* data, std::uint32_t stride, std::uint32_t count) { + const auto address = reinterpret_cast(data); + return {static_cast(address), static_cast((address >> 32u) & 0xffffu) | (stride << 16u), count, 0x01016facu}; +} + +void Draw(AgcDriver::VulkanDevice& device, float frontBias, float backBias) { + Pixels.fill(Kept); + for (std::size_t index = 0; index < Corners.size(); ++index) { + Vertices[index] = {Corners[index][0] * 2.0f / Width - 1.0f, 1.0f - Corners[index][1] * 2.0f / Height, SurfaceDepth, 1.0f}; + } + const auto target = device.Target(); + + std::vector vertexUserData(4, 0u); + const auto vertexBuffer = BufferDescriptor(Vertices.data(), 16u, static_cast(Vertices.size())); + std::copy(vertexBuffer.begin(), vertexBuffer.end(), vertexUserData.begin()); + const std::array vertexMemory{{{reinterpret_cast(VertexCode.data()), std::as_bytes(std::span(VertexCode))}}}; + ShaderRecompiler::RecompileRequest vertex{ + {ShaderStage::Vertex, reinterpret_cast(VertexCode.data()), VertexCode, 0, {}}, + {WaveSize, 0, vertexUserData, std::nullopt, std::nullopt, ShaderRecompiler::ShaderVertexStageInfo{}, vertexMemory}, + target, + {0, 0, 0, 64} + }; + vertex.useCache = false; + const auto vertexResult = ShaderRecompiler::Recompile(vertex); + const auto vertexPush = static_cast(vertexResult.pushConstants.size()); + + ShaderRecompiler::ShaderPixelStageInfo pixel{}; + pixel.inputAddr = ShaderRecompiler::PixelInputBit(ShaderRecompiler::PixelInput::PositionX) | ShaderRecompiler::PixelInputBit(ShaderRecompiler::PixelInput::PositionY); + pixel.posX = true; + pixel.posY = true; + pixel.targetOutputMode[0] = 9; + pixel.targetExportMapping.fill(0xe4u); + const std::vector pixelUserData(8, 0u); + const std::array pixelMemory{{{reinterpret_cast(PixelCode.data()), std::as_bytes(std::span(PixelCode))}}}; + ShaderRecompiler::RecompileRequest fragment{ + {ShaderStage::Fragment, reinterpret_cast(PixelCode.data()), PixelCode, 0, {}}, + {WaveSize, 0, pixelUserData, std::nullopt, pixel, std::nullopt, pixelMemory}, + target, + PixelPushLayout(vertexPush, target) + }; + fragment.useCache = false; + const auto pixelResult = ShaderRecompiler::Recompile(fragment); + const std::array shaders{{ + {ShaderStage::Vertex, &vertexResult, 0}, + {ShaderStage::Fragment, &pixelResult, PixelPushOffset(vertexPush, target)} + }}; + + AgcDriver::Graphics::State state{}; + state.stages = {AgcDriver::Graphics::ShaderPath::Vertex, 0u, WaveSize, WaveSize, std::nullopt, std::nullopt}; + state.color = {reinterpret_cast(Pixels.data()), {Width, Height}, VK_FORMAT_R8G8B8A8_UNORM, Pixels.size(), 0xe4u}; + state.colors = {state.color}; + state.hasColorTarget = true; + state.depth = AgcDriver::Graphics::DepthTarget{reinterpret_cast(DepthMemory.data()), 0, {Width, Height}, VK_FORMAT_D32_SFLOAT, SurfaceDepth, 0}; + state.depthTest = true; + state.depthWrite = false; + state.depthCompare = VK_COMPARE_OP_LESS; + state.depthBias = true; + state.depthBiasConstant = frontBias; + state.depthBiasPerFace = frontBias != backBias; + state.backDepthBiasConstant = backBias; + state.renderExtent = {Width, Height}; + state.topology = VK_PRIMITIVE_TOPOLOGY_TRIANGLE_LIST; + state.viewport = {0, static_cast(Height), static_cast(Width), -static_cast(Height), 0, 1}; + state.negativeOneToOne = false; + state.scissor = {{0, 0}, {Width, Height}}; + state.cullMode = VK_CULL_MODE_NONE; + state.frontFace = VK_FRONT_FACE_COUNTER_CLOCKWISE; + state.blend.colorWriteMask = 15; + state.blends = {state.blend}; + state.blendConstants = {}; + const AgcDriver::Pm4::DrawParameters draw{0, static_cast(Vertices.size()), 0, 1, 0, false}; + device.Draw(state, draw, shaders); + device.WaitIdle(); +} + +void Check(bool frontDrawn, bool backDrawn, std::string_view what) { + for (const auto& [x, y, front] : std::array, 2>{{{4, 4, true}, {20, 4, false}}}) { + const bool drawn = front ? frontDrawn : backDrawn; + const auto expected = drawn ? std::byte{255} : Kept; + const auto actual = Pixels[(y * Width + x) * 4u]; + Require(actual == expected, std::string(what) + ": the " + (front ? "front" : "back") + "-facing triangle at (" + std::to_string(x) + ", " + std::to_string(y) + ") is " + std::to_string(static_cast(actual)) + ", expected " + (drawn ? "drawn" : "depth-rejected")); + } +} + +} + +int main() { + try { + const auto device = OpenVulkanTestDevice(); + if (!device) return VulkanTestSkipped; + Draw(*device, -Bias, Bias); + Check(true, false, "front faces pulled closer, back faces pushed away"); + Draw(*device, Bias, -Bias); + Check(false, true, "front faces pushed away, back faces pulled closer"); + Draw(*device, -Bias, -Bias); + Check(true, true, "both faces pulled closer"); + Draw(*device, -Bias, 0.0f); + Check(true, false, "only front faces biased"); + std::puts("per-face depth bias tests passed"); + return 0; + } catch (const std::exception& error) { + std::cerr << error.what() << '\n'; + return 1; + } +} diff --git a/core/libs/prx/libSceAgcDriver/tests/execution/Division.cpp b/core/libs/prx/libSceAgcDriver/tests/execution/Division.cpp index 39bf75b41..09059269b 100644 --- a/core/libs/prx/libSceAgcDriver/tests/execution/Division.cpp +++ b/core/libs/prx/libSceAgcDriver/tests/execution/Division.cpp @@ -302,6 +302,13 @@ constexpr HelperVector HelperVectors[] = { {0x8c367fd8u, 0xbf976e24u, 0x8c367fd8u, 1u, 0xac367fd8u, 1u, 0x00000000u, 0x0c367fd8u}, {0x00f745cfu, 0x7f7fffffu, 0x00f745cfu, 0u, 0x00f745cfu, 1u, 0x40f745ceu, 0x00000000u}, {0xcb7cf32du, 0xbfe52415u, 0xbf6a1e62u, 0u, 0xcb7cf32du, 0u, 0x4be2692eu, 0x4b7cf32du}, + {0x3f800000u, 0x7f7fffffu, 0xff7fffffu, 1u, 0x1f800000u, 0u, 0x00000000u, 0xbf800000u}, + {0x3f800000u, 0xf1800000u, 0x71800000u, 1u, 0x3f800000u, 0u, 0x00000000u, 0xbf800000u}, + {0x7f7fffffu, 0xbf800000u, 0x7f7fffffu, 1u, 0x7f7fffffu, 1u, 0x00000000u, 0xff7fffffu}, + {0x3f800000u, 0xdf800000u, 0x5f800000u, 1u, 0x3f800000u, 0u, 0x00000000u, 0xbf800000u}, + {0x5f400000u, 0xdf200000u, 0x7ef00000u, 1u, 0x5f400000u, 0u, 0x00000000u, 0xdf400000u}, + {0xe40a117du, 0xbf800000u, 0xe40a117du, 1u, 0xe40a117du, 0u, 0x00000000u, 0x640a117du}, + {0x3f800000u, 0xdf7fffffu, 0x5f7fffffu, 1u, 0x3f800000u, 0u, 0x00000000u, 0xbf800000u}, }; struct DivisionVector { diff --git a/core/libs/prx/libSceAgcDriver/tests/execution/DrawOffsetIndices.cpp b/core/libs/prx/libSceAgcDriver/tests/execution/DrawOffsetIndices.cpp new file mode 100644 index 000000000..6754f0c84 --- /dev/null +++ b/core/libs/prx/libSceAgcDriver/tests/execution/DrawOffsetIndices.cpp @@ -0,0 +1,500 @@ +#include "prx/libSceAgcDriver/Execution/include/Driver/Driver.hpp" +#include "prx/libSceAgcDriver/Execution/include/VulkanDevice.hpp" +#include "prx/libSceAgcDriver/Graphics/include/Draw.hpp" +#include "prx/libc/include/GuestAllocations.hpp" +#include "Recompiler.hpp" +#include "VulkanTestDevice.hpp" +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace { + +using AgcDriver::Graphics::Require; +using ShaderRecompiler::ShaderStage; +using Path = AgcDriver::Graphics::IndirectDrawPath; +using Rule = AgcDriver::Pm4::DrawParameters::IndirectDraw::Rule; + +constexpr std::uint32_t BaseVertexSgpr = 16; +constexpr std::uint32_t StartInstanceSgpr = 17; +constexpr std::uint32_t Vertices = 3; +constexpr std::uint32_t Instances = 2; +constexpr std::uint32_t Slots = Vertices * Instances; +constexpr std::uint32_t Unwritten = 0xdeadbeefu; +constexpr std::uint32_t NoLocation = 0x280u; + +alignas(256) constexpr std::array VertexCode{ + 0x7e020305, 0x7e040308, + 0xd70f6a05, 0x00020a10, + 0xd70f6a08, 0x00021011, + 0xf4080506, 0xfa000000, + 0xf4080606, 0xfa000010, + 0x02060b08, + 0xbf8cc07f, + 0xe0002000, 0x80050a03, + 0xe0002000, 0x80060b03, + 0xbf8c3f70, + 0x4a08170a, 0x34080884, + 0x7e180301, 0x7e1a0302, 0x7e1c0305, 0x7e1e0308, + 0xe0781000, 0x80020c04, + 0x7e200280, 0x7e2202f2, + 0xf80008cf, 0x11101010, + 0xbf810000, +}; + +alignas(256) constexpr std::array PostAddCode{ + 0x7e020280, 0x7e040280, + 0xd70f7d05, 0x00020a10, + 0xd70f7d08, 0x00021011, + 0xf4080506, 0xfa000000, + 0xf4080606, 0xfa000010, + 0x02060b08, + 0xbf8cc07f, + 0xe0002000, 0x80050a03, + 0xe0002000, 0x80060b03, + 0xbf8c3f70, + 0x4a08170a, 0x34080884, + 0x7e180301, 0x7e1a0302, 0x7e1c0305, 0x7e1e0308, + 0xe0781000, 0x80020c04, + 0x7e200280, 0x7e2202f2, + 0xf80008cf, 0x11101010, + 0xbf810000, +}; + +alignas(256) constexpr std::array CarryCode{ + 0x7e020280, 0x7e040280, + 0xd70f7d05, 0x00020a10, + 0xd70f6a08, 0x00021011, + 0xf4080506, 0xfa000000, + 0xf4080606, 0xfa000010, + 0x02060b08, + 0xbf8cc07f, + 0xe0002000, 0x80050a03, + 0xe0002000, 0x80060b03, + 0xbf8c3f70, + 0x4a08170a, 0x34080884, + 0x7e180301, 0x7e1a0302, 0x7e1c0305, 0x7e1e0308, + 0xe0781000, 0x80020c04, + 0x7e200280, 0x7e2202f2, + 0xf80008cf, 0x11101010, + 0xbf810000, +}; + +alignas(256) constexpr std::array ClampCode{ + 0x7e020280, 0x7e040280, + 0xd55d8005, 0x04150010, + 0xd70f7d08, 0x00021011, + 0xf4080506, 0xfa000000, + 0xf4080606, 0xfa000010, + 0x02060b08, + 0xbf8cc07f, + 0xe0002000, 0x80050a03, + 0xe0002000, 0x80060b03, + 0xbf8c3f70, + 0x4a08170a, 0x34080884, + 0x7e180301, 0x7e1a0302, 0x7e1c0305, 0x7e1e0308, + 0xe0781000, 0x80020c04, + 0x7e200280, 0x7e2202f2, + 0xf80008cf, 0x11101010, + 0xbf810000, +}; + +alignas(256) constexpr std::array TwiceCode{ + 0x7e020280, 0x7e040280, + 0xd70f7d05, 0x00020a10, + 0xd70f7d05, 0x00020a10, + 0xd70f7d08, 0x00021011, + 0xf4080506, 0xfa000000, + 0xf4080606, 0xfa000010, + 0x02060b08, + 0xbf8cc07f, + 0xe0002000, 0x80050a03, + 0xe0002000, 0x80060b03, + 0xbf8c3f70, + 0x4a08170a, 0x34080884, + 0x7e180301, 0x7e1a0302, 0x7e1c0305, 0x7e1e0308, + 0xe0781000, 0x80020c04, + 0x7e200280, 0x7e2202f2, + 0xf80008cf, 0x11101010, + 0xbf810000, +}; + +alignas(256) constexpr std::array ExecCode{ + 0xbefe03c1, 0x7e040280, + 0xd70f7d05, 0x00020a10, + 0xd70f7d08, 0x00021011, + 0xf4080506, 0xfa000000, + 0xf4080606, 0xfa000010, + 0x02060b08, + 0xbf8cc07f, + 0xe0002000, 0x80050a03, + 0xe0002000, 0x80060b03, + 0xbf8c3f70, + 0x4a08170a, 0x34080884, + 0x7e180301, 0x7e1a0302, 0x7e1c0305, 0x7e1e0308, + 0xe0781000, 0x80020c04, + 0x7e200280, 0x7e2202f2, + 0xf80008cf, 0x11101010, + 0xbf810000, +}; + +alignas(256) constexpr std::array CarryWriteCode{ + 0xd70f1102, 0x00020100, + 0xd70f7d05, 0x00020a10, + 0xd70f7d08, 0x00021011, + 0xf4080506, 0xfa000000, + 0xf4080606, 0xfa000010, + 0x02060b08, + 0xbf8cc07f, + 0xe0002000, 0x80050a03, + 0xe0002000, 0x80060b03, + 0xbf8c3f70, + 0x4a08170a, 0x34080884, + 0x7e180301, 0x7e1a0302, 0x7e1c0305, 0x7e1e0308, + 0xe0781000, 0x80020c04, + 0x7e200280, 0x7e2202f2, + 0xf80008cf, 0x11101010, + 0xbf810000, +}; + +alignas(256) constexpr std::array OtherVgprCode{ + 0x7e020280, 0x7e040280, + 0xd70f7d00, 0x00020010, + 0xd70f7d08, 0x00021011, + 0xf4080506, 0xfa000000, + 0xf4080606, 0xfa000010, + 0x02060b08, + 0xbf8cc07f, + 0xe0002000, 0x80050a03, + 0xe0002000, 0x80060b03, + 0xbf8c3f70, + 0x4a08170a, 0x34080884, + 0x7e180301, 0x7e1a0302, 0x7e1c0305, 0x7e1e0308, + 0xe0781000, 0x80020c04, + 0x7e200280, 0x7e2202f2, + 0xf80008cf, 0x11101010, + 0xbf810000, +}; + +alignas(256) constexpr std::array PixelCode{0x7e0002f2u, 0x7e020280u, 0xf800180fu, 0x00010100u, 0xbf810000u}; + +constexpr std::uint32_t Width = 64; +constexpr std::uint32_t Height = 32; +alignas(256) std::array Pixels{}; +alignas(256) std::array Output{}; +alignas(256) std::array VertexSlots{}; +alignas(256) std::array InstanceSlots{}; +alignas(256) std::array BufferTable{}; +alignas(256) constexpr std::array Indices{2, 0, 1, 0}; + +std::array SlotDescriptor(const void* data, std::uint32_t records) { + const auto address = reinterpret_cast(data); + return {static_cast(address), static_cast((address >> 32u) & 0xffffu) | (4u << 16u), records, 0x01014facu}; +} + +std::array OutputDescriptor() { + const auto address = reinterpret_cast(Output.data()); + return {static_cast(address), static_cast((address >> 32u) & 0xffffu), static_cast(sizeof(Output)), 0x31016facu}; +} + +struct Expected { + std::uint32_t vertex; + std::uint32_t instance; + std::uint32_t vertexSum; + std::uint32_t instanceSum; +}; + +class Fixture { +public: + explicit Fixture(std::shared_ptr device) : device(std::move(device)) { + const auto vertex = SlotDescriptor(VertexSlots.data(), static_cast(VertexSlots.size())); + const auto instance = SlotDescriptor(InstanceSlots.data(), static_cast(InstanceSlots.size())); + std::copy(vertex.begin(), vertex.end(), BufferTable.begin()); + std::copy(instance.begin(), instance.end(), BufferTable.begin() + 4); + records = static_cast(::operator new(RecordBytes, std::align_val_t{65536})); + GuestAllocations::Mutation mutation; + mutation.Add(records, RecordBytes, true, true, true); + } + + std::vector UserData(std::uint32_t baseVertex, std::uint32_t startInstance) const { + std::vector userData(10, 0u); + const auto output = OutputDescriptor(); + std::copy(output.begin(), output.end(), userData.begin()); + const auto table = reinterpret_cast(BufferTable.data()); + userData[4] = static_cast(table); + userData[5] = static_cast(table >> 32u); + userData[BaseVertexSgpr - 8u] = baseVertex; + userData[StartInstanceSgpr - 8u] = startInstance; + return userData; + } + + ShaderRecompiler::RecompileResult Vertex(std::span code, const std::vector& userData) const { + ShaderRecompiler::ShaderVertexStageInfo info{}; + info.resourcesNum = 2; + info.resources[0].fields = {BufferTable[0], BufferTable[1], BufferTable[2], BufferTable[3]}; + info.resources[1].fields = {BufferTable[4], BufferTable[5], BufferTable[6], BufferTable[7]}; + info.resourcesDst[0] = {10, 1, 0, 0}; + info.resourcesDst[1] = {11, 1, 1, 1}; + info.fetchAttribReg = 6; + info.fetchBufferReg = 4; + info.fetchEmbedded = true; + const std::array memory{{ + {reinterpret_cast(code.data()), std::as_bytes(code)}, + {reinterpret_cast(BufferTable.data()), std::as_bytes(std::span(BufferTable))} + }}; + ShaderRecompiler::RecompileRequest request{ + {ShaderStage::Vertex, reinterpret_cast(code.data()), code, 0, {}}, + {32, 8, userData, std::nullopt, std::nullopt, info, memory}, + device->Target(), + {0, 0, 0, 64} + }; + request.useCache = false; + auto result = ShaderRecompiler::Recompile(request); + Require(result.vertexOffsetSgpr == static_cast(BaseVertexSgpr) && result.instanceOffsetSgpr == static_cast(StartInstanceSgpr), "the fetch adds of s16 and s17 were not folded into the draw (vertex " + std::to_string(result.vertexOffsetSgpr) + ", instance " + std::to_string(result.instanceOffsetSgpr) + ")"); + return result; + } + + AgcDriver::Graphics::State State() const { + AgcDriver::Graphics::State state{}; + state.stages = {AgcDriver::Graphics::ShaderPath::Vertex, 0u, 32u, 32u, std::nullopt, std::nullopt}; + state.color = {reinterpret_cast(Pixels.data()), {Width, Height}, VK_FORMAT_R8G8B8A8_UNORM, Pixels.size(), 0xe4u}; + state.colors = {state.color}; + state.hasColorTarget = true; + state.renderExtent = {Width, Height}; + state.topology = VK_PRIMITIVE_TOPOLOGY_TRIANGLE_LIST; + state.viewport = {0.0f, static_cast(Height), static_cast(Width), -static_cast(Height), 0.0f, 1.0f}; + state.scissor = {{0, 0}, {Width, Height}}; + state.cullMode = VK_CULL_MODE_NONE; + state.frontFace = VK_FRONT_FACE_COUNTER_CLOCKWISE; + state.blend.colorWriteMask = 15u; + state.blends = {state.blend}; + return state; + } + + void Draw(const ShaderRecompiler::RecompileResult& vertex, const AgcDriver::Pm4::DrawParameters& draw) { + const auto vertexPush = static_cast(vertex.pushConstants.size()); + ShaderRecompiler::ShaderPixelStageInfo pixelInfo{}; + pixelInfo.wave32 = true; + pixelInfo.targetOutputMode[0] = 9u; + pixelInfo.targetExportMapping.fill(0xe4u); + ShaderRecompiler::RecompileRequest pixelRequest{{ShaderStage::Fragment, reinterpret_cast(PixelCode.data()), PixelCode, 0, {}}, {32u, 0u, {}, std::nullopt, pixelInfo, std::nullopt, {}}, device->Target(), PixelPushLayout(vertexPush, device->Target())}; + pixelRequest.useCache = false; + const auto pixel = ShaderRecompiler::Recompile(pixelRequest); + const std::array shaders{{{ShaderStage::Vertex, &vertex, 0u}, {ShaderStage::Fragment, &pixel, PixelPushOffset(vertexPush, device->Target())}}}; + Output.fill(Unwritten); + device->Draw(State(), draw, shaders); + device->WaitIdle(); + } + + void Place(std::uint32_t firstVertexFetch, std::span fetchedVertices, std::uint32_t firstInstanceFetch) { + VertexSlots.fill(Unwritten); + InstanceSlots.fill(Unwritten); + for (std::uint32_t vertex = 0; vertex < fetchedVertices.size(); ++vertex) VertexSlots[firstVertexFetch + fetchedVertices[vertex]] = vertex; + for (std::uint32_t instance = 0; instance < Instances; ++instance) InstanceSlots[firstInstanceFetch + instance] = instance * Vertices; + } + + std::string Check(const std::string& what, std::span rawVertices, std::uint32_t baseVertex, std::uint32_t startInstance, bool checkRaw) const { + std::string failures; + for (std::uint32_t instance = 0; instance < Instances; ++instance) { + for (std::uint32_t vertex = 0; vertex < Vertices; ++vertex) { + const auto slot = instance * Vertices + vertex; + const Expected expected{rawVertices[vertex], instance, rawVertices[vertex] + baseVertex, instance + startInstance}; + const auto* actual = Output.data() + slot * 4u; + const bool same = actual[2] == expected.vertexSum && actual[3] == expected.instanceSum && (!checkRaw || (actual[0] == expected.vertex && actual[1] == expected.instance)); + char line[320]; + if (checkRaw) std::snprintf(line, sizeof(line), "%s, vertex %u instance %u: v5 %u v8 %u, after the fetch adds %u %u; expected %u %u, %u %u\n", what.c_str(), vertex, instance, actual[0], actual[1], actual[2], actual[3], expected.vertex, expected.instance, expected.vertexSum, expected.instanceSum); + else std::snprintf(line, sizeof(line), "%s, vertex %u instance %u: after the fetch adds %u %u; expected %u %u\n", what.c_str(), vertex, instance, actual[2], actual[3], expected.vertexSum, expected.instanceSum); + std::cout << line; + if (!same) failures += line; + } + } + return failures; + } + + std::uint32_t* Records() const { + return records; + } + + std::shared_ptr device; + +private: + static constexpr std::size_t RecordBytes = 65536; + std::uint32_t* records = nullptr; +}; + +AgcDriver::DriverDetail::DrawProgram Program(const std::vector& userData) { + AgcDriver::DriverDetail::DrawProgram program{}; + program.firstUserSgpr = 8; + program.userData = userData; + return program; +} + +AgcDriver::Pm4::DrawParameters FoldedDraw(const ShaderRecompiler::RecompileResult& vertex, const std::vector& userData, AgcDriver::Pm4::DrawParameters draw) { + AgcDriver::DriverDetail::Driver::FoldDrawOffsets(vertex, Program(userData), draw); + return draw; +} + +std::string PathName(const std::optional& path) { + return path ? "the CPU path (" + std::string(AgcDriver::Graphics::IndirectDrawPathName(*path)) + ")" : std::string("the GPU"); +} + +std::string Classified(Fixture& fixture, const std::string& what, const ShaderRecompiler::RecompileResult& vertex, const std::vector& userData, AgcDriver::Pm4::DrawParameters& draw, std::optional expectedPath, Rule expectedRule) { + const auto path = AgcDriver::DriverDetail::Driver::ClassifyIndirectDraw(vertex, fixture.State(), Program(userData), fixture.device, draw, false); + const auto& indirect = *draw.indirect; + const bool rules = expectedPath || (indirect.vertexRule == expectedRule && indirect.instanceRule == expectedRule); + std::cout << what << ": classified to " << PathName(path) << '\n'; + if (path == expectedPath && rules) return {}; + return what + ": classified to " + PathName(path) + ", expected " + PathName(expectedPath) + (expectedPath ? "" : expectedRule == Rule::InPlace ? " with both dimensions in place" : " with both dimensions constant") + "\n"; +} + +std::string Analysis(Fixture& fixture) { + struct Case { + const char* what; + std::span code; + bool vertex; + bool instance; + }; + const std::array cases{{ + {"a program that copies v5 and v8 before its fetch adds", VertexCode, true, true}, + {"a program whose fetch adds are the only readers of v5, v8, s16 and s17", PostAddCode, false, false}, + {"a program that reads the instance add's carry", CarryCode, false, true}, + {"a program whose vertex add is a clamped v_sad_u32", ClampCode, true, false}, + {"a program that adds s16 twice", TwiceCode, true, false}, + {"a program that writes EXEC before its fetch adds", ExecCode, true, true}, + {"a program that writes another add's carry to s17 before its fetch adds", CarryWriteCode, false, true}, + {"a program whose s16 add writes v0, not v5", OtherVgprCode, true, false}, + }}; + const auto userData = fixture.UserData(0x51u, 0x62u); + std::string failures; + for (const auto& entry : cases) { + const auto result = fixture.Vertex(entry.code, userData); + char line[256]; + std::snprintf(line, sizeof(line), "%s: folded value observable outside the fetch add: vertex %d instance %d; expected %d %d\n", entry.what, result.vertexOffsetShared ? 1 : 0, result.instanceOffsetShared ? 1 : 0, entry.vertex ? 1 : 0, entry.instance ? 1 : 0); + std::cout << line; + if (result.vertexOffsetShared != entry.vertex || result.instanceOffsetShared != entry.instance) failures += line; + } + return failures; +} + +std::string Run(Fixture& fixture) { + std::string failures = Analysis(fixture); + const std::array autoIndices{0, 1, 2}; + const std::array bufferIndices{Indices[0], Indices[1], Indices[2]}; + const auto indexAddress = reinterpret_cast(Indices.data()); + + { + constexpr std::uint32_t indxOffset = 5; + constexpr std::uint32_t baseVertex = 1000; + constexpr std::uint32_t startInstance = 40; + const auto userData = fixture.UserData(baseVertex, startInstance); + const auto vertex = fixture.Vertex(VertexCode, userData); + const auto draw = FoldedDraw(vertex, userData, {0, Vertices, 0, Instances, 0, false, indxOffset, 0}); + fixture.Place(indxOffset + baseVertex, autoIndices, startInstance); + fixture.Draw(vertex, draw); + const std::array raw{indxOffset, indxOffset + 1, indxOffset + 2}; + failures += fixture.Check("direct auto draw, GE_INDX_OFFSET 5, s16 1000, s17 40", raw, baseVertex, startInstance, true); + } + { + constexpr std::uint32_t baseVertex = 1000; + constexpr std::uint32_t startInstance = 40; + const auto userData = fixture.UserData(baseVertex, startInstance); + const auto vertex = fixture.Vertex(VertexCode, userData); + const auto draw = FoldedDraw(vertex, userData, {indexAddress, Vertices, 2, Instances, 0, true, 0, 0}); + fixture.Place(baseVertex, bufferIndices, startInstance); + fixture.Draw(vertex, draw); + failures += fixture.Check("direct indexed draw, s16 1000, s17 40", bufferIndices, baseVertex, startInstance, true); + } + + const auto staleUserData = fixture.UserData(0x51u, 0x62u); + const std::array indexedRecord{Vertices, Instances, 0, 700, 59455}; + std::copy(indexedRecord.begin(), indexedRecord.end(), fixture.Records()); + AgcDriver::Pm4::DrawParameters::IndirectDraw indexedIndirect{reinterpret_cast(fixture.Records()), 0x25u, 20u, 20u, 1u, false, 0u, 0x10u, 0x11u, NoLocation, false, 0u}; + indexedIndirect.baseVertexSgpr = static_cast(BaseVertexSgpr); + indexedIndirect.startInstanceSgpr = static_cast(StartInstanceSgpr); + { + const std::string what = "indexed indirect record whose program copies v5/v8 before its fetch adds, vertex offset 700, start instance 59455"; + AgcDriver::Pm4::DrawParameters packet{indexAddress, static_cast(Indices.size()), 2, 0, 0, true, 0, 0}; + packet.indirect = indexedIndirect; + failures += Classified(fixture, what, fixture.Vertex(VertexCode, staleUserData), staleUserData, packet, Path::NotFolded, Rule::Constant); + const auto arguments = AgcDriver::Pm4::ReadDrawArguments(indexedIndirect, 0); + const auto userData = fixture.UserData(arguments.vertexOffset, arguments.firstInstance); + const auto vertex = fixture.Vertex(VertexCode, userData); + const auto draw = FoldedDraw(vertex, userData, {indexAddress + arguments.firstVertexOrIndex * 2u, arguments.count, 2, arguments.instances, 0, true, 0, 0}); + fixture.Place(arguments.vertexOffset, bufferIndices, arguments.firstInstance); + fixture.Draw(vertex, draw); + failures += fixture.Check(what + ", on the CPU path", bufferIndices, arguments.vertexOffset, arguments.firstInstance, true); + } + { + const std::string what = "auto indirect record whose program copies v5/v8 before its fetch adds, start vertex 300, start instance 7"; + const std::array record{Vertices, Instances, 300, 7}; + std::copy(record.begin(), record.end(), fixture.Records() + 16); + AgcDriver::Pm4::DrawParameters::IndirectDraw indirect{reinterpret_cast(fixture.Records() + 16), 0x24u, 16u, 16u, 1u, false, 0u, 0x10u, 0x11u, NoLocation, false, 0u}; + indirect.baseVertexSgpr = static_cast(BaseVertexSgpr); + indirect.startInstanceSgpr = static_cast(StartInstanceSgpr); + AgcDriver::Pm4::DrawParameters packet{0, 0, 0, 0, 0, false, 0, 0}; + packet.indirect = indirect; + failures += Classified(fixture, what, fixture.Vertex(VertexCode, staleUserData), staleUserData, packet, Path::NotFolded, Rule::Constant); + const auto arguments = AgcDriver::Pm4::ReadDrawArguments(indirect, 0); + const auto userData = fixture.UserData(arguments.firstVertexOrIndex, arguments.firstInstance); + const auto vertex = fixture.Vertex(VertexCode, userData); + const auto draw = FoldedDraw(vertex, userData, {0, arguments.count, 0, arguments.instances, 0, false, indirect.indxOffset, 0}); + fixture.Place(arguments.firstVertexOrIndex, autoIndices, arguments.firstInstance); + fixture.Draw(vertex, draw); + failures += fixture.Check(what + ", on the CPU path", autoIndices, arguments.firstVertexOrIndex, arguments.firstInstance, true); + } + if (fixture.device->DrawIndirectSupport().firstInstance) { + { + const std::string what = "indexed indirect record whose fetch adds are the only readers of v5/v8, user data s16 0x51, s17 0x62"; + const auto vertex = fixture.Vertex(PostAddCode, staleUserData); + AgcDriver::Pm4::DrawParameters draw{indexAddress, static_cast(Indices.size()), 2, 0, 0, true, 0, 0}; + draw.indirect = indexedIndirect; + failures += Classified(fixture, what, vertex, staleUserData, draw, std::nullopt, Rule::InPlace); + fixture.Place(indexedRecord[3], bufferIndices, indexedRecord[4]); + fixture.Draw(vertex, draw); + failures += fixture.Check(what + ", drawn in place on the GPU", bufferIndices, indexedRecord[3], indexedRecord[4], false); + } + { + const std::string what = "indexed indirect record whose packet names no register for its offsets, user data s16 0x51, s17 0x62"; + const auto vertex = fixture.Vertex(VertexCode, staleUserData); + AgcDriver::Pm4::DrawParameters draw{indexAddress, static_cast(Indices.size()), 2, 0, 0, true, 0, 0}; + draw.indirect = indexedIndirect; + draw.indirect->baseVertexLocation = NoLocation; + draw.indirect->startInstanceLocation = NoLocation; + draw.indirect->baseVertexSgpr = -1; + draw.indirect->startInstanceSgpr = -1; + failures += Classified(fixture, what, vertex, staleUserData, draw, std::nullopt, Rule::Constant); + fixture.Place(0x51u, bufferIndices, 0x62u); + fixture.Draw(vertex, draw); + failures += fixture.Check(what + ", drawn on the GPU", bufferIndices, 0x51u, 0x62u, true); + } + } + return failures; +} + +} + +int main() { + try { + std::shared_ptr device = OpenVulkanTestDevice(); + if (!device) return VulkanTestSkipped; + Fixture fixture(device); + const auto failures = Run(fixture); + if (!failures.empty()) { + std::cerr << "vertex and instance indices differ from the guest's:\n" << failures; + return 1; + } + std::cout << "draw offset index tests passed\n"; + return 0; + } catch (const std::exception& error) { + std::cerr << error.what() << '\n'; + return 1; + } +} diff --git a/core/libs/prx/libSceAgcDriver/tests/execution/DualSourceBlend.cpp b/core/libs/prx/libSceAgcDriver/tests/execution/DualSourceBlend.cpp new file mode 100644 index 000000000..043d19548 --- /dev/null +++ b/core/libs/prx/libSceAgcDriver/tests/execution/DualSourceBlend.cpp @@ -0,0 +1,353 @@ +#include "prx/libSceAgcDriver/Execution/include/GuestMemory.hpp" +#include "prx/libSceAgcDriver/Execution/include/VulkanDevice.hpp" +#include "prx/libSceAgcDriver/Execution/include/Driver/Shaders/ShaderRegistry.hpp" +#include "prx/libSceAgcDriver/Graphics/include/ColorTargetLayout.hpp" +#include "prx/libSceAgcDriver/Graphics/include/Draw.hpp" +#include "prx/libSceAgcDriver/Graphics/include/ShaderInputState.hpp" +#include "prx/libSceAgcDriver/Graphics/include/State.hpp" +#include "prx/libSceAgcDriver/Graphics/include/Texture.hpp" +#include "prx/libc/include/GuestAllocations.hpp" +#include "prx/libc/include/GuestArena.hpp" +#include "prx/libc/include/GuestWriteWatch.hpp" +#include "Recompiler.hpp" +#include "Optimization/ResourceProgram.hpp" +#include "VulkanTestDevice.hpp" +#ifdef _WIN32 +#ifndef NOMINMAX +#define NOMINMAX +#endif +#include +#else +#include +#endif +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace { + +using AgcDriver::Graphics::ColorTargetLayout; +using AgcDriver::Graphics::ColorTileMode; +using AgcDriver::Graphics::Require; +using ShaderRecompiler::ShaderStage; +using Color = std::array; + +constexpr std::uint32_t Width = 256; +constexpr std::uint32_t Height = 128; +constexpr std::uint32_t ScissorWidth = 128; +constexpr std::size_t BlockBytes = 65536; +constexpr std::uint32_t Float16x4Info = 0x0730u; +constexpr std::uint32_t TiledAttrib3 = 0x4dc6c000u; +constexpr std::uint32_t Float32Export = 9; +constexpr std::uint32_t Float16Export = 4; + +alignas(256) constexpr std::array VertexCode{ + 0xe0382000, 0x80000005, 0xbf8c3f70, 0xf80008cf, 0x03020100, 0xbf810000, +}; + +constexpr std::array, 3> Triangle{{ + {-1.0f, -1.0f, 0.5f, 1.0f}, {3.0f, -1.0f, 0.5f, 1.0f}, {-1.0f, 3.0f, 0.5f, 1.0f} +}}; + +constexpr std::array Destinations{{ + {0.5f, 0.25f, 0.75f, 0.5f}, {1.0f, 0.0f, 0.125f, 0.25f}, {0.0625f, 0.875f, 0.5f, 1.0f}, {0.25f, 0.5f, 0.375f, 0.75f} +}}; + +alignas(256) std::array, 8> Programs{}; +std::size_t programCount = 0; + +std::uint16_t Half(float value) { + Require(value == 0.0f || (value >= 0.0625f && value <= 1.0f), "the test's half values are normal and at most 1"); + if (value == 0.0f) return 0; + const auto bits = std::bit_cast(value); + Require((bits & 0x1fffu) == 0, "the test's half values are exact"); + return static_cast(((((bits >> 23u) & 0xffu) - 112u) << 10u) | ((bits >> 13u) & 0x3ffu)); +} + +float FromHalf(std::uint16_t half) { + const auto exponent = (half >> 10u) & 0x1fu; + const auto mantissa = half & 0x3ffu; + Require((half & 0x8000u) == 0 && exponent != 31u, "a blended texel is negative, infinite or NaN"); + return exponent == 0 ? std::ldexp(static_cast(mantissa), -24) : std::ldexp(1.0f + static_cast(mantissa) / 1024.0f, static_cast(exponent) - 15); +} + +std::span Program(const Color& first, const Color& second, bool compressed) { + Require(programCount < Programs.size(), "too many test programs"); + auto& code = Programs[programCount++]; + std::size_t at = 0; + const auto move = [&](std::uint32_t vgpr, std::uint32_t value) { + code[at++] = 0x7e0002ffu | (vgpr << 17u); + code[at++] = value; + }; + if (compressed) { + move(4, Half(first[0]) | (Half(first[1]) << 16u)); + move(5, Half(first[2]) | (Half(first[3]) << 16u)); + move(6, Half(second[0]) | (Half(second[1]) << 16u)); + move(7, Half(second[2]) | (Half(second[3]) << 16u)); + code[at++] = 0xf800040fu; + code[at++] = 0x00000504u; + code[at++] = 0xf8001c1fu; + code[at++] = 0x00000706u; + } else { + for (std::uint32_t channel = 0; channel < 4; ++channel) move(4 + channel, std::bit_cast(first[channel])); + for (std::uint32_t channel = 0; channel < 4; ++channel) move(8 + channel, std::bit_cast(second[channel])); + code[at++] = 0xf800000fu; + code[at++] = 0x07060504u; + code[at++] = 0xf800181fu; + code[at++] = 0x0b0a0908u; + } + code[at++] = 0xbf810000u; + return std::span(code.data(), at); +} + +struct Block { + explicit Block(std::size_t bytes) : bytes(bytes), watched(AgcDriver::GuestMemory::WriteWatched()) { +#ifdef _WIN32 + if (watched) { + data = static_cast(GuestArena::GuestArenaAllocate_nid_postfix(bytes, BlockBytes)); + GuestArena::GuestArenaCommit_nid_postfix(data, bytes, PAGE_READWRITE, bytes); + } else { + data = static_cast(VirtualAlloc(nullptr, bytes, MEM_RESERVE | MEM_COMMIT, PAGE_READWRITE)); + } +#else + if (watched) { + void* raw = mmap(nullptr, bytes + BlockBytes, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + Require(raw != MAP_FAILED, "cannot map the color target"); + const auto begin = reinterpret_cast(raw); + const auto aligned = (begin + BlockBytes - 1) & ~(static_cast(BlockBytes) - 1); + if (aligned != begin) munmap(raw, aligned - begin); + if (aligned + bytes != begin + bytes + BlockBytes) munmap(reinterpret_cast(aligned + bytes), begin + BlockBytes - aligned); + data = reinterpret_cast(aligned); + GuestWriteWatch::GuestWriteWatchRegister_nid_postfix(data, bytes); + } else { + data = static_cast(std::aligned_alloc(BlockBytes, bytes)); + } +#endif + Require(data != nullptr, "cannot allocate the color target"); + Require(!watched || AgcDriver::GuestMemory::Watched(Address(), bytes), "the color target is not write-watched"); + GuestAllocations::Mutation mutation; + mutation.Add(data, bytes, true, true, true); + } + ~Block() { + AgcDriver::Graphics::StorageTexture::FlushPending(Address(), bytes, nullptr, "test release"); + { + GuestAllocations::Mutation mutation; + mutation.Remove(data); + } +#ifdef _WIN32 + if (watched) { + GuestArena::GuestArenaReset_nid_postfix(data, bytes); + GuestArena::GuestArenaRelease_nid_postfix(data, bytes); + } else { + VirtualFree(data, 0, MEM_RELEASE); + } +#else + if (watched) { + munmap(data, bytes); + GuestWriteWatch::GuestWriteWatchUnregister_nid_postfix(data, bytes); + } else { + std::free(data); + } +#endif + } + Block(const Block&) = delete; + Block& operator=(const Block&) = delete; + std::uint64_t Address() const { return reinterpret_cast(data); } + std::size_t bytes; + bool watched; + std::uint8_t* data = nullptr; +}; + +AgcDriver::Graphics::ColorTarget DecodeTarget(const Block& block) { + AgcDriver::Registers cx; + cx[0x318] = static_cast(block.Address() >> 8u); + cx[0x31b] = 0; + cx[0x31c] = Float16x4Info; + cx[0x31d] = 0; + cx[0x3b0] = ((Width - 1u) << 14u) | (Height - 1u); + cx[0x3b8] = TiledAttrib3; + cx[0x390] = static_cast(block.Address() >> 40u); + const auto color = AgcDriver::Graphics::DecodeColorBuffer(cx, 0); + Require(color.format == VK_FORMAT_R16G16B16A16_SFLOAT && color.elementBytes == 8 && color.bytes == block.bytes, "the 16_16_16_16 float target did not decode as a 256x128 RGBA16F surface"); + return color; +} + +VkPipelineColorBlendAttachmentState Blend(VkBlendFactor source, VkBlendFactor destination, VkBlendFactor sourceAlpha, VkBlendFactor destinationAlpha) { + VkPipelineColorBlendAttachmentState blend{}; + blend.blendEnable = VK_TRUE; + blend.srcColorBlendFactor = source; + blend.dstColorBlendFactor = destination; + blend.colorBlendOp = VK_BLEND_OP_ADD; + blend.srcAlphaBlendFactor = sourceAlpha; + blend.dstAlphaBlendFactor = destinationAlpha; + blend.alphaBlendOp = VK_BLEND_OP_ADD; + blend.colorWriteMask = 0xf; + return blend; +} + +void Draw(AgcDriver::VulkanDevice& device, const AgcDriver::Graphics::ColorTarget& color, const VkPipelineColorBlendAttachmentState& blend, std::uint32_t exportFormat, std::span pixelCode) { + const auto target = device.Target(); + constexpr std::uint32_t waveSize = 64; + std::vector vertexUserData(4, 0u); + const auto address = reinterpret_cast(Triangle.data()); + const std::array vertexBuffer{static_cast(address), static_cast((address >> 32u) & 0xffffu) | (16u << 16u), static_cast(Triangle.size()), 0x01016facu}; + std::copy(vertexBuffer.begin(), vertexBuffer.end(), vertexUserData.begin()); + const std::array vertexMemory{{{reinterpret_cast(VertexCode.data()), std::as_bytes(std::span(VertexCode))}}}; + ShaderRecompiler::RecompileRequest vertex{ + {ShaderStage::Vertex, reinterpret_cast(VertexCode.data()), VertexCode, 0, {}}, + {waveSize, 0, vertexUserData, std::nullopt, std::nullopt, ShaderRecompiler::ShaderVertexStageInfo{}, vertexMemory}, + target, + {0, 0, 0, 64} + }; + vertex.useCache = false; + const auto vertexResult = ShaderRecompiler::Recompile(vertex); + const auto vertexPush = static_cast(vertexResult.pushConstants.size()); + + AgcDriver::Graphics::State state{}; + state.stages = {AgcDriver::Graphics::ShaderPath::Vertex, 0u, waveSize, waveSize, std::nullopt, std::nullopt}; + state.colors = {color}; + state.color = color; + state.hasColorTarget = true; + state.renderExtent = {Width, Height}; + state.topology = VK_PRIMITIVE_TOPOLOGY_TRIANGLE_LIST; + state.viewport = {0, static_cast(Height), static_cast(Width), -static_cast(Height), 0, 1}; + state.negativeOneToOne = false; + state.scissor = {{0, 0}, {ScissorWidth, Height}}; + state.cullMode = VK_CULL_MODE_NONE; + state.frontFace = VK_FRONT_FACE_COUNTER_CLOCKWISE; + state.blends = {blend}; + state.blend = blend; + state.blendConstants = {}; + state.dualSourceBlend = true; + + const auto codeAddress = reinterpret_cast(pixelCode.data()); + const AgcDriver::Registers context{{0x1b6, 0x8000}, {0x1b3, 0}, {0x1b4, 0}, {0x203, 0}, {0x1c5, exportFormat | (exportFormat << 4u)}}; + const std::array pixelMemory{{{codeAddress, std::as_bytes(pixelCode)}}}; + ShaderRecompiler::RecompileRequest fragment{ + {ShaderStage::Fragment, codeAddress, pixelCode, 0, {}}, + {waveSize, 0, {}, std::nullopt, AgcDriver::Graphics::DecodePixelStageInfo(context, {}), std::nullopt, pixelMemory}, + target, + PixelPushLayout(vertexPush, target) + }; + const auto handle = ShaderRecompiler::PrepareShader(fragment); + AgcDriver::DriverDetail::ShaderSnapshot snapshot{codeAddress, 0, 1, {pixelCode.begin(), pixelCode.end()}, {}}; + snapshot.prepared->entries.push_back({0, handle}); + fragment.shader.code = snapshot.code; + fragment.context.pixel = AgcDriver::Graphics::DecodePixelStageInfo(context, AgcDriver::Graphics::ExportMappings(state)); + fragment.context.pixel->dualSourceBlend = state.dualSourceBlend; + const ShaderRecompiler::SrtRuntime runtime{}; + const auto materialize = [&](const ShaderRecompiler::RecompileRequest& request) { + const auto invocation = AgcDriver::DriverDetail::InvocationFor(snapshot, 0, request); + return invocation.Materialize(*invocation.Capture(runtime)); + }; + const auto pixelResult = materialize(fragment); + auto single = fragment; + single.context.pixel->dualSourceBlend = false; + const auto plain = materialize(single); + Require(pixelResult->variantId == ShaderRecompiler::GetPreparedArtifact(*handle).variantId && plain->variantId == pixelResult->variantId, "the draw did not use the registered pixel shader"); + Require(plain->PipelineVariantId() != pixelResult->PipelineVariantId() && materialize(fragment)->PipelineVariantId() == pixelResult->PipelineVariantId(), "dual-source blending did not key its own specialization"); + const std::array shaders{{ + {ShaderStage::Vertex, &vertexResult, 0}, + {ShaderStage::Fragment, pixelResult.get(), PixelPushOffset(vertexPush, target)} + }}; + const AgcDriver::Pm4::DrawParameters draw{0, static_cast(Triangle.size()), 0, 1, 0, false}; + device.Draw(state, draw, shaders); +} + +std::vector ReadBack(AgcDriver::VulkanDevice& device, const Block& block) { + AgcDriver::Graphics::StorageTexture::FlushPending(block.Address(), block.bytes, nullptr, "test read-back"); + device.WaitIdle(); + std::vector stored(block.bytes / 2u); + AgcDriver::GuestMemory::Read(block.Address(), std::as_writable_bytes(std::span(stored)), 1); + return stored; +} + +std::vector Fill(Block& block, const ColorTargetLayout& layout) { + std::vector halves(block.bytes / 2u); + for (std::uint32_t y = 0; y < Height; ++y) { + for (std::uint32_t x = 0; x < Width; ++x) { + const auto& color = Destinations[(x + y) % Destinations.size()]; + const auto index = layout.Offset(x, y) / 2u; + for (std::uint32_t channel = 0; channel < 4; ++channel) halves[index + channel] = Half(color[channel]); + } + } + std::memcpy(block.data, halves.data(), block.bytes); + return halves; +} + +void Check(AgcDriver::VulkanDevice& device, const Block& block, const ColorTargetLayout& layout, const std::vector& before, const std::function& blended, const std::string& what) { + const auto stored = ReadBack(device, block); + for (std::uint32_t y = 0; y < Height; ++y) { + for (std::uint32_t x = 0; x < Width; ++x) { + const auto index = layout.Offset(x, y) / 2u; + const auto expected = blended(Destinations[(x + y) % Destinations.size()]); + for (std::uint32_t channel = 0; channel < 4; ++channel) { + const bool inside = x < ScissorWidth; + const bool match = inside ? std::fabs(FromHalf(stored[index + channel]) - expected[channel]) <= 1.0f / 2048.0f : stored[index + channel] == before[index + channel]; + if (match) continue; + char detail[160]; + std::snprintf(detail, sizeof(detail), ": channel %u of pixel (%u, %u) %s the scissor holds half 0x%04x, expected %.6g", channel, x, y, inside ? "inside" : "outside", stored[index + channel], inside ? expected[channel] : FromHalf(before[index + channel])); + throw std::runtime_error(what + detail); + } + } + } +} + +struct BlendCase { + const char* name; + VkPipelineColorBlendAttachmentState blend; + std::function color; + std::function alpha; +}; + +} + +int main() { + try { + const auto device = OpenVulkanTestDevice(); + if (!device) return VulkanTestSkipped; + if (!device->DualSrcBlend()) { + std::puts("skipped, the device has no dual-source blending"); + return VulkanTestSkipped; + } + const ColorTargetLayout layout(Width, Height, ColorTileMode::RenderTarget, 8); + Block block(layout.Bytes()); + const Color first{0.25f, 0.5f, 0.125f, 0.5f}; + const Color second{0.5f, 0.25f, 1.0f, 0.75f}; + const std::array cases{{ + {"ONE + SRC1_COLOR", Blend(VK_BLEND_FACTOR_ONE, VK_BLEND_FACTOR_SRC1_COLOR, VK_BLEND_FACTOR_ONE, VK_BLEND_FACTOR_SRC1_COLOR), + [](float s, float t, float d, float) { return s + t * d; }, [](float s, float t, float d) { return s + t * d; }}, + {"SRC1_ALPHA + ONE_MINUS_SRC1_ALPHA", Blend(VK_BLEND_FACTOR_SRC1_ALPHA, VK_BLEND_FACTOR_ONE_MINUS_SRC1_ALPHA, VK_BLEND_FACTOR_ZERO, VK_BLEND_FACTOR_ONE_MINUS_SRC1_ALPHA), + [](float s, float, float d, float a) { return s * a + d * (1.0f - a); }, [](float, float t, float d) { return d * (1.0f - t); }}, + {"ONE_MINUS_SRC1_COLOR + ZERO", Blend(VK_BLEND_FACTOR_ONE_MINUS_SRC1_COLOR, VK_BLEND_FACTOR_ZERO, VK_BLEND_FACTOR_SRC1_ALPHA, VK_BLEND_FACTOR_ZERO), + [](float s, float t, float, float) { return s * (1.0f - t); }, [](float s, float t, float) { return s * t; }}, + }}; + for (const bool compressed : {false, true}) { + for (const auto& test : cases) { + const auto before = Fill(block, layout); + const auto color = DecodeTarget(block); + Draw(*device, color, test.blend, compressed ? Float16Export : Float32Export, Program(first, second, compressed)); + Check(*device, block, layout, before, [&](const Color& destination) { + Color result{}; + for (std::uint32_t channel = 0; channel < 3; ++channel) result[channel] = test.color(first[channel], second[channel], destination[channel], second[3]); + result[3] = test.alpha(first[3], second[3], destination[3]); + return result; + }, std::string(test.name) + (compressed ? " with FP16_ABGR exports" : " with 32_ABGR exports")); + } + } + std::puts("dual-source blend tests passed"); + return 0; + } catch (const std::exception& error) { + std::cerr << error.what() << '\n'; + return 1; + } +} diff --git a/core/libs/prx/libSceAgcDriver/tests/execution/DwordPatternFill.cpp b/core/libs/prx/libSceAgcDriver/tests/execution/DwordPatternFill.cpp new file mode 100644 index 000000000..9c0191c6f --- /dev/null +++ b/core/libs/prx/libSceAgcDriver/tests/execution/DwordPatternFill.cpp @@ -0,0 +1,142 @@ +#include "prx/libSceAgcDriver/Execution/include/Driver.hpp" +#include "prx/libSceAgcDriver/Execution/include/Driver/Memory/DwordPatternFill.hpp" +#include "prx/libSceAgcDriver/Execution/include/GuestMemory.hpp" +#include "prx/libSceAgcDriver/Graphics/include/Draw.hpp" +#include "VulkanTestDevice.hpp" +#include +#include +#include +#include +#include +#include +#include + +namespace { + +using AgcDriver::Graphics::Require; +using AgcDriver::DriverDetail::MatchDwordPatternFill; + +alignas(256) constexpr std::array Code{ + 0xbfa00003u, 0xd7460002u, 0x04010c0au, 0x7da80408u, 0xbf88003fu, 0x7e000c09u, 0xbf070980u, 0x858a807eu, + 0x7e005700u, 0x100000ffu, 0x4f800000u, 0x7e060f00u, 0xd5766a00u, 0x02020609u, 0x7d8a0280u, 0x4c020080u, + 0x02000101u, 0xd56a0001u, 0x00020700u, 0x4c000303u, 0x4a020303u, 0x02000101u, 0xd56a0000u, 0x00020500u, + 0xd5690001u, 0x00020009u, 0x4c060302u, 0x7d860609u, 0x7d8c02f9u, 0x06068c02u, 0x87ea6a0cu, 0x50000080u, + 0xd5286a00u, 0x003200c1u, 0xd5010000u, 0x002a00c1u, 0xd5690000u, 0x00020009u, 0x4c000102u, 0x7d0a0080u, + 0xbe88246au, 0xbf880015u, 0x7d0a0081u, 0xbe8a246au, 0xbf88000cu, 0x7d0a0082u, 0xbeea246au, 0xbf880003u, + 0x7e000207u, 0xe0102000u, 0x80000002u, 0x8afe7e6au, 0xbf880003u, 0x7e000206u, 0xe0102000u, 0x80000002u, + 0xbefe046au, 0x8afe7e0au, 0xbf880003u, 0x7e000205u, 0xe0102000u, 0x80000002u, 0xbefe040au, 0x8afe7e08u, + 0xbf880003u, 0x7e000204u, 0xe0102000u, 0x80000002u, 0xbf810000u}; +constexpr std::array Pattern{0x11111111u, 0x22222222u, 0x33333333u, 0x44444444u}; +constexpr std::uint32_t Untouched = 0xcafebabeu; +alignas(256) std::array Output{}; + +std::vector UserData(void* output, std::uint32_t records, std::uint32_t count, std::uint32_t period, std::uint32_t format = 0x00014004u) { + const auto address = reinterpret_cast(output); + return {static_cast(address), static_cast((address >> 32u) & 0xffffu) | (4u << 16u), records, format, Pattern[0], Pattern[1], Pattern[2], Pattern[3], count, period}; +} + +void MatchTests() { + const ShaderRecompiler::ShaderComputeStageInfo compute{{64, 1, 1}, 0, {true, false, false}, false, 1}; + const std::array packet{0xc0031500u, 4, 1, 1, 0x41u}; + const auto data = UserData(Output.data(), 1024, 256, 4); + const auto fill = MatchDwordPatternFill(packet, Code, data, compute); + Require(fill && fill->base == reinterpret_cast(Output.data()) && fill->bytes == 1024 && fill->pattern == Pattern, "the dword pattern-fill kernel was not recognized as a 16-byte pattern fill"); + for (const auto period : {1u, 2u}) { + const auto narrow = MatchDwordPatternFill(packet, Code, UserData(Output.data(), 1024, 256, period), compute); + Require(narrow && narrow->pattern[1] == Pattern[1 % period] && narrow->pattern[3] == Pattern[3 % period], "a shorter pattern period was not repeated over 16 bytes"); + } + for (std::size_t word = 0; word < Code.size(); ++word) { + auto altered = Code; + altered[word] ^= 1u; + Require(!MatchDwordPatternFill(packet, altered, data, compute), "a different shader matched the dword pattern-fill kernel"); + } + Require(!MatchDwordPatternFill(packet, std::span(Code).first(68), data, compute), "a truncated shader matched"); + Require(!MatchDwordPatternFill(packet, Code, UserData(Output.data(), 1024, 256, 3), compute), "a three-dword period matched a 16-byte pattern"); + Require(!MatchDwordPatternFill(packet, Code, UserData(Output.data(), 1024, 250, 4), compute), "a count of whole dwords not filling 16-byte units matched"); + Require(!MatchDwordPatternFill(packet, Code, UserData(Output.data() + 1, 1024, 256, 4), compute), "an unaligned destination matched"); + Require(!MatchDwordPatternFill(packet, Code, UserData(Output.data(), 1024, 256, 4, 0x00814004u), compute), "an add-tid descriptor matched"); + auto extra = data; + extra.push_back(0); + Require(!MatchDwordPatternFill(packet, Code, extra, compute), "a different user SGPR count matched (the group id moves)"); + auto partial = packet; + partial[4] |= 0x20u; + Require(!MatchDwordPatternFill(partial, Code, data, compute), "a partial dispatch matched"); + auto noGroup = compute; + noGroup.groupIdEnable[0] = false; + Require(!MatchDwordPatternFill(packet, Code, data, noGroup), "a dispatch without the group id matched"); + const auto clamped = MatchDwordPatternFill(packet, Code, UserData(Output.data(), 64, 256, 4), compute); + Require(clamped && clamped->bytes == 256, "the fill exceeded the descriptor's records"); +} + +std::vector Commands(const std::vector& data, std::uint32_t groups, bool partial, std::uint32_t threads) { + std::vector words; + const auto registers = [&](std::uint32_t first, std::span values) { + words.push_back(0xc0007600u | (static_cast(values.size()) << 16u)); + words.push_back(first); + words.insert(words.end(), values.begin(), values.end()); + }; + const std::array shape{64, 1, 1}; + registers(0x207, shape); + const auto address = reinterpret_cast(Code.data()); + const std::array program{static_cast(address >> 8u), static_cast(address >> 40u)}; + registers(0x20c, program); + const std::array resource{(static_cast(data.size()) << 1u) | (1u << 7u)}; + registers(0x213, resource); + registers(0x240, data); + words.insert(words.end(), {0xc0031500u, partial ? threads : groups, 1, 1, 0x41u | (partial ? 0x20u : 0u)}); + return words; +} + +void ExecuteTests() { + struct Case { + std::uint32_t groups, records, count, period; + bool partial; + std::uint32_t threads; + }; + constexpr std::array cases{{ + {4, 1024, 256, 4, false, 0}, + {4, 1024, 256, 2, false, 0}, + {4, 1024, 256, 1, false, 0}, + {16, 1024, 1024, 4, false, 0}, + {4, 1024, 100, 4, false, 0}, + {4, 1024, 250, 4, false, 0}, + {4, 64, 256, 4, false, 0}, + {4, 1024, 256, 3, false, 0}, + {4, 1024, 256, 5, false, 0}, + {0, 1024, 256, 4, true, 70}, + }}; + for (const auto& item : cases) { + Output.fill(Untouched); + const auto words = Commands(UserData(Output.data(), item.records, item.count, item.period), item.groups, item.partial, item.threads); + Packet packet{const_cast(words.data()), static_cast(words.size()), 0, {}}; + AgcDriver::Submit(&packet, 0); + AgcDriverWaitIdle_nid_postfix(); + AgcDriver::GuestMemory::FlushGpuWrites(reinterpret_cast(Output.data()), sizeof(Output)); + const auto launched = item.partial ? item.threads : item.groups * 64u; + const auto filled = std::min({launched, item.records, item.count}); + for (std::uint32_t i = 0; i < Output.size(); ++i) { + const auto expected = i < filled ? Pattern[std::min(i % item.period, 3u)] : Untouched; + Require(Output[i] == expected, "dword pattern fill mismatch at " + std::to_string(i) + " (period " + std::to_string(item.period) + ", count " + std::to_string(item.count) + ", records " + std::to_string(item.records) + "): got " + std::to_string(Output[i]) + ", expected " + std::to_string(expected)); + } + } +} + +} + +int main() { + try { + if (std::getenv("APS5_NO_FILL_HLE") == nullptr) MatchTests(); + { + const auto device = OpenVulkanTestDevice(); + if (!device) return VulkanTestSkipped; + } + ExecuteTests(); + AgcDriverShutdown_nid_postfix(); + std::puts("dword pattern fill tests passed"); + return 0; + } catch (const std::exception& error) { + std::cerr << error.what() << '\n'; + AgcDriverShutdown_nid_postfix(); + return 1; + } +} diff --git a/core/libs/prx/libSceAgcDriver/tests/execution/EmulatedColorCompare.cpp b/core/libs/prx/libSceAgcDriver/tests/execution/EmulatedColorCompare.cpp index 54514f210..c20841723 100644 --- a/core/libs/prx/libSceAgcDriver/tests/execution/EmulatedColorCompare.cpp +++ b/core/libs/prx/libSceAgcDriver/tests/execution/EmulatedColorCompare.cpp @@ -24,9 +24,11 @@ using ShaderRecompiler::ShaderStage; constexpr std::uint32_t Threads = 32; constexpr std::uint32_t Side = 2; constexpr std::uint32_t Format8888UNorm = 56; +constexpr std::uint32_t Format8888Srgb = 130; constexpr std::uint32_t Format8888UInt = 60; constexpr std::uint32_t Format32Float = 22; constexpr std::uint32_t Type2D = 9; +constexpr std::uint32_t TypeCube = 11; constexpr std::uint32_t LessEqual = 3; constexpr std::uint32_t ClampWrap = 0; constexpr std::uint32_t ClampMirror = 1; @@ -38,11 +40,13 @@ constexpr std::uint32_t BorderWhite = 2; constexpr std::uint32_t BorderTable = 3; constexpr std::uint32_t FilterPoint = 0; constexpr std::uint32_t FilterBilinear = 1; +constexpr std::uint32_t FilterAnisoPoint = 2; constexpr std::uint32_t FilterAnisoBilinear = 3; constexpr std::uint32_t ReductionMin = 1; constexpr std::array Red{0, 64, 128, 255}; alignas(256) std::array Input{}; +alignas(256) std::array CubeInput{}; alignas(256) std::array Output{}; alignas(256) std::array Extra{}; alignas(4096) std::array Texels{}; @@ -52,6 +56,11 @@ alignas(256) constexpr std::array Code{ 0x34160082, 0xe0701000, 0x8001040b, 0xbf810000, 0xbf810000, 0xbf810000, }; +alignas(256) constexpr std::array CubeCode{ + 0x16140090, 0xe0381000, 0x8000010a, 0xbf8c3f70, 0xf0bc0118, 0x00820401, 0xbf8c3f70, + 0x34160082, 0xe0701000, 0x8001040b, 0xbf810000, 0xbf810000, 0xbf810000, +}; + alignas(256) constexpr std::array OffsetCode{ 0x1614008c, 0xe03c1000, 0x8000030a, 0xe03c1000, 0x8005060a, 0xbf8c3f70, 0x7e020306, 0x7e040307, 0x7e0c0308, 0xf0f80108, 0x00820901, 0xbf8c3f70, 0x34160082, 0xe0701000, 0x8001090b, 0xbf810000, @@ -63,6 +72,7 @@ struct Sampler { std::uint32_t border = BorderBlack; std::uint32_t reduction = 0; std::uint32_t lodBias = 0; + bool disableCubeWrap = false; }; std::array BufferDescriptor(const void* data, std::uint32_t bytes) { @@ -70,20 +80,20 @@ std::array BufferDescriptor(const void* data, std::uint32_t by return {static_cast(address), static_cast((address >> 32u) & 0xffffu), bytes, 0x31016facu}; } -std::array TextureDescriptor(std::uint32_t format) { +std::array TextureDescriptor(std::uint32_t format, bool cube = false) { const auto address = static_cast(reinterpret_cast(Texels.data())); return { static_cast(address >> 8u), static_cast((address >> 40u) & 0xffu) | (format << 20u) | (((Side - 1u) & 3u) << 30u), ((Side - 1u) >> 2u) | ((Side - 1u) << 14u), - 0xfacu | (Type2D << 28u), - 0u, 0u, 0u, 0u, + 0xfacu | ((cube ? TypeCube : Type2D) << 28u), + cube ? 5u : 0u, 0u, 0u, 0u, }; } std::array SamplerDescriptor(const Sampler& sampler) { return { - sampler.clamp | (sampler.clamp << 3u) | (ClampEdge << 6u) | (LessEqual << 12u) | (sampler.reduction << 29u), + sampler.clamp | (sampler.clamp << 3u) | (ClampEdge << 6u) | (LessEqual << 12u) | (sampler.reduction << 29u) | (sampler.disableCubeWrap ? (1u << 28u) : 0u), 0u, sampler.lodBias | (sampler.filter << 20u) | (sampler.filter << 22u), sampler.border << 30u, @@ -106,6 +116,23 @@ void FillTexels() { } } +void FillCubeTexels() { + Texels.fill(0); + const auto descriptor = TextureDescriptor(Format8888UNorm, true); + const auto surface = AgcDriver::Graphics::DescribeSurface(AgcDriver::Graphics::DecodeTextureResource(descriptor)); + Require(surface.layers == 6u && surface.guestBytes <= Texels.size(), "cube descriptor did not expose six faces within texture storage"); + const auto& mip = surface.mips.at(0); + for (std::uint32_t face = 0; face < 6u; ++face) { + for (std::uint32_t y = 0; y < Side; ++y) { + for (std::uint32_t x = 0; x < Side; ++x) { + auto* texel = Texels.data() + surface.GuestLayerOffset(face) + mip.tiledOffset + y * mip.pitchBytes + x * 4u; + texel[0] = static_cast(face == 0u ? 255u : 0u); + texel[3] = 0xffu; + } + } + } +} + void FillInput() { for (std::uint32_t tid = 0; tid < Threads; ++tid) { Input[tid * 3u + 0u] = -0.6f + 0.07f * static_cast(tid); @@ -166,11 +193,11 @@ float Expected(std::uint32_t tid, const Sampler& sampler, bool offsets) { return top * (1.0f - b) + bottom * b; } -ShaderRecompiler::RecompileResult Compile(AgcDriver::VulkanDevice& device, std::uint32_t format, const Sampler& sampler, std::span code = Code, bool useCache = false, bool nativeSampleOffsets = true) { +ShaderRecompiler::RecompileResult Compile(AgcDriver::VulkanDevice& device, std::uint32_t format, const Sampler& sampler, std::span code = Code, bool useCache = false, bool nativeSampleOffsets = true, bool cube = false) { std::vector userData(24, 0u); - const auto input = BufferDescriptor(Input.data(), static_cast(sizeof(Input))); + const auto input = cube ? BufferDescriptor(CubeInput.data(), static_cast(sizeof(CubeInput))) : BufferDescriptor(Input.data(), static_cast(sizeof(Input))); const auto output = BufferDescriptor(Output.data(), static_cast(sizeof(Output))); - const auto texture = TextureDescriptor(format); + const auto texture = TextureDescriptor(format, cube); const auto samplerWords = SamplerDescriptor(sampler); std::copy(input.begin(), input.end(), userData.begin()); std::copy(output.begin(), output.end(), userData.begin() + 4); @@ -208,15 +235,36 @@ void Run(AgcDriver::VulkanDevice& device, const Sampler& sampler, const char* na } } +void RunCube(AgcDriver::VulkanDevice& device, const Sampler& sampler, const char* name) { + FillCubeTexels(); + for (std::uint32_t tid = 0; tid < Threads; ++tid) { + CubeInput[tid * 4u + 0u] = 0.5f; + CubeInput[tid * 4u + 1u] = 1.0f; + CubeInput[tid * 4u + 2u] = 0.0f; + CubeInput[tid * 4u + 3u] = 0.0f; + } + Output.fill(-1.0f); + const auto result = Compile(device, Format8888UNorm, sampler, CubeCode, false, true, true); + for (const auto& binding : result.bindings) { + Require(binding.role != ShaderRecompiler::DescriptorRole::GuestSamplers, "cube emulated comparison retained a sampler binding"); + Require(std::all_of(binding.imageSamplers.begin(), binding.imageSamplers.end(), [](auto mask) { return mask == 0u; }), "cube emulated comparison retained an unused sampler association"); + } + device.Dispatch(result, 1, 1, 1, {}, reinterpret_cast(CubeCode.data())); + device.WaitIdle(); + for (std::uint32_t tid = 0; tid < Threads; ++tid) { + Require(Output[tid] == 1.0f, std::string(name) + ": +X cube face comparison returned " + std::to_string(Output[tid]) + ", expected 1"); + } +} + bool BindsDepthCompare(const ShaderRecompiler::RecompileResult& result) { return std::any_of(result.bindings.begin(), result.bindings.end(), [](const ShaderRecompiler::DescriptorBinding& binding) { return std::any_of(binding.imageDepthCompare.begin(), binding.imageDepthCompare.end(), [](bool compare) { return compare; }); }); } -void Reject(AgcDriver::VulkanDevice& device, std::uint32_t format, const Sampler& sampler, std::string_view reason, std::span code = Code, bool nativeSampleOffsets = true) { +void Reject(AgcDriver::VulkanDevice& device, std::uint32_t format, const Sampler& sampler, std::string_view reason, std::span code = Code, bool nativeSampleOffsets = true, bool cube = false) { try { - static_cast(Compile(device, format, sampler, code, false, nativeSampleOffsets)); + static_cast(Compile(device, format, sampler, code, false, nativeSampleOffsets, cube)); } catch (const std::exception& error) { Require(std::string_view(error.what()).find(reason) != std::string_view::npos, std::string("unexpected rejection: ") + error.what()); return; @@ -232,9 +280,18 @@ int main() { if (!device) return VulkanTestSkipped; FillTexels(); FillInput(); + RunCube(*device, {ClampEdge, FilterPoint}, "point cube, clamp to edge"); + Reject(*device, Format8888UNorm, {ClampEdge, FilterBilinear}, "bilinear cube comparison requires DISABLE_CUBE_WRAP", CubeCode, true, true); + RunCube(*device, {ClampEdge, FilterBilinear, BorderBlack, 0u, 0u, true}, "bilinear cube with DISABLE_CUBE_WRAP"); + Reject(*device, Format8888Srgb, {ClampEdge, FilterPoint}, "implemented only for float, unorm and snorm formats"); + Reject(*device, Format8888UNorm, {ClampEdge, FilterAnisoBilinear}, "point or bilinear"); + Reject(*device, Format8888UNorm, {ClampEdge, FilterAnisoPoint}, "point or bilinear"); + FillTexels(); Require(BindsDepthCompare(Compile(*device, Format32Float, {ClampEdge, FilterBilinear})), "an R32 float texture left the native comparison path"); Require(!BindsDepthCompare(Compile(*device, Format8888UNorm, {ClampEdge, FilterBilinear})), "a color texture kept a depth-compare binding"); Run(*device, {ClampEdge, FilterPoint}, "point, clamp to edge"); + Run(*device, {ClampWrap, FilterPoint}, "point, wrap"); + Run(*device, {ClampWrap, FilterPoint}, "point, wrap with negative offsets", true); Run(*device, {ClampEdge, FilterBilinear}, "bilinear, clamp to edge"); Run(*device, {ClampWrap, FilterBilinear}, "bilinear, wrap"); Run(*device, {ClampBorder, FilterPoint, BorderWhite}, "point, white border"); @@ -254,7 +311,6 @@ int main() { Reject(*device, Format8888UNorm, {ClampMirror, FilterPoint}, "wrap, clamp-to-edge or clamp-to-border"); Reject(*device, Format8888UNorm, {ClampHalfBorder, FilterPoint}, "wrap, clamp-to-edge or clamp-to-border"); Reject(*device, Format8888UNorm, {ClampBorder, FilterPoint, BorderTable}, "border color table"); - Reject(*device, Format8888UNorm, {ClampEdge, FilterAnisoBilinear}, "point or bilinear"); Reject(*device, Format8888UNorm, {ClampEdge, FilterBilinear, BorderBlack, ReductionMin}, "min or max reduction"); std::puts("emulated color compare tests passed"); return 0; diff --git a/core/libs/prx/libSceAgcDriver/tests/execution/ExecHostLanes.cpp b/core/libs/prx/libSceAgcDriver/tests/execution/ExecHostLanes.cpp index 9d1cbec5b..2eb9752be 100644 --- a/core/libs/prx/libSceAgcDriver/tests/execution/ExecHostLanes.cpp +++ b/core/libs/prx/libSceAgcDriver/tests/execution/ExecHostLanes.cpp @@ -99,13 +99,13 @@ void Draw(AgcDriver::VulkanDevice& device) { {ShaderStage::Fragment, reinterpret_cast(PixelCode.data()), PixelCode, 0, {}}, {64, 0, pixelUserData, std::nullopt, pixel, std::nullopt, pixelMemory}, device.Target(), - {0, 0, vertexPush, 128 - vertexPush} + PixelPushLayout(vertexPush, device.Target()) }; fragment.useCache = false; const auto pixelResult = ShaderRecompiler::Recompile(fragment); const std::array shaders{{ {ShaderStage::Vertex, &vertexResult, 0}, - {ShaderStage::Fragment, &pixelResult, vertexPush} + {ShaderStage::Fragment, &pixelResult, PixelPushOffset(vertexPush, device.Target())} }}; AgcDriver::Graphics::State state{}; @@ -169,6 +169,8 @@ int main() { Check("compute lane", Threads); if (device->Target().subgroupSize < 32u) { std::printf("vertex draw skipped, subgroup size %u cannot hold a wave32\n", device->Target().subgroupSize); + } else if ((device->SubgroupStages() & VK_SHADER_STAGE_VERTEX_BIT) == 0u) { + std::puts("vertex draw skipped, the device has no subgroup operations in vertex shaders"); } else { Draw(*device); Check("vertex", Vertices); diff --git a/core/libs/prx/libSceAgcDriver/tests/execution/F16MinMaxNan.cpp b/core/libs/prx/libSceAgcDriver/tests/execution/F16MinMaxNan.cpp new file mode 100644 index 000000000..364b2c0da --- /dev/null +++ b/core/libs/prx/libSceAgcDriver/tests/execution/F16MinMaxNan.cpp @@ -0,0 +1,508 @@ +#include "prx/libSceAgcDriver/Execution/include/VulkanDevice.hpp" +#include "prx/libSceAgcDriver/Graphics/include/Draw.hpp" +#include "Recompiler.hpp" +#include "VulkanTestDevice.hpp" +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace { + +using AgcDriver::Graphics::Require; +using ShaderRecompiler::ShaderStage; + +constexpr std::uint32_t Threads = 32; +constexpr std::uint32_t Inputs = 4; +constexpr std::uint32_t Results = 16; +alignas(256) std::array Input{}; +alignas(256) std::array Output{}; + +alignas(256) constexpr std::array Code{ + 0x34020084u, 0x34060086u, 0xe0381000u, 0x80000401u, 0xbf8c3f70u, 0x7e140280u, 0x7e160280u, 0x7e180280u, + 0x7e1a0280u, 0x7e1c0280u, 0x7e1e0280u, 0x7e200280u, 0x7e220280u, 0x74140b04u, 0x72160b04u, 0xd751000cu, + 0x041a0b04u, 0xd754000du, 0x041a0b04u, 0xd757000eu, 0x041a0b04u, 0xd53a020fu, 0x22020b04u, 0xd5390110u, + 0x42020b04u, 0xe0781000u, 0x80010a03u, 0xe0781010u, 0x80010e03u, 0xbf810000u, +}; + +constexpr std::uint32_t Rows[128][4] = { + {0x12340000u, 0x56780000u, 0x9abcc000u, 0x00000000u}, + {0x12340000u, 0x56788000u, 0x9abc0001u, 0x00000000u}, + {0x12340000u, 0x56783c00u, 0x9abc7e00u, 0x00000000u}, + {0x12340000u, 0x5678c000u, 0x9abc3c00u, 0x00000000u}, + {0x12340000u, 0x56787c00u, 0x9abcfd23u, 0x00000000u}, + {0x12340000u, 0x5678fc00u, 0x9abcfc00u, 0x00000000u}, + {0x12340000u, 0x56787e00u, 0x9abc8000u, 0x00000000u}, + {0x12340000u, 0x5678fe55u, 0x9abc7c01u, 0x00000000u}, + {0x12340000u, 0x56787c01u, 0x9abc7c00u, 0x00000000u}, + {0x12340000u, 0x5678fd23u, 0x9abc0000u, 0x00000000u}, + {0x12340000u, 0x56780001u, 0x9abcfe55u, 0x00000000u}, + {0x12348000u, 0x56780000u, 0x9abcc000u, 0x00000000u}, + {0x12348000u, 0x56788000u, 0x9abc0001u, 0x00000000u}, + {0x12348000u, 0x56783c00u, 0x9abc7e00u, 0x00000000u}, + {0x12348000u, 0x5678c000u, 0x9abc3c00u, 0x00000000u}, + {0x12348000u, 0x56787c00u, 0x9abcfd23u, 0x00000000u}, + {0x12348000u, 0x5678fc00u, 0x9abcfc00u, 0x00000000u}, + {0x12348000u, 0x56787e00u, 0x9abc8000u, 0x00000000u}, + {0x12348000u, 0x5678fe55u, 0x9abc7c01u, 0x00000000u}, + {0x12348000u, 0x56787c01u, 0x9abc7c00u, 0x00000000u}, + {0x12348000u, 0x5678fd23u, 0x9abc0000u, 0x00000000u}, + {0x12348000u, 0x56780001u, 0x9abcfe55u, 0x00000000u}, + {0x12343c00u, 0x56780000u, 0x9abcc000u, 0x00000000u}, + {0x12343c00u, 0x56788000u, 0x9abc0001u, 0x00000000u}, + {0x12343c00u, 0x56783c00u, 0x9abc7e00u, 0x00000000u}, + {0x12343c00u, 0x5678c000u, 0x9abc3c00u, 0x00000000u}, + {0x12343c00u, 0x56787c00u, 0x9abcfd23u, 0x00000000u}, + {0x12343c00u, 0x5678fc00u, 0x9abcfc00u, 0x00000000u}, + {0x12343c00u, 0x56787e00u, 0x9abc8000u, 0x00000000u}, + {0x12343c00u, 0x5678fe55u, 0x9abc7c01u, 0x00000000u}, + {0x12343c00u, 0x56787c01u, 0x9abc7c00u, 0x00000000u}, + {0x12343c00u, 0x5678fd23u, 0x9abc0000u, 0x00000000u}, + {0x12343c00u, 0x56780001u, 0x9abcfe55u, 0x00000000u}, + {0x1234c000u, 0x56780000u, 0x9abcc000u, 0x00000000u}, + {0x1234c000u, 0x56788000u, 0x9abc0001u, 0x00000000u}, + {0x1234c000u, 0x56783c00u, 0x9abc7e00u, 0x00000000u}, + {0x1234c000u, 0x5678c000u, 0x9abc3c00u, 0x00000000u}, + {0x1234c000u, 0x56787c00u, 0x9abcfd23u, 0x00000000u}, + {0x1234c000u, 0x5678fc00u, 0x9abcfc00u, 0x00000000u}, + {0x1234c000u, 0x56787e00u, 0x9abc8000u, 0x00000000u}, + {0x1234c000u, 0x5678fe55u, 0x9abc7c01u, 0x00000000u}, + {0x1234c000u, 0x56787c01u, 0x9abc7c00u, 0x00000000u}, + {0x1234c000u, 0x5678fd23u, 0x9abc0000u, 0x00000000u}, + {0x1234c000u, 0x56780001u, 0x9abcfe55u, 0x00000000u}, + {0x12347c00u, 0x56780000u, 0x9abcc000u, 0x00000000u}, + {0x12347c00u, 0x56788000u, 0x9abc0001u, 0x00000000u}, + {0x12347c00u, 0x56783c00u, 0x9abc7e00u, 0x00000000u}, + {0x12347c00u, 0x5678c000u, 0x9abc3c00u, 0x00000000u}, + {0x12347c00u, 0x56787c00u, 0x9abcfd23u, 0x00000000u}, + {0x12347c00u, 0x5678fc00u, 0x9abcfc00u, 0x00000000u}, + {0x12347c00u, 0x56787e00u, 0x9abc8000u, 0x00000000u}, + {0x12347c00u, 0x5678fe55u, 0x9abc7c01u, 0x00000000u}, + {0x12347c00u, 0x56787c01u, 0x9abc7c00u, 0x00000000u}, + {0x12347c00u, 0x5678fd23u, 0x9abc0000u, 0x00000000u}, + {0x12347c00u, 0x56780001u, 0x9abcfe55u, 0x00000000u}, + {0x1234fc00u, 0x56780000u, 0x9abcc000u, 0x00000000u}, + {0x1234fc00u, 0x56788000u, 0x9abc0001u, 0x00000000u}, + {0x1234fc00u, 0x56783c00u, 0x9abc7e00u, 0x00000000u}, + {0x1234fc00u, 0x5678c000u, 0x9abc3c00u, 0x00000000u}, + {0x1234fc00u, 0x56787c00u, 0x9abcfd23u, 0x00000000u}, + {0x1234fc00u, 0x5678fc00u, 0x9abcfc00u, 0x00000000u}, + {0x1234fc00u, 0x56787e00u, 0x9abc8000u, 0x00000000u}, + {0x1234fc00u, 0x5678fe55u, 0x9abc7c01u, 0x00000000u}, + {0x1234fc00u, 0x56787c01u, 0x9abc7c00u, 0x00000000u}, + {0x1234fc00u, 0x5678fd23u, 0x9abc0000u, 0x00000000u}, + {0x1234fc00u, 0x56780001u, 0x9abcfe55u, 0x00000000u}, + {0x12347e00u, 0x56780000u, 0x9abcc000u, 0x00000000u}, + {0x12347e00u, 0x56788000u, 0x9abc0001u, 0x00000000u}, + {0x12347e00u, 0x56783c00u, 0x9abc7e00u, 0x00000000u}, + {0x12347e00u, 0x5678c000u, 0x9abc3c00u, 0x00000000u}, + {0x12347e00u, 0x56787c00u, 0x9abcfd23u, 0x00000000u}, + {0x12347e00u, 0x5678fc00u, 0x9abcfc00u, 0x00000000u}, + {0x12347e00u, 0x56787e00u, 0x9abc8000u, 0x00000000u}, + {0x12347e00u, 0x5678fe55u, 0x9abc7c01u, 0x00000000u}, + {0x12347e00u, 0x56787c01u, 0x9abc7c00u, 0x00000000u}, + {0x12347e00u, 0x5678fd23u, 0x9abc0000u, 0x00000000u}, + {0x12347e00u, 0x56780001u, 0x9abcfe55u, 0x00000000u}, + {0x1234fe55u, 0x56780000u, 0x9abcc000u, 0x00000000u}, + {0x1234fe55u, 0x56788000u, 0x9abc0001u, 0x00000000u}, + {0x1234fe55u, 0x56783c00u, 0x9abc7e00u, 0x00000000u}, + {0x1234fe55u, 0x5678c000u, 0x9abc3c00u, 0x00000000u}, + {0x1234fe55u, 0x56787c00u, 0x9abcfd23u, 0x00000000u}, + {0x1234fe55u, 0x5678fc00u, 0x9abcfc00u, 0x00000000u}, + {0x1234fe55u, 0x56787e00u, 0x9abc8000u, 0x00000000u}, + {0x1234fe55u, 0x5678fe55u, 0x9abc7c01u, 0x00000000u}, + {0x1234fe55u, 0x56787c01u, 0x9abc7c00u, 0x00000000u}, + {0x1234fe55u, 0x5678fd23u, 0x9abc0000u, 0x00000000u}, + {0x1234fe55u, 0x56780001u, 0x9abcfe55u, 0x00000000u}, + {0x12347c01u, 0x56780000u, 0x9abcc000u, 0x00000000u}, + {0x12347c01u, 0x56788000u, 0x9abc0001u, 0x00000000u}, + {0x12347c01u, 0x56783c00u, 0x9abc7e00u, 0x00000000u}, + {0x12347c01u, 0x5678c000u, 0x9abc3c00u, 0x00000000u}, + {0x12347c01u, 0x56787c00u, 0x9abcfd23u, 0x00000000u}, + {0x12347c01u, 0x5678fc00u, 0x9abcfc00u, 0x00000000u}, + {0x12347c01u, 0x56787e00u, 0x9abc8000u, 0x00000000u}, + {0x12347c01u, 0x5678fe55u, 0x9abc7c01u, 0x00000000u}, + {0x12347c01u, 0x56787c01u, 0x9abc7c00u, 0x00000000u}, + {0x12347c01u, 0x5678fd23u, 0x9abc0000u, 0x00000000u}, + {0x12347c01u, 0x56780001u, 0x9abcfe55u, 0x00000000u}, + {0x1234fd23u, 0x56780000u, 0x9abcc000u, 0x00000000u}, + {0x1234fd23u, 0x56788000u, 0x9abc0001u, 0x00000000u}, + {0x1234fd23u, 0x56783c00u, 0x9abc7e00u, 0x00000000u}, + {0x1234fd23u, 0x5678c000u, 0x9abc3c00u, 0x00000000u}, + {0x1234fd23u, 0x56787c00u, 0x9abcfd23u, 0x00000000u}, + {0x1234fd23u, 0x5678fc00u, 0x9abcfc00u, 0x00000000u}, + {0x1234fd23u, 0x56787e00u, 0x9abc8000u, 0x00000000u}, + {0x1234fd23u, 0x5678fe55u, 0x9abc7c01u, 0x00000000u}, + {0x1234fd23u, 0x56787c01u, 0x9abc7c00u, 0x00000000u}, + {0x1234fd23u, 0x5678fd23u, 0x9abc0000u, 0x00000000u}, + {0x1234fd23u, 0x56780001u, 0x9abcfe55u, 0x00000000u}, + {0x12340001u, 0x56780000u, 0x9abcc000u, 0x00000000u}, + {0x12340001u, 0x56788000u, 0x9abc0001u, 0x00000000u}, + {0x12340001u, 0x56783c00u, 0x9abc7e00u, 0x00000000u}, + {0x12340001u, 0x5678c000u, 0x9abc3c00u, 0x00000000u}, + {0x12340001u, 0x56787c00u, 0x9abcfd23u, 0x00000000u}, + {0x12340001u, 0x5678fc00u, 0x9abcfc00u, 0x00000000u}, + {0x12340001u, 0x56787e00u, 0x9abc8000u, 0x00000000u}, + {0x12340001u, 0x5678fe55u, 0x9abc7c01u, 0x00000000u}, + {0x12340001u, 0x56787c01u, 0x9abc7c00u, 0x00000000u}, + {0x12340001u, 0x5678fd23u, 0x9abc0000u, 0x00000000u}, + {0x12340001u, 0x56780001u, 0x9abcfe55u, 0x00000000u}, + {0x12347c01u, 0x5678fd23u, 0x9abc7e00u, 0x00000000u}, + {0x1234fd23u, 0x56787e00u, 0x9abc7c01u, 0x00000000u}, + {0x12347e00u, 0x56787c01u, 0x9abcfd23u, 0x00000000u}, + {0x1234fe55u, 0x56787e00u, 0x9abc7c01u, 0x00000000u}, + {0x12343c00u, 0x56787c01u, 0x9abcfd23u, 0x00000000u}, + {0x12347e00u, 0x5678fe55u, 0x9abc3c00u, 0x00000000u}, + {0x12340001u, 0x56788001u, 0x9abc7c01u, 0x00000000u} +}; +constexpr std::uint32_t Expected[128][7] = { + {0x00000000u, 0x00000000u, 0x0000c000u, 0x00000000u, 0x00000000u, 0x00008000u, 0x00000000u}, + {0x00008000u, 0x00000000u, 0x00008000u, 0x00000001u, 0x00000000u, 0x00008000u, 0x00000000u}, + {0x00000000u, 0x00003c00u, 0x00000000u, 0x00003c00u, 0x00000000u, 0x00008000u, 0x00000000u}, + {0x0000c000u, 0x00000000u, 0x0000c000u, 0x00003c00u, 0x00000000u, 0x00008000u, 0x00004000u}, + {0x00000000u, 0x00007c00u, 0x00000000u, 0x00007c00u, 0x00000000u, 0x00008000u, 0x00000000u}, + {0x0000fc00u, 0x00000000u, 0x0000fc00u, 0x00000000u, 0x0000fc00u, 0x00008000u, 0x00007c00u}, + {0x00000000u, 0x00000000u, 0x00008000u, 0x00000000u, 0x00008000u, 0x00008000u, 0x00000000u}, + {0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00008000u, 0x00000000u}, + {0x00000000u, 0x00000000u, 0x00000000u, 0x00007c00u, 0x00000000u, 0x00008000u, 0x00000000u}, + {0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00008000u, 0x00000000u}, + {0x00000000u, 0x00000001u, 0x00000000u, 0x00000001u, 0x00000000u, 0x00008000u, 0x00000000u}, + {0x00008000u, 0x00000000u, 0x0000c000u, 0x00000000u, 0x00008000u, 0x00000000u, 0x00000000u}, + {0x00008000u, 0x00008000u, 0x00008000u, 0x00000001u, 0x00008000u, 0x00000000u, 0x00000000u}, + {0x00008000u, 0x00003c00u, 0x00008000u, 0x00003c00u, 0x00008000u, 0x00000000u, 0x00000000u}, + {0x0000c000u, 0x00008000u, 0x0000c000u, 0x00003c00u, 0x00008000u, 0x00000000u, 0x00004000u}, + {0x00008000u, 0x00007c00u, 0x00008000u, 0x00007c00u, 0x00008000u, 0x00000000u, 0x00000000u}, + {0x0000fc00u, 0x00008000u, 0x0000fc00u, 0x00008000u, 0x0000fc00u, 0x00000000u, 0x00007c00u}, + {0x00008000u, 0x00008000u, 0x00008000u, 0x00008000u, 0x00008000u, 0x00000000u, 0x00000000u}, + {0x00008000u, 0x00008000u, 0x00008000u, 0x00008000u, 0x00008000u, 0x00000000u, 0x00000000u}, + {0x00008000u, 0x00008000u, 0x00008000u, 0x00007c00u, 0x00008000u, 0x00000000u, 0x00000000u}, + {0x00008000u, 0x00008000u, 0x00008000u, 0x00000000u, 0x00008000u, 0x00000000u, 0x00000000u}, + {0x00008000u, 0x00000001u, 0x00008000u, 0x00000001u, 0x00008000u, 0x00000000u, 0x00000000u}, + {0x00000000u, 0x00003c00u, 0x0000c000u, 0x00003c00u, 0x00000000u, 0x0000bc00u, 0x00003c00u}, + {0x00008000u, 0x00003c00u, 0x00008000u, 0x00003c00u, 0x00000001u, 0x0000bc00u, 0x00003c00u}, + {0x00003c00u, 0x00003c00u, 0x00003c00u, 0x00003c00u, 0x00003c00u, 0x0000bc00u, 0x00003c00u}, + {0x0000c000u, 0x00003c00u, 0x0000c000u, 0x00003c00u, 0x00003c00u, 0x0000bc00u, 0x00004000u}, + {0x00003c00u, 0x00007c00u, 0x00003c00u, 0x00007c00u, 0x00003c00u, 0x0000bc00u, 0x00003c00u}, + {0x0000fc00u, 0x00003c00u, 0x0000fc00u, 0x00003c00u, 0x0000fc00u, 0x0000bc00u, 0x00007c00u}, + {0x00003c00u, 0x00003c00u, 0x00008000u, 0x00003c00u, 0x00008000u, 0x0000bc00u, 0x00003c00u}, + {0x00003c00u, 0x00003c00u, 0x00003c00u, 0x00003c00u, 0x00003c00u, 0x0000bc00u, 0x00003c00u}, + {0x00003c00u, 0x00003c00u, 0x00003c00u, 0x00007c00u, 0x00003c00u, 0x0000bc00u, 0x00003c00u}, + {0x00003c00u, 0x00003c00u, 0x00000000u, 0x00003c00u, 0x00000000u, 0x0000bc00u, 0x00003c00u}, + {0x00000001u, 0x00003c00u, 0x00000001u, 0x00003c00u, 0x00000001u, 0x0000bc00u, 0x00003c00u}, + {0x0000c000u, 0x00000000u, 0x0000c000u, 0x00000000u, 0x0000c000u, 0x00000000u, 0x00004000u}, + {0x0000c000u, 0x00008000u, 0x0000c000u, 0x00000001u, 0x00008000u, 0x00000000u, 0x00004000u}, + {0x0000c000u, 0x00003c00u, 0x0000c000u, 0x00003c00u, 0x0000c000u, 0x00003c00u, 0x00004000u}, + {0x0000c000u, 0x0000c000u, 0x0000c000u, 0x00003c00u, 0x0000c000u, 0x00004000u, 0x00004000u}, + {0x0000c000u, 0x00007c00u, 0x0000c000u, 0x00007c00u, 0x0000c000u, 0x00004000u, 0x00004000u}, + {0x0000fc00u, 0x0000c000u, 0x0000fc00u, 0x0000c000u, 0x0000fc00u, 0x00004000u, 0x00007c00u}, + {0x0000c000u, 0x0000c000u, 0x0000c000u, 0x00008000u, 0x0000c000u, 0x00004000u, 0x00004000u}, + {0x0000c000u, 0x0000c000u, 0x0000c000u, 0x0000c000u, 0x0000c000u, 0x00004000u, 0x00004000u}, + {0x0000c000u, 0x0000c000u, 0x0000c000u, 0x00007c00u, 0x0000c000u, 0x00004000u, 0x00004000u}, + {0x0000c000u, 0x0000c000u, 0x0000c000u, 0x00000000u, 0x0000c000u, 0x00004000u, 0x00004000u}, + {0x0000c000u, 0x00000001u, 0x0000c000u, 0x00000001u, 0x0000c000u, 0x00000001u, 0x00004000u}, + {0x00000000u, 0x00007c00u, 0x0000c000u, 0x00007c00u, 0x00000000u, 0x0000fc00u, 0x00007c00u}, + {0x00008000u, 0x00007c00u, 0x00008000u, 0x00007c00u, 0x00000001u, 0x0000fc00u, 0x00007c00u}, + {0x00003c00u, 0x00007c00u, 0x00003c00u, 0x00007c00u, 0x00003c00u, 0x0000fc00u, 0x00007c00u}, + {0x0000c000u, 0x00007c00u, 0x0000c000u, 0x00007c00u, 0x00003c00u, 0x0000fc00u, 0x00007c00u}, + {0x00007c00u, 0x00007c00u, 0x00007c00u, 0x00007c00u, 0x00007c00u, 0x0000fc00u, 0x00007c00u}, + {0x0000fc00u, 0x00007c00u, 0x0000fc00u, 0x00007c00u, 0x0000fc00u, 0x0000fc00u, 0x00007c00u}, + {0x00007c00u, 0x00007c00u, 0x00008000u, 0x00007c00u, 0x00008000u, 0x0000fc00u, 0x00007c00u}, + {0x00007c00u, 0x00007c00u, 0x00007c00u, 0x00007c00u, 0x00007c00u, 0x0000fc00u, 0x00007c00u}, + {0x00007c00u, 0x00007c00u, 0x00007c00u, 0x00007c00u, 0x00007c00u, 0x0000fc00u, 0x00007c00u}, + {0x00007c00u, 0x00007c00u, 0x00000000u, 0x00007c00u, 0x00000000u, 0x0000fc00u, 0x00007c00u}, + {0x00000001u, 0x00007c00u, 0x00000001u, 0x00007c00u, 0x00000001u, 0x0000fc00u, 0x00007c00u}, + {0x0000fc00u, 0x00000000u, 0x0000fc00u, 0x00000000u, 0x0000c000u, 0x00000000u, 0x00007c00u}, + {0x0000fc00u, 0x00008000u, 0x0000fc00u, 0x00000001u, 0x00008000u, 0x00000000u, 0x00007c00u}, + {0x0000fc00u, 0x00003c00u, 0x0000fc00u, 0x00003c00u, 0x0000fc00u, 0x00003c00u, 0x00007c00u}, + {0x0000fc00u, 0x0000c000u, 0x0000fc00u, 0x00003c00u, 0x0000c000u, 0x00004000u, 0x00007c00u}, + {0x0000fc00u, 0x00007c00u, 0x0000fc00u, 0x00007c00u, 0x0000fc00u, 0x00007c00u, 0x00007c00u}, + {0x0000fc00u, 0x0000fc00u, 0x0000fc00u, 0x0000fc00u, 0x0000fc00u, 0x00007c00u, 0x00007c00u}, + {0x0000fc00u, 0x0000fc00u, 0x0000fc00u, 0x00008000u, 0x0000fc00u, 0x00007c00u, 0x00007c00u}, + {0x0000fc00u, 0x0000fc00u, 0x0000fc00u, 0x0000fc00u, 0x0000fc00u, 0x00007c00u, 0x00007c00u}, + {0x0000fc00u, 0x0000fc00u, 0x0000fc00u, 0x00007c00u, 0x0000fc00u, 0x00007c00u, 0x00007c00u}, + {0x0000fc00u, 0x0000fc00u, 0x0000fc00u, 0x00000000u, 0x0000fc00u, 0x00007c00u, 0x00007c00u}, + {0x0000fc00u, 0x00000001u, 0x0000fc00u, 0x00000001u, 0x0000fc00u, 0x00000001u, 0x00007c00u}, + {0x00000000u, 0x00000000u, 0x0000c000u, 0x00000000u, 0x0000c000u, 0x00000000u, 0x00008000u}, + {0x00008000u, 0x00008000u, 0x00008000u, 0x00000001u, 0x00008000u, 0x00000000u, 0x00000000u}, + {0x00003c00u, 0x00003c00u, 0x00003c00u, 0x00003c00u, 0x00003c00u, 0x00003c00u, 0x0000bc00u}, + {0x0000c000u, 0x0000c000u, 0x0000c000u, 0x00003c00u, 0x0000c000u, 0x00004000u, 0x00004000u}, + {0x00007c00u, 0x00007c00u, 0x00007c00u, 0x00007c00u, 0x00007c00u, 0x00007c00u, 0x0000fc00u}, + {0x0000fc00u, 0x0000fc00u, 0x0000fc00u, 0x0000fc00u, 0x0000fc00u, 0x00007c00u, 0x00007c00u}, + {0x00007e00u, 0x00007e00u, 0x00008000u, 0x00008000u, 0x00008000u, 0x0000fe00u, 0x00007e00u}, + {0x00007e00u, 0x00007e00u, 0x00007e00u, 0x00007e00u, 0x00007e00u, 0x0000fe00u, 0x00007e00u}, + {0x00007e00u, 0x00007e00u, 0x00007c00u, 0x00007c00u, 0x00007c00u, 0x0000fe00u, 0x00007e00u}, + {0x00007e00u, 0x00007e00u, 0x00000000u, 0x00000000u, 0x00000000u, 0x0000fe00u, 0x00007e00u}, + {0x00000001u, 0x00000001u, 0x00000001u, 0x00000001u, 0x00000001u, 0x00000001u, 0x00008001u}, + {0x00000000u, 0x00000000u, 0x0000c000u, 0x00000000u, 0x0000c000u, 0x00000000u, 0x00008000u}, + {0x00008000u, 0x00008000u, 0x00008000u, 0x00000001u, 0x00008000u, 0x00000000u, 0x00000000u}, + {0x00003c00u, 0x00003c00u, 0x00003c00u, 0x00003c00u, 0x00003c00u, 0x00003c00u, 0x0000bc00u}, + {0x0000c000u, 0x0000c000u, 0x0000c000u, 0x00003c00u, 0x0000c000u, 0x00004000u, 0x00004000u}, + {0x00007c00u, 0x00007c00u, 0x00007c00u, 0x00007c00u, 0x00007c00u, 0x00007c00u, 0x0000fc00u}, + {0x0000fc00u, 0x0000fc00u, 0x0000fc00u, 0x0000fc00u, 0x0000fc00u, 0x00007c00u, 0x00007c00u}, + {0x0000fe55u, 0x0000fe55u, 0x00008000u, 0x00008000u, 0x00008000u, 0x00007e55u, 0x00007e55u}, + {0x0000fe55u, 0x0000fe55u, 0x0000fe55u, 0x0000fe55u, 0x0000fe55u, 0x00007e55u, 0x00007e55u}, + {0x0000fe55u, 0x0000fe55u, 0x00007c00u, 0x00007c00u, 0x00007c00u, 0x00007e55u, 0x00007e55u}, + {0x0000fe55u, 0x0000fe55u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00007e55u, 0x00007e55u}, + {0x00000001u, 0x00000001u, 0x00000001u, 0x00000001u, 0x00000001u, 0x00000001u, 0x00008001u}, + {0x00000000u, 0x00000000u, 0x0000c000u, 0x00000000u, 0x0000c000u, 0x00000000u, 0x00008000u}, + {0x00008000u, 0x00008000u, 0x00008000u, 0x00000001u, 0x00008000u, 0x00000000u, 0x00000000u}, + {0x00003c00u, 0x00003c00u, 0x00003c00u, 0x00003c00u, 0x00003c00u, 0x00003c00u, 0x0000bc00u}, + {0x0000c000u, 0x0000c000u, 0x0000c000u, 0x00003c00u, 0x0000c000u, 0x00004000u, 0x00004000u}, + {0x00007c00u, 0x00007c00u, 0x00007c00u, 0x00007c00u, 0x00007c00u, 0x00007c00u, 0x0000fc00u}, + {0x0000fc00u, 0x0000fc00u, 0x0000fc00u, 0x0000fc00u, 0x0000fc00u, 0x00007c00u, 0x00007c00u}, + {0x00007c01u, 0x00007c01u, 0x00008000u, 0x00008000u, 0x00008000u, 0x0000fc01u, 0x00007c01u}, + {0x00007c01u, 0x00007c01u, 0x00007c01u, 0x00007c01u, 0x00007c01u, 0x0000fc01u, 0x00007c01u}, + {0x00007c01u, 0x00007c01u, 0x00007c00u, 0x00007c00u, 0x00007c00u, 0x0000fc01u, 0x00007c01u}, + {0x00007c01u, 0x00007c01u, 0x00000000u, 0x00000000u, 0x00000000u, 0x0000fc01u, 0x00007c01u}, + {0x00000001u, 0x00000001u, 0x00000001u, 0x00000001u, 0x00000001u, 0x00000001u, 0x00008001u}, + {0x00000000u, 0x00000000u, 0x0000c000u, 0x00000000u, 0x0000c000u, 0x00000000u, 0x00008000u}, + {0x00008000u, 0x00008000u, 0x00008000u, 0x00000001u, 0x00008000u, 0x00000000u, 0x00000000u}, + {0x00003c00u, 0x00003c00u, 0x00003c00u, 0x00003c00u, 0x00003c00u, 0x00003c00u, 0x0000bc00u}, + {0x0000c000u, 0x0000c000u, 0x0000c000u, 0x00003c00u, 0x0000c000u, 0x00004000u, 0x00004000u}, + {0x00007c00u, 0x00007c00u, 0x00007c00u, 0x00007c00u, 0x00007c00u, 0x00007c00u, 0x0000fc00u}, + {0x0000fc00u, 0x0000fc00u, 0x0000fc00u, 0x0000fc00u, 0x0000fc00u, 0x00007c00u, 0x00007c00u}, + {0x0000fd23u, 0x0000fd23u, 0x00008000u, 0x00008000u, 0x00008000u, 0x00007d23u, 0x00007d23u}, + {0x0000fd23u, 0x0000fd23u, 0x0000fd23u, 0x0000fd23u, 0x0000fd23u, 0x00007d23u, 0x00007d23u}, + {0x0000fd23u, 0x0000fd23u, 0x00007c00u, 0x00007c00u, 0x00007c00u, 0x00007d23u, 0x00007d23u}, + {0x0000fd23u, 0x0000fd23u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00007d23u, 0x00007d23u}, + {0x00000001u, 0x00000001u, 0x00000001u, 0x00000001u, 0x00000001u, 0x00000001u, 0x00008001u}, + {0x00000000u, 0x00000001u, 0x0000c000u, 0x00000001u, 0x00000000u, 0x00008001u, 0x00000001u}, + {0x00008000u, 0x00000001u, 0x00008000u, 0x00000001u, 0x00000001u, 0x00008001u, 0x00000001u}, + {0x00000001u, 0x00003c00u, 0x00000001u, 0x00003c00u, 0x00000001u, 0x00008001u, 0x00000001u}, + {0x0000c000u, 0x00000001u, 0x0000c000u, 0x00003c00u, 0x00000001u, 0x00008001u, 0x00004000u}, + {0x00000001u, 0x00007c00u, 0x00000001u, 0x00007c00u, 0x00000001u, 0x00008001u, 0x00000001u}, + {0x0000fc00u, 0x00000001u, 0x0000fc00u, 0x00000001u, 0x0000fc00u, 0x00008001u, 0x00007c00u}, + {0x00000001u, 0x00000001u, 0x00008000u, 0x00000001u, 0x00008000u, 0x00008001u, 0x00000001u}, + {0x00000001u, 0x00000001u, 0x00000001u, 0x00000001u, 0x00000001u, 0x00008001u, 0x00000001u}, + {0x00000001u, 0x00000001u, 0x00000001u, 0x00007c00u, 0x00000001u, 0x00008001u, 0x00000001u}, + {0x00000001u, 0x00000001u, 0x00000000u, 0x00000001u, 0x00000000u, 0x00008001u, 0x00000001u}, + {0x00000001u, 0x00000001u, 0x00000001u, 0x00000001u, 0x00000001u, 0x00008001u, 0x00000001u}, + {0x00007c01u, 0x00007c01u, 0x00007c01u, 0x00007c01u, 0x00007c01u, 0x0000fc01u, 0x00007c01u}, + {0x0000fd23u, 0x0000fd23u, 0x0000fd23u, 0x0000fd23u, 0x0000fd23u, 0x00007d23u, 0x00007d23u}, + {0x00007e00u, 0x00007e00u, 0x00007e00u, 0x00007e00u, 0x00007e00u, 0x0000fe00u, 0x00007e00u}, + {0x0000fe55u, 0x0000fe55u, 0x0000fe55u, 0x0000fe55u, 0x0000fe55u, 0x00007e55u, 0x00007e55u}, + {0x00003c00u, 0x00003c00u, 0x00003c00u, 0x00003c00u, 0x00003c00u, 0x0000bc00u, 0x00003c00u}, + {0x00007e00u, 0x00007e00u, 0x00003c00u, 0x00003c00u, 0x00003c00u, 0x0000fe00u, 0x00007e00u}, + {0x00008001u, 0x00000001u, 0x00008001u, 0x00000001u, 0x00008001u, 0x00008001u, 0x00000001u} +}; +constexpr std::uint32_t ExpectedIeee[128][7] = { + {0x00000000u, 0x00000000u, 0x0000c000u, 0x00000000u, 0x00000000u, 0x00008000u, 0x00000000u}, + {0x00008000u, 0x00000000u, 0x00008000u, 0x00000001u, 0x00000000u, 0x00008000u, 0x00000000u}, + {0x00000000u, 0x00003c00u, 0x00000000u, 0x00003c00u, 0x00000000u, 0x00008000u, 0x00000000u}, + {0x0000c000u, 0x00000000u, 0x0000c000u, 0x00003c00u, 0x00000000u, 0x00008000u, 0x00004000u}, + {0x00000000u, 0x00007c00u, 0x0000ff23u, 0x0000ff23u, 0x0000ff23u, 0x00008000u, 0x00000000u}, + {0x0000fc00u, 0x00000000u, 0x0000fc00u, 0x00000000u, 0x0000fc00u, 0x00008000u, 0x00007c00u}, + {0x00000000u, 0x00000000u, 0x00008000u, 0x00000000u, 0x00008000u, 0x00008000u, 0x00000000u}, + {0x00000000u, 0x00000000u, 0x00007e01u, 0x00007e01u, 0x00007e01u, 0x00008000u, 0x00000000u}, + {0x00007e01u, 0x00007e01u, 0x00007c00u, 0x00007c00u, 0x00007c00u, 0x00007e01u, 0x0000fe01u}, + {0x0000ff23u, 0x0000ff23u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00007f23u, 0x00007f23u}, + {0x00000000u, 0x00000001u, 0x00000000u, 0x00000001u, 0x00000000u, 0x00008000u, 0x00000000u}, + {0x00008000u, 0x00000000u, 0x0000c000u, 0x00000000u, 0x00008000u, 0x00000000u, 0x00000000u}, + {0x00008000u, 0x00008000u, 0x00008000u, 0x00000001u, 0x00008000u, 0x00000000u, 0x00000000u}, + {0x00008000u, 0x00003c00u, 0x00008000u, 0x00003c00u, 0x00008000u, 0x00000000u, 0x00000000u}, + {0x0000c000u, 0x00008000u, 0x0000c000u, 0x00003c00u, 0x00008000u, 0x00000000u, 0x00004000u}, + {0x00008000u, 0x00007c00u, 0x0000ff23u, 0x0000ff23u, 0x0000ff23u, 0x00000000u, 0x00000000u}, + {0x0000fc00u, 0x00008000u, 0x0000fc00u, 0x00008000u, 0x0000fc00u, 0x00000000u, 0x00007c00u}, + {0x00008000u, 0x00008000u, 0x00008000u, 0x00008000u, 0x00008000u, 0x00000000u, 0x00000000u}, + {0x00008000u, 0x00008000u, 0x00007e01u, 0x00007e01u, 0x00007e01u, 0x00000000u, 0x00000000u}, + {0x00007e01u, 0x00007e01u, 0x00007c00u, 0x00007c00u, 0x00007c00u, 0x00007e01u, 0x0000fe01u}, + {0x0000ff23u, 0x0000ff23u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00007f23u, 0x00007f23u}, + {0x00008000u, 0x00000001u, 0x00008000u, 0x00000001u, 0x00008000u, 0x00000000u, 0x00000000u}, + {0x00000000u, 0x00003c00u, 0x0000c000u, 0x00003c00u, 0x00000000u, 0x0000bc00u, 0x00003c00u}, + {0x00008000u, 0x00003c00u, 0x00008000u, 0x00003c00u, 0x00000001u, 0x0000bc00u, 0x00003c00u}, + {0x00003c00u, 0x00003c00u, 0x00003c00u, 0x00003c00u, 0x00003c00u, 0x0000bc00u, 0x00003c00u}, + {0x0000c000u, 0x00003c00u, 0x0000c000u, 0x00003c00u, 0x00003c00u, 0x0000bc00u, 0x00004000u}, + {0x00003c00u, 0x00007c00u, 0x0000ff23u, 0x0000ff23u, 0x0000ff23u, 0x0000bc00u, 0x00003c00u}, + {0x0000fc00u, 0x00003c00u, 0x0000fc00u, 0x00003c00u, 0x0000fc00u, 0x0000bc00u, 0x00007c00u}, + {0x00003c00u, 0x00003c00u, 0x00008000u, 0x00003c00u, 0x00008000u, 0x0000bc00u, 0x00003c00u}, + {0x00003c00u, 0x00003c00u, 0x00007e01u, 0x00007e01u, 0x00007e01u, 0x0000bc00u, 0x00003c00u}, + {0x00007e01u, 0x00007e01u, 0x00007c00u, 0x00007c00u, 0x00007c00u, 0x00007e01u, 0x0000fe01u}, + {0x0000ff23u, 0x0000ff23u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00007f23u, 0x00007f23u}, + {0x00000001u, 0x00003c00u, 0x00000001u, 0x00003c00u, 0x00000001u, 0x0000bc00u, 0x00003c00u}, + {0x0000c000u, 0x00000000u, 0x0000c000u, 0x00000000u, 0x0000c000u, 0x00000000u, 0x00004000u}, + {0x0000c000u, 0x00008000u, 0x0000c000u, 0x00000001u, 0x00008000u, 0x00000000u, 0x00004000u}, + {0x0000c000u, 0x00003c00u, 0x0000c000u, 0x00003c00u, 0x0000c000u, 0x00003c00u, 0x00004000u}, + {0x0000c000u, 0x0000c000u, 0x0000c000u, 0x00003c00u, 0x0000c000u, 0x00004000u, 0x00004000u}, + {0x0000c000u, 0x00007c00u, 0x0000ff23u, 0x0000ff23u, 0x0000ff23u, 0x00004000u, 0x00004000u}, + {0x0000fc00u, 0x0000c000u, 0x0000fc00u, 0x0000c000u, 0x0000fc00u, 0x00004000u, 0x00007c00u}, + {0x0000c000u, 0x0000c000u, 0x0000c000u, 0x00008000u, 0x0000c000u, 0x00004000u, 0x00004000u}, + {0x0000c000u, 0x0000c000u, 0x00007e01u, 0x00007e01u, 0x00007e01u, 0x00004000u, 0x00004000u}, + {0x00007e01u, 0x00007e01u, 0x00007c00u, 0x00007c00u, 0x00007c00u, 0x00007e01u, 0x0000fe01u}, + {0x0000ff23u, 0x0000ff23u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00007f23u, 0x00007f23u}, + {0x0000c000u, 0x00000001u, 0x0000c000u, 0x00000001u, 0x0000c000u, 0x00000001u, 0x00004000u}, + {0x00000000u, 0x00007c00u, 0x0000c000u, 0x00007c00u, 0x00000000u, 0x0000fc00u, 0x00007c00u}, + {0x00008000u, 0x00007c00u, 0x00008000u, 0x00007c00u, 0x00000001u, 0x0000fc00u, 0x00007c00u}, + {0x00003c00u, 0x00007c00u, 0x00003c00u, 0x00007c00u, 0x00003c00u, 0x0000fc00u, 0x00007c00u}, + {0x0000c000u, 0x00007c00u, 0x0000c000u, 0x00007c00u, 0x00003c00u, 0x0000fc00u, 0x00007c00u}, + {0x00007c00u, 0x00007c00u, 0x0000ff23u, 0x0000ff23u, 0x0000ff23u, 0x0000fc00u, 0x00007c00u}, + {0x0000fc00u, 0x00007c00u, 0x0000fc00u, 0x00007c00u, 0x0000fc00u, 0x0000fc00u, 0x00007c00u}, + {0x00007c00u, 0x00007c00u, 0x00008000u, 0x00007c00u, 0x00008000u, 0x0000fc00u, 0x00007c00u}, + {0x00007c00u, 0x00007c00u, 0x00007e01u, 0x00007e01u, 0x00007e01u, 0x0000fc00u, 0x00007c00u}, + {0x00007e01u, 0x00007e01u, 0x00007c00u, 0x00007c00u, 0x00007c00u, 0x00007e01u, 0x0000fe01u}, + {0x0000ff23u, 0x0000ff23u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00007f23u, 0x00007f23u}, + {0x00000001u, 0x00007c00u, 0x00000001u, 0x00007c00u, 0x00000001u, 0x0000fc00u, 0x00007c00u}, + {0x0000fc00u, 0x00000000u, 0x0000fc00u, 0x00000000u, 0x0000c000u, 0x00000000u, 0x00007c00u}, + {0x0000fc00u, 0x00008000u, 0x0000fc00u, 0x00000001u, 0x00008000u, 0x00000000u, 0x00007c00u}, + {0x0000fc00u, 0x00003c00u, 0x0000fc00u, 0x00003c00u, 0x0000fc00u, 0x00003c00u, 0x00007c00u}, + {0x0000fc00u, 0x0000c000u, 0x0000fc00u, 0x00003c00u, 0x0000c000u, 0x00004000u, 0x00007c00u}, + {0x0000fc00u, 0x00007c00u, 0x0000ff23u, 0x0000ff23u, 0x0000ff23u, 0x00007c00u, 0x00007c00u}, + {0x0000fc00u, 0x0000fc00u, 0x0000fc00u, 0x0000fc00u, 0x0000fc00u, 0x00007c00u, 0x00007c00u}, + {0x0000fc00u, 0x0000fc00u, 0x0000fc00u, 0x00008000u, 0x0000fc00u, 0x00007c00u, 0x00007c00u}, + {0x0000fc00u, 0x0000fc00u, 0x00007e01u, 0x00007e01u, 0x00007e01u, 0x00007c00u, 0x00007c00u}, + {0x00007e01u, 0x00007e01u, 0x00007c00u, 0x00007c00u, 0x00007c00u, 0x00007e01u, 0x0000fe01u}, + {0x0000ff23u, 0x0000ff23u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00007f23u, 0x00007f23u}, + {0x0000fc00u, 0x00000001u, 0x0000fc00u, 0x00000001u, 0x0000fc00u, 0x00000001u, 0x00007c00u}, + {0x00000000u, 0x00000000u, 0x0000c000u, 0x00000000u, 0x0000c000u, 0x00000000u, 0x00008000u}, + {0x00008000u, 0x00008000u, 0x00008000u, 0x00000001u, 0x00008000u, 0x00000000u, 0x00000000u}, + {0x00003c00u, 0x00003c00u, 0x00003c00u, 0x00003c00u, 0x00003c00u, 0x00003c00u, 0x0000bc00u}, + {0x0000c000u, 0x0000c000u, 0x0000c000u, 0x00003c00u, 0x0000c000u, 0x00004000u, 0x00004000u}, + {0x00007c00u, 0x00007c00u, 0x0000ff23u, 0x0000ff23u, 0x0000ff23u, 0x00007c00u, 0x0000fc00u}, + {0x0000fc00u, 0x0000fc00u, 0x0000fc00u, 0x0000fc00u, 0x0000fc00u, 0x00007c00u, 0x00007c00u}, + {0x00007e00u, 0x00007e00u, 0x00008000u, 0x00008000u, 0x00008000u, 0x0000fe00u, 0x00007e00u}, + {0x00007e00u, 0x00007e00u, 0x00007e01u, 0x00007e01u, 0x00007e01u, 0x0000fe00u, 0x00007e00u}, + {0x00007e01u, 0x00007e01u, 0x00007c00u, 0x00007c00u, 0x00007c00u, 0x00007e01u, 0x0000fe01u}, + {0x0000ff23u, 0x0000ff23u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00007f23u, 0x00007f23u}, + {0x00000001u, 0x00000001u, 0x00000001u, 0x00000001u, 0x00000001u, 0x00000001u, 0x00008001u}, + {0x00000000u, 0x00000000u, 0x0000c000u, 0x00000000u, 0x0000c000u, 0x00000000u, 0x00008000u}, + {0x00008000u, 0x00008000u, 0x00008000u, 0x00000001u, 0x00008000u, 0x00000000u, 0x00000000u}, + {0x00003c00u, 0x00003c00u, 0x00003c00u, 0x00003c00u, 0x00003c00u, 0x00003c00u, 0x0000bc00u}, + {0x0000c000u, 0x0000c000u, 0x0000c000u, 0x00003c00u, 0x0000c000u, 0x00004000u, 0x00004000u}, + {0x00007c00u, 0x00007c00u, 0x0000ff23u, 0x0000ff23u, 0x0000ff23u, 0x00007c00u, 0x0000fc00u}, + {0x0000fc00u, 0x0000fc00u, 0x0000fc00u, 0x0000fc00u, 0x0000fc00u, 0x00007c00u, 0x00007c00u}, + {0x0000fe55u, 0x0000fe55u, 0x00008000u, 0x00008000u, 0x00008000u, 0x00007e55u, 0x00007e55u}, + {0x0000fe55u, 0x0000fe55u, 0x00007e01u, 0x00007e01u, 0x00007e01u, 0x00007e55u, 0x00007e55u}, + {0x00007e01u, 0x00007e01u, 0x00007c00u, 0x00007c00u, 0x00007c00u, 0x00007e01u, 0x0000fe01u}, + {0x0000ff23u, 0x0000ff23u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00007f23u, 0x00007f23u}, + {0x00000001u, 0x00000001u, 0x00000001u, 0x00000001u, 0x00000001u, 0x00000001u, 0x00008001u}, + {0x00007e01u, 0x00007e01u, 0x0000c000u, 0x0000c000u, 0x0000c000u, 0x0000fe01u, 0x00007e01u}, + {0x00007e01u, 0x00007e01u, 0x00000001u, 0x00000001u, 0x00000001u, 0x0000fe01u, 0x00007e01u}, + {0x00007e01u, 0x00007e01u, 0x00007e01u, 0x00007e01u, 0x00007e01u, 0x0000fe01u, 0x00007e01u}, + {0x00007e01u, 0x00007e01u, 0x00003c00u, 0x00003c00u, 0x00003c00u, 0x0000fe01u, 0x00007e01u}, + {0x00007e01u, 0x00007e01u, 0x0000ff23u, 0x0000ff23u, 0x0000ff23u, 0x0000fe01u, 0x00007e01u}, + {0x00007e01u, 0x00007e01u, 0x0000fc00u, 0x0000fc00u, 0x0000fc00u, 0x0000fe01u, 0x00007e01u}, + {0x00007e01u, 0x00007e01u, 0x00008000u, 0x00008000u, 0x00008000u, 0x0000fe01u, 0x00007e01u}, + {0x00007e01u, 0x00007e01u, 0x00007e01u, 0x00007e01u, 0x00007e01u, 0x0000fe01u, 0x00007e01u}, + {0x00007e01u, 0x00007e01u, 0x00007c00u, 0x00007c00u, 0x00007c00u, 0x0000fe01u, 0x00007e01u}, + {0x00007e01u, 0x00007e01u, 0x00000000u, 0x00000000u, 0x00000000u, 0x0000fe01u, 0x00007e01u}, + {0x00007e01u, 0x00007e01u, 0x00007e01u, 0x00007e01u, 0x00007e01u, 0x0000fe01u, 0x00007e01u}, + {0x0000ff23u, 0x0000ff23u, 0x0000c000u, 0x0000c000u, 0x0000c000u, 0x00007f23u, 0x00007f23u}, + {0x0000ff23u, 0x0000ff23u, 0x00000001u, 0x00000001u, 0x00000001u, 0x00007f23u, 0x00007f23u}, + {0x0000ff23u, 0x0000ff23u, 0x0000ff23u, 0x0000ff23u, 0x0000ff23u, 0x00007f23u, 0x00007f23u}, + {0x0000ff23u, 0x0000ff23u, 0x00003c00u, 0x00003c00u, 0x00003c00u, 0x00007f23u, 0x00007f23u}, + {0x0000ff23u, 0x0000ff23u, 0x0000ff23u, 0x0000ff23u, 0x0000ff23u, 0x00007f23u, 0x00007f23u}, + {0x0000ff23u, 0x0000ff23u, 0x0000fc00u, 0x0000fc00u, 0x0000fc00u, 0x00007f23u, 0x00007f23u}, + {0x0000ff23u, 0x0000ff23u, 0x00008000u, 0x00008000u, 0x00008000u, 0x00007f23u, 0x00007f23u}, + {0x0000ff23u, 0x0000ff23u, 0x00007e01u, 0x00007e01u, 0x00007e01u, 0x00007f23u, 0x00007f23u}, + {0x0000ff23u, 0x0000ff23u, 0x00007c00u, 0x00007c00u, 0x00007c00u, 0x00007f23u, 0x00007f23u}, + {0x0000ff23u, 0x0000ff23u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00007f23u, 0x00007f23u}, + {0x0000ff23u, 0x0000ff23u, 0x0000ff23u, 0x0000ff23u, 0x0000ff23u, 0x00007f23u, 0x00007f23u}, + {0x00000000u, 0x00000001u, 0x0000c000u, 0x00000001u, 0x00000000u, 0x00008001u, 0x00000001u}, + {0x00008000u, 0x00000001u, 0x00008000u, 0x00000001u, 0x00000001u, 0x00008001u, 0x00000001u}, + {0x00000001u, 0x00003c00u, 0x00000001u, 0x00003c00u, 0x00000001u, 0x00008001u, 0x00000001u}, + {0x0000c000u, 0x00000001u, 0x0000c000u, 0x00003c00u, 0x00000001u, 0x00008001u, 0x00004000u}, + {0x00000001u, 0x00007c00u, 0x0000ff23u, 0x0000ff23u, 0x0000ff23u, 0x00008001u, 0x00000001u}, + {0x0000fc00u, 0x00000001u, 0x0000fc00u, 0x00000001u, 0x0000fc00u, 0x00008001u, 0x00007c00u}, + {0x00000001u, 0x00000001u, 0x00008000u, 0x00000001u, 0x00008000u, 0x00008001u, 0x00000001u}, + {0x00000001u, 0x00000001u, 0x00007e01u, 0x00007e01u, 0x00007e01u, 0x00008001u, 0x00000001u}, + {0x00007e01u, 0x00007e01u, 0x00007c00u, 0x00007c00u, 0x00007c00u, 0x00007e01u, 0x0000fe01u}, + {0x0000ff23u, 0x0000ff23u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00007f23u, 0x00007f23u}, + {0x00000001u, 0x00000001u, 0x00000001u, 0x00000001u, 0x00000001u, 0x00008001u, 0x00000001u}, + {0x00007e01u, 0x00007e01u, 0x00007e01u, 0x00007e01u, 0x00007e01u, 0x0000fe01u, 0x00007e01u}, + {0x0000ff23u, 0x0000ff23u, 0x00007e01u, 0x00007e01u, 0x00007e01u, 0x00007f23u, 0x00007f23u}, + {0x00007e01u, 0x00007e01u, 0x0000ff23u, 0x0000ff23u, 0x0000ff23u, 0x00007e01u, 0x0000fe01u}, + {0x0000fe55u, 0x0000fe55u, 0x00007e01u, 0x00007e01u, 0x00007e01u, 0x00007e55u, 0x00007e55u}, + {0x00007e01u, 0x00007e01u, 0x0000ff23u, 0x0000ff23u, 0x0000ff23u, 0x00007e01u, 0x0000fe01u}, + {0x00007e00u, 0x00007e00u, 0x00003c00u, 0x00003c00u, 0x00003c00u, 0x0000fe00u, 0x00007e00u}, + {0x00008001u, 0x00000001u, 0x00007e01u, 0x00007e01u, 0x00007e01u, 0x00008001u, 0x00000001u} +}; +constexpr std::uint32_t Chunks = 4; +std::uint32_t Chunk = 0; +bool Ieee = false; +constexpr const char* Names[7] = { + "v_min_f16 v10, v4, v5", + "v_max_f16 v11, v4, v5", + "v_min3_f16 v12, v4, v5, v6", + "v_max3_f16 v13, v4, v5, v6", + "v_med3_f16 v14, v4, v5, v6", + "v_min_f16_e64 v15, -v4, |v5|", + "v_max_f16_e64 v16, |v4|, -v5", +}; + +void Fill(std::uint32_t tid, std::uint32_t* words) { + std::copy(std::begin(Rows[Chunk * Threads + tid]), std::end(Rows[Chunk * Threads + tid]), words); +} + +std::array BufferDescriptor(const void* data, std::uint32_t bytes) { + const auto address = reinterpret_cast(data); + return {static_cast(address), static_cast((address >> 32u) & 0xffffu), bytes, 0x31016facu}; +} + +std::string Hex(std::uint32_t value) { + char text[16]; + std::snprintf(text, sizeof(text), "0x%08x", value); + return text; +} + +void Expect(std::uint32_t tid, std::uint32_t actual, std::uint32_t expected, const char* name) { + Require(actual == expected, std::string(Ieee ? "f16 min max nan, IEEE mode: row " : "f16 min max nan: row ") + std::to_string(tid) + " " + name + " is " + Hex(actual) + ", expected " + Hex(expected)); +} + +void Run(AgcDriver::VulkanDevice& device) { + for (std::uint32_t tid = 0; tid < Threads; ++tid) Fill(tid, &Input[tid * Inputs]); + Output.fill(0xdeadbeefu); + std::vector userData(8, 0u); + const auto input = BufferDescriptor(Input.data(), static_cast(Input.size() * 4u)); + const auto output = BufferDescriptor(Output.data(), static_cast(Output.size() * 4u)); + std::copy(input.begin(), input.end(), userData.begin()); + std::copy(output.begin(), output.end(), userData.begin() + 4); + const std::span code(Code); + const std::array memory{{{reinterpret_cast(code.data()), std::as_bytes(code)}}}; + const ShaderRecompiler::ShaderComputeStageInfo compute{{Threads, 1, 1}, 0u, {false, false, false}, false, 1}; + ShaderRecompiler::RecompileRequest request{ + {ShaderStage::Compute, reinterpret_cast(code.data()), code, 0, {}}, + {32, 0, userData, compute, std::nullopt, std::nullopt, memory}, + device.Target(), + {0, 0, 0, 128} + }; + request.useCache = false; + if (Ieee) request.context.floatMode = ShaderRecompiler::ShaderFloatMode{0xf0u, true, true, false}; + const auto result = ShaderRecompiler::Recompile(request); + device.Dispatch(result, 1, 1, 1, {}, reinterpret_cast(code.data())); + device.WaitIdle(); +} + +void Check() { + for (std::uint32_t tid = 0; tid < Threads; ++tid) { + const std::uint32_t* in = &Input[tid * Inputs]; + const std::uint32_t* out = &Output[tid * Results]; + for (std::uint32_t i = 0; i < 7; ++i) { + Expect(Chunk * Threads + tid, out[i], (Ieee ? ExpectedIeee : Expected)[Chunk * Threads + tid][i], Names[i]); + } + } +} + +} + +int main() { + try { + const auto device = OpenVulkanTestDevice(); + if (!device) return VulkanTestSkipped; + for (const bool ieee : {false, true}) { + Ieee = ieee; + for (Chunk = 0; Chunk < Chunks; ++Chunk) { + Run(*device); + Check(); + } + } + std::puts("f16 min max nan tests passed"); + return 0; + } catch (const std::exception& error) { + std::cerr << error.what() << '\n'; + return 1; + } +} diff --git a/core/libs/prx/libSceAgcDriver/tests/execution/F32ConstantFactorTininess.cpp b/core/libs/prx/libSceAgcDriver/tests/execution/F32ConstantFactorTininess.cpp new file mode 100644 index 000000000..c2a7dae1f --- /dev/null +++ b/core/libs/prx/libSceAgcDriver/tests/execution/F32ConstantFactorTininess.cpp @@ -0,0 +1,165 @@ +#include "prx/libSceAgcDriver/Execution/include/VulkanDevice.hpp" +#include "prx/libSceAgcDriver/Graphics/include/Draw.hpp" +#include "Recompiler.hpp" +#include "VulkanTestDevice.hpp" +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace { + +using AgcDriver::Graphics::Require; +using ShaderRecompiler::ShaderStage; + +constexpr std::uint32_t Threads = 32; +constexpr std::uint32_t Inputs = 4; +constexpr std::uint32_t Results = 16; +alignas(256) std::array Input{}; +alignas(256) std::array Output{}; + +alignas(256) constexpr std::array Code{ + 0x34020084u, 0x34060086u, 0xe0381000u, 0x80000401u, 0xbf8c3f70u, + 0x7e3402f4u, 0x7e3602f0u, 0x7e3802ffu, 0x3f7ffffeu, 0x7e3a02f2u, 0x7e3c02ffu, 0x00400000u, 0x7e3e02f6u, + 0x7e4002ffu, 0x3e800000u, 0x7e4202f4u, + 0x101408f4u, 0x1016091au, + 0x101808f0u, 0x101a091bu, + 0x101c08ffu, 0x3f7ffffeu, 0x101e091cu, + 0x102008ffu, 0x3f800000u, 0x1022091du, + 0x102408ffu, 0x00400000u, 0x1026091eu, + 0x0e2808f6u, 0x0e2a091fu, + 0x582c0b04u, 0x3e800000u, 0xd54b0017u, 0x04164104u, + 0xd54b0218u, 0x4415e904u, 0xd54b0219u, 0x44164304u, + 0xe0781000u, 0x80010a03u, 0xe0781010u, 0x80010e03u, 0xe0781020u, 0x80011203u, 0xe0781030u, 0x80011603u, + 0xbf810000u, +}; + +constexpr std::uint32_t Rows[32][2] = { + {0x00800000u, 0x00000000u}, {0x80800000u, 0x00000000u}, {0x00800001u, 0x00000000u}, {0x00ffffffu, 0x00000000u}, + {0x01000000u, 0x00000000u}, {0x017fffffu, 0x00000000u}, {0x01800000u, 0x00000000u}, {0x00000001u, 0x00000000u}, + {0x807fffffu, 0x00000000u}, {0x00000000u, 0x00000000u}, {0x80000000u, 0x80000000u}, {0x7f800000u, 0x00000000u}, + {0xff800000u, 0x00000000u}, {0x7fc00000u, 0x00000000u}, {0x7f7fffffu, 0x00000000u}, {0x3f800000u, 0x00000000u}, + {0xbf800000u, 0x3f800000u}, {0x00c00000u, 0x00000000u}, {0x80ffffffu, 0x80000000u}, {0x01000001u, 0x00000000u}, + {0x3f7fffffu, 0x00000000u}, {0x40490fdbu, 0x00000000u}, {0x00fffffeu, 0x00000000u}, {0x81800000u, 0x00000000u}, + {0x017ffffeu, 0x00000000u}, {0x01800001u, 0x00000000u}, {0x0080ffffu, 0x00000000u}, {0x7f000000u, 0x00000000u}, + {0x00800000u, 0x3f800000u}, {0x817fffffu, 0x80000000u}, {0x017fffffu, 0x00800000u}, {0x017fffffu, 0x80000001u} +}; + +constexpr const char* Names[8] = { + "v_mul_f32 2.0", + "v_mul_f32 0.5", + "v_mul_f32 0x3f7ffffe", + "v_mul_f32 0x3f800000", + "v_mul_f32 0x00400000", + "v_mul_legacy_f32 4.0", + "v_fmamk_f32 0x3e800000", + "v_fma_f32 -|2.0|" +}; + +struct Pinned { + std::uint32_t tid; + std::uint32_t pair; + std::uint32_t value; +}; + +constexpr std::array TinyProducts{{ + {0u, 2u, 0x00000000u}, + {3u, 1u, 0x00000000u}, + {18u, 1u, 0x80000000u}, + {5u, 6u, 0x00000000u}, + {29u, 6u, 0x80000000u}, + {31u, 6u, 0x00000000u}, +}}; + +std::array BufferDescriptor(const void* data, std::uint32_t bytes) { + const auto address = reinterpret_cast(data); + return {static_cast(address), static_cast((address >> 32u) & 0xffffu), bytes, 0x31016facu}; +} + +std::string Hex(std::uint32_t value) { + char text[16]; + std::snprintf(text, sizeof(text), "0x%08x", value); + return text; +} + +bool IsNan(std::uint32_t value) { + return (value & 0x7fffffffu) > 0x7f800000u; +} + +void Run(AgcDriver::VulkanDevice& device, const std::optional& floatMode) { + Input.fill(0u); + for (std::uint32_t tid = 0; tid < Threads; ++tid) std::copy(std::begin(Rows[tid]), std::end(Rows[tid]), &Input[tid * Inputs]); + Output.fill(0xdeadbeefu); + std::vector userData(8, 0u); + const auto input = BufferDescriptor(Input.data(), static_cast(Input.size() * 4u)); + const auto output = BufferDescriptor(Output.data(), static_cast(Output.size() * 4u)); + std::copy(input.begin(), input.end(), userData.begin()); + std::copy(output.begin(), output.end(), userData.begin() + 4); + const std::span code(Code); + const std::array memory{{{reinterpret_cast(code.data()), std::as_bytes(code)}}}; + const ShaderRecompiler::ShaderComputeStageInfo compute{{Threads, 1, 1}, 0u, {false, false, false}, false, 1}; + ShaderRecompiler::RecompileRequest request{ + {ShaderStage::Compute, reinterpret_cast(code.data()), code, 0, {}}, + {32, 0, userData, compute, std::nullopt, std::nullopt, memory}, + device.Target(), + {0, 0, 0, 128} + }; + request.useCache = false; + request.context.floatMode = floatMode; + const auto result = ShaderRecompiler::Recompile(request); + device.Dispatch(result, 1, 1, 1, {}, reinterpret_cast(code.data())); + device.WaitIdle(); +} + +void Check(const char* mode) { + for (std::uint32_t tid = 0; tid < Threads; ++tid) { + const std::uint32_t* out = &Output[tid * Results]; + for (std::uint32_t pair = 0; pair < 8u; ++pair) { + const std::uint32_t constant = out[pair * 2u]; + const std::uint32_t reg = out[pair * 2u + 1u]; + const std::string where = std::string("f32 constant factor: lane ") + std::to_string(tid) + " " + mode + " " + Names[pair]; + if (IsNan(reg)) { + Require(IsNan(constant), where + " is " + Hex(constant) + " with a constant and a NaN with a register"); + continue; + } + Require(constant == reg, where + " is " + Hex(constant) + " with a constant and " + Hex(reg) + " with a register"); + } + } +} + +void CheckTinyProducts(const char* mode) { + for (const Pinned& pinned : TinyProducts) { + const std::uint32_t actual = Output[pinned.tid * Results + pinned.pair * 2u]; + Require(actual == pinned.value, std::string("f32 constant factor: lane ") + std::to_string(pinned.tid) + " " + mode + " " + Names[pinned.pair] + " is " + Hex(actual) + ", expected the flushed tiny product " + Hex(pinned.value)); + } +} + +} + +int main() { + try { + const auto device = OpenVulkanTestDevice(); + if (!device) return VulkanTestSkipped; + Run(*device, std::nullopt); + Check("no float mode"); + CheckTinyProducts("no float mode"); + Run(*device, ShaderRecompiler::ShaderFloatMode{0xc0u, true, false, false}); + Check("IEEE=0 f32 denormals flushed"); + CheckTinyProducts("IEEE=0 f32 denormals flushed"); + Run(*device, ShaderRecompiler::ShaderFloatMode{0x00u, false, true, false}); + Check("IEEE=1 all denormals flushed"); + CheckTinyProducts("IEEE=1 all denormals flushed"); + Run(*device, ShaderRecompiler::ShaderFloatMode{0xf0u, true, false, false}); + Check("f32 denormals kept"); + std::puts("f32 constant factor tests passed"); + return 0; + } catch (const std::exception& error) { + std::cerr << error.what() << '\n'; + return 1; + } +} diff --git a/core/libs/prx/libSceAgcDriver/tests/execution/F32DenormalFlush.cpp b/core/libs/prx/libSceAgcDriver/tests/execution/F32DenormalFlush.cpp index 37427d269..63a393720 100644 --- a/core/libs/prx/libSceAgcDriver/tests/execution/F32DenormalFlush.cpp +++ b/core/libs/prx/libSceAgcDriver/tests/execution/F32DenormalFlush.cpp @@ -21,16 +21,17 @@ using ShaderRecompiler::ShaderStage; constexpr std::uint32_t Threads = 32; constexpr std::uint32_t Inputs = 4; -constexpr std::uint32_t Results = 16; +constexpr std::uint32_t Results = 19; alignas(256) std::array Input{}; alignas(256) std::array Output{}; -alignas(256) constexpr std::array Code{ - 0x34020084u, 0x34060086u, 0xe0381000u, 0x80000401u, 0xbf8c3f70u, 0x06140b04u, 0x08160905u, 0x10180b04u, - 0x0e1a0b04u, 0xd54b000eu, 0x041a0b04u, 0x7e1e0306u, 0x561e0b04u, 0x1e200b04u, 0xd5540011u, 0x041a0b04u, - 0xd5570012u, 0x041a0b04u, 0x7e264904u, 0x7e284505u, 0x7e2a8104u, 0x7c020b04u, 0xd5010016u, 0x01a90280u, - 0x7e2e1b04u, 0x7e307f05u, 0xd5080119u, 0x42020b04u, 0xe0781000u, 0x80010a03u, 0xe0781010u, 0x80010e03u, - 0xe0781020u, 0x80011203u, 0xe0781030u, 0x80011603u, 0xbf810000u, +alignas(256) constexpr std::array Code{ + 0x34020084u, 0x160600ffu, 0x0000004cu, 0xe0381000u, 0x80000401u, 0xbf8c3f70u, 0x06140b04u, 0x08160905u, + 0x10180b04u, 0x0e1a0b04u, 0xd54b000eu, 0x041a0b04u, 0x7e1e0306u, 0x561e0b04u, 0x1e200b04u, 0xd5540011u, + 0x041a0b04u, 0xd5570012u, 0x041a0b04u, 0x7e264904u, 0x7e284505u, 0x7e2a8104u, 0x7c020b04u, 0xd5010016u, + 0x01a90280u, 0x7e2e1b04u, 0x7e307f05u, 0xd5080119u, 0x42020b04u, 0xd55f001au, 0x041a0b04u, 0xd550001bu, + 0x041a0b04u, 0xd550001cu, 0x03ca0b04u, 0xe0781000u, 0x80010a03u, 0xe0781010u, 0x80010e03u, 0xe0781020u, + 0x80011203u, 0xe0781030u, 0x80011603u, 0xe07c1040u, 0x80011a03u, 0xbf810000u, }; constexpr std::uint32_t Rows[32][4] = { @@ -67,41 +68,41 @@ constexpr std::uint32_t Rows[32][4] = { {0x80000001u, 0x80000001u, 0x80000001u, 0x00000000u}, {0x00000001u, 0x00000001u, 0x00000001u, 0x00000000u} }; -constexpr std::uint32_t Expected[32][16] = { - {0x3f800000u, 0x3f800000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x3f800000u, 0x00000000u, 0x00000000u, 0x3f800000u, 0x00000000u, 0x00000001u, 0x00000000u, 0x00000001u, 0x80000000u}, - {0x3f800000u, 0x3f800000u, 0x80000000u, 0x00000000u, 0x00800000u, 0x00800000u, 0x80000000u, 0x3f800000u, 0x00800000u, 0x80000000u, 0x3f800000u, 0x80000000u, 0x00000001u, 0x00000000u, 0x00000001u, 0x80000000u}, - {0x3f800000u, 0xbf800000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x3f800000u, 0x00000000u, 0x3f800000u, 0x00000000u, 0x3f000000u, 0x00000000u, 0x00000001u, 0x00000000u, 0x80000000u}, - {0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x3f800000u, 0x3f800000u, 0x00000000u, 0x3f800000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x80000000u}, - {0x3f7fffffu, 0xbf7fffffu, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00800000u, 0x3f7fffffu, 0x00800000u, 0x00000000u, 0x3f800000u, 0x3f7fffffu, 0x00000000u, 0x00000000u, 0xffffff83u, 0x80000000u}, - {0xbf7fffffu, 0x3f7fffffu, 0x80000000u, 0x80000000u, 0x80000000u, 0x80000000u, 0xbf7fffffu, 0x00800000u, 0x80000000u, 0xbf800000u, 0x3f800000u, 0xbf7fffffu, 0x00000001u, 0xffffffffu, 0xffffff83u, 0x80000000u}, - {0x3f7fffffu, 0xbf7fffffu, 0x00800000u, 0x00800000u, 0x00800000u, 0x00800000u, 0x00800001u, 0x3f7fffffu, 0x00800001u, 0x00000000u, 0x3f800000u, 0x3f7fffffu, 0x00000000u, 0x00000000u, 0xffffff83u, 0x80800000u}, - {0x3f000000u, 0xbf000000u, 0x00800000u, 0x00800000u, 0x00800000u, 0x00800000u, 0x01000000u, 0x3f000000u, 0x01000000u, 0x00000000u, 0x3f800000u, 0x3f000000u, 0x00000000u, 0x00000000u, 0xffffff84u, 0x80800000u}, - {0x80000000u, 0x81000000u, 0x80000000u, 0x80000000u, 0x00800000u, 0x00800000u, 0x80800001u, 0x00800000u, 0x00800000u, 0x00000000u, 0x80000000u, 0x3f000000u, 0x00000000u, 0x00000000u, 0xffffff83u, 0x00000000u}, - {0x01000000u, 0x80000000u, 0x00000000u, 0x00000000u, 0x3f800000u, 0x3f800000u, 0x00800000u, 0x3f800000u, 0x00800001u, 0x00000000u, 0x3f800000u, 0x3f000001u, 0x00000000u, 0x00000000u, 0xffffff83u, 0x80000000u}, - {0x00000000u, 0x80000000u, 0x80000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x80000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x80000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u}, - {0x00000000u, 0x00000000u, 0x80000000u, 0x00000000u, 0x80000000u, 0x80000000u, 0x80000000u, 0x00000000u, 0x80000000u, 0x80000000u, 0x00000000u, 0x80000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x80000000u}, - {0x7f800000u, 0xff800000u, 0xffc00000u, 0x00000000u, 0xffc00000u, 0xffc00000u, 0x00000000u, 0x7f800000u, 0x3f800000u, 0x7f800000u, 0x00000000u, 0x7f800000u, 0x00000000u, 0x7fffffffu, 0x00000000u, 0xffc00000u}, - {0xff800000u, 0xff800000u, 0xffc00000u, 0x00000000u, 0xffc00000u, 0xffc00000u, 0xff800000u, 0x00000000u, 0x00000000u, 0x00000000u, 0xff800000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0xffc00000u}, - {0x7fc00000u, 0xffc00000u, 0x7fc00000u, 0x00000000u, 0x7fc00000u, 0x7fc00000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x7fc00000u, 0x00000000u, 0x7fc00000u, 0x00000000u, 0x7fffffffu, 0x00000000u, 0x7fc00000u}, - {0x40000000u, 0xc0000000u, 0x00000000u, 0x00000000u, 0x80800000u, 0x80800000u, 0x00000000u, 0x40000000u, 0x00000000u, 0x40000000u, 0x00000000u, 0x3f000000u, 0x00000000u, 0x00000002u, 0x00000000u, 0x80000000u}, - {0x40000000u, 0x00000000u, 0x3f800000u, 0x3f800000u, 0x3f800000u, 0x3f800000u, 0x3f800000u, 0x3f800000u, 0x3f800000u, 0x3f800000u, 0x3f800000u, 0x3f000000u, 0x00000000u, 0x00000001u, 0x00000001u, 0xbf800000u}, - {0xbf800000u, 0x3f800000u, 0x80000000u, 0x00000000u, 0x3f000000u, 0x3f000000u, 0xbf800000u, 0x3f000000u, 0x00000000u, 0xbf800000u, 0x00000000u, 0xbf000000u, 0x00000001u, 0xffffffffu, 0x00000000u, 0x80000000u}, - {0x34800000u, 0x00000000u, 0x28800000u, 0x28800000u, 0x28800000u, 0x28800000u, 0x34000000u, 0x34000000u, 0x34000000u, 0x00000000u, 0x3f800000u, 0x3f000000u, 0x00000000u, 0x00000000u, 0xffffffeau, 0xa8800000u}, - {0x20000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x1f800000u, 0x1f800000u, 0x1f800000u, 0x00000000u, 0x3f800000u, 0x3f000000u, 0x00000000u, 0x00000000u, 0xffffffc1u, 0x80000000u}, - {0x207fffffu, 0x00000000u, 0x00000000u, 0x00000000u, 0x00ffffffu, 0x00ffffffu, 0x1fffffffu, 0x1fffffffu, 0x1fffffffu, 0x00000000u, 0x3f800000u, 0x3f7fffffu, 0x00000000u, 0x00000000u, 0xffffffc1u, 0x80000000u}, - {0x20200000u, 0x9fc00000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x1f000000u, 0x20000000u, 0x1f000000u, 0x00000000u, 0x3f800000u, 0x3f000000u, 0x00000000u, 0x00000000u, 0xffffffc0u, 0x80000000u}, - {0x20400000u, 0x9f800000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x1f7fffffu, 0x20000000u, 0x1f7fffffu, 0x00000000u, 0x3f800000u, 0x3f000000u, 0x00000000u, 0x00000000u, 0xffffffc0u, 0x80000000u}, - {0x20400001u, 0x9f800002u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x1f7fffffu, 0x20000001u, 0x1f7fffffu, 0x00000000u, 0x3f800000u, 0x3f000001u, 0x00000000u, 0x00000000u, 0xffffffc0u, 0x80000000u}, - {0x9f800000u, 0x20400000u, 0x80000000u, 0x80000000u, 0x80000000u, 0x80000000u, 0xa0000000u, 0x1f7fffffu, 0x80000000u, 0xbf800000u, 0x3f800000u, 0xbf000000u, 0x00000001u, 0xffffffffu, 0xffffffc0u, 0x80000000u}, - {0x00000000u, 0xc0000000u, 0xbf800000u, 0xbf800000u, 0xbf800000u, 0xbf800000u, 0xbf800000u, 0x3f800000u, 0x00000000u, 0x3f800000u, 0xbf800000u, 0x3f000000u, 0x00000000u, 0x00000001u, 0x00000001u, 0x3f800000u}, - {0x3f000000u, 0x3f000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00800000u, 0x3f000000u, 0x00800000u, 0x00000000u, 0x3f800000u, 0x3f000000u, 0x00000001u, 0x00000000u, 0x00000000u, 0x80000000u}, - {0x3f7fffffu, 0x3f7fffffu, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00800000u, 0x3f7fffffu, 0x00800000u, 0x00000000u, 0x3f800000u, 0x3f000000u, 0x00000001u, 0x00000000u, 0x00000000u, 0x80000000u}, - {0x41200000u, 0xc1200000u, 0x00000000u, 0x00000000u, 0x3dcccccdu, 0x3dcccccdu, 0x00000000u, 0x41200000u, 0x3dcccccdu, 0x41200000u, 0x00000000u, 0x3f200000u, 0x00000000u, 0x0000000au, 0x00000000u, 0x80000000u}, - {0xc0490fdbu, 0x40490fdbu, 0x00000000u, 0x00000000u, 0x40490fdbu, 0x40490fdbu, 0xc0490fdbu, 0x40490fdbu, 0x80000000u, 0xc0800000u, 0x80000000u, 0xbf490fdbu, 0x00000001u, 0xfffffffcu, 0x00000000u, 0x00000000u}, - {0x80000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x80000000u, 0x80000000u, 0x80000000u, 0x80000000u, 0x80000000u, 0x80000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u}, - {0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x80000000u} +constexpr std::uint32_t Expected[32][19] = { + {0x3f800000u, 0x3f800000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x3f800000u, 0x00000000u, 0x00000000u, 0x3f800000u, 0x00000000u, 0x00000001u, 0x00000000u, 0x00000001u, 0x80000000u, 0x00000000u, 0xff7fffffu, 0x00000000u}, + {0x3f800000u, 0x3f800000u, 0x80000000u, 0x00000000u, 0x00800000u, 0x00800000u, 0x80000000u, 0x3f800000u, 0x00800000u, 0x80000000u, 0x3f800000u, 0x80000000u, 0x00000001u, 0x00000000u, 0x00000001u, 0x80000000u, 0x00400000u, 0x00000000u, 0x00000000u}, + {0x3f800000u, 0xbf800000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x3f800000u, 0x00000000u, 0x3f800000u, 0x00000000u, 0x3f000000u, 0x00000000u, 0x00000001u, 0x00000000u, 0x80000000u, 0xffc00000u, 0xff7fffffu, 0x00000000u}, + {0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x3f800000u, 0x3f800000u, 0x00000000u, 0x3f800000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x80000000u, 0x7f800000u, 0x00000000u, 0x00000000u}, + {0x3f7fffffu, 0xbf7fffffu, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00800000u, 0x3f7fffffu, 0x00800000u, 0x00000000u, 0x3f800000u, 0x3f7fffffu, 0x00000000u, 0x00000000u, 0xffffff83u, 0x80000000u, 0x00000000u, 0xff7fffffu, 0x00000000u}, + {0xbf7fffffu, 0x3f7fffffu, 0x80000000u, 0x80000000u, 0x80000000u, 0x80000000u, 0xbf7fffffu, 0x00800000u, 0x80000000u, 0xbf800000u, 0x3f800000u, 0xbf7fffffu, 0x00000001u, 0xffffffffu, 0xffffff83u, 0x80000000u, 0x80000000u, 0xff7fffffu, 0x80000000u}, + {0x3f7fffffu, 0xbf7fffffu, 0x00800000u, 0x00800000u, 0x00800000u, 0x00800000u, 0x00800001u, 0x3f7fffffu, 0x00800001u, 0x00000000u, 0x3f800000u, 0x3f7fffffu, 0x00000000u, 0x00000000u, 0xffffff83u, 0x80800000u, 0x00000000u, 0xff7fffffu, 0x00800000u}, + {0x3f000000u, 0xbf000000u, 0x00800000u, 0x00800000u, 0x00800000u, 0x00800000u, 0x01000000u, 0x3f000000u, 0x01000000u, 0x00000000u, 0x3f800000u, 0x3f000000u, 0x00000000u, 0x00000000u, 0xffffff84u, 0x80800000u, 0x80000000u, 0xff7fffffu, 0x00800000u}, + {0x80000000u, 0x81000000u, 0x80000000u, 0x80000000u, 0x00800000u, 0x00800000u, 0x80800001u, 0x00800000u, 0x00800000u, 0x00000000u, 0x80000000u, 0x3f000000u, 0x00000000u, 0x00000000u, 0xffffff83u, 0x00000000u, 0x80800000u, 0x80000000u, 0x80000000u}, + {0x01000000u, 0x80000000u, 0x00000000u, 0x00000000u, 0x3f800000u, 0x3f800000u, 0x00800000u, 0x3f800000u, 0x00800001u, 0x00000000u, 0x3f800000u, 0x3f000001u, 0x00000000u, 0x00000000u, 0xffffff83u, 0x80000000u, 0x00800001u, 0x00000000u, 0x00000000u}, + {0x00000000u, 0x80000000u, 0x80000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x80000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x80000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0xffc00000u, 0xff7fffffu, 0x00000000u}, + {0x00000000u, 0x00000000u, 0x80000000u, 0x00000000u, 0x80000000u, 0x80000000u, 0x80000000u, 0x00000000u, 0x80000000u, 0x80000000u, 0x00000000u, 0x80000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x80000000u, 0xffc00000u, 0xff7fffffu, 0x00000000u}, + {0x7f800000u, 0xff800000u, 0xffc00000u, 0x00000000u, 0xffc00000u, 0xffc00000u, 0x00000000u, 0x7f800000u, 0x3f800000u, 0x7f800000u, 0x00000000u, 0x7f800000u, 0x00000000u, 0x7fffffffu, 0x00000000u, 0xffc00000u, 0x7f800000u, 0x00000000u, 0x00000000u}, + {0xff800000u, 0xff800000u, 0xffc00000u, 0x00000000u, 0xffc00000u, 0xffc00000u, 0xff800000u, 0x00000000u, 0x00000000u, 0x00000000u, 0xff800000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0xffc00000u, 0x80000000u, 0xff7fffffu, 0xff7fffffu}, + {0x7fc00000u, 0xffc00000u, 0x7fc00000u, 0x00000000u, 0x7fc00000u, 0x7fc00000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x7fc00000u, 0x00000000u, 0x7fc00000u, 0x00000000u, 0x7fffffffu, 0x00000000u, 0x7fc00000u, 0xffc00000u, 0xff7fffffu, 0x00000000u}, + {0x40000000u, 0xc0000000u, 0x00000000u, 0x00000000u, 0x80800000u, 0x80800000u, 0x00000000u, 0x40000000u, 0x00000000u, 0x40000000u, 0x00000000u, 0x3f000000u, 0x00000000u, 0x00000002u, 0x00000000u, 0x80000000u, 0xff800000u, 0xff7fffffu, 0x00000000u}, + {0x40000000u, 0x00000000u, 0x3f800000u, 0x3f800000u, 0x3f800000u, 0x3f800000u, 0x3f800000u, 0x3f800000u, 0x3f800000u, 0x3f800000u, 0x3f800000u, 0x3f000000u, 0x00000000u, 0x00000001u, 0x00000001u, 0xbf800000u, 0x80000000u, 0xff7fffffu, 0x3f800000u}, + {0xbf800000u, 0x3f800000u, 0x80000000u, 0x00000000u, 0x3f000000u, 0x3f000000u, 0xbf800000u, 0x3f000000u, 0x00000000u, 0xbf800000u, 0x00000000u, 0xbf000000u, 0x00000001u, 0xffffffffu, 0x00000000u, 0x80000000u, 0x7f800000u, 0x00000000u, 0x00000000u}, + {0x34800000u, 0x00000000u, 0x28800000u, 0x28800000u, 0x28800000u, 0x28800000u, 0x34000000u, 0x34000000u, 0x34000000u, 0x00000000u, 0x3f800000u, 0x3f000000u, 0x00000000u, 0x00000000u, 0xffffffeau, 0xa8800000u, 0x00000000u, 0xff7fffffu, 0x28800000u}, + {0x20000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x1f800000u, 0x1f800000u, 0x1f800000u, 0x00000000u, 0x3f800000u, 0x3f000000u, 0x00000000u, 0x00000000u, 0xffffffc1u, 0x80000000u, 0x00000000u, 0xff7fffffu, 0x00000000u}, + {0x207fffffu, 0x00000000u, 0x00000000u, 0x00000000u, 0x00ffffffu, 0x00ffffffu, 0x1fffffffu, 0x1fffffffu, 0x1fffffffu, 0x00000000u, 0x3f800000u, 0x3f7fffffu, 0x00000000u, 0x00000000u, 0xffffffc1u, 0x80000000u, 0x1fffffffu, 0x00000000u, 0x00000000u}, + {0x20200000u, 0x9fc00000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x1f000000u, 0x20000000u, 0x1f000000u, 0x00000000u, 0x3f800000u, 0x3f000000u, 0x00000000u, 0x00000000u, 0xffffffc0u, 0x80000000u, 0x00000000u, 0xff7fffffu, 0x00000000u}, + {0x20400000u, 0x9f800000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x1f7fffffu, 0x20000000u, 0x1f7fffffu, 0x00000000u, 0x3f800000u, 0x3f000000u, 0x00000000u, 0x00000000u, 0xffffffc0u, 0x80000000u, 0x00000000u, 0xff7fffffu, 0x00000000u}, + {0x20400001u, 0x9f800002u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x1f7fffffu, 0x20000001u, 0x1f7fffffu, 0x00000000u, 0x3f800000u, 0x3f000001u, 0x00000000u, 0x00000000u, 0xffffffc0u, 0x80000000u, 0x00000000u, 0xff7fffffu, 0x00000000u}, + {0x9f800000u, 0x20400000u, 0x80000000u, 0x80000000u, 0x80000000u, 0x80000000u, 0xa0000000u, 0x1f7fffffu, 0x80000000u, 0xbf800000u, 0x3f800000u, 0xbf000000u, 0x00000001u, 0xffffffffu, 0xffffffc0u, 0x80000000u, 0x80000000u, 0xff7fffffu, 0x80000000u}, + {0x00000000u, 0xc0000000u, 0xbf800000u, 0xbf800000u, 0xbf800000u, 0xbf800000u, 0xbf800000u, 0x3f800000u, 0x00000000u, 0x3f800000u, 0xbf800000u, 0x3f000000u, 0x00000000u, 0x00000001u, 0x00000001u, 0x3f800000u, 0x80000000u, 0xff7fffffu, 0xbf800000u}, + {0x3f000000u, 0x3f000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00800000u, 0x3f000000u, 0x00800000u, 0x00000000u, 0x3f800000u, 0x3f000000u, 0x00000001u, 0x00000000u, 0x00000000u, 0x80000000u, 0x00000000u, 0xff7fffffu, 0x00000000u}, + {0x3f7fffffu, 0x3f7fffffu, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00800000u, 0x3f7fffffu, 0x00800000u, 0x00000000u, 0x3f800000u, 0x3f000000u, 0x00000001u, 0x00000000u, 0x00000000u, 0x80000000u, 0x00000000u, 0xff7fffffu, 0x00000000u}, + {0x41200000u, 0xc1200000u, 0x00000000u, 0x00000000u, 0x3dcccccdu, 0x3dcccccdu, 0x00000000u, 0x41200000u, 0x3dcccccdu, 0x41200000u, 0x00000000u, 0x3f200000u, 0x00000000u, 0x0000000au, 0x00000000u, 0x80000000u, 0x7f800000u, 0x00000000u, 0x00000000u}, + {0xc0490fdbu, 0x40490fdbu, 0x00000000u, 0x00000000u, 0x40490fdbu, 0x40490fdbu, 0xc0490fdbu, 0x40490fdbu, 0x80000000u, 0xc0800000u, 0x80000000u, 0xbf490fdbu, 0x00000001u, 0xfffffffcu, 0x00000000u, 0x00000000u, 0xff800000u, 0x00000000u, 0x00000000u}, + {0x80000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x80000000u, 0x80000000u, 0x80000000u, 0x80000000u, 0x80000000u, 0x80000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0xffc00000u, 0xff7fffffu, 0x00000000u}, + {0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x80000000u, 0xffc00000u, 0xff7fffffu, 0x00000000u} }; -constexpr const char* Names[16] = { +constexpr const char* Names[19] = { "v_add_f32 v10, v4, v5", "v_sub_f32 v11, v5, v4", "v_mul_f32 v12, v4, v5", @@ -117,7 +118,10 @@ constexpr const char* Names[16] = { "v_cmp_lt_f32 v4, v5", "v_cvt_flr_i32_f32 v23, v4", "v_frexp_exp_i32_f32 v24, v5", - "v_mul_f32 v25, |v4|, -v5" + "v_mul_f32 v25, |v4|, -v5", + "v_div_fixup_f32 v26, v4, v5, v6", + "v_mullit_f32 v27, v4, v5, v6", + "v_mullit_f32 v28, v4, v5, 1.0" }; void Fill(std::uint32_t tid, std::uint32_t* words) { @@ -170,8 +174,8 @@ void Run(AgcDriver::VulkanDevice& device, const std::optional= 15u; if (floatResult && IsNan(Expected[tid][i])) { Require(IsNan(out[i]), std::string("f32 denormal flush: lane ") + std::to_string(tid) + " " + mode + " " + Names[i] + " is " + Hex(out[i]) + ", expected a NaN"); continue; diff --git a/core/libs/prx/libSceAgcDriver/tests/execution/F64Arithmetic.cpp b/core/libs/prx/libSceAgcDriver/tests/execution/F64Arithmetic.cpp index c8e592d6f..38ff1defd 100644 --- a/core/libs/prx/libSceAgcDriver/tests/execution/F64Arithmetic.cpp +++ b/core/libs/prx/libSceAgcDriver/tests/execution/F64Arithmetic.cpp @@ -22,6 +22,7 @@ constexpr std::uint32_t Inputs = 4; constexpr std::uint32_t Results = 16; alignas(256) std::array Input{}; alignas(256) std::array Output{}; +bool Ieee = false; alignas(256) constexpr std::array Code{ 0x34020084, 0x34060086, 0xe0381000, 0x80000401, 0xbf8c3f70, 0xd564000a, 0x02020d04, 0xd565000c, @@ -98,6 +99,40 @@ constexpr std::uint32_t Expected[32][16] = { {0x00000000u, 0x7ffc0000u, 0x00000000u, 0x7ffc0000u, 0x00000000u, 0x7ffc0000u, 0x00000000u, 0x7ffc0000u, 0x00000000u, 0x7ffc0000u, 0x00000000u, 0x7ffc0000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x7ffc0000u}, {0x00000000u, 0x41e00000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x41e00000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x41e00000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x41e00000u} }; +constexpr std::uint32_t MinMaxNoIeee[32][4] = { + {0x00000000u, 0x3ff00000u, 0x80000000u, 0x40000000u}, + {0x00000000u, 0x3ff80000u, 0x7fffffffu, 0x40000000u}, + {0xfffff7ccu, 0x3ff00000u, 0xffffffffu, 0x7fefffffu}, + {0x00000000u, 0xbff00000u, 0xfffffbcdu, 0x3ff00000u}, + {0x00000000u, 0x3fe00000u, 0x00000401u, 0x3ff00000u}, + {0x00000000u, 0xfff00000u, 0xffc00000u, 0x41dfffffu}, + {0x12eaf8e8u, 0xc020f641u, 0x8a59a474u, 0xbefa45c5u}, + {0x00000000u, 0x3ff00000u, 0x00000000u, 0x3ff00000u}, + {0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u}, + {0x00000000u, 0x80000000u, 0x00000000u, 0x00000000u}, + {0x00000000u, 0xbff00000u, 0x00000000u, 0x7ff00000u}, + {0x00000000u, 0x80000000u, 0x00000000u, 0x80000000u}, + {0x00000000u, 0x7ff00000u, 0x00000000u, 0x7ff00000u}, + {0x00000000u, 0x41f00000u, 0x00000000u, 0x41f00000u}, + {0xffffffffu, 0xffefffffu, 0x00000000u, 0x3ff80000u}, + {0x00000000u, 0xfff00000u, 0x00000000u, 0xfff00000u}, + {0xc66ab30du, 0x8e9a2140u, 0x047b57e5u, 0x02f22376u}, + {0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u}, + {0x00000000u, 0x00000000u, 0x00000000u, 0x3ff00000u}, + {0x00000000u, 0xbff00000u, 0x00000000u, 0x00000000u}, + {0x00000000u, 0x00000000u, 0x00000000u, 0x3fe00000u}, + {0x00000000u, 0xbfe00000u, 0x00000000u, 0x00000000u}, + {0x00000000u, 0x00000000u, 0x00000000u, 0x3ff80000u}, + {0x00000000u, 0x00000000u, 0x00000000u, 0x40040000u}, + {0x00000000u, 0xc0040000u, 0x00000000u, 0x00000000u}, + {0x00000000u, 0x00000000u, 0xffffffffu, 0x7fefffffu}, + {0xffffffffu, 0xffefffffu, 0x00000000u, 0x00000000u}, + {0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u}, + {0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u}, + {0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u}, + {0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u}, + {0x00000000u, 0x00000000u, 0x00000000u, 0x41e00000u} +}; constexpr const char* Names[16] = {"add lo", "add hi", "mul lo", "mul hi", "fma lo", "fma hi", "fma error lo", "fma error hi", "min lo", "min hi", "max lo", "max hi", "ldexp lo", "ldexp hi", "add abs neg lo", "add abs neg hi"}; void Fill(std::uint32_t tid, std::uint32_t* words) { @@ -116,7 +151,7 @@ std::string Hex(std::uint32_t value) { } void Expect(std::uint32_t tid, std::uint32_t actual, std::uint32_t expected, const char* name) { - Require(actual == expected, std::string("f64 arithmetic: lane ") + std::to_string(tid) + " " + name + " is " + Hex(actual) + ", expected " + Hex(expected)); + Require(actual == expected, std::string(Ieee ? "f64 arithmetic, IEEE mode: lane " : "f64 arithmetic: lane ") + std::to_string(tid) + " " + name + " is " + Hex(actual) + ", expected " + Hex(expected)); } void Run(AgcDriver::VulkanDevice& device) { @@ -137,6 +172,7 @@ void Run(AgcDriver::VulkanDevice& device) { {0, 0, 0, 128} }; request.useCache = false; + if (Ieee) request.context.floatMode = ShaderRecompiler::ShaderFloatMode{0xf0u, true, true, false}; const auto result = ShaderRecompiler::Recompile(request); device.Dispatch(result, 1, 1, 1, {}, reinterpret_cast(code.data())); device.WaitIdle(); @@ -146,7 +182,11 @@ void Check() { for (std::uint32_t tid = 0; tid < Threads; ++tid) { const std::uint32_t* in = &Input[tid * Inputs]; const std::uint32_t* out = &Output[tid * Results]; - for (std::uint32_t i = 0; i < 16; ++i) Expect(tid, out[i], Expected[tid][i], Names[i]); + for (std::uint32_t i = 0; i < 16; ++i) { + std::uint32_t expected = !Ieee && i >= 8u && i < 12u ? MinMaxNoIeee[tid][i - 8u] : Expected[tid][i]; + if (!Ieee && i == 13u && tid == 13u) expected = in[1]; + Expect(tid, out[i], expected, Names[i]); + } } } @@ -160,8 +200,11 @@ int main() { std::puts("skipped, the device has no shaderFloat64"); return VulkanTestSkipped; } - Run(*device); - Check(); + for (const bool ieee : {false, true}) { + Ieee = ieee; + Run(*device); + Check(); + } std::puts("f64 arithmetic tests passed"); return 0; } catch (const std::exception& error) { diff --git a/core/libs/prx/libSceAgcDriver/tests/execution/F64Conversions.cpp b/core/libs/prx/libSceAgcDriver/tests/execution/F64Conversions.cpp index ab7877a62..67e8aab62 100644 --- a/core/libs/prx/libSceAgcDriver/tests/execution/F64Conversions.cpp +++ b/core/libs/prx/libSceAgcDriver/tests/execution/F64Conversions.cpp @@ -137,6 +137,7 @@ void Run(AgcDriver::VulkanDevice& device) { {0, 0, 0, 128} }; request.useCache = false; + request.context.floatMode = ShaderRecompiler::ShaderFloatMode{0xf0u, true, true, false}; const auto result = ShaderRecompiler::Recompile(request); device.Dispatch(result, 1, 1, 1, {}, reinterpret_cast(code.data())); device.WaitIdle(); diff --git a/core/libs/prx/libSceAgcDriver/tests/execution/F64Division.cpp b/core/libs/prx/libSceAgcDriver/tests/execution/F64Division.cpp index 84d1c6f68..24f644084 100644 --- a/core/libs/prx/libSceAgcDriver/tests/execution/F64Division.cpp +++ b/core/libs/prx/libSceAgcDriver/tests/execution/F64Division.cpp @@ -138,6 +138,7 @@ void Run(AgcDriver::VulkanDevice& device) { {0, 0, 0, 128} }; request.useCache = false; + request.context.floatMode = ShaderRecompiler::ShaderFloatMode{0xf0u, true, true, false}; const auto result = ShaderRecompiler::Recompile(request); device.Dispatch(result, 1, 1, 1, {}, reinterpret_cast(code.data())); device.WaitIdle(); diff --git a/core/libs/prx/libSceAgcDriver/tests/execution/F64DivisionModifiers.cpp b/core/libs/prx/libSceAgcDriver/tests/execution/F64DivisionModifiers.cpp index 16dcd8747..61f5616a9 100644 --- a/core/libs/prx/libSceAgcDriver/tests/execution/F64DivisionModifiers.cpp +++ b/core/libs/prx/libSceAgcDriver/tests/execution/F64DivisionModifiers.cpp @@ -138,6 +138,7 @@ void Run(AgcDriver::VulkanDevice& device) { {0, 0, 0, 128} }; request.useCache = false; + request.context.floatMode = ShaderRecompiler::ShaderFloatMode{0xf0u, true, true, false}; const auto result = ShaderRecompiler::Recompile(request); device.Dispatch(result, 1, 1, 1, {}, reinterpret_cast(code.data())); device.WaitIdle(); diff --git a/core/libs/prx/libSceAgcDriver/tests/execution/F64FmaSubnormals.cpp b/core/libs/prx/libSceAgcDriver/tests/execution/F64FmaSubnormals.cpp new file mode 100644 index 000000000..b77fe2b37 --- /dev/null +++ b/core/libs/prx/libSceAgcDriver/tests/execution/F64FmaSubnormals.cpp @@ -0,0 +1,157 @@ +#include "prx/libSceAgcDriver/Execution/include/VulkanDevice.hpp" +#include "prx/libSceAgcDriver/Graphics/include/Draw.hpp" +#include "Recompiler.hpp" +#include "VulkanTestDevice.hpp" +#include +#include +#include +#include +#include +#include +#include + +namespace { + +using AgcDriver::Graphics::Require; +using ShaderRecompiler::ShaderStage; + +constexpr std::uint32_t Threads = 32; +constexpr std::uint32_t Inputs = 8; +constexpr std::uint32_t Results = 2; +alignas(256) std::array Input{}; +alignas(256) std::array Output{}; + +alignas(256) constexpr std::array Code{ + 0x34020085, 0x34060083, 0xe0381000, 0x80000401, 0xe0381010, 0x80000801, 0xbf8c3f70, 0xd54c000c, + 0x04220d04, 0xe0741000, 0x80010c03, 0xbf810000, +}; + +alignas(256) constexpr std::array FmasCode{ + 0x34020085, 0x34060083, 0xe0381000, 0x80000401, 0xe0381010, 0x80000801, 0xbf8c3f70, 0xbeea03c1, + 0xd570000c, 0x04220d04, 0xe0741000, 0x80010c03, 0xbf810000, +}; + +constexpr std::uint64_t Rows[Threads][4] = { + {0xaf909f1570e40fb2ull, 0x10697a78dd1df49bull, 0x8000000000000000ull, 0x800d3be6095b3463ull}, + {0x1e50511fd30f5d67ull, 0x24c92a1b12df5283ull, 0x8329a9b24976c10bull, 0x0006cc7690f01937ull}, + {0x98cd9ffcf3ed94edull, 0x2a51579178c2f7a9ull, 0x03300e10060c0a21ull, 0x000826b2393bc091ull}, + {0x2c67bd24a078a9e3ull, 0x169c3293f5b93146ull, 0x8314eb0558ce14d3ull, 0x8002b781181390c7ull}, + {0xbcca08e86f6c243dull, 0x0647cf3a0cd21b23ull, 0x03235effea3a0f40ull, 0x00053433dc65a9cbull}, + {0x97cb9cf8c8a2fbb6ull, 0xab2cb31ef6264b43ull, 0x8308c3ea76c1ebe5ull, 0x00014c7d2a0da8cbull}, + {0x0946591c0410303cull, 0xba059dbfbeb6d4adull, 0x80099e28c52b6c3dull, 0x835e314afa5d1a07ull}, + {0xcee37832c306ee31ull, 0x80017e95cd7d98dfull, 0x8c168438ac98afa4ull, 0x0ecd18c0555dceefull}, + {0x800e8b91a326e01dull, 0xd020fb9f53a2df2cull, 0x8cb514f5ff6d74b0ull, 0x103ee09fedab559aull}, + {0x401fac71cda29715ull, 0x8008000020000000ull, 0x002fac724c545e4full, 0x0000000000000007ull}, + {0x2e4c000000000000ull, 0x11aeb6db849700ffull, 0x800d70000a021070ull, 0x8000000000000000ull}, + {0x41d0000000000008ull, 0x0000000040000002ull, 0x80761e2b2ef2fce7ull, 0x0084f0ea6986819dull}, + {0xfc21916e447cf802ull, 0x8004eec859174483ull, 0x3be2fe907aa80074ull, 0x3c26da145be5eaacull}, + {0x35d175fd70782c6cull, 0x0c6033e7239c8fe0ull, 0x0005cb65548dabd1ull, 0x0241aea1c6a7627dull}, + {0xb4fb1dc3f1ce6211ull, 0x8b80000000000100ull, 0x800000000b9b5f36ull, 0x008b1dc3f1b72d04ull}, + {0x0008000000000000ull, 0x8004000000000000ull, 0x0000000000000000ull, 0x8000000000000000ull}, + {0x0008000000000000ull, 0x0004000000000000ull, 0x0170000000000000ull, 0x0170000000000000ull}, + {0x4004000000000000ull, 0x0000000000000001ull, 0x0000000000000000ull, 0x0000000000000002ull}, + {0x3fe0000000000000ull, 0x0000000000000001ull, 0x0000000000000000ull, 0x0000000000000000ull}, + {0x3fe0000000000001ull, 0x0000000000000001ull, 0x0000000000000000ull, 0x0000000000000001ull}, + {0x3fefffffffffffffull, 0x0010000000000000ull, 0x0000000000000000ull, 0x0010000000000000ull}, + {0x1e60000000000000ull, 0x1e60000000000000ull, 0x8000000000000001ull, 0x0000000000000000ull}, + {0x3ff0000000000001ull, 0x0018000000000001ull, 0x8010000000000000ull, 0x0008000000000003ull}, + {0x3ff8000000000000ull, 0x4000000000000000ull, 0x3fd0000000000000ull, 0x400a000000000000ull}, + {0x3ff0000000000001ull, 0x3ff0000000000001ull, 0xbff0000000000000ull, 0x3cc0000000000000ull}, + {0xffefffffffffffffull, 0x3ff0000000000000ull, 0x000fffffffffffffull, 0xffefffffffffffffull}, + {0x5fefffffffffffffull, 0x5fefffffffffffffull, 0x0000000000000000ull, 0x7feffffffffffffeull}, + {0x000fffffffffffffull, 0x4000000000000000ull, 0x8000000000000001ull, 0x001ffffffffffffdull}, + {0x8000000000000003ull, 0x7fe8000000000000ull, 0x3ff0000000000000ull, 0x3fefffffffffffeeull}, + {0x1a75555555555555ull, 0x000aaaaaaaaaaaabull, 0x0000000000000000ull, 0x0000000000000000ull}, + {0x0173000000000000ull, 0x3e17000000000000ull, 0x80001b5000000000ull, 0x0000000000000000ull}, + {0xa000000000000001ull, 0x1fffffffffffffffull, 0x0010000000000000ull, 0x8000000000000000ull}, +}; + +constexpr std::uint64_t FmasRows[][4] = { + {0xcee37832c306ee31ull, 0x80017e95cd7d98dfull, 0x8c168438ac98afa4ull, 0x06cd18c0555dceefull}, + {0x800e8b91a326e01dull, 0xd020fb9f53a2df2cull, 0x8cb514f5ff6d74b0ull, 0x083ee09fedab559aull}, + {0xfc21916e447cf802ull, 0x8004eec859174483ull, 0x3be2fe907aa80074ull, 0x3426da145be5eaacull}, + {0x6456b4d74e778407ull, 0x0000000000000112ull, 0x20280ac49e7f906cull, 0x19b84d8e7e014d9full}, + {0x3b87a9e17b60d8beull, 0x0c5959dcff86b48aull, 0x026239792b5377a2ull, 0x0004afcb04199627ull}, + {0xaf25fa06e8714b4bull, 0x18d2b6bb683e8122ull, 0x02a53f820f7c716bull, 0x800cda229eb9a09dull}, + {0xb99c5fe31c736fbbull, 0x8e5264264dcfc027ull, 0x8022aa8efefbe52cull, 0x0008275eae4b3d41ull}, + {0xc6fdfbc0001ad5dfull, 0x00ffcab17e745bd1ull, 0x82e0e3a0b85211a8ull, 0x800ee4e6bd8a79e5ull}, + {0xc1d1a6cfa76ad3b9ull, 0x862ba38c1492703full, 0x017130645c1851d5ull, 0x000f3ef62c247005ull}, + {0xada41786a4b824bcull, 0x1a554baa4a83fe63ull, 0x0134fb8e603406d4ull, 0x800d5ef26b6f7a29ull}, + {0xb81b70578b6c14b9ull, 0x8fcdd6775e710cc1ull, 0x829c666c28b85110ull, 0x0006656d3ea3b537ull}, + {0x2c256baa0ef1c91bull, 0x9baa7695febd5f53ull, 0x82d3dfc5a191618bull, 0x800236db72e217c3ull}, + {0x355013dab21e2539ull, 0x1296e6732f928ca7ull, 0x82f4e48c5e9c126cull, 0x0005c0b776cad66bull}, + {0xc458817a2d2ea918ull, 0x03a473fd45d66f08ull, 0x03240e0c4f7cfcc4ull, 0x800fa9bfa078e4f1ull}, + {0x33291df738c3f676ull, 0x146a0194f3933c51ull, 0x0120115306c4e7b2ull, 0x000028d32d6f3f07ull}, + {0xb1d797d703ad9014ull, 0x9613cb63b23f1b8bull, 0x0244b2cb24e6fb3aull, 0x00074c0e33ad5c9dull}, + {0x4656b4266997b7f2ull, 0x819765c0b9bfcad0ull, 0x03c9a52d2b9f1094ull, 0x80084cd678e92eefull}, + {0x4513191709caec8cull, 0x02d3b5ddc4ac6b05ull, 0x826384dfaf4b9e81ull, 0x0005e1b7f963c7dbull}, + {0xba6ec9ec81f6efb7ull, 0x8d785fa355446f4eull, 0x0279c121fe09c0aaull, 0x0005dcdd7f8c7cd3ull}, + {0xbe667b0282fcc03cull, 0x897ce8a400290c2cull, 0x0378775e473d7a50ull, 0x000513c44ed72187ull}, +}; + +std::array BufferDescriptor(const void* data, std::uint32_t bytes) { + const auto address = reinterpret_cast(data); + return {static_cast(address), static_cast((address >> 32u) & 0xffffu), bytes, 0x31016facu}; +} + +std::string Hex(std::uint64_t value) { + char text[24]; + std::snprintf(text, sizeof(text), "0x%016llx", static_cast(value)); + return text; +} + +template +void Run(AgcDriver::VulkanDevice& device, const std::array& program, const std::uint64_t (&rows)[Count][4]) { + Input.fill(0u); + for (std::uint32_t tid = 0; tid < Count; ++tid) { + for (std::uint32_t operand = 0; operand < 3; ++operand) { + Input[tid * Inputs + operand * 2] = static_cast(rows[tid][operand]); + Input[tid * Inputs + operand * 2 + 1] = static_cast(rows[tid][operand] >> 32u); + } + } + Output.fill(0xdeadbeefu); + std::vector userData(8, 0u); + const auto input = BufferDescriptor(Input.data(), static_cast(Input.size() * 4u)); + const auto output = BufferDescriptor(Output.data(), static_cast(Output.size() * 4u)); + std::copy(input.begin(), input.end(), userData.begin()); + std::copy(output.begin(), output.end(), userData.begin() + 4); + const std::span code(program); + const std::array memory{{{reinterpret_cast(code.data()), std::as_bytes(code)}}}; + const ShaderRecompiler::ShaderComputeStageInfo compute{{Threads, 1, 1}, 0u, {false, false, false}, false, 1}; + ShaderRecompiler::RecompileRequest request{ + {ShaderStage::Compute, reinterpret_cast(code.data()), code, 0, {}}, + {32, 0, userData, compute, std::nullopt, std::nullopt, memory}, + device.Target(), + {0, 0, 0, 128} + }; + request.useCache = false; + const auto result = ShaderRecompiler::Recompile(request); + device.Dispatch(result, 1, 1, 1, {}, reinterpret_cast(code.data())); + device.WaitIdle(); +} + +template +void Check(const char* name, const std::uint64_t (&rows)[Count][4]) { + for (std::uint32_t tid = 0; tid < Count; ++tid) { + const std::uint64_t actual = Output[tid * Results] | (static_cast(Output[tid * Results + 1]) << 32u); + Require(actual == rows[tid][3], std::string(name) + ": lane " + std::to_string(tid) + " (" + Hex(rows[tid][0]) + ", " + Hex(rows[tid][1]) + ", " + Hex(rows[tid][2]) + ") is " + Hex(actual) + ", expected " + Hex(rows[tid][3])); + } +} + +} + +int main() { + try { + const auto device = OpenVulkanTestDevice(); + if (!device) return VulkanTestSkipped; + Run(*device, Code, Rows); + Check("v_fma_f64", Rows); + Run(*device, FmasCode, FmasRows); + Check("v_div_fmas_f64 with VCC set", FmasRows); + std::puts("v_fma_f64 subnormal tests passed"); + return 0; + } catch (const std::exception& error) { + std::cerr << error.what() << '\n'; + return 1; + } +} diff --git a/core/libs/prx/libSceAgcDriver/tests/execution/F64NanConversions.cpp b/core/libs/prx/libSceAgcDriver/tests/execution/F64NanConversions.cpp index 39d5cc579..074d91448 100644 --- a/core/libs/prx/libSceAgcDriver/tests/execution/F64NanConversions.cpp +++ b/core/libs/prx/libSceAgcDriver/tests/execution/F64NanConversions.cpp @@ -132,6 +132,7 @@ void Run(AgcDriver::VulkanDevice& device) { {0, 0, 0, 128} }; request.useCache = false; + request.context.floatMode = ShaderRecompiler::ShaderFloatMode{0xf0u, true, true, false}; const auto result = ShaderRecompiler::Recompile(request); device.Dispatch(result, 1, 1, 1, {}, reinterpret_cast(code.data())); device.WaitIdle(); diff --git a/core/libs/prx/libSceAgcDriver/tests/execution/F64Transcendental.cpp b/core/libs/prx/libSceAgcDriver/tests/execution/F64Transcendental.cpp index c3d8c21b3..f991f67fe 100644 --- a/core/libs/prx/libSceAgcDriver/tests/execution/F64Transcendental.cpp +++ b/core/libs/prx/libSceAgcDriver/tests/execution/F64Transcendental.cpp @@ -152,6 +152,7 @@ void Run(AgcDriver::VulkanDevice& device) { {0, 0, 0, 128} }; request.useCache = false; + request.context.floatMode = ShaderRecompiler::ShaderFloatMode{0xf0u, true, true, false}; const auto result = ShaderRecompiler::Recompile(request); device.Dispatch(result, 1, 1, 1, {}, reinterpret_cast(code.data())); device.WaitIdle(); diff --git a/core/libs/prx/libSceAgcDriver/tests/execution/FillCoverKeys.cpp b/core/libs/prx/libSceAgcDriver/tests/execution/FillCoverKeys.cpp new file mode 100644 index 000000000..7be773667 --- /dev/null +++ b/core/libs/prx/libSceAgcDriver/tests/execution/FillCoverKeys.cpp @@ -0,0 +1,234 @@ +#include "prx/libSceAgcDriver/Execution/include/GuestMemory.hpp" +#include "prx/libSceAgcDriver/Execution/include/VulkanDevice.hpp" +#include "prx/libSceAgcDriver/Graphics/include/DccMetadata.hpp" +#include "prx/libSceAgcDriver/Graphics/include/Texture.hpp" +#include "prx/libc/include/GuestAllocations.hpp" +#include "Recompiler.hpp" +#include "VulkanTestDevice.hpp" +#ifdef _WIN32 +#ifndef NOMINMAX +#define NOMINMAX +#endif +#include +#endif +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace { + +using AgcDriver::Graphics::DccKeyCount; +using AgcDriver::Graphics::DccKeys; +using AgcDriver::Graphics::Require; +using AgcDriver::Graphics::StorageTexture; +using Cover = StorageTexture::FillCover; +using ShaderRecompiler::ShaderStage; + +constexpr std::uint32_t Threads = 32; +constexpr std::uint32_t Side = 128; +constexpr std::uint32_t Format8888Unorm = 56; +constexpr std::uint32_t TileRenderTarget64KB = 0x1b; +constexpr std::uint32_t Type2D = 9; +constexpr std::size_t SurfaceBytes = 65536; +constexpr std::size_t KeyBytes = SurfaceBytes / 256; +// The pipe-aligned key extent of the 128x128 surface (one 4 KiB metadata block): longer than the +// one-key-per-256-bytes count, so a fill between the two is told apart from a key-sized one. +constexpr std::size_t KeyExtent = 4096; +// Each scenario's images sit in their own region, so no fill of one overlaps the other's images. +constexpr std::size_t Region = 0x40000; +constexpr std::size_t BlockBytes = 2 * Region; + +alignas(256) constexpr std::array WriteCode{ + 0x7e020280, 0x7e0402f2, 0x7e0602f2, 0x7e0802f2, 0x7e0a02f2, 0xf0201f08, 0x00010200, 0xbf810000, +}; + +std::uint64_t AddressOf(const void* data) { + return static_cast(reinterpret_cast(data)); +} + +// Guest memory for the surfaces, registered as allocated so the driver reads and writes it. +class GuestBlock { +public: + GuestBlock() { +#ifdef _WIN32 + block = static_cast(VirtualAlloc(nullptr, BlockBytes, MEM_RESERVE | MEM_COMMIT, PAGE_READWRITE)); +#else + block = static_cast(std::aligned_alloc(65536, BlockBytes)); +#endif + Require(block != nullptr, "fill cover test: cannot allocate the guest block"); + std::memset(block, 0, BlockBytes); + GuestAllocations::Mutation().Add(block, BlockBytes, true, true, true); + } + + ~GuestBlock() { + GuestAllocations::Mutation().Remove(block); +#ifdef _WIN32 + VirtualFree(block, 0, MEM_RELEASE); +#else + std::free(block); +#endif + } + + GuestBlock(const GuestBlock&) = delete; + GuestBlock& operator=(const GuestBlock&) = delete; + + std::uint8_t* Data() { return block; } + +private: + std::uint8_t* block = nullptr; +}; + +int failures = 0; + +void Expect(bool condition, const std::string& what) { + if (condition) return; + std::fprintf(stderr, "FAIL: %s\n", what.c_str()); + ++failures; +} + +const char* CoverName(Cover cover) { + switch (cover) { + case Cover::None: return "none"; + case Cover::Exact: return "exact"; + case Cover::Inside: return "inside"; + case Cover::Around: return "around"; + case Cover::Straddle: return "straddle"; + case Cover::Several: return "several"; + case Cover::Keys: return "keys"; + case Cover::Layer: return "layer"; + } + return "?"; +} + +void ExpectCover(std::uint64_t address, std::size_t bytes, Cover expected, const char* what) { + const auto coverage = StorageTexture::ClassifyFill(address, bytes); + if (coverage.cover == expected) return; + char text[256]; + std::snprintf(text, sizeof(text), "%s: fill 0x%llx+0x%zx is %s, expected %s", what, static_cast(address), bytes, CoverName(coverage.cover), CoverName(expected)); + Expect(false, text); +} + +// The 128x128 RGBA8 surface at `address` with its DCC keys at `keys` (none when 0), with the words +// DccKeysFirstWrite.cpp lays out; the keys are pipe-aligned so their extent is KeyExtent. +std::array SurfaceWords(std::uint64_t address, std::uint64_t keys) { + const auto meta = keys >> 8u; + const std::uint32_t metaBits = keys != 0 ? (1u << 19u) | (1u << 20u) | (1u << 21u) : 0u; + return { + static_cast(address >> 8u), + static_cast((address >> 40u) & 0xffu) | (Format8888Unorm << 20u) | (((Side - 1u) & 3u) << 30u), + ((Side - 1u) >> 2u) | ((Side - 1u) << 14u), + 0xfacu | (TileRenderTarget64KB << 20u) | (Type2D << 28u), + 0u, + 0u, + metaBits | (static_cast(meta & 0xffu) << 24u), + static_cast(meta >> 8u), + }; +} + +void Dispatch(AgcDriver::VulkanDevice& device, std::span code, const std::vector& userData) { + const std::array memory{{{reinterpret_cast(code.data()), std::as_bytes(code)}}}; + const ShaderRecompiler::ShaderComputeStageInfo compute{{Threads, 1, 1}, 0u, {false, false, false}, false, 1}; + ShaderRecompiler::RecompileRequest request{ + {ShaderStage::Compute, reinterpret_cast(code.data()), code, 0, {}}, + {32, 0, userData, compute, std::nullopt, std::nullopt, memory}, + device.Target(), + {0, 0, 0, 128} + }; + request.useCache = false; + const auto result = ShaderRecompiler::Recompile(request); + device.Dispatch(result, 1, 1, 1, {}, reinterpret_cast(code.data())); + device.WaitIdle(); +} + +// The storage image the title's first write creates for the surface at `address`: a dispatch that +// stores into it, flushed at once so that no later write-back of it covers memory another image is +// checked over. The GPU mutex is taken here, not held by the caller. +std::shared_ptr Create(AgcDriver::VulkanDevice& device, std::uint64_t address, std::uint64_t keys) { + std::vector userData(16, 0u); + const auto words = SurfaceWords(address, keys); + std::copy(words.begin(), words.end(), userData.begin() + 4); + std::lock_guard gpu(AgcDriver::GuestMemory::GpuMutex()); + Dispatch(device, WriteCode, userData); + StorageTexture::FlushPending(address, SurfaceBytes, nullptr, "fill cover test"); + device.WaitIdle(); + const auto image = StorageTexture::FindLive(address, SurfaceBytes); + Require(image != nullptr && image->Descriptor().dccAddress == keys, "fill cover test: the dispatch did not create the image under its keys"); + return image; +} + +// Scenario one: image B (no DCC) and the image C beside it cover [0, 0x20000) of the region; image A +// (texels at 0x20000) has its keys at 0x8000, inside B, so a fill there meets B and only B. +// Scenario two: image D (texels at the region's start) has its keys at 0x10000, outside every other +// image, so a fill there meets nothing. +void Run(AgcDriver::VulkanDevice& device, std::uint8_t* block) { + const auto base = AddressOf(block); + const auto aKeys = base + 0x8000; + const auto dKeys = base + Region + 0x10000; + std::vector> images; + + std::memset(block, 0x55, 0x30000); + images.push_back(Create(device, base, 0)); + images.push_back(Create(device, base + 0x10000, 0)); + // The keys go in after B's write-back, which would store B's texels over them. + std::memset(block + 0x8000, 0xff, KeyExtent); + images.push_back(Create(device, base + 0x20000, aKeys)); + const auto& keyed = *images.back(); + const auto extent = DccKeyCount(keyed.Descriptor(), keyed.GuestBytes()); + Require(extent == KeyExtent, "fill cover test: the keyed image's key extent is not the pipe-aligned one"); + + std::memset(block + Region, 0x55, SurfaceBytes); + std::memset(block + Region + 0x10000, 0xff, KeyExtent); + images.push_back(Create(device, base + Region, dKeys)); + const auto alone = images.back(); + + std::lock_guard gpu(AgcDriver::GuestMemory::GpuMutex()); + // (a) a key-sized fill (and one up to the whole key extent) at the keys of an image whose keys + // lie in another live image: the key fill the bloom target's keys get over dead aliased images. + ExpectCover(aKeys, KeyBytes, Cover::Keys, "a key-sized fill over the keys of an image inside another live image"); + ExpectCover(aKeys, extent, Cover::Keys, "a fill of the whole key extent over the keys of an image inside another live image"); + // (c) past the key extent, the same fill also writes the memory after the keys, which the + // overlapping image may own: not a key fill (inferred, see docs/dev/TechnicalDebt.md). + ExpectCover(aKeys, extent + 1, Cover::Inside, "a fill one byte past the key extent, over another live image"); + // (b) the same lengths not at a key address stay what they were. + ExpectCover(aKeys + 0x100, KeyBytes, Cover::Inside, "a key-sized fill not at a dccAddress, over another live image"); + ExpectCover(aKeys + 0x100, 0x10000, Cover::Several, "a fill not at a dccAddress over two live images"); + // The consumer applies the same match: a fill past the key extent notes no keys, a key-sized one + // notes them on its image. + Expect(StorageTexture::NoteKeysFill(aKeys, extent + 1, 0x00) == 0 && keyed.FilledKeys() == DccKeys::Uncompressed, "NoteKeysFill noted keys for a fill past the key extent over another live image"); + Expect(StorageTexture::NoteKeysFill(aKeys, KeyBytes, 0x00) == 1 && keyed.FilledKeys() == DccKeys::Clear0000, "NoteKeysFill did not note a key-sized fill over an image inside another live image"); + // (d) with nothing overlapping, the length rule alone decides, as before the bound. + ExpectCover(dKeys, KeyBytes, Cover::Keys, "a key-sized fill at the keys of an image nothing else overlaps"); + ExpectCover(dKeys, 2 * extent, Cover::Keys, "a fill past the key extent at the keys of an image nothing else overlaps"); + Expect(StorageTexture::NoteKeysFill(dKeys, 2 * extent, 0x00) == 1 && alone->FilledKeys() == DccKeys::Clear0000, "NoteKeysFill refused a fill past the key extent with nothing overlapping"); +} + +} + +int main() { + try { + // The block outlives the device: the device's images are created over it. + GuestBlock block; + const auto device = OpenVulkanTestDevice(); + if (!device) return VulkanTestSkipped; + Run(*device, block.Data()); + if (failures != 0) { + std::fprintf(stderr, "%d fill cover checks failed\n", failures); + return 1; + } + std::puts("fill cover keys tests passed"); + return 0; + } catch (const std::exception& error) { + std::cerr << error.what() << '\n'; + return 1; + } +} diff --git a/core/libs/prx/libSceAgcDriver/tests/execution/FlatCacheBits.cpp b/core/libs/prx/libSceAgcDriver/tests/execution/FlatCacheBits.cpp index 742faa65c..dede33116 100644 --- a/core/libs/prx/libSceAgcDriver/tests/execution/FlatCacheBits.cpp +++ b/core/libs/prx/libSceAgcDriver/tests/execution/FlatCacheBits.cpp @@ -86,7 +86,7 @@ public: block = static_cast(std::aligned_alloc(BlockBytes, BlockBytes)); #endif Require(block != nullptr, "flat cache bits: cannot allocate the guest block"); - GuestAllocations::Mutation().Add(block, BlockBytes, true, true); + GuestAllocations::Mutation().Add(block, BlockBytes, true, true, true); } ~GuestBlock() { diff --git a/core/libs/prx/libSceAgcDriver/tests/execution/FlatD16Loads.cpp b/core/libs/prx/libSceAgcDriver/tests/execution/FlatD16Loads.cpp index abc38e965..7f57139ab 100644 --- a/core/libs/prx/libSceAgcDriver/tests/execution/FlatD16Loads.cpp +++ b/core/libs/prx/libSceAgcDriver/tests/execution/FlatD16Loads.cpp @@ -482,7 +482,7 @@ public: block = static_cast(std::aligned_alloc(BlockBytes, BlockBytes)); #endif Require(block != nullptr, "flat d16 loads: cannot allocate the guest block"); - GuestAllocations::Mutation().Add(block, BlockBytes, true, false); + GuestAllocations::Mutation().Add(block, BlockBytes, true, false, true); } ~GuestBlock() { diff --git a/core/libs/prx/libSceAgcDriver/tests/execution/FlatLoads.cpp b/core/libs/prx/libSceAgcDriver/tests/execution/FlatLoads.cpp index e96afcd62..a6ed28747 100644 --- a/core/libs/prx/libSceAgcDriver/tests/execution/FlatLoads.cpp +++ b/core/libs/prx/libSceAgcDriver/tests/execution/FlatLoads.cpp @@ -94,7 +94,7 @@ public: block = static_cast(std::aligned_alloc(BlockBytes, BlockBytes)); #endif Require(block != nullptr, "flat loads: cannot allocate the guest block"); - GuestAllocations::Mutation().Add(block, BlockBytes, true, false); + GuestAllocations::Mutation().Add(block, BlockBytes, true, false, true); } ~GuestBlock() { diff --git a/core/libs/prx/libSceAgcDriver/tests/execution/FlatLoadsX2X3.cpp b/core/libs/prx/libSceAgcDriver/tests/execution/FlatLoadsX2X3.cpp index c85f891fd..ab29a898a 100644 --- a/core/libs/prx/libSceAgcDriver/tests/execution/FlatLoadsX2X3.cpp +++ b/core/libs/prx/libSceAgcDriver/tests/execution/FlatLoadsX2X3.cpp @@ -93,7 +93,7 @@ public: block = static_cast(std::aligned_alloc(BlockBytes, BlockBytes)); #endif Require(block != nullptr, "flat loads x2 x3: cannot allocate the guest block"); - GuestAllocations::Mutation().Add(block, BlockBytes, true, false); + GuestAllocations::Mutation().Add(block, BlockBytes, true, false, true); } ~GuestBlock() { diff --git a/core/libs/prx/libSceAgcDriver/tests/execution/FlatLoadsX4.cpp b/core/libs/prx/libSceAgcDriver/tests/execution/FlatLoadsX4.cpp index 330114afb..5c65c5973 100644 --- a/core/libs/prx/libSceAgcDriver/tests/execution/FlatLoadsX4.cpp +++ b/core/libs/prx/libSceAgcDriver/tests/execution/FlatLoadsX4.cpp @@ -93,7 +93,7 @@ public: block = static_cast(std::aligned_alloc(BlockBytes, BlockBytes)); #endif Require(block != nullptr, "flat loads x4: cannot allocate the guest block"); - GuestAllocations::Mutation().Add(block, BlockBytes, true, false); + GuestAllocations::Mutation().Add(block, BlockBytes, true, false, true); } ~GuestBlock() { diff --git a/core/libs/prx/libSceAgcDriver/tests/execution/FlatStore.cpp b/core/libs/prx/libSceAgcDriver/tests/execution/FlatStore.cpp index 25bcf9396..259929ce0 100644 --- a/core/libs/prx/libSceAgcDriver/tests/execution/FlatStore.cpp +++ b/core/libs/prx/libSceAgcDriver/tests/execution/FlatStore.cpp @@ -47,7 +47,7 @@ public: #endif Require(block != nullptr, "flat store: cannot allocate the guest block"); Clear(); - GuestAllocations::Mutation().Add(block, BlockBytes, true, writable); + GuestAllocations::Mutation().Add(block, BlockBytes, true, writable, true); } ~GuestBlock() { diff --git a/core/libs/prx/libSceAgcDriver/tests/execution/Float16FmaRounding.cpp b/core/libs/prx/libSceAgcDriver/tests/execution/Float16FmaRounding.cpp new file mode 100644 index 000000000..424059a6c --- /dev/null +++ b/core/libs/prx/libSceAgcDriver/tests/execution/Float16FmaRounding.cpp @@ -0,0 +1,232 @@ +#include "prx/libSceAgcDriver/Execution/include/VulkanDevice.hpp" +#include "prx/libSceAgcDriver/Graphics/include/Draw.hpp" +#include "Recompiler.hpp" +#include "VulkanTestDevice.hpp" +#include +#include +#include +#include +#include +#include +#include + +namespace { + +using AgcDriver::Graphics::Require; +using ShaderRecompiler::ShaderStage; + +constexpr std::uint32_t Threads = 64; +constexpr std::uint32_t Inputs = 4; +constexpr std::uint32_t Results = 8; +constexpr std::uint32_t Checks = 7; +alignas(256) std::array Input{}; +alignas(256) std::array Output{}; + +alignas(256) constexpr std::array Code{ + 0x34020084, 0x34060085, 0xe0381000, 0x80000401, 0xbf8c3f70, 0xd74b000a, 0x041a0b04, 0xd74b380b, + 0x041a0b04, 0x7e180306, 0x6c180b04, 0xcc0e400d, 0x1c1a0b04, 0x7e1c0306, 0x781c0b04, 0x701e0f04, + 0x00000001, 0x6e200f04, 0x00003d55, 0x7e220280, 0xe0781000, 0x80010a03, 0xe0781010, 0x80010e03, + 0xbf810000, +}; + +constexpr std::uint32_t Rows[Threads][Inputs] = { + {0x800062e9u, 0x3c00d159u, 0x800023ecu, 0x0000f9a3u}, + {0x0000d4e0u, 0xbc005ae0u, 0x0000881fu, 0x00001e01u}, + {0x4200c1b4u, 0x4400f580u, 0xca008544u, 0x00003700u}, + {0x7c005af7u, 0x4000bde3u, 0x3c00772cu, 0x000027e9u}, + {0x3c0053e4u, 0x3c005900u, 0xfc000955u, 0x00002500u}, + {0x5bff3464u, 0x5bff5542u, 0x3c0012ffu, 0x00006d19u}, + {0x3c00c6d2u, 0x3c01064eu, 0x800129d8u, 0x00003200u}, + {0x1400e568u, 0x14004b80u, 0x00000891u, 0x000069d9u}, + {0xba18f442u, 0x70c0b900u, 0x88810546u, 0x00000700u}, + {0xbba84092u, 0x6a804788u, 0x814512ffu, 0x00008ba1u}, + {0xbc2cd70cu, 0x6d3ad700u, 0xa7f90ee0u, 0x00002100u}, + {0xbf00562cu, 0x72cc3774u, 0x8497ee8cu, 0x0000c716u}, + {0x56b4e294u, 0xdf004f00u, 0x15fb0683u, 0x00001700u}, + {0x1d0dd0e0u, 0x0a56cea0u, 0x1828837fu, 0x00001a01u}, + {0x5a40f006u, 0xcad01f55u, 0x80e6ea0eu, 0x00000900u}, + {0xcb3cc300u, 0x9615e4bcu, 0xc4a80843u, 0x0000d0fbu}, + {0x9e945d6eu, 0x95793cf9u, 0x998179b2u, 0x00001a00u}, + {0x603451cdu, 0xcf80d341u, 0x022c2276u, 0x00009041u}, + {0x06fbc6d0u, 0xf697ed40u, 0x5ea10c65u, 0x000030c0u}, + {0xd416668bu, 0x0a5d0272u, 0xc2d759b3u, 0x000032c9u}, + {0x6836db60u, 0x4900c9e0u, 0x806a02f8u, 0x00001ca0u}, + {0x02637ad4u, 0xe2b43100u, 0xd7ad0608u, 0x00002007u}, + {0xb4544d00u, 0x70806a2eu, 0x84828f49u, 0x00002a74u}, + {0x560003e6u, 0xd8453e28u, 0x0245af54u, 0x00005100u}, + {0x9b9bd580u, 0x08356004u, 0x14eb1222u, 0x000021e8u}, + {0x9aec64f8u, 0x67e64440u, 0x01018892u, 0x0000ad81u}, + {0x4366c00eu, 0x6e00eddcu, 0x0c16abffu, 0x00003900u}, + {0x002142a6u, 0xc7c266dbu, 0x28409b9du, 0x00000f11u}, + {0x45e74af4u, 0x015b6900u, 0xb5681505u, 0x00002d00u}, + {0x18eb7a00u, 0x6bc32ce7u, 0x804202f8u, 0x00008001u}, + {0xe990f8f2u, 0xc1201b57u, 0x06f71bc1u, 0x00008280u}, + {0xd9001c4du, 0xddeaf293u, 0x165316e1u, 0x00000e2bu}, + {0x532aee50u, 0x4e7c41c0u, 0xa97e11aau, 0x000009c0u}, + {0x066e7100u, 0xf4fa45aeu, 0xeee58a84u, 0x00003274u}, + {0x4dabed80u, 0x143c493cu, 0xcc6d8b1au, 0x000009e8u}, + {0xbd10d940u, 0x6d205d18u, 0x87670b12u, 0x00002001u}, + {0x68704f00u, 0x4e60e80au, 0x11f601a7u, 0x0000289cu}, + {0x5c805cd0u, 0xd9d45820u, 0x977a112bu, 0x00008357u}, + {0xf3003d96u, 0x34946e00u, 0x07660459u, 0x00003a00u}, + {0x363c73c0u, 0xa5223f70u, 0xb7e30d44u, 0x0000083au}, + {0x73de9e78u, 0xbe00262fu, 0x8872ac8eu, 0x00005a80u}, + {0xe6d7ccfeu, 0xb8f8e900u, 0xa88286b3u, 0x00001561u}, + {0x2824d550u, 0xf1805ac0u, 0x00860ec2u, 0x000022c0u}, + {0x36442fcau, 0x770019c7u, 0x8c9f254au, 0x00005fd8u}, + {0x6e192033u, 0xa3ae06abu, 0x84228eadu, 0x00005a00u}, + {0xad87d9b0u, 0x51cad8e0u, 0xf3f304cbu, 0x0000f6e3u}, + {0x39da80abu, 0xbad699fdu, 0xdc3c807du, 0x000012a2u}, + {0x199a8735u, 0x724992a9u, 0x9b4f82d6u, 0x0000a863u}, + {0xe56f0101u, 0x22719dfau, 0xe53e820au, 0x00000e86u}, + {0x66400f97u, 0x4c280a53u, 0x14008394u, 0x00004c5cu}, + {0x54409989u, 0xce7880b9u, 0x0151825eu, 0x0000066eu}, + {0x54b00f7cu, 0x506091e1u, 0x00a08146u, 0x00008201u}, + {0x50d689e7u, 0x4237096cu, 0x0dfa831eu, 0x00006495u}, + {0x04288b6fu, 0x0fb3084eu, 0x01e9037fu, 0x0000a90cu}, + {0xa1eb1a63u, 0x853d01e1u, 0x015a03b9u, 0x00003ea2u}, + {0x8af80d78u, 0x98e105dau, 0x02ef8286u, 0x0000294bu}, + {0x216c08b8u, 0x86d30ac8u, 0x8336022fu, 0x00006f80u}, + {0x87f41c46u, 0x0c0688eau, 0x82e782efu, 0x0000c6b4u}, + {0x0a215be0u, 0x183e5c10u, 0x83618001u, 0x00001c60u}, + {0x03dcdbe0u, 0x96385c10u, 0x02650001u, 0x000092e8u}, + {0x13496ff2u, 0x92dd5328u, 0x0270cb02u, 0x00008a00u}, + {0x8b276d1au, 0x9f695348u, 0x8397f1e6u, 0x000048c4u}, + {0x9560ee89u, 0x817d02f3u, 0x0331d633u, 0x0000de07u}, + {0x0fb3d541u, 0x0a3c089du, 0x822a503fu, 0x00003c3du} +}; +constexpr std::uint32_t Expected[Threads][Checks] = { + {0xf89fu, 0x8000u, 0xf89fu, 0x8000f89fu, 0x8000f89fu, 0xfc00u, 0xf97eu}, + {0xf431u, 0x0000u, 0xf431u, 0x0000f431u, 0x0000f431u, 0xb751u, 0xd67fu}, + {0x7bd7u, 0x0000u, 0x7bd7u, 0x00007bd7u, 0x00007bd7u, 0xbcfdu, 0xc2bau}, + {0x7717u, 0x7c00u, 0x7717u, 0x7c007717u, 0x7c007717u, 0x46e3u, 0x5ca5u}, + {0x70efu, 0xfc00u, 0x70efu, 0xfc0070efu, 0xfc0070efu, 0x3cefu, 0x5543u}, + {0x4dc5u, 0x7bfeu, 0x4dc5u, 0x7bfe4dc5u, 0x7bfe4dc5u, 0x6598u, 0x6d19u}, + {0x29c3u, 0x3c01u, 0x29c3u, 0x3c0129c3u, 0x3c0129c3u, 0xbd1du, 0xc874u}, + {0xf511u, 0x0010u, 0xf511u, 0x0010f511u, 0x0010f511u, 0xfc00u, 0x647du}, + {0x7153u, 0xef3du, 0x7153u, 0xef3d7153u, 0xef3d7153u, 0xbf73u, 0xf5adu}, + {0x4c4du, 0xea39u, 0x4c4du, 0xea394c4du, 0xea394c4du, 0x905cu, 0x4217u}, + {0x722bu, 0xed73u, 0x722bu, 0xed73722bu, 0xed73722bu, 0xbc67u, 0xd8b2u}, + {0xee81u, 0xf5f3u, 0xee81u, 0xf5f3ee81u, 0xf5f3ee81u, 0xe177u, 0x57c9u}, + {0xf5c1u, 0xf9ddu, 0xf5c1u, 0xf9ddf5c1u, 0xf9ddf5c1u, 0xbdc1u, 0xe462u}, + {0x6409u, 0x1829u, 0x6409u, 0x18296409u, 0x18296409u, 0xaf51u, 0xd27fu}, + {0xea2bu, 0xe953u, 0xea2bu, 0xe953ea2bu, 0xe953ea2bu, 0xbd07u, 0xf15du}, + {0x6c25u, 0xc4a3u, 0x6c25u, 0xc4a36c25u, 0xc4a36c25u, 0x585cu, 0xd190u}, + {0x79bfu, 0x997du, 0x79bfu, 0x997d79bfu, 0x997d79bfu, 0x3c13u, 0x5f3du}, + {0xe943u, 0xf3e1u, 0xe943u, 0xf3e1e943u, 0xf3e1e943u, 0xa62bu, 0x53bbu}, + {0x7879u, 0x5e95u, 0x7879u, 0x5e957879u, 0x5e957879u, 0xbc0bu, 0xc877u}, + {0x59b3u, 0xc2ddu, 0x59b3u, 0xc2dd59b3u, 0xc2dd59b3u, 0x5d8du, 0x685du}, + {0x696bu, 0x7543u, 0x696bu, 0x7543696bu, 0x7543696bu, 0xbc43u, 0xdceau}, + {0x7045u, 0xd7adu, 0x7045u, 0xd7ad7045u, 0xd7ad7045u, 0x5ee0u, 0x7c00u}, + {0x7bb9u, 0xe8dfu, 0x7bb9u, 0xe8df7bb9u, 0xe8df7bb9u, 0x3c09u, 0x4eadu}, + {0xaf53u, 0xf267u, 0xaf53u, 0xf267af53u, 0xf267af53u, 0x18e0u, 0x5100u}, + {0xf985u, 0x14ebu, 0xf985u, 0x14ebf985u, 0x14ebf985u, 0xbc0fu, 0xd755u}, + {0x6d47u, 0xc6d5u, 0x6d47u, 0xc6d56d47u, 0xc6d56d47u, 0xd6d6u, 0x669fu}, + {0x71f1u, 0x758du, 0x71f1u, 0x758d71f1u, 0x758d71f1u, 0xbd11u, 0xc028u}, + {0x6db3u, 0x283fu, 0x6db3u, 0x283f6db3u, 0x283f6db3u, 0x15dfu, 0x446fu}, + {0x7859u, 0xb567u, 0x7859u, 0xb5677859u, 0xb5677859u, 0x3c59u, 0x4ca7u}, + {0x6b5bu, 0x48c5u, 0x6b5bu, 0x48c56b5bu, 0x48c56b5bu, 0x9a00u, 0x7bffu}, + {0xd889u, 0x6f21u, 0xd889u, 0x6f21d889u, 0x6f21d889u, 0x3e2fu, 0xfa98u}, + {0xd311u, 0x7b65u, 0xd311u, 0x7b65d311u, 0x7b65d311u, 0x001cu, 0x1e1eu}, + {0xf489u, 0x65cfu, 0xf489u, 0x65cff489u, 0x65cff489u, 0xbc89u, 0xf035u}, + {0x7b19u, 0xeee5u, 0x7b19u, 0xeee57b19u, 0xeee57b19u, 0x6809u, 0x72aau}, + {0xfb33u, 0xcc6bu, 0xfb33u, 0xcc6bfb33u, 0xcc6bfb33u, 0xbc0fu, 0xef55u}, + {0xfaafu, 0xee7du, 0xfaafu, 0xee7dfaafu, 0xee7dfaafu, 0xbd41u, 0xdaffu}, + {0xfb11u, 0x7b13u, 0xfb11u, 0x7b13fb11u, 0x7b13fb11u, 0x3c09u, 0x50acu}, + {0x78f7u, 0xfa8fu, 0x78f7u, 0xfa8f78f7u, 0xfa8f78f7u, 0xa405u, 0x5e6au}, + {0x7031u, 0xec01u, 0x7031u, 0xec017031u, 0xec017031u, 0x3c31u, 0x4139u}, + {0x7735u, 0xb801u, 0x7735u, 0xb8017735u, 0xb8017735u, 0x4018u, 0x752au}, + {0xac91u, 0xf5e7u, 0xac91u, 0xf5e7ac91u, 0xf5e7ac91u, 0xbd41u, 0x5a80u}, + {0x7a3du, 0x643fu, 0x7a3du, 0x643f7a3du, 0x643f7a3du, 0xa6b7u, 0xcea7u}, + {0xf47bu, 0xddb1u, 0xf47bu, 0xddb1f47bu, 0xddb1f47bu, 0xbc7bu, 0xd715u}, + {0x2561u, 0x717bu, 0x2561u, 0x717b2561u, 0x717b2561u, 0x53a3u, 0x5fd9u}, + {0x8ea9u, 0xd5dbu, 0x8ea9u, 0xd5db8ea9u, 0xd5db8ea9u, 0x3e4du, 0x5a00u}, + {0x76efu, 0xf3f3u, 0x76efu, 0xf3f376efu, 0xf3f376efu, 0x7c00u, 0xf6f2u}, + {0x807du, 0xdc3fu, 0x807du, 0xdc3f807du, 0xdc3f807du, 0x0001u, 0x1286u}, + {0x82d5u, 0x5067u, 0x82d5u, 0x506782d5u, 0x506782d5u, 0x0040u, 0xa868u}, + {0x820bu, 0xe54fu, 0x820bu, 0xe54f820bu, 0xe54f820bu, 0x0001u, 0x0edcu}, + {0x8393u, 0x767fu, 0x8393u, 0x767f8393u, 0x767f8393u, 0x2023u, 0x4c5cu}, + {0x825du, 0xe6dfu, 0x825du, 0xe6df825du, 0xe6df825du, 0x8003u, 0x9b2du}, + {0x814bu, 0x6921u, 0x814bu, 0x6921814bu, 0x6921814bu, 0x0001u, 0x10bdu}, + {0x831fu, 0x5783u, 0x831fu, 0x5783831fu, 0x5783831fu, 0xb2c3u, 0x6495u}, + {0x037fu, 0x01e9u, 0x037fu, 0x01e9037fu, 0x01e9037fu, 0x0097u, 0xa916u}, + {0x03bbu, 0x0169u, 0x03bbu, 0x016903bbu, 0x016903bbu, 0x1d4cu, 0x3ea6u}, + {0x8285u, 0x02f7u, 0x8285u, 0x02f78285u, 0x02f78285u, 0x00e9u, 0x295au}, + {0x022fu, 0x8349u, 0x022fu, 0x8349022fu, 0x8349022fu, 0x3c6du, 0x6f80u}, + {0x82f9u, 0x82e7u, 0x82f9u, 0x82e782f9u, 0x82e782f9u, 0xa729u, 0xc6b3u}, + {0x7bffu, 0x835bu, 0x7bffu, 0x835b7bffu, 0x835b7bffu, 0x3c4fu, 0x5d40u}, + {0xfbffu, 0x0263u, 0xfbffu, 0x0263fbffu, 0x0263fbffu, 0x32ccu, 0xdd40u}, + {0x7c00u, 0x0263u, 0x7c00u, 0x02637c00u, 0x02637c00u, 0xbdf5u, 0x714cu}, + {0x7c00u, 0x837du, 0x7c00u, 0x837d7c00u, 0x837d7c00u, 0x7a14u, 0x6ecfu}, + {0xd638u, 0x0331u, 0xd638u, 0x0331d638u, 0x0331d638u, 0x7c00u, 0xf08bu}, + {0x503fu, 0x8229u, 0x503fu, 0x8229503fu, 0x8229503fu, 0xd591u, 0xd6f0u} +}; +constexpr std::uint32_t Masks[Checks] = {0xffffu, 0xffffu, 0xffffu, 0xffffffffu, 0xffffffffu, 0xffffu, 0xffffu}; +constexpr const char* Names[Checks] = { + "v_fma_f16 v10, v4, v5, v6", + "v_fma_f16 v11, v4, v5, v6 op_sel:[1,1,1,0]", + "v_fmac_f16 v12, v4, v5", + "v_pk_fma_f16 v13, v4, v5, v6", + "v_pk_fmac_f16 v14, v4, v5", + "v_fmaak_f16 v15, v4, v7, 0x0001", + "v_fmamk_f16 v16, v4, 0x3d55, v7", +}; + +std::array BufferDescriptor(const void* data, std::uint32_t bytes) { + const auto address = reinterpret_cast(data); + return {static_cast(address), static_cast((address >> 32u) & 0xffffu), bytes, 0x31016facu}; +} + +std::string Hex(std::uint32_t value) { + char text[16]; + std::snprintf(text, sizeof(text), "0x%08x", value); + return text; +} + +void Run(AgcDriver::VulkanDevice& device) { + for (std::uint32_t tid = 0; tid < Threads; ++tid) std::copy(std::begin(Rows[tid]), std::end(Rows[tid]), &Input[tid * Inputs]); + Output.fill(0xdeadbeefu); + std::vector userData(8, 0u); + const auto input = BufferDescriptor(Input.data(), static_cast(Input.size() * 4u)); + const auto output = BufferDescriptor(Output.data(), static_cast(Output.size() * 4u)); + std::copy(input.begin(), input.end(), userData.begin()); + std::copy(output.begin(), output.end(), userData.begin() + 4); + const std::span code(Code); + const std::array memory{{{reinterpret_cast(code.data()), std::as_bytes(code)}}}; + const ShaderRecompiler::ShaderComputeStageInfo compute{{Threads, 1, 1}, 0u, {false, false, false}, false, 1}; + ShaderRecompiler::RecompileRequest request{ + {ShaderStage::Compute, reinterpret_cast(code.data()), code, 0, {}}, + {32, 0, userData, compute, std::nullopt, std::nullopt, memory}, + device.Target(), + {0, 0, 0, 128} + }; + request.useCache = false; + const auto result = ShaderRecompiler::Recompile(request); + device.Dispatch(result, 1, 1, 1, {}, reinterpret_cast(code.data())); + device.WaitIdle(); +} + +void Check() { + for (std::uint32_t tid = 0; tid < Threads; ++tid) { + for (std::uint32_t index = 0; index < Checks; ++index) { + const std::uint32_t actual = Output[tid * Results + index] & Masks[index]; + Require(actual == Expected[tid][index], std::string("f16 fma rounding: lane ") + std::to_string(tid) + " " + Names[index] + " is " + Hex(actual) + ", expected " + Hex(Expected[tid][index])); + } + } +} + +} + +int main() { + try { + const auto device = OpenVulkanTestDevice(); + if (!device) return VulkanTestSkipped; + Run(*device); + Check(); + std::puts("f16 fma rounding tests passed"); + return 0; + } catch (const std::exception& error) { + std::cerr << error.what() << '\n'; + return 1; + } +} diff --git a/core/libs/prx/libSceAgcDriver/tests/execution/FmaLegacy.cpp b/core/libs/prx/libSceAgcDriver/tests/execution/FmaLegacy.cpp index 53af35045..f4ae0b8dc 100644 --- a/core/libs/prx/libSceAgcDriver/tests/execution/FmaLegacy.cpp +++ b/core/libs/prx/libSceAgcDriver/tests/execution/FmaLegacy.cpp @@ -4,8 +4,6 @@ #include "VulkanTestDevice.hpp" #include #include -#include -#include #include #include #include @@ -32,29 +30,42 @@ alignas(256) constexpr std::array FmaLegacyCode{ struct Row { std::uint32_t a, b, c; + std::array expected; }; -constexpr std::array Rows{{ - {0x00000000u, 0x7f800000u, 0x3f800000u}, - {0x80000000u, 0x7f800000u, 0xc0200000u}, - {0x7f800000u, 0x00000000u, 0x3f400000u}, - {0xff800000u, 0x80000000u, 0x40400000u}, - {0x00000000u, 0x7fc00000u, 0x3fc00000u}, - {0x7fc00000u, 0x80000000u, 0xc0800000u}, - {0x00000000u, 0x40000000u, 0x7f800000u}, - {0x80000000u, 0xc0400000u, 0xff800000u}, - {0x00000000u, 0x3f800000u, 0x7fc00000u}, - {0x3fc00000u, 0x40100000u, 0xbf400000u}, - {0xc0c00000u, 0x3f000000u, 0x41200000u}, - {0x7f800000u, 0x40000000u, 0x3f800000u}, - {0x7f800000u, 0xbf800000u, 0x7f800000u}, - {0x7fc00000u, 0x3f800000u, 0x3f800000u}, - {0x40400000u, 0x40800000u, 0xc1400000u}, - {0x7f7fffffu, 0x40000000u, 0x3f800000u}, - {0x3ec00000u, 0xc1480000u, 0x42c88000u}, - {0xc4800000u, 0xbd800000u, 0xc2800000u}, - {0x40e00000u, 0x41100000u, 0x3f000000u}, - {0x80000000u, 0x80000000u, 0x40a00000u}, +constexpr std::array Rows{{ + {0x00000000u, 0x7f800000u, 0x3f800000u, {0x3f800000u, 0x3f800000u, 0xbf800000u, 0x3f800000u, 0x3f800000u}}, + {0x80000000u, 0x7f800000u, 0xc0200000u, {0xc0200000u, 0xc0200000u, 0x40200000u, 0xc0200000u, 0xc0200000u}}, + {0x7f800000u, 0x00000000u, 0x3f400000u, {0x3f400000u, 0x3f400000u, 0xbf400000u, 0x3f400000u, 0x3f400000u}}, + {0xff800000u, 0x80000000u, 0x40400000u, {0x40400000u, 0x40400000u, 0xc0400000u, 0x40400000u, 0x40400000u}}, + {0x00000000u, 0x7fc00000u, 0x3fc00000u, {0x3fc00000u, 0x3fc00000u, 0xbfc00000u, 0x3fc00000u, 0x3fc00000u}}, + {0x7fc00000u, 0x80000000u, 0xc0800000u, {0xc0800000u, 0xc0800000u, 0x40800000u, 0xc0800000u, 0xc0800000u}}, + {0x00000000u, 0x40000000u, 0x7f800000u, {0x7f800000u, 0x7f800000u, 0xff800000u, 0x7f800000u, 0x7f800000u}}, + {0x80000000u, 0xc0400000u, 0xff800000u, {0xff800000u, 0xff800000u, 0x7f800000u, 0xff800000u, 0xff800000u}}, + {0x00000000u, 0x3f800000u, 0x7fc00000u, {0x7fc00000u, 0x7fc00000u, 0xffc00000u, 0x7fc00000u, 0x7fc00000u}}, + {0x3fc00000u, 0x40100000u, 0xbf400000u, {0x40280000u, 0xc0840000u, 0x40840000u, 0xbf400000u, 0x40280000u}}, + {0xc0c00000u, 0x3f000000u, 0x41200000u, {0x40e00000u, 0x41500000u, 0xc0e00000u, 0x41200000u, 0x40e00000u}}, + {0x7f800000u, 0x40000000u, 0x3f800000u, {0x7f800000u, 0xff800000u, 0x7f800000u, 0x3f800000u, 0x7f800000u}}, + {0x7f800000u, 0xbf800000u, 0x7f800000u, {0xffc00000u, 0x7f800000u, 0xff800000u, 0x7f800000u, 0x7f800000u}}, + {0x7fc00000u, 0x3f800000u, 0x3f800000u, {0x7fc00000u, 0xffc00000u, 0x7fc00000u, 0x3f800000u, 0xffc00000u}}, + {0x40400000u, 0x40800000u, 0xc1400000u, {0x00000000u, 0xc1c00000u, 0x41c00000u, 0xc1400000u, 0x00000000u}}, + {0x7f7fffffu, 0x40000000u, 0x3f800000u, {0x7f800000u, 0xff800000u, 0x7f800000u, 0x3f800000u, 0x7f800000u}}, + {0x3ec00000u, 0xc1480000u, 0x42c88000u, {0x42bf2000u, 0x42d1e000u, 0xc2d1e000u, 0x42c88000u, 0x42d1e000u}}, + {0xc4800000u, 0xbd800000u, 0xc2800000u, {0x00000000u, 0xc3000000u, 0x00000000u, 0xc2800000u, 0xc3000000u}}, + {0x40e00000u, 0x41100000u, 0x3f000000u, {0x427e0000u, 0xc27a0000u, 0x427a0000u, 0x3f000000u, 0x427e0000u}}, + {0x80000000u, 0x80000000u, 0x40a00000u, {0x40a00000u, 0x40a00000u, 0xc0a00000u, 0x40a00000u, 0x40a00000u}}, + {0x3f800800u, 0x3f800800u, 0xbf800000u, {0x3a000000u, 0xc0000800u, 0x40000800u, 0xbf800000u, 0x3a000000u}}, + {0x3f800001u, 0x3f7fffffu, 0xbf800000u, {0x00000000u, 0xc0000000u, 0x40000000u, 0xbf800000u, 0x00000000u}}, + {0x3faaaaabu, 0x40400000u, 0xc0800000u, {0x00000000u, 0xc1000000u, 0x41000000u, 0xc0800000u, 0x00000000u}}, + {0x7f7fffffu, 0x40000000u, 0xff7fffffu, {0x7f800000u, 0xff800000u, 0x7f800000u, 0xff7fffffu, 0x7f800000u}}, + {0x00800000u, 0x3f000000u, 0x00000000u, {0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u}}, + {0x00000001u, 0x4b000000u, 0x00000000u, {0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u}}, + {0x00000001u, 0x3f800000u, 0x80000000u, {0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u}}, + {0x80000001u, 0x3f800000u, 0x00000000u, {0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u}}, + {0x00800000u, 0x3f800000u, 0x80400000u, {0x00800000u, 0x80800000u, 0x00800000u, 0x00000000u, 0x00800000u}}, + {0x3f800000u, 0x00400000u, 0x00000000u, {0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u}}, + {0x7f812345u, 0x3f800000u, 0x3f800000u, {0x7f812345u, 0xff812345u, 0x7f812345u, 0x3f800000u, 0xff812345u}}, + {0x3f800000u, 0x3f800000u, 0x7f812345u, {0x7f812345u, 0x7f812345u, 0xff812345u, 0x7f812345u, 0x7f812345u}}, }}; std::array BufferDescriptor(const void* data, std::uint32_t count) { @@ -96,33 +107,15 @@ std::string Hex(std::uint32_t value) { return text; } -bool IsNan(std::uint32_t bits) { - return (bits & 0x7fffffffu) > 0x7f800000u; -} - -std::uint32_t FmaLegacy(float a, float b, float c) { - if (a == 0.0f || b == 0.0f) { - return std::bit_cast(0.0f + c); - } - return std::bit_cast(std::fma(a, b, c)); -} - void Check() { constexpr std::array names{ - "v_fma_legacy_f32", "v_fma_legacy_f32 -src0", "v_fma_legacy_f32 |src0| -src2", "v_fma_legacy_f32 src0=0", "v_fma_legacy_f32 -src0 -|src1|", + "v_mad_legacy_f32", "v_mad_legacy_f32 -src0", "v_mad_legacy_f32 |src0| -src2", "v_mad_legacy_f32 src0=0", "v_mad_legacy_f32 -src0 -|src1|", }; for (std::uint32_t tid = 0; tid < Rows.size(); ++tid) { const auto& row = Rows[tid]; - const float a = std::bit_cast(row.a); - const float b = std::bit_cast(row.b); - const float c = std::bit_cast(row.c); - const std::array expected{ - FmaLegacy(a, b, c), FmaLegacy(-a, b, c), FmaLegacy(std::fabs(a), b, -c), FmaLegacy(0.0f, b, c), FmaLegacy(-a, -std::fabs(b), c), - }; - for (std::uint32_t j = 0; j < expected.size(); ++j) { + for (std::uint32_t j = 0; j < row.expected.size(); ++j) { const auto actual = Output[tid * Results + j]; - const bool matches = actual == expected[j] || (IsNan(actual) && IsNan(expected[j])); - Require(matches, std::string("fma legacy: lane ") + std::to_string(tid) + " (" + Hex(row.a) + ", " + Hex(row.b) + ", " + Hex(row.c) + ") " + names[j] + " is " + Hex(actual) + ", expected " + Hex(expected[j])); + Require(actual == row.expected[j], std::string("mad legacy: lane ") + std::to_string(tid) + " (" + Hex(row.a) + ", " + Hex(row.b) + ", " + Hex(row.c) + ") " + names[j] + " is " + Hex(actual) + ", expected " + Hex(row.expected[j])); } } } @@ -135,7 +128,7 @@ int main() { if (!device) return VulkanTestSkipped; Run(*device); Check(); - std::puts("fma legacy tests passed"); + std::puts("mad legacy tests passed"); return 0; } catch (const std::exception& error) { std::cerr << error.what() << '\n'; diff --git a/core/libs/prx/libSceAgcDriver/tests/execution/Fp16Overflow.cpp b/core/libs/prx/libSceAgcDriver/tests/execution/Fp16Overflow.cpp new file mode 100644 index 000000000..880b0b65b --- /dev/null +++ b/core/libs/prx/libSceAgcDriver/tests/execution/Fp16Overflow.cpp @@ -0,0 +1,241 @@ +#include "prx/libSceAgcDriver/Execution/include/VulkanDevice.hpp" +#include "prx/libSceAgcDriver/Graphics/include/Draw.hpp" +#include "Recompiler.hpp" +#include "VulkanTestDevice.hpp" +#include +#include +#include +#include +#include +#include +#include +#include + +namespace { + +using AgcDriver::Graphics::Require; +using ShaderRecompiler::ShaderStage; + +constexpr std::uint32_t Threads = 32; +constexpr std::uint32_t Inputs = 4; +constexpr std::uint32_t Results = 16; +alignas(256) std::array Input{}; +alignas(256) std::array Output{}; + +alignas(256) constexpr std::array Code{ + 0x34020084u, 0x34060086u, 0xe0381000u, 0x80000401u, 0xbf8c3f70u, 0x7e140280u, 0x7e160280u, 0x7e180280u, + 0x7e1a0280u, 0x7e1c0280u, 0x7e1e0280u, 0x7e200280u, 0x7e220280u, 0x7e240280u, 0x7e260280u, 0x7e280280u, + 0x7e2a0280u, 0x7e2c0280u, 0x7e2e0280u, 0x7e300280u, 0x7e320280u, 0x64140b04u, 0x66160b04u, 0x6a180b04u, + 0xd74b000du, 0x041a0b04u, 0xd75f000eu, 0x041a0b04u, 0xcc10400fu, 0x00020b04u, 0xcc0f4010u, 0x00020b04u, + 0xcc214011u, 0x1c1a0b04u, 0x64240af9u, 0x04041504u, 0x7e26b104u, 0x7e28af04u, 0x7e2aa904u, 0x7e2cad04u, + 0x762e0f04u, 0x7e301507u, 0x7e32a107u, 0xe0781000u, 0x80010a03u, 0xe0781010u, 0x80010e03u, 0xe0781020u, + 0x80011203u, 0xe0781030u, 0x80011603u, 0xbf810000u, +}; + +constexpr std::uint32_t Rows[32][4] = { + {0x00007bffu, 0x00004c00u, 0x00000000u, 0x477ff000u}, + {0x00007bffu, 0x00004bffu, 0x00000000u, 0x477fefffu}, + {0x00007bffu, 0x00004c01u, 0x00000000u, 0x0000ffffu}, + {0x00007bffu, 0x00007bffu, 0x00000000u, 0x0000fff0u}, + {0x0000fbffu, 0x0000fbffu, 0x00000000u, 0x0000ffefu}, + {0x00007800u, 0x00004000u, 0x00000000u, 0x00000001u}, + {0x00007800u, 0x0000c000u, 0x00000000u, 0x00000010u}, + {0x00007c00u, 0x00003c00u, 0x00000000u, 0x7f800000u}, + {0x00007c00u, 0x00004000u, 0x00003c00u, 0xff800000u}, + {0x00007c00u, 0x00000000u, 0x00000000u, 0x7f7fffffu}, + {0x00007c00u, 0x0000fc00u, 0x00000000u, 0x47800000u}, + {0x00007e00u, 0x00003c00u, 0x00003c00u, 0x47800001u}, + {0x00007d00u, 0x00003c00u, 0x00003c00u, 0xc7800000u}, + {0x00007bffu, 0x00003c00u, 0x00004c00u, 0x00000040u}, + {0x00007bffu, 0x00004000u, 0x0000fbffu, 0x0000ffc0u}, + {0x00007bffu, 0x00004000u, 0x0000fc00u, 0x477fe000u}, + {0x00003c00u, 0x00000000u, 0x00003c00u, 0x00000010u}, + {0x00003c00u, 0x00003c00u, 0x00007c00u, 0x0000000fu}, + {0x00007bffu, 0x00000001u, 0x00003c00u, 0x477ff001u}, + {0x00000000u, 0x00000000u, 0x00000000u, 0x4b800000u}, + {0x00008000u, 0x00003c00u, 0x00000000u, 0x5f800000u}, + {0x00000001u, 0x00000001u, 0x00000000u, 0x00000028u}, + {0x00000100u, 0x00003c00u, 0x00000000u, 0x00000020u}, + {0x00000200u, 0x00003c00u, 0x00000000u, 0x0000001fu}, + {0x00004c00u, 0x00003c00u, 0x00003c00u, 0x0000000cu}, + {0x00004bffu, 0x00003c00u, 0x00003c00u, 0x0000000cu}, + {0x00005640u, 0x00005640u, 0x00005640u, 0x3f800000u}, + {0x0000fc00u, 0x00003c00u, 0x00007c00u, 0x00008000u}, + {0x00007800u, 0x00004400u, 0x00000000u, 0x7fa00000u}, + {0x0000fbffu, 0x0000cc00u, 0x00000000u, 0x00007fffu}, + {0x00007bffu, 0x0000cc00u, 0x00000000u, 0x00000002u}, + {0x00007c00u, 0x00007c00u, 0x00007c00u, 0xfc000000u} +}; +constexpr std::uint32_t ExpectedOverflow[32][16] = { + {0x00007bffu, 0x00007bfeu, 0x00007bffu, 0x00007bffu, 0x00000000u, 0x7bff7bffu, 0x7bff7bffu, 0x00007bffu, 0x7bff0000u, 0x00007bffu, 0x00004c00u, 0x00000100u, 0x00001c00u, 0x00000000u, 0x00007bffu, 0x00007b80u}, + {0x00007bffu, 0x00007bffu, 0x00007bffu, 0x00007bffu, 0x00000000u, 0x7bff7bffu, 0x7bff7bffu, 0x00007bffu, 0x7bff0000u, 0x00007bffu, 0x00004c00u, 0x00000100u, 0x00001c00u, 0x00000000u, 0x00007bffu, 0x00007b80u}, + {0x00007bffu, 0x00007bfeu, 0x00007bffu, 0x00007bffu, 0x00000000u, 0x7bff7bffu, 0x7bff7bffu, 0x00007bffu, 0x7bff0000u, 0x00007bffu, 0x00004c00u, 0x00000100u, 0x00001c00u, 0x000077ffu, 0x00000000u, 0x00007bffu}, + {0x00007bffu, 0x00000000u, 0x00007bffu, 0x00007bffu, 0x00000000u, 0x7bff7bffu, 0x7bff7bffu, 0x00007bffu, 0x7bff0000u, 0x00007bffu, 0x00004c00u, 0x00000100u, 0x00001c00u, 0x00003bffu, 0x00000000u, 0x00007bffu}, + {0x0000fbffu, 0x00000000u, 0x00007bffu, 0x00007bffu, 0x00008000u, 0x7bff7bffu, 0xfbfffbffu, 0x00007bffu, 0xfbff0000u, 0x00000000u, 0x0000fe00u, 0x00008100u, 0x0000fe00u, 0x0000b7ffu, 0x00000000u, 0x00007bffu}, + {0x00007800u, 0x00007800u, 0x00007bffu, 0x00007bffu, 0x00000000u, 0x7bff7bffu, 0x78007800u, 0x00007bffu, 0x78000000u, 0x00007bffu, 0x00004b80u, 0x00000200u, 0x00001da8u, 0x00007bffu, 0x00000000u, 0x00003c00u}, + {0x00007800u, 0x00007800u, 0x0000fbffu, 0x0000fbffu, 0x00008000u, 0xfbfffbffu, 0x78007800u, 0x0000fbffu, 0x78000000u, 0x00007bffu, 0x00004b80u, 0x00000200u, 0x00001da8u, 0x00007bffu, 0x00000000u, 0x00004c00u}, + {0x00007c00u, 0x00007c00u, 0x00007c00u, 0x00007c00u, 0x00000000u, 0x7c007c00u, 0x7c007c00u, 0x00007c00u, 0x7c000000u, 0x00007c00u, 0x00007c00u, 0x00000000u, 0x00000000u, 0x00007c00u, 0x00007c00u, 0x00000000u}, + {0x00007c00u, 0x00007c00u, 0x00007c00u, 0x00007c00u, 0x00007bffu, 0x7c007c00u, 0x7c007c00u, 0x00007c00u, 0x7c000000u, 0x00007c00u, 0x00007c00u, 0x00000000u, 0x00000000u, 0x00007c00u, 0x0000fc00u, 0x00000000u}, + {0x00007c00u, 0x00007c00u, 0x0000fe00u, 0x0000fe00u, 0x0000fe00u, 0xfe00fe00u, 0x7c007c00u, 0x0000fe00u, 0x7c000000u, 0x00007c00u, 0x00007c00u, 0x00000000u, 0x00000000u, 0x00007c00u, 0x00007bffu, 0x00007bffu}, + {0x0000fe00u, 0x00007c00u, 0x0000fc00u, 0x0000fc00u, 0x00008000u, 0xfc00fc00u, 0xfe00fe00u, 0x0000fc00u, 0xfe000000u, 0x00007c00u, 0x00007c00u, 0x00000000u, 0x00000000u, 0x00007c00u, 0x00007bffu, 0x00000000u}, + {0x00007e00u, 0x00007e00u, 0x00007e00u, 0x00007e00u, 0x00007bffu, 0x7e007e00u, 0x7e007e00u, 0x00007e00u, 0x7e000000u, 0x00007e00u, 0x00007e00u, 0x00007e00u, 0x00007e00u, 0x00007e00u, 0x00007bffu, 0x00003c00u}, + {0x00007d00u, 0x00007d00u, 0x00007d00u, 0x00007d00u, 0x00007bffu, 0x7d007d00u, 0x7d007d00u, 0x00007f00u, 0x7d000000u, 0x00007d00u, 0x00007d00u, 0x00007d00u, 0x00007d00u, 0x00007d00u, 0x0000fbffu, 0x00000000u}, + {0x00007bffu, 0x00007bffu, 0x00007bffu, 0x00007bffu, 0x00007bffu, 0x7bff7bffu, 0x7bff7bffu, 0x00007bffu, 0x7bff0000u, 0x00007bffu, 0x00004c00u, 0x00000100u, 0x00001c00u, 0x00007bffu, 0x00000000u, 0x00005400u}, + {0x00007bffu, 0x00007bffu, 0x00007bffu, 0x00007bffu, 0x0000fbffu, 0x7bff7bffu, 0x7bff7bffu, 0x00007bffu, 0x7bff0000u, 0x00007bffu, 0x00004c00u, 0x00000100u, 0x00001c00u, 0x00000000u, 0x00000000u, 0x00007bfeu}, + {0x00007bffu, 0x00007bffu, 0x00007bffu, 0x0000fc00u, 0x0000fc00u, 0x7bff7bffu, 0x7bff7bffu, 0x0000fc00u, 0x7bff0000u, 0x00007bffu, 0x00004c00u, 0x00000100u, 0x00001c00u, 0x00000000u, 0x00007bffu, 0x00007b00u}, + {0x00003c00u, 0x00003c00u, 0x00000000u, 0x00003c00u, 0x00007c00u, 0x00000000u, 0x3c003c00u, 0x00003c00u, 0x3c000000u, 0x00004000u, 0x00000000u, 0x00003c00u, 0x00003c00u, 0x00007bffu, 0x00000000u, 0x00004c00u}, + {0x00004000u, 0x00000000u, 0x00003c00u, 0x00007c00u, 0x00007c00u, 0x3c003c00u, 0x40004000u, 0x00007c00u, 0x40000000u, 0x00004000u, 0x00000000u, 0x00003c00u, 0x00003c00u, 0x00007800u, 0x00000000u, 0x00004b80u}, + {0x00007bffu, 0x00007bffu, 0x00001bffu, 0x00003c04u, 0x00007bffu, 0x1bff1bffu, 0x7bff7bffu, 0x00003c04u, 0x7bff0000u, 0x00007bffu, 0x00004c00u, 0x00000100u, 0x00001c00u, 0x00000000u, 0x00007bffu, 0x00007b80u}, + {0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x0000fe00u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00003c00u, 0x0000fbffu, 0x00007bffu, 0x00007bffu, 0x00000000u, 0x00007bffu, 0x00000000u}, + {0x00003c00u, 0x0000bc00u, 0x00008000u, 0x00000000u, 0x00000000u, 0x80008000u, 0x3c003c00u, 0x00000000u, 0x3c000000u, 0x00003c00u, 0x0000fbffu, 0x0000fbffu, 0x0000fbffu, 0x00008000u, 0x00007bffu, 0x00000000u}, + {0x00000002u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00020002u, 0x00000000u, 0x00020000u, 0x00003c00u, 0x0000ce00u, 0x00007bffu, 0x00006c00u, 0x00007bffu, 0x00000000u, 0x00005100u}, + {0x00003c00u, 0x0000bc00u, 0x00000100u, 0x00000100u, 0x00000000u, 0x01000100u, 0x3c003c00u, 0x00000100u, 0x3c000000u, 0x00003c00u, 0x0000cc00u, 0x00007bffu, 0x00005c00u, 0x00007bffu, 0x00000000u, 0x00005000u}, + {0x00003c00u, 0x0000bc00u, 0x00000200u, 0x00000200u, 0x00000000u, 0x02000200u, 0x3c003c00u, 0x00000200u, 0x3c000000u, 0x00003c00u, 0x0000cb80u, 0x00007800u, 0x000059a8u, 0x00007bffu, 0x00000000u, 0x00004fc0u}, + {0x00004c40u, 0x00004b80u, 0x00004c00u, 0x00004c40u, 0x00004c00u, 0x4c004c00u, 0x4c404c40u, 0x00004c40u, 0x4c400000u, 0x00007bffu, 0x00004400u, 0x00002c00u, 0x00003400u, 0x00007bffu, 0x00000000u, 0x00004a00u}, + {0x00004c40u, 0x00004b7fu, 0x00004bffu, 0x00004c40u, 0x00004bffu, 0x4bff4bffu, 0x4c404c40u, 0x00004c40u, 0x4c400000u, 0x00007bf5u, 0x00004400u, 0x00002c01u, 0x00003400u, 0x00007bffu, 0x00000000u, 0x00004a00u}, + {0x00005a40u, 0x00000000u, 0x000070e2u, 0x000070eeu, 0x00005640u, 0x70e270e2u, 0x5a405a40u, 0x000070eeu, 0x5a400000u, 0x00007bffu, 0x000046a5u, 0x0000211fu, 0x00002e66u, 0x00005640u, 0x00003c00u, 0x00000000u}, + {0x0000fc00u, 0x0000fc00u, 0x0000fc00u, 0x0000fe00u, 0x00007c00u, 0xfc00fc00u, 0xfc00fc00u, 0x0000fe00u, 0xfc000000u, 0x00000000u, 0x0000fe00u, 0x00008000u, 0x0000fe00u, 0x0000fc00u, 0x00000000u, 0x00007800u}, + {0x00007800u, 0x00007800u, 0x00007bffu, 0x00007bffu, 0x00000000u, 0x7bff7bffu, 0x78007800u, 0x00007bffu, 0x78000000u, 0x00007bffu, 0x00004b80u, 0x00000200u, 0x00001da8u, 0x00007800u, 0x00007f00u, 0x00000000u}, + {0x0000fbffu, 0x0000fbfeu, 0x00007bffu, 0x00007bffu, 0x00008000u, 0x7bff7bffu, 0xfbfffbffu, 0x00007bffu, 0xfbff0000u, 0x00000000u, 0x0000fe00u, 0x00008100u, 0x0000fe00u, 0x0000fbffu, 0x00000000u, 0x00007800u}, + {0x00007bfeu, 0x00007bffu, 0x0000fbffu, 0x0000fbffu, 0x00008000u, 0xfbfffbffu, 0x7bfe7bfeu, 0x0000fbffu, 0x7bfe0000u, 0x00007bffu, 0x00004c00u, 0x00000100u, 0x00001c00u, 0x00007bffu, 0x00000000u, 0x00004000u}, + {0x00007c00u, 0x0000fe00u, 0x00007c00u, 0x00007c00u, 0x0000fe00u, 0x7c007c00u, 0x7c007c00u, 0x00007c00u, 0x7c000000u, 0x00007c00u, 0x00007c00u, 0x00000000u, 0x00000000u, 0x00007c00u, 0x0000fbffu, 0x00000000u} +}; +constexpr std::uint32_t ExpectedInfinity[32][16] = { + {0x00007c00u, 0x00007bfeu, 0x00007c00u, 0x00007c00u, 0x00000000u, 0x7c007c00u, 0x7c007c00u, 0x00007c00u, 0x7c000000u, 0x00007c00u, 0x00004c00u, 0x00000100u, 0x00001c00u, 0x00000000u, 0x00007c00u, 0x00007b80u}, + {0x00007bffu, 0x00007bffu, 0x00007c00u, 0x00007c00u, 0x00000000u, 0x7c007c00u, 0x7bff7bffu, 0x00007c00u, 0x7bff0000u, 0x00007c00u, 0x00004c00u, 0x00000100u, 0x00001c00u, 0x00000000u, 0x00007bffu, 0x00007b80u}, + {0x00007c00u, 0x00007bfeu, 0x00007c00u, 0x00007c00u, 0x00000000u, 0x7c007c00u, 0x7c007c00u, 0x00007c00u, 0x7c000000u, 0x00007c00u, 0x00004c00u, 0x00000100u, 0x00001c00u, 0x000077ffu, 0x00000000u, 0x00007c00u}, + {0x00007c00u, 0x00000000u, 0x00007c00u, 0x00007c00u, 0x00000000u, 0x7c007c00u, 0x7c007c00u, 0x00007c00u, 0x7c000000u, 0x00007c00u, 0x00004c00u, 0x00000100u, 0x00001c00u, 0x00003bffu, 0x00000000u, 0x00007c00u}, + {0x0000fc00u, 0x00000000u, 0x00007c00u, 0x00007c00u, 0x00008000u, 0x7c007c00u, 0xfc00fc00u, 0x00007c00u, 0xfc000000u, 0x00000000u, 0x0000fe00u, 0x00008100u, 0x0000fe00u, 0x0000b7ffu, 0x00000000u, 0x00007bffu}, + {0x00007800u, 0x00007800u, 0x00007c00u, 0x00007c00u, 0x00000000u, 0x7c007c00u, 0x78007800u, 0x00007c00u, 0x78000000u, 0x00007c00u, 0x00004b80u, 0x00000200u, 0x00001da8u, 0x00007c00u, 0x00000000u, 0x00003c00u}, + {0x00007800u, 0x00007800u, 0x0000fc00u, 0x0000fc00u, 0x00008000u, 0xfc00fc00u, 0x78007800u, 0x0000fc00u, 0x78000000u, 0x00007c00u, 0x00004b80u, 0x00000200u, 0x00001da8u, 0x00007c00u, 0x00000000u, 0x00004c00u}, + {0x00007c00u, 0x00007c00u, 0x00007c00u, 0x00007c00u, 0x00000000u, 0x7c007c00u, 0x7c007c00u, 0x00007c00u, 0x7c000000u, 0x00007c00u, 0x00007c00u, 0x00000000u, 0x00000000u, 0x00007c00u, 0x00007c00u, 0x00000000u}, + {0x00007c00u, 0x00007c00u, 0x00007c00u, 0x00007c00u, 0x00007c00u, 0x7c007c00u, 0x7c007c00u, 0x00007c00u, 0x7c000000u, 0x00007c00u, 0x00007c00u, 0x00000000u, 0x00000000u, 0x00007c00u, 0x0000fc00u, 0x00000000u}, + {0x00007c00u, 0x00007c00u, 0x0000fe00u, 0x0000fe00u, 0x0000fe00u, 0xfe00fe00u, 0x7c007c00u, 0x0000fe00u, 0x7c000000u, 0x00007c00u, 0x00007c00u, 0x00000000u, 0x00000000u, 0x00007c00u, 0x00007c00u, 0x00007c00u}, + {0x0000fe00u, 0x00007c00u, 0x0000fc00u, 0x0000fc00u, 0x00008000u, 0xfc00fc00u, 0xfe00fe00u, 0x0000fc00u, 0xfe000000u, 0x00007c00u, 0x00007c00u, 0x00000000u, 0x00000000u, 0x00007c00u, 0x00007c00u, 0x00000000u}, + {0x00007e00u, 0x00007e00u, 0x00007e00u, 0x00007e00u, 0x00007c00u, 0x7e007e00u, 0x7e007e00u, 0x00007e00u, 0x7e000000u, 0x00007e00u, 0x00007e00u, 0x00007e00u, 0x00007e00u, 0x00007e00u, 0x00007c00u, 0x00003c00u}, + {0x00007d00u, 0x00007d00u, 0x00007d00u, 0x00007d00u, 0x00007c00u, 0x7d007d00u, 0x7d007d00u, 0x00007f00u, 0x7d000000u, 0x00007d00u, 0x00007d00u, 0x00007d00u, 0x00007d00u, 0x00007d00u, 0x0000fc00u, 0x00000000u}, + {0x00007bffu, 0x00007bffu, 0x00007bffu, 0x00007c00u, 0x00007bffu, 0x7bff7bffu, 0x7bff7bffu, 0x00007c00u, 0x7bff0000u, 0x00007c00u, 0x00004c00u, 0x00000100u, 0x00001c00u, 0x00007c00u, 0x00000000u, 0x00005400u}, + {0x00007bffu, 0x00007bffu, 0x00007c00u, 0x00007bffu, 0x0000fbffu, 0x7c007c00u, 0x7bff7bffu, 0x00007bffu, 0x7bff0000u, 0x00007c00u, 0x00004c00u, 0x00000100u, 0x00001c00u, 0x00000000u, 0x00000000u, 0x00007bfeu}, + {0x00007bffu, 0x00007bffu, 0x00007c00u, 0x0000fc00u, 0x0000fc00u, 0x7c007c00u, 0x7bff7bffu, 0x0000fc00u, 0x7bff0000u, 0x00007c00u, 0x00004c00u, 0x00000100u, 0x00001c00u, 0x00000000u, 0x00007bffu, 0x00007b00u}, + {0x00003c00u, 0x00003c00u, 0x00000000u, 0x00003c00u, 0x00007c00u, 0x00000000u, 0x3c003c00u, 0x00003c00u, 0x3c000000u, 0x00004000u, 0x00000000u, 0x00003c00u, 0x00003c00u, 0x00007c00u, 0x00000000u, 0x00004c00u}, + {0x00004000u, 0x00000000u, 0x00003c00u, 0x00007c00u, 0x00007c00u, 0x3c003c00u, 0x40004000u, 0x00007c00u, 0x40000000u, 0x00004000u, 0x00000000u, 0x00003c00u, 0x00003c00u, 0x00007800u, 0x00000000u, 0x00004b80u}, + {0x00007bffu, 0x00007bffu, 0x00001bffu, 0x00003c04u, 0x00007bffu, 0x1bff1bffu, 0x7bff7bffu, 0x00003c04u, 0x7bff0000u, 0x00007c00u, 0x00004c00u, 0x00000100u, 0x00001c00u, 0x00000000u, 0x00007c00u, 0x00007b80u}, + {0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x0000fe00u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00003c00u, 0x0000fc00u, 0x00007c00u, 0x00007c00u, 0x00000000u, 0x00007c00u, 0x00000000u}, + {0x00003c00u, 0x0000bc00u, 0x00008000u, 0x00000000u, 0x00000000u, 0x80008000u, 0x3c003c00u, 0x00000000u, 0x3c000000u, 0x00003c00u, 0x0000fc00u, 0x0000fc00u, 0x0000fc00u, 0x00008000u, 0x00007c00u, 0x00000000u}, + {0x00000002u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00020002u, 0x00000000u, 0x00020000u, 0x00003c00u, 0x0000ce00u, 0x00007c00u, 0x00006c00u, 0x00007c00u, 0x00000000u, 0x00005100u}, + {0x00003c00u, 0x0000bc00u, 0x00000100u, 0x00000100u, 0x00000000u, 0x01000100u, 0x3c003c00u, 0x00000100u, 0x3c000000u, 0x00003c00u, 0x0000cc00u, 0x00007c00u, 0x00005c00u, 0x00007c00u, 0x00000000u, 0x00005000u}, + {0x00003c00u, 0x0000bc00u, 0x00000200u, 0x00000200u, 0x00000000u, 0x02000200u, 0x3c003c00u, 0x00000200u, 0x3c000000u, 0x00003c00u, 0x0000cb80u, 0x00007800u, 0x000059a8u, 0x00007c00u, 0x00000000u, 0x00004fc0u}, + {0x00004c40u, 0x00004b80u, 0x00004c00u, 0x00004c40u, 0x00004c00u, 0x4c004c00u, 0x4c404c40u, 0x00004c40u, 0x4c400000u, 0x00007c00u, 0x00004400u, 0x00002c00u, 0x00003400u, 0x00007c00u, 0x00000000u, 0x00004a00u}, + {0x00004c40u, 0x00004b7fu, 0x00004bffu, 0x00004c40u, 0x00004bffu, 0x4bff4bffu, 0x4c404c40u, 0x00004c40u, 0x4c400000u, 0x00007bf5u, 0x00004400u, 0x00002c01u, 0x00003400u, 0x00007bffu, 0x00000000u, 0x00004a00u}, + {0x00005a40u, 0x00000000u, 0x000070e2u, 0x000070eeu, 0x00005640u, 0x70e270e2u, 0x5a405a40u, 0x000070eeu, 0x5a400000u, 0x00007c00u, 0x000046a5u, 0x0000211fu, 0x00002e66u, 0x00005640u, 0x00003c00u, 0x00000000u}, + {0x0000fc00u, 0x0000fc00u, 0x0000fc00u, 0x0000fe00u, 0x00007c00u, 0xfc00fc00u, 0xfc00fc00u, 0x0000fe00u, 0xfc000000u, 0x00000000u, 0x0000fe00u, 0x00008000u, 0x0000fe00u, 0x0000fc00u, 0x00000000u, 0x00007800u}, + {0x00007800u, 0x00007800u, 0x00007c00u, 0x00007c00u, 0x00000000u, 0x7c007c00u, 0x78007800u, 0x00007c00u, 0x78000000u, 0x00007c00u, 0x00004b80u, 0x00000200u, 0x00001da8u, 0x00007800u, 0x00007f00u, 0x00000000u}, + {0x0000fc00u, 0x0000fbfeu, 0x00007c00u, 0x00007c00u, 0x00008000u, 0x7c007c00u, 0xfc00fc00u, 0x00007c00u, 0xfc000000u, 0x00000000u, 0x0000fe00u, 0x00008100u, 0x0000fe00u, 0x0000fc00u, 0x00000000u, 0x00007800u}, + {0x00007bfeu, 0x00007c00u, 0x0000fc00u, 0x0000fc00u, 0x00008000u, 0xfc00fc00u, 0x7bfe7bfeu, 0x0000fc00u, 0x7bfe0000u, 0x00007c00u, 0x00004c00u, 0x00000100u, 0x00001c00u, 0x00007c00u, 0x00000000u, 0x00004000u}, + {0x00007c00u, 0x0000fe00u, 0x00007c00u, 0x00007c00u, 0x0000fe00u, 0x7c007c00u, 0x7c007c00u, 0x00007c00u, 0x7c000000u, 0x00007c00u, 0x00007c00u, 0x00000000u, 0x00000000u, 0x00007c00u, 0x0000fc00u, 0x00000000u} +}; +constexpr const char* Names[16] = { + "v_add_f16", + "v_sub_f16", + "v_mul_f16", + "v_fma_f16", + "v_div_fixup_f16", + "v_pk_mul_f16 op_sel_hi:[0,0]", + "v_pk_add_f16 op_sel_hi:[0,0]", + "v_fma_mixlo_f16", + "v_add_f16_sdwa dst_sel:WORD_1", + "v_exp_f16", + "v_log_f16", + "v_rcp_f16", + "v_rsq_f16", + "v_ldexp_f16", + "v_cvt_f16_f32", + "v_cvt_f16_u16" +}; + +bool WholeDword(std::uint32_t column) { + return column == 5u || column == 6u || column == 8u; +} + +bool DriverNan(std::uint32_t column) { + return column != 4u && column != 5u && column != 6u && column != 13u && column != 15u; +} + +std::array BufferDescriptor(const void* data, std::uint32_t bytes) { + const auto address = reinterpret_cast(data); + return {static_cast(address), static_cast((address >> 32u) & 0xffffu), bytes, 0x31016facu}; +} + +std::string Hex(std::uint32_t value) { + char text[16]; + std::snprintf(text, sizeof(text), "0x%08x", value); + return text; +} + +bool IsNan16(std::uint32_t half) { + return (half & 0x7fffu) > 0x7c00u; +} + +void Run(AgcDriver::VulkanDevice& device, const std::optional& floatMode) { + for (std::uint32_t tid = 0; tid < Threads; ++tid) std::copy(std::begin(Rows[tid]), std::end(Rows[tid]), &Input[tid * Inputs]); + Output.fill(0xdeadbeefu); + std::vector userData(8, 0u); + const auto input = BufferDescriptor(Input.data(), static_cast(Input.size() * 4u)); + const auto output = BufferDescriptor(Output.data(), static_cast(Output.size() * 4u)); + std::copy(input.begin(), input.end(), userData.begin()); + std::copy(output.begin(), output.end(), userData.begin() + 4); + const std::span code(Code); + const std::array memory{{{reinterpret_cast(code.data()), std::as_bytes(code)}}}; + const ShaderRecompiler::ShaderComputeStageInfo compute{{Threads, 1, 1}, 0u, {false, false, false}, false, 1}; + ShaderRecompiler::RecompileRequest request{ + {ShaderStage::Compute, reinterpret_cast(code.data()), code, 0, {}}, + {32, 0, userData, compute, std::nullopt, std::nullopt, memory}, + device.Target(), + {0, 0, 0, 128} + }; + request.useCache = false; + request.context.floatMode = floatMode; + const auto result = ShaderRecompiler::Recompile(request); + device.Dispatch(result, 1, 1, 1, {}, reinterpret_cast(code.data())); + device.WaitIdle(); +} + +void Check(const std::uint32_t (&expected)[32][16], const char* mode) { + for (std::uint32_t tid = 0; tid < Threads; ++tid) { + for (std::uint32_t column = 0; column < Results; ++column) { + const std::uint32_t actual = Output[tid * Results + column]; + const std::uint32_t wanted = expected[tid][column]; + const std::string name = std::string("fp16 overflow: lane ") + std::to_string(tid) + " " + mode + " " + Names[column] + " is " + Hex(actual) + ", expected " + Hex(wanted); + for (std::uint32_t shift = 0; shift < 32u; shift += 16u) { + if (shift == 16u && !WholeDword(column)) break; + const std::uint32_t half = (actual >> shift) & 0xffffu; + const std::uint32_t expectedHalf = (wanted >> shift) & 0xffffu; + if (DriverNan(column) && IsNan16(expectedHalf)) { + Require(IsNan16(half), name + " (any NaN)"); + } else { + Require(half == expectedHalf, name); + } + } + } + } +} + +} + +int main() { + try { + const auto device = OpenVulkanTestDevice(); + if (!device) return VulkanTestSkipped; + Run(*device, std::nullopt); + Check(ExpectedInfinity, "no float mode"); + Run(*device, ShaderRecompiler::ShaderFloatMode{0xc0u, true, false, false}); + Check(ExpectedInfinity, "FP16_OVFL=0"); + Run(*device, ShaderRecompiler::ShaderFloatMode{0xc0u, true, false, true}); + Check(ExpectedOverflow, "FP16_OVFL=1"); + std::puts("fp16 overflow tests passed"); + return 0; + } catch (const std::exception& error) { + std::cerr << error.what() << '\n'; + return 1; + } +} diff --git a/core/libs/prx/libSceAgcDriver/tests/execution/GatedDispatchCapture.cpp b/core/libs/prx/libSceAgcDriver/tests/execution/GatedDispatchCapture.cpp new file mode 100644 index 000000000..fe73e2b3a --- /dev/null +++ b/core/libs/prx/libSceAgcDriver/tests/execution/GatedDispatchCapture.cpp @@ -0,0 +1,144 @@ +#include "VulkanTestDevice.hpp" +#include "prx/libSceAgcDriver/Execution/include/Driver.hpp" +#include "prx/libSceAgcDriver/Submit/include/Acb.hpp" +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace { + +void Require(bool condition, const std::string& message) { + if (!condition) throw std::runtime_error(message); +} + +alignas(256) std::array SpinCode{0xf4080200u, 0xfa000000u, 0xbe840381u, 0x9304ff04u, 0x0019660du, 0x8004ff04u, 0x3c6ef35fu, 0x80828102u, 0xbf078002u, 0xbf85fff9u, 0xbf8cc07fu, 0x7e000204u, 0xe0700000u, 0x80020000u, 0xbf810000u}; +alignas(256) std::array ReaderCode{0xf4040100u, 0xfa000000u, 0xf4080201u, 0xfa000000u, 0xbf8cc07fu, 0xf4000182u, 0xfa000000u, 0xbf8cc07fu, 0x7e000206u, 0xe0700000u, 0x80020000u, 0xbf810000u}; + +constexpr std::uint32_t StaleValue = 0x57a1e000u; +constexpr std::uint32_t FreshValue = 0xf7e54000u; + +struct Slot { + alignas(256) std::array stale{}; + alignas(256) std::array fresh{}; + alignas(256) std::array table{}; + alignas(256) std::array output{}; + alignas(256) std::array outputDescriptor{}; + alignas(256) std::array spinOutput{}; + alignas(256) std::array spinDescriptor{}; + alignas(256) volatile std::uint32_t label = 0; + alignas(256) volatile std::uint32_t done = 0; +}; + +std::array Slots; + +std::uint32_t Low(const volatile void* pointer) { + return static_cast(reinterpret_cast(pointer)); +} + +std::uint32_t High(const volatile void* pointer) { + return static_cast(static_cast(reinterpret_cast(pointer)) >> 32u); +} + +std::array Descriptor(const void* pointer, std::uint32_t bytes) { + return {Low(pointer), High(pointer) & 0xffffu, bytes, 0x31016facu}; +} + +void SetShaderRegister(std::vector& words, std::uint32_t offset, std::uint32_t value) { + words.insert(words.end(), {0xc0017600u, offset, value}); +} + +void Dispatch(std::vector& words, const void* code, std::initializer_list userData) { + const auto address = static_cast(reinterpret_cast(code)); + SetShaderRegister(words, 0x20c, static_cast(address >> 8u)); + SetShaderRegister(words, 0x20d, static_cast(address >> 40u) & 0xffu); + for (const std::uint32_t offset : {0x207u, 0x208u, 0x209u}) SetShaderRegister(words, offset, 1); + SetShaderRegister(words, 0x212, 0); + SetShaderRegister(words, 0x213, static_cast(userData.size()) << 1u); + std::uint32_t offset = 0x240; + for (const auto value : userData) SetShaderRegister(words, offset++, value); + words.insert(words.end(), {0xc0031500u, 1, 1, 1, 0x8041u}); +} + +void WriteData(std::vector& words, volatile std::uint32_t* target, std::uint32_t value) { + words.insert(words.end(), {0xc0033700u, 0x00100200u, Low(target), High(target), value}); +} + +void WaitEqual(std::vector& words, volatile std::uint32_t* target, std::uint32_t value) { + words.insert(words.end(), {0xc0053c00u, 0x13u, Low(target), High(target), value, 0xffffffffu, 0x19u}); +} + +void Submit(std::uint32_t queue, std::vector& words) { + Packet packet{words.data(), static_cast(words.size()), 0, {}}; + Require(sceAgcDriverSubmitAcb(queue, &packet) == 0, "compute submit failed"); +} + +enum class Outcome { Fresh, Stale, Inconclusive }; + +Outcome Run(Slot& slot, std::uint32_t iterations) { + slot.stale[0] = StaleValue; + slot.fresh[0] = FreshValue; + slot.table = {Low(slot.stale.data()), High(slot.stale.data())}; + slot.outputDescriptor = Descriptor(slot.output.data(), sizeof(slot.output)); + slot.spinDescriptor = Descriptor(slot.spinOutput.data(), sizeof(slot.spinOutput)); + std::vector producer; + Dispatch(producer, SpinCode.data(), {Low(slot.spinDescriptor.data()), High(slot.spinDescriptor.data()), iterations}); + WriteData(producer, &slot.label, 1); + std::vector consumer; + WaitEqual(consumer, &slot.label, 1); + Dispatch(consumer, ReaderCode.data(), {Low(slot.table.data()), High(slot.table.data()), Low(slot.outputDescriptor.data()), High(slot.outputDescriptor.data())}); + WriteData(consumer, &slot.done, 1); + Submit(0x28, producer); + Submit(0x20, consumer); + std::this_thread::sleep_for(std::chrono::milliseconds(300)); + slot.table = {Low(slot.fresh.data()), High(slot.fresh.data())}; + std::atomic_thread_fence(std::memory_order_seq_cst); + const bool landedBeforeFill = slot.label != 0; + const auto start = std::chrono::steady_clock::now(); + while (slot.done == 0) { + Require(std::chrono::steady_clock::now() - start < std::chrono::seconds(10), "the gated submissions did not finish"); + std::this_thread::sleep_for(std::chrono::microseconds(100)); + } + AgcDriverWaitIdle_nid_postfix(); + if (landedBeforeFill) return Outcome::Inconclusive; + Require(slot.output[0] == FreshValue || slot.output[0] == StaleValue, "the gated dispatch stored neither table's value: " + std::to_string(slot.output[0])); + return slot.output[0] == FreshValue ? Outcome::Fresh : Outcome::Stale; +} + +} + +int main() { + try { + auto device = OpenVulkanTestDevice(); + if (!device) return VulkanTestSkipped; + device.reset(); + std::uint32_t iterations = 1u << 26u; + for (auto& slot : Slots) { + const auto outcome = Run(slot, iterations); + Require(outcome != Outcome::Stale, "a dispatch behind a cross-queue WAIT_REG_MEM read its table before the producer's label reached memory, so it missed the table the CPU filled after submitting"); + if (outcome == Outcome::Fresh) { + std::cout << "gated dispatch capture test passed (" << iterations << " producer iterations)\n"; + AgcDriverShutdown_nid_postfix(); + return 0; + } + iterations *= 2u; + } + std::cout << "the producer's label always reached memory before the table was filled: gated dispatch capture not tested\n"; + AgcDriverShutdown_nid_postfix(); + return VulkanTestSkipped; + } catch (const std::exception& error) { + std::cerr << error.what() << '\n'; + try { + AgcDriverShutdown_nid_postfix(); + } catch (const std::exception& shutdownError) { + std::cerr << shutdownError.what() << '\n'; + } + return 1; + } +} diff --git a/core/libs/prx/libSceAgcDriver/tests/execution/GlobalAtomicCsub.cpp b/core/libs/prx/libSceAgcDriver/tests/execution/GlobalAtomicCsub.cpp index 02ac733f2..b54523dd6 100644 --- a/core/libs/prx/libSceAgcDriver/tests/execution/GlobalAtomicCsub.cpp +++ b/core/libs/prx/libSceAgcDriver/tests/execution/GlobalAtomicCsub.cpp @@ -89,7 +89,7 @@ public: block = static_cast(std::aligned_alloc(BlockBytes, BlockBytes)); #endif Require(block != nullptr, "global atomic csub: cannot allocate the guest block"); - GuestAllocations::Mutation().Add(block, BlockBytes, true, true); + GuestAllocations::Mutation().Add(block, BlockBytes, true, true, true); } ~GuestBlock() { diff --git a/core/libs/prx/libSceAgcDriver/tests/execution/GlobalAtomics.cpp b/core/libs/prx/libSceAgcDriver/tests/execution/GlobalAtomics.cpp index e36953f05..2daef9cd5 100644 --- a/core/libs/prx/libSceAgcDriver/tests/execution/GlobalAtomics.cpp +++ b/core/libs/prx/libSceAgcDriver/tests/execution/GlobalAtomics.cpp @@ -292,7 +292,7 @@ public: block = static_cast(std::aligned_alloc(BlockBytes, BlockBytes)); #endif Require(block != nullptr, "global atomics: cannot allocate the guest block"); - GuestAllocations::Mutation().Add(block, BlockBytes, true, writable); + GuestAllocations::Mutation().Add(block, BlockBytes, true, writable, true); } ~GuestBlock() { diff --git a/core/libs/prx/libSceAgcDriver/tests/execution/GlobalAtomics64.cpp b/core/libs/prx/libSceAgcDriver/tests/execution/GlobalAtomics64.cpp index 859f39c58..528beb55c 100644 --- a/core/libs/prx/libSceAgcDriver/tests/execution/GlobalAtomics64.cpp +++ b/core/libs/prx/libSceAgcDriver/tests/execution/GlobalAtomics64.cpp @@ -256,7 +256,7 @@ public: block = static_cast(std::aligned_alloc(BlockBytes, BlockBytes)); #endif Require(block != nullptr, "global atomics: cannot allocate the guest block"); - GuestAllocations::Mutation().Add(block, BlockBytes, true, writable); + GuestAllocations::Mutation().Add(block, BlockBytes, true, writable, true); } ~GuestBlock() { diff --git a/core/libs/prx/libSceAgcDriver/tests/execution/GlobalAtomicsFloat.cpp b/core/libs/prx/libSceAgcDriver/tests/execution/GlobalAtomicsFloat.cpp index 800fb3947..503ccf662 100644 --- a/core/libs/prx/libSceAgcDriver/tests/execution/GlobalAtomicsFloat.cpp +++ b/core/libs/prx/libSceAgcDriver/tests/execution/GlobalAtomicsFloat.cpp @@ -585,7 +585,7 @@ public: block = static_cast(std::aligned_alloc(BlockBytes, BlockBytes)); #endif Require(block != nullptr, "global float atomics: cannot allocate the guest block"); - GuestAllocations::Mutation().Add(block, BlockBytes, true, writable); + GuestAllocations::Mutation().Add(block, BlockBytes, true, writable, true); } ~GuestBlock() { diff --git a/core/libs/prx/libSceAgcDriver/tests/execution/GlobalAtomicsLanes.cpp b/core/libs/prx/libSceAgcDriver/tests/execution/GlobalAtomicsLanes.cpp index d0bee4f75..172394301 100644 --- a/core/libs/prx/libSceAgcDriver/tests/execution/GlobalAtomicsLanes.cpp +++ b/core/libs/prx/libSceAgcDriver/tests/execution/GlobalAtomicsLanes.cpp @@ -67,7 +67,7 @@ public: block = static_cast(std::aligned_alloc(BlockBytes, BlockBytes)); #endif Require(block != nullptr, "global atomics: cannot allocate the guest block"); - GuestAllocations::Mutation().Add(block, BlockBytes, true, writable); + GuestAllocations::Mutation().Add(block, BlockBytes, true, writable, true); } ~GuestBlock() { diff --git a/core/libs/prx/libSceAgcDriver/tests/execution/GlobalLoadAddtid.cpp b/core/libs/prx/libSceAgcDriver/tests/execution/GlobalLoadAddtid.cpp index e7875353c..34ccd295e 100644 --- a/core/libs/prx/libSceAgcDriver/tests/execution/GlobalLoadAddtid.cpp +++ b/core/libs/prx/libSceAgcDriver/tests/execution/GlobalLoadAddtid.cpp @@ -28,12 +28,13 @@ constexpr std::uint32_t MaskedExecLo = 0x55555555u; alignas(256) std::array Table{}; alignas(256) std::array Output{}; -alignas(256) constexpr std::array AddtidCode{ +alignas(256) constexpr std::array AddtidCode{ 0x34020085, 0x34040082, 0xbe880300, 0x8709ff01, 0x0000ffff, 0x800aff08, 0x00000200, 0x820b8009, - 0x7e080280, 0x460808c1, 0x480808c1, 0x7e0a02ff, 0xabcdabcd, 0x7e0c02ff, 0xabcdabcd, 0x7e0e02ff, - 0xabcdabcd, 0xe0301000, 0x80000802, 0xdc588040, 0x05080000, 0xdc588fe4, 0x060a0000, 0xbefe03ff, - 0x55555555, 0xdc588000, 0x07080000, 0xbefe03c1, 0xe0701000, 0x80010401, 0xe0701004, 0x80010501, - 0xe0701008, 0x80010601, 0xe070100c, 0x80010701, 0xe0701010, 0x80010801, 0xbf810000, + 0x7e080280, 0xd7650004, 0x000208c1, 0xd7660004, 0x000208c1, 0x7e0a02ff, 0xabcdabcd, 0x7e0c02ff, + 0xabcdabcd, 0x7e0e02ff, 0xabcdabcd, 0xe0301000, 0x80000802, 0xdc588040, 0x05080000, 0xdc588fe4, + 0x060a0000, 0xbefe03ff, 0x55555555, 0xdc588000, 0x07080000, 0xbefe03c1, 0xe0701000, 0x80010401, + 0xe0701004, 0x80010501, 0xe0701008, 0x80010601, 0xe070100c, 0x80010701, 0xe0701010, 0x80010801, + 0xbf810000, }; struct Workgroup { diff --git a/core/libs/prx/libSceAgcDriver/tests/execution/GlobalLoads.cpp b/core/libs/prx/libSceAgcDriver/tests/execution/GlobalLoads.cpp index 26938a5d3..0c288dbb1 100644 --- a/core/libs/prx/libSceAgcDriver/tests/execution/GlobalLoads.cpp +++ b/core/libs/prx/libSceAgcDriver/tests/execution/GlobalLoads.cpp @@ -94,7 +94,7 @@ public: block = static_cast(std::aligned_alloc(BlockBytes, BlockBytes)); #endif Require(block != nullptr, "global loads: cannot allocate the guest block"); - GuestAllocations::Mutation().Add(block, BlockBytes, true, false); + GuestAllocations::Mutation().Add(block, BlockBytes, true, false, true); } ~GuestBlock() { diff --git a/core/libs/prx/libSceAgcDriver/tests/execution/GlobalLoadsX2X3.cpp b/core/libs/prx/libSceAgcDriver/tests/execution/GlobalLoadsX2X3.cpp index 878bf6426..73eceacdb 100644 --- a/core/libs/prx/libSceAgcDriver/tests/execution/GlobalLoadsX2X3.cpp +++ b/core/libs/prx/libSceAgcDriver/tests/execution/GlobalLoadsX2X3.cpp @@ -93,7 +93,7 @@ public: block = static_cast(std::aligned_alloc(BlockBytes, BlockBytes)); #endif Require(block != nullptr, "global loads x2 x3: cannot allocate the guest block"); - GuestAllocations::Mutation().Add(block, BlockBytes, true, false); + GuestAllocations::Mutation().Add(block, BlockBytes, true, false, true); } ~GuestBlock() { diff --git a/core/libs/prx/libSceAgcDriver/tests/execution/GlobalLoadsX4.cpp b/core/libs/prx/libSceAgcDriver/tests/execution/GlobalLoadsX4.cpp index 6dc40863b..44a9af2f5 100644 --- a/core/libs/prx/libSceAgcDriver/tests/execution/GlobalLoadsX4.cpp +++ b/core/libs/prx/libSceAgcDriver/tests/execution/GlobalLoadsX4.cpp @@ -93,7 +93,7 @@ public: block = static_cast(std::aligned_alloc(BlockBytes, BlockBytes)); #endif Require(block != nullptr, "global loads x4: cannot allocate the guest block"); - GuestAllocations::Mutation().Add(block, BlockBytes, true, false); + GuestAllocations::Mutation().Add(block, BlockBytes, true, false, true); } ~GuestBlock() { diff --git a/core/libs/prx/libSceAgcDriver/tests/execution/GlobalStoreAddtid.cpp b/core/libs/prx/libSceAgcDriver/tests/execution/GlobalStoreAddtid.cpp index fcf2f94d3..7a9f4a744 100644 --- a/core/libs/prx/libSceAgcDriver/tests/execution/GlobalStoreAddtid.cpp +++ b/core/libs/prx/libSceAgcDriver/tests/execution/GlobalStoreAddtid.cpp @@ -61,7 +61,7 @@ public: block = static_cast(std::aligned_alloc(BlockBytes, BlockBytes)); #endif Require(block != nullptr, "global store addtid: cannot allocate the guest block"); - GuestAllocations::Mutation().Add(block, BlockBytes, true, true); + GuestAllocations::Mutation().Add(block, BlockBytes, true, true, true); } ~GuestBlock() { diff --git a/core/libs/prx/libSceAgcDriver/tests/execution/GlobalVccBase.cpp b/core/libs/prx/libSceAgcDriver/tests/execution/GlobalVccBase.cpp index e0e441989..76e40037f 100644 --- a/core/libs/prx/libSceAgcDriver/tests/execution/GlobalVccBase.cpp +++ b/core/libs/prx/libSceAgcDriver/tests/execution/GlobalVccBase.cpp @@ -56,7 +56,7 @@ public: block = static_cast(std::aligned_alloc(BlockBytes, BlockBytes)); #endif Require(block != nullptr, "global vcc base: cannot allocate the guest block"); - GuestAllocations::Mutation().Add(block, BlockBytes, true, true); + GuestAllocations::Mutation().Add(block, BlockBytes, true, true, true); } ~GuestBlock() { diff --git a/core/libs/prx/libSceAgcDriver/tests/execution/GuardedNullPointer.cpp b/core/libs/prx/libSceAgcDriver/tests/execution/GuardedNullPointer.cpp new file mode 100644 index 000000000..d9ad01a30 --- /dev/null +++ b/core/libs/prx/libSceAgcDriver/tests/execution/GuardedNullPointer.cpp @@ -0,0 +1,268 @@ +#include "prx/libSceAgcDriver/Execution/include/Driver/Dispatch/DispatchCache.hpp" +#include "prx/libSceAgcDriver/Execution/include/ShaderMemory.hpp" +#include "prx/libSceAgcDriver/Execution/include/VulkanDevice.hpp" +#include "Recompiler.hpp" +#include "VulkanTestDevice.hpp" +#include +#include +#include +#include +#include +#include +#include +#include +#ifdef _WIN32 +#ifndef NOMINMAX +#define NOMINMAX +#endif +#ifndef WIN32_LEAN_AND_MEAN +#define WIN32_LEAN_AND_MEAN +#endif +#include +#include +#else +#include +#include +#endif + +namespace { + +using AgcDriver::Graphics::Require; +using ShaderRecompiler::ShaderStage; + +constexpr std::uint32_t Lanes = 64; +constexpr std::uint32_t Default = 0x11111111u; +constexpr std::uint32_t Payload = 0x3f800000u; + +alignas(256) std::array Output{}; +alignas(256) std::array Root{}; +alignas(256) std::array Data{Payload, 0u, 0u, 0u}; +alignas(256) std::array Indirect{}; +alignas(256) std::array Table{}; + +alignas(256) constexpr std::array BranchCode{ + 0xf4080100u, 0xfa000000u, 0xf4000200u, 0xfa000010u, 0xf4040280u, 0xfa000018u, 0xbe8c03ffu, 0x11111111u, + 0xbf8cc07fu, 0xbf068008u, 0xbf850003u, 0xf4000305u, 0xfa000000u, 0xbf8cc07fu, 0x7e02020cu, 0x34040082u, + 0xe0701000u, 0x80010102u, 0xbf810000u, +}; + +alignas(256) constexpr std::array ExecCode{ + 0xf4080100u, 0xfa000000u, 0xf4000200u, 0xfa000010u, 0xf4040280u, 0xfa000018u, 0xbe8c03ffu, 0x11111111u, + 0xbf8cc07fu, 0x7e02020cu, 0x34040082u, 0x7da80008u, 0xbf880004u, 0xf4000305u, 0xfa000000u, 0xbf8cc07fu, + 0x7e02020cu, 0xbefe04c1u, 0xe0701000u, 0x80010102u, 0xbf810000u, +}; + +alignas(256) constexpr std::array EmptyExecCode{ + 0xf4080100u, 0xfa000000u, 0xf4000200u, 0xfa000010u, 0xf4040280u, 0xfa000018u, 0xbe8c03ffu, 0x11111111u, + 0xbf8cc07fu, 0x7e02020cu, 0x34040082u, 0x7da80008u, 0xbf800000u, 0xf4000305u, 0xfa000000u, 0xbf8cc07fu, + 0x7e02020cu, 0xbefe04c1u, 0xe0701000u, 0x80010102u, 0xbf810000u, +}; + +alignas(256) constexpr std::array ChainCode{ + 0xf4080100u, 0xfa000000u, 0xf4000200u, 0xfa000010u, 0xf4040280u, 0xfa000018u, 0xbe8c03ffu, 0x11111111u, + 0xbf8cc07fu, 0xf4040385u, 0xfa000000u, 0xbf8cc07fu, 0xbf068008u, 0xbf850003u, 0xf4000307u, 0xfa000000u, + 0xbf8cc07fu, 0x7e02020cu, 0x34040082u, 0xe0701000u, 0x80010102u, 0xbf810000u, +}; + +alignas(256) constexpr std::array DescriptorCode{ + 0xf4080100u, 0xfa000000u, 0xf4000200u, 0xfa000010u, 0xf4040280u, 0xfa000018u, 0x7e0202ffu, 0x11111111u, + 0xbf8cc07fu, 0xbf068008u, 0xbf850006u, 0xf4080405u, 0xfa000010u, 0xbf8cc07fu, 0xe0300000u, 0x80040100u, + 0xbf8c3f70u, 0x34040082u, 0xe0701000u, 0x80010102u, 0xbf810000u, +}; + +class StderrCapture { +public: + StderrCapture() { + std::fflush(stderr); + file = std::tmpfile(); + Require(file != nullptr, "cannot open a temporary file for stderr"); +#ifdef _WIN32 + saved = _dup(_fileno(stderr)); + _dup2(_fileno(file), _fileno(stderr)); +#else + saved = dup(fileno(stderr)); + dup2(fileno(file), fileno(stderr)); +#endif + } + std::string Finish() { + std::fflush(stderr); +#ifdef _WIN32 + _dup2(saved, _fileno(stderr)); + _close(saved); +#else + dup2(saved, fileno(stderr)); + close(saved); +#endif + std::rewind(file); + std::string text; + char buffer[4096]; + for (std::size_t read = 0; (read = std::fread(buffer, 1, sizeof(buffer), file)) != 0;) text.append(buffer, read); + std::fclose(file); + std::fwrite(text.data(), 1, text.size(), stderr); + return text; + } + +private: + std::FILE* file = nullptr; + int saved = -1; +}; + +struct Outcome { + std::vector words; + std::string log; + std::uint64_t variantId = 0; + std::uint64_t pipelineVariantId = 0; + std::uint32_t poisoned = 0; + bool cacheable = false; +}; + +template +Outcome Run(AgcDriver::VulkanDevice& device, const std::array& code, std::uint64_t pointer, std::uint32_t guard) { + const auto output = static_cast(reinterpret_cast(Output.data())); + Root = {}; + Root[0] = static_cast(output); + Root[1] = static_cast((output >> 32u) & 0xffffu); + Root[2] = static_cast(sizeof(Output)); + Root[3] = 0x30027facu; + Root[4] = guard; + Root[6] = static_cast(pointer); + Root[7] = static_cast(pointer >> 32u); + std::fill(Output.begin(), Output.end(), 0xdeadbeefu); + const auto root = static_cast(reinterpret_cast(Root.data())); + const std::array userData{static_cast(root), static_cast(root >> 32u)}; + const std::span span(code); + const ShaderRecompiler::ShaderComputeStageInfo compute{{Lanes, 1, 1}, 0u, {false, false, false}, false, 1}; + ShaderRecompiler::RecompileRequest request{ + {ShaderStage::Compute, reinterpret_cast(span.data()), span, 0, {}}, + {64, 0, userData, compute, std::nullopt, std::nullopt, {}}, + device.Target(), + {0, 0, 0, 128} + }; + AgcDriver::ShaderMemory memory({}); + const auto capture = memory.Capture(request); + const auto regions = memory.Regions(); + request.context.memory = regions; + const auto result = ShaderRecompiler::Recompile(request, *capture); + Outcome outcome; + outcome.variantId = result->variantId; + outcome.pipelineVariantId = result->PipelineVariantId(); + outcome.poisoned = result->poisonedSrtReads; + outcome.cacheable = AgcDriver::DriverDetail::CacheableResult(*result); + StderrCapture log; + try { + device.Dispatch(*result, 1, 1, 1, {}, reinterpret_cast(span.data())); + device.WaitIdle(); + } catch (...) { + static_cast(log.Finish()); + throw; + } + outcome.log = log.Finish(); + outcome.words.assign(Output.begin(), Output.end()); + return outcome; +} + +bool Faulted(const Outcome& outcome) { + return outcome.log.find("BDA access failed") != std::string::npos; +} + +std::string Fault(std::uint64_t address, std::uint32_t pc) { + char text[96]; + std::snprintf(text, sizeof(text), "address=0x%llx instruction=0x%x bytes=4", static_cast(address), pc); + return text; +} + +void RequireWords(const Outcome& outcome, std::uint32_t lanes, std::uint32_t inside, const std::string& what) { + for (std::uint32_t lane = 0; lane < Lanes; ++lane) { + const auto expected = lane < lanes ? inside : Default; + Require(outcome.words[lane] == expected, what + ": lane " + std::to_string(lane) + " wrote " + std::to_string(outcome.words[lane]) + ", expected " + std::to_string(expected)); + } +} + +void RunTests(AgcDriver::VulkanDevice& device) { + const auto valid = static_cast(reinterpret_cast(Data.data())); + void* reserved = nullptr; +#ifdef _WIN32 + reserved = VirtualAlloc(nullptr, 65536, MEM_RESERVE, PAGE_NOACCESS); +#else + reserved = mmap(nullptr, 65536, PROT_NONE, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + if (reserved == MAP_FAILED) reserved = nullptr; +#endif + Require(reserved != nullptr, "cannot reserve an inaccessible range"); + const auto unmapped = static_cast(reinterpret_cast(reserved)); + + const auto taken = Run(device, BranchCode, valid, 1u); + Require(!Faulted(taken) && taken.poisoned == 0u && taken.cacheable, "guarded pointer: a mapped pointer faulted, was poisoned or left the caches:\n" + taken.log); + RequireWords(taken, Lanes, Payload, "guarded pointer: a mapped pointer under a taken branch"); + const auto skippedNull = Run(device, BranchCode, 0u, 0u); + Require(!Faulted(skippedNull) && skippedNull.poisoned == 1u && !skippedNull.cacheable, "guarded pointer: a null pointer behind a branch that skips its read faulted or stayed cacheable:\n" + skippedNull.log); + RequireWords(skippedNull, 0u, 0u, "guarded pointer: a skipped read through a null pointer"); + const auto takenNull = Run(device, BranchCode, 0u, 1u); + Require(takenNull.pipelineVariantId == skippedNull.pipelineVariantId && takenNull.pipelineVariantId == taken.pipelineVariantId && takenNull.variantId == taken.variantId, "guarded pointer: the poison or the branch condition changed the pipeline variant"); + Require(Faulted(takenNull) && takenNull.log.find(Fault(0u, 0x2cu)) != std::string::npos && takenNull.log.find("reason=1") != std::string::npos, "guarded pointer: a read through a null pointer that runs did not fault at its pc and address:\n" + takenNull.log); + Require(std::all_of(takenNull.words.begin(), takenNull.words.end(), [](std::uint32_t word) { return word == 0u || word == 0xdeadbeefu; }), "guarded pointer: a faulting read did not return zero"); + const auto takenUnmapped = Run(device, BranchCode, unmapped, 1u); + Require(Faulted(takenUnmapped) && takenUnmapped.log.find(Fault(unmapped, 0x2cu)) != std::string::npos, "guarded pointer: a read through an unmapped pointer that runs did not fault at its address:\n" + takenUnmapped.log); + Require(takenUnmapped.pipelineVariantId == taken.pipelineVariantId, "guarded pointer: an unmapped pointer changed the pipeline variant"); + + const auto someLanes = Run(device, ExecCode, valid, 5u); + Require(!Faulted(someLanes), "guarded pointer: a mapped pointer under EXEC faulted:\n" + someLanes.log); + RequireWords(someLanes, 5u, Payload, "guarded pointer: a mapped pointer read under EXEC"); + const auto noLanes = Run(device, ExecCode, 0u, 0u); + Require(!Faulted(noLanes) && noLanes.poisoned == 1u, "guarded pointer: a null pointer read skipped by s_cbranch_execz faulted:\n" + noLanes.log); + RequireWords(noLanes, 0u, 0u, "guarded pointer: a read skipped by s_cbranch_execz"); + const auto oneLane = Run(device, ExecCode, 0u, 1u); + Require(Faulted(oneLane) && oneLane.log.find(Fault(0u, 0x34u)) != std::string::npos, "guarded pointer: a null pointer read with one EXEC lane did not fault:\n" + oneLane.log); + const auto emptyMapped = Run(device, EmptyExecCode, valid, 0u); + Require(!Faulted(emptyMapped), "guarded pointer: a mapped pointer read with an empty EXEC faulted:\n" + emptyMapped.log); + RequireWords(emptyMapped, 0u, 0u, "guarded pointer: a mapped pointer read with an empty EXEC"); + const auto emptyNull = Run(device, EmptyExecCode, 0u, 0u); + Require(Faulted(emptyNull) && emptyNull.log.find(Fault(0u, 0x34u)) != std::string::npos, "guarded pointer: a null pointer read that runs with an empty EXEC did not fault:\n" + emptyNull.log); + Require(noLanes.pipelineVariantId == someLanes.pipelineVariantId && oneLane.pipelineVariantId == someLanes.pipelineVariantId && emptyNull.pipelineVariantId == emptyMapped.pipelineVariantId, "guarded pointer: the poison changed an EXEC program's pipeline variant"); + + const auto chainSkipped = Run(device, ChainCode, 0u, 0u); + Require(chainSkipped.poisoned == 3u && Faulted(chainSkipped) && chainSkipped.log.find(Fault(0u, 0x24u)) != std::string::npos, "guarded pointer: a pointer read through a null pointer, used only by a skipped read, did not fault at its own pc:\n" + chainSkipped.log); + const auto chainTaken = Run(device, ChainCode, 0u, 1u); + Require(Faulted(chainTaken) && chainTaken.log.find(Fault(0u, 0x24u)) != std::string::npos, "guarded pointer: a read through a pointer read through a null pointer did not fault at the first read:\n" + chainTaken.log); + Indirect[0] = valid; + const auto chainMapped = Run(device, ChainCode, static_cast(reinterpret_cast(Indirect.data())), 1u); + Require(!Faulted(chainMapped) && chainMapped.poisoned == 0u && chainMapped.cacheable && chainMapped.pipelineVariantId == chainTaken.pipelineVariantId && chainSkipped.pipelineVariantId == chainTaken.pipelineVariantId, "guarded pointer: a chain's poison changed its pipeline variant, or a mapped chain faulted:\n" + chainMapped.log); + RequireWords(chainMapped, Lanes, Payload, "guarded pointer: a read through a mapped pointer chain"); + + Table = {0u, 0u, 0u, 0u, static_cast(valid), static_cast((valid >> 32u) & 0xffffu), static_cast(sizeof(Data)), 0x30027facu}; + const auto table = static_cast(reinterpret_cast(Table.data())); + const auto descriptorTaken = Run(device, DescriptorCode, table, 1u); + Require(!Faulted(descriptorTaken) && descriptorTaken.poisoned == 0u && descriptorTaken.cacheable, "guarded pointer: a V# behind a mapped pointer faulted, was poisoned or left the caches:\n" + descriptorTaken.log); + RequireWords(descriptorTaken, Lanes, Payload, "guarded pointer: a buffer read through a V# behind a mapped pointer"); + const auto descriptorSkipped = Run(device, DescriptorCode, 0u, 0u); + Require(!Faulted(descriptorSkipped) && descriptorSkipped.poisoned == 4u && !descriptorSkipped.cacheable, "guarded pointer: a V# behind a branch that skips its null pointer faulted or stayed cacheable:\n" + descriptorSkipped.log); + RequireWords(descriptorSkipped, 0u, 0u, "guarded pointer: a skipped V# load through a null pointer"); + const auto descriptorNull = Run(device, DescriptorCode, 0u, 1u); + Require(descriptorNull.pipelineVariantId == descriptorSkipped.pipelineVariantId && Faulted(descriptorNull) && descriptorNull.log.find(Fault(0x10u, 0x2cu)) != std::string::npos, "guarded pointer: a V# load through a null pointer that runs did not fault at its pc and address:\n" + descriptorNull.log); + const auto descriptorUnmapped = Run(device, DescriptorCode, unmapped, 1u); + Require(descriptorUnmapped.pipelineVariantId == descriptorNull.pipelineVariantId && Faulted(descriptorUnmapped) && descriptorUnmapped.log.find(Fault(unmapped + 0x10u, 0x2cu)) != std::string::npos, "guarded pointer: a V# load through an unmapped pointer that runs did not fault at its address:\n" + descriptorUnmapped.log); + Table = {}; + const auto descriptorZero = Run(device, DescriptorCode, table, 1u); + Require(!Faulted(descriptorZero) && descriptorZero.poisoned == 0u && descriptorZero.cacheable && descriptorZero.pipelineVariantId == descriptorNull.pipelineVariantId, "guarded pointer: a zero V# read from mapped memory faulted, or its pipeline variant differs from a V# zeroed by poison:\n" + descriptorZero.log); + RequireWords(descriptorZero, Lanes, 0u, "guarded pointer: a buffer read through a zero V#"); + +#ifdef _WIN32 + VirtualFree(reserved, 0, MEM_RELEASE); +#else + munmap(reserved, 65536); +#endif +} + +} + +int main() { + try { + const auto device = OpenVulkanTestDevice(); + if (!device) return VulkanTestSkipped; + RunTests(*device); + std::puts("guarded null pointer tests passed"); + return 0; + } catch (const std::exception& error) { + std::cerr << error.what() << '\n'; + return 1; + } +} diff --git a/core/libs/prx/libSceAgcDriver/tests/execution/HostImportTeardown.cpp b/core/libs/prx/libSceAgcDriver/tests/execution/HostImportTeardown.cpp index 72095307c..fb95ceae9 100644 --- a/core/libs/prx/libSceAgcDriver/tests/execution/HostImportTeardown.cpp +++ b/core/libs/prx/libSceAgcDriver/tests/execution/HostImportTeardown.cpp @@ -35,7 +35,7 @@ public: #endif Require(block != nullptr, "host import teardown: cannot allocate the guest block"); std::memset(block, 0, BlockBytes); - GuestAllocations::Mutation().Add(block, BlockBytes, true, true); + GuestAllocations::Mutation().Add(block, BlockBytes, true, true, true); } ~GuestBlock() { diff --git a/core/libs/prx/libSceAgcDriver/tests/execution/ImageBy.cpp b/core/libs/prx/libSceAgcDriver/tests/execution/ImageBy.cpp index fccae5c4f..47e0ea034 100644 --- a/core/libs/prx/libSceAgcDriver/tests/execution/ImageBy.cpp +++ b/core/libs/prx/libSceAgcDriver/tests/execution/ImageBy.cpp @@ -166,7 +166,7 @@ int main() { try { const auto device = OpenVulkanTestDevice(); if (!device) return VulkanTestSkipped; - GuestAllocations::Mutation().Add(Texels.data(), Bytes, true, true); + GuestAllocations::Mutation().Add(Texels.data(), Bytes, true, true, true); constexpr Format r8{R8UInt, 1u, 1u}; constexpr Format r16{R16UInt, 2u, 1u}; constexpr Format rg8{Rg8UInt, 1u, 2u}; diff --git a/core/libs/prx/libSceAgcDriver/tests/execution/ImageConvertedUnorm.cpp b/core/libs/prx/libSceAgcDriver/tests/execution/ImageConvertedUnorm.cpp index a74578e4d..6c036012f 100644 --- a/core/libs/prx/libSceAgcDriver/tests/execution/ImageConvertedUnorm.cpp +++ b/core/libs/prx/libSceAgcDriver/tests/execution/ImageConvertedUnorm.cpp @@ -103,7 +103,7 @@ public: block = static_cast(std::aligned_alloc(BlockBytes, BlockBytes)); #endif Require(block != nullptr, "image converted unorm: cannot allocate the guest block"); - GuestAllocations::Mutation().Add(block, BlockBytes, true, true); + GuestAllocations::Mutation().Add(block, BlockBytes, true, true, true); } ~GuestBlock() { diff --git a/core/libs/prx/libSceAgcDriver/tests/execution/ImagePckN.cpp b/core/libs/prx/libSceAgcDriver/tests/execution/ImagePckN.cpp index b16b20847..c080c817d 100644 --- a/core/libs/prx/libSceAgcDriver/tests/execution/ImagePckN.cpp +++ b/core/libs/prx/libSceAgcDriver/tests/execution/ImagePckN.cpp @@ -170,7 +170,7 @@ int main() { try { const auto device = OpenVulkanTestDevice(); if (!device) return VulkanTestSkipped; - GuestAllocations::Mutation().Add(Texels.data(), Bytes, true, true); + GuestAllocations::Mutation().Add(Texels.data(), Bytes, true, true, true); constexpr Format r8{R8UInt, 1u, 1u}; constexpr Format r16{R16UInt, 2u, 1u}; constexpr Format rg8{Rg8UInt, 1u, 2u}; diff --git a/core/libs/prx/libSceAgcDriver/tests/execution/ImagePipeBankXor.cpp b/core/libs/prx/libSceAgcDriver/tests/execution/ImagePipeBankXor.cpp new file mode 100644 index 000000000..caf7f8f01 --- /dev/null +++ b/core/libs/prx/libSceAgcDriver/tests/execution/ImagePipeBankXor.cpp @@ -0,0 +1,153 @@ +#include "prx/libSceAgcDriver/Execution/include/VulkanDevice.hpp" +#include "prx/libSceAgcDriver/Graphics/include/Draw.hpp" +#include "prx/libSceAgcDriver/Graphics/include/TextureSwizzleEquations.hpp" +#include "prx/libSceAgcDriver/Graphics/include/TextureTiling.hpp" +#include "Recompiler.hpp" +#include "VulkanTestDevice.hpp" +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace { + +using namespace AgcDriver::Graphics; +using ShaderRecompiler::ShaderStage; + +constexpr std::uint32_t Threads = 32; +constexpr std::uint32_t Width = 160; +constexpr std::uint32_t Height = 70; +constexpr std::uint32_t Format8888UNorm = 56; +constexpr std::uint32_t Type2D = 9; + +alignas(256) std::array Input{}; +alignas(256) std::array Output{}; +constexpr std::size_t TexelBytes = 1u << 20u; +alignas(4096) std::array TexelStorage{}; +const std::span Texels(TexelStorage.data() + ((0u - reinterpret_cast(TexelStorage.data())) & 0xffffu), TexelBytes); + +alignas(256) constexpr std::array Code{ + 0x34020084, 0xe0381000, 0x80000201, 0xbf8c3f70, 0xf0001f08, 0x00020602, 0xbf8c3f70, 0xe0781000, + 0x80010601, 0xbf810000, +}; + +std::array BufferDescriptor(const void* data, std::uint32_t bytes) { + const auto address = reinterpret_cast(data); + return {static_cast(address), static_cast((address >> 32u) & 0xffffu), bytes, 0x31016facu}; +} + +std::array TextureDescriptor(std::uint32_t tileMode, std::uint32_t pipeBankXor) { + const auto address = static_cast(reinterpret_cast(Texels.data())) + pipeBankXor; + return { + static_cast(address >> 8u), + static_cast((address >> 40u) & 0xffu) | (Format8888UNorm << 20u) | (((Width - 1u) & 3u) << 30u), + ((Width - 1u) >> 2u) | ((Height - 1u) << 14u), + 0xfacu | (tileMode << 20u) | (Type2D << 28u), + 0u, 0u, 0u, 0u, + }; +} + +std::array Texel(std::uint32_t x, std::uint32_t y) { + return {static_cast(x + 1u), static_cast(y * 3u + 2u), static_cast((x ^ y) & 0xffu), 0xff}; +} + +std::uint64_t EquationOffset(const TextureSwizzleEquation& equation, std::uint32_t x, std::uint32_t y) { + std::uint64_t offset = 0; + for (std::uint32_t bit = 0; bit < 16; ++bit) { + const auto mask = equation.bits[bit]; + const auto selected = (x & (mask & 0xfffu)) ^ ((y << 12) & (mask & 0xfff000u)); + offset |= static_cast(std::popcount(selected) & 1u) << bit; + } + return offset; +} + +void FillTexels(std::uint32_t tileMode, std::uint32_t pipeBankXor) { + std::ranges::fill(Texels, std::uint8_t{0}); + const auto descriptor = DecodeTextureResource(TextureDescriptor(tileMode, pipeBankXor)); + Require(descriptor.baseAddress == reinterpret_cast(Texels.data()) && descriptor.pipeBankXor == pipeBankXor, "the descriptor base did not split into an aligned base and its pipe/bank XOR"); + const auto geometry = DescribeSurface(descriptor); + Require(geometry.guestBytes <= Texels.size(), "texture does not fit the test allocation"); + const auto* equation = FindTextureSwizzleEquation(XorSwizzleMode(descriptor.tileMode), 4u); + Require(equation != nullptr, "swizzle equation is missing"); + const auto& mip = geometry.mips.at(0); + const auto layout = ThinBlockLayout(descriptor.tileMode, 4u); + for (std::uint32_t y = 0; y < Height; ++y) { + for (std::uint32_t x = 0; x < Width; ++x) { + auto offset = mip.tiledOffset; + if (mip.tail) offset += EquationOffset(*equation, x + mip.tailX, y + mip.tailY) ^ pipeBankXor; + else offset += (static_cast(y / layout[2]) * mip.blocksPerRow + x / layout[1]) * layout[0] + (EquationOffset(*equation, x, y) ^ pipeBankXor); + const auto texel = Texel(x, y); + std::copy(texel.begin(), texel.end(), Texels.begin() + static_cast(offset)); + } + } +} + +void FillInput() { + for (std::uint32_t tid = 0; tid < Threads; ++tid) { + Input[tid * 4u + 0u] = (tid * 37u + 11u) % Width; + Input[tid * 4u + 1u] = (tid * 23u + 5u) % Height; + Input[tid * 4u + 2u] = 0u; + Input[tid * 4u + 3u] = 0u; + } + Input[0] = 0u; Input[1] = 0u; + Input[4] = Width - 1u; Input[5] = Height - 1u; + Input[8] = 33u; Input[9] = 17u; +} + +void Run(AgcDriver::VulkanDevice& device, std::uint32_t tileMode, std::uint32_t pipeBankXor, const char* name) { + FillTexels(tileMode, pipeBankXor); + Output.fill(-1.0f); + std::vector userData(16, 0u); + const auto input = BufferDescriptor(Input.data(), static_cast(sizeof(Input))); + const auto output = BufferDescriptor(Output.data(), static_cast(sizeof(Output))); + const auto texture = TextureDescriptor(tileMode, pipeBankXor); + std::copy(input.begin(), input.end(), userData.begin()); + std::copy(output.begin(), output.end(), userData.begin() + 4); + std::copy(texture.begin(), texture.end(), userData.begin() + 8); + const std::span code(Code); + const std::array memory{{{reinterpret_cast(code.data()), std::as_bytes(code)}}}; + const ShaderRecompiler::ShaderComputeStageInfo compute{{Threads, 1, 1}, 0u, {false, false, false}, false, 1}; + ShaderRecompiler::RecompileRequest request{ + {ShaderStage::Compute, reinterpret_cast(code.data()), code, 0, {}}, + {32, 0, userData, compute, std::nullopt, std::nullopt, memory}, + device.Target(), + {0, 0, 0, 128} + }; + request.useCache = false; + const auto result = ShaderRecompiler::Recompile(request); + device.Dispatch(result, 1, 1, 1, {}, reinterpret_cast(code.data())); + device.WaitIdle(); + for (std::uint32_t tid = 0; tid < Threads; ++tid) { + const auto expected = Texel(Input[tid * 4u], Input[tid * 4u + 1u]); + for (std::uint32_t component = 0; component < 4u; ++component) { + const float scaled = Output[tid * 4u + component] * 255.0f; + Require(std::fabs(scaled - std::round(scaled)) < 1e-3f && std::lround(scaled) == expected[component], std::string(name) + ": texel (" + std::to_string(Input[tid * 4u]) + ", " + std::to_string(Input[tid * 4u + 1u]) + ") component " + std::to_string(component) + " is " + std::to_string(scaled) + "/255, expected " + std::to_string(expected[component]) + "/255"); + } + } +} + +} + +int main() { + try { + const auto device = OpenVulkanTestDevice(); + if (!device) return VulkanTestSkipped; + FillInput(); + Run(*device, 0x16u, 0x000u, "SW_4KB_D_X without a pipe/bank XOR"); + Run(*device, 0x16u, 0xa00u, "SW_4KB_D_X with pipe/bank XOR 0xa00"); + Run(*device, 0x19u, 0x5600u, "SW_64KB_S_X with pipe/bank XOR 0x5600"); + Run(*device, 0x1bu, 0xf100u, "SW_64KB_R_X with pipe/bank XOR 0xf100"); + std::puts("image pipe/bank XOR tests passed"); + return 0; + } catch (const std::exception& error) { + std::cerr << error.what() << '\n'; + return 1; + } +} diff --git a/core/libs/prx/libSceAgcDriver/tests/execution/ImagePlaneAddressOnLine.cpp b/core/libs/prx/libSceAgcDriver/tests/execution/ImagePlaneAddressOnLine.cpp new file mode 100644 index 000000000..4614998ce --- /dev/null +++ b/core/libs/prx/libSceAgcDriver/tests/execution/ImagePlaneAddressOnLine.cpp @@ -0,0 +1,198 @@ +#include "prx/libSceAgcDriver/Execution/include/VulkanDevice.hpp" +#include "prx/libSceAgcDriver/Graphics/include/Draw.hpp" +#include "prx/libSceAgcDriver/Graphics/include/TextureTiling.hpp" +#include "Recompiler.hpp" +#include "VulkanTestDevice.hpp" +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace { + +using AgcDriver::Graphics::Require; +using ShaderRecompiler::ShaderStage; + +constexpr std::uint32_t Threads = 32; +constexpr std::uint32_t Words = 32; +constexpr std::uint32_t Size = 16; +constexpr std::uint32_t Format8888UNorm = 56; +constexpr std::uint32_t Type1D = 8; +constexpr std::uint32_t Type2D = 9; +constexpr std::uint32_t SampleResult = 8; +constexpr std::uint32_t LoadResult = 12; +alignas(256) std::array Buffer{}; +alignas(256) std::array Line{}; +alignas(256) std::array Plane{}; + +alignas(256) constexpr std::array Code{ + 0x34020087, 0xe0301000, 0x80000201, 0xe0301004, 0x80000301, 0xe0301008, 0x80000401, 0xe030100c, + 0x80000501, 0xe0301010, 0x80000601, 0xe0301014, 0x80000701, 0xbf8c3f70, 0xf09c0f08, 0x00610c02, + 0xf0001f08, 0x00011005, 0xbf8c3f70, 0xe0701020, 0x80000c01, 0xe0701024, 0x80000d01, 0xe0701028, + 0x80000e01, 0xe070102c, 0x80000f01, 0xe0701030, 0x80001001, 0xe0701034, 0x80001101, 0xe0701038, + 0x80001201, 0xe070103c, 0x80001301, 0xbf810000, +}; + +alignas(256) constexpr std::array ExplicitLodCode{ + 0x34020087, 0xe0301000, 0x80000201, 0xe0301004, 0x80000301, 0xe0301008, 0x80000401, 0xe030100c, + 0x80000501, 0xbf8c3f70, 0xf0900f08, 0x00610c02, 0xbf8c3f70, 0xe0701020, 0x80000c01, 0xbf810000, +}; + +constexpr std::array LineV{0.0f, 0.5f, 1.0f, 1.5f, -0.25f, 3.0f, 0.25f, 0.75f}; +constexpr std::array LineY{0u, 1u, 2u, 3u, 0xffffffffu, 7u, 0u, 5u}; + +std::uint32_t TexelOf(std::uint32_t tid) { + return (tid * 7u + 3u) % Size; +} + +std::uint32_t Bits(float value) { + return std::bit_cast(value); +} + +void FillInput(bool line) { + Buffer.fill(0xdeadbeefu); + for (std::uint32_t tid = 0; tid < Threads; ++tid) { + const auto x = TexelOf(tid); + const auto u = Bits((static_cast(x) + 0.5f) / static_cast(Size)); + const auto v = line ? Bits(LineV[tid % LineV.size()]) : Bits((static_cast((tid * 5u + 1u) % Size) + 0.5f) / static_cast(Size)); + const auto y = line ? LineY[tid % LineY.size()] : (tid * 5u + 1u) % Size; + const std::array words{u, v, 0u, x, y, 0u}; + std::copy(words.begin(), words.end(), &Buffer[tid * Words]); + } +} + +std::array LineDescriptor() { + const auto address = static_cast(reinterpret_cast(Line.data())); + return { + static_cast(address >> 8u), + static_cast((address >> 40u) & 0xffu) | (Format8888UNorm << 20u) | (((Size - 1u) & 3u) << 30u), + (Size - 1u) >> 2u, + 0xfacu | (Type1D << 28u), + 0u, + 0u, + 0u, + 0u, + }; +} + +std::array PlaneDescriptor() { + const auto address = static_cast(reinterpret_cast(Plane.data())); + return { + static_cast(address >> 8u), + static_cast((address >> 40u) & 0xffu) | (Format8888UNorm << 20u) | (((Size - 1u) & 3u) << 30u), + ((Size - 1u) >> 2u) | ((Size - 1u) << 14u), + 0xfacu | (Type2D << 28u), + 0u, + 0u, + 0u, + 0u, + }; +} + +void FillTexture(std::span storage, const std::array& descriptor, std::uint32_t rows) { + const auto geometry = AgcDriver::Graphics::DescribeSurface(AgcDriver::Graphics::DecodeTextureResource(descriptor)); + Require(geometry.guestBytes <= storage.size(), "image plane address on a line: the texture does not fit the texel storage"); + const auto& mip = geometry.mips.at(0); + std::fill(storage.begin(), storage.end(), 0xeeu); + for (std::uint32_t y = 0; y < rows; ++y) { + for (std::uint32_t x = 0; x < Size; ++x) { + auto* texel = &storage[geometry.GuestLayerOffset(0) + mip.tiledOffset + static_cast(y) * mip.pitchBytes + x * 4u]; + texel[0] = 77u; + texel[1] = static_cast(x); + texel[2] = static_cast(y); + texel[3] = 255u; + } + } +} + +std::array BufferDescriptor(const void* data, std::uint32_t bytes) { + const auto address = reinterpret_cast(data); + return {static_cast(address), static_cast((address >> 32u) & 0xffffu), bytes, 0x31016facu}; +} + +std::array SamplerDescriptor() { + return {0u, 0xfffu << 12u, 1u << 26u, 0u}; +} + +void Run(AgcDriver::VulkanDevice& device, std::span code, const std::array& texture) { + std::vector userData(16, 0u); + const auto buffer = BufferDescriptor(Buffer.data(), static_cast(Buffer.size() * 4u)); + const auto sampler = SamplerDescriptor(); + std::copy(buffer.begin(), buffer.end(), userData.begin()); + std::copy(texture.begin(), texture.end(), userData.begin() + 4); + std::copy(sampler.begin(), sampler.end(), userData.begin() + 12); + const std::array memory{{{reinterpret_cast(code.data()), std::as_bytes(code)}}}; + const ShaderRecompiler::ShaderComputeStageInfo compute{{Threads, 1, 1}, 0u, {false, false, false}, false, 1}; + ShaderRecompiler::RecompileRequest request{ + {ShaderStage::Compute, reinterpret_cast(code.data()), code, 0, {}}, + {32, 0, userData, compute, std::nullopt, std::nullopt, memory}, + device.Target(), + {0, 0, 0, 128} + }; + request.useCache = false; + const auto result = ShaderRecompiler::Recompile(request); + device.Dispatch(result, 1, 1, 1, {}, reinterpret_cast(code.data())); + device.WaitIdle(); +} + +void Expect(std::uint32_t tid, std::uint32_t result, const std::array& expected, const std::string& name) { + for (std::uint32_t component = 0; component < 4u; ++component) { + const float value = std::bit_cast(Buffer[tid * Words + result + component]) * 255.0f; + Require(std::lround(value) == static_cast(expected[component]), name + ": thread " + std::to_string(tid) + " component " + std::to_string(component) + " is " + std::to_string(value) + ", expected " + std::to_string(expected[component])); + } +} + +void CheckLine() { + for (std::uint32_t tid = 0; tid < Threads; ++tid) { + const std::array texel{77u, TexelOf(tid), 0u, 255u}; + Expect(tid, SampleResult, texel, "image_sample_lz 2d on a 1D texture with v " + std::to_string(LineV[tid % LineV.size()])); + Expect(tid, LoadResult, texel, "image_load 2d on a 1D texture with y " + std::to_string(static_cast(LineY[tid % LineY.size()]))); + } +} + +void CheckPlane() { + for (std::uint32_t tid = 0; tid < Threads; ++tid) { + const std::array texel{77u, TexelOf(tid), (tid * 5u + 1u) % Size, 255u}; + Expect(tid, SampleResult, texel, "image_sample_lz 2d on a 2D texture"); + Expect(tid, LoadResult, texel, "image_load 2d on a 2D texture"); + } +} + +void RequireRefused(AgcDriver::VulkanDevice& device) { + std::string refusal; + try { + Run(device, ExplicitLodCode, LineDescriptor()); + } catch (const std::exception& error) { + refusal = error.what(); + } + Require(refusal.find("incompatible with the static runtime image interface") != std::string::npos, "image_sample_l 2d on a 1D texture was not refused: " + refusal); +} + +} + +int main() { + try { + const auto device = OpenVulkanTestDevice(); + if (!device) return VulkanTestSkipped; + FillTexture(Line, LineDescriptor(), 1u); + FillInput(true); + Run(*device, Code, LineDescriptor()); + CheckLine(); + FillTexture(Plane, PlaneDescriptor(), Size); + FillInput(false); + Run(*device, Code, PlaneDescriptor()); + CheckPlane(); + RequireRefused(*device); + std::puts("image plane address on line tests passed"); + return 0; + } catch (const std::exception& error) { + std::cerr << error.what() << '\n'; + return 1; + } +} diff --git a/core/libs/prx/libSceAgcDriver/tests/execution/ImageSampleAnisoOverride.cpp b/core/libs/prx/libSceAgcDriver/tests/execution/ImageSampleAnisoOverride.cpp new file mode 100644 index 000000000..401c860e3 --- /dev/null +++ b/core/libs/prx/libSceAgcDriver/tests/execution/ImageSampleAnisoOverride.cpp @@ -0,0 +1,189 @@ +#include "prx/libSceAgcDriver/Execution/include/VulkanDevice.hpp" +#include "prx/libSceAgcDriver/Graphics/include/Draw.hpp" +#include "prx/libSceAgcDriver/Graphics/include/TextureTiling.hpp" +#include "Recompiler.hpp" +#include "VulkanTestDevice.hpp" +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace { + +using AgcDriver::Graphics::Require; +using ShaderRecompiler::ShaderStage; + +constexpr std::uint32_t Threads = 32; +constexpr std::uint32_t Size = 8; +constexpr std::uint32_t Levels = 4; +constexpr std::uint32_t Format32SInt = 21; +constexpr std::uint32_t Format8888UNorm = 56; +constexpr std::uint32_t Type2D = 9; +constexpr std::uint32_t SwizzleX111 = 0x24cu; +constexpr std::uint32_t IdentitySwizzle = 0xfacu; +constexpr std::uint32_t SampleX = 3; +constexpr std::uint32_t SampleY = 5; +constexpr float Tolerance = 1.0f / 64.0f; +constexpr std::array AnisoSampler{0x00000092u | (4u << 9u), 0x00fff000u, (3u << 20u) | (3u << 22u), 0u}; +constexpr std::uint32_t AnisoOverride = 1u << 29u; + +alignas(256) std::array Output{}; +using Storage = std::array; +alignas(256) Storage SingleIntegers{}; +alignas(256) Storage SingleColors{}; +alignas(256) Storage MipIntegers{}; +alignas(256) Storage MipColors{}; + +alignas(256) constexpr std::array Code{ + 0x34060085u, 0x7e2802ffu, 0x3ef00000u, 0x7e2a02ffu, 0x3f300000u, 0xf09c0f08u, 0x00610a14u, 0xf09c0f08u, + 0x00640e14u, 0xbf8c3f70u, 0xe0781000u, 0x80000a03u, 0xe0781010u, 0x80000e03u, 0xbf810000u, 0xbf810000u, +}; + +struct Levelled { + std::uint32_t lastLevel; + std::uint32_t maxMip; +}; + +constexpr Levelled SingleLevel{0, 0}; +constexpr Levelled Mipmapped{Levels - 1u, Levels - 1u}; +constexpr Levelled ClampedToOneLevel{Levels - 1u, 0}; + +std::string Hex(std::uint32_t value) { + char text[16]; + std::snprintf(text, sizeof(text), "0x%08x", value); + return text; +} + +AgcDriver::Graphics::TileMipLayout BaseLevel(std::uint32_t format, std::uint32_t levels) { + const auto mips = AgcDriver::Graphics::ComputeMipLayout(AgcDriver::Graphics::TextureTileMode::kLinear, format, Size, Size, levels); + Require(AgcDriver::Graphics::ComputeSurfaceSize(mips, 1) <= Storage{}.size(), "image sample aniso override: a texture does not fit its storage"); + return mips[0]; +} + +std::int32_t Integer(std::uint32_t x, std::uint32_t y) { + return -static_cast(100u * x + 7u * y + 1u); +} + +void Fill(Storage& integers, Storage& colors, std::uint32_t levels) { + const auto integerLevel = BaseLevel(Format32SInt, levels); + const auto colorLevel = BaseLevel(Format8888UNorm, levels); + integers.fill(0x5au); + colors.fill(0x5au); + for (std::uint32_t y = 0; y < Size; ++y) { + for (std::uint32_t x = 0; x < Size; ++x) { + const auto bits = static_cast(Integer(x, y)); + auto* integer = &integers[integerLevel.linearOffset + static_cast(y) * integerLevel.pitchBytes + x * 4u]; + for (std::uint32_t byte = 0; byte < 4u; ++byte) integer[byte] = static_cast(bits >> (byte * 8u)); + auto* color = &colors[colorLevel.linearOffset + static_cast(y) * colorLevel.pitchBytes + x * 4u]; + color[0] = x > SampleX ? 255u : 0u; + color[1] = 0u; + color[2] = 0u; + color[3] = 255u; + } + } +} + +std::array BufferDescriptor(const void* data, std::uint32_t bytes) { + const auto address = reinterpret_cast(data); + return {static_cast(address), static_cast((address >> 32u) & 0xffffu), bytes, 0x31016facu}; +} + +std::array TextureDescriptor(const void* texels, std::uint32_t format, std::uint32_t swizzle, Levelled levels) { + const auto address = static_cast(reinterpret_cast(texels)); + return { + static_cast(address >> 8u), + static_cast((address >> 40u) & 0xffu) | (format << 20u) | (((Size - 1u) & 3u) << 30u), + ((Size - 1u) >> 2u) | ((Size - 1u) << 14u), + swizzle | (levels.lastLevel << 16u) | (Type2D << 28u), + 0u, + levels.maxMip << 4u, + 0u, + 0u, + }; +} + +std::optional Run(AgcDriver::VulkanDevice& device, Levelled first, Levelled second, std::uint32_t samplerFlags) { + Output.fill(0xdeadbeefu); + std::vector userData(24, 0u); + const auto output = BufferDescriptor(Output.data(), static_cast(Output.size() * 4u)); + const auto integers = TextureDescriptor(first.maxMip == 0 ? SingleIntegers.data() : MipIntegers.data(), Format32SInt, SwizzleX111, first); + const auto colors = TextureDescriptor(second.maxMip == 0 ? SingleColors.data() : MipColors.data(), Format8888UNorm, IdentitySwizzle, second); + auto sampler = AnisoSampler; + sampler[2] |= samplerFlags; + std::copy(output.begin(), output.end(), userData.begin()); + std::copy(integers.begin(), integers.end(), userData.begin() + 4); + std::copy(sampler.begin(), sampler.end(), userData.begin() + 12); + std::copy(colors.begin(), colors.end(), userData.begin() + 16); + const std::span code(Code); + const std::array memory{{{reinterpret_cast(code.data()), std::as_bytes(code)}}}; + const ShaderRecompiler::ShaderComputeStageInfo compute{{Threads, 1, 1}, 0u, {true, false, false}, false, 1}; + ShaderRecompiler::RecompileRequest request{ + {ShaderStage::Compute, reinterpret_cast(code.data()), code, 0, {}}, + {32, 0, userData, compute, std::nullopt, std::nullopt, memory}, + device.Target(), + {0, 0, 0, 128} + }; + request.useCache = false; + try { + const auto result = ShaderRecompiler::Recompile(request); + device.Dispatch(result, 1, 1, 1, {}, reinterpret_cast(code.data())); + device.WaitIdle(); + } catch (const std::runtime_error& error) { + return std::string(error.what()); + } + return std::nullopt; +} + +void CheckSamples(const std::string& what) { + const std::array integer{static_cast(Integer(SampleX, SampleY)), 1u, 1u, 1u}; + const std::array color{0.25f, 0.0f, 0.0f, 1.0f}; + for (std::uint32_t lane = 0; lane < Threads; ++lane) { + for (std::uint32_t component = 0; component < 4u; ++component) { + const auto actual = Output[lane * 8u + component]; + Require(actual == integer[component], what + ": 32_SINT lane " + std::to_string(lane) + " component " + std::to_string(component) + " is " + Hex(actual) + ", expected " + Hex(integer[component])); + const auto bits = Output[lane * 8u + 4u + component]; + Require(std::fabs(std::bit_cast(bits) - color[component]) <= Tolerance, what + ": 8_8_8_8_UNORM lane " + std::to_string(lane) + " component " + std::to_string(component) + " is " + Hex(bits) + ", expected " + std::to_string(color[component])); + } + } +} + +void ExpectRuns(AgcDriver::VulkanDevice& device, const std::string& what, Levelled first, Levelled second, std::uint32_t samplerFlags) { + const auto error = Run(device, first, second, samplerFlags); + Require(!error.has_value(), what + " was refused: " + error.value_or("")); + CheckSamples(what); +} + +void ExpectMixedRefusal(AgcDriver::VulkanDevice& device, const std::string& what, Levelled first, Levelled second) { + const auto error = Run(device, first, second, AnisoOverride); + Require(error.has_value() && error->find("both single-level and mipmapped") != std::string::npos, what + " was not refused as a mixed ANISO_OVERRIDE pairing" + (error.has_value() ? ": " + *error : std::string())); +} + +} + +int main() { + try { + const auto device = OpenVulkanTestDevice(); + if (!device) return VulkanTestSkipped; + Fill(SingleIntegers, SingleColors, 1u); + Fill(MipIntegers, MipColors, Levels); + ExpectRuns(*device, "an ANISO_OVERRIDE sampler shared by two single-level images", SingleLevel, SingleLevel, AnisoOverride); + ExpectRuns(*device, "an ANISO_OVERRIDE sampler shared by two mipmapped images", Mipmapped, Mipmapped, AnisoOverride); + ExpectRuns(*device, "an ANISO_OVERRIDE sampler shared by a mipmapped image and one whose LAST_LEVEL is clamped to MAX_MIP 0", ClampedToOneLevel, Mipmapped, AnisoOverride); + ExpectRuns(*device, "an anisotropic sampler without ANISO_OVERRIDE shared by a single-level and a mipmapped image", SingleLevel, Mipmapped, 0u); + ExpectMixedRefusal(*device, "an ANISO_OVERRIDE sampler shared by a single-level and a mipmapped image", SingleLevel, Mipmapped); + ExpectMixedRefusal(*device, "an ANISO_OVERRIDE sampler shared by a single-level image and one whose LAST_LEVEL is clamped to MAX_MIP 0", SingleLevel, ClampedToOneLevel); + std::puts("image sample aniso override tests passed"); + return 0; + } catch (const std::exception& error) { + std::cerr << error.what() << '\n'; + return 1; + } +} diff --git a/core/libs/prx/libSceAgcDriver/tests/execution/ImageSampleForceDegamma.cpp b/core/libs/prx/libSceAgcDriver/tests/execution/ImageSampleForceDegamma.cpp new file mode 100644 index 000000000..965994506 --- /dev/null +++ b/core/libs/prx/libSceAgcDriver/tests/execution/ImageSampleForceDegamma.cpp @@ -0,0 +1,171 @@ +#include "prx/libSceAgcDriver/Execution/include/VulkanDevice.hpp" +#include "prx/libSceAgcDriver/Graphics/include/TextureTiling.hpp" +#include "Recompiler.hpp" +#include "VulkanTestDevice.hpp" +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace { + +using AgcDriver::Graphics::Require; +using ShaderRecompiler::ShaderStage; + +constexpr std::uint32_t Threads = 32; +constexpr std::uint32_t Size = 4; +constexpr std::uint32_t Format8888UNorm = 56; +constexpr std::uint32_t Format8888Srgb = 130; +constexpr std::uint32_t Type2D = 9; +constexpr std::uint32_t FilterPoint = 0; +constexpr std::uint32_t FilterBilinear = 1; +constexpr std::uint32_t ForceDegamma = 1u << 20u; + +alignas(256) std::array Input{}; +alignas(256) std::array Output{}; +alignas(4096) std::array Texels{}; + +alignas(256) constexpr std::array Code{ + 0x1614008c, 0xe03c1000, 0x8000010a, 0xbf8c3f70, 0xf0900f08, 0x00820401, 0xbf8c3f70, + 0x34160084, 0xe0781000, 0x8001040b, 0xbf810000, 0xbf810000, 0xbf810000, +}; + +std::array BufferDescriptor(const void* data, std::uint32_t bytes) { + const auto address = reinterpret_cast(data); + return {static_cast(address), static_cast((address >> 32u) & 0xffffu), bytes, 0x31016facu}; +} + +std::array TextureDescriptor(std::uint32_t format) { + const auto address = static_cast(reinterpret_cast(Texels.data())); + return { + static_cast(address >> 8u), + static_cast((address >> 40u) & 0xffu) | (format << 20u) | (((Size - 1u) & 3u) << 30u), + ((Size - 1u) >> 2u) | ((Size - 1u) << 14u), + 0xfacu | (Type2D << 28u), + 0u, + 0u, + 0u, + 0u, + }; +} + +std::array SamplerDescriptor(std::uint32_t filter, std::uint32_t forced) { + return {0x92u | forced, 0x00fff000u, (filter << 20u) | (filter << 22u) | (1u << 24u), 0u}; +} + +std::uint8_t Code8(std::uint32_t x, std::uint32_t y, std::uint32_t channel) { + return static_cast((y * 61u + x * 17u + channel * 97u + 5u) % 256u); +} + +void FillTexels() { + const auto mips = AgcDriver::Graphics::ComputeMipLayout(AgcDriver::Graphics::TextureTileMode::kLinear, Format8888Srgb, Size, Size, 1); + Require(AgcDriver::Graphics::ComputeSurfaceSize(mips, 1) <= Texels.size(), "image sample force degamma: the texture does not fit its storage"); + Texels.fill(0); + for (std::uint32_t y = 0; y < Size; ++y) { + for (std::uint32_t x = 0; x < Size; ++x) { + auto* texel = &Texels[mips[0].tiledOffset + static_cast(y) * mips[0].pitchBytes + x * 4u]; + for (std::uint32_t channel = 0; channel < 4u; ++channel) texel[channel] = Code8(x, y, channel); + } + } +} + +void FillInput(bool centers) { + for (std::uint32_t tid = 0; tid < Threads; ++tid) { + const auto x = tid % Size; + const auto y = (tid / Size) % Size; + Input[tid * 3u + 0u] = centers ? (static_cast(x) + 0.5f) / static_cast(Size) : (static_cast((tid * 5u + 1u) % 13u) + 0.3f) / 13.0f; + Input[tid * 3u + 1u] = centers ? (static_cast(y) + 0.5f) / static_cast(Size) : (static_cast((tid * 7u + 3u) % 13u) + 0.6f) / 13.0f; + Input[tid * 3u + 2u] = 0.0f; + } +} + +float SrgbToLinear(std::uint8_t code) { + const float c = static_cast(code) / 255.0f; + return c <= 0.04045f ? c / 12.92f : std::pow((c + 0.055f) / 1.055f, 2.4f); +} + +std::array Sample(AgcDriver::VulkanDevice& device, std::uint32_t format, std::uint32_t filter, std::uint32_t forced) { + std::vector userData(20, 0u); + const auto input = BufferDescriptor(Input.data(), static_cast(sizeof(Input))); + const auto output = BufferDescriptor(Output.data(), static_cast(sizeof(Output))); + const auto texture = TextureDescriptor(format); + const auto sampler = SamplerDescriptor(filter, forced); + std::copy(input.begin(), input.end(), userData.begin()); + std::copy(output.begin(), output.end(), userData.begin() + 4); + std::copy(texture.begin(), texture.end(), userData.begin() + 8); + std::copy(sampler.begin(), sampler.end(), userData.begin() + 16); + const std::array memory{{{reinterpret_cast(Code.data()), std::as_bytes(std::span(Code))}}}; + const ShaderRecompiler::ShaderComputeStageInfo compute{{Threads, 1, 1}, 0u, {false, false, false}, false, 1}; + ShaderRecompiler::RecompileRequest request{ + {ShaderStage::Compute, reinterpret_cast(Code.data()), Code, 0, {}}, + {32, 0, userData, compute, std::nullopt, std::nullopt, memory}, + device.Target(), + {0, 0, 0, 128} + }; + request.useCache = false; + const auto result = ShaderRecompiler::Recompile(request); + Output.fill(-1.0f); + device.Dispatch(result, 1, 1, 1, {}, reinterpret_cast(Code.data())); + device.WaitIdle(); + return Output; +} + +void RequireSame(const std::array& forced, const std::array& plain, const char* name) { + for (std::uint32_t index = 0; index < forced.size(); ++index) { + Require(std::bit_cast(forced[index]) == std::bit_cast(plain[index]), std::string(name) + ": thread " + std::to_string(index / 4u) + " component " + std::to_string(index % 4u) + " sampled " + std::to_string(forced[index]) + " through FORCE_DEGAMMA, " + std::to_string(plain[index]) + " without it"); + } +} + +void RequireDecoded(const std::array& sampled) { + for (std::uint32_t tid = 0; tid < Threads; ++tid) { + const auto x = tid % Size; + const auto y = (tid / Size) % Size; + for (std::uint32_t channel = 0; channel < 4u; ++channel) { + const auto code = Code8(x, y, channel); + const float expected = channel == 3u ? static_cast(code) / 255.0f : SrgbToLinear(code); + const float value = sampled[tid * 4u + channel]; + Require(std::fabs(value - expected) <= 0.005f, "8_8_8_8_SRGB through FORCE_DEGAMMA: thread " + std::to_string(tid) + " component " + std::to_string(channel) + " sampled " + std::to_string(value) + ", expected " + std::to_string(expected)); + } + } +} + +void Reject(AgcDriver::VulkanDevice& device, std::uint32_t format, std::string_view reason) { + try { + static_cast(Sample(device, format, FilterPoint, ForceDegamma)); + } catch (const std::exception& error) { + Require(std::string_view(error.what()).find(reason) != std::string_view::npos, std::string("unexpected rejection: ") + error.what()); + return; + } + Require(false, std::string("expected rejection: ") + std::string(reason)); +} + +} + +int main() { + try { + const auto device = OpenVulkanTestDevice(); + if (!device) return VulkanTestSkipped; + FillTexels(); + FillInput(true); + const auto forcedPoint = Sample(*device, Format8888Srgb, FilterPoint, ForceDegamma); + RequireDecoded(forcedPoint); + RequireSame(forcedPoint, Sample(*device, Format8888Srgb, FilterPoint, 0u), "8_8_8_8_SRGB, point"); + FillInput(false); + const auto forcedBilinear = Sample(*device, Format8888Srgb, FilterBilinear, ForceDegamma); + RequireSame(forcedBilinear, Sample(*device, Format8888Srgb, FilterBilinear, 0u), "8_8_8_8_SRGB, bilinear"); + Reject(*device, Format8888UNorm, "guest format 56, which is not sRGB, is sampled through a sampler that forces sRGB decoding"); + std::puts("image sample force degamma tests passed"); + return 0; + } catch (const std::exception& error) { + std::cerr << error.what() << '\n'; + return 1; + } +} diff --git a/core/libs/prx/libSceAgcDriver/tests/execution/ImageSampleUnnormalized.cpp b/core/libs/prx/libSceAgcDriver/tests/execution/ImageSampleUnnormalized.cpp index cc0385031..4d1274912 100644 --- a/core/libs/prx/libSceAgcDriver/tests/execution/ImageSampleUnnormalized.cpp +++ b/core/libs/prx/libSceAgcDriver/tests/execution/ImageSampleUnnormalized.cpp @@ -28,7 +28,12 @@ constexpr std::int32_t Width = 13; constexpr std::int32_t Height = 7; constexpr std::uint32_t StorageLevels = 4; constexpr std::uint32_t Format8888UNorm = 56; +constexpr std::uint32_t Format32UInt = 20; +constexpr std::uint32_t Format32SInt = 21; +constexpr std::uint32_t Type1D = 8; constexpr std::uint32_t Type2D = 9; +constexpr std::uint32_t Type3D = 10; +constexpr std::uint32_t OneZeroOneZero = 0x041u; constexpr std::uint8_t OtherLevelTexel = 0x40u; alignas(256) std::array Buffer{}; alignas(256) std::array SingleLevel{}; @@ -44,6 +49,14 @@ alignas(256) constexpr std::array Code{ constexpr std::array Instructions{"image_sample_lz", "image_sample_l 2.7", "image_sample"}; +alignas(256) constexpr std::array ConstantCode{ + 0x34020086, 0xe0301000, 0x80000201, 0xe0301004, 0x80000301, 0xbf8c3f70, 0xf09c0608, 0x00610802, + 0xf11c0208, 0x00610a02, 0xf11c0408, 0x00610e02, 0xbf8c3f70, 0xe0701010, 0x80000801, 0xe0701014, + 0x80000901, 0xe0701018, 0x80000a01, 0xe070101c, 0x80000b01, 0xe0701020, 0x80000c01, 0xe0701024, + 0x80000d01, 0xe0701028, 0x80000e01, 0xe070102c, 0x80000f01, 0xe0701030, 0x80001001, 0xe0701034, + 0x80001101, 0xbf810000, +}; + struct SamplerCase { const char* name; std::array words; @@ -107,6 +120,12 @@ void FillTexture(std::span texels, std::uint32_t levels) { } } +std::string Hex(std::uint32_t value) { + char text[16]; + std::snprintf(text, sizeof(text), "0x%08x", value); + return text; +} + std::array BufferDescriptor(const void* data, std::uint32_t bytes) { const auto address = reinterpret_cast(data); return {static_cast(address), static_cast((address >> 32u) & 0xffffu), bytes, 0x31016facu}; @@ -128,7 +147,7 @@ std::array TextureDescriptor(const void* data, std::uint32_t l using Samples = std::vector>; -Samples Run(AgcDriver::VulkanDevice& device, const std::array& texture, const std::array& sampler, const std::vector& coordinates) { +Samples Run(AgcDriver::VulkanDevice& device, const std::array& texture, const std::array& sampler, const std::vector& coordinates, std::span code = Code) { Samples samples; for (std::size_t first = 0; first < coordinates.size(); first += Threads) { Buffer.fill(0xdeadbeefu); @@ -142,7 +161,6 @@ Samples Run(AgcDriver::VulkanDevice& device, const std::array& std::copy(buffer.begin(), buffer.end(), userData.begin()); std::copy(texture.begin(), texture.end(), userData.begin() + 4); std::copy(sampler.begin(), sampler.end(), userData.begin() + 12); - const std::span code(Code); const std::array memory{{{reinterpret_cast(code.data()), std::as_bytes(code)}}}; const ShaderRecompiler::ShaderComputeStageInfo compute{{Threads, 1, 1}, 0u, {false, false, false}, false, 1}; ShaderRecompiler::RecompileRequest request{ @@ -231,6 +249,43 @@ int main() { ExpectFailure(*device, TextureDescriptor(MultiLevel.data(), 2u, 2u), Samplers[0].words, "single-level", "a 3-level view"); ExpectFailure(*device, single, {0x00008092u, 0x00fff000u, 0x05100000u, 0u}, "different minification", "unequal minification and magnification filters"); ExpectFailure(*device, single, {0x00008090u, 0x00fff000u, 0x05500000u, 0u}, "clamp mode 0", "wrap on X"); + const auto constantView = [&](std::uint32_t type, std::uint32_t format, std::uint32_t swizzle) { + auto descriptor = TextureDescriptor(SingleLevel.data(), 0u, 0u); + descriptor[1] = (descriptor[1] & ~(0x1ffu << 20u)) | (format << 20u); + descriptor[3] = swizzle | (type << 28u); + return descriptor; + }; + for (const auto& sampler : Samplers) { + for (const auto& sample : Run(*device, constantView(Type3D, Format8888UNorm, OneZeroOneZero), sampler.words, coordinates)) { + for (std::uint32_t index = 0; index < Results; ++index) { + const std::uint32_t expected = index % 2u == 0u ? 0x3f800000u : 0u; + Require(sample[index] == expected, std::string(sampler.name) + ": a 3D view whose channels select (1, 0, 1, 0) returned " + Hex(sample[index]) + " for result " + std::to_string(index)); + } + } + } + struct ConstantCase { + const char* name; + std::array texture; + std::uint32_t one; + }; + const std::array constants{{ + {"a 3D UNORM view selecting (1, 0, 1, 0)", constantView(Type3D, Format8888UNorm, OneZeroOneZero), 0x3f800000u}, + {"a 32_UINT view selecting (1, 0, 1, 0)", constantView(Type2D, Format32UInt, OneZeroOneZero), 1u}, + {"a 32_SINT view selecting (1, 0, 1, 0)", constantView(Type2D, Format32SInt, OneZeroOneZero), 1u}, + {"a 1D view selecting (0, 0, 0, 0)", constantView(Type1D, Format8888UNorm, 0u), 0u}, + }}; + const std::array constantInstructions{"image_sample_lz dmask 0x6", "image_gather4_lz dmask 0x2", "image_gather4_lz dmask 0x4"}; + for (const auto& constant : constants) { + auto normalized = Samplers[0].words; + normalized[0] &= ~0x8000u; + for (const auto& sample : Run(*device, constant.texture, normalized, coordinates, ConstantCode)) { + const std::array expected{0u, constant.one, 0u, 0u, 0u, 0u, constant.one, constant.one, constant.one, constant.one}; + for (std::uint32_t index = 0; index < expected.size(); ++index) { + const auto instruction = constantInstructions[index < 2u ? 0u : index < 6u ? 1u : 2u]; + Require(sample[index] == expected[index], std::string(instruction) + ", " + constant.name + ": result " + std::to_string(index) + " is " + Hex(sample[index]) + ", expected " + Hex(expected[index])); + } + } + } std::puts("image sample unnormalized tests passed"); return 0; } catch (const std::exception& error) { diff --git a/core/libs/prx/libSceAgcDriver/tests/execution/ImageSrgbLoad.cpp b/core/libs/prx/libSceAgcDriver/tests/execution/ImageSrgbLoad.cpp index 560166355..a1f35fc45 100644 --- a/core/libs/prx/libSceAgcDriver/tests/execution/ImageSrgbLoad.cpp +++ b/core/libs/prx/libSceAgcDriver/tests/execution/ImageSrgbLoad.cpp @@ -82,7 +82,7 @@ public: block = static_cast(std::aligned_alloc(BlockBytes, BlockBytes)); #endif Require(block != nullptr, "image sRGB load: cannot allocate the guest block"); - GuestAllocations::Mutation().Add(block, BlockBytes, true, true); + GuestAllocations::Mutation().Add(block, BlockBytes, true, true, true); } ~GuestBlock() { diff --git a/core/libs/prx/libSceAgcDriver/tests/execution/ImageStoreLineThroughPlane.cpp b/core/libs/prx/libSceAgcDriver/tests/execution/ImageStoreLineThroughPlane.cpp new file mode 100644 index 000000000..6f6e1cfb2 --- /dev/null +++ b/core/libs/prx/libSceAgcDriver/tests/execution/ImageStoreLineThroughPlane.cpp @@ -0,0 +1,193 @@ +#include "prx/libSceAgcDriver/Execution/include/VulkanDevice.hpp" +#include "prx/libSceAgcDriver/Graphics/include/Draw.hpp" +#include "prx/libSceAgcDriver/Graphics/include/Texture.hpp" +#include "prx/libSceAgcDriver/Graphics/include/TextureTiling.hpp" +#include "prx/libc/include/GuestAllocations.hpp" +#include "Recompiler.hpp" +#include "VulkanTestDevice.hpp" +#ifdef _WIN32 +#ifndef NOMINMAX +#define NOMINMAX +#endif +#include +#endif +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace { + +using AgcDriver::Graphics::Require; +using ShaderRecompiler::ShaderStage; + +constexpr std::uint32_t Threads = 32; +constexpr std::uint32_t Width = 32; +constexpr std::uint32_t Type1D = 8; +constexpr std::uint32_t IdentitySwizzle = 0xfacu; +constexpr std::size_t TexelBytes = 4096; +constexpr std::size_t BlockBytes = 65536; + +alignas(256) std::array Input{}; + +alignas(256) constexpr std::array StoreCode{ + 0x34020084, 0xe0381000, 0x80001401, 0xbf8c3f70, 0x7e3c0300, 0x7e3e0280, 0xf0201f08, 0x0001141e, + 0x7e3e0281, 0xf0201508, 0x0001141e, 0xbf810000, +}; + +class GuestBlock { +public: + GuestBlock() { +#ifdef _WIN32 + block = static_cast(VirtualAlloc(nullptr, BlockBytes, MEM_RESERVE | MEM_COMMIT, PAGE_READWRITE)); +#else + block = static_cast(std::aligned_alloc(BlockBytes, BlockBytes)); +#endif + Require(block != nullptr, "image store line through plane: cannot allocate the guest block"); + GuestAllocations::Mutation().Add(block, BlockBytes, true, true, true); + } + + ~GuestBlock() { + GuestAllocations::Mutation().Remove(block); +#ifdef _WIN32 + VirtualFree(block, 0, MEM_RELEASE); +#else + std::free(block); +#endif + } + + GuestBlock(const GuestBlock&) = delete; + GuestBlock& operator=(const GuestBlock&) = delete; + + std::uint8_t* Data() { return block; } + +private: + std::uint8_t* block = nullptr; +}; + +struct Format { + const char* name; + std::uint32_t format; + std::uint32_t components; +}; + +constexpr std::array Formats{{ + {"32_SINT", 21u, 1u}, + {"32_32_32_32_SINT", 76u, 4u}, +}}; + +std::string Hex(std::uint32_t value) { + char text[16]; + std::snprintf(text, sizeof(text), "0x%08x", value); + return text; +} + +std::uint32_t Value(std::uint32_t tid, std::uint32_t index) { + return (tid * 4u + index + 1u) * 0x9e3779b1u; +} + +std::uint32_t SecondStore(std::uint32_t x, std::uint32_t component) { + constexpr std::uint32_t Dmask = 0x5u; + if (((Dmask >> component) & 1u) == 0u) return 0u; + std::uint32_t data = 0; + for (std::uint32_t lower = 0; lower < component; ++lower) data += (Dmask >> lower) & 1u; + return Value(x, data); +} + +std::uint64_t TexelOffset(const Format& format, std::uint32_t x) { + const auto mip = AgcDriver::Graphics::ComputeMipLayout(AgcDriver::Graphics::TextureTileMode::kLinear, format.format, Width, 1u, 1u).at(0); + return mip.tiledOffset + static_cast(x) * format.components * 4u; +} + +std::vector Initial() { + std::vector bytes(TexelBytes); + for (std::size_t index = 0; index < bytes.size(); ++index) bytes[index] = static_cast(0x80u + 37u * index + 11u * (index >> 4u)); + return bytes; +} + +std::array BufferDescriptor(const void* data, std::uint32_t bytes) { + const auto address = reinterpret_cast(data); + return {static_cast(address), static_cast((address >> 32u) & 0xffffu), bytes, 0x31016facu}; +} + +std::array TextureDescriptor(const void* data, std::uint32_t format) { + const auto address = static_cast(reinterpret_cast(data)); + return { + static_cast(address >> 8u), + static_cast((address >> 40u) & 0xffu) | (format << 20u) | (((Width - 1u) & 3u) << 30u), + (Width - 1u) >> 2u, + IdentitySwizzle | (Type1D << 28u), + 0u, + 0u, + 0u, + 0u, + }; +} + +void Dispatch(AgcDriver::VulkanDevice& device, std::uint8_t* texels, std::span code, std::uint32_t format) { + for (std::uint32_t tid = 0; tid < Threads; ++tid) { + for (std::uint32_t index = 0; index < 4u; ++index) Input[tid * 4u + index] = Value(tid, index); + } + const auto initial = Initial(); + std::copy(initial.begin(), initial.end(), texels); + std::vector userData(16, 0u); + const auto buffer = BufferDescriptor(Input.data(), static_cast(Input.size() * 4u)); + const auto texture = TextureDescriptor(texels, format); + std::copy(buffer.begin(), buffer.end(), userData.begin()); + std::copy(texture.begin(), texture.end(), userData.begin() + 4); + const std::array memory{{{reinterpret_cast(code.data()), std::as_bytes(code)}}}; + const ShaderRecompiler::ShaderComputeStageInfo compute{{Threads, 1, 1}, 0u, {false, false, false}, false, 1}; + ShaderRecompiler::RecompileRequest request{ + {ShaderStage::Compute, reinterpret_cast(code.data()), code, 0, {}}, + {32, 0, userData, compute, std::nullopt, std::nullopt, memory}, + device.Target(), + {0, 0, 0, 128} + }; + request.useCache = false; + const auto result = ShaderRecompiler::Recompile(request); + device.Dispatch(result, 1, 1, 1, {}, reinterpret_cast(code.data())); + device.WaitIdle(); + AgcDriver::Graphics::StorageTexture::FlushPending(reinterpret_cast(texels), TexelBytes, nullptr, "test"); + device.WaitIdle(); +} + +void CheckTexels(const std::uint8_t* texels, const Format& format) { + const auto initial = Initial(); + for (std::uint32_t x = 0; x < Width; ++x) { + const auto offset = TexelOffset(format, x); + for (std::uint32_t component = 0; component < format.components; ++component) { + std::uint32_t actual = 0; + std::memcpy(&actual, texels + offset + component * 4u, 4u); + const auto wanted = SecondStore(x, component); + Require(actual == wanted, std::string("image_store 2d through a 1D ") + format.name + " T#: texel " + std::to_string(x) + " component " + std::to_string(component) + " is " + Hex(actual) + ", expected the y = 1 store with dmask 0x5 " + Hex(wanted)); + } + } + const auto end = TexelOffset(format, Width); + Require(std::equal(texels + end, texels + TexelBytes, initial.begin() + static_cast(end)), std::string("image_store 2d through a 1D ") + format.name + " T#: bytes past the texture changed"); +} + +} + +int main() { + try { + const auto device = OpenVulkanTestDevice(); + if (!device) return VulkanTestSkipped; + GuestBlock block; + auto* texels = block.Data(); + for (const auto& format : Formats) { + Dispatch(*device, texels, StoreCode, format.format); + CheckTexels(texels, format); + } + std::puts("image store line through plane tests passed"); + return 0; + } catch (const std::exception& error) { + std::cerr << error.what() << '\n'; + return 1; + } +} diff --git a/core/libs/prx/libSceAgcDriver/tests/execution/ImageStorePacked.cpp b/core/libs/prx/libSceAgcDriver/tests/execution/ImageStorePacked.cpp index 0a3a058a5..427ebc94b 100644 --- a/core/libs/prx/libSceAgcDriver/tests/execution/ImageStorePacked.cpp +++ b/core/libs/prx/libSceAgcDriver/tests/execution/ImageStorePacked.cpp @@ -58,7 +58,7 @@ public: block = static_cast(std::aligned_alloc(BlockBytes, BlockBytes)); #endif Require(block != nullptr, "image store packed: cannot allocate the guest block"); - GuestAllocations::Mutation().Add(block, BlockBytes, true, true); + GuestAllocations::Mutation().Add(block, BlockBytes, true, true, true); } ~GuestBlock() { diff --git a/core/libs/prx/libSceAgcDriver/tests/execution/ImageStoreSint.cpp b/core/libs/prx/libSceAgcDriver/tests/execution/ImageStoreSint.cpp index 8f8e1752b..a178c8c16 100644 --- a/core/libs/prx/libSceAgcDriver/tests/execution/ImageStoreSint.cpp +++ b/core/libs/prx/libSceAgcDriver/tests/execution/ImageStoreSint.cpp @@ -76,7 +76,7 @@ public: block = static_cast(std::aligned_alloc(BlockBytes, BlockBytes)); #endif Require(block != nullptr, "image store sint: cannot allocate the guest block"); - GuestAllocations::Mutation().Add(block, BlockBytes, true, true); + GuestAllocations::Mutation().Add(block, BlockBytes, true, true, true); } ~GuestBlock() { diff --git a/core/libs/prx/libSceAgcDriver/tests/execution/ImageVolumeAddressOnPlane.cpp b/core/libs/prx/libSceAgcDriver/tests/execution/ImageVolumeAddressOnPlane.cpp new file mode 100644 index 000000000..a30fdcb5b --- /dev/null +++ b/core/libs/prx/libSceAgcDriver/tests/execution/ImageVolumeAddressOnPlane.cpp @@ -0,0 +1,214 @@ +#include "prx/libSceAgcDriver/Execution/include/VulkanDevice.hpp" +#include "prx/libSceAgcDriver/Graphics/include/Draw.hpp" +#include "prx/libSceAgcDriver/Graphics/include/TextureTiling.hpp" +#include "Recompiler.hpp" +#include "VulkanTestDevice.hpp" +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace { + +using AgcDriver::Graphics::Require; +using ShaderRecompiler::ShaderStage; + +constexpr std::uint32_t Threads = 32; +constexpr std::uint32_t Words = 32; +constexpr std::uint32_t Size = 16; +constexpr std::uint32_t Format8888UNorm = 56; +constexpr std::uint32_t Type2D = 9; +constexpr std::uint32_t Type3D = 10; +constexpr std::uint32_t Depth = 2; +constexpr std::uint32_t SampleResult = 8; +constexpr std::uint32_t LoadResult = 12; +alignas(256) std::array Buffer{}; +alignas(256) std::array Plane{}; +alignas(256) std::array Volume{}; + +alignas(256) constexpr std::array Code{ + 0x34020087, 0xe0301000, 0x80000201, 0xe0301004, 0x80000301, 0xe0301008, 0x80000401, 0xe030100c, + 0x80000501, 0xe0301010, 0x80000601, 0xe0301014, 0x80000701, 0xbf8c3f70, 0xf09c0f10, 0x00610c02, + 0xf0001f10, 0x00011005, 0xbf8c3f70, 0xe0701020, 0x80000c01, 0xe0701024, 0x80000d01, 0xe0701028, + 0x80000e01, 0xe070102c, 0x80000f01, 0xe0701030, 0x80001001, 0xe0701034, 0x80001101, 0xe0701038, + 0x80001201, 0xe070103c, 0x80001301, 0xbf810000, +}; + +alignas(256) constexpr std::array ExplicitLodCode{ + 0x34020087, 0xe0301000, 0x80000201, 0xe0301004, 0x80000301, 0xe0301008, 0x80000401, 0xe030100c, + 0x80000501, 0xbf8c3f70, 0xf0900f10, 0x00610c02, 0xbf8c3f70, 0xe0701020, 0x80000c01, 0xbf810000, +}; + +constexpr std::array PlaneR{0.0f, 0.5f, 1.0f, 1.5f, -0.25f, 3.0f, 0.25f, 0.75f}; +constexpr std::array PlaneZ{0u, 1u, 0u, 2u, 0xffffffffu, 4u, 0u, 3u}; + +struct Lane { + std::uint32_t x; + std::uint32_t y; +}; + +Lane LaneOf(std::uint32_t tid) { + return {(tid * 7u + 3u) % Size, (tid * 5u + 1u) % Size}; +} + +std::uint32_t Bits(float value) { + return std::bit_cast(value); +} + +void FillInput(bool volume) { + Buffer.fill(0xdeadbeefu); + for (std::uint32_t tid = 0; tid < Threads; ++tid) { + const auto lane = LaneOf(tid); + const auto u = Bits((static_cast(lane.x) + 0.5f) / static_cast(Size)); + const auto v = Bits((static_cast(lane.y) + 0.5f) / static_cast(Size)); + const auto r = volume ? Bits((static_cast(tid % Depth) + 0.5f) / static_cast(Depth)) : Bits(PlaneR[tid % PlaneR.size()]); + const auto z = volume ? tid % Depth : PlaneZ[tid % PlaneZ.size()]; + const std::array words{u, v, r, lane.x, lane.y, z}; + std::copy(words.begin(), words.end(), &Buffer[tid * Words]); + } +} + +std::uint8_t SliceMarker(std::uint32_t z) { + return static_cast(z * 100u + 7u); +} + +std::array PlaneDescriptor() { + const auto address = static_cast(reinterpret_cast(Plane.data())); + return { + static_cast(address >> 8u), + static_cast((address >> 40u) & 0xffu) | (Format8888UNorm << 20u) | (((Size - 1u) & 3u) << 30u), + ((Size - 1u) >> 2u) | ((Size - 1u) << 14u), + 0xfacu | (Type2D << 28u), + 0u, + 0u, + 0u, + 0u, + }; +} + +std::array VolumeDescriptor() { + const auto address = static_cast(reinterpret_cast(Volume.data())); + return { + static_cast(address >> 8u), + static_cast((address >> 40u) & 0xffu) | (Format8888UNorm << 20u) | (((Size - 1u) & 3u) << 30u), + ((Size - 1u) >> 2u) | ((Size - 1u) << 14u), + 0xfacu | (Type3D << 28u), + Depth - 1u, + 0u, + 0u, + 0u, + }; +} + +void FillTexture(std::span storage, const std::array& descriptor, std::uint32_t slices) { + const auto geometry = AgcDriver::Graphics::DescribeSurface(AgcDriver::Graphics::DecodeTextureResource(descriptor)); + Require(geometry.guestBytes <= storage.size(), "image volume address on a plane: the texture does not fit the texel storage"); + const auto& mip = geometry.mips.at(0); + std::fill(storage.begin(), storage.end(), 0xeeu); + for (std::uint32_t z = 0; z < slices; ++z) { + for (std::uint32_t y = 0; y < Size; ++y) { + for (std::uint32_t x = 0; x < Size; ++x) { + auto* texel = &storage[geometry.GuestLayerOffset(z) + mip.tiledOffset + static_cast(y) * mip.pitchBytes + x * 4u]; + texel[0] = SliceMarker(z); + texel[1] = static_cast(x); + texel[2] = static_cast(y); + texel[3] = 255u; + } + } + } +} + +std::array BufferDescriptor(const void* data, std::uint32_t bytes) { + const auto address = reinterpret_cast(data); + return {static_cast(address), static_cast((address >> 32u) & 0xffffu), bytes, 0x31016facu}; +} + +std::array SamplerDescriptor() { + return {0u, 0xfffu << 12u, 1u << 26u, 0u}; +} + +void Run(AgcDriver::VulkanDevice& device, std::span code, const std::array& texture) { + std::vector userData(16, 0u); + const auto buffer = BufferDescriptor(Buffer.data(), static_cast(Buffer.size() * 4u)); + const auto sampler = SamplerDescriptor(); + std::copy(buffer.begin(), buffer.end(), userData.begin()); + std::copy(texture.begin(), texture.end(), userData.begin() + 4); + std::copy(sampler.begin(), sampler.end(), userData.begin() + 12); + const std::array memory{{{reinterpret_cast(code.data()), std::as_bytes(code)}}}; + const ShaderRecompiler::ShaderComputeStageInfo compute{{Threads, 1, 1}, 0u, {false, false, false}, false, 1}; + ShaderRecompiler::RecompileRequest request{ + {ShaderStage::Compute, reinterpret_cast(code.data()), code, 0, {}}, + {32, 0, userData, compute, std::nullopt, std::nullopt, memory}, + device.Target(), + {0, 0, 0, 128} + }; + request.useCache = false; + const auto result = ShaderRecompiler::Recompile(request); + device.Dispatch(result, 1, 1, 1, {}, reinterpret_cast(code.data())); + device.WaitIdle(); +} + +void Expect(std::uint32_t tid, std::uint32_t result, const std::array& expected, const std::string& name) { + for (std::uint32_t component = 0; component < 4u; ++component) { + const float value = std::bit_cast(Buffer[tid * Words + result + component]) * 255.0f; + Require(std::lround(value) == static_cast(expected[component]), name + ": thread " + std::to_string(tid) + " component " + std::to_string(component) + " is " + std::to_string(value) + ", expected " + std::to_string(expected[component])); + } +} + +void CheckPlane() { + for (std::uint32_t tid = 0; tid < Threads; ++tid) { + const auto lane = LaneOf(tid); + const std::array texel{SliceMarker(0), lane.x, lane.y, 255u}; + const auto z = PlaneZ[tid % PlaneZ.size()]; + Expect(tid, SampleResult, texel, "image_sample_lz 3d on a 2D texture with r " + std::to_string(PlaneR[tid % PlaneR.size()])); + Expect(tid, LoadResult, z == 0u ? texel : std::array{}, "image_load 3d on a 2D texture with z " + std::to_string(static_cast(z))); + } +} + +void CheckVolume() { + for (std::uint32_t tid = 0; tid < Threads; ++tid) { + const auto lane = LaneOf(tid); + const std::array texel{SliceMarker(tid % Depth), lane.x, lane.y, 255u}; + Expect(tid, SampleResult, texel, "image_sample_lz 3d on a 3D texture"); + Expect(tid, LoadResult, texel, "image_load 3d on a 3D texture"); + } +} + +void RequireRefused(AgcDriver::VulkanDevice& device) { + std::string refusal; + try { + Run(device, ExplicitLodCode, PlaneDescriptor()); + } catch (const std::exception& error) { + refusal = error.what(); + } + Require(refusal.find("incompatible with the static runtime image interface") != std::string::npos, "image_sample_l 3d on a 2D texture was not refused: " + refusal); +} + +} + +int main() { + try { + const auto device = OpenVulkanTestDevice(); + if (!device) return VulkanTestSkipped; + FillTexture(Plane, PlaneDescriptor(), 1u); + FillInput(false); + Run(*device, Code, PlaneDescriptor()); + CheckPlane(); + FillTexture(Volume, VolumeDescriptor(), Depth); + FillInput(true); + Run(*device, Code, VolumeDescriptor()); + CheckVolume(); + RequireRefused(*device); + std::puts("image volume address on plane tests passed"); + return 0; + } catch (const std::exception& error) { + std::cerr << error.what() << '\n'; + return 1; + } +} diff --git a/core/libs/prx/libSceAgcDriver/tests/execution/ImageXorSwizzleVolume.cpp b/core/libs/prx/libSceAgcDriver/tests/execution/ImageXorSwizzleVolume.cpp index 3aaf8d19f..52c572810 100644 --- a/core/libs/prx/libSceAgcDriver/tests/execution/ImageXorSwizzleVolume.cpp +++ b/core/libs/prx/libSceAgcDriver/tests/execution/ImageXorSwizzleVolume.cpp @@ -29,6 +29,7 @@ constexpr std::uint32_t Format8888UNorm = 56; constexpr std::uint32_t Type3D = 10; constexpr std::uint32_t TileZ64KBX = 0x18; constexpr std::uint32_t TileS64KBX = 0x19; +constexpr std::uint32_t TileD64KBX = 0x1a; alignas(256) std::array Input{}; alignas(256) std::array Output{}; @@ -145,6 +146,7 @@ int main() { if (!device) return VulkanTestSkipped; FillInput(); Run(*device, TileS64KBX, "thick SW_64KB_S_X volume"); + Run(*device, TileD64KBX, "thick SW_64KB_D_X volume"); Run(*device, TileZ64KBX, "thin SW_64KB_Z_X volume"); std::puts("image XOR swizzle volume tests passed"); return 0; diff --git a/core/libs/prx/libSceAgcDriver/tests/execution/InterpolationF16.cpp b/core/libs/prx/libSceAgcDriver/tests/execution/InterpolationF16.cpp index 6cfadd21b..8b519385a 100644 --- a/core/libs/prx/libSceAgcDriver/tests/execution/InterpolationF16.cpp +++ b/core/libs/prx/libSceAgcDriver/tests/execution/InterpolationF16.cpp @@ -133,7 +133,7 @@ ShaderRecompiler::RecompileResult RecompilePixel(std::span {ShaderStage::Fragment, reinterpret_cast(code.data()), code, 0, {}}, {waveSize, 0, pixelUserData, std::nullopt, PixelStage(waveSize, input0), std::nullopt, pixelMemory}, target, - {0, 0, pushOffset, 128 - pushOffset} + PixelPushLayout(pushOffset, target) }; fragment.useCache = false; fragment.context.floatMode = ShaderRecompiler::ShaderFloatMode{0xf0u, true, true, false}; @@ -162,7 +162,7 @@ void Draw(AgcDriver::VulkanDevice& device, std::uint32_t waveSize) { const auto pixelResult = RecompilePixel(PixelCode, waveSize, Fp16Input, target, vertexPush); const std::array shaders{{ {ShaderStage::Vertex, &vertexResult, 0}, - {ShaderStage::Fragment, &pixelResult, vertexPush} + {ShaderStage::Fragment, &pixelResult, PixelPushOffset(vertexPush, target)} }}; AgcDriver::Graphics::State state{}; diff --git a/core/libs/prx/libSceAgcDriver/tests/execution/LdexpF16InlineConstants.cpp b/core/libs/prx/libSceAgcDriver/tests/execution/LdexpF16InlineConstants.cpp new file mode 100644 index 000000000..fd14f4ee2 --- /dev/null +++ b/core/libs/prx/libSceAgcDriver/tests/execution/LdexpF16InlineConstants.cpp @@ -0,0 +1,139 @@ +#include "prx/libSceAgcDriver/Execution/include/VulkanDevice.hpp" +#include "prx/libSceAgcDriver/Graphics/include/Draw.hpp" +#include "Recompiler.hpp" +#include "VulkanTestDevice.hpp" +#include +#include +#include +#include +#include +#include +#include + +namespace { + +using AgcDriver::Graphics::Require; +using ShaderRecompiler::ShaderStage; + +constexpr std::uint32_t Threads = 32; +constexpr std::uint32_t Results = 16; +alignas(256) std::array Input{}; +alignas(256) std::array Output{}; + +alignas(256) constexpr std::array LdexpCode{ + 0x34060084, 0xe0302000, 0x80000400, 0xbe8803f0, 0x7e0a02ff, 0x00003800, 0xbf8c3f70, 0xd53b000a, + 0x0201e104, 0xd53b000b, 0x0201e304, 0xd53b000c, 0x0201e504, 0xd53b000d, 0x0201e704, 0xd53b000e, + 0x0201e904, 0xd53b000f, 0x0201eb04, 0xd53b0010, 0x0201ed04, 0xd53b0011, 0x0201ef04, 0xd53b0012, + 0x0201f104, 0xd53b0013, 0x02010504, 0xd53b0014, 0x02018304, 0xd53b0015, 0x02018104, 0xd53b0016, + 0x0201a104, 0xd53b0017, 0x0201ff04, 0x3f000000, 0xd53b0018, 0x02001104, 0xd53b0019, 0x02020b04, + 0xe0702000, 0x80010a03, 0xe0702004, 0x80010b03, 0xe0702008, 0x80010c03, 0xe070200c, 0x80010d03, + 0xe0702010, 0x80010e03, 0xe0702014, 0x80010f03, 0xe0702018, 0x80011003, 0xe070201c, 0x80011103, + 0xe0702020, 0x80011203, 0xe0702024, 0x80011303, 0xe0702028, 0x80011403, 0xe070202c, 0x80011503, + 0xe0702030, 0x80011603, 0xe0702034, 0x80011703, 0xe0702038, 0x80011803, 0xe070203c, 0x80011903, + 0xbf810000, +}; + +struct Row { + std::uint32_t half; + std::array expected; +}; + +constexpr std::array Rows{{ + {0x3c00u, {0x7c00u, 0x0000u, 0x7c00u, 0x0000u, 0x7c00u, 0x0000u, 0x7c00u, 0x0000u, 0x7c00u, 0x4400u, 0x3800u, 0x7c00u, 0x0100u, 0x3c00u, 0x3c00u, 0x7c00u}}, + {0x3555u, {0x7c00u, 0x0000u, 0x7c00u, 0x0000u, 0x7c00u, 0x0000u, 0x7c00u, 0x0000u, 0x7c00u, 0x3d55u, 0x3155u, 0x7c00u, 0x0055u, 0x3555u, 0x3555u, 0x7c00u}}, + {0x4248u, {0x7c00u, 0x0000u, 0x7c00u, 0x0000u, 0x7c00u, 0x0000u, 0x7c00u, 0x0000u, 0x7c00u, 0x4a48u, 0x3e48u, 0x7c00u, 0x0324u, 0x4248u, 0x4248u, 0x7c00u}}, + {0x7bffu, {0x7c00u, 0x0000u, 0x7c00u, 0x0000u, 0x7c00u, 0x0000u, 0x7c00u, 0x0000u, 0x7c00u, 0x7c00u, 0x77ffu, 0x7c00u, 0x3bffu, 0x7bffu, 0x7bffu, 0x7c00u}}, + {0x0400u, {0x7c00u, 0x0000u, 0x7c00u, 0x0000u, 0x7c00u, 0x0000u, 0x7c00u, 0x0000u, 0x7c00u, 0x0c00u, 0x0200u, 0x7c00u, 0x0000u, 0x0400u, 0x0400u, 0x7c00u}}, + {0x2e66u, {0x7c00u, 0x0000u, 0x7c00u, 0x0000u, 0x7c00u, 0x0000u, 0x7c00u, 0x0000u, 0x7c00u, 0x3666u, 0x2a66u, 0x7c00u, 0x001au, 0x2e66u, 0x2e66u, 0x7c00u}}, + {0x5640u, {0x7c00u, 0x0000u, 0x7c00u, 0x0000u, 0x7c00u, 0x0000u, 0x7c00u, 0x0000u, 0x7c00u, 0x5e40u, 0x5240u, 0x7c00u, 0x1640u, 0x5640u, 0x5640u, 0x7c00u}}, + {0x3800u, {0x7c00u, 0x0000u, 0x7c00u, 0x0000u, 0x7c00u, 0x0000u, 0x7c00u, 0x0000u, 0x7c00u, 0x4000u, 0x3400u, 0x7c00u, 0x0080u, 0x3800u, 0x3800u, 0x7c00u}}, + {0x07ffu, {0x7c00u, 0x0000u, 0x7c00u, 0x0000u, 0x7c00u, 0x0000u, 0x7c00u, 0x0000u, 0x7c00u, 0x0fffu, 0x0400u, 0x7c00u, 0x0000u, 0x07ffu, 0x07ffu, 0x7c00u}}, + {0x0800u, {0x7c00u, 0x0000u, 0x7c00u, 0x0000u, 0x7c00u, 0x0000u, 0x7c00u, 0x0000u, 0x7c00u, 0x1000u, 0x0400u, 0x7c00u, 0x0000u, 0x0800u, 0x0800u, 0x7c00u}}, + {0xbc00u, {0xfc00u, 0x8000u, 0xfc00u, 0x8000u, 0xfc00u, 0x8000u, 0xfc00u, 0x8000u, 0xfc00u, 0xc400u, 0xb800u, 0xfc00u, 0x8100u, 0xbc00u, 0xbc00u, 0xfc00u}}, + {0xb555u, {0xfc00u, 0x8000u, 0xfc00u, 0x8000u, 0xfc00u, 0x8000u, 0xfc00u, 0x8000u, 0xfc00u, 0xbd55u, 0xb155u, 0xfc00u, 0x8055u, 0xb555u, 0xb555u, 0xfc00u}}, + {0xc248u, {0xfc00u, 0x8000u, 0xfc00u, 0x8000u, 0xfc00u, 0x8000u, 0xfc00u, 0x8000u, 0xfc00u, 0xca48u, 0xbe48u, 0xfc00u, 0x8324u, 0xc248u, 0xc248u, 0xfc00u}}, + {0xfbffu, {0xfc00u, 0x8000u, 0xfc00u, 0x8000u, 0xfc00u, 0x8000u, 0xfc00u, 0x8000u, 0xfc00u, 0xfc00u, 0xf7ffu, 0xfc00u, 0xbbffu, 0xfbffu, 0xfbffu, 0xfc00u}}, + {0x8400u, {0xfc00u, 0x8000u, 0xfc00u, 0x8000u, 0xfc00u, 0x8000u, 0xfc00u, 0x8000u, 0xfc00u, 0x8c00u, 0x8200u, 0xfc00u, 0x8000u, 0x8400u, 0x8400u, 0xfc00u}}, + {0xae66u, {0xfc00u, 0x8000u, 0xfc00u, 0x8000u, 0xfc00u, 0x8000u, 0xfc00u, 0x8000u, 0xfc00u, 0xb666u, 0xaa66u, 0xfc00u, 0x801au, 0xae66u, 0xae66u, 0xfc00u}}, + {0xd640u, {0xfc00u, 0x8000u, 0xfc00u, 0x8000u, 0xfc00u, 0x8000u, 0xfc00u, 0x8000u, 0xfc00u, 0xde40u, 0xd240u, 0xfc00u, 0x9640u, 0xd640u, 0xd640u, 0xfc00u}}, + {0x8800u, {0xfc00u, 0x8000u, 0xfc00u, 0x8000u, 0xfc00u, 0x8000u, 0xfc00u, 0x8000u, 0xfc00u, 0x9000u, 0x8400u, 0xfc00u, 0x8000u, 0x8800u, 0x8800u, 0xfc00u}}, + {0x0001u, {0x7c00u, 0x0000u, 0x7c00u, 0x0000u, 0x7c00u, 0x0000u, 0x7c00u, 0x0000u, 0x7c00u, 0x0004u, 0x0000u, 0x7c00u, 0x0000u, 0x0001u, 0x0001u, 0x7c00u}}, + {0x0002u, {0x7c00u, 0x0000u, 0x7c00u, 0x0000u, 0x7c00u, 0x0000u, 0x7c00u, 0x0000u, 0x7c00u, 0x0008u, 0x0001u, 0x7c00u, 0x0000u, 0x0002u, 0x0002u, 0x7c00u}}, + {0x0003u, {0x7c00u, 0x0000u, 0x7c00u, 0x0000u, 0x7c00u, 0x0000u, 0x7c00u, 0x0000u, 0x7c00u, 0x000cu, 0x0002u, 0x7c00u, 0x0000u, 0x0003u, 0x0003u, 0x7c00u}}, + {0x0100u, {0x7c00u, 0x0000u, 0x7c00u, 0x0000u, 0x7c00u, 0x0000u, 0x7c00u, 0x0000u, 0x7c00u, 0x0400u, 0x0080u, 0x7c00u, 0x0000u, 0x0100u, 0x0100u, 0x7c00u}}, + {0x0155u, {0x7c00u, 0x0000u, 0x7c00u, 0x0000u, 0x7c00u, 0x0000u, 0x7c00u, 0x0000u, 0x7c00u, 0x0554u, 0x00aau, 0x7c00u, 0x0000u, 0x0155u, 0x0155u, 0x7c00u}}, + {0x0200u, {0x7c00u, 0x0000u, 0x7c00u, 0x0000u, 0x7c00u, 0x0000u, 0x7c00u, 0x0000u, 0x7c00u, 0x0800u, 0x0100u, 0x7c00u, 0x0000u, 0x0200u, 0x0200u, 0x7c00u}}, + {0x03ffu, {0x7c00u, 0x0000u, 0x7c00u, 0x0000u, 0x7c00u, 0x0000u, 0x7c00u, 0x0000u, 0x7c00u, 0x0bfeu, 0x0200u, 0x7c00u, 0x0000u, 0x03ffu, 0x03ffu, 0x7c00u}}, + {0x8001u, {0xfc00u, 0x8000u, 0xfc00u, 0x8000u, 0xfc00u, 0x8000u, 0xfc00u, 0x8000u, 0xfc00u, 0x8004u, 0x8000u, 0xfc00u, 0x8000u, 0x8001u, 0x8001u, 0xfc00u}}, + {0x8003u, {0xfc00u, 0x8000u, 0xfc00u, 0x8000u, 0xfc00u, 0x8000u, 0xfc00u, 0x8000u, 0xfc00u, 0x800cu, 0x8002u, 0xfc00u, 0x8000u, 0x8003u, 0x8003u, 0xfc00u}}, + {0x8155u, {0xfc00u, 0x8000u, 0xfc00u, 0x8000u, 0xfc00u, 0x8000u, 0xfc00u, 0x8000u, 0xfc00u, 0x8554u, 0x80aau, 0xfc00u, 0x8000u, 0x8155u, 0x8155u, 0xfc00u}}, + {0x8200u, {0xfc00u, 0x8000u, 0xfc00u, 0x8000u, 0xfc00u, 0x8000u, 0xfc00u, 0x8000u, 0xfc00u, 0x8800u, 0x8100u, 0xfc00u, 0x8000u, 0x8200u, 0x8200u, 0xfc00u}}, + {0x83ffu, {0xfc00u, 0x8000u, 0xfc00u, 0x8000u, 0xfc00u, 0x8000u, 0xfc00u, 0x8000u, 0xfc00u, 0x8bfeu, 0x8200u, 0xfc00u, 0x8000u, 0x83ffu, 0x83ffu, 0xfc00u}}, + {0x0000u, {0x0000u, 0x0000u, 0x0000u, 0x0000u, 0x0000u, 0x0000u, 0x0000u, 0x0000u, 0x0000u, 0x0000u, 0x0000u, 0x0000u, 0x0000u, 0x0000u, 0x0000u, 0x0000u}}, + {0x8000u, {0x8000u, 0x8000u, 0x8000u, 0x8000u, 0x8000u, 0x8000u, 0x8000u, 0x8000u, 0x8000u, 0x8000u, 0x8000u, 0x8000u, 0x8000u, 0x8000u, 0x8000u, 0x8000u}}, +}}; + +constexpr std::array Names{ + "0.5", "-0.5", "1.0", "-1.0", "2.0", "-2.0", "4.0", "-4.0", "0.15915494", "2", "-1", "64", "-16", "literal 0x3f000000", "s8 = 0x3f000000", "v5 = 0x3800", +}; + +std::array BufferDescriptor(const void* data, std::uint32_t count) { + const auto address = reinterpret_cast(data); + return {static_cast(address), static_cast((address >> 32u) & 0xffffu) | (4u << 16u), count, 0x11016facu}; +} + +void Run(AgcDriver::VulkanDevice& device) { + for (std::uint32_t tid = 0; tid < Threads; ++tid) Input[tid] = Rows[tid].half; + Output.fill(0xdeadbeefu); + std::vector userData(8, 0u); + const auto input = BufferDescriptor(Input.data(), static_cast(Input.size())); + const auto output = BufferDescriptor(Output.data(), static_cast(Output.size())); + std::copy(input.begin(), input.end(), userData.begin()); + std::copy(output.begin(), output.end(), userData.begin() + 4); + const std::span code(LdexpCode); + const std::array memory{{{reinterpret_cast(code.data()), std::as_bytes(code)}}}; + const ShaderRecompiler::ShaderComputeStageInfo compute{{Threads, 1, 1}, 0, {false, false, false}, false, 1}; + ShaderRecompiler::RecompileRequest request{ + {ShaderStage::Compute, reinterpret_cast(code.data()), code, 0, {}}, + {32, 0, userData, compute, std::nullopt, std::nullopt, memory}, + device.Target(), + {0, 0, 0, 128} + }; + request.useCache = false; + request.context.floatMode = ShaderRecompiler::ShaderFloatMode{0xc0u, true, false, false}; + const auto result = ShaderRecompiler::Recompile(request); + device.Dispatch(result, 1, 1, 1, {}, reinterpret_cast(code.data())); + device.WaitIdle(); +} + +std::string Hex(std::uint32_t value) { + char text[16]; + std::snprintf(text, sizeof(text), "0x%04x", value); + return text; +} + +void Check() { + for (std::uint32_t tid = 0; tid < Threads; ++tid) { + const auto& row = Rows[tid]; + for (std::uint32_t column = 0; column < Results; ++column) { + const auto actual = Output[tid * Results + column] & 0xffffu; + Require(actual == row.expected[column], std::string("ldexp f16 inline constants: v_ldexp_f16 ") + Hex(row.half) + ", " + Names[column] + " is " + Hex(actual) + ", expected " + Hex(row.expected[column])); + } + } +} + +} + +int main() { + try { + const auto device = OpenVulkanTestDevice(); + if (!device) return VulkanTestSkipped; + Run(*device); + Check(); + std::puts("ldexp f16 inline constants tests passed"); + return 0; + } catch (const std::exception& error) { + std::cerr << error.what() << '\n'; + return 1; + } +} diff --git a/core/libs/prx/libSceAgcDriver/tests/execution/LdsAtomics64.cpp b/core/libs/prx/libSceAgcDriver/tests/execution/LdsAtomics64.cpp index 3ca9e67f2..e6bf5cd89 100644 --- a/core/libs/prx/libSceAgcDriver/tests/execution/LdsAtomics64.cpp +++ b/core/libs/prx/libSceAgcDriver/tests/execution/LdsAtomics64.cpp @@ -90,7 +90,12 @@ constexpr std::uint64_t Sign = 0x8000000000000000ull; bool Nan(std::uint64_t bits) { return (bits & ~Sign) > 0x7ff0000000000000ull; } std::uint64_t OrderKey(std::uint64_t bits) { return (bits & Sign) != 0u ? ~bits : bits | Sign; } +constexpr std::uint64_t Quiet = 0x0008000000000000ull; + +bool Signaling(std::uint64_t bits) { return Nan(bits) && (bits & Quiet) == 0u; } std::uint64_t MinMax(std::uint64_t old, std::uint64_t data, bool max) { + if (Signaling(data)) return data | Quiet; + if (Signaling(old)) return old | Quiet; if (Nan(old)) return data; if (Nan(data)) return old; return (max ? OrderKey(data) > OrderKey(old) : OrderKey(data) < OrderKey(old)) ? data : old; @@ -114,7 +119,7 @@ void Fill(std::uint32_t tid, std::uint32_t* words) { std::array values{}; if (kind == 0u) values = {seed, seed * 0xbf58476d1ce4e5b9ull, seed ^ 0x94d049bb133111ebull}; if (kind == 1u) values = {tid % 7u, (tid * 3u) % 7u, tid % 5u}; - if (kind == 2u) values = {floats[tid % 12u], floats[(tid / 2u + 3u) % 12u], floats[(tid + 5u) % 12u]}; + if (kind == 2u) values = {floats[tid % 12u], floats[(tid / 4u + 9u) % 12u], floats[(tid + 5u) % 12u]}; if (kind == 3u) values = {seed, (tid & 4u) != 0u ? seed : seed + 0x100000000ull, seed * 3u}; for (std::uint32_t i = 0; i < values.size(); ++i) { words[i * 2u] = static_cast(values[i]); diff --git a/core/libs/prx/libSceAgcDriver/tests/execution/LdsBeyondDeviceLimit.cpp b/core/libs/prx/libSceAgcDriver/tests/execution/LdsBeyondDeviceLimit.cpp new file mode 100644 index 000000000..8b37c4591 --- /dev/null +++ b/core/libs/prx/libSceAgcDriver/tests/execution/LdsBeyondDeviceLimit.cpp @@ -0,0 +1,227 @@ +#include "prx/libSceAgcDriver/Execution/include/VulkanDevice.hpp" +#include "prx/libSceAgcDriver/Graphics/include/Draw.hpp" +#include "Recompiler.hpp" +#include "VulkanTestDevice.hpp" +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace { + +using AgcDriver::Graphics::Require; +using ShaderRecompiler::ShaderStage; + +constexpr std::uint32_t Threads = 64; +constexpr std::uint32_t Results = 16; +constexpr std::uint32_t Checked = 12; +constexpr std::uint32_t MaxGroups = 16; +constexpr std::uint32_t AboveLimitBytes = 4096; +constexpr std::uint32_t BelowLimitBytes = 512; +constexpr std::uint32_t Untouched = 0xdeadbeefu; +constexpr std::uint32_t OpTypeInt = 21; +constexpr std::uint32_t OpTypeFloat = 22; +constexpr std::uint32_t OpTypeVector = 23; +constexpr std::uint32_t OpTypeArray = 28; +constexpr std::uint32_t OpTypeStruct = 30; +constexpr std::uint32_t OpTypePointer = 32; +constexpr std::uint32_t OpConstant = 43; +constexpr std::uint32_t OpVariable = 59; +constexpr std::uint32_t StorageClassWorkgroup = 4; +alignas(256) std::array Input{}; +alignas(256) std::array Output{}; +alignas(256) std::array IndirectGroups{}; + +alignas(256) constexpr std::array Code{ + 0x34020084, 0xe0381000, 0x80000401, 0x7e100208, 0x7e3e0209, 0x343e3e82, 0x4a103f08, 0x7e40020a, + 0x34404083, 0x4a104108, 0x34121090, 0x4a121300, 0x4a1412ff, 0x10000000, 0x7e160280, 0x7e180280, + 0x7e1a0281, 0x7e1e0281, 0x7e200280, 0xbf8c0000, 0xd9340000, 0x00000b06, 0xd834000c, 0x00000b06, + 0xd8340000, 0x00000904, 0xd8340000, 0x00000a05, 0xd8340000, 0x00000907, 0xbf8c0000, 0xbf8a0000, + 0xd880000c, 0x28000d06, 0xd9800000, 0x29000f06, 0x3a260884, 0x3a2a0aff, 0x00000080, 0x363608ff, + 0xfffffff8, 0xbf8c0000, 0xbf8a0000, 0xd8d80000, 0x2b000013, 0xd8d80000, 0x2c000015, 0xd8d80000, + 0x2d000007, 0xd8d8000c, 0x2e000006, 0xd9d80000, 0x2f000006, 0xd9d80000, 0x3100001b, 0xd8d80000, + 0x33000005, 0x343c1086, 0x4a3c3d00, 0x343c3c86, 0xbf8c0000, 0xe0781000, 0x8001281e, 0xe0781010, + 0x80012c1e, 0xe0781020, 0x8001301e, 0xbf8c0000, 0xbf810000, +}; + +constexpr const char* Names[Checked] = { + "ds_add_rtn_u32 result", "ds_add_rtn_u64 result low", "ds_add_rtn_u64 result high", "neighbour's word above the limit", + "other wave's word below the limit", "out-of-range read", "final 32-bit counter", "final 64-bit counter low", + "final 64-bit counter high", "ds_read_b64 above the limit, low", "ds_read_b64 above the limit, high", "own word below the limit", +}; + +std::array BufferDescriptor(const void* data, std::uint32_t bytes) { + const auto address = reinterpret_cast(data); + return {static_cast(address), static_cast((address >> 32u) & 0xffffu), bytes, 0x31016facu}; +} + +std::string Hex(std::uint32_t value) { + char text[16]; + std::snprintf(text, sizeof(text), "0x%08x", value); + return text; +} + +std::uint64_t WorkgroupBytes(std::span words) { + std::unordered_map> types; + std::unordered_map constants; + std::vector pointers; + for (std::size_t at = 5; at < words.size();) { + const auto count = words[at] >> 16u; + Require(count != 0u && at + count <= words.size(), "lds beyond the device limit: malformed SPIR-V at word " + std::to_string(at)); + const auto instruction = words.subspan(at, count); + const auto opcode = instruction[0] & 0xffffu; + if (opcode == OpTypeInt || opcode == OpTypeFloat || opcode == OpTypeVector || opcode == OpTypeArray || opcode == OpTypeStruct || opcode == OpTypePointer) types[instruction[1]] = instruction; + if (opcode == OpConstant) constants[instruction[2]] = instruction[3]; + if (opcode == OpVariable && instruction[3] == StorageClassWorkgroup) pointers.push_back(instruction[1]); + at += count; + } + const std::function size = [&](std::uint32_t id) -> std::uint64_t { + const auto type = types.find(id); + Require(type != types.end(), "lds beyond the device limit: a Workgroup variable has a type the test does not size"); + const auto& definition = type->second; + switch (definition[0] & 0xffffu) { + case OpTypeInt: + case OpTypeFloat: + return definition[2] / 8u; + case OpTypeVector: + return size(definition[2]) * definition[3]; + case OpTypeArray: { + const auto length = constants.find(definition[3]); + Require(length != constants.end(), "lds beyond the device limit: a Workgroup array has a length that is not a constant"); + return size(definition[2]) * length->second; + } + case OpTypeStruct: { + std::uint64_t bytes = 0; + for (const auto member : definition.subspan(2)) bytes += size(member); + return bytes; + } + default: + throw std::runtime_error("lds beyond the device limit: a Workgroup variable has a type the test does not size"); + } + }; + std::uint64_t total = 0; + for (const auto pointer : pointers) { + const auto type = types.find(pointer); + Require(type != types.end() && (type->second[0] & 0xffffu) == OpTypePointer, "lds beyond the device limit: a Workgroup variable is not a pointer"); + total += size(type->second[3]); + } + return total; +} + +ShaderRecompiler::RecompileResult Compile(std::uint32_t waveSize, std::uint32_t ldsBytes, const ShaderRecompiler::SpirvTarget& target) { + std::vector userData(8, 0u); + const auto input = BufferDescriptor(Input.data(), static_cast(Input.size() * 4u)); + const auto output = BufferDescriptor(Output.data(), static_cast(Output.size() * 4u)); + std::copy(input.begin(), input.end(), userData.begin()); + std::copy(output.begin(), output.end(), userData.begin() + 4); + const std::array memory{{{reinterpret_cast(Code.data()), std::as_bytes(std::span(Code))}}}; + const ShaderRecompiler::ShaderComputeStageInfo compute{{Threads, 1, 1}, ldsBytes / 4u, {true, true, true}, false, 1}; + ShaderRecompiler::RecompileRequest request{ + {ShaderStage::Compute, reinterpret_cast(Code.data()), Code, 0, {}}, + {waveSize, 0, userData, compute, std::nullopt, std::nullopt, memory}, + target, + {0, 0, 0, 128} + }; + request.useCache = false; + return ShaderRecompiler::Recompile(request); +} + +std::uint32_t Group(std::uint32_t x, std::uint32_t y, std::uint32_t z) { + return (z << 3u) | (y << 2u) | x; +} + +void Run(AgcDriver::VulkanDevice& device, std::uint32_t waveSize, const ShaderRecompiler::SpirvTarget& target, std::uint32_t groupsX, std::uint32_t groupsY, std::uint32_t groupsZ, bool indirect, const std::string& run) { + const auto limit = target.maxWorkgroupSharedMemoryBytes; + Require(limit % 8u == 0u && limit >= 2048u, "lds beyond the device limit: unexpected maxComputeSharedMemorySize " + std::to_string(limit)); + const auto fits = [&](const ShaderRecompiler::RecompileResult& compiled, const std::string& what) { + const auto bytes = WorkgroupBytes(compiled.spirv.Words()); + Require(bytes <= limit, "lds beyond the device limit " + run + ": " + what + " declares " + std::to_string(bytes) + " bytes of Workgroup memory, over the device's " + std::to_string(limit)); + }; + const auto within = Compile(waveSize, limit - BelowLimitBytes, target); + Require(within.workgroupMemoryDwords == 0u, "lds beyond the device limit " + run + ": LDS below the device limit left workgroup memory"); + fits(within, "LDS below the device limit"); + const auto atLimit = Compile(waveSize, limit, target); + Require(atLimit.workgroupMemoryDwords == limit / 4u + 1u, "lds beyond the device limit " + run + ": LDS at the device limit gave " + std::to_string(atLimit.workgroupMemoryDwords) + " device memory dwords per workgroup"); + fits(atLimit, "LDS at the device limit"); + const auto bytes = limit + AboveLimitBytes; + for (std::uint32_t tid = 0; tid < Threads; ++tid) { + Input[tid * 4u + 0u] = limit + tid * 4u; + Input[tid * 4u + 1u] = 1024u + tid * 4u; + Input[tid * 4u + 2u] = bytes - 16u; + Input[tid * 4u + 3u] = bytes + tid * 4u; + } + Output.fill(Untouched); + const auto result = Compile(waveSize, bytes, target); + Require(result.workgroupMemoryDwords == bytes / 4u + 1u, "lds beyond the device limit " + run + ": " + std::to_string(bytes) + " bytes of LDS gave " + std::to_string(result.workgroupMemoryDwords) + " device memory dwords per workgroup"); + fits(result, "LDS above the device limit"); + if (indirect) { + IndirectGroups = {groupsX, groupsY, groupsZ, 0u}; + const auto outcome = device.DispatchIndirect(result, reinterpret_cast(IndirectGroups.data()), {}, reinterpret_cast(Code.data())); + Require(outcome.cpuReason == 8, "lds beyond the device limit " + run + ": the indirect group counts were not read on the CPU (reason " + std::to_string(outcome.cpuReason) + ")"); + } else { + device.Dispatch(result, groupsX, groupsY, groupsZ, {}, reinterpret_cast(Code.data())); + } + device.WaitIdle(); + std::vector dispatched(MaxGroups, false); + for (std::uint32_t z = 0; z < groupsZ; ++z) { + for (std::uint32_t y = 0; y < groupsY; ++y) { + for (std::uint32_t x = 0; x < groupsX; ++x) dispatched[Group(x, y, z)] = true; + } + } + for (std::uint32_t group = 0; group < MaxGroups; ++group) { + const std::string where = "lds beyond the device limit " + run + ", group " + std::to_string(group); + std::vector counter32; + std::vector counter64; + for (std::uint32_t tid = 0; tid < Threads; ++tid) { + const auto* lane = &Output[(group * Threads + tid) * Results]; + if (!dispatched[group]) { + Require(std::all_of(lane, lane + Results, [](std::uint32_t word) { return word == Untouched; }), where + ": lane " + std::to_string(tid) + " of a group that was not dispatched was written"); + continue; + } + const auto value = [&](std::uint32_t laneId) { return (group << 16u) | laneId; }; + const std::array expected{ + lane[0], lane[1], 0u, value(tid ^ 1u), + value(tid ^ 32u) + 0x10000000u, 0u, Threads, Threads, + 0u, value(tid & ~1u), value(tid | 1u), value(tid) + 0x10000000u, + }; + for (std::uint32_t index = 0; index < Checked; ++index) { + Require(lane[index] == expected[index], where + ": lane " + std::to_string(tid) + " " + Names[index] + " is " + Hex(lane[index]) + ", expected " + Hex(expected[index])); + } + counter32.push_back(lane[0]); + counter64.push_back(lane[1]); + } + if (!dispatched[group]) continue; + std::sort(counter32.begin(), counter32.end()); + std::sort(counter64.begin(), counter64.end()); + for (std::uint32_t tid = 0; tid < Threads; ++tid) { + Require(counter32[tid] == tid && counter64[tid] == tid, where + ": the LDS atomics returned " + Hex(counter32[tid]) + " / " + Hex(counter64[tid]) + " as their " + std::to_string(tid) + "th smallest old value"); + } + } +} + +} + +int main() { + try { + const auto device = OpenVulkanTestDevice(); + if (!device) return VulkanTestSkipped; + Run(*device, 32, device->Target(), 3, 2, 1, false, "wave32 3x2"); + Run(*device, 32, device->Target(), 4, 2, 1, false, "wave32 4x2"); + Run(*device, 64, device->Target(), 3, 2, 1, false, "wave64 3x2"); + Run(*device, 64, device->ComputeTarget(32), 4, 2, 1, false, "wave64 split 4x2"); + Run(*device, 64, device->Target(), 2, 2, 2, false, "wave64 2x2x2"); + Run(*device, 32, device->Target(), 2, 2, 2, true, "wave32 indirect 2x2x2"); + std::puts("lds beyond the device limit tests passed"); + return 0; + } catch (const std::exception& error) { + std::cerr << error.what() << '\n'; + return 1; + } +} diff --git a/core/libs/prx/libSceAgcDriver/tests/execution/LdsFloatAtomicModes.cpp b/core/libs/prx/libSceAgcDriver/tests/execution/LdsFloatAtomicModes.cpp new file mode 100644 index 000000000..0233d2c12 --- /dev/null +++ b/core/libs/prx/libSceAgcDriver/tests/execution/LdsFloatAtomicModes.cpp @@ -0,0 +1,480 @@ +#include "prx/libSceAgcDriver/Execution/include/VulkanDevice.hpp" +#include "prx/libSceAgcDriver/Graphics/include/Draw.hpp" +#include "Recompiler.hpp" +#include "VulkanTestDevice.hpp" +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace { + +using AgcDriver::Graphics::Require; +using ShaderRecompiler::ShaderStage; + +constexpr std::uint32_t Threads = 32; +constexpr std::uint32_t Inputs = 8; +constexpr std::uint32_t Results = 32; +constexpr std::uint32_t Columns = 27; +alignas(256) std::array Input{}; +alignas(256) std::array Output{}; + +alignas(256) constexpr std::array Code{ + 0x34020085u, 0x34040087u, 0xe0381000u, 0x80000401u, 0xe0381010u, 0x80000801u, 0xbf8c3f70u, 0x7e180280u, + 0x7e1a0280u, 0x7e1c0280u, 0x7e1e0280u, 0x7e200280u, 0x7e220280u, 0x7e240280u, 0x7e260280u, 0x7e280280u, + 0x7e2a0280u, 0x7e2c0280u, 0x7e2e0280u, 0x7e300280u, 0x7e320280u, 0x7e340280u, 0x7e360280u, 0x7e380280u, + 0x7e3a0280u, 0x7e3c0280u, 0x7e3e0280u, 0x7e400280u, 0x7e420280u, 0x7e440280u, 0x7e460280u, 0x7e480280u, + 0x7e4a0280u, 0x7e4c0280u, 0x7e4e0280u, 0x7e5002ffu, 0x45464748u, 0x7e5202ffu, 0x41424344u, 0x34060083u, + 0xd8340000u, 0x00000403u, 0xbf8cc07fu, 0xd8c80000u, 0x0d000503u, 0xbf8cc07fu, 0xd8d80000u, 0x0c000003u, + 0xbf8cc07fu, 0xd8340000u, 0x00000403u, 0xbf8cc07fu, 0xd8cc0000u, 0x0f000503u, 0xbf8cc07fu, 0xd8d80000u, + 0x0e000003u, 0xbf8cc07fu, 0xd8340000u, 0x00000403u, 0xbf8cc07fu, 0xd8c40000u, 0x110a0503u, 0xbf8cc07fu, + 0xd8d80000u, 0x10000003u, 0xbf8cc07fu, 0xd8340000u, 0x00000403u, 0xbf8cc07fu, 0xd8480000u, 0x00000503u, + 0xbf8cc07fu, 0xd8d80000u, 0x12000003u, 0xbf8cc07fu, 0xd8340000u, 0x00000403u, 0xbf8cc07fu, 0xd84c0000u, + 0x00000503u, 0xbf8cc07fu, 0xd8d80000u, 0x13000003u, 0xbf8cc07fu, 0xd8340000u, 0x00000403u, 0xbf8cc07fu, + 0xd8440000u, 0x000a0503u, 0xbf8cc07fu, 0xd8d80000u, 0x14000003u, 0xbf8cc07fu, 0xd9340000u, 0x00000603u, + 0xbf8cc07fu, 0xd9c80000u, 0x18000803u, 0xbf8cc07fu, 0xd9d80000u, 0x16000003u, 0xbf8cc07fu, 0xd9340000u, + 0x00000603u, 0xbf8cc07fu, 0xd9cc0000u, 0x1c000803u, 0xbf8cc07fu, 0xd9d80000u, 0x1a000003u, 0xbf8cc07fu, + 0xd9340000u, 0x00000603u, 0xbf8cc07fu, 0xd9480000u, 0x00000803u, 0xbf8cc07fu, 0xd9d80000u, 0x1e000003u, + 0xbf8cc07fu, 0xd9340000u, 0x00000603u, 0xbf8cc07fu, 0xd94c0000u, 0x00000803u, 0xbf8cc07fu, 0xd9d80000u, + 0x20000003u, 0xbf8cc07fu, 0xd9340000u, 0x00000603u, 0xbf8cc07fu, 0xd9c40000u, 0x24280803u, 0xbf8cc07fu, + 0xd9d80000u, 0x22000003u, 0xbf8cc07fu, 0xd9340000u, 0x00000603u, 0xbf8cc07fu, 0xd9440000u, 0x00280803u, + 0xbf8cc07fu, 0xd9d80000u, 0x26000003u, 0xbf8cc07fu, 0xe0781000u, 0x80010c02u, 0xe0781010u, 0x80011002u, + 0xe0781020u, 0x80011402u, 0xe0781030u, 0x80011802u, 0xe0781040u, 0x80011c02u, 0xe0781050u, 0x80012002u, + 0xe0781060u, 0x80012402u, 0xbf810000u, +}; + +constexpr std::uint32_t Rows[32][8] = { + {0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x45464748u, 0x00000000u}, + {0x00000000u, 0x80000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x80000000u, 0x45464748u, 0x00000000u}, + {0x80000000u, 0x00000000u, 0x00000000u, 0x80000000u, 0x00000000u, 0x00000000u, 0x45464748u, 0x00000000u}, + {0x3f800000u, 0xbf800000u, 0x00000000u, 0x3ff00000u, 0x00000000u, 0xbff00000u, 0x45464748u, 0x00000000u}, + {0xbf800000u, 0x3f800000u, 0x00000000u, 0xbff00000u, 0x00000000u, 0x3ff00000u, 0x45464748u, 0x00000000u}, + {0x7f800000u, 0xff800000u, 0x00000000u, 0x7ff00000u, 0x00000000u, 0xfff00000u, 0x45464748u, 0x00000000u}, + {0xff800000u, 0x7f800000u, 0x00000000u, 0xfff00000u, 0x00000000u, 0x7ff00000u, 0x45464748u, 0x00000000u}, + {0x00000001u, 0x00000000u, 0x00000001u, 0x00000000u, 0x00000000u, 0x00000000u, 0x45464748u, 0x00000000u}, + {0x00000000u, 0x00000001u, 0x00000000u, 0x00000000u, 0x00000001u, 0x00000000u, 0x45464748u, 0x00000000u}, + {0x80000001u, 0x80000000u, 0x00000001u, 0x80000000u, 0x00000000u, 0x80000000u, 0x45464748u, 0x00000000u}, + {0x00000001u, 0x80000001u, 0x00000001u, 0x00000000u, 0x00000001u, 0x80000000u, 0x45464748u, 0x00000000u}, + {0x3f800000u, 0x00000001u, 0x00000000u, 0x3ff00000u, 0x00000001u, 0x00000000u, 0x45464748u, 0x00000000u}, + {0x7fc00000u, 0x3f800000u, 0x00000000u, 0x7ff80000u, 0x00000000u, 0x3ff00000u, 0x45464748u, 0x00000000u}, + {0x3f800000u, 0x7fc00000u, 0x00000000u, 0x3ff00000u, 0x00000000u, 0x7ff80000u, 0x45464748u, 0x00000000u}, + {0x7fc00000u, 0x7fc00123u, 0x00000000u, 0x7ff80000u, 0x00000123u, 0x7ff80000u, 0x45464748u, 0x00000000u}, + {0x7fc00123u, 0x7fc00000u, 0x00000123u, 0x7ff80000u, 0x00000000u, 0x7ff80000u, 0x45464748u, 0x00000000u}, + {0xffc00000u, 0x7fc00000u, 0x00000000u, 0xfff80000u, 0x00000000u, 0x7ff80000u, 0x45464748u, 0x00000000u}, + {0x7fc00000u, 0x7fc00000u, 0x00000000u, 0x7ff80000u, 0x00000000u, 0x7ff80000u, 0x45464748u, 0x00000000u}, + {0x7f800001u, 0x3f800000u, 0x00000001u, 0x7ff00000u, 0x00000000u, 0x3ff00000u, 0x45464748u, 0x00000000u}, + {0x3f800000u, 0x7f800001u, 0x00000000u, 0x3ff00000u, 0x00000001u, 0x7ff00000u, 0x45464748u, 0x00000000u}, + {0x7f800001u, 0x7fc00000u, 0x00000001u, 0x7ff00000u, 0x00000000u, 0x7ff80000u, 0x45464748u, 0x00000000u}, + {0x7fc00000u, 0x7f800001u, 0x00000000u, 0x7ff80000u, 0x00000001u, 0x7ff00000u, 0x45464748u, 0x00000000u}, + {0x7f800001u, 0x7f800123u, 0x00000001u, 0x7ff00000u, 0x00000123u, 0x7ff00000u, 0x45464748u, 0x00000000u}, + {0x7f800123u, 0x7f800001u, 0x00000123u, 0x7ff00000u, 0x00000001u, 0x7ff00000u, 0x45464748u, 0x00000000u}, + {0xff800001u, 0x7f800001u, 0x00000001u, 0xfff00000u, 0x00000001u, 0x7ff00000u, 0x45464748u, 0x00000000u}, + {0x7f800001u, 0xff800001u, 0x00000001u, 0x7ff00000u, 0x00000001u, 0xfff00000u, 0x45464748u, 0x00000000u}, + {0x00000001u, 0x7f800001u, 0x00000001u, 0x00000000u, 0x00000001u, 0x7ff00000u, 0x45464748u, 0x00000000u}, + {0x7f800001u, 0x00000001u, 0x00000001u, 0x7ff00000u, 0x00000001u, 0x00000000u, 0x45464748u, 0x00000000u}, + {0x7f800000u, 0x7f800001u, 0x00000000u, 0x7ff00000u, 0x00000001u, 0x7ff00000u, 0x45464748u, 0x00000000u}, + {0x7f800001u, 0x7f800000u, 0x00000001u, 0x7ff00000u, 0x00000000u, 0x7ff00000u, 0x45464748u, 0x00000000u}, + {0x00000000u, 0x7f800001u, 0x00000000u, 0x00000000u, 0x00000001u, 0x7ff00000u, 0x45464748u, 0x00000000u}, + {0x7f800001u, 0x00000000u, 0x00000001u, 0x7ff00000u, 0x00000000u, 0x00000000u, 0x45464748u, 0x00000000u} +}; + +constexpr std::uint32_t ExpectedTitles[32][Columns] = { + {0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x45464748u, 0x00000000u, 0x00000000u, 0x00000000u, 0x45464748u, + 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, + 0x00000000u, 0x00000000u, 0x00000000u, 0x45464748u, 0x41424344u, 0x00000000u, 0x00000000u, 0x45464748u, 0x41424344u}, + {0x80000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x45464748u, 0x00000000u, 0x80000000u, 0x00000000u, 0x45464748u, + 0x00000000u, 0x80000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, + 0x80000000u, 0x00000000u, 0x00000000u, 0x45464748u, 0x41424344u, 0x00000000u, 0x00000000u, 0x45464748u, 0x41424344u}, + {0x80000000u, 0x80000000u, 0x00000000u, 0x80000000u, 0x45464748u, 0x80000000u, 0x80000000u, 0x00000000u, 0x45464748u, + 0x00000000u, 0x80000000u, 0x00000000u, 0x80000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x80000000u, 0x00000000u, + 0x80000000u, 0x00000000u, 0x00000000u, 0x45464748u, 0x41424344u, 0x00000000u, 0x80000000u, 0x45464748u, 0x41424344u}, + {0xbf800000u, 0x3f800000u, 0x3f800000u, 0x3f800000u, 0x3f800000u, 0x3f800000u, 0xbf800000u, 0x3f800000u, 0x3f800000u, + 0x00000000u, 0xbff00000u, 0x00000000u, 0x3ff00000u, 0x00000000u, 0x3ff00000u, 0x00000000u, 0x3ff00000u, 0x00000000u, + 0xbff00000u, 0x00000000u, 0x3ff00000u, 0x00000000u, 0x3ff00000u, 0x00000000u, 0x3ff00000u, 0x00000000u, 0x3ff00000u}, + {0xbf800000u, 0xbf800000u, 0x3f800000u, 0xbf800000u, 0xbf800000u, 0xbf800000u, 0xbf800000u, 0x3f800000u, 0xbf800000u, + 0x00000000u, 0xbff00000u, 0x00000000u, 0xbff00000u, 0x00000000u, 0x3ff00000u, 0x00000000u, 0xbff00000u, 0x00000000u, + 0xbff00000u, 0x00000000u, 0x3ff00000u, 0x00000000u, 0xbff00000u, 0x00000000u, 0xbff00000u, 0x00000000u, 0xbff00000u}, + {0xff800000u, 0x7f800000u, 0x7f800000u, 0x7f800000u, 0x7f800000u, 0x7f800000u, 0xff800000u, 0x7f800000u, 0x7f800000u, + 0x00000000u, 0xfff00000u, 0x00000000u, 0x7ff00000u, 0x00000000u, 0x7ff00000u, 0x00000000u, 0x7ff00000u, 0x00000000u, + 0xfff00000u, 0x00000000u, 0x7ff00000u, 0x00000000u, 0x7ff00000u, 0x00000000u, 0x7ff00000u, 0x00000000u, 0x7ff00000u}, + {0xff800000u, 0xff800000u, 0x7f800000u, 0xff800000u, 0xff800000u, 0xff800000u, 0xff800000u, 0x7f800000u, 0xff800000u, + 0x00000000u, 0xfff00000u, 0x00000000u, 0xfff00000u, 0x00000000u, 0x7ff00000u, 0x00000000u, 0xfff00000u, 0x00000000u, + 0xfff00000u, 0x00000000u, 0x7ff00000u, 0x00000000u, 0xfff00000u, 0x00000000u, 0xfff00000u, 0x00000000u, 0xfff00000u}, + {0x00000000u, 0x00000001u, 0x00000000u, 0x00000001u, 0x45464748u, 0x00000001u, 0x00000000u, 0x00000000u, 0x45464748u, + 0x00000000u, 0x00000000u, 0x00000001u, 0x00000000u, 0x00000001u, 0x00000000u, 0x00000001u, 0x00000000u, 0x00000000u, + 0x00000000u, 0x00000001u, 0x00000000u, 0x00000001u, 0x00000000u, 0x00000001u, 0x00000000u, 0x00000001u, 0x00000000u}, + {0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x45464748u, 0x00000000u, 0x00000000u, 0x00000000u, 0x45464748u, + 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000001u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, + 0x00000000u, 0x00000001u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u}, + {0x80000000u, 0x80000001u, 0x80000000u, 0x80000001u, 0x45464748u, 0x80000001u, 0x80000000u, 0x80000000u, 0x45464748u, + 0x00000001u, 0x80000000u, 0x00000001u, 0x80000000u, 0x00000000u, 0x80000000u, 0x00000001u, 0x80000000u, 0x00000001u, + 0x80000000u, 0x00000000u, 0x80000000u, 0x00000001u, 0x80000000u, 0x00000001u, 0x80000000u, 0x00000001u, 0x80000000u}, + {0x80000000u, 0x00000001u, 0x00000000u, 0x00000001u, 0x45464748u, 0x00000001u, 0x80000000u, 0x00000000u, 0x45464748u, + 0x00000001u, 0x80000000u, 0x00000001u, 0x00000000u, 0x00000001u, 0x00000000u, 0x00000001u, 0x00000000u, 0x00000001u, + 0x80000000u, 0x00000001u, 0x00000000u, 0x00000001u, 0x00000000u, 0x00000001u, 0x00000000u, 0x00000001u, 0x00000000u}, + {0x00000000u, 0x3f800000u, 0x3f800000u, 0x3f800000u, 0x3f800000u, 0x3f800000u, 0x00000000u, 0x3f800000u, 0x3f800000u, + 0x00000001u, 0x00000000u, 0x00000000u, 0x3ff00000u, 0x00000000u, 0x3ff00000u, 0x00000000u, 0x3ff00000u, 0x00000001u, + 0x00000000u, 0x00000000u, 0x3ff00000u, 0x00000000u, 0x3ff00000u, 0x00000000u, 0x3ff00000u, 0x00000000u, 0x3ff00000u}, + {0x3f800000u, 0x7fc00000u, 0x3f800000u, 0x7fc00000u, 0x7fc00000u, 0x7fc00000u, 0x3f800000u, 0x3f800000u, 0x7fc00000u, + 0x00000000u, 0x3ff00000u, 0x00000000u, 0x7ff80000u, 0x00000000u, 0x3ff00000u, 0x00000000u, 0x7ff80000u, 0x00000000u, + 0x3ff00000u, 0x00000000u, 0x3ff00000u, 0x00000000u, 0x7ff80000u, 0x00000000u, 0x7ff80000u, 0x00000000u, 0x7ff80000u}, + {0x3f800000u, 0x3f800000u, 0x3f800000u, 0x3f800000u, 0x3f800000u, 0x3f800000u, 0x3f800000u, 0x3f800000u, 0x3f800000u, + 0x00000000u, 0x3ff00000u, 0x00000000u, 0x3ff00000u, 0x00000000u, 0x3ff00000u, 0x00000000u, 0x3ff00000u, 0x00000000u, + 0x3ff00000u, 0x00000000u, 0x3ff00000u, 0x00000000u, 0x3ff00000u, 0x00000000u, 0x3ff00000u, 0x00000000u, 0x3ff00000u}, + {0x7fc00123u, 0x7fc00000u, 0x7fc00123u, 0x7fc00000u, 0x7fc00000u, 0x7fc00000u, 0x7fc00123u, 0x7fc00123u, 0x7fc00000u, + 0x00000123u, 0x7ff80000u, 0x00000000u, 0x7ff80000u, 0x00000123u, 0x7ff80000u, 0x00000000u, 0x7ff80000u, 0x00000123u, + 0x7ff80000u, 0x00000123u, 0x7ff80000u, 0x00000000u, 0x7ff80000u, 0x00000000u, 0x7ff80000u, 0x00000000u, 0x7ff80000u}, + {0x7fc00000u, 0x7fc00123u, 0x7fc00000u, 0x7fc00123u, 0x7fc00123u, 0x7fc00123u, 0x7fc00000u, 0x7fc00000u, 0x7fc00123u, + 0x00000000u, 0x7ff80000u, 0x00000123u, 0x7ff80000u, 0x00000000u, 0x7ff80000u, 0x00000123u, 0x7ff80000u, 0x00000000u, + 0x7ff80000u, 0x00000000u, 0x7ff80000u, 0x00000123u, 0x7ff80000u, 0x00000123u, 0x7ff80000u, 0x00000123u, 0x7ff80000u}, + {0x7fc00000u, 0xffc00000u, 0x7fc00000u, 0xffc00000u, 0xffc00000u, 0xffc00000u, 0x7fc00000u, 0x7fc00000u, 0xffc00000u, + 0x00000000u, 0x7ff80000u, 0x00000000u, 0xfff80000u, 0x00000000u, 0x7ff80000u, 0x00000000u, 0xfff80000u, 0x00000000u, + 0x7ff80000u, 0x00000000u, 0x7ff80000u, 0x00000000u, 0xfff80000u, 0x00000000u, 0xfff80000u, 0x00000000u, 0xfff80000u}, + {0x7fc00000u, 0x7fc00000u, 0x7fc00000u, 0x7fc00000u, 0x7fc00000u, 0x7fc00000u, 0x7fc00000u, 0x7fc00000u, 0x7fc00000u, + 0x00000000u, 0x7ff80000u, 0x00000000u, 0x7ff80000u, 0x00000000u, 0x7ff80000u, 0x00000000u, 0x7ff80000u, 0x00000000u, + 0x7ff80000u, 0x00000000u, 0x7ff80000u, 0x00000000u, 0x7ff80000u, 0x00000000u, 0x7ff80000u, 0x00000000u, 0x7ff80000u}, + {0x7fc00001u, 0x7f800001u, 0x7fc00001u, 0x7f800001u, 0x7f800001u, 0x7f800001u, 0x7fc00001u, 0x7fc00001u, 0x7f800001u, + 0x00000001u, 0x7ff80000u, 0x00000001u, 0x7ff00000u, 0x00000001u, 0x7ff80000u, 0x00000001u, 0x7ff00000u, 0x00000001u, + 0x7ff80000u, 0x00000001u, 0x7ff80000u, 0x00000001u, 0x7ff00000u, 0x00000001u, 0x7ff00000u, 0x00000001u, 0x7ff00000u}, + {0x7fc00001u, 0x3f800000u, 0x7fc00001u, 0x3f800000u, 0x3f800000u, 0x3f800000u, 0x7fc00001u, 0x7fc00001u, 0x3f800000u, + 0x00000001u, 0x7ff80000u, 0x00000000u, 0x3ff00000u, 0x00000001u, 0x7ff80000u, 0x00000000u, 0x3ff00000u, 0x00000001u, + 0x7ff80000u, 0x00000001u, 0x7ff80000u, 0x00000000u, 0x3ff00000u, 0x00000000u, 0x3ff00000u, 0x00000000u, 0x3ff00000u}, + {0x7fc00001u, 0x7f800001u, 0x7fc00001u, 0x7f800001u, 0x7f800001u, 0x7f800001u, 0x7fc00001u, 0x7fc00001u, 0x7f800001u, + 0x00000001u, 0x7ff80000u, 0x00000001u, 0x7ff00000u, 0x00000001u, 0x7ff80000u, 0x00000001u, 0x7ff00000u, 0x00000001u, + 0x7ff80000u, 0x00000001u, 0x7ff80000u, 0x00000001u, 0x7ff00000u, 0x00000001u, 0x7ff00000u, 0x00000001u, 0x7ff00000u}, + {0x7fc00001u, 0x7fc00000u, 0x7fc00001u, 0x7fc00000u, 0x7fc00000u, 0x7fc00000u, 0x7fc00001u, 0x7fc00001u, 0x7fc00000u, + 0x00000001u, 0x7ff80000u, 0x00000000u, 0x7ff80000u, 0x00000001u, 0x7ff80000u, 0x00000000u, 0x7ff80000u, 0x00000001u, + 0x7ff80000u, 0x00000001u, 0x7ff80000u, 0x00000000u, 0x7ff80000u, 0x00000000u, 0x7ff80000u, 0x00000000u, 0x7ff80000u}, + {0x7fc00123u, 0x7f800001u, 0x7fc00123u, 0x7f800001u, 0x7f800001u, 0x7f800001u, 0x7fc00123u, 0x7fc00123u, 0x7f800001u, + 0x00000123u, 0x7ff80000u, 0x00000001u, 0x7ff00000u, 0x00000123u, 0x7ff80000u, 0x00000001u, 0x7ff00000u, 0x00000123u, + 0x7ff80000u, 0x00000123u, 0x7ff80000u, 0x00000001u, 0x7ff00000u, 0x00000001u, 0x7ff00000u, 0x00000001u, 0x7ff00000u}, + {0x7fc00001u, 0x7f800123u, 0x7fc00001u, 0x7f800123u, 0x7f800123u, 0x7f800123u, 0x7fc00001u, 0x7fc00001u, 0x7f800123u, + 0x00000001u, 0x7ff80000u, 0x00000123u, 0x7ff00000u, 0x00000001u, 0x7ff80000u, 0x00000123u, 0x7ff00000u, 0x00000001u, + 0x7ff80000u, 0x00000001u, 0x7ff80000u, 0x00000123u, 0x7ff00000u, 0x00000123u, 0x7ff00000u, 0x00000123u, 0x7ff00000u}, + {0x7fc00001u, 0xff800001u, 0x7fc00001u, 0xff800001u, 0xff800001u, 0xff800001u, 0x7fc00001u, 0x7fc00001u, 0xff800001u, + 0x00000001u, 0x7ff80000u, 0x00000001u, 0xfff00000u, 0x00000001u, 0x7ff80000u, 0x00000001u, 0xfff00000u, 0x00000001u, + 0x7ff80000u, 0x00000001u, 0x7ff80000u, 0x00000001u, 0xfff00000u, 0x00000001u, 0xfff00000u, 0x00000001u, 0xfff00000u}, + {0xffc00001u, 0x7f800001u, 0xffc00001u, 0x7f800001u, 0x7f800001u, 0x7f800001u, 0xffc00001u, 0xffc00001u, 0x7f800001u, + 0x00000001u, 0xfff80000u, 0x00000001u, 0x7ff00000u, 0x00000001u, 0xfff80000u, 0x00000001u, 0x7ff00000u, 0x00000001u, + 0xfff80000u, 0x00000001u, 0xfff80000u, 0x00000001u, 0x7ff00000u, 0x00000001u, 0x7ff00000u, 0x00000001u, 0x7ff00000u}, + {0x7fc00001u, 0x00000001u, 0x7fc00001u, 0x00000001u, 0x00000001u, 0x00000001u, 0x7fc00001u, 0x7fc00001u, 0x00000001u, + 0x00000001u, 0x7ff80000u, 0x00000001u, 0x00000000u, 0x00000001u, 0x7ff80000u, 0x00000001u, 0x00000000u, 0x00000001u, + 0x7ff80000u, 0x00000001u, 0x7ff80000u, 0x00000001u, 0x00000000u, 0x00000001u, 0x00000000u, 0x00000001u, 0x00000000u}, + {0x7fc00001u, 0x7f800001u, 0x7fc00001u, 0x7f800001u, 0x7f800001u, 0x7f800001u, 0x7fc00001u, 0x7fc00001u, 0x7f800001u, + 0x00000001u, 0x7ff80000u, 0x00000001u, 0x7ff00000u, 0x00000001u, 0x7ff80000u, 0x00000001u, 0x7ff00000u, 0x00000001u, + 0x7ff80000u, 0x00000001u, 0x7ff80000u, 0x00000001u, 0x7ff00000u, 0x00000001u, 0x7ff00000u, 0x00000001u, 0x7ff00000u}, + {0x7fc00001u, 0x7f800000u, 0x7fc00001u, 0x7f800000u, 0x7f800000u, 0x7f800000u, 0x7fc00001u, 0x7fc00001u, 0x7f800000u, + 0x00000001u, 0x7ff80000u, 0x00000000u, 0x7ff00000u, 0x00000001u, 0x7ff80000u, 0x00000000u, 0x7ff00000u, 0x00000001u, + 0x7ff80000u, 0x00000001u, 0x7ff80000u, 0x00000000u, 0x7ff00000u, 0x00000000u, 0x7ff00000u, 0x00000000u, 0x7ff00000u}, + {0x7fc00001u, 0x7f800001u, 0x7fc00001u, 0x7f800001u, 0x7f800001u, 0x7f800001u, 0x7fc00001u, 0x7fc00001u, 0x7f800001u, + 0x00000001u, 0x7ff80000u, 0x00000001u, 0x7ff00000u, 0x00000001u, 0x7ff80000u, 0x00000001u, 0x7ff00000u, 0x00000001u, + 0x7ff80000u, 0x00000001u, 0x7ff80000u, 0x00000001u, 0x7ff00000u, 0x00000001u, 0x7ff00000u, 0x00000001u, 0x7ff00000u}, + {0x7fc00001u, 0x00000000u, 0x7fc00001u, 0x00000000u, 0x00000000u, 0x00000000u, 0x7fc00001u, 0x7fc00001u, 0x00000000u, + 0x00000001u, 0x7ff80000u, 0x00000000u, 0x00000000u, 0x00000001u, 0x7ff80000u, 0x00000000u, 0x00000000u, 0x00000001u, + 0x7ff80000u, 0x00000001u, 0x7ff80000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u}, + {0x7fc00001u, 0x7f800001u, 0x7fc00001u, 0x7f800001u, 0x7f800001u, 0x7f800001u, 0x7fc00001u, 0x7fc00001u, 0x7f800001u, + 0x00000001u, 0x7ff80000u, 0x00000001u, 0x7ff00000u, 0x00000001u, 0x7ff80000u, 0x00000001u, 0x7ff00000u, 0x00000001u, + 0x7ff80000u, 0x00000001u, 0x7ff80000u, 0x00000001u, 0x7ff00000u, 0x00000001u, 0x7ff00000u, 0x00000001u, 0x7ff00000u} +}; + +constexpr std::uint32_t ExpectedKept[32][Columns] = { + {0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x45464748u, 0x00000000u, 0x00000000u, 0x00000000u, 0x45464748u, + 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, + 0x00000000u, 0x00000000u, 0x00000000u, 0x45464748u, 0x41424344u, 0x00000000u, 0x00000000u, 0x45464748u, 0x41424344u}, + {0x80000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x45464748u, 0x00000000u, 0x80000000u, 0x00000000u, 0x45464748u, + 0x00000000u, 0x80000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, + 0x80000000u, 0x00000000u, 0x00000000u, 0x45464748u, 0x41424344u, 0x00000000u, 0x00000000u, 0x45464748u, 0x41424344u}, + {0x80000000u, 0x80000000u, 0x00000000u, 0x80000000u, 0x45464748u, 0x80000000u, 0x80000000u, 0x00000000u, 0x45464748u, + 0x00000000u, 0x80000000u, 0x00000000u, 0x80000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x80000000u, 0x00000000u, + 0x80000000u, 0x00000000u, 0x00000000u, 0x45464748u, 0x41424344u, 0x00000000u, 0x80000000u, 0x45464748u, 0x41424344u}, + {0xbf800000u, 0x3f800000u, 0x3f800000u, 0x3f800000u, 0x3f800000u, 0x3f800000u, 0xbf800000u, 0x3f800000u, 0x3f800000u, + 0x00000000u, 0xbff00000u, 0x00000000u, 0x3ff00000u, 0x00000000u, 0x3ff00000u, 0x00000000u, 0x3ff00000u, 0x00000000u, + 0xbff00000u, 0x00000000u, 0x3ff00000u, 0x00000000u, 0x3ff00000u, 0x00000000u, 0x3ff00000u, 0x00000000u, 0x3ff00000u}, + {0xbf800000u, 0xbf800000u, 0x3f800000u, 0xbf800000u, 0xbf800000u, 0xbf800000u, 0xbf800000u, 0x3f800000u, 0xbf800000u, + 0x00000000u, 0xbff00000u, 0x00000000u, 0xbff00000u, 0x00000000u, 0x3ff00000u, 0x00000000u, 0xbff00000u, 0x00000000u, + 0xbff00000u, 0x00000000u, 0x3ff00000u, 0x00000000u, 0xbff00000u, 0x00000000u, 0xbff00000u, 0x00000000u, 0xbff00000u}, + {0xff800000u, 0x7f800000u, 0x7f800000u, 0x7f800000u, 0x7f800000u, 0x7f800000u, 0xff800000u, 0x7f800000u, 0x7f800000u, + 0x00000000u, 0xfff00000u, 0x00000000u, 0x7ff00000u, 0x00000000u, 0x7ff00000u, 0x00000000u, 0x7ff00000u, 0x00000000u, + 0xfff00000u, 0x00000000u, 0x7ff00000u, 0x00000000u, 0x7ff00000u, 0x00000000u, 0x7ff00000u, 0x00000000u, 0x7ff00000u}, + {0xff800000u, 0xff800000u, 0x7f800000u, 0xff800000u, 0xff800000u, 0xff800000u, 0xff800000u, 0x7f800000u, 0xff800000u, + 0x00000000u, 0xfff00000u, 0x00000000u, 0xfff00000u, 0x00000000u, 0x7ff00000u, 0x00000000u, 0xfff00000u, 0x00000000u, + 0xfff00000u, 0x00000000u, 0x7ff00000u, 0x00000000u, 0xfff00000u, 0x00000000u, 0xfff00000u, 0x00000000u, 0xfff00000u}, + {0x00000000u, 0x00000001u, 0x00000001u, 0x00000001u, 0x00000001u, 0x00000001u, 0x00000000u, 0x00000001u, 0x00000001u, + 0x00000000u, 0x00000000u, 0x00000001u, 0x00000000u, 0x00000001u, 0x00000000u, 0x00000001u, 0x00000000u, 0x00000000u, + 0x00000000u, 0x00000001u, 0x00000000u, 0x00000001u, 0x00000000u, 0x00000001u, 0x00000000u, 0x00000001u, 0x00000000u}, + {0x00000000u, 0x00000000u, 0x00000001u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000001u, 0x00000000u, + 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000001u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, + 0x00000000u, 0x00000001u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u}, + {0x80000001u, 0x80000001u, 0x80000000u, 0x80000001u, 0x80000001u, 0x80000001u, 0x80000001u, 0x80000000u, 0x80000001u, + 0x00000001u, 0x80000000u, 0x00000001u, 0x80000000u, 0x00000000u, 0x80000000u, 0x00000001u, 0x80000000u, 0x00000001u, + 0x80000000u, 0x00000000u, 0x80000000u, 0x00000001u, 0x80000000u, 0x00000001u, 0x80000000u, 0x00000001u, 0x80000000u}, + {0x80000001u, 0x00000001u, 0x00000001u, 0x00000001u, 0x00000001u, 0x00000001u, 0x80000001u, 0x00000001u, 0x00000001u, + 0x00000001u, 0x80000000u, 0x00000001u, 0x00000000u, 0x00000001u, 0x00000000u, 0x00000001u, 0x00000000u, 0x00000001u, + 0x80000000u, 0x00000001u, 0x00000000u, 0x00000001u, 0x00000000u, 0x00000001u, 0x00000000u, 0x00000001u, 0x00000000u}, + {0x00000001u, 0x3f800000u, 0x3f800000u, 0x3f800000u, 0x3f800000u, 0x3f800000u, 0x00000001u, 0x3f800000u, 0x3f800000u, + 0x00000001u, 0x00000000u, 0x00000000u, 0x3ff00000u, 0x00000000u, 0x3ff00000u, 0x00000000u, 0x3ff00000u, 0x00000001u, + 0x00000000u, 0x00000000u, 0x3ff00000u, 0x00000000u, 0x3ff00000u, 0x00000000u, 0x3ff00000u, 0x00000000u, 0x3ff00000u}, + {0x3f800000u, 0x7fc00000u, 0x3f800000u, 0x7fc00000u, 0x7fc00000u, 0x7fc00000u, 0x3f800000u, 0x3f800000u, 0x7fc00000u, + 0x00000000u, 0x3ff00000u, 0x00000000u, 0x7ff80000u, 0x00000000u, 0x3ff00000u, 0x00000000u, 0x7ff80000u, 0x00000000u, + 0x3ff00000u, 0x00000000u, 0x3ff00000u, 0x00000000u, 0x7ff80000u, 0x00000000u, 0x7ff80000u, 0x00000000u, 0x7ff80000u}, + {0x3f800000u, 0x3f800000u, 0x3f800000u, 0x3f800000u, 0x3f800000u, 0x3f800000u, 0x3f800000u, 0x3f800000u, 0x3f800000u, + 0x00000000u, 0x3ff00000u, 0x00000000u, 0x3ff00000u, 0x00000000u, 0x3ff00000u, 0x00000000u, 0x3ff00000u, 0x00000000u, + 0x3ff00000u, 0x00000000u, 0x3ff00000u, 0x00000000u, 0x3ff00000u, 0x00000000u, 0x3ff00000u, 0x00000000u, 0x3ff00000u}, + {0x7fc00123u, 0x7fc00000u, 0x7fc00123u, 0x7fc00000u, 0x7fc00000u, 0x7fc00000u, 0x7fc00123u, 0x7fc00123u, 0x7fc00000u, + 0x00000123u, 0x7ff80000u, 0x00000000u, 0x7ff80000u, 0x00000123u, 0x7ff80000u, 0x00000000u, 0x7ff80000u, 0x00000123u, + 0x7ff80000u, 0x00000123u, 0x7ff80000u, 0x00000000u, 0x7ff80000u, 0x00000000u, 0x7ff80000u, 0x00000000u, 0x7ff80000u}, + {0x7fc00000u, 0x7fc00123u, 0x7fc00000u, 0x7fc00123u, 0x7fc00123u, 0x7fc00123u, 0x7fc00000u, 0x7fc00000u, 0x7fc00123u, + 0x00000000u, 0x7ff80000u, 0x00000123u, 0x7ff80000u, 0x00000000u, 0x7ff80000u, 0x00000123u, 0x7ff80000u, 0x00000000u, + 0x7ff80000u, 0x00000000u, 0x7ff80000u, 0x00000123u, 0x7ff80000u, 0x00000123u, 0x7ff80000u, 0x00000123u, 0x7ff80000u}, + {0x7fc00000u, 0xffc00000u, 0x7fc00000u, 0xffc00000u, 0xffc00000u, 0xffc00000u, 0x7fc00000u, 0x7fc00000u, 0xffc00000u, + 0x00000000u, 0x7ff80000u, 0x00000000u, 0xfff80000u, 0x00000000u, 0x7ff80000u, 0x00000000u, 0xfff80000u, 0x00000000u, + 0x7ff80000u, 0x00000000u, 0x7ff80000u, 0x00000000u, 0xfff80000u, 0x00000000u, 0xfff80000u, 0x00000000u, 0xfff80000u}, + {0x7fc00000u, 0x7fc00000u, 0x7fc00000u, 0x7fc00000u, 0x7fc00000u, 0x7fc00000u, 0x7fc00000u, 0x7fc00000u, 0x7fc00000u, + 0x00000000u, 0x7ff80000u, 0x00000000u, 0x7ff80000u, 0x00000000u, 0x7ff80000u, 0x00000000u, 0x7ff80000u, 0x00000000u, + 0x7ff80000u, 0x00000000u, 0x7ff80000u, 0x00000000u, 0x7ff80000u, 0x00000000u, 0x7ff80000u, 0x00000000u, 0x7ff80000u}, + {0x7fc00001u, 0x7f800001u, 0x7fc00001u, 0x7f800001u, 0x7f800001u, 0x7f800001u, 0x7fc00001u, 0x7fc00001u, 0x7f800001u, + 0x00000001u, 0x7ff80000u, 0x00000001u, 0x7ff00000u, 0x00000001u, 0x7ff80000u, 0x00000001u, 0x7ff00000u, 0x00000001u, + 0x7ff80000u, 0x00000001u, 0x7ff80000u, 0x00000001u, 0x7ff00000u, 0x00000001u, 0x7ff00000u, 0x00000001u, 0x7ff00000u}, + {0x7fc00001u, 0x3f800000u, 0x7fc00001u, 0x3f800000u, 0x3f800000u, 0x3f800000u, 0x7fc00001u, 0x7fc00001u, 0x3f800000u, + 0x00000001u, 0x7ff80000u, 0x00000000u, 0x3ff00000u, 0x00000001u, 0x7ff80000u, 0x00000000u, 0x3ff00000u, 0x00000001u, + 0x7ff80000u, 0x00000001u, 0x7ff80000u, 0x00000000u, 0x3ff00000u, 0x00000000u, 0x3ff00000u, 0x00000000u, 0x3ff00000u}, + {0x7fc00001u, 0x7f800001u, 0x7fc00001u, 0x7f800001u, 0x7f800001u, 0x7f800001u, 0x7fc00001u, 0x7fc00001u, 0x7f800001u, + 0x00000001u, 0x7ff80000u, 0x00000001u, 0x7ff00000u, 0x00000001u, 0x7ff80000u, 0x00000001u, 0x7ff00000u, 0x00000001u, + 0x7ff80000u, 0x00000001u, 0x7ff80000u, 0x00000001u, 0x7ff00000u, 0x00000001u, 0x7ff00000u, 0x00000001u, 0x7ff00000u}, + {0x7fc00001u, 0x7fc00000u, 0x7fc00001u, 0x7fc00000u, 0x7fc00000u, 0x7fc00000u, 0x7fc00001u, 0x7fc00001u, 0x7fc00000u, + 0x00000001u, 0x7ff80000u, 0x00000000u, 0x7ff80000u, 0x00000001u, 0x7ff80000u, 0x00000000u, 0x7ff80000u, 0x00000001u, + 0x7ff80000u, 0x00000001u, 0x7ff80000u, 0x00000000u, 0x7ff80000u, 0x00000000u, 0x7ff80000u, 0x00000000u, 0x7ff80000u}, + {0x7fc00123u, 0x7f800001u, 0x7fc00123u, 0x7f800001u, 0x7f800001u, 0x7f800001u, 0x7fc00123u, 0x7fc00123u, 0x7f800001u, + 0x00000123u, 0x7ff80000u, 0x00000001u, 0x7ff00000u, 0x00000123u, 0x7ff80000u, 0x00000001u, 0x7ff00000u, 0x00000123u, + 0x7ff80000u, 0x00000123u, 0x7ff80000u, 0x00000001u, 0x7ff00000u, 0x00000001u, 0x7ff00000u, 0x00000001u, 0x7ff00000u}, + {0x7fc00001u, 0x7f800123u, 0x7fc00001u, 0x7f800123u, 0x7f800123u, 0x7f800123u, 0x7fc00001u, 0x7fc00001u, 0x7f800123u, + 0x00000001u, 0x7ff80000u, 0x00000123u, 0x7ff00000u, 0x00000001u, 0x7ff80000u, 0x00000123u, 0x7ff00000u, 0x00000001u, + 0x7ff80000u, 0x00000001u, 0x7ff80000u, 0x00000123u, 0x7ff00000u, 0x00000123u, 0x7ff00000u, 0x00000123u, 0x7ff00000u}, + {0x7fc00001u, 0xff800001u, 0x7fc00001u, 0xff800001u, 0xff800001u, 0xff800001u, 0x7fc00001u, 0x7fc00001u, 0xff800001u, + 0x00000001u, 0x7ff80000u, 0x00000001u, 0xfff00000u, 0x00000001u, 0x7ff80000u, 0x00000001u, 0xfff00000u, 0x00000001u, + 0x7ff80000u, 0x00000001u, 0x7ff80000u, 0x00000001u, 0xfff00000u, 0x00000001u, 0xfff00000u, 0x00000001u, 0xfff00000u}, + {0xffc00001u, 0x7f800001u, 0xffc00001u, 0x7f800001u, 0x7f800001u, 0x7f800001u, 0xffc00001u, 0xffc00001u, 0x7f800001u, + 0x00000001u, 0xfff80000u, 0x00000001u, 0x7ff00000u, 0x00000001u, 0xfff80000u, 0x00000001u, 0x7ff00000u, 0x00000001u, + 0xfff80000u, 0x00000001u, 0xfff80000u, 0x00000001u, 0x7ff00000u, 0x00000001u, 0x7ff00000u, 0x00000001u, 0x7ff00000u}, + {0x7fc00001u, 0x00000001u, 0x7fc00001u, 0x00000001u, 0x00000001u, 0x00000001u, 0x7fc00001u, 0x7fc00001u, 0x00000001u, + 0x00000001u, 0x7ff80000u, 0x00000001u, 0x00000000u, 0x00000001u, 0x7ff80000u, 0x00000001u, 0x00000000u, 0x00000001u, + 0x7ff80000u, 0x00000001u, 0x7ff80000u, 0x00000001u, 0x00000000u, 0x00000001u, 0x00000000u, 0x00000001u, 0x00000000u}, + {0x7fc00001u, 0x7f800001u, 0x7fc00001u, 0x7f800001u, 0x7f800001u, 0x7f800001u, 0x7fc00001u, 0x7fc00001u, 0x7f800001u, + 0x00000001u, 0x7ff80000u, 0x00000001u, 0x7ff00000u, 0x00000001u, 0x7ff80000u, 0x00000001u, 0x7ff00000u, 0x00000001u, + 0x7ff80000u, 0x00000001u, 0x7ff80000u, 0x00000001u, 0x7ff00000u, 0x00000001u, 0x7ff00000u, 0x00000001u, 0x7ff00000u}, + {0x7fc00001u, 0x7f800000u, 0x7fc00001u, 0x7f800000u, 0x7f800000u, 0x7f800000u, 0x7fc00001u, 0x7fc00001u, 0x7f800000u, + 0x00000001u, 0x7ff80000u, 0x00000000u, 0x7ff00000u, 0x00000001u, 0x7ff80000u, 0x00000000u, 0x7ff00000u, 0x00000001u, + 0x7ff80000u, 0x00000001u, 0x7ff80000u, 0x00000000u, 0x7ff00000u, 0x00000000u, 0x7ff00000u, 0x00000000u, 0x7ff00000u}, + {0x7fc00001u, 0x7f800001u, 0x7fc00001u, 0x7f800001u, 0x7f800001u, 0x7f800001u, 0x7fc00001u, 0x7fc00001u, 0x7f800001u, + 0x00000001u, 0x7ff80000u, 0x00000001u, 0x7ff00000u, 0x00000001u, 0x7ff80000u, 0x00000001u, 0x7ff00000u, 0x00000001u, + 0x7ff80000u, 0x00000001u, 0x7ff80000u, 0x00000001u, 0x7ff00000u, 0x00000001u, 0x7ff00000u, 0x00000001u, 0x7ff00000u}, + {0x7fc00001u, 0x00000000u, 0x7fc00001u, 0x00000000u, 0x00000000u, 0x00000000u, 0x7fc00001u, 0x7fc00001u, 0x00000000u, + 0x00000001u, 0x7ff80000u, 0x00000000u, 0x00000000u, 0x00000001u, 0x7ff80000u, 0x00000000u, 0x00000000u, 0x00000001u, + 0x7ff80000u, 0x00000001u, 0x7ff80000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u}, + {0x7fc00001u, 0x7f800001u, 0x7fc00001u, 0x7f800001u, 0x7f800001u, 0x7f800001u, 0x7fc00001u, 0x7fc00001u, 0x7f800001u, + 0x00000001u, 0x7ff80000u, 0x00000001u, 0x7ff00000u, 0x00000001u, 0x7ff80000u, 0x00000001u, 0x7ff00000u, 0x00000001u, + 0x7ff80000u, 0x00000001u, 0x7ff80000u, 0x00000001u, 0x7ff00000u, 0x00000001u, 0x7ff00000u, 0x00000001u, 0x7ff00000u} +}; + +constexpr std::uint32_t ExpectedFlushed[32][Columns] = { + {0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x45464748u, 0x00000000u, 0x00000000u, 0x00000000u, 0x45464748u, + 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, + 0x00000000u, 0x00000000u, 0x00000000u, 0x45464748u, 0x41424344u, 0x00000000u, 0x00000000u, 0x45464748u, 0x41424344u}, + {0x80000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x45464748u, 0x00000000u, 0x80000000u, 0x00000000u, 0x45464748u, + 0x00000000u, 0x80000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, + 0x80000000u, 0x00000000u, 0x00000000u, 0x45464748u, 0x41424344u, 0x00000000u, 0x00000000u, 0x45464748u, 0x41424344u}, + {0x80000000u, 0x80000000u, 0x00000000u, 0x80000000u, 0x45464748u, 0x80000000u, 0x80000000u, 0x00000000u, 0x45464748u, + 0x00000000u, 0x80000000u, 0x00000000u, 0x80000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x80000000u, 0x00000000u, + 0x80000000u, 0x00000000u, 0x00000000u, 0x45464748u, 0x41424344u, 0x00000000u, 0x80000000u, 0x45464748u, 0x41424344u}, + {0xbf800000u, 0x3f800000u, 0x3f800000u, 0x3f800000u, 0x3f800000u, 0x3f800000u, 0xbf800000u, 0x3f800000u, 0x3f800000u, + 0x00000000u, 0xbff00000u, 0x00000000u, 0x3ff00000u, 0x00000000u, 0x3ff00000u, 0x00000000u, 0x3ff00000u, 0x00000000u, + 0xbff00000u, 0x00000000u, 0x3ff00000u, 0x00000000u, 0x3ff00000u, 0x00000000u, 0x3ff00000u, 0x00000000u, 0x3ff00000u}, + {0xbf800000u, 0xbf800000u, 0x3f800000u, 0xbf800000u, 0xbf800000u, 0xbf800000u, 0xbf800000u, 0x3f800000u, 0xbf800000u, + 0x00000000u, 0xbff00000u, 0x00000000u, 0xbff00000u, 0x00000000u, 0x3ff00000u, 0x00000000u, 0xbff00000u, 0x00000000u, + 0xbff00000u, 0x00000000u, 0x3ff00000u, 0x00000000u, 0xbff00000u, 0x00000000u, 0xbff00000u, 0x00000000u, 0xbff00000u}, + {0xff800000u, 0x7f800000u, 0x7f800000u, 0x7f800000u, 0x7f800000u, 0x7f800000u, 0xff800000u, 0x7f800000u, 0x7f800000u, + 0x00000000u, 0xfff00000u, 0x00000000u, 0x7ff00000u, 0x00000000u, 0x7ff00000u, 0x00000000u, 0x7ff00000u, 0x00000000u, + 0xfff00000u, 0x00000000u, 0x7ff00000u, 0x00000000u, 0x7ff00000u, 0x00000000u, 0x7ff00000u, 0x00000000u, 0x7ff00000u}, + {0xff800000u, 0xff800000u, 0x7f800000u, 0xff800000u, 0xff800000u, 0xff800000u, 0xff800000u, 0x7f800000u, 0xff800000u, + 0x00000000u, 0xfff00000u, 0x00000000u, 0xfff00000u, 0x00000000u, 0x7ff00000u, 0x00000000u, 0xfff00000u, 0x00000000u, + 0xfff00000u, 0x00000000u, 0x7ff00000u, 0x00000000u, 0xfff00000u, 0x00000000u, 0xfff00000u, 0x00000000u, 0xfff00000u}, + {0x00000000u, 0x00000001u, 0x00000000u, 0x00000001u, 0x45464748u, 0x00000001u, 0x00000000u, 0x00000000u, 0x45464748u, + 0x00000000u, 0x00000000u, 0x00000001u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000001u, 0x00000000u, 0x00000000u, + 0x00000000u, 0x00000000u, 0x00000000u, 0x45464748u, 0x41424344u, 0x00000001u, 0x00000000u, 0x45464748u, 0x41424344u}, + {0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x45464748u, 0x00000000u, 0x00000000u, 0x00000000u, 0x45464748u, + 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, + 0x00000000u, 0x00000000u, 0x00000000u, 0x45464748u, 0x41424344u, 0x00000000u, 0x00000000u, 0x45464748u, 0x41424344u}, + {0x80000000u, 0x80000001u, 0x80000000u, 0x80000001u, 0x45464748u, 0x80000001u, 0x80000000u, 0x80000000u, 0x45464748u, + 0x00000000u, 0x80000000u, 0x00000001u, 0x80000000u, 0x00000000u, 0x80000000u, 0x00000001u, 0x80000000u, 0x00000000u, + 0x80000000u, 0x00000000u, 0x80000000u, 0x45464748u, 0x41424344u, 0x00000001u, 0x80000000u, 0x45464748u, 0x41424344u}, + {0x80000000u, 0x00000001u, 0x00000000u, 0x00000001u, 0x45464748u, 0x00000001u, 0x80000000u, 0x00000000u, 0x45464748u, + 0x00000000u, 0x80000000u, 0x00000001u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000001u, 0x00000000u, 0x00000000u, + 0x80000000u, 0x00000000u, 0x00000000u, 0x45464748u, 0x41424344u, 0x00000001u, 0x00000000u, 0x45464748u, 0x41424344u}, + {0x00000000u, 0x3f800000u, 0x3f800000u, 0x3f800000u, 0x3f800000u, 0x3f800000u, 0x00000000u, 0x3f800000u, 0x3f800000u, + 0x00000000u, 0x00000000u, 0x00000000u, 0x3ff00000u, 0x00000000u, 0x3ff00000u, 0x00000000u, 0x3ff00000u, 0x00000000u, + 0x00000000u, 0x00000000u, 0x3ff00000u, 0x00000000u, 0x3ff00000u, 0x00000000u, 0x3ff00000u, 0x00000000u, 0x3ff00000u}, + {0x3f800000u, 0x7fc00000u, 0x3f800000u, 0x7fc00000u, 0x7fc00000u, 0x7fc00000u, 0x3f800000u, 0x3f800000u, 0x7fc00000u, + 0x00000000u, 0x3ff00000u, 0x00000000u, 0x7ff80000u, 0x00000000u, 0x3ff00000u, 0x00000000u, 0x7ff80000u, 0x00000000u, + 0x3ff00000u, 0x00000000u, 0x3ff00000u, 0x00000000u, 0x7ff80000u, 0x00000000u, 0x7ff80000u, 0x00000000u, 0x7ff80000u}, + {0x3f800000u, 0x3f800000u, 0x3f800000u, 0x3f800000u, 0x3f800000u, 0x3f800000u, 0x3f800000u, 0x3f800000u, 0x3f800000u, + 0x00000000u, 0x3ff00000u, 0x00000000u, 0x3ff00000u, 0x00000000u, 0x3ff00000u, 0x00000000u, 0x3ff00000u, 0x00000000u, + 0x3ff00000u, 0x00000000u, 0x3ff00000u, 0x00000000u, 0x3ff00000u, 0x00000000u, 0x3ff00000u, 0x00000000u, 0x3ff00000u}, + {0x7fc00123u, 0x7fc00000u, 0x7fc00123u, 0x7fc00000u, 0x7fc00000u, 0x7fc00000u, 0x7fc00123u, 0x7fc00123u, 0x7fc00000u, + 0x00000123u, 0x7ff80000u, 0x00000000u, 0x7ff80000u, 0x00000123u, 0x7ff80000u, 0x00000000u, 0x7ff80000u, 0x00000123u, + 0x7ff80000u, 0x00000123u, 0x7ff80000u, 0x00000000u, 0x7ff80000u, 0x00000000u, 0x7ff80000u, 0x00000000u, 0x7ff80000u}, + {0x7fc00000u, 0x7fc00123u, 0x7fc00000u, 0x7fc00123u, 0x7fc00123u, 0x7fc00123u, 0x7fc00000u, 0x7fc00000u, 0x7fc00123u, + 0x00000000u, 0x7ff80000u, 0x00000123u, 0x7ff80000u, 0x00000000u, 0x7ff80000u, 0x00000123u, 0x7ff80000u, 0x00000000u, + 0x7ff80000u, 0x00000000u, 0x7ff80000u, 0x00000123u, 0x7ff80000u, 0x00000123u, 0x7ff80000u, 0x00000123u, 0x7ff80000u}, + {0x7fc00000u, 0xffc00000u, 0x7fc00000u, 0xffc00000u, 0xffc00000u, 0xffc00000u, 0x7fc00000u, 0x7fc00000u, 0xffc00000u, + 0x00000000u, 0x7ff80000u, 0x00000000u, 0xfff80000u, 0x00000000u, 0x7ff80000u, 0x00000000u, 0xfff80000u, 0x00000000u, + 0x7ff80000u, 0x00000000u, 0x7ff80000u, 0x00000000u, 0xfff80000u, 0x00000000u, 0xfff80000u, 0x00000000u, 0xfff80000u}, + {0x7fc00000u, 0x7fc00000u, 0x7fc00000u, 0x7fc00000u, 0x7fc00000u, 0x7fc00000u, 0x7fc00000u, 0x7fc00000u, 0x7fc00000u, + 0x00000000u, 0x7ff80000u, 0x00000000u, 0x7ff80000u, 0x00000000u, 0x7ff80000u, 0x00000000u, 0x7ff80000u, 0x00000000u, + 0x7ff80000u, 0x00000000u, 0x7ff80000u, 0x00000000u, 0x7ff80000u, 0x00000000u, 0x7ff80000u, 0x00000000u, 0x7ff80000u}, + {0x7fc00001u, 0x7f800001u, 0x7fc00001u, 0x7f800001u, 0x7f800001u, 0x7f800001u, 0x7fc00001u, 0x7fc00001u, 0x7f800001u, + 0x00000001u, 0x7ff80000u, 0x00000001u, 0x7ff00000u, 0x00000001u, 0x7ff80000u, 0x00000001u, 0x7ff00000u, 0x00000001u, + 0x7ff80000u, 0x00000001u, 0x7ff80000u, 0x00000001u, 0x7ff00000u, 0x00000001u, 0x7ff00000u, 0x00000001u, 0x7ff00000u}, + {0x7fc00001u, 0x3f800000u, 0x7fc00001u, 0x3f800000u, 0x3f800000u, 0x3f800000u, 0x7fc00001u, 0x7fc00001u, 0x3f800000u, + 0x00000001u, 0x7ff80000u, 0x00000000u, 0x3ff00000u, 0x00000001u, 0x7ff80000u, 0x00000000u, 0x3ff00000u, 0x00000001u, + 0x7ff80000u, 0x00000001u, 0x7ff80000u, 0x00000000u, 0x3ff00000u, 0x00000000u, 0x3ff00000u, 0x00000000u, 0x3ff00000u}, + {0x7fc00001u, 0x7f800001u, 0x7fc00001u, 0x7f800001u, 0x7f800001u, 0x7f800001u, 0x7fc00001u, 0x7fc00001u, 0x7f800001u, + 0x00000001u, 0x7ff80000u, 0x00000001u, 0x7ff00000u, 0x00000001u, 0x7ff80000u, 0x00000001u, 0x7ff00000u, 0x00000001u, + 0x7ff80000u, 0x00000001u, 0x7ff80000u, 0x00000001u, 0x7ff00000u, 0x00000001u, 0x7ff00000u, 0x00000001u, 0x7ff00000u}, + {0x7fc00001u, 0x7fc00000u, 0x7fc00001u, 0x7fc00000u, 0x7fc00000u, 0x7fc00000u, 0x7fc00001u, 0x7fc00001u, 0x7fc00000u, + 0x00000001u, 0x7ff80000u, 0x00000000u, 0x7ff80000u, 0x00000001u, 0x7ff80000u, 0x00000000u, 0x7ff80000u, 0x00000001u, + 0x7ff80000u, 0x00000001u, 0x7ff80000u, 0x00000000u, 0x7ff80000u, 0x00000000u, 0x7ff80000u, 0x00000000u, 0x7ff80000u}, + {0x7fc00123u, 0x7f800001u, 0x7fc00123u, 0x7f800001u, 0x7f800001u, 0x7f800001u, 0x7fc00123u, 0x7fc00123u, 0x7f800001u, + 0x00000123u, 0x7ff80000u, 0x00000001u, 0x7ff00000u, 0x00000123u, 0x7ff80000u, 0x00000001u, 0x7ff00000u, 0x00000123u, + 0x7ff80000u, 0x00000123u, 0x7ff80000u, 0x00000001u, 0x7ff00000u, 0x00000001u, 0x7ff00000u, 0x00000001u, 0x7ff00000u}, + {0x7fc00001u, 0x7f800123u, 0x7fc00001u, 0x7f800123u, 0x7f800123u, 0x7f800123u, 0x7fc00001u, 0x7fc00001u, 0x7f800123u, + 0x00000001u, 0x7ff80000u, 0x00000123u, 0x7ff00000u, 0x00000001u, 0x7ff80000u, 0x00000123u, 0x7ff00000u, 0x00000001u, + 0x7ff80000u, 0x00000001u, 0x7ff80000u, 0x00000123u, 0x7ff00000u, 0x00000123u, 0x7ff00000u, 0x00000123u, 0x7ff00000u}, + {0x7fc00001u, 0xff800001u, 0x7fc00001u, 0xff800001u, 0xff800001u, 0xff800001u, 0x7fc00001u, 0x7fc00001u, 0xff800001u, + 0x00000001u, 0x7ff80000u, 0x00000001u, 0xfff00000u, 0x00000001u, 0x7ff80000u, 0x00000001u, 0xfff00000u, 0x00000001u, + 0x7ff80000u, 0x00000001u, 0x7ff80000u, 0x00000001u, 0xfff00000u, 0x00000001u, 0xfff00000u, 0x00000001u, 0xfff00000u}, + {0xffc00001u, 0x7f800001u, 0xffc00001u, 0x7f800001u, 0x7f800001u, 0x7f800001u, 0xffc00001u, 0xffc00001u, 0x7f800001u, + 0x00000001u, 0xfff80000u, 0x00000001u, 0x7ff00000u, 0x00000001u, 0xfff80000u, 0x00000001u, 0x7ff00000u, 0x00000001u, + 0xfff80000u, 0x00000001u, 0xfff80000u, 0x00000001u, 0x7ff00000u, 0x00000001u, 0x7ff00000u, 0x00000001u, 0x7ff00000u}, + {0x7fc00001u, 0x00000001u, 0x7fc00001u, 0x00000001u, 0x00000001u, 0x00000001u, 0x7fc00001u, 0x7fc00001u, 0x00000001u, + 0x00000001u, 0x7ff80000u, 0x00000001u, 0x00000000u, 0x00000001u, 0x7ff80000u, 0x00000001u, 0x00000000u, 0x00000001u, + 0x7ff80000u, 0x00000001u, 0x7ff80000u, 0x00000001u, 0x00000000u, 0x00000001u, 0x00000000u, 0x00000001u, 0x00000000u}, + {0x7fc00001u, 0x7f800001u, 0x7fc00001u, 0x7f800001u, 0x7f800001u, 0x7f800001u, 0x7fc00001u, 0x7fc00001u, 0x7f800001u, + 0x00000001u, 0x7ff80000u, 0x00000001u, 0x7ff00000u, 0x00000001u, 0x7ff80000u, 0x00000001u, 0x7ff00000u, 0x00000001u, + 0x7ff80000u, 0x00000001u, 0x7ff80000u, 0x00000001u, 0x7ff00000u, 0x00000001u, 0x7ff00000u, 0x00000001u, 0x7ff00000u}, + {0x7fc00001u, 0x7f800000u, 0x7fc00001u, 0x7f800000u, 0x7f800000u, 0x7f800000u, 0x7fc00001u, 0x7fc00001u, 0x7f800000u, + 0x00000001u, 0x7ff80000u, 0x00000000u, 0x7ff00000u, 0x00000001u, 0x7ff80000u, 0x00000000u, 0x7ff00000u, 0x00000001u, + 0x7ff80000u, 0x00000001u, 0x7ff80000u, 0x00000000u, 0x7ff00000u, 0x00000000u, 0x7ff00000u, 0x00000000u, 0x7ff00000u}, + {0x7fc00001u, 0x7f800001u, 0x7fc00001u, 0x7f800001u, 0x7f800001u, 0x7f800001u, 0x7fc00001u, 0x7fc00001u, 0x7f800001u, + 0x00000001u, 0x7ff80000u, 0x00000001u, 0x7ff00000u, 0x00000001u, 0x7ff80000u, 0x00000001u, 0x7ff00000u, 0x00000001u, + 0x7ff80000u, 0x00000001u, 0x7ff80000u, 0x00000001u, 0x7ff00000u, 0x00000001u, 0x7ff00000u, 0x00000001u, 0x7ff00000u}, + {0x7fc00001u, 0x00000000u, 0x7fc00001u, 0x00000000u, 0x00000000u, 0x00000000u, 0x7fc00001u, 0x7fc00001u, 0x00000000u, + 0x00000001u, 0x7ff80000u, 0x00000000u, 0x00000000u, 0x00000001u, 0x7ff80000u, 0x00000000u, 0x00000000u, 0x00000001u, + 0x7ff80000u, 0x00000001u, 0x7ff80000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u}, + {0x7fc00001u, 0x7f800001u, 0x7fc00001u, 0x7f800001u, 0x7f800001u, 0x7f800001u, 0x7fc00001u, 0x7fc00001u, 0x7f800001u, + 0x00000001u, 0x7ff80000u, 0x00000001u, 0x7ff00000u, 0x00000001u, 0x7ff80000u, 0x00000001u, 0x7ff00000u, 0x00000001u, + 0x7ff80000u, 0x00000001u, 0x7ff80000u, 0x00000001u, 0x7ff00000u, 0x00000001u, 0x7ff00000u, 0x00000001u, 0x7ff00000u} +}; + +constexpr const char* Names[18] = { + "ds_min_rtn_f32 memory", + "ds_min_rtn_f32 returned", + "ds_max_rtn_f32 memory", + "ds_max_rtn_f32 returned", + "ds_cmpst_rtn_f32 memory", + "ds_cmpst_rtn_f32 returned", + "ds_min_f32 memory", + "ds_max_f32 memory", + "ds_cmpst_f32 memory", + "ds_min_rtn_f64 memory", + "ds_min_rtn_f64 returned", + "ds_max_rtn_f64 memory", + "ds_max_rtn_f64 returned", + "ds_min_f64 memory", + "ds_max_f64 memory", + "ds_cmpst_rtn_f64 memory", + "ds_cmpst_rtn_f64 returned", + "ds_cmpst_f64 memory" +}; + +constexpr std::uint32_t Columns64 = 9; + +std::array BufferDescriptor(const void* data, std::uint32_t bytes) { + const auto address = reinterpret_cast(data); + return {static_cast(address), static_cast((address >> 32u) & 0xffffu), bytes, 0x31016facu}; +} + +std::string Hex(std::uint32_t value) { + char text[16]; + std::snprintf(text, sizeof(text), "0x%08x", value); + return text; +} + +const char* ColumnName(std::uint32_t column) { + return column < Columns64 ? Names[column] : Names[Columns64 + (column - Columns64) / 2u]; +} + +void Run(AgcDriver::VulkanDevice& device, const std::optional& floatMode) { + for (std::uint32_t tid = 0; tid < Threads; ++tid) std::copy(std::begin(Rows[tid]), std::end(Rows[tid]), &Input[tid * Inputs]); + Output.fill(0xdeadbeefu); + std::vector userData(8, 0u); + const auto input = BufferDescriptor(Input.data(), static_cast(Input.size() * 4u)); + const auto output = BufferDescriptor(Output.data(), static_cast(Output.size() * 4u)); + std::copy(input.begin(), input.end(), userData.begin()); + std::copy(output.begin(), output.end(), userData.begin() + 4); + const std::span code(Code); + const std::array memory{{reinterpret_cast(code.data()), std::as_bytes(code)}}; + const ShaderRecompiler::ShaderComputeStageInfo compute{{Threads, 1, 1}, 64u, {false, false, false}, false, 1}; + ShaderRecompiler::RecompileRequest request{ + {ShaderStage::Compute, reinterpret_cast(code.data()), code, 0, {}}, + {32, 0, userData, compute, std::nullopt, std::nullopt, memory}, + device.Target(), + {0, 0, 0, 128} + }; + request.useCache = false; + request.context.floatMode = floatMode; + const auto result = ShaderRecompiler::Recompile(request); + device.Dispatch(result, 1, 1, 1, {}, reinterpret_cast(code.data())); + device.WaitIdle(); +} + +void Check(const std::uint32_t (&expected)[32][Columns], const char* mode) { + for (std::uint32_t tid = 0; tid < Threads; ++tid) { + const std::uint32_t* out = &Output[tid * Results]; + for (std::uint32_t column = 0; column < Columns; ++column) { + const std::uint32_t actual = out[column < Columns64 ? column : column + 1u]; + Require(actual == expected[tid][column], std::string("lds float atomic modes: ") + mode + " lane " + std::to_string(tid) + " " + ColumnName(column) + + (column >= Columns64 && (column - Columns64) % 2u ? " high" : column >= Columns64 ? " low" : "") + " is " + Hex(actual) + ", expected " + Hex(expected[tid][column])); + } + } +} + +} + +int main() { + try { + const auto device = OpenVulkanTestDevice(); + if (!device) return VulkanTestSkipped; + Run(*device, std::nullopt); + Check(ExpectedTitles, "no float mode"); + Run(*device, ShaderRecompiler::ShaderFloatMode{0xc0u, true, false, false}); + Check(ExpectedTitles, "FLOAT_MODE 0xc0"); + Run(*device, ShaderRecompiler::ShaderFloatMode{0xf0u, true, false, false}); + Check(ExpectedKept, "FLOAT_MODE 0xf0"); + Run(*device, ShaderRecompiler::ShaderFloatMode{0x00u, true, false, false}); + Check(ExpectedFlushed, "FLOAT_MODE 0x00"); + Run(*device, ShaderRecompiler::ShaderFloatMode{0x50u, true, false, false}); + Check(ExpectedKept, "FLOAT_MODE 0x50"); + Run(*device, ShaderRecompiler::ShaderFloatMode{0xa0u, true, false, false}); + Check(ExpectedFlushed, "FLOAT_MODE 0xa0"); + std::puts("lds float atomic modes tests passed"); + return 0; + } catch (const std::exception& error) { + std::cerr << error.what() << '\n'; + return 1; + } +} diff --git a/core/libs/prx/libSceAgcDriver/tests/execution/MacLegacy.cpp b/core/libs/prx/libSceAgcDriver/tests/execution/MacLegacy.cpp new file mode 100644 index 000000000..2c3178731 --- /dev/null +++ b/core/libs/prx/libSceAgcDriver/tests/execution/MacLegacy.cpp @@ -0,0 +1,137 @@ +#include "prx/libSceAgcDriver/Execution/include/VulkanDevice.hpp" +#include "prx/libSceAgcDriver/Graphics/include/Draw.hpp" +#include "Recompiler.hpp" +#include "VulkanTestDevice.hpp" +#include +#include +#include +#include +#include +#include +#include + +namespace { + +using AgcDriver::Graphics::Require; +using ShaderRecompiler::ShaderStage; + +constexpr std::uint32_t Threads = 32; +constexpr std::uint32_t Inputs = 8; +constexpr std::uint32_t Results = 8; +alignas(256) std::array Input{}; +alignas(256) std::array Output{}; + +alignas(256) constexpr std::array MacLegacyCode{ + 0x34020083, 0x34060083, 0xe0302000, 0x80000401, 0xe0302004, 0x80000501, 0xe0302008, 0x80000601, + 0xbf8c3f70, 0x7e140306, 0x0c140b04, 0x7e160306, 0xd506000b, 0x20020b04, 0x7e180306, 0xd506030c, + 0x40020b04, 0x7e1a0306, 0x0c1a0a80, 0xe0702000, 0x80010a03, 0xe0702004, 0x80010b03, 0xe0702008, + 0x80010c03, 0xe070200c, 0x80010d03, 0xbf810000, +}; + +struct Row { + std::uint32_t a, b, c; + std::array expected; +}; + +constexpr std::array Rows{{ + {0x00000000u, 0x7f800000u, 0x3f800000u, {0x3f800000u, 0x3f800000u, 0x3f800000u, 0x3f800000u}}, + {0x80000000u, 0x7f800000u, 0xc0200000u, {0xc0200000u, 0xc0200000u, 0xc0200000u, 0xc0200000u}}, + {0x7f800000u, 0x00000000u, 0x3f400000u, {0x3f400000u, 0x3f400000u, 0x3f400000u, 0x3f400000u}}, + {0xff800000u, 0x80000000u, 0x40400000u, {0x40400000u, 0x40400000u, 0x40400000u, 0x40400000u}}, + {0x00000000u, 0x7fc00000u, 0x3fc00000u, {0x3fc00000u, 0x3fc00000u, 0x3fc00000u, 0x3fc00000u}}, + {0x7fc00000u, 0x80000000u, 0xc0800000u, {0xc0800000u, 0xc0800000u, 0xc0800000u, 0xc0800000u}}, + {0x00000000u, 0x40000000u, 0x7f800000u, {0x7f800000u, 0x7f800000u, 0x7f800000u, 0x7f800000u}}, + {0x80000000u, 0xc0400000u, 0xff800000u, {0xff800000u, 0xff800000u, 0xff800000u, 0xff800000u}}, + {0x00000000u, 0x3f800000u, 0x7fc00000u, {0x7fc00000u, 0x7fc00000u, 0x7fc00000u, 0x7fc00000u}}, + {0x3fc00000u, 0x40100000u, 0xbf400000u, {0x40280000u, 0xc0840000u, 0xc0840000u, 0xbf400000u}}, + {0xc0c00000u, 0x3f000000u, 0x41200000u, {0x40e00000u, 0x41500000u, 0x40e00000u, 0x41200000u}}, + {0x7f800000u, 0x40000000u, 0x3f800000u, {0x7f800000u, 0xff800000u, 0xff800000u, 0x3f800000u}}, + {0x7f800000u, 0xbf800000u, 0x7f800000u, {0xffc00000u, 0x7f800000u, 0xffc00000u, 0x7f800000u}}, + {0x7fc00000u, 0x3f800000u, 0x3f800000u, {0x7fc00000u, 0xffc00000u, 0x7fc00000u, 0x3f800000u}}, + {0x40400000u, 0x40800000u, 0xc1400000u, {0x00000000u, 0xc1c00000u, 0xc1c00000u, 0xc1400000u}}, + {0x7f7fffffu, 0x40000000u, 0x3f800000u, {0x7f800000u, 0xff800000u, 0xff800000u, 0x3f800000u}}, + {0x3ec00000u, 0xc1480000u, 0x42c88000u, {0x42bf2000u, 0x42d1e000u, 0x42bf2000u, 0x42c88000u}}, + {0xc4800000u, 0xbd800000u, 0xc2800000u, {0x00000000u, 0xc3000000u, 0xc3000000u, 0xc2800000u}}, + {0x40e00000u, 0x41100000u, 0x3f000000u, {0x427e0000u, 0xc27a0000u, 0xc27a0000u, 0x3f000000u}}, + {0x80000000u, 0x80000000u, 0x40a00000u, {0x40a00000u, 0x40a00000u, 0x40a00000u, 0x40a00000u}}, + {0x3f800800u, 0x3f800800u, 0xbf800000u, {0x3a000000u, 0xc0000800u, 0xc0000800u, 0xbf800000u}}, + {0x3f800001u, 0x3f7fffffu, 0xbf800000u, {0x00000000u, 0xc0000000u, 0xc0000000u, 0xbf800000u}}, + {0x3faaaaabu, 0x40400000u, 0xc0800000u, {0x00000000u, 0xc1000000u, 0xc1000000u, 0xc0800000u}}, + {0x7f7fffffu, 0x40000000u, 0xff7fffffu, {0x7f800000u, 0xff800000u, 0xff800000u, 0xff7fffffu}}, + {0x00800000u, 0x3f000000u, 0x00000000u, {0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u}}, + {0x00000001u, 0x4b000000u, 0x00000000u, {0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u}}, + {0x00000001u, 0x3f800000u, 0x80000000u, {0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u}}, + {0x80000001u, 0x3f800000u, 0x00000000u, {0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u}}, + {0x00800000u, 0x3f800000u, 0x80400000u, {0x00800000u, 0x80800000u, 0x80800000u, 0x00000000u}}, + {0x3f800000u, 0x00400000u, 0x00000000u, {0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u}}, + {0x7f812345u, 0x3f800000u, 0x3f800000u, {0x7f812345u, 0xff812345u, 0x7f812345u, 0x3f800000u}}, + {0x3f800000u, 0x3f800000u, 0x7f812345u, {0x7f812345u, 0x7f812345u, 0x7f812345u, 0x7f812345u}}, +}}; + +std::array BufferDescriptor(const void* data, std::uint32_t count) { + const auto address = reinterpret_cast(data); + return {static_cast(address), static_cast((address >> 32u) & 0xffffu) | (4u << 16u), count, 0x11016facu}; +} + +void Run(AgcDriver::VulkanDevice& device) { + Input.fill(0u); + for (std::uint32_t tid = 0; tid < Rows.size(); ++tid) { + const auto& row = Rows[tid]; + const std::array words{row.a, row.b, row.c}; + std::copy(words.begin(), words.end(), Input.begin() + tid * Inputs); + } + Output.fill(0xdeadbeefu); + std::vector userData(8, 0u); + const auto input = BufferDescriptor(Input.data(), static_cast(Input.size())); + const auto output = BufferDescriptor(Output.data(), static_cast(Output.size())); + std::copy(input.begin(), input.end(), userData.begin()); + std::copy(output.begin(), output.end(), userData.begin() + 4); + const std::span code(MacLegacyCode); + const std::array memory{{{reinterpret_cast(code.data()), std::as_bytes(code)}}}; + const ShaderRecompiler::ShaderComputeStageInfo compute{{Threads, 1, 1}, 0, {false, false, false}, false, 1}; + ShaderRecompiler::RecompileRequest request{ + {ShaderStage::Compute, reinterpret_cast(code.data()), code, 0, {}}, + {32, 0, userData, compute, std::nullopt, std::nullopt, memory}, + device.Target(), + {0, 0, 0, 128} + }; + request.useCache = false; + const auto result = ShaderRecompiler::Recompile(request); + device.Dispatch(result, 1, 1, 1, {}, reinterpret_cast(code.data())); + device.WaitIdle(); +} + +std::string Hex(std::uint32_t value) { + char text[16]; + std::snprintf(text, sizeof(text), "0x%08x", value); + return text; +} + +void Check() { + constexpr std::array names{ + "v_mac_legacy_f32", "v_mac_legacy_f32_e64 -src0", "v_mac_legacy_f32_e64 |src0| -|src1|", "v_mac_legacy_f32 src0=0", + }; + for (std::uint32_t tid = 0; tid < Rows.size(); ++tid) { + const auto& row = Rows[tid]; + for (std::uint32_t j = 0; j < row.expected.size(); ++j) { + const auto actual = Output[tid * Results + j]; + Require(actual == row.expected[j], std::string("mac legacy: lane ") + std::to_string(tid) + " (" + Hex(row.a) + ", " + Hex(row.b) + ", " + Hex(row.c) + ") " + names[j] + " is " + Hex(actual) + ", expected " + Hex(row.expected[j])); + } + } +} + +} + +int main() { + try { + const auto device = OpenVulkanTestDevice(); + if (!device) return VulkanTestSkipped; + Run(*device); + Check(); + std::puts("mac legacy tests passed"); + return 0; + } catch (const std::exception& error) { + std::cerr << error.what() << '\n'; + return 1; + } +} diff --git a/core/libs/prx/libSceAgcDriver/tests/execution/MaskedGlobalStore.cpp b/core/libs/prx/libSceAgcDriver/tests/execution/MaskedGlobalStore.cpp new file mode 100644 index 000000000..83dc78c5b --- /dev/null +++ b/core/libs/prx/libSceAgcDriver/tests/execution/MaskedGlobalStore.cpp @@ -0,0 +1,94 @@ +#include "prx/libSceAgcDriver/Execution/include/VulkanDevice.hpp" +#include "prx/libSceAgcDriver/Graphics/include/Draw.hpp" +#include "prx/libc/include/GuestAllocations.hpp" +#include "Recompiler.hpp" +#include "VulkanTestDevice.hpp" +#ifdef _WIN32 +#ifndef NOMINMAX +#define NOMINMAX +#endif +#include +#endif +#include +#include +#include +#include +#include +#include +#include +#include + +namespace { + +using AgcDriver::Graphics::Require; +using ShaderRecompiler::ShaderStage; + +constexpr std::size_t BlockBytes = 64 * 1024; + +alignas(256) constexpr std::array MaskedStoreCode{0xd7460000u, 0x04010c06u, 0x34000084u, 0xdc388000u, 0x04020000u, 0xbf8c3f70u, 0xd7710008u, 0x00120805u, 0xd7710009u, 0x00120a05u, 0xd771000au, 0x00120c05u, 0xd771000bu, 0x00120e05u, 0xdc788000u, 0x00020800u, 0xbf810000u}; + +class GuestBlock { +public: + GuestBlock() { +#ifdef _WIN32 + block = static_cast(VirtualAlloc(nullptr, BlockBytes, MEM_RESERVE | MEM_COMMIT, PAGE_READWRITE)); +#else + block = static_cast(std::aligned_alloc(BlockBytes, BlockBytes)); +#endif + Require(block != nullptr, "masked global store: cannot allocate the guest block"); + GuestAllocations::Mutation().Add(block, BlockBytes, true, true, true); + } + ~GuestBlock() { GuestAllocations::Mutation().Remove(block); } + GuestBlock(const GuestBlock&) = delete; + GuestBlock& operator=(const GuestBlock&) = delete; + std::uint8_t* Data() { return block; } + +private: + std::uint8_t* block = nullptr; +}; + +void Run(AgcDriver::VulkanDevice& device, GuestBlock& guest, std::uint32_t waveSize, std::uint32_t value, std::uint32_t keep) { + constexpr std::uint32_t groups = 64; + constexpr std::size_t words = groups * 64u * 4u; + auto* data = reinterpret_cast(guest.Data()); + for (std::size_t index = 0; index < BlockBytes / 4u; ++index) data[index] = static_cast(index * 0x9e3779b1u); + const auto address = static_cast(reinterpret_cast(guest.Data())); + const std::vector userData{0, 0, static_cast(address), static_cast(address >> 32u), value, keep}; + const std::span code(MaskedStoreCode); + const std::array memory{{{reinterpret_cast(code.data()), std::as_bytes(code)}}}; + const ShaderRecompiler::ShaderComputeStageInfo compute{{64, 1, 1}, 0, {true, false, false}, false, 1}; + ShaderRecompiler::RecompileRequest request{ + {ShaderStage::Compute, reinterpret_cast(code.data()), code, 0, {}}, + {waveSize, 0, userData, compute, std::nullopt, std::nullopt, memory}, + device.Target(), + {0, 0, 0, 128} + }; + request.useCache = false; + const auto result = ShaderRecompiler::Recompile(request); + device.Dispatch(result, groups, 1, 1, {}, reinterpret_cast(code.data())); + device.WaitIdle(); + for (std::size_t index = 0; index < BlockBytes / 4u; ++index) { + const auto original = static_cast(index * 0x9e3779b1u); + const auto expected = index < words ? (original & keep) | value : original; + Require(data[index] == expected, "a masked global_store_dwordx4 left dword " + std::to_string(index) + " incorrect"); + } +} + +} + +int main() { + try { + const auto device = OpenVulkanTestDevice(); + if (!device) return VulkanTestSkipped; + GuestBlock guest; + for (const auto waveSize : {64u, 32u}) { + Run(*device, guest, waveSize, 0u, 0x00000ff0u); + Run(*device, guest, waveSize, 0xfffff00fu, 0x00000ff0u); + } + std::puts("masked global store tests passed"); + return 0; + } catch (const std::exception& error) { + std::cerr << error.what() << '\n'; + return 1; + } +} diff --git a/core/libs/prx/libSceAgcDriver/tests/execution/MixPrecision.cpp b/core/libs/prx/libSceAgcDriver/tests/execution/MixPrecision.cpp index 1d7b3d44a..d8c1506e3 100644 --- a/core/libs/prx/libSceAgcDriver/tests/execution/MixPrecision.cpp +++ b/core/libs/prx/libSceAgcDriver/tests/execution/MixPrecision.cpp @@ -2,11 +2,13 @@ #include "prx/libSceAgcDriver/Graphics/include/Draw.hpp" #include "Recompiler.hpp" #include "VulkanTestDevice.hpp" +#include #include #include #include #include #include +#include #include #include @@ -15,6 +17,7 @@ namespace { using AgcDriver::Graphics::Require; using ShaderRecompiler::ShaderStage; +constexpr std::uint32_t Float64Capability = 10; constexpr std::uint32_t Threads = 64; constexpr std::uint32_t Inputs = 4; constexpr std::uint32_t Results = 16; @@ -31,6 +34,87 @@ alignas(256) constexpr std::array MixCode{ 0x80011103, 0xe0702020, 0x80011203, 0xbf810000, }; +alignas(256) constexpr std::array RoundingCode{ + 0x34020082, 0x34060084, 0xe0302000, 0x80000401, 0xe0302004, 0x80000501, 0xe0302008, 0x80000601, + 0x7e1402ff, 0xabcd0000, 0x7e1602ff, 0x0000abcd, 0x7e1802ff, 0xabcd0000, 0x7e1a02ff, 0xabcd0000, + 0x7e1c02ff, 0xabcd0000, 0x7e1e02ff, 0xabcd0000, 0x7e2002ff, 0xabcd0000, 0xbf8c3f70, 0xcc21000a, + 0x041a0b04, 0xcc22000b, 0x041a0b04, 0xcc21400c, 0x1c1a0b04, 0xcc21000d, 0x1c1a0b04, 0xcc21400e, + 0x041a0b04, 0xcc21400f, 0x0c1a0b04, 0xcc210010, 0x141a0b04, 0xe0702000, 0x80010a03, 0xe0702004, + 0x80010b03, 0xe0702008, 0x80010c03, 0xe070200c, 0x80010d03, 0xe0702010, 0x80010e03, 0xe0702014, + 0x80010f03, 0xe0702018, 0x80011003, 0xbf810000, +}; + +struct RoundingRow { + std::uint32_t a, b, c, result, expected; +}; + +constexpr std::array RoundingRows{{ + {0x000062e9u, 0x0000d159u, 0x000023ecu, 2u, 0xf89fu}, + {0xbfc00000u, 0x33800000u, 0x00800000u, 0u, 0x8001u}, + {0x3bec0000u, 0xc73c0000u, 0xb767d928u, 0u, 0xdd6bu}, + {0xb4e70000u, 0xc5500000u, 0x8d000000u, 0u, 0x15ddu}, + {0xcc750000u, 0xb0500000u, 0x2f800000u, 0u, 0x2a39u}, + {0xc7b30000u, 0x35500000u, 0x0d000000u, 0u, 0xac8bu}, + {0x36400000u, 0x4c414000u, 0xb3800000u, 0u, 0x5887u}, + {0x4ae00000u, 0x3604c000u, 0x00800000u, 0u, 0x4b43u}, + {0xcf840000u, 0x36fc0000u, 0x31054000u, 0u, 0xf80fu}, + {0x43320000u, 0x41580000u, 0xaf800000u, 0u, 0x68b1u}, + {0x38400000u, 0x34406b26u, 0x43ab9000u, 0u, 0x5d5du}, + {0x00007780u, 0x00001478u, 0x00000002u, 2u, 0x5031u}, + {0x0000e520u, 0x0000cc10u, 0x00000002u, 2u, 0x7535u}, + {0x00004138u, 0x0000e440u, 0x00000001u, 2u, 0xe98bu}, + {0x000020ccu, 0x0000d880u, 0x00000001u, 2u, 0xbd65u}, + {0x00004880u, 0x0000d81cu, 0x00000002u, 2u, 0xe49fu}, + {0x0000cad4u, 0x00005900u, 0x00008002u, 2u, 0xe845u}, + {0x00001836u, 0x0000b5b3u, 0x00003a9au, 2u, 0x3a99u}, + {0x000063f0u, 0x00005340u, 0x00008002u, 2u, 0x7b31u}, + {0x0000e700u, 0x0000cca4u, 0x00008100u, 2u, 0x780fu}, + {0x00000003u, 0x0000f89du, 0x0d000000u, 3u, 0x9eebu}, + {0x00009d20u, 0x000036a0u, 0x8d000000u, 3u, 0x983fu}, + {0x000008a5u, 0x0000b400u, 0x80800000u, 3u, 0x8253u}, + {0x00009400u, 0x00002935u, 0x8d000000u, 3u, 0x829bu}, + {0x0000c5c0u, 0x00004630u, 0xaf800000u, 3u, 0xd073u}, + {0x0000140du, 0x00000c40u, 0x42e11000u, 3u, 0x5709u}, + {0x00008800u, 0x0000b439u, 0x00800000u, 3u, 0x021du}, + {0x00006c80u, 0x00003c04u, 0x33800000u, 3u, 0x6c85u}, + {0x00002840u, 0x0000f018u, 0x33800000u, 3u, 0xdc59u}, + {0x4f200000u, 0x34d98000u, 0x00008100u, 4u, 0x643fu}, + {0xced08000u, 0x37900000u, 0x00008100u, 4u, 0xf755u}, + {0xb4b63000u, 0xce800000u, 0x00008100u, 4u, 0x5db1u}, + {0xca094000u, 0x38a00000u, 0x00008002u, 4u, 0xd95du}, + {0x37fb0000u, 0xcdb00000u, 0x00008100u, 4u, 0xf165u}, + {0x46c00000u, 0xbe374000u, 0x00000002u, 4u, 0xec4bu}, + {0x48580000u, 0xbdf60000u, 0x00008822u, 4u, 0xf67du}, + {0x40186000u, 0x46c00000u, 0x00000001u, 4u, 0x7b25u}, + {0xcc500000u, 0x33bd0000u, 0x00008001u, 4u, 0xc4cdu}, + {0x00001889u, 0x45c00000u, 0x00008002u, 5u, 0x4acdu}, + {0x0000a007u, 0xc4c00000u, 0x00000001u, 5u, 0x4a0bu}, + {0x0000a87du, 0x42c00000u, 0x00000002u, 5u, 0xc2bbu}, + {0x00003018u, 0xc6680000u, 0x00000100u, 5u, 0xe76bu}, + {0x0000d61eu, 0x3dc00000u, 0x00008001u, 5u, 0xc897u}, + {0x00002888u, 0xc7480000u, 0x00008100u, 5u, 0xe715u}, + {0x0000c064u, 0xc0900000u, 0x00000001u, 5u, 0x48f1u}, + {0x000074deu, 0xbe200000u, 0x00000001u, 5u, 0xea15u}, + {0x0000d86du, 0xc1400000u, 0x00008100u, 5u, 0x66a3u}, + {0x48c00000u, 0x000018dfu, 0x00800000u, 6u, 0x634fu}, + {0x41e00000u, 0x0000a002u, 0x2f800000u, 6u, 0xb303u}, + {0xb2f61000u, 0x00004094u, 0xc23ed000u, 6u, 0xd1f7u}, + {0xb5200000u, 0x0000941eu, 0xc575b000u, 6u, 0xebadu}, + {0xc1600000u, 0x000003dau, 0x0d000000u, 6u, 0x92bdu}, + {0x4a500000u, 0x00001894u, 0x33800000u, 6u, 0x6f71u}, + {0x3ce00000u, 0x0000681au, 0xaf800000u, 6u, 0x532du}, + {0x43a00000u, 0x00002a02u, 0x34079c43u, 6u, 0x4b83u}, + {0x43c00000u, 0x0000c81bu, 0xb3800000u, 6u, 0xea29u}, + {0x1f000000u, 0x1f000000u, 0x3f801000u, 0u, 0x3c01u}, + {0x1f000000u, 0x9f000000u, 0x3f801000u, 0u, 0x3c00u}, + {0x00800000u, 0x3e800000u, 0x3f801000u, 0u, 0x3c01u}, + {0x33000000u, 0x3f800000u, 0x00800000u, 0u, 0x0001u}, + {0x33000000u, 0x3f800000u, 0x80800000u, 0u, 0x0000u}, + {0x477ff000u, 0x3f800000u, 0xa6800000u, 0u, 0x7bffu}, + {0x7f7fffffu, 0x40000000u, 0xff7fffffu, 0u, 0x7c00u}, + {0x00003c01u, 0x00004200u, 0x00800000u, 3u, 0x4202u}, +}}; + float Quarter(std::uint32_t index) { return static_cast(static_cast(index % 17u) - 8) / 4.0f; } @@ -84,14 +168,21 @@ std::array BufferDescriptor(const void* data, std::uint32_t co return {static_cast(address), static_cast((address >> 32u) & 0xffffu) | (4u << 16u), count, 0x11016facu}; } -void Run(AgcDriver::VulkanDevice& device) { +void FillRoundingInput() { + Input.fill(0u); + for (std::uint32_t tid = 0; tid < Threads; ++tid) { + const auto& row = RoundingRows[tid]; + std::copy_n(std::array{row.a, row.b, row.c}.begin(), 3, Input.begin() + tid * Inputs); + } +} + +void Run(AgcDriver::VulkanDevice& device, std::span code) { Output.fill(0xdeadbeefu); std::vector userData(8, 0u); const auto input = BufferDescriptor(Input.data(), static_cast(Input.size())); const auto output = BufferDescriptor(Output.data(), static_cast(Output.size())); std::copy(input.begin(), input.end(), userData.begin()); std::copy(output.begin(), output.end(), userData.begin() + 4); - const std::span code(MixCode); const std::array memory{{{reinterpret_cast(code.data()), std::as_bytes(code)}}}; const ShaderRecompiler::ShaderComputeStageInfo compute{{Threads, 1, 1}, 0, {false, false, false}, false, 1}; ShaderRecompiler::RecompileRequest request{ @@ -131,15 +222,36 @@ void Check() { } } +void CheckRounding() { + for (std::uint32_t tid = 0; tid < Threads; ++tid) { + const auto& row = RoundingRows[tid]; + const auto* out = &Output[tid * Results]; + const std::uint32_t expected = 0xabcd0000u | row.expected; + const auto where = "mix rounding: thread " + std::to_string(tid) + " (" + Hex(row.a) + ", " + Hex(row.b) + ", " + Hex(row.c) + ") result " + std::to_string(row.result); + Require(out[row.result] == expected, where + " is " + Hex(out[row.result]) + ", expected " + Hex(expected)); + if (row.result == 0u) { + Require(out[1] == ((row.expected << 16u) | 0xabcdu), where + ": v_fma_mixhi_f16 is " + Hex(out[1])); + } + } +} + } int main() { try { const auto device = OpenVulkanTestDevice(); if (!device) return VulkanTestSkipped; + const auto capabilities = device->Target().supportedCapabilities; + if (std::find(capabilities.begin(), capabilities.end(), Float64Capability) == capabilities.end()) { + std::puts("skipped, v_fma_mixlo_f16 and v_fma_mixhi_f16 are computed in f64 and the device has no shaderFloat64"); + return VulkanTestSkipped; + } FillInput(); - Run(*device); + Run(*device, MixCode); Check(); + FillRoundingInput(); + Run(*device, RoundingCode); + CheckRounding(); std::puts("mix precision tests passed"); return 0; } catch (const std::exception& error) { diff --git a/core/libs/prx/libSceAgcDriver/tests/execution/NanQuieting.cpp b/core/libs/prx/libSceAgcDriver/tests/execution/NanQuieting.cpp index 07de164aa..b14810e75 100644 --- a/core/libs/prx/libSceAgcDriver/tests/execution/NanQuieting.cpp +++ b/core/libs/prx/libSceAgcDriver/tests/execution/NanQuieting.cpp @@ -21,6 +21,7 @@ using ShaderRecompiler::ShaderStage; constexpr std::uint32_t Threads = 32; constexpr std::uint32_t Inputs = 4; constexpr std::uint32_t Results = 16; +constexpr std::uint32_t Float64Capability = 10; alignas(256) std::array Input{}; alignas(256) std::array Output{}; @@ -34,6 +35,73 @@ alignas(256) constexpr std::array Code{ 0x80011203u, 0xe0781030u, 0x80011603u, 0xbf810000u, }; +alignas(256) constexpr std::array CodeRounding{ + 0x34020084u, 0x34060086u, 0xe0381000u, 0x80000401u, 0xbf8c3f70u, 0x7e140280u, 0x7e160280u, 0x7e180280u, + 0x7e1a0280u, 0x7e1c0280u, 0x7e1e0280u, 0x7e200280u, 0x7e220280u, 0x7e240280u, 0x7e260280u, 0x7e280280u, + 0x7e2a0280u, 0x7e2c0280u, 0x7e2e0280u, 0x7e300280u, 0x7e320280u, 0x7e147d04u, 0x7e183504u, 0x7e1c3104u, + 0x7e202f04u, 0x7e243304u, 0x7e287b04u, 0xd5680016u, 0x02020d04u, 0x7e302107u, 0xe0781000u, 0x80010a03u, + 0xe0781010u, 0x80010e03u, 0xe0781020u, 0x80011203u, 0xe0781030u, 0x80011603u, 0xbf810000u, +}; + +alignas(256) constexpr std::array CodeApproximate{ + 0x34020084u, 0x34060086u, 0xe0381000u, 0x80000401u, 0xbf8c3f70u, 0x7e140280u, 0x7e160280u, 0x7e180280u, + 0x7e1a0280u, 0x7e1c0280u, 0x7e1e0280u, 0x7e200280u, 0x7e220280u, 0x7e240280u, 0x7e260280u, 0x7e280280u, + 0x7e2a0280u, 0x7e2c0280u, 0x7e2e0280u, 0x7e300280u, 0x7e320280u, 0x7e145f04u, 0x7e186304u, 0x7e1c6904u, + 0xd5600010u, 0x03ca08f2u, 0xd5600012u, 0x0411e4f2u, 0xd5600014u, 0x841208f2u, 0x7e2c1f04u, 0xe0781000u, + 0x80010a03u, 0xe0781010u, 0x80010e03u, 0xe0781020u, 0x80011203u, 0xe0781030u, 0x80011603u, 0xbf810000u, +}; + +constexpr std::uint32_t RowsF64[8][4] = { + {0x00000001u, 0x7ff00000u, 0x00000001u, 0x7f800001u}, + {0x12345678u, 0xfff00000u, 0x00000001u, 0xff812345u}, + {0x00000000u, 0x7ff40000u, 0x00000000u, 0x7fa00000u}, + {0x00000000u, 0x7ff80000u, 0x00000001u, 0x7fc00000u}, + {0x12345678u, 0xfff80000u, 0x00000001u, 0xffc12345u}, + {0x00000000u, 0x7ff00000u, 0x00000001u, 0x7f800000u}, + {0x00000000u, 0x3ff80000u, 0x00000001u, 0x3f800000u}, + {0x00000000u, 0xc0040000u, 0x00000002u, 0xc0200000u} +}; +constexpr std::uint32_t ExpectedRoundingIeee[8][16] = { + {0x00000001u, 0x7ff80000u, 0x00000001u, 0x7ff80000u, 0x00000001u, 0x7ff80000u, 0x00000001u, 0x7ff80000u, 0x00000001u, 0x7ff80000u, 0x00000001u, 0x7ff80000u, 0x00000001u, 0x7ff80000u, 0x20000000u, 0x7ff80000u}, + {0x12345678u, 0xfff80000u, 0x12345678u, 0xfff80000u, 0x12345678u, 0xfff80000u, 0x12345678u, 0xfff80000u, 0x12345678u, 0xfff80000u, 0x12345678u, 0xfff80000u, 0x12345678u, 0xfff80000u, 0xa0000000u, 0xfff82468u}, + {0x00000000u, 0x7ffc0000u, 0x00000000u, 0x7ffc0000u, 0x00000000u, 0x7ffc0000u, 0x00000000u, 0x7ffc0000u, 0x00000000u, 0x7ffc0000u, 0x00000000u, 0x7ffc0000u, 0x00000000u, 0x7ffc0000u, 0x00000000u, 0x7ffc0000u}, + {0x00000000u, 0x7ff80000u, 0x00000000u, 0x7ff80000u, 0x00000000u, 0x7ff80000u, 0x00000000u, 0x7ff80000u, 0x00000000u, 0x7ff80000u, 0x00000000u, 0x7ff80000u, 0x00000000u, 0x7ff80000u, 0x00000000u, 0x7ff80000u}, + {0x12345678u, 0xfff80000u, 0x12345678u, 0xfff80000u, 0x12345678u, 0xfff80000u, 0x12345678u, 0xfff80000u, 0x12345678u, 0xfff80000u, 0x12345678u, 0xfff80000u, 0x12345678u, 0xfff80000u, 0xa0000000u, 0xfff82468u}, + {0x00000000u, 0xfff80000u, 0x00000000u, 0x7ff00000u, 0x00000000u, 0x7ff00000u, 0x00000000u, 0x7ff00000u, 0x00000000u, 0x7ff00000u, 0x00000000u, 0x7ff00000u, 0x00000000u, 0x7ff00000u, 0x00000000u, 0x7ff00000u}, + {0x00000000u, 0x3fe00000u, 0x00000000u, 0x3ff00000u, 0x00000000u, 0x40000000u, 0x00000000u, 0x3ff00000u, 0x00000000u, 0x40000000u, 0x00000000u, 0x3fe80000u, 0x00000000u, 0x40080000u, 0x00000000u, 0x3ff00000u}, + {0x00000000u, 0x3fe00000u, 0x00000000u, 0xc0080000u, 0x00000000u, 0xc0000000u, 0x00000000u, 0xc0000000u, 0x00000000u, 0xc0000000u, 0x00000000u, 0xbfe40000u, 0x00000000u, 0xc0240000u, 0x00000000u, 0xc0040000u} +}; +constexpr std::uint32_t ExpectedRoundingNoIeee[8][16] = { + {0x00000001u, 0x7ff00000u, 0x00000001u, 0x7ff00000u, 0x00000001u, 0x7ff00000u, 0x00000001u, 0x7ff00000u, 0x00000001u, 0x7ff00000u, 0x00000001u, 0x7ff00000u, 0x00000001u, 0x7ff00000u, 0x20000000u, 0x7ff00000u}, + {0x12345678u, 0xfff00000u, 0x12345678u, 0xfff00000u, 0x12345678u, 0xfff00000u, 0x12345678u, 0xfff00000u, 0x12345678u, 0xfff00000u, 0x12345678u, 0xfff00000u, 0x12345678u, 0xfff00000u, 0xa0000000u, 0xfff02468u}, + {0x00000000u, 0x7ff40000u, 0x00000000u, 0x7ff40000u, 0x00000000u, 0x7ff40000u, 0x00000000u, 0x7ff40000u, 0x00000000u, 0x7ff40000u, 0x00000000u, 0x7ff40000u, 0x00000000u, 0x7ff40000u, 0x00000000u, 0x7ff40000u}, + {0x00000000u, 0x7ff80000u, 0x00000000u, 0x7ff80000u, 0x00000000u, 0x7ff80000u, 0x00000000u, 0x7ff80000u, 0x00000000u, 0x7ff80000u, 0x00000000u, 0x7ff80000u, 0x00000000u, 0x7ff80000u, 0x00000000u, 0x7ff80000u}, + {0x12345678u, 0xfff80000u, 0x12345678u, 0xfff80000u, 0x12345678u, 0xfff80000u, 0x12345678u, 0xfff80000u, 0x12345678u, 0xfff80000u, 0x12345678u, 0xfff80000u, 0x12345678u, 0xfff80000u, 0xa0000000u, 0xfff82468u}, + {0x00000000u, 0xfff80000u, 0x00000000u, 0x7ff00000u, 0x00000000u, 0x7ff00000u, 0x00000000u, 0x7ff00000u, 0x00000000u, 0x7ff00000u, 0x00000000u, 0x7ff00000u, 0x00000000u, 0x7ff00000u, 0x00000000u, 0x7ff00000u}, + {0x00000000u, 0x3fe00000u, 0x00000000u, 0x3ff00000u, 0x00000000u, 0x40000000u, 0x00000000u, 0x3ff00000u, 0x00000000u, 0x40000000u, 0x00000000u, 0x3fe80000u, 0x00000000u, 0x40080000u, 0x00000000u, 0x3ff00000u}, + {0x00000000u, 0x3fe00000u, 0x00000000u, 0xc0080000u, 0x00000000u, 0xc0000000u, 0x00000000u, 0xc0000000u, 0x00000000u, 0xc0000000u, 0x00000000u, 0xbfe40000u, 0x00000000u, 0xc0240000u, 0x00000000u, 0xc0040000u} +}; +constexpr std::uint32_t ExpectedApproximateIeee[8][16] = { + {0x00000001u, 0x7ff80000u, 0x00000001u, 0x7ff80000u, 0x00000001u, 0x7ff80000u, 0x00000001u, 0x7ff80000u, 0x00000001u, 0x7ff80000u, 0x00000001u, 0xfff80000u, 0x7fc00000u, 0x00000000u, 0x00000000u, 0x00000000u}, + {0x12345678u, 0xfff80000u, 0x12345678u, 0xfff80000u, 0x12345678u, 0xfff80000u, 0x12345678u, 0xfff80000u, 0x12345678u, 0xfff80000u, 0x12345678u, 0x7ff80000u, 0xffc00000u, 0x00000000u, 0x00000000u, 0x00000000u}, + {0x00000000u, 0x7ffc0000u, 0x00000000u, 0x7ffc0000u, 0x00000000u, 0x7ffc0000u, 0x00000000u, 0x7ffc0000u, 0x00000000u, 0x7ffc0000u, 0x00000000u, 0xfffc0000u, 0x7fe00000u, 0x00000000u, 0x00000000u, 0x00000000u}, + {0x00000000u, 0x7ff80000u, 0x00000000u, 0x7ff80000u, 0x00000000u, 0x7ff80000u, 0x00000000u, 0x7ff80000u, 0x00000000u, 0x7ff80000u, 0x00000000u, 0xfff80000u, 0x7fc00000u, 0x00000000u, 0x00000000u, 0x00000000u}, + {0x12345678u, 0xfff80000u, 0x12345678u, 0xfff80000u, 0x12345678u, 0xfff80000u, 0x12345678u, 0xfff80000u, 0x12345678u, 0xfff80000u, 0x12345678u, 0x7ff80000u, 0xffc00000u, 0x00000000u, 0x00000000u, 0x00000000u}, + {0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x7ff00000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x7ff00000u, 0x00000000u, 0xfff80000u, 0x7f800000u, 0x00000000u, 0x00000000u, 0x00000000u}, + {0x54000000u, 0x3fe55555u, 0x74000000u, 0x3fea20bdu, 0x14000000u, 0x3ff3988eu, 0x00000000u, 0x3ff00000u, 0x00000000u, 0x3ff00000u, 0x00000000u, 0xbff00000u, 0x3fc00000u, 0x00000000u, 0x00000000u, 0x00000000u}, + {0x9c000000u, 0xbfd99999u, 0x00000000u, 0xfff80000u, 0x00000000u, 0xfff80000u, 0x00000000u, 0xbff00000u, 0x00000000u, 0xbff00000u, 0x00000000u, 0xbff00000u, 0xc0200000u, 0x00000000u, 0x00000000u, 0x00000000u} +}; +constexpr std::uint32_t ExpectedApproximateNoIeee[8][16] = { + {0x00000001u, 0x7ff00000u, 0x00000001u, 0x7ff00000u, 0x00000001u, 0x7ff00000u, 0x00000001u, 0x7ff00000u, 0x00000001u, 0x7ff00000u, 0x00000001u, 0xfff00000u, 0x7fc00000u, 0x00000000u, 0x00000000u, 0x00000000u}, + {0x12345678u, 0xfff00000u, 0x12345678u, 0xfff00000u, 0x12345678u, 0xfff00000u, 0x12345678u, 0xfff00000u, 0x12345678u, 0xfff00000u, 0x12345678u, 0x7ff00000u, 0xffc00000u, 0x00000000u, 0x00000000u, 0x00000000u}, + {0x00000000u, 0x7ff40000u, 0x00000000u, 0x7ff40000u, 0x00000000u, 0x7ff40000u, 0x00000000u, 0x7ff40000u, 0x00000000u, 0x7ff40000u, 0x00000000u, 0xfff40000u, 0x7fe00000u, 0x00000000u, 0x00000000u, 0x00000000u}, + {0x00000000u, 0x7ff80000u, 0x00000000u, 0x7ff80000u, 0x00000000u, 0x7ff80000u, 0x00000000u, 0x7ff80000u, 0x00000000u, 0x7ff80000u, 0x00000000u, 0xfff80000u, 0x7fc00000u, 0x00000000u, 0x00000000u, 0x00000000u}, + {0x12345678u, 0xfff80000u, 0x12345678u, 0xfff80000u, 0x12345678u, 0xfff80000u, 0x12345678u, 0xfff80000u, 0x12345678u, 0xfff80000u, 0x12345678u, 0x7ff80000u, 0xffc00000u, 0x00000000u, 0x00000000u, 0x00000000u}, + {0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x7ff00000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x7ff00000u, 0x00000000u, 0xfff80000u, 0x7f800000u, 0x00000000u, 0x00000000u, 0x00000000u}, + {0x54000000u, 0x3fe55555u, 0x74000000u, 0x3fea20bdu, 0x14000000u, 0x3ff3988eu, 0x00000000u, 0x3ff00000u, 0x00000000u, 0x3ff00000u, 0x00000000u, 0xbff00000u, 0x3fc00000u, 0x00000000u, 0x00000000u, 0x00000000u}, + {0x9c000000u, 0xbfd99999u, 0x00000000u, 0xfff80000u, 0x00000000u, 0xfff80000u, 0x00000000u, 0xbff00000u, 0x00000000u, 0xbff00000u, 0x00000000u, 0xbff00000u, 0xc0200000u, 0x00000000u, 0x00000000u, 0x00000000u} +}; + constexpr std::uint32_t Rows[32][4] = { {0x7f812345u, 0x40000000u, 0x40400000u, 0x40007d23u}, {0xff800001u, 0x40000000u, 0x40400000u, 0x4000fc01u}, @@ -154,10 +222,27 @@ constexpr const char* Names[16] = { "v_trunc_f32", "v_rndne_f16" }; +constexpr const char* NamesRounding[8] = { + "v_fract_f64", + "v_floor_f64", + "v_ceil_f64", + "v_trunc_f64", + "v_rndne_f64", + "v_frexp_mant_f64", + "v_ldexp_f64", + "v_cvt_f64_f32" +}; +constexpr const char* NamesApproximate[8] = { + "v_rcp_f64", + "v_rsq_f64", + "v_sqrt_f64", + "v_div_fixup_f64 denominator", + "v_div_fixup_f64 numerator", + "v_div_fixup_f64 both", + "v_cvt_f32_f64", + "unused" +}; -void Fill(std::uint32_t tid, std::uint32_t* words) { - std::copy(std::begin(Rows[tid]), std::end(Rows[tid]), words); -} std::array BufferDescriptor(const void* data, std::uint32_t bytes) { const auto address = reinterpret_cast(data); @@ -182,15 +267,20 @@ bool IsNan16(std::uint32_t value) { return (value & 0x7fffu) > 0x7c00u; } -void Run(AgcDriver::VulkanDevice& device, const std::optional& floatMode) { - for (std::uint32_t tid = 0; tid < Threads; ++tid) Fill(tid, &Input[tid * Inputs]); +bool IsNan64(std::uint32_t low, std::uint32_t high) { + return (high & 0x7fffffffu) > 0x7ff00000u || ((high & 0x7fffffffu) == 0x7ff00000u && low != 0u); +} + +template +void Run(AgcDriver::VulkanDevice& device, const std::array& program, const std::uint32_t (&rows)[Count][4], const std::optional& floatMode) { + for (std::uint32_t tid = 0; tid < Threads; ++tid) std::copy(std::begin(rows[tid % Count]), std::end(rows[tid % Count]), &Input[tid * Inputs]); Output.fill(0xdeadbeefu); std::vector userData(8, 0u); const auto input = BufferDescriptor(Input.data(), static_cast(Input.size() * 4u)); const auto output = BufferDescriptor(Output.data(), static_cast(Output.size() * 4u)); std::copy(input.begin(), input.end(), userData.begin()); std::copy(output.begin(), output.end(), userData.begin() + 4); - const std::span code(Code); + const std::span code(program); const std::array memory{{{reinterpret_cast(code.data()), std::as_bytes(code)}}}; const ShaderRecompiler::ShaderComputeStageInfo compute{{Threads, 1, 1}, 0u, {false, false, false}, false, 1}; ShaderRecompiler::RecompileRequest request{ @@ -221,18 +311,49 @@ void Check(const std::uint32_t (&expected)[32][16], const char* mode) { } } +void CheckF64(const std::uint32_t (&expected)[8][16], const char* const (&names)[8], std::uint32_t approximate, const char* mode) { + for (std::uint32_t tid = 0; tid < Threads; ++tid) { + const std::uint32_t* out = &Output[tid * Results]; + const std::uint32_t* want = expected[tid % 8u]; + for (std::uint32_t i = 0; i < 16; ++i) { + const std::uint32_t pair = i / 2u; + if (pair < approximate && !IsNan64(want[pair * 2u], want[pair * 2u + 1u])) continue; + const std::string name = std::string(mode) + " " + names[pair] + (i % 2u ? " high" : " low"); + Expect(tid, out[i], want[i], name.c_str()); + } + } +} + +void CheckF64Modes(AgcDriver::VulkanDevice& device) { + const std::optional modes[3] = {std::nullopt, ShaderRecompiler::ShaderFloatMode{0xc0u, true, false, false}, ShaderRecompiler::ShaderFloatMode{0xc0u, true, true, false}}; + const char* labels[3] = {"no float mode", "IEEE=0", "IEEE=1"}; + for (std::uint32_t mode = 0; mode < 3; ++mode) { + const bool ieee = mode == 2u; + Run(device, CodeRounding, RowsF64, modes[mode]); + CheckF64(ieee ? ExpectedRoundingIeee : ExpectedRoundingNoIeee, NamesRounding, 0u, labels[mode]); + Run(device, CodeApproximate, RowsF64, modes[mode]); + CheckF64(ieee ? ExpectedApproximateIeee : ExpectedApproximateNoIeee, NamesApproximate, 3u, labels[mode]); + } +} + } int main() { try { const auto device = OpenVulkanTestDevice(); if (!device) return VulkanTestSkipped; - Run(*device, std::nullopt); + Run(*device, Code, Rows, std::nullopt); Check(ExpectedNoIeee, "no float mode"); - Run(*device, ShaderRecompiler::ShaderFloatMode{0xc0u, true, false, false}); + Run(*device, Code, Rows, ShaderRecompiler::ShaderFloatMode{0xc0u, true, false, false}); Check(ExpectedNoIeee, "IEEE=0"); - Run(*device, ShaderRecompiler::ShaderFloatMode{0xc0u, true, true, false}); + Run(*device, Code, Rows, ShaderRecompiler::ShaderFloatMode{0xc0u, true, true, false}); Check(ExpectedIeee, "IEEE=1"); + const auto capabilities = device->Target().supportedCapabilities; + if (std::find(capabilities.begin(), capabilities.end(), Float64Capability) == capabilities.end()) { + std::puts("f64 nan quieting skipped, the device has no shaderFloat64"); + } else { + CheckF64Modes(*device); + } std::puts("nan quieting tests passed"); return 0; } catch (const std::exception& error) { diff --git a/core/libs/prx/libSceAgcDriver/tests/execution/NggVertexWaveInfo.cpp b/core/libs/prx/libSceAgcDriver/tests/execution/NggVertexWaveInfo.cpp index ba1c0124f..ad361921f 100644 --- a/core/libs/prx/libSceAgcDriver/tests/execution/NggVertexWaveInfo.cpp +++ b/core/libs/prx/libSceAgcDriver/tests/execution/NggVertexWaveInfo.cpp @@ -68,13 +68,13 @@ void Draw(AgcDriver::VulkanDevice& device, std::uint32_t waveSize) { {ShaderStage::Fragment, reinterpret_cast(PixelCode.data()), PixelCode, 0, {}}, {waveSize, 0, pixelUserData, std::nullopt, pixel, std::nullopt, pixelMemory}, target, - {0, 0, vertexPush, 128 - vertexPush} + PixelPushLayout(vertexPush, target) }; fragment.useCache = false; const auto pixelResult = ShaderRecompiler::Recompile(fragment); const std::array shaders{{ {ShaderStage::Vertex, &vertexResult, 0}, - {ShaderStage::Fragment, &pixelResult, vertexPush} + {ShaderStage::Fragment, &pixelResult, PixelPushOffset(vertexPush, target)} }}; AgcDriver::Graphics::State state{}; diff --git a/core/libs/prx/libSceAgcDriver/tests/execution/PackedAlu.cpp b/core/libs/prx/libSceAgcDriver/tests/execution/PackedAlu.cpp index a54e9e63f..d5f6b199f 100644 --- a/core/libs/prx/libSceAgcDriver/tests/execution/PackedAlu.cpp +++ b/core/libs/prx/libSceAgcDriver/tests/execution/PackedAlu.cpp @@ -23,7 +23,7 @@ constexpr std::uint32_t Results = 64; alignas(256) std::array Input{}; alignas(256) std::array Output{}; -alignas(256) constexpr std::array Code{ +alignas(256) constexpr std::array Code{ 0x34020084, 0x34060088, 0xe0381000, 0x80000401, 0xbf8c3f70, 0xcc00400a, 0x1c1a0b04, 0xcc00280b, 0x141a0b04, 0xcc09400c, 0x1c1a0b04, 0xcc09300d, 0x0c1a0b04, 0xcc01400e, 0x1a020b04, 0xcc01580f, 0x02020b04, 0xcc024010, 0x1a020b04, 0xcc024811, 0x1a020b04, 0xcc0a4012, 0x1a020b04, 0xcc0a4013, @@ -35,10 +35,11 @@ alignas(256) constexpr std::array Code{ 0x1c1a0b04, 0xcc0e5828, 0x041a0b04, 0xcc0e4229, 0x3c1a0b04, 0xcc0f402a, 0x1a020b04, 0xcc0f482b, 0x1a020b04, 0xcc0f412c, 0x5a020b04, 0xcc10402d, 0x1a020b04, 0xcc10402e, 0x02020b04, 0xcc10402f, 0x7a020b04, 0xcc114030, 0x1a020b04, 0xcc115831, 0x1a020b04, 0xcc114232, 0x3a020b04, 0xcc124033, - 0x1a020b04, 0xcc124834, 0x12020b04, 0xcc124135, 0x5a020b04, 0xe0781000, 0x80010a03, 0xe0781010, + 0x1a020b04, 0xcc124834, 0x12020b04, 0xcc124135, 0x5a020b04, 0xcc11c036, 0x1a020b04, 0xcc11c837, + 0x12020b04, 0xcc12c038, 0x1a020b04, 0xcc12c139, 0x5a020b04, 0xe0781000, 0x80010a03, 0xe0781010, 0x80010e03, 0xe0781020, 0x80011203, 0xe0781030, 0x80011603, 0xe0781040, 0x80011a03, 0xe0781050, 0x80011e03, 0xe0781060, 0x80012203, 0xe0781070, 0x80012603, 0xe0781080, 0x80012a03, 0xe0781090, - 0x80012e03, 0xe07810a0, 0x80013203, 0xbf810000, + 0x80012e03, 0xe07810a0, 0x80013203, 0xe07810b0, 0x80013603, 0xbf810000, }; constexpr std::uint32_t Rows[32][4] = { @@ -75,75 +76,75 @@ constexpr std::uint32_t Rows[32][4] = { {0x00007c00u, 0xfc000000u, 0x7d557e55u, 0x00000000u}, {0x7c000000u, 0x00007c00u, 0xfe557d55u, 0x00000000u}, }; -constexpr std::uint32_t Expected[32][44] = { - {0x3c003c00u, 0x3c003c00u, 0x3c003c00u, 0x3c003c00u, 0x00000000u, 0x00000000u, 0x7c007c00u, 0x7c00fc00u, 0x7c007c00u, 0x7c007c00u, 0x7c007c00u, 0x7c007c00u, 0x7c007c00u, 0xfc00fc00u, 0x80000000u, 0x00008000u, 0x80000000u, 0x00008000u, 0x80000000u, 0x80000000u, 0xfc007c00u, 0x7c007c00u, 0x80000000u, 0x80008000u, 0xfc007c00u, 0x8000fc00u, 0x80000000u, 0x00007c00u, 0xfe00fe00u, 0xfe00fe00u, 0xfe00fe00u, 0xfe00fe00u, 0xfc007c00u, 0xfc00fc00u, 0x7c007c00u, 0xfe00fe00u, 0xfe00fe00u, 0xfe00fe00u, 0xfc000000u, 0xfc00fc00u, 0xfc00fc00u, 0x80007c00u, 0x7c000000u, 0x7c007c00u}, - {0xbc007c00u, 0x7c00bc00u, 0xbc007c00u, 0x7c00bc00u, 0x00000000u, 0x00000000u, 0x78007800u, 0x7800f800u, 0x78007800u, 0x78007800u, 0x80008000u, 0x80008000u, 0x80008000u, 0x00000000u, 0x7c00fc00u, 0xfc007c00u, 0x7c00fc00u, 0xfc007c00u, 0x7c00fc00u, 0x7c00fc00u, 0x7c007c00u, 0x7c007c00u, 0xfc00fc00u, 0xfc00fc00u, 0xfc00fc00u, 0x7c00fc00u, 0x7c007c00u, 0xfc007c00u, 0xfc00fe00u, 0xfc00fe00u, 0xfc00fe00u, 0x7c007c00u, 0xfe00fe00u, 0xfe00fc00u, 0x7c007c00u, 0xfc00fc00u, 0xfc00fc00u, 0xfc00fc00u, 0xfc00fc00u, 0xfc00fc00u, 0xfc00fc00u, 0x7c007c00u, 0x7c00fc00u, 0x7c007c00u}, - {0x7bffc000u, 0xc000fbffu, 0x7bffc000u, 0x7fff3c00u, 0x43ff0400u, 0x040043ffu, 0xbc00fbffu, 0xbc003c00u, 0xbc00fbffu, 0xfbfffbffu, 0xbc02fc01u, 0xbc02bc02u, 0xbc02fc01u, 0x7c013c02u, 0xfffeffffu, 0xfffe7fffu, 0x3fffffffu, 0xffff3fffu, 0x3fffffffu, 0x3fffffffu, 0x7fffffffu, 0xffffffffu, 0x3c01fc00u, 0x3c013c01u, 0x7fffffffu, 0xfc00ffffu, 0x3c01fc00u, 0x3c017fffu, 0x7fffffffu, 0x7fffffffu, 0xffff7fffu, 0xffffffffu, 0x7fffffffu, 0x7fffffffu, 0x7fff7fffu, 0x7fffffffu, 0xffffffffu, 0x7fff7fffu, 0x3c01fc00u, 0x3c013c01u, 0x3c017c00u, 0x3c01fc00u, 0xfc003c01u, 0xbc01fc00u}, - {0xc0009400u, 0x1400c000u, 0xc0009400u, 0x1400c000u, 0x00008000u, 0x80000000u, 0x1400ffffu, 0x14009400u, 0x1400ffffu, 0xffffffffu, 0x1400ffffu, 0x14001400u, 0x1400ffffu, 0x7fff9400u, 0x00000000u, 0x80000000u, 0x00000001u, 0x00010000u, 0x0000ffffu, 0x00008000u, 0x14007fffu, 0x7fff7fffu, 0x00008000u, 0x00000000u, 0x14008000u, 0x7fff8000u, 0x00007fffu, 0x14000000u, 0xc0007fffu, 0xc0007fffu, 0x7fff1400u, 0xc000ffffu, 0x14007fffu, 0x14001400u, 0x94007fffu, 0x00007fffu, 0x7fff7fffu, 0x0000ffffu, 0x00008000u, 0x00000000u, 0x80008000u, 0x14008000u, 0x00001400u, 0x00000000u}, - {0x7e00d5feu, 0x7ffe8000u, 0x7e00d5feu, 0x81fe7e00u, 0x8000d600u, 0xd6008000u, 0x7fff7c55u, 0x7fff7dffu, 0x7fff7c55u, 0x7c557c55u, 0x7fff8055u, 0x7fff7fffu, 0x7fff8055u, 0x7e5581ffu, 0x0000c000u, 0x00000000u, 0x000103f0u, 0x03f00001u, 0xffff03f0u, 0xffff0000u, 0xffff7e00u, 0x7e007e00u, 0x8000fe55u, 0x80008000u, 0xfffffe55u, 0xfe55ffffu, 0x80007e00u, 0x7e008000u, 0xfffffe55u, 0xfffffe55u, 0xfe55ffffu, 0xffff7e55u, 0xfffffe55u, 0xffffffffu, 0x7ffffe55u, 0xfffffe55u, 0xfe55fe55u, 0xffff7e55u, 0x8000fe55u, 0x80008000u, 0x00007e55u, 0x8000fe55u, 0x8000ffffu, 0x8000fe55u}, - {0x80007656u, 0x4254c3feu, 0x80007656u, 0x4254c3feu, 0x00017801u, 0x78010001u, 0xfffe7802u, 0xfffebc00u, 0xfffe7802u, 0x78027802u, 0x00000000u, 0x00000000u, 0x00000000u, 0xbc0243feu, 0x80007802u, 0x8000fffeu, 0x00001e00u, 0x1e000000u, 0x00001e00u, 0x00000000u, 0x7fff3c01u, 0x3c013c01u, 0x7fff3c01u, 0x7fff7fffu, 0x7fff3c01u, 0x7fff7fffu, 0x7fff3c01u, 0x3c013c01u, 0x7ffffe55u, 0x7ffffe55u, 0x7fff7fffu, 0x7ffffe55u, 0x7fff4001u, 0x7fff7fffu, 0xffff0000u, 0x7fff3c02u, 0x3c023c02u, 0x7fff3c02u, 0x7fff3c01u, 0x7fff7fffu, 0x7fffbc01u, 0x7fff3c01u, 0x3c013c01u, 0xffff3c01u}, - {0x7bff6000u, 0x60007bffu, 0x7bff6000u, 0x60007bffu, 0x00000000u, 0x00000000u, 0xedcc6dccu, 0xedcc6dccu, 0xedcc6dccu, 0x6dcc6dccu, 0xedcc6dccu, 0xedccedccu, 0xedcc6dccu, 0xedcc6dccu, 0x00000000u, 0x00000000u, 0x00000008u, 0x00080000u, 0x0000fff8u, 0x0000fff8u, 0x0000edccu, 0xedccedccu, 0xedcc8000u, 0xedccedccu, 0xedccedccu, 0xedccedccu, 0x00008000u, 0x80000000u, 0x7bff6000u, 0x7bff6000u, 0x7bff6000u, 0x7bff6000u, 0xedccedccu, 0xedccedccu, 0x6dccedccu, 0x80000000u, 0x00000000u, 0x80000000u, 0xedccedccu, 0xedccedccu, 0xedcc8000u, 0x00008000u, 0x00008000u, 0x6dcc0000u}, - {0x00001dccu, 0xedccd000u, 0x00001dccu, 0xbdcc0000u, 0x00003000u, 0x30000000u, 0x8000ce34u, 0x80009634u, 0x8000ce34u, 0xce34ce34u, 0x8800a9ccu, 0x88008800u, 0x8800a9ccu, 0xc00071ccu, 0xfc001234u, 0x1234fc00u, 0xfc001234u, 0x1234fc00u, 0xfc001234u, 0xfc001234u, 0xfc001234u, 0x12341234u, 0x8400bc00u, 0x84008400u, 0xfc00bc00u, 0xfc008400u, 0x84001234u, 0x1234bc00u, 0x7c00edccu, 0x7c00edccu, 0x92347c00u, 0xfc00edccu, 0xfc00bbfeu, 0xfc0011b4u, 0xfc00bc01u, 0x7c009234u, 0x92349234u, 0x7c009234u, 0xfc00bc00u, 0xfc00fc00u, 0x84001234u, 0x84001234u, 0xbc001234u, 0x04009234u}, - {0xc99f5e41u, 0x4ceb74ddu, 0xc99f5e41u, 0xcd1df4abu, 0xcb9f0801u, 0x0801cb9fu, 0xb278f7feu, 0xb278f922u, 0xb278f7feu, 0xf7fef7feu, 0x47ce0000u, 0x47ce47ceu, 0x47ce0000u, 0x46aa0124u, 0xaaa88000u, 0xdff88000u, 0x06aa0000u, 0x000006aau, 0x06aa0000u, 0x06aa0f7fu, 0x7d237bffu, 0x7bff7bffu, 0x35557bffu, 0x35553555u, 0x7d237bffu, 0x7bff7d23u, 0x35557bffu, 0x7bff3555u, 0x7f237c00u, 0x7f237c00u, 0xfe007f23u, 0x7f23fc00u, 0x7f237c00u, 0x7f237f23u, 0xff230000u, 0x7f237c00u, 0x7c007c00u, 0x7f237c00u, 0x7f237bffu, 0x7f237f23u, 0x7f23fbffu, 0x7f237bffu, 0x7bff7f23u, 0xff237bffu}, - {0x00008000u, 0x80000000u, 0x00008000u, 0x80000000u, 0x80008000u, 0x80008000u, 0xb555fc01u, 0xb555b555u, 0xb555fc01u, 0xfc01fc01u, 0xb555fc01u, 0xb555b555u, 0xb555fc01u, 0xfc01b555u, 0x00000000u, 0x00000000u, 0x04004000u, 0x40000400u, 0xfc00c000u, 0xfc00fc00u, 0x35557c01u, 0x7c017c01u, 0x80008000u, 0x80008000u, 0x80008000u, 0x80008000u, 0x35557c01u, 0x35557c01u, 0x80007e01u, 0x80007e01u, 0x7e010000u, 0x0000fe01u, 0x35557e01u, 0x35553555u, 0xb5557e01u, 0x80007e01u, 0x7e017e01u, 0x8000fe01u, 0x80007e01u, 0x80008000u, 0x0000fe01u, 0x35557e01u, 0x7e013555u, 0x80007e01u}, - {0x03ff0000u, 0xa2480000u, 0x03ff0000u, 0x0000a248u, 0x03ff0000u, 0x000003ffu, 0xfc00c248u, 0xfc007fffu, 0xfc00c248u, 0xc248c248u, 0x03fec248u, 0x03fe03feu, 0x03fec248u, 0x46477fffu, 0x80000000u, 0x00000100u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x7fff0000u, 0x00000000u, 0x7c01c248u, 0x7c017c01u, 0x7fffc248u, 0xc2487fffu, 0x7c010000u, 0x00007c01u, 0x7fff0000u, 0x7fff0000u, 0x00007fffu, 0x7fff0000u, 0x7fffc248u, 0x7fff7fffu, 0xffffc248u, 0x7fff8000u, 0x80008000u, 0x7fff8000u, 0x7e01c248u, 0x7e017e01u, 0xfe010000u, 0x7e010000u, 0x7e010000u, 0x7e018000u}, - {0xc2337fffu, 0x87fd7fffu, 0xc2337fffu, 0xffff07fdu, 0x42348000u, 0x80004234u, 0x4e35bffeu, 0x4e35d234u, 0x4e35bffeu, 0xbffebffeu, 0xd6333ffeu, 0xd633d633u, 0xd6333ffeu, 0xc3fd5234u, 0xc0100000u, 0x00004000u, 0x03c00003u, 0x000303c0u, 0x03c0ffffu, 0x03c0fc00u, 0x3c01fffeu, 0xfffefffeu, 0x1234c000u, 0x12341234u, 0x3c01fffeu, 0xfffec000u, 0x1234c000u, 0x12343c01u, 0x7ffffffeu, 0x7ffffffeu, 0xfffeffffu, 0x7fff7ffeu, 0x3c02fffeu, 0x3c02bfffu, 0x3c00fffeu, 0x1236fffeu, 0xfffefffeu, 0x12367ffeu, 0x1234c000u, 0x12341234u, 0xbc01c000u, 0x3c01c000u, 0x3c011234u, 0x3c014000u}, - {0x9c00a56eu, 0xcffef560u, 0x9c00a56eu, 0x795e4c00u, 0x2000a570u, 0xa5702000u, 0xc6480000u, 0xc6483014u, 0xc6480000u, 0x00000000u, 0xbe482468u, 0xbe48be48u, 0xbe482468u, 0x8e34547cu, 0x0000dcc0u, 0xcc004000u, 0x00840edcu, 0x0edc0084u, 0xff84fedcu, 0xff84ffedu, 0x42481234u, 0x12341234u, 0x8400edccu, 0x84008400u, 0x8400edccu, 0x8400edccu, 0x42481234u, 0x42481234u, 0x7c00fffeu, 0x7c00fffeu, 0x7c00fffeu, 0x7c00fffeu, 0x4248edccu, 0x4248edcbu, 0xc2486dccu, 0x8a48c47fu, 0xc47fc47fu, 0x8a48c47fu, 0x8400edccu, 0x84008400u, 0x0400edccu, 0x42481234u, 0x12344248u, 0x84006dccu}, - {0x04023a00u, 0x3a000402u, 0x04023a00u, 0x38020600u, 0x00020200u, 0x02000002u, 0xfffd7dffu, 0xfffd7ffdu, 0xfffd7dffu, 0x7dff7dffu, 0xffff7e01u, 0xffffffffu, 0xffff7e01u, 0xfe017fffu, 0xc0007fffu, 0xc000ffffu, 0x00037fffu, 0x7fff0003u, 0xffff7fffu, 0xffff0001u, 0xffff7fffu, 0x7fff7fffu, 0xfffefe00u, 0xfffefffeu, 0xfffffe00u, 0xfffffffeu, 0xfffe7fffu, 0x7ffffe00u, 0xfffefe00u, 0xfffefe00u, 0xfe00fffeu, 0xfffe7e00u, 0xfffefe00u, 0xfffefffeu, 0x7ffefe00u, 0xfffefe00u, 0xfe00fe00u, 0xfffe7e00u, 0xfffefe00u, 0xfffefffeu, 0xfffe7e00u, 0xfffefe00u, 0xfe00fffeu, 0x7ffefe00u}, - {0x96003bb8u, 0xee00ebccu, 0x96003bb8u, 0x6bcc6e00u, 0x9800bdb8u, 0xbdb89800u, 0x9034c247u, 0x90349233u, 0x9034c247u, 0xc247c247u, 0x9434c249u, 0x94349434u, 0x9434c249u, 0xc4489235u, 0xe000ff00u, 0xfff00000u, 0x07e0007fu, 0x007f07e0u, 0x07e0007fu, 0x07e007ffu, 0x7e007fffu, 0x7fff7fffu, 0x12344248u, 0x12341234u, 0x7e007fffu, 0x7e007fffu, 0x12344248u, 0x12344248u, 0x7e007fffu, 0x7e007fffu, 0x7fff7e00u, 0xfe007fffu, 0x7e007fffu, 0x7e007fffu, 0x7e00ffffu, 0x7e007fffu, 0x7fff7fffu, 0x7e00ffffu, 0x12344248u, 0x12341234u, 0x1234c248u, 0x12344248u, 0x42481234u, 0x92344248u}, - {0x80000800u, 0x08006340u, 0x80000800u, 0xeb408000u, 0x00000000u, 0x00000000u, 0x35555240u, 0x35558b95u, 0x35555240u, 0x52405240u, 0x3555a5c0u, 0x35553555u, 0x3555a5c0u, 0xfc00df15u, 0x00005640u, 0xc8000000u, 0x00005640u, 0x56400000u, 0x00005640u, 0x000002b2u, 0x35555640u, 0x56405640u, 0x0000fc00u, 0x00000000u, 0x3555fc00u, 0xfc005640u, 0x00005640u, 0x35550000u, 0x0000fc00u, 0x0000fc00u, 0xfc000800u, 0x80007c00u, 0x3555fc00u, 0x35555645u, 0xb555fc00u, 0x0000fc00u, 0xfc00fc00u, 0x0000fc00u, 0x0000fc00u, 0x00000000u, 0x80005640u, 0x35555640u, 0x00005640u, 0x0000d640u}, - {0xdd70b2c8u, 0x611b56e4u, 0xdd70b2c8u, 0x9ee3191cu, 0xa57032c9u, 0x32c9a570u, 0x0000fa46u, 0x00006aefu, 0x0000fa46u, 0xfa46fa46u, 0xdb980000u, 0xdb98db98u, 0xdb980000u, 0x6aef70a9u, 0x4000e918u, 0x300091a0u, 0x00010fa4u, 0x0fa40001u, 0x00010fa4u, 0x00010007u, 0x12347d23u, 0x7d237d23u, 0xedcc7d23u, 0xedccedccu, 0xedcc7d23u, 0x7d23edccu, 0x12347d23u, 0x7d231234u, 0xc3fd7f23u, 0xc3fd7f23u, 0x7f237fffu, 0x44ffff23u, 0xedcc7f23u, 0xedcc7f23u, 0x6dcc7f23u, 0xc47f7f23u, 0x7f237f23u, 0xc47fff23u, 0xedcc7f23u, 0xedccedccu, 0xedccff23u, 0x12347f23u, 0x7f237f23u, 0x6dcc7f23u}, - {0xc000d640u, 0xd6413c00u, 0xc000d640u, 0x5640bc01u, 0x84008000u, 0x80008400u, 0xfbff7fffu, 0xfbfffc00u, 0xfbff7fffu, 0x7fff7fffu, 0xfc017fffu, 0xfc01fc01u, 0xfc017fffu, 0x8000fc00u, 0x7fff0000u, 0x80008000u, 0x7fff0001u, 0x00017fffu, 0x7fffffffu, 0x7fff8000u, 0x7fffffffu, 0xffffffffu, 0x7c008000u, 0x7c007c00u, 0x7fffffffu, 0xffff8000u, 0x7c008000u, 0x7c007fffu, 0x7fffffffu, 0x7fffffffu, 0xffff7fffu, 0xffff7fffu, 0x7fffffffu, 0x7fff7c00u, 0x7fffffffu, 0x7fffffffu, 0xffffffffu, 0x7fff7fffu, 0x7c008000u, 0x7c007c00u, 0x7c008000u, 0x7c008000u, 0xffff7c00u, 0xfc000000u}, - {0xfffecd56u, 0x38002153u, 0xfffecd56u, 0x5955fffeu, 0x00009556u, 0x95560000u, 0x3c013553u, 0x3c017156u, 0x3c013553u, 0x35533553u, 0x3c01caa9u, 0x3c013c01u, 0x3c01caa9u, 0xfffe06acu, 0x00004000u, 0x6aaa0000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00001aaau, 0x3c013555u, 0x35553555u, 0x0000fffeu, 0x00000000u, 0x3c01fffeu, 0xfffe3c01u, 0x00003555u, 0x35550000u, 0xfffefffeu, 0xfffefffeu, 0xfffe3800u, 0xfffe7ffeu, 0x3c01fffeu, 0x3c013d56u, 0xbc01fffeu, 0x0000fffeu, 0xfffefffeu, 0x00007ffeu, 0x00003555u, 0x00000000u, 0x80003555u, 0x3c013555u, 0x00003c01u, 0x0000b555u}, - {0x31b86c40u, 0xd5b84840u, 0x31b86c40u, 0x6c40b1b8u, 0x1db83440u, 0x34401db8u, 0x3e478490u, 0x3e470490u, 0x3e478490u, 0x84908490u, 0x46490000u, 0x46494649u, 0x46490000u, 0xc6498000u, 0xff004800u, 0x4800ff00u, 0x007b0042u, 0x0042007bu, 0x007b0042u, 0x007b0042u, 0x7bff4248u, 0x42484248u, 0xc2484248u, 0xc248c248u, 0xc2484248u, 0x7bffc248u, 0x7bff4248u, 0x42484248u, 0xfc00492fu, 0xfc00492fu, 0x48effc00u, 0x7c00c8afu, 0x7bff4648u, 0x7bff0000u, 0x7bff0000u, 0xfc0048efu, 0x48ef48efu, 0xfc0048efu, 0xc2484248u, 0xc248c248u, 0xfbffc248u, 0x7bff4248u, 0x7bff4248u, 0x7bff4248u}, - {0x9a908000u, 0xdcbc4000u, 0x9a908000u, 0x80009cbcu, 0x5a900000u, 0x00005a90u, 0xdb98fe55u, 0xdb98edccu, 0xdb98fe55u, 0xfe55fe55u, 0x0000fe55u, 0x00000000u, 0x0000fe55u, 0x1089edccu, 0xc0000000u, 0x0000b980u, 0x000e0000u, 0x0000000eu, 0xfffe0000u, 0xfffe0000u, 0xedcc0000u, 0x00000000u, 0xedccfe55u, 0xedccedccu, 0xedccfe55u, 0xfe55edccu, 0xedcc0000u, 0x0000edccu, 0x7c00fe55u, 0x7c00fe55u, 0xfe557c00u, 0xfc007e55u, 0xf1ccfe55u, 0xf1ccedccu, 0x0000fe55u, 0x7c00fe55u, 0xfe55fe55u, 0x7c007e55u, 0xedcc0000u, 0xedccedccu, 0xedcc0000u, 0xedcc0000u, 0xedcc0000u, 0x6dcc8000u}, - {0x12344400u, 0x14000a34u, 0x12344400u, 0x0c001234u, 0x00003000u, 0x30000000u, 0xfffef1ccu, 0xfffe03feu, 0xfffef1ccu, 0xf1ccf1ccu, 0xfffee9ccu, 0xfffefffeu, 0xfffee9ccu, 0xedccfbfeu, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000400u, 0x04000400u, 0xfffeedccu, 0xfffefffeu, 0xfffeedccu, 0xedccfffeu, 0x00000400u, 0x04000000u, 0xfffeb5c8u, 0xfffeb5c8u, 0xb5c9fffeu, 0xfffe35d0u, 0xfffeedccu, 0xfffefffeu, 0x7ffeedccu, 0xfffeb5ccu, 0xb5ccb5ccu, 0xfffeb5ccu, 0x0000edccu, 0x00000000u, 0x80000400u, 0x00000400u, 0x00000400u, 0x00008400u}, - {0x7fff9c00u, 0xbc00c69eu, 0x7fff9c00u, 0x029f7fffu, 0x0000e000u, 0xe0000000u, 0x7d235e55u, 0x7d237b78u, 0x7d235e55u, 0x5e555e55u, 0x7d2361abu, 0x7d237d23u, 0x7d2361abu, 0x60007eceu, 0x0000fe55u, 0xf2a80000u, 0x0000fe55u, 0xfe550000u, 0x0000fe55u, 0x0000ffcau, 0x7d236000u, 0x60006000u, 0x0000fe55u, 0x00000000u, 0x7d23fe55u, 0x6000fe55u, 0x00006000u, 0x7d230000u, 0x7f23fe55u, 0x7f23fe55u, 0xfe557f23u, 0x7f23fe55u, 0x7f23fe55u, 0x7f237f23u, 0xff237e55u, 0x7f23fe55u, 0xfe55fe55u, 0x7f237e55u, 0x7f236000u, 0x7f237f23u, 0x7f23e000u, 0x7f236000u, 0x60007f23u, 0xff236000u}, - {0x3c008400u, 0x84007c00u, 0x3c008400u, 0x84007c00u, 0xc0000000u, 0x0000c000u, 0x3c018000u, 0x3c017c01u, 0x3c018000u, 0x80008000u, 0xbc018000u, 0xbc01bc01u, 0xbc018000u, 0xc0007c01u, 0x80000000u, 0x0000c000u, 0x60000000u, 0x00006000u, 0xe0000000u, 0xe0000000u, 0x7c010000u, 0x00000000u, 0xc0008000u, 0xc000c000u, 0xc0008000u, 0xc0007c01u, 0x7c010000u, 0x00008000u, 0x7e018400u, 0x7e018400u, 0x7c007e01u, 0x7e018400u, 0x7e010000u, 0x7e017e01u, 0xfe018000u, 0x7e018000u, 0x80008000u, 0x7e018000u, 0x7e018000u, 0x7e017e01u, 0x7e010000u, 0x7e010000u, 0x80007e01u, 0xfe018000u}, - {0x40006fe8u, 0xaa007de8u, 0x40006fe8u, 0xe7e84000u, 0x2c00f1e8u, 0xf1e82c00u, 0xb155c09du, 0xb155f79du, 0xb155c09du, 0xc09dc09du, 0xb9553c0du, 0xb955b955u, 0xb9553c0du, 0x8255730du, 0x80004900u, 0x49008000u, 0x03e00612u, 0x061203e0u, 0x03e0fe12u, 0x03e0fe12u, 0x7c00fe55u, 0xfe55fe55u, 0x3555c248u, 0x35553555u, 0x7c00fe55u, 0xfe55c248u, 0x3555c248u, 0x35557c00u, 0x7c00fe55u, 0x7c00fe55u, 0xfe557e00u, 0xfc007e55u, 0x7c00fe55u, 0x7c00c19du, 0x7c00fe55u, 0x7c00fe55u, 0xfe55fe55u, 0x7c007e55u, 0x3555c248u, 0x35553555u, 0xfc00c248u, 0x7c00c248u, 0x7c003555u, 0x7c004248u}, - {0x755fffffu, 0x5e480e33u, 0x755fffffu, 0x4e341e47u, 0x7960c3ffu, 0xc3ff7960u, 0x30143c00u, 0x3014edcbu, 0x30143c00u, 0x3c003c00u, 0xab843c02u, 0xab84ab84u, 0xab843c02u, 0xf9b9edcdu, 0x8000fffeu, 0xf0008490u, 0x00047fffu, 0x7fff0004u, 0x0004ffffu, 0x0004ffffu, 0x42483c01u, 0x3c013c01u, 0xedccffffu, 0xedccedccu, 0xedccffffu, 0x4248ffffu, 0x42483c01u, 0xedcc3c01u, 0xfc00ffffu, 0xfc00ffffu, 0xfffff48du, 0xf9b8ffffu, 0xedcbffffu, 0xedcbffffu, 0x6dcd7fffu, 0xf48dffffu, 0xffffffffu, 0xf48d7fffu, 0xedcc3c01u, 0xedccedccu, 0xedccbc01u, 0x42483c01u, 0x4248edccu, 0x6dcc3c01u}, - {0x8132edccu, 0x70a9fe55u, 0x8132edccu, 0xedcc8132u, 0x82dd0000u, 0x000082ddu, 0x7d227d23u, 0x7d227d23u, 0x7d227d23u, 0x7d237d23u, 0x7d247d23u, 0x7d247d24u, 0x7d247d23u, 0x7d247d23u, 0xfff80000u, 0x0000fff8u, 0x1fff0000u, 0x00001fffu, 0xffff0000u, 0xffff0000u, 0x7d237d23u, 0x7d237d23u, 0xffff0000u, 0xffffffffu, 0xffff7d23u, 0xffff7d23u, 0x7d230000u, 0x00007d23u, 0x7f237f23u, 0x7f237f23u, 0x7f237f23u, 0x7f23ff23u, 0x7f237f23u, 0x7f237f23u, 0xff237f23u, 0x7f237f23u, 0x7f237f23u, 0x7f23ff23u, 0x7f237f23u, 0x7f237f23u, 0x7f23ff23u, 0x7f237f23u, 0x7f237f23u, 0xff237f23u}, - {0x00008000u, 0x80002000u, 0x00008000u, 0xa0000000u, 0x00000000u, 0x00000000u, 0xc2480400u, 0xc248c648u, 0xc2480400u, 0x04000400u, 0xc248fc00u, 0xc248c248u, 0xc248fc00u, 0x0000be48u, 0x00000400u, 0x00000000u, 0x00000400u, 0x04000000u, 0x00000400u, 0x00000004u, 0x00000400u, 0x04000400u, 0xc2480000u, 0xc248c248u, 0xc2480400u, 0x0000c248u, 0x00000000u, 0x04000000u, 0x00000000u, 0x00000000u, 0x00008000u, 0x00008000u, 0xc2480400u, 0xc248c248u, 0x42488400u, 0x80000000u, 0x00000000u, 0x80000000u, 0xc2480000u, 0xc248c248u, 0xc2488000u, 0x00000400u, 0x00000400u, 0x42480000u}, - {0xcaab5400u, 0x9634ac00u, 0xcaab5400u, 0x30001234u, 0xcaabd000u, 0xd000caabu, 0x3554e9ccu, 0x35543155u, 0x3554e9ccu, 0xe9cce9ccu, 0x3556f1ccu, 0x35563556u, 0x3556f1ccu, 0xedcd3955u, 0xffe00000u, 0x8000f000u, 0x07ff000fu, 0x000f07ffu, 0xffffffffu, 0xffffffe0u, 0x3555fc00u, 0xfc00fc00u, 0xffffedccu, 0xffffffffu, 0xfffffc00u, 0xfffffc00u, 0x3555edccu, 0x3555edccu, 0xffff7c00u, 0xffff7c00u, 0x7c00ffffu, 0x7ffffc00u, 0xfffffc00u, 0xfffffc00u, 0xffff7c00u, 0xffff7c00u, 0x7c007c00u, 0xffff7c00u, 0x3555fc00u, 0x35553555u, 0x3555fc00u, 0x3555edccu, 0xedcc3555u, 0xb5557c00u}, - {0x8400c000u, 0xa11c8400u, 0x8400c000u, 0xc000651cu, 0x00000000u, 0x00000000u, 0x7d231234u, 0x7d230000u, 0x7d231234u, 0x12341234u, 0x82dd1234u, 0x82dd82ddu, 0x82dd1234u, 0x95110000u, 0x7d230000u, 0x0000d230u, 0x7d230000u, 0x00007d23u, 0x7d230000u, 0x7d230000u, 0x7d231234u, 0x12341234u, 0x00000000u, 0x00000000u, 0x7d231234u, 0x7d230000u, 0x00000000u, 0x00001234u, 0x7f23c000u, 0x7f23c000u, 0x84007f23u, 0xff23c000u, 0x7f231234u, 0x7f230000u, 0x7f231234u, 0x7f230000u, 0x00000000u, 0x7f230000u, 0x7f230000u, 0x7f237f23u, 0xff239234u, 0x7f231234u, 0x7f230000u, 0x7f231234u}, - {0x7d557e55u, 0x7e557d55u, 0x7d557e55u, 0x7e557d55u, 0x00000000u, 0x00000000u, 0xfc007c00u, 0xfc000000u, 0xfc007c00u, 0x7c007c00u, 0x04007c00u, 0x04000400u, 0x04007c00u, 0x80000000u, 0xfc000000u, 0x0000fc00u, 0xfc000000u, 0x0000fc00u, 0xfc000000u, 0xfc000000u, 0x00007c00u, 0x7c007c00u, 0xfc000000u, 0xfc00fc00u, 0xfc007c00u, 0xfc000000u, 0x00000000u, 0x00007c00u, 0xfe00fe00u, 0xfe00fe00u, 0xfe00fe00u, 0xfe00fe00u, 0xfc007c00u, 0xfc000000u, 0xfc007c00u, 0xfe00fe00u, 0xfe00fe00u, 0xfe00fe00u, 0xfc000000u, 0xfc00fc00u, 0x0000fc00u, 0x00007c00u, 0x7c000000u, 0x80007c00u}, - {0xfe557d55u, 0x7d55fe55u, 0xfe557d55u, 0x7d55fe55u, 0x00000000u, 0x00000000u, 0x7c007c00u, 0x7c00f800u, 0x7c007c00u, 0x7c007c00u, 0x7c008400u, 0x7c007c00u, 0x7c008400u, 0x00000000u, 0x00007c00u, 0x7c000000u, 0x00007c00u, 0x7c000000u, 0x00007c00u, 0x00007c00u, 0x7c007c00u, 0x7c007c00u, 0x00000000u, 0x00000000u, 0x7c007c00u, 0x00007c00u, 0x00000000u, 0x7c000000u, 0xfe00fe00u, 0xfe00fe00u, 0xfe00fe00u, 0xfe00fe00u, 0x7c007c00u, 0x7c007c00u, 0xfc00fc00u, 0xfe00fe00u, 0xfe00fe00u, 0xfe00fe00u, 0x00000000u, 0x00000000u, 0x80008000u, 0x7c007c00u, 0x00007c00u, 0x00000000u}, +constexpr std::uint32_t Expected[32][48] = { + {0x3c003c00u, 0x3c003c00u, 0x3c003c00u, 0x3c003c00u, 0x00000000u, 0x00000000u, 0x7c007c00u, 0x7c00fc00u, 0x7c007c00u, 0x7c007c00u, 0x7c007c00u, 0x7c007c00u, 0x7c007c00u, 0xfc00fc00u, 0x80000000u, 0x00008000u, 0x80000000u, 0x00008000u, 0x80000000u, 0x80000000u, 0xfc007c00u, 0x7c007c00u, 0x80000000u, 0x80008000u, 0xfc007c00u, 0x8000fc00u, 0x80000000u, 0x00007c00u, 0xfe00fe00u, 0xfe00fe00u, 0xfe00fe00u, 0xfe00fe00u, 0xfc007c00u, 0xfc00fc00u, 0x7c007c00u, 0xfe00fe00u, 0xfe00fe00u, 0xfe00fe00u, 0xfc000000u, 0xfc00fc00u, 0xfc00fc00u, 0x80007c00u, 0x7c000000u, 0x7c007c00u, 0x00000000u, 0x00000000u, 0x00003c00u, 0x3c003c00u}, + {0xbc007c00u, 0x7c00bc00u, 0xbc007c00u, 0x7c00bc00u, 0x00000000u, 0x00000000u, 0x78007800u, 0x7800f800u, 0x78007800u, 0x78007800u, 0x80008000u, 0x80008000u, 0x80008000u, 0x00000000u, 0x7c00fc00u, 0xfc007c00u, 0x7c00fc00u, 0xfc007c00u, 0x7c00fc00u, 0x7c00fc00u, 0x7c007c00u, 0x7c007c00u, 0xfc00fc00u, 0xfc00fc00u, 0xfc00fc00u, 0x7c00fc00u, 0x7c007c00u, 0xfc007c00u, 0xfc00fe00u, 0xfc00fe00u, 0xfc00fe00u, 0x7c007c00u, 0xfe00fe00u, 0xfe00fc00u, 0x7c007c00u, 0xfc00fc00u, 0xfc00fc00u, 0xfc00fc00u, 0xfc00fc00u, 0xfc00fc00u, 0xfc00fc00u, 0x7c007c00u, 0x7c00fc00u, 0x7c007c00u, 0x00000000u, 0x3c000000u, 0x3c003c00u, 0x3c003c00u}, + {0x7bffc000u, 0xc000fbffu, 0x7bffc000u, 0x7fff3c00u, 0x43ff0400u, 0x040043ffu, 0xbc00fbffu, 0xbc003c00u, 0xbc00fbffu, 0xfbfffbffu, 0xbc02fc01u, 0xbc02bc02u, 0xbc02fc01u, 0x7c013c02u, 0xfffeffffu, 0xfffe7fffu, 0x3fffffffu, 0xffff3fffu, 0x3fffffffu, 0x3fffffffu, 0x7fffffffu, 0xffffffffu, 0x3c01fc00u, 0x3c013c01u, 0x7fffffffu, 0xfc00ffffu, 0x3c01fc00u, 0x3c017fffu, 0x7fffffffu, 0x7fffffffu, 0xffff7fffu, 0xffffffffu, 0x7fffffffu, 0x7fffffffu, 0x7fff7fffu, 0x7fffffffu, 0xffffffffu, 0x7fff7fffu, 0x3c01fc00u, 0x3c013c01u, 0x3c017c00u, 0x3c01fc00u, 0xfc003c01u, 0xbc01fc00u, 0x3c000000u, 0x00003c00u, 0x3c000000u, 0x00000000u}, + {0xc0009400u, 0x1400c000u, 0xc0009400u, 0x1400c000u, 0x00008000u, 0x80000000u, 0x1400ffffu, 0x14009400u, 0x1400ffffu, 0xffffffffu, 0x1400ffffu, 0x14001400u, 0x1400ffffu, 0x7fff9400u, 0x00000000u, 0x80000000u, 0x00000001u, 0x00010000u, 0x0000ffffu, 0x00008000u, 0x14007fffu, 0x7fff7fffu, 0x00008000u, 0x00000000u, 0x14008000u, 0x7fff8000u, 0x00007fffu, 0x14000000u, 0xc0007fffu, 0xc0007fffu, 0x7fff1400u, 0xc000ffffu, 0x14007fffu, 0x14001400u, 0x94007fffu, 0x00007fffu, 0x7fff7fffu, 0x0000ffffu, 0x00008000u, 0x00000000u, 0x80008000u, 0x14008000u, 0x00001400u, 0x00000000u, 0x00000000u, 0x00000000u, 0x14000000u, 0x00000000u}, + {0x7e00d5feu, 0x7ffe8000u, 0x7e00d5feu, 0x81fe7e00u, 0x8000d600u, 0xd6008000u, 0x7fff7c55u, 0x7fff7dffu, 0x7fff7c55u, 0x7c557c55u, 0x7fff8055u, 0x7fff7fffu, 0x7fff8055u, 0x7e5581ffu, 0x0000c000u, 0x00000000u, 0x000103f0u, 0x03f00001u, 0xffff03f0u, 0xffff0000u, 0xffff7e00u, 0x7e007e00u, 0x8000fe55u, 0x80008000u, 0xfffffe55u, 0xfe55ffffu, 0x80007e00u, 0x7e008000u, 0xfffffe55u, 0xfffffe55u, 0xfe55ffffu, 0xffff7e55u, 0xfffffe55u, 0xffffffffu, 0x7ffffe55u, 0xfffffe55u, 0xfe55fe55u, 0xffff7e55u, 0x8000fe55u, 0x80008000u, 0x00007e55u, 0x8000fe55u, 0x8000ffffu, 0x8000fe55u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u}, + {0x80007656u, 0x4254c3feu, 0x80007656u, 0x4254c3feu, 0x00017801u, 0x78010001u, 0xfffe7802u, 0xfffebc00u, 0xfffe7802u, 0x78027802u, 0x00000000u, 0x00000000u, 0x00000000u, 0xbc0243feu, 0x80007802u, 0x8000fffeu, 0x00001e00u, 0x1e000000u, 0x00001e00u, 0x00000000u, 0x7fff3c01u, 0x3c013c01u, 0x7fff3c01u, 0x7fff7fffu, 0x7fff3c01u, 0x7fff7fffu, 0x7fff3c01u, 0x3c013c01u, 0x7ffffe55u, 0x7ffffe55u, 0x7fff7fffu, 0x7ffffe55u, 0x7fff4001u, 0x7fff7fffu, 0xffff0000u, 0x7fff3c02u, 0x3c023c02u, 0x7fff3c02u, 0x7fff3c01u, 0x7fff7fffu, 0x7fffbc01u, 0x7fff3c01u, 0x3c013c01u, 0xffff3c01u, 0x00003c00u, 0x3c003c00u, 0x00003c00u, 0x00003c00u}, + {0x7bff6000u, 0x60007bffu, 0x7bff6000u, 0x60007bffu, 0x00000000u, 0x00000000u, 0xedcc6dccu, 0xedcc6dccu, 0xedcc6dccu, 0x6dcc6dccu, 0xedcc6dccu, 0xedccedccu, 0xedcc6dccu, 0xedcc6dccu, 0x00000000u, 0x00000000u, 0x00000008u, 0x00080000u, 0x0000fff8u, 0x0000fff8u, 0x0000edccu, 0xedccedccu, 0xedcc8000u, 0xedccedccu, 0xedccedccu, 0xedccedccu, 0x00008000u, 0x80000000u, 0x7bff6000u, 0x7bff6000u, 0x7bff6000u, 0x7bff6000u, 0xedccedccu, 0xedccedccu, 0x6dccedccu, 0x80000000u, 0x00000000u, 0x80000000u, 0xedccedccu, 0xedccedccu, 0xedcc8000u, 0x00008000u, 0x00008000u, 0x6dcc0000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x3c000000u}, + {0x00001dccu, 0xedccd000u, 0x00001dccu, 0xbdcc0000u, 0x00003000u, 0x30000000u, 0x8000ce34u, 0x80009634u, 0x8000ce34u, 0xce34ce34u, 0x8800a9ccu, 0x88008800u, 0x8800a9ccu, 0xc00071ccu, 0xfc001234u, 0x1234fc00u, 0xfc001234u, 0x1234fc00u, 0xfc001234u, 0xfc001234u, 0xfc001234u, 0x12341234u, 0x8400bc00u, 0x84008400u, 0xfc00bc00u, 0xfc008400u, 0x84001234u, 0x1234bc00u, 0x7c00edccu, 0x7c00edccu, 0x92347c00u, 0xfc00edccu, 0xfc00bbfeu, 0xfc0011b4u, 0xfc00bc01u, 0x7c009234u, 0x92349234u, 0x7c009234u, 0xfc00bc00u, 0xfc00fc00u, 0x84001234u, 0x84001234u, 0xbc001234u, 0x04009234u, 0x00000000u, 0x00000000u, 0x00001234u, 0x04000000u}, + {0xc99f5e41u, 0x4ceb74ddu, 0xc99f5e41u, 0xcd1df4abu, 0xcb9f0801u, 0x0801cb9fu, 0xb278f7feu, 0xb278f922u, 0xb278f7feu, 0xf7fef7feu, 0x47ce0000u, 0x47ce47ceu, 0x47ce0000u, 0x46aa0124u, 0xaaa88000u, 0xdff88000u, 0x06aa0000u, 0x000006aau, 0x06aa0000u, 0x06aa0f7fu, 0x7d237bffu, 0x7bff7bffu, 0x35557bffu, 0x35553555u, 0x7d237bffu, 0x7bff7d23u, 0x35557bffu, 0x7bff3555u, 0x7f237c00u, 0x7f237c00u, 0xfe007f23u, 0x7f23fc00u, 0x7f237c00u, 0x7f237f23u, 0xff230000u, 0x7f237c00u, 0x7c007c00u, 0x7f237c00u, 0x7f237bffu, 0x7f237f23u, 0x7f23fbffu, 0x7f237bffu, 0x7bff7f23u, 0xff237bffu, 0x00003c00u, 0x35550000u, 0x00003c00u, 0x00003c00u}, + {0x00008000u, 0x80000000u, 0x00008000u, 0x80000000u, 0x80008000u, 0x80008000u, 0xb555fc01u, 0xb555b555u, 0xb555fc01u, 0xfc01fc01u, 0xb555fc01u, 0xb555b555u, 0xb555fc01u, 0xfc01b555u, 0x00000000u, 0x00000000u, 0x04004000u, 0x40000400u, 0xfc00c000u, 0xfc00fc00u, 0x35557c01u, 0x7c017c01u, 0x80008000u, 0x80008000u, 0x80008000u, 0x80008000u, 0x35557c01u, 0x35557c01u, 0x80007e01u, 0x80007e01u, 0x7e010000u, 0x0000fe01u, 0x35557e01u, 0x35553555u, 0xb5557e01u, 0x80007e01u, 0x7e017e01u, 0x8000fe01u, 0x80007e01u, 0x80008000u, 0x0000fe01u, 0x35557e01u, 0x7e013555u, 0x80007e01u, 0x00000000u, 0x00000000u, 0x35550000u, 0x00000000u}, + {0x03ff0000u, 0xa2480000u, 0x03ff0000u, 0x0000a248u, 0x03ff0000u, 0x000003ffu, 0xfc00c248u, 0xfc007fffu, 0xfc00c248u, 0xc248c248u, 0x03fec248u, 0x03fe03feu, 0x03fec248u, 0x46477fffu, 0x80000000u, 0x00000100u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x7fff0000u, 0x00000000u, 0x7c01c248u, 0x7c017c01u, 0x7fffc248u, 0xc2487fffu, 0x7c010000u, 0x00007c01u, 0x7fff0000u, 0x7fff0000u, 0x00007fffu, 0x7fff0000u, 0x7fffc248u, 0x7fff7fffu, 0xffffc248u, 0x7fff8000u, 0x80008000u, 0x7fff8000u, 0x7e01c248u, 0x7e017e01u, 0xfe010000u, 0x7e010000u, 0x7e010000u, 0x7e018000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u}, + {0xc2337fffu, 0x87fd7fffu, 0xc2337fffu, 0xffff07fdu, 0x42348000u, 0x80004234u, 0x4e35bffeu, 0x4e35d234u, 0x4e35bffeu, 0xbffebffeu, 0xd6333ffeu, 0xd633d633u, 0xd6333ffeu, 0xc3fd5234u, 0xc0100000u, 0x00004000u, 0x03c00003u, 0x000303c0u, 0x03c0ffffu, 0x03c0fc00u, 0x3c01fffeu, 0xfffefffeu, 0x1234c000u, 0x12341234u, 0x3c01fffeu, 0xfffec000u, 0x1234c000u, 0x12343c01u, 0x7ffffffeu, 0x7ffffffeu, 0xfffeffffu, 0x7fff7ffeu, 0x3c02fffeu, 0x3c02bfffu, 0x3c00fffeu, 0x1236fffeu, 0xfffefffeu, 0x12367ffeu, 0x1234c000u, 0x12341234u, 0xbc01c000u, 0x3c01c000u, 0x3c011234u, 0x3c014000u, 0x12340000u, 0x3c000000u, 0x3c000000u, 0x3c003c00u}, + {0x9c00a56eu, 0xcffef560u, 0x9c00a56eu, 0x795e4c00u, 0x2000a570u, 0xa5702000u, 0xc6480000u, 0xc6483014u, 0xc6480000u, 0x00000000u, 0xbe482468u, 0xbe48be48u, 0xbe482468u, 0x8e34547cu, 0x0000dcc0u, 0xcc004000u, 0x00840edcu, 0x0edc0084u, 0xff84fedcu, 0xff84ffedu, 0x42481234u, 0x12341234u, 0x8400edccu, 0x84008400u, 0x8400edccu, 0x8400edccu, 0x42481234u, 0x42481234u, 0x7c00fffeu, 0x7c00fffeu, 0x7c00fffeu, 0x7c00fffeu, 0x4248edccu, 0x4248edcbu, 0xc2486dccu, 0x8a48c47fu, 0xc47fc47fu, 0x8a48c47fu, 0x8400edccu, 0x84008400u, 0x0400edccu, 0x42481234u, 0x12344248u, 0x84006dccu, 0x00000000u, 0x00000000u, 0x3c001234u, 0x00003c00u}, + {0x04023a00u, 0x3a000402u, 0x04023a00u, 0x38020600u, 0x00020200u, 0x02000002u, 0xfffd7dffu, 0xfffd7ffdu, 0xfffd7dffu, 0x7dff7dffu, 0xffff7e01u, 0xffffffffu, 0xffff7e01u, 0xfe017fffu, 0xc0007fffu, 0xc000ffffu, 0x00037fffu, 0x7fff0003u, 0xffff7fffu, 0xffff0001u, 0xffff7fffu, 0x7fff7fffu, 0xfffefe00u, 0xfffefffeu, 0xfffffe00u, 0xfffffffeu, 0xfffe7fffu, 0x7ffffe00u, 0xfffefe00u, 0xfffefe00u, 0xfe00fffeu, 0xfffe7e00u, 0xfffefe00u, 0xfffefffeu, 0x7ffefe00u, 0xfffefe00u, 0xfe00fe00u, 0xfffe7e00u, 0xfffefe00u, 0xfffefffeu, 0xfffe7e00u, 0xfffefe00u, 0xfe00fffeu, 0x7ffefe00u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u}, + {0x96003bb8u, 0xee00ebccu, 0x96003bb8u, 0x6bcc6e00u, 0x9800bdb8u, 0xbdb89800u, 0x9034c247u, 0x90349233u, 0x9034c247u, 0xc247c247u, 0x9434c249u, 0x94349434u, 0x9434c249u, 0xc4489235u, 0xe000ff00u, 0xfff00000u, 0x07e0007fu, 0x007f07e0u, 0x07e0007fu, 0x07e007ffu, 0x7e007fffu, 0x7fff7fffu, 0x12344248u, 0x12341234u, 0x7e007fffu, 0x7e007fffu, 0x12344248u, 0x12344248u, 0x7e007fffu, 0x7e007fffu, 0x7fff7e00u, 0xfe007fffu, 0x7e007fffu, 0x7e007fffu, 0x7e00ffffu, 0x7e007fffu, 0x7fff7fffu, 0x7e00ffffu, 0x12344248u, 0x12341234u, 0x1234c248u, 0x12344248u, 0x42481234u, 0x92344248u, 0x12343c00u, 0x3c001234u, 0x12343c00u, 0x00003c00u}, + {0x80000800u, 0x08006340u, 0x80000800u, 0xeb408000u, 0x00000000u, 0x00000000u, 0x35555240u, 0x35558b95u, 0x35555240u, 0x52405240u, 0x3555a5c0u, 0x35553555u, 0x3555a5c0u, 0xfc00df15u, 0x00005640u, 0xc8000000u, 0x00005640u, 0x56400000u, 0x00005640u, 0x000002b2u, 0x35555640u, 0x56405640u, 0x0000fc00u, 0x00000000u, 0x3555fc00u, 0xfc005640u, 0x00005640u, 0x35550000u, 0x0000fc00u, 0x0000fc00u, 0xfc000800u, 0x80007c00u, 0x3555fc00u, 0x35555645u, 0xb555fc00u, 0x0000fc00u, 0xfc00fc00u, 0x0000fc00u, 0x0000fc00u, 0x00000000u, 0x80005640u, 0x35555640u, 0x00005640u, 0x0000d640u, 0x00000000u, 0x00003555u, 0x35553c00u, 0x00000000u}, + {0xdd70b2c8u, 0x611b56e4u, 0xdd70b2c8u, 0x9ee3191cu, 0xa57032c9u, 0x32c9a570u, 0x0000fa46u, 0x00006aefu, 0x0000fa46u, 0xfa46fa46u, 0xdb980000u, 0xdb98db98u, 0xdb980000u, 0x6aef70a9u, 0x4000e918u, 0x300091a0u, 0x00010fa4u, 0x0fa40001u, 0x00010fa4u, 0x00010007u, 0x12347d23u, 0x7d237d23u, 0xedcc7d23u, 0xedccedccu, 0xedcc7d23u, 0x7d23edccu, 0x12347d23u, 0x7d231234u, 0xc3fd7f23u, 0xc3fd7f23u, 0x7f237fffu, 0x44ffff23u, 0xedcc7f23u, 0xedcc7f23u, 0x6dcc7f23u, 0xc47f7f23u, 0x7f237f23u, 0xc47fff23u, 0xedcc7f23u, 0xedccedccu, 0xedccff23u, 0x12347f23u, 0x7f237f23u, 0x6dcc7f23u, 0x00000000u, 0x00000000u, 0x12340000u, 0x3c000000u}, + {0xc000d640u, 0xd6413c00u, 0xc000d640u, 0x5640bc01u, 0x84008000u, 0x80008400u, 0xfbff7fffu, 0xfbfffc00u, 0xfbff7fffu, 0x7fff7fffu, 0xfc017fffu, 0xfc01fc01u, 0xfc017fffu, 0x8000fc00u, 0x7fff0000u, 0x80008000u, 0x7fff0001u, 0x00017fffu, 0x7fffffffu, 0x7fff8000u, 0x7fffffffu, 0xffffffffu, 0x7c008000u, 0x7c007c00u, 0x7fffffffu, 0xffff8000u, 0x7c008000u, 0x7c007fffu, 0x7fffffffu, 0x7fffffffu, 0xffff7fffu, 0xffff7fffu, 0x7fffffffu, 0x7fff7c00u, 0x7fffffffu, 0x7fffffffu, 0xffffffffu, 0x7fff7fffu, 0x7c008000u, 0x7c007c00u, 0x7c008000u, 0x7c008000u, 0xffff7c00u, 0xfc000000u, 0x3c000000u, 0x00000000u, 0x3c000000u, 0x00000000u}, + {0xfffecd56u, 0x38002153u, 0xfffecd56u, 0x5955fffeu, 0x00009556u, 0x95560000u, 0x3c013553u, 0x3c017156u, 0x3c013553u, 0x35533553u, 0x3c01caa9u, 0x3c013c01u, 0x3c01caa9u, 0xfffe06acu, 0x00004000u, 0x6aaa0000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00001aaau, 0x3c013555u, 0x35553555u, 0x0000fffeu, 0x00000000u, 0x3c01fffeu, 0xfffe3c01u, 0x00003555u, 0x35550000u, 0xfffefffeu, 0xfffefffeu, 0xfffe3800u, 0xfffe7ffeu, 0x3c01fffeu, 0x3c013d56u, 0xbc01fffeu, 0x0000fffeu, 0xfffefffeu, 0x00007ffeu, 0x00003555u, 0x00000000u, 0x80003555u, 0x3c013555u, 0x00003c01u, 0x0000b555u, 0x00003555u, 0x00003555u, 0x3c003555u, 0x00000000u}, + {0x31b86c40u, 0xd5b84840u, 0x31b86c40u, 0x6c40b1b8u, 0x1db83440u, 0x34401db8u, 0x3e478490u, 0x3e470490u, 0x3e478490u, 0x84908490u, 0x46490000u, 0x46494649u, 0x46490000u, 0xc6498000u, 0xff004800u, 0x4800ff00u, 0x007b0042u, 0x0042007bu, 0x007b0042u, 0x007b0042u, 0x7bff4248u, 0x42484248u, 0xc2484248u, 0xc248c248u, 0xc2484248u, 0x7bffc248u, 0x7bff4248u, 0x42484248u, 0xfc00492fu, 0xfc00492fu, 0x48effc00u, 0x7c00c8afu, 0x7bff4648u, 0x7bff0000u, 0x7bff0000u, 0xfc0048efu, 0x48ef48efu, 0xfc0048efu, 0xc2484248u, 0xc248c248u, 0xfbffc248u, 0x7bff4248u, 0x7bff4248u, 0x7bff4248u, 0x00003c00u, 0x3c000000u, 0x3c003c00u, 0x3c003c00u}, + {0x9a908000u, 0xdcbc4000u, 0x9a908000u, 0x80009cbcu, 0x5a900000u, 0x00005a90u, 0xdb98fe55u, 0xdb98edccu, 0xdb98fe55u, 0xfe55fe55u, 0x0000fe55u, 0x00000000u, 0x0000fe55u, 0x1089edccu, 0xc0000000u, 0x0000b980u, 0x000e0000u, 0x0000000eu, 0xfffe0000u, 0xfffe0000u, 0xedcc0000u, 0x00000000u, 0xedccfe55u, 0xedccedccu, 0xedccfe55u, 0xfe55edccu, 0xedcc0000u, 0x0000edccu, 0x7c00fe55u, 0x7c00fe55u, 0xfe557c00u, 0xfc007e55u, 0xf1ccfe55u, 0xf1ccedccu, 0x0000fe55u, 0x7c00fe55u, 0xfe55fe55u, 0x7c007e55u, 0xedcc0000u, 0xedccedccu, 0xedcc0000u, 0xedcc0000u, 0xedcc0000u, 0x6dcc8000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x3c000000u}, + {0x12344400u, 0x14000a34u, 0x12344400u, 0x0c001234u, 0x00003000u, 0x30000000u, 0xfffef1ccu, 0xfffe03feu, 0xfffef1ccu, 0xf1ccf1ccu, 0xfffee9ccu, 0xfffefffeu, 0xfffee9ccu, 0xedccfbfeu, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000400u, 0x04000400u, 0xfffeedccu, 0xfffefffeu, 0xfffeedccu, 0xedccfffeu, 0x00000400u, 0x04000000u, 0xfffeb5c8u, 0xfffeb5c8u, 0xb5c9fffeu, 0xfffe35d0u, 0xfffeedccu, 0xfffefffeu, 0x7ffeedccu, 0xfffeb5ccu, 0xb5ccb5ccu, 0xfffeb5ccu, 0x0000edccu, 0x00000000u, 0x80000400u, 0x00000400u, 0x00000400u, 0x00008400u, 0x00000000u, 0x00000400u, 0x00000400u, 0x00000000u}, + {0x7fff9c00u, 0xbc00c69eu, 0x7fff9c00u, 0x029f7fffu, 0x0000e000u, 0xe0000000u, 0x7d235e55u, 0x7d237b78u, 0x7d235e55u, 0x5e555e55u, 0x7d2361abu, 0x7d237d23u, 0x7d2361abu, 0x60007eceu, 0x0000fe55u, 0xf2a80000u, 0x0000fe55u, 0xfe550000u, 0x0000fe55u, 0x0000ffcau, 0x7d236000u, 0x60006000u, 0x0000fe55u, 0x00000000u, 0x7d23fe55u, 0x6000fe55u, 0x00006000u, 0x7d230000u, 0x7f23fe55u, 0x7f23fe55u, 0xfe557f23u, 0x7f23fe55u, 0x7f23fe55u, 0x7f237f23u, 0xff237e55u, 0x7f23fe55u, 0xfe55fe55u, 0x7f237e55u, 0x7f236000u, 0x7f237f23u, 0x7f23e000u, 0x7f236000u, 0x60007f23u, 0xff236000u, 0x00003c00u, 0x00000000u, 0x00003c00u, 0x00003c00u}, + {0x3c008400u, 0x84007c00u, 0x3c008400u, 0x84007c00u, 0xc0000000u, 0x0000c000u, 0x3c018000u, 0x3c017c01u, 0x3c018000u, 0x80008000u, 0xbc018000u, 0xbc01bc01u, 0xbc018000u, 0xc0007c01u, 0x80000000u, 0x0000c000u, 0x60000000u, 0x00006000u, 0xe0000000u, 0xe0000000u, 0x7c010000u, 0x00000000u, 0xc0008000u, 0xc000c000u, 0xc0008000u, 0xc0007c01u, 0x7c010000u, 0x00008000u, 0x7e018400u, 0x7e018400u, 0x7c007e01u, 0x7e018400u, 0x7e010000u, 0x7e017e01u, 0xfe018000u, 0x7e018000u, 0x80008000u, 0x7e018000u, 0x7e018000u, 0x7e017e01u, 0x7e010000u, 0x7e010000u, 0x80007e01u, 0xfe018000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u}, + {0x40006fe8u, 0xaa007de8u, 0x40006fe8u, 0xe7e84000u, 0x2c00f1e8u, 0xf1e82c00u, 0xb155c09du, 0xb155f79du, 0xb155c09du, 0xc09dc09du, 0xb9553c0du, 0xb955b955u, 0xb9553c0du, 0x8255730du, 0x80004900u, 0x49008000u, 0x03e00612u, 0x061203e0u, 0x03e0fe12u, 0x03e0fe12u, 0x7c00fe55u, 0xfe55fe55u, 0x3555c248u, 0x35553555u, 0x7c00fe55u, 0xfe55c248u, 0x3555c248u, 0x35557c00u, 0x7c00fe55u, 0x7c00fe55u, 0xfe557e00u, 0xfc007e55u, 0x7c00fe55u, 0x7c00c19du, 0x7c00fe55u, 0x7c00fe55u, 0xfe55fe55u, 0x7c007e55u, 0x3555c248u, 0x35553555u, 0xfc00c248u, 0x7c00c248u, 0x7c003555u, 0x7c004248u, 0x35550000u, 0x3c000000u, 0x3c000000u, 0x3c003c00u}, + {0x755fffffu, 0x5e480e33u, 0x755fffffu, 0x4e341e47u, 0x7960c3ffu, 0xc3ff7960u, 0x30143c00u, 0x3014edcbu, 0x30143c00u, 0x3c003c00u, 0xab843c02u, 0xab84ab84u, 0xab843c02u, 0xf9b9edcdu, 0x8000fffeu, 0xf0008490u, 0x00047fffu, 0x7fff0004u, 0x0004ffffu, 0x0004ffffu, 0x42483c01u, 0x3c013c01u, 0xedccffffu, 0xedccedccu, 0xedccffffu, 0x4248ffffu, 0x42483c01u, 0xedcc3c01u, 0xfc00ffffu, 0xfc00ffffu, 0xfffff48du, 0xf9b8ffffu, 0xedcbffffu, 0xedcbffffu, 0x6dcd7fffu, 0xf48dffffu, 0xffffffffu, 0xf48d7fffu, 0xedcc3c01u, 0xedccedccu, 0xedccbc01u, 0x42483c01u, 0x4248edccu, 0x6dcc3c01u, 0x00003c00u, 0x3c000000u, 0x3c003c00u, 0x3c003c00u}, + {0x8132edccu, 0x70a9fe55u, 0x8132edccu, 0xedcc8132u, 0x82dd0000u, 0x000082ddu, 0x7d227d23u, 0x7d227d23u, 0x7d227d23u, 0x7d237d23u, 0x7d247d23u, 0x7d247d24u, 0x7d247d23u, 0x7d247d23u, 0xfff80000u, 0x0000fff8u, 0x1fff0000u, 0x00001fffu, 0xffff0000u, 0xffff0000u, 0x7d237d23u, 0x7d237d23u, 0xffff0000u, 0xffffffffu, 0xffff7d23u, 0xffff7d23u, 0x7d230000u, 0x00007d23u, 0x7f237f23u, 0x7f237f23u, 0x7f237f23u, 0x7f23ff23u, 0x7f237f23u, 0x7f237f23u, 0xff237f23u, 0x7f237f23u, 0x7f237f23u, 0x7f23ff23u, 0x7f237f23u, 0x7f237f23u, 0x7f23ff23u, 0x7f237f23u, 0x7f237f23u, 0xff237f23u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u}, + {0x00008000u, 0x80002000u, 0x00008000u, 0xa0000000u, 0x00000000u, 0x00000000u, 0xc2480400u, 0xc248c648u, 0xc2480400u, 0x04000400u, 0xc248fc00u, 0xc248c248u, 0xc248fc00u, 0x0000be48u, 0x00000400u, 0x00000000u, 0x00000400u, 0x04000000u, 0x00000400u, 0x00000004u, 0x00000400u, 0x04000400u, 0xc2480000u, 0xc248c248u, 0xc2480400u, 0x0000c248u, 0x00000000u, 0x04000000u, 0x00000000u, 0x00000000u, 0x00008000u, 0x00008000u, 0xc2480400u, 0xc248c248u, 0x42488400u, 0x80000000u, 0x00000000u, 0x80000000u, 0xc2480000u, 0xc248c248u, 0xc2488000u, 0x00000400u, 0x00000400u, 0x42480000u, 0x00000000u, 0x00000000u, 0x00000400u, 0x3c000000u}, + {0xcaab5400u, 0x9634ac00u, 0xcaab5400u, 0x30001234u, 0xcaabd000u, 0xd000caabu, 0x3554e9ccu, 0x35543155u, 0x3554e9ccu, 0xe9cce9ccu, 0x3556f1ccu, 0x35563556u, 0x3556f1ccu, 0xedcd3955u, 0xffe00000u, 0x8000f000u, 0x07ff000fu, 0x000f07ffu, 0xffffffffu, 0xffffffe0u, 0x3555fc00u, 0xfc00fc00u, 0xffffedccu, 0xffffffffu, 0xfffffc00u, 0xfffffc00u, 0x3555edccu, 0x3555edccu, 0xffff7c00u, 0xffff7c00u, 0x7c00ffffu, 0x7ffffc00u, 0xfffffc00u, 0xfffffc00u, 0xffff7c00u, 0xffff7c00u, 0x7c007c00u, 0xffff7c00u, 0x3555fc00u, 0x35553555u, 0x3555fc00u, 0x3555edccu, 0xedcc3555u, 0xb5557c00u, 0x35550000u, 0x00000000u, 0x35550000u, 0x00003c00u}, + {0x8400c000u, 0xa11c8400u, 0x8400c000u, 0xc000651cu, 0x00000000u, 0x00000000u, 0x7d231234u, 0x7d230000u, 0x7d231234u, 0x12341234u, 0x82dd1234u, 0x82dd82ddu, 0x82dd1234u, 0x95110000u, 0x7d230000u, 0x0000d230u, 0x7d230000u, 0x00007d23u, 0x7d230000u, 0x7d230000u, 0x7d231234u, 0x12341234u, 0x00000000u, 0x00000000u, 0x7d231234u, 0x7d230000u, 0x00000000u, 0x00001234u, 0x7f23c000u, 0x7f23c000u, 0x84007f23u, 0xff23c000u, 0x7f231234u, 0x7f230000u, 0x7f231234u, 0x7f230000u, 0x00000000u, 0x7f230000u, 0x7f230000u, 0x7f237f23u, 0xff239234u, 0x7f231234u, 0x7f230000u, 0x7f231234u, 0x00000000u, 0x00000000u, 0x00001234u, 0x00001234u}, + {0x7d557e55u, 0x7e557d55u, 0x7d557e55u, 0x7e557d55u, 0x00000000u, 0x00000000u, 0xfc007c00u, 0xfc000000u, 0xfc007c00u, 0x7c007c00u, 0x04007c00u, 0x04000400u, 0x04007c00u, 0x80000000u, 0xfc000000u, 0x0000fc00u, 0xfc000000u, 0x0000fc00u, 0xfc000000u, 0xfc000000u, 0x00007c00u, 0x7c007c00u, 0xfc000000u, 0xfc00fc00u, 0xfc007c00u, 0xfc000000u, 0x00000000u, 0x00007c00u, 0xfe00fe00u, 0xfe00fe00u, 0xfe00fe00u, 0xfe00fe00u, 0xfc007c00u, 0xfc000000u, 0xfc007c00u, 0xfe00fe00u, 0xfe00fe00u, 0xfe00fe00u, 0xfc000000u, 0xfc00fc00u, 0x0000fc00u, 0x00007c00u, 0x7c000000u, 0x80007c00u, 0x00000000u, 0x00000000u, 0x00003c00u, 0x00003c00u}, + {0xfe557d55u, 0x7d55fe55u, 0xfe557d55u, 0x7d55fe55u, 0x00000000u, 0x00000000u, 0x7c007c00u, 0x7c00f800u, 0x7c007c00u, 0x7c007c00u, 0x7c008400u, 0x7c007c00u, 0x7c008400u, 0x00000000u, 0x00007c00u, 0x7c000000u, 0x00007c00u, 0x7c000000u, 0x00007c00u, 0x00007c00u, 0x7c007c00u, 0x7c007c00u, 0x00000000u, 0x00000000u, 0x7c007c00u, 0x00007c00u, 0x00000000u, 0x7c000000u, 0xfe00fe00u, 0xfe00fe00u, 0xfe00fe00u, 0xfe00fe00u, 0x7c007c00u, 0x7c007c00u, 0xfc00fc00u, 0xfe00fe00u, 0xfe00fe00u, 0xfe00fe00u, 0x00000000u, 0x00000000u, 0x80008000u, 0x7c007c00u, 0x00007c00u, 0x00000000u, 0x00000000u, 0x00003c00u, 0x3c003c00u, 0x00000000u}, }; -constexpr std::uint32_t ExpectedIeeeOff[32][44] = { - {0x3c003c00u, 0x3c003c00u, 0x3c003c00u, 0x3c003c00u, 0x00000000u, 0x00000000u, 0x7c007c00u, 0x7c00fc00u, 0x7c007c00u, 0x7c007c00u, 0x7c007c00u, 0x7c007c00u, 0x7c007c00u, 0xfc00fc00u, 0x80000000u, 0x00008000u, 0x80000000u, 0x00008000u, 0x80000000u, 0x80000000u, 0xfc007c00u, 0x7c007c00u, 0x80000000u, 0x80008000u, 0xfc007c00u, 0x8000fc00u, 0x80000000u, 0x00007c00u, 0xfe00fe00u, 0xfe00fe00u, 0xfe00fe00u, 0xfe00fe00u, 0xfc007c00u, 0xfc00fc00u, 0x7c007c00u, 0xfe00fe00u, 0xfe00fe00u, 0xfe00fe00u, 0xfc000000u, 0xfc00fc00u, 0xfc00fc00u, 0x80007c00u, 0x7c000000u, 0x7c007c00u}, - {0xbc007c00u, 0x7c00bc00u, 0xbc007c00u, 0x7c00bc00u, 0x00000000u, 0x00000000u, 0x78007800u, 0x7800f800u, 0x78007800u, 0x78007800u, 0x80008000u, 0x80008000u, 0x80008000u, 0x00000000u, 0x7c00fc00u, 0xfc007c00u, 0x7c00fc00u, 0xfc007c00u, 0x7c00fc00u, 0x7c00fc00u, 0x7c007c00u, 0x7c007c00u, 0xfc00fc00u, 0xfc00fc00u, 0xfc00fc00u, 0x7c00fc00u, 0x7c007c00u, 0xfc007c00u, 0xfc00fe00u, 0xfc00fe00u, 0xfc00fe00u, 0x7c007c00u, 0xfe00fe00u, 0xfe00fc00u, 0x7c007c00u, 0xfc00fc00u, 0xfc00fc00u, 0xfc00fc00u, 0xfc00fc00u, 0xfc00fc00u, 0xfc00fc00u, 0x7c007c00u, 0x7c00fc00u, 0x7c007c00u}, - {0x7bffc000u, 0xc000fbffu, 0x7bffc000u, 0x7fff3c00u, 0x43ff0400u, 0x040043ffu, 0xbc00fbffu, 0xbc003c00u, 0xbc00fbffu, 0xfbfffbffu, 0xbc02fc01u, 0xbc02bc02u, 0xbc02fc01u, 0x7c013c02u, 0xfffeffffu, 0xfffe7fffu, 0x3fffffffu, 0xffff3fffu, 0x3fffffffu, 0x3fffffffu, 0x7fffffffu, 0xffffffffu, 0x3c01fc00u, 0x3c013c01u, 0x7fffffffu, 0xfc00ffffu, 0x3c01fc00u, 0x3c017fffu, 0x7fffffffu, 0x7fffffffu, 0xffff7fffu, 0xffffffffu, 0x7fffffffu, 0x7fffffffu, 0x7fff7fffu, 0x7fffffffu, 0xffffffffu, 0x7fff7fffu, 0x3c01fc00u, 0x3c013c01u, 0x3c017c00u, 0x3c01fc00u, 0xfc003c01u, 0xbc01fc00u}, - {0xc0009400u, 0x1400c000u, 0xc0009400u, 0x1400c000u, 0x00008000u, 0x80000000u, 0x1400ffffu, 0x14009400u, 0x1400ffffu, 0xffffffffu, 0x1400ffffu, 0x14001400u, 0x1400ffffu, 0x7fff9400u, 0x00000000u, 0x80000000u, 0x00000001u, 0x00010000u, 0x0000ffffu, 0x00008000u, 0x14007fffu, 0x7fff7fffu, 0x00008000u, 0x00000000u, 0x14008000u, 0x7fff8000u, 0x00007fffu, 0x14000000u, 0xc0007fffu, 0xc0007fffu, 0x7fff1400u, 0xc000ffffu, 0x14007fffu, 0x14001400u, 0x94007fffu, 0x00007fffu, 0x7fff7fffu, 0x0000ffffu, 0x00008000u, 0x00000000u, 0x80008000u, 0x14008000u, 0x00001400u, 0x00000000u}, - {0x7e00d5feu, 0x7ffe8000u, 0x7e00d5feu, 0x81fe7e00u, 0x8000d600u, 0xd6008000u, 0x7fff7c55u, 0x7fff7dffu, 0x7fff7c55u, 0x7c557c55u, 0x7fff8055u, 0x7fff7fffu, 0x7fff8055u, 0x7e5581ffu, 0x0000c000u, 0x00000000u, 0x000103f0u, 0x03f00001u, 0xffff03f0u, 0xffff0000u, 0xffff7e00u, 0x7e007e00u, 0x8000fe55u, 0x80008000u, 0xfffffe55u, 0xfe55ffffu, 0x80007e00u, 0x7e008000u, 0xfffffe55u, 0xfffffe55u, 0xfe55ffffu, 0xffff7e55u, 0xfffffe55u, 0xffffffffu, 0x7ffffe55u, 0xfffffe55u, 0xfe55fe55u, 0xffff7e55u, 0x8000fe55u, 0x80008000u, 0x00007e55u, 0x8000fe55u, 0x8000ffffu, 0x8000fe55u}, - {0x80007656u, 0x4254c3feu, 0x80007656u, 0x4254c3feu, 0x00017801u, 0x78010001u, 0xfffe7802u, 0xfffebc00u, 0xfffe7802u, 0x78027802u, 0x00000000u, 0x00000000u, 0x00000000u, 0xbc0243feu, 0x80007802u, 0x8000fffeu, 0x00001e00u, 0x1e000000u, 0x00001e00u, 0x00000000u, 0x7fff3c01u, 0x3c013c01u, 0x7fff3c01u, 0x7fff7fffu, 0x7fff3c01u, 0x7fff7fffu, 0x7fff3c01u, 0x3c013c01u, 0x7ffffe55u, 0x7ffffe55u, 0x7fff7fffu, 0x7ffffe55u, 0x7fff4001u, 0x7fff7fffu, 0xffff0000u, 0x7fff3c02u, 0x3c023c02u, 0x7fff3c02u, 0x7fff3c01u, 0x7fff7fffu, 0x7fffbc01u, 0x7fff3c01u, 0x3c013c01u, 0xffff3c01u}, - {0x7bff6000u, 0x60007bffu, 0x7bff6000u, 0x60007bffu, 0x00000000u, 0x00000000u, 0xedcc6dccu, 0xedcc6dccu, 0xedcc6dccu, 0x6dcc6dccu, 0xedcc6dccu, 0xedccedccu, 0xedcc6dccu, 0xedcc6dccu, 0x00000000u, 0x00000000u, 0x00000008u, 0x00080000u, 0x0000fff8u, 0x0000fff8u, 0x0000edccu, 0xedccedccu, 0xedcc8000u, 0xedccedccu, 0xedccedccu, 0xedccedccu, 0x00008000u, 0x80000000u, 0x7bff6000u, 0x7bff6000u, 0x7bff6000u, 0x7bff6000u, 0xedccedccu, 0xedccedccu, 0x6dccedccu, 0x80000000u, 0x00000000u, 0x80000000u, 0xedccedccu, 0xedccedccu, 0xedcc8000u, 0x00008000u, 0x00008000u, 0x6dcc0000u}, - {0x00001dccu, 0xedccd000u, 0x00001dccu, 0xbdcc0000u, 0x00003000u, 0x30000000u, 0x8000ce34u, 0x80009634u, 0x8000ce34u, 0xce34ce34u, 0x8800a9ccu, 0x88008800u, 0x8800a9ccu, 0xc00071ccu, 0xfc001234u, 0x1234fc00u, 0xfc001234u, 0x1234fc00u, 0xfc001234u, 0xfc001234u, 0xfc001234u, 0x12341234u, 0x8400bc00u, 0x84008400u, 0xfc00bc00u, 0xfc008400u, 0x84001234u, 0x1234bc00u, 0x7c00edccu, 0x7c00edccu, 0x92347c00u, 0xfc00edccu, 0xfc00bbfeu, 0xfc0011b4u, 0xfc00bc01u, 0x7c009234u, 0x92349234u, 0x7c009234u, 0xfc00bc00u, 0xfc00fc00u, 0x84001234u, 0x84001234u, 0xbc001234u, 0x04009234u}, - {0xc99f5e41u, 0x4ceb74ddu, 0xc99f5e41u, 0xcd1df4abu, 0xcb9f0801u, 0x0801cb9fu, 0xb278f7feu, 0xb278f922u, 0xb278f7feu, 0xf7fef7feu, 0x47ce0000u, 0x47ce47ceu, 0x47ce0000u, 0x46aa0124u, 0xaaa88000u, 0xdff88000u, 0x06aa0000u, 0x000006aau, 0x06aa0000u, 0x06aa0f7fu, 0x7d237bffu, 0x7bff7bffu, 0x35557bffu, 0x35553555u, 0x7d237bffu, 0x7bff7d23u, 0x35557bffu, 0x7bff3555u, 0x7d237c00u, 0x7d237c00u, 0xfe007d23u, 0x7d23fc00u, 0x7d237c00u, 0x7d237d23u, 0xfd230000u, 0x7d237c00u, 0x7c007c00u, 0x7d237c00u, 0x35557bffu, 0x35553555u, 0xb555fbffu, 0x35557bffu, 0x7bff7bffu, 0x35557bffu}, - {0x00008000u, 0x80000000u, 0x00008000u, 0x80000000u, 0x80008000u, 0x80008000u, 0xb555fc01u, 0xb555b555u, 0xb555fc01u, 0xfc01fc01u, 0xb555fc01u, 0xb555b555u, 0xb555fc01u, 0xfc01b555u, 0x00000000u, 0x00000000u, 0x04004000u, 0x40000400u, 0xfc00c000u, 0xfc00fc00u, 0x35557c01u, 0x7c017c01u, 0x80008000u, 0x80008000u, 0x80008000u, 0x80008000u, 0x35557c01u, 0x35557c01u, 0x80007c01u, 0x80007c01u, 0x7c010000u, 0x0000fc01u, 0x35557c01u, 0x35553555u, 0xb5557c01u, 0x80007c01u, 0x7c017c01u, 0x8000fc01u, 0x80008000u, 0x80008000u, 0x00008000u, 0x35558000u, 0x80003555u, 0x80000000u}, - {0x03ff0000u, 0xa2480000u, 0x03ff0000u, 0x0000a248u, 0x03ff0000u, 0x000003ffu, 0xfc00c248u, 0xfc007fffu, 0xfc00c248u, 0xc248c248u, 0x03fec248u, 0x03fe03feu, 0x03fec248u, 0x46477fffu, 0x80000000u, 0x00000100u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x7fff0000u, 0x00000000u, 0x7c01c248u, 0x7c017c01u, 0x7fffc248u, 0xc2487fffu, 0x7c010000u, 0x00007c01u, 0x7fff0000u, 0x7fff0000u, 0x00007fffu, 0x7fff0000u, 0x7fffc248u, 0x7fff7fffu, 0xffffc248u, 0x7fff8000u, 0x80008000u, 0x7fff8000u, 0x7fffc248u, 0x7fff7fffu, 0x7fff0000u, 0x7fff0000u, 0xc2480000u, 0xffff8000u}, - {0xc2337fffu, 0x87fd7fffu, 0xc2337fffu, 0xffff07fdu, 0x42348000u, 0x80004234u, 0x4e35bffeu, 0x4e35d234u, 0x4e35bffeu, 0xbffebffeu, 0xd6333ffeu, 0xd633d633u, 0xd6333ffeu, 0xc3fd5234u, 0xc0100000u, 0x00004000u, 0x03c00003u, 0x000303c0u, 0x03c0ffffu, 0x03c0fc00u, 0x3c01fffeu, 0xfffefffeu, 0x1234c000u, 0x12341234u, 0x3c01fffeu, 0xfffec000u, 0x1234c000u, 0x12343c01u, 0x7ffffffeu, 0x7ffffffeu, 0xfffeffffu, 0x7fff7ffeu, 0x3c02fffeu, 0x3c02bfffu, 0x3c00fffeu, 0x1236fffeu, 0xfffefffeu, 0x12367ffeu, 0x1234c000u, 0x12341234u, 0xbc01c000u, 0x3c01c000u, 0x3c011234u, 0x3c014000u}, - {0x9c00a56eu, 0xcffef560u, 0x9c00a56eu, 0x795e4c00u, 0x2000a570u, 0xa5702000u, 0xc6480000u, 0xc6483014u, 0xc6480000u, 0x00000000u, 0xbe482468u, 0xbe48be48u, 0xbe482468u, 0x8e34547cu, 0x0000dcc0u, 0xcc004000u, 0x00840edcu, 0x0edc0084u, 0xff84fedcu, 0xff84ffedu, 0x42481234u, 0x12341234u, 0x8400edccu, 0x84008400u, 0x8400edccu, 0x8400edccu, 0x42481234u, 0x42481234u, 0x7c00fffeu, 0x7c00fffeu, 0x7c00fffeu, 0x7c00fffeu, 0x4248edccu, 0x4248edcbu, 0xc2486dccu, 0x8a48c47fu, 0xc47fc47fu, 0x8a48c47fu, 0x8400edccu, 0x84008400u, 0x0400edccu, 0x42481234u, 0x12344248u, 0x84006dccu}, - {0x04023a00u, 0x3a000402u, 0x04023a00u, 0x38020600u, 0x00020200u, 0x02000002u, 0xfffd7dffu, 0xfffd7ffdu, 0xfffd7dffu, 0x7dff7dffu, 0xffff7e01u, 0xffffffffu, 0xffff7e01u, 0xfe017fffu, 0xc0007fffu, 0xc000ffffu, 0x00037fffu, 0x7fff0003u, 0xffff7fffu, 0xffff0001u, 0xffff7fffu, 0x7fff7fffu, 0xfffefe00u, 0xfffefffeu, 0xfffffe00u, 0xfffffffeu, 0xfffe7fffu, 0x7ffffe00u, 0xfffefe00u, 0xfffefe00u, 0xfe00fffeu, 0xfffe7e00u, 0xfffefe00u, 0xfffefffeu, 0x7ffefe00u, 0xfffefe00u, 0xfe00fe00u, 0xfffe7e00u, 0xfffefe00u, 0xfffefffeu, 0xfffe7e00u, 0xfffefe00u, 0xfe00fffeu, 0x7ffefe00u}, - {0x96003bb8u, 0xee00ebccu, 0x96003bb8u, 0x6bcc6e00u, 0x9800bdb8u, 0xbdb89800u, 0x9034c247u, 0x90349233u, 0x9034c247u, 0xc247c247u, 0x9434c249u, 0x94349434u, 0x9434c249u, 0xc4489235u, 0xe000ff00u, 0xfff00000u, 0x07e0007fu, 0x007f07e0u, 0x07e0007fu, 0x07e007ffu, 0x7e007fffu, 0x7fff7fffu, 0x12344248u, 0x12341234u, 0x7e007fffu, 0x7e007fffu, 0x12344248u, 0x12344248u, 0x7e007fffu, 0x7e007fffu, 0x7fff7e00u, 0xfe007fffu, 0x7e007fffu, 0x7e007fffu, 0x7e00ffffu, 0x7e007fffu, 0x7fff7fffu, 0x7e00ffffu, 0x12344248u, 0x12341234u, 0x1234c248u, 0x12344248u, 0x42481234u, 0x92344248u}, - {0x80000800u, 0x08006340u, 0x80000800u, 0xeb408000u, 0x00000000u, 0x00000000u, 0x35555240u, 0x35558b95u, 0x35555240u, 0x52405240u, 0x3555a5c0u, 0x35553555u, 0x3555a5c0u, 0xfc00df15u, 0x00005640u, 0xc8000000u, 0x00005640u, 0x56400000u, 0x00005640u, 0x000002b2u, 0x35555640u, 0x56405640u, 0x0000fc00u, 0x00000000u, 0x3555fc00u, 0xfc005640u, 0x00005640u, 0x35550000u, 0x0000fc00u, 0x0000fc00u, 0xfc000800u, 0x80007c00u, 0x3555fc00u, 0x35555645u, 0xb555fc00u, 0x0000fc00u, 0xfc00fc00u, 0x0000fc00u, 0x0000fc00u, 0x00000000u, 0x80005640u, 0x35555640u, 0x00005640u, 0x0000d640u}, - {0xdd70b2c8u, 0x611b56e4u, 0xdd70b2c8u, 0x9ee3191cu, 0xa57032c9u, 0x32c9a570u, 0x0000fa46u, 0x00006aefu, 0x0000fa46u, 0xfa46fa46u, 0xdb980000u, 0xdb98db98u, 0xdb980000u, 0x6aef70a9u, 0x4000e918u, 0x300091a0u, 0x00010fa4u, 0x0fa40001u, 0x00010fa4u, 0x00010007u, 0x12347d23u, 0x7d237d23u, 0xedcc7d23u, 0xedccedccu, 0xedcc7d23u, 0x7d23edccu, 0x12347d23u, 0x7d231234u, 0xc3fd7d23u, 0xc3fd7d23u, 0x7d237fffu, 0x44fffd23u, 0xedcc7d23u, 0xedcc7d23u, 0x6dcc7d23u, 0xc47f7d23u, 0x7d237d23u, 0xc47ffd23u, 0xedcc7d23u, 0xedccedccu, 0xedccfd23u, 0x12347d23u, 0x1234edccu, 0x6dcc7d23u}, - {0xc000d640u, 0xd6413c00u, 0xc000d640u, 0x5640bc01u, 0x84008000u, 0x80008400u, 0xfbff7fffu, 0xfbfffc00u, 0xfbff7fffu, 0x7fff7fffu, 0xfc017fffu, 0xfc01fc01u, 0xfc017fffu, 0x8000fc00u, 0x7fff0000u, 0x80008000u, 0x7fff0001u, 0x00017fffu, 0x7fffffffu, 0x7fff8000u, 0x7fffffffu, 0xffffffffu, 0x7c008000u, 0x7c007c00u, 0x7fffffffu, 0xffff8000u, 0x7c008000u, 0x7c007fffu, 0x7fffffffu, 0x7fffffffu, 0xffff7fffu, 0xffff7fffu, 0x7fffffffu, 0x7fff7c00u, 0x7fffffffu, 0x7fffffffu, 0xffffffffu, 0x7fff7fffu, 0x7c008000u, 0x7c007c00u, 0x7c008000u, 0x7c008000u, 0xffff7c00u, 0xfc000000u}, - {0xfffecd56u, 0x38002153u, 0xfffecd56u, 0x5955fffeu, 0x00009556u, 0x95560000u, 0x3c013553u, 0x3c017156u, 0x3c013553u, 0x35533553u, 0x3c01caa9u, 0x3c013c01u, 0x3c01caa9u, 0xfffe06acu, 0x00004000u, 0x6aaa0000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00001aaau, 0x3c013555u, 0x35553555u, 0x0000fffeu, 0x00000000u, 0x3c01fffeu, 0xfffe3c01u, 0x00003555u, 0x35550000u, 0xfffefffeu, 0xfffefffeu, 0xfffe3800u, 0xfffe7ffeu, 0x3c01fffeu, 0x3c013d56u, 0xbc01fffeu, 0x0000fffeu, 0xfffefffeu, 0x00007ffeu, 0x00003555u, 0x00000000u, 0x80003555u, 0x3c013555u, 0x00003c01u, 0x0000b555u}, - {0x31b86c40u, 0xd5b84840u, 0x31b86c40u, 0x6c40b1b8u, 0x1db83440u, 0x34401db8u, 0x3e478490u, 0x3e470490u, 0x3e478490u, 0x84908490u, 0x46490000u, 0x46494649u, 0x46490000u, 0xc6498000u, 0xff004800u, 0x4800ff00u, 0x007b0042u, 0x0042007bu, 0x007b0042u, 0x007b0042u, 0x7bff4248u, 0x42484248u, 0xc2484248u, 0xc248c248u, 0xc2484248u, 0x7bffc248u, 0x7bff4248u, 0x42484248u, 0xfc00492fu, 0xfc00492fu, 0x48effc00u, 0x7c00c8afu, 0x7bff4648u, 0x7bff0000u, 0x7bff0000u, 0xfc0048efu, 0x48ef48efu, 0xfc0048efu, 0xc2484248u, 0xc248c248u, 0xfbffc248u, 0x7bff4248u, 0x7bff4248u, 0x7bff4248u}, - {0x9a908000u, 0xdcbc4000u, 0x9a908000u, 0x80009cbcu, 0x5a900000u, 0x00005a90u, 0xdb98fe55u, 0xdb98edccu, 0xdb98fe55u, 0xfe55fe55u, 0x0000fe55u, 0x00000000u, 0x0000fe55u, 0x1089edccu, 0xc0000000u, 0x0000b980u, 0x000e0000u, 0x0000000eu, 0xfffe0000u, 0xfffe0000u, 0xedcc0000u, 0x00000000u, 0xedccfe55u, 0xedccedccu, 0xedccfe55u, 0xfe55edccu, 0xedcc0000u, 0x0000edccu, 0x7c00fe55u, 0x7c00fe55u, 0xfe557c00u, 0xfc007e55u, 0xf1ccfe55u, 0xf1ccedccu, 0x0000fe55u, 0x7c00fe55u, 0xfe55fe55u, 0x7c007e55u, 0xedcc0000u, 0xedccedccu, 0xedcc0000u, 0xedcc0000u, 0xedcc0000u, 0x6dcc8000u}, - {0x12344400u, 0x14000a34u, 0x12344400u, 0x0c001234u, 0x00003000u, 0x30000000u, 0xfffef1ccu, 0xfffe03feu, 0xfffef1ccu, 0xf1ccf1ccu, 0xfffee9ccu, 0xfffefffeu, 0xfffee9ccu, 0xedccfbfeu, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000400u, 0x04000400u, 0xfffeedccu, 0xfffefffeu, 0xfffeedccu, 0xedccfffeu, 0x00000400u, 0x04000000u, 0xfffeb5c8u, 0xfffeb5c8u, 0xb5c9fffeu, 0xfffe35d0u, 0xfffeedccu, 0xfffefffeu, 0x7ffeedccu, 0xfffeb5ccu, 0xb5ccb5ccu, 0xfffeb5ccu, 0x0000edccu, 0x00000000u, 0x80000400u, 0x00000400u, 0x00000400u, 0x00008400u}, - {0x7fff9c00u, 0xbc00c69eu, 0x7fff9c00u, 0x029f7fffu, 0x0000e000u, 0xe0000000u, 0x7d235e55u, 0x7d237b78u, 0x7d235e55u, 0x5e555e55u, 0x7d2361abu, 0x7d237d23u, 0x7d2361abu, 0x60007eceu, 0x0000fe55u, 0xf2a80000u, 0x0000fe55u, 0xfe550000u, 0x0000fe55u, 0x0000ffcau, 0x7d236000u, 0x60006000u, 0x0000fe55u, 0x00000000u, 0x7d23fe55u, 0x6000fe55u, 0x00006000u, 0x7d230000u, 0x7d23fe55u, 0x7d23fe55u, 0xfe557d23u, 0x7d23fe55u, 0x7d23fe55u, 0x7d237d23u, 0xfd237e55u, 0x7d23fe55u, 0xfe55fe55u, 0x7d237e55u, 0x00006000u, 0x00000000u, 0x8000e000u, 0x00006000u, 0x60007d23u, 0x00006000u}, - {0x3c008400u, 0x84007c00u, 0x3c008400u, 0x84007c00u, 0xc0000000u, 0x0000c000u, 0x3c018000u, 0x3c017c01u, 0x3c018000u, 0x80008000u, 0xbc018000u, 0xbc01bc01u, 0xbc018000u, 0xc0007c01u, 0x80000000u, 0x0000c000u, 0x60000000u, 0x00006000u, 0xe0000000u, 0xe0000000u, 0x7c010000u, 0x00000000u, 0xc0008000u, 0xc000c000u, 0xc0008000u, 0xc0007c01u, 0x7c010000u, 0x00008000u, 0x7c018400u, 0x7c018400u, 0x7c007c01u, 0x7c018400u, 0x7c010000u, 0x7c017c01u, 0xfc018000u, 0x7c018000u, 0x80008000u, 0x7c018000u, 0xc0008000u, 0xc000c000u, 0x40000000u, 0xc0000000u, 0x80000000u, 0xc0008000u}, - {0x40006fe8u, 0xaa007de8u, 0x40006fe8u, 0xe7e84000u, 0x2c00f1e8u, 0xf1e82c00u, 0xb155c09du, 0xb155f79du, 0xb155c09du, 0xc09dc09du, 0xb9553c0du, 0xb955b955u, 0xb9553c0du, 0x8255730du, 0x80004900u, 0x49008000u, 0x03e00612u, 0x061203e0u, 0x03e0fe12u, 0x03e0fe12u, 0x7c00fe55u, 0xfe55fe55u, 0x3555c248u, 0x35553555u, 0x7c00fe55u, 0xfe55c248u, 0x3555c248u, 0x35557c00u, 0x7c00fe55u, 0x7c00fe55u, 0xfe557e00u, 0xfc007e55u, 0x7c00fe55u, 0x7c00c19du, 0x7c00fe55u, 0x7c00fe55u, 0xfe55fe55u, 0x7c007e55u, 0x3555c248u, 0x35553555u, 0xfc00c248u, 0x7c00c248u, 0x7c003555u, 0x7c004248u}, - {0x755fffffu, 0x5e480e33u, 0x755fffffu, 0x4e341e47u, 0x7960c3ffu, 0xc3ff7960u, 0x30143c00u, 0x3014edcbu, 0x30143c00u, 0x3c003c00u, 0xab843c02u, 0xab84ab84u, 0xab843c02u, 0xf9b9edcdu, 0x8000fffeu, 0xf0008490u, 0x00047fffu, 0x7fff0004u, 0x0004ffffu, 0x0004ffffu, 0x42483c01u, 0x3c013c01u, 0xedccffffu, 0xedccedccu, 0xedccffffu, 0x4248ffffu, 0x42483c01u, 0xedcc3c01u, 0xfc00ffffu, 0xfc00ffffu, 0xfffff48du, 0xf9b8ffffu, 0xedcbffffu, 0xedcbffffu, 0x6dcd7fffu, 0xf48dffffu, 0xffffffffu, 0xf48d7fffu, 0xedcc3c01u, 0xedccedccu, 0xedccbc01u, 0x42483c01u, 0x4248edccu, 0x6dcc3c01u}, - {0x8132edccu, 0x70a9fe55u, 0x8132edccu, 0xedcc8132u, 0x82dd0000u, 0x000082ddu, 0x7d227d23u, 0x7d227d23u, 0x7d227d23u, 0x7d237d23u, 0x7d247d23u, 0x7d247d24u, 0x7d247d23u, 0x7d247d23u, 0xfff80000u, 0x0000fff8u, 0x1fff0000u, 0x00001fffu, 0xffff0000u, 0xffff0000u, 0x7d237d23u, 0x7d237d23u, 0xffff0000u, 0xffffffffu, 0xffff7d23u, 0xffff7d23u, 0x7d230000u, 0x00007d23u, 0x7d237d23u, 0x7d237d23u, 0x7d237d23u, 0x7d23fd23u, 0x7d237d23u, 0x7d237d23u, 0xfd237d23u, 0x7d237d23u, 0x7d237d23u, 0x7d23fd23u, 0x7d230000u, 0x7d237d23u, 0x7d230000u, 0x7d230000u, 0x7d230000u, 0xfd238000u}, - {0x00008000u, 0x80002000u, 0x00008000u, 0xa0000000u, 0x00000000u, 0x00000000u, 0xc2480400u, 0xc248c648u, 0xc2480400u, 0x04000400u, 0xc248fc00u, 0xc248c248u, 0xc248fc00u, 0x0000be48u, 0x00000400u, 0x00000000u, 0x00000400u, 0x04000000u, 0x00000400u, 0x00000004u, 0x00000400u, 0x04000400u, 0xc2480000u, 0xc248c248u, 0xc2480400u, 0x0000c248u, 0x00000000u, 0x04000000u, 0x00000000u, 0x00000000u, 0x00008000u, 0x00008000u, 0xc2480400u, 0xc248c248u, 0x42488400u, 0x80000000u, 0x00000000u, 0x80000000u, 0xc2480000u, 0xc248c248u, 0xc2488000u, 0x00000400u, 0x00000400u, 0x42480000u}, - {0xcaab5400u, 0x9634ac00u, 0xcaab5400u, 0x30001234u, 0xcaabd000u, 0xd000caabu, 0x3554e9ccu, 0x35543155u, 0x3554e9ccu, 0xe9cce9ccu, 0x3556f1ccu, 0x35563556u, 0x3556f1ccu, 0xedcd3955u, 0xffe00000u, 0x8000f000u, 0x07ff000fu, 0x000f07ffu, 0xffffffffu, 0xffffffe0u, 0x3555fc00u, 0xfc00fc00u, 0xffffedccu, 0xffffffffu, 0xfffffc00u, 0xfffffc00u, 0x3555edccu, 0x3555edccu, 0xffff7c00u, 0xffff7c00u, 0x7c00ffffu, 0x7ffffc00u, 0xfffffc00u, 0xfffffc00u, 0xffff7c00u, 0xffff7c00u, 0x7c007c00u, 0xffff7c00u, 0x3555fc00u, 0x35553555u, 0x3555fc00u, 0x3555edccu, 0xedcc3555u, 0xb5557c00u}, - {0x8400c000u, 0xa11c8400u, 0x8400c000u, 0xc000651cu, 0x00000000u, 0x00000000u, 0x7d231234u, 0x7d230000u, 0x7d231234u, 0x12341234u, 0x82dd1234u, 0x82dd82ddu, 0x82dd1234u, 0x95110000u, 0x7d230000u, 0x0000d230u, 0x7d230000u, 0x00007d23u, 0x7d230000u, 0x7d230000u, 0x7d231234u, 0x12341234u, 0x00000000u, 0x00000000u, 0x7d231234u, 0x7d230000u, 0x00000000u, 0x00001234u, 0x7d23c000u, 0x7d23c000u, 0x84007d23u, 0xfd23c000u, 0x7d231234u, 0x7d230000u, 0x7d231234u, 0x7d230000u, 0x00000000u, 0x7d230000u, 0x00000000u, 0x00000000u, 0x00009234u, 0x00001234u, 0x12340000u, 0x80001234u}, - {0x7d557e55u, 0x7e557d55u, 0x7d557e55u, 0x7e557d55u, 0x00000000u, 0x00000000u, 0xfc007c00u, 0xfc000000u, 0xfc007c00u, 0x7c007c00u, 0x04007c00u, 0x04000400u, 0x04007c00u, 0x80000000u, 0xfc000000u, 0x0000fc00u, 0xfc000000u, 0x0000fc00u, 0xfc000000u, 0xfc000000u, 0x00007c00u, 0x7c007c00u, 0xfc000000u, 0xfc00fc00u, 0xfc007c00u, 0xfc000000u, 0x00000000u, 0x00007c00u, 0xfe00fe00u, 0xfe00fe00u, 0xfe00fe00u, 0xfe00fe00u, 0xfc007c00u, 0xfc000000u, 0xfc007c00u, 0xfe00fe00u, 0xfe00fe00u, 0xfe00fe00u, 0xfc000000u, 0xfc00fc00u, 0x0000fc00u, 0x00007c00u, 0x7c000000u, 0x80007c00u}, - {0xfe557d55u, 0x7d55fe55u, 0xfe557d55u, 0x7d55fe55u, 0x00000000u, 0x00000000u, 0x7c007c00u, 0x7c00f800u, 0x7c007c00u, 0x7c007c00u, 0x7c008400u, 0x7c007c00u, 0x7c008400u, 0x00000000u, 0x00007c00u, 0x7c000000u, 0x00007c00u, 0x7c000000u, 0x00007c00u, 0x00007c00u, 0x7c007c00u, 0x7c007c00u, 0x00000000u, 0x00000000u, 0x7c007c00u, 0x00007c00u, 0x00000000u, 0x7c000000u, 0xfe00fe00u, 0xfe00fe00u, 0xfe00fe00u, 0xfe00fe00u, 0x7c007c00u, 0x7c007c00u, 0xfc00fc00u, 0xfe00fe00u, 0xfe00fe00u, 0xfe00fe00u, 0x00000000u, 0x00000000u, 0x80008000u, 0x7c007c00u, 0x00007c00u, 0x00000000u}, +constexpr std::uint32_t ExpectedIeeeOff[32][48] = { + {0x3c003c00u, 0x3c003c00u, 0x3c003c00u, 0x3c003c00u, 0x00000000u, 0x00000000u, 0x7c007c00u, 0x7c00fc00u, 0x7c007c00u, 0x7c007c00u, 0x7c007c00u, 0x7c007c00u, 0x7c007c00u, 0xfc00fc00u, 0x80000000u, 0x00008000u, 0x80000000u, 0x00008000u, 0x80000000u, 0x80000000u, 0xfc007c00u, 0x7c007c00u, 0x80000000u, 0x80008000u, 0xfc007c00u, 0x8000fc00u, 0x80000000u, 0x00007c00u, 0xfe00fe00u, 0xfe00fe00u, 0xfe00fe00u, 0xfe00fe00u, 0xfc007c00u, 0xfc00fc00u, 0x7c007c00u, 0xfe00fe00u, 0xfe00fe00u, 0xfe00fe00u, 0xfc000000u, 0xfc00fc00u, 0xfc00fc00u, 0x80007c00u, 0x7c000000u, 0x7c007c00u, 0x00000000u, 0x00000000u, 0x00003c00u, 0x3c003c00u}, + {0xbc007c00u, 0x7c00bc00u, 0xbc007c00u, 0x7c00bc00u, 0x00000000u, 0x00000000u, 0x78007800u, 0x7800f800u, 0x78007800u, 0x78007800u, 0x80008000u, 0x80008000u, 0x80008000u, 0x00000000u, 0x7c00fc00u, 0xfc007c00u, 0x7c00fc00u, 0xfc007c00u, 0x7c00fc00u, 0x7c00fc00u, 0x7c007c00u, 0x7c007c00u, 0xfc00fc00u, 0xfc00fc00u, 0xfc00fc00u, 0x7c00fc00u, 0x7c007c00u, 0xfc007c00u, 0xfc00fe00u, 0xfc00fe00u, 0xfc00fe00u, 0x7c007c00u, 0xfe00fe00u, 0xfe00fc00u, 0x7c007c00u, 0xfc00fc00u, 0xfc00fc00u, 0xfc00fc00u, 0xfc00fc00u, 0xfc00fc00u, 0xfc00fc00u, 0x7c007c00u, 0x7c00fc00u, 0x7c007c00u, 0x00000000u, 0x3c000000u, 0x3c003c00u, 0x3c003c00u}, + {0x7bffc000u, 0xc000fbffu, 0x7bffc000u, 0x7fff3c00u, 0x43ff0400u, 0x040043ffu, 0xbc00fbffu, 0xbc003c00u, 0xbc00fbffu, 0xfbfffbffu, 0xbc02fc01u, 0xbc02bc02u, 0xbc02fc01u, 0x7c013c02u, 0xfffeffffu, 0xfffe7fffu, 0x3fffffffu, 0xffff3fffu, 0x3fffffffu, 0x3fffffffu, 0x7fffffffu, 0xffffffffu, 0x3c01fc00u, 0x3c013c01u, 0x7fffffffu, 0xfc00ffffu, 0x3c01fc00u, 0x3c017fffu, 0x7fffffffu, 0x7fffffffu, 0xffff7fffu, 0xffffffffu, 0x7fffffffu, 0x7fffffffu, 0x7fff7fffu, 0x7fffffffu, 0xffffffffu, 0x7fff7fffu, 0x3c01fc00u, 0x3c013c01u, 0x3c017c00u, 0x3c01fc00u, 0xfc003c01u, 0xbc01fc00u, 0x3c000000u, 0x00003c00u, 0x3c000000u, 0x00000000u}, + {0xc0009400u, 0x1400c000u, 0xc0009400u, 0x1400c000u, 0x00008000u, 0x80000000u, 0x1400ffffu, 0x14009400u, 0x1400ffffu, 0xffffffffu, 0x1400ffffu, 0x14001400u, 0x1400ffffu, 0x7fff9400u, 0x00000000u, 0x80000000u, 0x00000001u, 0x00010000u, 0x0000ffffu, 0x00008000u, 0x14007fffu, 0x7fff7fffu, 0x00008000u, 0x00000000u, 0x14008000u, 0x7fff8000u, 0x00007fffu, 0x14000000u, 0xc0007fffu, 0xc0007fffu, 0x7fff1400u, 0xc000ffffu, 0x14007fffu, 0x14001400u, 0x94007fffu, 0x00007fffu, 0x7fff7fffu, 0x0000ffffu, 0x00008000u, 0x00000000u, 0x80008000u, 0x14008000u, 0x00001400u, 0x00000000u, 0x00000000u, 0x00000000u, 0x14000000u, 0x00000000u}, + {0x7e00d5feu, 0x7ffe8000u, 0x7e00d5feu, 0x81fe7e00u, 0x8000d600u, 0xd6008000u, 0x7fff7c55u, 0x7fff7dffu, 0x7fff7c55u, 0x7c557c55u, 0x7fff8055u, 0x7fff7fffu, 0x7fff8055u, 0x7e5581ffu, 0x0000c000u, 0x00000000u, 0x000103f0u, 0x03f00001u, 0xffff03f0u, 0xffff0000u, 0xffff7e00u, 0x7e007e00u, 0x8000fe55u, 0x80008000u, 0xfffffe55u, 0xfe55ffffu, 0x80007e00u, 0x7e008000u, 0xfffffe55u, 0xfffffe55u, 0xfe55ffffu, 0xffff7e55u, 0xfffffe55u, 0xffffffffu, 0x7ffffe55u, 0xfffffe55u, 0xfe55fe55u, 0xffff7e55u, 0x8000fe55u, 0x80008000u, 0x00007e55u, 0x8000fe55u, 0x8000ffffu, 0x8000fe55u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u}, + {0x80007656u, 0x4254c3feu, 0x80007656u, 0x4254c3feu, 0x00017801u, 0x78010001u, 0xfffe7802u, 0xfffebc00u, 0xfffe7802u, 0x78027802u, 0x00000000u, 0x00000000u, 0x00000000u, 0xbc0243feu, 0x80007802u, 0x8000fffeu, 0x00001e00u, 0x1e000000u, 0x00001e00u, 0x00000000u, 0x7fff3c01u, 0x3c013c01u, 0x7fff3c01u, 0x7fff7fffu, 0x7fff3c01u, 0x7fff7fffu, 0x7fff3c01u, 0x3c013c01u, 0x7ffffe55u, 0x7ffffe55u, 0x7fff7fffu, 0x7ffffe55u, 0x7fff4001u, 0x7fff7fffu, 0xffff0000u, 0x7fff3c02u, 0x3c023c02u, 0x7fff3c02u, 0x7fff3c01u, 0x7fff7fffu, 0x7fffbc01u, 0x7fff3c01u, 0x3c013c01u, 0xffff3c01u, 0x00003c00u, 0x3c003c00u, 0x00003c00u, 0x00003c00u}, + {0x7bff6000u, 0x60007bffu, 0x7bff6000u, 0x60007bffu, 0x00000000u, 0x00000000u, 0xedcc6dccu, 0xedcc6dccu, 0xedcc6dccu, 0x6dcc6dccu, 0xedcc6dccu, 0xedccedccu, 0xedcc6dccu, 0xedcc6dccu, 0x00000000u, 0x00000000u, 0x00000008u, 0x00080000u, 0x0000fff8u, 0x0000fff8u, 0x0000edccu, 0xedccedccu, 0xedcc8000u, 0xedccedccu, 0xedccedccu, 0xedccedccu, 0x00008000u, 0x80000000u, 0x7bff6000u, 0x7bff6000u, 0x7bff6000u, 0x7bff6000u, 0xedccedccu, 0xedccedccu, 0x6dccedccu, 0x80000000u, 0x00000000u, 0x80000000u, 0xedccedccu, 0xedccedccu, 0xedcc8000u, 0x00008000u, 0x00008000u, 0x6dcc0000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x3c000000u}, + {0x00001dccu, 0xedccd000u, 0x00001dccu, 0xbdcc0000u, 0x00003000u, 0x30000000u, 0x8000ce34u, 0x80009634u, 0x8000ce34u, 0xce34ce34u, 0x8800a9ccu, 0x88008800u, 0x8800a9ccu, 0xc00071ccu, 0xfc001234u, 0x1234fc00u, 0xfc001234u, 0x1234fc00u, 0xfc001234u, 0xfc001234u, 0xfc001234u, 0x12341234u, 0x8400bc00u, 0x84008400u, 0xfc00bc00u, 0xfc008400u, 0x84001234u, 0x1234bc00u, 0x7c00edccu, 0x7c00edccu, 0x92347c00u, 0xfc00edccu, 0xfc00bbfeu, 0xfc0011b4u, 0xfc00bc01u, 0x7c009234u, 0x92349234u, 0x7c009234u, 0xfc00bc00u, 0xfc00fc00u, 0x84001234u, 0x84001234u, 0xbc001234u, 0x04009234u, 0x00000000u, 0x00000000u, 0x00001234u, 0x04000000u}, + {0xc99f5e41u, 0x4ceb74ddu, 0xc99f5e41u, 0xcd1df4abu, 0xcb9f0801u, 0x0801cb9fu, 0xb278f7feu, 0xb278f922u, 0xb278f7feu, 0xf7fef7feu, 0x47ce0000u, 0x47ce47ceu, 0x47ce0000u, 0x46aa0124u, 0xaaa88000u, 0xdff88000u, 0x06aa0000u, 0x000006aau, 0x06aa0000u, 0x06aa0f7fu, 0x7d237bffu, 0x7bff7bffu, 0x35557bffu, 0x35553555u, 0x7d237bffu, 0x7bff7d23u, 0x35557bffu, 0x7bff3555u, 0x7d237c00u, 0x7d237c00u, 0xfe007d23u, 0x7d23fc00u, 0x7d237c00u, 0x7d237d23u, 0xfd230000u, 0x7d237c00u, 0x7c007c00u, 0x7d237c00u, 0x35557bffu, 0x35553555u, 0xb555fbffu, 0x35557bffu, 0x7bff7bffu, 0x35557bffu, 0x35553c00u, 0x35553c00u, 0x35553c00u, 0x35553c00u}, + {0x00008000u, 0x80000000u, 0x00008000u, 0x80000000u, 0x80008000u, 0x80008000u, 0xb555fc01u, 0xb555b555u, 0xb555fc01u, 0xfc01fc01u, 0xb555fc01u, 0xb555b555u, 0xb555fc01u, 0xfc01b555u, 0x00000000u, 0x00000000u, 0x04004000u, 0x40000400u, 0xfc00c000u, 0xfc00fc00u, 0x35557c01u, 0x7c017c01u, 0x80008000u, 0x80008000u, 0x80008000u, 0x80008000u, 0x35557c01u, 0x35557c01u, 0x80007c01u, 0x80007c01u, 0x7c010000u, 0x0000fc01u, 0x35557c01u, 0x35553555u, 0xb5557c01u, 0x80007c01u, 0x7c017c01u, 0x8000fc01u, 0x80008000u, 0x80008000u, 0x00008000u, 0x35558000u, 0x80003555u, 0x80000000u, 0x00000000u, 0x00000000u, 0x35550000u, 0x00000000u}, + {0x03ff0000u, 0xa2480000u, 0x03ff0000u, 0x0000a248u, 0x03ff0000u, 0x000003ffu, 0xfc00c248u, 0xfc007fffu, 0xfc00c248u, 0xc248c248u, 0x03fec248u, 0x03fe03feu, 0x03fec248u, 0x46477fffu, 0x80000000u, 0x00000100u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x7fff0000u, 0x00000000u, 0x7c01c248u, 0x7c017c01u, 0x7fffc248u, 0xc2487fffu, 0x7c010000u, 0x00007c01u, 0x7fff0000u, 0x7fff0000u, 0x00007fffu, 0x7fff0000u, 0x7fffc248u, 0x7fff7fffu, 0xffffc248u, 0x7fff8000u, 0x80008000u, 0x7fff8000u, 0x7fffc248u, 0x7fff7fffu, 0x7fff0000u, 0x7fff0000u, 0xc2480000u, 0xffff8000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u}, + {0xc2337fffu, 0x87fd7fffu, 0xc2337fffu, 0xffff07fdu, 0x42348000u, 0x80004234u, 0x4e35bffeu, 0x4e35d234u, 0x4e35bffeu, 0xbffebffeu, 0xd6333ffeu, 0xd633d633u, 0xd6333ffeu, 0xc3fd5234u, 0xc0100000u, 0x00004000u, 0x03c00003u, 0x000303c0u, 0x03c0ffffu, 0x03c0fc00u, 0x3c01fffeu, 0xfffefffeu, 0x1234c000u, 0x12341234u, 0x3c01fffeu, 0xfffec000u, 0x1234c000u, 0x12343c01u, 0x7ffffffeu, 0x7ffffffeu, 0xfffeffffu, 0x7fff7ffeu, 0x3c02fffeu, 0x3c02bfffu, 0x3c00fffeu, 0x1236fffeu, 0xfffefffeu, 0x12367ffeu, 0x1234c000u, 0x12341234u, 0xbc01c000u, 0x3c01c000u, 0x3c011234u, 0x3c014000u, 0x12340000u, 0x3c000000u, 0x3c000000u, 0x3c003c00u}, + {0x9c00a56eu, 0xcffef560u, 0x9c00a56eu, 0x795e4c00u, 0x2000a570u, 0xa5702000u, 0xc6480000u, 0xc6483014u, 0xc6480000u, 0x00000000u, 0xbe482468u, 0xbe48be48u, 0xbe482468u, 0x8e34547cu, 0x0000dcc0u, 0xcc004000u, 0x00840edcu, 0x0edc0084u, 0xff84fedcu, 0xff84ffedu, 0x42481234u, 0x12341234u, 0x8400edccu, 0x84008400u, 0x8400edccu, 0x8400edccu, 0x42481234u, 0x42481234u, 0x7c00fffeu, 0x7c00fffeu, 0x7c00fffeu, 0x7c00fffeu, 0x4248edccu, 0x4248edcbu, 0xc2486dccu, 0x8a48c47fu, 0xc47fc47fu, 0x8a48c47fu, 0x8400edccu, 0x84008400u, 0x0400edccu, 0x42481234u, 0x12344248u, 0x84006dccu, 0x00000000u, 0x00000000u, 0x3c001234u, 0x00003c00u}, + {0x04023a00u, 0x3a000402u, 0x04023a00u, 0x38020600u, 0x00020200u, 0x02000002u, 0xfffd7dffu, 0xfffd7ffdu, 0xfffd7dffu, 0x7dff7dffu, 0xffff7e01u, 0xffffffffu, 0xffff7e01u, 0xfe017fffu, 0xc0007fffu, 0xc000ffffu, 0x00037fffu, 0x7fff0003u, 0xffff7fffu, 0xffff0001u, 0xffff7fffu, 0x7fff7fffu, 0xfffefe00u, 0xfffefffeu, 0xfffffe00u, 0xfffffffeu, 0xfffe7fffu, 0x7ffffe00u, 0xfffefe00u, 0xfffefe00u, 0xfe00fffeu, 0xfffe7e00u, 0xfffefe00u, 0xfffefffeu, 0x7ffefe00u, 0xfffefe00u, 0xfe00fe00u, 0xfffe7e00u, 0xfffefe00u, 0xfffefffeu, 0xfffe7e00u, 0xfffefe00u, 0xfe00fffeu, 0x7ffefe00u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u}, + {0x96003bb8u, 0xee00ebccu, 0x96003bb8u, 0x6bcc6e00u, 0x9800bdb8u, 0xbdb89800u, 0x9034c247u, 0x90349233u, 0x9034c247u, 0xc247c247u, 0x9434c249u, 0x94349434u, 0x9434c249u, 0xc4489235u, 0xe000ff00u, 0xfff00000u, 0x07e0007fu, 0x007f07e0u, 0x07e0007fu, 0x07e007ffu, 0x7e007fffu, 0x7fff7fffu, 0x12344248u, 0x12341234u, 0x7e007fffu, 0x7e007fffu, 0x12344248u, 0x12344248u, 0x7e007fffu, 0x7e007fffu, 0x7fff7e00u, 0xfe007fffu, 0x7e007fffu, 0x7e007fffu, 0x7e00ffffu, 0x7e007fffu, 0x7fff7fffu, 0x7e00ffffu, 0x12344248u, 0x12341234u, 0x1234c248u, 0x12344248u, 0x42481234u, 0x92344248u, 0x12343c00u, 0x3c001234u, 0x12343c00u, 0x00003c00u}, + {0x80000800u, 0x08006340u, 0x80000800u, 0xeb408000u, 0x00000000u, 0x00000000u, 0x35555240u, 0x35558b95u, 0x35555240u, 0x52405240u, 0x3555a5c0u, 0x35553555u, 0x3555a5c0u, 0xfc00df15u, 0x00005640u, 0xc8000000u, 0x00005640u, 0x56400000u, 0x00005640u, 0x000002b2u, 0x35555640u, 0x56405640u, 0x0000fc00u, 0x00000000u, 0x3555fc00u, 0xfc005640u, 0x00005640u, 0x35550000u, 0x0000fc00u, 0x0000fc00u, 0xfc000800u, 0x80007c00u, 0x3555fc00u, 0x35555645u, 0xb555fc00u, 0x0000fc00u, 0xfc00fc00u, 0x0000fc00u, 0x0000fc00u, 0x00000000u, 0x80005640u, 0x35555640u, 0x00005640u, 0x0000d640u, 0x00000000u, 0x00003555u, 0x35553c00u, 0x00000000u}, + {0xdd70b2c8u, 0x611b56e4u, 0xdd70b2c8u, 0x9ee3191cu, 0xa57032c9u, 0x32c9a570u, 0x0000fa46u, 0x00006aefu, 0x0000fa46u, 0xfa46fa46u, 0xdb980000u, 0xdb98db98u, 0xdb980000u, 0x6aef70a9u, 0x4000e918u, 0x300091a0u, 0x00010fa4u, 0x0fa40001u, 0x00010fa4u, 0x00010007u, 0x12347d23u, 0x7d237d23u, 0xedcc7d23u, 0xedccedccu, 0xedcc7d23u, 0x7d23edccu, 0x12347d23u, 0x7d231234u, 0xc3fd7d23u, 0xc3fd7d23u, 0x7d237fffu, 0x44fffd23u, 0xedcc7d23u, 0xedcc7d23u, 0x6dcc7d23u, 0xc47f7d23u, 0x7d237d23u, 0xc47ffd23u, 0xedcc7d23u, 0xedccedccu, 0xedccfd23u, 0x12347d23u, 0x1234edccu, 0x6dcc7d23u, 0x00000000u, 0x12340000u, 0x12340000u, 0x3c000000u}, + {0xc000d640u, 0xd6413c00u, 0xc000d640u, 0x5640bc01u, 0x84008000u, 0x80008400u, 0xfbff7fffu, 0xfbfffc00u, 0xfbff7fffu, 0x7fff7fffu, 0xfc017fffu, 0xfc01fc01u, 0xfc017fffu, 0x8000fc00u, 0x7fff0000u, 0x80008000u, 0x7fff0001u, 0x00017fffu, 0x7fffffffu, 0x7fff8000u, 0x7fffffffu, 0xffffffffu, 0x7c008000u, 0x7c007c00u, 0x7fffffffu, 0xffff8000u, 0x7c008000u, 0x7c007fffu, 0x7fffffffu, 0x7fffffffu, 0xffff7fffu, 0xffff7fffu, 0x7fffffffu, 0x7fff7c00u, 0x7fffffffu, 0x7fffffffu, 0xffffffffu, 0x7fff7fffu, 0x7c008000u, 0x7c007c00u, 0x7c008000u, 0x7c008000u, 0xffff7c00u, 0xfc000000u, 0x3c000000u, 0x00000000u, 0x3c000000u, 0x00000000u}, + {0xfffecd56u, 0x38002153u, 0xfffecd56u, 0x5955fffeu, 0x00009556u, 0x95560000u, 0x3c013553u, 0x3c017156u, 0x3c013553u, 0x35533553u, 0x3c01caa9u, 0x3c013c01u, 0x3c01caa9u, 0xfffe06acu, 0x00004000u, 0x6aaa0000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00001aaau, 0x3c013555u, 0x35553555u, 0x0000fffeu, 0x00000000u, 0x3c01fffeu, 0xfffe3c01u, 0x00003555u, 0x35550000u, 0xfffefffeu, 0xfffefffeu, 0xfffe3800u, 0xfffe7ffeu, 0x3c01fffeu, 0x3c013d56u, 0xbc01fffeu, 0x0000fffeu, 0xfffefffeu, 0x00007ffeu, 0x00003555u, 0x00000000u, 0x80003555u, 0x3c013555u, 0x00003c01u, 0x0000b555u, 0x00003555u, 0x00003555u, 0x3c003555u, 0x00000000u}, + {0x31b86c40u, 0xd5b84840u, 0x31b86c40u, 0x6c40b1b8u, 0x1db83440u, 0x34401db8u, 0x3e478490u, 0x3e470490u, 0x3e478490u, 0x84908490u, 0x46490000u, 0x46494649u, 0x46490000u, 0xc6498000u, 0xff004800u, 0x4800ff00u, 0x007b0042u, 0x0042007bu, 0x007b0042u, 0x007b0042u, 0x7bff4248u, 0x42484248u, 0xc2484248u, 0xc248c248u, 0xc2484248u, 0x7bffc248u, 0x7bff4248u, 0x42484248u, 0xfc00492fu, 0xfc00492fu, 0x48effc00u, 0x7c00c8afu, 0x7bff4648u, 0x7bff0000u, 0x7bff0000u, 0xfc0048efu, 0x48ef48efu, 0xfc0048efu, 0xc2484248u, 0xc248c248u, 0xfbffc248u, 0x7bff4248u, 0x7bff4248u, 0x7bff4248u, 0x00003c00u, 0x3c000000u, 0x3c003c00u, 0x3c003c00u}, + {0x9a908000u, 0xdcbc4000u, 0x9a908000u, 0x80009cbcu, 0x5a900000u, 0x00005a90u, 0xdb98fe55u, 0xdb98edccu, 0xdb98fe55u, 0xfe55fe55u, 0x0000fe55u, 0x00000000u, 0x0000fe55u, 0x1089edccu, 0xc0000000u, 0x0000b980u, 0x000e0000u, 0x0000000eu, 0xfffe0000u, 0xfffe0000u, 0xedcc0000u, 0x00000000u, 0xedccfe55u, 0xedccedccu, 0xedccfe55u, 0xfe55edccu, 0xedcc0000u, 0x0000edccu, 0x7c00fe55u, 0x7c00fe55u, 0xfe557c00u, 0xfc007e55u, 0xf1ccfe55u, 0xf1ccedccu, 0x0000fe55u, 0x7c00fe55u, 0xfe55fe55u, 0x7c007e55u, 0xedcc0000u, 0xedccedccu, 0xedcc0000u, 0xedcc0000u, 0xedcc0000u, 0x6dcc8000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x3c000000u}, + {0x12344400u, 0x14000a34u, 0x12344400u, 0x0c001234u, 0x00003000u, 0x30000000u, 0xfffef1ccu, 0xfffe03feu, 0xfffef1ccu, 0xf1ccf1ccu, 0xfffee9ccu, 0xfffefffeu, 0xfffee9ccu, 0xedccfbfeu, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000400u, 0x04000400u, 0xfffeedccu, 0xfffefffeu, 0xfffeedccu, 0xedccfffeu, 0x00000400u, 0x04000000u, 0xfffeb5c8u, 0xfffeb5c8u, 0xb5c9fffeu, 0xfffe35d0u, 0xfffeedccu, 0xfffefffeu, 0x7ffeedccu, 0xfffeb5ccu, 0xb5ccb5ccu, 0xfffeb5ccu, 0x0000edccu, 0x00000000u, 0x80000400u, 0x00000400u, 0x00000400u, 0x00008400u, 0x00000000u, 0x00000400u, 0x00000400u, 0x00000000u}, + {0x7fff9c00u, 0xbc00c69eu, 0x7fff9c00u, 0x029f7fffu, 0x0000e000u, 0xe0000000u, 0x7d235e55u, 0x7d237b78u, 0x7d235e55u, 0x5e555e55u, 0x7d2361abu, 0x7d237d23u, 0x7d2361abu, 0x60007eceu, 0x0000fe55u, 0xf2a80000u, 0x0000fe55u, 0xfe550000u, 0x0000fe55u, 0x0000ffcau, 0x7d236000u, 0x60006000u, 0x0000fe55u, 0x00000000u, 0x7d23fe55u, 0x6000fe55u, 0x00006000u, 0x7d230000u, 0x7d23fe55u, 0x7d23fe55u, 0xfe557d23u, 0x7d23fe55u, 0x7d23fe55u, 0x7d237d23u, 0xfd237e55u, 0x7d23fe55u, 0xfe55fe55u, 0x7d237e55u, 0x00006000u, 0x00000000u, 0x8000e000u, 0x00006000u, 0x60007d23u, 0x00006000u, 0x00003c00u, 0x00000000u, 0x00003c00u, 0x00003c00u}, + {0x3c008400u, 0x84007c00u, 0x3c008400u, 0x84007c00u, 0xc0000000u, 0x0000c000u, 0x3c018000u, 0x3c017c01u, 0x3c018000u, 0x80008000u, 0xbc018000u, 0xbc01bc01u, 0xbc018000u, 0xc0007c01u, 0x80000000u, 0x0000c000u, 0x60000000u, 0x00006000u, 0xe0000000u, 0xe0000000u, 0x7c010000u, 0x00000000u, 0xc0008000u, 0xc000c000u, 0xc0008000u, 0xc0007c01u, 0x7c010000u, 0x00008000u, 0x7c018400u, 0x7c018400u, 0x7c007c01u, 0x7c018400u, 0x7c010000u, 0x7c017c01u, 0xfc018000u, 0x7c018000u, 0x80008000u, 0x7c018000u, 0xc0008000u, 0xc000c000u, 0x40000000u, 0xc0000000u, 0x80000000u, 0xc0008000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u}, + {0x40006fe8u, 0xaa007de8u, 0x40006fe8u, 0xe7e84000u, 0x2c00f1e8u, 0xf1e82c00u, 0xb155c09du, 0xb155f79du, 0xb155c09du, 0xc09dc09du, 0xb9553c0du, 0xb955b955u, 0xb9553c0du, 0x8255730du, 0x80004900u, 0x49008000u, 0x03e00612u, 0x061203e0u, 0x03e0fe12u, 0x03e0fe12u, 0x7c00fe55u, 0xfe55fe55u, 0x3555c248u, 0x35553555u, 0x7c00fe55u, 0xfe55c248u, 0x3555c248u, 0x35557c00u, 0x7c00fe55u, 0x7c00fe55u, 0xfe557e00u, 0xfc007e55u, 0x7c00fe55u, 0x7c00c19du, 0x7c00fe55u, 0x7c00fe55u, 0xfe55fe55u, 0x7c007e55u, 0x3555c248u, 0x35553555u, 0xfc00c248u, 0x7c00c248u, 0x7c003555u, 0x7c004248u, 0x35550000u, 0x3c000000u, 0x3c000000u, 0x3c003c00u}, + {0x755fffffu, 0x5e480e33u, 0x755fffffu, 0x4e341e47u, 0x7960c3ffu, 0xc3ff7960u, 0x30143c00u, 0x3014edcbu, 0x30143c00u, 0x3c003c00u, 0xab843c02u, 0xab84ab84u, 0xab843c02u, 0xf9b9edcdu, 0x8000fffeu, 0xf0008490u, 0x00047fffu, 0x7fff0004u, 0x0004ffffu, 0x0004ffffu, 0x42483c01u, 0x3c013c01u, 0xedccffffu, 0xedccedccu, 0xedccffffu, 0x4248ffffu, 0x42483c01u, 0xedcc3c01u, 0xfc00ffffu, 0xfc00ffffu, 0xfffff48du, 0xf9b8ffffu, 0xedcbffffu, 0xedcbffffu, 0x6dcd7fffu, 0xf48dffffu, 0xffffffffu, 0xf48d7fffu, 0xedcc3c01u, 0xedccedccu, 0xedccbc01u, 0x42483c01u, 0x4248edccu, 0x6dcc3c01u, 0x00003c00u, 0x3c000000u, 0x3c003c00u, 0x3c003c00u}, + {0x8132edccu, 0x70a9fe55u, 0x8132edccu, 0xedcc8132u, 0x82dd0000u, 0x000082ddu, 0x7d227d23u, 0x7d227d23u, 0x7d227d23u, 0x7d237d23u, 0x7d247d23u, 0x7d247d24u, 0x7d247d23u, 0x7d247d23u, 0xfff80000u, 0x0000fff8u, 0x1fff0000u, 0x00001fffu, 0xffff0000u, 0xffff0000u, 0x7d237d23u, 0x7d237d23u, 0xffff0000u, 0xffffffffu, 0xffff7d23u, 0xffff7d23u, 0x7d230000u, 0x00007d23u, 0x7d237d23u, 0x7d237d23u, 0x7d237d23u, 0x7d23fd23u, 0x7d237d23u, 0x7d237d23u, 0xfd237d23u, 0x7d237d23u, 0x7d237d23u, 0x7d23fd23u, 0x7d230000u, 0x7d237d23u, 0x7d230000u, 0x7d230000u, 0x7d230000u, 0xfd238000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u}, + {0x00008000u, 0x80002000u, 0x00008000u, 0xa0000000u, 0x00000000u, 0x00000000u, 0xc2480400u, 0xc248c648u, 0xc2480400u, 0x04000400u, 0xc248fc00u, 0xc248c248u, 0xc248fc00u, 0x0000be48u, 0x00000400u, 0x00000000u, 0x00000400u, 0x04000000u, 0x00000400u, 0x00000004u, 0x00000400u, 0x04000400u, 0xc2480000u, 0xc248c248u, 0xc2480400u, 0x0000c248u, 0x00000000u, 0x04000000u, 0x00000000u, 0x00000000u, 0x00008000u, 0x00008000u, 0xc2480400u, 0xc248c248u, 0x42488400u, 0x80000000u, 0x00000000u, 0x80000000u, 0xc2480000u, 0xc248c248u, 0xc2488000u, 0x00000400u, 0x00000400u, 0x42480000u, 0x00000000u, 0x00000000u, 0x00000400u, 0x3c000000u}, + {0xcaab5400u, 0x9634ac00u, 0xcaab5400u, 0x30001234u, 0xcaabd000u, 0xd000caabu, 0x3554e9ccu, 0x35543155u, 0x3554e9ccu, 0xe9cce9ccu, 0x3556f1ccu, 0x35563556u, 0x3556f1ccu, 0xedcd3955u, 0xffe00000u, 0x8000f000u, 0x07ff000fu, 0x000f07ffu, 0xffffffffu, 0xffffffe0u, 0x3555fc00u, 0xfc00fc00u, 0xffffedccu, 0xffffffffu, 0xfffffc00u, 0xfffffc00u, 0x3555edccu, 0x3555edccu, 0xffff7c00u, 0xffff7c00u, 0x7c00ffffu, 0x7ffffc00u, 0xfffffc00u, 0xfffffc00u, 0xffff7c00u, 0xffff7c00u, 0x7c007c00u, 0xffff7c00u, 0x3555fc00u, 0x35553555u, 0x3555fc00u, 0x3555edccu, 0xedcc3555u, 0xb5557c00u, 0x35550000u, 0x00000000u, 0x35550000u, 0x00003c00u}, + {0x8400c000u, 0xa11c8400u, 0x8400c000u, 0xc000651cu, 0x00000000u, 0x00000000u, 0x7d231234u, 0x7d230000u, 0x7d231234u, 0x12341234u, 0x82dd1234u, 0x82dd82ddu, 0x82dd1234u, 0x95110000u, 0x7d230000u, 0x0000d230u, 0x7d230000u, 0x00007d23u, 0x7d230000u, 0x7d230000u, 0x7d231234u, 0x12341234u, 0x00000000u, 0x00000000u, 0x7d231234u, 0x7d230000u, 0x00000000u, 0x00001234u, 0x7d23c000u, 0x7d23c000u, 0x84007d23u, 0xfd23c000u, 0x7d231234u, 0x7d230000u, 0x7d231234u, 0x7d230000u, 0x00000000u, 0x7d230000u, 0x00000000u, 0x00000000u, 0x00009234u, 0x00001234u, 0x12340000u, 0x80001234u, 0x00000000u, 0x12340000u, 0x00001234u, 0x00001234u}, + {0x7d557e55u, 0x7e557d55u, 0x7d557e55u, 0x7e557d55u, 0x00000000u, 0x00000000u, 0xfc007c00u, 0xfc000000u, 0xfc007c00u, 0x7c007c00u, 0x04007c00u, 0x04000400u, 0x04007c00u, 0x80000000u, 0xfc000000u, 0x0000fc00u, 0xfc000000u, 0x0000fc00u, 0xfc000000u, 0xfc000000u, 0x00007c00u, 0x7c007c00u, 0xfc000000u, 0xfc00fc00u, 0xfc007c00u, 0xfc000000u, 0x00000000u, 0x00007c00u, 0xfe00fe00u, 0xfe00fe00u, 0xfe00fe00u, 0xfe00fe00u, 0xfc007c00u, 0xfc000000u, 0xfc007c00u, 0xfe00fe00u, 0xfe00fe00u, 0xfe00fe00u, 0xfc000000u, 0xfc00fc00u, 0x0000fc00u, 0x00007c00u, 0x7c000000u, 0x80007c00u, 0x00000000u, 0x00000000u, 0x00003c00u, 0x00003c00u}, + {0xfe557d55u, 0x7d55fe55u, 0xfe557d55u, 0x7d55fe55u, 0x00000000u, 0x00000000u, 0x7c007c00u, 0x7c00f800u, 0x7c007c00u, 0x7c007c00u, 0x7c008400u, 0x7c007c00u, 0x7c008400u, 0x00000000u, 0x00007c00u, 0x7c000000u, 0x00007c00u, 0x7c000000u, 0x00007c00u, 0x00007c00u, 0x7c007c00u, 0x7c007c00u, 0x00000000u, 0x00000000u, 0x7c007c00u, 0x00007c00u, 0x00000000u, 0x7c000000u, 0xfe00fe00u, 0xfe00fe00u, 0xfe00fe00u, 0xfe00fe00u, 0x7c007c00u, 0x7c007c00u, 0xfc00fc00u, 0xfe00fe00u, 0xfe00fe00u, 0xfe00fe00u, 0x00000000u, 0x00000000u, 0x80008000u, 0x7c007c00u, 0x00007c00u, 0x00000000u, 0x00000000u, 0x00003c00u, 0x3c003c00u, 0x00000000u}, }; -constexpr const char* Names[44] = { +constexpr const char* Names[48] = { "v_pk_mad_i16 v4, v5, v6", "v_pk_mad_i16 v4, v5, v6 op_sel:[1,0,1] op_sel_hi:[0,1,0]", "v_pk_mad_u16 v4, v5, v6", @@ -188,6 +189,10 @@ constexpr const char* Names[44] = { "v_pk_max_f16 v4, v5", "v_pk_max_f16 v4, v5 op_sel:[1,0] op_sel_hi:[0,1]", "v_pk_max_f16 v4, v5 neg_lo:[0,1] neg_hi:[1,0]", + "v_pk_min_f16 v4, v5 clamp", + "v_pk_min_f16 v4, v5 op_sel:[1,0] op_sel_hi:[0,1] clamp", + "v_pk_max_f16 v4, v5 clamp", + "v_pk_max_f16 v4, v5 neg_lo:[0,1] neg_hi:[1,0] clamp", }; bool Ieee = false; @@ -241,7 +246,7 @@ void Check() { for (std::uint32_t tid = 0; tid < Threads; ++tid) { const std::uint32_t* in = &Input[tid * Inputs]; const std::uint32_t* out = &Output[tid * Results]; - for (std::uint32_t i = 0; i < 44; ++i) { + for (std::uint32_t i = 0; i < 48; ++i) { Expect(tid, out[i], (Ieee ? Expected : ExpectedIeeeOff)[tid][i], Names[i]); } } diff --git a/core/libs/prx/libSceAgcDriver/tests/execution/PackedUnormColorTarget.cpp b/core/libs/prx/libSceAgcDriver/tests/execution/PackedUnormColorTarget.cpp new file mode 100644 index 000000000..31bf88c82 --- /dev/null +++ b/core/libs/prx/libSceAgcDriver/tests/execution/PackedUnormColorTarget.cpp @@ -0,0 +1,326 @@ +#include "prx/libSceAgcDriver/Execution/include/GuestMemory.hpp" +#include "prx/libSceAgcDriver/Execution/include/VulkanDevice.hpp" +#include "prx/libSceAgcDriver/Execution/include/Driver/Shaders/ShaderRegistry.hpp" +#include "prx/libSceAgcDriver/Graphics/include/ColorTargetLayout.hpp" +#include "prx/libSceAgcDriver/Graphics/include/Draw.hpp" +#include "prx/libSceAgcDriver/Graphics/include/ShaderInputState.hpp" +#include "prx/libSceAgcDriver/Graphics/include/State.hpp" +#include "prx/libSceAgcDriver/Graphics/include/Texture.hpp" +#include "prx/libc/include/GuestAllocations.hpp" +#include "prx/libc/include/GuestArena.hpp" +#include "prx/libc/include/GuestWriteWatch.hpp" +#include "Recompiler.hpp" +#include "Optimization/ResourceProgram.hpp" +#include "VulkanTestDevice.hpp" +#ifdef _WIN32 +#ifndef NOMINMAX +#define NOMINMAX +#endif +#include +#else +#include +#endif +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace { + +using AgcDriver::Graphics::ColorTargetLayout; +using AgcDriver::Graphics::ColorTileMode; +using AgcDriver::Graphics::Require; +using ShaderRecompiler::ShaderStage; + +constexpr std::uint32_t Width = 256; +constexpr std::uint32_t Height = 128; +constexpr std::uint32_t ScissorWidth = 128; +constexpr std::size_t BlockBytes = 65536; +constexpr std::uint32_t Unorm10_11_11Info = 0x8018u; +constexpr std::uint32_t TiledAttrib3 = 0x4dc6c000u; +constexpr std::uint32_t LinearAttrib3 = 0x09000000u; +constexpr std::uint32_t Float32Export = 9; +constexpr std::uint32_t Float16Export = 4; + +alignas(256) constexpr std::array VertexCode{ + 0xe0382000, 0x80000005, 0xbf8c3f70, 0xf80008cf, 0x03020100, 0xbf810000, +}; + +constexpr std::array, 3> Triangle{{ + {-1.0f, -1.0f, 0.5f, 1.0f}, {3.0f, -1.0f, 0.5f, 1.0f}, {-1.0f, 3.0f, 0.5f, 1.0f} +}}; + +alignas(256) std::array, 32> Programs{}; +std::size_t programCount = 0; + +std::uint32_t Unorm(float value, std::uint32_t bits) { + const auto maximum = (1u << bits) - 1u; + if (!(value > 0.0f)) return 0; + if (value >= 1.0f) return maximum; + return static_cast(std::nearbyint(static_cast(value) * maximum)); +} + +std::uint32_t Packed(std::array rgb) { + return Unorm(rgb[0], 11) | (Unorm(rgb[1], 11) << 11u) | (Unorm(rgb[2], 10) << 22u); +} + +std::uint32_t Half(float value) { + Require(value == 0.0f || (value >= 0.0625f && value <= 1.0f), "the test's half values are normal and at most 1"); + if (value == 0.0f) return 0; + const auto bits = std::bit_cast(value); + Require((bits & 0x1fffu) == 0, "the test's half values are exact"); + return ((((bits >> 23u) & 0xffu) - 112u) << 10u) | ((bits >> 13u) & 0x3ffu); +} + +std::span Program(const std::array& vgprs, bool compressed) { + Require(programCount < Programs.size(), "too many test programs"); + auto& code = Programs[programCount++]; + std::size_t at = 0; + for (std::uint32_t vgpr = 0; vgpr < (compressed ? 2u : 4u); ++vgpr) { + code[at++] = 0x7e0002ffu | ((4u + vgpr) << 17u); + code[at++] = vgprs[vgpr]; + } + code[at++] = compressed ? 0xf8001c0fu : 0xf800180fu; + code[at++] = compressed ? 0x00000504u : 0x07060504u; + code[at++] = 0xbf810000u; + return std::span(code.data(), at); +} + +std::span Float32Program(std::array rgb) { + return Program({std::bit_cast(rgb[0]), std::bit_cast(rgb[1]), std::bit_cast(rgb[2]), std::bit_cast(1.0f)}, false); +} + +std::span Float16Program(std::array rgb) { + return Program({Half(rgb[0]) | (Half(rgb[1]) << 16u), Half(rgb[2]) | (Half(1.0f) << 16u), 0, 0}, true); +} + +struct Block { + explicit Block(std::size_t bytes) : bytes(bytes), watched(AgcDriver::GuestMemory::WriteWatched()) { +#ifdef _WIN32 + if (watched) { + data = static_cast(GuestArena::GuestArenaAllocate_nid_postfix(bytes, BlockBytes)); + GuestArena::GuestArenaCommit_nid_postfix(data, bytes, PAGE_READWRITE, bytes); + } else { + data = static_cast(VirtualAlloc(nullptr, bytes, MEM_RESERVE | MEM_COMMIT, PAGE_READWRITE)); + } +#else + if (watched) { + void* raw = mmap(nullptr, bytes + BlockBytes, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + Require(raw != MAP_FAILED, "cannot map the color target"); + const auto begin = reinterpret_cast(raw); + const auto aligned = (begin + BlockBytes - 1) & ~(static_cast(BlockBytes) - 1); + if (aligned != begin) munmap(raw, aligned - begin); + if (aligned + bytes != begin + bytes + BlockBytes) munmap(reinterpret_cast(aligned + bytes), begin + BlockBytes - aligned); + data = reinterpret_cast(aligned); + GuestWriteWatch::GuestWriteWatchRegister_nid_postfix(data, bytes); + } else { + data = static_cast(std::aligned_alloc(BlockBytes, bytes)); + } +#endif + Require(data != nullptr, "cannot allocate the color target"); + Require(!watched || AgcDriver::GuestMemory::Watched(Address(), bytes), "the color target is not write-watched"); + GuestAllocations::Mutation mutation; + mutation.Add(data, bytes, true, true, true); + } + ~Block() { + AgcDriver::Graphics::StorageTexture::FlushPending(Address(), bytes, nullptr, "test release"); + { + GuestAllocations::Mutation mutation; + mutation.Remove(data); + } +#ifdef _WIN32 + if (watched) { + GuestArena::GuestArenaReset_nid_postfix(data, bytes); + GuestArena::GuestArenaRelease_nid_postfix(data, bytes); + } else { + VirtualFree(data, 0, MEM_RELEASE); + } +#else + if (watched) { + munmap(data, bytes); + GuestWriteWatch::GuestWriteWatchUnregister_nid_postfix(data, bytes); + } else { + std::free(data); + } +#endif + } + Block(const Block&) = delete; + Block& operator=(const Block&) = delete; + std::uint64_t Address() const { return reinterpret_cast(data); } + std::size_t bytes; + bool watched; + std::uint8_t* data = nullptr; +}; + +AgcDriver::Graphics::ColorTarget DecodeTarget(const Block& block, std::uint32_t attrib3) { + AgcDriver::Registers cx; + cx[0x318] = static_cast(block.Address() >> 8u); + cx[0x31b] = 0; + cx[0x31c] = Unorm10_11_11Info; + cx[0x31d] = 0; + cx[0x3b0] = ((Width - 1u) << 14u) | (Height - 1u); + cx[0x3b8] = attrib3; + cx[0x390] = static_cast(block.Address() >> 40u); + auto color = AgcDriver::Graphics::DecodeColorBuffer(cx, 0); + Require(color.format == VK_FORMAT_R32_UINT && color.elementBytes == 4 && color.packing == ShaderRecompiler::ColorExportPacking::Unorm10_11_11 && color.bytes == block.bytes, "the 10_11_11 unorm target did not decode as a packed R32_UINT surface"); + return color; +} + +void Draw(AgcDriver::VulkanDevice& device, const AgcDriver::Graphics::ColorTarget& color, std::uint32_t exportFormat, std::span pixelCode) { + const auto target = device.Target(); + constexpr std::uint32_t waveSize = 64; + std::vector vertexUserData(4, 0u); + const auto address = reinterpret_cast(Triangle.data()); + const std::array vertexBuffer{static_cast(address), static_cast((address >> 32u) & 0xffffu) | (16u << 16u), static_cast(Triangle.size()), 0x01016facu}; + std::copy(vertexBuffer.begin(), vertexBuffer.end(), vertexUserData.begin()); + const std::array vertexMemory{{{reinterpret_cast(VertexCode.data()), std::as_bytes(std::span(VertexCode))}}}; + ShaderRecompiler::RecompileRequest vertex{ + {ShaderStage::Vertex, reinterpret_cast(VertexCode.data()), VertexCode, 0, {}}, + {waveSize, 0, vertexUserData, std::nullopt, std::nullopt, ShaderRecompiler::ShaderVertexStageInfo{}, vertexMemory}, + target, + {0, 0, 0, 64} + }; + vertex.useCache = false; + const auto vertexResult = ShaderRecompiler::Recompile(vertex); + const auto vertexPush = static_cast(vertexResult.pushConstants.size()); + + AgcDriver::Graphics::State state{}; + state.stages = {AgcDriver::Graphics::ShaderPath::Vertex, 0u, waveSize, waveSize, std::nullopt, std::nullopt}; + state.colors = {color}; + state.color = color; + state.hasColorTarget = true; + state.renderExtent = {Width, Height}; + state.topology = VK_PRIMITIVE_TOPOLOGY_TRIANGLE_LIST; + state.viewport = {0, static_cast(Height), static_cast(Width), -static_cast(Height), 0, 1}; + state.negativeOneToOne = false; + state.scissor = {{0, 0}, {ScissorWidth, Height}}; + state.cullMode = VK_CULL_MODE_NONE; + state.frontFace = VK_FRONT_FACE_COUNTER_CLOCKWISE; + VkPipelineColorBlendAttachmentState blend{}; + blend.colorWriteMask = VK_COLOR_COMPONENT_R_BIT | VK_COLOR_COMPONENT_G_BIT | VK_COLOR_COMPONENT_B_BIT; + state.blends = {blend}; + state.blend = blend; + state.blendConstants = {}; + + const auto codeAddress = reinterpret_cast(pixelCode.data()); + const AgcDriver::Registers context{{0x1b6, 0x8000}, {0x1b3, 0}, {0x1b4, 0}, {0x203, 0}, {0x1c5, exportFormat}}; + const std::array pixelMemory{{{codeAddress, std::as_bytes(pixelCode)}}}; + ShaderRecompiler::RecompileRequest fragment{ + {ShaderStage::Fragment, codeAddress, pixelCode, 0, {}}, + {waveSize, 0, {}, std::nullopt, AgcDriver::Graphics::DecodePixelStageInfo(context, {}), std::nullopt, pixelMemory}, + target, + PixelPushLayout(vertexPush, target) + }; + const auto handle = ShaderRecompiler::PrepareShader(fragment); + AgcDriver::DriverDetail::ShaderSnapshot snapshot{codeAddress, 0, 1, {pixelCode.begin(), pixelCode.end()}, {}}; + snapshot.prepared->entries.push_back({0, handle}); + fragment.shader.code = snapshot.code; + fragment.context.pixel = AgcDriver::Graphics::DecodePixelStageInfo(context, AgcDriver::Graphics::ExportMappings(state)); + fragment.context.pixel->targetExportPacking = AgcDriver::Graphics::ExportPackings(state); + const ShaderRecompiler::SrtRuntime runtime{}; + const auto materialize = [&](const ShaderRecompiler::RecompileRequest& request) { + const auto invocation = AgcDriver::DriverDetail::InvocationFor(snapshot, 0, request); + return invocation.Materialize(*invocation.Capture(runtime)); + }; + const auto pixelResult = materialize(fragment); + auto unpacked = fragment; + unpacked.context.pixel->targetExportPacking.fill(ShaderRecompiler::ColorExportPacking::None); + const auto plain = materialize(unpacked); + Require(pixelResult->variantId == ShaderRecompiler::GetPreparedArtifact(*handle).variantId && plain->variantId == pixelResult->variantId, "the draw did not use the registered pixel shader"); + Require(plain->PipelineVariantId() != pixelResult->PipelineVariantId() && materialize(fragment)->PipelineVariantId() == pixelResult->PipelineVariantId(), "the 10_11_11 unorm packing did not key its own specialization"); + const std::array shaders{{ + {ShaderStage::Vertex, &vertexResult, 0}, + {ShaderStage::Fragment, pixelResult.get(), PixelPushOffset(vertexPush, target)} + }}; + const AgcDriver::Pm4::DrawParameters draw{0, static_cast(Triangle.size()), 0, 1, 0, false}; + device.Draw(state, draw, shaders); +} + +std::vector ReadBack(AgcDriver::VulkanDevice& device, const Block& block) { + AgcDriver::Graphics::StorageTexture::FlushPending(block.Address(), block.bytes, nullptr, "test read-back"); + device.WaitIdle(); + std::vector stored(block.bytes / 4u); + AgcDriver::GuestMemory::Read(block.Address(), std::as_writable_bytes(std::span(stored)), 1); + return stored; +} + +void ExpectSurface(AgcDriver::VulkanDevice& device, const Block& block, const ColorTargetLayout& layout, const std::vector& before, std::uint32_t word, const std::string& what) { + const auto stored = ReadBack(device, block); + for (std::uint32_t y = 0; y < Height; ++y) { + for (std::uint32_t x = 0; x < Width; ++x) { + const auto index = layout.Offset(x, y) / 4u; + const bool inside = x < ScissorWidth; + const auto expected = inside ? word : before[index]; + if (stored[index] == expected) continue; + char detail[160]; + std::snprintf(detail, sizeof(detail), ": pixel (%u, %u) %s the scissor holds 0x%08x, expected 0x%08x", x, y, inside ? "inside" : "outside", stored[index], expected); + throw std::runtime_error(what + detail); + } + } +} + +std::vector Fill(Block& block) { + std::vector words(block.bytes / 4u); + for (std::size_t index = 0; index < words.size(); ++index) words[index] = static_cast(index * 0x9e3779b9u); + std::memcpy(block.data, words.data(), block.bytes); + return words; +} + +std::string Describe(std::array rgb) { + char text[96]; + std::snprintf(text, sizeof(text), "(%.9g, %.9g, %.9g)", rgb[0], rgb[1], rgb[2]); + return text; +} + +void TargetTests(AgcDriver::VulkanDevice& device, Block& block, std::uint32_t attrib3, ColorTileMode mode, const char* kind) { + const ColorTargetLayout layout(Width, Height, mode, 4); + Require(layout.Bytes() == block.bytes, "the test surface size disagrees with its layout"); + const auto color = DecodeTarget(block, attrib3); + auto before = Fill(block); + const float nan = std::numeric_limits::quiet_NaN(); + const float infinity = std::numeric_limits::infinity(); + const std::array, 8> cases{{ + {0.0f, 1.0f, 0.5f}, + {0.5f, 0.25f, 1.0f / 3.0f}, + {nan, -1.0f, 2.0f}, + {infinity, -0.0f, std::numeric_limits::denorm_min()}, + {0.5f / 2047.0f, 1.5f / 2047.0f, 0.5f / 1023.0f}, + {2046.5f / 2047.0f, 1023.5f / 2047.0f, 1022.5f / 1023.0f}, + {-infinity, std::numeric_limits::min(), 0.999999940f}, + {0.1f, 0.7f, 0.9f}, + }}; + for (const auto& rgb : cases) { + Draw(device, color, Float32Export, Float32Program(rgb)); + ExpectSurface(device, block, layout, before, Packed(rgb), std::string("a 32_ABGR export of ") + Describe(rgb) + " into a " + kind + " 10_11_11 unorm target"); + before = ReadBack(device, block); + } + const std::array halves{0.5f, 0.75f, 0.125f}; + Draw(device, color, Float16Export, Float16Program(halves)); + ExpectSurface(device, block, layout, before, Packed(halves), std::string("an FP16_ABGR export of ") + Describe(halves) + " into a " + kind + " 10_11_11 unorm target"); +} + +} + +int main() { + try { + const auto device = OpenVulkanTestDevice(); + if (!device) return VulkanTestSkipped; + Block tiled(ColorTargetLayout(Width, Height, ColorTileMode::RenderTarget, 4).Bytes()); + Block linear(ColorTargetLayout(Width, Height, ColorTileMode::Linear, 4).Bytes()); + TargetTests(*device, tiled, TiledAttrib3, ColorTileMode::RenderTarget, "tiled"); + TargetTests(*device, linear, LinearAttrib3, ColorTileMode::Linear, "linear"); + std::puts("10_11_11 unorm color target tests passed"); + return 0; + } catch (const std::exception& error) { + std::cerr << error.what() << '\n'; + return 1; + } +} diff --git a/core/libs/prx/libSceAgcDriver/tests/execution/PatternBufferFill.cpp b/core/libs/prx/libSceAgcDriver/tests/execution/PatternBufferFill.cpp new file mode 100644 index 000000000..9e0872423 --- /dev/null +++ b/core/libs/prx/libSceAgcDriver/tests/execution/PatternBufferFill.cpp @@ -0,0 +1,208 @@ +#include "prx/libSceAgcDriver/Execution/include/Driver.hpp" +#include "prx/libSceAgcDriver/Execution/include/GuestMemory.hpp" +#include "prx/libSceAgcDriver/Execution/include/Driver/Memory/BufferFill.hpp" +#include "prx/libSceAgcDriver/Graphics/include/Draw.hpp" +#include "VulkanTestDevice.hpp" +#include +#include +#include +#include +#include + +namespace { + +using AgcDriver::Graphics::Require; +using namespace AgcDriver::DriverDetail; +alignas(256) constexpr auto Code = PatternFillKernel; +alignas(256) std::array Pattern{0, 0x12345678u, 0xdeadbeefu, 0x7fc12345u}; +alignas(256) std::array Control{}; +alignas(256) std::array Output{}; + +std::array UserData(void* output, std::uint32_t records) { + std::array data{}; + const auto descriptor = [&](std::size_t first, const void* pointer, std::uint32_t stride, std::uint32_t count, std::uint32_t format) { + const auto address = reinterpret_cast(pointer); + data[first] = static_cast(address); + data[first + 1] = static_cast(address >> 32u) | (stride << 16u); + data[first + 2] = count; + data[first + 3] = format; + }; + descriptor(0, Pattern.data(), 4, Pattern.size(), 0x14004u); + descriptor(4, output, 4, records, 0x14004u); + descriptor(8, Control.data(), 16, 1, 0x4dfacu); + return data; +} + +void RangeTests() { + const ShaderRecompiler::ShaderComputeStageInfo compute{{64, 1, 1}, 0, {true, false, false}, false, 1}; + const auto data = UserData(Output.data(), 256); + Require(MatchesPatternFillKernel(Code, data, compute), "pattern fill kernel was not recognized"); + for (std::size_t word = 0; word < Code.size(); ++word) { + auto altered = Code; + altered[word] ^= 1; + Require(!MatchesPatternFillKernel(altered, data, compute), "a different shader matched the fill kernel"); + } + Require(!MatchesPatternFillKernel(std::span(Code).first(17), data, compute), "truncated fill shader matched"); + for (const auto word : {1u, 3u, 5u, 7u, 9u, 11u}) { + auto altered = data; + altered[word] ^= word % 4 == 1 ? 0x80000000u : 0x1000u; + Require(!MatchesPatternFillKernel(Code, altered, compute), "unsupported fill descriptor matched"); + } + auto disabled = compute; + disabled.groupIdEnable[0] = false; + Require(!MatchesPatternFillKernel(Code, data, disabled), "fill with no group id matched"); + disabled = compute; + disabled.numThreads[0] = 32; + Require(!MatchesPatternFillKernel(Code, data, disabled), "different fill group size matched"); + std::array packet{0xc0031500u, 2, 1, 1, 0x41}; + auto range = DecodePatternFillRange(data, packet); + Require(range && range->UniformBytes(256, 0) == 512, "fill exceeded launched threads"); + Require(range->UniformBytes(80, 0) == 320, "fill exceeded its control count"); + Require(!range->UniformBytes(80, 1), "nonuniform pattern matched a uniform fill"); + Require(!range->UniformBytes(79, 0), "partial 16-byte fill matched"); + Require(range->UniformBytes(0, 0) == 0, "empty control count was not empty"); + packet[1] = 68; + packet[4] |= 0x20; + range = DecodePatternFillRange(data, packet); + Require(range && range->UniformBytes(256, 0) == 272, "partial workgroup was rounded up"); + range->records = 64; + Require(range->UniformBytes(256, 0) == 256, "fill exceeded output descriptor"); + range->source = range->destination + 16; + Require(!range->UniformBytes(256, 0), "aliased pattern was accepted"); + range->source = 0x1000; + range->control = range->destination - 4; + Require(!range->UniformBytes(256, 0), "aliased control was accepted"); + packet[1] = 0; + Require(DecodePatternFillRange(data, packet)->invocations == 0, "empty dispatch gained invocations"); + packet[1] = 0x4000001u; + packet[4] = 0x41; + Require(!DecodePatternFillRange(data, packet), "wrapped thread indices were accepted"); + packet[1] = 1; + packet[2] = 2; + Require(!DecodePatternFillRange(data, packet), "multidimensional fill was accepted"); +} + +std::vector Commands(std::span data, std::uint32_t threads, std::uint32_t wave, bool partial, bool indirect) { + std::vector words; + const auto registers = [&](std::uint32_t first, std::span values) { + words.push_back(0xc0007600u | (static_cast(values.size()) << 16u)); + words.push_back(first); + words.insert(words.end(), values.begin(), values.end()); + }; + const std::array shape{64, 1, 1}; + registers(0x207, shape); + const auto address = reinterpret_cast(Code.data()); + const std::array program{static_cast(address >> 8u), static_cast(address >> 40u)}; + registers(0x20c, program); + const std::array resource{(12u << 1u) | (1u << 7u)}; + registers(0x213, resource); + registers(0x240, data); + const auto initiator = 0x41u | (wave == 32 ? 0x8000u : 0u) | (partial ? 0x20u : 0u); + if (indirect) { + static std::array arguments; + arguments = {threads / 64, 1, 1}; + const auto pointer = reinterpret_cast(arguments.data()); + words.insert(words.end(), {0xc0021600u, static_cast(pointer), static_cast(pointer >> 32u), initiator}); + } else { + words.insert(words.end(), {0xc0031500u, partial ? threads : threads / 64, 1, 1, initiator}); + } + return words; +} + +void Submit(const std::vector& words, void* output = Output.data(), std::size_t bytes = sizeof(Output)) { + Packet packet{const_cast(words.data()), static_cast(words.size()), 0, {}}; + AgcDriver::Submit(&packet, 0); + AgcDriverWaitIdle_nid_postfix(); + AgcDriver::GuestMemory::FlushGpuWrites(reinterpret_cast(output), bytes); +} + +void ExecuteTests() { + struct Case { std::uint32_t pattern, count, records, threads, mask; bool partial, indirect; }; + constexpr std::array cases{{ + {0, 256, 256, 256, 0, false, false}, + {0xffffffffu, 256, 256, 128, 0, false, false}, + {0x7fc12345u, 80, 256, 256, 0, false, false}, + {0x80000000u, 256, 64, 256, 0, false, false}, + {0xdeadbeefu, 256, 256, 68, 0, true, false}, + {0x3f800000u, 0, 256, 64, 0, false, false}, + {0x80808080u, 256, 256, 0, 0, false, false}, + {0xffffffffu, 1, 256, 64, 0, false, false}, + {0, 256, 7, 64, 0, false, false}, + {0x01010101u, 128, 256, 128, 0, false, true}, + {0, 128, 256, 128, 3, false, false} + }}; + for (const auto wave : {32u, 64u}) { + for (const auto& item : cases) { + Output.fill(0xcafebabeu); + Pattern[0] = item.pattern; + Control = {item.count, item.mask, 0, 0}; + const auto words = Commands(UserData(Output.data(), item.records), item.threads, wave, item.partial, item.indirect); + Submit(words); + const auto count = std::min({item.count, item.records, item.threads}); + for (std::size_t i = 0; i < Output.size(); ++i) { + const auto expected = i < count ? Pattern[i & item.mask] : 0xcafebabeu; + Require(Output[i] == expected, "pattern fill mismatch at " + std::to_string(i) + " wave " + std::to_string(wave) + " count " + std::to_string(item.count) + " got " + std::to_string(Output[i]) + " expected " + std::to_string(expected)); + } + } + } + + Output.fill(0xcafebabeu); + Pattern[0] = 0x40404040u; + const auto firstPattern = Pattern; + Control = {256, 3, 0, 0}; + auto words = Commands(UserData(Output.data(), 256), 256, 32, false, false); + const auto label = [&](std::uint32_t* target, std::uint32_t value) { + const auto address = reinterpret_cast(target); + words.insert(words.end(), {0xc0064900u, 0x514u, (1u << 29u) | (2u << 24u), static_cast(address), static_cast(address >> 32u), value, 0, 0}); + }; + label(&Control[0], 128); + label(&Control[1], 0); + label(&Pattern[0], 0xababababu); + const auto next = Commands(UserData(Output.data(), 256), 256, 32, false, false); + words.insert(words.end(), next.begin(), next.end()); + Submit(words); + for (std::size_t i = 0; i < Output.size(); ++i) { + const auto expected = i < 128 ? 0xababababu : i < 256 ? firstPattern[i & 3u] : 0xcafebabeu; + Require(Output[i] == expected, "fill did not observe preceding GPU results and labels at " + std::to_string(i)); + } +} + +void Benchmark() { + constexpr std::size_t bytes = 64u << 20u; + std::vector storage(bytes / 4 + 4); + auto* output = reinterpret_cast((reinterpret_cast(storage.data()) + 15u) & ~std::uintptr_t{15}); + Pattern[0] = 0xffffffffu; + Control = {bytes / 4, 0, 0, 0}; + const auto words = Commands(UserData(output, bytes / 4), bytes / 4, 32, false, false); + Submit(words, output, bytes); + std::array times; + for (auto& ms : times) { + const auto start = std::chrono::steady_clock::now(); + Submit(words, output, bytes); + ms = std::chrono::duration(std::chrono::steady_clock::now() - start).count(); + Require(std::all_of(output, output + bytes / 4, [](auto value) { return value == 0xffffffffu; }), "benchmark fill was incomplete"); + } + std::sort(times.begin(), times.end()); + std::printf("64 MiB fill median %.3f ms, min %.3f, max %.3f\n", times[4], times.front(), times.back()); +} + +} + +int main(int argc, char** argv) { + try { + RangeTests(); + { + const auto device = OpenVulkanTestDevice(); + if (!device) return VulkanTestSkipped; + } + if (argc == 2 && std::string_view(argv[1]) == "--benchmark") Benchmark(); + else ExecuteTests(); + AgcDriverShutdown_nid_postfix(); + std::puts("pattern buffer fill tests passed"); + return 0; + } catch (const std::exception& error) { + std::cerr << error.what() << '\n'; + AgcDriverShutdown_nid_postfix(); + return 1; + } +} diff --git a/core/libs/prx/libSceAgcDriver/tests/execution/PendingExactSurface.cpp b/core/libs/prx/libSceAgcDriver/tests/execution/PendingExactSurface.cpp new file mode 100644 index 000000000..9386584c9 --- /dev/null +++ b/core/libs/prx/libSceAgcDriver/tests/execution/PendingExactSurface.cpp @@ -0,0 +1,130 @@ +#include "prx/libSceAgcDriver/Execution/include/GuestMemory.hpp" +#include "prx/libc/include/GuestAllocations.hpp" +#include "prx/libSceAgcDriver/Execution/include/VulkanDevice.hpp" +#include "prx/libSceAgcDriver/Graphics/include/GuestTextureResource.hpp" +#include "prx/libSceAgcDriver/Graphics/include/ShaderResources.hpp" +#include "prx/libSceAgcDriver/Graphics/include/Texture.hpp" +#include "prx/libSceAgcDriver/Graphics/include/TextureTiling.hpp" +#include "Recompiler.hpp" +#include "VulkanTestDevice.hpp" +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace { + +using AgcDriver::Graphics::Require; +using AgcDriver::Graphics::StorageTexture; +using ShaderRecompiler::ShaderStage; + +constexpr std::uint32_t Threads = 8; +constexpr std::uint32_t Side = 256; +constexpr std::uint32_t Format32UInt = 20; +constexpr std::uint32_t TileR64KBX = 0x1b; +constexpr std::uint32_t Type2D = 9; +constexpr std::size_t BlockBytes = 0x60000; +constexpr std::uint32_t Untouched = 0xdeadbeefu; + +alignas(256) constexpr std::array StoreCode{ + 0x7e040300, 0x7e060280, 0x7e0202ff, 0x12345678u, 0xf0200108, 0x00020102, 0xbf810000, +}; + +std::uint64_t AddressOf(const void* data) { + return static_cast(reinterpret_cast(data)); +} + +std::array TextureDescriptor(const std::uint32_t* texels, std::uint32_t maxMip) { + const auto address = AddressOf(texels); + return { + static_cast(address >> 8u), + static_cast((address >> 40u) & 0xffu) | (Format32UInt << 20u) | (((Side - 1u) & 3u) << 30u), + ((Side - 1u) >> 2u) | ((Side - 1u) << 14u), + 0xfacu | (TileR64KBX << 20u) | (Type2D << 28u), + 0u, + maxMip << 4u, + 0u, + 0u, + }; +} + +std::uint64_t GuestBytes(const std::uint32_t* texels, std::uint32_t maxMip) { + const auto words = TextureDescriptor(texels, maxMip); + return AgcDriver::Graphics::DescribeSurface(AgcDriver::Graphics::DecodeTextureResource(words)).guestBytes; +} + +void Store(AgcDriver::VulkanDevice& device, const std::uint32_t* texels, std::uint32_t maxMip) { + std::vector userData(16, 0u); + const auto texture = TextureDescriptor(texels, maxMip); + std::copy(texture.begin(), texture.end(), userData.begin() + 8); + const std::span code(StoreCode); + const std::array memory{{{reinterpret_cast(code.data()), std::as_bytes(code)}}}; + const ShaderRecompiler::ShaderComputeStageInfo compute{{Threads, 1, 1}, 0u, {false, false, false}, false, 1}; + ShaderRecompiler::RecompileRequest request{ + {ShaderStage::Compute, reinterpret_cast(code.data()), code, 0, {}}, + {32, 0, userData, compute, std::nullopt, std::nullopt, memory}, + device.Target(), + {0, 0, 0, 128} + }; + request.useCache = false; + const auto result = ShaderRecompiler::Recompile(request); + device.Dispatch(result, 1, 1, 1, {}, reinterpret_cast(code.data())); + device.WaitIdle(); +} + +std::string Describe(const std::shared_ptr& image) { + if (image == nullptr) return "no pending image"; + return "the pending image of " + std::to_string(image->GuestBytes()) + " bytes and " + std::to_string(image->Descriptor().mipCount) + " mips"; +} + +} + +int main() { + try { + const auto device = OpenVulkanTestDevice(); + if (!device) return VulkanTestSkipped; + std::vector storage((BlockBytes + 0x10000u) / 4u); + auto* texels = reinterpret_cast((reinterpret_cast(storage.data()) + 0xffffu) & ~std::uintptr_t{0xffffu}); + std::fill(texels, texels + BlockBytes / 4u, Untouched); + { + GuestAllocations::Mutation mutation; + mutation.Add(texels, BlockBytes, true, true, true); + } + const auto surface = AddressOf(texels); + const auto chainBytes = GuestBytes(texels, 1u); + const auto singleBytes = GuestBytes(texels, 0u); + Require(singleBytes < chainBytes && chainBytes <= BlockBytes, "the 2-mip chain must cover more guest bytes than the 1-mip surface"); + + Store(*device, texels, 1u); + const auto chain = StorageTexture::FindPending(surface, chainBytes); + Require(chain != nullptr && chain->GuestBytes() == chainBytes, "the write through the 2-mip chain left " + Describe(chain) + " at the surface"); + const auto prefix = StorageTexture::FindPending(surface, singleBytes); + Require(prefix == chain, "a lookup of the chain's first mip found " + Describe(prefix) + " instead of the pending chain"); + + Store(*device, texels, 0u); + Require(StorageTexture::FindPending(surface, chainBytes) == chain, "the write through the 1-mip surface flushed the pending chain"); + const auto single = StorageTexture::FindPending(surface, singleBytes); + Require(single != nullptr && single != chain && single->GuestBytes() == singleBytes, "a lookup of the 1-mip surface found " + Describe(single) + " instead of the pending 1-mip image"); + + { + std::lock_guard lock(AgcDriver::GuestMemory::GpuMutex()); + AgcDriver::Graphics::FlushCachedTextures(device->Device()); + } + device->WaitIdle(); + { + GuestAllocations::Mutation mutation; + mutation.Remove(texels); + } + std::puts("pending exact surface tests passed"); + return 0; + } catch (const std::exception& error) { + std::cerr << error.what() << '\n'; + return 1; + } +} diff --git a/core/libs/prx/libSceAgcDriver/tests/execution/PixelBarycentricEmulation.cpp b/core/libs/prx/libSceAgcDriver/tests/execution/PixelBarycentricEmulation.cpp new file mode 100644 index 000000000..3a9d3178e --- /dev/null +++ b/core/libs/prx/libSceAgcDriver/tests/execution/PixelBarycentricEmulation.cpp @@ -0,0 +1,265 @@ +#include "prx/libSceAgcDriver/Execution/include/VulkanDevice.hpp" +#include "prx/libSceAgcDriver/Graphics/include/Draw.hpp" +#include "Recompiler.hpp" +#include "VulkanTestDevice.hpp" +#include +#include +#include +#include +#include +#include +#include +#include + +namespace { + +using AgcDriver::Graphics::Require; +using ShaderRecompiler::PixelInput; +using ShaderRecompiler::PixelInputBit; +using ShaderRecompiler::ShaderStage; + +constexpr std::uint32_t Width = 64; +constexpr std::uint32_t Height = 48; +constexpr float Unwritten = -7.0f; +alignas(256) std::array Pixels{}; + +alignas(256) constexpr std::array VertexCode{ + 0xe0382000, 0x80000005, 0xbf8c3f70, 0xf800020f, 0x03020100, 0xf80008cf, 0x03020100, 0xbf810000, +}; + +alignas(256) constexpr std::array BarycentricPixelCode{ + 0xf800180f, 0x01000100, 0xbf810000, +}; + +alignas(256) constexpr std::array VertexValuePixelCode{ + 0xc80a0000, 0xc80e0001, 0xc8120002, 0xf800180f, 0x04040302, 0xbf810000, +}; + +alignas(256) constexpr std::array InterpolatedPixelCode{ + 0xc8080000, 0xc8090001, 0xf800180f, 0x02020202, 0xbf810000, +}; + +struct Vertex { + float x; + float y; + float w; +}; + +constexpr std::array Corners{{{-0.8f, -0.8f, 1.0f}, {0.9f, -0.6f, 2.5f}, {-0.5f, 0.85f, 0.5f}}}; + +std::array, 3> ClipPositions() { + std::array, 3> positions{}; + for (std::size_t i = 0; i < Corners.size(); ++i) positions[i] = {Corners[i].x * Corners[i].w, Corners[i].y * Corners[i].w, 0.5f * Corners[i].w, Corners[i].w}; + return positions; +} + +std::array BufferDescriptor(const void* data, std::uint32_t stride, std::uint32_t count) { + const auto address = reinterpret_cast(data); + return {static_cast(address), static_cast((address >> 32u) & 0xffffu) | (stride << 16u), count, 0x01016facu}; +} + +ShaderRecompiler::RecompileResult Draw(AgcDriver::VulkanDevice& device, std::span pixelCode, std::uint32_t inputs) { + Pixels.fill(Unwritten); + auto target = device.Target(); + target.fragmentShaderBarycentricEnabled = false; + static const auto positions = ClipPositions(); + + std::vector vertexUserData(4, 0u); + const auto vertexBuffer = BufferDescriptor(positions.data(), 16u, static_cast(positions.size())); + std::copy(vertexBuffer.begin(), vertexBuffer.end(), vertexUserData.begin()); + const std::array vertexMemory{{{reinterpret_cast(VertexCode.data()), std::as_bytes(std::span(VertexCode))}}}; + ShaderRecompiler::RecompileRequest vertex{ + {ShaderStage::Vertex, reinterpret_cast(VertexCode.data()), VertexCode, 0, {}}, + {64u, 0, vertexUserData, std::nullopt, std::nullopt, ShaderRecompiler::ShaderVertexStageInfo{}, vertexMemory}, + target, + {0, 0, 0, 64} + }; + vertex.useCache = false; + const auto vertexResult = ShaderRecompiler::Recompile(vertex); + const auto vertexPush = static_cast(vertexResult.pushConstants.size()); + + ShaderRecompiler::ShaderPixelStageInfo pixel{}; + pixel.interpolatorCount = 1; + pixel.inputAddr = inputs; + pixel.hasPerspectiveCenterVgpr = (inputs & PixelInputBit(PixelInput::PerspectiveCenter)) != 0u; + pixel.noPerspective = (inputs & PixelInputBit(PixelInput::LinearCenter)) != 0u; + pixel.executeOnNoop = true; + pixel.targetOutputMode[0] = 9; + pixel.targetExportMapping.fill(0xe4u); + const std::vector pixelUserData; + const std::array pixelMemory{{{reinterpret_cast(pixelCode.data()), std::as_bytes(pixelCode)}}}; + ShaderRecompiler::RecompileRequest fragment{ + {ShaderStage::Fragment, reinterpret_cast(pixelCode.data()), pixelCode, 0, {}}, + {64u, 0, pixelUserData, std::nullopt, pixel, std::nullopt, pixelMemory}, + target, + PixelPushLayout(vertexPush, target) + }; + fragment.useCache = false; + auto pixelResult = ShaderRecompiler::Recompile(fragment); + std::vector shaders{{ShaderStage::Vertex, &vertexResult, 0}}; + ShaderRecompiler::RecompileResult geometry; + if (pixelResult.barycentricEmulation.active) { + geometry = ShaderRecompiler::BuildBarycentricGeometryShader(vertexResult, pixelResult, target, device.GeometryLimits()); + shaders.push_back({ShaderStage::Geometry, &geometry, 0}); + } + shaders.push_back({ShaderStage::Fragment, &pixelResult, PixelPushOffset(vertexPush, target)}); + + AgcDriver::Graphics::State state{}; + state.stages = {AgcDriver::Graphics::ShaderPath::Vertex, 0u, 64u, 64u, std::nullopt, std::nullopt}; + state.color = {reinterpret_cast(Pixels.data()), {Width, Height}, VK_FORMAT_R32G32B32A32_SFLOAT, sizeof(Pixels), 0xe4u, AgcDriver::Graphics::ColorTileMode::Linear, 16u}; + state.colors = {state.color}; + state.hasColorTarget = true; + state.renderExtent = {Width, Height}; + state.topology = VK_PRIMITIVE_TOPOLOGY_TRIANGLE_LIST; + state.viewport = {0, static_cast(Height), static_cast(Width), -static_cast(Height), 0, 1}; + state.negativeOneToOne = false; + state.scissor = {{0, 0}, {Width, Height}}; + state.cullMode = VK_CULL_MODE_NONE; + state.frontFace = VK_FRONT_FACE_COUNTER_CLOCKWISE; + state.blend.colorWriteMask = 15; + state.blends = {state.blend}; + state.blendConstants = {}; + const AgcDriver::Pm4::DrawParameters draw{0, static_cast(positions.size()), 0, 1, 0, false}; + device.Draw(state, draw, shaders); + device.WaitIdle(); + return pixelResult; +} + +struct Expected { + bool inside; + std::array linear; + std::array perspective; +}; + +Expected Barycentrics(std::uint32_t x, std::uint32_t y, bool snapped) { + std::array, 3> window{}; + for (std::size_t i = 0; i < Corners.size(); ++i) window[i] = {(Corners[i].x + 1.0) * Width / 2.0, (1.0 - Corners[i].y) * Height / 2.0}; + if (snapped) { + for (auto& corner : window) { + for (auto& coordinate : corner) coordinate = std::round(coordinate * 256.0) / 256.0; + } + } + const double px = x + 0.5; + const double py = y + 0.5; + const auto edge = [](const std::array& a, const std::array& b, double cx, double cy) { return (b[0] - a[0]) * (cy - a[1]) - (b[1] - a[1]) * (cx - a[0]); }; + const double area = edge(window[0], window[1], window[2][0], window[2][1]); + const std::array b{edge(window[1], window[2], px, py) / area, edge(window[2], window[0], px, py) / area, edge(window[0], window[1], px, py) / area}; + Expected expected{std::min({b[0], b[1], b[2]}) > 0.02, {}, {}}; + double sum = 0.0; + for (std::size_t i = 0; i < 3; ++i) sum += b[i] / Corners[i].w; + for (std::size_t i = 0; i < 3; ++i) { + expected.linear[i] = static_cast(b[i]); + expected.perspective[i] = static_cast(b[i] / Corners[i].w / sum); + } + return expected; +} + +Expected BarycentricsUnsnapped(std::uint32_t x, std::uint32_t y) { + return Barycentrics(x, y, false); +} + +Expected BarycentricsSnapped(std::uint32_t x, std::uint32_t y) { + return Barycentrics(x, y, true); +} + +bool MatchesEither(float value, float unsnapped, float snapped) { + return std::fabs(value - unsnapped) <= 1e-4f || std::fabs(value - snapped) <= 1e-4f; +} + +float Pixel(std::uint32_t x, std::uint32_t y, std::uint32_t channel) { + return Pixels[(y * Width + x) * 4u + channel]; +} + +std::uint32_t CheckBarycentrics(const std::string& what, bool perspective) { + std::uint32_t checked = 0; + for (std::uint32_t y = 0; y < Height; ++y) { + for (std::uint32_t x = 0; x < Width; ++x) { + const auto expected = BarycentricsUnsnapped(x, y); + if (!expected.inside) continue; + const auto snapped = BarycentricsSnapped(x, y); + const auto& weights = perspective ? expected.perspective : expected.linear; + const auto& snappedWeights = perspective ? snapped.perspective : snapped.linear; + for (std::uint32_t channel = 0; channel < 4; ++channel) { + const auto value = Pixel(x, y, channel); + const auto reference = weights[1u + (channel & 1u)]; + const auto snappedReference = snappedWeights[1u + (channel & 1u)]; + Require(MatchesEither(value, reference, snappedReference), what + ": pixel (" + std::to_string(x) + ", " + std::to_string(y) + ") channel " + std::to_string(channel) + " is " + std::to_string(value) + ", expected " + std::to_string(reference) + " or " + std::to_string(snappedReference)); + } + ++checked; + } + } + Require(checked > 200u, what + ": too few interior pixels were checked"); + return checked; +} + +void CheckVertexValues() { + const auto positions = ClipPositions(); + const auto p0 = positions[0][0]; + const std::array expected{positions[1][0] - p0, positions[2][0] - p0, p0}; + std::uint32_t checked = 0; + for (std::uint32_t y = 0; y < Height; ++y) { + for (std::uint32_t x = 0; x < Width; ++x) { + if (!BarycentricsUnsnapped(x, y).inside) continue; + for (std::uint32_t channel = 0; channel < 3; ++channel) { + Require(Pixel(x, y, channel) == expected[channel], "v_interp_mov: pixel (" + std::to_string(x) + ", " + std::to_string(y) + ") channel " + std::to_string(channel) + " is " + std::to_string(Pixel(x, y, channel)) + ", expected " + std::to_string(expected[channel])); + } + ++checked; + } + } + Require(checked > 200u, "v_interp_mov: too few interior pixels were checked"); +} + +void CheckInterpolated() { + const auto positions = ClipPositions(); + std::uint32_t checked = 0; + for (std::uint32_t y = 0; y < Height; ++y) { + for (std::uint32_t x = 0; x < Width; ++x) { + const auto expected = BarycentricsUnsnapped(x, y); + if (!expected.inside) continue; + const auto snapped = BarycentricsSnapped(x, y); + float reference = 0.0f; + float snappedReference = 0.0f; + for (std::size_t i = 0; i < 3; ++i) { + reference += expected.perspective[i] * positions[i][0]; + snappedReference += snapped.perspective[i] * positions[i][0]; + } + for (std::uint32_t channel = 0; channel < 4; ++channel) { + Require(MatchesEither(Pixel(x, y, channel), reference, snappedReference), "v_interp_p1/p2: pixel (" + std::to_string(x) + ", " + std::to_string(y) + ") channel " + std::to_string(channel) + " is " + std::to_string(Pixel(x, y, channel)) + ", expected " + std::to_string(reference) + " or " + std::to_string(snappedReference)); + } + ++checked; + } + } + Require(checked > 200u, "v_interp_p1/p2: too few interior pixels were checked"); +} + +} + +int main() { + try { + const auto device = OpenVulkanTestDevice(); + if (!device) return VulkanTestSkipped; + if (!device->GeometryLimits()) { + std::puts("skipped, the device lacks geometryShader"); + return VulkanTestSkipped; + } + const auto perspective = PixelInputBit(PixelInput::PerspectiveCenter); + const auto linear = PixelInputBit(PixelInput::LinearCenter); + auto result = Draw(*device, BarycentricPixelCode, perspective); + Require(result.barycentricEmulation.active && result.barycentricEmulation.smooth && !result.barycentricEmulation.linear, "reading PERSP_CENTER I/J did not emulate perspective barycentrics"); + const auto smooth = CheckBarycentrics("PERSP_CENTER I/J", true); + result = Draw(*device, BarycentricPixelCode, linear); + Require(result.barycentricEmulation.active && !result.barycentricEmulation.smooth && result.barycentricEmulation.linear, "reading LINEAR_CENTER I/J did not emulate linear barycentrics"); + const auto noPerspective = CheckBarycentrics("LINEAR_CENTER I/J", false); + result = Draw(*device, VertexValuePixelCode, perspective); + Require(result.barycentricEmulation.active && result.fragmentParameters.size() == 1 && result.fragmentParameters[0].perVertex, "v_interp_mov of P10 and P20 did not emulate per-vertex parameters"); + CheckVertexValues(); + result = Draw(*device, InterpolatedPixelCode, perspective); + Require(!result.barycentricEmulation.active, "plain v_interp_p1/p2 needed a geometry stage"); + CheckInterpolated(); + std::printf("barycentric emulation tests passed: %u perspective and %u linear I/J pixels, raw vertex values and host interpolation\n", smooth, noPerspective); + return 0; + } catch (const std::exception& error) { + std::cerr << error.what() << '\n'; + return 1; + } +} diff --git a/core/libs/prx/libSceAgcDriver/tests/execution/PixelFixedPointPosition.cpp b/core/libs/prx/libSceAgcDriver/tests/execution/PixelFixedPointPosition.cpp new file mode 100644 index 000000000..60aa9b05e --- /dev/null +++ b/core/libs/prx/libSceAgcDriver/tests/execution/PixelFixedPointPosition.cpp @@ -0,0 +1,254 @@ +#include "prx/libSceAgcDriver/Execution/include/GuestMemory.hpp" +#include "prx/libSceAgcDriver/Execution/include/VulkanDevice.hpp" +#include "prx/libSceAgcDriver/Graphics/include/ColorTargetLayout.hpp" +#include "prx/libSceAgcDriver/Graphics/include/Draw.hpp" +#include "prx/libSceAgcDriver/Graphics/include/State.hpp" +#include "prx/libSceAgcDriver/Graphics/include/Texture.hpp" +#include "prx/libc/include/GuestAllocations.hpp" +#include "prx/libc/include/GuestArena.hpp" +#include "prx/libc/include/GuestWriteWatch.hpp" +#include "Recompiler.hpp" +#include "VulkanTestDevice.hpp" +#ifdef _WIN32 +#ifndef NOMINMAX +#define NOMINMAX +#endif +#include +#else +#include +#endif +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace { + +using AgcDriver::Graphics::Require; +using ShaderRecompiler::ShaderStage; + +constexpr std::uint32_t Width = 64; +constexpr std::uint32_t Height = 16; +constexpr std::size_t BlockBytes = 65536; +constexpr std::uint32_t Rgba32FloatInfo = 0x8738u; +constexpr std::uint32_t LinearAttrib3 = 0x09000000u; +constexpr std::uint32_t Float32AbgrExport = 9; +constexpr std::uint32_t NoInput = 0xffffffffu; + +alignas(256) constexpr std::array VertexCode{ + 0xe0382000, 0x80000005, 0xbf8c3f70, 0xf80008cf, 0x03020100, 0xbf810000, +}; + +constexpr std::array, 3> Triangle{{ + {-1.0f, -1.0f, 0.5f, 1.0f}, {3.0f, -1.0f, 0.5f, 1.0f}, {-1.0f, 3.0f, 0.5f, 1.0f} +}}; + +alignas(256) std::array, 4> Programs{}; +std::size_t programCount = 0; + +std::array BufferDescriptor(const void* data, std::uint32_t stride, std::uint32_t count) { + const auto address = reinterpret_cast(data); + return {static_cast(address), static_cast((address >> 32u) & 0xffffu) | (stride << 16u), count, 0x01016facu}; +} + +std::span Program(std::uint32_t fixedVgpr, std::uint32_t stippleVgpr) { + Require(programCount < Programs.size(), "too many test programs"); + auto& code = Programs[programCount++]; + std::size_t at = 0; + code[at++] = 0x36000000u | (20u << 17u) | (fixedVgpr << 9u) | 0xffu; + code[at++] = 0x0000ffffu; + code[at++] = 0x2c000000u | (21u << 17u) | (fixedVgpr << 9u) | 0x90u; + code[at++] = 0x7e000c00u | (20u << 17u) | (256u + 20u); + code[at++] = 0x7e000c00u | (21u << 17u) | (256u + 21u); + code[at++] = stippleVgpr == NoInput ? 0x7e000280u | (22u << 17u) : 0x7e000c00u | (22u << 17u) | (256u + stippleVgpr); + code[at++] = 0x7e0002f2u | (23u << 17u); + code[at++] = 0xf800180fu; + code[at++] = 0x17161514u; + code[at++] = 0xbf810000u; + return std::span(code.data(), at); +} + +struct Block { + explicit Block(std::size_t bytes) : bytes(bytes), watched(AgcDriver::GuestMemory::WriteWatched()) { +#ifdef _WIN32 + if (watched) { + data = static_cast(GuestArena::GuestArenaAllocate_nid_postfix(bytes, BlockBytes)); + GuestArena::GuestArenaCommit_nid_postfix(data, bytes, PAGE_READWRITE, bytes); + } else { + data = static_cast(VirtualAlloc(nullptr, bytes, MEM_RESERVE | MEM_COMMIT, PAGE_READWRITE)); + } +#else + if (watched) { + void* raw = mmap(nullptr, bytes + BlockBytes, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + Require(raw != MAP_FAILED, "cannot map the color target"); + const auto begin = reinterpret_cast(raw); + const auto aligned = (begin + BlockBytes - 1) & ~(static_cast(BlockBytes) - 1); + if (aligned != begin) munmap(raw, aligned - begin); + if (aligned + bytes != begin + bytes + BlockBytes) munmap(reinterpret_cast(aligned + bytes), begin + BlockBytes - aligned); + data = reinterpret_cast(aligned); + GuestWriteWatch::GuestWriteWatchRegister_nid_postfix(data, bytes); + } else { + data = static_cast(std::aligned_alloc(BlockBytes, bytes)); + } +#endif + Require(data != nullptr, "cannot allocate the color target"); + Require(!watched || AgcDriver::GuestMemory::Watched(Address(), bytes), "the color target is not write-watched"); + GuestAllocations::Mutation mutation; + mutation.Add(data, bytes, true, true, true); + } + ~Block() { + AgcDriver::Graphics::StorageTexture::FlushPending(Address(), bytes, nullptr, "test release"); + { + GuestAllocations::Mutation mutation; + mutation.Remove(data); + } +#ifdef _WIN32 + if (watched) { + GuestArena::GuestArenaReset_nid_postfix(data, bytes); + GuestArena::GuestArenaRelease_nid_postfix(data, bytes); + } else { + VirtualFree(data, 0, MEM_RELEASE); + } +#else + if (watched) { + munmap(data, bytes); + GuestWriteWatch::GuestWriteWatchUnregister_nid_postfix(data, bytes); + } else { + std::free(data); + } +#endif + } + Block(const Block&) = delete; + Block& operator=(const Block&) = delete; + std::uint64_t Address() const { return reinterpret_cast(data); } + std::size_t bytes; + bool watched; + std::uint8_t* data = nullptr; +}; + +AgcDriver::Graphics::ColorTarget DecodeTarget(const Block& block, std::uint32_t info) { + AgcDriver::Registers cx; + cx[0x318] = static_cast(block.Address() >> 8u); + cx[0x31b] = 0; + cx[0x31c] = info; + cx[0x31d] = 0; + cx[0x323] = 0; + cx[0x324] = 0; + cx[0x3b0] = ((Width - 1u) << 14u) | (Height - 1u); + cx[0x3b8] = LinearAttrib3; + cx[0x390] = static_cast(block.Address() >> 40u); + auto color = AgcDriver::Graphics::DecodeColorBuffer(cx, 0); + color.exportIndex = 0; + return color; +} + +void Draw(AgcDriver::VulkanDevice& device, const AgcDriver::Graphics::ColorTarget& color, std::uint32_t inputAddr, std::span pixelCode) { + const auto target = device.Target(); + constexpr std::uint32_t waveSize = 64; + std::vector vertexUserData(4, 0u); + const auto vertexBuffer = BufferDescriptor(Triangle.data(), 16u, static_cast(Triangle.size())); + std::copy(vertexBuffer.begin(), vertexBuffer.end(), vertexUserData.begin()); + const std::array vertexMemory{{{reinterpret_cast(VertexCode.data()), std::as_bytes(std::span(VertexCode))}}}; + ShaderRecompiler::RecompileRequest vertex{ + {ShaderStage::Vertex, reinterpret_cast(VertexCode.data()), VertexCode, 0, {}}, + {waveSize, 0, vertexUserData, std::nullopt, std::nullopt, ShaderRecompiler::ShaderVertexStageInfo{}, vertexMemory}, + target, + {0, 0, 0, 64} + }; + vertex.useCache = false; + const auto vertexResult = ShaderRecompiler::Recompile(vertex); + const auto vertexPush = static_cast(vertexResult.pushConstants.size()); + + ShaderRecompiler::ShaderPixelStageInfo pixel{}; + pixel.inputAddr = inputAddr; + pixel.targetOutputMode.at(0) = Float32AbgrExport; + pixel.targetExportMapping.fill(0xe4u); + const std::vector pixelUserData(8, 0u); + const std::array pixelMemory{{{reinterpret_cast(pixelCode.data()), std::as_bytes(pixelCode)}}}; + ShaderRecompiler::RecompileRequest fragment{ + {ShaderStage::Fragment, reinterpret_cast(pixelCode.data()), pixelCode, 0, {}}, + {waveSize, 0, pixelUserData, std::nullopt, pixel, std::nullopt, pixelMemory}, + target, + PixelPushLayout(vertexPush, target) + }; + fragment.useCache = false; + const auto pixelResult = ShaderRecompiler::Recompile(fragment); + const std::array shaders{{ + {ShaderStage::Vertex, &vertexResult, 0}, + {ShaderStage::Fragment, &pixelResult, PixelPushOffset(vertexPush, target)} + }}; + + VkPipelineColorBlendAttachmentState blend{}; + blend.blendEnable = VK_FALSE; + blend.colorWriteMask = 0xf; + AgcDriver::Graphics::State state{}; + state.stages = {AgcDriver::Graphics::ShaderPath::Vertex, 0u, waveSize, waveSize, std::nullopt, std::nullopt}; + state.colors = {color}; + state.color = color; + state.hasColorTarget = true; + state.renderExtent = {Width, Height}; + state.topology = VK_PRIMITIVE_TOPOLOGY_TRIANGLE_LIST; + state.viewport = {0, static_cast(Height), static_cast(Width), -static_cast(Height), 0, 1}; + state.negativeOneToOne = false; + state.scissor = {{0, 0}, {Width, Height}}; + state.cullMode = VK_CULL_MODE_NONE; + state.frontFace = VK_FRONT_FACE_COUNTER_CLOCKWISE; + state.blends = {blend}; + state.blend = blend; + state.blendConstants = {}; + const AgcDriver::Pm4::DrawParameters draw{0, static_cast(Triangle.size()), 0, 1, 0, false}; + device.Draw(state, draw, shaders); +} + +std::vector ReadBack(AgcDriver::VulkanDevice& device, std::uint64_t address, std::size_t bytes) { + AgcDriver::Graphics::StorageTexture::FlushPending(address, bytes, nullptr, "test read-back"); + device.WaitIdle(); + std::vector stored(bytes); + AgcDriver::GuestMemory::Read(address, std::as_writable_bytes(std::span(stored)), 1); + return stored; +} + +void Expect(AgcDriver::VulkanDevice& device, std::uint32_t inputAddr, std::uint32_t fixedVgpr, std::uint32_t stippleVgpr, const std::string& what) { + constexpr std::size_t bytes = std::size_t{Width} * Height * 16u; + Require(AgcDriver::Graphics::ColorTargetLayout(Width, Height, AgcDriver::Graphics::ColorTileMode::Linear, 16).Bytes() == bytes, what + ": the linear target is padded"); + Block block(bytes); + std::fill(block.data, block.data + bytes, std::uint8_t{0xff}); + const auto color = DecodeTarget(block, Rgba32FloatInfo); + Require(color.format == VK_FORMAT_R32G32B32A32_SFLOAT && color.elementBytes == 16 && color.bytes == bytes, what + ": the target decoded to the wrong format"); + Draw(device, color, inputAddr, Program(fixedVgpr, stippleVgpr)); + const auto stored = ReadBack(device, block.Address(), bytes); + for (std::uint32_t y = 0; y < Height; ++y) { + for (std::uint32_t x = 0; x < Width; ++x) { + std::array texel{}; + std::memcpy(texel.data(), stored.data() + (static_cast(y) * Width + x) * 16u, sizeof(texel)); + const std::array expected{static_cast(x), static_cast(y), 0.0f, 1.0f}; + Require(texel == expected, what + ": pixel (" + std::to_string(x) + ", " + std::to_string(y) + ") holds (" + std::to_string(texel[0]) + ", " + std::to_string(texel[1]) + ", " + std::to_string(texel[2]) + ", " + std::to_string(texel[3]) + ")"); + } + } +} + +} + +int main() { + try { + using ShaderRecompiler::PixelInput; + using ShaderRecompiler::PixelInputBit; + const auto device = OpenVulkanTestDevice(); + if (!device) return VulkanTestSkipped; + Expect(*device, PixelInputBit(PixelInput::PositionFixedPoint), 0u, NoInput, "POS_FIXED_PT alone"); + Expect(*device, PixelInputBit(PixelInput::LineStipple) | PixelInputBit(PixelInput::PositionFixedPoint), 1u, 0u, "LINE_STIPPLE_TEX and POS_FIXED_PT"); + Expect(*device, PixelInputBit(PixelInput::PerspectiveCenter) | PixelInputBit(PixelInput::LineStipple) | PixelInputBit(PixelInput::PositionFixedPoint), 3u, 2u, "POS_FIXED_PT after the perspective center and LINE_STIPPLE_TEX"); + std::puts("pixel fixed-point position tests passed"); + return 0; + } catch (const std::exception& error) { + std::cerr << error.what() << '\n'; + return 1; + } +} diff --git a/core/libs/prx/libSceAgcDriver/tests/execution/PixelHelperLanes.cpp b/core/libs/prx/libSceAgcDriver/tests/execution/PixelHelperLanes.cpp index 0660b8036..606fcdb41 100644 --- a/core/libs/prx/libSceAgcDriver/tests/execution/PixelHelperLanes.cpp +++ b/core/libs/prx/libSceAgcDriver/tests/execution/PixelHelperLanes.cpp @@ -87,13 +87,13 @@ void Draw(AgcDriver::VulkanDevice& device, std::uint32_t waveSize, const ShaderR {ShaderStage::Fragment, reinterpret_cast(PixelCode.data()), PixelCode, 0, {}}, {waveSize, 0, pixelUserData, std::nullopt, pixel, std::nullopt, pixelMemory}, target, - {0, 0, vertexPush, 128 - vertexPush} + PixelPushLayout(vertexPush, target) }; fragment.useCache = false; const auto pixelResult = ShaderRecompiler::Recompile(fragment); const std::array shaders{{ {ShaderStage::Vertex, &vertexResult, 0}, - {ShaderStage::Fragment, &pixelResult, vertexPush} + {ShaderStage::Fragment, &pixelResult, PixelPushOffset(vertexPush, target)} }}; AgcDriver::Graphics::State state{}; diff --git a/core/libs/prx/libSceAgcDriver/tests/execution/PixelInterlock.cpp b/core/libs/prx/libSceAgcDriver/tests/execution/PixelInterlock.cpp index ea1bd10b2..116235fa1 100644 --- a/core/libs/prx/libSceAgcDriver/tests/execution/PixelInterlock.cpp +++ b/core/libs/prx/libSceAgcDriver/tests/execution/PixelInterlock.cpp @@ -58,7 +58,7 @@ public: block = static_cast(std::aligned_alloc(GuestBytes, 2 * GuestBytes)); #endif Require(block != nullptr, "pixel interlock: cannot allocate the guest block"); - GuestAllocations::Mutation().Add(block, GuestBytes, true, true); + GuestAllocations::Mutation().Add(block, GuestBytes, true, true, true); } ~GuestCounters() { @@ -137,13 +137,13 @@ void Draw(AgcDriver::VulkanDevice& device, std::uint32_t waveSize, std::span(pixelCode.data()), pixelCode, 0, {}}, {waveSize, 0, pixelUserData, std::nullopt, pixel, std::nullopt, pixelMemory}, target, - {0, 0, vertexPush, 128 - vertexPush} + PixelPushLayout(vertexPush, target) }; fragment.useCache = false; const auto pixelResult = ShaderRecompiler::Recompile(fragment); const std::array shaders{{ {ShaderStage::Vertex, &vertexResult, 0}, - {ShaderStage::Fragment, &pixelResult, vertexPush} + {ShaderStage::Fragment, &pixelResult, PixelPushOffset(vertexPush, target)} }}; AgcDriver::Graphics::State state{}; diff --git a/core/libs/prx/libSceAgcDriver/tests/execution/PixelLds.cpp b/core/libs/prx/libSceAgcDriver/tests/execution/PixelLds.cpp index ba7b210d8..174ef5e52 100644 --- a/core/libs/prx/libSceAgcDriver/tests/execution/PixelLds.cpp +++ b/core/libs/prx/libSceAgcDriver/tests/execution/PixelLds.cpp @@ -102,13 +102,13 @@ void Draw(AgcDriver::VulkanDevice& device, std::uint32_t waveSize, bool bounded) {ShaderStage::Fragment, reinterpret_cast(pixelCode.data()), pixelCode, 0, {}}, {waveSize, 0, pixelUserData, std::nullopt, pixel, std::nullopt, pixelMemory}, target, - {0, 0, vertexPush, 128 - vertexPush} + PixelPushLayout(vertexPush, target) }; fragment.useCache = false; const auto pixelResult = ShaderRecompiler::Recompile(fragment); const std::array shaders{{ {ShaderStage::Vertex, &vertexResult, 0}, - {ShaderStage::Fragment, &pixelResult, vertexPush} + {ShaderStage::Fragment, &pixelResult, PixelPushOffset(vertexPush, target)} }}; AgcDriver::Graphics::State state{}; diff --git a/core/libs/prx/libSceAgcDriver/tests/execution/PixelLdsLaneSlots.cpp b/core/libs/prx/libSceAgcDriver/tests/execution/PixelLdsLaneSlots.cpp index a1aa594a1..8311db542 100644 --- a/core/libs/prx/libSceAgcDriver/tests/execution/PixelLdsLaneSlots.cpp +++ b/core/libs/prx/libSceAgcDriver/tests/execution/PixelLdsLaneSlots.cpp @@ -95,13 +95,13 @@ void Draw(AgcDriver::VulkanDevice& device, std::uint32_t waveSize) { {ShaderStage::Fragment, reinterpret_cast(pixelCode.data()), pixelCode, 0, {}}, {waveSize, 0, pixelUserData, std::nullopt, pixel, std::nullopt, pixelMemory}, target, - {0, 0, vertexPush, 128 - vertexPush} + PixelPushLayout(vertexPush, target) }; fragment.useCache = false; const auto pixelResult = ShaderRecompiler::Recompile(fragment); const std::array shaders{{ {ShaderStage::Vertex, &vertexResult, 0}, - {ShaderStage::Fragment, &pixelResult, vertexPush} + {ShaderStage::Fragment, &pixelResult, PixelPushOffset(vertexPush, target)} }}; AgcDriver::Graphics::State state{}; diff --git a/core/libs/prx/libSceAgcDriver/tests/execution/PixelUint16Export.cpp b/core/libs/prx/libSceAgcDriver/tests/execution/PixelUint16Export.cpp new file mode 100644 index 000000000..f354ffc86 --- /dev/null +++ b/core/libs/prx/libSceAgcDriver/tests/execution/PixelUint16Export.cpp @@ -0,0 +1,272 @@ +#include "prx/libSceAgcDriver/Execution/include/GuestMemory.hpp" +#include "prx/libSceAgcDriver/Execution/include/VulkanDevice.hpp" +#include "prx/libSceAgcDriver/Graphics/include/ColorTargetLayout.hpp" +#include "prx/libSceAgcDriver/Graphics/include/Draw.hpp" +#include "prx/libSceAgcDriver/Graphics/include/State.hpp" +#include "prx/libSceAgcDriver/Graphics/include/Texture.hpp" +#include "prx/libc/include/GuestAllocations.hpp" +#include "prx/libc/include/GuestArena.hpp" +#include "prx/libc/include/GuestWriteWatch.hpp" +#include "Recompiler.hpp" +#include "VulkanTestDevice.hpp" +#ifdef _WIN32 +#ifndef NOMINMAX +#define NOMINMAX +#endif +#include +#else +#include +#endif +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace { + +using AgcDriver::Graphics::Require; +using ShaderRecompiler::ShaderStage; + +constexpr std::uint32_t Width = 64; +constexpr std::uint32_t ElementBytes = 8; +constexpr std::size_t RowBytes = std::size_t{Width} * ElementBytes; +constexpr std::size_t BlockBytes = 65536; +constexpr std::uint32_t Uint16x4Info = 0x8430u; +constexpr std::uint32_t LinearAttrib3 = 0x09000000u; +constexpr std::uint8_t Kept = 0x5a; + +struct Row { + std::uint32_t low; + std::uint32_t high; +}; + +constexpr std::array Rows{{ + {0x0000ffffu, 0xffff0000u}, + {0x00010002u, 0xffffffffu}, + {0xffffffffu, 0x00010002u}, + {0xffff0000u, 0x0000ffffu}, + {0x22221111u, 0x44443333u}, +}}; +constexpr std::uint32_t Height = Rows.size() + 1; +constexpr std::size_t TargetBytes = RowBytes * Height; + +alignas(256) constexpr std::array VertexCode{ + 0xe0382000, 0x80000005, 0xbf8c3f70, 0xf80008cf, 0x03020100, 0xbf810000, +}; + +constexpr std::array, 3> Triangle{{ + {-1.0f, -1.0f, 0.5f, 1.0f}, {3.0f, -1.0f, 0.5f, 1.0f}, {-1.0f, 3.0f, 0.5f, 1.0f} +}}; + +alignas(256) std::array, Rows.size()> Programs{}; + +std::array BufferDescriptor(const void* data, std::uint32_t stride, std::uint32_t count) { + const auto address = reinterpret_cast(data); + return {static_cast(address), static_cast((address >> 32u) & 0xffffu) | (stride << 16u), count, 0x01016facu}; +} + +std::span Program(std::size_t index) { + auto& code = Programs[index]; + code = {0x7e0802ffu, Rows[index].low, 0x7e0a02ffu, Rows[index].high, 0xf8001c0fu, 0x00000504u, 0xbf810000u, 0}; + return std::span(code.data(), 7); +} + +std::array Exported(const Row& row) { + return {static_cast(row.low), static_cast(row.low >> 16u), static_cast(row.high), static_cast(row.high >> 16u)}; +} + +struct Block { + explicit Block(std::size_t bytes) : bytes(bytes), watched(AgcDriver::GuestMemory::WriteWatched()) { +#ifdef _WIN32 + if (watched) { + data = static_cast(GuestArena::GuestArenaAllocate_nid_postfix(bytes, BlockBytes)); + GuestArena::GuestArenaCommit_nid_postfix(data, bytes, PAGE_READWRITE, bytes); + } else { + data = static_cast(VirtualAlloc(nullptr, bytes, MEM_RESERVE | MEM_COMMIT, PAGE_READWRITE)); + } +#else + if (watched) { + void* raw = mmap(nullptr, bytes + BlockBytes, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + Require(raw != MAP_FAILED, "cannot map the color target"); + const auto begin = reinterpret_cast(raw); + const auto aligned = (begin + BlockBytes - 1) & ~(static_cast(BlockBytes) - 1); + if (aligned != begin) munmap(raw, aligned - begin); + if (aligned + bytes != begin + bytes + BlockBytes) munmap(reinterpret_cast(aligned + bytes), begin + BlockBytes - aligned); + data = reinterpret_cast(aligned); + GuestWriteWatch::GuestWriteWatchRegister_nid_postfix(data, bytes); + } else { + data = static_cast(std::aligned_alloc(BlockBytes, bytes)); + } +#endif + Require(data != nullptr, "cannot allocate the color target"); + Require(!watched || AgcDriver::GuestMemory::Watched(Address(), bytes), "the color target is not write-watched"); + GuestAllocations::Mutation mutation; + mutation.Add(data, bytes, true, true, true); + } + ~Block() { + AgcDriver::Graphics::StorageTexture::FlushPending(Address(), bytes, nullptr, "test release"); + { + GuestAllocations::Mutation mutation; + mutation.Remove(data); + } +#ifdef _WIN32 + if (watched) { + GuestArena::GuestArenaReset_nid_postfix(data, bytes); + GuestArena::GuestArenaRelease_nid_postfix(data, bytes); + } else { + VirtualFree(data, 0, MEM_RELEASE); + } +#else + if (watched) { + munmap(data, bytes); + GuestWriteWatch::GuestWriteWatchUnregister_nid_postfix(data, bytes); + } else { + std::free(data); + } +#endif + } + Block(const Block&) = delete; + Block& operator=(const Block&) = delete; + std::uint64_t Address() const { return reinterpret_cast(data); } + std::size_t bytes; + bool watched; + std::uint8_t* data = nullptr; +}; + +AgcDriver::Graphics::ColorTarget DecodeTarget(const Block& block) { + AgcDriver::Registers cx; + cx[0x318] = static_cast(block.Address() >> 8u); + cx[0x31b] = 0; + cx[0x31c] = Uint16x4Info; + cx[0x31d] = 0; + cx[0x323] = 0; + cx[0x324] = 0; + cx[0x3b0] = ((Width - 1u) << 14u) | (Height - 1u); + cx[0x3b8] = LinearAttrib3; + cx[0x390] = static_cast(block.Address() >> 40u); + auto color = AgcDriver::Graphics::DecodeColorBuffer(cx, 0); + color.exportIndex = 0; + color.uintExport = true; + return color; +} + +void Draw(AgcDriver::VulkanDevice& device, const AgcDriver::Graphics::ColorTarget& color, std::uint8_t mapping, std::size_t row) { + const auto target = device.Target(); + constexpr std::uint32_t waveSize = 64; + std::vector vertexUserData(4, 0u); + const auto vertexBuffer = BufferDescriptor(Triangle.data(), 16u, static_cast(Triangle.size())); + std::copy(vertexBuffer.begin(), vertexBuffer.end(), vertexUserData.begin()); + const std::array vertexMemory{{{reinterpret_cast(VertexCode.data()), std::as_bytes(std::span(VertexCode))}}}; + ShaderRecompiler::RecompileRequest vertex{ + {ShaderStage::Vertex, reinterpret_cast(VertexCode.data()), VertexCode, 0, {}}, + {waveSize, 0, vertexUserData, std::nullopt, std::nullopt, ShaderRecompiler::ShaderVertexStageInfo{}, vertexMemory}, + target, + {0, 0, 0, 64} + }; + vertex.useCache = false; + const auto vertexResult = ShaderRecompiler::Recompile(vertex); + const auto vertexPush = static_cast(vertexResult.pushConstants.size()); + + const auto pixelCode = Program(row); + ShaderRecompiler::ShaderPixelStageInfo pixel{}; + pixel.inputAddr = ShaderRecompiler::PixelInputBit(ShaderRecompiler::PixelInput::PositionX) | ShaderRecompiler::PixelInputBit(ShaderRecompiler::PixelInput::PositionY); + pixel.posX = true; + pixel.posY = true; + pixel.targetOutputMode[0] = 7; + pixel.targetExportMapping.fill(0xe4u); + pixel.targetExportMapping[0] = mapping; + const std::vector pixelUserData(8, 0u); + const std::array pixelMemory{{{reinterpret_cast(pixelCode.data()), std::as_bytes(pixelCode)}}}; + ShaderRecompiler::RecompileRequest fragment{ + {ShaderStage::Fragment, reinterpret_cast(pixelCode.data()), pixelCode, 0, {}}, + {waveSize, 0, pixelUserData, std::nullopt, pixel, std::nullopt, pixelMemory}, + target, + PixelPushLayout(vertexPush, target) + }; + fragment.useCache = false; + const auto pixelResult = ShaderRecompiler::Recompile(fragment); + const std::array shaders{{ + {ShaderStage::Vertex, &vertexResult, 0}, + {ShaderStage::Fragment, &pixelResult, PixelPushOffset(vertexPush, target)} + }}; + + AgcDriver::Graphics::State state{}; + state.stages = {AgcDriver::Graphics::ShaderPath::Vertex, 0u, waveSize, waveSize, std::nullopt, std::nullopt}; + state.colors = {color}; + state.color = color; + state.hasColorTarget = true; + state.renderExtent = {Width, Height}; + state.topology = VK_PRIMITIVE_TOPOLOGY_TRIANGLE_LIST; + state.viewport = {0, static_cast(Height), static_cast(Width), -static_cast(Height), 0, 1}; + state.negativeOneToOne = false; + state.scissor = {{0, static_cast(row)}, {Width, 1}}; + state.cullMode = VK_CULL_MODE_NONE; + state.frontFace = VK_FRONT_FACE_COUNTER_CLOCKWISE; + VkPipelineColorBlendAttachmentState blend{}; + blend.colorWriteMask = 0xf; + state.blends = {blend}; + state.blend = blend; + state.blendConstants = {}; + const AgcDriver::Pm4::DrawParameters draw{0, static_cast(Triangle.size()), 0, 1, 0, false}; + device.Draw(state, draw, shaders); +} + +std::vector ReadBack(AgcDriver::VulkanDevice& device, std::uint64_t address, std::size_t bytes) { + AgcDriver::Graphics::StorageTexture::FlushPending(address, bytes, nullptr, "test read-back"); + device.WaitIdle(); + std::vector stored(bytes); + AgcDriver::GuestMemory::Read(address, std::as_writable_bytes(std::span(stored)), 1); + return stored; +} + +std::string Hex(std::uint32_t value) { + char text[16]; + std::snprintf(text, sizeof(text), "0x%x", value); + return text; +} + +void ExportTests(AgcDriver::VulkanDevice& device, std::uint8_t mapping, const std::string& name) { + Block block(BlockBytes); + std::memset(block.data, Kept, BlockBytes); + const auto color = DecodeTarget(block); + Require(color.format == VK_FORMAT_R16G16B16A16_UINT && color.elementBytes == ElementBytes, "the 16_16_16_16 UINT target did not decode as R16G16B16A16_UINT"); + Require(AgcDriver::Graphics::ColorTargetLayout(Width, Height, AgcDriver::Graphics::ColorTileMode::Linear, ElementBytes).Bytes() == TargetBytes, "the linear 16_16_16_16 target is padded"); + for (std::size_t row = 0; row < Rows.size(); ++row) Draw(device, color, mapping, row); + const auto stored = ReadBack(device, block.Address(), TargetBytes); + for (std::size_t row = 0; row < Height; ++row) { + for (std::uint32_t x = 0; x < Width; ++x) { + const auto* texel = stored.data() + row * RowBytes + std::size_t{x} * ElementBytes; + for (std::uint32_t component = 0; component < 4; ++component) { + std::uint16_t actual = 0; + std::memcpy(&actual, texel + component * 2u, 2); + const auto expected = row < Rows.size() ? Exported(Rows[row])[(mapping >> (component * 2u)) & 3u] : static_cast(Kept * 0x101u); + if (actual == expected) continue; + const auto words = row < Rows.size() ? " from the packed words " + Hex(Rows[row].low) + ", " + Hex(Rows[row].high) : std::string(" outside the drawn rows"); + throw std::runtime_error("a UINT16_ABGR export under the " + name + " mapping stored " + Hex(actual) + " in component " + std::to_string(component) + " of pixel (" + std::to_string(x) + ", " + std::to_string(row) + ")" + words + ", expected " + Hex(expected)); + } + } + } +} + +} + +int main() { + try { + const auto device = OpenVulkanTestDevice(); + if (!device) return VulkanTestSkipped; + ExportTests(*device, 0xe4u, "identity"); + ExportTests(*device, 0x1bu, "STD_REV"); + ExportTests(*device, 0x93u, "ALT_REV"); + std::puts("UINT16_ABGR export tests passed"); + return 0; + } catch (const std::exception& error) { + std::cerr << error.what() << '\n'; + return 1; + } +} diff --git a/core/libs/prx/libSceAgcDriver/tests/execution/PixelValidMask.cpp b/core/libs/prx/libSceAgcDriver/tests/execution/PixelValidMask.cpp index 914ba8e97..2b8f7b61f 100644 --- a/core/libs/prx/libSceAgcDriver/tests/execution/PixelValidMask.cpp +++ b/core/libs/prx/libSceAgcDriver/tests/execution/PixelValidMask.cpp @@ -71,13 +71,13 @@ void Draw(AgcDriver::VulkanDevice& device, std::uint32_t waveSize, std::span(pixelCode.data()), pixelCode, 0, {}}, {waveSize, 0, pixelUserData, std::nullopt, pixel, std::nullopt, pixelMemory}, target, - {0, 0, vertexPush, 128 - vertexPush} + PixelPushLayout(vertexPush, target) }; fragment.useCache = false; const auto pixelResult = ShaderRecompiler::Recompile(fragment); const std::array shaders{{ {ShaderStage::Vertex, &vertexResult, 0}, - {ShaderStage::Fragment, &pixelResult, vertexPush} + {ShaderStage::Fragment, &pixelResult, PixelPushOffset(vertexPush, target)} }}; AgcDriver::Graphics::State state{}; diff --git a/core/libs/prx/libSceAgcDriver/tests/execution/PixelWaveReduction.cpp b/core/libs/prx/libSceAgcDriver/tests/execution/PixelWaveReduction.cpp index efdfff7d9..83f21e2b2 100644 --- a/core/libs/prx/libSceAgcDriver/tests/execution/PixelWaveReduction.cpp +++ b/core/libs/prx/libSceAgcDriver/tests/execution/PixelWaveReduction.cpp @@ -105,13 +105,13 @@ void Draw(AgcDriver::VulkanDevice& device, std::span pixelC {ShaderStage::Fragment, reinterpret_cast(pixelCode.data()), pixelCode, 0, {}}, {64u, 0, pixelUserData, std::nullopt, pixel, std::nullopt, pixelMemory}, target, - {0, 0, vertexPush, 128 - vertexPush} + PixelPushLayout(vertexPush, target) }; fragment.useCache = false; const auto pixelResult = ShaderRecompiler::Recompile(fragment); const std::array shaders{{ {ShaderStage::Vertex, &vertexResult, 0}, - {ShaderStage::Fragment, &pixelResult, vertexPush} + {ShaderStage::Fragment, &pixelResult, PixelPushOffset(vertexPush, target)} }}; AgcDriver::Graphics::State state{}; diff --git a/core/libs/prx/libSceAgcDriver/tests/execution/PreparedGraphics.cpp b/core/libs/prx/libSceAgcDriver/tests/execution/PreparedGraphics.cpp index 770819753..eb1a511a1 100644 --- a/core/libs/prx/libSceAgcDriver/tests/execution/PreparedGraphics.cpp +++ b/core/libs/prx/libSceAgcDriver/tests/execution/PreparedGraphics.cpp @@ -5,6 +5,7 @@ #include "prx/libSceAgcDriver/Graphics/include/ShaderInputState.hpp" #include "Optimization/ResourceProgram.hpp" #include +#include #include #include #include @@ -125,6 +126,7 @@ void Check(AgcDriver::VulkanDevice& device, AgcDriver::Graphics::ShaderPath path target.supportedCapabilities = capabilities; target.supportedExtensions = extensions; target.mesh = MeshTargetLimits{{128, 1, 1}, 128, 32768, 256, 256, 128, 32768, 1, 1}; + target.spirvVersion = 0x00010400u; } const auto stages = PrepareGraphicsStages(prepared, target); Require(stages.size() == (tessellation ? 4u : 2u), "graphics preparation compiled the wrong stages"); @@ -182,6 +184,18 @@ void Check(AgcDriver::VulkanDevice& device, AgcDriver::Graphics::ShaderPath path ResolvePreparedGraphics(*front.snapshot, {}, 7, target); Require(front.snapshot->prepared->fragments.size() == 1, "expired helper link was not removed"); Require(front.snapshot->prepared->rectangleProgress.size() == 1 && !front.snapshot->prepared->rectangleProgress.front().fragment.expired(), "expired rectangle progress was not removed"); + auto drawOnly = prepared; + drawOnly.pixel.interpolatorSettings[0] = 0x405u; + const auto drawStages = PrepareGraphicsStages(drawOnly, target); + auto unlinked = std::make_shared(); + unlinked->type = 1; + unlinked->prepared->entries.push_back(drawStages.back().entry); + const auto drawPixelId = GetPreparedArtifact(*drawStages.back().entry.handle).variantId; + const auto drawVertexId = GetPreparedArtifact(*drawStages.front().entry.handle).variantId; + Require(drawVertexId == GetPreparedArtifact(*stages.front().entry.handle).variantId && drawPixelId != pixelArtifact.variantId && drawPixelId != newPixelId, "draw-only fragment test did not isolate a new fragment variant"); + const auto drawRectangle = DrawRectangle(*front.snapshot, unlinked, drawVertexId, drawPixelId, target); + Require(!drawRectangle.control.spirv.empty() && !drawRectangle.evaluation.spirv.empty(), "a rect-list draw did not prepare the rectangle of a fragment variant prepared at draw"); + static_cast(PreparedRectangle(*front.snapshot, drawVertexId, drawPixelId)); } if (!dump.empty()) { for (std::size_t index = 0; index < stages.size(); ++index) { @@ -234,6 +248,44 @@ void Check(AgcDriver::VulkanDevice& device, AgcDriver::Graphics::ShaderPath path Reject([&] { DecodeGraphicsPrograms(invalid, queue, registry, true, true); }, "reserved graphics program address"); } +void NullPixelMatchesRegistration(AgcDriver::VulkanDevice& device) { + using namespace AgcDriver::DriverDetail; + Fixture front; + front.Initialize(2u); + const auto null = std::make_shared(PrepareNullPixelProgram(device)); + const auto registered = null->prepared->entries; + Require(!registered.empty(), "the null pixel program was not prepared at registration"); + ShaderRegistry registry; + registry.emplace(front.snapshot->codeAddress, front.snapshot); + registry.emplace(null->codeAddress, null); + AgcDriver::QueueState queue{}; + queue.context[0x8e] = 0xfu; + queue.context[0x8f] = 0xfu; + front.Bind(queue, 0xc8u, 0x8bu); + alignas(8) const std::array merged{}; + const auto mergedAddress = reinterpret_cast(merged.data()); + queue.shader[0x82u] = static_cast(mergedAddress); + queue.shader[0x83u] = static_cast(mergedAddress >> 32u); + queue.shader[0x008u] = 0u; + queue.shader[0x009u] = 0u; + queue.shader[0x00bu] = 4u << 1u; + queue.context[0x1b3u] = 0x30u; + queue.context[0x1b4u] = 0x30u; + DrawDecode draw{}; + draw.state.stages.path = AgcDriver::Graphics::ShaderPath::Vertex; + draw.state.stages.vertexWaveSize = 32u; + draw.state.stages.fragmentWaveSize = 64u; + std::array mappings{}; + mappings.fill(0xe4u); + draw.pixel = AgcDriver::Graphics::DecodePixelStageInfo(queue.context, mappings, true); + DecodeGraphicsPrograms(draw, queue, registry, false, true); + const auto& fragment = draw.programs.back(); + Require(fragment.snapshot == null && fragment.userData.empty(), "a null pixel draw took the last pixel shader's user SGPRs"); + const auto stages = PrepareGraphicsStages(draw, device.Target()); + const auto& handle = stages.back().entry.handle; + Require(std::ranges::any_of(registered, [&](const auto& entry) { return entry.handle == handle; }), "a null pixel draw did not match the variant prepared at registration"); +} + } int main(int argc, char** argv) { @@ -242,6 +294,7 @@ int main(int argc, char** argv) { if (!device) return VulkanTestSkipped; Require(argc <= 2, "invalid test arguments"); const auto dump = argc == 2 ? std::filesystem::path(argv[1]) : std::filesystem::path{}; + NullPixelMatchesRegistration(*device); Check(*device, AgcDriver::Graphics::ShaderPath::Vertex, dump); Check(*device, AgcDriver::Graphics::ShaderPath::Geometry, dump); Check(*device, AgcDriver::Graphics::ShaderPath::Tessellation, dump); diff --git a/core/libs/prx/libSceAgcDriver/tests/execution/PreparedShaders.cpp b/core/libs/prx/libSceAgcDriver/tests/execution/PreparedShaders.cpp index 977f26c1a..695192b8d 100644 --- a/core/libs/prx/libSceAgcDriver/tests/execution/PreparedShaders.cpp +++ b/core/libs/prx/libSceAgcDriver/tests/execution/PreparedShaders.cpp @@ -1,11 +1,17 @@ #include "VulkanTestDevice.hpp" #include "Optimization/ResourceProgram.hpp" +#include "ControlFlow/RequestSerializer.hpp" #include "prx/libSceAgcDriver/Execution/include/Driver/Shaders/ShaderRegistry.hpp" +#include "prx/libSceAgcDriver/Graphics/include/ShaderInputState.hpp" #include "prx/libSceAgcDriver/Execution/include/Driver.hpp" +#include "prx/libSceAgcDriver/Execution/include/ShaderPreparation.hpp" #include "prx/libSceAgcDriver/Submit/include/Acb.hpp" #include #include #include +#include +#include +#include #include #include #include @@ -13,6 +19,11 @@ #include #include #include +#include +#include +#include +#include +#include namespace { @@ -58,7 +69,24 @@ void RunUnregistered(AgcDriver::VulkanDevice& device, ShaderRecompiler::Recompil } } +void NullPixelAtDraw(AgcDriver::VulkanDevice& device) { + std::vector code(64, 0); + code.front() = 0xbf810000u; + AgcDriver::DriverDetail::ShaderSnapshot snapshot{AgcDriver::DriverDetail::NullPixelProgramAddress(), 0, 1, code, {}}; + snapshot.header.resize(sizeof(Shader)); + std::array users{}; + const auto pixel = AgcDriver::Graphics::DecodePixelStageInfo({}, {}, true); + const ShaderRecompiler::RecompileRequest registered{{ShaderRecompiler::ShaderStage::Fragment, snapshot.codeAddress, snapshot.code, 0, {}}, {64, 0, {}, {}, pixel, {}, {}}, device.Target(), {0, 0, 0, 128}}; + snapshot.prepared->entries.push_back({0, ShaderRecompiler::PrepareShader(registered)}); + ShaderRecompiler::RecompileRequest request{{ShaderRecompiler::ShaderStage::Fragment, snapshot.codeAddress, snapshot.code, 0, {}}, {32, 0, users, {}, pixel, {}, {}}, device.Target(), {0, 0, 20, 108}}; + static_cast(AgcDriver::DriverDetail::InvocationFor(snapshot, 0, request)); + Require(snapshot.prepared->entries.size() == 2, "the null pixel program was not prepared at draw"); + static_cast(AgcDriver::DriverDetail::InvocationFor(snapshot, 0, request)); + Require(AgcDriver::DriverDetail::SourceHandleFor(snapshot, 0, request) == snapshot.prepared->entries.back().handle && snapshot.prepared->entries.size() == 2, "the null pixel program was prepared again for the same draw"); +} + void Run(AgcDriver::VulkanDevice& device) { + NullPixelAtDraw(device); alignas(256) std::array code{0xbf810000u}; std::array users{}; const ShaderRecompiler::ShaderComputeStageInfo compute{{1, 1, 1}, 0, {false, false, false}, false, 1, {}}; @@ -138,6 +166,52 @@ void Run(AgcDriver::VulkanDevice& device) { ExpectFailure([&] { static_cast(AgcDriver::DriverDetail::SourceHandleFor(snapshot, 0, request)); }, "artifact is missing"); } +void FailureCapture(AgcDriver::VulkanDevice& device) { + using namespace ShaderRecompiler; + const std::array code{0xffffffffu}; + const std::vector header(2048, std::byte{0x5a}); + RecompileRequest request{{ShaderStage::Compute, 0x12345cafeull, code, 0x20000, header}, + {32, 0, {}, ShaderComputeStageInfo{{1, 1, 1}, 0, {}, false, 1, {}}, {}, {}, {}}, device.ComputeTarget(32), {0, 0, 0, 128}}; + const auto failure = [&](auto action) { + try { action(); } + catch (const std::exception& error) { return std::string(error.what()); } + throw std::runtime_error("invalid shader preparation succeeded"); + }; + const auto expected = failure([&] { static_cast(PrepareShader(request)); }); + const char* previous = std::getenv("APS5_DUMP_SHADERS"); + const std::string saved = previous != nullptr ? previous : ""; + const std::string path = "shader_12345cafe.req"; + std::remove(path.c_str()); +#ifdef _WIN32 + Require(_putenv_s("APS5_DUMP_SHADERS", "") == 0, "cannot disable shader capture"); +#else + Require(unsetenv("APS5_DUMP_SHADERS") == 0, "cannot disable shader capture"); +#endif + Require(failure([&] { static_cast(AgcDriver::DriverDetail::PrepareShaderWithDiagnostics(request)); }) == expected, "disabled shader capture changed the preparation failure"); + Require(!std::ifstream(path, std::ios::binary).is_open(), "disabled shader capture created a request"); +#ifdef _WIN32 + Require(_putenv_s("APS5_DUMP_SHADERS", "1") == 0, "cannot enable shader capture"); +#else + Require(setenv("APS5_DUMP_SHADERS", "1", 1) == 0, "cannot enable shader capture"); +#endif + const auto actual = failure([&] { static_cast(AgcDriver::DriverDetail::PrepareShaderWithDiagnostics(request)); }); +#ifdef _WIN32 + Require(_putenv_s("APS5_DUMP_SHADERS", saved.c_str()) == 0, "cannot restore shader capture"); +#else + Require((previous != nullptr ? setenv("APS5_DUMP_SHADERS", saved.c_str(), 1) : unsetenv("APS5_DUMP_SHADERS")) == 0, "cannot restore shader capture"); +#endif + Require(actual == expected, "shader capture changed the preparation failure"); + std::ifstream file(path, std::ios::binary); + std::ostringstream text; + text << file.rdbuf(); + file.close(); + std::remove(path.c_str()); + const RequestSerializer serializer; + Require(text.str().size() > 1024 && text.str() == serializer.Serialize(request), "shader failure capture was incomplete"); + const auto replay = serializer.Deserialize(text.str()); + Require(serializer.Serialize(replay.request) == text.str(), "shader failure capture changed on replay"); +} + void PrepareMultisampledStorage(AgcDriver::VulkanDevice& device) { using namespace ShaderRecompiler; std::array code{0xd7460000u, 0x0401060cu, 0xd7460001u, 0x0405060du, 0x7e04020eu, 0x7e060280u, 0x7e080208u, 0x7e0a0209u, 0x7e0c020au, 0x7e0e020bu, 0xf0200f38u, 0x00000400u, 0xbf810000u}; @@ -148,6 +222,9 @@ void PrepareMultisampledStorage(AgcDriver::VulkanDevice& device) { request.context.waveSize = 32; request.context.compute = ShaderComputeStageInfo{{8, 8, 1}, 0, {true, true, false}, false, 2, {}}; request.target = device.ComputeTarget(32); + std::vector multisampleCapabilities(request.target.supportedCapabilities.begin(), request.target.supportedCapabilities.end()); + if (std::ranges::find(multisampleCapabilities, spv::CapabilityStorageImageMultisample) == multisampleCapabilities.end()) multisampleCapabilities.push_back(spv::CapabilityStorageImageMultisample); + request.target.supportedCapabilities = multisampleCapabilities; request.layout.pushConstantSizeBytes = 128; request.useCache = false; for (std::uint32_t variant = 0; variant < 5; ++variant) { @@ -189,7 +266,100 @@ void PrepareMultisampledStorage(AgcDriver::VulkanDevice& device) { ExpectFailure([&] { static_cast(PrepareShader(request)); }, "storage image multisampling is unavailable on the target device"); } +void SettleLater(AgcDriver::DriverDetail::PreparedShaders& prepared, std::vector entries, std::exception_ptr failure) { + std::this_thread::sleep_for(std::chrono::milliseconds(50)); + { + std::lock_guard lock(prepared.mutex); + prepared.entries.insert(prepared.entries.end(), entries.begin(), entries.end()); + prepared.failure = failure; + prepared.pending = false; + } + prepared.settled.notify_all(); +} + +std::exception_ptr InjectedFailure() { + try { + throw std::runtime_error("injected registration preparation failure"); + } catch (...) { + return std::current_exception(); + } +} + +void PendingRegistrationPreparation(AgcDriver::VulkanDevice& device) { + alignas(256) const std::array code{0xbf810000u}; + std::array users{}; + const ShaderRecompiler::ShaderComputeStageInfo compute{{1, 1, 1}, 0, {false, false, false}, false, 1, {}}; + const auto address = reinterpret_cast(code.data()); + for (const bool source : {true, false}) { + AgcDriver::DriverDetail::ShaderSnapshot snapshot{address, 0, 0, {code.begin(), code.end()}, {}}; + snapshot.header.resize(sizeof(Shader)); + ShaderRecompiler::RecompileRequest request{{ShaderRecompiler::ShaderStage::Compute, address, snapshot.code, 0, {}}, {32, 0, users, compute, {}, {}, {}}, device.ComputeTarget(32), {0, 0, 0, 128}}; + const auto handle = ShaderRecompiler::PrepareShader(request); + snapshot.prepared->pending = true; + auto settle = std::async(std::launch::async, SettleLater, std::ref(*snapshot.prepared), std::vector{{0, handle}}, nullptr); + if (source) Require(AgcDriver::DriverDetail::SourceHandleFor(snapshot, 0, request) == handle, "first use did not wait for registration preparation"); + else static_cast(AgcDriver::DriverDetail::InvocationFor(snapshot, 0, request)); + settle.get(); + Require(snapshot.prepared->entries.size() == 1, "first use prepared a pending registration again"); + } + AgcDriver::DriverDetail::ShaderSnapshot snapshot{address, 0, 0, {code.begin(), code.end()}, {}}; + snapshot.header.resize(sizeof(Shader)); + snapshot.prepared->pending = true; + auto settle = std::async(std::launch::async, SettleLater, std::ref(*snapshot.prepared), std::vector{}, InjectedFailure()); + { + AgcDriver::DriverDetail::ShaderPreparationTransaction transaction; + ExpectFailure([&] { static_cast(transaction.Read(snapshot)); }, "injected registration preparation failure"); + } + settle.get(); + ShaderRecompiler::RecompileRequest request{{ShaderRecompiler::ShaderStage::Compute, address, snapshot.code, 0, {}}, {32, 0, users, compute, {}, {}, {}}, device.ComputeTarget(32), {0, 0, 0, 128}}; + ExpectFailure([&] { static_cast(AgcDriver::DriverDetail::SourceHandleFor(snapshot, 0, request)); }, "injected registration preparation failure"); + ExpectFailure([&] { static_cast(AgcDriver::DriverDetail::InvocationFor(snapshot, 0, request)); }, "injected registration preparation failure"); +} + +void UnsupportedTypeRegistration() { + alignas(256) static const std::array code{0xbf810000u}; + struct Header { + Shader shader{}; + std::array registers{{{0, 0x1218}, {1, 0x40104004}, {2, 0xa0}, {5, 0}, {3, 6}, {4, 0xc}, {6, 0}, {7, 0}}}; + } header; + header.shader.file_header = 0x34333231u; + header.shader.version = 0x18; + header.shader.header_size = sizeof(header); + header.shader.shader_size = sizeof(code); + header.shader.code = code.data(); + header.shader.sh_registers = header.registers.data(); + header.shader.num_sh_registers = header.registers.size(); + header.shader.type = 8; + AgcDriverRegisterShader_nid_postfix(&header.shader); + AgcDriverRegisterShader_nid_postfix(&header.shader); +} + +void RegistrationWithoutSpecials() { + alignas(256) std::array code{0xbf810000u}; + struct Header { + Shader shader{}; + std::array registers{}; + } header; + const auto address = reinterpret_cast(code.data()); + header.shader.file_header = 0x34333231u; + header.shader.version = 0x18; + header.shader.header_size = sizeof(header); + header.shader.shader_size = sizeof(code); + header.shader.code = code.data(); + header.shader.sh_registers = header.registers.data(); + header.shader.num_sh_registers = header.registers.size(); + header.registers = {{{0x20c, static_cast(address >> 8u)}, {0x20d, static_cast(address >> 40u)}, {0x207, 1}, {0x208, 1}, {0x209, 1}, {0x212, 0}, {0x213, 0}}}; + AgcDriverRegisterShader_nid_postfix(&header.shader); + std::vector commands; + for (const auto reg : header.registers) commands.insert(commands.end(), {0xc0017600u, reg.offset, reg.value}); + for (const std::uint32_t initiator : {0x8041u, 0x41u}) commands.insert(commands.end(), {0xc0031500u, 1, 1, 1, initiator}); + Packet packet{commands.data(), static_cast(commands.size()), 0, {}}; + sceAgcDriverSubmitAcb(0x20, &packet); + AgcDriverWaitIdle_nid_postfix(); +} + void Registration(bool indirect) { + UnsupportedTypeRegistration(); alignas(256) std::array code{0xbf810000u}; struct Header { Shader shader{}; @@ -224,6 +394,15 @@ void Registration(bool indirect) { })); } for (auto& registration : registrations) registration.get(); + Shader copy = header.shader; + copy.user_data = reinterpret_cast(©); + AgcDriverResolveShaderAbi_nid_postfix(©, {}, {}); + alignas(Shader) std::array padded{}; + std::memcpy(padded.data(), ©, sizeof(Shader)); + padded.back() = std::byte{0x7d}; + AgcDriverResolveShaderAbi_nid_postfix(reinterpret_cast(padded.data()), {}, {}); + copy.target ^= 1u; + ExpectFailure([&] { AgcDriverResolveShaderAbi_nid_postfix(©, {}, {}); }, "replaced shader header"); header.registers[1].value |= 0x100u; ExpectFailure([&] { AgcDriverRegisterShader_nid_postfix(&header.shader); }, "invalid registered program address"); header.registers[1].value &= 0xffu; @@ -261,17 +440,88 @@ void Registration(bool indirect) { Require(destination == 0, "failed dispatch executed a subsequent memory write"); } +void DeferredRegistration() { + alignas(256) std::array code{0xbe842104u, 0xbf810000u}; + struct Header { + Shader shader{}; + std::array registers{}; + ShaderSpecialRegs specials{}; + } header; + const auto address = reinterpret_cast(code.data()); + header.shader.file_header = 0x34333231u; + header.shader.version = 0x18; + header.shader.header_size = sizeof(header); + header.shader.shader_size = sizeof(code); + header.shader.code = code.data(); + header.shader.sh_registers = header.registers.data(); + header.shader.num_sh_registers = header.registers.size(); + header.shader.specials = &header.specials; + header.specials.dispatch_modifier = 0x8000; + header.registers = {{{0x20c, static_cast(address >> 8u)}, {0x20d, static_cast(address >> 40u)}, {0x207, 1}, {0x208, 1}, {0x209, 1}, {0x212, 0}, {0x213, 12}, {0x207, 1}}}; + AgcDriverRegisterShader_nid_postfix(&header.shader); + std::vector commands; + for (const auto reg : header.registers) commands.insert(commands.end(), {0xc0017600u, reg.offset, reg.value}); + commands.insert(commands.end(), {0xc0031500u, 1, 1, 1, 0x8041}); + Packet packet{commands.data(), static_cast(commands.size()), 0, {}}; + sceAgcDriverSubmitAcb(0x20, &packet); + ExpectFailure([] { AgcDriverWaitIdle_nid_postfix(); }, "not statically resolvable"); + ExpectFailure([] { AgcDriverShutdown_nid_postfix(); }, "not statically resolvable"); +} + +void DeferredUndecodableRegistration() { + alignas(256) std::array code{0xbe842104u, 0xf1949f05u, 0x00060018u}; + struct Header { + Shader shader{}; + std::array registers{}; + ShaderSpecialRegs specials{}; + } header; + const auto address = reinterpret_cast(code.data()); + header.shader.file_header = 0x34333231u; + header.shader.version = 0x18; + header.shader.header_size = sizeof(header); + header.shader.shader_size = sizeof(code); + header.shader.code = code.data(); + header.shader.sh_registers = header.registers.data(); + header.shader.num_sh_registers = header.registers.size(); + header.shader.specials = &header.specials; + header.specials.dispatch_modifier = 0x8000; + header.registers = {{{0x20c, static_cast(address >> 8u)}, {0x20d, static_cast(address >> 40u)}, {0x207, 1}, {0x208, 1}, {0x209, 1}, {0x212, 0}, {0x213, 12}, {0x207, 1}}}; + AgcDriverRegisterShader_nid_postfix(&header.shader); + std::vector commands; + for (const auto reg : header.registers) commands.insert(commands.end(), {0xc0017600u, reg.offset, reg.value}); + commands.insert(commands.end(), {0xc0031500u, 1, 1, 1, 0x8041}); + Packet packet{commands.data(), static_cast(commands.size()), 0, {}}; + sceAgcDriverSubmitAcb(0x20, &packet); + ExpectFailure([] { AgcDriverWaitIdle_nid_postfix(); }, "unsupported MIMG opcode"); + ExpectFailure([] { AgcDriverShutdown_nid_postfix(); }, "unsupported MIMG opcode"); +} + } int main(int argc, char** argv) { try { - Require(argc == 1 || (argc == 2 && (std::string_view(argv[1]) == "--indirect" || std::string_view(argv[1]) == "--fail-before-registration")), "invalid test arguments"); + Require(argc == 1 || (argc == 2 && (std::string_view(argv[1]) == "--indirect" || std::string_view(argv[1]) == "--fail-before-registration" || std::string_view(argv[1]) == "--deferred" || std::string_view(argv[1]) == "--deferred-undecodable")), "invalid test arguments"); + if (argc == 2 && std::string_view(argv[1]) == "--deferred") { + if (!OpenVulkanTestDevice()) return VulkanTestSkipped; + DeferredRegistration(); + std::cout << "deferred shader preparation tests passed\n"; + return 0; + } + if (argc == 2 && std::string_view(argv[1]) == "--deferred-undecodable") { + if (!OpenVulkanTestDevice()) return VulkanTestSkipped; + DeferredUndecodableRegistration(); + std::cout << "deferred undecodable shader tests passed\n"; + return 0; + } auto device = OpenVulkanTestDevice(); if (!device) return VulkanTestSkipped; Run(*device); + FailureCapture(*device); + PendingRegistrationPreparation(*device); Require(argc != 2 || std::string_view(argv[1]) != "--fail-before-registration", "injected failure before registration"); PrepareMultisampledStorage(*device); device.reset(); + RegistrationWithoutSpecials(); Registration(argc == 2); std::cout << "prepared shader and transactional registration tests passed\n"; return 0; diff --git a/core/libs/prx/libSceAgcDriver/tests/execution/ProvokingVertex.cpp b/core/libs/prx/libSceAgcDriver/tests/execution/ProvokingVertex.cpp new file mode 100644 index 000000000..f1651a590 --- /dev/null +++ b/core/libs/prx/libSceAgcDriver/tests/execution/ProvokingVertex.cpp @@ -0,0 +1,142 @@ +#include "prx/libSceAgcDriver/Execution/include/VulkanDevice.hpp" +#include "prx/libSceAgcDriver/Graphics/include/Draw.hpp" +#include "VulkanTestDevice.hpp" +#include "ProvokingVertex_vert_spv.h" +#include "ProvokingVertex_frag_spv.h" +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace { + +using AgcDriver::Graphics::Require; +constexpr std::uint32_t Panel = 64; +constexpr std::uint32_t Width = Panel * 3; +constexpr std::uint32_t Height = Panel; +constexpr std::byte Sentinel{0x40}; +alignas(256) std::array Pixels{}; +using Color = std::array; +constexpr Color Red{255, 0, 0, 255}; +constexpr Color Green{0, 255, 0, 255}; +constexpr Color Blue{0, 0, 255, 255}; +constexpr Color Yellow{255, 255, 0, 255}; +constexpr Color Magenta{255, 0, 255, 255}; + +ShaderRecompiler::RecompileResult Shader(std::span words, std::uint64_t variant) { + ShaderRecompiler::RecompileResult result{}; + result.spirv = std::vector(words.begin(), words.end()); + result.variantId = variant; + return result; +} + +ShaderRecompiler::RecompileResult GuestPixel(const ShaderRecompiler::SpirvTarget& target) { + alignas(256) static constexpr std::array code{0xc8120002u, 0xc8160102u, 0xc81a0202u, 0xc81e0302u, 0xf800180fu, 0x07060504u, 0xbf810000u}; + ShaderRecompiler::ShaderPixelStageInfo pixel{}; + pixel.interpolatorCount = 1; + pixel.interpolatorSettings[0] = 0x400u; + pixel.inputAddr = 2; + pixel.hasPerspectiveCenterVgpr = true; + pixel.targetOutputMode[0] = 9; + pixel.targetExportMapping.fill(0xe4u); + ShaderRecompiler::RecompileRequest request{}; + request.shader = {ShaderRecompiler::ShaderStage::Fragment, reinterpret_cast(code.data()), code, 0, {}}; + request.context.waveSize = 64; + request.context.pixel = pixel; + request.target = target; + request.layout.pushConstantSizeBytes = 128; + request.useCache = false; + auto result = ShaderRecompiler::Recompile(request); + result.variantId = 3; + Require(result.fragmentParameters.size() == 1 && result.fragmentParameters[0].flat && !result.fragmentParameters[0].perVertex, "guest P0 shader did not produce a flat input"); + return result; +} + +void CheckPixel(std::uint32_t x, std::uint32_t y, const Color& expected, const std::string& name) { + for (std::uint32_t channel = 0; channel < expected.size(); ++channel) { + const auto actual = std::to_integer(Pixels[(y * Width + x) * 4 + channel]); + Require(actual == expected[channel], name + ": pixel (" + std::to_string(x) + ", " + std::to_string(y) + ") channel " + std::to_string(channel) + " is " + std::to_string(actual) + ", expected " + std::to_string(expected[channel])); + } +} + +void Run(AgcDriver::VulkanDevice& device, VkPrimitiveTopology topology, std::span shaders, bool useLast = true, std::string_view shaderName = "GLSL") { + Pixels.fill(Sentinel); + AgcDriver::Graphics::State state{}; + state.stages.path = AgcDriver::Graphics::ShaderPath::Vertex; + state.color = {reinterpret_cast(Pixels.data()), {Width, Height}, VK_FORMAT_R8G8B8A8_UNORM, Pixels.size(), 0xe4u}; + state.colors = {state.color}; + state.hasColorTarget = true; + state.renderExtent = {Width, Height}; + state.topology = topology; + state.scissor = {{0, 0}, {Width, Height}}; + state.frontFace = VK_FRONT_FACE_COUNTER_CLOCKWISE; + state.blend.colorWriteMask = 15; + state.blends = {state.blend}; + const bool strip = topology == VK_PRIMITIVE_TOPOLOGY_TRIANGLE_STRIP; + const AgcDriver::Pm4::DrawParameters draw{0, strip ? 4u : 6u, 0, 1, 0, false}; + for (std::uint32_t panel = 0; panel < 3; ++panel) { + state.viewport = {static_cast(panel * Panel), static_cast(Height), static_cast(Panel), -static_cast(Height), 0, 1}; + state.provokingVertexMode = panel == 1 && useLast ? VK_PROVOKING_VERTEX_MODE_LAST_VERTEX_EXT : VK_PROVOKING_VERTEX_MODE_FIRST_VERTEX_EXT; + device.Draw(state, draw, shaders); + } + device.WaitIdle(); + const auto name = std::string(shaderName) + (strip ? " triangle strip" : " triangle list"); + for (std::uint32_t panel = 0; panel < 3; ++panel) { + const bool last = panel == 1 && useLast; + const auto firstTriangle = last ? Blue : Red; + const auto secondTriangle = strip ? (last ? Yellow : Green) : (last ? Magenta : Yellow); + for (std::uint32_t offset = 0; offset < 4; ++offset) { + CheckPixel(panel * Panel + 16 + offset, 16, firstTriangle, name); + CheckPixel(panel * Panel + 48 - offset, 48, secondTriangle, name); + } + CheckPixel(panel * Panel, 0, {64, 64, 64, 64}, name + " background"); + } +} + +} + +int main() { + try { + const auto device = OpenVulkanTestDevice(); + if (!device) return VulkanTestSkipped; + const auto vertex = Shader(PROVOKING_VERTEX_VERT_SPV, 1); + const auto fragment = Shader(PROVOKING_VERTEX_FRAG_SPV, 2); + const std::array shaders{{ + {ShaderRecompiler::ShaderStage::Vertex, &vertex, 0}, + {ShaderRecompiler::ShaderStage::Fragment, &fragment, 0} + }}; + if (!device->ProvokingVertexLast()) { + for (const auto topology : {VK_PRIMITIVE_TOPOLOGY_TRIANGLE_LIST, VK_PRIMITIVE_TOPOLOGY_TRIANGLE_STRIP}) Run(*device, topology, shaders, false); + bool rejected = false; + try { + Run(*device, VK_PRIMITIVE_TOPOLOGY_TRIANGLE_LIST, shaders); + } catch (const std::exception& error) { + Require(std::string(error.what()).find("last provoking vertex requires VK_EXT_provoking_vertex with provokingVertexLast enabled") != std::string::npos, std::string("unexpected last-vertex rejection: ") + error.what()); + rejected = true; + } + Require(rejected, "last provoking vertex was drawn without device support"); + device->WaitIdle(); + Require(std::getenv("ANYPS5_REQUIRE_VULKAN") == nullptr, "required provoking vertex rendering coverage is unavailable: provokingVertexLast is not supported"); + std::puts("skipped last-vertex rendering, provokingVertexLast is unavailable; first-vertex rendering and feature rejection passed"); + return VulkanTestSkipped; + } + for (const auto topology : {VK_PRIMITIVE_TOPOLOGY_TRIANGLE_LIST, VK_PRIMITIVE_TOPOLOGY_TRIANGLE_STRIP}) Run(*device, topology, shaders); + const auto guestPixel = GuestPixel(device->Target()); + const std::array guestShaders{{ + {ShaderRecompiler::ShaderStage::Vertex, &vertex, 0}, + {ShaderRecompiler::ShaderStage::Fragment, &guestPixel, 0} + }}; + for (const auto topology : {VK_PRIMITIVE_TOPOLOGY_TRIANGLE_LIST, VK_PRIMITIVE_TOPOLOGY_TRIANGLE_STRIP}) Run(*device, topology, guestShaders, true, "guest P0"); + std::puts("provoking vertex rendering tests passed: GLSL and guest P0, triangle list and strip, first/last/first"); + return 0; + } catch (const std::exception& error) { + std::cerr << error.what() << '\n'; + return 1; + } +} diff --git a/core/libs/prx/libSceAgcDriver/tests/execution/SampledCacheBudget.cpp b/core/libs/prx/libSceAgcDriver/tests/execution/SampledCacheBudget.cpp index cf910ab2c..e7318fa76 100644 --- a/core/libs/prx/libSceAgcDriver/tests/execution/SampledCacheBudget.cpp +++ b/core/libs/prx/libSceAgcDriver/tests/execution/SampledCacheBudget.cpp @@ -1,6 +1,7 @@ #include "prx/libSceAgcDriver/Execution/include/BdaFeatures.hpp" #include "prx/libSceAgcDriver/Execution/include/GuestMemory.hpp" #include "prx/libSceAgcDriver/Graphics/include/GuestBufferMemory.hpp" +#include "prx/libSceAgcDriver/Graphics/include/Recorder.hpp" #include "prx/libSceAgcDriver/Graphics/include/ShaderResources.hpp" #include "prx/libSceAgcDriver/Graphics/include/Texture.hpp" #include "prx/libSceAgcDriver/Graphics/include/TextureDetiler.hpp" @@ -25,6 +26,7 @@ #include #include #include +#include #include namespace { @@ -39,6 +41,7 @@ constexpr std::size_t SurfaceBytes = std::size_t{Side} * Side * 4u; constexpr std::size_t Surfaces = 12; constexpr std::size_t BlockBytes = SurfaceBytes * Surfaces; constexpr std::uint64_t Budget = 1ull << 20u; +constexpr std::uint64_t Ceiling = Budget + Budget / 4u; class Device { public: @@ -160,7 +163,7 @@ public: #endif Require(memory != nullptr, "cannot allocate the texture block"); std::memset(memory, 0x5a, BlockBytes); - GuestAllocations::Mutation().Add(memory, BlockBytes, true, true); + GuestAllocations::Mutation().Add(memory, BlockBytes, true, true, true); } ~Block() { @@ -278,6 +281,162 @@ void snapshotTests(const Context& context) { Require(TextureCacheUsage().sampledBytes == tiny->AllocationBytes(), "a snapshot whose " + std::to_string(tiny->AllocationBytes()) + "-byte image is smaller than its " + std::to_string(guestBytes) + " guest bytes counted " + std::to_string(TextureCacheUsage().sampledBytes) + " bytes, not its allocation"); } +void heldTests(const Context& context) { + Block block(context); + auto held = Sampled(context, block.Surface(0)); + const auto bytes = TextureCacheUsage().sampledBytes; + const auto capacity = static_cast(Budget / bytes); + Require((capacity + 1u) * bytes <= Ceiling && capacity + 2u < Surfaces, "a snapshot of the 256 KiB surface counts " + std::to_string(bytes) + " bytes, which the test's ceiling does not fit as planned"); + std::vector> released; + for (std::size_t surface = 1; surface < capacity; ++surface) released.push_back(Sampled(context, block.Surface(surface))); + Sampled(context, block.Surface(capacity)); + Require(Sampled(context, block.Surface(0)) == held, "the sampled texture cache evicted a texture its caller still holds"); + Require(released.front().expired(), "the least recently used texture nobody holds was not evicted in place of the held one"); + held.reset(); + ClearCachedTextures(context.device); + std::vector> all; + for (std::size_t surface = 0; surface <= capacity; ++surface) all.push_back(Sampled(context, block.Surface(surface))); + Require(TextureCacheUsage().sampledEntries == capacity + 1u && TextureCacheUsage().sampledBytes > Budget, "textures their callers hold left a cache over its budget"); + for (std::size_t surface = 0; surface < all.size(); ++surface) Require(Sampled(context, block.Surface(surface)) == all[surface], "surface " + std::to_string(surface) + " was made again while its texture was held"); + all.push_back(Sampled(context, block.Surface(capacity + 1u))); + Require(TextureCacheUsage().sampledEntries == capacity + 1u && TextureCacheUsage().sampledBytes <= Ceiling, "held textures took the sampled texture cache past a quarter over its budget"); + for (std::size_t surface = 1; surface < all.size(); ++surface) Require(Sampled(context, block.Surface(surface)) == all[surface], "past the ceiling, surface " + std::to_string(surface) + " left in place of the least recently used held texture"); + all.clear(); + Sampled(context, block.Surface(capacity + 2u)); + Require(TextureCacheUsage().sampledEntries == capacity && TextureCacheUsage().sampledBytes <= Budget, "released textures were not evicted down to the budget"); +} + +void rescanTests(const Context& context) { + constexpr std::uint32_t SmallSide = 128; + constexpr std::size_t SmallBytes = std::size_t{SmallSide} * SmallSide * 4u; + Block block(context); + const auto small = [&](std::size_t surface) { return Sampled(context, block.Surface(0) + surface * SmallBytes, SmallSide); }; + std::vector> all{small(0)}; + const auto bytes = TextureCacheUsage().sampledBytes; + const auto capacity = static_cast(Budget / bytes); + Require((capacity + 3u) * bytes <= Ceiling && (capacity + 3u) * SmallBytes <= BlockBytes, "a snapshot of the 64 KiB surface counts " + std::to_string(bytes) + " bytes, which the test's ceiling does not fit as planned"); + for (std::size_t surface = 1; surface <= capacity; ++surface) all.push_back(small(surface)); + const std::weak_ptr newest = all.back(); + all.pop_back(); + all.push_back(small(capacity + 1u)); + std::this_thread::sleep_for(std::chrono::milliseconds(1100)); + all.push_back(small(capacity + 2u)); + Require(newest.expired(), "a released texture behind held ones was still cached a second later"); + Require(TextureCacheUsage().sampledEntries == capacity + 2u, "the rescan left " + std::to_string(TextureCacheUsage().sampledEntries) + " entries, not the " + std::to_string(capacity + 2u) + " textures still held"); +} + +void viewEvictionTests(const Context& context) { + if (context.hostImportAlignment == 0) return; + Block snapshots(context); + Block imported(context); + if (HostImportFor(context, imported.Surface(0), BlockBytes) == nullptr) return; + const std::weak_ptr view = Sampled(context, imported.Surface(0)); + Require(!view.expired() && TextureCacheUsage().sampledBytes == 0, "the view of a cached storage image did not enter the sampled texture cache at 0 bytes"); + Sampled(context, snapshots.Surface(0)); + const auto capacity = static_cast(Budget / TextureCacheUsage().sampledBytes); + for (std::size_t surface = 1; surface <= capacity; ++surface) Sampled(context, snapshots.Surface(surface)); + Require(!view.expired(), "the sampled texture cache evicted a view of a cached storage image, which frees no memory"); +} + +void frameTests(const Context& context) { + Block block(context); + ClearCachedTextures(context.device); + Recorder::CountPresent(); + const auto bytes = Sampled(context, block.Surface(0))->AllocationBytes(); + const auto capacity = static_cast(Budget / bytes); + const auto frame = capacity + 2u; + Require(capacity >= 2u && 2u * frame <= Surfaces && TextureCacheUsage().sampledEntries == 1u, "a snapshot of the 256 KiB surface holds " + std::to_string(bytes) + " bytes, which the test's 1 MiB budget does not fit as planned"); + std::vector> made(frame); + for (std::size_t surface = 0; surface < frame; ++surface) made[surface] = Sampled(context, block.Surface(surface)); + Require(TextureCacheUsage().sampledEntries == capacity, "a frame of " + std::to_string(frame) + " textures left " + std::to_string(TextureCacheUsage().sampledEntries) + " cached, not the " + std::to_string(capacity) + " the budget holds"); + for (std::size_t surface = capacity; surface < frame; ++surface) Require(made[surface].expired(), "surface " + std::to_string(surface) + ", used once in a frame whose textures fill the cache, was cached in place of an earlier one"); + Recorder::CountPresent(); + std::size_t hits = 0; + for (std::size_t surface = 0; surface < frame; ++surface) { + const auto again = Sampled(context, block.Surface(surface)); + hits += again == made[surface].lock(); + } + Require(hits == capacity, "the next frame over the same " + std::to_string(frame) + " textures found " + std::to_string(hits) + " of them cached, not the " + std::to_string(capacity) + " the cache kept"); + auto admitted = Sampled(context, block.Surface(frame - 1u)); + Require(Sampled(context, block.Surface(frame - 1u)) == admitted, "a texture used a second time in the frame was not cached"); + Require(TextureCacheUsage().sampledEntries == capacity && TextureCacheUsage().sampledBytes <= Budget, "the texture admitted on its second use left the cache over its budget"); + admitted.reset(); + Recorder::CountPresent(); + std::vector> moved(frame); + for (std::size_t surface = 0; surface < frame; ++surface) moved[surface] = Sampled(context, block.Surface(frame + surface)); + for (std::size_t surface = 0; surface < frame; ++surface) Require(moved[surface].expired(), "surface " + std::to_string(frame + surface) + ", first used in the frame after the cached textures were, was cached in place of one of them"); + Require(TextureCacheUsage().sampledEntries == capacity, "a frame over other textures left " + std::to_string(TextureCacheUsage().sampledEntries) + " of the previous frame's textures cached"); + Recorder::CountPresent(); + for (std::size_t surface = 0; surface < frame; ++surface) moved[surface] = Sampled(context, block.Surface(frame + surface)); + for (std::size_t surface = 0; surface < capacity; ++surface) Require(!moved[surface].expired(), "surface " + std::to_string(frame + surface) + " of a second frame over other textures did not replace the textures last used two frames before"); + Require(TextureCacheUsage().sampledEntries == capacity, "a second frame over other textures left " + std::to_string(TextureCacheUsage().sampledEntries) + " entries cached"); +} + +void earlyMissTests(const Context& context) { + Block block(context); + ClearCachedTextures(context.device); + Recorder::CountPresent(); + const auto bytes = Sampled(context, block.Surface(0))->AllocationBytes(); + const auto capacity = static_cast(Budget / bytes); + Require(capacity >= 2u && capacity < Surfaces && TextureCacheUsage().sampledEntries == 1u, "a snapshot of the 256 KiB surface holds " + std::to_string(bytes) + " bytes, which the test's 1 MiB budget does not fit as planned"); + std::vector> cached(capacity); + for (std::size_t surface = 0; surface < capacity; ++surface) cached[surface] = Sampled(context, block.Surface(surface)); + Recorder::CountPresent(); + for (std::size_t surface = 0; surface < capacity; ++surface) { + const auto again = Sampled(context, block.Surface(surface)); + Require(again == cached[surface].lock(), "surface " + std::to_string(surface) + " missed in a frame over the textures the cache holds"); + } + Recorder::CountPresent(); + Sampled(context, block.Surface(capacity)); + std::size_t hits = 0; + for (std::size_t surface = 0; surface < capacity; ++surface) { + const auto again = Sampled(context, block.Surface(surface)); + hits += again == cached[surface].lock(); + } + Require(hits == capacity, "a frame that first uses a new texture before its " + std::to_string(capacity) + " cached ones found " + std::to_string(hits) + " of them cached"); +} + +constexpr std::uint64_t KeyHashBasis = 14695981039346656037ull; +constexpr std::uint64_t KeyHashPrime = 1099511628211ull; + +std::array, 2> CollidingDescriptors(VkDevice device, std::uint64_t address) { + const auto words = Descriptor(address); + auto prefix = KeyHashBasis ^ reinterpret_cast(device); + for (std::size_t word = 0; word < 6; ++word) prefix = (prefix ^ words[word]) * KeyHashPrime; + std::unordered_map word6ByHighHalf; + for (std::uint32_t candidate = 0; candidate < 0x200u; ++candidate) { + const auto word6 = ((candidate >> 1u) << 24u) | (candidate & 1u); + const auto hash = (prefix ^ word6) * KeyHashPrime; + const auto [match, added] = word6ByHighHalf.emplace(static_cast(hash >> 32u), word6); + if (added) continue; + auto first = words; + auto second = words; + first[6] = match->second; + second[6] = word6; + second[7] = static_cast(hash ^ ((prefix ^ match->second) * KeyHashPrime)); + return {first, second}; + } + throw std::runtime_error("no two descriptors of the surface that differ only in word 6's statistics counter and the unused meta address collide in TextureKeyHash"); +} + +void collidingKeyTests(const Context& context) { + Block block(context); + ClearCachedTextures(context.device); + Recorder::CountPresent(); + const auto bytes = Sampled(context, block.Surface(0))->AllocationBytes(); + const auto capacity = static_cast(Budget / bytes); + Require(capacity >= 2u && capacity < Surfaces && TextureCacheUsage().sampledEntries == 1u, "a snapshot of the 256 KiB surface holds " + std::to_string(bytes) + " bytes, which the test's 1 MiB budget does not fit as planned"); + std::vector> cached(capacity); + for (std::size_t surface = 0; surface < capacity; ++surface) cached[surface] = Sampled(context, block.Surface(surface)); + Recorder::CountPresent(); + const auto [first, second] = CollidingDescriptors(context.device, block.Surface(capacity)); + CachedSampledTexture(context, first); + CachedSampledTexture(context, second); + std::size_t kept = 0; + for (const auto& texture : cached) kept += !texture.expired(); + Require(kept == capacity && TextureCacheUsage().sampledEntries == capacity, "the first use of a descriptor whose cache key hash equals that of another descriptor first used in the frame left " + std::to_string(kept) + " of the " + std::to_string(capacity) + " textures the last frame used cached"); +} + constexpr std::uint64_t ReportedBudget = 64ull << 30u; constexpr std::uint64_t ReportedUsage = 12ull << 30u; @@ -350,7 +509,13 @@ int main(int argc, char** argv) { std::lock_guard gpu(AgcDriver::GuestMemory::GpuMutex()); reportedBudgetTests(context); snapshotTests(context); + heldTests(context); + rescanTests(context); + viewEvictionTests(context); viewTests(context); + frameTests(context); + earlyMissTests(context); + collidingKeyTests(context); ClearCachedTextures(context.device); } std::puts("sampled texture cache budget tests passed"); diff --git a/core/libs/prx/libSceAgcDriver/tests/execution/ScalarSwappcFetch.cpp b/core/libs/prx/libSceAgcDriver/tests/execution/ScalarSwappcFetch.cpp index 158fb8d50..a5ad23d65 100644 --- a/core/libs/prx/libSceAgcDriver/tests/execution/ScalarSwappcFetch.cpp +++ b/core/libs/prx/libSceAgcDriver/tests/execution/ScalarSwappcFetch.cpp @@ -2,6 +2,7 @@ #include "prx/libSceAgcDriver/Graphics/include/Draw.hpp" #include "Recompiler.hpp" #include "VulkanTestDevice.hpp" +#include #include #include #include @@ -35,9 +36,11 @@ constexpr std::array, 3> Triangle{{ {-1.0f, -1.0f, 0.5f, 1.0f}, {3.0f, -1.0f, 0.25f, 1.0f}, {-1.0f, 3.0f, 0.75f, 1.0f} }}; -std::array VertexBufferDescriptor(std::uint32_t select) { +alignas(256) constexpr std::array RestartIndices{0xffff, 0xffff, 0xffff, 0xffff, 0xffff, 0xffff}; + +std::array VertexBufferDescriptor(std::uint32_t select, std::uint32_t records) { const auto address = reinterpret_cast(Triangle.data()); - return {static_cast(address), static_cast((address >> 32u) & 0xffffu) | (16u << 16u), static_cast(Triangle.size()), 0x0104d000u | select}; + return {static_cast(address), static_cast((address >> 32u) & 0xffffu) | (16u << 16u), records, 0x0104d000u | select}; } std::array OutputDescriptor() { @@ -56,10 +59,10 @@ ShaderRecompiler::RecompileRequest VertexRequest(const AgcDriver::VulkanDevice& return request; } -ShaderRecompiler::ShaderVertexStageInfo VertexInfo(std::uint32_t select, std::int32_t registers) { +ShaderRecompiler::ShaderVertexStageInfo VertexInfo(std::uint32_t select, std::int32_t registers, std::uint32_t records = static_cast(Triangle.size())) { ShaderRecompiler::ShaderVertexStageInfo info{}; info.resourcesNum = 1; - info.resources[0].fields = VertexBufferDescriptor(select); + info.resources[0].fields = VertexBufferDescriptor(select, records); info.resourcesDst[0] = {8, registers, 0, 0}; info.fetchAttribReg = 8; info.fetchBufferReg = 10; @@ -76,12 +79,12 @@ std::vector UserData() { return userData; } -void Draw(AgcDriver::VulkanDevice& device, std::uint32_t select) { +void Draw(AgcDriver::VulkanDevice& device, std::uint32_t select, bool onlyRestart = false) { Pixels.fill(std::byte{0x40}); Output.fill(0xdeadbeefu); const auto userData = UserData(); const std::array vertexMemory{{{reinterpret_cast(VertexCode.data()), std::as_bytes(std::span(VertexCode))}}}; - const auto vertexResult = ShaderRecompiler::Recompile(VertexRequest(device, userData, VertexInfo(select, 4), vertexMemory)); + const auto vertexResult = ShaderRecompiler::Recompile(VertexRequest(device, userData, VertexInfo(select, 4, onlyRestart ? 0u : static_cast(Triangle.size())), vertexMemory)); const auto vertexPush = static_cast(vertexResult.pushConstants.size()); ShaderRecompiler::ShaderPixelStageInfo pixel{}; @@ -96,13 +99,13 @@ void Draw(AgcDriver::VulkanDevice& device, std::uint32_t select) { {ShaderStage::Fragment, reinterpret_cast(PixelCode.data()), PixelCode, 0, {}}, {64u, 0, pixelUserData, std::nullopt, pixel, std::nullopt, pixelMemory}, device.Target(), - {0, 0, vertexPush, 128 - vertexPush} + PixelPushLayout(vertexPush, device.Target()) }; fragment.useCache = false; const auto pixelResult = ShaderRecompiler::Recompile(fragment); const std::array shaders{{ {ShaderStage::Vertex, &vertexResult, 0}, - {ShaderStage::Fragment, &pixelResult, vertexPush} + {ShaderStage::Fragment, &pixelResult, PixelPushOffset(vertexPush, device.Target())} }}; AgcDriver::Graphics::State state{}; @@ -111,7 +114,8 @@ void Draw(AgcDriver::VulkanDevice& device, std::uint32_t select) { state.colors = {state.color}; state.hasColorTarget = true; state.renderExtent = {Width, Height}; - state.topology = VK_PRIMITIVE_TOPOLOGY_TRIANGLE_LIST; + state.topology = onlyRestart ? VK_PRIMITIVE_TOPOLOGY_TRIANGLE_STRIP : VK_PRIMITIVE_TOPOLOGY_TRIANGLE_LIST; + state.primitiveRestart = onlyRestart; state.viewport = {0, static_cast(Height), static_cast(Width), -static_cast(Height), 0, 1}; state.negativeOneToOne = false; state.scissor = {{0, 0}, {Width, Height}}; @@ -120,7 +124,7 @@ void Draw(AgcDriver::VulkanDevice& device, std::uint32_t select) { state.blend.colorWriteMask = 15; state.blends = {state.blend}; state.blendConstants = {}; - const AgcDriver::Pm4::DrawParameters draw{0, static_cast(Triangle.size()), 0, 1, 0, false}; + const auto draw = onlyRestart ? AgcDriver::Pm4::DrawParameters{reinterpret_cast(RestartIndices.data()), static_cast(RestartIndices.size()), 2, 1, 0, true} : AgcDriver::Pm4::DrawParameters{0, static_cast(Triangle.size()), 0, 1, 0, false}; device.Draw(state, draw, shaders); device.WaitIdle(); } @@ -159,6 +163,8 @@ int main() { Draw(*device, ReverseSelect); Check(ReverseSelect, "fetch-shader call, reversed dst_sel"); CheckRegisterLimit(*device); + Draw(*device, IdentitySelect, true); + Require(std::all_of(Output.begin(), Output.end(), [](std::uint32_t word) { return word == 0xdeadbeefu; }) && std::all_of(Pixels.begin(), Pixels.end(), [](std::byte value) { return value == std::byte{0x40}; }), "an indexed draw of only restart indices ran its vertices or wrote pixels"); std::puts("scalar swappc fetch tests passed"); return 0; } catch (const std::exception& error) { diff --git a/core/libs/prx/libSceAgcDriver/tests/execution/ScalarWideLoads.cpp b/core/libs/prx/libSceAgcDriver/tests/execution/ScalarWideLoads.cpp index 97f04ea30..ba7fa6821 100644 --- a/core/libs/prx/libSceAgcDriver/tests/execution/ScalarWideLoads.cpp +++ b/core/libs/prx/libSceAgcDriver/tests/execution/ScalarWideLoads.cpp @@ -159,7 +159,7 @@ public: block = static_cast(std::aligned_alloc(BlockBytes, BlockBytes)); #endif Require(block != nullptr, "scalar wide loads: cannot allocate the guest block"); - GuestAllocations::Mutation().Add(block, BlockBytes, true, false); + GuestAllocations::Mutation().Add(block, BlockBytes, true, false, true); } ~GuestBlock() { diff --git a/core/libs/prx/libSceAgcDriver/tests/execution/SdwaDestination.cpp b/core/libs/prx/libSceAgcDriver/tests/execution/SdwaDestination.cpp index 59a1b5490..7a27e485a 100644 --- a/core/libs/prx/libSceAgcDriver/tests/execution/SdwaDestination.cpp +++ b/core/libs/prx/libSceAgcDriver/tests/execution/SdwaDestination.cpp @@ -17,21 +17,22 @@ using ShaderRecompiler::ShaderStage; constexpr std::uint32_t Threads = 64; constexpr std::uint32_t Inputs = 4; -constexpr std::uint32_t Results = 8; +constexpr std::uint32_t Results = 16; constexpr std::uint32_t Garbage = 0xabcd1234u; alignas(256) std::array Input{}; alignas(256) std::array Output{}; -alignas(256) constexpr std::array SdwaCode{ - 0x34020082, 0x34060083, 0xe0302000, 0x80000401, 0xe0302004, 0x80000501, 0xe0302008, 0x80000601, +alignas(256) constexpr std::array SdwaCode{ + 0x34020082, 0x34060084, 0xe0302000, 0x80000401, 0xe0302004, 0x80000501, 0xe0302008, 0x80000601, 0x7e1402ff, 0xabcd1234, 0x7e1602ff, 0xabcd1234, 0x7e1802ff, 0xabcd1234, 0x7e1a02ff, 0xabcd1234, 0x7e1c02ff, 0xabcd1234, 0x7e1e02ff, 0xabcd1234, 0x7e2002ff, 0xabcd1234, 0x7e2202ff, 0xabcd1234, 0x7e2402ff, 0xabcd1234, 0xbf8c3f70, 0x7e1414f9, 0x00060604, 0x7e161504, 0x7e1814f9, 0x00061505, 0x7e1a14f9, 0x00060604, 0x7e1c14f9, 0x00060605, 0xd746000d, 0x0435210e, 0x641e0cf9, 0x05040606, 0x7e2002f9, 0x00061004, 0x7e2202f9, 0x00061204, 0x7e2402f9, 0x00060104, + 0x7e2602ff, 0xabcd1234, 0x7e266ef9, 0x00061404, 0xe0702000, 0x80010a03, 0xe0702004, 0x80010b03, 0xe0702008, 0x80010c03, 0xe070200c, 0x80010d03, 0xe0702010, 0x80010f03, 0xe0702014, 0x80011003, 0xe0702018, 0x80011103, - 0xe070201c, 0x80011203, 0xbf810000, + 0xe070201c, 0x80011203, 0xe0702020, 0x80011303, 0xbf810000, }; float Quarter(std::uint32_t index) { @@ -106,6 +107,8 @@ void Check() { Require(out[5] == ((Garbage & 0xffffff00u) | low), where(5) + ", expected " + Hex((Garbage & 0xffffff00u) | low) + ": v_mov_b32_sdwa dst_sel:BYTE_0 dst_unused:UNUSED_PRESERVE must keep the other bytes"); Require(out[6] == ((Garbage & 0xff00ffffu) | (low << 16u)), where(6) + ", expected " + Hex((Garbage & 0xff00ffffu) | (low << 16u)) + ": v_mov_b32_sdwa dst_sel:BYTE_2 dst_unused:UNUSED_PRESERVE must keep the other bytes"); Require(out[7] == (low << 8u), where(7) + ", expected " + Hex(low << 8u) + ": v_mov_b32_sdwa dst_sel:BYTE_1 dst_unused:UNUSED_PAD must zero the other bytes"); + const auto inverted = ~in[0] & 0xffffu; + Require(out[8] == ((Garbage & 0xffff0000u) | inverted), where(8) + ", expected " + Hex((Garbage & 0xffff0000u) | inverted) + ": v_not_b32_sdwa dst_sel:WORD_0 dst_unused:UNUSED_PRESERVE must keep the high half"); } } diff --git a/core/libs/prx/libSceAgcDriver/tests/execution/ShaderClock.cpp b/core/libs/prx/libSceAgcDriver/tests/execution/ShaderClock.cpp index 88557c5e5..e3d6569b4 100644 --- a/core/libs/prx/libSceAgcDriver/tests/execution/ShaderClock.cpp +++ b/core/libs/prx/libSceAgcDriver/tests/execution/ShaderClock.cpp @@ -22,6 +22,7 @@ constexpr std::uint32_t Inputs = 4; constexpr std::uint32_t Results = 16; alignas(256) std::array Input{}; alignas(256) std::array Output{}; +bool WrappingClock = false; alignas(256) constexpr std::array Code{ 0x34020084, 0x34060086, 0xe0301000, 0x80000401, 0xf4900200, 0x00000000, 0xf4940280, 0x00000000, @@ -41,6 +42,11 @@ std::uint64_t Pair(const std::uint32_t* words) { return static_cast(words[0]) | (static_cast(words[1]) << 32u); } +bool Advanced(const std::uint32_t* before, const std::uint32_t* after) { + if (WrappingClock) return static_cast(after[0] - before[0]) < 0x80000000u; + return Pair(after) >= Pair(before); +} + std::array BufferDescriptor(const void* data, std::uint32_t bytes) { const auto address = reinterpret_cast(data); return {static_cast(address), static_cast((address >> 32u) & 0xffffu), bytes, 0x31016facu}; @@ -62,8 +68,8 @@ void Check() { const std::uint32_t* out = &Output[tid * Results]; const std::uint32_t* first = &Output[(tid & ~7u) * Results]; for (std::uint32_t index = 0; index < 8; ++index) Expect(tid, out[index], first[index], "clock read uniform over the wave"); - Require(Pair(out + 4) >= Pair(out), "memtime: lane " + std::to_string(tid) + " went back from " + std::to_string(Pair(out)) + " to " + std::to_string(Pair(out + 4))); - Require(Pair(out + 6) >= Pair(out + 2), "memrealtime: lane " + std::to_string(tid) + " went back from " + std::to_string(Pair(out + 2)) + " to " + std::to_string(Pair(out + 6))); + Require(Advanced(out, out + 4), "memtime: lane " + std::to_string(tid) + " went back from " + std::to_string(Pair(out)) + " to " + std::to_string(Pair(out + 4))); + Require(Advanced(out + 2, out + 6), "memrealtime: lane " + std::to_string(tid) + " went back from " + std::to_string(Pair(out + 2)) + " to " + std::to_string(Pair(out + 6))); Require(Pair(out + 4) != 0u && Pair(out + 6) != 0u, "clocks read zero"); std::uint32_t sum = 0u; for (std::uint32_t step = 0; step < 2000u; ++step) sum = (sum & 0xffffffu) * 3u + in[0]; @@ -104,6 +110,7 @@ int main() { try { const auto device = OpenVulkanTestDevice(); if (!device) return VulkanTestSkipped; + WrappingClock = device->DeviceName().starts_with("llvmpipe"); if (!TargetHasCapability(device->Target(), spv::CapabilityShaderClockKHR)) { std::puts("skipped, the device lacks shaderSubgroupClock or shaderDeviceClock"); return VulkanTestSkipped; diff --git a/core/libs/prx/libSceAgcDriver/tests/execution/SinCosNearZero.cpp b/core/libs/prx/libSceAgcDriver/tests/execution/SinCosNearZero.cpp new file mode 100644 index 000000000..5b5f48130 --- /dev/null +++ b/core/libs/prx/libSceAgcDriver/tests/execution/SinCosNearZero.cpp @@ -0,0 +1,190 @@ +#include "prx/libSceAgcDriver/Execution/include/VulkanDevice.hpp" +#include "prx/libSceAgcDriver/Graphics/include/Draw.hpp" +#include "Recompiler.hpp" +#include "VulkanTestDevice.hpp" +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace { + +using AgcDriver::Graphics::Require; +using ShaderRecompiler::ShaderStage; + +constexpr std::uint32_t Threads = 32; +constexpr std::uint32_t Inputs = 4; +constexpr std::uint32_t Results = 4; +constexpr std::uint32_t Columns = 4; +alignas(256) std::array Input{}; +alignas(256) std::array Output{}; + +alignas(256) constexpr std::array Code{ + 0x34020084u, 0xe0381000u, 0x80000401u, 0xbf8c3f70u, 0x7e140280u, 0x7e160280u, 0x7e180280u, 0x7e1a0280u, + 0x7e146b04u, 0x7e166d04u, 0x7e18c105u, 0x7e1ac305u, 0xe0781000u, 0x80010a01u, 0xbf810000u, +}; + +constexpr std::uint32_t Rows[Threads][4] = { + {0x80800000u, 0x00008001u, 0x00000000u, 0x00000000u}, + {0x8d800000u, 0x00008400u, 0x00000000u, 0x00000000u}, + {0xb0800000u, 0x00009000u, 0x00000000u, 0x00000000u}, + {0xb58637bdu, 0x0000868eu, 0x00000000u, 0x00000000u}, + {0x00800000u, 0x00000001u, 0x00000000u, 0x00000000u}, + {0x358637bdu, 0x00000400u, 0x00000000u, 0x00000000u}, + {0xb87fda40u, 0x000083ffu, 0x00000000u, 0x00000000u}, + {0x387fda40u, 0x000003ffu, 0x00000000u, 0x00000000u}, + {0xbeffffffu, 0x0000b7ffu, 0x00000000u, 0x00000000u}, + {0x3effffffu, 0x000037ffu, 0x00000000u, 0x00000000u}, + {0x3f000000u, 0x00003800u, 0x00000000u, 0x00000000u}, + {0xbf000000u, 0x0000b800u, 0x00000000u, 0x00000000u}, + {0x3e7fffffu, 0x000033ffu, 0x00000000u, 0x00000000u}, + {0xbe7fffffu, 0x0000b3ffu, 0x00000000u, 0x00000000u}, + {0x3e800000u, 0x00003400u, 0x00000000u, 0x00000000u}, + {0x3f400000u, 0x00003a00u, 0x00000000u, 0x00000000u}, + {0x437f8000u, 0x00005bfcu, 0x00000000u, 0x00000000u}, + {0x43802000u, 0x00005c01u, 0x00000000u, 0x00000000u}, + {0xc47a2000u, 0x0000e3d1u, 0x00000000u, 0x00000000u}, + {0x7f7fc99eu, 0x00007bffu, 0x00000000u, 0x00000000u}, + {0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u}, + {0x80000000u, 0x00008000u, 0x00000000u, 0x00000000u}, + {0x7f800000u, 0x00007c00u, 0x00000000u, 0x00000000u}, + {0x7fc00000u, 0x00007e00u, 0x00000000u, 0x00000000u}, + {0x3dcccccdu, 0x00002e66u, 0x00000000u, 0x00000000u}, + {0xbdcccccdu, 0x0000ae66u, 0x00000000u, 0x00000000u}, + {0x3e99999au, 0x000034cdu, 0x00000000u, 0x00000000u}, + {0xbf333333u, 0x0000b99au, 0x00000000u, 0x00000000u}, + {0x3fa66666u, 0x00003d33u, 0x00000000u, 0x00000000u}, + {0xc0266666u, 0x0000c133u, 0x00000000u, 0x00000000u}, + {0x41240000u, 0x00004920u, 0x00000000u, 0x00000000u}, + {0xc2c84000u, 0x0000d642u, 0x00000000u, 0x00000000u} +}; + +constexpr std::uint32_t Expected[Threads][Columns] = { + {0x81c90fd5u, 0x3f800000u, 0x00008006u, 0x00003c00u}, + {0x8ec90fd5u, 0x3f800000u, 0x00008e48u, 0x00003c00u}, + {0xb1c90fd5u, 0x3f800000u, 0x00009a48u, 0x00003c00u}, + {0xb6d2d421u, 0x3f800000u, 0x00009126u, 0x00003c00u}, + {0x01c90fd5u, 0x3f800000u, 0x00000006u, 0x00003c00u}, + {0x36d2d421u, 0x3f800000u, 0x00000e48u, 0x00003c00u}, + {0xb9c8f22fu, 0x3f7ffffeu, 0x00008e47u, 0x00003c00u}, + {0x39c8f22fu, 0x3f7ffffeu, 0x00000e47u, 0x00003c00u}, + {0xb4490fdcu, 0xbf800000u, 0x00009648u, 0x0000bc00u}, + {0x34490fdcu, 0xbf800000u, 0x00001648u, 0x0000bc00u}, + {0x00000000u, 0xbf800000u, 0x00000000u, 0x0000bc00u}, + {0x00000000u, 0xbf800000u, 0x00000000u, 0x0000bc00u}, + {0x3f800000u, 0x33c90fd9u, 0x00003c00u, 0x00001248u}, + {0xbf800000u, 0x33c90fd9u, 0x0000bc00u, 0x00001248u}, + {0x3f800000u, 0x00000000u, 0x00003c00u, 0x00000000u}, + {0xbf800000u, 0x00000000u, 0x0000bc00u, 0x00000000u}, + {0x00000000u, 0xbf800000u, 0x00000000u, 0x0000bc00u}, + {0x3f800000u, 0x00000000u, 0x00003c00u, 0x00000000u}, + {0x00000000u, 0xbf800000u, 0x00000000u, 0x0000bc00u}, + {0x00000000u, 0x3f800000u, 0x00000000u, 0x00003c00u}, + {0x00000000u, 0x3f800000u, 0x00000000u, 0x00003c00u}, + {0x80000000u, 0x3f800000u, 0x00008000u, 0x00003c00u}, + {0xffc00000u, 0xffc00000u, 0x0000fe00u, 0x0000fe00u}, + {0x7fc00000u, 0x7fc00000u, 0x00007e00u, 0x00007e00u}, + {0x3f167919u, 0x3f4f1bbdu, 0x000038b4u, 0x00003a79u}, + {0xbf167919u, 0x3f4f1bbdu, 0x0000b8b4u, 0x00003a79u}, + {0x3f737870u, 0xbe9e377cu, 0x00003b9cu, 0x0000b4f3u}, + {0x3f737870u, 0xbe9e377cu, 0x00003b9du, 0x0000b4edu}, + {0x3f737872u, 0xbe9e3770u, 0x00003b9du, 0x0000b4edu}, + {0x3f167910u, 0xbf4f1bc3u, 0x000038b0u, 0x0000ba7cu}, + {0x3f800000u, 0x00000000u, 0x00003c00u, 0x00000000u}, + {0xbf3504f4u, 0x3f3504f3u, 0x0000b9a8u, 0x000039a8u} +}; + +constexpr const char* Names[Columns] = {"v_sin_f32", "v_cos_f32", "v_sin_f16", "v_cos_f16"}; +constexpr std::uint32_t Tolerance[Threads] = {0u, 0u, 0u, 0u, 0u, 0u, 2u, 2u, 1u, 1u, 0u, 0u, 2u, 2u, 0u, 0u, 0u, 0u, 0u, 0u, 0u, 0u, 0u, 0u, 12u, 12u, 12u, 12u, 12u, 12u, 12u, 12u}; + +std::array BufferDescriptor(const void* data, std::uint32_t bytes) { + const auto address = reinterpret_cast(data); + return {static_cast(address), static_cast((address >> 32u) & 0xffffu), bytes, 0x31016facu}; +} + +std::string Hex(std::uint32_t value) { + char text[16]; + std::snprintf(text, sizeof(text), "0x%08x", value); + return text; +} + +std::uint32_t Ordered(std::uint32_t bits, std::uint32_t width) { + const std::uint32_t sign = 1u << (width - 1u); + return (bits & sign) != 0u ? sign - (bits & ~sign) : bits | sign; +} + +std::uint32_t Distance(std::uint32_t actual, std::uint32_t expected, std::uint32_t width) { + const std::uint32_t a = Ordered(actual, width); + const std::uint32_t e = Ordered(expected, width); + return a > e ? a - e : e - a; +} + +bool IsPinned(std::uint32_t bits, std::uint32_t width) { + const std::uint32_t magnitude = width == 16u ? bits & 0x7fffu : bits & 0x7fffffffu; + const std::uint32_t infinity = width == 16u ? 0x7c00u : 0x7f800000u; + return magnitude == 0u || magnitude >= infinity; +} + +void Run(AgcDriver::VulkanDevice& device, const std::optional& floatMode) { + for (std::uint32_t tid = 0; tid < Threads; ++tid) std::copy(std::begin(Rows[tid]), std::end(Rows[tid]), &Input[tid * Inputs]); + Output.fill(0xdeadbeefu); + std::vector userData(8, 0u); + const auto input = BufferDescriptor(Input.data(), static_cast(Input.size() * 4u)); + const auto output = BufferDescriptor(Output.data(), static_cast(Output.size() * 4u)); + std::copy(input.begin(), input.end(), userData.begin()); + std::copy(output.begin(), output.end(), userData.begin() + 4); + const std::span code(Code); + const std::array memory{{reinterpret_cast(code.data()), std::as_bytes(code)}}; + const ShaderRecompiler::ShaderComputeStageInfo compute{{Threads, 1, 1}, 0u, {false, false, false}, false, 1}; + ShaderRecompiler::RecompileRequest request{ + {ShaderStage::Compute, reinterpret_cast(code.data()), code, 0, {}}, + {32, 0, userData, compute, std::nullopt, std::nullopt, memory}, + device.Target(), + {0, 0, 0, 128} + }; + request.useCache = false; + request.context.floatMode = floatMode; + const auto result = ShaderRecompiler::Recompile(request); + device.Dispatch(result, 1, 1, 1, {}, reinterpret_cast(code.data())); + device.WaitIdle(); +} + +void Check() { + for (std::uint32_t tid = 0; tid < Threads; ++tid) { + const std::uint32_t* out = &Output[tid * Results]; + for (std::uint32_t column = 0; column < Columns; ++column) { + const std::uint32_t width = column < 2u ? 32u : 16u; + const std::uint32_t mask = width == 16u ? 0xffffu : 0xffffffffu; + const std::uint32_t actual = out[column] & mask; + const std::uint32_t expected = Expected[tid][column] & mask; + const std::string name = "sin cos near zero: lane " + std::to_string(tid) + " " + Names[column] + " is " + Hex(actual) + ", expected " + Hex(expected); + if (IsPinned(expected, width)) { + Require(actual == expected, name); + continue; + } + Require(Distance(actual, expected, width) <= Tolerance[tid], name + " (tolerance " + std::to_string(Tolerance[tid]) + " ulp)"); + } + } +} + +} + +int main() { + try { + const auto device = OpenVulkanTestDevice(); + if (!device) return VulkanTestSkipped; + Run(*device, ShaderRecompiler::ShaderFloatMode{0xc0u, true, false, false}); + Check(); + std::puts("sin cos near zero tests passed"); + return 0; + } catch (const std::exception& error) { + std::cerr << error.what() << '\n'; + return 1; + } +} diff --git a/core/libs/prx/libSceAgcDriver/tests/execution/SinCosReduction.cpp b/core/libs/prx/libSceAgcDriver/tests/execution/SinCosReduction.cpp new file mode 100644 index 000000000..f4e1ed0b1 --- /dev/null +++ b/core/libs/prx/libSceAgcDriver/tests/execution/SinCosReduction.cpp @@ -0,0 +1,145 @@ +#include "prx/libSceAgcDriver/Execution/include/VulkanDevice.hpp" +#include "prx/libSceAgcDriver/Graphics/include/Draw.hpp" +#include "Recompiler.hpp" +#include "VulkanTestDevice.hpp" +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace { + +using AgcDriver::Graphics::Require; +using ShaderRecompiler::ShaderStage; + +constexpr std::uint32_t Threads = 32; +constexpr std::uint32_t Results = 2; +alignas(256) std::array Input{}; +alignas(256) std::array Output{}; + +alignas(256) constexpr std::array Code{ + 0x34020081, 0xe0302000, 0x80000400, 0xbf8c3f70, 0x7e146b04, 0x7e166d04, 0xe0702000, 0x80010a01, + 0xe0702004, 0x80010b01, 0xbf810000, +}; + +struct Row { + std::uint32_t x, sin, cos; +}; + +constexpr std::array Rows{{ + {0x80800000u, 0x81c90fd5u, 0x3f800000u}, + {0x8f800000u, 0x90c90fd5u, 0x3f800000u}, + {0xa3d42337u, 0xa5269cc5u, 0x3f800000u}, + {0xb0800000u, 0xb1c90fd5u, 0x3f800000u}, + {0xb2800000u, 0xb3c90fd5u, 0x3f800000u}, + {0xb3000000u, 0xb4490fd5u, 0x3f800000u}, + {0xb3800000u, 0xb4c90fd5u, 0x3f800000u}, + {0xb3c00000u, 0xb516cbe0u, 0x3f800000u}, + {0xb8732b55u, 0xb9befc10u, 0x3f7ffffeu}, + {0xbc23d70au, 0xbd80984du, 0x3f7f7eadu}, + {0xbe82a4d7u, 0xbf7fdd81u, 0xbd04dfbau}, + {0xbe000000u, 0xbf3504f4u, 0x3f3504f3u}, + {0xbeb33333u, 0xbf4f1bbdu, 0xbf167918u}, + {0xbef5c28fu, 0xbe005760u, 0xbf7dfb3au}, + {0x00800000u, 0x01c90fd5u, 0x3f800000u}, + {0x30800000u, 0x31c90fd5u, 0x3f800000u}, + {0x33800000u, 0x34c90fd5u, 0x3f800000u}, + {0x3c23d70au, 0x3d80984du, 0x3f7f7eadu}, + {0x3e82a4d7u, 0x3f7fdd81u, 0xbd04dfbau}, + {0x3f3d70a4u, 0xbf7f7eadu, 0xbd809845u}, + {0xbf3d70a4u, 0x3f7f7eadu, 0xbd809845u}, + {0x42c8999au, 0x3f73780du, 0xbe9e39ddu}, + {0xc2c8999au, 0xbf73780du, 0xbe9e39ddu}, + {0xc348b333u, 0x3f73780du, 0xbe9e39ddu}, + {0x80000000u, 0x80000000u, 0x3f800000u}, + {0x00000000u, 0x00000000u, 0x3f800000u}, + {0xbf000000u, 0x00000000u, 0xbf800000u}, + {0xbe800000u, 0xbf800000u, 0x00000000u}, + {0xbf800000u, 0x00000000u, 0x3f800000u}, + {0xcb000001u, 0x00000000u, 0x3f800000u}, + {0x7f7fffffu, 0x00000000u, 0x3f800000u}, + {0xff800000u, 0xffc00000u, 0xffc00000u}, +}}; + +std::array BufferDescriptor(const void* data, std::uint32_t count) { + const auto address = reinterpret_cast(data); + return {static_cast(address), static_cast((address >> 32u) & 0xffffu) | (4u << 16u), count, 0x11016facu}; +} + +void Run(AgcDriver::VulkanDevice& device) { + for (std::uint32_t tid = 0; tid < Threads; ++tid) Input[tid] = Rows[tid].x; + Output.fill(0xdeadbeefu); + std::vector userData(8, 0u); + const auto input = BufferDescriptor(Input.data(), static_cast(Input.size())); + const auto output = BufferDescriptor(Output.data(), static_cast(Output.size())); + std::copy(input.begin(), input.end(), userData.begin()); + std::copy(output.begin(), output.end(), userData.begin() + 4); + const std::span code(Code); + const std::array memory{{{reinterpret_cast(code.data()), std::as_bytes(code)}}}; + const ShaderRecompiler::ShaderComputeStageInfo compute{{Threads, 1, 1}, 0, {false, false, false}, false, 1}; + ShaderRecompiler::RecompileRequest request{ + {ShaderStage::Compute, reinterpret_cast(code.data()), code, 0, {}}, + {32, 0, userData, compute, std::nullopt, std::nullopt, memory}, + device.Target(), + {0, 0, 0, 128} + }; + request.useCache = false; + const auto result = ShaderRecompiler::Recompile(request); + device.Dispatch(result, 1, 1, 1, {}, reinterpret_cast(code.data())); + device.WaitIdle(); +} + +std::string Hex(std::uint32_t value) { + char text[16]; + std::snprintf(text, sizeof(text), "0x%08x", value); + return text; +} + +bool Near(std::uint32_t actual, std::uint32_t expected) { + const float value = std::bit_cast(expected); + return std::abs(static_cast(std::bit_cast(actual)) - value) <= std::abs(value) * 0x1p-10; +} + +void Expect(std::uint32_t tid, const char* name, std::uint32_t actual, std::uint32_t expected, bool accurate) { + const float value = std::bit_cast(expected); + const bool exact = !std::isfinite(value) || value == 0.0f; + Require(exact ? actual == expected : !accurate || Near(actual, expected), std::string("sin cos reduction: lane ") + std::to_string(tid) + " " + name + " of " + Hex(Rows[tid].x) + " is " + Hex(actual) + (exact ? ", expected " : ", expected near ") + Hex(expected)); +} + +void Check() { + bool accurate = true; + for (std::uint32_t tid = 0; tid < Threads; ++tid) { + const float x = std::bit_cast(Rows[tid].x); + if (x > 0.0f && x < 0.5f) { + accurate = accurate && Near(Output[tid * Results], Rows[tid].sin) && Near(Output[tid * Results + 1], Rows[tid].cos); + } + } + if (!accurate) { + std::puts("the device's sin and cos are not within 2^-10 of the hardware for small positive inputs, so only zero, infinite and NaN results are checked"); + } + for (std::uint32_t tid = 0; tid < Threads; ++tid) { + Expect(tid, "v_sin_f32", Output[tid * Results], Rows[tid].sin, accurate); + Expect(tid, "v_cos_f32", Output[tid * Results + 1], Rows[tid].cos, accurate); + } +} + +} + +int main() { + try { + const auto device = OpenVulkanTestDevice(); + if (!device) return VulkanTestSkipped; + Run(*device); + Check(); + std::puts("sin cos reduction tests passed"); + return 0; + } catch (const std::exception& error) { + std::cerr << error.what() << '\n'; + return 1; + } +} diff --git a/core/libs/prx/libSceAgcDriver/tests/execution/Srgb8ColorTarget.cpp b/core/libs/prx/libSceAgcDriver/tests/execution/Srgb8ColorTarget.cpp index c1315cbbd..58de25068 100644 --- a/core/libs/prx/libSceAgcDriver/tests/execution/Srgb8ColorTarget.cpp +++ b/core/libs/prx/libSceAgcDriver/tests/execution/Srgb8ColorTarget.cpp @@ -127,7 +127,7 @@ struct Block { Require(data != nullptr, "cannot allocate the color target"); Require(!watched || AgcDriver::GuestMemory::Watched(Address(), bytes), "the color target is not write-watched"); GuestAllocations::Mutation mutation; - mutation.Add(data, bytes, true, true); + mutation.Add(data, bytes, true, true, true); } ~Block() { AgcDriver::Graphics::StorageTexture::FlushPending(Address(), bytes, nullptr, "test release"); @@ -227,13 +227,13 @@ void Draw(AgcDriver::VulkanDevice& device, std::vector(pixelCode.data()), pixelCode, 0, {}}, {waveSize, 0, pixelUserData, std::nullopt, pixel, std::nullopt, pixelMemory}, target, - {0, 0, vertexPush, 128 - vertexPush} + PixelPushLayout(vertexPush, target) }; fragment.useCache = false; const auto pixelResult = ShaderRecompiler::Recompile(fragment); const std::array shaders{{ {ShaderStage::Vertex, &vertexResult, 0}, - {ShaderStage::Fragment, &pixelResult, vertexPush} + {ShaderStage::Fragment, &pixelResult, PixelPushOffset(vertexPush, target)} }}; AgcDriver::Graphics::State state{}; diff --git a/core/libs/prx/libSceAgcDriver/tests/execution/StorageCacheBudget.cpp b/core/libs/prx/libSceAgcDriver/tests/execution/StorageCacheBudget.cpp index 8cd839635..fcab5f29a 100644 --- a/core/libs/prx/libSceAgcDriver/tests/execution/StorageCacheBudget.cpp +++ b/core/libs/prx/libSceAgcDriver/tests/execution/StorageCacheBudget.cpp @@ -3,11 +3,11 @@ #include "prx/libc/include/GuestAllocations.hpp" #include "Recompiler.hpp" #include "VulkanTestDevice.hpp" -#include #include #include #include #include +#include #include #include #include @@ -16,19 +16,18 @@ namespace { -using AgcDriver::Graphics::Check; using AgcDriver::Graphics::Require; using AgcDriver::Graphics::StorageTexture; using ShaderRecompiler::ShaderStage; -constexpr std::uint32_t Side = 4096; +constexpr std::uint32_t Side = 512; constexpr std::uint32_t Format32UInt = 20; constexpr std::uint32_t TileR64KBX = 0x1b; constexpr std::uint32_t Type2D = 9; constexpr std::uint64_t SurfaceBytes = std::uint64_t{Side} * Side * 4u; constexpr std::uint64_t Stride = 0x10000; -constexpr std::uint64_t FixedBudget = 2048ull << 20u; -constexpr std::uint64_t EvictionLimit = 4ull << 30u; +constexpr std::uint64_t FixedBudget = 16ull << 20u; +constexpr std::uint64_t TestBudget = 32ull << 20u; constexpr std::uint32_t MaxSurfaces = 80; alignas(256) constexpr std::array LoadStoreCode{ @@ -41,72 +40,6 @@ std::uint32_t Marker(std::uint32_t surface) { return 0x5a000000u | surface; } -template -TFunction InstanceFunction(PFN_vkGetInstanceProcAddr resolve, VkInstance instance, const char* name) { - const auto result = reinterpret_cast(resolve(instance, name)); - Require(result != nullptr, name); - return result; -} - -struct SelectedDevice { - VkPhysicalDeviceMemoryProperties memory{}; - VkPhysicalDeviceType type = VK_PHYSICAL_DEVICE_TYPE_OTHER; -}; - -SelectedDevice SelectDevice() { -#ifdef _WIN32 - void* library = SDL_LoadObject("vulkan-1.dll"); -#else - void* library = SDL_LoadObject("libvulkan.so.1"); -#endif - Require(library != nullptr, "cannot load Vulkan"); - const auto resolve = reinterpret_cast(SDL_LoadFunction(library, "vkGetInstanceProcAddr")); - VkInstance instance = VK_NULL_HANDLE; - SelectedDevice chosen; - try { - Require(resolve != nullptr, "missing Vulkan instance resolver"); - VkApplicationInfo application{VK_STRUCTURE_TYPE_APPLICATION_INFO}; - application.apiVersion = VK_API_VERSION_1_1; - VkInstanceCreateInfo info{VK_STRUCTURE_TYPE_INSTANCE_CREATE_INFO}; - info.pApplicationInfo = &application; - Check(InstanceFunction(resolve, VK_NULL_HANDLE, "vkCreateInstance")(&info, nullptr, &instance), "vkCreateInstance"); - const auto enumerate = InstanceFunction(resolve, instance, "vkEnumeratePhysicalDevices"); - std::uint32_t count = 0; - Check(enumerate(instance, &count, nullptr), "vkEnumeratePhysicalDevices"); - std::vector devices(count); - Check(enumerate(instance, &count, devices.data()), "vkEnumeratePhysicalDevices"); - devices.resize(count); - const auto properties = InstanceFunction(resolve, instance, "vkGetPhysicalDeviceProperties"); - const auto queues = InstanceFunction(resolve, instance, "vkGetPhysicalDeviceQueueFamilyProperties"); - constexpr auto graphicsCompute = VK_QUEUE_GRAPHICS_BIT | VK_QUEUE_COMPUTE_BIT; - VkPhysicalDevice selected = VK_NULL_HANDLE; - int selectedRank = -1; - for (const auto physical : devices) { - VkPhysicalDeviceProperties described{}; - properties(physical, &described); - const int rank = described.deviceType == VK_PHYSICAL_DEVICE_TYPE_DISCRETE_GPU ? 3 : described.deviceType == VK_PHYSICAL_DEVICE_TYPE_INTEGRATED_GPU ? 2 : described.deviceType == VK_PHYSICAL_DEVICE_TYPE_VIRTUAL_GPU ? 1 : 0; - if (described.apiVersion < VK_API_VERSION_1_1 || rank <= selectedRank) continue; - std::uint32_t familyCount = 0; - queues(physical, &familyCount, nullptr); - std::vector families(familyCount); - queues(physical, &familyCount, families.data()); - if (std::none_of(families.begin(), families.end(), [](const VkQueueFamilyProperties& family) { return family.queueCount != 0 && (family.queueFlags & graphicsCompute) == graphicsCompute; })) continue; - selected = physical; - selectedRank = rank; - chosen.type = described.deviceType; - } - Require(selected != VK_NULL_HANDLE, "no Vulkan 1.1 graphics and compute device"); - InstanceFunction(resolve, instance, "vkGetPhysicalDeviceMemoryProperties")(selected, &chosen.memory); - } catch (...) { - if (instance != VK_NULL_HANDLE) InstanceFunction(resolve, instance, "vkDestroyInstance")(instance, nullptr); - SDL_UnloadObject(library); - throw; - } - InstanceFunction(resolve, instance, "vkDestroyInstance")(instance, nullptr); - SDL_UnloadObject(library); - return chosen; -} - std::array TextureDescriptor(std::uint64_t address) { return { static_cast(address >> 8u), @@ -152,10 +85,14 @@ std::shared_ptr Load(AgcDriver::VulkanDevice& device, std::uint6 int main() { try { +#ifdef _WIN32 + Require(_putenv_s("APS5_TEXTURE_CACHE_MIB", "32") == 0, "cannot set the test cache budget"); +#else + Require(setenv("APS5_TEXTURE_CACHE_MIB", "32", 1) == 0, "cannot set the test cache budget"); +#endif const auto device = OpenVulkanTestDevice(); if (!device) return VulkanTestSkipped; - const auto selected = SelectDevice(); - const auto budget = AgcDriver::Graphics::TextureCacheBudget(selected.memory); + const auto budget = TestBudget; std::vector storage((SurfaceBytes + MaxSurfaces * Stride + Stride) / 4u, 0u); auto* texels = reinterpret_cast((reinterpret_cast(storage.data()) + Stride - 1u) & ~std::uintptr_t{Stride - 1u}); for (std::uint32_t surface = 0; surface < MaxSurfaces; ++surface) texels[surface * Stride / 4u] = Marker(surface); @@ -163,7 +100,7 @@ int main() { const auto blockBytes = static_cast(SurfaceBytes + (MaxSurfaces - 1u) * Stride); { GuestAllocations::Mutation mutation; - mutation.Add(texels, blockBytes, true, true); + mutation.Add(texels, blockBytes, true, true, true); } const auto release = [&] { AgcDriver::Graphics::ClearCachedTextures(device->Device()); @@ -175,12 +112,8 @@ int main() { Require(first != nullptr, "surface 0 has no cached storage image"); const auto held = std::max(first->AllocationBytes(), first->GuestBytes()); const auto reused = static_cast(FixedBudget / held) + 1u; - std::printf("texture cache budget %llu MiB; each 4096x4096 32_UINT surface holds %llu MiB; cyclic set of %u surfaces (%llu MiB)\n", static_cast(budget >> 20u), static_cast(held >> 20u), reused, static_cast((reused * held) >> 20u)); - if (budget < reused * held) { - std::printf("skipped, the device-local heap gives the texture caches less than the cyclic set\n"); - release(); - return VulkanTestSkipped; - } + std::printf("texture cache budget %llu MiB; each 512x512 32_UINT surface holds %llu MiB; cyclic set of %u surfaces (%llu MiB)\n", static_cast(budget >> 20u), static_cast(held >> 20u), reused, static_cast((reused * held) >> 20u)); + Require(budget >= reused * held, "the cyclic set exceeds the test cache budget"); std::vector> images(reused); images[0] = first; for (std::uint32_t surface = 1; surface < reused; ++surface) images[surface] = Load(*device, base, surface); @@ -191,14 +124,11 @@ int main() { } Require(remade == 0, std::to_string(remade) + " of " + std::to_string(reused) + " storage images were made again on the second pass over a " + std::to_string((reused * held) >> 20u) + " MiB cyclic set under a " + std::to_string(budget >> 20u) + " MiB budget"); const auto capacity = static_cast(budget / held); - if (selected.type == VK_PHYSICAL_DEVICE_TYPE_CPU || budget > EvictionLimit || capacity + 2u > MaxSurfaces) { - std::printf("eviction past the budget not tested on a CPU device or over a 4 GiB budget\n"); - } else { - for (std::uint32_t surface = reused; surface < capacity + 2u; ++surface) Load(*device, base, surface); - for (std::uint32_t surface = 0; surface < capacity + 2u; ++surface) { - const bool cached = StorageTexture::FindLive(base + surface * Stride, SurfaceBytes) != nullptr; - Require(cached == (surface >= 2u), "surface " + std::to_string(surface) + (cached ? " is still cached" : " was evicted") + " after " + std::to_string(capacity + 2u) + " surfaces under a budget of " + std::to_string(capacity) + " of them"); - } + Require(capacity + 2u <= MaxSurfaces, "the eviction set exceeds the test storage"); + for (std::uint32_t surface = reused; surface < capacity + 2u; ++surface) Load(*device, base, surface); + for (std::uint32_t surface = 0; surface < capacity + 2u; ++surface) { + const bool cached = StorageTexture::FindLive(base + surface * Stride, SurfaceBytes) != nullptr; + Require(cached == (surface >= 2u), "surface " + std::to_string(surface) + (cached ? " is still cached" : " was evicted") + " after " + std::to_string(capacity + 2u) + " surfaces under a budget of " + std::to_string(capacity) + " of them"); } } catch (...) { release(); diff --git a/core/libs/prx/libSceAgcDriver/tests/execution/Uint8x4ColorTarget.cpp b/core/libs/prx/libSceAgcDriver/tests/execution/Uint8x4ColorTarget.cpp new file mode 100644 index 000000000..f03fe5dbb --- /dev/null +++ b/core/libs/prx/libSceAgcDriver/tests/execution/Uint8x4ColorTarget.cpp @@ -0,0 +1,250 @@ +#include "prx/libSceAgcDriver/Execution/include/GuestMemory.hpp" +#include "prx/libSceAgcDriver/Execution/include/VulkanDevice.hpp" +#include "prx/libSceAgcDriver/Graphics/include/ColorTargetLayout.hpp" +#include "prx/libSceAgcDriver/Graphics/include/Draw.hpp" +#include "prx/libSceAgcDriver/Graphics/include/State.hpp" +#include "prx/libSceAgcDriver/Graphics/include/Texture.hpp" +#include "prx/libc/include/GuestAllocations.hpp" +#include "prx/libc/include/GuestArena.hpp" +#include "prx/libc/include/GuestWriteWatch.hpp" +#include "Recompiler.hpp" +#include "VulkanTestDevice.hpp" +#ifdef _WIN32 +#ifndef NOMINMAX +#define NOMINMAX +#endif +#include +#else +#include +#endif +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace { + +using AgcDriver::Graphics::Require; +using ShaderRecompiler::ShaderStage; + +constexpr std::uint32_t Width = 256; +constexpr std::uint32_t Height = 64; +constexpr std::uint32_t ScissorWidth = 128; +constexpr std::size_t BlockBytes = 65536; +constexpr std::uint32_t Rgba8UintInfo = 0x8428u; +constexpr std::uint32_t LinearAttrib3 = 0x09000000u; +constexpr std::uint32_t Uint16AbgrExport = 7; +constexpr std::uint8_t Kept = 0x5a; + +alignas(256) constexpr std::array VertexCode{ + 0xe0382000, 0x80000005, 0xbf8c3f70, 0xf80008cf, 0x03020100, 0xbf810000, +}; + +constexpr std::array, 3> Triangle{{ + {-1.0f, -1.0f, 0.5f, 1.0f}, {3.0f, -1.0f, 0.5f, 1.0f}, {-1.0f, 3.0f, 0.5f, 1.0f} +}}; + +alignas(256) std::array, 4> Programs{}; +std::size_t programCount = 0; + +std::array BufferDescriptor(const void* data, std::uint32_t stride, std::uint32_t count) { + const auto address = reinterpret_cast(data); + return {static_cast(address), static_cast((address >> 32u) & 0xffffu) | (stride << 16u), count, 0x01016facu}; +} + +std::span Program(std::array abgr) { + Require(programCount < Programs.size(), "too many test programs"); + auto& code = Programs[programCount++]; + std::size_t at = 0; + const auto move = [&](std::uint32_t vgpr, std::uint32_t value) { + code[at++] = 0x7e0002ffu | (vgpr << 17u); + code[at++] = value; + }; + move(4, static_cast(abgr[0]) | (static_cast(abgr[1]) << 16u)); + move(5, static_cast(abgr[2]) | (static_cast(abgr[3]) << 16u)); + code[at++] = 0xf8001c0fu; + code[at++] = 0x00000504u; + code[at++] = 0xbf810000u; + return std::span(code.data(), at); +} + +struct Block { + explicit Block(std::size_t bytes) : bytes(bytes), watched(AgcDriver::GuestMemory::WriteWatched()) { +#ifdef _WIN32 + if (watched) { + data = static_cast(GuestArena::GuestArenaAllocate_nid_postfix(bytes, BlockBytes)); + GuestArena::GuestArenaCommit_nid_postfix(data, bytes, PAGE_READWRITE, bytes); + } else { + data = static_cast(VirtualAlloc(nullptr, bytes, MEM_RESERVE | MEM_COMMIT, PAGE_READWRITE)); + } +#else + if (watched) { + void* raw = mmap(nullptr, bytes + BlockBytes, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + Require(raw != MAP_FAILED, "cannot map the color target"); + const auto begin = reinterpret_cast(raw); + const auto aligned = (begin + BlockBytes - 1) & ~(static_cast(BlockBytes) - 1); + if (aligned != begin) munmap(raw, aligned - begin); + if (aligned + bytes != begin + bytes + BlockBytes) munmap(reinterpret_cast(aligned + bytes), begin + BlockBytes - aligned); + data = reinterpret_cast(aligned); + GuestWriteWatch::GuestWriteWatchRegister_nid_postfix(data, bytes); + } else { + data = static_cast(std::aligned_alloc(BlockBytes, bytes)); + } +#endif + Require(data != nullptr, "cannot allocate the color target"); + Require(!watched || AgcDriver::GuestMemory::Watched(Address(), bytes), "the color target is not write-watched"); + GuestAllocations::Mutation mutation; + mutation.Add(data, bytes, true, true, true); + } + ~Block() { + AgcDriver::Graphics::StorageTexture::FlushPending(Address(), bytes, nullptr, "test release"); + { + GuestAllocations::Mutation mutation; + mutation.Remove(data); + } +#ifdef _WIN32 + if (watched) { + GuestArena::GuestArenaReset_nid_postfix(data, bytes); + GuestArena::GuestArenaRelease_nid_postfix(data, bytes); + } else { + VirtualFree(data, 0, MEM_RELEASE); + } +#else + if (watched) { + munmap(data, bytes); + GuestWriteWatch::GuestWriteWatchUnregister_nid_postfix(data, bytes); + } else { + std::free(data); + } +#endif + } + Block(const Block&) = delete; + Block& operator=(const Block&) = delete; + std::uint64_t Address() const { return reinterpret_cast(data); } + std::size_t bytes; + bool watched; + std::uint8_t* data = nullptr; +}; + +AgcDriver::Graphics::ColorTarget DecodeTarget(const Block& block, std::uint32_t info) { + AgcDriver::Registers cx; + cx[0x318] = static_cast(block.Address() >> 8u); + cx[0x31b] = 0; + cx[0x31c] = info; + cx[0x31d] = 0; + cx[0x323] = 0; + cx[0x324] = 0; + cx[0x3b0] = ((Width - 1u) << 14u) | (Height - 1u); + cx[0x3b8] = LinearAttrib3; + cx[0x390] = static_cast(block.Address() >> 40u); + auto color = AgcDriver::Graphics::DecodeColorBuffer(cx, 0); + color.exportIndex = 0; + color.uintExport = true; + return color; +} + +void Draw(AgcDriver::VulkanDevice& device, const AgcDriver::Graphics::ColorTarget& color, std::span pixelCode) { + const auto target = device.Target(); + constexpr std::uint32_t waveSize = 64; + std::vector vertexUserData(4, 0u); + const auto vertexBuffer = BufferDescriptor(Triangle.data(), 16u, static_cast(Triangle.size())); + std::copy(vertexBuffer.begin(), vertexBuffer.end(), vertexUserData.begin()); + const std::array vertexMemory{{{reinterpret_cast(VertexCode.data()), std::as_bytes(std::span(VertexCode))}}}; + ShaderRecompiler::RecompileRequest vertex{ + {ShaderStage::Vertex, reinterpret_cast(VertexCode.data()), VertexCode, 0, {}}, + {waveSize, 0, vertexUserData, std::nullopt, std::nullopt, ShaderRecompiler::ShaderVertexStageInfo{}, vertexMemory}, + target, + {0, 0, 0, 64} + }; + vertex.useCache = false; + const auto vertexResult = ShaderRecompiler::Recompile(vertex); + const auto vertexPush = static_cast(vertexResult.pushConstants.size()); + + ShaderRecompiler::ShaderPixelStageInfo pixel{}; + pixel.inputAddr = ShaderRecompiler::PixelInputBit(ShaderRecompiler::PixelInput::PositionX) | ShaderRecompiler::PixelInputBit(ShaderRecompiler::PixelInput::PositionY); + pixel.posX = true; + pixel.posY = true; + pixel.targetOutputMode.at(0) = Uint16AbgrExport; + pixel.targetExportMapping.fill(0xe4u); + const std::vector pixelUserData(8, 0u); + const std::array pixelMemory{{{reinterpret_cast(pixelCode.data()), std::as_bytes(pixelCode)}}}; + ShaderRecompiler::RecompileRequest fragment{ + {ShaderStage::Fragment, reinterpret_cast(pixelCode.data()), pixelCode, 0, {}}, + {waveSize, 0, pixelUserData, std::nullopt, pixel, std::nullopt, pixelMemory}, + target, + PixelPushLayout(vertexPush, target) + }; + fragment.useCache = false; + const auto pixelResult = ShaderRecompiler::Recompile(fragment); + const std::array shaders{{ + {ShaderStage::Vertex, &vertexResult, 0}, + {ShaderStage::Fragment, &pixelResult, PixelPushOffset(vertexPush, target)} + }}; + + VkPipelineColorBlendAttachmentState blend{}; + blend.blendEnable = VK_FALSE; + blend.colorWriteMask = 0xf; + AgcDriver::Graphics::State state{}; + state.stages = {AgcDriver::Graphics::ShaderPath::Vertex, 0u, waveSize, waveSize, std::nullopt, std::nullopt}; + state.colors = {color}; + state.color = color; + state.hasColorTarget = true; + state.renderExtent = {Width, Height}; + state.topology = VK_PRIMITIVE_TOPOLOGY_TRIANGLE_LIST; + state.viewport = {0, static_cast(Height), static_cast(Width), -static_cast(Height), 0, 1}; + state.negativeOneToOne = false; + state.scissor = {{0, 0}, {ScissorWidth, Height}}; + state.cullMode = VK_CULL_MODE_NONE; + state.frontFace = VK_FRONT_FACE_COUNTER_CLOCKWISE; + state.blends = {blend}; + state.blend = blend; + state.blendConstants = {}; + const AgcDriver::Pm4::DrawParameters draw{0, static_cast(Triangle.size()), 0, 1, 0, false}; + device.Draw(state, draw, shaders); +} + +std::vector ReadBack(AgcDriver::VulkanDevice& device, std::uint64_t address, std::size_t bytes) { + AgcDriver::Graphics::StorageTexture::FlushPending(address, bytes, nullptr, "test read-back"); + device.WaitIdle(); + std::vector stored(bytes); + AgcDriver::GuestMemory::Read(address, std::as_writable_bytes(std::span(stored)), 1); + return stored; +} + +void Expect(AgcDriver::VulkanDevice& device, std::uint32_t info, VkFormat format, std::uint32_t elementBytes, std::array exported, const std::string& what) { + const auto bytes = static_cast(Width) * Height * elementBytes; + Require(AgcDriver::Graphics::ColorTargetLayout(Width, Height, AgcDriver::Graphics::ColorTileMode::Linear, elementBytes).Bytes() == bytes, what + ": the linear target is padded"); + Block block(bytes); + std::memset(block.data, Kept, bytes); + const auto color = DecodeTarget(block, info); + Require(color.format == format && color.elementBytes == elementBytes && color.bytes == bytes, what + ": the target decoded to the wrong format"); + Draw(device, color, Program(exported)); + const auto stored = ReadBack(device, block.Address(), bytes); + for (std::size_t offset = 0; offset < bytes; ++offset) { + const bool inside = (offset / elementBytes) % Width < ScissorWidth; + const auto expected = inside ? static_cast(exported[offset % elementBytes]) : Kept; + Require(stored[offset] == expected, what + ": byte " + std::to_string(offset) + " is " + std::to_string(stored[offset]) + ", expected " + std::to_string(expected)); + } +} + +} + +int main() { + try { + const auto device = OpenVulkanTestDevice(); + if (!device) return VulkanTestSkipped; + Expect(*device, Rgba8UintInfo, VK_FORMAT_R8G8B8A8_UINT, 4, {0x12u, 0xbeu, 0x42u, 0x7fu}, "a UINT16_ABGR export into an 8_8_8_8_UINT target"); + std::puts("8_8_8_8_UINT color target tests passed"); + return 0; + } catch (const std::exception& error) { + std::cerr << error.what() << '\n'; + return 1; + } +} diff --git a/core/libs/prx/libSceAgcDriver/tests/execution/UnimportedLabelOrder.cpp b/core/libs/prx/libSceAgcDriver/tests/execution/UnimportedLabelOrder.cpp new file mode 100644 index 000000000..2cfd3dd65 --- /dev/null +++ b/core/libs/prx/libSceAgcDriver/tests/execution/UnimportedLabelOrder.cpp @@ -0,0 +1,113 @@ +#include "prx/libSceAgcDriver/Execution/include/GuestMemory.hpp" +#include "prx/libSceAgcDriver/Execution/include/VulkanDevice.hpp" +#include "prx/libSceAgcDriver/Graphics/include/Context.hpp" +#include "prx/libc/include/GuestAllocations.hpp" +#include "VulkanTestDevice.hpp" +#ifdef _WIN32 +#ifndef NOMINMAX +#define NOMINMAX +#endif +#include +#endif +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace { + +using AgcDriver::Graphics::Require; + +constexpr std::size_t BlockBytes = 65536; +constexpr std::uint32_t ReleaseMemHeader = 0xc0064900u; + +class GuestBlock { +public: + GuestBlock() { +#ifdef _WIN32 + block = static_cast(VirtualAlloc(nullptr, BlockBytes, MEM_RESERVE | MEM_COMMIT, PAGE_READWRITE)); +#else + block = static_cast(std::aligned_alloc(BlockBytes, BlockBytes)); +#endif + Require(block != nullptr, "unimported label order: cannot allocate the guest block"); + std::memset(block, 0, BlockBytes); + GuestAllocations::Mutation().Add(block, BlockBytes, true, true, true); + } + + ~GuestBlock() { + GuestAllocations::Mutation().Remove(block); +#ifdef _WIN32 + VirtualFree(block, 0, MEM_RELEASE); +#else + std::free(block); +#endif + } + + GuestBlock(const GuestBlock&) = delete; + GuestBlock& operator=(const GuestBlock&) = delete; + + volatile std::uint32_t* Words() { return reinterpret_cast(block); } + + std::uint64_t Address() const { return static_cast(reinterpret_cast(block)); } + +private: + std::uint8_t* block = nullptr; +}; + +alignas(64) volatile std::uint32_t commandMemory[16] = {}; + +std::uint64_t AddressOf(volatile const std::uint32_t* word) { + return static_cast(reinterpret_cast(word)); +} + +int StoreLabel(AgcDriver::VulkanDevice& device, std::uint64_t address, std::uint32_t value, std::uint64_t stamp) { + const auto bytes = std::as_bytes(std::span(&value, 1)); + const int reason = device.WriteLabelOnGpu(address, bytes, stamp, 0); + if (reason >= 1 && reason <= 4) { + if (reason != 1) device.WaitIdle(); + AgcDriver::GuestMemory::Write(address, bytes, 4); + } + return reason; +} + +} + +int main() { + try { + GuestBlock labels; + auto device = OpenVulkanTestDevice(); + if (!device) return VulkanTestSkipped; + std::lock_guard lock(AgcDriver::GuestMemory::GpuMutex()); + const std::array pattern{0x11111111u, 0x22222222u, 0x33333333u, 0x44444444u}; + if (!device->FillBuffer(labels.Address() + 4096, 4096, pattern)) { + std::printf("skipped, the device does not import guest memory\n"); + return VulkanTestSkipped; + } + StoreLabel(*device, AddressOf(&commandMemory[8]), 1, 1); + StoreLabel(*device, labels.Address() + 4, 2, 2); + device->SubmitRecorded(true); + const auto deadline = std::chrono::steady_clock::now() + std::chrono::seconds(10); + while (labels.Words()[1] != 2) { + Require(std::chrono::steady_clock::now() < deadline, "unimported label order: the later label never landed"); + std::this_thread::sleep_for(std::chrono::microseconds(100)); + } + const std::uint32_t seen = commandMemory[8]; + commandMemory[8] = ReleaseMemHeader; + device->WaitIdle(); + const std::uint32_t reused = commandMemory[8]; + std::array message{}; + std::snprintf(message.data(), message.size(), "unimported label order: the label held 0x%08x when a later label had landed, and the guest's reused word holds 0x%08x instead of 0x%08x", seen, reused, ReleaseMemHeader); + Require(seen == 1 && reused == ReleaseMemHeader, message.data()); + std::puts("unimported label order tests passed"); + return 0; + } catch (const std::exception& error) { + std::cerr << error.what() << '\n'; + return 1; + } +} diff --git a/core/libs/prx/libSceAgcDriver/tests/execution/ViewPastLastMip.cpp b/core/libs/prx/libSceAgcDriver/tests/execution/ViewPastLastMip.cpp index c2b535c5d..ec229885e 100644 --- a/core/libs/prx/libSceAgcDriver/tests/execution/ViewPastLastMip.cpp +++ b/core/libs/prx/libSceAgcDriver/tests/execution/ViewPastLastMip.cpp @@ -95,7 +95,7 @@ int main() { std::fill(texels, texels + SurfaceBytes / 4u, Untouched); { GuestAllocations::Mutation mutation; - mutation.Add(texels, SurfaceBytes, true, true); + mutation.Add(texels, SurfaceBytes, true, true, true); } Store(*device, texels, 3u, StorePastCode); Check(texels, false, "store through level 3"); diff --git a/core/libs/prx/libSceAgcDriver/tests/execution/Vop3AddSubClamp.cpp b/core/libs/prx/libSceAgcDriver/tests/execution/Vop3AddSubClamp.cpp new file mode 100644 index 000000000..b297825ec --- /dev/null +++ b/core/libs/prx/libSceAgcDriver/tests/execution/Vop3AddSubClamp.cpp @@ -0,0 +1,167 @@ +#include "prx/libSceAgcDriver/Execution/include/VulkanDevice.hpp" +#include "prx/libSceAgcDriver/Graphics/include/Draw.hpp" +#include "Recompiler.hpp" +#include "VulkanTestDevice.hpp" +#include +#include +#include +#include +#include +#include +#include +#include + +namespace { + +using AgcDriver::Graphics::Require; +using ShaderRecompiler::ShaderStage; + +constexpr std::uint32_t Threads = 32; +constexpr std::uint32_t Inputs = 4; +constexpr std::uint32_t Results = 16; +constexpr std::uint32_t Checked = 8; +alignas(256) std::array Input{}; +alignas(256) std::array Output{}; + +alignas(256) constexpr std::array Code{ + 0x34020084u, 0x34040086u, 0xe0381000u, 0x80000401u, 0xbf8c3f70u, 0x7e140280u, 0x7e160280u, 0x7e180280u, + 0x7e1a0280u, 0x7e1c0280u, 0x7e1e0280u, 0x7e200280u, 0x7e220280u, 0x7e240280u, 0x7e260280u, 0x7e280280u, + 0x7e2a0280u, 0x7e2c0280u, 0x7e2e0280u, 0x7e300280u, 0x7e320280u, 0xd525800au, 0x00020b04u, 0xd526800bu, + 0x00020b04u, 0xd527800cu, 0x00020b04u, 0xd703800du, 0x00020b04u, 0xd704800eu, 0x00020b04u, 0xd70d800fu, + 0x00020b04u, 0xd70e8010u, 0x00020b04u, 0x4a220b04u, 0xe0781000u, 0x80010a02u, 0xe0781010u, 0x80010e02u, + 0xe0781020u, 0x80011202u, 0xe0781030u, 0x80011602u, 0xbf8c3f70u, 0xbf810000u, +}; + +constexpr std::uint32_t Rows[32][4] = { + {0xffffffffu, 0x00000001u, 0x00000000u, 0x00000000u}, + {0xfffffffeu, 0x00000001u, 0x00000000u, 0x00000000u}, + {0x80000000u, 0x80000000u, 0x00000000u, 0x00000000u}, + {0x7fffffffu, 0x80000000u, 0x00000000u, 0x00000000u}, + {0x00000000u, 0x00000001u, 0x00000000u, 0x00000000u}, + {0x00000001u, 0x00000000u, 0x00000000u, 0x00000000u}, + {0x00000005u, 0x00000005u, 0x00000000u, 0x00000000u}, + {0x12345678u, 0xfedcba98u, 0x00000000u, 0x00000000u}, + {0xfedcba98u, 0x12345678u, 0x00000000u, 0x00000000u}, + {0x00007fffu, 0x00000001u, 0x00000000u, 0x00000000u}, + {0x00008000u, 0x0000ffffu, 0x00000000u, 0x00000000u}, + {0x00008000u, 0x00000001u, 0x00000000u, 0x00000000u}, + {0x0000ffffu, 0x00000001u, 0x00000000u, 0x00000000u}, + {0x0000ffffu, 0x0000ffffu, 0x00000000u, 0x00000000u}, + {0x00007fffu, 0x00007fffu, 0x00000000u, 0x00000000u}, + {0x00008000u, 0x00008000u, 0x00000000u, 0x00000000u}, + {0x00000000u, 0x00008000u, 0x00000000u, 0x00000000u}, + {0x00000000u, 0x00007fffu, 0x00000000u, 0x00000000u}, + {0x1234ffffu, 0xabcd0001u, 0x00000000u, 0x00000000u}, + {0xabcd7fffu, 0x12340001u, 0x00000000u, 0x00000000u}, + {0xffff8000u, 0x00017fffu, 0x00000000u, 0x00000000u}, + {0x00010000u, 0x00000001u, 0x00000000u, 0x00000000u}, + {0x00000001u, 0x00010000u, 0x00000000u, 0x00000000u}, + {0x0000fffeu, 0x0000ffffu, 0x00000000u, 0x00000000u}, + {0x00007ffeu, 0x0000fffeu, 0x00000000u, 0x00000000u}, + {0x00008001u, 0x00000002u, 0x00000000u, 0x00000000u}, + {0x00000002u, 0x00008001u, 0x00000000u, 0x00000000u}, + {0xffffffffu, 0xffffffffu, 0x00000000u, 0x00000000u}, + {0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u}, + {0x00000000u, 0xffffffffu, 0x00000000u, 0x00000000u}, + {0x7fff8000u, 0x8000ffffu, 0x00000000u, 0x00000000u}, + {0x40000000u, 0xc0000000u, 0x00000000u, 0x00000000u} +}; +constexpr std::uint32_t Expected[32][Checked] = { + {0xffffffffu, 0xfffffffeu, 0x00000000u, 0x0000ffffu, 0x0000fffeu, 0x00000000u, 0x0000fffeu, 0x00000000u}, + {0xffffffffu, 0xfffffffdu, 0x00000000u, 0x0000ffffu, 0x0000fffdu, 0x0000ffffu, 0x0000fffdu, 0xffffffffu}, + {0xffffffffu, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u}, + {0xffffffffu, 0x00000000u, 0x00000001u, 0x0000ffffu, 0x0000ffffu, 0x0000ffffu, 0x0000ffffu, 0xffffffffu}, + {0x00000001u, 0x00000000u, 0x00000001u, 0x00000001u, 0x00000000u, 0x00000001u, 0x0000ffffu, 0x00000001u}, + {0x00000001u, 0x00000001u, 0x00000000u, 0x00000001u, 0x00000001u, 0x00000001u, 0x00000001u, 0x00000001u}, + {0x0000000au, 0x00000000u, 0x00000000u, 0x0000000au, 0x00000000u, 0x0000000au, 0x00000000u, 0x0000000au}, + {0xffffffffu, 0x00000000u, 0xeca86420u, 0x0000ffffu, 0x00000000u, 0x00001110u, 0x00007fffu, 0x11111110u}, + {0xffffffffu, 0xeca86420u, 0x00000000u, 0x0000ffffu, 0x00006420u, 0x00001110u, 0x00008000u, 0x11111110u}, + {0x00008000u, 0x00007ffeu, 0x00000000u, 0x00008000u, 0x00007ffeu, 0x00007fffu, 0x00007ffeu, 0x00008000u}, + {0x00017fffu, 0x00000000u, 0x00007fffu, 0x0000ffffu, 0x00000000u, 0x00008000u, 0x00008001u, 0x00017fffu}, + {0x00008001u, 0x00007fffu, 0x00000000u, 0x00008001u, 0x00007fffu, 0x00008001u, 0x00008000u, 0x00008001u}, + {0x00010000u, 0x0000fffeu, 0x00000000u, 0x0000ffffu, 0x0000fffeu, 0x00000000u, 0x0000fffeu, 0x00010000u}, + {0x0001fffeu, 0x00000000u, 0x00000000u, 0x0000ffffu, 0x00000000u, 0x0000fffeu, 0x00000000u, 0x0001fffeu}, + {0x0000fffeu, 0x00000000u, 0x00000000u, 0x0000fffeu, 0x00000000u, 0x00007fffu, 0x00000000u, 0x0000fffeu}, + {0x00010000u, 0x00000000u, 0x00000000u, 0x0000ffffu, 0x00000000u, 0x00008000u, 0x00000000u, 0x00010000u}, + {0x00008000u, 0x00000000u, 0x00008000u, 0x00008000u, 0x00000000u, 0x00008000u, 0x00007fffu, 0x00008000u}, + {0x00007fffu, 0x00000000u, 0x00007fffu, 0x00007fffu, 0x00000000u, 0x00007fffu, 0x00008001u, 0x00007fffu}, + {0xbe020000u, 0x00000000u, 0x99980002u, 0x0000ffffu, 0x0000fffeu, 0x00000000u, 0x0000fffeu, 0xbe020000u}, + {0xbe018000u, 0x99997ffeu, 0x00000000u, 0x00008000u, 0x00007ffeu, 0x00007fffu, 0x00007ffeu, 0xbe018000u}, + {0xffffffffu, 0xfffe0001u, 0x00000000u, 0x0000ffffu, 0x00000001u, 0x0000ffffu, 0x00008000u, 0x0000ffffu}, + {0x00010001u, 0x0000ffffu, 0x00000000u, 0x00000001u, 0x00000000u, 0x00000001u, 0x0000ffffu, 0x00010001u}, + {0x00010001u, 0x00000000u, 0x0000ffffu, 0x00000001u, 0x00000001u, 0x00000001u, 0x00000001u, 0x00010001u}, + {0x0001fffdu, 0x00000000u, 0x00000001u, 0x0000ffffu, 0x00000000u, 0x0000fffdu, 0x0000ffffu, 0x0001fffdu}, + {0x00017ffcu, 0x00000000u, 0x00008000u, 0x0000ffffu, 0x00000000u, 0x00007ffcu, 0x00007fffu, 0x00017ffcu}, + {0x00008003u, 0x00007fffu, 0x00000000u, 0x00008003u, 0x00007fffu, 0x00008003u, 0x00008000u, 0x00008003u}, + {0x00008003u, 0x00000000u, 0x00007fffu, 0x00008003u, 0x00000000u, 0x00008003u, 0x00007fffu, 0x00008003u}, + {0xffffffffu, 0x00000000u, 0x00000000u, 0x0000ffffu, 0x00000000u, 0x0000fffeu, 0x00000000u, 0xfffffffeu}, + {0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u}, + {0xffffffffu, 0x00000000u, 0xffffffffu, 0x0000ffffu, 0x00000000u, 0x0000ffffu, 0x00000001u, 0xffffffffu}, + {0xffffffffu, 0x00000000u, 0x00017fffu, 0x0000ffffu, 0x00000000u, 0x00008000u, 0x00008001u, 0x00007fffu}, + {0xffffffffu, 0x00000000u, 0x80000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u} +}; +constexpr const char* Names[Checked] = { + "v_add_nc_u32 v10, v4, v5 clamp", + "v_sub_nc_u32 v11, v4, v5 clamp", + "v_subrev_nc_u32 v12, v4, v5 clamp", + "v_add_nc_u16 v13, v4, v5 clamp", + "v_sub_nc_u16 v14, v4, v5 clamp", + "v_add_nc_i16 v15, v4, v5 clamp", + "v_sub_nc_i16 v16, v4, v5 clamp", + "v_add_nc_u32 v17, v4, v5" +}; + +std::array BufferDescriptor(const void* data, std::uint32_t bytes) { + const auto address = reinterpret_cast(data); + return {static_cast(address), static_cast((address >> 32u) & 0xffffu), bytes, 0x31016facu}; +} + +std::string Hex(std::uint32_t value) { + char text[16]; + std::snprintf(text, sizeof(text), "0x%08x", value); + return text; +} + +void Run(AgcDriver::VulkanDevice& device) { + for (std::uint32_t tid = 0; tid < Threads; ++tid) std::copy(std::begin(Rows[tid]), std::end(Rows[tid]), &Input[tid * Inputs]); + Output.fill(0xdeadbeefu); + std::vector userData(8, 0u); + const auto input = BufferDescriptor(Input.data(), static_cast(Input.size() * 4u)); + const auto output = BufferDescriptor(Output.data(), static_cast(Output.size() * 4u)); + std::copy(input.begin(), input.end(), userData.begin()); + std::copy(output.begin(), output.end(), userData.begin() + 4); + const std::span code(Code); + const std::array memory{{{reinterpret_cast(code.data()), std::as_bytes(code)}}}; + const ShaderRecompiler::ShaderComputeStageInfo compute{{Threads, 1, 1}, 0u, {false, false, false}, false, 1}; + ShaderRecompiler::RecompileRequest request{ + {ShaderStage::Compute, reinterpret_cast(code.data()), code, 0, {}}, + {32, 0, userData, compute, std::nullopt, std::nullopt, memory}, + device.Target(), + {0, 0, 0, 128} + }; + request.useCache = false; + const auto result = ShaderRecompiler::Recompile(request); + device.Dispatch(result, 1, 1, 1, {}, reinterpret_cast(code.data())); + device.WaitIdle(); +} + +} + +int main() { + try { + const auto device = OpenVulkanTestDevice(); + if (!device) return VulkanTestSkipped; + Run(*device); + for (std::uint32_t tid = 0; tid < Threads; ++tid) { + for (std::uint32_t i = 0; i < Checked; ++i) { + const std::uint32_t actual = Output[tid * Results + i]; + Require(actual == Expected[tid][i], std::string("VOP3 add/sub clamp: lane ") + std::to_string(tid) + " " + Names[i] + " is " + Hex(actual) + ", expected " + Hex(Expected[tid][i])); + } + } + std::puts("VOP3 add/sub clamp tests passed"); + return 0; + } catch (const std::exception& error) { + std::cerr << error.what() << '\n'; + return 1; + } +} diff --git a/core/libs/prx/libSceAgcDriver/tests/execution/Vop3CubeClamp.cpp b/core/libs/prx/libSceAgcDriver/tests/execution/Vop3CubeClamp.cpp index d655f5f83..10857e6d3 100644 --- a/core/libs/prx/libSceAgcDriver/tests/execution/Vop3CubeClamp.cpp +++ b/core/libs/prx/libSceAgcDriver/tests/execution/Vop3CubeClamp.cpp @@ -9,6 +9,7 @@ #include #include #include +#include #include #include #include @@ -101,6 +102,40 @@ constexpr std::uint32_t Expected[32][16] = { {0x40000000u, 0x00000000u, 0x7f800001u, 0x00000000u, 0x3f800000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x40000000u, 0x00000000u, 0x7f800001u, 0x00000000u, 0x3f800000u, 0x00000000u, 0x00000000u, 0x00000000u}, {0x40000000u, 0x00000000u, 0xff812345u, 0x00000000u, 0x3f800000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x40000000u, 0x00000000u, 0x7f812345u, 0x00000000u, 0x3f800000u, 0x00000000u, 0x00000000u, 0x00000000u} }; +constexpr std::uint32_t ExpectedIeee[32][16] = { + {0x40800000u, 0x80000000u, 0x00000000u, 0x00000000u, 0x3f800000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x40800000u, 0x00000000u, 0x80000000u, 0x00000000u, 0x3f800000u, 0x00000000u, 0x00000000u, 0x00000000u}, + {0x40800000u, 0x3f800000u, 0x3f800000u, 0x40000000u, 0x3f800000u, 0x3f800000u, 0x3f800000u, 0x3f800000u, 0x40800000u, 0xbf800000u, 0xbf800000u, 0xc0000000u, 0x3f800000u, 0x3f800000u, 0x3f800000u, 0x3f800000u}, + {0x00000000u, 0xbf000000u, 0xbf000000u, 0x7fc00001u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x3f000000u, 0x3f000000u, 0x7fc00001u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u}, + {0x00000000u, 0x7fc12345u, 0xbf000000u, 0x40800000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x3f800000u, 0x3f800000u, 0xffc12345u, 0x3f000000u, 0x40800000u, 0x3f800000u, 0x00000000u, 0x00000000u, 0x00000000u}, + {0x00000000u, 0xffc00000u, 0xffc00000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x7fc00000u, 0x7fc00000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u}, + {0x00000000u, 0xffc00001u, 0x7fc00000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x7fc00001u, 0xffc00000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u}, + {0x3f800000u, 0x00000007u, 0x00000007u, 0xff800000u, 0x3f800000u, 0x00000007u, 0x00000007u, 0x00000000u, 0x00000000u, 0x00000007u, 0x80000007u, 0x7f800000u, 0x00000000u, 0x00000007u, 0x00000007u, 0x3f800000u}, + {0x00000000u, 0x7f800000u, 0x80000007u, 0x7fc00000u, 0x00000000u, 0x3f800000u, 0x00000000u, 0x00000000u, 0x00000000u, 0xff800000u, 0x00000007u, 0x7fc00000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u}, + {0x3f800000u, 0x80000007u, 0x80000000u, 0xbf800000u, 0x3f800000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x80000007u, 0x00000000u, 0x3f800000u, 0x00000000u, 0x00000007u, 0x00000000u, 0x3f800000u}, + {0x40000000u, 0x00000007u, 0x7fc00001u, 0x00000000u, 0x3f800000u, 0x00000007u, 0x00000000u, 0x00000000u, 0x40000000u, 0x00000007u, 0x7fc00001u, 0x00000000u, 0x3f800000u, 0x00000007u, 0x00000000u, 0x00000000u}, + {0x40000000u, 0x00000007u, 0x80000007u, 0x7f800000u, 0x3f800000u, 0x00000007u, 0x00000000u, 0x3f800000u, 0x40000000u, 0x00000007u, 0x80000007u, 0x7f800000u, 0x3f800000u, 0x00000007u, 0x00000007u, 0x3f800000u}, + {0x40800000u, 0x00000007u, 0x80000007u, 0x00000000u, 0x3f800000u, 0x00000007u, 0x00000000u, 0x00000000u, 0x40800000u, 0x00000007u, 0x00000007u, 0x00000000u, 0x3f800000u, 0x00000007u, 0x00000000u, 0x00000000u}, + {0x40400000u, 0xff800000u, 0xc0000000u, 0xff800000u, 0x3f800000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x40000000u, 0x7f800000u, 0x40000000u, 0xff800000u, 0x3f800000u, 0x00000000u, 0x3f800000u, 0x3f800000u}, + {0x40a00000u, 0xc0000000u, 0x40200000u, 0xc0a00000u, 0x3f800000u, 0x00000000u, 0x3f800000u, 0x00000000u, 0x40a00000u, 0x40000000u, 0xc0200000u, 0x40a00000u, 0x3f800000u, 0x3f800000u, 0x3f800000u, 0x00000000u}, + {0x00000000u, 0x00000000u, 0x7f800000u, 0x7fc00000u, 0x00000000u, 0x00000000u, 0x3f800000u, 0x00000000u, 0x00000000u, 0x80000000u, 0xff800000u, 0x7fc00000u, 0x00000000u, 0x00000000u, 0x3f800000u, 0x00000000u}, + {0x40800000u, 0xc0200000u, 0xff800000u, 0x7f800000u, 0x3f800000u, 0x00000000u, 0x00000000u, 0x3f800000u, 0x40800000u, 0xc0200000u, 0x7f800000u, 0xff800000u, 0x3f800000u, 0x00000000u, 0x00000000u, 0x3f800000u}, + {0x40800000u, 0xbf000000u, 0x3f000000u, 0x3f800000u, 0x3f800000u, 0x00000000u, 0x3f000000u, 0x3f800000u, 0x40800000u, 0xbf000000u, 0xbf000000u, 0xbf800000u, 0x3f800000u, 0x00000000u, 0x3f000000u, 0x3f800000u}, + {0x00000000u, 0x40200000u, 0x7f800000u, 0x7fc00000u, 0x00000000u, 0x3f800000u, 0x3f800000u, 0x00000000u, 0x00000000u, 0xc0200000u, 0xff800000u, 0x7fc00000u, 0x00000000u, 0x00000000u, 0x3f800000u, 0x00000000u}, + {0x40400000u, 0x00000000u, 0x80000000u, 0xff800000u, 0x3f800000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x40000000u, 0x00000000u, 0x00000000u, 0xff800000u, 0x3f800000u, 0x00000000u, 0x00000000u, 0x3f800000u}, + {0x40400000u, 0x00000000u, 0x3f000000u, 0xff800000u, 0x3f800000u, 0x00000000u, 0x3f000000u, 0x00000000u, 0x40000000u, 0x00000000u, 0x3f000000u, 0xff800000u, 0x3f800000u, 0x00000000u, 0x00000000u, 0x3f800000u}, + {0x40a00000u, 0x7f800000u, 0x80000000u, 0xff800000u, 0x3f800000u, 0x3f800000u, 0x00000000u, 0x00000000u, 0x40a00000u, 0x7f800000u, 0x00000000u, 0x7f800000u, 0x3f800000u, 0x00000000u, 0x00000000u, 0x00000000u}, + {0x40000000u, 0xff800000u, 0x00000000u, 0x7f800000u, 0x3f800000u, 0x00000000u, 0x00000000u, 0x3f800000u, 0x40000000u, 0x7f800000u, 0x80000000u, 0x7f800000u, 0x3f800000u, 0x00000000u, 0x00000000u, 0x3f800000u}, + {0x40000000u, 0xff800000u, 0xbf000000u, 0x7f800000u, 0x3f800000u, 0x00000000u, 0x00000000u, 0x3f800000u, 0x40000000u, 0x7f800000u, 0xbf000000u, 0x7f800000u, 0x3f800000u, 0x00000000u, 0x3f000000u, 0x3f800000u}, + {0x40a00000u, 0x7f800000u, 0x7f800000u, 0xff800000u, 0x3f800000u, 0x3f800000u, 0x3f800000u, 0x00000000u, 0x40a00000u, 0x7f800000u, 0xff800000u, 0x7f800000u, 0x3f800000u, 0x00000000u, 0x3f800000u, 0x00000000u}, + {0x40400000u, 0xff800000u, 0x3f000000u, 0xff800000u, 0x3f800000u, 0x00000000u, 0x3f000000u, 0x00000000u, 0x40000000u, 0x7f800000u, 0x3f000000u, 0xff800000u, 0x3f800000u, 0x00000000u, 0x00000000u, 0x3f800000u}, + {0x3f800000u, 0xbf000000u, 0x3f000000u, 0xff800000u, 0x3f800000u, 0x00000000u, 0x3f000000u, 0x00000000u, 0x00000000u, 0xbf000000u, 0xbf000000u, 0x7f800000u, 0x00000000u, 0x3f000000u, 0x3f000000u, 0x3f800000u}, + {0x40800000u, 0x00000000u, 0x80000000u, 0x00000000u, 0x3f800000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x40800000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x3f800000u, 0x00000000u, 0x00000000u, 0x00000000u}, + {0x40800000u, 0x00000000u, 0x80000000u, 0x00000000u, 0x3f800000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x40800000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x3f800000u, 0x00000000u, 0x00000000u, 0x00000000u}, + {0x40000000u, 0x00000000u, 0x7fc00000u, 0x00000000u, 0x3f800000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x40000000u, 0x00000000u, 0x7fc00000u, 0x00000000u, 0x3f800000u, 0x00000000u, 0x00000000u, 0x00000000u}, + {0x40000000u, 0x00000000u, 0xffc00000u, 0x00000000u, 0x3f800000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x40000000u, 0x00000000u, 0x7fc00000u, 0x00000000u, 0x3f800000u, 0x00000000u, 0x00000000u, 0x00000000u}, + {0x40000000u, 0x00000000u, 0x7fc00001u, 0x00000000u, 0x3f800000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x40000000u, 0x00000000u, 0x7fc00001u, 0x00000000u, 0x3f800000u, 0x00000000u, 0x00000000u, 0x00000000u}, + {0x40000000u, 0x00000000u, 0xffc12345u, 0x00000000u, 0x3f800000u, 0x00000000u, 0x00000000u, 0x00000000u, 0x40000000u, 0x00000000u, 0x7fc12345u, 0x00000000u, 0x3f800000u, 0x00000000u, 0x00000000u, 0x00000000u} +}; constexpr const char* Names[16] = { "v_cubeid_f32 v10, v4, v5, v6", "v_cubesc_f32 v11, v4, v5, v6", @@ -139,7 +174,7 @@ void Expect(std::uint32_t tid, std::uint32_t actual, std::uint32_t expected, con Require(actual == expected, std::string("vop3 cube clamp: lane ") + std::to_string(tid) + " " + name + " is " + Hex(actual) + ", expected " + Hex(expected)); } -ShaderRecompiler::RecompileResult Compile(AgcDriver::VulkanDevice& device, std::span code) { +ShaderRecompiler::RecompileResult Compile(AgcDriver::VulkanDevice& device, std::span code, const std::optional& floatMode = std::nullopt) { std::vector userData(8, 0u); const auto input = BufferDescriptor(Input.data(), static_cast(Input.size() * 4u)); const auto output = BufferDescriptor(Output.data(), static_cast(Output.size() * 4u)); @@ -154,14 +189,15 @@ ShaderRecompiler::RecompileResult Compile(AgcDriver::VulkanDevice& device, std:: {0, 0, 0, 128} }; request.useCache = false; + request.context.floatMode = floatMode; return ShaderRecompiler::Recompile(request); } -void Run(AgcDriver::VulkanDevice& device) { +void Run(AgcDriver::VulkanDevice& device, const std::optional& floatMode) { for (std::uint32_t tid = 0; tid < Threads; ++tid) Fill(tid, &Input[tid * Inputs]); Output.fill(0xdeadbeefu); const std::span code(Code); - const auto result = Compile(device, code); + const auto result = Compile(device, code, floatMode); device.Dispatch(result, 1, 1, 1, {}, reinterpret_cast(code.data())); device.WaitIdle(); } @@ -179,12 +215,11 @@ void CheckOmodRefused(AgcDriver::VulkanDevice& device) { } } -void Check() { +void Check(const std::uint32_t (&expected)[32][16], const char* mode) { for (std::uint32_t tid = 0; tid < Threads; ++tid) { - const std::uint32_t* in = &Input[tid * Inputs]; const std::uint32_t* out = &Output[tid * Results]; for (std::uint32_t i = 0; i < 16; ++i) { - Expect(tid, out[i], Expected[tid][i], Names[i]); + Expect(tid, out[i], expected[tid][i], (std::string(mode) + " " + Names[i]).c_str()); } } } @@ -195,8 +230,12 @@ int main() { try { const auto device = OpenVulkanTestDevice(); if (!device) return VulkanTestSkipped; - Run(*device); - Check(); + Run(*device, std::nullopt); + Check(Expected, "no float mode"); + Run(*device, ShaderRecompiler::ShaderFloatMode{0xc0u, true, false, false}); + Check(Expected, "IEEE=0"); + Run(*device, ShaderRecompiler::ShaderFloatMode{0xc0u, true, true, false}); + Check(ExpectedIeee, "IEEE=1"); CheckOmodRefused(*device); std::puts("vop3 cube clamp tests passed"); return 0; diff --git a/core/libs/prx/libSceAgcDriver/tests/execution/Vop3PackLdexp.cpp b/core/libs/prx/libSceAgcDriver/tests/execution/Vop3PackLdexp.cpp index 181109a98..e20d2a860 100644 --- a/core/libs/prx/libSceAgcDriver/tests/execution/Vop3PackLdexp.cpp +++ b/core/libs/prx/libSceAgcDriver/tests/execution/Vop3PackLdexp.cpp @@ -9,6 +9,8 @@ #include #include #include +#include +#include #include #include @@ -102,8 +104,123 @@ constexpr std::uint32_t Expected[32][14] = { }; constexpr const char* Names[14] = {"cvt_pknorm_i16_f320", "cvt_pknorm_i16_f321", "cvt_pknorm_u16_f322", "cvt_pknorm_u16_f323", "ldexp_f324", "ldexp_f325", "ldexp_f326", "lerp_u87", "pack_b32_f168", "pack_b32_f169", "pack_b32_f1610", "pack_b32_f1611", "mul_lo_i32_0", "mul_lo_i32_1"}; -void Fill(std::uint32_t tid, std::uint32_t* words) { - std::copy(std::begin(Rows[tid]), std::end(Rows[tid]), words); +alignas(256) constexpr std::array PackCode{ + 0x34020084u, 0x34060086u, 0xe0381000u, 0x80000401u, 0xbf8c3f70u, 0xd711000au, 0x02020b04u, 0xd711180bu, + 0x02020b04u, 0xd711020cu, 0x22020b04u, 0xd711090du, 0x42020905u, 0xe0781000u, 0x80010a03u, 0xe0781010u, + 0x80010e03u, 0xe0781020u, 0x80011203u, 0xe0781030u, 0x80011603u, 0xbf810000u, +}; + +constexpr std::uint32_t PackRows[32][4] = { + {0xfd237c01u, 0xfc017dffu, 0x00000000u, 0x00000000u}, + {0x7e00fd23u, 0x00007c00u, 0x00000000u, 0x00000000u}, + {0x7c007dffu, 0xfd230001u, 0x00000000u, 0x00000000u}, + {0x0000fc01u, 0xfc00fd23u, 0x00000000u, 0x00000000u}, + {0x83ff7e00u, 0xc1007fffu, 0x00000000u, 0x00000000u}, + {0x7c01fe55u, 0x7fff8000u, 0x00000000u, 0x00000000u}, + {0xfc017fffu, 0x83ff7c01u, 0x00000000u, 0x00000000u}, + {0x7fff7c00u, 0x7e00fe55u, 0x00000000u, 0x00000000u}, + {0x3c00fc00u, 0x80000000u, 0x00000000u, 0x00000000u}, + {0x00013c00u, 0x7dffc100u, 0x00000000u, 0x00000000u}, + {0xc1000000u, 0x3c007e00u, 0x00000000u, 0x00000000u}, + {0x7dff8000u, 0x7c013c00u, 0x00000000u, 0x00000000u}, + {0xfe550001u, 0x7c007bffu, 0x00000000u, 0x00000000u}, + {0xfc0083ffu, 0x7bfffc01u, 0x00000000u, 0x00000000u}, + {0x80007bffu, 0xfe55fc00u, 0x00000000u, 0x00000000u}, + {0x7bffc100u, 0x000183ffu, 0x00000000u, 0x00000000u}, + {0xfd237dffu, 0xfc017c01u, 0x00000000u, 0x00000000u}, + {0x7e007c00u, 0x0000fd23u, 0x00000000u, 0x00000000u}, + {0x7c000001u, 0xfd237dffu, 0x00000000u, 0x00000000u}, + {0x0000fd23u, 0xfc00fc01u, 0x00000000u, 0x00000000u}, + {0x83ff7fffu, 0xc1007e00u, 0x00000000u, 0x00000000u}, + {0x7c018000u, 0x7ffffe55u, 0x00000000u, 0x00000000u}, + {0xfc017c01u, 0x83ff7fffu, 0x00000000u, 0x00000000u}, + {0x7ffffe55u, 0x7e007c00u, 0x00000000u, 0x00000000u}, + {0x3c000000u, 0x8000fc00u, 0x00000000u, 0x00000000u}, + {0x0001c100u, 0x7dff3c00u, 0x00000000u, 0x00000000u}, + {0xc1007e00u, 0x3c000000u, 0x00000000u, 0x00000000u}, + {0x7dff3c00u, 0x7c018000u, 0x00000000u, 0x00000000u}, + {0xfe557bffu, 0x7c000001u, 0x00000000u, 0x00000000u}, + {0xfc00fc01u, 0x7bff83ffu, 0x00000000u, 0x00000000u}, + {0x8000fc00u, 0xfe557bffu, 0x00000000u, 0x00000000u}, + {0x7bff83ffu, 0x0001c100u, 0x00000000u, 0x00000000u} +}; +constexpr std::uint32_t PackExpected[32][4] = { + {0x7dff7c01u, 0xfc01fd23u, 0x7dfffc01u, 0xfc017c01u}, + {0x7c00fd23u, 0x00007e00u, 0x7c007d23u, 0x7d230000u}, + {0x00017dffu, 0xfd237c00u, 0x0001fdffu, 0xfdff7d23u}, + {0xfd23fc01u, 0xfc000000u, 0x7d237c01u, 0x7c017c00u}, + {0x7fff7e00u, 0xc10083ffu, 0x7ffffe00u, 0xfe004100u}, + {0x8000fe55u, 0x7fff7c01u, 0x00007e55u, 0x7e557fffu}, + {0x7c017fffu, 0x83fffc01u, 0x7c01ffffu, 0xffff03ffu}, + {0xfe557c00u, 0x7e007fffu, 0x7e55fc00u, 0xfc007e00u}, + {0x0000fc00u, 0x80003c00u, 0x00007c00u, 0x7c000000u}, + {0xc1003c00u, 0x7dff0001u, 0x4100bc00u, 0xbc007dffu}, + {0x7e000000u, 0x3c00c100u, 0x7e008000u, 0x80003c00u}, + {0x3c008000u, 0x7c017dffu, 0x3c000000u, 0x00007c01u}, + {0x7bff0001u, 0x7c00fe55u, 0x7bff8001u, 0x80017c00u}, + {0xfc0183ffu, 0x7bfffc00u, 0x7c0103ffu, 0x03ff7bffu}, + {0xfc007bffu, 0xfe558000u, 0x7c00fbffu, 0xfbff7e55u}, + {0x83ffc100u, 0x00017bffu, 0x03ff4100u, 0x41000001u}, + {0x7c017dffu, 0xfc01fd23u, 0x7c01fdffu, 0xfdff7c01u}, + {0xfd237c00u, 0x00007e00u, 0x7d23fc00u, 0xfc000000u}, + {0x7dff0001u, 0xfd237c00u, 0x7dff8001u, 0x80017d23u}, + {0xfc01fd23u, 0xfc000000u, 0x7c017d23u, 0x7d237c00u}, + {0x7e007fffu, 0xc10083ffu, 0x7e00ffffu, 0xffff4100u}, + {0xfe558000u, 0x7fff7c01u, 0x7e550000u, 0x00007fffu}, + {0x7fff7c01u, 0x83fffc01u, 0x7ffffc01u, 0xfc0103ffu}, + {0x7c00fe55u, 0x7e007fffu, 0x7c007e55u, 0x7e557e00u}, + {0xfc000000u, 0x80003c00u, 0x7c008000u, 0x80000000u}, + {0x3c00c100u, 0x7dff0001u, 0x3c004100u, 0x41007dffu}, + {0x00007e00u, 0x3c00c100u, 0x0000fe00u, 0xfe003c00u}, + {0x80003c00u, 0x7c017dffu, 0x0000bc00u, 0xbc007c01u}, + {0x00017bffu, 0x7c00fe55u, 0x0001fbffu, 0xfbff7c00u}, + {0x83fffc01u, 0x7bfffc00u, 0x03ff7c01u, 0x7c017bffu}, + {0x7bfffc00u, 0xfe558000u, 0x7bff7c00u, 0x7c007e55u}, + {0xc10083ffu, 0x00017bffu, 0x410003ffu, 0x03ff0001u} +}; +constexpr std::uint32_t PackExpectedIeee[32][4] = { + {0x7fff7e01u, 0xfe01ff23u, 0x7ffffe01u, 0xfe017e01u}, + {0x7c00ff23u, 0x00007e00u, 0x7c007f23u, 0x7f230000u}, + {0x00017fffu, 0xff237c00u, 0x0001ffffu, 0xffff7f23u}, + {0xff23fe01u, 0xfc000000u, 0x7f237e01u, 0x7e017c00u}, + {0x7fff7e00u, 0xc10083ffu, 0x7ffffe00u, 0xfe004100u}, + {0x8000fe55u, 0x7fff7e01u, 0x00007e55u, 0x7e557fffu}, + {0x7e017fffu, 0x83fffe01u, 0x7e01ffffu, 0xffff03ffu}, + {0xfe557c00u, 0x7e007fffu, 0x7e55fc00u, 0xfc007e00u}, + {0x0000fc00u, 0x80003c00u, 0x00007c00u, 0x7c000000u}, + {0xc1003c00u, 0x7fff0001u, 0x4100bc00u, 0xbc007fffu}, + {0x7e000000u, 0x3c00c100u, 0x7e008000u, 0x80003c00u}, + {0x3c008000u, 0x7e017fffu, 0x3c000000u, 0x00007e01u}, + {0x7bff0001u, 0x7c00fe55u, 0x7bff8001u, 0x80017c00u}, + {0xfe0183ffu, 0x7bfffc00u, 0x7e0103ffu, 0x03ff7bffu}, + {0xfc007bffu, 0xfe558000u, 0x7c00fbffu, 0xfbff7e55u}, + {0x83ffc100u, 0x00017bffu, 0x03ff4100u, 0x41000001u}, + {0x7e017fffu, 0xfe01ff23u, 0x7e01ffffu, 0xffff7e01u}, + {0xff237c00u, 0x00007e00u, 0x7f23fc00u, 0xfc000000u}, + {0x7fff0001u, 0xff237c00u, 0x7fff8001u, 0x80017f23u}, + {0xfe01ff23u, 0xfc000000u, 0x7e017f23u, 0x7f237c00u}, + {0x7e007fffu, 0xc10083ffu, 0x7e00ffffu, 0xffff4100u}, + {0xfe558000u, 0x7fff7e01u, 0x7e550000u, 0x00007fffu}, + {0x7fff7e01u, 0x83fffe01u, 0x7ffffe01u, 0xfe0103ffu}, + {0x7c00fe55u, 0x7e007fffu, 0x7c007e55u, 0x7e557e00u}, + {0xfc000000u, 0x80003c00u, 0x7c008000u, 0x80000000u}, + {0x3c00c100u, 0x7fff0001u, 0x3c004100u, 0x41007fffu}, + {0x00007e00u, 0x3c00c100u, 0x0000fe00u, 0xfe003c00u}, + {0x80003c00u, 0x7e017fffu, 0x0000bc00u, 0xbc007e01u}, + {0x00017bffu, 0x7c00fe55u, 0x0001fbffu, 0xfbff7c00u}, + {0x83fffe01u, 0x7bfffc00u, 0x03ff7e01u, 0x7e017bffu}, + {0x7bfffc00u, 0xfe558000u, 0x7bff7c00u, 0x7c007e55u}, + {0xc10083ffu, 0x00017bffu, 0x410003ffu, 0x03ff0001u} +}; +constexpr const char* PackNames[4] = { + "v_pack_b32_f16 v10, v4, v5", + "v_pack_b32_f16 v11, v4, v5 op_sel:[1,1,0]", + "v_pack_b32_f16 v12, -v4, |v5|", + "v_pack_b32_f16 v13, |v5|, -v4 op_sel:[1,0,0]" +}; + +void Fill(const std::uint32_t (&rows)[32][4]) { + for (std::uint32_t tid = 0; tid < Threads; ++tid) std::copy(std::begin(rows[tid]), std::end(rows[tid]), &Input[tid * Inputs]); } std::array BufferDescriptor(const void* data, std::uint32_t bytes) { @@ -121,15 +238,14 @@ void Expect(std::uint32_t tid, std::uint32_t actual, std::uint32_t expected, con Require(actual == expected, std::string("vop3 pack ldexp: lane ") + std::to_string(tid) + " " + name + " is " + Hex(actual) + ", expected " + Hex(expected)); } -void Run(AgcDriver::VulkanDevice& device) { - for (std::uint32_t tid = 0; tid < Threads; ++tid) Fill(tid, &Input[tid * Inputs]); +void Run(AgcDriver::VulkanDevice& device, std::span code, const std::uint32_t (&rows)[32][4], const std::optional& floatMode) { + Fill(rows); Output.fill(0xdeadbeefu); std::vector userData(8, 0u); const auto input = BufferDescriptor(Input.data(), static_cast(Input.size() * 4u)); const auto output = BufferDescriptor(Output.data(), static_cast(Output.size() * 4u)); std::copy(input.begin(), input.end(), userData.begin()); std::copy(output.begin(), output.end(), userData.begin() + 4); - const std::span code(Code); const std::array memory{{{reinterpret_cast(code.data()), std::as_bytes(code)}}}; const ShaderRecompiler::ShaderComputeStageInfo compute{{Threads, 1, 1}, 0u, {false, false, false}, false, 1}; ShaderRecompiler::RecompileRequest request{ @@ -139,6 +255,7 @@ void Run(AgcDriver::VulkanDevice& device) { {0, 0, 0, 128} }; request.useCache = false; + request.context.floatMode = floatMode; const auto result = ShaderRecompiler::Recompile(request); device.Dispatch(result, 1, 1, 1, {}, reinterpret_cast(code.data())); device.WaitIdle(); @@ -152,14 +269,27 @@ void Check() { } } +void CheckPack(const std::uint32_t (&expected)[32][4], const char* mode) { + for (std::uint32_t tid = 0; tid < Threads; ++tid) { + const std::uint32_t* out = &Output[tid * Results]; + for (std::uint32_t i = 0; i < 4; ++i) Expect(tid, out[i], expected[tid][i], (std::string(mode) + " " + PackNames[i]).c_str()); + } +} + } int main() { try { const auto device = OpenVulkanTestDevice(); if (!device) return VulkanTestSkipped; - Run(*device); + Run(*device, Code, Rows, std::nullopt); Check(); + Run(*device, PackCode, PackRows, std::nullopt); + CheckPack(PackExpected, "no float mode"); + Run(*device, PackCode, PackRows, ShaderRecompiler::ShaderFloatMode{0xc0u, true, false, false}); + CheckPack(PackExpected, "IEEE=0"); + Run(*device, PackCode, PackRows, ShaderRecompiler::ShaderFloatMode{0xc0u, true, true, false}); + CheckPack(PackExpectedIeee, "IEEE=1"); std::puts("vop3 pack ldexp tests passed"); return 0; } catch (const std::exception& error) { diff --git a/core/libs/prx/libSceAgcDriver/tests/execution/VulkanTestDevice.hpp b/core/libs/prx/libSceAgcDriver/tests/execution/VulkanTestDevice.hpp index bab6cab86..0b7c23d83 100644 --- a/core/libs/prx/libSceAgcDriver/tests/execution/VulkanTestDevice.hpp +++ b/core/libs/prx/libSceAgcDriver/tests/execution/VulkanTestDevice.hpp @@ -27,4 +27,13 @@ inline std::unique_ptr OpenVulkanTestDevice() { } } +inline std::uint32_t PixelPushOffset(std::uint32_t vertexPush, const ShaderRecompiler::SpirvTarget& target) { + return AgcDriver::Graphics::StagePushOffset(vertexPush, ShaderRecompiler::ShaderStage::Fragment, target.fixedPushSlots); +} + +inline ShaderRecompiler::BindingLayout PixelPushLayout(std::uint32_t vertexPush, const ShaderRecompiler::SpirvTarget& target) { + const auto offset = PixelPushOffset(vertexPush, target); + return {0, 0, offset, AgcDriver::Graphics::PipelinePushSlotBytes - offset % AgcDriver::Graphics::PipelinePushSlotBytes}; +} + #endif diff --git a/core/libs/prx/libSceAgcDriver/tests/execution/ZeroStrideVertexFetch.cpp b/core/libs/prx/libSceAgcDriver/tests/execution/ZeroStrideVertexFetch.cpp new file mode 100644 index 000000000..50e546f07 --- /dev/null +++ b/core/libs/prx/libSceAgcDriver/tests/execution/ZeroStrideVertexFetch.cpp @@ -0,0 +1,153 @@ +#include "prx/libSceAgcDriver/Execution/include/VulkanDevice.hpp" +#include "prx/libSceAgcDriver/Graphics/include/Draw.hpp" +#include "Recompiler.hpp" +#include "VulkanTestDevice.hpp" +#include +#include +#include +#include +#include +#include +#include +#include + +namespace { + +using AgcDriver::Graphics::Require; +using ShaderRecompiler::ShaderStage; + +constexpr std::uint32_t Width = 64; +constexpr std::uint32_t Height = 32; +constexpr std::uint32_t Vertices = 3; +alignas(256) std::array Pixels{}; +alignas(256) std::array Output{}; + +alignas(256) constexpr std::array VertexCode{ + 0xbe822100, 0x34020a84, 0xe0781000, 0x80010801, 0xf80008cf, 0x0b0a0908, 0xbf810000, +}; + +alignas(256) constexpr std::array PixelCode{ + 0x7e0e02f2, 0xf800180f, 0x07070707, 0xbf810000, +}; + +alignas(256) constexpr std::array Element{0.25f, -2.0f, 7.5f, 1.0f}; + +struct Case { + std::uint32_t select; + std::uint32_t records; + bool inRange; + const char* what; +}; + +std::array VertexBufferDescriptor(const Case& value) { + const auto address = reinterpret_cast(Element.data()); + return {static_cast(address), static_cast((address >> 32u) & 0xffffu), value.records, 0x0104dfacu | (value.select << 28u)}; +} + +std::array OutputDescriptor() { + const auto address = reinterpret_cast(Output.data()); + return {static_cast(address), static_cast((address >> 32u) & 0xffffu), static_cast(sizeof(Output)), 0x31016facu}; +} + +void Draw(AgcDriver::VulkanDevice& device, const Case& value) { + Pixels.fill(std::byte{0x40}); + Output.fill(0xdeadbeefu); + std::vector userData(12, 0u); + userData[0] = 0x10000000u; + userData[1] = 0x00000001u; + const auto output = OutputDescriptor(); + std::copy(output.begin(), output.end(), userData.begin() + 4); + ShaderRecompiler::ShaderVertexStageInfo info{}; + info.resourcesNum = 1; + info.resources[0].fields = VertexBufferDescriptor(value); + info.resourcesDst[0] = {8, 4, 0, 0}; + info.fetchAttribReg = 8; + info.fetchBufferReg = 10; + info.fetchEmbedded = true; + const std::array vertexMemory{{{reinterpret_cast(VertexCode.data()), std::as_bytes(std::span(VertexCode))}}}; + ShaderRecompiler::RecompileRequest vertexRequest{ + {ShaderStage::Vertex, reinterpret_cast(VertexCode.data()), VertexCode, 0, {}}, + {64u, 0, userData, std::nullopt, std::nullopt, info, vertexMemory}, + device.Target(), + {0, 0, 0, 64} + }; + vertexRequest.useCache = false; + const auto vertexResult = ShaderRecompiler::Recompile(vertexRequest); + Require(vertexResult.vertexAttributes.size() == 1u, std::string(value.what) + ": the fetch did not become a vertex attribute"); + const auto vertexPush = static_cast(vertexResult.pushConstants.size()); + + ShaderRecompiler::ShaderPixelStageInfo pixel{}; + pixel.inputAddr = ShaderRecompiler::PixelInputBit(ShaderRecompiler::PixelInput::PositionX) | ShaderRecompiler::PixelInputBit(ShaderRecompiler::PixelInput::PositionY); + pixel.posX = true; + pixel.posY = true; + pixel.targetOutputMode[0] = 9; + pixel.targetExportMapping.fill(0xe4u); + const std::vector pixelUserData(8, 0u); + const std::array pixelMemory{{{reinterpret_cast(PixelCode.data()), std::as_bytes(std::span(PixelCode))}}}; + ShaderRecompiler::RecompileRequest fragment{ + {ShaderStage::Fragment, reinterpret_cast(PixelCode.data()), PixelCode, 0, {}}, + {64u, 0, pixelUserData, std::nullopt, pixel, std::nullopt, pixelMemory}, + device.Target(), + PixelPushLayout(vertexPush, device.Target()) + }; + fragment.useCache = false; + const auto pixelResult = ShaderRecompiler::Recompile(fragment); + const std::array shaders{{ + {ShaderStage::Vertex, &vertexResult, 0}, + {ShaderStage::Fragment, &pixelResult, PixelPushOffset(vertexPush, device.Target())} + }}; + + AgcDriver::Graphics::State state{}; + state.stages = {AgcDriver::Graphics::ShaderPath::Vertex, 0u, 64u, 64u, std::nullopt, std::nullopt}; + state.color = {reinterpret_cast(Pixels.data()), {Width, Height}, VK_FORMAT_R8G8B8A8_UNORM, Pixels.size(), 0xe4u}; + state.colors = {state.color}; + state.hasColorTarget = true; + state.renderExtent = {Width, Height}; + state.topology = VK_PRIMITIVE_TOPOLOGY_TRIANGLE_LIST; + state.viewport = {0, static_cast(Height), static_cast(Width), -static_cast(Height), 0, 1}; + state.negativeOneToOne = false; + state.scissor = {{0, 0}, {Width, Height}}; + state.cullMode = VK_CULL_MODE_NONE; + state.frontFace = VK_FRONT_FACE_COUNTER_CLOCKWISE; + state.blend.colorWriteMask = 15; + state.blends = {state.blend}; + state.blendConstants = {}; + const AgcDriver::Pm4::DrawParameters draw{0, Vertices, 0, 1, 0, false}; + device.Draw(state, draw, shaders); + device.WaitIdle(); +} + +void Check(const Case& value) { + for (std::uint32_t vertex = 0; vertex < Vertices; ++vertex) { + for (std::uint32_t component = 0; component < 4u; ++component) { + const auto expected = value.inRange ? std::bit_cast(Element[component]) : 0u; + const auto actual = Output[vertex * 4u + component]; + Require(actual == expected, std::string(value.what) + ": vertex " + std::to_string(vertex) + " v" + std::to_string(8u + component) + " is " + std::to_string(actual) + ", expected " + std::to_string(expected)); + } + } +} + +} + +int main() { + try { + const auto device = OpenVulkanTestDevice(); + if (!device) return VulkanTestSkipped; + constexpr std::array cases{{ + {2u, 1u, true, "OOB_SELECT 2, one record"}, + {2u, 0u, false, "OOB_SELECT 2, no records"}, + {3u, 16u, true, "OOB_SELECT 3, whole element in range"}, + {3u, 0u, false, "OOB_SELECT 3, no records"}, + {0u, 16u, true, "OOB_SELECT 0, whole element in range"}, + }}; + for (const auto& value : cases) { + Draw(*device, value); + Check(value); + } + std::puts("zero-stride vertex fetch tests passed"); + return 0; + } catch (const std::exception& error) { + std::cerr << error.what() << '\n'; + return 1; + } +} diff --git a/core/libs/prx/libSceAgcDriver/tests/shaders/Cover.frag b/core/libs/prx/libSceAgcDriver/tests/shaders/Cover.frag new file mode 100644 index 000000000..4bb961d02 --- /dev/null +++ b/core/libs/prx/libSceAgcDriver/tests/shaders/Cover.frag @@ -0,0 +1,5 @@ +#version 450 +layout(location = 0) out vec4 color; +void main() { + color = vec4(1.0); +} diff --git a/core/libs/prx/libSceAgcDriver/tests/shaders/Cover.vert b/core/libs/prx/libSceAgcDriver/tests/shaders/Cover.vert new file mode 100644 index 000000000..4a5591d03 --- /dev/null +++ b/core/libs/prx/libSceAgcDriver/tests/shaders/Cover.vert @@ -0,0 +1,6 @@ +#version 450 +out gl_PerVertex { vec4 gl_Position; }; +void main() { + vec2 position = vec2(float((gl_VertexIndex << 1) & 2), float(gl_VertexIndex & 2)); + gl_Position = vec4(position * 2.0 - 1.0, 0.5, 1.0); +} diff --git a/core/libs/prx/libSceAgcDriver/tests/shaders/ProvokingVertex.frag b/core/libs/prx/libSceAgcDriver/tests/shaders/ProvokingVertex.frag new file mode 100644 index 000000000..966977ddc --- /dev/null +++ b/core/libs/prx/libSceAgcDriver/tests/shaders/ProvokingVertex.frag @@ -0,0 +1,6 @@ +#version 450 +layout(location = 0) flat in vec4 color; +layout(location = 0) out vec4 outputColor; +void main() { + outputColor = color; +} diff --git a/core/libs/prx/libSceAgcDriver/tests/shaders/ProvokingVertex.vert b/core/libs/prx/libSceAgcDriver/tests/shaders/ProvokingVertex.vert new file mode 100644 index 000000000..c561c90e0 --- /dev/null +++ b/core/libs/prx/libSceAgcDriver/tests/shaders/ProvokingVertex.vert @@ -0,0 +1,13 @@ +#version 450 +out gl_PerVertex { vec4 gl_Position; }; +layout(location = 0) flat out vec4 color; +void main() { + vec2 positions[6] = vec2[]( + vec2(-0.75, 0.75), vec2(-0.75, -0.75), vec2(0.75, 0.75), + vec2(0.75, -0.75), vec2(0.75, 0.75), vec2(-0.75, -0.75)); + vec4 colors[6] = vec4[]( + vec4(1, 0, 0, 1), vec4(0, 1, 0, 1), vec4(0, 0, 1, 1), + vec4(1, 1, 0, 1), vec4(0, 1, 1, 1), vec4(1, 0, 1, 1)); + gl_Position = vec4(positions[gl_VertexIndex], 0.5, 1.0); + color = colors[gl_VertexIndex]; +} diff --git a/core/libs/prx/libSceAjm.native/CMakeLists.txt b/core/libs/prx/libSceAjm.native/CMakeLists.txt index aa4e24f83..a231a1529 100644 --- a/core/libs/prx/libSceAjm.native/CMakeLists.txt +++ b/core/libs/prx/libSceAjm.native/CMakeLists.txt @@ -36,6 +36,13 @@ if(BUILD_TESTING) target_link_libraries(guest_ajm_tests PRIVATE libSceAjm.native libc) configure_windows_unwind(guest_ajm_tests) add_test(NAME guest_ajm COMMAND guest_ajm_tests) + if(UNIX) + add_test_executable(guest_ajm_teardown_tests ${LIBS_INCLUDE_DIR}/tests/GuestAjmTeardown.cpp) + target_include_directories(guest_ajm_teardown_tests PRIVATE ${LIBS_INCLUDE_DIR}) + target_link_libraries(guest_ajm_teardown_tests PRIVATE libc ${CMAKE_DL_LIBS}) + add_dependencies(guest_ajm_teardown_tests libSceAjm.native) + add_test(NAME guest_ajm_teardown COMMAND guest_ajm_teardown_tests $) + endif() if(WIN32) set_tests_properties(guest_ajm PROPERTIES ENVIRONMENT_MODIFICATION "PATH=path_list_prepend:$") diff --git a/core/libs/prx/libSceAjm.native/src/Ajm.cpp b/core/libs/prx/libSceAjm.native/src/Ajm.cpp index b7649e151..89dc47bc0 100644 --- a/core/libs/prx/libSceAjm.native/src/Ajm.cpp +++ b/core/libs/prx/libSceAjm.native/src/Ajm.cpp @@ -124,6 +124,7 @@ struct Instance { std::uint32_t superframeRemaining = 0; std::uint32_t frameInSuperframe = 0; std::uint64_t totalDecodedSamples = 0; + std::uint64_t segmentSamples = 0; SidebandGaplessDecode gapless{}; bool flagsReported = false; AVCodecContext* mp3 = nullptr; @@ -144,13 +145,13 @@ struct Instance { } }; -std::mutex g_lock; -std::map> g_instances; +std::mutex& g_lock = *new std::mutex(); +std::map>& g_instances = *new std::map>(); std::atomic g_nextContext{1}; std::atomic g_nextInstance{1}; std::atomic g_nextBatch{1}; -std::mutex g_batchLock; -std::set g_batches; +std::mutex& g_batchLock = *new std::mutex(); +std::set& g_batches = *new std::set(); enum class JobKind : std::uint32_t { Initialize = 1, @@ -193,6 +194,7 @@ static_assert(sizeof(PackedJob) == 32); int Append(AjmBatchInfo* info, const JobHeader& header, const AjmBuffer* inputs, const AjmBuffer* outputs) { if (!info || !info->p_buffer) return SCE_AJM_ERROR_INVALID_PARAMETER; if (header.inputCount > 0xffu || header.outputCount > 0xffu || header.parameterSize > sizeof(header.parameters) || header.sidebandSize > 0xffffffffu) return SCE_AJM_ERROR_INVALID_PARAMETER; + if ((header.inputCount && !inputs) || (header.outputCount && !outputs)) NotImplemented_nid_no_patch("AJM batch job with a null buffer list and a nonzero buffer count"); const std::size_t parameterBytes = (header.parameterSize + 7u) & ~std::size_t{7}; const std::size_t bytes = sizeof(PackedJob) + (header.inputCount + header.outputCount) * sizeof(AjmBuffer) + parameterBytes; if (info->offset > info->size || bytes > info->size - info->offset) return SCE_AJM_ERROR_OUT_OF_RESOURCES; @@ -200,9 +202,9 @@ int Append(AjmBatchInfo* info, const JobHeader& header, const AjmBuffer* inputs, const PackedJob record{static_cast(header.kind), static_cast(header.inputCount), static_cast(header.outputCount), static_cast(header.parameterSize), header.instance, header.flags, header.sideband, static_cast(header.sidebandSize), static_cast(bytes)}; std::memcpy(cursor, &record, sizeof(record)); cursor += sizeof(record); - if (header.inputCount) std::memcpy(cursor, inputs, header.inputCount * sizeof(AjmBuffer)); + if (inputs) std::memcpy(cursor, inputs, header.inputCount * sizeof(AjmBuffer)); cursor += header.inputCount * sizeof(AjmBuffer); - if (header.outputCount) std::memcpy(cursor, outputs, header.outputCount * sizeof(AjmBuffer)); + if (outputs) std::memcpy(cursor, outputs, header.outputCount * sizeof(AjmBuffer)); cursor += header.outputCount * sizeof(AjmBuffer); if (header.parameterSize) std::memcpy(cursor, header.parameters, header.parameterSize); info->offset += bytes; @@ -375,6 +377,7 @@ std::int32_t InitializeInstance(Instance& instance, const std::uint8_t* paramete instance.opusPending.clear(); instance.initialized = true; instance.totalDecodedSamples = 0; + instance.segmentSamples = 0; instance.gapless = {}; return 0; } @@ -392,6 +395,7 @@ std::int32_t InitializeInstance(Instance& instance, const std::uint8_t* paramete } instance.initialized = true; instance.totalDecodedSamples = 0; + instance.segmentSamples = 0; instance.gapless = {}; return 0; } @@ -545,9 +549,10 @@ void EmitFrame(Instance& instance, PcmOutputs& outputs, const std::uint8_t* pcm, count -= skip; } if (instance.gapless.totalSamples != 0) { - const std::uint64_t remaining = instance.gapless.totalSamples > instance.totalDecodedSamples ? instance.gapless.totalSamples - instance.totalDecodedSamples : 0; + const std::uint64_t remaining = instance.gapless.totalSamples > instance.segmentSamples ? instance.gapless.totalSamples - instance.segmentSamples : 0; count = static_cast(std::min(count, remaining)); } + instance.segmentSamples += count; instance.totalDecodedSamples += count; if (!Resampling(instance)) { outputs.Emit(pcm + first * channels * sampleBytes, count * channels * sampleBytes); @@ -627,7 +632,9 @@ void RunAt9(Instance& instance, const JobHeader& job, const AjmBuffer* inputs, c const auto superframeSize = static_cast(instance.info.superframeSize); const auto framesInSuperframe = static_cast(std::max(1, instance.info.framesInSuperframe)); if (Resampling(instance)) ResamplerProduce(instance, pcmOutputs, channels, encoding, sampleBytes); + const auto segmentEnded = [&] { return instance.gapless.totalSamples != 0 && instance.segmentSamples >= instance.gapless.totalSamples; }; for (;;) { + if (segmentEnded()) break; if (instance.superframeRemaining == 0) consumed += RiffDataOffset(input.data() + consumed, input.size() - consumed); const std::size_t needed = instance.superframeRemaining == 0 ? superframeSize : instance.superframeRemaining; if (input.size() - consumed < needed) { @@ -659,8 +666,8 @@ void RunAt9(Instance& instance, const JobHeader& job, const AjmBuffer* inputs, c ++frames; EmitFrame(instance, pcmOutputs, pcm.data(), frameSamples, channels, sampleBytes); - if ((job.flags & RUN_MULTIPLE_FRAMES) == 0) break; } + if (segmentEnded()) instance.segmentSamples = 0; if (TraceEnabled()) { std::fprintf(stderr, "[ajm] instance %u run flags 0x%llx: %zu input bytes in %u buffers, %zu output bytes in %u buffers, sideband %llu bytes; superframe %d (%d frames), %d channels, %d samples/frame, %d Hz, config %02x %02x %02x %02x -> result 0x%x, decoder status 0x%x, %u frames, consumed %zu, produced %zu, total samples %llu, gapless total %u skip %u skipped %u, input", @@ -1059,6 +1066,7 @@ void RunOpus(Instance& instance, const JobHeader& job, const AjmBuffer* inputs, void ClearContext(Instance& instance) { instance.totalDecodedSamples = 0; + instance.segmentSamples = 0; instance.gapless.skippedSamples = 0; if (HasAt9Decoder(instance) && instance.initialized) ResetDecoder(instance); if (instance.mp3) avcodec_flush_buffers(instance.mp3); @@ -1133,7 +1141,10 @@ void Execute(const JobHeader& job, const AjmBuffer* inputs, const AjmBuffer* out AJM_TRACE("[ajm] instance %u set gapless decode: total %u, skip %u, reset %llu (sideband %llu bytes)\n", job.instance, gapless.totalSamples, gapless.skipSamples, static_cast(job.flags), static_cast(job.sidebandSize)); instance->gapless.totalSamples = gapless.totalSamples; instance->gapless.skipSamples = gapless.skipSamples; - if (job.flags) instance->gapless.skippedSamples = 0; + if (job.flags) { + instance->gapless.skippedSamples = 0; + instance->segmentSamples = 0; + } WriteResult(job.sideband, job.sidebandSize, 0); break; } diff --git a/core/libs/prx/libSceAmpr/Export.cpp b/core/libs/prx/libSceAmpr/Export.cpp index 6105b3e46..89561ff3f 100644 --- a/core/libs/prx/libSceAmpr/Export.cpp +++ b/core/libs/prx/libSceAmpr/Export.cpp @@ -802,4 +802,113 @@ int APS5_VABI sceAmprAmmWaitCommandBufferCompletion(std::uint32_t id) { return AmmSubmitted_nid_no_patch(id) ? 0 : SCE_KERNEL_ERROR_ESRCH; } +int APS5_VABI _ZNSt8ios_base7_AddstdEPS__nid_postfix(void) { + NotImplemented_nid_no_patch(__func__); + return 0; +} + +int APS5_VABI _ZSt16_Throw_Cpp_errori_nid_postfix(void) { + NotImplemented_nid_no_patch(__func__); + return 0; +} + +int APS5_VABI _Thrd_join_nid_postfix(void) { + NotImplemented_nid_no_patch(__func__); + return 0; +} + +int APS5_VABI _ZNSt4_Pad8_ReleaseEv_nid_postfix(void) { + NotImplemented_nid_no_patch(__func__); + return 0; +} + +int APS5_VABI _ZNSt4_PadC2Ev_nid_postfix(void) { + NotImplemented_nid_no_patch(__func__); + return 0; +} + +int APS5_VABI _ZNSt4_PadD2Ev_nid_postfix(void) { + NotImplemented_nid_no_patch(__func__); + return 0; +} + +int APS5_VABI _Thrd_id_nid_postfix(void) { + NotImplemented_nid_no_patch(__func__); + return 0; +} + +int APS5_VABI _ZNSt4_Pad7_LaunchEPP7pthread_nid_postfix(void) { + NotImplemented_nid_no_patch(__func__); + return 0; +} + +int APS5_VABI _ZNSt7num_getIcSt19istreambuf_iteratorIcSt11char_traitsIcEEE2idE_nid_postfix(void) { + NotImplemented_nid_no_patch(__func__); + return 0; +} + +int APS5_VABI _Cnd_signal_nid_postfix(void) { + NotImplemented_nid_no_patch(__func__); + return 0; +} + +int APS5_VABI _Unlock_shared_ptr_spin_lock_nid_postfix(void) { + NotImplemented_nid_no_patch(__func__); + return 0; +} + +int APS5_VABI _Cnd_init_with_name_nid_postfix(void) { + NotImplemented_nid_no_patch(__func__); + return 0; +} + +int APS5_VABI _ZTVN10__cxxabiv120__function_type_infoE_nid_postfix(void) { + NotImplemented_nid_no_patch(__func__); + return 0; +} + +int APS5_VABI _ZTVSt7num_getIcSt19istreambuf_iteratorIcSt11char_traitsIcEEE_nid_postfix(void) { + NotImplemented_nid_no_patch(__func__); + return 0; +} + +int APS5_VABI _ZNKSt8time_getIcSt19istreambuf_iteratorIcSt11char_traitsIcEEE3getES3_S3_RSt8ios_baseRNSt5_IosbIiE8_IostateEP2tmPKcSE__nid_postfix(void) { + NotImplemented_nid_no_patch(__func__); + return 0; +} + +int APS5_VABI _ZNSt8time_getIcSt19istreambuf_iteratorIcSt11char_traitsIcEEE2idE_nid_postfix(void) { + NotImplemented_nid_no_patch(__func__); + return 0; +} + +int APS5_VABI _ZTVN10__cxxabiv119__pointer_type_infoE_nid_postfix(void) { + NotImplemented_nid_no_patch(__func__); + return 0; +} + +int APS5_VABI _ZNSt8time_getIcSt19istreambuf_iteratorIcSt11char_traitsIcEEE7_GetcatEPPKNSt6locale5facetEPKS5__nid_postfix(void) { + NotImplemented_nid_no_patch(__func__); + return 0; +} + +int APS5_VABI _Lock_shared_ptr_spin_lock_nid_postfix(void) { + NotImplemented_nid_no_patch(__func__); + return 0; +} + +int APS5_VABI _Thrd_sleep_nid_postfix(void) { + NotImplemented_nid_no_patch(__func__); + return 0; +} + +int APS5_VABI wcstombs_s_nid_postfix(void) { + NotImplemented_nid_no_patch(__func__); + return 0; +} + +int APS5_VABI __cxa_call_unexpected_nid_postfix(void) { + NotImplemented_nid_no_patch(__func__); + return 0; +} } diff --git a/core/libs/prx/libSceAudio3d/Export.cpp b/core/libs/prx/libSceAudio3d/Export.cpp index 5254ad3c7..b366da306 100644 --- a/core/libs/prx/libSceAudio3d/Export.cpp +++ b/core/libs/prx/libSceAudio3d/Export.cpp @@ -7,9 +7,9 @@ #include #include #include -#include #include "SceTypes.hpp" #include "prx/libc/include/General.hpp" +#include "prx/libc/include/PreciseWait.hpp" namespace { @@ -27,6 +27,7 @@ constexpr std::uint32_t AUDIO3D_SAMPLE_RATE = 48000; constexpr std::uint32_t AUDIO3D_BUFFER_ADVANCE_AND_PUSH = 2; constexpr std::uint32_t AUDIO3D_BLOCKING_ASYNC = 0; constexpr std::uint32_t AUDIO3D_BLOCKING_SYNC = 1; +constexpr std::uint32_t AUDIO3D_ATTRIBUTE_RESET_STATE = 0x20000; constexpr std::uint32_t AUDIO3D_ATTRIBUTE_LATE_REVERB_LEVEL = 0x10001; constexpr std::uint32_t AUDIO3D_ATTRIBUTE_DOWNMIX_SPREAD_RADIUS = 0x10002; constexpr std::uint32_t AUDIO3D_ATTRIBUTE_DOWNMIX_SPREAD_HEIGHT_AWARE = 0x10003; @@ -167,7 +168,7 @@ int APS5_VABI sceAudio3dPortPush(uint32_t port_id, uint32_t blocking) { if (blocking == AUDIO3D_BLOCKING_ASYNC || level < g_port.queue_depth) return 0; wait_until = g_port.playing[level - g_port.queue_depth]; } - while (Clock::now() < wait_until) std::this_thread::sleep_until(wait_until); + PreciseSleepUntil(wait_until); return 0; } @@ -207,4 +208,24 @@ int APS5_VABI sceAudio3dObjectUnreserve(uint32_t port_id, uint32_t object_id) { return 0; } +int APS5_VABI sceAudio3dObjectSetAttributes(uint32_t port_id, uint32_t object_id, uint64_t num_attributes, const Audio3dAttribute* attribute_array) { + std::lock_guard lock(g_mutex); + if (!PortIsOpen(port_id)) return AUDIO3D_ERROR_INVALID_PORT; + if (num_attributes == 0 || attribute_array == nullptr) return AUDIO3D_ERROR_INVALID_PARAMETER; + if (!g_port.objects.contains(object_id)) return AUDIO3D_ERROR_INVALID_OBJECT; + for (std::uint64_t i = 0; i < num_attributes; ++i) { + if (attribute_array[i].attribute_id == AUDIO3D_ATTRIBUTE_RESET_STATE) continue; + if (attribute_array[i].value == nullptr) return AUDIO3D_ERROR_INVALID_PARAMETER; + } + return 0; +} + +int APS5_VABI sceAudio3dPortFlush(uint32_t port_id) { + std::lock_guard lock(g_mutex); + if (!PortIsOpen(port_id)) return AUDIO3D_ERROR_INVALID_PORT; + g_port.advanced = 0; + g_port.playing.clear(); + return 0; +} + } diff --git a/core/libs/prx/libSceAudioIn/Export.cpp b/core/libs/prx/libSceAudioIn/Export.cpp index 1899b5ab7..4ba03302f 100644 --- a/core/libs/prx/libSceAudioIn/Export.cpp +++ b/core/libs/prx/libSceAudioIn/Export.cpp @@ -2,13 +2,13 @@ #include #include "SceTypes.hpp" #include "prx/libc/include/General.hpp" +#include "prx/libc/include/PreciseWait.hpp" #include "SDL.h" #include #include #include #include #include -#include namespace { @@ -76,7 +76,7 @@ Port* find(int handle) { } std::size_t capture(SDL_AudioDeviceID device, std::uint8_t* dest, std::size_t bytes, Clock::time_point deadline) { - while (SDL_GetQueuedAudioSize(device) < bytes && Clock::now() < deadline) std::this_thread::sleep_for(std::chrono::milliseconds(1)); + while (SDL_GetQueuedAudioSize(device) < bytes && Clock::now() < deadline) PreciseSleepUs(1000); const std::size_t queued = SDL_GetQueuedAudioSize(device); std::array discard; for (std::size_t excess = queued > bytes * MAX_QUEUED_BLOCKS ? queued - bytes * MAX_QUEUED_BLOCKS : 0; excess > 0;) { @@ -122,7 +122,7 @@ int APS5_VABI sceAudioInInput(int handle, void* dest) { if (device != 0) { captured = capture(device, out, bytes, deadline); } else { - while (Clock::now() < wake) std::this_thread::sleep_until(wake); + PreciseSleepUntil(wake); } std::memset(out + captured, 0, bytes - captured); lock.lock(); diff --git a/core/libs/prx/libSceAudioOut/CMakeLists.txt b/core/libs/prx/libSceAudioOut/CMakeLists.txt index c3d0bd85a..d42ddf36a 100644 --- a/core/libs/prx/libSceAudioOut/CMakeLists.txt +++ b/core/libs/prx/libSceAudioOut/CMakeLists.txt @@ -66,6 +66,15 @@ if(BUILD_TESTING) set_tests_properties(audio_out2_port_layouts PROPERTIES ENVIRONMENT_MODIFICATION "PATH=path_list_prepend:$") endif() + add_test_executable(audio_out2_port_buffers_tests tests/AudioOut2PortBuffers.cpp) + target_include_directories(audio_out2_port_buffers_tests PRIVATE ${LIBS_INCLUDE_DIR}) + target_link_libraries(audio_out2_port_buffers_tests PRIVATE libSceAudioOut libc) + configure_windows_unwind(audio_out2_port_buffers_tests) + add_test(NAME audio_out2_port_buffers COMMAND audio_out2_port_buffers_tests) + if(WIN32) + set_tests_properties(audio_out2_port_buffers PROPERTIES ENVIRONMENT_MODIFICATION + "PATH=path_list_prepend:$") + endif() add_test_executable(audio_out2_timing_tests tests/AudioOut2Timing.cpp) target_include_directories(audio_out2_timing_tests PRIVATE ${LIBS_INCLUDE_DIR}) target_link_libraries(audio_out2_timing_tests PRIVATE libSceAudioOut libc) diff --git a/core/libs/prx/libSceAudioOut/src/AudioOut.cpp b/core/libs/prx/libSceAudioOut/src/AudioOut.cpp index a6697a1a0..5dcdd2821 100644 --- a/core/libs/prx/libSceAudioOut/src/AudioOut.cpp +++ b/core/libs/prx/libSceAudioOut/src/AudioOut.cpp @@ -13,6 +13,7 @@ #include "SDL.h" #include "SceTypes.hpp" +#include "prx/libc/include/PreciseWait.hpp" #include "prx/libkernel/Time/include/Time.hpp" static constexpr int PORT_TYPE_MAIN = 0; @@ -95,6 +96,7 @@ struct Port { }; static std::mutex g_mutex; +static std::mutex g_outputMutex; static Port g_ports[PORTS_MAX]; static bool g_sdlInitialized = false; @@ -190,7 +192,13 @@ static const void* prepareBuffer(const Port& port, const void* data, std::vector return buf.data(); } -static void queueAudio(Port& port, const void* data) { +static void sleepUs(std::unique_lock& lock, std::uint64_t us) { + lock.unlock(); + PreciseSleepUs(us); + lock.lock(); +} + +static void queueAudio(Port& port, const void* data, std::unique_lock& lock) { if (data == nullptr) { const std::uint64_t waitStart = sceKernelGetProcessTime(); while (SDL_GetQueuedAudioSize(port.device) > 0) { @@ -198,10 +206,7 @@ static void queueAudio(Port& port, const void* data) { SDL_ClearQueuedAudio(port.device); break; } - struct timespec req{}; - req.tv_sec = 0; - req.tv_nsec = static_cast(DRAIN_SLEEP_US * 1000ULL); - nanosleep(&req, nullptr); + sleepUs(lock, DRAIN_SLEEP_US); } return; } @@ -251,10 +256,7 @@ static void queueAudio(Port& port, const void* data) { SDL_ClearQueuedAudio(port.device); break; } - struct timespec req{}; - req.tv_sec = 0; - req.tv_nsec = static_cast(DRAIN_SLEEP_US * 1000ULL); - nanosleep(&req, nullptr); + sleepUs(lock, DRAIN_SLEEP_US); } if (SDL_QueueAudio(port.device, queueData, queueSize) < 0) { @@ -263,22 +265,15 @@ static void queueAudio(Port& port, const void* data) { // APS5_LOG_OUT("device=%u type=%d bytes=%u queued=%u", port.device, port.type, queueSize, SDL_GetQueuedAudioSize(port.device)); } -static void sleepUs(std::uint64_t us) { - struct timespec req{}; - req.tv_sec = static_cast(us / 1000000ULL); - req.tv_nsec = static_cast((us % 1000000ULL) * 1000ULL); - nanosleep(&req, nullptr); -} - -static void paceVirtualPort(Port& port, bool hasData) { +static void paceVirtualPort(Port& port, bool hasData, std::unique_lock& lock) { const std::uint64_t now = sceKernelGetProcessTime(); const auto queued = static_cast(port.virtualQueueEnd - now); if (queued <= 0) port.virtualQueueEnd = now; if (!hasData) { - if (queued > 0) sleepUs(static_cast(queued)); + if (queued > 0) sleepUs(lock, static_cast(queued)); return; } - if (queued > static_cast(TARGET_LATENCY_US)) sleepUs(static_cast(queued) - TARGET_LATENCY_US); + if (queued > static_cast(TARGET_LATENCY_US)) sleepUs(lock, static_cast(queued) - TARGET_LATENCY_US); port.virtualQueueEnd += (1000000ULL * port.samplesNum) / port.freq; } @@ -361,6 +356,7 @@ int APS5_VABI sceAudioOutOpen(int userId, int type, int index, std::uint32_t len } int APS5_VABI sceAudioOutClose(int handle) { + std::lock_guard outputLock(g_outputMutex); std::lock_guard lock(g_mutex); Port* port = getPort(handle); if (port == nullptr) { @@ -372,16 +368,17 @@ int APS5_VABI sceAudioOutClose(int handle) { } int APS5_VABI sceAudioOutOutput(int handle, const void* ptr) { - std::lock_guard lock(g_mutex); + std::lock_guard outputLock(g_outputMutex); + std::unique_lock lock(g_mutex); Port* port = getPort(handle); if (port == nullptr) { return -2144993277; } if (port->device == 0) { - paceVirtualPort(*port, ptr != nullptr); + paceVirtualPort(*port, ptr != nullptr, lock); } else { - queueAudio(*port, ptr); + queueAudio(*port, ptr, lock); } if (ptr != nullptr) port->lastDataOutputTime = sceKernelGetProcessTime(); return static_cast(port->samplesNum); @@ -392,7 +389,8 @@ int APS5_VABI sceAudioOutOutputs(AudioOutOutputParam* param, std::uint32_t num) return -2144993276; } - std::lock_guard lock(g_mutex); + std::lock_guard outputLock(g_outputMutex); + std::unique_lock lock(g_mutex); for (std::uint32_t i = 0; i < num; i++) { if (getPort(param[i].handle) == nullptr) { @@ -404,9 +402,9 @@ int APS5_VABI sceAudioOutOutputs(AudioOutOutputParam* param, std::uint32_t num) for (std::uint32_t i = 0; i < num; i++) { Port& port = *getPort(param[i].handle); if (port.device == 0) { - paceVirtualPort(port, param[i].ptr != nullptr); + paceVirtualPort(port, param[i].ptr != nullptr, lock); } else { - queueAudio(port, param[i].ptr); + queueAudio(port, param[i].ptr, lock); } } diff --git a/core/libs/prx/libSceAudioOut/src/AudioOut2Context.cpp b/core/libs/prx/libSceAudioOut/src/AudioOut2Context.cpp index 9dc8f44d6..383ad018f 100644 --- a/core/libs/prx/libSceAudioOut/src/AudioOut2Context.cpp +++ b/core/libs/prx/libSceAudioOut/src/AudioOut2Context.cpp @@ -12,6 +12,7 @@ #include #include "SceTypes.hpp" #include "prx/libc/include/General.hpp" +#include "prx/libc/include/PreciseWait.hpp" #include "AudioOut2Internal.hpp" // An AudioOut2 context is the hardware output queue: every push appends a grain (num_grains samples) @@ -324,7 +325,7 @@ int APS5_VABI sceAudioOut2ContextPush(AudioOut2ContextHandle ctx, uint32_t block } if (now - waitStart > FULL_WAIT_TIMEOUT) break; lock.unlock(); - std::this_thread::sleep_for(FULL_WAIT_STEP); + PreciseSleepUs(std::chrono::microseconds(FULL_WAIT_STEP).count()); lock.lock(); now = Clock::now(); } diff --git a/core/libs/prx/libSceAudioOut/src/AudioOut2Internal.hpp b/core/libs/prx/libSceAudioOut/src/AudioOut2Internal.hpp index 96b9ed331..1a6f8cc7e 100644 --- a/core/libs/prx/libSceAudioOut/src/AudioOut2Internal.hpp +++ b/core/libs/prx/libSceAudioOut/src/AudioOut2Internal.hpp @@ -63,6 +63,9 @@ struct AudioOut2Port { bool int16 = false; const AudioOut2StereoFold* fold = nullptr; const void* data = nullptr; + std::deque queuedData; + const void* lastBuffer = nullptr; + const void* repeatedData = nullptr; float volume[AUDIO_OUT2_PORT_CHANNELS_MAX] = {1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f, 1.0f}; std::uint64_t dataSets = 0; std::uint64_t attributeTraces = 0; diff --git a/core/libs/prx/libSceAudioOut/src/AudioOut2Port.cpp b/core/libs/prx/libSceAudioOut/src/AudioOut2Port.cpp index 03248d3e0..0deb7d563 100644 --- a/core/libs/prx/libSceAudioOut/src/AudioOut2Port.cpp +++ b/core/libs/prx/libSceAudioOut/src/AudioOut2Port.cpp @@ -23,6 +23,7 @@ static constexpr std::uint64_t ATTRIBUTE_TRACE_EVERY = 2000; // 8 and 9 (a u32, a u32 and a byte on object and voice ports) are not understood and ignored. static constexpr std::uint32_t ATTRIBUTE_DATA = 0; static constexpr std::uint32_t ATTRIBUTE_VOLUME = 1; +static constexpr std::size_t QUEUED_DATA_MAX = 16; static constexpr std::uint32_t FORMAT_CHANNELS_SHIFT = 8; static constexpr std::uint32_t FORMAT_CHANNELS_MASK = 0xFu; @@ -102,8 +103,14 @@ AudioOut2Grain AudioOut2CaptureGrain(const AudioOut2Context& context) { std::lock_guard lock(g_portsLock); AudioOut2Grain grain; for (std::size_t index = 0; index < g_ports.size(); index++) { - const auto& port = g_ports[index]; - if (port.used && port.context == &context && port.data != nullptr && port.channels != 0) grain.push_back({index, port.generation, port.data}); + auto& port = g_ports[index]; + if (!port.used || port.context != &context || port.channels == 0) continue; + const void* data = port.repeatedData; + if (!port.queuedData.empty()) { + data = port.queuedData.front(); + port.queuedData.pop_front(); + } + if (data != nullptr) grain.push_back({index, port.generation, data}); } return grain; } @@ -115,7 +122,11 @@ std::uint32_t AudioOut2MixPorts(const AudioOut2Context& context, const AudioOut2 if (index >= g_ports.size()) continue; const auto& port = g_ports[index]; if (!port.used || port.generation != generation || port.context != &context || port.channels == 0) continue; - const auto route = padOut != nullptr ? AudioOut2RouteForPort(port.type, port.channels) : AudioOut2Route::Main; + auto route = AudioOut2RouteForPort(port.type, port.channels); + if (padOut == nullptr) { + if (route == AudioOut2Route::PadVibration) continue; + route = AudioOut2Route::Main; + } if (route == AudioOut2Route::Main) AccumulatePort(port, data, out, frames); else AccumulatePadPort(port, data, route, padOut, frames); mixed++; @@ -234,6 +245,10 @@ int APS5_VABI sceAudioOut2PortSetAttributes(AudioOut2PortHandle port, const Audi verdict = "null value"; } else if (attribute.attribute_id == ATTRIBUTE_DATA && attribute.value_size == sizeof(entry->data)) { std::memcpy(&entry->data, attribute.value, sizeof(entry->data)); + entry->queuedData.push_back(entry->data); + if (entry->queuedData.size() > QUEUED_DATA_MAX) entry->queuedData.pop_front(); + entry->repeatedData = entry->data != nullptr && (entry->lastBuffer == nullptr || entry->lastBuffer == entry->data) ? entry->data : nullptr; + if (entry->data != nullptr) entry->lastBuffer = entry->data; entry->dataSets++; verdict = "pcm data pointer"; } else if (attribute.attribute_id == ATTRIBUTE_VOLUME && entry->channels != 0 && attribute.value_size == entry->channels * sizeof(float)) { diff --git a/core/libs/prx/libSceAudioOut/tests/AudioOut2PortBuffers.cpp b/core/libs/prx/libSceAudioOut/tests/AudioOut2PortBuffers.cpp new file mode 100644 index 000000000..54f467b33 --- /dev/null +++ b/core/libs/prx/libSceAudioOut/tests/AudioOut2PortBuffers.cpp @@ -0,0 +1,172 @@ +#include "SceTypes.hpp" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +extern "C" { +int APS5_VABI sceAudioOut2ContextResetParam(AudioOut2ContextParam*); +int APS5_VABI sceAudioOut2ContextCreate(const AudioOut2ContextParam*, void*, std::size_t, AudioOut2ContextHandle*); +int APS5_VABI sceAudioOut2ContextDestroy(AudioOut2ContextHandle); +int APS5_VABI sceAudioOut2ContextPush(AudioOut2ContextHandle, std::uint32_t); +int APS5_VABI sceAudioOut2PortCreate(AudioOut2ContextHandle, const AudioOut2PortParam*, AudioOut2PortHandle*); +int APS5_VABI sceAudioOut2PortDestroy(AudioOut2PortHandle); +int APS5_VABI sceAudioOut2PortSetAttributes(AudioOut2PortHandle, const AudioOut2Attribute*, std::uint32_t); +} + +static void Require(bool value) { if (!value) std::abort(); } + +static void SetEnvironment(const char* name, const std::string& value) { +#ifdef _WIN32 + _putenv_s(name, value.c_str()); +#else + setenv(name, value.c_str(), 1); +#endif +} + +namespace { + +constexpr std::uint32_t grain = 256; +constexpr std::uint32_t channels = 2; +constexpr std::uint32_t frequency = 48000; +constexpr std::uint16_t portTypeMain = 0; +constexpr std::uint32_t attributeData = 0; +constexpr std::uint32_t formatStereoFloat = 2u << 8; +constexpr std::uint32_t silentGrains = 32; +constexpr float masterGain = 0.5f; +constexpr float tolerance = 1e-6f; + +void SetData(AudioOut2PortHandle port, const void* data) { + const AudioOut2Attribute attribute{attributeData, 0, &data, sizeof(data)}; + Require(sceAudioOut2PortSetAttributes(port, &attribute, 1) == 0); +} + +std::vector Play(const std::function& feed) { + const auto path = std::filesystem::temp_directory_path() / ("anyps5_audio_out2_port_buffers-" + std::to_string(std::random_device{}()) + ".raw"); + std::filesystem::remove(path); + SetEnvironment("SDL_DISKAUDIOFILE", path.string()); + + AudioOut2ContextParam params{}; + Require(sceAudioOut2ContextResetParam(¶ms) == 0); + params.num_grains = grain; + params.queue_depth = 1; + AudioOut2ContextHandle context = 0; + Require(sceAudioOut2ContextCreate(¶ms, nullptr, 0, &context) == 0); + AudioOut2PortParam portParams{}; + portParams.port_type = portTypeMain; + portParams.data_format = formatStereoFloat; + portParams.sampling_freq = frequency; + AudioOut2PortHandle port = 0; + Require(sceAudioOut2PortCreate(context, &portParams, &port) == 0); + feed(context, port); + for (std::uint32_t push = 0; push < silentGrains; push++) Require(sceAudioOut2ContextPush(context, 1) == 0); + Require(sceAudioOut2PortDestroy(port) == 0); + Require(sceAudioOut2ContextDestroy(context) == 0); + + std::ifstream file(path, std::ios::binary); + const std::vector bytes{std::istreambuf_iterator(file), std::istreambuf_iterator()}; + file.close(); + std::filesystem::remove(path); + Require(bytes.size() % sizeof(float) == 0); + const auto* samples = reinterpret_cast(bytes.data()); + std::size_t first = 0; + std::size_t last = bytes.size() / sizeof(float); + while (first < last && samples[first] == 0.0f) first++; + while (last > first && samples[last - 1] == 0.0f) last--; + return {samples + first, samples + last}; +} + +std::vector Grain(float level) { + std::vector data(static_cast(grain) * channels); + for (std::uint32_t frame = 0; frame < grain; frame++) { + const float sign = frame % 2 == 0 ? 1.0f : -1.0f; + data[frame * channels] = sign * level * static_cast(1 + frame % 5); + data[frame * channels + 1] = -sign * level * static_cast(1 + frame % 3); + } + return data; +} + +void RequirePlayed(const std::vector& played, const std::vector*>& grains) { + Require(played.size() == grains.size() * grain * channels); + for (std::size_t index = 0; index < grains.size(); index++) { + const auto& data = *grains[index]; + for (std::size_t sample = 0; sample < data.size(); sample++) { + Require(std::fabs(played[index * data.size() + sample] - data[sample] * masterGain) <= tolerance); + } + } +} + +void TestBuffersSetBeforeOnePushPlayInOrder() { + const auto first = Grain(0.01f); + const auto second = Grain(0.02f); + const auto third = Grain(0.03f); + const auto played = Play([&](AudioOut2ContextHandle context, AudioOut2PortHandle port) { + SetData(port, first.data()); + SetData(port, second.data()); + SetData(port, third.data()); + Require(sceAudioOut2ContextPush(context, 1) == 0); + Require(sceAudioOut2ContextPush(context, 1) == 0); + Require(sceAudioOut2ContextPush(context, 1) == 0); + }); + RequirePlayed(played, {&first, &second, &third}); +} + +void TestBurstThenGapKeepsEveryBuffer() { + const auto first = Grain(0.01f); + const auto second = Grain(0.02f); + const auto third = Grain(0.03f); + const auto played = Play([&](AudioOut2ContextHandle context, AudioOut2PortHandle port) { + SetData(port, first.data()); + Require(sceAudioOut2ContextPush(context, 1) == 0); + SetData(port, second.data()); + SetData(port, third.data()); + Require(sceAudioOut2ContextPush(context, 1) == 0); + Require(sceAudioOut2ContextPush(context, 1) == 0); + }); + RequirePlayed(played, {&first, &second, &third}); +} + +void TestAdvancingPortWithoutNewBufferPlaysSilence() { + const auto first = Grain(0.01f); + const auto second = Grain(0.02f); + const auto played = Play([&](AudioOut2ContextHandle context, AudioOut2PortHandle port) { + SetData(port, first.data()); + Require(sceAudioOut2ContextPush(context, 1) == 0); + SetData(port, second.data()); + Require(sceAudioOut2ContextPush(context, 1) == 0); + }); + RequirePlayed(played, {&first, &second}); +} + +void TestRewrittenBufferRepeats() { + auto buffer = Grain(0.01f); + const auto first = buffer; + const auto second = Grain(0.02f); + const auto played = Play([&](AudioOut2ContextHandle context, AudioOut2PortHandle port) { + SetData(port, buffer.data()); + Require(sceAudioOut2ContextPush(context, 1) == 0); + Require(sceAudioOut2ContextPush(context, 1) == 0); + buffer = second; + SetData(port, nullptr); + }); + RequirePlayed(played, {&first, &second}); +} + +} + +int main() { + SetEnvironment("SDL_AUDIODRIVER", "disk"); + TestBuffersSetBeforeOnePushPlayInOrder(); + TestBurstThenGapKeepsEveryBuffer(); + TestAdvancingPortWithoutNewBufferPlaysSilence(); + TestRewrittenBufferRepeats(); + return 0; +} diff --git a/core/libs/prx/libSceAudioOut/tests/AudioOut2PortLayouts.cpp b/core/libs/prx/libSceAudioOut/tests/AudioOut2PortLayouts.cpp index 56648e1b0..bf6c33e17 100644 --- a/core/libs/prx/libSceAudioOut/tests/AudioOut2PortLayouts.cpp +++ b/core/libs/prx/libSceAudioOut/tests/AudioOut2PortLayouts.cpp @@ -37,6 +37,8 @@ namespace { constexpr std::uint32_t grain = 256; constexpr std::uint32_t frequency = 48000; constexpr std::uint16_t portTypeMain = 0; +constexpr std::uint16_t portTypePadSpeaker = 0x3; +constexpr std::uint16_t portTypePadVibration = 0x6; constexpr std::uint32_t attributeData = 0; constexpr std::uint32_t attributeVolume = 1; constexpr std::uint32_t formatFloat = 0; @@ -73,9 +75,9 @@ AudioOut2ContextHandle CreateContext() { return context; } -int CreatePort(AudioOut2ContextHandle context, std::uint32_t format, AudioOut2PortHandle* port) { +int CreatePort(AudioOut2ContextHandle context, std::uint32_t format, AudioOut2PortHandle* port, std::uint16_t type = portTypeMain) { AudioOut2PortParam params{}; - params.port_type = portTypeMain; + params.port_type = type; params.data_format = format; params.sampling_freq = frequency; return sceAudioOut2PortCreate(context, ¶ms, port); @@ -91,20 +93,31 @@ void SetVolume(AudioOut2PortHandle port, const std::vector& volume) { Require(sceAudioOut2PortSetAttributes(port, &attribute, 1) == 0); } -std::vector Play(std::uint32_t format, const void* data, const std::vector& volume) { +struct PortData { + std::uint16_t type; + std::uint32_t format; + const void* data; + std::vector volume; +}; + +std::vector Play(const std::vector& inputs) { const auto path = std::filesystem::temp_directory_path() / ("anyps5_audio_out2_port_layouts-" + std::to_string(std::random_device{}()) + ".raw"); std::filesystem::remove(path); SetEnvironment("SDL_DISKAUDIOFILE", path.string()); const auto context = CreateContext(); - AudioOut2PortHandle port = 0; - Require(CreatePort(context, format, &port) == 0); - SetVolume(port, volume); - SetData(port, data); + std::vector ports; + for (const auto& input : inputs) { + AudioOut2PortHandle port = 0; + Require(CreatePort(context, input.format, &port, input.type) == 0); + SetVolume(port, input.volume); + SetData(port, input.data); + ports.push_back(port); + } Require(sceAudioOut2ContextPush(context, 1) == 0); - SetData(port, nullptr); + for (const auto port : ports) SetData(port, nullptr); for (std::uint32_t push = 0; push < silentGrains; push++) Require(sceAudioOut2ContextPush(context, 1) == 0); - Require(sceAudioOut2PortDestroy(port) == 0); + for (const auto port : ports) Require(sceAudioOut2PortDestroy(port) == 0); Require(sceAudioOut2ContextDestroy(context) == 0); std::ifstream file(path, std::ios::binary); @@ -121,6 +134,10 @@ std::vector Play(std::uint32_t format, const void* data, const std::vecto return {samples + first, samples + last}; } +std::vector Play(std::uint32_t format, const void* data, const std::vector& volume) { + return Play({{portTypeMain, format, data, volume}}); +} + float Sample(std::uint32_t channel, std::uint32_t frame) { const float sign = frame % 2 == 0 ? 1.0f : -1.0f; return sign * 0.01f * static_cast(channel + 1) * static_cast(1 + frame % 5); @@ -204,6 +221,18 @@ void TestDroppedLfe() { RequireFold(Play(Format(surround714.channels, formatFloat), data.data(), volume), surround714, expected, volume, 1.0f); } +void TestPadPortsWithoutPadDevice() { + const auto speaker = FloatGrain(mono.channels); + std::vector vibration(static_cast(grain) * stereo.channels, 0.75f); + const std::vector unity{1.0f, 1.0f}; + const auto played = Play({ + {portTypePadVibration, Format(stereo.channels, formatFloat), vibration.data(), unity}, + {portTypePadSpeaker, Format(mono.channels, formatFloat), speaker.data(), {1.0f}}, + }); + RequireFold(played, mono, speaker, {1.0f}, 1.0f); + Require(Play({{portTypePadVibration, Format(stereo.channels, formatFloat), vibration.data(), unity}}).empty()); +} + void TestMeasuredMainPortFormats() { struct Case { std::uint32_t format; @@ -270,6 +299,7 @@ int main() { } TestHeightChannels(); TestDroppedLfe(); + TestPadPortsWithoutPadDevice(); TestMeasuredMainPortFormats(); return 0; } diff --git a/core/libs/prx/libSceAudioOut/tests/AudioOutLastOutputTime.cpp b/core/libs/prx/libSceAudioOut/tests/AudioOutLastOutputTime.cpp index c9463eccc..c06cc14d1 100644 --- a/core/libs/prx/libSceAudioOut/tests/AudioOutLastOutputTime.cpp +++ b/core/libs/prx/libSceAudioOut/tests/AudioOutLastOutputTime.cpp @@ -1,6 +1,7 @@ #include "SceTypes.hpp" #include "prx/libc/include/general/VabiMacros.hpp" +#include #include #include #include @@ -13,6 +14,7 @@ int APS5_VABI sceAudioOutOpen(int, int, int, std::uint32_t, std::uint32_t, std:: int APS5_VABI sceAudioOutClose(int); int APS5_VABI sceAudioOutOutput(int, const void*); int APS5_VABI sceAudioOutOutputs(AudioOutOutputParam*, std::uint32_t); +int APS5_VABI sceAudioOutGetPortState(int, AudioOutPortState*); int APS5_VABI sceAudioOutGetLastOutputTime(int, std::uint64_t*); std::uint64_t APS5_VABI sceKernelGetProcessTime(); } @@ -154,6 +156,30 @@ void TestErrors() { Require(destination == untouched, "a rejected call must not write the destination"); } +void TestQueriesDuringOutput(bool batch, bool drain) { + const std::vector block(frequency, 256); + const int handle = sceAudioOutOpen(user, portTypeVibration, 0, frequency, frequency, formatS16Mono); + Require(handle > 0, "port must open"); + Require(sceAudioOutOutput(handle, block.data()) == static_cast(frequency), "output must accept the block"); + const auto initialTime = LastOutputTime(handle); + std::atomic started = false; + std::thread output([&] { + AudioOutOutputParam param{handle, drain ? nullptr : block.data()}; + started = true; + const int result = batch ? sceAudioOutOutputs(¶m, 1) : sceAudioOutOutput(handle, param.ptr); + Require(result == static_cast(frequency), "pending output must complete before close"); + }); + while (!started) std::this_thread::yield(); + std::this_thread::sleep_for(std::chrono::milliseconds(50)); + const auto before = std::chrono::steady_clock::now(); + AudioOutPortState state{}; + Require(sceAudioOutGetPortState(handle, &state) == 0, "pending output must allow port state queries"); + Require(LastOutputTime(handle) == initialTime, "pending output must not change its timestamp yet"); + Require(std::chrono::steady_clock::now() - before < std::chrono::milliseconds(300), "audio queries must not wait for output pacing or draining"); + Require(sceAudioOutClose(handle) == 0, "close must safely wait for pending output"); + output.join(); +} + } int main() { @@ -170,5 +196,9 @@ int main() { TestReopenedPort(); TestErrors(); TestPortWithoutDeviceKeepsRealTime(); + TestQueriesDuringOutput(false, false); + TestQueriesDuringOutput(false, true); + TestQueriesDuringOutput(true, false); + TestQueriesDuringOutput(true, true); return 0; } diff --git a/core/libs/prx/libSceAudioPropagation/AudioPropagation.hpp b/core/libs/prx/libSceAudioPropagation/AudioPropagation.hpp index 31904b0c5..602d531cf 100644 --- a/core/libs/prx/libSceAudioPropagation/AudioPropagation.hpp +++ b/core/libs/prx/libSceAudioPropagation/AudioPropagation.hpp @@ -81,6 +81,7 @@ std::int32_t APS5_VABI sceAudioPropagationSourceGetAudioPath(AudioPropagationHan std::int32_t APS5_VABI sceAudioPropagationSourceGetRays(AudioPropagationHandle source, void* rays, std::uint32_t* count); std::int32_t APS5_VABI sceAudioPropagationSourceCalculateAudioPaths(AudioPropagationHandle source, const void* rays, std::uint32_t rayCount, std::uint32_t flags, void* paths, std::uint32_t pathCount); std::int32_t APS5_VABI sceAudioPropagationSourceSetAudioPaths(AudioPropagationHandle source, const void* entries, std::uint32_t count); +std::int32_t APS5_VABI sceAudioPropagationSourceSetAudioPath(AudioPropagationHandle path, const void* data, float gain); std::int32_t APS5_VABI sceAudioPropagationSourceRender(AudioPropagationHandle system, const AudioPropagation::RenderInfo* infos, std::uint32_t count); } diff --git a/core/libs/prx/libSceAudioPropagation/Export.cpp b/core/libs/prx/libSceAudioPropagation/Export.cpp index 59e0a7b18..c3cdff194 100644 --- a/core/libs/prx/libSceAudioPropagation/Export.cpp +++ b/core/libs/prx/libSceAudioPropagation/Export.cpp @@ -291,6 +291,14 @@ std::int32_t APS5_VABI sceAudioPropagationSourceSetAudioPaths(AudioPropagationHa return 0; } +std::int32_t APS5_VABI sceAudioPropagationSourceSetAudioPath(AudioPropagationHandle path, const void* data, float gain) { + static_cast(path); + static_cast(data); + static_cast(gain); + NotImplemented_nid_no_patch(__func__); + return 0; +} + std::int32_t APS5_VABI sceAudioPropagationSourceRender(AudioPropagationHandle system, const RenderInfo* infos, std::uint32_t count) { require(infos != nullptr && count != 0, __func__); auto& state = registry(); diff --git a/core/libs/prx/libSceAudiodec/src/Codecs.cpp b/core/libs/prx/libSceAudiodec/src/Codecs.cpp index dfa428f9d..aca46d88d 100644 --- a/core/libs/prx/libSceAudiodec/src/Codecs.cpp +++ b/core/libs/prx/libSceAudiodec/src/Codecs.cpp @@ -82,6 +82,12 @@ protected: return result; } + void reopen(std::vector replacement) { + avcodec_free_context(&context); + extradata = std::move(replacement); + open(); + } + private: void open() { const AVCodec* codec = avcodec_find_decoder(id); @@ -118,7 +124,8 @@ public: class AacDecoder final : public FfmpegDecoder { public: - AacDecoder(std::int32_t wordSize, bool adts, std::vector extradata) : FfmpegDecoder(AV_CODEC_ID_AAC, wordSize, std::move(extradata)), adts(adts) {} + AacDecoder(std::int32_t wordSize, bool adts, std::vector> configs) + : FfmpegDecoder(AV_CODEC_ID_AAC, wordSize, configs.empty() ? std::vector{} : configs.front()), adts(adts), configs(std::move(configs)) {} DecodeResult Decode(const std::uint8_t* data, std::size_t size, std::uint8_t* pcm, std::size_t pcmSize) override { std::size_t frameBytes = size; @@ -128,11 +135,22 @@ public: if (frameBytes < 7) return {DecodeStatus::InvalidData}; if (frameBytes > size) return {DecodeStatus::PartialInput}; } - return decodePacket(data, frameBytes, pcm, pcmSize); + if (configs.size() <= 1) return decodePacket(data, frameBytes, pcm, pcmSize); + for (std::size_t index = 0; index < configs.size(); ++index) { + if (index != 0) reopen(configs[index]); + const DecodeResult result = decodePacket(data, frameBytes, pcm, pcmSize); + if (result.status != DecodeStatus::InvalidData) { + configs = {configs[index]}; + return result; + } + } + reopen(configs.front()); + return {DecodeStatus::InvalidData}; } private: bool adts; + std::vector> configs; }; class At9Decoder final : public Decoder { @@ -222,13 +240,19 @@ std::unique_ptr CreateMp3(std::int32_t wordSize) { } std::unique_ptr CreateAac(std::int32_t wordSize, bool adts, std::uint32_t samplingFreqIndex, std::uint32_t channels) { - std::vector config; + std::vector> configs; if (!adts) { const std::uint32_t lowComplexity = 2; - config = {static_cast(lowComplexity << 3 | samplingFreqIndex >> 1), - static_cast((samplingFreqIndex & 1) << 7 | channels << 3)}; + const auto config = [&](std::uint32_t channelConfiguration) { + return std::vector{static_cast(lowComplexity << 3 | samplingFreqIndex >> 1), + static_cast((samplingFreqIndex & 1) << 7 | channelConfiguration << 3)}; + }; + configs.push_back(config(channels < 2 ? 1 : 2)); + for (std::uint32_t channelConfiguration = 3; channelConfiguration <= 7; ++channelConfiguration) { + if ((channelConfiguration == 7 ? 8 : channelConfiguration) <= channels) configs.push_back(config(channelConfiguration)); + } } - return std::make_unique(wordSize, adts, std::move(config)); + return std::make_unique(wordSize, adts, std::move(configs)); } std::unique_ptr CreateAt9(std::int32_t wordSize, const std::uint8_t (&config)[4], At9Format& format) { diff --git a/core/libs/prx/libSceAvPlayer/Export.cpp b/core/libs/prx/libSceAvPlayer/Export.cpp index 25dfc65c5..fc78bf88b 100644 --- a/core/libs/prx/libSceAvPlayer/Export.cpp +++ b/core/libs/prx/libSceAvPlayer/Export.cpp @@ -2,7 +2,9 @@ // SPDX-License-Identifier: GPL-2.0-or-later #include +#include #include +#include #include #include "SceTypes.hpp" #include "prx/libSceAvPlayer/include/AvPlayer.hpp" @@ -19,6 +21,10 @@ std::uint32_t VideoBufferCount(std::int32_t requested) { return static_cast(std::clamp(requested, 2, 16)); } +constexpr std::size_t Ps5InitDataExSize = 0x230; +constexpr std::size_t Ps5AutoStartOffset = 0x74; +constexpr std::size_t Ps5VideoFrameBuffersOffset = 0x228; + } #pragma GCC visibility push(default) @@ -97,8 +103,14 @@ int APS5_VABI sceAvPlayerInitEx(const AvPlayerInitDataEx* init_ex, AvPlayerInter init.file_replacement = init_ex->file_replacement; init.event_replacement = init_ex->event_replacement; init.debug_level = init_ex->debug_level; - init.num_output_video_framebuffers = init_ex->num_output_video_framebuffers; - init.auto_start = init_ex->auto_start; + if (init_ex->this_size >= Ps5InitDataExSize) { + const auto* bytes = reinterpret_cast(init_ex); + std::memcpy(&init.num_output_video_framebuffers, bytes + Ps5VideoFrameBuffersOffset, sizeof(init.num_output_video_framebuffers)); + init.auto_start = bytes[Ps5AutoStartOffset] != 0; + } else { + init.num_output_video_framebuffers = init_ex->num_output_video_framebuffers; + init.auto_start = init_ex->auto_start; + } init.default_language = init_ex->default_language; *handle = new Player(init, VideoBufferCount(init.num_output_video_framebuffers)); return SCE_OK; diff --git a/core/libs/prx/libSceAvPlayer/src/Source.cpp b/core/libs/prx/libSceAvPlayer/src/Source.cpp index 8990d5d57..d6baf7b3d 100644 --- a/core/libs/prx/libSceAvPlayer/src/Source.cpp +++ b/core/libs/prx/libSceAvPlayer/src/Source.cpp @@ -509,9 +509,9 @@ private: presentationClock.Rebase(static_cast(front.info.timestamp)); clockEpoch = front.epoch; } - if (synced) { + if (synced || audioDriving) { const auto now = presentationClock.Now(); - if (static_cast(front.info.timestamp) > now) return false; + if (synced && static_cast(front.info.timestamp) > now) return false; while (frames.size() > 1 && frames[1].epoch == frames.front().epoch && static_cast(frames[1].info.timestamp) <= now) recycleFront(video); } present(video, info); diff --git a/core/libs/prx/libSceFont/src/Layout.cpp b/core/libs/prx/libSceFont/src/Layout.cpp index 68d242d53..8a067968b 100644 --- a/core/libs/prx/libSceFont/src/Layout.cpp +++ b/core/libs/prx/libSceFont/src/Layout.cpp @@ -428,9 +428,12 @@ int Font::GetCharGlyphMetrics(FontHandle handle, std::uint32_t code, FontGlyphMe metrics->Horizontal.bearingX = static_cast(slot->metrics.horiBearingX) / 64.0f * ratioX; metrics->Horizontal.bearingY = static_cast(slot->metrics.horiBearingY) / 64.0f * ratioY; metrics->Horizontal.advance = static_cast(slot->metrics.horiAdvance) / 64.0f * ratioX; - metrics->Vertical.bearingX = 0.0f; - metrics->Vertical.bearingY = 0.0f; - metrics->Vertical.advance = 0.0f; + const bool vertical = FT_HAS_VERTICAL(face); + const FT_Pos vertBearingX = vertical ? slot->metrics.vertBearingX : slot->metrics.horiBearingX - slot->metrics.horiAdvance / 2; + const FT_Pos vertBearingY = vertical ? slot->metrics.vertBearingY : (slot->metrics.vertAdvance - slot->metrics.height) / 2; + metrics->Vertical.bearingX = static_cast(vertBearingX) / 64.0f * ratioX; + metrics->Vertical.bearingY = static_cast(vertBearingY) / 64.0f * ratioY; + metrics->Vertical.advance = static_cast(slot->metrics.vertAdvance) / 64.0f * ratioY; return finish(SCE_FONT_OK); } diff --git a/core/libs/prx/libSceFont/src/State.cpp b/core/libs/prx/libSceFont/src/State.cpp index 6911fa24f..f2b1bc905 100644 --- a/core/libs/prx/libSceFont/src/State.cpp +++ b/core/libs/prx/libSceFont/src/State.cpp @@ -12,6 +12,7 @@ #include #include +#include "prx/libc/include/PreciseWait.hpp" #include "prx/libSceFont/include/FontInternal.hpp" namespace { @@ -56,7 +57,7 @@ Font::FontState::~FontState() { } void Font::Backoff() { - std::this_thread::sleep_for(std::chrono::microseconds(30)); + PreciseSleepUs(30); } std::uint32_t Font::AcquireWordLock(std::uint32_t& word) { diff --git a/core/libs/prx/libSceGameLiveStreaming/Export.cpp b/core/libs/prx/libSceGameLiveStreaming/Export.cpp index 11d59f501..b28b535c8 100644 --- a/core/libs/prx/libSceGameLiveStreaming/Export.cpp +++ b/core/libs/prx/libSceGameLiveStreaming/Export.cpp @@ -2,6 +2,20 @@ #include #include "SceTypes.hpp" #include "prx/libc/include/General.hpp" +#include "prx/libSceUserService/UserService.hpp" +#include + +struct GameLiveStreamingStatus2 { + std::int32_t userId; + bool isOnAir; + std::uint8_t align[3]; + std::uint32_t spectatorCounts; + std::uint32_t textMessageCounts; + std::uint32_t commandMessageCounts; + std::uint32_t broadcastVideoResolution; + std::uint8_t reserved[48]; +}; +static_assert(sizeof(GameLiveStreamingStatus2) == 72); extern "C" { @@ -14,8 +28,10 @@ int APS5_VABI sceGameLiveStreamingTerminate(void) { return 0; } -int APS5_VABI sceGameLiveStreamingGetCurrentStatus2() { - NotImplemented_nid_no_patch(__func__); +int APS5_VABI sceGameLiveStreamingGetCurrentStatus2(GameLiveStreamingStatus2* status) { + if (status == nullptr) APS5_INVALID_ARG_EX; + std::memset(status, 0, sizeof(*status)); + status->userId = USER_SERVICE_USER_ID_INVALID; return 0; } diff --git a/core/libs/prx/libSceHmd/CMakeLists.txt b/core/libs/prx/libSceHmd/CMakeLists.txt new file mode 100644 index 000000000..d5a15e9ca --- /dev/null +++ b/core/libs/prx/libSceHmd/CMakeLists.txt @@ -0,0 +1,14 @@ +add_library(libSceHmd SHARED EXCLUDE_FROM_ALL Export.cpp) +target_include_directories(libSceHmd PRIVATE ${LIBS_INCLUDE_DIR}) +configure_windows_unwind(libSceHmd) + +if(BUILD_TESTING) + add_test_executable(guest_hmd_tests ${LIBS_INCLUDE_DIR}/tests/GuestHmd.cpp) + target_include_directories(guest_hmd_tests PRIVATE ${LIBS_INCLUDE_DIR}) + target_link_libraries(guest_hmd_tests PRIVATE libSceHmd) + add_test(NAME guest_hmd COMMAND guest_hmd_tests) + if(WIN32) + set_tests_properties(guest_hmd PROPERTIES ENVIRONMENT_MODIFICATION + "PATH=path_list_prepend:$") + endif() +endif() diff --git a/core/libs/prx/libSceHttp/Export.cpp b/core/libs/prx/libSceHttp/Export.cpp index 76a577451..22c0d2e5e 100644 --- a/core/libs/prx/libSceHttp/Export.cpp +++ b/core/libs/prx/libSceHttp/Export.cpp @@ -415,4 +415,8 @@ int APS5_VABI sceHttpSetRedirectCallback(int id, HttpRedirectCallback cbfunc, vo return 0; } +int APS5_VABI sceHttpAbortWaitRequest(void) { + NotImplemented_nid_no_patch(__func__); + return 0; +} } diff --git a/core/libs/prx/libSceHttp/src/Uri.cpp b/core/libs/prx/libSceHttp/src/Uri.cpp index 8bee28c3e..6aac69db3 100644 --- a/core/libs/prx/libSceHttp/src/Uri.cpp +++ b/core/libs/prx/libSceHttp/src/Uri.cpp @@ -298,7 +298,7 @@ int APS5_VABI sceHttpUriParse(SceHttpUriElement* out, const char* src_url, void* char* strings[7]; for (size_t i = 0; i < 7; ++i) { strings[i] = next; - std::memcpy(next, fields[i].data(), fields[i].size()); + if (!fields[i].empty()) std::memcpy(next, fields[i].data(), fields[i].size()); next[fields[i].size()] = '\0'; next += fields[i].size() + 1; } diff --git a/core/libs/prx/libSceIme/Export.cpp b/core/libs/prx/libSceIme/Export.cpp index b105f365b..4675c980f 100644 --- a/core/libs/prx/libSceIme/Export.cpp +++ b/core/libs/prx/libSceIme/Export.cpp @@ -35,6 +35,11 @@ int APS5_VABI sceImeClose_nid_postfix(void) { return ErrorNotOpened; } +int APS5_VABI sceImeGetPanelPositionAndForm(PositionAndForm* form) { + (void)form; + return ErrorNotOpened; +} + int APS5_VABI sceImeGetPanelSize(const Param* param, uint32_t* width, uint32_t* height) { if (!param || !width || !height) return ErrorInvalidAddress; if (param->type > TypeNumber) return ErrorInvalidType; diff --git a/core/libs/prx/libSceJson2/Export.cpp b/core/libs/prx/libSceJson2/Export.cpp index 70675ee05..be6afe76f 100644 --- a/core/libs/prx/libSceJson2/Export.cpp +++ b/core/libs/prx/libSceJson2/Export.cpp @@ -780,7 +780,6 @@ int APS5_VABI _ZN3sce4Json17InitParameterRttiC1EPNS0_14AllocParamRttiEPvm(void) NotImplemented_nid_no_patch(__func__); return 0; } - struct InitParameter2 { void* allocator; void* userData; @@ -817,4 +816,13 @@ int APS5_VABI _ZN3sce4Json12MemAllocator11notifyErrorEimPv(void) { return 0; } +int APS5_VABI _ZN3sce4Json14InitParameter2C2Ev(void) { + NotImplemented_nid_no_patch(__func__); + return 0; +} + +int APS5_VABI _ZN3sce4Json18InitParameterRtti216setAllocatorRttiEPNS0_14AllocParamRttiEPv(void) { + NotImplemented_nid_no_patch(__func__); + return 0; +} } diff --git a/core/libs/prx/libSceNet/CMakeLists.txt b/core/libs/prx/libSceNet/CMakeLists.txt index 2516da1e1..84e0b910f 100644 --- a/core/libs/prx/libSceNet/CMakeLists.txt +++ b/core/libs/prx/libSceNet/CMakeLists.txt @@ -8,7 +8,7 @@ if(BUILD_TESTING) target_link_libraries(guest_sce_net_tests PRIVATE libSceNet libSceNetCtl libkernel libc) configure_windows_unwind(guest_sce_net_tests) add_test(NAME guest_sce_net COMMAND guest_sce_net_tests) - set_tests_properties(guest_sce_net PROPERTIES TIMEOUT 20) + set_tests_properties(guest_sce_net PROPERTIES TIMEOUT 20 SKIP_RETURN_CODE 77) if(WIN32) set_tests_properties(guest_sce_net PROPERTIES ENVIRONMENT_MODIFICATION "PATH=path_list_prepend:$") diff --git a/core/libs/prx/libSceNet/Export.cpp b/core/libs/prx/libSceNet/Export.cpp index 05a06ea00..5e9d23160 100644 --- a/core/libs/prx/libSceNet/Export.cpp +++ b/core/libs/prx/libSceNet/Export.cpp @@ -2,6 +2,7 @@ #define WIN32_LEAN_AND_MEAN #include #include +#include #else #include #include @@ -112,6 +113,7 @@ int native_error() { case WSAEINTR: return 4; case WSAEINVAL: return NET_EINVAL; case WSAENOTSOCK: return NET_ENOTSOCK; + case WSAESHUTDOWN: return 32; default: return 5; } } @@ -224,8 +226,58 @@ struct Sock { int snd_timeout_us = 0; }; -std::mutex g_mutex; -std::condition_variable g_cv; +#ifdef _WIN32 +struct HostMutex { + SRWLOCK native = SRWLOCK_INIT; + + HostMutex() = default; + HostMutex(const HostMutex&) = delete; + HostMutex& operator=(const HostMutex&) = delete; + + void lock() { AcquireSRWLockExclusive(&native); } + void unlock() { ReleaseSRWLockExclusive(&native); } +}; +#else +using HostMutex = std::mutex; +#endif + +class HostCondition { +public: + HostCondition() = default; + HostCondition(const HostCondition&) = delete; + HostCondition& operator=(const HostCondition&) = delete; + + void NotifyAll() { +#ifdef _WIN32 + WakeAllConditionVariable(&native); +#else + native.notify_all(); +#endif + } + + void WaitUntil(std::unique_lock& lock, std::chrono::steady_clock::time_point deadline) { +#ifdef _WIN32 + const auto now = std::chrono::steady_clock::now(); + if (deadline <= now) return; + const auto milliseconds = std::chrono::ceil(deadline - now).count(); + if (!SleepConditionVariableSRW(&native, &lock.mutex()->native, static_cast(milliseconds), 0) && + GetLastError() != ERROR_TIMEOUT) + throw std::runtime_error("sceNetEpollWait: native condition wait failed"); +#else + native.wait_until(lock, deadline); +#endif + } + +private: +#ifdef _WIN32 + CONDITION_VARIABLE native = CONDITION_VARIABLE_INIT; +#else + std::condition_variable native; +#endif +}; + +HostMutex g_mutex; +HostCondition g_cv; std::map g_socks; std::set g_epolls; std::map> g_epoll_socks; @@ -255,7 +307,7 @@ int fail(int err) { } void set_resolver_error(int rid, int error) { - std::lock_guard lk(g_mutex); + std::lock_guard lk(g_mutex); const auto resolver = g_resolvers.find(rid); if (resolver != g_resolvers.end()) resolver->second = error; } @@ -325,7 +377,7 @@ int* APS5_VABI sceNetErrnoLoc(void) { int PollSockets(KernelSocketPoll::Entry* entries, int count, int timeoutMilliseconds) { std::vector> natives(static_cast(count)); { - std::lock_guard lk(g_mutex); + std::lock_guard lk(g_mutex); for (int index = 0; index < count; ++index) { const auto socket = g_socks.find(entries[index].descriptor); if (socket != g_socks.end()) natives[static_cast(index)] = socket->second.native; @@ -382,13 +434,13 @@ int PollSockets(KernelSocketPoll::Entry* entries, int count, int timeoutMillisec const bool g_socketPollerRegistered = (KernelSetSocketPoller_nid_no_patch(&PollSockets), true); int APS5_VABI sceNetInit_nid_postfix(void) { - std::lock_guard lk(g_mutex); + std::lock_guard lk(g_mutex); g_net_inited = true; return 0; } int APS5_VABI sceNetTerm(void) { - std::lock_guard lk(g_mutex); + std::lock_guard lk(g_mutex); g_net_inited = false; return 0; } @@ -399,14 +451,14 @@ int APS5_VABI sceNetPoolCreate(const char* name, int size, int flags) { if (size <= 0) { return fail(NET_EINVAL); } - std::lock_guard lk(g_mutex); + std::lock_guard lk(g_mutex); const int id = g_next_pool++; g_pools.emplace(id, static_cast(size)); return id; } int APS5_VABI sceNetPoolDestroy(int memid) { - std::lock_guard lk(g_mutex); + std::lock_guard lk(g_mutex); return g_pools.erase(memid) != 0 ? 0 : fail(NET_EBADF); } @@ -428,7 +480,7 @@ int APS5_VABI sceNetSocket(const char* name, int family, int type, int protocol) close_socket(native); return fail(12); } - std::lock_guard lk(g_mutex); + std::lock_guard lk(g_mutex); const int fd = g_next_sock++; Sock s; s.family = family; @@ -441,7 +493,7 @@ int APS5_VABI sceNetSocket(const char* name, int family, int type, int protocol) int APS5_VABI sceNetSocketClose(int s) { std::shared_ptr native; { - std::lock_guard lk(g_mutex); + std::lock_guard lk(g_mutex); const auto it = g_socks.find(s); if (it == g_socks.end()) return fail(NET_EBADF); native = it->second.native; @@ -454,7 +506,7 @@ int APS5_VABI sceNetSocketClose(int s) { ::shutdown(native->value, SHUT_RDWR); #endif } - g_cv.notify_all(); + g_cv.NotifyAll(); return 0; } @@ -462,7 +514,7 @@ int APS5_VABI sceNetSocketAbort(int s, int flags) { (void)flags; std::shared_ptr native; { - std::lock_guard lk(g_mutex); + std::lock_guard lk(g_mutex); auto it = g_socks.find(s); if (it == g_socks.end()) return fail(NET_EBADF); it->second.aborted = true; @@ -473,14 +525,14 @@ int APS5_VABI sceNetSocketAbort(int s, int flags) { #else if (native) ::shutdown(native->value, SHUT_RDWR); #endif - g_cv.notify_all(); + g_cv.NotifyAll(); return 0; } int APS5_VABI sceNetBind_nid_postfix(int s, const void* addr, uint32_t addrlen) { Sock socket; { - std::lock_guard lk(g_mutex); + std::lock_guard lk(g_mutex); auto it = g_socks.find(s); if (it == g_socks.end()) return fail(NET_EBADF); socket = it->second; @@ -492,7 +544,7 @@ int APS5_VABI sceNetBind_nid_postfix(int s, const void* addr, uint32_t addrlen) if (native.ss_family != (socket.family == NET_AF_INET ? AF_INET : AF_INET6)) return fail(NET_EAFNOSUPPORT); if (::bind(socket.native->value, reinterpret_cast(&native), native_length) != 0) return fail(native_error()); - std::lock_guard lk(g_mutex); + std::lock_guard lk(g_mutex); auto it = g_socks.find(s); if (it != g_socks.end()) it->second.bound = true; return 0; @@ -501,14 +553,14 @@ int APS5_VABI sceNetBind_nid_postfix(int s, const void* addr, uint32_t addrlen) int APS5_VABI sceNetListen(int s, int backlog) { Sock socket; { - std::lock_guard lk(g_mutex); + std::lock_guard lk(g_mutex); auto it = g_socks.find(s); if (it == g_socks.end()) return fail(NET_EBADF); socket = it->second; } if (socket.type != NET_SOCK_STREAM) return fail(NET_EOPNOTSUPP); if (::listen(socket.native->value, backlog) != 0) return fail(native_error()); - std::lock_guard lk(g_mutex); + std::lock_guard lk(g_mutex); auto it = g_socks.find(s); if (it != g_socks.end()) { it->second.bound = true; @@ -520,7 +572,7 @@ int APS5_VABI sceNetListen(int s, int backlog) { int APS5_VABI sceNetAccept(int s, void* addr, uint32_t* addrlen) { Sock listener; { - std::lock_guard lk(g_mutex); + std::lock_guard lk(g_mutex); auto it = g_socks.find(s); if (it == g_socks.end()) return fail(NET_EBADF); listener = it->second; @@ -545,7 +597,7 @@ int APS5_VABI sceNetAccept(int s, void* addr, uint32_t* addrlen) { connection.native = std::move(handle); int descriptor; { - std::lock_guard lk(g_mutex); + std::lock_guard lk(g_mutex); descriptor = g_next_sock++; g_socks.emplace(descriptor, connection); } @@ -559,7 +611,7 @@ int APS5_VABI sceNetAccept(int s, void* addr, uint32_t* addrlen) { int APS5_VABI sceNetConnect(int s, const void* addr, uint32_t addrlen) { Sock socket; { - std::lock_guard lk(g_mutex); + std::lock_guard lk(g_mutex); auto it = g_socks.find(s); if (it == g_socks.end()) return fail(NET_EBADF); socket = it->second; @@ -581,7 +633,7 @@ int64_t APS5_VABI sceNetRecv(int s, void* buf, size_t len, int flags) { if (len > INT_MAX) return fail(NET_EMSGSIZE); Sock socket; { - std::lock_guard lk(g_mutex); + std::lock_guard lk(g_mutex); auto it = g_socks.find(s); if (it == g_socks.end()) return fail(NET_EBADF); socket = it->second; @@ -598,7 +650,7 @@ int64_t APS5_VABI sceNetRecvfrom(int s, void* buf, size_t len, int flags, void* if (from && !fromlen) return fail(NET_EINVAL); Sock socket; { - std::lock_guard lk(g_mutex); + std::lock_guard lk(g_mutex); auto it = g_socks.find(s); if (it == g_socks.end()) return fail(NET_EBADF); socket = it->second; @@ -619,7 +671,7 @@ int64_t APS5_VABI sceNetSend(int s, const void* buf, size_t len, int flags) { if (len > INT_MAX) return fail(NET_EMSGSIZE); Sock socket; { - std::lock_guard lk(g_mutex); + std::lock_guard lk(g_mutex); auto it = g_socks.find(s); if (it == g_socks.end()) return fail(NET_EBADF); socket = it->second; @@ -634,7 +686,7 @@ int64_t APS5_VABI sceNetSendto(int s, const void* buf, size_t len, int flags, co if (len > INT_MAX) return fail(NET_EMSGSIZE); Sock socket; { - std::lock_guard lk(g_mutex); + std::lock_guard lk(g_mutex); auto it = g_socks.find(s); if (it == g_socks.end()) return fail(NET_EBADF); socket = it->second; @@ -678,7 +730,7 @@ int64_t APS5_VABI sceNetRecvmsg(int s, NetMsghdr* msg, int flags) { if (flags != 0 && flags != NET_MSG_PEEK) return fail(NET_EOPNOTSUPP); Sock socket; { - std::lock_guard lk(g_mutex); + std::lock_guard lk(g_mutex); auto it = g_socks.find(s); if (it == g_socks.end()) return fail(NET_EBADF); socket = it->second; @@ -722,7 +774,7 @@ int APS5_VABI sceNetShutdown(int s, int how) { if (how < 0 || how > 2) return fail(NET_EINVAL); Sock socket; { - std::lock_guard lk(g_mutex); + std::lock_guard lk(g_mutex); auto it = g_socks.find(s); if (it == g_socks.end()) return fail(NET_EBADF); socket = it->second; @@ -733,7 +785,7 @@ int APS5_VABI sceNetShutdown(int s, int how) { int APS5_VABI sceNetSetsockopt(int s, int level, int optname, const void* optval, uint32_t optlen) { Sock socket; { - std::lock_guard lk(g_mutex); + std::lock_guard lk(g_mutex); auto it = g_socks.find(s); if (it == g_socks.end()) return fail(NET_EBADF); socket = it->second; @@ -770,7 +822,7 @@ int APS5_VABI sceNetSetsockopt(int s, int level, int optname, const void* optval } else { return fail(NET_EOPNOTSUPP); } - std::lock_guard lk(g_mutex); + std::lock_guard lk(g_mutex); auto it = g_socks.find(s); if (it != g_socks.end()) { if (optname == NET_SO_NBIO) it->second.nonblock = value != 0; @@ -783,7 +835,7 @@ int APS5_VABI sceNetSetsockopt(int s, int level, int optname, const void* optval int APS5_VABI sceNetGetsockopt(int s, int level, int optname, void* optval, uint32_t* optlen) { Sock socket; { - std::lock_guard lk(g_mutex); + std::lock_guard lk(g_mutex); auto it = g_socks.find(s); if (it == g_socks.end()) return fail(NET_EBADF); socket = it->second; @@ -807,7 +859,7 @@ int APS5_VABI sceNetGetsockopt(int s, int level, int optname, void* optval, uint int APS5_VABI sceNetGetsockname(int s, void* addr, uint32_t* addrlen) { Sock socket; { - std::lock_guard lk(g_mutex); + std::lock_guard lk(g_mutex); auto it = g_socks.find(s); if (it == g_socks.end()) return fail(NET_EBADF); socket = it->second; @@ -823,7 +875,7 @@ int APS5_VABI sceNetGetsockname(int s, void* addr, uint32_t* addrlen) { int APS5_VABI sceNetGetpeername(int s, void* addr, uint32_t* addrlen) { Sock socket; { - std::lock_guard lk(g_mutex); + std::lock_guard lk(g_mutex); auto it = g_socks.find(s); if (it == g_socks.end()) return fail(NET_EBADF); socket = it->second; @@ -840,14 +892,14 @@ int APS5_VABI sceNetGetSockInfo(int s, void* info, int n, int flags) { (void)info; (void)n; (void)flags; - std::lock_guard lk(g_mutex); + std::lock_guard lk(g_mutex); return g_socks.count(s) != 0 ? 0 : fail(NET_EBADF); } int APS5_VABI sceNetEpollCreate(const char* name, int flags) { (void)name; (void)flags; - std::lock_guard lk(g_mutex); + std::lock_guard lk(g_mutex); const int id = g_next_epoll++; g_epolls.insert(id); g_epoll_socks.emplace(id, std::map{}); @@ -858,7 +910,7 @@ int APS5_VABI sceNetGetMemoryPoolStats(int memid, NetMemoryPoolStats* stats) { if (stats == nullptr) { return fail(NET_EINVAL); } - std::lock_guard lk(g_mutex); + std::lock_guard lk(g_mutex); const auto pool = g_pools.find(memid); if (pool == g_pools.end()) { return fail(NET_EBADF); @@ -868,29 +920,29 @@ int APS5_VABI sceNetGetMemoryPoolStats(int memid, NetMemoryPoolStats* stats) { } int APS5_VABI sceNetEpollDestroy(int eid) { - std::lock_guard lk(g_mutex); + std::lock_guard lk(g_mutex); if (g_epolls.erase(eid) == 0) { return fail(NET_EBADF); } g_epoll_socks.erase(eid); g_epoll_aborted.erase(eid); - g_cv.notify_all(); + g_cv.NotifyAll(); return 0; } int APS5_VABI sceNetEpollAbort(int eid, int flags) { (void)flags; - std::lock_guard lk(g_mutex); + std::lock_guard lk(g_mutex); if (g_epolls.count(eid) == 0) { return fail(NET_EBADF); } g_epoll_aborted.insert(eid); - g_cv.notify_all(); + g_cv.NotifyAll(); return 0; } int APS5_VABI sceNetEpollControl(int eid, int op, int id, const NetEpollEvent* event) { - std::lock_guard lk(g_mutex); + std::lock_guard lk(g_mutex); if (g_epolls.count(eid) == 0) { return fail(NET_EBADF); } @@ -907,7 +959,7 @@ int APS5_VABI sceNetEpollControl(int eid, int op, int id, const NetEpollEvent* e if (registration == registrations.end()) return fail(NET_ENOENT); registration->second = *event; } - g_cv.notify_all(); + g_cv.NotifyAll(); return 0; } @@ -920,7 +972,7 @@ int APS5_VABI sceNetEpollWait(int eid, NetEpollEvent* events, int maxevents, int std::vector registered_events; std::vector> native_sockets; { - std::lock_guard lk(g_mutex); + std::lock_guard lk(g_mutex); if (g_epolls.count(eid) == 0) return fail(NET_EBADF); if (g_epoll_aborted.erase(eid) != 0) return fail(NET_ECONNABORTED); const auto registration = g_epoll_socks.find(eid); @@ -935,12 +987,12 @@ int APS5_VABI sceNetEpollWait(int eid, NetEpollEvent* events, int maxevents, int } if (registered_events.empty()) { if (timeout == 0) return 0; - std::unique_lock lk(g_mutex); + std::unique_lock lk(g_mutex); const auto now = std::chrono::steady_clock::now(); if (timeout > 0 && now >= deadline) return 0; const auto slice = std::chrono::milliseconds(100); const auto wake = timeout > 0 ? std::min(deadline, now + slice) : now + slice; - g_cv.wait_until(lk, wake); + g_cv.WaitUntil(lk, wake); continue; } @@ -974,7 +1026,7 @@ int APS5_VABI sceNetEpollWait(int eid, NetEpollEvent* events, int maxevents, int #endif if (ready_count < 0) return fail(native_error()); { - std::lock_guard lk(g_mutex); + std::lock_guard lk(g_mutex); if (g_epolls.count(eid) == 0) return fail(NET_EBADF); if (g_epoll_aborted.erase(eid) != 0) return fail(NET_ECONNABORTED); } @@ -1077,20 +1129,20 @@ int APS5_VABI sceNetResolverCreate(const char* name, int memid, int flags) { (void)name; (void)memid; (void)flags; - std::lock_guard lk(g_mutex); + std::lock_guard lk(g_mutex); const int id = g_next_resolver++; g_resolvers[id] = 0; return id; } int APS5_VABI sceNetResolverDestroy(int rid) { - std::lock_guard lk(g_mutex); + std::lock_guard lk(g_mutex); return g_resolvers.erase(rid) != 0 ? 0 : fail(NET_EBADF); } int lookup_ipv4(int rid, const char* hostname, const char* func, std::vector& addresses) { { - std::lock_guard lk(g_mutex); + std::lock_guard lk(g_mutex); if (g_resolvers.count(rid) == 0) return fail(NET_EBADF); } if (!initialize_sockets()) return fail(5); @@ -1164,7 +1216,7 @@ int APS5_VABI sceNetResolverStartAton(int rid, const void* addr, char* hostname, (void)flags; if (!addr || !hostname || len <= 0) return fail(NET_EINVAL); { - std::lock_guard lk(g_mutex); + std::lock_guard lk(g_mutex); if (g_resolvers.count(rid) == 0) return fail(NET_EBADF); } if (!initialize_sockets()) return fail(5); @@ -1185,7 +1237,7 @@ int APS5_VABI sceNetResolverStartAton(int rid, const void* addr, char* hostname, int APS5_VABI sceNetResolverGetError(int rid, int* status) { if (!status) return fail(NET_EINVAL); - std::lock_guard lk(g_mutex); + std::lock_guard lk(g_mutex); const auto resolver = g_resolvers.find(rid); if (resolver == g_resolvers.end()) return fail(NET_EBADF); *status = resolver->second; diff --git a/core/libs/prx/libSceNetCtlApDialog/Export.cpp b/core/libs/prx/libSceNetCtlApDialog/Export.cpp new file mode 100644 index 000000000..0dc6cd822 --- /dev/null +++ b/core/libs/prx/libSceNetCtlApDialog/Export.cpp @@ -0,0 +1,38 @@ +#include +#include +#include "SceTypes.hpp" +#include "prx/libc/include/General.hpp" + +extern "C" { + +int APS5_VABI sceNetCtlApDialogInitialize(void) { + NotImplemented_nid_no_patch(__func__); + return 0; +} + +int APS5_VABI sceNetCtlApDialogTerminate(void) { + NotImplemented_nid_no_patch(__func__); + return 0; +} + +int APS5_VABI sceNetCtlApDialogOpen(void) { + NotImplemented_nid_no_patch(__func__); + return 0; +} + +int APS5_VABI sceNetCtlApDialogClose(void) { + NotImplemented_nid_no_patch(__func__); + return 0; +} + +int APS5_VABI sceNetCtlApDialogUpdateStatus(void) { + NotImplemented_nid_no_patch(__func__); + return 0; +} + +int APS5_VABI sceNetCtlApDialogGetResult(void) { + NotImplemented_nid_no_patch(__func__); + return 0; +} + +} diff --git a/core/libs/prx/libSceNgs2.native/Ngs2Library.cmake b/core/libs/prx/libSceNgs2.native/Ngs2Library.cmake index 453e4d6a1..06a20f0d4 100644 --- a/core/libs/prx/libSceNgs2.native/Ngs2Library.cmake +++ b/core/libs/prx/libSceNgs2.native/Ngs2Library.cmake @@ -3,8 +3,10 @@ function(add_sce_ngs2_library target) add_library(${target} SHARED EXCLUDE_FROM_ALL ${ngs2Dir}/src/Atrac9.cpp ${ngs2Dir}/src/Custom.cpp + ${ngs2Dir}/src/Pan.cpp ${ngs2Dir}/src/Rack.cpp ${ngs2Dir}/src/Render.cpp + ${ngs2Dir}/src/Reverb.cpp ${ngs2Dir}/src/System.cpp ${ngs2Dir}/src/Unimplemented.cpp ${ngs2Dir}/src/Voice.cpp diff --git a/core/libs/prx/libSceNgs2.native/include/Ngs2Types.hpp b/core/libs/prx/libSceNgs2.native/include/Ngs2Types.hpp index 3496f022f..531b51834 100644 --- a/core/libs/prx/libSceNgs2.native/include/Ngs2Types.hpp +++ b/core/libs/prx/libSceNgs2.native/include/Ngs2Types.hpp @@ -17,6 +17,7 @@ static constexpr int SCE_NGS2_ERROR_UNKNOWN_WAVEFORM_FORMAT = static_cast(0 static constexpr std::uint32_t SCE_NGS2_RACK_ID_SAMPLER = 0x1000; static constexpr std::uint32_t SCE_NGS2_RACK_ID_SUBMIXER = 0x2000; +static constexpr std::uint32_t SCE_NGS2_RACK_ID_REVERB = 0x2001; static constexpr std::uint32_t SCE_NGS2_RACK_ID_MASTERING = 0x3000; static constexpr std::uint32_t SCE_NGS2_RACK_ID_CUSTOM_SUBMIXER = 0x4002; @@ -50,6 +51,8 @@ static constexpr std::uint32_t SCE_NGS2_SAMPLER_VOICE_PARAM_PITCH = 0x10000005; static constexpr std::uint32_t SCE_NGS2_SAMPLER_VOICE_PARAM_FILTER = 0x1000000a; static constexpr std::uint32_t SCE_NGS2_SUBMIXER_VOICE_PARAM_SETUP = 0x20000000; static constexpr std::uint32_t SCE_NGS2_SUBMIXER_VOICE_PARAM_USER_FX = 0x20000004; +static constexpr std::uint32_t SCE_NGS2_REVERB_VOICE_PARAM_SETUP = 0x20010000; +static constexpr std::uint32_t SCE_NGS2_REVERB_VOICE_PARAM_I3DL2 = 0x20010001; static constexpr std::uint32_t SCE_NGS2_MASTERING_VOICE_PARAM_SETUP = 0x30000000; static constexpr std::uint32_t SCE_NGS2_MASTERING_VOICE_PARAM_GAIN = 0x30000004; static constexpr std::uint32_t SCE_NGS2_MASTERING_VOICE_PARAM_OUTPUT = 0x30000005; @@ -159,6 +162,12 @@ struct Ngs2MasteringRackOption { std::uint32_t num_peak_meter_blocks; }; +struct Ngs2ReverbRackOption { + Ngs2RackOption rack_option; + std::uint32_t max_channels; + std::uint32_t reverb_size; +}; + struct Ngs2CustomModuleOption { std::uint32_t size; }; @@ -421,6 +430,40 @@ struct Ngs2SubmixerVoiceSetupParam { }; static_assert(sizeof(Ngs2SubmixerVoiceSetupParam) == 16); +struct Ngs2ReverbVoiceSetupParam { + Ngs2VoiceParamHeader header; + std::uint32_t num_input_channels; + std::uint32_t num_output_channels; + std::uint32_t flags; + std::uint32_t reserved; +}; +static_assert(sizeof(Ngs2ReverbVoiceSetupParam) == 24); + +struct Ngs2ReverbI3DL2Param { + float wet; + float dry; + std::int32_t room; + std::int32_t room_hf; + std::uint32_t reflection_pattern; + float decay_time; + float decay_hf_ratio; + std::int32_t reflections; + float reflections_delay; + std::int32_t reverb; + float reverb_delay; + float diffusion; + float density; + float hf_reference; + std::uint32_t reserved[8]; +}; +static_assert(sizeof(Ngs2ReverbI3DL2Param) == 88); + +struct Ngs2ReverbVoiceI3DL2Param { + Ngs2VoiceParamHeader header; + Ngs2ReverbI3DL2Param i3dl2; +}; +static_assert(sizeof(Ngs2ReverbVoiceI3DL2Param) == 96); + struct Ngs2SubmixerVoiceUserFxParam { Ngs2VoiceParamHeader header; Ngs2UserFxProcessHandler handler; @@ -548,12 +591,19 @@ struct Ngs2WaveformInfo { static_assert(sizeof(Ngs2WaveformInfo) == 232); struct Ngs2PanParam { - std::uint32_t reserved[16]; + float angle; + float distance; + float fbw_level; + float lfe_level; }; +static_assert(sizeof(Ngs2PanParam) == 16); struct Ngs2PanWork { - std::uint32_t reserved[64]; + float speaker_angles[8]; + float unit_angle; + std::uint32_t num_speakers; }; +static_assert(sizeof(Ngs2PanWork) == 40); struct Ngs2GeomListenerParam { std::uint32_t reserved[32]; diff --git a/core/libs/prx/libSceNgs2.native/src/Atrac9.cpp b/core/libs/prx/libSceNgs2.native/src/Atrac9.cpp index 6403f9c10..240c0a7a9 100644 --- a/core/libs/prx/libSceNgs2.native/src/Atrac9.cpp +++ b/core/libs/prx/libSceNgs2.native/src/Atrac9.cpp @@ -1,6 +1,7 @@ #include #include #include +#include #include #include #include @@ -67,42 +68,134 @@ void Ngs2RestartAtrac9(Ngs2Voice& voice) { atrac9.decoder.reset(Atrac9GetHandle()); if (atrac9.decoder == nullptr || Atrac9InitDecoder(atrac9.decoder.get(), atrac9.config) != 0) throw std::runtime_error("NGS2: ATRAC9 decoder init failed"); atrac9.window.clear(); - atrac9.windowStart = 0; + atrac9.windowCursor = 0; + atrac9.input.clear(); + atrac9.remainingSamples = 0; + atrac9.skipSamples = 0; + atrac9.finishOnDrain = false; } -static void DecodeSuperframe(Ngs2Voice& voice, Ngs2Block& block) { +static std::size_t BufferedFrames(const Ngs2Voice& voice) { + return voice.atrac9.window.size() / voice.channels - voice.atrac9.windowCursor; +} + +static bool GatherSuperframe(Ngs2Voice& voice) { auto& atrac9 = voice.atrac9; - if (block.dataCursor + atrac9.superframeBytes > block.info.data_size) throw std::invalid_argument("NGS2: the ATRAC9 block ends inside a superframe"); + while (atrac9.input.size() < atrac9.superframeBytes) { + if (voice.blocks.empty()) { + if (!voice.acceptsBlocks && atrac9.remainingSamples != 0) + throw std::invalid_argument("NGS2: the ATRAC9 stream ends before its declared samples"); + return false; + } + auto& block = voice.blocks.front(); + if (!block.started) { + if (block.info.num_samples != 0) { + if (atrac9.remainingSamples != 0 || !atrac9.input.empty()) + throw std::invalid_argument("NGS2: a new ATRAC9 waveform interrupts an incomplete stream"); + atrac9.remainingSamples = block.info.num_samples; + atrac9.skipSamples = block.info.num_skip_samples; + } else if (atrac9.remainingSamples == 0) { + throw std::invalid_argument("NGS2: an ATRAC9 data-only block has no waveform to continue"); + } + block.started = true; + } + const auto count = std::min(atrac9.superframeBytes - atrac9.input.size(), block.info.data_size - block.dataCursor); + atrac9.input.insert(atrac9.input.end(), block.data + block.dataCursor, block.data + block.dataCursor + count); + block.dataCursor += count; + voice.decodedBytes += count; + if (atrac9.input.size() == atrac9.superframeBytes) return true; + if (!Ngs2FinishBlock(voice)) return false; + } + return true; +} + +static bool DecodeSuperframe(Ngs2Voice& voice) { + if (!GatherSuperframe(voice)) return false; + auto& atrac9 = voice.atrac9; + if (atrac9.windowCursor != 0) { + atrac9.window.erase(atrac9.window.begin(), atrac9.window.begin() + static_cast(atrac9.windowCursor) * voice.channels); + atrac9.windowCursor = 0; + } const std::size_t frameValues = static_cast(atrac9.frameSamples) * voice.channels; - const std::size_t superframeValues = frameValues * atrac9.framesInSuperframe; - if (atrac9.window.size() >= 2 * superframeValues) { - atrac9.window.erase(atrac9.window.begin(), atrac9.window.begin() + static_cast(superframeValues)); - atrac9.windowStart += atrac9.frameSamples * atrac9.framesInSuperframe; - } - std::size_t end = atrac9.window.size(); - atrac9.window.resize(end + superframeValues); - const auto* superframe = block.data + block.dataCursor; + const auto superframeSamples = atrac9.frameSamples * atrac9.framesInSuperframe; + const auto end = atrac9.window.size(); + atrac9.window.resize(end + static_cast(superframeSamples) * voice.channels); std::size_t consumed = 0; - for (std::uint32_t frame = 0; frame < atrac9.framesInSuperframe; frame++, end += frameValues) { + for (std::uint32_t frame = 0; frame < atrac9.framesInSuperframe; ++frame) { int used = 0; - const int status = Atrac9DecodeF32(atrac9.decoder.get(), superframe + consumed, static_cast(atrac9.superframeBytes - consumed), atrac9.window.data() + end, &used, 0); - consumed += static_cast(std::max(used, 0)); - if (status != 0 || consumed > atrac9.superframeBytes) throw std::runtime_error("NGS2: ATRAC9 decode failed with " + Ngs2Hex(static_cast(status))); + const int status = Atrac9DecodeF32(atrac9.decoder.get(), atrac9.input.data() + consumed, + static_cast(atrac9.superframeBytes - consumed), atrac9.window.data() + end + frame * frameValues, &used, 0); + if (status != 0 || used <= 0 || static_cast(used) > atrac9.superframeBytes - consumed) + throw std::runtime_error("NGS2: ATRAC9 decode failed with " + Ngs2Hex(static_cast(status))); + consumed += static_cast(used); } - block.dataCursor += atrac9.superframeBytes; - voice.decodedBytes += atrac9.superframeBytes; + atrac9.input.clear(); + const auto skipped = std::min(atrac9.skipSamples, superframeSamples); + const auto samples = std::min(atrac9.remainingSamples, superframeSamples - skipped); + atrac9.skipSamples -= skipped; + atrac9.remainingSamples -= samples; + std::memmove(atrac9.window.data() + end, atrac9.window.data() + end + static_cast(skipped) * voice.channels, + static_cast(samples) * voice.channels * sizeof(float)); + atrac9.window.resize(end + static_cast(samples) * voice.channels); + if (atrac9.remainingSamples == 0) { + atrac9.finishOnDrain = true; + } else if (voice.blocks.front().dataCursor == voice.blocks.front().info.data_size) { + if (!Ngs2FinishBlock(voice)) return false; + } + return true; } -const float* Ngs2Atrac9Frame(Ngs2Voice& voice, Ngs2Block& block, std::uint32_t frame) { +static bool EnsureFrames(Ngs2Voice& voice, std::size_t count) { auto& atrac9 = voice.atrac9; - const std::uint32_t position = block.info.num_skip_samples + frame; - if (block.dataCursor == 0) { - atrac9.window.clear(); - atrac9.windowStart = 0; + while (voice.state == Ngs2PlayState::Playing && BufferedFrames(voice) < count) { + if (atrac9.finishOnDrain) { + if (BufferedFrames(voice) != 0) return false; + atrac9.finishOnDrain = false; + if (!Ngs2FinishBlock(voice)) return false; + } + if (!DecodeSuperframe(voice)) return false; } - if (position < atrac9.windowStart) throw std::logic_error("NGS2: ATRAC9 frames are read backwards"); - while (position >= atrac9.windowStart + atrac9.window.size() / voice.channels) DecodeSuperframe(voice, block); - return atrac9.window.data() + static_cast(position - atrac9.windowStart) * voice.channels; + return voice.state == Ngs2PlayState::Playing; +} + +static bool AdvanceAtrac9(Ngs2Voice& voice) { + while ((voice.phase >> 32) != 0 && voice.state == Ngs2PlayState::Playing) { + if (!EnsureFrames(voice, 1)) return false; + const auto count = std::min(voice.phase >> 32, BufferedFrames(voice)); + voice.atrac9.windowCursor += static_cast(count); + voice.decodedSamples += count; + voice.phase -= count << 32; + if (BufferedFrames(voice) == 0 && voice.atrac9.finishOnDrain) { + voice.atrac9.finishOnDrain = false; + if (!Ngs2FinishBlock(voice)) return false; + } + } + return voice.state == Ngs2PlayState::Playing; +} + +void Ngs2ConsumeAtrac9(Ngs2Voice& voice, std::uint32_t grain, std::uint32_t systemRate) { + constexpr double phaseOne = 4294967296.0; + const auto initialRevision = voice.waveformRevision; + const auto step = static_cast(std::llround(voice.sampleRate * static_cast(voice.pitch) / systemRate * phaseOne)); + std::uint32_t written = 0; + for (; written < grain; ++written) { + if (!AdvanceAtrac9(voice) || !EnsureFrames(voice, 1)) break; + const auto revision = voice.waveformRevision; + const auto fraction = static_cast(voice.phase / phaseOne); + if (fraction != 0.0f) EnsureFrames(voice, 2); + if (voice.state != Ngs2PlayState::Playing || voice.waveformRevision != revision) break; + const auto* current = voice.atrac9.window.data() + static_cast(voice.atrac9.windowCursor) * voice.channels; + const auto* next = BufferedFrames(voice) > 1 ? current + voice.channels : current; + for (std::uint32_t channel = 0; channel < voice.channels; ++channel) + voice.samples[static_cast(channel) * grain + written] = std::lerp(current[channel], next[channel], fraction); + voice.phase += step; + } + voice.hasSamples = written != 0; + if (voice.state != Ngs2PlayState::Playing || voice.waveformRevision != initialRevision) return; + AdvanceAtrac9(voice); + if (voice.state != Ngs2PlayState::Playing || voice.waveformRevision != initialRevision) return; + if (!voice.acceptsBlocks && voice.blocks.empty() && BufferedFrames(voice) == 0 && voice.atrac9.remainingSamples != 0) + throw std::invalid_argument("NGS2: the ATRAC9 stream ends before its declared samples"); } static std::uint16_t ReadLe16(const std::uint8_t* data) { diff --git a/core/libs/prx/libSceNgs2.native/src/Ngs2Internal.hpp b/core/libs/prx/libSceNgs2.native/src/Ngs2Internal.hpp index ab1486207..9bed17834 100644 --- a/core/libs/prx/libSceNgs2.native/src/Ngs2Internal.hpp +++ b/core/libs/prx/libSceNgs2.native/src/Ngs2Internal.hpp @@ -41,6 +41,7 @@ struct Ngs2Block { bool streaming = false; std::uint64_t availableFrames = 0; std::vector pieces; + bool started = false; }; struct Ngs2Atrac9DecoderDeleter { @@ -54,7 +55,17 @@ struct Ngs2Atrac9 { std::uint32_t framesInSuperframe = 0; std::uint32_t superframeBytes = 0; std::vector window; - std::uint32_t windowStart = 0; + std::uint32_t windowCursor = 0; + std::vector input; + std::uint32_t remainingSamples = 0; + std::uint32_t skipSamples = 0; + bool finishOnDrain = false; +}; + +struct Ngs2ReverbState; + +struct Ngs2ReverbDeleter { + void operator()(Ngs2ReverbState* reverb) const; }; struct Ngs2Voice; @@ -98,12 +109,14 @@ struct Ngs2Voice { std::uint32_t sampleRate = 0; std::uint32_t waveformType = 0; Ngs2Atrac9 atrac9; + std::unique_ptr reverb; float pitch = 1.0f; std::uint64_t phase = 0; std::deque blocks; bool acceptsBlocks = true; std::uint64_t decodedSamples = 0; std::uint64_t decodedBytes = 0; + std::uint64_t waveformRevision = 0; const std::uint8_t* waveformEnd = nullptr; Ngs2VoiceCallbackHandler callback = nullptr; std::uintptr_t callbackData = 0; @@ -174,13 +187,21 @@ int Ngs2ReleaseBuffer(const Ngs2BufferAllocator& allocator, Ngs2ContextBufferInf void Ngs2SetupAtrac9(Ngs2Voice& voice, const Ngs2WaveformFormat& format); std::size_t Ngs2Atrac9BlockBytes(const Ngs2Voice& voice, const Ngs2WaveformBlock& block); void Ngs2RestartAtrac9(Ngs2Voice& voice); -const float* Ngs2Atrac9Frame(Ngs2Voice& voice, Ngs2Block& block, std::uint32_t frame); +void Ngs2ConsumeAtrac9(Ngs2Voice& voice, std::uint32_t grain, std::uint32_t systemRate); +bool Ngs2FinishBlock(Ngs2Voice& voice); void Ngs2CheckCustomRack(const Ngs2CustomRackOption& option); void Ngs2SetupUserFx(Ngs2Rack& rack, const Ngs2CustomRackOption& option); void Ngs2CleanupUserFx(Ngs2Rack& rack); void Ngs2ApplyCustomParam(Ngs2Voice& voice, const Ngs2VoiceParamHeader& param); void Ngs2ProcessUserFx(Ngs2Voice& voice, std::uint32_t grain, std::uint32_t sampleRate); +void Ngs2SetReverbParams(Ngs2Voice& voice, const Ngs2ReverbI3DL2Param& params); +void Ngs2SetupReverb(Ngs2Voice& voice); +void Ngs2ClearReverb(Ngs2Voice& voice); +bool Ngs2ProcessReverb(Ngs2Voice& voice, std::uint32_t grain, std::uint32_t sampleRate); void Ngs2ProcessLegacyUserFx(Ngs2Voice& voice, std::uint32_t grain, std::uint32_t sampleRate); void Ngs2RenderSystem(Ngs2System& system, const Ngs2RenderBufferInfo* bufferInfo, std::uint32_t numBufferInfo); +float Ngs2DefaultLevel(std::uint32_t sourceChannels, std::uint32_t source, std::uint32_t destChannels, std::uint32_t dest); +int Ngs2PanInit(Ngs2PanWork* work, const float* speaker_angles, float unit_angle, uint32_t num_speakers); +int Ngs2PanGetVolumeMatrix(Ngs2PanWork* work, const Ngs2PanParam* params, uint32_t num_params, uint32_t matrix_format, float* out_volume_matrix); #endif diff --git a/core/libs/prx/libSceNgs2.native/src/Pan.cpp b/core/libs/prx/libSceNgs2.native/src/Pan.cpp new file mode 100644 index 000000000..af88b2320 --- /dev/null +++ b/core/libs/prx/libSceNgs2.native/src/Pan.cpp @@ -0,0 +1,135 @@ +#include +#include +#include +#include +#include +#include +#include +#include + +#include "prx/libc/include/General.hpp" +#include "Ngs2Internal.hpp" + +static constexpr float DEGREES = 360.0f; +static constexpr float RADIANS = 2.0f * std::numbers::pi_v; +static constexpr std::uint32_t MAX_PAN_PARAMS = 8; +static constexpr std::uint32_t LFE_CHANNEL = 3; + +struct PanLayout { + std::uint32_t inputOrder[7]; + std::uint32_t channels[7]; + float defaultDegrees[7]; + std::uint32_t mixChannels; +}; + +static constexpr PanLayout STEREO_LAYOUT{{1, 0}, {1, 0}, {90, 270}, 2}; +static constexpr PanLayout QUAD_LAYOUT{{1, 3, 2, 0}, {1, 5, 4, 0}, {40, 110, 250, 320}, 6}; +static constexpr PanLayout FIVE_LAYOUT{{2, 1, 4, 3, 0}, {2, 1, 5, 4, 0}, {0, 40, 110, 250, 320}, 6}; +static constexpr PanLayout SEVEN_LAYOUT{{2, 1, 4, 6, 5, 3, 0}, {2, 1, 5, 7, 6, 4, 0}, {0, 40, 110, 150, 210, 250, 320}, 8}; + +static const PanLayout* LayoutFor(std::uint32_t numSpeakers) { + switch (numSpeakers) { + case 2: return &STEREO_LAYOUT; + case 4: return &QUAD_LAYOUT; + case 5: return &FIVE_LAYOUT; + case 7: return &SEVEN_LAYOUT; + default: return nullptr; + } +} + +static bool IsUnitAngle(float unitAngle) { + return unitAngle == DEGREES || unitAngle == RADIANS; +} + +static float EqualPower(float position) { + return std::sin(position * (std::numbers::pi_v / 2.0f)); +} + +static void AddBetweenSpeakers(const Ngs2PanWork& work, std::uint32_t speaker, float angle, float weight, float* gains) { + const float from = work.speaker_angles[speaker - 1]; + const float to = work.speaker_angles[speaker]; + if (!(angle >= from) || !(to > angle)) return; + const float position = (angle - from) / (to - from); + gains[speaker - 1] += EqualPower(1.0f - position) * weight; + gains[speaker] += EqualPower(position) * weight; +} + +int Ngs2PanInit(Ngs2PanWork* work, const float* speaker_angles, float unit_angle, uint32_t num_speakers) { + if (work == nullptr) APS5_INVALID_ARG_EX; + if (speaker_angles != nullptr && !IsUnitAngle(unit_angle)) APS5_INVALID_ARG_EX; + const auto* layout = LayoutFor(num_speakers); + if (layout == nullptr) APS5_INVALID_ARG_EX; + *work = {}; + work->num_speakers = num_speakers; + if (speaker_angles == nullptr) { + for (std::uint32_t i = 0; i < num_speakers; i++) work->speaker_angles[i] = layout->defaultDegrees[i]; + work->speaker_angles[num_speakers] = layout->defaultDegrees[0] + DEGREES; + work->unit_angle = DEGREES; + if (unit_angle == RADIANS) { + for (std::uint32_t i = 0; i < num_speakers; i++) work->speaker_angles[i] = work->speaker_angles[i] / 180.0f * std::numbers::pi_v; + work->unit_angle = RADIANS; + } + return SCE_NGS2_OK; + } + for (std::uint32_t i = 0; i < num_speakers; i++) { + float angle = speaker_angles[layout->inputOrder[i]]; + if (!std::isfinite(angle)) APS5_INVALID_ARG_EX; + while (angle < 0.0f) { + if (angle + unit_angle == angle) throw std::runtime_error("NGS2: pan angles too large to wrap are not implemented"); + angle += unit_angle; + } + while (angle >= unit_angle) { + if (!(unit_angle < angle)) APS5_INVALID_ARG_EX; + if (angle - unit_angle == angle) throw std::runtime_error("NGS2: pan angles too large to wrap are not implemented"); + angle -= unit_angle; + } + work->speaker_angles[i] = angle; + } + work->speaker_angles[num_speakers] = work->speaker_angles[0] + unit_angle; + work->unit_angle = unit_angle; + return SCE_NGS2_OK; +} + +int Ngs2PanGetVolumeMatrix(Ngs2PanWork* work, const Ngs2PanParam* params, uint32_t num_params, uint32_t matrix_format, float* out_volume_matrix) { + if (work == nullptr || params == nullptr) APS5_INVALID_ARG_EX; + if (out_volume_matrix == nullptr) return SCE_NGS2_ERROR_INVALID_OUT_ADDRESS; + if (matrix_format != 1 && matrix_format != 2 && matrix_format != 6 && matrix_format != 8) APS5_INVALID_ARG_EX; + const auto* layout = LayoutFor(work->num_speakers); + if (layout == nullptr || !IsUnitAngle(work->unit_angle) || !std::isfinite(work->speaker_angles[0]) || num_params > MAX_PAN_PARAMS) APS5_INVALID_ARG_EX; + for (std::uint32_t p = 0; p < num_params; p++) { + if (!std::isfinite(params[p].angle) || std::isnan(params[p].distance)) APS5_INVALID_ARG_EX; + } + std::memset(out_volume_matrix, 0, static_cast(num_params) * matrix_format * sizeof(float)); + const std::uint32_t numSpeakers = work->num_speakers; + const float unit = work->unit_angle; + const float first = work->speaker_angles[0]; + for (std::uint32_t p = 0; p < num_params; p++) { + const auto& param = params[p]; + const float distance = std::max(-1.0f, std::min(1.0f, param.distance)); + const float towardWeight = std::sin((distance + 1.0f) * 0.5f * std::numbers::pi_v * 0.5f); + const float oppositeWeight = std::sin((1.0f - distance) * 0.5f * std::numbers::pi_v * 0.5f); + float angle = param.angle; + while (angle < first || angle >= first + unit) { + const float before = angle; + if (angle >= first + unit) angle -= unit; + if (angle < first) angle += unit; + if (angle == before) throw std::runtime_error("NGS2: pan angles too large to wrap are not implemented"); + } + float opposite = angle + unit * 0.5f; + if (!(opposite < first + unit)) opposite -= unit; + float gains[8] = {}; + for (std::uint32_t speaker = 1; speaker <= numSpeakers; speaker++) { + AddBetweenSpeakers(*work, speaker, angle, towardWeight, gains); + AddBetweenSpeakers(*work, speaker, opposite, oppositeWeight, gains); + } + float mix[8] = {}; + mix[layout->channels[0]] = gains[0] + gains[numSpeakers]; + for (std::uint32_t speaker = 1; speaker < numSpeakers; speaker++) mix[layout->channels[speaker]] = gains[speaker]; + for (std::uint32_t channel = 0; channel < 8; channel++) mix[channel] = channel == LFE_CHANNEL ? param.lfe_level : mix[channel] * param.fbw_level; + for (std::uint32_t dest = 0; dest < matrix_format; dest++) { + float& level = out_volume_matrix[static_cast(dest) * num_params + p]; + for (std::uint32_t channel = 0; channel < layout->mixChannels; channel++) level += Ngs2DefaultLevel(layout->mixChannels, channel, matrix_format, dest) * mix[channel]; + } + } + return SCE_NGS2_OK; +} diff --git a/core/libs/prx/libSceNgs2.native/src/Rack.cpp b/core/libs/prx/libSceNgs2.native/src/Rack.cpp index 04d922208..1acc40d6b 100644 --- a/core/libs/prx/libSceNgs2.native/src/Rack.cpp +++ b/core/libs/prx/libSceNgs2.native/src/Rack.cpp @@ -16,6 +16,7 @@ union RackOptions { Ngs2SamplerRackOption sampler; Ngs2SubmixerRackOption submixer; Ngs2MasteringRackOption mastering; + Ngs2ReverbRackOption reverb; Ngs2CustomSubmixerRackOption customSubmixer; }; @@ -24,6 +25,7 @@ static std::size_t RackOptionSize(std::uint32_t rackId) { case SCE_NGS2_RACK_ID_SAMPLER: return sizeof(Ngs2SamplerRackOption); case SCE_NGS2_RACK_ID_SUBMIXER: return sizeof(Ngs2SubmixerRackOption); case SCE_NGS2_RACK_ID_MASTERING: return sizeof(Ngs2MasteringRackOption); + case SCE_NGS2_RACK_ID_REVERB: return sizeof(Ngs2ReverbRackOption); case SCE_NGS2_RACK_ID_CUSTOM_SUBMIXER: return sizeof(Ngs2CustomSubmixerRackOption); default: throw std::runtime_error("NGS2: rack id " + Ngs2Hex(rackId) + " is not implemented"); } @@ -59,6 +61,10 @@ static RackOptions DefaultRackOption(std::uint32_t rackId) { options.submixer.max_inputs = 1; options.submixer.num_peak_meter_blocks = 8; break; + case SCE_NGS2_RACK_ID_REVERB: + options.reverb.max_channels = 8; + options.reverb.reverb_size = 1; + break; default: common.max_matrices = 0; common.max_ports = 0; @@ -72,6 +78,7 @@ static RackOptions DefaultRackOption(std::uint32_t rackId) { static std::uint32_t RackMaxChannels(std::uint32_t rackId, const RackOptions& options) { if (rackId == SCE_NGS2_RACK_ID_SUBMIXER) return options.submixer.max_channels; if (rackId == SCE_NGS2_RACK_ID_MASTERING) return options.mastering.max_channels; + if (rackId == SCE_NGS2_RACK_ID_REVERB) return options.reverb.max_channels; if (rackId == SCE_NGS2_RACK_ID_CUSTOM_SUBMIXER) return options.customSubmixer.max_channels; return NGS2_MAX_CHANNELS; } @@ -87,6 +94,9 @@ static RackOptions CheckedRackOption(std::uint32_t rackId, const Ngs2RackOption* throw std::invalid_argument("NGS2: invalid rack voice or channel count"); } if (rackId == SCE_NGS2_RACK_ID_CUSTOM_SUBMIXER) Ngs2CheckCustomRack(options.customSubmixer.custom_rack_option); + if (rackId == SCE_NGS2_RACK_ID_REVERB && options.reverb.reverb_size != 1) { + throw std::runtime_error("NGS2: reverb rack size " + std::to_string(options.reverb.reverb_size) + " is not implemented"); + } return options; } diff --git a/core/libs/prx/libSceNgs2.native/src/Render.cpp b/core/libs/prx/libSceNgs2.native/src/Render.cpp index b0133ed2f..5c39dd53d 100644 --- a/core/libs/prx/libSceNgs2.native/src/Render.cpp +++ b/core/libs/prx/libSceNgs2.native/src/Render.cpp @@ -3,6 +3,7 @@ #include #include #include +#include #include #include #include @@ -38,12 +39,6 @@ static void ReadPcm(const Ngs2Voice& voice, const std::uint8_t* frame, float* ou static void ReadFrames(Ngs2Voice& voice, float* current, float* next) { auto& block = voice.blocks.front(); - if (voice.waveformType == SCE_NGS2_WAVEFORM_TYPE_ATRAC9) { - std::memcpy(current, Ngs2Atrac9Frame(voice, block, block.cursor), voice.channels * sizeof(float)); - if (block.cursor + 1 < block.info.num_samples) std::memcpy(next, Ngs2Atrac9Frame(voice, block, block.cursor + 1), voice.channels * sizeof(float)); - else std::memcpy(next, current, voice.channels * sizeof(float)); - return; - } ReadPcm(voice, FrameAt(voice, block, block.cursor), current); if (block.streaming && block.cursor + 1 >= block.availableFrames) std::memcpy(next, current, voice.channels * sizeof(float)); else if (block.cursor + 1 < block.info.num_samples) ReadPcm(voice, FrameAt(voice, block, block.cursor + 1), next); @@ -52,13 +47,14 @@ static void ReadFrames(Ngs2Voice& voice, float* current, float* next) { else std::memcpy(next, current, voice.channels * sizeof(float)); } -static void FinishBlock(Ngs2Voice& voice) { +bool Ngs2FinishBlock(Ngs2Voice& voice) { auto& block = voice.blocks.front(); const bool repeat = block.info.num_repeats != 0; if (repeat) { if (block.info.num_repeats != UINT32_MAX) block.info.num_repeats--; block.numRepeated++; block.cursor = 0; + block.started = false; if (voice.waveformType == SCE_NGS2_WAVEFORM_TYPE_ATRAC9) { block.dataCursor = 0; Ngs2RestartAtrac9(voice); @@ -80,7 +76,9 @@ static void FinishBlock(Ngs2Voice& voice) { voice.waveformEnd = voice.waveformType == SCE_NGS2_WAVEFORM_TYPE_ATRAC9 ? block.data + block.dataCursor : block.streaming ? Ngs2StreamEnd(voice, block) : FrameAt(voice, block, block.info.num_samples); voice.blocks.pop_front(); } + const auto revision = voice.waveformRevision; if (voice.callback != nullptr && (voice.callbackFlags & info.flag) != 0) voice.callback(&info); + return voice.waveformRevision == revision && voice.state == Ngs2PlayState::Playing; } static void Advance(Ngs2Voice& voice, std::uint64_t frames) { @@ -96,7 +94,7 @@ static void Advance(Ngs2Voice& voice, std::uint64_t frames) { if (block.streaming) { while (block.pieces.size() > 1 && block.cursor >= block.pieces.front().firstFrame + block.pieces.front().frames) block.pieces.erase(block.pieces.begin()); } - if (block.cursor == block.info.num_samples || (block.streaming && !voice.acceptsBlocks && block.cursor == block.availableFrames)) FinishBlock(voice); + if (block.cursor == block.info.num_samples || (block.streaming && !voice.acceptsBlocks && block.cursor == block.availableFrames)) Ngs2FinishBlock(voice); } } @@ -147,25 +145,73 @@ static void ApplyFilter(Ngs2Voice& voice, Ngs2Filter& filter, std::uint32_t grai } static void RenderSampler(Ngs2Voice& voice, std::uint32_t grain, std::uint32_t systemRate) { - ConsumeSamples(voice, grain, systemRate); + if (voice.waveformType == SCE_NGS2_WAVEFORM_TYPE_ATRAC9) Ngs2ConsumeAtrac9(voice, grain, systemRate); + else ConsumeSamples(voice, grain, systemRate); for (auto& filter : voice.filters) { if (!filter.enabled || (!voice.hasSamples && !HasHistory(filter))) continue; ApplyFilter(voice, filter, grain); voice.hasSamples = true; } - if (voice.state == Ngs2PlayState::Playing && voice.blocks.empty() && !voice.acceptsBlocks && std::none_of(voice.filters.begin(), voice.filters.end(), HasHistory)) { + const bool buffered = voice.waveformType == SCE_NGS2_WAVEFORM_TYPE_ATRAC9 && voice.atrac9.windowCursor * voice.channels < voice.atrac9.window.size(); + if (voice.state == Ngs2PlayState::Playing && voice.blocks.empty() && !buffered && !voice.acceptsBlocks && std::none_of(voice.filters.begin(), voice.filters.end(), HasHistory)) { voice.state = Ngs2PlayState::Empty; } } static void RenderVoice(Ngs2Voice& voice, const std::vector& voices, std::uint32_t grain, std::uint32_t systemRate); +struct Ngs2DefaultRow { + float mono; + float stereo[2]; + float surround51[6]; + float surround71[8]; +}; + +static constexpr float HALF_POWER = std::numbers::sqrt2_v / 2.0f; +static constexpr float QUARTER_POWER = std::numbers::sqrt2_v / 4.0f; + +static constexpr Ngs2DefaultRow MONO_CENTER{1.0f, {HALF_POWER, HALF_POWER}, {0, 0, 1, 0, 0, 0}, {0, 0, 1, 0, 0, 0, 0, 0}}; +static constexpr Ngs2DefaultRow FRONT_LEFT{HALF_POWER, {1, 0}, {1, 0, 0, 0, 0, 0}, {1, 0, 0, 0, 0, 0, 0, 0}}; +static constexpr Ngs2DefaultRow FRONT_RIGHT{HALF_POWER, {0, 1}, {0, 1, 0, 0, 0, 0}, {0, 1, 0, 0, 0, 0, 0, 0}}; +static constexpr Ngs2DefaultRow LFE{0.0f, {0, 0}, {0, 0, 0, 1, 0, 0}, {0, 0, 0, 1, 0, 0, 0, 0}}; +static constexpr Ngs2DefaultRow QUAD_LEFT{0.5f, {HALF_POWER, 0}, {0, 0, 0, 0, 1, 0}, {0, 0, 0, 0, 1, 0, 0, 0}}; +static constexpr Ngs2DefaultRow QUAD_RIGHT{0.5f, {0, HALF_POWER}, {0, 0, 0, 0, 0, 1}, {0, 0, 0, 0, 0, 1, 0, 0}}; +static constexpr Ngs2DefaultRow FIVE_SURROUND_LEFT{0.5f, {0, 0}, {0, 0, 0, 0, 1, 0}, {0, 0, 0, 0, 1, 0, 0, 0}}; +static constexpr Ngs2DefaultRow FIVE_SURROUND_RIGHT{0.5f, {0, 0}, {0, 0, 0, 0, 0, 1}, {0, 0, 0, 0, 0, 1, 0, 0}}; +static constexpr Ngs2DefaultRow SURROUND_LEFT{0.5f, {0.5f, 0}, {0, 0, 0, 0, 1, 0}, {0, 0, 0, 0, 1, 0, 0, 0}}; +static constexpr Ngs2DefaultRow SURROUND_RIGHT{0.5f, {0, 0.5f}, {0, 0, 0, 0, 0, 1}, {0, 0, 0, 0, 0, 1, 0, 0}}; +static constexpr Ngs2DefaultRow BACK_CENTER{QUARTER_POWER, {QUARTER_POWER, QUARTER_POWER}, {0, 0, 0, 0, HALF_POWER, HALF_POWER}, {0, 0, 0, 0, 0, 0, HALF_POWER, HALF_POWER}}; +static constexpr Ngs2DefaultRow BACK_LEFT{0.5f, {0.5f, 0}, {0, 0, 0, 0, 1, 0}, {0, 0, 0, 0, 0, 0, 1, 0}}; +static constexpr Ngs2DefaultRow BACK_RIGHT{0.5f, {0, 0.5f}, {0, 0, 0, 0, 0, 1}, {0, 0, 0, 0, 0, 0, 0, 1}}; + +static constexpr Ngs2DefaultRow DEFAULT_MAP[NGS2_MAX_CHANNELS][NGS2_MAX_CHANNELS] = { + {MONO_CENTER}, + {FRONT_LEFT, FRONT_RIGHT}, + {FRONT_LEFT, FRONT_RIGHT, LFE}, + {FRONT_LEFT, FRONT_RIGHT, QUAD_LEFT, QUAD_RIGHT}, + {FRONT_LEFT, FRONT_RIGHT, MONO_CENTER, FIVE_SURROUND_LEFT, FIVE_SURROUND_RIGHT}, + {FRONT_LEFT, FRONT_RIGHT, MONO_CENTER, LFE, SURROUND_LEFT, SURROUND_RIGHT}, + {FRONT_LEFT, FRONT_RIGHT, MONO_CENTER, LFE, SURROUND_LEFT, SURROUND_RIGHT, BACK_CENTER}, + {FRONT_LEFT, FRONT_RIGHT, MONO_CENTER, LFE, SURROUND_LEFT, SURROUND_RIGHT, BACK_LEFT, BACK_RIGHT}, +}; + +float Ngs2DefaultLevel(std::uint32_t sourceChannels, std::uint32_t source, std::uint32_t destChannels, std::uint32_t dest) { + const auto& row = DEFAULT_MAP[sourceChannels - 1][source]; + switch (destChannels) { + case 1: return row.mono; + case 2: return row.stereo[dest]; + case 6: return row.surround51[dest]; + case 8: return row.surround71[dest]; + default: throw std::runtime_error("NGS2: the default channel map into " + std::to_string(destChannels) + " channels is not implemented"); + } +} + static void MixPort(Ngs2Voice& voice, const Ngs2Voice& source, const Ngs2Port& port, std::uint32_t grain) { - const auto* matrix = port.matrix < 0 ? nullptr : &source.matrices[port.matrix]; + const auto* matrix = port.matrix < 0 || source.matrices[port.matrix].empty() ? nullptr : &source.matrices[port.matrix]; const std::size_t outputs = matrix == nullptr ? voice.channels : std::min(voice.channels, matrix->size() / source.channels); for (std::size_t dst = 0; dst < outputs; dst++) { for (std::uint32_t src = 0; src < source.channels; src++) { - const float level = port.volume * (matrix == nullptr ? (src == dst ? 1.0f : 0.0f) : (*matrix)[dst * source.channels + src]); + const float level = port.volume * (matrix == nullptr ? Ngs2DefaultLevel(source.channels, src, voice.channels, static_cast(dst)) : (*matrix)[dst * source.channels + src]); if (level == 0.0f) continue; for (std::uint32_t i = 0; i < grain; i++) voice.samples[dst * grain + i] += source.samples[src * grain + i] * level; } @@ -197,6 +243,7 @@ static void RenderVoice(Ngs2Voice& voice, const std::vector& voices, if (voice.state == Ngs2PlayState::Playing && voice.channels != 0) { if (voice.rack->rackId == SCE_NGS2_RACK_ID_SAMPLER) RenderSampler(voice, grain, systemRate); else MixInputs(voice, voices, grain, systemRate); + if (voice.rack->rackId == SCE_NGS2_RACK_ID_REVERB && voice.reverb) voice.hasSamples = Ngs2ProcessReverb(voice, grain, systemRate); Ngs2ProcessLegacyUserFx(voice, grain, systemRate); if (voice.hasSamples) Ngs2ProcessUserFx(voice, grain, systemRate); } diff --git a/core/libs/prx/libSceNgs2.native/src/Reverb.cpp b/core/libs/prx/libSceNgs2.native/src/Reverb.cpp new file mode 100644 index 000000000..131fb321f --- /dev/null +++ b/core/libs/prx/libSceNgs2.native/src/Reverb.cpp @@ -0,0 +1,384 @@ +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "prx/libc/include/General.hpp" +#include "Ngs2Internal.hpp" + +static constexpr std::uint32_t TAP_RING_SAMPLES = 14400; +static constexpr std::uint32_t EARLY_TAPS = 4; +static constexpr std::uint32_t COMBS = 16; +static constexpr std::uint32_t ALLPASSES = 4; +static constexpr std::uint32_t LFE_CHANNEL = 3; + +static constexpr float TAP_TIMES[8][EARLY_TAPS] = { + {0, 11, 23, 35}, {0, 14.75f, 26.5499992f, 38.0299988f}, {0, 6, 17, 29}, {0, 8, 20, 53}, + {0, 11, 17, 30}, {0, 16, 30, 55}, {0, 20, 29, 40}, {0, 37, 47, 62}, +}; +static constexpr float TAP_GAINS[8][EARLY_TAPS] = { + {0.7f, 0.4f, 0.134f, 0.1f}, {0.7f, 0.4f, 0.3f, 0.1f}, {0.7f, 0.45f, -0.6f, 0.2f}, {0.7f, 0.55f, -0.6f, 0.2f}, + {0.7f, -0.45f, 0.6f, 0.3f}, {0.7f, 0.7f, 0.7f, 0.4f}, {0.7f, 0.7f, -0.4f, 0.3f}, {0.7f, 0.7f, -0.4f, 0.4f}, +}; +static constexpr float COMB_TIMES[COMBS] = { + 22.0100002f, 23.25f, 24.75f, 26.6000004f, 28.25f, 29.6299992f, 31.0599995f, 32.4399986f, + 33.6899986f, 34.9500008f, 36.2900009f, 37.6500015f, 38.9900017f, 40.3499985f, 41.6699982f, 43.0499992f, +}; +static constexpr float ALLPASS_TIMES[ALLPASSES] = {11.5799999f, 9.18999958f, 7.0999999f, 4.69000006f}; +static constexpr std::int32_t ALLPASS_OFFSETS[8][ALLPASSES] = { + {0, -32, -6, -11}, {-10, -5, 0, -34}, {-8, -3, -33, -5}, {-7, -9, -5, -8}, + {-33, -8, -8, 0}, {-11, -21, -10, -7}, {-9, -10, -27, -3}, {-6, -11, -9, -24}, +}; +static constexpr std::uint16_t CHANNEL_COMBS[8] = {0xaaaa, 0x5555, 0xcccc, 0x3333, 0x0ff0, 0x9ab2, 0x3c3c, 0x399c}; +static constexpr Ngs2ReverbI3DL2Param SETUP_PARAMS{1.0f, 0.0f, -10000, 0, 0, 1.0f, 0.5f, -10000, 0.02f, -10000, 0.04f, 100.0f, 100.0f, 5000.0f, {}}; + +struct Ngs2ReverbTaps { + std::array delays{}; + std::array gains{}; +}; + +struct Ngs2Comb { + std::vector buffer; + std::uint32_t cursor = 0; + float feedback = 0.0f; + float lowpassIn = 0.0f; + float lowpassBack = 0.0f; + float direct = 0.0f; + float filtered = 0.0f; + float emphasis = 0.0f; + float emphasisBack = 0.0f; + float previous = 0.0f; + float lowpass = 0.0f; +}; + +struct Ngs2Allpass { + std::vector buffer; + std::uint32_t cursor = 0; +}; + +struct Ngs2ReverbChannel { + std::array allpasses; + float y1 = 0.0f; + float y2 = 0.0f; + float wet = 0.0f; + float dry = 0.0f; +}; + +struct Ngs2ReverbState { + Ngs2ReverbI3DL2Param params{}; + bool reset = true; + bool changed = true; + std::uint32_t sampleRate = 0; + std::vector ring; + std::uint32_t ringCursor = 0; + Ngs2ReverbTaps taps; + float earlyGain = 0.0f; + float lateGain = 0.0f; + float density = NAN; + std::array combs; + std::array channels; + bool roomFilter = false; + float roomB0 = 0.0f; + float roomA1 = 0.0f; + float roomA2 = 0.0f; +}; + +void Ngs2ReverbDeleter::operator()(Ngs2ReverbState* reverb) const { + delete reverb; +} + +static float Millibels(float millibels) { + if (millibels < -10000.0f) return 0.0f; + return static_cast(std::pow(10.0, static_cast(millibels * 0.0005f))); +} + +static float Clamp(float value, float low, float high) { + return value < low ? low : std::min(high, value); +} + +static std::int32_t ClampMillibels(std::int32_t value, std::int32_t high) { + return std::max(-10000, std::min(value, high)); +} + +static std::uint32_t Samples(float milliseconds, float rate) { + return static_cast(rate * milliseconds * 0.001f + 0.5f); +} + +static void Ramp(float* samples, float from, float to, std::uint32_t grain, const float* source, bool accumulate) { + const float step = (to - from) / static_cast(grain); + float gain = from; + for (std::uint32_t i = 0; i < grain; i++) { + const float value = source[i] * (from == to ? to : gain); + samples[i] = accumulate ? samples[i] + value : value; + gain += step; + } +} + +static void RunTaps(const Ngs2ReverbState& reverb, const Ngs2ReverbTaps& taps, const float* input, std::uint32_t grain, float* early, float* late) { + const auto size = static_cast(reverb.ring.size()); + std::fill(early, early + grain, 0.0f); + std::fill(late, late + grain, 0.0f); + for (std::uint32_t tap = 0; tap <= EARLY_TAPS; tap++) { + float* out = tap < EARLY_TAPS ? early : late; + const float gain = taps.gains[tap]; + const std::uint32_t delay = taps.delays[tap]; + std::uint32_t read = (reverb.ringCursor + size - delay) % size; + for (std::uint32_t i = 0; i < grain; i++) { + if (i < delay) { + out[i] += reverb.ring[read] * gain; + read = read + 1 == size ? 0 : read + 1; + } else { + out[i] += input[i - delay] * gain; + } + } + } +} + +static Ngs2ReverbTaps TapsFor(const Ngs2ReverbI3DL2Param& params, std::uint32_t size, float rate) { + Ngs2ReverbTaps taps; + const std::uint32_t pattern = params.reflection_pattern < 8 ? params.reflection_pattern : params.reflection_pattern - 8; + const auto early = std::min(static_cast(params.reflections_delay * rate * 1000.0f * 0.001f + 0.5f), size); + const auto late = std::min(static_cast((params.reflections_delay + params.reverb_delay) * rate * 1000.0f * 0.001f + 0.5f), size); + const auto last = static_cast(rate * TAP_TIMES[pattern][EARLY_TAPS - 1] * 0.001f + 0.5f); + const std::uint32_t limit = params.reflection_pattern > 7 ? size : late; + const float scale = early + last > limit ? static_cast(limit - early) / static_cast(last) : 1.0f; + taps.delays[0] = early; + taps.gains[0] = TAP_GAINS[pattern][0]; + for (std::uint32_t tap = 1; tap < EARLY_TAPS; tap++) { + taps.delays[tap] = static_cast(scale * TAP_TIMES[pattern][tap] * rate * 0.001f + 0.5f) + early; + taps.gains[tap] = taps.delays[tap] == taps.delays[tap - 1] ? 0.0f : TAP_GAINS[pattern][tap]; + } + taps.delays[EARLY_TAPS] = std::min(early + static_cast(params.reverb_delay * rate * 1000.0f * 0.001f + 0.5f), size); + taps.gains[EARLY_TAPS] = 1.0f; + return taps; +} + +static void SetupComb(Ngs2Comb& comb, std::uint32_t length, const Ngs2ReverbI3DL2Param& params, float rate, bool clear) { + if (clear || comb.buffer.size() != length) { + comb.buffer.assign(length, 0.0f); + if (comb.cursor >= length) comb.cursor = 0; + } + comb.feedback = std::pow(10.0f, static_cast(length) / rate / params.decay_time * -3.0f); + const float ratio = params.decay_hf_ratio; + if (ratio > 1.0f) { + const float high = std::pow(10.0f, (ratio - 1.0f) * std::log10(comb.feedback)); + comb.lowpassIn = 0.0506209731f; + comb.lowpassBack = 0.949379027f; + comb.direct = 1.0f; + comb.filtered = 1.0f - high; + } else { + float shaped = ratio * ratio; + if (0.8f > ratio) { + shaped = (0.8f - ratio) * -0.857142925f + 0.8f; + shaped *= shaped; + } + const float back = std::min(0.99977994f, std::log10(comb.feedback) * 2.30258512f * 0.25f * (1.0f - 1.0f / shaped)); + comb.lowpassIn = 1.0f - back; + comb.lowpassBack = back; + comb.direct = 0.0f; + comb.filtered = 1.0f; + } + const float root = std::sqrt(ratio); + const float tilt = (1.0f - root) / (root + 1.0f); + comb.emphasis = tilt / (1.0f - tilt) + 1.0f; + comb.emphasisBack = -tilt / (1.0f - tilt); +} + +static void Reset(Ngs2ReverbState& reverb) { + reverb.ring.assign(TAP_RING_SAMPLES, 0.0f); + reverb.ringCursor = 0; + reverb.taps = {}; + reverb.earlyGain = 1.0f; + reverb.lateGain = 1.0f; + reverb.density = NAN; + reverb.combs = {}; + reverb.channels = {}; +} + +static void ApplyParams(Ngs2ReverbState& reverb, float rate) { + const auto& params = reverb.params; + reverb.taps = TapsFor(params, static_cast(reverb.ring.size()), rate); + const float room = std::min(1.0f, Millibels(static_cast(params.room))); + reverb.earlyGain = Millibels(static_cast(ClampMillibels(params.reflections, 1000))) * room; + reverb.lateGain = Millibels(static_cast(ClampMillibels(params.reverb, 2000) - 2400)) * room; + const bool densityChanged = !(reverb.density == params.density); + reverb.density = params.density; + const float densityScale = params.density * -3.0f / 100.0f + 4.0f; + for (std::uint32_t i = 0; i < COMBS; i++) { + auto length = static_cast(rate * COMB_TIMES[i] / densityScale * 0.001f + 0.5f); + length += ~length & 1; + SetupComb(reverb.combs[i], length, params, rate, densityChanged); + } + for (std::uint32_t channel = 0; channel < 8; channel++) { + for (std::uint32_t stage = 0; stage < ALLPASSES; stage++) { + auto& allpass = reverb.channels[channel].allpasses[stage]; + const auto signedLength = static_cast(Samples(ALLPASS_TIMES[stage], rate)) + ALLPASS_OFFSETS[channel][stage]; + if (signedLength < 1) throw std::runtime_error("NGS2: reverb at a sample rate of " + std::to_string(reverb.sampleRate) + " Hz is not implemented"); + const auto length = static_cast(signedLength); + if (allpass.buffer.size() != length) allpass.buffer.assign(length, 0.0f); + if (allpass.cursor >= length) allpass.cursor = 0; + } + } +} + +static void UpdateRoomFilter(Ngs2ReverbState& reverb, float rate) { + const auto& params = reverb.params; + const float roomHf = Millibels(static_cast(ClampMillibels(params.room_hf, 0))); + reverb.roomFilter = 1.0f > roomHf; + if (!reverb.roomFilter) return; + const float cosine = std::cos(params.hf_reference * 6.28318548f / rate); + const float root = std::sqrt((roomHf + roomHf) * (1.0f - cosine) - roomHf * roomHf * (1.0f - cosine * cosine)); + const float pole = ((1.0f - roomHf * cosine) - root) / (1.0f - roomHf); + reverb.roomB0 = (1.0f - pole) * (1.0f - pole); + reverb.roomA1 = pole + pole; + reverb.roomA2 = pole * -pole; +} + +static void RunComb(Ngs2Comb& comb, const float* input, float* output, std::uint32_t grain) { + const auto length = static_cast(comb.buffer.size()); + for (std::uint32_t i = 0; i < grain; i++) { + const float emphasized = input[i] * comb.emphasis + comb.previous * comb.emphasisBack; + comb.previous = input[i]; + const float delayed = comb.buffer[comb.cursor]; + comb.lowpass = delayed * comb.lowpassIn + comb.lowpass * comb.lowpassBack; + const float back = (delayed * comb.direct + comb.lowpass * comb.filtered) * comb.feedback; + comb.buffer[comb.cursor] = emphasized + back; + output[i] = back; + comb.cursor = comb.cursor + 1 == length ? 0 : comb.cursor + 1; + } +} + +static void RunAllpass(Ngs2Allpass& allpass, float gain, float* samples, std::uint32_t grain) { + const auto length = static_cast(allpass.buffer.size()); + for (std::uint32_t i = 0; i < grain; i++) { + const float delayed = allpass.buffer[allpass.cursor]; + const float stored = delayed * gain + samples[i]; + allpass.buffer[allpass.cursor] = stored; + samples[i] = delayed - gain * stored; + allpass.cursor = allpass.cursor + 1 == length ? 0 : allpass.cursor + 1; + } +} + +static void MixDown(const float* input, std::uint32_t channels, std::uint32_t grain, float* mono) { + const float* channel[8]; + for (std::uint32_t c = 0; c < channels; c++) channel[c] = input + static_cast(c) * grain; + if (channels == 1) { + std::copy(channel[0], channel[0] + grain, mono); + return; + } + const float gain = channels == 2 ? 0.707000017f : channels == 6 ? 0.316000015f : 0.26699999f; + for (std::uint32_t i = 0; i < grain; i++) mono[i] = channel[0][i] * gain; + for (std::uint32_t c = 1; c < channels; c++) { + if (c == LFE_CHANNEL) continue; + for (std::uint32_t i = 0; i < grain; i++) mono[i] += channel[c][i] * gain; + } +} + +bool Ngs2ProcessReverb(Ngs2Voice& voice, std::uint32_t grain, std::uint32_t sampleRate) { + auto& reverb = *voice.reverb; + const float rate = static_cast(sampleRate); + const std::uint32_t channels = voice.channels; + if (reverb.sampleRate != sampleRate) { + reverb.sampleRate = sampleRate; + reverb.reset = true; + } + const bool reset = reverb.reset; + if (reset) { + Reset(reverb); + reverb.changed = true; + } + const std::vector input = voice.samples; + std::vector mono(grain), early(grain), late(grain); + MixDown(input.data(), channels, grain, mono.data()); + if (reverb.changed) { + std::vector oldEarly(grain), oldLate(grain); + RunTaps(reverb, reverb.taps, mono.data(), grain, oldEarly.data(), oldLate.data()); + ApplyParams(reverb, rate); + RunTaps(reverb, reverb.taps, mono.data(), grain, early.data(), late.data()); + Ramp(early.data(), 0.0f, 1.0f, grain, early.data(), false); + Ramp(early.data(), 1.0f, 0.0f, grain, oldEarly.data(), true); + Ramp(late.data(), 0.0f, 1.0f, grain, late.data(), false); + Ramp(late.data(), 1.0f, 0.0f, grain, oldLate.data(), true); + } else { + RunTaps(reverb, reverb.taps, mono.data(), grain, early.data(), late.data()); + } + for (std::uint32_t i = 0; i < grain; i++) { + reverb.ring[reverb.ringCursor] = mono[i]; + reverb.ringCursor = reverb.ringCursor + 1 == reverb.ring.size() ? 0 : reverb.ringCursor + 1; + } + std::vector combs(static_cast(COMBS) * grain); + for (std::uint32_t i = 0; i < COMBS; i++) RunComb(reverb.combs[i], late.data(), combs.data() + static_cast(i) * grain, grain); + UpdateRoomFilter(reverb, rate); + const auto& params = reverb.params; + bool audible = false; + for (std::uint32_t channel = 0; channel < channels; channel++) { + auto& state = reverb.channels[channel]; + float* out = voice.samples.data() + static_cast(channel) * grain; + const float* in = input.data() + static_cast(channel) * grain; + if (channel == LFE_CHANNEL) { + std::copy(in, in + grain, out); + } else { + std::vector tail(grain, 0.0f); + std::uint32_t selected = 0; + for (std::uint32_t i = 0; i < COMBS; i++) { + if ((CHANNEL_COMBS[channel] >> i & 1) == 0) continue; + const float weight = (selected & 1) != 0 ? 1.0f : 1.0f / (static_cast(selected) * 0.5f * 20.0f + 40.0f) * (params.density + -100.0f) + 1.0f; + const float* comb = combs.data() + static_cast(i) * grain; + for (std::uint32_t n = 0; n < grain; n++) tail[n] += comb[n] * weight; + selected++; + } + for (auto& allpass : state.allpasses) RunAllpass(allpass, params.diffusion * 0.005f, tail.data(), grain); + for (std::uint32_t n = 0; n < grain; n++) out[n] = early[n] * reverb.earlyGain + tail[n] * reverb.lateGain; + for (std::uint32_t n = 0; n < grain; n++) { + const float y = reverb.roomFilter ? reverb.roomB0 * out[n] + reverb.roomA1 * state.y1 + reverb.roomA2 * state.y2 : out[n]; + state.y2 = state.y1; + state.y1 = y; + out[n] = y; + } + } + const float wet = channel == LFE_CHANNEL ? 0.0f : params.wet; + Ramp(out, reset ? wet : state.wet, wet, grain, out, false); + Ramp(out, reset ? params.dry : state.dry, params.dry, grain, in, true); + state.wet = wet; + state.dry = params.dry; + for (std::uint32_t n = 0; n < grain && !audible; n++) audible = out[n] != 0.0f; + } + reverb.reset = false; + reverb.changed = false; + return audible; +} + +void Ngs2SetReverbParams(Ngs2Voice& voice, const Ngs2ReverbI3DL2Param& params) { + if (std::isnan(params.wet) || std::isnan(params.dry) || std::isnan(params.decay_time) || std::isnan(params.decay_hf_ratio) || std::isnan(params.reflections_delay) || + std::isnan(params.reverb_delay) || std::isnan(params.diffusion) || std::isnan(params.density) || std::isnan(params.hf_reference)) { + APS5_INVALID_ARG_EX; + } + if (!voice.reverb) voice.reverb.reset(new Ngs2ReverbState{}); + auto& stored = voice.reverb->params; + stored = params; + stored.wet = Clamp(params.wet, 0.0f, 32.0f); + stored.dry = Clamp(params.dry, 0.0f, 32.0f); + stored.reflection_pattern = params.reflection_pattern < 16 ? params.reflection_pattern : 0; + stored.decay_time = Clamp(params.decay_time, 0.1f, 20.0f); + stored.decay_hf_ratio = Clamp(params.decay_hf_ratio, 0.00999999978f, 2.0f); + stored.reflections_delay = Clamp(params.reflections_delay, 0.0f, 0.300000012f); + stored.reverb_delay = Clamp(params.reverb_delay, 0.0f, 0.100000001f); + stored.diffusion = Clamp(params.diffusion, 0.0f, 100.0f); + stored.density = Clamp(params.density, 0.0f, 100.0f); + stored.hf_reference = Clamp(params.hf_reference, 20.0f, 20000.0f); + voice.reverb->changed = true; +} + +void Ngs2SetupReverb(Ngs2Voice& voice) { + voice.reverb.reset(new Ngs2ReverbState{}); + Ngs2SetReverbParams(voice, SETUP_PARAMS); +} + +void Ngs2ClearReverb(Ngs2Voice& voice) { + if (voice.reverb) voice.reverb->reset = true; +} diff --git a/core/libs/prx/libSceNgs2.native/src/Unimplemented.cpp b/core/libs/prx/libSceNgs2.native/src/Unimplemented.cpp index 15fb19513..62b848071 100644 --- a/core/libs/prx/libSceNgs2.native/src/Unimplemented.cpp +++ b/core/libs/prx/libSceNgs2.native/src/Unimplemented.cpp @@ -37,25 +37,6 @@ int APS5_VABI sceNgs2GeomResetSourceParam(Ngs2GeomSourceParam* out_source_param) return 0; } -int APS5_VABI sceNgs2PanGetVolumeMatrix(Ngs2PanWork* work, const Ngs2PanParam* params, uint32_t num_params, uint32_t matrix_format, float* out_volume_matrix) { - (void)work; - (void)params; - (void)num_params; - (void)matrix_format; - (void)out_volume_matrix; - NotImplemented_nid_no_patch(__func__); - return 0; -} - -int APS5_VABI sceNgs2PanInit(Ngs2PanWork* work, const float* speaker_angles, float unit_angle, uint32_t num_speakers) { - (void)work; - (void)speaker_angles; - (void)unit_angle; - (void)num_speakers; - NotImplemented_nid_no_patch(__func__); - return 0; -} - } #pragma GCC visibility pop diff --git a/core/libs/prx/libSceNgs2.native/src/Voice.cpp b/core/libs/prx/libSceNgs2.native/src/Voice.cpp index fb21bc22d..233011108 100644 --- a/core/libs/prx/libSceNgs2.native/src/Voice.cpp +++ b/core/libs/prx/libSceNgs2.native/src/Voice.cpp @@ -18,6 +18,7 @@ static constexpr std::uint8_t COMMAND_TYPE_FLOAT = 1; static constexpr std::uint8_t COMMAND_TYPE_INT = 3; static constexpr std::uint8_t COMMAND_TYPE_UINT = 4; static constexpr std::uint8_t COMMAND_TYPE_FLOAT_ARRAY = 0x11; +static constexpr float MAX_MATRIX_LEVEL = 4.0f; void Ngs2Voice::SetEvent(std::uint32_t eventId) { switch (eventId) { @@ -34,6 +35,7 @@ void Ngs2Voice::SetEvent(std::uint32_t eventId) { case SCE_NGS2_VOICE_EVENT_STOP_IMM: case SCE_NGS2_VOICE_EVENT_KILL: state = Ngs2PlayState::Empty; + Ngs2ClearReverb(*this); break; case SCE_NGS2_VOICE_EVENT_PAUSE: if (state == Ngs2PlayState::Playing) state = Ngs2PlayState::Paused; @@ -46,6 +48,7 @@ void Ngs2Voice::SetEvent(std::uint32_t eventId) { } void Ngs2Voice::ResetSetup() { + ++waveformRevision; SetEvent(SCE_NGS2_VOICE_EVENT_STOP_IMM); std::fill(ports.begin(), ports.end(), Ngs2Port{}); for (auto& matrix : matrices) matrix.clear(); @@ -92,10 +95,12 @@ static Ngs2Port& PortAt(Ngs2Voice& voice, std::uint32_t port) { } static void SetMatrixLevels(Ngs2Voice& voice, std::uint32_t matrixId, const float* levels, std::uint32_t numLevels) { - if (matrixId >= voice.matrices.size() || numLevels > NGS2_MAX_CHANNELS * NGS2_MAX_CHANNELS || (levels == nullptr && numLevels != 0)) { + if (matrixId >= voice.matrices.size() || numLevels == 0 || numLevels > NGS2_MAX_CHANNELS * NGS2_MAX_CHANNELS || levels == nullptr) { APS5_INVALID_ARG_EX; } - voice.matrices[matrixId].assign(levels, levels + numLevels); + auto& matrix = voice.matrices[matrixId]; + matrix.resize(numLevels); + for (std::uint32_t i = 0; i < numLevels; i++) matrix[i] = std::clamp(levels[i], -MAX_MATRIX_LEVEL, MAX_MATRIX_LEVEL); } static void SetPortMatrix(Ngs2Voice& voice, std::uint32_t port, std::int32_t matrixId) { @@ -166,6 +171,20 @@ static void SetupSampler(Ngs2Voice& voice, const Ngs2WaveformFormat& format) { } static void AppendStreamData(Ngs2Voice& voice, const Ngs2SamplerVoiceWaveformBlocksParam& param, std::size_t frameBytes) { + if (voice.waveformType == SCE_NGS2_WAVEFORM_TYPE_ATRAC9) { + const bool pendingWaveform = voice.atrac9.remainingSamples != 0 || + std::any_of(voice.blocks.begin(), voice.blocks.end(), [](const Ngs2Block& block) { return block.info.num_samples != 0; }); + if (!pendingWaveform) throw std::invalid_argument("NGS2: appending compressed data requires a waveform"); + for (std::uint32_t i = 0; i < param.num_blocks; ++i) { + auto block = param.blocks[i]; + if (block.data_size == 0) continue; + block.num_samples = 0; + block.num_skip_samples = 0; + block.num_repeats = 0; + voice.blocks.push_back({static_cast(param.data) + block.data_offset, block}); + } + return; + } if (voice.blocks.empty() || !voice.blocks.back().streaming) throw std::runtime_error("NGS2: appending waveform data to a voice without a streaming waveform is not implemented"); auto& stream = voice.blocks.back(); for (std::uint32_t i = 0; i < param.num_blocks; i++) { @@ -186,6 +205,7 @@ static void AddWaveformBlocks(Ngs2Voice& voice, const Ngs2SamplerVoiceWaveformBl const bool reset = (param.flags & SCE_NGS2_WAVEFORM_BLOCKS_FLAG_RESET) != 0; if (!voice.acceptsBlocks && !reset) throw std::invalid_argument("NGS2: the voice waveform was already closed"); if (reset) { + ++voice.waveformRevision; voice.blocks.clear(); voice.phase = 0; voice.waveformEnd = nullptr; @@ -193,7 +213,6 @@ static void AddWaveformBlocks(Ngs2Voice& voice, const Ngs2SamplerVoiceWaveformBl } const std::size_t frameBytes = voice.channels * sizeof(std::int16_t); if ((param.flags & SCE_NGS2_WAVEFORM_BLOCKS_FLAG_APPEND) != 0) { - if (voice.waveformType != SCE_NGS2_WAVEFORM_TYPE_PCM_I16L) throw std::runtime_error("NGS2: appending waveform data to a voice that is not 16-bit PCM is not implemented"); AppendStreamData(voice, param, frameBytes); voice.acceptsBlocks = (param.flags & SCE_NGS2_WAVEFORM_BLOCKS_FLAG_CONTINUE) != 0; return; @@ -206,7 +225,8 @@ static void AddWaveformBlocks(Ngs2Voice& voice, const Ngs2SamplerVoiceWaveformBl : (static_cast(block.num_skip_samples) + block.num_samples) * frameBytes; const bool streaming = block.num_samples != 0 && bytes > block.data_size && voice.waveformType == SCE_NGS2_WAVEFORM_TYPE_PCM_I16L && block.num_skip_samples == 0 && block.num_repeats == 0 && block.data_size >= frameBytes && block.data_size % frameBytes == 0; - if (!streaming && (block.num_samples == 0 || bytes > block.data_size)) { + const bool compressedStream = voice.waveformType == SCE_NGS2_WAVEFORM_TYPE_ATRAC9 && voice.acceptsBlocks && block.num_repeats == 0; + if (block.num_samples == 0 || (bytes > block.data_size && !streaming && !compressedStream)) { throw std::invalid_argument("NGS2: waveform block " + std::to_string(i) + " does not fit its data"); } voice.blocks.push_back({static_cast(param.data) + block.data_offset, block}); @@ -304,6 +324,21 @@ static void ApplyParam(Ngs2Voice& voice, const Ngs2VoiceParamHeader& param) { if (ParamAs(param).flags != 0) throw std::runtime_error("NGS2: submixer setup flags are not implemented"); SetupMixer(voice, ParamAs(param).num_io_channels); return; + case SCE_NGS2_RACK_ID_REVERB: + if (param.id == SCE_NGS2_REVERB_VOICE_PARAM_SETUP) { + const auto& setup = ParamAs(param); + if (setup.flags != 0) throw std::runtime_error("NGS2: reverb setup flags are not implemented"); + if (setup.num_input_channels != setup.num_output_channels) throw std::runtime_error("NGS2: reverb channel conversion is not implemented"); + if (setup.num_output_channels != 1 && setup.num_output_channels != 2 && setup.num_output_channels != 6 && setup.num_output_channels != 8) APS5_INVALID_ARG_EX; + SetupMixer(voice, setup.num_output_channels); + Ngs2SetupReverb(voice); + return; + } + if (param.id == SCE_NGS2_REVERB_VOICE_PARAM_I3DL2) { + Ngs2SetReverbParams(voice, ParamAs(param).i3dl2); + return; + } + break; case SCE_NGS2_RACK_ID_MASTERING: if (param.id == SCE_NGS2_MASTERING_VOICE_PARAM_SETUP) { SetupMixer(voice, ParamAs(param).num_io_channels); @@ -386,6 +421,12 @@ int APS5_VABI sceNgs2VoiceGetState(uintptr_t voice_handle, Ngs2VoiceState* state sampler.waveform_data = voice.WaveformData(); return SCE_NGS2_OK; } + case SCE_NGS2_RACK_ID_REVERB: { + if (state_size != sizeof(Ngs2VoiceState)) return SCE_NGS2_ERROR_INVALID_OUT_SIZE; + *state = {}; + state->state_flags = voice.stateFlags; + return SCE_NGS2_OK; + } case SCE_NGS2_RACK_ID_SUBMIXER: { if (state_size != sizeof(Ngs2SubmixerVoiceState)) return SCE_NGS2_ERROR_INVALID_OUT_SIZE; auto& submixer = *reinterpret_cast(state); @@ -431,6 +472,14 @@ int APS5_VABI sceNgs2VoiceGetStateFlags(uintptr_t voice_handle, uint32_t* state_ return SCE_NGS2_OK; } +int APS5_VABI sceNgs2PanInit(Ngs2PanWork* work, const float* speaker_angles, float unit_angle, uint32_t num_speakers) { + return Ngs2PanInit(work, speaker_angles, unit_angle, num_speakers); +} + +int APS5_VABI sceNgs2PanGetVolumeMatrix(Ngs2PanWork* work, const Ngs2PanParam* params, uint32_t num_params, uint32_t matrix_format, float* out_volume_matrix) { + return Ngs2PanGetVolumeMatrix(work, params, num_params, matrix_format, out_volume_matrix); +} + } #pragma GCC visibility pop diff --git a/core/libs/prx/libSceNgs2.native/tests/Ngs2Atrac9.cpp b/core/libs/prx/libSceNgs2.native/tests/Ngs2Atrac9.cpp index 44290013c..a6e94c2a9 100644 --- a/core/libs/prx/libSceNgs2.native/tests/Ngs2Atrac9.cpp +++ b/core/libs/prx/libSceNgs2.native/tests/Ngs2Atrac9.cpp @@ -2,7 +2,9 @@ #include "libatrac9.h" +#include #include +#include #include #include #include @@ -334,6 +336,151 @@ static void TestCorruptSuperframeAtPageEnd() { Require(sceNgs2SystemDestroy(system, nullptr) == SCE_NGS2_OK); } +static uintptr_t EmptySampler(uintptr_t system) { + const auto voice = Voice(CreateRack(system, SCE_NGS2_RACK_ID_SAMPLER)); + Control(voice, SCE_NGS2_SAMPLER_VOICE_PARAM_SETUP, Ngs2SamplerVoiceSetupParam{{}, {SCE_NGS2_WAVEFORM_TYPE_ATRAC9, 1, 48000, Config, 0, 0}}); + Patch(voice, Mastering(system, 1)); + Event(voice, SCE_NGS2_VOICE_EVENT_PLAY); + return voice; +} + +static void Queue(uintptr_t voice, const void* data, std::uint32_t bytes, std::uint32_t flags, std::uint32_t samples = 0, std::uint32_t skip = 0) { + const Ngs2WaveformBlock block{0, bytes, 0, skip, samples, 0, bytes}; + Control(voice, SCE_NGS2_SAMPLER_VOICE_PARAM_ADD_WAVEFORM_BLOCKS, Ngs2SamplerVoiceWaveformBlocksParam{{}, data, flags, 1, &block}); +} + +static void TestStreamPartitions() { + const auto reference = Reference(2); + std::vector data(Superframe, Superframe + SuperframeBytes); + data.insert(data.end(), Superframe, Superframe + SuperframeBytes); + for (const auto skip : {100u, 1100u}) { + for (const float pitch : {0.5f, 1.0f, 1.5f}) { + const auto system = CreateSystem(); + const auto voice = EmptySampler(system); + Control(voice, SCE_NGS2_VOICE_PARAM_CALLBACK, + Ngs2VoiceCallbackParam{{}, OnBlock, 0, SCE_NGS2_VOICE_CALLBACK_FLAG_BLOCK_END, 0}); + Control(voice, SCE_NGS2_SAMPLER_VOICE_PARAM_PITCH, Ngs2SamplerVoicePitchParam{{}, pitch}); + callbackFlags.clear(); + const std::uint32_t sizes[]{1, 17, 61, 93, 340}; + std::uint32_t offset = 0; + for (const auto size : sizes) { + const auto flags = offset == 0 ? SCE_NGS2_WAVEFORM_BLOCKS_FLAG_CONTINUE + : SCE_NGS2_WAVEFORM_BLOCKS_FLAG_APPEND | (offset + size == data.size() ? 0 : SCE_NGS2_WAVEFORM_BLOCKS_FLAG_CONTINUE); + Queue(voice, data.data() + offset, size, flags, 800, skip); + offset += size; + } + const auto output = Render(system, 1664); + for (std::size_t i = 0; i < output.size(); ++i) { + const float position = i * pitch; + float expected = 0.0f; + if (position < 800) { + const auto index = static_cast(position); + const auto next = std::min(index + 1, 799u); + const auto fraction = position - index; + expected = reference[skip + index] * (1 - fraction) + reference[skip + next] * fraction; + } + Require(std::abs(output[i] - expected) < 1e-6f); + } + Require(callbackFlags.size() == 5 && Flags(voice) == 0); + Ngs2SamplerVoiceState state{}; + Require(sceNgs2VoiceGetState(voice, &state.voice_state, sizeof(state)) == SCE_NGS2_OK); + Require(state.num_decoded_samples == 800); + Require(sceNgs2SystemDestroy(system, nullptr) == 0); + } + } +} + +struct RefillState { int calls = 0; bool pause = false; bool reset = false; }; + +static void APS5_VABI Refill(const Ngs2VoiceCallbackInfo* info) { + auto& state = *reinterpret_cast(info->callback_data); + ++state.calls; + Require(info->flag == SCE_NGS2_VOICE_CALLBACK_FLAG_BLOCK_END); + Require(info->user_data == info->block_size); + if (state.calls != 1) return; + Require(info->block_size == 13 && info->block_data == Superframe); + if (state.reset) Queue(info->voice_handle, Superframe, SuperframeBytes, SCE_NGS2_WAVEFORM_BLOCKS_FLAG_RESET, 500, 100); + else Queue(info->voice_handle, Superframe + 13, SuperframeBytes - 13, SCE_NGS2_WAVEFORM_BLOCKS_FLAG_APPEND, 0xffffffffu, 0xffffffffu); + if (state.pause) Event(info->voice_handle, SCE_NGS2_VOICE_EVENT_PAUSE); +} + +static void TestRefill(const std::vector& reference) { + for (int mode = 0; mode < 3; ++mode) { + const auto system = CreateSystem(); + const auto voice = EmptySampler(system); + RefillState state{0, mode == 1, mode == 2}; + Control(voice, SCE_NGS2_VOICE_PARAM_CALLBACK, + Ngs2VoiceCallbackParam{{}, Refill, reinterpret_cast(&state), SCE_NGS2_VOICE_CALLBACK_FLAG_BLOCK_END, 0}); + Queue(voice, Superframe, 13, SCE_NGS2_WAVEFORM_BLOCKS_FLAG_CONTINUE, 500, 100); + if (mode != 0) { + for (float value : Render(system, Grain)) Require(value == 0.0f); + Require(state.calls == 1); + if (state.pause) Event(voice, SCE_NGS2_VOICE_EVENT_RESUME); + } + const auto output = Render(system, 512); + for (std::size_t i = 0; i < output.size(); ++i) Require(output[i] == (i < 500 ? reference[100 + i] : 0.0f)); + Require(state.calls == 2 && Flags(voice) == 0); + Require(sceNgs2SystemDestroy(system, nullptr) == 0); + } +} + +static void TestStarvationAndTruncation(const std::vector& reference) { + for (bool truncate : {false, true}) { + const auto system = CreateSystem(); + const auto voice = EmptySampler(system); + Queue(voice, Superframe, 13, SCE_NGS2_WAVEFORM_BLOCKS_FLAG_CONTINUE, 500, 100); + for (float value : Render(system, Grain * 3)) Require(value == 0.0f); + Require((Flags(voice) & SCE_NGS2_VOICE_STATE_FLAG_PLAYING) != 0); + if (truncate) { + Control(voice, SCE_NGS2_SAMPLER_VOICE_PARAM_ADD_WAVEFORM_BLOCKS, Ngs2SamplerVoiceWaveformBlocksParam{{}, nullptr, 0, 0, nullptr}); + bool rejected = false; + try { Render(system, Grain); } catch (const std::invalid_argument&) { rejected = true; } + Require(rejected); + } else { + Queue(voice, Superframe + 13, SuperframeBytes - 13, SCE_NGS2_WAVEFORM_BLOCKS_FLAG_APPEND); + const auto output = Render(system, 512); + for (std::size_t i = 0; i < output.size(); ++i) Require(output[i] == (i < 500 ? reference[100 + i] : 0.0f)); + Require(Flags(voice) == 0); + } + Require(sceNgs2SystemDestroy(system, nullptr) == 0); + } +} + +static void TestAppendBoundaries(const std::vector& reference) { + for (std::uint32_t split = 1; split < SuperframeBytes; ++split) { + const auto system = CreateSystem(); + const auto voice = EmptySampler(system); + Queue(voice, Superframe, split, SCE_NGS2_WAVEFORM_BLOCKS_FLAG_CONTINUE, 500, 100); + for (float value : Render(system, Grain)) Require(value == 0.0f); + const Ngs2WaveformBlock block{split, SuperframeBytes - split, UINT32_MAX, UINT32_MAX, UINT32_MAX, 0, 0}; + Control(voice, SCE_NGS2_SAMPLER_VOICE_PARAM_ADD_WAVEFORM_BLOCKS, + Ngs2SamplerVoiceWaveformBlocksParam{{}, Superframe, SCE_NGS2_WAVEFORM_BLOCKS_FLAG_APPEND, 1, &block}); + const auto output = Render(system, 512); + for (std::size_t i = 0; i < output.size(); ++i) Require(output[i] == (i < 500 ? reference[100 + i] : 0.0f)); + Ngs2SamplerVoiceState state{}; + Require(sceNgs2VoiceGetState(voice, &state.voice_state, sizeof(state)) == SCE_NGS2_OK); + Require(state.num_decoded_samples == 500 && state.decoded_data_size == SuperframeBytes); + Require(state.waveform_data == Superframe + SuperframeBytes && Flags(voice) == 0); + Require(sceNgs2SystemDestroy(system, nullptr) == SCE_NGS2_OK); + } +} + +static void TestAppendWithoutWaveform() { + const auto system = CreateSystem(); + const auto voice = EmptySampler(system); + bool rejected = false; + try { + Queue(voice, Superframe, SuperframeBytes, SCE_NGS2_WAVEFORM_BLOCKS_FLAG_APPEND, 500); + } catch (const std::invalid_argument&) { + rejected = true; + } + Require(rejected); + Queue(voice, Superframe, SuperframeBytes, 0, 500); + Render(system, 512); + Require(Flags(voice) == 0); + Require(sceNgs2SystemDestroy(system, nullptr) == SCE_NGS2_OK); +} + int main() { const auto reference = Reference(); TestSkipAndBlockEnd(reference); @@ -343,5 +490,10 @@ int main() { TestMalformedLoops(); TestCalcBlock(); TestCorruptSuperframeAtPageEnd(); + TestStreamPartitions(); + TestRefill(reference); + TestStarvationAndTruncation(reference); + TestAppendBoundaries(reference); + TestAppendWithoutWaveform(); return 0; } diff --git a/core/libs/prx/libSceNgs2.native/tests/Ngs2Render.cpp b/core/libs/prx/libSceNgs2.native/tests/Ngs2Render.cpp index ee375ba7a..eac7d3fbd 100644 --- a/core/libs/prx/libSceNgs2.native/tests/Ngs2Render.cpp +++ b/core/libs/prx/libSceNgs2.native/tests/Ngs2Render.cpp @@ -1,10 +1,13 @@ #include "Ngs2Test.hpp" +#include #include #include #include #include #include +#include +#include #include #include #include @@ -144,6 +147,110 @@ static void TestPcmBlockEnd() { Require(sceNgs2SystemDestroy(system, nullptr) == SCE_NGS2_OK); } +static bool Near(float a, float b) { + return std::fabs(a - b) < 1e-4f; +} + +static bool PanThrows(Ngs2PanWork* work, const Ngs2PanParam* params, std::uint32_t numParams, std::uint32_t format) { + float out[64] = {}; + try { sceNgs2PanGetVolumeMatrix(work, params, numParams, format, out); } catch (const std::exception&) { return true; } + return false; +} + +static bool PanInitThrows(Ngs2PanWork* work, const float* angles, float unitAngle, std::uint32_t numSpeakers) { + try { sceNgs2PanInit(work, angles, unitAngle, numSpeakers); } catch (const std::exception&) { return true; } + return false; +} + +struct PanCase { + std::vector angles; + float unitAngle; + std::uint32_t numSpeakers; + std::vector params; + std::uint32_t format; + std::vector expected; +}; + +static void TestPan() { + const float pi = std::numbers::pi_v; + Ngs2PanWork work{}; + Require(sceNgs2PanInit(&work, nullptr, 360.0f, 2) == SCE_NGS2_OK); + Require(work.num_speakers == 2 && work.unit_angle == 360.0f); + Require(work.speaker_angles[0] == 90.0f && work.speaker_angles[1] == 270.0f && work.speaker_angles[2] == 450.0f); + Require(sceNgs2PanInit(&work, nullptr, 2.0f * pi, 2) == SCE_NGS2_OK && work.unit_angle == 2.0f * pi); + Require(work.speaker_angles[0] == 90.0f / 180.0f * pi && work.speaker_angles[1] == 270.0f / 180.0f * pi && work.speaker_angles[2] == 450.0f); + Require(sceNgs2PanInit(&work, nullptr, 100.0f, 4) == SCE_NGS2_OK && work.unit_angle == 360.0f && work.speaker_angles[4] == 400.0f); + const float frontBack[] = {-30.0f, 30.0f, 0.0f, -110.0f, 470.0f}; + Require(sceNgs2PanInit(&work, frontBack, 360.0f, 5) == SCE_NGS2_OK); + const float circle[] = {0.0f, 30.0f, 110.0f, 250.0f, 330.0f, 360.0f}; + for (std::uint32_t i = 0; i < 6; i++) Require(work.speaker_angles[i] == circle[i]); + + const std::vector cases = { + {{}, 360.0f, 2, {{0, 1, 1, 0}, {45, 1, 1, 0}, {90, 1, 1, 0}, {135, 1, 1, 0}, {180, 1, 1, 0}, {225, 1, 1, 0}, {270, 1, 1, 0}, {315, 1, 1, 0}}, 2, + {0.707106769f, 0.382683456f, 0.0f, 0.382683456f, 0.707106769f, 0.923879504f, 1.0f, 0.923879504f, 0.707106769f, 0.923879504f, 1.0f, 0.923879504f, 0.707106769f, 0.382683456f, 0.0f, 0.382683456f}}, + {{}, 360.0f, 2, {{30, 0, 1, 0}, {30, -1, 1, 0}, {30, 0.5f, 0.5f, 0}, {30, 3, 1, 0}, {-30, -0.25f, 1, 0}}, 2, + {0.965925813f, 0.866025448f, 0.39667666f, 0.49999997f, 0.896872759f, 0.965925813f, 0.49999997f, 0.495722443f, 0.866025448f, 0.997858942f}}, + {{}, 360.0f, 2, {{60, 1, 1, 0.5f}}, 1, {0.866025388f}}, + {{}, 360.0f, 2, {{60, 1, 1, 0.5f}}, 6, {0.258819073f, 0.965925813f, 0.0f, 0.0f, 0.0f, 0.0f}}, + {{}, 2.0f * pi, 2, {{0.25f * pi, 1, 1, 0}, {1.5f * pi, 1, 1, 0}}, 2, {0.999965429f, 1.0f, 0.00831161533f, 0.0f}}, + {{}, 360.0f, 4, {{100, 1, 1, 0.25f}}, 6, {0.0f, 0.222520918f, 0.0f, 0.25f, 0.0f, 0.974927902f}}, + {{}, 360.0f, 4, {{100, 1, 1, 0.25f}}, 2, {0.0f, 0.709984899f}}, + {{}, 360.0f, 5, {{70, 1, 0.8f, 0.5f}, {200, 0.5f, 1, 0}}, 6, + {0.0f, 0.0f, 0.625465155f, 0.270598054f, 0.0f, 0.270598054f, 0.5f, 0.0f, 0.0f, 0.782271147f, 0.498791903f, 0.491533577f}}, + {{}, 360.0f, 5, {{70, 1, 0.8f, 0.5f}}, 1, {0.691666603f}}, + {{}, 360.0f, 7, {{180, 1, 1, 0.25f}}, 8, {0.0f, 0.0f, 0.0f, 0.25f, 0.0f, 0.0f, 0.707106769f, 0.707106769f}}, + {{}, 360.0f, 7, {{180, 1, 1, 0.25f}}, 6, {0.0f, 0.0f, 0.0f, 0.25f, 0.707106769f, 0.707106769f}}, + {{-pi / 6, pi / 6, 0, -110 * pi / 180, 110 * pi / 180}, 2.0f * pi, 5, {{1.0f, 1, 1, 0}, {-2.5f, 0.2f, 1, 0.1f}}, 6, + {0.0f, 0.0f, 0.859783232f, 0.582614243f, 0.0f, 0.0f, 0.0f, 0.100000001f, 0.0f, 0.753403723f, 0.510659218f, 0.372568965f}}, + {{-30, 30, 0, -90, 90, -150, 150}, 360.0f, 7, {{-120, 1, 1, 0}}, 1, {0.707106769f}}, + {{-45, 45}, 360.0f, 2, {{10, 1, 1, 0}, {720 + 170, 1, 1, 0}}, 2, {0.57357651f, 0.664795876f, 0.819152057f, 0.747025073f}}, + {{-40, 40, -120, 120}, 360.0f, 4, {{-80, 0.75f, 1, 0}}, 8, {0.69351995f, 0.0746578425f, 0.0f, 0.0f, 0.69351995f, 0.180239946f, 0.0f, 0.0f}}, + }; + for (const auto& panCase : cases) { + Require(sceNgs2PanInit(&work, panCase.angles.empty() ? nullptr : panCase.angles.data(), panCase.unitAngle, panCase.numSpeakers) == SCE_NGS2_OK); + std::vector out(panCase.expected.size(), -1.0f); + Require(sceNgs2PanGetVolumeMatrix(&work, panCase.params.data(), static_cast(panCase.params.size()), panCase.format, out.data()) == SCE_NGS2_OK); + for (std::size_t i = 0; i < out.size(); i++) Require(std::fabs(out[i] - panCase.expected[i]) < 1e-6f); + } + + const Ngs2PanParam front{0.0f, 1.0f, 1.0f, 0.0f}; + float out[2] = {}; + Require(sceNgs2PanInit(&work, nullptr, 360.0f, 2) == SCE_NGS2_OK); + Require(sceNgs2PanGetVolumeMatrix(&work, &front, 1, 2, nullptr) == SCE_NGS2_ERROR_INVALID_OUT_ADDRESS); + Require(sceNgs2PanGetVolumeMatrix(&work, &front, 0, 2, out) == SCE_NGS2_OK); + Require(PanThrows(nullptr, &front, 1, 2) && PanThrows(&work, nullptr, 0, 2)); + Require(PanThrows(&work, &front, 1, 4) && PanThrows(&work, &front, 1, 3) && !PanThrows(&work, &front, 1, 8)); + const Ngs2PanParam many[9] = {}; + Require(PanThrows(&work, many, 9, 2) && !PanThrows(&work, many, 8, 2)); + const Ngs2PanParam nanAngle{NAN, 1.0f, 1.0f, 0.0f}; + Require(PanThrows(&work, &nanAngle, 1, 2)); + const Ngs2PanParam laterNan[2] = {front, nanAngle}; + float untouched[4] = {-1.0f, -1.0f, -1.0f, -1.0f}; + bool rejected = false; + try { sceNgs2PanGetVolumeMatrix(&work, laterNan, 2, 2, untouched); } catch (const std::invalid_argument&) { rejected = true; } + Require(rejected && std::all_of(std::begin(untouched), std::end(untouched), [](float level) { return level == -1.0f; })); + const Ngs2PanParam huge{1e30f, 1.0f, 1.0f, 0.0f}; + Require(PanThrows(&work, &huge, 1, 2)); + const float hugeSpeaker[] = {1e30f, 30.0f}; + const float hugeNegative[] = {-1e30f, 30.0f}; + Require(PanInitThrows(&work, hugeSpeaker, 360.0f, 2) && PanInitThrows(&work, hugeNegative, 360.0f, 2)); + Require(sceNgs2PanInit(&work, nullptr, 360.0f, 2) == SCE_NGS2_OK); + work.speaker_angles[0] = -INFINITY; + Require(PanThrows(&work, &front, 1, 2)); + Require(sceNgs2PanInit(&work, nullptr, 360.0f, 2) == SCE_NGS2_OK); + Require(PanInitThrows(nullptr, nullptr, 360.0f, 2)); + for (std::uint32_t count : {0u, 1u, 3u, 6u, 8u}) Require(PanInitThrows(&work, nullptr, 360.0f, count)); + const float pair[] = {-30.0f, 30.0f}; + Require(PanInitThrows(&work, pair, 180.0f, 2) && !PanInitThrows(&work, pair, 2.0f * pi, 2)); + const float invalid[] = {0.0f, NAN}; + Require(PanInitThrows(&work, invalid, 360.0f, 2)); + const float onUnit[] = {720.0f, 30.0f}; + const float belowZero[] = {-360.0f, 30.0f}; + Require(PanInitThrows(&work, onUnit, 360.0f, 2) && !PanInitThrows(&work, belowZero, 360.0f, 2)); + work.unit_angle = 100.0f; + Require(PanThrows(&work, &front, 1, 2)); +} + static void TestPitchAndRepeat() { const auto system = CreateSystem(); const auto master = Mastering(system, 1); @@ -201,6 +308,49 @@ static void TestSubmixerMatrix() { Require(sceNgs2SystemDestroy(system, nullptr) == SCE_NGS2_OK); } +static std::vector MatrixLevelFrame(const std::vector& levels, bool command) { + const auto system = CreateSystem(); + const auto master = Mastering(system, 2); + const std::vector pcm(Grain, 16384); + const auto sampler = Sampler(system, pcm, 0); + Patch(sampler, master); + if (command) { + const Ngs2VoiceCommand set{5, 0, 0x11, static_cast(levels.size()), {.levels = levels.data()}}; + Require(sceNgs2VoiceRunCommands(sampler, &set, 1) == SCE_NGS2_OK); + } else { + Control(sampler, SCE_NGS2_VOICE_PARAM_MATRIX_LEVELS, Ngs2VoiceMatrixLevelsParam{{}, 0, static_cast(levels.size()), levels.data()}); + } + Control(sampler, SCE_NGS2_VOICE_PARAM_PORT_MATRIX, Ngs2VoicePortMatrixParam{{}, 0, 0}); + Event(sampler, SCE_NGS2_VOICE_EVENT_PLAY); + std::vector out(Grain * 2, -1.0f); + const Ngs2RenderBufferInfo info{out.data(), out.size() * sizeof(float), SCE_NGS2_WAVEFORM_TYPE_PCM_F32L, 2}; + Require(sceNgs2SystemRender(system, &info, 1) == SCE_NGS2_OK); + Require(sceNgs2SystemDestroy(system, nullptr) == SCE_NGS2_OK); + return {out[0], out[1]}; +} + +static void TestMatrixLevelClamp() { + const float inf = INFINITY; + for (bool command : {false, true}) { + Require(MatrixLevelFrame({4.0f, -4.0f}, command) == (std::vector{2.0f, -2.0f})); + Require(MatrixLevelFrame({4.0001f, -4.5f}, command) == (std::vector{2.0f, -2.0f})); + Require(MatrixLevelFrame({100.0f, -100.0f}, command) == (std::vector{2.0f, -2.0f})); + Require(MatrixLevelFrame({inf, -inf}, command) == (std::vector{2.0f, -2.0f})); + Require(MatrixLevelFrame({3.5f, -0.25f}, command) == (std::vector{1.75f, -0.125f})); + const auto nan = MatrixLevelFrame({NAN, 0.5f}, command); + Require(std::isnan(nan[0]) && nan[1] == 0.25f); + } + + const auto system = CreateSystem(); + const std::vector silence(Grain, 0); + const auto sampler = Sampler(system, silence, 0); + const float levels[2] = {1.0f, 1.0f}; + bool empty = false; + try { Control(sampler, SCE_NGS2_VOICE_PARAM_MATRIX_LEVELS, Ngs2VoiceMatrixLevelsParam{{}, 0, 0, levels}); } catch (const std::invalid_argument&) { empty = true; } + Require(empty); + Require(sceNgs2SystemDestroy(system, nullptr) == SCE_NGS2_OK); +} + static int allocations = 0; static std::int32_t APS5_VABI Allocate(Ngs2ContextBufferInfo* info) { Require(info->host_buffer == nullptr && info->host_buffer_size != 0 && info->user_data == 9); @@ -215,6 +365,185 @@ static std::int32_t APS5_VABI Release(Ngs2ContextBufferInfo* info) { return SCE_NGS2_OK; } +struct ReverbFrame { + std::uint32_t frame; + std::vector samples; +}; + +struct ReverbCase { + std::uint32_t channels; + float amplitude; + bool noise; + Ngs2ReverbI3DL2Param params; + std::int32_t changeGrain; + Ngs2ReverbI3DL2Param change; + std::vector expected; +}; + +static std::vector RenderReverb(const ReverbCase& reverbCase, std::uint32_t frames) { + const auto firstBuffer = usedBuffers; + const auto system = CreateSystem(); + Require(sceNgs2SystemSetGrainSamples(system, 256) == SCE_NGS2_OK); + const auto channels = reverbCase.channels; + const auto master = Mastering(system, channels); + const auto reverb = Voice(CreateRack(system, SCE_NGS2_RACK_ID_REVERB)); + Control(reverb, SCE_NGS2_REVERB_VOICE_PARAM_SETUP, Ngs2ReverbVoiceSetupParam{{}, channels, channels, 0, 0}); + Control(reverb, SCE_NGS2_REVERB_VOICE_PARAM_I3DL2, Ngs2ReverbVoiceI3DL2Param{{}, reverbCase.params}); + Patch(reverb, master); + Event(reverb, SCE_NGS2_VOICE_EVENT_PLAY); + std::vector pcm(4096 * channels, 0); + std::uint32_t seed = 1; + for (std::uint32_t i = 0; reverbCase.noise && i + 3 * channels < pcm.size(); i++) { + seed = seed * 1664525u + 1013904223u; + if (i < 3000 * channels) pcm[i + 3 * channels] = static_cast(static_cast((seed >> 16) & 0x3fff) - 0x2000); + } + for (std::uint32_t c = 0; !reverbCase.noise && c < channels; c++) pcm[3 * channels + c] = static_cast((16384 - 4000 * static_cast(c)) * reverbCase.amplitude); + const auto sampler = Voice(CreateRack(system, SCE_NGS2_RACK_ID_SAMPLER)); + Control(sampler, SCE_NGS2_SAMPLER_VOICE_PARAM_SETUP, Ngs2SamplerVoiceSetupParam{{}, {SCE_NGS2_WAVEFORM_TYPE_PCM_I16L, channels, 48000, 0, 0, 0}}); + const Ngs2WaveformBlock block{0, pcm.size() * sizeof(std::int16_t), 0, 0, 4096, 0, 0}; + Control(sampler, SCE_NGS2_SAMPLER_VOICE_PARAM_ADD_WAVEFORM_BLOCKS, Ngs2SamplerVoiceWaveformBlocksParam{{}, pcm.data(), 0, 1, &block}); + Patch(sampler, reverb); + std::vector identity(channels * channels, 0.0f); + for (std::uint32_t c = 0; c < channels; c++) identity[c * channels + c] = 1.0f; + Control(sampler, SCE_NGS2_VOICE_PARAM_MATRIX_LEVELS, Ngs2VoiceMatrixLevelsParam{{}, 0, channels * channels, identity.data()}); + Control(sampler, SCE_NGS2_VOICE_PARAM_PORT_MATRIX, Ngs2VoicePortMatrixParam{{}, 0, 0}); + Event(sampler, SCE_NGS2_VOICE_EVENT_PLAY); + std::vector all; + std::vector out(256 * channels); + for (std::uint32_t grain = 0; grain * 256 < frames; grain++) { + if (static_cast(grain) == reverbCase.changeGrain) Control(reverb, SCE_NGS2_REVERB_VOICE_PARAM_I3DL2, Ngs2ReverbVoiceI3DL2Param{{}, reverbCase.change}); + const Ngs2RenderBufferInfo info{out.data(), out.size() * sizeof(float), SCE_NGS2_WAVEFORM_TYPE_PCM_F32L, channels}; + Require(sceNgs2SystemRender(system, &info, 1) == SCE_NGS2_OK); + all.insert(all.end(), out.begin(), out.end()); + } + Require(sceNgs2SystemDestroy(system, nullptr) == SCE_NGS2_OK); + usedBuffers = firstBuffer; + return all; +} + +static void TestReverb() { + const Ngs2ReverbI3DL2Param cave{1.0f, 0.0f, -1000, 0, 0, 2.91f, 1.3f, -602, 0.015f, -302, 0.022f, 100.0f, 100.0f, 5000.0f, {}}; + auto changed = cave; + changed.reflection_pattern = 5; + changed.decay_time = 0.7f; + changed.decay_hf_ratio = 0.6f; + changed.reflections_delay = 0.004f; + changed.reverb_delay = 0.03f; + changed.room_hf = -900; + changed.wet = 0.6f; + changed.dry = 0.4f; + auto denser = cave; + denser.density = 99.999f; + const std::vector cases = { + {2, 1.0f, false, cave, -1, {}, {{100, {0, 0}}, {723, {0.0687032491f, 0.0687032491f}}, {1055, {0.0392590016f, 0.0392590016f}}, {1417, {0.0131517649f, 0.0131517649f}}, + {1779, {0.0098147504f, 0.0098147504f}}, {2500, {0, 0}}, {4000, {1.45101382e-07f, -1.15403088e-06f}}, {6000, {0.000655628915f, 3.03641864e-05f}}, + {9000, {0.000400578661f, -0.000684358703f}}, {12000, {0.000191548112f, 0.000220489033f}}, {16000, {0.000210488462f, 0.000146183695f}}, + {20000, {2.41150738e-05f, -0.000864534522f}}, {26000, {0.000125532999f, -0.000234790568f}}, {32000, {-0.000462158496f, 4.56995949e-05f}}}}, + {2, 1.0f, false, {1.0f, 0.0f, -500, -800, 3, 1.2f, 0.5f, -300, 0.01f, 0, 0.03f, 80.0f, 60.0f, 3000.0f, {}}, -1, {}, + {{100, {0, 0}}, {723, {0.000166983227f, 0.000166983227f}}, {1055, {-3.41919585e-05f, -3.41919585e-05f}}, {1417, {0, 0}}, {1779, {0, 0}}, + {2500, {0, 1.55222626e-08f}}, {4000, {-0.000106364969f, 9.90879998e-05f}}, {6000, {0.00013573296f, 7.22560799e-06f}}, + {9000, {-7.91876082e-05f, 0.000225067721f}}, {12000, {0.000150469248f, 0.00036288987f}}, {16000, {-7.86288219e-05f, -0.000195921995f}}, + {20000, {1.01910318e-05f, 5.79599109e-05f}}, {26000, {2.14232614e-06f, -2.05506512e-05f}}, {32000, {-4.31467561e-06f, 2.34100444e-05f}}}}, + {6, 1.0f, false, {1.0f, 0.2f, -200, -300, 12, 1.0f, 0.9f, -100, 0.03f, -100, 0.04f, 70.0f, 90.0f, 4000.0f, {}}, -1, {}, + {{6000, {-0.000167909253f, -0.000179917421f, -0.000647808949f, 0, 3.22720189e-05f, 0.000213803389f}}, + {9000, {8.47857882e-05f, 8.62322413e-05f, -0.000100753634f, 0, 0.000109430795f, -0.000122895101f}}, + {12000, {4.37544441e-05f, 3.58103216e-06f, -0.00013746327f, 0, 9.3237948e-05f, -0.00013529828f}}, + {16000, {8.63046807e-08f, 1.62214419e-05f, 8.89167641e-05f, 0, 1.02178528e-05f, 0.000201705057f}}, + {20000, {-9.6526619e-06f, -2.73812566e-05f, -1.92096536e-07f, 0, -0.000116049101f, 6.94858772e-06f}}, + {26000, {1.27750207e-06f, -1.59169917e-06f, 5.30674515e-05f, 0, -3.59285696e-05f, 5.16466662e-06f}}, + {32000, {3.5737969e-06f, 4.19655044e-06f, -2.30460955e-07f, 0, 1.16596939e-05f, 1.14889262e-05f}}}}, + {2, 1.0f, false, cave, 6, changed, {{723, {0.0687032491f, 0.0687032491f}}, {1055, {0.0392590016f, 0.0392590016f}}, {1417, {0.0131517649f, 0.0131517649f}}, + {1779, {4.38156894e-05f, 4.38156894e-05f}}, {2500, {0, 0}}, {4000, {-1.60370582e-05f, -4.44955149e-05f}}, {6000, {1.02755057e-05f, 8.34046841e-06f}}, + {9000, {1.1641363e-05f, -4.69685165e-06f}}, {12000, {3.57917179e-06f, -2.34550089e-06f}}, {16000, {-8.00212234e-08f, 3.19533115e-06f}}, + {20000, {5.66619519e-07f, 4.26969848e-07f}}, {26000, {1.46052287e-07f, -2.44572629e-08f}}, {32000, {-4.45668853e-08f, 1.51349102e-08f}}}}, + {2, 0.02f, false, {40.0f, -3.0f, 5000, 500, 20, 50.0f, 5.0f, 3000, 0.9f, 5000, 0.9f, 300.0f, 300.0f, 50000.0f, {}}, -1, {}, + {{14403, {0.877262115f, 0.877262115f}}, {14500, {0, 0}}, {15000, {0, 0}}, {15500, {0, 7.50572099e-07f}}, {16000, {4.95207075e-07f, 6.37215771e-06f}}, + {20000, {0.00482429564f, -0.00149071764f}}, {26000, {-0.0096597122f, 0.00718426611f}}, {32000, {0.0107425917f, -0.000738018774f}}}}, + {2, 1.0f, false, {1.0f, 0.3f, 0, -200, 15, 0.8f, 1.0f, 0, 0.002f, -500, 0.004f, 60.0f, 80.0f, 6000.0f, {}}, -1, {}, + {{60, {0, 0}}, {99, {0.0947210863f, 0.0947210863f}}, {100, {0.0472047739f, 0.0472047739f}}, {150, {0, 0}}, {200, {0, 0}}, {300, {0, 0}}, + {2000, {-9.86956729e-06f, 7.96923473e-07f}}, {3000, {-0.000152820328f, -4.59704825e-06f}}, {5000, {0.000203833857f, -0.000208990226f}}, + {8000, {-3.69777154e-05f, -0.000259475899f}}}}, + {2, 1.0f, true, cave, 6, changed, {{1540, {0.0222691782f, 0.0242232569f}}, {1600, {0.0201471522f, 0.0186609477f}}, {1700, {-0.0211425349f, -0.0235042125f}}, + {1780, {-0.0168117173f, 0.00417749817f}}, {1800, {-0.0866222829f, -0.0945812687f}}, {2000, {0.0131965755f, -0.0707511827f}}, + {2500, {0.0942443311f, -0.100531064f}}, {3500, {0.000985965831f, 0.000946713379f}}, {8000, {-0.000968122098f, 0.000556072395f}}, + {15000, {0.000227724231f, 0.000151095577f}}}}, + {2, 1.0f, true, cave, 20, denser, {{5200, {-0.00942458585f, -0.0064059021f}}, {6000, {0.00248076022f, 0.00317249796f}}, + {8000, {0.00102439919f, 0.00105210941f}}, {12000, {1.63059376e-05f, 6.10309735e-06f}}, {20000, {-1.08853078e-06f, -2.99162899e-08f}}}}, + }; + for (const auto& reverbCase : cases) { + const auto out = RenderReverb(reverbCase, 32256); + for (const auto& expected : reverbCase.expected) { + for (std::uint32_t c = 0; c < reverbCase.channels; c++) { + if (c == 3) continue; + const float value = out[expected.frame * reverbCase.channels + c]; + Require(std::fabs(value - expected.samples[c]) <= 2e-7f + std::fabs(expected.samples[c]) * 1e-5f); + } + } + } +} + +static void TestReverbSetup() { + const auto system = CreateSystem(); + const auto master = Mastering(system, 2); + const auto reverb = Voice(CreateRack(system, SCE_NGS2_RACK_ID_REVERB)); + for (std::uint32_t channels : {3u, 4u, 5u, 7u}) { + bool rejected = false; + try { Control(reverb, SCE_NGS2_REVERB_VOICE_PARAM_SETUP, Ngs2ReverbVoiceSetupParam{{}, channels, channels, 0, 0}); } catch (const std::invalid_argument&) { rejected = true; } + Require(rejected); + } + bool conversion = false; + try { Control(reverb, SCE_NGS2_REVERB_VOICE_PARAM_SETUP, Ngs2ReverbVoiceSetupParam{{}, 1, 2, 0, 0}); } catch (const std::runtime_error&) { conversion = true; } + Require(conversion); + bool flags = false; + try { Control(reverb, SCE_NGS2_REVERB_VOICE_PARAM_SETUP, Ngs2ReverbVoiceSetupParam{{}, 2, 2, 1, 0}); } catch (const std::runtime_error&) { flags = true; } + Require(flags); + Control(reverb, SCE_NGS2_REVERB_VOICE_PARAM_SETUP, Ngs2ReverbVoiceSetupParam{{}, 2, 2, 0, 0}); + Ngs2ReverbI3DL2Param invalid{1.0f, 0.0f, 0, 0, 0, 1.0f, 1.0f, 0, 0.0f, 0, 0.0f, 100.0f, 100.0f, 5000.0f, {}}; + invalid.decay_time = NAN; + bool nan = false; + try { Control(reverb, SCE_NGS2_REVERB_VOICE_PARAM_I3DL2, Ngs2ReverbVoiceI3DL2Param{{}, invalid}); } catch (const std::invalid_argument&) { nan = true; } + Require(nan); + Patch(reverb, master); + Event(reverb, SCE_NGS2_VOICE_EVENT_PLAY); + const std::vector pcm(Grain * 64, 16384); + const auto sampler = Sampler(system, pcm, 0); + Patch(sampler, reverb); + Event(sampler, SCE_NGS2_VOICE_EVENT_PLAY); + std::vector out(Grain * 2); + const Ngs2RenderBufferInfo info{out.data(), out.size() * sizeof(float), SCE_NGS2_WAVEFORM_TYPE_PCM_F32L, 2}; + float loudest = 0.0f; + for (std::uint32_t grain = 0; grain < 600; grain++) { + Require(sceNgs2SystemRender(system, &info, 1) == SCE_NGS2_OK); + for (float sample : out) loudest = std::max(loudest, std::fabs(sample)); + } + Require(loudest > 0.0f && loudest < 1e-8f); + Ngs2VoiceState state{}; + Require(sceNgs2VoiceGetState(reverb, &state, sizeof(state)) == SCE_NGS2_OK && (state.state_flags & SCE_NGS2_VOICE_STATE_FLAG_INUSE) != 0); + Require(sceNgs2SystemSetSampleRate(system, 7000) == SCE_NGS2_OK); + bool lowRate = false; + try { sceNgs2SystemRender(system, &info, 1); } catch (const std::runtime_error&) { lowRate = true; } + Require(lowRate); + Require(sceNgs2SystemDestroy(system, nullptr) == SCE_NGS2_OK); + + Ngs2ReverbRackOption option{}; + option.rack_option.size = sizeof(option); + option.rack_option.max_grain_samples = 512; + option.rack_option.max_voices = 1; + option.rack_option.max_input_delay_blocks = 1; + option.rack_option.max_matrices = 1; + option.rack_option.max_ports = 8; + option.max_channels = 8; + option.reverb_size = 1; + Ngs2ContextBufferInfo query{}; + Require(sceNgs2RackQueryBufferSize(SCE_NGS2_RACK_ID_REVERB, &option.rack_option, &query) == SCE_NGS2_OK); + for (std::uint32_t size : {0u, 2u}) { + option.reverb_size = size; + bool unsupported = false; + try { sceNgs2RackQueryBufferSize(SCE_NGS2_RACK_ID_REVERB, &option.rack_option, &query); } catch (const std::runtime_error&) { unsupported = true; } + Require(unsupported); + } +} + static void TestSampleRate() { const auto system = CreateSystem(); Require(sceNgs2SystemSetSampleRate(0x1234, 96000) == SCE_NGS2_ERROR_INVALID_SYSTEM_HANDLE); @@ -423,6 +752,116 @@ static void TestLock() { Require(sceNgs2SystemDestroy(system, nullptr) == SCE_NGS2_OK); } +static constexpr float HalfPower = std::numbers::sqrt2_v / 2.0f; + +static std::vector MixFrame(std::uint32_t sourceChannels, std::uint32_t sourceChannel, std::uint32_t destChannels) { + const auto firstBuffer = usedBuffers; + const auto system = CreateSystem(); + const auto master = Mastering(system, destChannels); + const auto voice = Voice(CreateRack(system, SCE_NGS2_RACK_ID_SAMPLER)); + Control(voice, SCE_NGS2_SAMPLER_VOICE_PARAM_SETUP, Ngs2SamplerVoiceSetupParam{{}, {SCE_NGS2_WAVEFORM_TYPE_PCM_I16L, sourceChannels, 48000, 0, 0, 0}}); + std::vector frames(Grain * sourceChannels, 0); + for (std::uint32_t i = 0; i < Grain; i++) frames[i * sourceChannels + sourceChannel] = 16384; + const Ngs2WaveformBlock block{0, frames.size() * sizeof(std::int16_t), 0, 0, Grain, 0, 0}; + Control(voice, SCE_NGS2_SAMPLER_VOICE_PARAM_ADD_WAVEFORM_BLOCKS, Ngs2SamplerVoiceWaveformBlocksParam{{}, frames.data(), 0, 1, &block}); + Patch(voice, master); + Event(voice, SCE_NGS2_VOICE_EVENT_PLAY); + std::vector out(Grain * destChannels, -1.0f); + const Ngs2RenderBufferInfo info{out.data(), out.size() * sizeof(float), SCE_NGS2_WAVEFORM_TYPE_PCM_F32L, destChannels}; + Require(sceNgs2SystemRender(system, &info, 1) == SCE_NGS2_OK); + Require(sceNgs2SystemDestroy(system, nullptr) == SCE_NGS2_OK); + usedBuffers = firstBuffer; + return std::vector(out.begin(), out.begin() + destChannels); +} + +static void TestDefaultChannelMap() { + const float half = 0.5f * HalfPower; + const float quarter = 0.5f * (std::numbers::sqrt2_v / 4.0f); + Require(MixFrame(1, 0, 1) == std::vector{0.5f}); + Require(MixFrame(1, 0, 2) == (std::vector{half, half})); + Require(MixFrame(1, 0, 6) == (std::vector{0, 0, 0.5f, 0, 0, 0})); + Require(MixFrame(1, 0, 8) == (std::vector{0, 0, 0.5f, 0, 0, 0, 0, 0})); + Require(MixFrame(2, 0, 1) == std::vector{half} && MixFrame(2, 1, 1) == std::vector{half}); + Require(MixFrame(2, 1, 2) == (std::vector{0, 0.5f})); + Require(MixFrame(2, 1, 6) == (std::vector{0, 0.5f, 0, 0, 0, 0})); + Require(MixFrame(3, 2, 2) == (std::vector{0, 0})); + Require(MixFrame(3, 2, 6) == (std::vector{0, 0, 0, 0.5f, 0, 0})); + Require(MixFrame(4, 2, 1) == std::vector{0.25f}); + Require(MixFrame(4, 2, 2) == (std::vector{half, 0})); + Require(MixFrame(4, 3, 6) == (std::vector{0, 0, 0, 0, 0, 0.5f})); + Require(MixFrame(5, 2, 2) == (std::vector{half, half})); + Require(MixFrame(5, 3, 1) == std::vector{0.25f}); + Require(MixFrame(5, 3, 2) == (std::vector{0, 0})); + Require(MixFrame(6, 3, 1) == std::vector{0}); + Require(MixFrame(6, 4, 2) == (std::vector{0.25f, 0})); + Require(MixFrame(6, 5, 8) == (std::vector{0, 0, 0, 0, 0, 0.5f, 0, 0})); + Require(MixFrame(7, 6, 1) == std::vector{quarter}); + Require(MixFrame(7, 6, 2) == (std::vector{quarter, quarter})); + Require(MixFrame(7, 6, 6) == (std::vector{0, 0, 0, 0, half, half})); + Require(MixFrame(7, 6, 8) == (std::vector{0, 0, 0, 0, 0, 0, half, half})); + Require(MixFrame(8, 6, 2) == (std::vector{0.25f, 0})); + Require(MixFrame(8, 7, 6) == (std::vector{0, 0, 0, 0, 0, 0.5f})); + Require(MixFrame(8, 7, 8) == (std::vector{0, 0, 0, 0, 0, 0, 0, 0.5f})); + + const auto system = CreateSystem(); + const auto master = Mastering(system, 2); + const auto submixer = Voice(CreateRack(system, SCE_NGS2_RACK_ID_SUBMIXER)); + Control(submixer, SCE_NGS2_SUBMIXER_VOICE_PARAM_SETUP, Ngs2SubmixerVoiceSetupParam{{}, 4, 0}); + Patch(submixer, master); + Event(submixer, SCE_NGS2_VOICE_EVENT_PLAY); + const std::vector pcm(Grain, 16384); + const auto sampler = Sampler(system, pcm, 0); + Patch(sampler, submixer); + Event(sampler, SCE_NGS2_VOICE_EVENT_PLAY); + std::vector out(Grain * 2, -1.0f); + const Ngs2RenderBufferInfo info{out.data(), out.size() * sizeof(float), SCE_NGS2_WAVEFORM_TYPE_PCM_F32L, 2}; + bool unsupported = false; + try { sceNgs2SystemRender(system, &info, 1); } catch (const std::runtime_error&) { unsupported = true; } + Require(unsupported); + Require(sceNgs2SystemDestroy(system, nullptr) == SCE_NGS2_OK); +} + +static void TestUnsetMatrix() { + const auto system = CreateSystem(); + const auto master = Mastering(system, 2); + const auto submixer = Voice(CreateRack(system, SCE_NGS2_RACK_ID_SUBMIXER)); + Control(submixer, SCE_NGS2_SUBMIXER_VOICE_PARAM_SETUP, Ngs2SubmixerVoiceSetupParam{{}, 2, 0}); + Patch(submixer, master); + Event(submixer, SCE_NGS2_VOICE_EVENT_PLAY); + + const std::vector pcm(Grain, 16384); + const auto sampler = Sampler(system, pcm, 0); + Patch(sampler, submixer); + Control(sampler, SCE_NGS2_VOICE_PARAM_PORT_MATRIX, Ngs2VoicePortMatrixParam{{}, 0, 0}); + Control(sampler, SCE_NGS2_VOICE_PARAM_PORT_VOLUME, Ngs2VoicePortVolumeParam{{}, 0, 0.5f}); + Event(sampler, SCE_NGS2_VOICE_EVENT_PLAY); + + std::vector out(Grain * 2, -1.0f); + const Ngs2RenderBufferInfo info{out.data(), out.size() * sizeof(float), SCE_NGS2_WAVEFORM_TYPE_PCM_F32L, 2}; + Require(sceNgs2SystemRender(system, &info, 1) == SCE_NGS2_OK); + const float spread = 0.25f * HalfPower; + for (std::uint32_t i = 0; i < Grain; i++) Require(out[i * 2] == spread && out[i * 2 + 1] == spread); + Require(sceNgs2SystemDestroy(system, nullptr) == SCE_NGS2_OK); + + const auto stereoSystem = CreateSystem(); + const auto stereoMaster = Mastering(stereoSystem, 2); + const auto stereo = Voice(CreateRack(stereoSystem, SCE_NGS2_RACK_ID_SAMPLER)); + Control(stereo, SCE_NGS2_SAMPLER_VOICE_PARAM_SETUP, Ngs2SamplerVoiceSetupParam{{}, {SCE_NGS2_WAVEFORM_TYPE_PCM_I16L, 2, 48000, 0, 0, 0}}); + std::vector frames; + for (std::uint32_t i = 0; i < Grain; i++) { + frames.push_back(16384); + frames.push_back(8192); + } + const Ngs2WaveformBlock block{0, frames.size() * sizeof(std::int16_t), 0, 0, Grain, 0, 0}; + Control(stereo, SCE_NGS2_SAMPLER_VOICE_PARAM_ADD_WAVEFORM_BLOCKS, Ngs2SamplerVoiceWaveformBlocksParam{{}, frames.data(), 0, 1, &block}); + Patch(stereo, stereoMaster); + Control(stereo, SCE_NGS2_VOICE_PARAM_PORT_MATRIX, Ngs2VoicePortMatrixParam{{}, 0, 0}); + Event(stereo, SCE_NGS2_VOICE_EVENT_PLAY); + Require(sceNgs2SystemRender(stereoSystem, &info, 1) == SCE_NGS2_OK); + for (std::uint32_t i = 0; i < Grain; i++) Require(out[i * 2] == 0.5f && out[i * 2 + 1] == 0.25f); + Require(sceNgs2SystemDestroy(stereoSystem, nullptr) == SCE_NGS2_OK); +} + static void TestAllocator() { const Ngs2BufferAllocator allocator{Allocate, Release, 9}; uintptr_t system = 0; @@ -448,13 +887,19 @@ int main() { Require(std::atexit(RenderAfterStaticTeardown) == 0); TestErrorsAndInfo(); TestPcmBlockEnd(); + TestPan(); TestPitchAndRepeat(); TestSubmixerMatrix(); + TestReverb(); + TestReverbSetup(); TestSampleRate(); TestUserData(); TestMasteringGain(); + TestMatrixLevelClamp(); TestStereoIntoSurround(); TestLock(); + TestDefaultChannelMap(); + TestUnsetMatrix(); TestAllocator(); exitSystem = CreateSystem(); return 0; diff --git a/core/libs/prx/libSceNgs2.native/tests/Ngs2Test.hpp b/core/libs/prx/libSceNgs2.native/tests/Ngs2Test.hpp index 97843654b..51a50d512 100644 --- a/core/libs/prx/libSceNgs2.native/tests/Ngs2Test.hpp +++ b/core/libs/prx/libSceNgs2.native/tests/Ngs2Test.hpp @@ -25,6 +25,8 @@ int APS5_VABI sceNgs2SystemGetUserData(uintptr_t, uintptr_t*); int APS5_VABI sceNgs2SystemLock(uintptr_t); int APS5_VABI sceNgs2SystemUnlock(uintptr_t); int APS5_VABI sceNgs2SystemRender(uintptr_t, const Ngs2RenderBufferInfo*, uint32_t); +int APS5_VABI sceNgs2PanInit(Ngs2PanWork*, const float*, float, uint32_t); +int APS5_VABI sceNgs2PanGetVolumeMatrix(Ngs2PanWork*, const Ngs2PanParam*, uint32_t, uint32_t, float*); int APS5_VABI sceNgs2RackQueryBufferSize(uint32_t, const Ngs2RackOption*, Ngs2ContextBufferInfo*); int APS5_VABI sceNgs2RackCreate(uintptr_t, uint32_t, const Ngs2RackOption*, const Ngs2ContextBufferInfo*, uintptr_t*); int APS5_VABI sceNgs2RackCreateWithAllocator(uintptr_t, uint32_t, const Ngs2RackOption*, const Ngs2BufferAllocator*, uintptr_t*); diff --git a/core/libs/prx/libSceNpManager/Export.cpp b/core/libs/prx/libSceNpManager/Export.cpp index 511383f4c..e086b9ebf 100644 --- a/core/libs/prx/libSceNpManager/Export.cpp +++ b/core/libs/prx/libSceNpManager/Export.cpp @@ -8,6 +8,7 @@ // PSN is not emulated: the user is reported as signed out and online queries fail. static constexpr int SCE_NP_ERROR_INVALID_ARGUMENT = static_cast(0x80550003); static constexpr int SCE_NP_ERROR_SIGNED_OUT = static_cast(0x80550006); +static constexpr int SCE_NP_ERROR_USER_NOT_FOUND = static_cast(0x80550007); static constexpr int SCE_NP_ERROR_CALLBACK_ALREADY_REGISTERED = static_cast(0x80550008); static constexpr int SCE_NP_ERROR_CALLBACK_NOT_REGISTERED = static_cast(0x80550009); static constexpr uint32_t NP_STATE_SIGNED_OUT = 1; @@ -33,12 +34,10 @@ int APS5_VABI sceNpCheckCallback(void) { return 0; } -int APS5_VABI sceNpCheckNpAvailability(int req_id, const char* user, void* result) { +int APS5_VABI sceNpCheckNpAvailability(int req_id, const NpOnlineId* online_id) { (void)req_id; - (void)user; - (void)result; - NotImplemented_nid_no_patch(__func__); - return 0; + if (!online_id) return SCE_NP_ERROR_INVALID_ARGUMENT; + return SCE_NP_ERROR_USER_NOT_FOUND; } int APS5_VABI sceNpCheckNpReachability(int req_id, int user_id) { diff --git a/core/libs/prx/libSceNpPartner001/Export.cpp b/core/libs/prx/libSceNpPartner001/Export.cpp index ef5b77a1f..74e81b2df 100644 --- a/core/libs/prx/libSceNpPartner001/Export.cpp +++ b/core/libs/prx/libSceNpPartner001/Export.cpp @@ -2,15 +2,13 @@ extern "C" { -APS5_EXPORT("7CxI50-xlCk", sceNpPartnerUnknown_7CxI50_MxlCk); -int APS5_VABI sceNpPartnerUnknown_7CxI50_MxlCk(void) { - NotImplemented_nid_no_patch("7CxI50-xlCk"); +int APS5_VABI sceNpEAAccessInitialize(void) { + NotImplemented_nid_no_patch(__func__); return 0; } -APS5_EXPORT("pMxXhNozUX8", sceNpPartnerUnknown_pMxXhNozUX8); -int APS5_VABI sceNpPartnerUnknown_pMxXhNozUX8(void) { - NotImplemented_nid_no_patch("pMxXhNozUX8"); +int APS5_VABI sceNpEAAccessTerminate(void) { + NotImplemented_nid_no_patch(__func__); return 0; } diff --git a/core/libs/prx/libSceOpusDec/CMakeLists.txt b/core/libs/prx/libSceOpusDec/CMakeLists.txt new file mode 100644 index 000000000..4179bce7f --- /dev/null +++ b/core/libs/prx/libSceOpusDec/CMakeLists.txt @@ -0,0 +1,16 @@ +add_library(libSceOpusDec SHARED EXCLUDE_FROM_ALL ${CMAKE_CURRENT_SOURCE_DIR}/Export.cpp) +target_include_directories(libSceOpusDec PRIVATE ${LIBS_INCLUDE_DIR}) +target_link_libraries(libSceOpusDec PRIVATE ffmpeg) +configure_windows_unwind(libSceOpusDec) + +if(BUILD_TESTING) + add_test_executable(guest_opus_dec_tests ${LIBS_INCLUDE_DIR}/tests/GuestOpusDec.cpp) + target_include_directories(guest_opus_dec_tests PRIVATE ${LIBS_INCLUDE_DIR}) + target_link_libraries(guest_opus_dec_tests PRIVATE libSceOpusDec libc) + configure_windows_unwind(guest_opus_dec_tests) + add_test(NAME guest_opus_dec COMMAND guest_opus_dec_tests) + if(WIN32) + set_tests_properties(guest_opus_dec PROPERTIES ENVIRONMENT_MODIFICATION + "PATH=path_list_prepend:$") + endif() +endif() diff --git a/core/libs/prx/libSceOpusDec/Export.cpp b/core/libs/prx/libSceOpusDec/Export.cpp new file mode 100644 index 000000000..3e1729acb --- /dev/null +++ b/core/libs/prx/libSceOpusDec/Export.cpp @@ -0,0 +1,138 @@ +#include "prx/libc/include/general/VabiMacros.hpp" +#include +#include +#include +#include +#include +#include +#include +#include +#include + +extern "C" { +#include +} + +namespace { + +struct Decoder { + std::uint32_t context; + int channels; + AVCodecContext* codec = nullptr; + ~Decoder() { avcodec_free_context(&codec); } +}; + +std::mutex mutex; +std::unordered_map contexts; +std::unordered_map> decoders; +std::uint32_t nextContext = 1; + +int PacketSamples(const std::uint8_t* packet, int size) { + const unsigned config = packet[0] >> 3; + const int frameSamples = config < 12 ? ((config & 3) == 3 ? 2880 : 480 << (config & 3)) + : config < 16 ? 480 << (config & 1) : 120 << (config & 3); + int frames = 1; + switch (packet[0] & 3) { + case 1: + case 2: frames = 2; break; + case 3: + if (size < 2) throw std::runtime_error("OpusDec: truncated packet frame count"); + frames = packet[1] & 63; + break; + } + if (frames == 0 || frames * frameSamples > 5760) + throw std::runtime_error("OpusDec: invalid packet duration"); + return frames * frameSamples; +} + +} // namespace + +extern "C" { + +int APS5_VABI sceOpusDecInitialize(std::uint32_t* context) { + if (!context) throw std::runtime_error("OpusDec: null context output"); + std::lock_guard lock(mutex); + for (const auto& [id, owner] : contexts) + if (owner == context) throw std::runtime_error("OpusDec: context already initialized"); + if (nextContext == 0) throw std::runtime_error("OpusDec: context identifiers exhausted"); + const auto id = nextContext++; + contexts.emplace(id, context); + *context = id; + return 0; +} + +int APS5_VABI sceOpusDecTerminate(std::uint32_t* context) { + if (!context) throw std::runtime_error("OpusDec: null context"); + std::lock_guard lock(mutex); + if (!contexts.contains(*context)) throw std::runtime_error("OpusDec: unknown context"); + for (const auto& [state, decoder] : decoders) + if (decoder->context == *context) throw std::runtime_error("OpusDec: termination with live decoders is not implemented"); + contexts.erase(*context); + return 0; +} + +int APS5_VABI sceOpusDecGetSize(int channels) { + if (channels != 1 && channels != 2) throw std::runtime_error("OpusDec: only mono and stereo are supported"); + return 640; +} + +int APS5_VABI sceOpusDecCreateEx(std::uint32_t* context, void* state, int sampleRate, int channels) { + if (!context || !state) throw std::runtime_error("OpusDec: null context or state"); + sceOpusDecGetSize(channels); + if (sampleRate != 48000) throw std::runtime_error("OpusDec: sample rates other than 48000 Hz are not implemented"); + std::lock_guard lock(mutex); + if (!contexts.contains(*context)) throw std::runtime_error("OpusDec: unknown context"); + if (decoders.contains(state)) throw std::runtime_error("OpusDec: state already owns a decoder"); + auto decoder = std::make_unique(); + decoder->context = *context; + decoder->channels = channels; + const auto* codec = avcodec_find_decoder(AV_CODEC_ID_OPUS); + if (!codec) throw std::runtime_error("OpusDec: FFmpeg Opus decoder unavailable"); + decoder->codec = avcodec_alloc_context3(codec); + if (!decoder->codec) throw std::bad_alloc(); + av_channel_layout_default(&decoder->codec->ch_layout, channels); + decoder->codec->sample_rate = 48000; + if (avcodec_open2(decoder->codec, codec, nullptr) < 0) + throw std::runtime_error("OpusDec: cannot open FFmpeg decoder"); + decoders.emplace(state, std::move(decoder)); + return 0; +} + +int APS5_VABI sceOpusDecDecode(void* state, const std::uint8_t* data, int bytes, std::int16_t* pcm, int capacity) { + if (!data || bytes <= 0 || bytes > std::numeric_limits::max() - AV_INPUT_BUFFER_PADDING_SIZE || !pcm || capacity < 0) + throw std::runtime_error("OpusDec: invalid packet or PCM buffer; packet loss concealment is not implemented"); + std::lock_guard lock(mutex); + const auto found = decoders.find(state); + if (found == decoders.end()) throw std::runtime_error("OpusDec: unknown decoder state"); + auto& decoder = *found->second; + const int samples = PacketSamples(data, bytes); + const int outputBytes = samples * decoder.channels * 2; + if (capacity / 2 * 2 < outputBytes) throw std::runtime_error("OpusDec: PCM buffer is too small"); + std::unique_ptr packet(av_packet_alloc(), [](AVPacket* p) { av_packet_free(&p); }); + std::unique_ptr frame(av_frame_alloc(), [](AVFrame* f) { av_frame_free(&f); }); + if (!packet || !frame || av_new_packet(packet.get(), bytes) < 0) throw std::bad_alloc(); + std::memcpy(packet->data, data, bytes); + if (avcodec_send_packet(decoder.codec, packet.get()) < 0 || avcodec_receive_frame(decoder.codec, frame.get()) < 0) + throw std::runtime_error("OpusDec: FFmpeg rejected the packet"); + const auto format = static_cast(frame->format); + if (frame->nb_samples != samples || frame->ch_layout.nb_channels != decoder.channels || frame->sample_rate != 48000 || + (format != AV_SAMPLE_FMT_FLTP && format != AV_SAMPLE_FMT_FLT)) + throw std::runtime_error("OpusDec: unexpected FFmpeg PCM format"); + for (int sample = 0; sample < samples; ++sample) { + for (int channel = 0; channel < decoder.channels; ++channel) { + const auto* source = reinterpret_cast(frame->extended_data[format == AV_SAMPLE_FMT_FLTP ? channel : 0]); + const float value = source[format == AV_SAMPLE_FMT_FLTP ? sample : sample * decoder.channels + channel]; + const auto converted = static_cast(std::lrint(std::clamp(value * 32768.0f, -32768.0f, 32767.0f))); + std::memcpy(reinterpret_cast(pcm) + (sample * decoder.channels + channel) * 2, &converted, 2); + } + } + return outputBytes; +} + +int APS5_VABI sceOpusDecDestroy(void* state) { + std::lock_guard lock(mutex); + if (decoders.erase(state) == 0) throw std::runtime_error("OpusDec: unknown decoder state"); + return 0; +} + +} diff --git a/core/libs/prx/libScePad/Export.cpp b/core/libs/prx/libScePad/Export.cpp index 26bc9347e..7dc22890e 100644 --- a/core/libs/prx/libScePad/Export.cpp +++ b/core/libs/prx/libScePad/Export.cpp @@ -268,9 +268,10 @@ int APS5_VABI scePadIsRemoteController(int handle, bool* remote) { } -int APS5_VABI scePadSetAngularVelocityBiasCorrectionState() { - NotImplemented_nid_no_patch(__func__); - return 0; +int APS5_VABI scePadSetAngularVelocityBiasCorrectionState(int handle, bool enabled) { + if (handle != PAD_HANDLE) return PAD_ERROR_INVALID_HANDLE; + if (enabled) NotImplemented_nid_no_patch(__func__); + return PAD_OK; } } diff --git a/core/libs/prx/libScePosixForWebKit/CMakeLists.txt b/core/libs/prx/libScePosixForWebKit/CMakeLists.txt index 5ab1740e0..c786173f6 100644 --- a/core/libs/prx/libScePosixForWebKit/CMakeLists.txt +++ b/core/libs/prx/libScePosixForWebKit/CMakeLists.txt @@ -1,16 +1,29 @@ -add_library(libScePosixForWebKit SHARED EXCLUDE_FROM_ALL Export.cpp Resolver.cpp) +add_library(libScePosixForWebKit SHARED EXCLUDE_FROM_ALL Export.cpp Resolver.cpp Filesystem.cpp) target_include_directories(libScePosixForWebKit PRIVATE ${LIBS_INCLUDE_DIR}) +target_link_libraries(libScePosixForWebKit PRIVATE libSceNet) if(WIN32) target_link_libraries(libScePosixForWebKit PRIVATE ws2_32) endif() configure_windows_unwind(libScePosixForWebKit) if(BUILD_TESTING) + add_test_executable(guest_posix_filesystem_tests ${LIBS_INCLUDE_DIR}/tests/GuestPosixFilesystem.cpp) + target_include_directories(guest_posix_filesystem_tests PRIVATE ${LIBS_INCLUDE_DIR}) + target_link_libraries(guest_posix_filesystem_tests PRIVATE libScePosixForWebKit libkernel libc) + configure_windows_unwind(guest_posix_filesystem_tests) + add_test(NAME guest_posix_filesystem COMMAND guest_posix_filesystem_tests) + set_tests_properties(guest_posix_filesystem PROPERTIES TIMEOUT 30) + if(WIN32) + set_tests_properties(guest_posix_filesystem PROPERTIES ENVIRONMENT_MODIFICATION + "PATH=path_list_prepend:$") + endif() add_test_executable(guest_resolver_tests ${LIBS_INCLUDE_DIR}/tests/GuestResolver.cpp) target_include_directories(guest_resolver_tests PRIVATE ${LIBS_INCLUDE_DIR}) - target_link_libraries(guest_resolver_tests PRIVATE libScePosixForWebKit) + target_link_libraries(guest_resolver_tests PRIVATE libScePosixForWebKit libSceNet libc) + configure_windows_unwind(guest_resolver_tests) add_test(NAME guest_resolver COMMAND guest_resolver_tests) if(WIN32) + target_link_libraries(guest_resolver_tests PRIVATE ws2_32) set_tests_properties(guest_resolver PROPERTIES ENVIRONMENT_MODIFICATION "PATH=path_list_prepend:$") endif() diff --git a/core/libs/prx/libScePosixForWebKit/Filesystem.cpp b/core/libs/prx/libScePosixForWebKit/Filesystem.cpp new file mode 100644 index 000000000..b5245f5b1 --- /dev/null +++ b/core/libs/prx/libScePosixForWebKit/Filesystem.cpp @@ -0,0 +1,85 @@ +#include "prx/libc/include/GuestArena.hpp" +#include "prx/libc/include/general/VabiMacros.hpp" +#include "prx/libkernel/File/include/FileFlags.hpp" +#include "prx/libkernel/Socket/include/SocketRuntime.hpp" +#include +#include +#include +#include +#ifdef _WIN32 +#include +#include +extern "C" _invalid_parameter_handler _set_thread_local_invalid_parameter_handler(_invalid_parameter_handler); +#else +#include +#endif + +extern "C" { +int* APS5_VABI __error_nid_postfix(); +int APS5_VABI open_nid_postfix(const char*, int, int); +} + +namespace { +constexpr char Characters[] = "0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz"; + +int Failure(int error) { + *__error_nid_postfix() = error; + return -1; +} + +#ifdef _WIN32 +void IgnoreInvalidParameter(const wchar_t*, const wchar_t*, const wchar_t*, unsigned int, std::uintptr_t) {} +#endif +} + +extern "C" int APS5_VABI mkstemp_nid_postfix(char* pattern) { + if (!pattern) return Failure(14); + const auto length = std::strlen(pattern); + if (length == 0) return Failure(22); + if (length >= 1024) return Failure(63); + auto start = length; + while (start > 0 && pattern[start - 1] == 'X') --start; + const GuestArena::HostWrite destination(pattern + start, length - start); + if (!destination.Open()) return Failure(14); + if (start != length) { + std::random_device random; + std::uniform_int_distribution character(0, sizeof(Characters) - 2); + for (auto position = start; position < length; ++position) + pattern[position] = Characters[character(random)]; + } + const std::string initial(pattern + start, length - start); + for (;;) { + const int descriptor = open_nid_postfix(pattern, SCE_KERNEL_O_RDWR | SCE_KERNEL_O_CREAT | SCE_KERNEL_O_EXCL, 0600); + if (descriptor >= 0 || *__error_nid_postfix() != 17) return descriptor; + auto position = start; + for (; position < length; ++position) { + const auto index = std::strchr(Characters, pattern[position]) - Characters; + pattern[position] = Characters[(index + 1) % (sizeof(Characters) - 1)]; + if (pattern[position] != initial[position - start]) break; + } + if (position == length) return -1; + } +} + +extern "C" int APS5_VABI isatty_nid_postfix(int descriptor) { + if (descriptor >= GuestSockets::FirstDescriptor) { + Failure(GuestSocketIsOpen_nid_no_patch(descriptor) ? 25 : 9); + return 0; + } +#ifdef _WIN32 + const auto previous = _set_thread_local_invalid_parameter_handler(IgnoreInvalidParameter); + const auto handle = reinterpret_cast(::_get_osfhandle(descriptor)); + _set_thread_local_invalid_parameter_handler(previous); + if (handle == INVALID_HANDLE_VALUE) { + Failure(9); + return 0; + } + DWORD mode = 0; + if (::GetConsoleMode(handle, &mode)) return 1; + Failure(25); +#else + if (::isatty(descriptor)) return 1; + Failure(errno == EBADF ? 9 : 25); +#endif + return 0; +} diff --git a/core/libs/prx/libScePosixForWebKit/GuestResolver.hpp b/core/libs/prx/libScePosixForWebKit/GuestResolver.hpp index d14d4524e..7ed54e716 100644 --- a/core/libs/prx/libScePosixForWebKit/GuestResolver.hpp +++ b/core/libs/prx/libScePosixForWebKit/GuestResolver.hpp @@ -3,6 +3,18 @@ #include namespace GuestResolver { +struct HostEntry { + char* name; + char** aliases; + std::int32_t family; + std::int32_t length; + char** addresses; +}; +static_assert(sizeof(HostEntry) == 32); +static_assert(offsetof(HostEntry, family) == 16); +static_assert(offsetof(HostEntry, length) == 20); +static_assert(offsetof(HostEntry, addresses) == 24); + struct AddressInfo { std::int32_t flags; std::int32_t family; diff --git a/core/libs/prx/libScePosixForWebKit/Resolver.cpp b/core/libs/prx/libScePosixForWebKit/Resolver.cpp index 5952e6276..7c5254fee 100644 --- a/core/libs/prx/libScePosixForWebKit/Resolver.cpp +++ b/core/libs/prx/libScePosixForWebKit/Resolver.cpp @@ -9,11 +9,68 @@ #endif #include "prx/libc/include/general/VabiMacros.hpp" #include +#include #include #include +#include +#include +#include +#include #include "GuestResolver.hpp" +extern "C" int* APS5_VABI sceNetErrnoLoc(); + namespace { +struct HostLookup { + GuestResolver::HostEntry entry{}; + std::string name; + std::vector aliasNames; + std::vector aliases; + std::vector addressBytes; + std::vector addresses; +}; + +bool NumericAddress(const char* name, std::array& address) { + std::array parts{}; + std::size_t count = 0; + std::uint64_t value; + bool digit; + for (;;) { + if (*name < '0' || *name > '9') return false; + value = 0; + int base = 10; + digit = false; + if (*name == '0') { + ++name; + if (*name == 'x' || *name == 'X') { base = 16; ++name; } + else { base = 8; digit = true; } + } + for (;;) { + const int next = *name >= '0' && *name <= '9' ? *name - '0' : + *name >= 'a' && *name <= 'f' ? *name - 'a' + 10 : + *name >= 'A' && *name <= 'F' ? *name - 'A' + 10 : -1; + if (next < 0 || next >= base) break; + value = value * base + next; + digit = true; + ++name; + } + if (*name != '.') break; + if (count == parts.size() || value > 255) return false; + parts[count++] = static_cast(value); + ++name; + } + if (!digit) return false; + if (*name != '\0' && *name != ' ' && *name != '\t' && *name != '\n' && + *name != '\r' && *name != '\f' && *name != '\v') return false; + if (count && value >= (std::uint64_t{1} << (8 * (4 - count)))) return false; + auto result = static_cast(value); + for (std::size_t i = 0; i < count; ++i) + result |= static_cast(parts[i]) << (24 - 8 * i); + for (std::size_t i = 0; i < address.size(); ++i) + address[i] = static_cast(result >> (24 - 8 * i)); + return true; +} + bool Ready() { #ifdef _WIN32 static const int status = [] { WSADATA data{}; return WSAStartup(MAKEWORD(2, 2), &data); }(); @@ -41,6 +98,61 @@ int GuestError(int error) { } extern "C" { +GuestResolver::HostEntry* APS5_VABI gethostbyname_nid_postfix(const char* name) { + if (!name || !Ready()) { *sceNetErrnoLoc() = 3; return nullptr; } + try { + std::array numericAddress{}; + char* numericAliases[]{nullptr}; + char* numericAddresses[]{reinterpret_cast(numericAddress.data()), nullptr}; + hostent nativeEntry{const_cast(name), numericAliases, AF_INET, 4, numericAddresses}; + hostent* native = &nativeEntry; + const bool numeric = NumericAddress(name, numericAddress); +#ifdef _WIN32 + if (!numeric) native = ::gethostbyname(name); + if (!native) { + const auto error = WSAGetLastError(); + *sceNetErrnoLoc() = error == WSAHOST_NOT_FOUND ? 1 : error == WSATRY_AGAIN ? 2 : error == WSANO_DATA ? 4 : 3; + return nullptr; + } +#else + std::vector buffer; + int hostError = 0; + int error = 0; + if (!numeric) { + buffer.resize(1024); + while ((error = ::gethostbyname_r(name, &nativeEntry, buffer.data(), buffer.size(), &native, &hostError)) == ERANGE) + buffer.resize(buffer.size() * 2); + } + if (error != 0 || !native) { + *sceNetErrnoLoc() = hostError != 0 ? hostError : 3; + if (hostError == -1) errno = error == EAGAIN ? 35 : error; + return nullptr; + } +#endif + static thread_local HostLookup lookup; + lookup.name = native->h_name; + lookup.aliasNames.clear(); + lookup.aliases.clear(); + lookup.addressBytes.clear(); + lookup.addresses.clear(); + for (auto** alias = native->h_aliases; *alias; ++alias) lookup.aliasNames.emplace_back(*alias); + for (auto& alias : lookup.aliasNames) lookup.aliases.push_back(alias.data()); + lookup.aliases.push_back(nullptr); + for (auto** address = native->h_addr_list; *address; ++address) + lookup.addressBytes.emplace_back(*address, native->h_length); + for (auto& address : lookup.addressBytes) lookup.addresses.push_back(address.data()); + lookup.addresses.push_back(nullptr); + lookup.entry = {lookup.name.data(), lookup.aliases.data(), native->h_addrtype == AF_INET6 ? 28 : 2, + native->h_length, lookup.addresses.data()}; + if (numeric) *sceNetErrnoLoc() = 0; + return &lookup.entry; + } catch (const std::bad_alloc&) { + *sceNetErrnoLoc() = -1; + errno = ENOMEM; + return nullptr; + } +} + void APS5_VABI freeaddrinfo_nid_postfix(GuestResolver::AddressInfo* first) { while (first) { auto* next = first->next; diff --git a/core/libs/prx/libScePsml/CMakeLists.txt b/core/libs/prx/libScePsml/CMakeLists.txt new file mode 100644 index 000000000..121b19bff --- /dev/null +++ b/core/libs/prx/libScePsml/CMakeLists.txt @@ -0,0 +1,2 @@ +include(${CMAKE_CURRENT_SOURCE_DIR}/../libScePsml_debug/PsmlLibrary.cmake) +add_sce_psml_library(libScePsml) diff --git a/core/libs/prx/libScePsml_debug/CMakeLists.txt b/core/libs/prx/libScePsml_debug/CMakeLists.txt index 3fd929d07..ae4867fdc 100644 --- a/core/libs/prx/libScePsml_debug/CMakeLists.txt +++ b/core/libs/prx/libScePsml_debug/CMakeLists.txt @@ -1,6 +1,5 @@ -add_library(libScePsml_debug SHARED EXCLUDE_FROM_ALL Export.cpp) -target_include_directories(libScePsml_debug PRIVATE ${LIBS_INCLUDE_DIR}) -configure_windows_unwind(libScePsml_debug) +include(${CMAKE_CURRENT_SOURCE_DIR}/PsmlLibrary.cmake) +add_sce_psml_library(libScePsml_debug) if(BUILD_TESTING) add_test_executable(guest_psml_tests ${LIBS_INCLUDE_DIR}/tests/GuestPsml.cpp) diff --git a/core/libs/prx/libScePsml_debug/Export.cpp b/core/libs/prx/libScePsml_debug/Export.cpp index 3eaf69d18..d85faa0ae 100644 --- a/core/libs/prx/libScePsml_debug/Export.cpp +++ b/core/libs/prx/libScePsml_debug/Export.cpp @@ -75,6 +75,11 @@ int APS5_VABI scePsmlMfsrIsCaptureInProgress() { return 0; } +int APS5_VABI scePsmlMfsrGetDispatchMfsrPacket1000() { + NotImplemented_nid_no_patch(__func__); + return 0; +} + int APS5_VABI scePsmlMfsrGetDispatchMfsrPacket900() { NotImplemented_nid_no_patch(__func__); return 0; diff --git a/core/libs/prx/libScePsml_debug/PsmlLibrary.cmake b/core/libs/prx/libScePsml_debug/PsmlLibrary.cmake new file mode 100644 index 000000000..69fe16dbf --- /dev/null +++ b/core/libs/prx/libScePsml_debug/PsmlLibrary.cmake @@ -0,0 +1,5 @@ +function(add_sce_psml_library target) + add_library(${target} SHARED EXCLUDE_FROM_ALL ${CMAKE_CURRENT_FUNCTION_LIST_DIR}/Export.cpp) + target_include_directories(${target} PRIVATE ${LIBS_INCLUDE_DIR}) + configure_windows_unwind(${target}) +endfunction() diff --git a/core/libs/prx/libSceRtc/Export.cpp b/core/libs/prx/libSceRtc/Export.cpp index 7772cb1fb..2e213f70b 100644 --- a/core/libs/prx/libSceRtc/Export.cpp +++ b/core/libs/prx/libSceRtc/Export.cpp @@ -376,12 +376,16 @@ int APS5_VABI sceRtcGetCurrentClockLocalTime(RtcDateTime* time) { int APS5_VABI sceRtcConvertUtcToLocalTime(const RtcTick* utc, RtcTick* local_time) { if (!utc || !local_time) return SCE_RTC_ERROR_INVALID_POINTER; + if (utc->tick > MAX_TICK) return SCE_RTC_ERROR_INVALID_VALUE; return addTicks(local_time, utc, localOffsetSeconds(utc->tick), TICKS_PER_SECOND); } int APS5_VABI sceRtcConvertLocalTimeToUtc(const RtcTick* local_time, RtcTick* utc) { if (!local_time || !utc) return SCE_RTC_ERROR_INVALID_POINTER; - const std::int64_t offset = localOffsetSeconds(local_time->tick - static_cast(localOffsetSeconds(local_time->tick) * TICKS_PER_SECOND)); + if (local_time->tick > MAX_TICK) return SCE_RTC_ERROR_INVALID_VALUE; + RtcTick estimate{}; + if (const int result = addTicks(&estimate, local_time, -localOffsetSeconds(local_time->tick), TICKS_PER_SECOND); result != 0) return result; + const std::int64_t offset = localOffsetSeconds(estimate.tick); return addTicks(utc, local_time, -offset, TICKS_PER_SECOND); } @@ -460,6 +464,7 @@ int APS5_VABI sceRtcFormatRFC3339(char* date_time, const RtcTick* utc, int time_ int APS5_VABI sceRtcFormatRFC3339LocalTime(char* date_time, const RtcTick* utc) { if (!date_time || !utc) return SCE_RTC_ERROR_INVALID_POINTER; + if (utc->tick > MAX_TICK) return SCE_RTC_ERROR_INVALID_VALUE; return sceRtcFormatRFC3339(date_time, utc, localOffsetMinutes(*utc)); } @@ -477,6 +482,7 @@ int APS5_VABI sceRtcFormatRFC2822(char* date_time, const RtcTick* utc, int time_ int APS5_VABI sceRtcFormatRFC2822LocalTime(char* date_time, const RtcTick* utc) { if (!date_time || !utc) return SCE_RTC_ERROR_INVALID_POINTER; + if (utc->tick > MAX_TICK) return SCE_RTC_ERROR_INVALID_VALUE; return sceRtcFormatRFC2822(date_time, utc, localOffsetMinutes(*utc)); } diff --git a/core/libs/prx/libSceRudp/Export.cpp b/core/libs/prx/libSceRudp/Export.cpp index 7b8eb5082..efe18931f 100644 --- a/core/libs/prx/libSceRudp/Export.cpp +++ b/core/libs/prx/libSceRudp/Export.cpp @@ -47,14 +47,17 @@ int APS5_VABI sceRudpInit_nid_postfix(void* mem_pool, int mem_pool_size) { } int APS5_VABI sceRudpActivate() { - NotImplemented_nid_no_patch(__func__); return 0; } int APS5_VABI sceRudpGetStatus(void* status, std::size_t size) { - (void)status; - (void)size; - NotImplemented_nid_no_patch(__func__); + std::lock_guard lk(g_mutex); + if (!g_inited) { + return RUDP_ERROR_NOT_INITIALIZED; + } + if (status != nullptr && size != 0) { + std::memset(status, 0, size); // state 0 = idle, no connections + } return 0; } diff --git a/core/libs/prx/libSceSaveData.native/CMakeLists.txt b/core/libs/prx/libSceSaveData.native/CMakeLists.txt index 496c0b9bc..3b596955e 100644 --- a/core/libs/prx/libSceSaveData.native/CMakeLists.txt +++ b/core/libs/prx/libSceSaveData.native/CMakeLists.txt @@ -40,4 +40,14 @@ if(BUILD_TESTING) set_tests_properties(guest_savedata_ps4 PROPERTIES ENVIRONMENT_MODIFICATION "PATH=path_list_prepend:$") endif() + + add_test_executable(guest_savedata_cancel_tests ${LIBS_INCLUDE_DIR}/tests/GuestSaveDataCancel.cpp) + target_include_directories(guest_savedata_cancel_tests PRIVATE ${LIBS_INCLUDE_DIR}) + target_link_libraries(guest_savedata_cancel_tests PRIVATE libSceSaveData.native libc) + configure_windows_unwind(guest_savedata_cancel_tests) + add_test(NAME guest_savedata_cancel COMMAND guest_savedata_cancel_tests) + if(WIN32) + set_tests_properties(guest_savedata_cancel PROPERTIES ENVIRONMENT_MODIFICATION + "PATH=path_list_prepend:$") + endif() endif() diff --git a/core/libs/prx/libSceSaveData.native/Export.cpp b/core/libs/prx/libSceSaveData.native/Export.cpp index f6270b5dd..4f96392ed 100644 --- a/core/libs/prx/libSceSaveData.native/Export.cpp +++ b/core/libs/prx/libSceSaveData.native/Export.cpp @@ -71,7 +71,7 @@ namespace { // with a small .param sidecar holding the last SaveDataParam the title wrote. Keying by user as well as slot // keeps two users' in-memory saves from colliding on the same console. constexpr char MEM_DIR[] = "_sd_mem"; -constexpr std::size_t MEM_MAX_SIZE = 0x1000000; // 16 MiB +constexpr std::size_t MEM_MAX_SIZE = 32u * 1024u * 1024u; std::mutex g_mem_mutex; std::string mem_path(std::int32_t user_id, std::uint32_t slot, const char* ext) { @@ -140,7 +140,7 @@ SaveDataParam load_param(const std::string& real_path) { std::error_code ec; const auto written = std::filesystem::last_write_time(real_path, ec); if (!ec) { - const auto system = std::chrono::clock_cast(written); + const auto system = std::filesystem::file_time_type::clock::to_sys(written); param.mtime = std::chrono::duration_cast(system.time_since_epoch()).count(); } } @@ -749,4 +749,9 @@ int APS5_VABI sceSaveDataGetConvertProgress() { NotImplemented_nid_no_patch(__func__); return 0; } + +int APS5_VABI sceSaveDataCancel() { + NotImplemented_nid_no_patch(__func__); + return 0; +} } diff --git a/core/libs/prx/libSceShare/Export.cpp b/core/libs/prx/libSceShare/Export.cpp index 4545d4f8e..0c609afb1 100644 --- a/core/libs/prx/libSceShare/Export.cpp +++ b/core/libs/prx/libSceShare/Export.cpp @@ -118,8 +118,10 @@ int APS5_VABI sceShareGetRunningStatus(uint32_t* status) { return 0; } -int APS5_VABI sceShareSetContentParamForApplicationTitle(void) { - NotImplemented_nid_no_patch(__func__); +int APS5_VABI sceShareSetContentParamForApplicationTitle(const char* application_title) { + if (application_title == nullptr) { + return ERROR_INVALID_PARAM; + } return 0; } diff --git a/core/libs/prx/libSceSysmodule/Export.cpp b/core/libs/prx/libSceSysmodule/Export.cpp index 85b64a2ed..83f56fb54 100644 --- a/core/libs/prx/libSceSysmodule/Export.cpp +++ b/core/libs/prx/libSceSysmodule/Export.cpp @@ -18,6 +18,8 @@ namespace { +constexpr int SCE_SYSMODULE_ERROR_UNLOADED = static_cast(0x805A1001); + const char* findModuleName(const std::uint32_t id) { const auto it = kModuleTable.find(id); return it != kModuleTable.end() ? it->second : nullptr; @@ -112,7 +114,7 @@ int APS5_VABI sceSysmoduleIsLoaded(std::uint16_t id) { std::lock_guard lock(gMutex); auto it = gLoadCount.find(id); if (it == gLoadCount.end() || it->second < 1) { - return 0x80A90002; + return SCE_SYSMODULE_ERROR_UNLOADED; } return 0; } @@ -160,7 +162,7 @@ int APS5_VABI sceSysmoduleUnloadModule(std::uint16_t id) { std::lock_guard lock(gMutex); auto it = gLoadCount.find(id); if (it == gLoadCount.end() || it->second < 1) { - return 0x80A90003; + return SCE_SYSMODULE_ERROR_UNLOADED; } it->second--; return 0; @@ -188,7 +190,7 @@ int APS5_VABI sceSysmoduleUnloadModuleInternal(std::uint32_t id) { std::lock_guard lock(gMutex); auto it = gLoadCount.find(id); if (it == gLoadCount.end() || it->second < 1) { - return 0x80A90003; + return SCE_SYSMODULE_ERROR_UNLOADED; } it->second--; return 0; diff --git a/core/libs/prx/libSceSystemService/Export.cpp b/core/libs/prx/libSceSystemService/Export.cpp index c3c3c4709..ab607a86e 100644 --- a/core/libs/prx/libSceSystemService/Export.cpp +++ b/core/libs/prx/libSceSystemService/Export.cpp @@ -5,6 +5,7 @@ #include "prx/libc/include/Shutdown.hpp" #include "SceTypes.hpp" #include "prx/libc/include/General.hpp" +#include "prx/libkernel/KernelErrors.hpp" #include "prx/libSceSystemService/SystemService.hpp" extern "C" { @@ -133,6 +134,12 @@ int APS5_VABI sceSystemServiceLaunchPlayerDialog(const void* param) { return SYSTEM_SERVICE_OK; } +int APS5_VABI sceSystemServiceLaunchWebBrowser(const char* uri, void* param) { + (void)uri; + (void)param; + return SCE_KERNEL_ERROR_EOPNOTSUPP; +} + int APS5_VABI sceSystemServiceDisableMusicPlayer(void) { return SYSTEM_SERVICE_OK; } diff --git a/core/libs/prx/libSceUlt/CMakeLists.txt b/core/libs/prx/libSceUlt/CMakeLists.txt index e9ce2a507..c9005a3cb 100644 --- a/core/libs/prx/libSceUlt/CMakeLists.txt +++ b/core/libs/prx/libSceUlt/CMakeLists.txt @@ -20,7 +20,7 @@ if(BUILD_TESTING) target_include_directories(guest_ult_thread_tests PRIVATE ${LIBS_INCLUDE_DIR}) target_link_libraries(guest_ult_thread_tests PRIVATE libSceUlt libc) configure_windows_unwind(guest_ult_thread_tests) - foreach(testCase IN ITEMS outstanding validation) + foreach(testCase IN ITEMS outstanding validation tryjoin destroy) add_test(NAME guest_ult_thread_${testCase} COMMAND guest_ult_thread_tests ${testCase}) set_tests_properties(guest_ult_thread_${testCase} PROPERTIES TIMEOUT 20) if(WIN32) diff --git a/core/libs/prx/libSceUlt/Export.cpp b/core/libs/prx/libSceUlt/Export.cpp index 979033e5a..d87bd7e5f 100644 --- a/core/libs/prx/libSceUlt/Export.cpp +++ b/core/libs/prx/libSceUlt/Export.cpp @@ -108,7 +108,9 @@ void ReleaseQueue(UltQueueState& state) { void* APS5_VABI ulthreadRunner(void* arg) { auto* state = static_cast(arg); - return reinterpret_cast(static_cast(state->_entry(state->_arg))); + const auto result = state->_entry(state->_arg); + state->_exited.store(true, std::memory_order_release); + return reinterpret_cast(static_cast(result)); } } @@ -489,6 +491,7 @@ int APS5_VABI sceUltUlthreadCreate(void* ulthread, const char* name, UltUlthread auto state = std::make_shared(); state->_entry = entry; state->_arg = arg; + state->_runtime = runtime; { std::lock_guard lock(gMutex); if (gRuntimes.find(runtime) == gRuntimes.end()) { @@ -534,4 +537,51 @@ int APS5_VABI sceUltUlthreadJoin(void* ulthread, std::int32_t* status) { return ULT_OK; } +int APS5_VABI sceUltUlthreadTryJoin(void* ulthread, std::int32_t* status) { + if (ulthread == nullptr) { + return ULT_ERROR_NULL; + } + { + std::lock_guard lock(gMutex); + auto it = gUlthreads.find(ulthread); + if (it == gUlthreads.end()) { + return ULT_ERROR_STATE; + } + if (!it->second->_exited.load(std::memory_order_acquire)) { + return ULT_ERROR_BUSY; + } + } + return sceUltUlthreadJoin(ulthread, status); +} + +int APS5_VABI sceUltUlthreadRuntimeDestroy(void* runtime) { + if (runtime == nullptr) { + return ULT_ERROR_NULL; + } + std::lock_guard lock(gMutex); + auto it = gRuntimes.find(runtime); + if (it == gRuntimes.end()) { + return ULT_ERROR_STATE; + } + for (const auto& entry : gUlthreads) { + if (entry.second->_runtime == runtime) { + return ULT_ERROR_BUSY; + } + } + gRuntimes.erase(it); + return ULT_OK; +} + +int APS5_VABI _sceUltUlthreadRuntimeOptParamInitialize(UltUlthreadRuntimeOptParam* optParam, std::uint32_t buildVersion) { + return sceUltUlthreadRuntimeOptParamInitialize(optParam, buildVersion); +} + +int APS5_VABI _sceUltUlthreadRuntimeCreate(void* runtime, const char* name, std::uint32_t maxNumUlthread, std::uint32_t numWorkerThread, void* workArea, const void* optParam, std::uint32_t buildVersion) { + return sceUltUlthreadRuntimeCreate(runtime, name, maxNumUlthread, numWorkerThread, workArea, optParam, buildVersion); +} + +int APS5_VABI _sceUltUlthreadCreate(void* ulthread, const char* name, UltUlthreadEntry entry, std::uint64_t arg, void* context, std::uint64_t sizeContext, void* runtime, const void* optParam, std::uint32_t buildVersion) { + return sceUltUlthreadCreate(ulthread, name, entry, arg, context, sizeContext, runtime, optParam, buildVersion); +} + } diff --git a/core/libs/prx/libSceUlt/SceUltTypes.hpp b/core/libs/prx/libSceUlt/SceUltTypes.hpp index 1a8a0c8d5..0a314a9d5 100644 --- a/core/libs/prx/libSceUlt/SceUltTypes.hpp +++ b/core/libs/prx/libSceUlt/SceUltTypes.hpp @@ -1,6 +1,7 @@ #ifndef CORE_LIBS_PRX_LIBSCEULT_SCEULTTYPES_HPP #define CORE_LIBS_PRX_LIBSCEULT_SCEULTTYPES_HPP +#include #include #include #include @@ -70,6 +71,8 @@ struct UltUlthreadState { UltUlthreadEntry _entry = nullptr; std::uint64_t _arg = 0; Pthread _thread = nullptr; + void* _runtime = nullptr; + std::atomic _exited{false}; }; #endif diff --git a/core/libs/prx/libSceVdecsw/CMakeLists.txt b/core/libs/prx/libSceVdecsw/CMakeLists.txt new file mode 100644 index 000000000..6d41dc24e --- /dev/null +++ b/core/libs/prx/libSceVdecsw/CMakeLists.txt @@ -0,0 +1,20 @@ +add_library(libSceVdecsw SHARED EXCLUDE_FROM_ALL + ${CMAKE_CURRENT_SOURCE_DIR}/Export.cpp +) + +target_include_directories(libSceVdecsw PRIVATE ${LIBS_INCLUDE_DIR}) +configure_windows_unwind(libSceVdecsw) + +target_link_libraries(libSceVdecsw PRIVATE libSceVideodec2) + +if(BUILD_TESTING) + add_test_executable(guest_vdecsw_tests ${LIBS_INCLUDE_DIR}/tests/GuestVdecsw.cpp) + target_include_directories(guest_vdecsw_tests PRIVATE ${LIBS_INCLUDE_DIR}) + target_link_libraries(guest_vdecsw_tests PRIVATE libSceVdecsw libSceVideodec2 libc) + configure_windows_unwind(guest_vdecsw_tests) + add_test(NAME guest_vdecsw COMMAND guest_vdecsw_tests) + if(WIN32) + set_tests_properties(guest_vdecsw PROPERTIES ENVIRONMENT_MODIFICATION + "PATH=path_list_prepend:$") + endif() +endif() diff --git a/core/libs/prx/libSceVdecsw/Export.cpp b/core/libs/prx/libSceVdecsw/Export.cpp index a0391dc7c..d56d38b16 100644 --- a/core/libs/prx/libSceVdecsw/Export.cpp +++ b/core/libs/prx/libSceVdecsw/Export.cpp @@ -1,73 +1,389 @@ -#include +#include #include +#include +#include +#include +#include +#include +#include +#include +#include #include "SceTypes.hpp" #include "prx/libc/include/General.hpp" +namespace { + +constexpr int VdecswErrorOutputPending = static_cast(0x81510115u); +constexpr int VdecswErrorInputQueueEmpty = static_cast(0x81510116u); +constexpr int VdecswErrorDecodePending = static_cast(0x81510117u); +constexpr std::size_t MaxQueuedPictures = 4; + +struct InputData { + std::uint64_t thisSize; + const std::uint8_t* auData; + std::uint64_t auSize; + std::uint64_t ptsData; + std::uint64_t dtsData; + std::uint64_t attachedData; +}; + +struct InputResult { + std::uint64_t thisSize; + const void* decodedAu; + std::uint32_t outputFrameCount; + std::uint32_t reserved; +}; + +struct FrameBuffer { + std::uint64_t thisSize; + void* frameBuffer; + std::uint64_t frameBufferSize; +}; + +struct OutputInfo { + std::uint64_t thisSize; + bool isValid; + bool isLastFrame; + bool isErrorFrame; + std::uint8_t pictureCount; + std::uint32_t codecType; + std::uint32_t frameWidth; + std::uint32_t framePitch; + std::uint32_t frameHeight; + bool isDiscardedFrame; + void* frameBuffer; + std::uint64_t frameBufferSize; + std::uint32_t frameFormat; + std::uint32_t framePitchInBytes; +}; + +struct AvcPictureInfo { + std::uint64_t thisSize; + bool isValid; + std::uint64_t ptsData; + std::uint64_t dtsData; + std::uint64_t attachedData; + std::uint8_t idrPictureFlag; + std::uint8_t profileIdc; + std::uint8_t levelIdc; + std::uint32_t picWidthInMbsMinus1; + std::uint32_t picHeightInMapUnitsMinus1; + std::uint8_t frameMbsOnlyFlag; + std::uint8_t frameCroppingFlag; + std::uint32_t frameCropLeftOffset; + std::uint32_t frameCropRightOffset; + std::uint32_t frameCropTopOffset; + std::uint32_t frameCropBottomOffset; + std::uint8_t vui[44]; +}; + +static_assert(sizeof(InputData) == 0x30 && sizeof(InputResult) == 0x18 && sizeof(FrameBuffer) == 0x18); +static_assert(sizeof(OutputInfo) == 0x38 && offsetof(OutputInfo, isLastFrame) == 9 && offsetof(OutputInfo, pictureCount) == 0xb && offsetof(OutputInfo, isDiscardedFrame) == 0x1c); +static_assert(sizeof(AvcPictureInfo) == 0x78 && offsetof(AvcPictureInfo, picWidthInMbsMinus1) == 0x2c && offsetof(AvcPictureInfo, frameCropBottomOffset) == 0x44); + +struct Videodec2FrameBuffer { + std::uint64_t thisSize; + void* frameBuffer; + std::uint64_t frameBufferSize; + bool isAccepted; +}; + +struct Videodec2OutputInfo { + std::uint64_t thisSize; + bool isValid; + bool isErrorFrame; + std::uint8_t pictureCount; + std::uint32_t codecType; + std::uint32_t frameWidth; + std::uint32_t framePitch; + std::uint32_t frameHeight; + void* frameBuffer; + std::uint64_t frameBufferSize; + std::uint32_t frameFormat; + std::uint32_t framePitchInBytes; +}; + +struct Videodec2AvcPictureInfo { + std::uint64_t thisSize; + bool isValid; + std::uint64_t ptsData; + std::uint64_t dtsData; + std::uint64_t attachedData; + std::uint8_t idrPictureFlag; + std::uint8_t profileIdc; + std::uint8_t levelIdc; + std::uint32_t picWidthInLumaSamples; + std::uint32_t picHeightInLumaSamples; +}; + +struct DecoderMemoryInfo { + std::uint64_t thisSize; + std::uint64_t cpuMemorySize; + void* cpuMemory; + std::uint64_t gpuMemorySize; + void* gpuMemory; + std::uint64_t cpuGpuMemorySize; + void* cpuGpuMemory; + std::uint64_t maxFrameBufferSize; + std::uint32_t frameBufferAlignment; +}; + +} + +extern "C" { +int APS5_VABI sceVideodec2QueryComputeMemoryInfo_nid_postfix(void* info); +int APS5_VABI sceVideodec2AllocateComputeQueue_nid_postfix(const void* config, const void* memory, std::uint64_t* queue); +int APS5_VABI sceVideodec2ReleaseComputeQueue_nid_postfix(std::uint64_t queue); +int APS5_VABI sceVideodec2QueryDecoderMemoryInfo_nid_postfix(const void* config, void* memory); +int APS5_VABI sceVideodec2CreateDecoder_nid_postfix(const void* config, const void* memory, std::uint64_t* handle); +int APS5_VABI sceVideodec2DeleteDecoder_nid_postfix(std::uint64_t handle); +int APS5_VABI sceVideodec2Decode_nid_postfix(std::uint64_t handle, const InputData* input, Videodec2FrameBuffer* frame, Videodec2OutputInfo* output); +int APS5_VABI sceVideodec2Flush_nid_postfix(std::uint64_t handle, Videodec2FrameBuffer* frame, Videodec2OutputInfo* output); +int APS5_VABI sceVideodec2Reset_nid_postfix(std::uint64_t handle); +int APS5_VABI sceVideodec2GetAvcPictureInfo_nid_postfix(const Videodec2OutputInfo* output, Videodec2AvcPictureInfo* first, Videodec2AvcPictureInfo* second); +} + +namespace { + +struct PendingInput { + InputData input; + std::vector data; +}; + +struct ConsumedInput { + const void* au; + std::uint32_t frames; +}; + +struct Picture { + std::vector pixels; + Videodec2OutputInfo output; + AvcPictureInfo info; +}; + +struct Decoder { + std::mutex mutex; + std::uint64_t frameBytes = 0; + std::deque inputs; + std::deque consumed; + std::deque outputs; + std::deque pictures; + std::map delivered; + bool finalized = false; + bool drained = false; +}; + +std::mutex decodersMutex; +std::map> decoders; + +std::shared_ptr findDecoder(std::uint64_t handle) { + std::lock_guard lock(decodersMutex); + const auto found = decoders.find(handle); + if (found == decoders.end()) throw std::runtime_error("Vdecsw: invalid decoder handle"); + return found->second; +} + +AvcPictureInfo pictureInfo(const Videodec2OutputInfo& output) { + Videodec2AvcPictureInfo source{sizeof(Videodec2AvcPictureInfo)}; + sceVideodec2GetAvcPictureInfo_nid_postfix(&output, &source, nullptr); + if (!source.isValid) throw std::runtime_error("Vdecsw: decoded picture has no picture info"); + const auto widthInMbs = (output.frameWidth + 15u) / 16u; + const auto heightInMbs = (output.frameHeight + 15u) / 16u; + AvcPictureInfo info{}; + info.thisSize = sizeof(AvcPictureInfo); + info.isValid = true; + info.ptsData = source.ptsData; + info.dtsData = source.dtsData; + info.attachedData = source.attachedData; + info.idrPictureFlag = source.idrPictureFlag; + info.profileIdc = source.profileIdc; + info.levelIdc = source.levelIdc; + info.picWidthInMbsMinus1 = widthInMbs - 1u; + info.picHeightInMapUnitsMinus1 = heightInMbs - 1u; + info.frameMbsOnlyFlag = 1; + info.frameCroppingFlag = widthInMbs * 16u != output.frameWidth || heightInMbs * 16u != output.frameHeight; + info.frameCropRightOffset = (widthInMbs * 16u - output.frameWidth) / 2u; + info.frameCropBottomOffset = (heightInMbs * 16u - output.frameHeight) / 2u; + return info; +} + +void advance(std::uint64_t handle, Decoder& decoder) { + while (decoder.pictures.size() < MaxQueuedPictures) { + std::vector pixels(decoder.frameBytes); + Videodec2FrameBuffer frame{sizeof(Videodec2FrameBuffer), pixels.data(), pixels.size(), false}; + Videodec2OutputInfo output{sizeof(Videodec2OutputInfo)}; + if (!decoder.inputs.empty()) { + const auto* au = static_cast(decoder.inputs.front().input.auData); + auto input = decoder.inputs.front().input; + input.auData = decoder.inputs.front().data.data(); + sceVideodec2Decode_nid_postfix(handle, &input, &frame, &output); + decoder.inputs.pop_front(); + decoder.consumed.push_back({au, output.isValid ? 1u : 0u}); + } else if (decoder.finalized && !decoder.drained) { + sceVideodec2Flush_nid_postfix(handle, &frame, &output); + decoder.drained = !output.isValid; + } else { + return; + } + if (output.isValid) { + const auto info = pictureInfo(output); + decoder.pictures.push_back({std::move(pixels), output, info}); + } + } +} + +} + extern "C" { -int APS5_VABI sceVdecswCreateDecoder() { - NotImplemented_nid_no_patch(__func__); +int APS5_VABI sceVdecswQueryComputeMemoryInfo(void* info) { + return sceVideodec2QueryComputeMemoryInfo_nid_postfix(info); +} + +int APS5_VABI sceVdecswAllocateComputeQueue(const void* config, const void* memory, std::uint64_t* queue) { + return sceVideodec2AllocateComputeQueue_nid_postfix(config, memory, queue); +} + +int APS5_VABI sceVdecswReleaseComputeQueue(std::uint64_t queue) { + return sceVideodec2ReleaseComputeQueue_nid_postfix(queue); +} + +int APS5_VABI sceVdecswQueryDecoderMemoryInfo(const void* config, void* memory) { + return sceVideodec2QueryDecoderMemoryInfo_nid_postfix(config, memory); +} + +int APS5_VABI sceVdecswCreateDecoder(const void* config, const DecoderMemoryInfo* memory, std::uint64_t* handle) { + if (handle == nullptr || memory == nullptr || memory->thisSize < sizeof(DecoderMemoryInfo) || memory->maxFrameBufferSize == 0) throw std::runtime_error("Vdecsw: invalid decoder creation arguments"); + const auto result = sceVideodec2CreateDecoder_nid_postfix(config, memory, handle); + if (result != 0) return result; + auto decoder = std::make_shared(); + decoder->frameBytes = memory->maxFrameBufferSize; + std::lock_guard lock(decodersMutex); + decoders.emplace(*handle, std::move(decoder)); return 0; } -int APS5_VABI sceVdecswDeleteDecoder() { - NotImplemented_nid_no_patch(__func__); +int APS5_VABI sceVdecswDeleteDecoder(std::uint64_t handle) { + { + std::lock_guard lock(decodersMutex); + if (decoders.erase(handle) == 0) throw std::runtime_error("Vdecsw: invalid decoder handle"); + } + return sceVideodec2DeleteDecoder_nid_postfix(handle); +} + +int APS5_VABI sceVdecswSetDecodeInput(std::uint64_t handle, const InputData* input) { + if (input == nullptr || input->thisSize < sizeof(InputData)) throw std::runtime_error("Vdecsw: invalid decode input"); + const auto decoder = findDecoder(handle); + std::lock_guard lock(decoder->mutex); + if (decoder->finalized) throw std::runtime_error("Vdecsw: decode input after the sequence was finalized"); + if (input->auData == nullptr || input->auSize == 0) throw std::runtime_error("Vdecsw: empty access unit"); + decoder->inputs.push_back({*input, std::vector(input->auData, input->auData + input->auSize)}); + advance(handle, *decoder); return 0; } -int APS5_VABI sceVdecswQueryComputeMemoryInfo() { - NotImplemented_nid_no_patch(__func__); +int APS5_VABI sceVdecswTrySyncDecodeInput(std::uint64_t handle, InputResult* result) { + if (result == nullptr || result->thisSize < sizeof(InputResult)) throw std::runtime_error("Vdecsw: invalid decode input result"); + const auto decoder = findDecoder(handle); + std::lock_guard lock(decoder->mutex); + advance(handle, *decoder); + result->decodedAu = nullptr; + result->outputFrameCount = 0; + result->reserved = 0; + if (!decoder->consumed.empty()) { + result->decodedAu = decoder->consumed.front().au; + result->outputFrameCount = decoder->consumed.front().frames; + decoder->consumed.pop_front(); + return 0; + } + return decoder->inputs.empty() ? VdecswErrorInputQueueEmpty : VdecswErrorDecodePending; +} + +int APS5_VABI sceVdecswSetDecodeOutput(std::uint64_t handle, const FrameBuffer* frame) { + if (frame == nullptr || frame->thisSize < sizeof(FrameBuffer) || frame->frameBuffer == nullptr) throw std::runtime_error("Vdecsw: invalid decode output"); + const auto decoder = findDecoder(handle); + std::lock_guard lock(decoder->mutex); + decoder->outputs.push_back(*frame); return 0; } -int APS5_VABI sceVdecswFinalizeDecodeSequence() { - NotImplemented_nid_no_patch(__func__); +int APS5_VABI sceVdecswTrySyncDecodeOutput(std::uint64_t handle, OutputInfo* output) { + if (output == nullptr || output->thisSize < sizeof(OutputInfo)) throw std::runtime_error("Vdecsw: invalid decode output info"); + const auto decoder = findDecoder(handle); + std::lock_guard lock(decoder->mutex); + advance(handle, *decoder); + if (decoder->outputs.empty()) throw std::runtime_error("Vdecsw: output sync without an output buffer"); + const bool ended = decoder->finalized && decoder->drained; + if (decoder->pictures.size() < (ended ? 1u : 2u)) { + if (!ended) return VdecswErrorOutputPending; + std::memset(reinterpret_cast(output) + sizeof(output->thisSize), 0, sizeof(OutputInfo) - sizeof(output->thisSize)); + output->isLastFrame = true; + decoder->outputs.pop_front(); + return 0; + } + const auto target = decoder->outputs.front(); + auto picture = std::move(decoder->pictures.front()); + decoder->pictures.pop_front(); + if (target.frameBufferSize < picture.output.frameBufferSize) throw std::runtime_error("Vdecsw: output buffer is smaller than the decoded picture"); + std::memcpy(target.frameBuffer, picture.pixels.data(), picture.output.frameBufferSize); + decoder->outputs.pop_front(); + output->isValid = true; + output->isLastFrame = ended && decoder->pictures.empty(); + output->isErrorFrame = picture.output.isErrorFrame; + output->pictureCount = picture.output.pictureCount; + output->codecType = picture.output.codecType; + output->frameWidth = picture.output.frameWidth; + output->framePitch = picture.output.framePitch; + output->frameHeight = picture.output.frameHeight; + output->isDiscardedFrame = false; + output->frameBuffer = target.frameBuffer; + output->frameBufferSize = picture.output.frameBufferSize; + output->frameFormat = picture.output.frameFormat; + output->framePitchInBytes = picture.output.framePitchInBytes; + decoder->delivered[target.frameBuffer] = picture.info; return 0; } -int APS5_VABI sceVdecswQueryDecoderMemoryInfo() { - NotImplemented_nid_no_patch(__func__); +int APS5_VABI sceVdecswGetAvcPictureInfo(const OutputInfo* output, AvcPictureInfo* first, AvcPictureInfo* second) { + if (output == nullptr || first == nullptr || first->thisSize < sizeof(std::uint64_t)) throw std::runtime_error("Vdecsw: invalid argument pointer"); + if (output->pictureCount > 1) throw std::runtime_error("Vdecsw: second field picture info is not implemented"); + if (output->pictureCount == 0) return 0; + static_cast(second); + std::vector> all; + { + std::lock_guard lock(decodersMutex); + for (const auto& [handle, decoder] : decoders) all.push_back(decoder); + } + for (const auto& decoder : all) { + std::lock_guard lock(decoder->mutex); + const auto found = decoder->delivered.find(output->frameBuffer); + if (found == decoder->delivered.end()) continue; + const auto bytes = std::min(first->thisSize, sizeof(AvcPictureInfo)) - sizeof(std::uint64_t); + std::memcpy(reinterpret_cast(first) + sizeof(std::uint64_t), reinterpret_cast(&found->second) + sizeof(std::uint64_t), bytes); + return 0; + } + throw std::runtime_error("Vdecsw: picture info requested for a frame buffer no decoder returned"); +} + +int APS5_VABI sceVdecswFinalizeDecodeSequence(std::uint64_t handle) { + const auto decoder = findDecoder(handle); + std::lock_guard lock(decoder->mutex); + decoder->finalized = true; + advance(handle, *decoder); return 0; } -int APS5_VABI sceVdecswSetDecodeInput() { - NotImplemented_nid_no_patch(__func__); - return 0; -} - -int APS5_VABI sceVdecswReleaseComputeQueue() { - NotImplemented_nid_no_patch(__func__); - return 0; -} - -int APS5_VABI sceVdecswAllocateComputeQueue() { - NotImplemented_nid_no_patch(__func__); - return 0; -} - -int APS5_VABI sceVdecswGetAvcPictureInfo() { - NotImplemented_nid_no_patch(__func__); - return 0; -} - -int APS5_VABI sceVdecswTrySyncDecodeOutput() { - NotImplemented_nid_no_patch(__func__); - return 0; -} - -int APS5_VABI sceVdecswTrySyncDecodeInput() { - NotImplemented_nid_no_patch(__func__); - return 0; -} - -int APS5_VABI sceVdecswSetDecodeOutput() { - NotImplemented_nid_no_patch(__func__); - return 0; -} - -int APS5_VABI sceVdecswResetDecoder() { - NotImplemented_nid_no_patch(__func__); - return 0; +int APS5_VABI sceVdecswResetDecoder(std::uint64_t handle) { + const auto decoder = findDecoder(handle); + std::lock_guard lock(decoder->mutex); + decoder->inputs.clear(); + decoder->consumed.clear(); + decoder->outputs.clear(); + decoder->pictures.clear(); + decoder->delivered.clear(); + decoder->finalized = false; + decoder->drained = false; + return sceVideodec2Reset_nid_postfix(handle); } } diff --git a/core/libs/prx/libSceVideoOut/CMakeLists.txt b/core/libs/prx/libSceVideoOut/CMakeLists.txt index 2a15963e5..cf2807919 100644 --- a/core/libs/prx/libSceVideoOut/CMakeLists.txt +++ b/core/libs/prx/libSceVideoOut/CMakeLists.txt @@ -16,7 +16,7 @@ target_include_directories(libSceVideoOut PRIVATE ${CMAKE_SOURCE_DIR}/3rdparty/SDL2/include ${CMAKE_SOURCE_DIR}/3rdparty/Vulkan-Headers/include ) -target_link_libraries(libSceVideoOut PRIVATE libSceAgcDriver libScePad libSceMouse libSceKeyboard libc libkernel SDL2-static) +target_link_libraries(libSceVideoOut PRIVATE libSceAgcDriver libScePad libSceMouse libSceKeyboard libc libkernel decoder_png SDL2-static) if(WIN32) target_link_libraries(libSceVideoOut PRIVATE comctl32) endif() @@ -31,6 +31,21 @@ target_include_directories(video_out_flip_tests PRIVATE ${LIBS_INCLUDE_DIR} ${CM target_link_libraries(video_out_flip_tests PRIVATE libSceVideoOut libSceAgcDriver libkernel libc) set_target_properties(video_out_flip_tests PROPERTIES CXX_EXTENSIONS OFF) configure_windows_unwind(video_out_flip_tests) +if(BUILD_TESTING) + set_target_properties(video_out_flip_tests PROPERTIES EXCLUDE_FROM_ALL OFF) + foreach(flipMode IN ITEMS lifetime reopen decode controls overlay) + if(flipMode STREQUAL "lifetime") + add_test(NAME video_out_flip_${flipMode} COMMAND video_out_flip_tests) + else() + add_test(NAME video_out_flip_${flipMode} COMMAND video_out_flip_tests ${flipMode}) + endif() + set_tests_properties(video_out_flip_${flipMode} PROPERTIES TIMEOUT 30 SKIP_RETURN_CODE 77 RESOURCE_LOCK videoout_app0 ENVIRONMENT "SDL_VIDEODRIVER=offscreen") + if(WIN32) + set_tests_properties(video_out_flip_${flipMode} PROPERTIES ENVIRONMENT_MODIFICATION + "PATH=path_list_prepend:$") + endif() + endforeach() +endif() if(BUILD_TESTING) add_test_executable(pad_sdl_tests tests/PadSdl.cpp) @@ -44,17 +59,43 @@ if(BUILD_TESTING) target_link_libraries(guest_video_out_wqhd_detection_tests PRIVATE libSceVideoOut libSceAgcDriver libkernel libc) configure_windows_unwind(guest_video_out_wqhd_detection_tests) add_test(NAME guest_video_out_wqhd_detection COMMAND guest_video_out_wqhd_detection_tests) - set_tests_properties(guest_video_out_wqhd_detection PROPERTIES TIMEOUT 20 SKIP_RETURN_CODE 77) + set_tests_properties(guest_video_out_wqhd_detection PROPERTIES TIMEOUT 20 SKIP_RETURN_CODE 77 RESOURCE_LOCK videoout_app0) if(WIN32) set_tests_properties(guest_video_out_wqhd_detection PROPERTIES ENVIRONMENT_MODIFICATION "PATH=path_list_prepend:$") add_test(NAME guest_video_out_wqhd_detection_no_vulkan COMMAND guest_video_out_wqhd_detection_tests) - set_tests_properties(guest_video_out_wqhd_detection_no_vulkan PROPERTIES TIMEOUT 20 SKIP_RETURN_CODE 77 + set_tests_properties(guest_video_out_wqhd_detection_no_vulkan PROPERTIES TIMEOUT 20 SKIP_RETURN_CODE 77 RESOURCE_LOCK videoout_app0 ENVIRONMENT "VK_DRIVER_FILES=${CMAKE_CURRENT_BINARY_DIR}/missing_icd.json;VK_ICD_FILENAMES=${CMAKE_CURRENT_BINARY_DIR}/missing_icd.json" ENVIRONMENT_MODIFICATION "PATH=path_list_prepend:$") endif() endif() +if(BUILD_TESTING) + add_test_executable(guest_video_out_resolution_status_tests ${LIBS_INCLUDE_DIR}/tests/GuestVideoOutResolutionStatus.cpp) + target_include_directories(guest_video_out_resolution_status_tests PRIVATE ${LIBS_INCLUDE_DIR}) + target_link_libraries(guest_video_out_resolution_status_tests PRIVATE libSceVideoOut libSceAgcDriver libkernel libc) + configure_windows_unwind(guest_video_out_resolution_status_tests) + add_test(NAME guest_video_out_resolution_status COMMAND guest_video_out_resolution_status_tests) + set_tests_properties(guest_video_out_resolution_status PROPERTIES TIMEOUT 20 SKIP_RETURN_CODE 77 RESOURCE_LOCK videoout_app0) + if(WIN32) + set_tests_properties(guest_video_out_resolution_status PROPERTIES ENVIRONMENT_MODIFICATION + "PATH=path_list_prepend:$") + endif() +endif() + +if(BUILD_TESTING) + add_test_executable(guest_video_out_output_mode_tests ${LIBS_INCLUDE_DIR}/tests/GuestVideoOutOutputMode.cpp) + target_include_directories(guest_video_out_output_mode_tests PRIVATE ${LIBS_INCLUDE_DIR}) + target_link_libraries(guest_video_out_output_mode_tests PRIVATE libSceVideoOut libSceAgcDriver libkernel libc) + configure_windows_unwind(guest_video_out_output_mode_tests) + add_test(NAME guest_video_out_output_mode COMMAND guest_video_out_output_mode_tests) + set_tests_properties(guest_video_out_output_mode PROPERTIES TIMEOUT 20 SKIP_RETURN_CODE 77 RESOURCE_LOCK videoout_app0) + if(WIN32) + set_tests_properties(guest_video_out_output_mode PROPERTIES ENVIRONMENT_MODIFICATION + "PATH=path_list_prepend:$") + endif() +endif() + if(BUILD_TESTING) add_test_executable(guest_buffer_reuse_tests ${LIBS_INCLUDE_DIR}/tests/GuestBufferReuse.cpp) target_include_directories(guest_buffer_reuse_tests PRIVATE ${LIBS_INCLUDE_DIR}) diff --git a/core/libs/prx/libSceVideoOut/Unimplemented.cpp b/core/libs/prx/libSceVideoOut/Unimplemented.cpp index ed9002366..ddeb807c6 100644 --- a/core/libs/prx/libSceVideoOut/Unimplemented.cpp +++ b/core/libs/prx/libSceVideoOut/Unimplemented.cpp @@ -7,6 +7,11 @@ extern "C" { +int APS5_VABI sceVideoOutGetOutputStatusWithVrr(int handle, void* status) { + NotImplemented_nid_no_patch(__func__); + return 0; +} + int APS5_VABI sceVideoOutAdjustColor_(int handle, const VideoOutColorSettings* settings, uint32_t settings_size) { if (settings_size < sizeof(VideoOutColorSettings)) throw std::runtime_error("sceVideoOutAdjustColor_: VIDEO_OUT_ERROR_INVALID_VALUE"); return sceVideoOutAdjustColor(handle, settings); diff --git a/core/libs/prx/libSceVideoOut/include/Event.hpp b/core/libs/prx/libSceVideoOut/include/Event.hpp index 8b07233df..af8d10be4 100644 --- a/core/libs/prx/libSceVideoOut/include/Event.hpp +++ b/core/libs/prx/libSceVideoOut/include/Event.hpp @@ -12,6 +12,7 @@ int APS5_VABI sceVideoOutAddFlipEvent(KernelEqueue eq, int handle, void* udata); int APS5_VABI sceVideoOutAddOutputModeEvent(KernelEqueue eq, int handle, void* udata); int APS5_VABI sceVideoOutAddPreVblankStartEvent(KernelEqueue eq, int handle, void* udata); int APS5_VABI sceVideoOutAddVblankEvent(KernelEqueue eq, int handle, void* udata); +int APS5_VABI sceVideoOutAddVrrActiveStatusEvent(KernelEqueue eq, int handle, void* udata); int APS5_VABI sceVideoOutDeleteFlipEvent(KernelEqueue eq, int handle); int APS5_VABI sceVideoOutDeletePreVblankStartEvent(KernelEqueue eq, int handle); diff --git a/core/libs/prx/libSceVideoOut/include/Output.hpp b/core/libs/prx/libSceVideoOut/include/Output.hpp index 91689f906..4bac79d39 100644 --- a/core/libs/prx/libSceVideoOut/include/Output.hpp +++ b/core/libs/prx/libSceVideoOut/include/Output.hpp @@ -17,6 +17,8 @@ int APS5_VABI sceVideoOutSetFlipRate(int handle, int rate); int APS5_VABI sceVideoOutSetWindowModeMargins(int handle, int top, int bottom); int APS5_VABI sceVideoOutGetFlipStatus(int handle, VideoOutFlipStatus* status); int APS5_VABI sceVideoOutGetOutputStatus(int handle, VideoOutOutputStatus* status); +int APS5_VABI sceVideoOutGetOutputStatusWithVrr(int handle, void* status); +int APS5_VABI sceVideoOutGetResolutionStatus(int handle, VideoOutResolutionStatus* status); int APS5_VABI sceVideoOutGetVblankStatus(int handle, VideoOutVblankStatus* status); int APS5_VABI sceVideoOutIsFlipPending(int handle); int APS5_VABI sceVideoOutWaitVblank(int handle); diff --git a/core/libs/prx/libSceVideoOut/include/VideoOutDriver.hpp b/core/libs/prx/libSceVideoOut/include/VideoOutDriver.hpp index 02adf6ea0..66df2fa8c 100644 --- a/core/libs/prx/libSceVideoOut/include/VideoOutDriver.hpp +++ b/core/libs/prx/libSceVideoOut/include/VideoOutDriver.hpp @@ -37,6 +37,7 @@ static constexpr int VIDEO_OUT_ERROR_FLIP_QUEUE_FULL = -2144796654; static constexpr int VIDEO_OUT_ERROR_UNSUPPORTED_OUTPUT_MODE = -2144796650; static constexpr int VIDEO_OUT_ERROR_UNAVAILABLE_OUTPUT_MODE = -2144796647; static constexpr int VIDEO_OUT_ERROR_INVALID_EVENT = -2144796659; +static constexpr int VIDEO_OUT_ERROR_UNKNOWN_OUTPUT_MODE = -2144796642; static constexpr int VIDEO_OUT_BUS_TYPE_MAIN = 0; static constexpr int VIDEO_OUT_BUS_TYPE_OVERLAY = 1; @@ -74,6 +75,7 @@ static constexpr int VIDEO_OUT_EVENT_FLIP = 0; static constexpr int VIDEO_OUT_EVENT_VBLANK = 1; static constexpr int VIDEO_OUT_EVENT_PRE_VBLANK_START = 2; static constexpr int VIDEO_OUT_EVENT_SET_MODE = 8; +static constexpr int VIDEO_OUT_EVENT_VRR_STATUS = 16; static constexpr int VIDEO_OUT_FLIP_MODE_VSYNC = 1; static constexpr int VIDEO_OUT_FLIP_MODE_VSYNC_MULTI = 4; @@ -116,10 +118,12 @@ struct VideoOutConfig { std::vector vblankEvents; std::vector preVblankEvents; std::vector outputModeEvents; + std::vector vrrStatusEvents; uint32_t width = VIDEO_OUT_DEFAULT_WIDTH; uint32_t height = VIDEO_OUT_DEFAULT_HEIGHT; uint64_t generation = 0; + int busType = VIDEO_OUT_BUS_TYPE_MAIN; bool opened = false; bool closing = false; std::exception_ptr failure; @@ -139,10 +143,16 @@ struct VideoOutConfig { std::array bufferReuse; std::array groups{}; - void Check() const { + bool Closed() const { return !opened || closing; } + + void CheckAlive() const { if (failure) std::rethrow_exception(failure); if (shutdownToken.stop_requested()) throw ProcessShutdown{}; - if (!opened || closing) throw std::runtime_error("VideoOut: port is closed"); + } + + void Check() const { + CheckAlive(); + if (Closed()) throw std::runtime_error("VideoOut: port is closed"); } }; @@ -163,6 +173,7 @@ struct FlipRequest final : AgcDriver::IFlipRequest, std::enable_shared_from_this VideoOutBuffer buffer; BufferAttributeGroup group; bool reserved = false; + bool unregistered = false; bool ready = false; bool gpuComplete = false; bool terminal = false; @@ -173,6 +184,8 @@ struct FlipRequest final : AgcDriver::IFlipRequest, std::enable_shared_from_this ~FlipRequest() override; void GpuReady(const std::shared_ptr& frameTiming) override; void Fail(std::exception_ptr error) noexcept override; + void Cancel() noexcept; + void ReleaseLocked() noexcept; }; struct FlipQueue { @@ -199,6 +212,7 @@ public: bool Close(int handle); std::shared_ptr GetConfig(int handle); bool IsOpen(int handle); + bool HasConfig(int handle); // 0, or VIDEO_OUT_ERROR_FLIP_QUEUE_FULL when the title has VIDEO_OUT_FLIP_QUEUE_CAPACITY flips pending. int SubmitFlip(int handle, int index, int flipMode, int64_t flipArg); diff --git a/core/libs/prx/libSceVideoOut/src/DisplayWindow.cpp b/core/libs/prx/libSceVideoOut/src/DisplayWindow.cpp index 1b60c53ec..5cd47dbc8 100644 --- a/core/libs/prx/libSceVideoOut/src/DisplayWindow.cpp +++ b/core/libs/prx/libSceVideoOut/src/DisplayWindow.cpp @@ -3,8 +3,10 @@ #include "prx/libSceAgcDriver/Execution/include/Driver.hpp" #include "prx/libkernel/AppMetadata/include/AppMetadata.hpp" #include "prx/libkernel/Time/include/Time.hpp" +#include "Decoder/Png.hpp" #include "SDL_vulkan.h" #include +#include #include #include @@ -20,6 +22,17 @@ void require(bool condition, const char* reason) { if (!condition) throw std::runtime_error(std::string("DisplayWindow: ") + reason); } +void applyAppIcon(SDL_Window* window) { + if (!HasAppIcon_nid_postfix()) return; + const auto data = GetAppIconData_nid_postfix(); + auto image = Decoder::Png::Decode(std::span(data.bytes, static_cast(data.size))); + require(image.has_value(), "failed to decode icon0.png"); + SDL_Surface* icon = SDL_CreateRGBSurfaceWithFormatFrom(image->pixels.data(), static_cast(image->width), static_cast(image->height), 32, static_cast(image->width * 4), SDL_PIXELFORMAT_RGBA32); + require(icon != nullptr, SDL_GetError()); + SDL_SetWindowIcon(window, icon); + SDL_FreeSurface(icon); +} + #ifdef _WIN32 constexpr UINT_PTR DisplayWindowSubclassId = 0x41505335u; #endif @@ -51,6 +64,7 @@ void DisplayWindow::create(std::uint32_t sourceWidth, std::uint32_t sourceHeight AgcDriverUnlockVulkanLoader_nid_postfix(); require(window != nullptr, SDL_GetError()); SDL_SetWindowMinimumSize(window, static_cast(DisplayWindowMinimumWidth), static_cast(DisplayWindowMinimumHeight)); + applyAppIcon(window); installSubclass(); } diff --git a/core/libs/prx/libSceVideoOut/src/Event.cpp b/core/libs/prx/libSceVideoOut/src/Event.cpp index c70ba7557..adef7235c 100644 --- a/core/libs/prx/libSceVideoOut/src/Event.cpp +++ b/core/libs/prx/libSceVideoOut/src/Event.cpp @@ -13,6 +13,7 @@ static std::vector* getEventList(VideoOutConfig& cfg, int16_t if (eventKind == VIDEO_OUT_EVENT_VBLANK) return &cfg.vblankEvents; if (eventKind == VIDEO_OUT_EVENT_PRE_VBLANK_START) return &cfg.preVblankEvents; if (eventKind == VIDEO_OUT_EVENT_SET_MODE) return &cfg.outputModeEvents; + if (eventKind == VIDEO_OUT_EVENT_VRR_STATUS) return &cfg.vrrStatusEvents; throw std::runtime_error("getEventList: unknown event kind"); } @@ -120,10 +121,7 @@ int APS5_VABI sceVideoOutAddVblankEvent(KernelEqueue eq, int handle, void* udata } int APS5_VABI sceVideoOutAddPreVblankStartEvent(KernelEqueue eq, int handle, void* udata) try { - static_cast(eq); - static_cast(udata); - VideoOutDriver::Get().GetConfig(handle); - throw std::runtime_error("VideoOut: physical pre-vblank timing is not implemented"); + return registerVideoOutEvent(handle, eq, VIDEO_OUT_EVENT_PRE_VBLANK_START, udata); } catch (const ProcessShutdown&) { LibcAwaitExit_nid_postfix(); } @@ -134,6 +132,12 @@ int APS5_VABI sceVideoOutAddOutputModeEvent(KernelEqueue eq, int handle, void* u LibcAwaitExit_nid_postfix(); } +int APS5_VABI sceVideoOutAddVrrActiveStatusEvent(KernelEqueue eq, int handle, void* udata) try { + return registerVideoOutEvent(handle, eq, VIDEO_OUT_EVENT_VRR_STATUS, udata); +} catch (const ProcessShutdown&) { + LibcAwaitExit_nid_postfix(); +} + int APS5_VABI sceVideoOutDeleteFlipEvent(KernelEqueue eq, int handle) try { return deleteVideoOutEvent(handle, eq, VIDEO_OUT_EVENT_FLIP); } catch (const ProcessShutdown&) { @@ -160,7 +164,7 @@ int APS5_VABI sceVideoOutGetEventId(const KernelEvent* ev) { throw std::runtime_error(std::string(__func__) + ": VIDEO_OUT_ERROR_INVALID_EVENT"); } const int ident = static_cast(ev->ident); - if (ident != VIDEO_OUT_EVENT_FLIP && ident != VIDEO_OUT_EVENT_VBLANK && ident != VIDEO_OUT_EVENT_PRE_VBLANK_START && ident != VIDEO_OUT_EVENT_SET_MODE) { + if (ident != VIDEO_OUT_EVENT_FLIP && ident != VIDEO_OUT_EVENT_VBLANK && ident != VIDEO_OUT_EVENT_PRE_VBLANK_START && ident != VIDEO_OUT_EVENT_SET_MODE && ident != VIDEO_OUT_EVENT_VRR_STATUS) { throw std::runtime_error(std::string(__func__) + ": VIDEO_OUT_ERROR_INVALID_EVENT"); } return ident; diff --git a/core/libs/prx/libSceVideoOut/src/Output.cpp b/core/libs/prx/libSceVideoOut/src/Output.cpp index af435d9a3..412ab1bf2 100644 --- a/core/libs/prx/libSceVideoOut/src/Output.cpp +++ b/core/libs/prx/libSceVideoOut/src/Output.cpp @@ -11,18 +11,26 @@ #include "prx/libc/include/General.hpp" #include "prx/libSceVideoOut/include/VideoOutDriver.hpp" +static constexpr std::uint32_t VIDEO_OUT_OUTPUT_OPTIONS_FIRST_WORD = 0x00FF0000; +static constexpr std::size_t VIDEO_OUT_OUTPUT_OPTIONS_FREE_WORD = 3; + static int validateOutputConfig(int handle, uint64_t mode, const VideoOutOutputOptions* options, void* reservedPtr, uint64_t reserved) { - if (!VideoOutDriver::Get().IsOpen(handle)) { - throw std::runtime_error(std::string(__func__) + ": VIDEO_OUT_ERROR_INVALID_HANDLE"); - } - if (reservedPtr != nullptr || reserved != 0) { - throw std::runtime_error(std::string(__func__) + ": VIDEO_OUT_ERROR_INVALID_VALUE"); + if (!VideoOutDriver::Get().HasConfig(handle) || !VideoOutDriver::Get().IsOpen(handle)) return VIDEO_OUT_ERROR_INVALID_HANDLE; + if (reservedPtr != nullptr || reserved != 0) return VIDEO_OUT_ERROR_INVALID_VALUE; + switch (mode) { + case VIDEO_OUT_OUTPUT_MODE_DEFAULT: + break; + case 0x4: case 0x7: case 0x8: case 0xC: case 0xD: case 0xE: case VIDEO_OUT_OUTPUT_MODE_119_88HZ: case 0x10: case 0x11: case 0x13: + return VIDEO_OUT_ERROR_UNSUPPORTED_OUTPUT_MODE; + case 0x12: + throw std::runtime_error(std::string(__func__) + ": output mode 0x12 answers differently on two consoles"); + default: + return VIDEO_OUT_ERROR_UNKNOWN_OUTPUT_MODE; } if (options != nullptr) { - for (auto v : options->internalData) { - if (v != 0) { - throw std::runtime_error(std::string(__func__) + ": VIDEO_OUT_ERROR_INVALID_OPTION"); - } + for (std::size_t index = 0; index < std::size(options->internalData); ++index) { + const std::uint32_t expected = index == 0 ? VIDEO_OUT_OUTPUT_OPTIONS_FIRST_WORD : 0; + if (index != VIDEO_OUT_OUTPUT_OPTIONS_FREE_WORD && options->internalData[index] != expected) return VIDEO_OUT_ERROR_INVALID_VALUE; } } return 0; @@ -156,6 +164,27 @@ int APS5_VABI sceVideoOutGetOutputStatus(int handle, VideoOutOutputStatus* statu LibcAwaitExit_nid_postfix(); } +int APS5_VABI sceVideoOutGetResolutionStatus(int handle, VideoOutResolutionStatus* status) try { + if (status == nullptr) { + throw std::runtime_error(std::string(__func__) + ": VIDEO_OUT_ERROR_INVALID_ADDRESS"); + } + auto cfg = VideoOutDriver::Get().GetConfig(handle); + if (cfg == nullptr) { + throw std::runtime_error(std::string(__func__) + ": VIDEO_OUT_ERROR_INVALID_HANDLE"); + } + std::unique_lock lock(cfg->mutex); + cfg->Check(); + *status = VideoOutResolutionStatus{}; + status->fullWidth = cfg->width; + status->fullHeight = cfg->height; + status->paneWidth = cfg->width; + status->paneHeight = cfg->height; + status->refreshRate = (cfg->outputMode == VIDEO_OUT_OUTPUT_MODE_119_88HZ) ? VIDEO_OUT_REFRESH_RATE_119_88HZ : VIDEO_OUT_REFRESH_RATE_59_94HZ; + return 0; +} catch (const ProcessShutdown&) { + LibcAwaitExit_nid_postfix(); +} + int APS5_VABI sceVideoOutIsFlipPending(int handle) try { auto cfg = VideoOutDriver::Get().GetConfig(handle); if (cfg == nullptr) { @@ -184,10 +213,9 @@ int APS5_VABI sceVideoOutWaitVblank(int handle) try { } int APS5_VABI sceVideoOutInitializeOutputOptions(VideoOutOutputOptions* options) { - if (options == nullptr) { - throw std::runtime_error(std::string(__func__) + ": VIDEO_OUT_ERROR_INVALID_ADDRESS"); - } + if (options == nullptr) return VIDEO_OUT_ERROR_INVALID_ADDRESS; std::memset(options, 0, sizeof(VideoOutOutputOptions)); + options->internalData[0] = VIDEO_OUT_OUTPUT_OPTIONS_FIRST_WORD; return 0; } @@ -196,7 +224,7 @@ int APS5_VABI sceVideoOutIsOutputSupported(int handle, uint64_t mode, const Vide if (result != 0) { return result; } - return mode == VIDEO_OUT_OUTPUT_MODE_DEFAULT ? 1 : 0; + return 1; } catch (const ProcessShutdown&) { LibcAwaitExit_nid_postfix(); } @@ -206,9 +234,6 @@ int APS5_VABI sceVideoOutConfigureOutput(int handle, uint64_t mode, const VideoO if (supported < 0) { return supported; } - if (supported == 0) { - return VIDEO_OUT_ERROR_UNAVAILABLE_OUTPUT_MODE; - } auto cfg = VideoOutDriver::Get().GetConfig(handle); if (cfg == nullptr) { throw std::runtime_error(std::string(__func__) + ": VIDEO_OUT_ERROR_INVALID_HANDLE"); @@ -290,20 +315,8 @@ int APS5_VABI sceVideoOutVrrPegToFixedRate() try { LibcAwaitExit_nid_postfix(); } -APS5_EXPORT("kP2L8t3j-aM", sceVideoOutUnknown00); -int APS5_VABI sceVideoOutUnknown00() try { - NotImplemented_nid_no_patch(__func__); +int APS5_VABI sceVideoOutAddVrrStatusFlagsPrivilege() { return 0; -} catch (const ProcessShutdown&) { - LibcAwaitExit_nid_postfix(); -} - -APS5_EXPORT("LibwuIonIBw", sceVideoOutUnknown01); -int APS5_VABI sceVideoOutUnknown01() try { - NotImplemented_nid_no_patch(__func__); - return 0; -} catch (const ProcessShutdown&) { - LibcAwaitExit_nid_postfix(); } } diff --git a/core/libs/prx/libSceVideoOut/src/VideoOutDriver.cpp b/core/libs/prx/libSceVideoOut/src/VideoOutDriver.cpp index 8f185445a..2b52be2cd 100644 --- a/core/libs/prx/libSceVideoOut/src/VideoOutDriver.cpp +++ b/core/libs/prx/libSceVideoOut/src/VideoOutDriver.cpp @@ -6,6 +6,7 @@ #include #include #include +#include #include "SDL.h" #include "SDL_vulkan.h" @@ -64,6 +65,30 @@ void checkConfig(const VideoOutConfig& cfg) { cfg.Check(); } +void resolveFlipBuffer(FlipRequest& request, const VideoOutConfig& cfg) { + if (request.index < 0) { + request.width = cfg.width; + request.height = cfg.height; + return; + } + request.buffer = cfg.buffers[request.index]; + if (!request.buffer.Occupied()) { + static std::array, VIDEO_OUT_BUFFER_NUM_MAX> reported{}; + if (!reported[request.index].exchange(true)) std::fprintf(stderr, "[videoout] GPU flip of unregistered buffer %d on handle %u presents black\n", request.index, request.outputHandle); + request.unregistered = true; + request.width = cfg.width; + request.height = cfg.height; + return; + } + require(request.buffer.groupIndex < VIDEO_OUT_BUFFER_ATTRIBUTE_NUM_MAX, "invalid buffer group"); + request.group = cfg.groups[request.buffer.groupIndex]; + require(request.group.occupied, "buffer group is not registered"); + require(request.buffer.dataAddress != 0, "null registered buffer address"); + static_cast(DescribeVideoOutBuffer(request.buffer, request.group)); + request.width = request.group.attribute.width; + request.height = request.group.attribute.height; +} + class RenderingWait final : public AgcDriver::IRenderingWait { std::shared_ptr _config; std::uint32_t _index; @@ -74,10 +99,10 @@ public: void Wait() override { std::unique_lock lock(_config->mutex); _config->vblankCond.wait(lock, _config->shutdownToken, [&] { - return _config->failure || _config->closing || !_config->opened || + return _config->failure || _config->Closed() || _config->bufferReuse[_index].IsComplete(_ticket); }); - checkConfig(*_config); + _config->CheckAlive(); } }; @@ -118,20 +143,6 @@ public: if (queue->stopping || cfg->shutdownToken.stop_requested()) throw ProcessShutdown{}; std::lock_guard lock(cfg->mutex); checkConfig(*cfg); - if (info.index >= 0) { - request->buffer = cfg->buffers[info.index]; - require(request->buffer.Occupied(), "flip buffer is not registered"); - require(request->buffer.groupIndex < VIDEO_OUT_BUFFER_ATTRIBUTE_NUM_MAX, "invalid buffer group"); - request->group = cfg->groups[request->buffer.groupIndex]; - require(request->group.occupied, "buffer group is not registered"); - require(request->buffer.dataAddress != 0, "null registered buffer address"); - static_cast(DescribeVideoOutBuffer(request->buffer, request->group)); - request->width = request->group.attribute.width; - request->height = request->group.attribute.height; - } else { - request->width = cfg->width; - request->height = cfg->height; - } request->generation = cfg->generation; request->flipRate = cfg->flipRate; if (info.index >= 0) request->reuseTicket = cfg->bufferReuse[info.index].Reserve(); @@ -166,15 +177,27 @@ private: FlipRequest::~FlipRequest() { if (!reserved) return; std::lock_guard lock(cfg->mutex); - if (!terminal) { - --cfg->flipStatus.flipPendingNum; - --queue->reservations; - if (index >= 0) { - --cfg->bufferPending[index]; - cfg->bufferReuse[index].Complete(reuseTicket); - } - cfg->vblankCond.notify_all(); + ReleaseLocked(); +} + +void FlipRequest::ReleaseLocked() noexcept { + if (!reserved || terminal) return; + --cfg->flipStatus.flipPendingNum; + --queue->reservations; + if (index >= 0) { + --cfg->bufferPending[index]; + cfg->bufferReuse[index].Complete(reuseTicket); } + terminal = true; + cfg->vblankCond.notify_all(); +} + +void FlipRequest::Cancel() noexcept { + { + std::lock_guard lock(cfg->mutex); + ReleaseLocked(); + } + queue->changed.notify_all(); } void FlipRequest::GpuReady(const std::shared_ptr& frameTiming) { @@ -187,8 +210,13 @@ void FlipRequest::GpuReady(const std::shared_ptr& frameT if (queue->failure) std::rethrow_exception(queue->failure); if (queue->stopping || cfg->shutdownToken.stop_requested()) throw ProcessShutdown{}; std::lock_guard lock(cfg->mutex); - checkConfig(*cfg); + cfg->CheckAlive(); + if (cfg->Closed()) { + ReleaseLocked(); + return; + } require(reserved && !ready && !terminal && cfg->generation == generation, "invalid flip readiness transition"); + resolveFlipBuffer(*this, *cfg); readiness.Mark("locks_validate"); queuedAt = AgcDriver::FrameTiming::Clock::now(); queue->requests.push_back(shared_from_this()); @@ -212,16 +240,7 @@ void FlipRequest::Fail(std::exception_ptr error) noexcept { if (!error) std::terminate(); std::lock_guard lock(cfg->mutex); if (!cfg->failure) cfg->failure = error; - if (reserved && !terminal) { - --cfg->flipStatus.flipPendingNum; - --queue->reservations; - if (index >= 0) { - --cfg->bufferPending[index]; - cfg->bufferReuse[index].Complete(reuseTicket); - } - terminal = true; - } - cfg->vblankCond.notify_all(); + ReleaseLocked(); queue->changed.notify_all(); } @@ -308,6 +327,7 @@ int VideoOutDriver::Open(int busType) { } auto cfg = std::make_shared(); cfg->generation = generation; + cfg->busType = busType; cfg->opened = true; cfg->flipStatus.flipArg = -1; cfg->flipStatus.currentBuffer = -1; @@ -342,10 +362,12 @@ bool VideoOutDriver::close(int handle) { removeEvents(cfg->vblankEvents, VIDEO_OUT_EVENT_VBLANK); removeEvents(cfg->preVblankEvents, VIDEO_OUT_EVENT_PRE_VBLANK_START); removeEvents(cfg->outputModeEvents, VIDEO_OUT_EVENT_SET_MODE); + removeEvents(cfg->vrrStatusEvents, VIDEO_OUT_EVENT_VRR_STATUS); cfg->flipEvents.clear(); cfg->vblankEvents.clear(); cfg->preVblankEvents.clear(); cfg->outputModeEvents.clear(); + cfg->vrrStatusEvents.clear(); cfg->vblankCond.notify_all(); flipQueue->changed.notify_all(); return true; @@ -369,6 +391,17 @@ bool VideoOutDriver::IsOpen(int handle) { return GetConfig(handle) != nullptr; } +bool VideoOutDriver::HasConfig(int handle) { + std::shared_ptr cfg; + { + std::lock_guard lock(mutex); + if (handle <= 0 || handle >= VIDEO_OUT_NUM_MAX || contexts[handle] == nullptr) return false; + cfg = contexts[handle]; + } + std::lock_guard cfgLock(cfg->mutex); + return cfg->opened && !cfg->closing; +} + int VideoOutDriver::SubmitFlip(int handle, int index, int flipMode, int64_t flipArg) { if (LibcShutdownToken_nid_postfix().stop_requested()) throw ProcessShutdown{}; // A title that does not pace on flipPendingNum can run ahead of the presenter now that the queue @@ -406,6 +439,10 @@ void VideoOutDriver::vblankEnd() { std::lock_guard cfgLock(cfg->mutex); if (!cfg->opened || cfg->failure) continue; require(cfg->vblankStatus.count != std::numeric_limits::max(), "vblank counter overflow"); + ++cfg->preVblankStatus.count; + cfg->preVblankStatus.processTime = sceKernelGetProcessTime(); + cfg->preVblankStatus.processTimeCounter = sceKernelGetProcessTimeCounter(); + triggerEvents(*cfg, VIDEO_OUT_EVENT_PRE_VBLANK_START, reinterpret_cast(cfg->preVblankStatus.count)); ++cfg->vblankStatus.count; cfg->vblankStatus.processTime = sceKernelGetProcessTime(); cfg->vblankStatus.processTimeCounter = sceKernelGetProcessTimeCounter(); @@ -420,7 +457,11 @@ void VideoOutDriver::processFlip(FlipRequest& req) { { std::unique_lock lock(req.cfg->mutex); timing.Mark("config_mutex_wait"); - checkConfig(*req.cfg); + req.cfg->CheckAlive(); + if (req.cfg->Closed()) { + req.ReleaseLocked(); + return; + } require(req.ready && !req.terminal && req.generation == req.cfg->generation, "stale or incomplete flip request"); const auto interval = static_cast(req.flipRate + 1); require(req.cfg->lastFlipVblank <= std::numeric_limits::max() - interval, "flip interval overflow"); @@ -428,34 +469,52 @@ void VideoOutDriver::processFlip(FlipRequest& req) { timing.Mark("validate"); req.cfg->vblankCond.wait(lock, req.cfg->shutdownToken, [&] { return req.cfg->vblankStatus.count >= target || req.cfg->failure || req.cfg->closing; }); timing.Mark("vblank_wait"); - checkConfig(*req.cfg); + req.cfg->CheckAlive(); + if (req.cfg->Closed()) { + req.ReleaseLocked(); + return; + } req.cfg->lastFlipVblank = req.cfg->vblankStatus.count; } require(req.width != 0 && req.height != 0 && req.width <= static_cast(std::numeric_limits::max()) && req.height <= static_cast(std::numeric_limits::max()), "invalid window dimensions"); - window.Ensure(req.width, req.height); - const auto extensions = windowExtensions(window.Handle()); - const AgcDriver::PresentationWindow target{window.Handle(), extensions, &createWindowSurface, &windowDrawableSize, req.width, req.height, req.timing}; - timing.Mark("window_prepare"); - const auto gpuReady = [](void* context) { - auto& request = *static_cast(context); - std::lock_guard lock(request.cfg->mutex); - checkConfig(*request.cfg); - require(!request.terminal && !request.gpuComplete, "invalid GPU completion transition"); - request.gpuComplete = true; - request.cfg->vblankCond.notify_all(); - }; - if (req.index >= 0) { - const auto display = DescribeVideoOutBuffer(req.buffer, req.group); - AgcDriverPresentBuffer_nid_postfix(target, display, gpuReady, &req); + if (req.cfg->busType == VIDEO_OUT_BUS_TYPE_OVERLAY) { + std::lock_guard lock(req.cfg->mutex); + req.cfg->CheckAlive(); + if (!req.cfg->Closed()) { + require(!req.terminal && !req.gpuComplete, "invalid GPU completion transition"); + req.gpuComplete = true; + } } else { - AgcDriverPresentClear_nid_postfix(target, req.index == VIDEO_OUT_BUFFER_INDEX_BLACK, gpuReady, &req); + window.Ensure(req.width, req.height); + const auto extensions = windowExtensions(window.Handle()); + const AgcDriver::PresentationWindow target{window.Handle(), extensions, &createWindowSurface, &windowDrawableSize, req.width, req.height, req.timing}; + timing.Mark("window_prepare"); + const auto gpuReady = [](void* context) { + auto& request = *static_cast(context); + std::lock_guard lock(request.cfg->mutex); + request.cfg->CheckAlive(); + if (request.cfg->Closed()) return; + require(!request.terminal && !request.gpuComplete, "invalid GPU completion transition"); + request.gpuComplete = true; + request.cfg->vblankCond.notify_all(); + }; + if (req.index >= 0 && !req.unregistered) { + const auto display = DescribeVideoOutBuffer(req.buffer, req.group); + AgcDriverPresentBuffer_nid_postfix(target, display, gpuReady, &req); + } else { + AgcDriverPresentClear_nid_postfix(target, req.unregistered || req.index == VIDEO_OUT_BUFFER_INDEX_BLACK, gpuReady, &req); + } + timing.Mark("present"); + window.UpdateTitle(); + timing.Mark("window_title"); } - timing.Mark("present"); - window.UpdateTitle(); - timing.Mark("window_title"); std::lock_guard lock(req.cfg->mutex); timing.Mark("completion_mutex_wait"); - checkConfig(*req.cfg); + req.cfg->CheckAlive(); + if (req.cfg->Closed()) { + req.ReleaseLocked(); + return; + } require(!req.terminal && req.cfg->generation == req.generation, "flip cancelled during presentation"); require(req.gpuComplete, "flip submitted before GPU completion"); require(req.cfg->flipStatus.count != std::numeric_limits::max(), "flip counter overflow"); @@ -574,7 +633,12 @@ void VideoOutDriver::presentLoop(std::stop_token token, std::promise& star cancelled.swap(flipQueue->requests); } } - AgcDriverShutdown_nid_postfix(); + try { + AgcDriverShutdown_nid_postfix(); + } catch (...) { + std::lock_guard lock(flipQueue->mutex); + if (!flipQueue->failure) flipQueue->failure = std::current_exception(); + } window.Destroy(); } diff --git a/core/libs/prx/libSceVideoOut/tests/Flip.cpp b/core/libs/prx/libSceVideoOut/tests/Flip.cpp index ed59a5177..fa4a07c6f 100644 --- a/core/libs/prx/libSceVideoOut/tests/Flip.cpp +++ b/core/libs/prx/libSceVideoOut/tests/Flip.cpp @@ -13,6 +13,9 @@ #include #include #include +#include +#include +#include #include #include #include @@ -96,24 +99,78 @@ void testLifetime(bool reopen) { check(replacement != cfg && replacement->generation > cfg->generation, "reopen reused old port state"); } gate->Release(); - std::exception_ptr failure; { std::unique_lock lock(cfg->mutex); - check(cfg->vblankCond.wait_for(lock, std::chrono::seconds(10), [&] { return cfg->failure != nullptr; }), "flip failure did not wake waiters"); - failure = cfg->failure; - check(cfg->flipStatus.count == 0 && cfg->flipStatus.flipPendingNum == 0, "failed flip has successful or pending status"); + check(cfg->vblankCond.wait_for(lock, std::chrono::seconds(10), [&] { return cfg->flipStatus.flipPendingNum == 0; }), "closed port kept its pending flip"); + check(cfg->failure == nullptr, "closing a port failed the driver"); + check(cfg->flipStatus.count == 0, "cancelled flip has successful status"); } - const auto message = expectFailure([&] { std::rethrow_exception(failure); }); - check(message.find("closed") != std::string::npos, "flip used a closed port"); if (replacement) { std::lock_guard lock(replacement->mutex); check(replacement->flipStatus.flipPendingNum == 0 && replacement->flipStatus.count == 0, "old request changed new port counters"); + } else { + check(expectFailure([&] { sceVideoOutWaitVblank(handle); }).find("closed") != std::string::npos, "closed port accepted a vblank wait"); } - check(expectFailure([&] { sceVideoOutWaitVblank(handle); }).find("closed") != std::string::npos, "VideoOut lost worker failure"); AgcDriverUnregisterVideoOutput_nid_postfix(7, gate); if (reopen) sceVideoOutClose(handle); - const auto shutdown = expectFailure([] { LibcRunShutdown_nid_postfix(); }); - check(shutdown.find("closed") != std::string::npos, "shutdown lost asynchronous error"); + LibcRunShutdown_nid_postfix(); +} + +void testDeferredBuffer() { + const int handle = sceVideoOutOpen(255, 0, 0, nullptr); + auto cfg = VideoOutDriver::Get().GetConfig(handle); + auto gate = std::make_shared(); + AgcDriverRegisterVideoOutput_nid_postfix(7, gate); + std::array held{0xc004105c, 7, 0xfffffffeu, 1, 0, 0}; + Packet heldPacket{held.data(), 6, 0, {}}; + check(sceAgcDriverSubmitDcb(&heldPacket) == 0, "the gated flip was not accepted"); + std::array flip{0xc004105c, static_cast(handle), 0, 1, 3, 0}; + Packet flipPacket{flip.data(), 6, 0, {}}; + check(sceAgcDriverSubmitDcb(&flipPacket) == 0, "a GPU flip of a buffer registered before the GPU reaches it was rejected at submit"); + std::vector allocation(65536 + 65535); + const auto storage = alignedBuffer(allocation); + VideoOutBuffers buffer{storage.data(), nullptr, {nullptr, nullptr}}; + VideoOutBufferAttribute2 attribute{}; + sceVideoOutSetBufferAttribute2(&attribute, 0x8000000000000000ull, 0, 64, 64, 0, 0, 0); + check(sceVideoOutRegisterBuffers2(handle, 0, 0, &buffer, 1, &attribute, 0, nullptr) == 0, "buffer registration failed"); + gate->Release(); + { + std::unique_lock lock(cfg->mutex); + const bool done = cfg->vblankCond.wait_for(lock, std::chrono::seconds(30), [&] { return cfg->failure || cfg->flipStatus.count == 1; }); + if (cfg->failure) std::rethrow_exception(cfg->failure); + check(done, "the deferred flip did not complete"); + check(cfg->flipStatus.flipArg == 3 && cfg->flipStatus.currentBuffer == 0, "the deferred flip presented the wrong request"); + } + AgcDriverWaitIdle_nid_postfix(); + sceVideoOutUnregisterBuffers(handle, 0); + sceVideoOutClose(handle); + AgcDriverUnregisterVideoOutput_nid_postfix(7, gate); + LibcRunShutdown_nid_postfix(); +} + +void testUnregisteredBuffer() { + const int handle = sceVideoOutOpen(255, 0, 0, nullptr); + auto cfg = VideoOutDriver::Get().GetConfig(handle); + std::vector allocation(65536 + 65535); + const auto storage = alignedBuffer(allocation); + VideoOutBuffers buffer{storage.data(), nullptr, {nullptr, nullptr}}; + VideoOutBufferAttribute2 attribute{}; + sceVideoOutSetBufferAttribute2(&attribute, 0x8000000000000000ull, 0, 64, 64, 0, 0, 0); + check(sceVideoOutRegisterBuffers2(handle, 0, 0, &buffer, 1, &attribute, 0, nullptr) == 0, "buffer registration failed"); + check(sceVideoOutUnregisterBuffers(handle, 0) == 0, "buffer unregistration failed"); + std::array flip{0xc004105c, static_cast(handle), 0, 1, 7, 0}; + Packet flipPacket{flip.data(), 6, 0, {}}; + check(sceAgcDriverSubmitDcb(&flipPacket) == 0, "a GPU flip of an unregistered buffer was rejected at submit"); + { + std::unique_lock lock(cfg->mutex); + const bool done = cfg->vblankCond.wait_for(lock, std::chrono::seconds(30), [&] { return cfg->failure || cfg->flipStatus.count == 1; }); + if (cfg->failure) std::rethrow_exception(cfg->failure); + check(done, "a GPU flip of an unregistered buffer did not complete"); + check(cfg->flipStatus.flipPendingNum == 0 && cfg->flipStatus.flipArg == 7 && cfg->bufferPending[0] == 0, "a GPU flip of an unregistered buffer left pending state"); + } + AgcDriverWaitIdle_nid_postfix(); + sceVideoOutClose(handle); + LibcRunShutdown_nid_postfix(); } std::size_t tiledOffset(uint32_t x, uint32_t y, uint32_t width) { @@ -232,10 +289,10 @@ void testControls() { check(handle >= 0, "open with the highest priority on every CPU failed"); const auto cfg = VideoOutDriver::Get().GetConfig(handle); check(sceVideoOutIsOutputSupported(handle, VIDEO_OUT_OUTPUT_MODE_DEFAULT, nullptr, nullptr, 0) == 1, "default output mode is unsupported"); - for (const uint64_t unsupported : std::initializer_list{VIDEO_OUT_OUTPUT_MODE_119_88HZ, 0xd000000aull}) { - check(sceVideoOutIsOutputSupported(handle, unsupported, nullptr, nullptr, 0) == 0, "unavailable output mode is supported"); - check(sceVideoOutConfigureOutput(handle, unsupported, nullptr, nullptr, 0) == VIDEO_OUT_ERROR_UNAVAILABLE_OUTPUT_MODE, "unavailable output mode was configured"); - } + check(sceVideoOutIsOutputSupported(handle, VIDEO_OUT_OUTPUT_MODE_119_88HZ, nullptr, nullptr, 0) == VIDEO_OUT_ERROR_UNSUPPORTED_OUTPUT_MODE, "unavailable output mode is supported"); + check(sceVideoOutConfigureOutput(handle, VIDEO_OUT_OUTPUT_MODE_119_88HZ, nullptr, nullptr, 0) == VIDEO_OUT_ERROR_UNSUPPORTED_OUTPUT_MODE, "unavailable output mode was configured"); + check(sceVideoOutIsOutputSupported(handle, 0xd000000aull, nullptr, nullptr, 0) == VIDEO_OUT_ERROR_UNKNOWN_OUTPUT_MODE, "unknown output mode is supported"); + check(sceVideoOutConfigureOutput(handle, 0xd000000aull, nullptr, nullptr, 0) == VIDEO_OUT_ERROR_UNKNOWN_OUTPUT_MODE, "unknown output mode was configured"); check(sceVideoOutConfigureOutput(handle, VIDEO_OUT_OUTPUT_MODE_DEFAULT, nullptr, nullptr, 0) == 0, "default output mode was rejected"); for (int rate = 0; rate <= 2; ++rate) { check(sceVideoOutSetFlipRate(handle, rate) == 0 && cfg->flipRate == rate, "flip rate was not applied"); @@ -256,8 +313,20 @@ void testControls() { std::lock_guard lock(cfg->mutex); check(cfg->vblankEvents.size() == 1, "duplicate vblank subscription"); } + check(sceVideoOutAddVrrActiveStatusEvent(queue, handle, &settings) == 0, "VRR status subscription failed"); + { + std::lock_guard lock(cfg->mutex); + check(cfg->vrrStatusEvents.size() == 1, "VRR status subscription was not recorded"); + } check(sceVideoOutWaitVblank(handle) == 0, "vblank wait failed"); check(owner->GetTriggeredEvents(&event, 1) == 1 && event.udata == &settings && sceVideoOutGetEventId(&event) == VIDEO_OUT_EVENT_VBLANK, "vblank event or updated user data missing"); + check(sceVideoOutAddPreVblankStartEvent(queue, handle, &mode) == 0, "pre-vblank subscription failed"); + check(sceVideoOutWaitVblank(handle) == 0, "vblank wait failed"); + KernelEvent events[2]{}; + check(owner->GetTriggeredEvents(events, 2) == 2, "pre-vblank or vblank event missing"); + const auto& preVblank = sceVideoOutGetEventId(&events[0]) == VIDEO_OUT_EVENT_PRE_VBLANK_START ? events[0] : events[1]; + check(sceVideoOutGetEventId(&preVblank) == VIDEO_OUT_EVENT_PRE_VBLANK_START && preVblank.udata == &mode, "pre-vblank event missing"); + check(sceVideoOutDeletePreVblankStartEvent(queue, handle) == 0, "pre-vblank unsubscription failed"); std::vector allocation(65536 + 65535); const auto storage = alignedBuffer(allocation); VideoOutBuffers buffer{storage.data(), nullptr, {allocation.data(), storage.data()}}; @@ -267,6 +336,11 @@ void testControls() { attribute.pixel_format = 0x8000000000000000ull; check(sceVideoOutRegisterBuffers2(handle, 0, 0, &buffer, 1, &attribute, 0, nullptr) == 0, "buffers with set reserved pointers were rejected"); check(sceVideoOutUnregisterBuffers(handle, 0) == 0, "buffer unregistration failed"); + check(owner->GetTriggeredEvents(&event, 1) == 0, "VRR status event triggered without a VRR change"); + KernelEvent vrrStatus{}; + vrrStatus.ident = VIDEO_OUT_EVENT_VRR_STATUS; + vrrStatus.filter = EVFILT_VIDEO_OUT; + check(sceVideoOutGetEventId(&vrrStatus) == VIDEO_OUT_EVENT_VRR_STATUS, "VRR status event id rejected"); sceVideoOutClose(handle); check(owner->GetTriggeredEvents(&event, 1) == 0, "closed port retained pending events"); check(sceKernelDeleteEqueue(queue) == 0, "event queue deletion failed"); @@ -312,6 +386,58 @@ void testPresentation() { LibcRunShutdown_nid_postfix(); } +void testOverlay() { + const int mainHandle = sceVideoOutOpen(255, 0, 0, nullptr); + const int overlayHandle = sceVideoOutOpen(255, 1, 0, nullptr); + check(mainHandle != overlayHandle, "the overlay bus shares the main port"); + auto mainCfg = VideoOutDriver::Get().GetConfig(mainHandle); + auto overlayCfg = VideoOutDriver::Get().GetConfig(overlayHandle); + std::vector mainAllocation(6 * 65536 + 65535); + std::vector overlayAllocation(6 * 65536 + 65535); + const auto mainStorage = alignedBuffer(mainAllocation); + const auto overlayStorage = alignedBuffer(overlayAllocation); + fillBuffer(mainStorage, 259, 137); + fillBuffer(overlayStorage, 259, 137); + VideoOutBuffers mainBuffer{mainStorage.data(), nullptr, {nullptr, nullptr}}; + VideoOutBuffers overlayBuffer{overlayStorage.data(), nullptr, {nullptr, nullptr}}; + VideoOutBufferAttribute2 attribute{}; + sceVideoOutSetBufferAttribute2(&attribute, 0x8000000000000000ull, 0, 259, 137, 0, 0, 0); + check(sceVideoOutRegisterBuffers2(mainHandle, 0, 0, &mainBuffer, 1, &attribute, 0, nullptr) == 0, "main buffer registration failed"); + check(sceVideoOutRegisterBuffers2(overlayHandle, 0, 0, &overlayBuffer, 1, &attribute, 0, nullptr) == 0, "overlay buffer registration failed"); + const auto flip = [](int handle, const std::shared_ptr& cfg, std::int64_t argument) { + std::uint64_t target; + { + std::lock_guard lock(cfg->mutex); + target = cfg->flipStatus.count + 1; + } + check(sceVideoOutSubmitFlip(handle, 0, 1, argument) == 0, "the flip was not accepted"); + std::unique_lock lock(cfg->mutex); + check(cfg->vblankCond.wait_for(lock, std::chrono::seconds(15), [&] { return (cfg->failure && cfg->flipStatus.flipPendingNum == 0) || cfg->flipStatus.count == target; }), "the flip did not complete"); + if (cfg->failure) std::rethrow_exception(cfg->failure); + check(cfg->flipStatus.flipArg == argument && cfg->flipStatus.currentBuffer == 0 && cfg->flipStatus.flipPendingNum == 0, "flip status is wrong"); + }; + const auto presentsAfter = [](std::uint64_t previous) { + const auto deadline = std::chrono::steady_clock::now() + std::chrono::seconds(5); + while (AgcDriver::VulkanDevice::PresentCounts().presents == previous && std::chrono::steady_clock::now() < deadline) std::this_thread::sleep_for(std::chrono::milliseconds(10)); + return AgcDriver::VulkanDevice::PresentCounts().presents; + }; + const auto start = AgcDriver::VulkanDevice::PresentCounts().presents; + flip(mainHandle, mainCfg, 1); + const auto afterMain = presentsAfter(start); + check(afterMain > start, "a main-bus flip was not presented"); + flip(overlayHandle, overlayCfg, 2); + flip(overlayHandle, overlayCfg, 3); + std::this_thread::sleep_for(std::chrono::milliseconds(500)); + check(AgcDriver::VulkanDevice::PresentCounts().presents == afterMain, "an overlay-bus flip replaced the main output"); + flip(mainHandle, mainCfg, 4); + check(presentsAfter(afterMain) > afterMain, "a main-bus flip after overlay flips was not presented"); + sceVideoOutUnregisterBuffers(overlayHandle, 0); + sceVideoOutUnregisterBuffers(mainHandle, 0); + sceVideoOutClose(overlayHandle); + sceVideoOutClose(mainHandle); + LibcRunShutdown_nid_postfix(); +} + void testUnavailable() { const auto message = expectFailure([] { sceVideoOutOpen(255, 0, 0, nullptr); }); check(expectFailure([] { sceVideoOutOpen(255, 0, 0, nullptr); }) == message, "a second open lost the presentation failure"); @@ -532,6 +658,8 @@ void testOneDevice() { int main(int argc, char** argv) { try { + std::filesystem::create_directories("app0/sce_sys"); + std::ofstream("app0/sce_sys/param.json", std::ios::binary) << R"({"titleId":"PPSA00000","localizedParameters":{"en-US":{"titleName":"Example"}},"downloadDataSize":0})"; if (argc == 2 && std::string(argv[1]) == "decode") testDecode(); else if (argc == 2 && std::string(argv[1]) == "controls") testControls(); else if (argc == 2 && std::string(argv[1]) == "present") testPresentation(); @@ -541,6 +669,9 @@ int main(int argc, char** argv) { else if (argc == 2 && std::string(argv[1]) == "aftergpu") testFlipAfterGpuWork(); else if (argc == 2 && std::string(argv[1]) == "unavailable") testUnavailable(); else if (argc == 2 && std::string(argv[1]) == "onedevice") testOneDevice(); + else if (argc == 2 && std::string(argv[1]) == "overlay") testOverlay(); + else if (argc == 2 && std::string(argv[1]) == "deferred") testDeferredBuffer(); + else if (argc == 2 && std::string(argv[1]) == "unregistered") testUnregisteredBuffer(); else testLifetime(argc == 2 && std::string(argv[1]) == "reopen"); std::puts("VideoOut flip tests passed"); return 0; @@ -548,6 +679,10 @@ int main(int argc, char** argv) { std::fprintf(stderr, "%s\n", error.what()); try { LibcRunShutdown_nid_postfix(); } catch (const std::exception& shutdown) { std::fprintf(stderr, "shutdown: %s\n", shutdown.what()); } + if (std::string(error.what()).find("Vulkan support") != std::string::npos && !std::getenv("ANYPS5_REQUIRE_DISPLAY")) { + std::printf("skipped, no display or Vulkan device: %s\n", error.what()); + return 77; + } return 1; } } diff --git a/core/libs/prx/libSceZlib/CMakeLists.txt b/core/libs/prx/libSceZlib/CMakeLists.txt new file mode 100644 index 000000000..6beeb7625 --- /dev/null +++ b/core/libs/prx/libSceZlib/CMakeLists.txt @@ -0,0 +1,19 @@ +add_library(libSceZlib SHARED EXCLUDE_FROM_ALL ${CMAKE_CURRENT_SOURCE_DIR}/Export.cpp) +target_include_directories(libSceZlib PRIVATE ${LIBS_INCLUDE_DIR} + ${CMAKE_SOURCE_DIR}/3rdparty/libjpeg-turbo/src/spng/zlib) +set_target_properties(libSceZlib PROPERTIES CXX_EXTENSIONS OFF) +configure_windows_unwind(libSceZlib) +target_link_libraries(libSceZlib PRIVATE jpeg_turbo) + +if(BUILD_TESTING) + add_test_executable(guest_zlib_tests ${LIBS_INCLUDE_DIR}/tests/GuestZlib.cpp) + target_include_directories(guest_zlib_tests PRIVATE ${LIBS_INCLUDE_DIR}) + target_link_libraries(guest_zlib_tests PRIVATE libSceZlib libc) + configure_windows_unwind(guest_zlib_tests) + add_test(NAME guest_zlib COMMAND guest_zlib_tests) + set_tests_properties(guest_zlib PROPERTIES TIMEOUT 30) + if(WIN32) + set_tests_properties(guest_zlib PROPERTIES ENVIRONMENT_MODIFICATION + "PATH=path_list_prepend:$") + endif() +endif() diff --git a/core/libs/prx/libSceZlib/Export.cpp b/core/libs/prx/libSceZlib/Export.cpp new file mode 100644 index 000000000..ba1067f8b --- /dev/null +++ b/core/libs/prx/libSceZlib/Export.cpp @@ -0,0 +1,100 @@ +#include "prx/libc/include/general/VabiMacros.hpp" +#include "zlib.h" +#include +#include +#include +#include +#include + +namespace { + +static_assert(sizeof(std::uint32_t) == 4 && sizeof(std::uint64_t) == 8 && sizeof(std::int32_t) == 4); + +constexpr std::int32_t NOT_INITIALIZED = static_cast(0x81120032); +constexpr std::int32_t ALREADY_INITIALIZED = static_cast(0x81120033); +constexpr std::int32_t NO_SPACE = static_cast(0x8112001C); +constexpr std::int32_t FATAL = static_cast(0x811200FF); + +struct Result { + std::uint64_t id; + std::uint32_t produced; + std::int32_t status; + bool retrieved = false; +}; + +std::mutex serviceMutex; +bool initialized = false; +std::uint64_t nextId = 1; +std::optional result; + +} // namespace + +extern "C" { + +std::int32_t APS5_VABI sceZlibInitialize(const void* buffer, std::uint64_t length) { + std::lock_guard lock(serviceMutex); + if (initialized) return ALREADY_INITIALIZED; + if ((buffer == nullptr) != (length == 0)) throw std::runtime_error("sceZlibInitialize: invalid work buffer"); + initialized = true; + return 0; +} + +std::int32_t APS5_VABI sceZlibFinalize() { + std::lock_guard lock(serviceMutex); + if (!initialized) return NOT_INITIALIZED; + if (result && !result->retrieved) throw std::runtime_error("sceZlibFinalize: outstanding result semantics unknown"); + result.reset(); + initialized = false; + return 0; +} + +std::int32_t APS5_VABI sceZlibInflate(const void* source, std::uint32_t sourceLength, + void* destination, std::uint32_t destinationLength, + std::uint64_t* requestId) { + std::lock_guard lock(serviceMutex); + if (!initialized) return NOT_INITIALIZED; + if (!source || !sourceLength || !destination || !requestId || !destinationLength || + destinationLength > 65536 || reinterpret_cast(destination) % 2048 != 0) + throw std::runtime_error("sceZlibInflate: invalid buffer or size"); + if (result && !result->retrieved) throw std::runtime_error("sceZlibInflate: multiple outstanding requests unsupported"); + if (nextId == std::numeric_limits::max()) throw std::runtime_error("sceZlibInflate: request IDs exhausted"); + z_stream stream{}; + stream.next_in = const_cast(static_cast(source)); + stream.avail_in = sourceLength; + stream.next_out = static_cast(destination); + stream.avail_out = destinationLength; + if (inflateInit(&stream) != Z_OK) throw std::runtime_error("sceZlibInflate: decoder initialization failed"); + const int decoded = inflate(&stream, Z_FINISH); + const auto produced = static_cast(stream.total_out); + const std::int32_t status = decoded == Z_STREAM_END ? 0 : + (decoded == Z_BUF_ERROR && stream.avail_out == 0 ? NO_SPACE : FATAL); + const int ended = inflateEnd(&stream); + if (ended != Z_OK) throw std::runtime_error("sceZlibInflate: decoder cleanup failed"); + result = Result{nextId++, produced, status}; + *requestId = result->id; + return 0; +} + +std::int32_t APS5_VABI sceZlibWaitForDone(std::uint64_t* requestId, std::uint32_t* timeout) { + std::lock_guard lock(serviceMutex); + if (!initialized) return NOT_INITIALIZED; + if (timeout) throw std::runtime_error("sceZlibWaitForDone: timeout semantics unknown"); + if (!requestId || !result || result->retrieved) throw std::runtime_error("sceZlibWaitForDone: no outstanding request"); + *requestId = result->id; + return 0; +} + +std::int32_t APS5_VABI sceZlibGetResult(std::uint64_t requestId, std::uint32_t* destinationLength, + std::int32_t* status) { + std::lock_guard lock(serviceMutex); + if (!initialized) return NOT_INITIALIZED; + if (!destinationLength || !status || !result || result->id != requestId) + throw std::runtime_error("sceZlibGetResult: invalid request or output"); + if (result->retrieved) throw std::runtime_error("sceZlibGetResult: repeated retrieval semantics unknown"); + *destinationLength = result->produced; + *status = result->status; + result->retrieved = true; + return 0; +} + +} diff --git a/core/libs/prx/libc/CMakeLists.txt b/core/libs/prx/libc/CMakeLists.txt index 30aaf71ba..24a870408 100644 --- a/core/libs/prx/libc/CMakeLists.txt +++ b/core/libs/prx/libc/CMakeLists.txt @@ -9,6 +9,7 @@ target_sources(libc PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/src/AllocatingStrings.cpp ${CMAKE_CURRENT_SOURCE_DIR}/src/WideConversion.cpp ${CMAKE_CURRENT_SOURCE_DIR}/src/Multibyte.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/src/RuneLocale.cpp ${CMAKE_CURRENT_SOURCE_DIR}/src/StringExtensions.cpp ${CMAKE_CURRENT_SOURCE_DIR}/src/Math.cpp ${CMAKE_CURRENT_SOURCE_DIR}/src/Time.cpp @@ -32,7 +33,9 @@ target_sources(libc PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/src/Environment.cpp ${CMAKE_CURRENT_SOURCE_DIR}/src/Directory.cpp ${CMAKE_CURRENT_SOURCE_DIR}/src/Filesystem.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/src/Rename.cpp ${CMAKE_CURRENT_SOURCE_DIR}/src/Errors.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/src/Syslog.cpp ${CMAKE_CURRENT_SOURCE_DIR}/src/CxxAbiSupport.cpp ${CMAKE_CURRENT_SOURCE_DIR}/src/exception/Exports.cpp ) @@ -56,6 +59,16 @@ if(BUILD_TESTING) set_tests_properties(guest_time_conversion PROPERTIES ENVIRONMENT_MODIFICATION "PATH=path_list_prepend:$") endif() + add_test_executable(precise_wait_tests ${LIBS_INCLUDE_DIR}/tests/PreciseWait.cpp) + target_include_directories(precise_wait_tests PRIVATE ${LIBS_INCLUDE_DIR}) + target_link_libraries(precise_wait_tests PRIVATE libc) + configure_windows_unwind(precise_wait_tests) + add_test(NAME precise_wait COMMAND precise_wait_tests) + set_tests_properties(precise_wait PROPERTIES TIMEOUT 10) + if(WIN32) + set_tests_properties(precise_wait PROPERTIES ENVIRONMENT_MODIFICATION + "PATH=path_list_prepend:$") + endif() add_test_executable(guest_scan_count_tests ${LIBS_INCLUDE_DIR}/tests/GuestScanCount.cpp) target_include_directories(guest_scan_count_tests PRIVATE ${LIBS_INCLUDE_DIR}) target_link_libraries(guest_scan_count_tests PRIVATE libc) @@ -224,6 +237,16 @@ if(BUILD_TESTING) set_tests_properties(guest_standard_streams PROPERTIES ENVIRONMENT_MODIFICATION "PATH=path_list_prepend:$") endif() + add_test_executable(file_streams_tests ${LIBS_INCLUDE_DIR}/tests/FileStreams.cpp) + target_include_directories(file_streams_tests PRIVATE ${LIBS_INCLUDE_DIR}) + target_link_libraries(file_streams_tests PRIVATE libc) + configure_windows_unwind(file_streams_tests) + add_test(NAME file_streams COMMAND file_streams_tests file_streams.tmp) + set_tests_properties(file_streams PROPERTIES TIMEOUT 20) + if(WIN32) + set_tests_properties(file_streams PROPERTIES ENVIRONMENT_MODIFICATION + "PATH=path_list_prepend:$") + endif() add_test_executable(guest_error_tests ${LIBS_INCLUDE_DIR}/tests/GuestErrors.cpp) target_include_directories(guest_error_tests PRIVATE ${LIBS_INCLUDE_DIR}) target_link_libraries(guest_error_tests PRIVATE libc) @@ -233,6 +256,15 @@ if(BUILD_TESTING) set_tests_properties(guest_errors PROPERTIES ENVIRONMENT_MODIFICATION "PATH=path_list_prepend:$") endif() + add_test_executable(guest_syslog_tests ${LIBS_INCLUDE_DIR}/tests/GuestSyslog.cpp) + target_include_directories(guest_syslog_tests PRIVATE ${LIBS_INCLUDE_DIR}) + target_link_libraries(guest_syslog_tests PRIVATE libc) + configure_windows_unwind(guest_syslog_tests) + add_test(NAME guest_syslog COMMAND guest_syslog_tests) + if(WIN32) + set_tests_properties(guest_syslog PROPERTIES ENVIRONMENT_MODIFICATION + "PATH=path_list_prepend:$") + endif() add_test_executable(mspace_tests ${LIBS_INCLUDE_DIR}/tests/Mspace.cpp) target_include_directories(mspace_tests PRIVATE ${LIBS_INCLUDE_DIR}) target_link_libraries(mspace_tests PRIVATE libc) @@ -251,6 +283,15 @@ if(BUILD_TESTING) set_tests_properties(guest_formatting PROPERTIES ENVIRONMENT_MODIFICATION "PATH=path_list_prepend:$") endif() + add_test_executable(guest_new_handler_tests ${LIBS_INCLUDE_DIR}/tests/GuestNewHandler.cpp) + target_include_directories(guest_new_handler_tests PRIVATE ${LIBS_INCLUDE_DIR}) + target_link_libraries(guest_new_handler_tests PRIVATE libc) + configure_windows_unwind(guest_new_handler_tests) + add_test(NAME guest_new_handler COMMAND guest_new_handler_tests) + if(WIN32) + set_tests_properties(guest_new_handler PROPERTIES ENVIRONMENT_MODIFICATION + "PATH=path_list_prepend:$") + endif() add_test_executable(guest_search_tests ${LIBS_INCLUDE_DIR}/tests/GuestSearch.cpp) target_include_directories(guest_search_tests PRIVATE ${LIBS_INCLUDE_DIR}) target_link_libraries(guest_search_tests PRIVATE libc) @@ -278,6 +319,15 @@ if(BUILD_TESTING) set_tests_properties(guest_strings PROPERTIES ENVIRONMENT_MODIFICATION "PATH=path_list_prepend:$") endif() + add_test_executable(guest_wcsrtombs_tests ${LIBS_INCLUDE_DIR}/tests/GuestWcsrtombs.cpp) + target_include_directories(guest_wcsrtombs_tests PRIVATE ${LIBS_INCLUDE_DIR}) + target_link_libraries(guest_wcsrtombs_tests PRIVATE libc) + configure_windows_unwind(guest_wcsrtombs_tests) + add_test(NAME guest_wcsrtombs COMMAND guest_wcsrtombs_tests) + if(WIN32) + set_tests_properties(guest_wcsrtombs PROPERTIES ENVIRONMENT_MODIFICATION + "PATH=path_list_prepend:$") + endif() add_test_executable(guest_wcsrtombs_s_tests ${LIBS_INCLUDE_DIR}/tests/GuestWcsrtombsS.cpp) target_include_directories(guest_wcsrtombs_s_tests PRIVATE ${LIBS_INCLUDE_DIR}) target_link_libraries(guest_wcsrtombs_s_tests PRIVATE libc) @@ -296,6 +346,10 @@ if(BUILD_TESTING) target_include_directories(exception_personality_tests PRIVATE ${LIBS_INCLUDE_DIR}) target_link_libraries(exception_personality_tests PRIVATE libc) add_test(NAME exception_personality COMMAND exception_personality_tests) + add_test_executable(guest_exit_order_tests ${LIBS_INCLUDE_DIR}/tests/GuestExitOrder.cpp) + target_include_directories(guest_exit_order_tests PRIVATE ${LIBS_INCLUDE_DIR}) + target_link_libraries(guest_exit_order_tests PRIVATE libc) + add_test(NAME guest_exit_order COMMAND guest_exit_order_tests) if(NOT APPLE) enable_language(C) add_test_executable(exception_c_cleanup_tests ${LIBS_INCLUDE_DIR}/tests/ExceptionCCleanup.cpp ${LIBS_INCLUDE_DIR}/tests/ExceptionCCleanup.c) @@ -338,6 +392,17 @@ if(BUILD_TESTING) set_tests_properties(application_heap_replacement application_heap_invalid_metadata application_heap_default application_heap_exit_order PROPERTIES ENVIRONMENT_MODIFICATION "PATH=path_list_prepend:$") endif() + add_test_executable(application_heap_stats_tests ${LIBS_INCLUDE_DIR}/tests/ApplicationHeapStats.cpp) + target_include_directories(application_heap_stats_tests PRIVATE ${LIBS_INCLUDE_DIR}) + target_link_libraries(application_heap_stats_tests PRIVATE libc) + configure_windows_unwind(application_heap_stats_tests) + add_test(NAME application_heap_stats COMMAND application_heap_stats_tests) + add_test(NAME application_heap_stats_default COMMAND application_heap_stats_tests default) + add_test(NAME application_heap_stats_missing COMMAND application_heap_stats_tests missing) + if(WIN32) + set_tests_properties(application_heap_stats application_heap_stats_default application_heap_stats_missing + PROPERTIES ENVIRONMENT_MODIFICATION "PATH=path_list_prepend:$") + endif() if(CMAKE_SYSTEM_NAME STREQUAL "Linux") add_test_executable(cpu_topology_tests ${LIBS_INCLUDE_DIR}/tests/CpuTopology.cpp) target_include_directories(cpu_topology_tests PRIVATE ${LIBS_INCLUDE_DIR}) @@ -372,7 +437,44 @@ if(BUILD_TESTING) endif() endif() +add_test_executable(guest_rune_locale_tests EXCLUDE_FROM_ALL ${LIBS_INCLUDE_DIR}/tests/GuestRuneLocale.cpp) +target_include_directories(guest_rune_locale_tests PRIVATE ${LIBS_INCLUDE_DIR}) +target_link_libraries(guest_rune_locale_tests PRIVATE libc) +configure_windows_unwind(guest_rune_locale_tests) +if(BUILD_TESTING) + set_target_properties(guest_rune_locale_tests PROPERTIES EXCLUDE_FROM_ALL FALSE) + add_test(NAME guest_rune_locale COMMAND guest_rune_locale_tests) + if(WIN32) + set_tests_properties(guest_rune_locale PROPERTIES ENVIRONMENT_MODIFICATION "PATH=path_list_prepend:$") + endif() +endif() + if(MINGW) + if(BUILD_TESTING) + enable_language(ASM) + add_test_executable(guest_message_destructor_abi_tests + ${LIBS_INCLUDE_DIR}/tests/GuestMessageDestructorAbi.cpp + ${LIBS_INCLUDE_DIR}/tests/GuestMessageDestructorAbi.S) + target_include_directories(guest_message_destructor_abi_tests PRIVATE ${LIBS_INCLUDE_DIR}) + target_link_libraries(guest_message_destructor_abi_tests PRIVATE libc) + configure_windows_unwind(guest_message_destructor_abi_tests) + foreach(mode IN ITEMS control decoy zero release regex-decoy regex-zero + unshared-decoy unshared-zero regex-unshared-decoy regex-unshared-zero) + add_test(NAME guest_message_destructor_${mode} COMMAND guest_message_destructor_abi_tests ${mode}) + set_tests_properties(guest_message_destructor_${mode} PROPERTIES TIMEOUT 10 + ENVIRONMENT_MODIFICATION "PATH=path_list_prepend:$") + endforeach() + foreach(kind IN ITEMS message plain) + foreach(operation IN ITEMS what destroy delete) + foreach(register IN ITEMS decoy zero) + set(mode "vtable-${kind}-${operation}-${register}") + add_test(NAME guest_exception_${mode} COMMAND guest_message_destructor_abi_tests ${mode}) + set_tests_properties(guest_exception_${mode} PROPERTIES TIMEOUT 10 + ENVIRONMENT_MODIFICATION "PATH=path_list_prepend:$") + endforeach() + endforeach() + endforeach() + endif() foreach(runtimeLinkFlag IN ITEMS -static-libgcc -static-libstdc++ -static) string(REPLACE "${runtimeLinkFlag}" "" CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS}") endforeach() diff --git a/core/libs/prx/libc/Export.cpp b/core/libs/prx/libc/Export.cpp index df3cffffd..fb67f24bb 100644 --- a/core/libs/prx/libc/Export.cpp +++ b/core/libs/prx/libc/Export.cpp @@ -27,17 +27,15 @@ int APS5_VABI _ZSt14_Atomic_assertPKcS0__nid_postfix() { // Live Cyberpunk 2077 import used in an fopen/fseek/ftell-like file-size // idiom as (handle, 0, 2); returning 0 reports success. -APS5_EXPORT("rWSuTWY2JN0", libcCyberUnknown18); -int APS5_VABI libcCyberUnknown18(void) { - NotImplemented_nid_no_patch("rWSuTWY2JN0"); +int APS5_VABI vsnprintf_s_nid_postfix() { + NotImplemented_nid_no_patch(__func__); return 0; } // Live Cyberpunk 2077 import used as (handle, 0, 0) in the same file-size // idiom; returning 0 reports success. -APS5_EXPORT("tfNbpqL3D0M", libcCyberUnknown19); -int APS5_VABI libcCyberUnknown19(void) { - NotImplemented_nid_no_patch("tfNbpqL3D0M"); +int APS5_VABI vsscanf_s_nid_postfix() { + NotImplemented_nid_no_patch(__func__); return 0; } @@ -48,21 +46,18 @@ std::uint64_t APS5_VABI libcCyberUnknown02(void) { return 0; } -APS5_EXPORT("5Lf51jvohTQ", libcCyberUnknown03); -std::uint64_t APS5_VABI libcCyberUnknown03(void) { - NotImplemented_nid_no_patch("5Lf51jvohTQ"); +std::uint64_t APS5_VABI _Mtx_destroy_nid_postfix() { + NotImplemented_nid_no_patch(__func__); return 0; } -APS5_EXPORT("7yMFgcS8EPA", libcCyberUnknown05); -std::uint64_t APS5_VABI libcCyberUnknown05(void) { - NotImplemented_nid_no_patch("7yMFgcS8EPA"); +std::uint64_t APS5_VABI _Cnd_destroy_nid_postfix() { + NotImplemented_nid_no_patch(__func__); return 0; } -APS5_EXPORT("CyXs2l-1kNA", libcCyberUnknown07); -std::uint64_t APS5_VABI libcCyberUnknown07(void) { - NotImplemented_nid_no_patch("CyXs2l-1kNA"); +std::uint64_t APS5_VABI _Iswctype_nid_postfix() { + NotImplemented_nid_no_patch(__func__); return 0; } @@ -72,45 +67,43 @@ std::uint64_t APS5_VABI libcCyberUnknown08(void) { return 0; } -APS5_EXPORT("JhVR7D4Ax6Y", libcCyberUnknown09); -std::uint64_t APS5_VABI libcCyberUnknown09(void) { - NotImplemented_nid_no_patch("JhVR7D4Ax6Y"); +std::uint64_t APS5_VABI _WStoul_nid_postfix() { + NotImplemented_nid_no_patch(__func__); return 0; } -APS5_EXPORT("SreZybSRWpU", libcCyberUnknown10); -std::uint64_t APS5_VABI libcCyberUnknown10(void) { - NotImplemented_nid_no_patch("SreZybSRWpU"); +std::uint64_t APS5_VABI _Cnd_init_nid_postfix() { + NotImplemented_nid_no_patch(__func__); return 0; } -APS5_EXPORT("VsP3daJgmVA", libcCyberUnknown11); -std::uint64_t APS5_VABI libcCyberUnknown11(void) { - NotImplemented_nid_no_patch("VsP3daJgmVA"); +std::uint64_t APS5_VABI _Cnd_broadcast_nid_postfix() { + NotImplemented_nid_no_patch(__func__); return 0; } -APS5_EXPORT("YaHc3GS7y7g", libcCyberUnknown12); -std::uint64_t APS5_VABI libcCyberUnknown12(void) { - NotImplemented_nid_no_patch("YaHc3GS7y7g"); +std::uint64_t APS5_VABI _Mtx_init_nid_postfix() { + NotImplemented_nid_no_patch(__func__); return 0; } -APS5_EXPORT("gTuXQwP9rrs", libcCyberUnknown13); -std::uint64_t APS5_VABI libcCyberUnknown13(void) { - NotImplemented_nid_no_patch("gTuXQwP9rrs"); +std::uint64_t APS5_VABI _Mtx_unlock_nid_postfix() { + NotImplemented_nid_no_patch(__func__); return 0; } -APS5_EXPORT("iS4aWbUonl0", libcCyberUnknown14); -std::uint64_t APS5_VABI libcCyberUnknown14(void) { - NotImplemented_nid_no_patch("iS4aWbUonl0"); +std::uint64_t APS5_VABI _Mtx_lock_nid_postfix() { + NotImplemented_nid_no_patch(__func__); return 0; } -APS5_EXPORT("vEaqE-7IZYc", libcCyberUnknown16); -std::uint64_t APS5_VABI libcCyberUnknown16(void) { - NotImplemented_nid_no_patch("vEaqE-7IZYc"); +std::uint64_t APS5_VABI _Cnd_wait_nid_postfix() { + NotImplemented_nid_no_patch(__func__); + return 0; +} + +int APS5_VABI swscanf_s_nid_postfix(void) { + NotImplemented_nid_no_patch(__func__); return 0; } } diff --git a/core/libs/prx/libc/include/ApplicationHeap.hpp b/core/libs/prx/libc/include/ApplicationHeap.hpp index 79398ada9..22212a31d 100644 --- a/core/libs/prx/libc/include/ApplicationHeap.hpp +++ b/core/libs/prx/libc/include/ApplicationHeap.hpp @@ -15,6 +15,7 @@ void* ApplicationHeapAlign_nid_no_patch(std::size_t alignment, std::size_t bytes void* ApplicationHeapRealign_nid_no_patch(void* pointer, std::size_t bytes, std::size_t alignment); void* ApplicationHeapCalloc_nid_no_patch(std::size_t count, std::size_t bytes); int ApplicationHeapPosixAlign_nid_no_patch(void** pointer, std::size_t alignment, std::size_t bytes); +int ApplicationHeapStatsFast_nid_no_patch(void* stats); } diff --git a/core/libs/prx/libc/include/FileStream.hpp b/core/libs/prx/libc/include/FileStream.hpp index 9c9c4e87a..d97cdf3b3 100644 --- a/core/libs/prx/libc/include/FileStream.hpp +++ b/core/libs/prx/libc/include/FileStream.hpp @@ -2,7 +2,10 @@ #define CORE_LIBS_PRX_LIBC_INCLUDE_FILESTREAM_HPP #include +#include #include +#include +#include #include #include #include @@ -66,11 +69,17 @@ public: } GuestFilePrefix& GuestState() { return _guest; } - bool Reopen(const char* filename, const char* mode) { + bool Reopen(const std::filesystem::path& filename, const char* mode) { auto* previous = GetHandle(); _guest = {}; encodingError = false; - _handle = std::freopen(filename, mode, previous); +#ifdef _WIN32 + std::wstring wideMode; + for (const char character : std::string_view(mode)) wideMode.push_back(static_cast(character)); + _handle = ::_wfreopen(filename.c_str(), wideMode.c_str(), previous); +#else + _handle = std::freopen(filename.c_str(), mode, previous); +#endif if (!_handle) return false; _guest.flags = 0x10; #ifdef _WIN32 diff --git a/core/libs/prx/libc/include/FilesystemError.hpp b/core/libs/prx/libc/include/FilesystemError.hpp new file mode 100644 index 000000000..e31bedd2a --- /dev/null +++ b/core/libs/prx/libc/include/FilesystemError.hpp @@ -0,0 +1,25 @@ +#ifndef CORE_LIBS_PRX_LIBC_INCLUDE_FILESYSTEMERROR_HPP +#define CORE_LIBS_PRX_LIBC_INCLUDE_FILESYSTEMERROR_HPP + +#include + +inline int FilesystemError(const std::error_code& error) { + if (error == std::errc::no_such_file_or_directory) return 2; + if (error == std::errc::permission_denied) return 13; + if (error == std::errc::operation_not_permitted) return 1; + if (error == std::errc::not_a_directory) return 20; + if (error == std::errc::is_a_directory) return 21; + if (error == std::errc::directory_not_empty) return 66; + if (error == std::errc::device_or_resource_busy) return 16; + if (error == std::errc::read_only_file_system) return 30; + if (error == std::errc::filename_too_long) return 63; + if (error == std::errc::too_many_symbolic_link_levels) return 62; + if (error == std::errc::not_enough_memory) return 12; + if (error == std::errc::invalid_argument) return 22; + if (error == std::errc::cross_device_link) return 18; + if (error == std::errc::file_exists) return 17; + if (error == std::errc::no_space_on_device) return 28; + return 5; +} + +#endif diff --git a/core/libs/prx/libc/include/General.hpp b/core/libs/prx/libc/include/General.hpp index 3aa9dcbc6..7dc00888d 100644 --- a/core/libs/prx/libc/include/General.hpp +++ b/core/libs/prx/libc/include/General.hpp @@ -11,6 +11,7 @@ extern "C" void NotImplemented_nid_no_patch(const char* funcName); extern "C" void CxaFinalize_nid_no_patch(void* dsoHandle); +extern "C" bool StopAtBinaryPrefix_nid_no_patch(const char* str, char** endptr, int base); extern "C" std::filesystem::path ResolvePath_nid_no_patch(const char* path); // Guest path aliases: a guest prefix such as "/savedata0" (a save-data mount point) resolves to a host diff --git a/core/libs/prx/libc/include/GuestAllocations.hpp b/core/libs/prx/libc/include/GuestAllocations.hpp index 81c711b9e..90c911034 100644 --- a/core/libs/prx/libc/include/GuestAllocations.hpp +++ b/core/libs/prx/libc/include/GuestAllocations.hpp @@ -18,6 +18,7 @@ struct Range { std::uint64_t allocationAddress; std::size_t allocationBytes; bool releasable = true; + bool gpu = false; }; using Lease = std::vector>; @@ -25,17 +26,22 @@ using Lease = std::vector>; extern "C" { void* GuestAllocationsBegin_nid_postfix(); void GuestAllocationsRegisterMainImage_nid_postfix(void* mutation); +#ifdef _WIN32 +void GuestAllocationsRegisterImage_nid_postfix(void* mutation, const void* image); +void GuestAllocationsUnregisterImage_nid_postfix(void* mutation, const void* image); +#endif void GuestAllocationsEnd_nid_postfix(void* mutation) noexcept; -void GuestAllocationsAdd_nid_postfix(void* mutation, void* pointer, std::size_t bytes, bool readable, bool writable); +void GuestAllocationsAdd_nid_postfix(void* mutation, void* pointer, std::size_t bytes, bool readable, bool writable, bool gpu); void GuestAllocationsRequireUnpinned_nid_postfix(void* mutation, const void* pointer, std::size_t bytes); void GuestAllocationsRequireAvailable_nid_postfix(void* mutation, const void* pointer, std::size_t bytes); bool GuestAllocationsCovers_nid_postfix(void* mutation, const void* pointer, std::size_t bytes); bool GuestAllocationsOverlaps_nid_postfix(void* mutation, const void* pointer, std::size_t bytes); Range GuestAllocationsFind_nid_postfix(void* mutation, const void* pointer); void GuestAllocationsRemove_nid_postfix(void* mutation, const void* pointer); -void GuestAllocationsProtect_nid_postfix(void* mutation, const void* pointer, std::size_t bytes, bool readable, bool writable, const std::function& apply); +void GuestAllocationsProtect_nid_postfix(void* mutation, const void* pointer, std::size_t bytes, bool readable, bool writable, bool gpu, const std::function& apply); void GuestAllocationsUnmap_nid_postfix(void* mutation, const void* pointer, std::size_t bytes, const std::function& apply); Lease GuestAllocationsAcquire_nid_postfix(); +Lease GuestAllocationsAcquireAll_nid_postfix(); // Changes whenever a mutation ends, so callers can cache facts about guest mappings between changes. std::uint64_t GuestAllocationsGeneration_nid_postfix(); // Precise invalidation for such caches: the callback runs, after the generation changed, for every @@ -47,7 +53,7 @@ void GuestAllocationsInvalidate_nid_postfix(std::uintptr_t address, std::size_t // and finishes the GPU work that holds leases; it returns whether it finished any (a round that found // nothing to wait for counts against the same bound as a plain spin). Without one the mutation spins // until the lease is dropped by another thread. -void GuestAllocationsSetPinWaiter_nid_postfix(bool (*callback)()); +void GuestAllocationsSetPinWaiter_nid_postfix(bool (*callback)(std::uintptr_t address, std::size_t bytes)); } class Mutation { @@ -57,7 +63,11 @@ public: Mutation(const Mutation&) = delete; Mutation& operator=(const Mutation&) = delete; void RegisterMainImage() { GuestAllocationsRegisterMainImage_nid_postfix(handle); } - void Add(void* pointer, std::size_t bytes, bool readable, bool writable) { GuestAllocationsAdd_nid_postfix(handle, pointer, bytes, readable, writable); } +#ifdef _WIN32 + void RegisterImage(const void* image) { GuestAllocationsRegisterImage_nid_postfix(handle, image); } + void UnregisterImage(const void* image) { GuestAllocationsUnregisterImage_nid_postfix(handle, image); } +#endif + void Add(void* pointer, std::size_t bytes, bool readable, bool writable, bool gpu) { GuestAllocationsAdd_nid_postfix(handle, pointer, bytes, readable, writable, gpu); } void RequireUnpinned(const void* pointer, std::size_t bytes) const { GuestAllocationsRequireUnpinned_nid_postfix(handle, pointer, bytes); } void RequireAvailable(const void* pointer, std::size_t bytes) const { GuestAllocationsRequireAvailable_nid_postfix(handle, pointer, bytes); } bool Covers(const void* pointer, std::size_t bytes) const { return GuestAllocationsCovers_nid_postfix(handle, pointer, bytes); } @@ -65,7 +75,7 @@ public: Range Find(const void* pointer) const { return GuestAllocationsFind_nid_postfix(handle, pointer); } void Remove(const void* pointer) { GuestAllocationsRemove_nid_postfix(handle, pointer); } void Unmap(const void* pointer, std::size_t bytes, const std::function& apply) { GuestAllocationsUnmap_nid_postfix(handle, pointer, bytes, apply); } - void Protect(const void* pointer, std::size_t bytes, bool readable, bool writable, const std::function& apply) { GuestAllocationsProtect_nid_postfix(handle, pointer, bytes, readable, writable, apply); } + void Protect(const void* pointer, std::size_t bytes, bool readable, bool writable, bool gpu, const std::function& apply) { GuestAllocationsProtect_nid_postfix(handle, pointer, bytes, readable, writable, gpu, apply); } private: void* handle; diff --git a/core/libs/prx/libc/include/GuestArena.hpp b/core/libs/prx/libc/include/GuestArena.hpp index 0cbeac68d..2f02aeebd 100644 --- a/core/libs/prx/libc/include/GuestArena.hpp +++ b/core/libs/prx/libc/include/GuestArena.hpp @@ -10,6 +10,7 @@ namespace GuestArena { #ifndef _WIN32 using SharedBackingResolver = bool (*)(std::uintptr_t address, std::size_t bytes, int* file, std::uint64_t* offset); +using SharedBackingWriter = bool (*)(std::uintptr_t address, const void* source, std::size_t bytes); #endif extern "C" { @@ -23,8 +24,10 @@ void GuestArenaRelease_nid_postfix(const void* pointer, std::size_t bytes); // The reserved range, and whether it was reserved with page write watching (Windows MEM_WRITE_WATCH). void GuestArenaRange_nid_postfix(std::uintptr_t* base, std::size_t* bytes); bool GuestArenaWriteWatched_nid_postfix(); -#ifdef _WIN32 void GuestArenaSetProtection_nid_postfix(std::uintptr_t address, std::size_t bytes, std::uint32_t protection); +void GuestArenaCommit_nid_postfix(void* pointer, std::size_t bytes, std::uint32_t protection, std::size_t granule); +void GuestArenaReset_nid_postfix(void* pointer, std::size_t bytes); +#ifdef _WIN32 bool GuestArenaHandleWrite_nid_postfix(std::uintptr_t address); void GuestArenaPinWritable_nid_postfix(const void* pointer, std::size_t bytes); void GuestArenaUnpinWritable_nid_postfix(const void* pointer, std::size_t bytes); @@ -33,13 +36,13 @@ bool GuestArenaCollectWrites_nid_postfix(std::uintptr_t address, std::size_t byt bool GuestArenaHostRegionOverlaps_nid_postfix(std::uintptr_t address, std::size_t bytes); std::uint64_t GuestArenaCommitGeneration_nid_postfix(); void GuestArenaSetPrivateMappingObserver_nid_postfix(void (*callback)(std::uintptr_t address, std::size_t bytes, std::uint64_t generation)); -void GuestArenaCommit_nid_postfix(void* pointer, std::size_t bytes, std::uint32_t protection, std::size_t granule); -void GuestArenaReset_nid_postfix(void* pointer, std::size_t bytes); void GuestArenaMap_nid_postfix(void* pointer, std::size_t bytes, void* section, std::uint64_t offset, std::uint32_t protection); void* GuestArenaMapAlias_nid_postfix(std::uintptr_t address, std::size_t bytes); void GuestArenaUnmapAlias_nid_postfix(void* alias); #else void GuestArenaSetSharedBacking_nid_postfix(SharedBackingResolver resolver); +void GuestArenaSetSharedBackingWriter_nid_no_patch(SharedBackingWriter writer); +bool GuestArenaWriteSharedBacking_nid_no_patch(std::uintptr_t address, const void* source, std::size_t bytes); bool GuestArenaSharedBacking_nid_postfix(std::uintptr_t address, std::size_t bytes, int* file, std::uint64_t* offset); #endif bool GuestArenaBeginHostWrite_nid_postfix(void* pointer, std::size_t bytes); diff --git a/core/libs/prx/libc/include/GuestTime.hpp b/core/libs/prx/libc/include/GuestTime.hpp new file mode 100644 index 000000000..99150ba72 --- /dev/null +++ b/core/libs/prx/libc/include/GuestTime.hpp @@ -0,0 +1,20 @@ +#pragma once +#include +#include + +struct GuestTm { + int tm_sec; + int tm_min; + int tm_hour; + int tm_mday; + int tm_mon; + int tm_year; + int tm_wday; + int tm_yday; + int tm_isdst; + std::int64_t tm_gmtoff; + const char* tm_zone; +}; +static_assert(offsetof(GuestTm, tm_gmtoff) == 40); +static_assert(offsetof(GuestTm, tm_zone) == 48); +static_assert(sizeof(GuestTm) == 56); diff --git a/core/libs/prx/libc/include/GuestWriteWatch.hpp b/core/libs/prx/libc/include/GuestWriteWatch.hpp index 4e5d98dd5..95d5df1fc 100644 --- a/core/libs/prx/libc/include/GuestWriteWatch.hpp +++ b/core/libs/prx/libc/include/GuestWriteWatch.hpp @@ -13,6 +13,12 @@ void GuestWriteWatchRegister_nid_postfix(const void* pointer, std::size_t bytes) bool GuestWriteWatchUnregister_nid_postfix(const void* pointer, std::size_t bytes); bool GuestWriteWatchCovers_nid_postfix(std::uintptr_t address, std::size_t bytes); bool GuestWriteWatchCollect_nid_postfix(std::uintptr_t address, std::size_t bytes, void (*written)(void* context, std::uintptr_t begin, std::uintptr_t end), void* context); +void GuestWriteWatchBeginHostWrite_nid_postfix(const void* pointer, std::size_t bytes); +void GuestWriteWatchEndHostWrite_nid_postfix(const void* pointer, std::size_t bytes); +void GuestWriteWatchProtectionChanged_nid_postfix(const void* pointer, std::size_t bytes, int protection); +int GuestWriteWatchProtection_nid_postfix(std::uintptr_t address, std::uintptr_t limit, std::uintptr_t* runEnd); +bool GuestWriteWatchCollectArmed_nid_postfix(std::uintptr_t address, std::size_t bytes, void (*written)(void* context, std::uintptr_t begin, std::uintptr_t end), void* context); +bool GuestWriteWatchCollectFresh_nid_postfix(std::uintptr_t address, std::size_t bytes, void (*written)(void* context, std::uintptr_t begin, std::uintptr_t end), void* context); } diff --git a/core/libs/prx/libc/include/PreciseWait.hpp b/core/libs/prx/libc/include/PreciseWait.hpp index 06196d05f..e9fc32421 100644 --- a/core/libs/prx/libc/include/PreciseWait.hpp +++ b/core/libs/prx/libc/include/PreciseWait.hpp @@ -37,4 +37,11 @@ inline void PreciseSleepUs(unsigned long long micros) { std::this_thread::sleep_for(std::chrono::microseconds(micros)); } +template +inline void PreciseSleepUntil(const std::chrono::time_point& deadline) { + for (auto now = TClock::now(); now < deadline; now = TClock::now()) { + PreciseSleepUs(static_cast(std::chrono::ceil(deadline - now).count())); + } +} + #endif diff --git a/core/libs/prx/libc/include/Shutdown.hpp b/core/libs/prx/libc/include/Shutdown.hpp index dc4007098..a7618a032 100644 --- a/core/libs/prx/libc/include/Shutdown.hpp +++ b/core/libs/prx/libc/include/Shutdown.hpp @@ -12,5 +12,6 @@ extern "C" void LibcRequestShutdown_nid_postfix(); extern "C" void LibcRequestExit_nid_postfix(int code); extern "C" [[noreturn]] void LibcAwaitExit_nid_postfix(); extern "C" [[noreturn]] void LibcExit_nid_no_patch(int code); +extern "C" [[noreturn]] void LibcTerminate_nid_no_patch(int code); #endif diff --git a/core/libs/prx/libc/include/WindowsFormatting.hpp b/core/libs/prx/libc/include/WindowsFormatting.hpp index 43c99d111..45dbd178e 100644 --- a/core/libs/prx/libc/include/WindowsFormatting.hpp +++ b/core/libs/prx/libc/include/WindowsFormatting.hpp @@ -210,7 +210,8 @@ inline int FormatWindows(char* buffer, size_t size, const char* format, const vo } else if ((conversion == 'c' && length == "l") || (conversion == 'C' && length.empty())) { std::string utf8; AppendUtf8(utf8, static_cast(args.Next())); - output.Value(spec + 's', utf8.c_str()); + if (utf8[0] == 0) output.Value(spec + 'c', 0); + else output.Value(spec + 's', utf8.c_str()); } else if (conversion == 's' && (length.empty() || length == "h")) { const char* value = args.Next(); if (!value) value = "(null)"; diff --git a/core/libs/prx/libc/include/WindowsMappings.hpp b/core/libs/prx/libc/include/WindowsMappings.hpp index 3c423d148..b518d9cb8 100644 --- a/core/libs/prx/libc/include/WindowsMappings.hpp +++ b/core/libs/prx/libc/include/WindowsMappings.hpp @@ -221,6 +221,7 @@ public: const auto capacity = *count; *count = 0; const auto end = address + bytes; + if (clear && !collectable(address, end)) return false; for (auto cursor = address; cursor < end;) { const auto nextClean = cleanRanges.upper_bound(cursor); if (nextClean != cleanRanges.begin()) { @@ -275,6 +276,21 @@ public: } private: + bool collectable(std::uintptr_t address, std::uintptr_t end) { + for (auto cursor = address; cursor < end;) { + const auto base = cursor & ~(pageBytes - 1); + if (const auto found = views.find(base); found != views.end()) { + if (found->second.protection == PAGE_NOACCESS) return false; + cursor = std::min(end, base + pageBytes); + continue; + } + const auto memory = query(cursor); + if (memory.State != MEM_COMMIT || memory.Type != MEM_PRIVATE) return false; + cursor = std::min(end, reinterpret_cast(memory.BaseAddress) + memory.RegionSize); + } + return true; + } + static constexpr std::size_t pageBytes = 0x4000; struct SharedPage { std::uint64_t generation = 1; diff --git a/core/libs/prx/libc/include/WindowsWideFormatting.hpp b/core/libs/prx/libc/include/WindowsWideFormatting.hpp deleted file mode 100644 index 8d4baba34..000000000 --- a/core/libs/prx/libc/include/WindowsWideFormatting.hpp +++ /dev/null @@ -1,99 +0,0 @@ -#ifndef CORE_LIBS_PRX_LIBC_INCLUDE_WINDOWSWIDEFORMATTING_HPP -#define CORE_LIBS_PRX_LIBC_INCLUDE_WINDOWSWIDEFORMATTING_HPP - -#include "WindowsFormatting.hpp" -#include "General.hpp" -#include -#include -#include - -namespace LibcDetail { - -inline int FormatWideWindows(wchar_t* output, size_t capacity, const wchar_t* format, const void* source) { - if (!capacity) return -1; - if (!output || !format || !source) { errno = 22; return -1; } - static_assert(sizeof(wchar_t) == 2); - FormatArguments args(source); - std::wstring translated; - std::wstring result; - std::vector slots; - std::deque extended; - auto append = [&slots](auto value) { - static_assert(sizeof(value) <= 8); - std::uint64_t bits = 0; - std::memcpy(&bits, &value, sizeof(value)); - slots.push_back(bits); - }; - while (*format) { - const wchar_t value = *format++; - if (value != L'%') { result += value; continue; } - if (*format == L'%') { result += *format++; continue; } - translated = L"%"; - slots.clear(); - extended.clear(); - bool nullCharacter = false; - while (*format && std::wcschr(L"-+ #0", *format)) translated += *format++; - if (*format == L'*') { translated += *format++; append(args.Next()); } - else while (*format >= L'0' && *format <= L'9') translated += *format++; - if (*format == L'.') { - translated += *format++; - if (*format == L'*') { translated += *format++; append(args.Next()); } - else while (*format >= L'0' && *format <= L'9') translated += *format++; - } - std::wstring length; - if (*format && std::wcschr(L"hljztL", *format)) { - length += *format++; - if ((length == L"h" && *format == L'h') || (length == L"l" && *format == L'l')) length += *format++; - } - const wchar_t conversion = *format; - if (!conversion) { errno = 22; return -1; } - if (!std::wcschr(L"diouxXaAeEfFgGcspn", conversion)) { - NotImplemented_nid_no_patch("swprintf format conversion"); - return -1; - } - ++format; - if (std::wcschr(L"diouxXn", conversion) && (length == L"l" || length == L"j" || length == L"z" || length == L"t")) - translated += L"ll"; - else if (conversion == L's' && length.empty()) translated += L"h"; - else if (conversion == L'c' && length.empty()) translated += L"l"; - else translated += length; - translated += conversion; - if (conversion == L'n') { - void* pointer = args.Next(); - if (length == L"hh") *static_cast(pointer) = static_cast(result.size()); - else if (length == L"h") *static_cast(pointer) = static_cast(result.size()); - else if (length.empty()) *static_cast(pointer) = static_cast(result.size()); - else *static_cast(pointer) = static_cast(result.size()); - continue; - } - if (conversion == L's' || conversion == L'p') append(args.Next()); - else if (conversion == L'c') { - const int character = args.Next(); - const int converted = length.empty() ? static_cast(static_cast(character)) : character; - nullCharacter = converted == 0; - append(nullCharacter ? 1 : converted); - } - else if (std::wcschr(L"aAeEfFgG", conversion)) { - if (length == L"L") { - extended.push_back(args.Next()); - append(&extended.back()); - } else append(args.Next()); - } else if (length.empty() || length == L"h" || length == L"hh") append(args.Next()); - else append(args.Next()); - if (result.size() >= capacity) return -1; - std::vector part(capacity - result.size()); - const int count = std::vswprintf(part.data(), part.size(), translated.c_str(), reinterpret_cast(slots.data())); - if (count < 0 || static_cast(count) >= part.size()) return -1; - if (nullCharacter) { - for (int i = 0; i < count; ++i) if (part[i] == 1) part[i] = 0; - } - result.append(part.data(), static_cast(count)); - } - if (result.size() >= capacity || result.size() > INT_MAX) return -1; - std::memcpy(output, result.c_str(), (result.size() + 1) * sizeof(wchar_t)); - return static_cast(result.size()); -} - -} - -#endif diff --git a/core/libs/prx/libc/include/general/LogMacros.hpp b/core/libs/prx/libc/include/general/LogMacros.hpp index f9c5f24c2..9142a9c73 100644 --- a/core/libs/prx/libc/include/general/LogMacros.hpp +++ b/core/libs/prx/libc/include/general/LogMacros.hpp @@ -60,7 +60,7 @@ _APS5_LOG_IMPL_NF(stdout, fmt); \ #define APS5_LOG_CHARS_OUT_TIMING(fmt) APS5_LOG_CHARS_OUT_IF(APS5_ENABLE_TIMING_LOG, fmt) #ifdef APS5_DEBUG_LOG -#define APS5_LOG_DEBUG(fmt, ...) _APS5_LOG_IMPL(stdout, fmt, __VA_ARGS__) +#define APS5_LOG_OUT_DEBUG(fmt, ...) _APS5_LOG_IMPL(stdout, fmt, __VA_ARGS__) #define APS5_LOG_CHARS_OUT_DEBUG(fmt) _APS5_LOG_IMPL_NF(stdout, fmt) #else #define APS5_LOG_OUT_DEBUG(fmt, ...) ((void)0) diff --git a/core/libs/prx/libc/src/ApplicationHeap.cpp b/core/libs/prx/libc/src/ApplicationHeap.cpp index cc12f6571..3affde65e 100644 --- a/core/libs/prx/libc/src/ApplicationHeap.cpp +++ b/core/libs/prx/libc/src/ApplicationHeap.cpp @@ -22,6 +22,7 @@ using Align = void* (APS5_VABI *)(std::size_t, std::size_t); using Realign = void* (APS5_VABI *)(void*, std::size_t, std::size_t); using PosixAlign = int (APS5_VABI *)(void**, std::size_t, std::size_t); using Initialize = void (APS5_VABI *)(); +using Stats = int (APS5_VABI *)(void*); std::mutex heapMutex; std::array heapApi{}; @@ -46,6 +47,7 @@ int APS5_VABI defaultPosixAlign(void** pointer, std::size_t alignment, std::size if (pointer == nullptr || alignment < sizeof(void*) || (alignment & (alignment - 1)) != 0) return 22; try { *pointer = defaultAlign(alignment, bytes); return 0; } catch (const std::bad_alloc&) { return 12; } + catch (const std::length_error&) { return 12; } } std::array defaultApi() { @@ -233,3 +235,17 @@ int ApplicationHeapPosixAlign_nid_no_patch(void** pointer, std::size_t alignment *pointer = result; return 0; } + +int ApplicationHeapStatsFast_nid_no_patch(void* stats) { + Stats statsFast; + { + std::lock_guard lock(heapMutex); + if (heapFailure) std::rethrow_exception(heapFailure); + if (heapFinalized) throw std::runtime_error("application heap: allocator has been finalized"); + if (heapApi[0] == nullptr) throw std::runtime_error("application heap: allocator API is not registered"); + if (heapApi[8] == nullptr) throw std::runtime_error("malloc_stats_fast: implemented only for an application allocator that provides it"); + std::memcpy(&statsFast, &heapApi[8], sizeof(statsFast)); + } + CallbackScope scope; + return statsFast(stats); +} diff --git a/core/libs/prx/libc/src/Directory.cpp b/core/libs/prx/libc/src/Directory.cpp index 0cc8ae972..b6fff287b 100644 --- a/core/libs/prx/libc/src/Directory.cpp +++ b/core/libs/prx/libc/src/Directory.cpp @@ -20,12 +20,40 @@ int DirectoryError(int error) { default: return 5; } } +#ifdef _WIN32 +using NativeDirectory = _WDIR; +NativeDirectory* OpenNative(const std::filesystem::path& path) { return ::_wopendir(path.c_str()); } +int CloseNative(NativeDirectory* native) { return ::_wclosedir(native); } +void RewindNative(NativeDirectory* native) { ::_wrewinddir(native); } +bool ReadNative(NativeDirectory* native, std::filesystem::path& component, std::string& name, std::uint64_t& inode) { + auto* entry = ::_wreaddir(native); + if (!entry) return false; + component = entry->d_name; + const auto utf8 = component.u8string(); + name.assign(utf8.begin(), utf8.end()); + inode = static_cast(entry->d_ino); + return true; +} +#else +using NativeDirectory = DIR; +NativeDirectory* OpenNative(const std::filesystem::path& path) { return ::opendir(path.c_str()); } +int CloseNative(NativeDirectory* native) { return ::closedir(native); } +void RewindNative(NativeDirectory* native) { ::rewinddir(native); } +bool ReadNative(NativeDirectory* native, std::filesystem::path& component, std::string& name, std::uint64_t& inode) { + auto* entry = ::readdir(native); + if (!entry) return false; + name = entry->d_name; + component = name; + inode = static_cast(entry->d_ino); + return true; +} +#endif struct Directory { - DIR* native = nullptr; + NativeDirectory* native = nullptr; std::filesystem::path path; GuestDirectoryEntry entry{}; std::mutex mutex; - ~Directory() { if (native) ::closedir(native); } + ~Directory() { if (native) CloseNative(native); } }; std::uint8_t DirectoryType(const std::filesystem::path& path) { std::error_code error; @@ -49,7 +77,7 @@ void* APS5_VABI opendir_nid_postfix(const char* path) { try { auto directory = std::make_unique(); directory->path = ResolvePath_nid_no_patch(path); - directory->native = ::opendir(directory->path.string().c_str()); + directory->native = OpenNative(directory->path); if (!directory->native) { errno = DirectoryError(errno); return nullptr; } return directory.release(); } catch (const std::bad_alloc&) { errno = 12; return nullptr; } @@ -62,22 +90,23 @@ GuestDirectoryEntry* APS5_VABI readdir_nid_postfix(void* handle) { std::lock_guard lock(directory.mutex); const int savedError = errno; errno = 0; - auto* entry = ::readdir(directory.native); - if (!entry) { + std::filesystem::path component; + std::string name; + std::uint64_t inode = 0; + if (!ReadNative(directory.native, component, name, inode)) { errno = errno ? DirectoryError(errno) : savedError; return nullptr; } - const auto length = std::strlen(entry->d_name); + const auto length = name.size(); if (length > 255) { errno = 63; return nullptr; } try { directory.entry = {}; // MinGW reports no inode identity (zero); do not fabricate one. - const auto inode = static_cast(entry->d_ino); directory.entry.fileNumber = inode <= UINT32_MAX ? static_cast(inode) : 0; directory.entry.recordLength = static_cast(8 + ((length + 1 + 3) & ~3)); directory.entry.nameLength = static_cast(length); - directory.entry.type = DirectoryType(directory.path / entry->d_name); - std::memcpy(directory.entry.name, entry->d_name, length + 1); + directory.entry.type = DirectoryType(directory.path / component); + std::memcpy(directory.entry.name, name.c_str(), length + 1); errno = savedError; return &directory.entry; } catch (const std::bad_alloc&) { errno = 12; return nullptr; } @@ -87,7 +116,7 @@ GuestDirectoryEntry* APS5_VABI readdir_nid_postfix(void* handle) { int APS5_VABI closedir_nid_postfix(void* handle) { if (!handle) { errno = 9; return -1; } std::unique_ptr directory(static_cast(handle)); - const int result = ::closedir(directory->native); + const int result = CloseNative(directory->native); directory->native = nullptr; if (result) errno = DirectoryError(errno); return result; @@ -97,6 +126,6 @@ void APS5_VABI rewinddir_nid_postfix(void* handle) { if (!handle) { errno = 9; return; } auto& directory = *static_cast(handle); std::lock_guard lock(directory.mutex); - ::rewinddir(directory.native); + RewindNative(directory.native); } } diff --git a/core/libs/prx/libc/src/FileAndHeap.cpp b/core/libs/prx/libc/src/FileAndHeap.cpp index 7c15cffde..f295fdb7f 100644 --- a/core/libs/prx/libc/src/FileAndHeap.cpp +++ b/core/libs/prx/libc/src/FileAndHeap.cpp @@ -3,6 +3,7 @@ #include #include #include +#include #include #include #include @@ -14,16 +15,39 @@ #include "prx/libc/include/General.hpp" #include "prx/libc/include/GuestArena.hpp" -static std::string NativeFileMode(const char* mode) { - std::string result(mode); +struct GuestFileMode { + std::string native; + bool writes = false; + bool exclusive = false; +}; + +static std::optional ParseFileMode(const char* mode) { + const char access = mode[0]; + if (access != 'r' && access != 'w' && access != 'a') return std::nullopt; + bool update = false; + bool exclusive = false; + for (const char* flag = mode + 1; *flag != '\0' && std::strchr("b+xev", *flag) != nullptr; ++flag) { + if (*flag == '+') update = true; + if (*flag == 'x') exclusive = true; + } + if (exclusive && access == 'r' && !update) return std::nullopt; + GuestFileMode result{std::string(1, access), access != 'r' || update, exclusive}; + if (update) result.native += '+'; #ifdef _WIN32 - if (!result.empty() && result.find('b') == std::string::npos) result.insert(1, 1, 'b'); + result.native += 'b'; #endif + if (exclusive) result.native += 'x'; return result; } -static bool WritesFile(const char* mode) { - return std::strpbrk(mode, "wa+") != nullptr; +static std::FILE* OpenNative(const std::filesystem::path& path, const std::string& nativeMode) { +#ifdef _WIN32 + std::wstring wideMode; + for (const char character : nativeMode) wideMode.push_back(static_cast(character)); + return ::_wfopen(path.c_str(), wideMode.c_str()); +#else + return std::fopen(path.c_str(), nativeMode.c_str()); +#endif } extern "C" { @@ -56,15 +80,12 @@ FileStream* APS5_VABI fdopen_nid_postfix(int descriptor, const char* mode) { FileStream* APS5_VABI freopen_nid_postfix(const char* filename, const char* mode, FileStream* stream) { if (!stream || !mode) { errno = 22; return nullptr; } if (!filename) { errno = 45; return nullptr; } // Mode-only reopening is not supported. - const char* supported[] = {"r", "w", "a", "rb", "wb", "ab", "r+", "w+", "a+", - "rb+", "wb+", "ab+", "r+b", "w+b", "a+b"}; - bool valid = false; - for (const auto* candidate : supported) if (std::strcmp(mode, candidate) == 0) valid = true; - if (!valid) { errno = 22; return nullptr; } + const auto parsed = ParseFileMode(mode); + if (!parsed) { errno = 22; return nullptr; } try { - const auto path = *filename ? ResolvePath_nid_no_patch(filename).string() : std::string{}; - if (stream->Reopen(path.c_str(), NativeFileMode(mode).c_str())) { - if (!path.empty() && WritesFile(mode)) RecordWrittenPath_nid_no_patch(path); + const std::filesystem::path path = *filename ? ResolvePath_nid_no_patch(filename) : std::filesystem::path{}; + if (stream->Reopen(path, parsed->native.c_str())) { + if (!path.empty() && parsed->writes) RecordWrittenPath_nid_no_patch(path); return stream; } const int error = errno; @@ -77,12 +98,17 @@ FileStream* APS5_VABI freopen_nid_postfix(const char* filename, const char* mode FileStream* APS5_VABI fopen_nid_postfix(const char* filename, const char* mode) { if (!filename || !mode) throw std::runtime_error(std::string(__func__) + ": " + FOPEN_MSG_NULL_ARG); + const auto parsed = ParseFileMode(mode); + if (!parsed) { + errno = EINVAL; + return nullptr; + } const std::filesystem::path fpath = ResolvePath_nid_no_patch(filename); const auto abs_path = fpath.string(); - std::unique_ptr handle(std::fopen(abs_path.c_str(), NativeFileMode(mode).c_str()), std::fclose); + std::unique_ptr handle(OpenNative(fpath, parsed->native), std::fclose); if (!handle) { const int error = errno; - if (error == ENOENT) { + if (error == ENOENT || (parsed->exclusive && error == EEXIST)) { errno = error; return nullptr; } @@ -103,7 +129,7 @@ FileStream* APS5_VABI fopen_nid_postfix(const char* filename, const char* mode) throw std::runtime_error(std::string(__func__) + ": " + FOPEN_MSG_OPEN_FAILED + ": \"" + abs_path + "\": " + reason); } // APS5_LOG_OUT("success: \"%s\"", abs_path.c_str()); - if (WritesFile(mode)) RecordWrittenPath_nid_no_patch(fpath); + if (parsed->writes) RecordWrittenPath_nid_no_patch(fpath); auto stream = std::make_unique(handle.get(), true); handle.release(); return stream.release(); @@ -232,6 +258,10 @@ int APS5_VABI fflush_nid_postfix(FileStream* stream) { return 0; } +int APS5_VABI malloc_stats_fast_nid_postfix(void* stats) { + return ApplicationHeapStatsFast_nid_no_patch(stats); +} + void* APS5_VABI malloc_nid_postfix(size_t size) { return ApplicationHeapAllocate_nid_no_patch(size); } diff --git a/core/libs/prx/libc/src/Filesystem.cpp b/core/libs/prx/libc/src/Filesystem.cpp index 2e2d2bd3b..d59f14d39 100644 --- a/core/libs/prx/libc/src/Filesystem.cpp +++ b/core/libs/prx/libc/src/Filesystem.cpp @@ -1,3 +1,4 @@ +#include "prx/libc/include/FilesystemError.hpp" #include "prx/libc/include/General.hpp" #include #include @@ -10,27 +11,6 @@ #include #endif -namespace { -int FilesystemError(const std::error_code& error) { - if (error == std::errc::no_such_file_or_directory) return 2; - if (error == std::errc::permission_denied) return 13; - if (error == std::errc::operation_not_permitted) return 1; - if (error == std::errc::not_a_directory) return 20; - if (error == std::errc::is_a_directory) return 21; - if (error == std::errc::directory_not_empty) return 66; - if (error == std::errc::device_or_resource_busy) return 16; - if (error == std::errc::read_only_file_system) return 30; - if (error == std::errc::filename_too_long) return 63; - if (error == std::errc::too_many_symbolic_link_levels) return 62; - if (error == std::errc::not_enough_memory) return 12; - if (error == std::errc::invalid_argument) return 22; - if (error == std::errc::cross_device_link) return 18; - if (error == std::errc::file_exists) return 17; - if (error == std::errc::no_space_on_device) return 28; - return 5; -} -} - extern "C" int APS5_VABI access_nid_postfix(const char* path, int mode) { if (!path) { errno = 14; return -1; } if (mode < 0 || (mode & ~7)) { errno = 22; return -1; } @@ -90,25 +70,6 @@ extern "C" int APS5_VABI access_nid_postfix(const char* path, int mode) { } } -extern "C" int APS5_VABI rename_nid_postfix(const char* from, const char* to) { - if (!from || !to) { errno = 14; return -1; } - if (!*from || !*to) { errno = 2; return -1; } - try { - const auto source = ResolvePath_nid_no_patch(from); - const auto destination = ResolvePath_nid_no_patch(to); - std::error_code error; - std::filesystem::rename(source, destination, error); - if (error) { errno = FilesystemError(error); return -1; } - RecordWrittenPath_nid_no_patch(source); - RecordWrittenPath_nid_no_patch(destination); - return 0; - } catch (const std::bad_alloc&) { errno = 12; return -1; } - catch (const std::filesystem::filesystem_error& error) { - errno = FilesystemError(error.code()); - return -1; - } -} - extern "C" int APS5_VABI remove_nid_postfix(const char* path) { if (!path) { errno = 14; return -1; } if (!*path) { errno = 2; return -1; } @@ -119,8 +80,16 @@ extern "C" int APS5_VABI remove_nid_postfix(const char* path) { const DWORD attributes = GetFileAttributesW(resolved.c_str()); bool removed = false; if (attributes != INVALID_FILE_ATTRIBUTES) { + const DWORD writable = attributes & ~FILE_ATTRIBUTE_READONLY; + const bool unlocked = writable != attributes && + SetFileAttributesW(resolved.c_str(), writable ? writable : FILE_ATTRIBUTE_NORMAL); removed = (attributes & FILE_ATTRIBUTE_DIRECTORY) ? RemoveDirectoryW(resolved.c_str()) != 0 : DeleteFileW(resolved.c_str()) != 0; + if (!removed && unlocked) { + const DWORD nativeError = GetLastError(); + SetFileAttributesW(resolved.c_str(), attributes); + SetLastError(nativeError); + } } if (!removed) { const DWORD nativeError = GetLastError(); diff --git a/core/libs/prx/libc/src/Formatting.cpp b/core/libs/prx/libc/src/Formatting.cpp index 7395115b5..37a59d7d2 100644 --- a/core/libs/prx/libc/src/Formatting.cpp +++ b/core/libs/prx/libc/src/Formatting.cpp @@ -15,14 +15,28 @@ #include "prx/libc/include/FileStream.hpp" #include "prx/libc/include/ApplicationHeap.hpp" -#ifdef _WIN32 #include "prx/libc/include/WindowsFormatting.hpp" + +#ifdef _WIN32 #include "prx/libc/include/WindowsScanning.hpp" -#include "prx/libc/include/WindowsWideFormatting.hpp" #endif namespace { +#ifndef _WIN32 +bool HasGuestWideFormat(const char* format) { + if (format == nullptr) return false; + while ((format = std::strchr(format, '%')) != nullptr) { + ++format; + if (*format == '%') { ++format; continue; } + while (*format && std::strchr("-+ #0'0123456789.*$", *format)) ++format; + if (*format == 'S' || *format == 'C' || (*format == 'l' && (format[1] == 's' || format[1] == 'c'))) return true; + if (*format) ++format; + } + return false; +} +#endif + std::string HostLength(const std::string& length, char conversion) { #ifdef _WIN32 if (length == "l" && std::strchr("diouxXn", conversion) != nullptr) return "ll"; @@ -142,21 +156,6 @@ int ScanGuest(const char* buffer, const char* format, bool secure, NextPointer n extern "C" { -int APS5_VABI swprintf_nid_postfix(wchar_t* output, size_t capacity, const wchar_t* format, ...) { -#ifdef _WIN32 - __builtin_sysv_va_list args; - __builtin_sysv_va_start(args, format); - const int result = LibcDetail::FormatWideWindows(output, capacity, format, args); - __builtin_sysv_va_end(args); -#else - std::va_list args; - va_start(args, format); - const int result = std::vswprintf(output, capacity, format, args); - va_end(args); -#endif - return result; -} - int APS5_VABI vasprintf_nid_postfix(char** destination, const char* format, VaList* args) { if (!destination) { errno = 22; return -1; } *destination = nullptr; @@ -167,14 +166,16 @@ int APS5_VABI vasprintf_nid_postfix(char** destination, const char* format, VaLi const int count = LibcDetail::FormatWindows(nullptr, 0, format, args, &text); const char* source = text.c_str(); #else + std::string guestText; + const bool guestWide = HasGuestWideFormat(format); char* text = nullptr; std::va_list copy; va_copy(copy, *reinterpret_cast(args)); - const int count = ::vasprintf(&text, format, copy); + const int count = guestWide ? LibcDetail::FormatWindows(nullptr, 0, format, args, &guestText) : ::vasprintf(&text, format, copy); va_end(copy); std::unique_ptr owner(text, std::free); if (count < 0) return -1; - const char* source = text; + const char* source = guestWide ? guestText.c_str() : text; #endif auto* output = static_cast(ApplicationHeapAllocate_nid_no_patch(static_cast(count) + 1)); std::memcpy(output, source, static_cast(count) + 1); @@ -194,6 +195,13 @@ int APS5_VABI vfprintf_nid_postfix(FileStream* stream, const char* format, VaLis const int result = std::fwrite(buffer.data(), 1, static_cast(count), native) == static_cast(count) ? count : -1; #else + if (HasGuestWideFormat(format)) { + std::string buffer; + const int count = LibcDetail::FormatWindows(nullptr, 0, format, args, &buffer); + const int result = std::fwrite(buffer.data(), 1, static_cast(count), native) == static_cast(count) ? count : -1; + stream->SyncStatus(); + return result; + } std::va_list copy; va_copy(copy, *reinterpret_cast(args)); const int result = std::vfprintf(native, format, copy); @@ -276,7 +284,7 @@ int APS5_VABI sprintf_nid_postfix(char* buffer, const char* format, ...) { int APS5_VABI printf_nid_postfix(const char* format, ...) { std::va_list args; va_start(args, format); - const int result = std::vprintf(format, args); + const int result = HasGuestWideFormat(format) ? LibcDetail::PrintWindows(format, args) : std::vprintf(format, args); va_end(args); return result; } @@ -284,7 +292,7 @@ int APS5_VABI printf_nid_postfix(const char* format, ...) { int APS5_VABI libc_printf_nid_postfix(const char* format, ...) { std::va_list args; va_start(args, format); - const int result = std::vprintf(format, args); + const int result = HasGuestWideFormat(format) ? LibcDetail::PrintWindows(format, args) : std::vprintf(format, args); va_end(args); return result; } @@ -292,7 +300,7 @@ int APS5_VABI libc_printf_nid_postfix(const char* format, ...) { int APS5_VABI snprintf_nid_postfix(char* buffer, size_t size, const char* format, ...) { std::va_list args; va_start(args, format); - const int result = std::vsnprintf(buffer, size, format, args); + const int result = HasGuestWideFormat(format) ? LibcDetail::FormatWindows(buffer, size, format, args) : std::vsnprintf(buffer, size, format, args); va_end(args); return result; } @@ -300,7 +308,7 @@ int APS5_VABI snprintf_nid_postfix(char* buffer, size_t size, const char* format int APS5_VABI sprintf_nid_postfix(char* buffer, const char* format, ...) { std::va_list args; va_start(args, format); - const int result = std::vsprintf(buffer, format, args); + const int result = HasGuestWideFormat(format) ? LibcDetail::FormatWindows(buffer, SIZE_MAX, format, args) : std::vsprintf(buffer, format, args); va_end(args); return result; } @@ -363,6 +371,7 @@ int APS5_VABI vprintf_nid_postfix(const char* str, VaList* c) { #ifdef _WIN32 return LibcDetail::PrintWindows(str, c); #else + if (HasGuestWideFormat(str)) return LibcDetail::PrintWindows(str, c); std::va_list copy; va_copy(copy, *reinterpret_cast(c)); const int result = std::vprintf(str, copy); @@ -375,6 +384,7 @@ int APS5_VABI vsprintf_nid_postfix(char* str, const char* format, VaList* args) #ifdef _WIN32 return LibcDetail::FormatWindows(str, SIZE_MAX, format, args); #else + if (HasGuestWideFormat(format)) return LibcDetail::FormatWindows(str, SIZE_MAX, format, args); std::va_list copy; va_copy(copy, *reinterpret_cast(args)); const int result = std::vsprintf(str, format, copy); @@ -387,6 +397,7 @@ int APS5_VABI vsnprintf_nid_postfix(char* str, size_t size, const char* format, #ifdef _WIN32 return LibcDetail::FormatWindows(str, size, format, c); #else + if (HasGuestWideFormat(format)) return LibcDetail::FormatWindows(str, size, format, c); std::va_list copy; va_copy(copy, *reinterpret_cast(c)); const int result = std::vsnprintf(str, size, format, copy); @@ -416,6 +427,7 @@ int APS5_VABI printf_s_nid_postfix(const char* format, ...) { #else int APS5_VABI vsprintf_s_nid_postfix(char* buffer, size_t size, const char* format, VaList* args) { + if (HasGuestWideFormat(format)) return LibcDetail::FormatWindows(buffer, size, format, args); std::va_list copy; va_copy(copy, *reinterpret_cast(args)); const int result = std::vsnprintf(buffer, size, format, copy); @@ -426,7 +438,7 @@ int APS5_VABI vsprintf_s_nid_postfix(char* buffer, size_t size, const char* form int APS5_VABI sprintf_s_nid_postfix(char* buffer, size_t size, const char* format, ...) { std::va_list args; va_start(args, format); - const int result = std::vsnprintf(buffer, size, format, args); + const int result = HasGuestWideFormat(format) ? LibcDetail::FormatWindows(buffer, size, format, args) : std::vsnprintf(buffer, size, format, args); va_end(args); return result; } @@ -434,7 +446,7 @@ int APS5_VABI sprintf_s_nid_postfix(char* buffer, size_t size, const char* forma int APS5_VABI snprintf_s_nid_postfix(char* buffer, size_t size, const char* format, ...) { std::va_list args; va_start(args, format); - const int result = std::vsnprintf(buffer, size, format, args); + const int result = HasGuestWideFormat(format) ? LibcDetail::FormatWindows(buffer, size, format, args) : std::vsnprintf(buffer, size, format, args); va_end(args); return result; } @@ -442,7 +454,7 @@ int APS5_VABI snprintf_s_nid_postfix(char* buffer, size_t size, const char* form int APS5_VABI printf_s_nid_postfix(const char* format, ...) { std::va_list args; va_start(args, format); - const int result = std::vprintf(format, args); + const int result = HasGuestWideFormat(format) ? LibcDetail::PrintWindows(format, args) : std::vprintf(format, args); va_end(args); return result; } diff --git a/core/libs/prx/libc/src/FormattingWide.cpp b/core/libs/prx/libc/src/FormattingWide.cpp index 7808ba4ba..e3a18f1b6 100644 --- a/core/libs/prx/libc/src/FormattingWide.cpp +++ b/core/libs/prx/libc/src/FormattingWide.cpp @@ -214,6 +214,13 @@ int APS5_VABI vswprintf_nid_postfix(char16_t* buffer, std::size_t size, const ch } } +int APS5_VABI swprintf_nid_postfix(char16_t* buffer, std::size_t size, const char16_t* format, ...) { + APS5_VA_BEGIN(format); + const int result = vswprintf_nid_postfix(buffer, size, format, reinterpret_cast(args)); + APS5_VA_END(); + return result; +} + int APS5_VABI snwprintf_s_nid_postfix(char16_t* buffer, std::size_t size, const char16_t* format, ...) { constexpr std::size_t RsizeMax = SIZE_MAX >> 1; int result = -1; diff --git a/core/libs/prx/libc/src/General.cpp b/core/libs/prx/libc/src/General.cpp index a190e8fcc..fd554d147 100644 --- a/core/libs/prx/libc/src/General.cpp +++ b/core/libs/prx/libc/src/General.cpp @@ -21,10 +21,13 @@ #endif #include #else +#include +#include +#include #include #endif #include "prx/libc/include/General.hpp" -#include "prx/libc/include/GuestHeap.hpp" +#include "prx/libc/include/ApplicationHeap.hpp" namespace { // Guest prefixes (without leading slashes) mapped to host directories, e.g. save-data mount points: @@ -125,27 +128,43 @@ std::filesystem::path Resolve(WorkingDirectory& state, const char* path) { if (auto aliased = ResolveAlias(guest.relative_path().generic_string())) return *aliased; return ResolveHostPath(state.root, guest.relative_path()).make_preferred(); } +#ifdef _WIN32 +using WriteMark = FILETIME; +WriteMark Now() { + FILETIME now; + GetSystemTimeAsFileTime(&now); + return now; +} +bool NotBefore(const WriteMark& time, const WriteMark& mark) { + return CompareFileTime(&time, &mark) >= 0; +} +#else +using WriteMark = timespec; +WriteMark Now() { + timespec now; + clock_gettime(CLOCK_REALTIME_COARSE, &now); + return now; +} +bool NotBefore(const WriteMark& time, const WriteMark& mark) { + if (time.tv_sec == mark.tv_sec) return time.tv_nsec >= mark.tv_nsec; + return time.tv_sec >= mark.tv_sec; +} +#endif struct WrittenPathRegistry { std::mutex syncMutex; std::mutex mutex; std::map dirty; -#ifdef _WIN32 - FILETIME mark = [] { FILETIME now; GetSystemTimeAsFileTime(&now); return now; }(); -#endif + WriteMark mark = Now(); }; WrittenPathRegistry& Written() { static WrittenPathRegistry registry; return registry; } #ifdef _WIN32 -std::optional WriteTime(const std::filesystem::path& path) { +std::optional WriteTime(const std::filesystem::path& path) { WIN32_FILE_ATTRIBUTE_DATA data; if (GetFileAttributesExW(path.c_str(), GetFileExInfoStandard, &data)) return data.ftLastWriteTime; const auto error = GetLastError(); if (error == ERROR_FILE_NOT_FOUND || error == ERROR_PATH_NOT_FOUND) return std::nullopt; throw std::system_error(static_cast(error), std::system_category(), "Reading the write time of " + path.string()); } -bool ChangedSince(const std::filesystem::path& path, const FILETIME& mark) { - const auto time = WriteTime(path); - return time && CompareFileTime(&*time, &mark) >= 0; -} void FlushPath(const std::filesystem::path& path) { const auto handle = CreateFileW(path.c_str(), GENERIC_WRITE, FILE_SHARE_READ | FILE_SHARE_WRITE | FILE_SHARE_DELETE, nullptr, OPEN_EXISTING, FILE_FLAG_BACKUP_SEMANTICS, nullptr); @@ -159,7 +178,29 @@ void FlushPath(const std::filesystem::path& path) { if (!FlushFileBuffers(handle)) throw std::system_error(static_cast(GetLastError()), std::system_category(), "Flushing " + path.string()); } +#else +std::optional WriteTime(const std::filesystem::path& path) { + struct stat data; + if (::stat(path.c_str(), &data) == 0) return data.st_mtim; + if (errno == ENOENT || errno == ENOTDIR) return std::nullopt; + throw std::system_error(errno, std::generic_category(), "Reading the write time of " + path.string()); +} +void FlushPath(const std::filesystem::path& path) { + const int descriptor = ::open(path.c_str(), O_RDONLY | O_NONBLOCK | O_CLOEXEC); + if (descriptor < 0) { + if (errno == ENOENT || errno == ENOTDIR || errno == EACCES) return; + throw std::system_error(errno, std::generic_category(), "Opening " + path.string() + " for sync"); + } + const int result = ::fsync(descriptor); + const int error = errno; + ::close(descriptor); + if (result != 0 && error != EINVAL) throw std::system_error(error, std::generic_category(), "Flushing " + path.string()); +} #endif +bool ChangedSince(const std::filesystem::path& path, const WriteMark& mark) { + const auto time = WriteTime(path); + return time && NotBefore(*time, mark); +} int DirectoryFailure(const std::error_code& error) { if (error == std::errc::permission_denied) return 13; if (error == std::errc::not_a_directory) return 20; @@ -214,10 +255,8 @@ extern "C" std::vector WrittenPaths_nid_no_patch() { extern "C" void SyncWrittenPaths_nid_no_patch() { auto& registry = Written(); std::lock_guard serial(registry.syncMutex); -#ifdef _WIN32 - FILETIME start; - GetSystemTimeAsFileTime(&start); - FILETIME mark; + const auto start = Now(); + WriteMark mark; std::vector> entries; { std::lock_guard lock(registry.mutex); @@ -233,7 +272,7 @@ extern "C" void SyncWrittenPaths_nid_no_patch() { for (const auto& [path, dirty] : entries) { const auto time = WriteTime(path); if (!time) missing.push_back(path); - if (!dirty && !(time && CompareFileTime(&*time, &mark) >= 0)) continue; + if (!dirty && !(time && NotBefore(*time, mark))) continue; if (time && flushed.insert(path).second) FlushPath(path); for (auto ancestor = path.parent_path(); ancestor.has_relative_path() && !flushed.contains(ancestor) && ChangedSince(ancestor, mark); ancestor = ancestor.parent_path()) { @@ -246,11 +285,6 @@ extern "C" void SyncWrittenPaths_nid_no_patch() { const auto entry = registry.dirty.find(path); if (entry != registry.dirty.end() && !entry->second) registry.dirty.erase(entry); } -#else - ::sync(); - std::lock_guard lock(registry.mutex); - registry.dirty.clear(); -#endif } extern "C" void BlockPathAlias_nid_no_patch(const char* guestPrefix) { @@ -304,7 +338,8 @@ extern "C" char* APS5_VABI getcwd_nid_postfix(char* buffer, std::size_t size) { const auto path = relative == "." ? std::string("/") : "/" + relative.generic_string(); const auto required = path.size() + 1; if ((buffer || size) && size < required) { errno = 34; return nullptr; } - if (!buffer) buffer = static_cast(GuestHeap::GuestHeapAllocate_nid_postfix(size ? size : required)); + if (!buffer) buffer = static_cast(ApplicationHeapAllocate_nid_no_patch(size ? size : required)); + if (!buffer) { errno = 12; return nullptr; } std::memcpy(buffer, path.c_str(), required); return buffer; } catch (const std::bad_alloc&) { errno = 12; return nullptr; } diff --git a/core/libs/prx/libc/src/GuestAllocations.cpp b/core/libs/prx/libc/src/GuestAllocations.cpp index a29a4cf1c..8d77fa23d 100644 --- a/core/libs/prx/libc/src/GuestAllocations.cpp +++ b/core/libs/prx/libc/src/GuestAllocations.cpp @@ -34,6 +34,7 @@ struct Registry { std::mutex mutex; std::map> ranges; bool mainImageRegistered = false; + std::map registeredImages; }; Registry& registry() { @@ -43,7 +44,7 @@ Registry& registry() { std::atomic generation{1}; std::atomic invalidator{nullptr}; -std::atomic pinWaiter{nullptr}; +std::atomic pinWaiter{nullptr}; std::chrono::milliseconds pinWait() { static const std::chrono::milliseconds value{[] { @@ -129,22 +130,21 @@ void GuestAllocationsInvalidate_nid_postfix(std::uintptr_t address, std::size_t if (const auto callback = invalidator.load(std::memory_order_acquire)) callback(address, bytes); } -void GuestAllocationsSetPinWaiter_nid_postfix(bool (*callback)()) { +void GuestAllocationsSetPinWaiter_nid_postfix(bool (*callback)(std::uintptr_t, std::size_t)) { pinWaiter.store(callback, std::memory_order_release); } #ifdef _WIN32 -void GuestAllocationsRegisterMainImage_nid_postfix(void*) { +namespace { +void registerImage(const void* image) { auto& state = registry(); - if (state.mainImageRegistered) return; - const auto image = GetModuleHandleW(nullptr); - require(image != nullptr, "cannot locate the main guest image"); + if (state.registeredImages.contains(reinterpret_cast(image))) return; auto replacement = state.ranges; auto cursor = reinterpret_cast(image); bool registered = false; for (;;) { MEMORY_BASIC_INFORMATION memory{}; - require(VirtualQuery(reinterpret_cast(cursor), &memory, sizeof(memory)) == sizeof(memory), "cannot query the main guest image"); + require(VirtualQuery(reinterpret_cast(cursor), &memory, sizeof(memory)) == sizeof(memory), "cannot query a guest image"); if (memory.AllocationBase != image) break; require(memory.Type == MEM_IMAGE && (memory.State == MEM_COMMIT || memory.State == MEM_RESERVE), "unsupported guest image mapping"); require(reinterpret_cast(memory.BaseAddress) == cursor && memory.RegionSize != 0 && memory.RegionSize <= std::numeric_limits::max() - cursor, "invalid guest image range"); @@ -165,10 +165,36 @@ void GuestAllocationsRegisterMainImage_nid_postfix(void*) { } cursor += memory.RegionSize; } - require(registered, "main guest image has no committed pages"); + require(registered, "guest image has no committed pages"); state.ranges.swap(replacement); + state.registeredImages.emplace(reinterpret_cast(image), cursor); +} +} + +void GuestAllocationsRegisterMainImage_nid_postfix(void*) { + auto& state = registry(); + if (state.mainImageRegistered) return; + const auto image = GetModuleHandleW(nullptr); + require(image != nullptr, "cannot locate the main guest image"); + registerImage(image); state.mainImageRegistered = true; } + +void GuestAllocationsRegisterImage_nid_postfix(void*, const void* image) { + require(image != nullptr, "cannot register a guest image without a base"); + registerImage(image); +} + +void GuestAllocationsUnregisterImage_nid_postfix(void* mutation, const void* image) { + auto& state = registry(); + const auto found = state.registeredImages.find(reinterpret_cast(image)); + if (found == state.registeredImages.end()) return; + const auto [begin, end] = *found; + GuestAllocationsRequireUnpinned_nid_postfix(mutation, image, end - begin); + recordChange(mutation, image, end - begin); + std::erase_if(state.ranges, [&](const auto& entry) { return !entry.second->releasable && entry.second->allocationAddress >= begin && entry.second->allocationAddress < end; }); + state.registeredImages.erase(found); +} #else void GuestAllocationsRegisterMainImage_nid_postfix(void*) { auto& state = registry(); @@ -216,7 +242,7 @@ void GuestAllocationsRegisterMainImage_nid_postfix(void*) { } #endif -void GuestAllocationsAdd_nid_postfix(void* mutation, void* pointer, std::size_t bytes, bool readable, bool writable) { +void GuestAllocationsAdd_nid_postfix(void* mutation, void* pointer, std::size_t bytes, bool readable, bool writable, bool gpu) { recordChange(mutation, pointer, bytes); const auto address = reinterpret_cast(pointer); require(address != 0 && bytes <= std::numeric_limits::max() - address, "invalid guest allocation range"); @@ -228,12 +254,12 @@ void GuestAllocationsAdd_nid_postfix(void* mutation, void* pointer, std::size_t const auto& previous = *std::prev(next)->second; require(previous.address + previous.bytes <= address, "overlapping guest allocation"); } - ranges.emplace(address, std::make_shared(Range{address, bytes, readable, writable, address, bytes})); + ranges.emplace(address, std::make_shared(Range{address, bytes, readable, writable, address, bytes, true, gpu})); } [[noreturn]] void PinnedFailure(std::uintptr_t address, std::size_t bytes, const char* why) { char message[160]; - std::snprintf(message, sizeof(message), "guest allocation 0x%llx+0x%llx is owned by an active GPU command (%s)", static_cast(address), static_cast(bytes), why); + std::snprintf(message, sizeof(message), "guest allocation 0x%llx+0x%llx is still pinned by GPU work or an import (%s)", static_cast(address), static_cast(bytes), why); throw std::runtime_error(message); } @@ -269,7 +295,7 @@ void GuestAllocationsRequireUnpinned_nid_postfix(void* mutation, const void* poi bool progressed = false; if (state != nullptr && state->lock.owns_lock()) { state->lock.unlock(); - if (waiter != nullptr) progressed = waiter(); + if (waiter != nullptr) progressed = waiter(address, bytes); else std::this_thread::yield(); state->lock.lock(); } else { @@ -329,12 +355,12 @@ Range GuestAllocationsFind_nid_postfix(void*, const void* pointer) { if (exact != registry().ranges.end() && exact->second->allocationAddress == address && exact->second->allocationBytes == exact->second->bytes) { const auto& range = *exact->second; require(range.releasable, "guest image memory is not a releasable allocation"); - return {address, range.allocationBytes, range.readable, range.writable, address, range.allocationBytes, range.releasable}; + return {address, range.allocationBytes, range.readable, range.writable, address, range.allocationBytes, range.releasable, range.gpu}; } for (const auto& [base, range] : registry().ranges) { if (range->allocationAddress == address) { require(range->releasable, "guest image memory is not a releasable allocation"); - return {address, range->allocationBytes, range->readable, range->writable, address, range->allocationBytes, range->releasable}; + return {address, range->allocationBytes, range->readable, range->writable, address, range->allocationBytes, range->releasable, range->gpu}; } } throw std::runtime_error("guest allocation is not registered"); @@ -355,7 +381,7 @@ void GuestAllocationsRemove_nid_postfix(void* mutation, const void* pointer) { namespace { -std::map> replaceRange(const void* pointer, std::size_t bytes, bool remove, bool readable, bool writable) { +std::map> replaceRange(const void* pointer, std::size_t bytes, bool remove, bool readable, bool writable, bool gpu) { const auto address = reinterpret_cast(pointer); require(bytes != 0 && bytes <= std::numeric_limits::max() - address, "invalid guest protection or unmap range"); require(!writable || readable, "writable guest allocation must be readable"); @@ -369,12 +395,12 @@ std::map> replaceRange(const void* p if (base >= end) break; require(base <= cursor, "guest protection or unmap range has a hole"); replacement.erase(base); - const auto insert = [&](std::uint64_t first, std::uint64_t last, bool canRead, bool canWrite) { - if (first < last) replacement.emplace(first, std::make_shared(Range{first, static_cast(last - first), canRead, canWrite, range.allocationAddress, range.allocationBytes, range.releasable})); + const auto insert = [&](std::uint64_t first, std::uint64_t last, bool canRead, bool canWrite, bool gpuMapped) { + if (first < last) replacement.emplace(first, std::make_shared(Range{first, static_cast(last - first), canRead, canWrite, range.allocationAddress, range.allocationBytes, range.releasable, gpuMapped})); }; - insert(base, std::max(base, address), range.readable, range.writable); - if (!remove) insert(std::max(base, address), std::min(finish, end), readable, writable); - insert(std::min(finish, end), finish, range.readable, range.writable); + insert(base, std::max(base, address), range.readable, range.writable, range.gpu); + if (!remove) insert(std::max(base, address), std::min(finish, end), readable, writable, gpu); + insert(std::min(finish, end), finish, range.readable, range.writable, range.gpu); cursor = std::min(finish, end); } require(cursor == end, "guest protection or unmap range is not registered"); @@ -383,10 +409,10 @@ std::map> replaceRange(const void* p } -void GuestAllocationsProtect_nid_postfix(void* mutation, const void* pointer, std::size_t bytes, bool readable, bool writable, const std::function& apply) { +void GuestAllocationsProtect_nid_postfix(void* mutation, const void* pointer, std::size_t bytes, bool readable, bool writable, bool gpu, const std::function& apply) { GuestAllocationsRequireUnpinned_nid_postfix(mutation, pointer, bytes); recordChange(mutation, pointer, bytes); - auto replacement = replaceRange(pointer, bytes, false, readable, writable); + auto replacement = replaceRange(pointer, bytes, false, readable, writable, gpu); apply(); registry().ranges.swap(replacement); } @@ -412,7 +438,7 @@ void GuestAllocationsUnmap_nid_postfix(void* mutation, const void* pointer, std: const auto range = *containing; require(range.releasable, "guest image memory cannot be unmapped"); const auto pieceEnd = std::min(end, range.allocationAddress + range.allocationBytes); - auto replacement = replaceRange(reinterpret_cast(cursor), pieceEnd - cursor, true, false, false); + auto replacement = replaceRange(reinterpret_cast(cursor), pieceEnd - cursor, true, false, false, false); bool last = true; for (const auto& [base, entry] : replacement) { if (entry->allocationAddress == range.allocationAddress) last = false; @@ -434,4 +460,14 @@ Lease GuestAllocationsAcquire_nid_postfix() { return result; } +Lease GuestAllocationsAcquireAll_nid_postfix() { + std::lock_guard lock(registry().mutex); + Lease result; + result.reserve(registry().ranges.size()); + for (const auto& [address, range] : registry().ranges) { + if (range->bytes != 0) result.push_back(range); + } + return result; +} + } diff --git a/core/libs/prx/libc/src/GuestArena.cpp b/core/libs/prx/libc/src/GuestArena.cpp index b815e0fab..fea67b8cc 100644 --- a/core/libs/prx/libc/src/GuestArena.cpp +++ b/core/libs/prx/libc/src/GuestArena.cpp @@ -1,8 +1,16 @@ #include "prx/libc/include/GuestArena.hpp" +#include "prx/libc/include/GuestWriteWatch.hpp" #include "prx/libc/include/WindowsMappings.hpp" #include #include #include +#ifndef _WIN32 +#include +#include +#include +#include +#include +#endif #include #include #include @@ -71,6 +79,22 @@ public: return reinterpret_cast(candidate); } + bool MarkUsedForCommit(const void* pointer, std::size_t bytes) { + std::lock_guard lock(_lock); + const auto start = reinterpret_cast(pointer); + const auto end = start + bytes; + const auto exact = _used.find(start); + if (exact != _used.end() && exact->second >= end) return false; + const auto next = _used.lower_bound(start); + if ((next != _used.end() && next->first < end) || (next != _used.begin() && std::prev(next)->second > start)) { + char message[160]; + std::snprintf(message, sizeof(message), "guest arena commit 0x%llx+0x%zx overlaps a used range", static_cast(start), bytes); + throw std::runtime_error(message); + } + _used.emplace(start, end); + return true; + } + void MarkUsed(const void* pointer, std::size_t bytes) { std::lock_guard lock(_lock); const auto start = reinterpret_cast(pointer); @@ -107,7 +131,9 @@ public: private: Arena() { -#ifdef _WIN32 +#if defined(__linux__) + SeedLinuxUsedRanges(); +#elif defined(_WIN32) _writeWatched = std::getenv("APS5_NO_WRITE_WATCH") == nullptr; SYSTEM_INFO system{}; GetSystemInfo(&system); @@ -138,6 +164,66 @@ private: #endif } +#if defined(__linux__) + void SeedLinuxUsedRanges() { + std::ifstream maps("/proc/self/maps"); + if (!maps.is_open()) throw std::runtime_error("guest arena: cannot open /proc/self/maps"); + std::string line; + while (std::getline(maps, line)) { + std::istringstream fields(line); + std::uintptr_t begin = 0, end = 0; + char dash = 0; + if (!(fields >> std::hex >> begin >> dash >> end) || dash != '-' || end <= begin) throw std::runtime_error("guest arena: malformed /proc/self/maps line: " + line); + if (end <= ArenaStart || begin >= ApplicationAreaEnd) continue; + const auto first = std::max(begin, ArenaStart); + const auto last = std::min(end, ApplicationAreaEnd); + _used.emplace(first, last); + _hostRegions.emplace_back(first, last); + } + _used.emplace(SystemReservedStart, SystemReservedEnd); + _hostRegions.emplace_back(SystemReservedStart, SystemReservedEnd); + ReserveFreeRanges(); + _base = ArenaStart; + _end = ApplicationAreaEnd; + } + + static void TryReserve(std::vector>& occupied, std::uintptr_t from, std::uintptr_t to) { + constexpr std::uintptr_t kChunk = 1ULL << 30; + for (std::uintptr_t cursor = from; cursor < to; cursor += kChunk) { + const auto end = std::min(cursor + kChunk, to); + void* const wanted = reinterpret_cast(cursor); + void* const placed = mmap(wanted, end - cursor, PROT_NONE, + MAP_PRIVATE | MAP_ANONYMOUS | MAP_NORESERVE | MAP_FIXED_NOREPLACE, -1, 0); + if (placed == wanted) continue; + const int failed = placed == MAP_FAILED ? errno : EINVAL; + if (placed != MAP_FAILED) munmap(placed, end - cursor); + if (failed != EEXIST) { + char message[128]; + std::snprintf(message, sizeof(message), "guest arena: cannot reserve 0x%llx+0x%llx", + static_cast(cursor), static_cast(end - cursor)); + throw std::system_error(failed, std::system_category(), message); + } + occupied.emplace_back(cursor, end); + } + } + + void ReserveFreeRanges() { + std::vector> occupied; + std::uintptr_t cursor = ArenaStart; + for (const auto& [usedStart, usedEnd] : _used) { + if (usedStart > ApplicationAreaEnd) break; + if (usedStart > cursor) TryReserve(occupied, cursor, std::min(usedStart, ApplicationAreaEnd)); + if (usedEnd > cursor) cursor = usedEnd; + if (cursor >= ApplicationAreaEnd) break; + } + if (cursor < ApplicationAreaEnd) TryReserve(occupied, cursor, ApplicationAreaEnd); + for (const auto& [from, to] : occupied) { + _used.emplace(from, to); + _hostRegions.emplace_back(from, to); + } + } +#endif + std::mutex _lock; std::map _used; std::vector> _holes; @@ -191,6 +277,16 @@ void GuestArenaRange_nid_postfix(std::uintptr_t* base, std::size_t* bytes) { *bytes = Arena::Get().Size(); } +namespace { + +std::invalid_argument OutsideArena(const char* operation, const void* pointer, std::size_t bytes) { + char message[128]; + std::snprintf(message, sizeof(message), "%s 0x%llx+0x%zx outside the guest arena", operation, static_cast(reinterpret_cast(pointer)), bytes); + return std::invalid_argument(message); +} + +} + #ifdef _WIN32 void GuestArenaSetProtection_nid_postfix(std::uintptr_t address, std::size_t bytes, std::uint32_t protection) { WindowsMappings::Get().SetProtection(address, bytes, protection); @@ -222,16 +318,6 @@ bool GuestArenaHostRegionOverlaps_nid_postfix(std::uintptr_t address, std::size_ return Arena::Get().OverlapsHostRegion(address, bytes); } -namespace { - -std::invalid_argument OutsideArena(const char* operation, const void* pointer, std::size_t bytes) { - char message[128]; - std::snprintf(message, sizeof(message), "%s 0x%llx+0x%zx outside the guest arena", operation, static_cast(reinterpret_cast(pointer)), bytes); - return std::invalid_argument(message); -} - -} - std::uint64_t GuestArenaCommitGeneration_nid_postfix() { return commitGeneration.load(std::memory_order_acquire); } @@ -268,6 +354,52 @@ void GuestArenaUnmapAlias_nid_postfix(void* alias) { } #endif +#if defined(__linux__) +namespace { +int PosixProtection(std::uint32_t windowsProtection) { + switch (windowsProtection) { + case 0x01: return PROT_NONE; + case 0x02: return PROT_READ; + case 0x04: return PROT_READ | PROT_WRITE; + case 0x08: return PROT_READ | PROT_WRITE; + case 0x10: return PROT_EXEC; + case 0x20: return PROT_READ | PROT_EXEC; + case 0x40: return PROT_READ | PROT_WRITE | PROT_EXEC; + default: throw std::runtime_error("guest arena: unmapped protection value"); + } +} + +void* PlaceAt(void* pointer, std::size_t bytes, std::uint32_t protection) { + return mmap(pointer, bytes, PosixProtection(protection), MAP_PRIVATE | MAP_ANONYMOUS | MAP_FIXED, -1, 0); +} +} +#endif + +#if defined(__linux__) +void GuestArenaSetProtection_nid_postfix(std::uintptr_t address, std::size_t bytes, std::uint32_t protection) { + if (!Arena::Get().Contains(reinterpret_cast(address), bytes)) throw OutsideArena("set protection", reinterpret_cast(address), bytes); + if (mprotect(reinterpret_cast(address), bytes, PosixProtection(protection)) != 0) + throw std::system_error(errno, std::generic_category(), "guest arena protection failed"); +} + +void GuestArenaCommit_nid_postfix(void* pointer, std::size_t bytes, std::uint32_t protection, std::size_t granule) { + (void)granule; + if (!Arena::Get().Contains(pointer, bytes)) throw OutsideArena("commit", pointer, bytes); + const bool marked = Arena::Get().MarkUsedForCommit(pointer, bytes); + try { + if (PlaceAt(pointer, bytes, protection) == MAP_FAILED) throw std::system_error(errno, std::generic_category(), "guest arena commit failed"); + } catch (...) { + if (marked) Arena::Get().Release(pointer, bytes); + throw; + } +} + +void GuestArenaReset_nid_postfix(void* pointer, std::size_t bytes) { + if (!Arena::Get().Contains(pointer, bytes)) throw OutsideArena("reset", pointer, bytes); + if (PlaceAt(pointer, bytes, 0x01) == MAP_FAILED) throw std::system_error(errno, std::generic_category(), "guest arena reset failed"); +} +#endif + bool GuestArenaWriteWatched_nid_postfix() { return Arena::Get().WriteWatched(); } @@ -276,8 +408,7 @@ bool GuestArenaBeginHostWrite_nid_postfix(void* pointer, std::size_t bytes) { #ifdef _WIN32 return WindowsMappings::Get().BeginHostWrite(reinterpret_cast(pointer), bytes); #else - (void)pointer; - (void)bytes; + GuestWriteWatch::GuestWriteWatchBeginHostWrite_nid_postfix(pointer, bytes); return true; #endif } @@ -286,8 +417,7 @@ void GuestArenaEndHostWrite_nid_postfix(void* pointer, std::size_t bytes) { #ifdef _WIN32 WindowsMappings::Get().EndHostWrite(reinterpret_cast(pointer), bytes); #else - (void)pointer; - (void)bytes; + GuestWriteWatch::GuestWriteWatchEndHostWrite_nid_postfix(pointer, bytes); #endif } @@ -295,6 +425,7 @@ void GuestArenaEndHostWrite_nid_postfix(void* pointer, std::size_t bytes) { namespace { std::atomic sharedBackingResolver{nullptr}; +std::atomic sharedBackingWriter{nullptr}; } @@ -302,6 +433,15 @@ void GuestArenaSetSharedBacking_nid_postfix(SharedBackingResolver resolver) { sharedBackingResolver.store(resolver, std::memory_order_release); } +void GuestArenaSetSharedBackingWriter_nid_no_patch(SharedBackingWriter writer) { + sharedBackingWriter.store(writer, std::memory_order_release); +} + +bool GuestArenaWriteSharedBacking_nid_no_patch(std::uintptr_t address, const void* source, std::size_t bytes) { + const auto writer = sharedBackingWriter.load(std::memory_order_acquire); + return writer != nullptr && writer(address, source, bytes); +} + bool GuestArenaSharedBacking_nid_postfix(std::uintptr_t address, std::size_t bytes, int* file, std::uint64_t* offset) { const auto resolver = sharedBackingResolver.load(std::memory_order_acquire); return resolver != nullptr && resolver(address, bytes, file, offset); diff --git a/core/libs/prx/libc/src/GuestHeap.cpp b/core/libs/prx/libc/src/GuestHeap.cpp index ae9d1674c..24442855c 100644 --- a/core/libs/prx/libc/src/GuestHeap.cpp +++ b/core/libs/prx/libc/src/GuestHeap.cpp @@ -22,9 +22,12 @@ namespace GuestHeap { namespace { +constexpr std::uint32_t ReadWriteProtection = 0x04; + // Each block is preceded by a header holding the raw block address and its size in bytes. constexpr std::size_t HeaderBytes = 2 * sizeof(void*); -constexpr std::size_t MinimumAlignment = 16; +constexpr std::size_t RawAlignment = 16; +constexpr std::size_t MinimumAlignment = 32; constexpr std::size_t PageBytes = 0x4000; constexpr std::size_t SpanBytes = 1u << 20; constexpr std::size_t MaximumSmallBytes = 64u * 1024u; @@ -109,12 +112,7 @@ private: } static void commit(void* pointer, std::size_t bytes) { -#ifdef _WIN32 - GuestArena::GuestArenaCommit_nid_postfix(pointer, bytes, PAGE_READWRITE, bytes); -#else - (void)pointer; - (void)bytes; -#endif + GuestArena::GuestArenaCommit_nid_postfix(pointer, bytes, ReadWriteProtection, bytes); } // Freed large blocks stay committed and are handed out again for the same block size: the game @@ -167,10 +165,8 @@ private: } static void release(void* raw, std::size_t bytes) { -#ifdef _WIN32 GuestArena::GuestArenaReset_nid_postfix(raw, bytes); GuestAllocations::GuestAllocationsInvalidate_nid_postfix(reinterpret_cast(raw), bytes); -#endif GuestArena::GuestArenaRelease_nid_postfix(raw, bytes); } @@ -200,7 +196,7 @@ void rawFree(void* raw, std::size_t blockBytes) { void* allocate(GuestAllocations::Mutation& mutation, std::size_t alignment, std::size_t bytes) { if (alignment == 0 || (alignment & (alignment - 1)) != 0) throw std::invalid_argument("invalid guest heap alignment"); alignment = std::max(alignment, MinimumAlignment); - const std::size_t padding = alignment > MinimumAlignment ? alignment : 0; + const std::size_t padding = alignment > RawAlignment ? alignment : 0; if (bytes > std::numeric_limits::max() - HeaderBytes - padding) throw std::length_error("guest heap allocation overflow"); std::size_t blockBytes = 0; void* raw = rawAllocate(bytes + HeaderBytes + padding, blockBytes); @@ -209,7 +205,7 @@ void* allocate(GuestAllocations::Mutation& mutation, std::size_t alignment, std: reinterpret_cast(pointer)[-2] = raw; reinterpret_cast(pointer)[-1] = blockBytes; try { - mutation.Add(pointer, bytes, true, true); + mutation.Add(pointer, bytes, true, true, true); } catch (...) { rawFree(raw, blockBytes); throw; diff --git a/core/libs/prx/libc/src/GuestWriteWatch.cpp b/core/libs/prx/libc/src/GuestWriteWatch.cpp index afde2dfc5..0345de9a4 100644 --- a/core/libs/prx/libc/src/GuestWriteWatch.cpp +++ b/core/libs/prx/libc/src/GuestWriteWatch.cpp @@ -22,6 +22,22 @@ #include #endif +#if defined(__APPLE__) +#include +#include +#include +#include +#include +#include +#include +#include +#include +#endif + +#if (defined(__linux__) && defined(PAGEMAP_SCAN) && defined(UFFD_FEATURE_WP_ASYNC)) || defined(__APPLE__) +#define APS5_GUEST_WRITE_WATCH 1 +#endif + namespace GuestWriteWatch { namespace { @@ -79,10 +95,56 @@ public: begin &= ~(PageBytes - 1); end = (end + PageBytes - 1) & ~(PageBytes - 1); if (end <= begin) return true; + if (!takeFresh(begin, end, true, written, context)) return false; + return scanWritten(begin, end, written, context); + } + + bool CollectFresh(std::uintptr_t begin, std::uintptr_t end, void (*written)(void*, std::uintptr_t, std::uintptr_t), void* context) { + if (!Available()) return false; + begin &= ~(PageBytes - 1); + end = (end + PageBytes - 1) & ~(PageBytes - 1); + if (end <= begin) return true; + { + std::shared_lock lock(_lock); + auto it = _fresh.upper_bound(begin); + if (it != _fresh.begin()) --it; + while (it != _fresh.end() && it->first < end && it->second <= begin) ++it; + if (it == _fresh.end() || it->first >= end) return true; + } + return takeFresh(begin, end, false, written, context); + } + + bool CollectArmed(std::uintptr_t begin, std::uintptr_t end, void (*written)(void*, std::uintptr_t, std::uintptr_t), void* context) { + if (!Available()) return false; + begin &= ~(PageBytes - 1); + end = (end + PageBytes - 1) & ~(PageBytes - 1); + if (end <= begin) return true; + std::vector> armed; + { + std::shared_lock lock(_lock); + if (!covers(begin, end)) return false; + auto cursor = begin; + auto it = _fresh.upper_bound(begin); + if (it != _fresh.begin()) --it; + for (; it != _fresh.end() && it->first < end; ++it) { + if (it->second <= cursor) continue; + if (it->first > cursor) armed.emplace_back(cursor, it->first); + cursor = std::max(cursor, it->second); + } + if (cursor < end) armed.emplace_back(cursor, end); + } + for (const auto& [from, to] : armed) { + if (!scanWritten(from, to, written, context)) return false; + } + return true; + } + +private: + bool takeFresh(std::uintptr_t begin, std::uintptr_t end, bool whole, void (*written)(void*, std::uintptr_t, std::uintptr_t), void* context) { std::vector> fresh; { std::unique_lock lock(_lock); - if (!covers(begin, end)) return false; + if (whole && !covers(begin, end)) return false; auto it = _fresh.upper_bound(begin); if (it != _fresh.begin()) --it; for (; it != _fresh.end() && it->first < end; ++it) { @@ -90,7 +152,7 @@ public: const auto to = std::min(it->second, end); if (from < to) fresh.emplace_back(from, to); } - if (!fresh.empty()) remove(_fresh, begin, end); + for (const auto& [from, to] : fresh) remove(_fresh, from, to); } for (const auto& [from, to] : fresh) { written(context, from, to); @@ -101,9 +163,14 @@ public: remove(_fresh, left, right); insert(_fresh, left, right); } - written(context, begin, end); + if (whole) written(context, begin, end); + else for (const auto& [left, right] : fresh) written(context, left, right); return false; } + return true; + } + + bool scanWritten(std::uintptr_t begin, std::uintptr_t end, void (*written)(void*, std::uintptr_t, std::uintptr_t), void* context) const { std::array regions; auto cursor = begin; while (cursor < end) { @@ -249,13 +316,294 @@ private: std::map _fresh; }; +const bool g_opened = (Watch::Get(), true); +#elif defined(__APPLE__) +class Watch { +public: + static Watch& Get() { + static Watch watch; + return watch; + } + + bool Available() const { + return _available; + } + + void Register(std::uintptr_t begin, std::uintptr_t end) { + begin = pageStart(begin); + end = pageEnd(end); + if (!_available || end <= begin) return; + std::vector pages((end - begin) / _page, Written); + for (auto cursor = begin; cursor < end;) { + mach_vm_address_t address = cursor; + mach_vm_size_t size = 0; + vm_region_basic_info_data_64_t info{}; + mach_msg_type_number_t count = VM_REGION_BASIC_INFO_COUNT_64; + mach_port_t object = MACH_PORT_NULL; + if (mach_vm_region(mach_task_self(), &address, &size, VM_REGION_BASIC_INFO_64, reinterpret_cast(&info), &count, &object) != KERN_SUCCESS || address >= end) break; + const auto first = std::max(cursor, address); + const auto last = std::min(end, address + size); + const std::uint8_t protection = ((info.protection & VM_PROT_READ) ? PROT_READ : 0) | ((info.protection & VM_PROT_WRITE) ? PROT_WRITE : 0) | ((info.protection & VM_PROT_EXECUTE) ? PROT_EXEC : 0); + for (auto page = first; page < last; page += _page) pages[(page - begin) / _page] = Written | protection; + cursor = last; + } + const Lock lock(_lock); + forEachPage(begin, end, [&](std::uintptr_t page, std::uint8_t& state) { + pages[(page - begin) / _page] = static_cast(Written | (state & ProtectionMask)); + }); + erase(begin, end); + _ranges.emplace(begin, Range{end, std::move(pages)}); + } + + bool Unregister(std::uintptr_t begin, std::uintptr_t end) { + begin = pageStart(begin); + end = pageEnd(end); + if (!_available || end <= begin) return false; + const Lock lock(_lock); + return erase(begin, end); + } + + bool Covers(std::uintptr_t begin, std::uintptr_t end) { + if (!_available) return false; + const Lock lock(_lock); + return covers(begin, end); + } + + bool Collect(std::uintptr_t begin, std::uintptr_t end, void (*written)(void*, std::uintptr_t, std::uintptr_t), void* context) { + if (!_available) return false; + begin = pageStart(begin); + end = pageEnd(end); + if (end <= begin) return true; + std::vector> runs; + { + const Lock lock(_lock); + if (!covers(begin, end)) return false; + forEachPage(begin, end, [&](std::uintptr_t page, std::uint8_t& state) { + if ((state & Written) == 0) return; + if (!runs.empty() && runs.back().second == page) runs.back().second = page + _page; + else runs.emplace_back(page, page + _page); + for (const auto& [first, last] : _hostWrites) + if (page >= first && page < last) return; + state &= static_cast(~Written); + if ((state & PROT_WRITE) != 0) state |= Protected; + }); + for (const auto& [first, last] : runs) protect(first, last); + } + for (const auto& [first, last] : runs) written(context, first, last); + return true; + } + + void ProtectionChanged(std::uintptr_t begin, std::uintptr_t end, int protection) { + begin = pageStart(begin); + end = pageEnd(end); + if (!_available || end <= begin) return; + const Lock lock(_lock); + forEachPage(begin, end, [&](std::uintptr_t, std::uint8_t& state) { + state = static_cast(Written | (protection & ProtectionMask)); + }); + } + + void BeginHostWrite(std::uintptr_t begin, std::uintptr_t end) { + begin = pageStart(begin); + end = pageEnd(end); + if (!_available || end <= begin) return; + const Lock lock(_lock); + _hostWrites.emplace_back(begin, end); + forEachPage(begin, end, [&](std::uintptr_t page, std::uint8_t& state) { + if ((state & Protected) != 0) ::mprotect(reinterpret_cast(page), _page, state & ProtectionMask); + state = static_cast((state & ProtectionMask) | Written); + }); + } + + void EndHostWrite(std::uintptr_t begin, std::uintptr_t end) { + begin = pageStart(begin); + end = pageEnd(end); + if (!_available || end <= begin) return; + const Lock lock(_lock); + const auto found = std::find(_hostWrites.begin(), _hostWrites.end(), std::pair{begin, end}); + if (found != _hostWrites.end()) _hostWrites.erase(found); + } + + int Protection(std::uintptr_t address, std::uintptr_t limit, std::uintptr_t* runEnd) { + const auto page = pageStart(address); + const Lock lock(_lock); + auto it = _ranges.upper_bound(page); + if (_available && it != _ranges.begin() && page < std::prev(it)->second.end) { + --it; + const auto& pages = it->second.pages; + const std::uint8_t protection = pages[(page - it->first) / _page] & ProtectionMask; + auto end = page + _page; + while (end < it->second.end && end < limit && (pages[(end - it->first) / _page] & ProtectionMask) == protection) end += _page; + *runEnd = std::min(end, limit); + return protection; + } + *runEnd = it != _ranges.end() ? std::min(it->first, limit) : limit; + return -1; + } + +private: + static constexpr std::uint8_t ProtectionMask = PROT_READ | PROT_WRITE | PROT_EXEC; + static constexpr std::uint8_t Written = 0x10; + static constexpr std::uint8_t Protected = 0x20; + + struct Range { + std::uintptr_t end; + std::vector pages; + }; + + struct Turns { + std::atomic next{0}; + std::atomic serving{0}; + }; + + struct Lock { + explicit Lock(Turns& turns) : _turns(turns), _turn(turns.next.fetch_add(1, std::memory_order_relaxed)) { + while (_turns.serving.load(std::memory_order_acquire) != _turn) ::sched_yield(); + } + ~Lock() { + _turns.serving.store(_turn + 1, std::memory_order_release); + } + Turns& _turns; + std::uint64_t _turn; + }; + + Watch() { + if (std::getenv("APS5_NO_WRITE_WATCH") != nullptr) return; + struct sigaction action {}; + action.sa_sigaction = &Watch::fault; + action.sa_flags = SA_SIGINFO | SA_RESTART; + sigemptyset(&action.sa_mask); + if (sigaction(SIGBUS, &action, &_previousBus) != 0 || sigaction(SIGSEGV, &action, &_previousSegv) != 0) { + std::fprintf(stderr, "[memory] write watch: cannot install the fault handler (%s); guest buffers are fully re-read every use\n", std::strerror(errno)); + return; + } + _available = true; + } + + static void fault(int signal, siginfo_t* info, void* context) { + auto& watch = Get(); + if (watch.record(reinterpret_cast(info->si_addr))) return; + const auto& previous = signal == SIGBUS ? watch._previousBus : watch._previousSegv; + if ((previous.sa_flags & SA_SIGINFO) != 0 && previous.sa_sigaction != nullptr) { + previous.sa_sigaction(signal, info, context); + return; + } + if (previous.sa_handler != SIG_DFL && previous.sa_handler != SIG_IGN && previous.sa_handler != nullptr) { + previous.sa_handler(signal); + return; + } + ::signal(signal, SIG_DFL); + } + + bool record(std::uintptr_t address) { + const auto page = pageStart(address); + const Lock lock(_lock); + auto it = _ranges.upper_bound(page); + if (it == _ranges.begin()) return false; + --it; + if (page >= it->second.end) return false; + auto& state = it->second.pages[(page - it->first) / _page]; + if ((state & Protected) != 0) { + state = static_cast((state & ProtectionMask) | Written); + ::mprotect(reinterpret_cast(page), _page, state & ProtectionMask); + return true; + } + return (state & PROT_WRITE) != 0; + } + + std::uintptr_t pageStart(std::uintptr_t address) const { + return address & ~(_page - 1); + } + + std::uintptr_t pageEnd(std::uintptr_t address) const { + return (address + _page - 1) & ~(_page - 1); + } + + template + void forEachPage(std::uintptr_t begin, std::uintptr_t end, TFunction function) { + auto it = _ranges.upper_bound(begin); + if (it != _ranges.begin()) --it; + for (; it != _ranges.end() && it->first < end; ++it) { + const auto first = std::max(begin, it->first); + const auto last = std::min(end, it->second.end); + for (auto page = first; page < last; page += _page) function(page, it->second.pages[(page - it->first) / _page]); + } + } + + void protect(std::uintptr_t begin, std::uintptr_t end) { + std::uintptr_t runStart = 0; + std::uintptr_t previous = begin; + int runProtection = -1; + const auto flush = [&](std::uintptr_t runEnd) { + if (runProtection >= 0) ::mprotect(reinterpret_cast(runStart), runEnd - runStart, runProtection); + runProtection = -1; + }; + forEachPage(begin, end, [&](std::uintptr_t page, std::uint8_t& state) { + const int wanted = (state & Protected) != 0 ? (state & ProtectionMask & ~PROT_WRITE) : -1; + if (wanted != runProtection || page != previous) { + flush(previous); + runStart = page; + runProtection = wanted; + } + previous = page + _page; + }); + flush(previous); + } + + bool covers(std::uintptr_t begin, std::uintptr_t end) const { + auto it = _ranges.upper_bound(begin); + if (it == _ranges.begin()) return false; + --it; + auto cursor = begin; + for (; it != _ranges.end() && it->first <= cursor; ++it) { + if (it->second.end > cursor) cursor = it->second.end; + if (cursor >= end) return true; + } + return false; + } + + bool erase(std::uintptr_t begin, std::uintptr_t end) { + bool removed = false; + auto it = _ranges.upper_bound(begin); + if (it != _ranges.begin()) --it; + while (it != _ranges.end() && it->first < end) { + const auto rangeBegin = it->first; + const auto rangeEnd = it->second.end; + if (rangeEnd <= begin) { + ++it; + continue; + } + removed = true; + auto pages = std::move(it->second.pages); + it = _ranges.erase(it); + const auto cutBegin = std::max(begin, rangeBegin); + const auto cutEnd = std::min(end, rangeEnd); + for (auto page = cutBegin; page < cutEnd; page += _page) { + const auto state = pages[(page - rangeBegin) / _page]; + if ((state & Protected) != 0) ::mprotect(reinterpret_cast(page), _page, state & ProtectionMask); + } + if (rangeBegin < cutBegin) _ranges.emplace(rangeBegin, Range{cutBegin, std::vector(pages.begin(), pages.begin() + (cutBegin - rangeBegin) / _page)}); + if (cutEnd < rangeEnd) it = _ranges.emplace(cutEnd, Range{rangeEnd, std::vector(pages.begin() + (cutEnd - rangeBegin) / _page, pages.end())}).first; + } + return removed; + } + + const std::uintptr_t _page = static_cast(::getpagesize()); + bool _available = false; + Turns _lock; + std::map _ranges; + std::vector> _hostWrites; + struct sigaction _previousBus {}; + struct sigaction _previousSegv {}; +}; + const bool g_opened = (Watch::Get(), true); #endif } bool GuestWriteWatchAvailable_nid_postfix() { -#if defined(__linux__) && defined(PAGEMAP_SCAN) && defined(UFFD_FEATURE_WP_ASYNC) +#ifdef APS5_GUEST_WRITE_WATCH return Watch::Get().Available(); #else return false; @@ -263,7 +611,7 @@ bool GuestWriteWatchAvailable_nid_postfix() { } void GuestWriteWatchRegister_nid_postfix(const void* pointer, std::size_t bytes) { -#if defined(__linux__) && defined(PAGEMAP_SCAN) && defined(UFFD_FEATURE_WP_ASYNC) +#ifdef APS5_GUEST_WRITE_WATCH const auto begin = reinterpret_cast(pointer); Watch::Get().Register(begin, begin + bytes); #else @@ -273,7 +621,7 @@ void GuestWriteWatchRegister_nid_postfix(const void* pointer, std::size_t bytes) } bool GuestWriteWatchUnregister_nid_postfix(const void* pointer, std::size_t bytes) { -#if defined(__linux__) && defined(PAGEMAP_SCAN) && defined(UFFD_FEATURE_WP_ASYNC) +#ifdef APS5_GUEST_WRITE_WATCH const auto begin = reinterpret_cast(pointer); return Watch::Get().Unregister(begin, begin + bytes); #else @@ -284,7 +632,7 @@ bool GuestWriteWatchUnregister_nid_postfix(const void* pointer, std::size_t byte } bool GuestWriteWatchCovers_nid_postfix(std::uintptr_t address, std::size_t bytes) { -#if defined(__linux__) && defined(PAGEMAP_SCAN) && defined(UFFD_FEATURE_WP_ASYNC) +#ifdef APS5_GUEST_WRITE_WATCH return bytes != 0 && address + bytes > address && Watch::Get().Covers(address, address + bytes); #else static_cast(address); @@ -294,7 +642,81 @@ bool GuestWriteWatchCovers_nid_postfix(std::uintptr_t address, std::size_t bytes } bool GuestWriteWatchCollect_nid_postfix(std::uintptr_t address, std::size_t bytes, void (*written)(void* context, std::uintptr_t begin, std::uintptr_t end), void* context) { +#ifdef APS5_GUEST_WRITE_WATCH + if (bytes == 0 || address + bytes < address) return false; + return Watch::Get().Collect(address, address + bytes, written, context); +#else + static_cast(address); + static_cast(bytes); + static_cast(written); + static_cast(context); + return false; +#endif +} + +void GuestWriteWatchBeginHostWrite_nid_postfix(const void* pointer, std::size_t bytes) { +#if defined(__APPLE__) + const auto begin = reinterpret_cast(pointer); + Watch::Get().BeginHostWrite(begin, begin + bytes); +#else + static_cast(pointer); + static_cast(bytes); +#endif +} + +void GuestWriteWatchEndHostWrite_nid_postfix(const void* pointer, std::size_t bytes) { +#if defined(__APPLE__) + const auto begin = reinterpret_cast(pointer); + Watch::Get().EndHostWrite(begin, begin + bytes); +#else + static_cast(pointer); + static_cast(bytes); +#endif +} + +void GuestWriteWatchProtectionChanged_nid_postfix(const void* pointer, std::size_t bytes, int protection) { +#if defined(__APPLE__) + const auto begin = reinterpret_cast(pointer); + Watch::Get().ProtectionChanged(begin, begin + bytes, protection); +#else + static_cast(pointer); + static_cast(bytes); + static_cast(protection); +#endif +} + +int GuestWriteWatchProtection_nid_postfix(std::uintptr_t address, std::uintptr_t limit, std::uintptr_t* runEnd) { +#if defined(__APPLE__) + return Watch::Get().Protection(address, limit, runEnd); +#else + static_cast(address); + *runEnd = limit; + return -1; +#endif +} + +bool GuestWriteWatchCollectFresh_nid_postfix(std::uintptr_t address, std::size_t bytes, void (*written)(void* context, std::uintptr_t begin, std::uintptr_t end), void* context) { #if defined(__linux__) && defined(PAGEMAP_SCAN) && defined(UFFD_FEATURE_WP_ASYNC) + if (bytes == 0 || address + bytes < address) return false; + return Watch::Get().CollectFresh(address, address + bytes, written, context); +#elif defined(__APPLE__) + static_cast(written); + static_cast(context); + return bytes != 0 && address + bytes >= address; +#else + static_cast(address); + static_cast(bytes); + static_cast(written); + static_cast(context); + return false; +#endif +} + +bool GuestWriteWatchCollectArmed_nid_postfix(std::uintptr_t address, std::size_t bytes, void (*written)(void* context, std::uintptr_t begin, std::uintptr_t end), void* context) { +#if defined(__linux__) && defined(PAGEMAP_SCAN) && defined(UFFD_FEATURE_WP_ASYNC) + if (bytes == 0 || address + bytes < address) return false; + return Watch::Get().CollectArmed(address, address + bytes, written, context); +#elif defined(__APPLE__) if (bytes == 0 || address + bytes < address) return false; return Watch::Get().Collect(address, address + bytes, written, context); #else diff --git a/core/libs/prx/libc/src/Math.cpp b/core/libs/prx/libc/src/Math.cpp index 1ae4fb350..2828d8ed4 100644 --- a/core/libs/prx/libc/src/Math.cpp +++ b/core/libs/prx/libc/src/Math.cpp @@ -14,6 +14,10 @@ std::lldiv_t APS5_VABI lldiv_nid_postfix(long long numerator, long long denomina return std::lldiv(numerator, denominator); } +std::lldiv_t APS5_VABI ldiv_nid_postfix(std::int64_t numerator, std::int64_t denominator) { + return std::lldiv(numerator, denominator); +} + float APS5_VABI fmodf_nid_postfix(float x, float y) { return std::fmod(x, y); } float APS5_VABI asinf_nid_postfix(float x) { return std::asin(x); } float APS5_VABI acosf_nid_postfix(float x) { return std::acos(x); } @@ -105,6 +109,16 @@ short APS5_VABI _FDtest_nid_postfix(const float* value) { return Finite; } int APS5_VABI __isnanf_nid_postfix(float x) { return std::isnan(x) ? 1 : 0; } +int APS5_VABI __fpclassifyf_nid_postfix(float x) { + constexpr int Infinite = 0x01, NotANumber = 0x02, Normal = 0x04, Subnormal = 0x08, Zero = 0x10; + std::uint32_t bits; + std::memcpy(&bits, &x, sizeof(bits)); + const auto exponent = bits & 0x7f800000u; + const auto fraction = bits & 0x007fffffu; + if (exponent == 0) return fraction != 0 ? Subnormal : Zero; + if (exponent == 0x7f800000u) return fraction != 0 ? NotANumber : Infinite; + return Normal; +} int APS5_VABI __signbitf_nid_postfix(float x) { return std::signbit(x) ? 1 : 0; } static std::mutex g_randLock; diff --git a/core/libs/prx/libc/src/Multibyte.cpp b/core/libs/prx/libc/src/Multibyte.cpp index 2a76fc7dc..10ccdb457 100644 --- a/core/libs/prx/libc/src/Multibyte.cpp +++ b/core/libs/prx/libc/src/Multibyte.cpp @@ -50,6 +50,26 @@ std::size_t APS5_VABI wcrtomb_nid_postfix(char* destination, std::uint16_t value return 1; } +std::size_t APS5_VABI wcsrtombs_nid_postfix(char* destination, const std::uint16_t** source, std::size_t capacity, void* state) { + const auto* wide = *source; + std::size_t converted = 0; + while (!destination || converted < capacity) { + char byte; + if (wcrtomb_nid_postfix(&byte, wide[converted], state) == static_cast(-1)) { + if (destination) *source = wide + converted; + return static_cast(-1); + } + if (destination) destination[converted] = byte; + if (byte == '\0') { + if (destination) *source = nullptr; + return converted; + } + ++converted; + } + *source = wide + converted; + return converted; +} + int APS5_VABI wcsrtombs_s_nid_postfix( std::size_t* result, char* destination, std::size_t capacity, const std::uint16_t** source, std::size_t limit, void* state diff --git a/core/libs/prx/libc/src/Process.cpp b/core/libs/prx/libc/src/Process.cpp index d8ce47284..ce307b378 100644 --- a/core/libs/prx/libc/src/Process.cpp +++ b/core/libs/prx/libc/src/Process.cpp @@ -8,6 +8,9 @@ #include #include #include +#ifdef _WIN32 +#include +#endif #include #include #include "prx/libc/include/Shutdown.hpp" @@ -106,6 +109,13 @@ extern "C" { std::fprintf(stderr, "[libc] _Exit(%d) called from %p\n", code, __builtin_return_address(0)); std::fflush(stderr); } + LibcTerminate_nid_no_patch(code); +} + +[[noreturn]] void LibcTerminate_nid_no_patch(int code) { +#ifdef _WIN32 + TerminateProcess(GetCurrentProcess(), static_cast(code)); +#endif std::_Exit(code); } @@ -113,6 +123,7 @@ extern "C" { exitRequested.store(true); if (exitStarted.exchange(true)) LibcAwaitExit_nid_postfix(); LibcRunShutdown_nid_postfix(); + CxaFinalize_nid_no_patch(nullptr); std::exit(code); } diff --git a/core/libs/prx/libc/src/Rename.cpp b/core/libs/prx/libc/src/Rename.cpp new file mode 100644 index 000000000..3e5c5658c --- /dev/null +++ b/core/libs/prx/libc/src/Rename.cpp @@ -0,0 +1,23 @@ +#include "prx/libc/include/FilesystemError.hpp" +#include "prx/libc/include/General.hpp" +#include +#include + +extern "C" int APS5_VABI rename_nid_postfix(const char* from, const char* to) { + if (!from || !to) { errno = 14; return -1; } + if (!*from || !*to) { errno = 2; return -1; } + try { + const auto source = ResolvePath_nid_no_patch(from); + const auto destination = ResolvePath_nid_no_patch(to); + std::error_code error; + std::filesystem::rename(source, destination, error); + if (error) { errno = FilesystemError(error); return -1; } + RecordWrittenPath_nid_no_patch(source); + RecordWrittenPath_nid_no_patch(destination); + return 0; + } catch (const std::bad_alloc&) { errno = 12; return -1; } + catch (const std::filesystem::filesystem_error& error) { + errno = FilesystemError(error.code()); + return -1; + } +} diff --git a/core/libs/prx/libc/src/RuneLocale.cpp b/core/libs/prx/libc/src/RuneLocale.cpp new file mode 100644 index 000000000..cba8bd792 --- /dev/null +++ b/core/libs/prx/libc/src/RuneLocale.cpp @@ -0,0 +1,144 @@ +#include "prx/libc/include/general/VabiMacros.hpp" +#include +#include +#include + +namespace GuestRune { + +struct Range { + std::int32_t count; + void* entries; +}; + +struct Locale { + char magic[8]; + char encoding[32]; + void* getRune; + void* putRune; + std::int32_t invalidRune; + std::uint64_t types[256]; + std::int32_t lower[256]; + std::int32_t upper[256]; + Range typeRanges; + Range lowerRanges; + Range upperRanges; + void* variable; + std::int32_t variableLength; +}; +static_assert(offsetof(Locale, invalidRune) == 56); +static_assert(offsetof(Locale, types) == 64); +static_assert(offsetof(Locale, lower) == 2112); +static_assert(offsetof(Locale, upper) == 3136); +static_assert(offsetof(Locale, typeRanges) == 4160); +static_assert(offsetof(Locale, variable) == 4208); +static_assert(sizeof(Locale) == 4224); + +constexpr std::uint64_t Alpha = 0x100; +constexpr std::uint64_t Control = 0x200; +constexpr std::uint64_t Digit = 0x400; +constexpr std::uint64_t Graph = 0x800; +constexpr std::uint64_t Lower = 0x1000; +constexpr std::uint64_t Punct = 0x2000; +constexpr std::uint64_t Space = 0x4000; +constexpr std::uint64_t Upper = 0x8000; +constexpr std::uint64_t Hex = 0x10000; +constexpr std::uint64_t Blank = 0x20000; +constexpr std::uint64_t Print = 0x40000; +constexpr std::uint64_t Number = 0x400000; + +constexpr std::uint64_t Type(int c) { + if (c == '\t') return Control | Space | Blank; + if (c >= '\n' && c <= '\r') return Control | Space; + if (c < ' ' || c == 0x7f) return Control; + if (c == ' ') return Space | Blank | Print; + if (c >= '0' && c <= '9') return Digit | Print | Graph | Hex | Number | static_cast(c - '0'); + if (c >= 'A' && c <= 'F') return Upper | Hex | Print | Graph | Alpha | static_cast(c - 'A' + 10); + if (c >= 'a' && c <= 'f') return Lower | Hex | Print | Graph | Alpha | static_cast(c - 'a' + 10); + if (c >= 'A' && c <= 'Z') return Upper | Print | Graph | Alpha; + if (c >= 'a' && c <= 'z') return Lower | Print | Graph | Alpha; + if (c < 0x7f) return Punct | Print | Graph; + return 0; +} + +constexpr Locale MakeDefault() { + Locale locale{{'R', 'u', 'n', 'e', 'M', 'a', 'g', 'i'}, "NONE", nullptr, nullptr, 0xfffd, {}, {}, {}, {}, {}, {}, nullptr, 0}; + for (int c = 0; c < 256; ++c) { + locale.types[c] = Type(c); + locale.lower[c] = c >= 'A' && c <= 'Z' ? c + ('a' - 'A') : c; + locale.upper[c] = c >= 'a' && c <= 'z' ? c - ('a' - 'A') : c; + } + return locale; +} + +} + +extern "C" { + +extern const GuestRune::Locale _DefaultRuneLocale_nid_postfix; +const GuestRune::Locale _DefaultRuneLocale_nid_postfix = GuestRune::MakeDefault(); +const GuestRune::Locale* _CurrentRuneLocale_nid_postfix = &_DefaultRuneLocale_nid_postfix; +int __mb_sb_limit_nid_postfix = 256; + +} + +namespace GuestRune { + +constexpr std::uint32_t Weof = 0xffffffff; + +int Is(std::uint32_t c, std::uint64_t mask) { + return c < 256 && (_CurrentRuneLocale_nid_postfix->types[c] & mask) != 0; +} + +} + +extern "C" { + +int APS5_VABI iswalnum_nid_postfix(std::uint32_t c) { return GuestRune::Is(c, GuestRune::Alpha | GuestRune::Number); } +int APS5_VABI iswalpha_nid_postfix(std::uint32_t c) { return GuestRune::Is(c, GuestRune::Alpha); } +int APS5_VABI iswblank_nid_postfix(std::uint32_t c) { return GuestRune::Is(c, GuestRune::Blank); } +int APS5_VABI iswcntrl_nid_postfix(std::uint32_t c) { return GuestRune::Is(c, GuestRune::Control); } +int APS5_VABI iswdigit_nid_postfix(std::uint32_t c) { return GuestRune::Is(c, GuestRune::Digit); } +int APS5_VABI iswgraph_nid_postfix(std::uint32_t c) { return GuestRune::Is(c, GuestRune::Graph); } +int APS5_VABI iswlower_nid_postfix(std::uint32_t c) { return GuestRune::Is(c, GuestRune::Lower); } +int APS5_VABI iswprint_nid_postfix(std::uint32_t c) { return GuestRune::Is(c, GuestRune::Print); } +int APS5_VABI iswpunct_nid_postfix(std::uint32_t c) { return GuestRune::Is(c, GuestRune::Punct); } +int APS5_VABI iswspace_nid_postfix(std::uint32_t c) { return GuestRune::Is(c, GuestRune::Space); } +int APS5_VABI iswupper_nid_postfix(std::uint32_t c) { return GuestRune::Is(c, GuestRune::Upper); } +int APS5_VABI iswxdigit_nid_postfix(std::uint32_t c) { return GuestRune::Is(c, GuestRune::Hex); } +int APS5_VABI iswctype_nid_postfix(std::uint32_t c, std::uint64_t mask) { return GuestRune::Is(c, mask); } + +std::uint64_t APS5_VABI wctype_nid_postfix(const char* property) { + static constexpr struct { + const char* name; + std::uint64_t mask; + } properties[] = { + {"alnum", GuestRune::Alpha | GuestRune::Number}, {"alpha", GuestRune::Alpha}, {"blank", GuestRune::Blank}, + {"cntrl", GuestRune::Control}, {"digit", GuestRune::Digit}, {"graph", GuestRune::Graph}, + {"lower", GuestRune::Lower}, {"print", GuestRune::Print}, {"punct", GuestRune::Punct}, + {"space", GuestRune::Space}, {"upper", GuestRune::Upper}, {"xdigit", GuestRune::Hex}, + {"ideogram", 0x80000}, {"special", 0x100000}, {"phonogram", 0x200000}, + {"number", GuestRune::Number}, {"rune", 0xffffff00}, + }; + for (const auto& entry : properties) { + if (std::strcmp(property, entry.name) == 0) return entry.mask; + } + return 0; +} + +std::uint32_t APS5_VABI towlower_nid_postfix(std::uint32_t c) { + return c < 256 ? static_cast(_CurrentRuneLocale_nid_postfix->lower[c]) : c; +} + +std::uint32_t APS5_VABI towupper_nid_postfix(std::uint32_t c) { + return c < 256 ? static_cast(_CurrentRuneLocale_nid_postfix->upper[c]) : c; +} + +std::uint32_t APS5_VABI btowc_nid_postfix(int c) { + return c == -1 ? GuestRune::Weof : static_cast(c); +} + +int APS5_VABI wctob_nid_postfix(std::uint32_t c) { + return c < 256 ? static_cast(c) : -1; +} + +} diff --git a/core/libs/prx/libc/src/RuntimeSupport.cpp b/core/libs/prx/libc/src/RuntimeSupport.cpp index e6bf99f61..4943567c8 100644 --- a/core/libs/prx/libc/src/RuntimeSupport.cpp +++ b/core/libs/prx/libc/src/RuntimeSupport.cpp @@ -1,5 +1,6 @@ #include #include +#include #include #include #include @@ -94,10 +95,12 @@ unsigned int APS5_VABI _Atomic_load_4_nid_postfix(volatile unsigned int* target, } std::intmax_t APS5_VABI strtoimax_nid_postfix(const char* str, char** endptr, int base) { + if (StopAtBinaryPrefix_nid_no_patch(str, endptr, base)) return 0; return std::strtoimax(str, endptr, base); } std::uintmax_t APS5_VABI strtoumax_nid_postfix(const char* str, char** endptr, int base) { + if (StopAtBinaryPrefix_nid_no_patch(str, endptr, base)) return 0; return std::strtoumax(str, endptr, base); } @@ -130,8 +133,9 @@ void APS5_VABI _Unlockfilelock_nid_postfix(FileStream* stream) { #endif } -unsigned long APS5_VABI _Stoul_nid_postfix(const char* str, char** endptr, int base) { - return std::strtoul(str, endptr, base); +std::uint64_t APS5_VABI _Stoul_nid_postfix(const char* str, char** endptr, int base) { + if (StopAtBinaryPrefix_nid_no_patch(str, endptr, base)) return 0; + return std::strtoull(str, endptr, base); } void APS5_VABI _Locksyslock_nid_postfix() { diff --git a/core/libs/prx/libc/src/StdioExtras.cpp b/core/libs/prx/libc/src/StdioExtras.cpp index 0b35feabc..0e74f5986 100644 --- a/core/libs/prx/libc/src/StdioExtras.cpp +++ b/core/libs/prx/libc/src/StdioExtras.cpp @@ -1,3 +1,4 @@ +#include #include #include #include @@ -18,10 +19,19 @@ namespace { using GuestNewHandler = void (APS5_VABI*)(); -GuestNewHandler g_newHandler = nullptr; +std::atomic g_newHandler{nullptr}; void* Allocate(std::size_t size) { - return ApplicationHeapAllocate_nid_no_patch(size == 0 ? 1 : size); + for (;;) { + GuestNewHandler handler = nullptr; + try { + return ApplicationHeapAllocate_nid_no_patch(size == 0 ? 1 : size); + } catch (const std::bad_alloc&) { + handler = g_newHandler.load(); + if (handler == nullptr) throw; + } + handler(); + } } } @@ -58,13 +68,11 @@ const char* APS5_VABI setlocale_nid_postfix(int category, const char* locale) { } GuestNewHandler APS5_VABI _ZSt15set_new_handlerPFvvE_nid_postfix(GuestNewHandler handler) { - const auto previous = g_newHandler; - g_newHandler = handler; - return previous; + return g_newHandler.exchange(handler); } GuestNewHandler APS5_VABI _ZSt15get_new_handlerv_nid_postfix() { - return g_newHandler; + return g_newHandler.load(); } unsigned char _ZSt7nothrow_nid_postfix = 0; diff --git a/core/libs/prx/libc/src/StringExtensions.cpp b/core/libs/prx/libc/src/StringExtensions.cpp index a9a759763..155ec1443 100644 --- a/core/libs/prx/libc/src/StringExtensions.cpp +++ b/core/libs/prx/libc/src/StringExtensions.cpp @@ -1,5 +1,6 @@ #include "prx/libc/include/general/VabiMacros.hpp" #include +#include #include #include #include @@ -20,6 +21,22 @@ char* APS5_VABI basename_nid_postfix(const char* path) { return buffer; } +char* APS5_VABI __inet_ntoa_nid_postfix(unsigned int address) { + thread_local char buffer[sizeof("255.255.255.255")]; + unsigned char bytes[sizeof(address)]; + std::memcpy(bytes, &address, sizeof(bytes)); + char* end = buffer; + for (std::size_t index = 0; index < sizeof(bytes); ++index) { + if (index != 0) *end++ = '.'; + const unsigned value = bytes[index]; + if (value >= 100) *end++ = static_cast('0' + value / 100); + if (value >= 10) *end++ = static_cast('0' + value / 10 % 10); + *end++ = static_cast('0' + value % 10); + } + *end = 0; + return buffer; +} + std::size_t APS5_VABI strnlen_nid_postfix(const char* text, std::size_t limit) { std::size_t length = 0; while (length < limit && text[length] != '\0') ++length; @@ -54,6 +71,12 @@ std::size_t APS5_VABI strlcat_nid_postfix(char* destination, const char* source, return destinationLength + sourceLength; } +char* APS5_VABI stpcpy_nid_postfix(char* destination, const char* source) { + const auto length = std::strlen(source); + std::memcpy(destination, source, length + 1); + return destination + length; +} + char* APS5_VABI strtok_r_nid_postfix(char* text, const char* delimiters, char** state) { if (text == nullptr) text = *state; if (text == nullptr) return nullptr; @@ -73,4 +96,60 @@ char* APS5_VABI strtok_nid_postfix(char* text, const char* delimiters) { return strtok_r_nid_postfix(text, delimiters, &state); } + +int APS5_VABI __inet_aton_nid_postfix(const char* text, void* address) { + std::uint8_t parts[3]; + std::size_t count = 0; + std::uint64_t value = 0; + bool digit = false; + char c = *text; + for (;;) { + if (c < '0' || c > '9') return 0; + value = 0; + unsigned base = 10; + digit = false; + if (c == '0') { + c = *++text; + if (c == 'x' || c == 'X') { + base = 16; + c = *++text; + } else { + base = 8; + digit = true; + } + } + for (;; c = *++text, digit = true) { + const char lower = static_cast(c | 0x20); + if (c >= '0' && c <= '9') { + if (base == 8 && c >= '8') return 0; + value = value * base + static_cast(c - '0'); + } else if (base == 16 && lower >= 'a' && lower <= 'f') { + value = (value << 4) | static_cast(lower - 'a' + 10); + } else { + break; + } + } + if (c != '.') break; + if (count == 3 || value > 0xff) return 0; + parts[count++] = static_cast(value); + c = *++text; + } + if (c != '\0' && c != ' ' && (c < '\t' || c > '\r')) return 0; + if (!digit) return 0; + constexpr std::uint64_t limits[] = {UINT64_MAX, 0xffffff, 0xffff, 0xff}; + if (value > limits[count]) return 0; + for (std::size_t index = 0; index < count; ++index) value |= std::uint64_t{parts[index]} << (24 - 8 * index); + if (address) { + const std::uint8_t bytes[4] = {static_cast(value >> 24), static_cast(value >> 16), + static_cast(value >> 8), static_cast(value)}; + std::memcpy(address, bytes, sizeof(bytes)); + } + return 1; +} + +std::uint32_t APS5_VABI __inet_addr_nid_postfix(const char* text) { + std::uint32_t address; + return __inet_aton_nid_postfix(text, &address) ? address : 0xffffffff; +} + } diff --git a/core/libs/prx/libc/src/Strings.cpp b/core/libs/prx/libc/src/Strings.cpp index 533611a17..f60b7ea24 100644 --- a/core/libs/prx/libc/src/Strings.cpp +++ b/core/libs/prx/libc/src/Strings.cpp @@ -119,19 +119,32 @@ size_t APS5_VABI strlcpy_nid_postfix(char* dest, const char* src, size_t size) { return srcLen; } +bool StopAtBinaryPrefix_nid_no_patch(const char* str, char** endptr, int base) { + if (base != 0 && base != 2) return false; + while (*str == ' ' || (*str >= '\t' && *str <= '\r')) ++str; + if (*str == '+' || *str == '-') ++str; + if (str[0] != '0' || (str[1] != 'b' && str[1] != 'B') || (str[2] != '0' && str[2] != '1')) return false; + if (endptr) *endptr = const_cast(str + 1); + return true; +} + std::int64_t APS5_VABI strtol_nid_postfix(const char* str, char** endptr, int base) { + if (StopAtBinaryPrefix_nid_no_patch(str, endptr, base)) return 0; return std::strtoll(str, endptr, base); } std::uint64_t APS5_VABI strtoul_nid_postfix(const char* str, char** endptr, int base) { + if (StopAtBinaryPrefix_nid_no_patch(str, endptr, base)) return 0; return std::strtoull(str, endptr, base); } long long APS5_VABI strtoll_nid_postfix(const char* str, char** endptr, int base) { + if (StopAtBinaryPrefix_nid_no_patch(str, endptr, base)) return 0; return std::strtoll(str, endptr, base); } unsigned long long APS5_VABI strtoull_nid_postfix(const char* str, char** endptr, int base) { + if (StopAtBinaryPrefix_nid_no_patch(str, endptr, base)) return 0; return std::strtoull(str, endptr, base); } @@ -316,6 +329,7 @@ char* APS5_VABI strnstr_nid_postfix(const char* haystack, const char* needle, si } unsigned long long APS5_VABI _Stoull_nid_postfix(const char* str, char** endptr, int base) { + if (StopAtBinaryPrefix_nid_no_patch(str, endptr, base)) return 0; return std::strtoull(str, endptr, base); } @@ -409,19 +423,19 @@ long double APS5_VABI wcstold_nid_postfix(const char16_t* str, char16_t** endptr } long long APS5_VABI wcstol_nid_postfix(const char16_t* str, char16_t** endptr, int base) { - return ParseAsciiPrefix(str, endptr, [base](const char* text, char** end) { return std::strtoll(text, end, base); }); + return ParseAsciiPrefix(str, endptr, [base](const char* text, char** end) { return StopAtBinaryPrefix_nid_no_patch(text, end, base) ? 0LL : std::strtoll(text, end, base); }); } long long APS5_VABI wcstoll_nid_postfix(const char16_t* str, char16_t** endptr, int base) { - return ParseAsciiPrefix(str, endptr, [base](const char* text, char** end) { return std::strtoll(text, end, base); }); + return ParseAsciiPrefix(str, endptr, [base](const char* text, char** end) { return StopAtBinaryPrefix_nid_no_patch(text, end, base) ? 0LL : std::strtoll(text, end, base); }); } unsigned long long APS5_VABI wcstoul_nid_postfix(const char16_t* str, char16_t** endptr, int base) { - return ParseAsciiPrefix(str, endptr, [base](const char* text, char** end) { return std::strtoull(text, end, base); }); + return ParseAsciiPrefix(str, endptr, [base](const char* text, char** end) { return StopAtBinaryPrefix_nid_no_patch(text, end, base) ? 0ULL : std::strtoull(text, end, base); }); } unsigned long long APS5_VABI wcstoull_nid_postfix(const char16_t* str, char16_t** endptr, int base) { - return ParseAsciiPrefix(str, endptr, [base](const char* text, char** end) { return std::strtoull(text, end, base); }); + return ParseAsciiPrefix(str, endptr, [base](const char* text, char** end) { return StopAtBinaryPrefix_nid_no_patch(text, end, base) ? 0ULL : std::strtoull(text, end, base); }); } int APS5_VABI wcscoll_nid_postfix(const char16_t* first, const char16_t* second) { @@ -438,8 +452,4 @@ size_t APS5_VABI strxfrm_nid_postfix(char* destination, const char* source, size return std::strxfrm(destination, source, count); } -size_t APS5_VABI wcsrtombs_nid_postfix(char* destination, const wchar_t** source, size_t count, mbstate_t* state) { - return std::wcsrtombs(destination, source, count, state); -} - } diff --git a/core/libs/prx/libc/src/Syslog.cpp b/core/libs/prx/libc/src/Syslog.cpp new file mode 100644 index 000000000..c7cc00b3e --- /dev/null +++ b/core/libs/prx/libc/src/Syslog.cpp @@ -0,0 +1,106 @@ +#include "prx/libc/include/general/VabiMacros.hpp" +#include "SceTypes.hpp" +#include +#include +#include +#include +#include +#ifdef _WIN32 +#include "prx/libc/include/WindowsFormatting.hpp" +#endif + +extern "C" { +int* APS5_VABI __error_nid_postfix(); +int APS5_VABI strerror_r_nid_postfix(int error, char* buffer, std::size_t length); +} + +namespace { +constexpr int priorityMask = 0x07; +constexpr int facilityMask = 0x3f8; +constexpr int userFacility = 1 << 3; +constexpr int internalLogPriority = 3 | 0x02 | 0x20 | 0x01; + +std::mutex& OutputMutex() { + static std::mutex mutex; + return mutex; +} + +void Emit(int priority, const std::string& message) { + std::lock_guard lock(OutputMutex()); + std::fprintf(stderr, "[syslog:%d] %s", priority, message.c_str()); + if (message.empty() || message.back() != '\n') std::fputc('\n', stderr); + std::fflush(stderr); +} + +std::string ExpandErrorText(const char* format, int error) { + std::string expanded; + for (const char* cursor = format; *cursor; ++cursor) { + if (cursor[0] == '%' && cursor[1] == 'm') { + char text[128]; + strerror_r_nid_postfix(error, text, sizeof(text)); + for (const char* character = text; *character; ++character) { + expanded += *character; + if (*character == '%') expanded += '%'; + } + ++cursor; + } else if (cursor[0] == '%' && cursor[1] == '%') { + expanded += "%%"; + ++cursor; + } else { + expanded += *cursor; + } + } + return expanded; +} + +std::string Format(const char* format, VaList* args) { +#ifdef _WIN32 + std::string buffer; + LibcDetail::FormatWindows(nullptr, 0, format, args, &buffer); + return buffer; +#else + std::va_list copy; + va_copy(copy, *reinterpret_cast(args)); + const int size = std::vsnprintf(nullptr, 0, format, copy); + va_end(copy); + if (size <= 0) return {}; + std::string buffer(static_cast(size) + 1, '\0'); + va_copy(copy, *reinterpret_cast(args)); + std::vsnprintf(buffer.data(), buffer.size(), format, copy); + va_end(copy); + buffer.resize(static_cast(size)); + return buffer; +#endif +} +} + +extern "C" { +void APS5_VABI vsyslog_nid_postfix(int priority, const char* format, VaList* args) { + const int saved = *__error_nid_postfix(); + if (priority & ~(priorityMask | facilityMask)) { + char complaint[64]; + std::snprintf(complaint, sizeof(complaint), "syslog: unknown facility/priority: %x", priority); + Emit(internalLogPriority, complaint); + priority &= priorityMask | facilityMask; + } + if ((priority & facilityMask) == 0) priority |= userFacility; + Emit(priority, format ? Format(ExpandErrorText(format, saved).c_str(), args) : std::string()); + *__error_nid_postfix() = saved; +} + +void APS5_VABI syslog_nid_postfix(int priority, const char* format, ...) { +#ifdef _WIN32 + __builtin_sysv_va_list args; + __builtin_sysv_va_start(args, format); +#else + std::va_list args; + va_start(args, format); +#endif + vsyslog_nid_postfix(priority, format, reinterpret_cast(args)); +#ifdef _WIN32 + __builtin_sysv_va_end(args); +#else + va_end(args); +#endif +} +} diff --git a/core/libs/prx/libc/src/Time.cpp b/core/libs/prx/libc/src/Time.cpp index 4710acdd0..f46add6e9 100644 --- a/core/libs/prx/libc/src/Time.cpp +++ b/core/libs/prx/libc/src/Time.cpp @@ -2,17 +2,105 @@ #include #include #include +#include #ifdef _WIN32 #include #endif #include "prx/libc/include/General.hpp" +#include "prx/libc/include/GuestTime.hpp" + +namespace { + +constexpr std::int64_t SECONDS_PER_DAY = 86400; +constexpr std::int64_t DAYS_FROM_MARCH_TO_YEAR_END = 306; + +void copyCalendarFields(const std::tm& host, GuestTm& guest) { + guest.tm_sec = host.tm_sec; + guest.tm_min = host.tm_min; + guest.tm_hour = host.tm_hour; + guest.tm_mday = host.tm_mday; + guest.tm_mon = host.tm_mon; + guest.tm_year = host.tm_year; + guest.tm_wday = host.tm_wday; + guest.tm_yday = host.tm_yday; + guest.tm_isdst = host.tm_isdst; +} + +std::tm toHostTm(const GuestTm& guest) { + std::tm host{}; + host.tm_sec = guest.tm_sec; + host.tm_min = guest.tm_min; + host.tm_hour = guest.tm_hour; + host.tm_mday = guest.tm_mday; + host.tm_mon = guest.tm_mon; + host.tm_year = guest.tm_year; + host.tm_wday = guest.tm_wday; + host.tm_yday = guest.tm_yday; + host.tm_isdst = guest.tm_isdst; +#ifndef _WIN32 + host.tm_gmtoff = static_cast(guest.tm_gmtoff); + host.tm_zone = guest.tm_zone; +#endif + return host; +} + +bool isLeapYear(std::int64_t year) { + return year % 4 == 0 && (year % 100 != 0 || year % 400 == 0); +} + +bool utcCalendarFields(std::int64_t time, GuestTm& result) { + std::int64_t days = time / SECONDS_PER_DAY; + std::int64_t secondOfDay = time % SECONDS_PER_DAY; + if (secondOfDay < 0) { + secondOfDay += SECONDS_PER_DAY; + --days; + } + const std::int64_t shifted = days + 719468; + const std::int64_t era = (shifted >= 0 ? shifted : shifted - 146096) / 146097; + const std::int64_t dayOfEra = shifted - era * 146097; + const std::int64_t yearOfEra = (dayOfEra - dayOfEra / 1460 + dayOfEra / 36524 - dayOfEra / 146096) / 365; + const std::int64_t dayOfMarchYear = dayOfEra - (365 * yearOfEra + yearOfEra / 4 - yearOfEra / 100); + const std::int64_t marchMonth = (5 * dayOfMarchYear + 2) / 153; + const std::int64_t month = marchMonth < 10 ? marchMonth + 3 : marchMonth - 9; + const std::int64_t year = yearOfEra + era * 400 + (month <= 2 ? 1 : 0); + const std::int64_t tmYear = year - 1900; + if (tmYear < std::numeric_limits::min() || tmYear > std::numeric_limits::max()) return false; + result.tm_sec = static_cast(secondOfDay % 60); + result.tm_min = static_cast(secondOfDay / 60 % 60); + result.tm_hour = static_cast(secondOfDay / 3600); + result.tm_mday = static_cast(dayOfMarchYear - (153 * marchMonth + 2) / 5 + 1); + result.tm_mon = static_cast(month - 1); + result.tm_year = static_cast(tmYear); + result.tm_wday = static_cast((days % 7 + 11) % 7); + result.tm_yday = static_cast(month <= 2 + ? dayOfMarchYear - DAYS_FROM_MARCH_TO_YEAR_END + : dayOfMarchYear + 59 + (isLeapYear(year) ? 1 : 0)); + result.tm_isdst = 0; + return true; +} + +} + +#ifdef __linux__ +#include + +namespace { + +const bool timeZoneFixed = [] { + if (std::getenv("TZ") == nullptr) ::setenv("TZ", ":/etc/localtime", 0); + ::tzset(); + return true; +}(); + +} // namespace +#endif extern "C" { -std::tm* APS5_VABI localtime_s_nid_postfix(const int64_t* timer, std::tm* result); -std::tm* APS5_VABI gmtime_s_nid_postfix(const int64_t* timer, std::tm* result); +GuestTm* APS5_VABI localtime_s_nid_postfix(const int64_t* timer, GuestTm* result); +GuestTm* APS5_VABI gmtime_s_nid_postfix(const int64_t* timer, GuestTm* result); int64_t APS5_VABI libc_time_nid_postfix(int64_t* timer) { std::time_t t = std::time(nullptr); @@ -45,64 +133,80 @@ double APS5_VABI difftime_nid_postfix(int64_t time1, int64_t time0) { return std::difftime(static_cast(time1), static_cast(time0)); } -std::tm* APS5_VABI libc_gmtime_nid_postfix(const int64_t* timer) { - static thread_local std::tm result; +GuestTm* APS5_VABI libc_gmtime_nid_postfix(const int64_t* timer) { + static thread_local GuestTm result; return gmtime_s_nid_postfix(timer, &result); } -std::tm* APS5_VABI libc_localtime_nid_postfix(const int64_t* timer) { - static thread_local std::tm result; +GuestTm* APS5_VABI libc_localtime_nid_postfix(const int64_t* timer) { + static thread_local GuestTm result; return localtime_s_nid_postfix(timer, &result); } -std::tm* APS5_VABI localtime_nid_postfix(const int64_t* timer) { +GuestTm* APS5_VABI localtime_nid_postfix(const int64_t* timer) { return libc_localtime_nid_postfix(timer); } -std::tm* APS5_VABI localtime_s_nid_postfix(const int64_t* timer, std::tm* result) { +GuestTm* APS5_VABI localtime_s_nid_postfix(const int64_t* timer, GuestTm* result) { if (timer == nullptr || result == nullptr) return nullptr; const std::time_t t = static_cast(*timer); + std::tm host{}; #ifdef _WIN32 - if (localtime_s(result, &t) != 0) return nullptr; + if (localtime_s(&host, &t) != 0) return nullptr; + copyCalendarFields(host, *result); + long standardBias = 0; + long daylightBias = 0; + _get_timezone(&standardBias); + _get_dstbias(&daylightBias); + const bool daylight = host.tm_isdst > 0; + result->tm_gmtoff = -static_cast(standardBias + (daylight ? daylightBias : 0)); + result->tm_zone = _tzname[daylight ? 1 : 0]; #else - if (localtime_r(&t, result) == nullptr) return nullptr; + if (localtime_r(&t, &host) == nullptr) return nullptr; + copyCalendarFields(host, *result); + result->tm_gmtoff = host.tm_gmtoff; + result->tm_zone = host.tm_zone; #endif return result; } -std::tm* APS5_VABI gmtime_s_nid_postfix(const int64_t* timer, std::tm* result) { +GuestTm* APS5_VABI gmtime_s_nid_postfix(const int64_t* timer, GuestTm* result) { if (timer == nullptr || result == nullptr) return nullptr; - const std::time_t t = static_cast(*timer); -#ifdef _WIN32 - if (gmtime_s(result, &t) != 0) return nullptr; -#else - if (gmtime_r(&t, result) == nullptr) return nullptr; -#endif + if (!utcCalendarFields(*timer, *result)) return nullptr; + result->tm_gmtoff = 0; + result->tm_zone = "UTC"; return result; } -std::tm* APS5_VABI gmtime_nid_postfix(const int64_t* timer) { +GuestTm* APS5_VABI gmtime_nid_postfix(const int64_t* timer) { return libc_gmtime_nid_postfix(timer); } -int64_t APS5_VABI libc_mktime_nid_postfix(std::tm* timeptr) { - return static_cast(std::mktime(timeptr)); +int64_t APS5_VABI libc_mktime_nid_postfix(GuestTm* timeptr) { + std::tm host = toHostTm(*timeptr); + const std::time_t t = std::mktime(&host); + if (t == static_cast(-1)) return -1; + const int64_t time = static_cast(t); + localtime_s_nid_postfix(&time, timeptr); + return time; } -int64_t APS5_VABI mktime_nid_postfix(std::tm* timeptr) { - return static_cast(std::mktime(timeptr)); +int64_t APS5_VABI mktime_nid_postfix(GuestTm* timeptr) { + return libc_mktime_nid_postfix(timeptr); } -size_t APS5_VABI libc_strftime_nid_postfix(char* str, size_t count, const char* format, const std::tm* timeptr) { - return std::strftime(str, count, format, timeptr); +size_t APS5_VABI libc_strftime_nid_postfix(char* str, size_t count, const char* format, const GuestTm* timeptr) { + const std::tm host = toHostTm(*timeptr); + return std::strftime(str, count, format, &host); } -char* APS5_VABI asctime_nid_postfix(const std::tm* timeptr) { - return std::asctime(timeptr); +char* APS5_VABI asctime_nid_postfix(const GuestTm* timeptr) { + const std::tm host = toHostTm(*timeptr); + return std::asctime(&host); } -size_t APS5_VABI strftime_nid_postfix(char* str, size_t count, const char* format, const std::tm* timeptr) { - return std::strftime(str, count, format, timeptr); +size_t APS5_VABI strftime_nid_postfix(char* str, size_t count, const char* format, const GuestTm* timeptr) { + return libc_strftime_nid_postfix(str, count, format, timeptr); } // The guest's CLOCKS_PER_SEC is 1000000: clock() reports process CPU time in microseconds. diff --git a/core/libs/prx/libc/src/exception/ExceptionPointer.cpp b/core/libs/prx/libc/src/exception/ExceptionPointer.cpp index eba668749..447d85ac4 100644 --- a/core/libs/prx/libc/src/exception/ExceptionPointer.cpp +++ b/core/libs/prx/libc/src/exception/ExceptionPointer.cpp @@ -1,12 +1,23 @@ #include #ifndef _LIBCPP_VERSION +#include + extern "C" { void ExceptionPointerAddref(std::exception_ptr* self) noexcept asm("_ZNSt15__exception_ptr13exception_ptr9_M_addrefEv"); void ExceptionPointerRelease(std::exception_ptr* self) noexcept asm("_ZNSt15__exception_ptr13exception_ptr10_M_releaseEv"); void* ExceptionPointerGet(const std::exception_ptr* self) noexcept asm("_ZNKSt15__exception_ptr13exception_ptr6_M_getEv"); void ExceptionPointerConstruct(std::exception_ptr* self, void* exception) noexcept asm("_ZNSt15__exception_ptr13exception_ptrC1EPv"); const std::type_info* ExceptionPointerType(const std::exception_ptr* self) noexcept asm("_ZNKSt15__exception_ptr13exception_ptr20__cxa_exception_typeEv"); +void FutureResultDestroy(std::__future_base::_Result_base* self) noexcept asm("_ZNSt13__future_base12_Result_baseD2Ev"); + +void FutureResultDestroy(std::__future_base::_Result_base* self) noexcept { + self->_M_error.~exception_ptr(); +} + +void APS5_VABI _ZNSt13__future_base12_Result_baseD2Ev_nid_postfix(std::__future_base::_Result_base* self) noexcept { + FutureResultDestroy(self); +} void APS5_VABI _ZNSt15__exception_ptr13exception_ptr9_M_addrefEv_nid_postfix(std::exception_ptr* self) noexcept { ExceptionPointerAddref(self); diff --git a/core/libs/prx/libc/src/exception/Exports.cpp b/core/libs/prx/libc/src/exception/Exports.cpp index 462c2c479..2bf7ef211 100644 --- a/core/libs/prx/libc/src/exception/Exports.cpp +++ b/core/libs/prx/libc/src/exception/Exports.cpp @@ -76,5 +76,13 @@ unsigned NativeUncaughtExceptions() asm("__cxa_uncaught_exceptions"); unsigned NativeUncaughtExceptions() { return LibcException::globals.uncaught; } bool NativeUncaughtException() asm("__cxa_uncaught_exception"); bool NativeUncaughtException() { return LibcException::globals.uncaught != 0; } +bool NativeStdUncaughtException() asm("_ZSt18uncaught_exceptionv"); +bool NativeStdUncaughtException() { return LibcException::globals.uncaught != 0; } +int NativeStdUncaughtExceptions() asm("_ZSt19uncaught_exceptionsv"); +int NativeStdUncaughtExceptions() { return static_cast(LibcException::globals.uncaught); } +void* NativeInitPrimaryException(void* object, std::type_info* type, void (*destructor)(void*)) asm("__cxa_init_primary_exception"); +void* NativeInitPrimaryException(void* object, std::type_info* type, void (*destructor)(void*)) { + return __cxa_init_primary_exception_nid_postfix(object, type, destructor); +} } #endif diff --git a/core/libs/prx/libc/src/exception/Runtime.cpp b/core/libs/prx/libc/src/exception/Runtime.cpp index 244bc1f6f..b9e26d77f 100644 --- a/core/libs/prx/libc/src/exception/Runtime.cpp +++ b/core/libs/prx/libc/src/exception/Runtime.cpp @@ -248,6 +248,7 @@ void APS5_VABI __cxa_free_exception_nid_postfix(void* object) { LibcException::Header* __cxa_init_primary_exception_nid_postfix(void* object, std::type_info* type, void (*destructor)(void*)) { using namespace LibcException; auto* header = FromObject(object); + AllocationOf(header)->references.store(0, std::memory_order_relaxed); header->type = type; header->destructor = destructor; header->terminate = terminateHandler.load(std::memory_order_acquire); diff --git a/core/libs/prx/libc/src/exception/StandardExceptions.cpp b/core/libs/prx/libc/src/exception/StandardExceptions.cpp index b8bf43028..cef6c4e56 100644 --- a/core/libs/prx/libc/src/exception/StandardExceptions.cpp +++ b/core/libs/prx/libc/src/exception/StandardExceptions.cpp @@ -9,9 +9,9 @@ struct ExceptionObject { const void* vtable; const char* message; }; struct ExceptionVtable { std::ptrdiff_t offset; const TypeRecord* type; - void (*destroy)(ExceptionObject*); - void (*deleteObject)(ExceptionObject*); - const char* (*what)(const ExceptionObject*); + void (APS5_VABI *destroy)(ExceptionObject*); + void (APS5_VABI *deleteObject)(ExceptionObject*); + const char* (APS5_VABI *what)(const ExceptionObject*); }; struct Message { std::size_t length; @@ -71,6 +71,9 @@ void ReleaseMessage(const char* message) { void DestroyPlain(ExceptionObject*) {} void DeletePlain(ExceptionObject* object) { std::free(object); } void DestroyMessage(ExceptionObject* object) { ReleaseMessage(object->message); object->message = nullptr; } +static void APS5_VABI DestroyGuestMessage_nid_no_patch(void* object) { + DestroyMessage(static_cast(object)); +} void DeleteMessage(ExceptionObject* object) { DestroyMessage(object); std::free(object); } const char* PlainWhat(const ExceptionObject* object) { auto* table = static_cast(object->vtable); @@ -86,6 +89,12 @@ const char* PlainWhat(const ExceptionObject* object) { return name; } const char* MessageWhat(const ExceptionObject* object) { return object->message ? object->message : ""; } +static void APS5_VABI GuestDestroyPlain_nid_no_patch(ExceptionObject* object) { DestroyPlain(object); } +static void APS5_VABI GuestDeletePlain_nid_no_patch(ExceptionObject* object) { DeletePlain(object); } +static const char* APS5_VABI GuestPlainWhat_nid_no_patch(const ExceptionObject* object) { return PlainWhat(object); } +static void APS5_VABI GuestDestroyMessage_nid_no_patch(ExceptionObject* object) { DestroyMessage(object); } +static void APS5_VABI GuestDeleteMessage_nid_no_patch(ExceptionObject* object) { DeleteMessage(object); } +static const char* APS5_VABI GuestMessageWhat_nid_no_patch(const ExceptionObject* object) { return MessageWhat(object); } void Construct(ExceptionObject* object, const ExceptionVtable& table, const char* message) { object->vtable = &table.destroy; object->message = CopyMessage(message); @@ -102,7 +111,8 @@ ExceptionObject* Assign(ExceptionObject* object, const ExceptionObject* source) [[noreturn]] void ThrowMessage(const ExceptionVtable& table, const char* message) { auto* object = static_cast(__cxa_allocate_exception_nid_postfix(sizeof(ExceptionObject))); Construct(object, table, message); - __cxa_throw_nid_postfix(object, reinterpret_cast(const_cast(table.type)), [](void* p) { DestroyMessage(static_cast(p)); }); + __cxa_throw_nid_postfix(object, reinterpret_cast(const_cast(table.type)), + reinterpret_cast(DestroyGuestMessage_nid_no_patch)); } [[noreturn]] void ThrowPlain(const ExceptionVtable& table) { auto* object = static_cast(__cxa_allocate_exception_nid_postfix(sizeof(void*))); @@ -114,7 +124,7 @@ ExceptionObject* Assign(ExceptionObject* object, const ExceptionObject* source) extern "C" { LibcException::TypeRecord _ZTISt8ios_base_nid_postfix {LibcException::ClassTypeVtable + 2, "St8ios_base", nullptr}; LibcException::TypeRecord _ZTISt9exception_nid_postfix {LibcException::ClassTypeVtable + 2, "St9exception", nullptr}; -LibcException::ExceptionVtable _ZTVSt9exception_nid_postfix {0, &_ZTISt9exception_nid_postfix, LibcException::DestroyPlain, LibcException::DeletePlain, LibcException::PlainWhat}; +LibcException::ExceptionVtable _ZTVSt9exception_nid_postfix {0, &_ZTISt9exception_nid_postfix, LibcException::GuestDestroyPlain_nid_no_patch, LibcException::GuestDeletePlain_nid_no_patch, LibcException::GuestPlainWhat_nid_no_patch}; void APS5_VABI _ZNSt9exceptionD1Ev_nid_postfix(LibcException::ExceptionObject* self) { LibcException::DestroyPlain(self); } void APS5_VABI _ZNSt9exceptionD2Ev_nid_postfix(LibcException::ExceptionObject* self) { LibcException::DestroyPlain(self); } void APS5_VABI _ZNSt9exceptionD0Ev_nid_postfix(LibcException::ExceptionObject* self) { LibcException::DeletePlain(self); } @@ -132,7 +142,7 @@ void APS5_VABI _ZNSt9exceptionC1Ev_nid_postfix(LibcException::ExceptionObject* s void APS5_VABI _ZNSt9exceptionC2Ev_nid_postfix(LibcException::ExceptionObject* self) { self->vtable = &_ZTVSt9exception_nid_postfix.destroy; } LibcException::TypeRecord _ZTISt8bad_cast_nid_postfix {LibcException::SingleTypeVtable + 2, "St8bad_cast", &_ZTISt9exception_nid_postfix}; -LibcException::ExceptionVtable _ZTVSt8bad_cast_nid_postfix {0, &_ZTISt8bad_cast_nid_postfix, LibcException::DestroyPlain, LibcException::DeletePlain, LibcException::PlainWhat}; +LibcException::ExceptionVtable _ZTVSt8bad_cast_nid_postfix {0, &_ZTISt8bad_cast_nid_postfix, LibcException::GuestDestroyPlain_nid_no_patch, LibcException::GuestDeletePlain_nid_no_patch, LibcException::GuestPlainWhat_nid_no_patch}; void APS5_VABI _ZNSt8bad_castD1Ev_nid_postfix(LibcException::ExceptionObject* self) { LibcException::DestroyPlain(self); } void APS5_VABI _ZNSt8bad_castD2Ev_nid_postfix(LibcException::ExceptionObject* self) { LibcException::DestroyPlain(self); } void APS5_VABI _ZNSt8bad_castD0Ev_nid_postfix(LibcException::ExceptionObject* self) { LibcException::DeletePlain(self); } @@ -141,7 +151,7 @@ void APS5_VABI _ZNSt8bad_castC1Ev_nid_postfix(LibcException::ExceptionObject* se void APS5_VABI _ZNSt8bad_castC2Ev_nid_postfix(LibcException::ExceptionObject* self) { self->vtable = &_ZTVSt8bad_cast_nid_postfix.destroy; } LibcException::TypeRecord _ZTISt10bad_typeid_nid_postfix {LibcException::SingleTypeVtable + 2, "St10bad_typeid", &_ZTISt9exception_nid_postfix}; -LibcException::ExceptionVtable _ZTVSt10bad_typeid_nid_postfix {0, &_ZTISt10bad_typeid_nid_postfix, LibcException::DestroyPlain, LibcException::DeletePlain, LibcException::PlainWhat}; +LibcException::ExceptionVtable _ZTVSt10bad_typeid_nid_postfix {0, &_ZTISt10bad_typeid_nid_postfix, LibcException::GuestDestroyPlain_nid_no_patch, LibcException::GuestDeletePlain_nid_no_patch, LibcException::GuestPlainWhat_nid_no_patch}; void APS5_VABI _ZNSt10bad_typeidD1Ev_nid_postfix(LibcException::ExceptionObject* self) { LibcException::DestroyPlain(self); } void APS5_VABI _ZNSt10bad_typeidD2Ev_nid_postfix(LibcException::ExceptionObject* self) { LibcException::DestroyPlain(self); } void APS5_VABI _ZNSt10bad_typeidD0Ev_nid_postfix(LibcException::ExceptionObject* self) { LibcException::DeletePlain(self); } @@ -150,7 +160,7 @@ void APS5_VABI _ZNSt10bad_typeidC1Ev_nid_postfix(LibcException::ExceptionObject* void APS5_VABI _ZNSt10bad_typeidC2Ev_nid_postfix(LibcException::ExceptionObject* self) { self->vtable = &_ZTVSt10bad_typeid_nid_postfix.destroy; } LibcException::TypeRecord _ZTISt9bad_alloc_nid_postfix {LibcException::SingleTypeVtable + 2, "St9bad_alloc", &_ZTISt9exception_nid_postfix}; -LibcException::ExceptionVtable _ZTVSt9bad_alloc_nid_postfix {0, &_ZTISt9bad_alloc_nid_postfix, LibcException::DestroyPlain, LibcException::DeletePlain, LibcException::PlainWhat}; +LibcException::ExceptionVtable _ZTVSt9bad_alloc_nid_postfix {0, &_ZTISt9bad_alloc_nid_postfix, LibcException::GuestDestroyPlain_nid_no_patch, LibcException::GuestDeletePlain_nid_no_patch, LibcException::GuestPlainWhat_nid_no_patch}; void APS5_VABI _ZNSt9bad_allocD1Ev_nid_postfix(LibcException::ExceptionObject* self) { LibcException::DestroyPlain(self); } void APS5_VABI _ZNSt9bad_allocD2Ev_nid_postfix(LibcException::ExceptionObject* self) { LibcException::DestroyPlain(self); } void APS5_VABI _ZNSt9bad_allocD0Ev_nid_postfix(LibcException::ExceptionObject* self) { LibcException::DeletePlain(self); } @@ -159,7 +169,7 @@ void APS5_VABI _ZNSt9bad_allocC1Ev_nid_postfix(LibcException::ExceptionObject* s void APS5_VABI _ZNSt9bad_allocC2Ev_nid_postfix(LibcException::ExceptionObject* self) { self->vtable = &_ZTVSt9bad_alloc_nid_postfix.destroy; } LibcException::TypeRecord _ZTISt20bad_array_new_length_nid_postfix {LibcException::SingleTypeVtable + 2, "St20bad_array_new_length", &_ZTISt9bad_alloc_nid_postfix}; -LibcException::ExceptionVtable _ZTVSt20bad_array_new_length_nid_postfix {0, &_ZTISt20bad_array_new_length_nid_postfix, LibcException::DestroyPlain, LibcException::DeletePlain, LibcException::PlainWhat}; +LibcException::ExceptionVtable _ZTVSt20bad_array_new_length_nid_postfix {0, &_ZTISt20bad_array_new_length_nid_postfix, LibcException::GuestDestroyPlain_nid_no_patch, LibcException::GuestDeletePlain_nid_no_patch, LibcException::GuestPlainWhat_nid_no_patch}; void APS5_VABI _ZNSt20bad_array_new_lengthD1Ev_nid_postfix(LibcException::ExceptionObject* self) { LibcException::DestroyPlain(self); } void APS5_VABI _ZNSt20bad_array_new_lengthD2Ev_nid_postfix(LibcException::ExceptionObject* self) { LibcException::DestroyPlain(self); } void APS5_VABI _ZNSt20bad_array_new_lengthD0Ev_nid_postfix(LibcException::ExceptionObject* self) { LibcException::DeletePlain(self); } @@ -168,7 +178,7 @@ void APS5_VABI _ZNSt20bad_array_new_lengthC1Ev_nid_postfix(LibcException::Except void APS5_VABI _ZNSt20bad_array_new_lengthC2Ev_nid_postfix(LibcException::ExceptionObject* self) { self->vtable = &_ZTVSt20bad_array_new_length_nid_postfix.destroy; } LibcException::TypeRecord _ZTISt17bad_function_call_nid_postfix {LibcException::SingleTypeVtable + 2, "St17bad_function_call", &_ZTISt9exception_nid_postfix}; -LibcException::ExceptionVtable _ZTVSt17bad_function_call_nid_postfix {0, &_ZTISt17bad_function_call_nid_postfix, LibcException::DestroyPlain, LibcException::DeletePlain, LibcException::PlainWhat}; +LibcException::ExceptionVtable _ZTVSt17bad_function_call_nid_postfix {0, &_ZTISt17bad_function_call_nid_postfix, LibcException::GuestDestroyPlain_nid_no_patch, LibcException::GuestDeletePlain_nid_no_patch, LibcException::GuestPlainWhat_nid_no_patch}; void APS5_VABI _ZNSt17bad_function_callD1Ev_nid_postfix(LibcException::ExceptionObject* self) { LibcException::DestroyPlain(self); } void APS5_VABI _ZNSt17bad_function_callD2Ev_nid_postfix(LibcException::ExceptionObject* self) { LibcException::DestroyPlain(self); } void APS5_VABI _ZNSt17bad_function_callD0Ev_nid_postfix(LibcException::ExceptionObject* self) { LibcException::DeletePlain(self); } @@ -177,14 +187,14 @@ void APS5_VABI _ZNSt17bad_function_callC1Ev_nid_postfix(LibcException::Exception void APS5_VABI _ZNSt17bad_function_callC2Ev_nid_postfix(LibcException::ExceptionObject* self) { self->vtable = &_ZTVSt17bad_function_call_nid_postfix.destroy; } LibcException::TypeRecord _ZTISt12bad_weak_ptr_nid_postfix {LibcException::SingleTypeVtable + 2, "St12bad_weak_ptr", &_ZTISt9exception_nid_postfix}; -LibcException::ExceptionVtable _ZTVSt12bad_weak_ptr_nid_postfix {0, &_ZTISt12bad_weak_ptr_nid_postfix, LibcException::DestroyPlain, LibcException::DeletePlain, LibcException::PlainWhat}; +LibcException::ExceptionVtable _ZTVSt12bad_weak_ptr_nid_postfix {0, &_ZTISt12bad_weak_ptr_nid_postfix, LibcException::GuestDestroyPlain_nid_no_patch, LibcException::GuestDeletePlain_nid_no_patch, LibcException::GuestPlainWhat_nid_no_patch}; void APS5_VABI _ZNSt12bad_weak_ptrD1Ev_nid_postfix(LibcException::ExceptionObject* self) { LibcException::DestroyPlain(self); } void APS5_VABI _ZNSt12bad_weak_ptrD2Ev_nid_postfix(LibcException::ExceptionObject* self) { LibcException::DestroyPlain(self); } void APS5_VABI _ZNSt12bad_weak_ptrD0Ev_nid_postfix(LibcException::ExceptionObject* self) { LibcException::DeletePlain(self); } const char* APS5_VABI _ZNKSt12bad_weak_ptr4whatEv_nid_postfix(const LibcException::ExceptionObject* self) { return LibcException::PlainWhat(self); } LibcException::TypeRecord _ZTISt11logic_error_nid_postfix {LibcException::SingleTypeVtable + 2, "St11logic_error", &_ZTISt9exception_nid_postfix}; -LibcException::ExceptionVtable _ZTVSt11logic_error_nid_postfix {0, &_ZTISt11logic_error_nid_postfix, LibcException::DestroyMessage, LibcException::DeleteMessage, LibcException::MessageWhat}; +LibcException::ExceptionVtable _ZTVSt11logic_error_nid_postfix {0, &_ZTISt11logic_error_nid_postfix, LibcException::GuestDestroyMessage_nid_no_patch, LibcException::GuestDeleteMessage_nid_no_patch, LibcException::GuestMessageWhat_nid_no_patch}; void APS5_VABI _ZNSt11logic_errorD1Ev_nid_postfix(LibcException::ExceptionObject* self) { LibcException::DestroyMessage(self); } void APS5_VABI _ZNSt11logic_errorD2Ev_nid_postfix(LibcException::ExceptionObject* self) { LibcException::DestroyMessage(self); } void APS5_VABI _ZNSt11logic_errorD0Ev_nid_postfix(LibcException::ExceptionObject* self) { LibcException::DeleteMessage(self); } @@ -196,7 +206,7 @@ void APS5_VABI _ZNSt11logic_errorC2ERKS__nid_postfix(LibcException::ExceptionObj LibcException::ExceptionObject* APS5_VABI _ZNSt11logic_erroraSERKS__nid_postfix(LibcException::ExceptionObject* self, const LibcException::ExceptionObject* source) { return LibcException::Assign(self, source); } LibcException::TypeRecord _ZTISt13runtime_error_nid_postfix {LibcException::SingleTypeVtable + 2, "St13runtime_error", &_ZTISt9exception_nid_postfix}; -LibcException::ExceptionVtable _ZTVSt13runtime_error_nid_postfix {0, &_ZTISt13runtime_error_nid_postfix, LibcException::DestroyMessage, LibcException::DeleteMessage, LibcException::MessageWhat}; +LibcException::ExceptionVtable _ZTVSt13runtime_error_nid_postfix {0, &_ZTISt13runtime_error_nid_postfix, LibcException::GuestDestroyMessage_nid_no_patch, LibcException::GuestDeleteMessage_nid_no_patch, LibcException::GuestMessageWhat_nid_no_patch}; void APS5_VABI _ZNSt13runtime_errorD1Ev_nid_postfix(LibcException::ExceptionObject* self) { LibcException::DestroyMessage(self); } void APS5_VABI _ZNSt13runtime_errorD2Ev_nid_postfix(LibcException::ExceptionObject* self) { LibcException::DestroyMessage(self); } void APS5_VABI _ZNSt13runtime_errorD0Ev_nid_postfix(LibcException::ExceptionObject* self) { LibcException::DeleteMessage(self); } @@ -208,7 +218,7 @@ void APS5_VABI _ZNSt13runtime_errorC2ERKS__nid_postfix(LibcException::ExceptionO LibcException::ExceptionObject* APS5_VABI _ZNSt13runtime_erroraSERKS__nid_postfix(LibcException::ExceptionObject* self, const LibcException::ExceptionObject* source) { return LibcException::Assign(self, source); } LibcException::TypeRecord _ZTISt16invalid_argument_nid_postfix {LibcException::SingleTypeVtable + 2, "St16invalid_argument", &_ZTISt11logic_error_nid_postfix}; -LibcException::ExceptionVtable _ZTVSt16invalid_argument_nid_postfix {0, &_ZTISt16invalid_argument_nid_postfix, LibcException::DestroyMessage, LibcException::DeleteMessage, LibcException::MessageWhat}; +LibcException::ExceptionVtable _ZTVSt16invalid_argument_nid_postfix {0, &_ZTISt16invalid_argument_nid_postfix, LibcException::GuestDestroyMessage_nid_no_patch, LibcException::GuestDeleteMessage_nid_no_patch, LibcException::GuestMessageWhat_nid_no_patch}; void APS5_VABI _ZNSt16invalid_argumentD1Ev_nid_postfix(LibcException::ExceptionObject* self) { LibcException::DestroyMessage(self); } void APS5_VABI _ZNSt16invalid_argumentD2Ev_nid_postfix(LibcException::ExceptionObject* self) { LibcException::DestroyMessage(self); } void APS5_VABI _ZNSt16invalid_argumentD0Ev_nid_postfix(LibcException::ExceptionObject* self) { LibcException::DeleteMessage(self); } @@ -220,7 +230,7 @@ void APS5_VABI _ZNSt16invalid_argumentC2ERKS__nid_postfix(LibcException::Excepti LibcException::ExceptionObject* APS5_VABI _ZNSt16invalid_argumentaSERKS__nid_postfix(LibcException::ExceptionObject* self, const LibcException::ExceptionObject* source) { return LibcException::Assign(self, source); } LibcException::TypeRecord _ZTISt12out_of_range_nid_postfix {LibcException::SingleTypeVtable + 2, "St12out_of_range", &_ZTISt11logic_error_nid_postfix}; -LibcException::ExceptionVtable _ZTVSt12out_of_range_nid_postfix {0, &_ZTISt12out_of_range_nid_postfix, LibcException::DestroyMessage, LibcException::DeleteMessage, LibcException::MessageWhat}; +LibcException::ExceptionVtable _ZTVSt12out_of_range_nid_postfix {0, &_ZTISt12out_of_range_nid_postfix, LibcException::GuestDestroyMessage_nid_no_patch, LibcException::GuestDeleteMessage_nid_no_patch, LibcException::GuestMessageWhat_nid_no_patch}; void APS5_VABI _ZNSt12out_of_rangeD1Ev_nid_postfix(LibcException::ExceptionObject* self) { LibcException::DestroyMessage(self); } void APS5_VABI _ZNSt12out_of_rangeD2Ev_nid_postfix(LibcException::ExceptionObject* self) { LibcException::DestroyMessage(self); } void APS5_VABI _ZNSt12out_of_rangeD0Ev_nid_postfix(LibcException::ExceptionObject* self) { LibcException::DeleteMessage(self); } @@ -232,7 +242,7 @@ void APS5_VABI _ZNSt12out_of_rangeC2ERKS__nid_postfix(LibcException::ExceptionOb LibcException::ExceptionObject* APS5_VABI _ZNSt12out_of_rangeaSERKS__nid_postfix(LibcException::ExceptionObject* self, const LibcException::ExceptionObject* source) { return LibcException::Assign(self, source); } LibcException::TypeRecord _ZTISt12domain_error_nid_postfix {LibcException::SingleTypeVtable + 2, "St12domain_error", &_ZTISt11logic_error_nid_postfix}; -LibcException::ExceptionVtable _ZTVSt12domain_error_nid_postfix {0, &_ZTISt12domain_error_nid_postfix, LibcException::DestroyMessage, LibcException::DeleteMessage, LibcException::MessageWhat}; +LibcException::ExceptionVtable _ZTVSt12domain_error_nid_postfix {0, &_ZTISt12domain_error_nid_postfix, LibcException::GuestDestroyMessage_nid_no_patch, LibcException::GuestDeleteMessage_nid_no_patch, LibcException::GuestMessageWhat_nid_no_patch}; void APS5_VABI _ZNSt12domain_errorD1Ev_nid_postfix(LibcException::ExceptionObject* self) { LibcException::DestroyMessage(self); } void APS5_VABI _ZNSt12domain_errorD2Ev_nid_postfix(LibcException::ExceptionObject* self) { LibcException::DestroyMessage(self); } void APS5_VABI _ZNSt12domain_errorD0Ev_nid_postfix(LibcException::ExceptionObject* self) { LibcException::DeleteMessage(self); } @@ -244,7 +254,7 @@ void APS5_VABI _ZNSt12domain_errorC2ERKS__nid_postfix(LibcException::ExceptionOb LibcException::ExceptionObject* APS5_VABI _ZNSt12domain_erroraSERKS__nid_postfix(LibcException::ExceptionObject* self, const LibcException::ExceptionObject* source) { return LibcException::Assign(self, source); } LibcException::TypeRecord _ZTISt12length_error_nid_postfix {LibcException::SingleTypeVtable + 2, "St12length_error", &_ZTISt11logic_error_nid_postfix}; -LibcException::ExceptionVtable _ZTVSt12length_error_nid_postfix {0, &_ZTISt12length_error_nid_postfix, LibcException::DestroyMessage, LibcException::DeleteMessage, LibcException::MessageWhat}; +LibcException::ExceptionVtable _ZTVSt12length_error_nid_postfix {0, &_ZTISt12length_error_nid_postfix, LibcException::GuestDestroyMessage_nid_no_patch, LibcException::GuestDeleteMessage_nid_no_patch, LibcException::GuestMessageWhat_nid_no_patch}; void APS5_VABI _ZNSt12length_errorD1Ev_nid_postfix(LibcException::ExceptionObject* self) { LibcException::DestroyMessage(self); } void APS5_VABI _ZNSt12length_errorD2Ev_nid_postfix(LibcException::ExceptionObject* self) { LibcException::DestroyMessage(self); } void APS5_VABI _ZNSt12length_errorD0Ev_nid_postfix(LibcException::ExceptionObject* self) { LibcException::DeleteMessage(self); } @@ -256,7 +266,7 @@ void APS5_VABI _ZNSt12length_errorC2ERKS__nid_postfix(LibcException::ExceptionOb LibcException::ExceptionObject* APS5_VABI _ZNSt12length_erroraSERKS__nid_postfix(LibcException::ExceptionObject* self, const LibcException::ExceptionObject* source) { return LibcException::Assign(self, source); } LibcException::TypeRecord _ZTISt12system_error_nid_postfix {LibcException::SingleTypeVtable + 2, "St12system_error", &_ZTISt13runtime_error_nid_postfix}; -LibcException::ExceptionVtable _ZTVSt12system_error_nid_postfix {0, &_ZTISt12system_error_nid_postfix, LibcException::DestroyMessage, LibcException::DeleteMessage, LibcException::MessageWhat}; +LibcException::ExceptionVtable _ZTVSt12system_error_nid_postfix {0, &_ZTISt12system_error_nid_postfix, LibcException::GuestDestroyMessage_nid_no_patch, LibcException::GuestDeleteMessage_nid_no_patch, LibcException::GuestMessageWhat_nid_no_patch}; void APS5_VABI _ZNSt12system_errorD1Ev_nid_postfix(LibcException::ExceptionObject* self) { LibcException::DestroyMessage(self); } void APS5_VABI _ZNSt12system_errorD2Ev_nid_postfix(LibcException::ExceptionObject* self) { LibcException::DestroyMessage(self); } void APS5_VABI _ZNSt12system_errorD0Ev_nid_postfix(LibcException::ExceptionObject* self) { LibcException::DeleteMessage(self); } @@ -266,7 +276,7 @@ void APS5_VABI _ZNSt12system_errorC2ERKS__nid_postfix(LibcException::ExceptionOb LibcException::ExceptionObject* APS5_VABI _ZNSt12system_erroraSERKS__nid_postfix(LibcException::ExceptionObject* self, const LibcException::ExceptionObject* source) { return LibcException::Assign(self, source); } LibcException::TypeRecord _ZTISt11regex_error_nid_postfix {LibcException::SingleTypeVtable + 2, "St11regex_error", &_ZTISt13runtime_error_nid_postfix}; -LibcException::ExceptionVtable _ZTVSt11regex_error_nid_postfix {0, &_ZTISt11regex_error_nid_postfix, LibcException::DestroyMessage, LibcException::DeleteMessage, LibcException::MessageWhat}; +LibcException::ExceptionVtable _ZTVSt11regex_error_nid_postfix {0, &_ZTISt11regex_error_nid_postfix, LibcException::GuestDestroyMessage_nid_no_patch, LibcException::GuestDeleteMessage_nid_no_patch, LibcException::GuestMessageWhat_nid_no_patch}; void APS5_VABI _ZNSt11regex_errorD1Ev_nid_postfix(LibcException::ExceptionObject* self) { LibcException::DestroyMessage(self); } void APS5_VABI _ZNSt11regex_errorD2Ev_nid_postfix(LibcException::ExceptionObject* self) { LibcException::DestroyMessage(self); } void APS5_VABI _ZNSt11regex_errorD0Ev_nid_postfix(LibcException::ExceptionObject* self) { LibcException::DeleteMessage(self); } @@ -276,13 +286,13 @@ void APS5_VABI _ZNSt11regex_errorC2ERKS__nid_postfix(LibcException::ExceptionObj LibcException::ExceptionObject* APS5_VABI _ZNSt11regex_erroraSERKS__nid_postfix(LibcException::ExceptionObject* self, const LibcException::ExceptionObject* source) { return LibcException::Assign(self, source); } LibcException::TypeRecord _ZTISt11range_error_nid_postfix {LibcException::SingleTypeVtable + 2, "St11range_error", &_ZTISt13runtime_error_nid_postfix}; -LibcException::ExceptionVtable _ZTVSt11range_error_nid_postfix {0, &_ZTISt11range_error_nid_postfix, LibcException::DestroyMessage, LibcException::DeleteMessage, LibcException::MessageWhat}; +LibcException::ExceptionVtable _ZTVSt11range_error_nid_postfix {0, &_ZTISt11range_error_nid_postfix, LibcException::GuestDestroyMessage_nid_no_patch, LibcException::GuestDeleteMessage_nid_no_patch, LibcException::GuestMessageWhat_nid_no_patch}; void APS5_VABI _ZNSt11range_errorD1Ev_nid_postfix(LibcException::ExceptionObject* self) { LibcException::DestroyMessage(self); } void APS5_VABI _ZNSt11range_errorD2Ev_nid_postfix(LibcException::ExceptionObject* self) { LibcException::DestroyMessage(self); } void APS5_VABI _ZNSt11range_errorD0Ev_nid_postfix(LibcException::ExceptionObject* self) { LibcException::DeleteMessage(self); } LibcException::TypeRecord _ZTINSt8ios_base7failureE_nid_postfix {LibcException::SingleTypeVtable + 2, "NSt8ios_base7failureE", &_ZTISt12system_error_nid_postfix}; -LibcException::ExceptionVtable _ZTVNSt8ios_base7failureE_nid_postfix {0, &_ZTINSt8ios_base7failureE_nid_postfix, LibcException::DestroyMessage, LibcException::DeleteMessage, LibcException::MessageWhat}; +LibcException::ExceptionVtable _ZTVNSt8ios_base7failureE_nid_postfix {0, &_ZTINSt8ios_base7failureE_nid_postfix, LibcException::GuestDestroyMessage_nid_no_patch, LibcException::GuestDeleteMessage_nid_no_patch, LibcException::GuestMessageWhat_nid_no_patch}; void APS5_VABI _ZNSt8ios_base7failureD1Ev_nid_postfix(LibcException::ExceptionObject* self) { LibcException::DestroyMessage(self); } void APS5_VABI _ZNSt8ios_base7failureD2Ev_nid_postfix(LibcException::ExceptionObject* self) { LibcException::DestroyMessage(self); } void APS5_VABI _ZNSt8ios_base7failureD0Ev_nid_postfix(LibcException::ExceptionObject* self) { LibcException::DeleteMessage(self); } @@ -304,6 +314,7 @@ const char* APS5_VABI _ZNKSt8ios_base7failure4whatEv_nid_postfix(const LibcExcep auto* object = static_cast(__cxa_allocate_exception_nid_postfix(sizeof(RegexObject))); LibcException::Construct(&object->base, _ZTVSt11regex_error_nid_postfix, "regular expression error"); object->code = code; - __cxa_throw_nid_postfix(object, reinterpret_cast(&_ZTISt11regex_error_nid_postfix), [](void* p) { LibcException::DestroyMessage(static_cast(p)); }); + __cxa_throw_nid_postfix(object, reinterpret_cast(&_ZTISt11regex_error_nid_postfix), + reinterpret_cast(LibcException::DestroyGuestMessage_nid_no_patch)); } } diff --git a/core/libs/prx/libc/src/specifics/x86_64/RegisterContext.cpp b/core/libs/prx/libc/src/specifics/x86_64/RegisterContext.cpp index aece550d8..b4538a0cf 100644 --- a/core/libs/prx/libc/src/specifics/x86_64/RegisterContext.cpp +++ b/core/libs/prx/libc/src/specifics/x86_64/RegisterContext.cpp @@ -4,27 +4,39 @@ extern "C" void LibcCaptureRegisters(std::uintptr_t*); extern "C" [[noreturn]] void LibcRestoreRegisters(const std::uintptr_t*); -#if defined(__linux__) && defined(__x86_64__) +#if (defined(__linux__) || defined(__APPLE__)) && defined(__x86_64__) asm( ".text\n" +#ifdef __APPLE__ +".private_extern _LibcCaptureRegisters\n" +"_LibcCaptureRegisters:\n" +#else ".hidden LibcCaptureRegisters\n" ".type LibcCaptureRegisters,@function\n" "LibcCaptureRegisters:\n" +#endif "movq %rax,0(%rdi)\nmovq %rdx,8(%rdi)\nmovq %rcx,16(%rdi)\nmovq %rbx,24(%rdi)\n" "movq %rsi,32(%rdi)\nmovq %rdi,40(%rdi)\nmovq %rbp,48(%rdi)\n" "leaq 8(%rsp),%rax\nmovq %rax,56(%rdi)\n" "movq %r8,64(%rdi)\nmovq %r9,72(%rdi)\nmovq %r10,80(%rdi)\nmovq %r11,88(%rdi)\n" "movq %r12,96(%rdi)\nmovq %r13,104(%rdi)\nmovq %r14,112(%rdi)\nmovq %r15,120(%rdi)\n" "movq (%rsp),%rax\nmovq %rax,128(%rdi)\nret\n" +#ifdef __APPLE__ +".private_extern _LibcRestoreRegisters\n" +"_LibcRestoreRegisters:\n" +#else ".size LibcCaptureRegisters,.-LibcCaptureRegisters\n" ".hidden LibcRestoreRegisters\n.type LibcRestoreRegisters,@function\n" "LibcRestoreRegisters:\n" +#endif "movq %rdi,%r10\nmovq 128(%r10),%r11\n" "movq 0(%r10),%rax\nmovq 8(%r10),%rdx\nmovq 16(%r10),%rcx\nmovq 24(%r10),%rbx\n" "movq 32(%r10),%rsi\nmovq 40(%r10),%rdi\nmovq 48(%r10),%rbp\nmovq 56(%r10),%rsp\n" "movq 64(%r10),%r8\nmovq 72(%r10),%r9\nmovq 96(%r10),%r12\n" "movq 104(%r10),%r13\nmovq 112(%r10),%r14\nmovq 120(%r10),%r15\njmp *%r11\n" +#ifndef __APPLE__ ".size LibcRestoreRegisters,.-LibcRestoreRegisters\n" +#endif ); #elif defined(_WIN32) && defined(__x86_64__) asm( diff --git a/core/libs/prx/libkernel/Aio/src/Aio.cpp b/core/libs/prx/libkernel/Aio/src/Aio.cpp index bee34799c..040773e22 100644 --- a/core/libs/prx/libkernel/Aio/src/Aio.cpp +++ b/core/libs/prx/libkernel/Aio/src/Aio.cpp @@ -10,13 +10,10 @@ #include "SceTypes.hpp" #include "prx/libc/include/General.hpp" #include "prx/libc/include/GuestArena.hpp" +#include "prx/libkernel/File/include/File.hpp" #include "prx/libkernel/KernelErrors.hpp" -#ifdef _WIN32 -#include -#include -#include -#else +#ifndef _WIN32 #include #include #endif @@ -54,35 +51,6 @@ void SetState(std::int32_t id, std::int32_t state) { g_states[id] = state; } -#ifdef _WIN32 -std::int64_t NativePositioned(std::int32_t fd, void* buf, std::size_t nbyte, std::int64_t offset, bool write, const char* name) { - if (nbyte > static_cast(std::numeric_limits::max())) { - throw std::runtime_error(std::string(name) + ": nbytes exceeds platform limit"); - } - if (offset < 0) { - errno = EINVAL; - return -1; - } - const auto handle = reinterpret_cast(::_get_osfhandle(fd)); - if (handle == INVALID_HANDLE_VALUE) { - errno = EBADF; - return -1; - } - OVERLAPPED overlapped{}; - overlapped.Offset = static_cast(offset); - overlapped.OffsetHigh = static_cast(static_cast(offset) >> 32u); - DWORD done = 0; - const BOOL ok = write ? ::WriteFile(handle, buf, static_cast(nbyte), &done, &overlapped) - : ::ReadFile(handle, buf, static_cast(nbyte), &done, &overlapped); - if (!ok) { - if (!write && ::GetLastError() == ERROR_HANDLE_EOF) return 0; - errno = EIO; - return -1; - } - return done; -} -#endif - std::int64_t NativePread(std::int32_t fd, void* buf, std::size_t nbyte, std::int64_t offset) { const GuestArena::HostWrite destination(buf, nbyte); if (!destination.Open()) { @@ -90,7 +58,11 @@ std::int64_t NativePread(std::int32_t fd, void* buf, std::size_t nbyte, std::int return -1; } #ifdef _WIN32 - return NativePositioned(fd, buf, nbyte, offset, false, "sceKernelAioSubmitReadCommands"); + if (offset < 0) { + errno = EINVAL; + return -1; + } + return NativePositioned_nid_no_patch(fd, buf, nbyte, offset, false); #else return static_cast(::pread(fd, buf, nbyte, static_cast(offset))); #endif @@ -98,7 +70,11 @@ std::int64_t NativePread(std::int32_t fd, void* buf, std::size_t nbyte, std::int std::int64_t NativePwrite(std::int32_t fd, const void* buf, std::size_t nbyte, std::int64_t offset) { #ifdef _WIN32 - return NativePositioned(fd, const_cast(buf), nbyte, offset, true, "sceKernelAioSubmitWriteCommands"); + if (offset < 0) { + errno = EINVAL; + return -1; + } + return NativePositioned_nid_no_patch(fd, const_cast(buf), nbyte, offset, true); #else return static_cast(::pwrite(fd, buf, nbyte, static_cast(offset))); #endif @@ -113,6 +89,7 @@ bool RunRequest(KernelAioRwRequest& req, bool write) { req.result->return_value = static_cast(SCE_KERNEL_ERROR_EIO); if (error == EBADF) req.result->return_value = static_cast(SCE_KERNEL_ERROR_EBADF); if (error == EFAULT) req.result->return_value = static_cast(SCE_KERNEL_ERROR_EFAULT); + if (error == EINVAL) req.result->return_value = static_cast(SCE_KERNEL_ERROR_EINVAL); req.result->state = AioAborted; return false; } @@ -305,8 +282,11 @@ int APS5_VABI sceKernelAioWaitRequest(int32_t id, int32_t* state, uint32_t* usec } } -int APS5_VABI sceKernelAioPollRequests() { - NotImplemented_nid_no_patch(__func__); +int APS5_VABI sceKernelAioPollRequests(int32_t* id, int32_t num, int32_t* state) { + const int error = ValidateIds(id, num, state, "sceKernelAioPollRequests"); + if (error != 0) return error; + std::lock_guard lock(g_mutex); + for (int32_t i = 0; i < num; ++i) state[i] = g_states[id[i]]; return 0; } diff --git a/core/libs/prx/libkernel/AppMetadata/src/AppMetadata.cpp b/core/libs/prx/libkernel/AppMetadata/src/AppMetadata.cpp index bc423779a..79df79827 100644 --- a/core/libs/prx/libkernel/AppMetadata/src/AppMetadata.cpp +++ b/core/libs/prx/libkernel/AppMetadata/src/AppMetadata.cpp @@ -42,6 +42,18 @@ void ensureTitleLoaded() { g_titleLoaded = true; } +const bool g_downloadDataMounted = [] { + try { + const auto run = std::filesystem::current_path(); + const auto paramJson = run / "app0" / "sce_sys" / "param.json"; + if (!std::filesystem::exists(paramJson) || parseParamJson(paramJson).downloadDataSizeMiB == 0) return false; + std::filesystem::create_directories(run / "download0"); + return true; + } catch (const std::exception&) { + return false; + } +}(); + void ensureIconLoaded() { if (g_iconAttempted) return; g_iconAttempted = true; diff --git a/core/libs/prx/libkernel/Apr/src/Apr.cpp b/core/libs/prx/libkernel/Apr/src/Apr.cpp index a363671b9..1417f6fd4 100644 --- a/core/libs/prx/libkernel/Apr/src/Apr.cpp +++ b/core/libs/prx/libkernel/Apr/src/Apr.cpp @@ -2,6 +2,7 @@ #include #include "SceTypes.hpp" #include "prx/libc/include/General.hpp" +#include "prx/libc/include/PreciseWait.hpp" #include "prx/libc/include/GuestArena.hpp" #include "prx/libkernel/Apr/include/AprCommandBuffer.hpp" #include "prx/libkernel/DirectMemory/DirectMemory.hpp" @@ -39,15 +40,15 @@ struct AprFile { std::uint64_t size; }; -std::mutex g_filesLock; -std::vector g_files; -std::unordered_map g_idsByPath; +std::mutex& g_filesLock = *new std::mutex(); +std::vector& g_files = *new std::vector(); +std::unordered_map& g_idsByPath = *new std::unordered_map(); // File sizes by directory, listed on first use. Titles resolve thousands of package paths at startup, // and one size query per path took over 20 s; a listing costs one directory read. Names are compared // case-insensitively, like the host file system. -std::mutex g_directoriesLock; -std::unordered_map> g_directories; +std::mutex& g_directoriesLock = *new std::mutex(); +std::unordered_map>& g_directories = *new std::unordered_map>(); std::string _foldCase(std::string text) { for (auto& character : text) { @@ -164,7 +165,7 @@ void _writeAddress(const Apr::WriteAddressCommand& command) { std::array, 256> g_counters{}; -std::mutex g_counterWrites; +std::mutex& g_counterWrites = *new std::mutex(); std::uint32_t _counter(std::uint32_t index) { return g_counters[index % g_counters.size()].load(std::memory_order_acquire); @@ -263,7 +264,7 @@ struct AmmState { }; AmmState& _amm() { - static AmmState state; + static auto& state = *new AmmState(); return state; } @@ -585,7 +586,7 @@ void _execute(const Apr::CommandBufferObject& buffer) { return std::atomic_ref(*reinterpret_cast(command.address)).load(std::memory_order_acquire); }; const std::uint64_t reference = (command.reference & command.mask) << unused; - while (!_waitSatisfied(command.compare, (current() & command.mask) << unused, reference)) std::this_thread::sleep_for(std::chrono::microseconds(50)); + while (!_waitSatisfied(command.compare, (current() & command.mask) << unused, reference)) PreciseSleepUs(50); break; } case Apr::Opcode::WriteKernelEventQueue: { @@ -704,7 +705,7 @@ int APS5_VABI sceKernelAprGetFileStat(uint32_t id, FileStat* stat) { int APS5_VABI sceKernelAprSubmitCommandBuffer(const Apr::CommandBufferObject* buffer, uint32_t priority) { (void)priority; - if (!buffer || buffer->type != Apr::BufferType::Apr) return _fail(GUEST_EINVAL); + if (!buffer) return _fail(GUEST_EINVAL); _execute(*buffer); return 0; } diff --git a/core/libs/prx/libkernel/CMakeLists.txt b/core/libs/prx/libkernel/CMakeLists.txt index 1fe4e1e79..6938ffc9a 100644 --- a/core/libs/prx/libkernel/CMakeLists.txt +++ b/core/libs/prx/libkernel/CMakeLists.txt @@ -4,9 +4,15 @@ add_library(libkernel SHARED EXCLUDE_FROM_ALL ${CMAKE_CURRENT_SOURCE_DIR}/Equeue/Equeue.cpp ${CMAKE_CURRENT_SOURCE_DIR}/Equeue/src/EventAccessors.cpp ${CMAKE_CURRENT_SOURCE_DIR}/File/src/DirectoryDescriptor.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/File/src/FileLock.cpp ${CMAKE_CURRENT_SOURCE_DIR}/File/src/NativeStat.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/File/src/Terminal.cpp ${CMAKE_CURRENT_SOURCE_DIR}/File/src/Open.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/File/src/OpenFileLimit.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/File/src/SharedMemory.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/File/src/RandomDevice.cpp ${CMAKE_CURRENT_SOURCE_DIR}/File/src/Stdio.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/../libc/src/Rename.cpp ${CMAKE_CURRENT_SOURCE_DIR}/Semaphore/src/Semaphore.cpp ${CMAKE_CURRENT_SOURCE_DIR}/SyncOnAddress/src/SyncOnAddress.cpp ${CMAKE_CURRENT_SOURCE_DIR}/DirectMemory/Export.cpp @@ -45,8 +51,10 @@ add_library(libkernel SHARED EXCLUDE_FROM_ALL ${CMAKE_CURRENT_SOURCE_DIR}/System/src/Sanitizer.cpp ${CMAKE_CURRENT_SOURCE_DIR}/System/src/Rtld.cpp ${CMAKE_CURRENT_SOURCE_DIR}/System/src/Process.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/System/src/Sandbox.cpp ${CMAKE_CURRENT_SOURCE_DIR}/System/src/SystemConfiguration.cpp ${CMAKE_CURRENT_SOURCE_DIR}/System/src/Signals.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/System/src/SignalSets.cpp ${CMAKE_CURRENT_SOURCE_DIR}/System/src/Notification.cpp ${CMAKE_CURRENT_SOURCE_DIR}/Time/Time.cpp ${CMAKE_CURRENT_SOURCE_DIR}/Time/TimedWait.cpp @@ -59,12 +67,35 @@ target_include_directories(libkernel PRIVATE ${LIBS_INCLUDE_DIR}/nid/include) target_link_libraries(libkernel PRIVATE ${CMAKE_DL_LIBS}) target_link_libraries(libkernel PRIVATE libc) if(WIN32) - target_link_libraries(libkernel PRIVATE ws2_32) + target_link_libraries(libkernel PRIVATE ws2_32 shell32) target_compile_definitions(libkernel PRIVATE _WIN32_WINNT=0x0602) endif() configure_windows_unwind(libkernel) if(BUILD_TESTING) + add_test_executable(download_data_mount_tests ${LIBS_INCLUDE_DIR}/tests/GuestDownloadDataMount.cpp) + target_include_directories(download_data_mount_tests PRIVATE ${LIBS_INCLUDE_DIR}) + target_link_libraries(download_data_mount_tests PRIVATE libkernel libc) + configure_windows_unwind(download_data_mount_tests) + foreach(downloadDataCase IN ITEMS declared:256 none:0 invalid:256) + string(REPLACE ":" ";" downloadDataCase ${downloadDataCase}) + list(GET downloadDataCase 0 downloadDataName) + list(GET downloadDataCase 1 downloadDataSize) + set(downloadDataRun ${CMAKE_CURRENT_BINARY_DIR}/download_data_mount_${downloadDataName}) + if(downloadDataName STREQUAL "invalid") + file(WRITE ${downloadDataRun}/app0/sce_sys/param.json "{\"downloadDataSize\":${downloadDataSize}}") + else() + file(WRITE ${downloadDataRun}/app0/sce_sys/param.json + "{\"titleId\":\"PPSA00000\",\"localizedParameters\":{\"en-US\":{\"titleName\":\"Example\"}},\"downloadDataSize\":${downloadDataSize}," + "\"userDefinedParam1\":0,\"userDefinedParam2\":0,\"userDefinedParam3\":0,\"userDefinedParam4\":0}") + endif() + add_test(NAME download_data_mount_${downloadDataName} COMMAND download_data_mount_tests ${downloadDataName} + WORKING_DIRECTORY ${downloadDataRun}) + if(WIN32) + set_tests_properties(download_data_mount_${downloadDataName} PROPERTIES ENVIRONMENT_MODIFICATION + "PATH=path_list_prepend:$") + endif() + endforeach() add_test_executable(param_json_parser_tests ${LIBS_INCLUDE_DIR}/tests/ParamJsonParser.cpp) target_include_directories(param_json_parser_tests PRIVATE ${LIBS_INCLUDE_DIR}) target_link_libraries(param_json_parser_tests PRIVATE libkernel libc) @@ -84,12 +115,32 @@ if(BUILD_TESTING) set_tests_properties(guest_posix_sem_init guest_posix_sem_post PROPERTIES ENVIRONMENT_MODIFICATION "PATH=path_list_prepend:$") endif() + add_test_executable(guest_process_time_tests ${LIBS_INCLUDE_DIR}/tests/GuestProcessTime.cpp) + target_include_directories(guest_process_time_tests PRIVATE ${LIBS_INCLUDE_DIR}) + target_link_libraries(guest_process_time_tests PRIVATE libkernel libc) + configure_windows_unwind(guest_process_time_tests) + add_test(NAME guest_process_time COMMAND guest_process_time_tests) + add_test(NAME guest_process_time_counter COMMAND guest_process_time_tests counter) + if(WIN32) + set_tests_properties(guest_process_time guest_process_time_counter PROPERTIES ENVIRONMENT_MODIFICATION + "PATH=path_list_prepend:$") + endif() + add_test_executable(guest_process_arguments_tests ${LIBS_INCLUDE_DIR}/tests/GuestProcessArguments.cpp) + target_include_directories(guest_process_arguments_tests PRIVATE ${LIBS_INCLUDE_DIR}) + target_link_libraries(guest_process_arguments_tests PRIVATE libkernel libc) + configure_windows_unwind(guest_process_arguments_tests) + add_test(NAME guest_process_arguments COMMAND guest_process_arguments_tests plain "two words" "ü日") + if(WIN32) + set_tests_properties(guest_process_arguments PROPERTIES ENVIRONMENT_MODIFICATION + "PATH=path_list_prepend:$") + endif() add_library(guest_module_fixture SHARED ${LIBS_INCLUDE_DIR}/tests/GuestModuleFixture.cpp) target_include_directories(guest_module_fixture PRIVATE ${LIBS_INCLUDE_DIR}) add_custom_command(TARGET guest_module_fixture POST_BUILD COMMAND $ guest_module_fixture $ VERBATIM) add_dependencies(guest_module_fixture nid_patcher) + configure_windows_unwind(guest_module_fixture) set_target_properties(guest_module_fixture PROPERTIES BUILD_WITH_INSTALL_RPATH ON INSTALL_RPATH "$ORIGIN") find_package(Python3 QUIET COMPONENTS Interpreter) if(Python3_Interpreter_FOUND AND NOT WIN32) @@ -114,14 +165,26 @@ if(BUILD_TESTING) set_tests_properties(guest_dynamic_loader PROPERTIES ENVIRONMENT_MODIFICATION "PATH=path_list_prepend:$") endif() - if(NOT WIN32) - add_test_executable(guest_module_info_tests ${LIBS_INCLUDE_DIR}/tests/GuestModuleInfo.cpp) - target_include_directories(guest_module_info_tests PRIVATE ${LIBS_INCLUDE_DIR}) - target_link_libraries(guest_module_info_tests PRIVATE libkernel libc) - add_dependencies(guest_module_info_tests guest_module_fixture) - add_test(NAME guest_module_info COMMAND guest_module_info_tests $) - set_tests_properties(guest_module_info PROPERTIES - WORKING_DIRECTORY "$") + add_test_executable(guest_module_info_tests ${LIBS_INCLUDE_DIR}/tests/GuestModuleInfo.cpp) + target_include_directories(guest_module_info_tests PRIVATE ${LIBS_INCLUDE_DIR}) + target_link_libraries(guest_module_info_tests PRIVATE libkernel libc) + configure_windows_unwind(guest_module_info_tests) + add_dependencies(guest_module_info_tests guest_module_fixture) + add_test(NAME guest_module_info COMMAND guest_module_info_tests $) + set_tests_properties(guest_module_info PROPERTIES + WORKING_DIRECTORY "$") + if(WIN32) + set_tests_properties(guest_module_info PROPERTIES ENVIRONMENT_MODIFICATION + "PATH=path_list_prepend:$") + endif() + add_test_executable(guest_file_timestamp_tests ${LIBS_INCLUDE_DIR}/tests/GuestFileTimestamps.cpp) + target_include_directories(guest_file_timestamp_tests PRIVATE ${LIBS_INCLUDE_DIR}) + target_link_libraries(guest_file_timestamp_tests PRIVATE libkernel libc) + configure_windows_unwind(guest_file_timestamp_tests) + add_test(NAME guest_file_timestamps COMMAND guest_file_timestamp_tests) + if(WIN32) + set_tests_properties(guest_file_timestamps PROPERTIES ENVIRONMENT_MODIFICATION + "PATH=path_list_prepend:$") endif() add_test_executable(guest_filesystem_tests ${LIBS_INCLUDE_DIR}/tests/GuestFilesystem.cpp) target_include_directories(guest_filesystem_tests PRIVATE ${LIBS_INCLUDE_DIR}) @@ -132,6 +195,16 @@ if(BUILD_TESTING) set_tests_properties(guest_filesystem PROPERTIES ENVIRONMENT_MODIFICATION "PATH=path_list_prepend:$") endif() + add_test_executable(guest_terminal_tests ${LIBS_INCLUDE_DIR}/tests/GuestTerminal.cpp) + target_include_directories(guest_terminal_tests PRIVATE ${LIBS_INCLUDE_DIR}) + target_link_libraries(guest_terminal_tests PRIVATE libkernel libc) + configure_windows_unwind(guest_terminal_tests) + add_test(NAME guest_terminal COMMAND guest_terminal_tests) + set_tests_properties(guest_terminal PROPERTIES TIMEOUT 20) + if(WIN32) + set_tests_properties(guest_terminal PROPERTIES ENVIRONMENT_MODIFICATION + "PATH=path_list_prepend:$") + endif() add_test_executable(guest_kernel_sync_tests ${LIBS_INCLUDE_DIR}/tests/GuestKernelSync.cpp) target_include_directories(guest_kernel_sync_tests PRIVATE ${LIBS_INCLUDE_DIR}) target_link_libraries(guest_kernel_sync_tests PRIVATE libkernel libc) @@ -141,6 +214,16 @@ if(BUILD_TESTING) set_tests_properties(guest_kernel_sync PROPERTIES ENVIRONMENT_MODIFICATION "PATH=path_list_prepend:$") endif() + add_test_executable(guest_kernel_rename_tests ${LIBS_INCLUDE_DIR}/tests/GuestKernelRename.cpp) + target_include_directories(guest_kernel_rename_tests PRIVATE ${LIBS_INCLUDE_DIR}) + target_link_libraries(guest_kernel_rename_tests PRIVATE libkernel libc) + target_compile_definitions(guest_kernel_rename_tests PRIVATE KERNEL_PRX_PATH="$") + configure_windows_unwind(guest_kernel_rename_tests) + add_test(NAME guest_kernel_rename COMMAND guest_kernel_rename_tests) + if(WIN32) + set_tests_properties(guest_kernel_rename PROPERTIES ENVIRONMENT_MODIFICATION + "PATH=path_list_prepend:$") + endif() add_test_executable(guest_aio_tests ${LIBS_INCLUDE_DIR}/tests/GuestAio.cpp) target_include_directories(guest_aio_tests PRIVATE ${LIBS_INCLUDE_DIR}) target_link_libraries(guest_aio_tests PRIVATE libkernel libc) @@ -190,6 +273,16 @@ if(BUILD_TESTING) set_tests_properties(guest_thread_self PROPERTIES ENVIRONMENT_MODIFICATION "PATH=path_list_prepend:$") endif() + add_test_executable(guest_pthread_getname_tests ${LIBS_INCLUDE_DIR}/tests/GuestPthreadGetname.cpp) + target_include_directories(guest_pthread_getname_tests PRIVATE ${LIBS_INCLUDE_DIR}) + target_link_libraries(guest_pthread_getname_tests PRIVATE libkernel libc) + configure_windows_unwind(guest_pthread_getname_tests) + add_test(NAME guest_pthread_getname COMMAND guest_pthread_getname_tests) + set_tests_properties(guest_pthread_getname PROPERTIES TIMEOUT 20) + if(WIN32) + set_tests_properties(guest_pthread_getname PROPERTIES ENVIRONMENT_MODIFICATION + "PATH=path_list_prepend:$") + endif() add_test_executable(guest_pthread_attr_tests ${LIBS_INCLUDE_DIR}/tests/GuestPthreadAttr.cpp) target_include_directories(guest_pthread_attr_tests PRIVATE ${LIBS_INCLUDE_DIR}) target_link_libraries(guest_pthread_attr_tests PRIVATE libkernel libc) @@ -200,6 +293,26 @@ if(BUILD_TESTING) set_tests_properties(guest_pthread_attr PROPERTIES ENVIRONMENT_MODIFICATION "PATH=path_list_prepend:$") endif() + add_test_executable(guest_pthread_start_tests ${LIBS_INCLUDE_DIR}/tests/PthreadStart.cpp) + target_include_directories(guest_pthread_start_tests PRIVATE ${LIBS_INCLUDE_DIR}) + target_link_libraries(guest_pthread_start_tests PRIVATE libkernel libc) + configure_windows_unwind(guest_pthread_start_tests) + add_test(NAME guest_pthread_start COMMAND guest_pthread_start_tests) + set_tests_properties(guest_pthread_start PROPERTIES TIMEOUT 20) + if(WIN32) + set_tests_properties(guest_pthread_start PROPERTIES TIMEOUT 60 ENVIRONMENT_MODIFICATION + "PATH=path_list_prepend:$") + endif() + add_test_executable(guest_pthread_stack_tests ${LIBS_INCLUDE_DIR}/tests/GuestPthreadStack.cpp) + target_include_directories(guest_pthread_stack_tests PRIVATE ${LIBS_INCLUDE_DIR}) + target_link_libraries(guest_pthread_stack_tests PRIVATE libkernel libc) + configure_windows_unwind(guest_pthread_stack_tests) + add_test(NAME guest_pthread_stack COMMAND guest_pthread_stack_tests) + set_tests_properties(guest_pthread_stack PROPERTIES TIMEOUT 20) + if(WIN32) + set_tests_properties(guest_pthread_stack PROPERTIES ENVIRONMENT_MODIFICATION + "PATH=path_list_prepend:$") + endif() add_test_executable(guest_cond_timedwait_tests ${LIBS_INCLUDE_DIR}/tests/GuestCondTimedwait.cpp) target_include_directories(guest_cond_timedwait_tests PRIVATE ${LIBS_INCLUDE_DIR}) target_link_libraries(guest_cond_timedwait_tests PRIVATE libkernel libc) @@ -210,6 +323,16 @@ if(BUILD_TESTING) set_tests_properties(guest_cond_timedwait PROPERTIES ENVIRONMENT_MODIFICATION "PATH=path_list_prepend:$") endif() + add_test_executable(guest_cond_recursive_mutex_tests ${LIBS_INCLUDE_DIR}/tests/GuestCondRecursiveMutex.cpp) + target_include_directories(guest_cond_recursive_mutex_tests PRIVATE ${LIBS_INCLUDE_DIR}) + target_link_libraries(guest_cond_recursive_mutex_tests PRIVATE libkernel libc) + configure_windows_unwind(guest_cond_recursive_mutex_tests) + add_test(NAME guest_cond_recursive_mutex COMMAND guest_cond_recursive_mutex_tests) + set_tests_properties(guest_cond_recursive_mutex PROPERTIES TIMEOUT 20) + if(WIN32) + set_tests_properties(guest_cond_recursive_mutex PROPERTIES ENVIRONMENT_MODIFICATION + "PATH=path_list_prepend:$") + endif() add_test_executable(guest_cpu_time_clocks_tests ${LIBS_INCLUDE_DIR}/tests/GuestCpuTimeClocks.cpp) target_include_directories(guest_cpu_time_clocks_tests PRIVATE ${LIBS_INCLUDE_DIR}) target_link_libraries(guest_cpu_time_clocks_tests PRIVATE libkernel libc) @@ -230,6 +353,16 @@ if(BUILD_TESTING) set_tests_properties(guest_freebsd_clocks PROPERTIES ENVIRONMENT_MODIFICATION "PATH=path_list_prepend:$") endif() + add_test_executable(guest_thread_cpu_clock_tests ${LIBS_INCLUDE_DIR}/tests/GuestThreadCpuClock.cpp) + target_include_directories(guest_thread_cpu_clock_tests PRIVATE ${LIBS_INCLUDE_DIR}) + target_link_libraries(guest_thread_cpu_clock_tests PRIVATE libkernel libc) + configure_windows_unwind(guest_thread_cpu_clock_tests) + add_test(NAME guest_thread_cpu_clock COMMAND guest_thread_cpu_clock_tests) + set_tests_properties(guest_thread_cpu_clock PROPERTIES TIMEOUT 30) + if(WIN32) + set_tests_properties(guest_thread_cpu_clock PROPERTIES ENVIRONMENT_MODIFICATION + "PATH=path_list_prepend:$") + endif() add_test_executable(guest_nanosleep_tests ${LIBS_INCLUDE_DIR}/tests/GuestNanosleep.cpp) target_include_directories(guest_nanosleep_tests PRIVATE ${LIBS_INCLUDE_DIR}) target_link_libraries(guest_nanosleep_tests PRIVATE libkernel libc) @@ -250,6 +383,16 @@ if(BUILD_TESTING) set_tests_properties(guest_pthread_sem PROPERTIES ENVIRONMENT_MODIFICATION "PATH=path_list_prepend:$") endif() + add_test_executable(guest_pthread_cancel_tests ${LIBS_INCLUDE_DIR}/tests/GuestPthreadCancel.cpp) + target_include_directories(guest_pthread_cancel_tests PRIVATE ${LIBS_INCLUDE_DIR}) + target_link_libraries(guest_pthread_cancel_tests PRIVATE libkernel libc) + configure_windows_unwind(guest_pthread_cancel_tests) + add_test(NAME guest_pthread_cancel COMMAND guest_pthread_cancel_tests) + set_tests_properties(guest_pthread_cancel PROPERTIES TIMEOUT 20) + if(WIN32) + set_tests_properties(guest_pthread_cancel PROPERTIES ENVIRONMENT_MODIFICATION + "PATH=path_list_prepend:$") + endif() add_test_executable(guest_semaphore_tests ${LIBS_INCLUDE_DIR}/tests/GuestSemaphore.cpp) target_include_directories(guest_semaphore_tests PRIVATE ${LIBS_INCLUDE_DIR}) target_link_libraries(guest_semaphore_tests PRIVATE libkernel libc) @@ -289,6 +432,35 @@ if(BUILD_TESTING) set_tests_properties(guest_signals PROPERTIES ENVIRONMENT_MODIFICATION "PATH=path_list_prepend:$") endif() + add_test_executable(guest_kill_tests ${LIBS_INCLUDE_DIR}/tests/GuestKill.cpp) + target_include_directories(guest_kill_tests PRIVATE ${LIBS_INCLUDE_DIR}) + target_link_libraries(guest_kill_tests PRIVATE libkernel libc) + configure_windows_unwind(guest_kill_tests) + add_test(NAME guest_kill COMMAND guest_kill_tests) + set_tests_properties(guest_kill PROPERTIES TIMEOUT 20) + if(WIN32) + set_tests_properties(guest_kill PROPERTIES ENVIRONMENT_MODIFICATION + "PATH=path_list_prepend:$") + endif() + add_test_executable(guest_signal_set_tests ${LIBS_INCLUDE_DIR}/tests/GuestSignalSets.cpp) + target_include_directories(guest_signal_set_tests PRIVATE ${LIBS_INCLUDE_DIR}) + target_link_libraries(guest_signal_set_tests PRIVATE libkernel libc) + configure_windows_unwind(guest_signal_set_tests) + add_test(NAME guest_signal_sets COMMAND guest_signal_set_tests) + set_tests_properties(guest_signal_sets PROPERTIES TIMEOUT 20) + if(WIN32) + set_tests_properties(guest_signal_sets PROPERTIES ENVIRONMENT_MODIFICATION + "PATH=path_list_prepend:$") + endif() + add_test_executable(guest_sigaltstack_tests ${LIBS_INCLUDE_DIR}/tests/GuestSigaltstack.cpp) + target_include_directories(guest_sigaltstack_tests PRIVATE ${LIBS_INCLUDE_DIR}) + target_link_libraries(guest_sigaltstack_tests PRIVATE libkernel libc) + configure_windows_unwind(guest_sigaltstack_tests) + add_test(NAME guest_sigaltstack COMMAND guest_sigaltstack_tests) + if(WIN32) + set_tests_properties(guest_sigaltstack PROPERTIES ENVIRONMENT_MODIFICATION + "PATH=path_list_prepend:$") + endif() add_test_executable(guest_exception_handler_tests ${LIBS_INCLUDE_DIR}/tests/GuestExceptionHandler.cpp) target_include_directories(guest_exception_handler_tests PRIVATE ${LIBS_INCLUDE_DIR}) target_link_libraries(guest_exception_handler_tests PRIVATE libkernel libc) @@ -306,6 +478,13 @@ if(BUILD_TESTING) add_test(NAME guest_raise_exception COMMAND guest_raise_exception_tests) set_tests_properties(guest_raise_exception PROPERTIES TIMEOUT 30 ENVIRONMENT_MODIFICATION "PATH=path_list_prepend:$") + add_test_executable(guest_raise_nested_sleep_tests ${LIBS_INCLUDE_DIR}/tests/GuestRaiseNestedSleep.cpp) + target_include_directories(guest_raise_nested_sleep_tests PRIVATE ${LIBS_INCLUDE_DIR}) + target_link_libraries(guest_raise_nested_sleep_tests PRIVATE libkernel libc) + configure_windows_unwind(guest_raise_nested_sleep_tests) + add_test(NAME guest_raise_nested_sleep COMMAND guest_raise_nested_sleep_tests) + set_tests_properties(guest_raise_nested_sleep PROPERTIES TIMEOUT 30 ENVIRONMENT_MODIFICATION + "PATH=path_list_prepend:$") endif() if(WIN32) add_library(guest_unwind_module_info_fixture SHARED ${LIBS_INCLUDE_DIR}/tests/GuestUnwindModuleInfoFixture.cpp) @@ -318,6 +497,26 @@ if(BUILD_TESTING) add_test(NAME guest_unwind_module_info COMMAND guest_unwind_module_info_tests) set_tests_properties(guest_unwind_module_info PROPERTIES ENVIRONMENT_MODIFICATION "PATH=path_list_prepend:$;PATH=path_list_prepend:$") + elseif(CMAKE_SYSTEM_NAME STREQUAL "Linux") + add_library(guest_unwind_module_info_fixture SHARED ${LIBS_INCLUDE_DIR}/tests/GuestModuleFixture.cpp) + target_include_directories(guest_unwind_module_info_fixture PRIVATE ${LIBS_INCLUDE_DIR}) + set_target_properties(guest_unwind_module_info_fixture PROPERTIES PREFIX "" OUTPUT_NAME guest_unwind_module_info_fixture.prx SUFFIX ".guest.prx") + add_library(guest_unwind_no_header_fixture SHARED ${LIBS_INCLUDE_DIR}/tests/GuestModuleFixture.cpp) + target_include_directories(guest_unwind_no_header_fixture PRIVATE ${LIBS_INCLUDE_DIR}) + target_link_options(guest_unwind_no_header_fixture PRIVATE -Wl,--no-eh-frame-hdr) + set_target_properties(guest_unwind_no_header_fixture PROPERTIES PREFIX "" OUTPUT_NAME guest_unwind_no_header_fixture.prx SUFFIX ".guest.prx") + add_test_executable(guest_unwind_module_info_tests ${LIBS_INCLUDE_DIR}/tests/GuestUnwindModuleInfoLinux.cpp) + target_include_directories(guest_unwind_module_info_tests PRIVATE ${LIBS_INCLUDE_DIR}) + target_link_libraries(guest_unwind_module_info_tests PRIVATE libkernel libc) + add_dependencies(guest_unwind_module_info_tests guest_unwind_module_info_fixture guest_unwind_no_header_fixture) + add_test(NAME guest_unwind_module_info COMMAND guest_unwind_module_info_tests guest_unwind_module_info_fixture.prx guest_unwind_no_header_fixture.prx) + set_tests_properties(guest_unwind_module_info PROPERTIES + WORKING_DIRECTORY "$") + add_test_executable(guest_unwind_module_info_failure_tests ${LIBS_INCLUDE_DIR}/tests/GuestUnwindModuleInfoFailureLinux.cpp) + target_include_directories(guest_unwind_module_info_failure_tests PRIVATE ${LIBS_INCLUDE_DIR}) + target_link_libraries(guest_unwind_module_info_failure_tests PRIVATE libkernel libc) + set_target_properties(guest_unwind_module_info_failure_tests PROPERTIES ENABLE_EXPORTS ON) + add_test(NAME guest_unwind_module_info_failure COMMAND guest_unwind_module_info_failure_tests) endif() add_test_executable(guest_sync_on_address_tests ${LIBS_INCLUDE_DIR}/tests/GuestSyncOnAddress.cpp) target_include_directories(guest_sync_on_address_tests PRIVATE ${LIBS_INCLUDE_DIR}) @@ -339,6 +538,16 @@ if(BUILD_TESTING) set_tests_properties(guest_available_cpumask PROPERTIES ENVIRONMENT_MODIFICATION "PATH=path_list_prepend:$") endif() + add_test_executable(guest_current_cpu_tests ${LIBS_INCLUDE_DIR}/tests/GuestCurrentCpu.cpp) + target_include_directories(guest_current_cpu_tests PRIVATE ${LIBS_INCLUDE_DIR}) + target_link_libraries(guest_current_cpu_tests PRIVATE libkernel libc) + configure_windows_unwind(guest_current_cpu_tests) + add_test(NAME guest_current_cpu COMMAND guest_current_cpu_tests) + set_tests_properties(guest_current_cpu PROPERTIES TIMEOUT 20) + if(WIN32) + set_tests_properties(guest_current_cpu PROPERTIES ENVIRONMENT_MODIFICATION + "PATH=path_list_prepend:$") + endif() add_test_executable(guest_getrusage_tests ${LIBS_INCLUDE_DIR}/tests/GuestGetrusage.cpp) target_include_directories(guest_getrusage_tests PRIVATE ${LIBS_INCLUDE_DIR}) target_link_libraries(guest_getrusage_tests PRIVATE libkernel libc) @@ -349,6 +558,26 @@ if(BUILD_TESTING) set_tests_properties(guest_getrusage PROPERTIES ENVIRONMENT_MODIFICATION "PATH=path_list_prepend:$") endif() + add_test_executable(guest_sched_priority_tests ${LIBS_INCLUDE_DIR}/tests/GuestSchedPriority.cpp) + target_include_directories(guest_sched_priority_tests PRIVATE ${LIBS_INCLUDE_DIR}) + target_link_libraries(guest_sched_priority_tests PRIVATE libkernel libc) + configure_windows_unwind(guest_sched_priority_tests) + add_test(NAME guest_sched_priority COMMAND guest_sched_priority_tests) + set_tests_properties(guest_sched_priority PROPERTIES TIMEOUT 20) + if(WIN32) + set_tests_properties(guest_sched_priority PROPERTIES ENVIRONMENT_MODIFICATION + "PATH=path_list_prepend:$") + endif() + add_test_executable(guest_waitpid_tests ${LIBS_INCLUDE_DIR}/tests/GuestWaitpid.cpp) + target_include_directories(guest_waitpid_tests PRIVATE ${LIBS_INCLUDE_DIR}) + target_link_libraries(guest_waitpid_tests PRIVATE libkernel libc) + configure_windows_unwind(guest_waitpid_tests) + add_test(NAME guest_waitpid COMMAND guest_waitpid_tests) + set_tests_properties(guest_waitpid PROPERTIES TIMEOUT 20) + if(WIN32) + set_tests_properties(guest_waitpid PROPERTIES ENVIRONMENT_MODIFICATION + "PATH=path_list_prepend:$") + endif() add_test_executable(guest_kernel_error_tests ${LIBS_INCLUDE_DIR}/tests/GuestKernelErrors.cpp) target_include_directories(guest_kernel_error_tests PRIVATE ${LIBS_INCLUDE_DIR}) target_link_libraries(guest_kernel_error_tests PRIVATE libkernel libc) @@ -359,6 +588,16 @@ if(BUILD_TESTING) set_tests_properties(guest_kernel_errors PROPERTIES ENVIRONMENT_MODIFICATION "PATH=path_list_prepend:$") endif() + add_test_executable(guest_getrlimit_tests ${LIBS_INCLUDE_DIR}/tests/GuestGetrlimit.cpp) + target_include_directories(guest_getrlimit_tests PRIVATE ${LIBS_INCLUDE_DIR}) + target_link_libraries(guest_getrlimit_tests PRIVATE libkernel libc) + configure_windows_unwind(guest_getrlimit_tests) + add_test(NAME guest_getrlimit COMMAND guest_getrlimit_tests) + set_tests_properties(guest_getrlimit PROPERTIES TIMEOUT 20) + if(WIN32) + set_tests_properties(guest_getrlimit PROPERTIES ENVIRONMENT_MODIFICATION + "PATH=path_list_prepend:$") + endif() add_test_executable(guest_equeue_event_tests ${LIBS_INCLUDE_DIR}/tests/GuestEqueueEvents.cpp) target_include_directories(guest_equeue_event_tests PRIVATE ${LIBS_INCLUDE_DIR}) target_link_libraries(guest_equeue_event_tests PRIVATE libkernel libc) @@ -369,6 +608,15 @@ if(BUILD_TESTING) set_tests_properties(guest_equeue_events PROPERTIES ENVIRONMENT_MODIFICATION "PATH=path_list_prepend:$") endif() + add_test_executable(guest_shm_open_tests ${LIBS_INCLUDE_DIR}/tests/GuestShmOpen.cpp) + target_include_directories(guest_shm_open_tests PRIVATE ${LIBS_INCLUDE_DIR}) + target_link_libraries(guest_shm_open_tests PRIVATE libkernel libc) + configure_windows_unwind(guest_shm_open_tests) + add_test(NAME guest_shm_open COMMAND guest_shm_open_tests) + if(WIN32) + set_tests_properties(guest_shm_open PROPERTIES ENVIRONMENT_MODIFICATION + "PATH=path_list_prepend:$") + endif() add_test_executable(guest_system_configuration_tests ${LIBS_INCLUDE_DIR}/tests/GuestSystemConfiguration.cpp) target_include_directories(guest_system_configuration_tests PRIVATE ${LIBS_INCLUDE_DIR}) target_link_libraries(guest_system_configuration_tests PRIVATE libkernel libc) @@ -378,6 +626,15 @@ if(BUILD_TESTING) set_tests_properties(guest_system_configuration PROPERTIES ENVIRONMENT_MODIFICATION "PATH=path_list_prepend:$") endif() + add_test_executable(guest_system_command_tests ${LIBS_INCLUDE_DIR}/tests/GuestSystemCommand.cpp) + target_include_directories(guest_system_command_tests PRIVATE ${LIBS_INCLUDE_DIR}) + target_link_libraries(guest_system_command_tests PRIVATE libkernel libc) + configure_windows_unwind(guest_system_command_tests) + add_test(NAME guest_system_command COMMAND guest_system_command_tests) + if(WIN32) + set_tests_properties(guest_system_command PROPERTIES ENVIRONMENT_MODIFICATION + "PATH=path_list_prepend:$") + endif() add_test_executable(guest_udp_tests ${LIBS_INCLUDE_DIR}/tests/GuestUdp.cpp) target_include_directories(guest_udp_tests PRIVATE ${LIBS_INCLUDE_DIR}) target_link_libraries(guest_udp_tests PRIVATE libkernel libc) @@ -388,6 +645,16 @@ if(BUILD_TESTING) set_tests_properties(guest_udp PROPERTIES ENVIRONMENT_MODIFICATION "PATH=path_list_prepend:$") endif() + add_test_executable(guest_socket_read_write_tests ${LIBS_INCLUDE_DIR}/tests/GuestSocketReadWrite.cpp) + target_include_directories(guest_socket_read_write_tests PRIVATE ${LIBS_INCLUDE_DIR}) + target_link_libraries(guest_socket_read_write_tests PRIVATE libkernel libc) + configure_windows_unwind(guest_socket_read_write_tests) + add_test(NAME guest_socket_read_write COMMAND guest_socket_read_write_tests) + set_tests_properties(guest_socket_read_write PROPERTIES TIMEOUT 20) + if(WIN32) + set_tests_properties(guest_socket_read_write PROPERTIES ENVIRONMENT_MODIFICATION + "PATH=path_list_prepend:$") + endif() add_test_executable(guest_poll_tests ${LIBS_INCLUDE_DIR}/tests/GuestPoll.cpp) target_include_directories(guest_poll_tests PRIVATE ${LIBS_INCLUDE_DIR}) target_link_libraries(guest_poll_tests PRIVATE libkernel libc) @@ -428,6 +695,16 @@ if(BUILD_TESTING) set_tests_properties(guest_elf_phdr_match PROPERTIES ENVIRONMENT_MODIFICATION "PATH=path_list_prepend:$") endif() + add_test_executable(guest_credentials_tests ${LIBS_INCLUDE_DIR}/tests/GuestCredentials.cpp) + target_include_directories(guest_credentials_tests PRIVATE ${LIBS_INCLUDE_DIR}) + target_link_libraries(guest_credentials_tests PRIVATE libkernel libc) + configure_windows_unwind(guest_credentials_tests) + add_test(NAME guest_credentials COMMAND guest_credentials_tests) + set_tests_properties(guest_credentials PROPERTIES TIMEOUT 20) + if(WIN32) + set_tests_properties(guest_credentials PROPERTIES ENVIRONMENT_MODIFICATION + "PATH=path_list_prepend:$") + endif() add_test_executable(guest_address_tests ${LIBS_INCLUDE_DIR}/tests/GuestAddressConversion.cpp) target_include_directories(guest_address_tests PRIVATE ${LIBS_INCLUDE_DIR}) target_link_libraries(guest_address_tests PRIVATE libkernel libc) @@ -443,9 +720,33 @@ if(BUILD_TESTING) configure_windows_unwind(guest_memory_tests) add_test(NAME guest_memory COMMAND guest_memory_tests) if(WIN32) + add_library(guest_memory_module SHARED ${LIBS_INCLUDE_DIR}/tests/GuestMemoryModule.cpp) + set_target_properties(guest_memory_module PROPERTIES PREFIX "" SUFFIX ".guest.prx") + add_dependencies(guest_memory_tests guest_memory_module) + target_compile_definitions(guest_memory_tests PRIVATE GUEST_MEMORY_MODULE="$") set_tests_properties(guest_memory PROPERTIES ENVIRONMENT_MODIFICATION "PATH=path_list_prepend:$") endif() + add_test_executable(guest_mlock_tests ${LIBS_INCLUDE_DIR}/tests/GuestMlock.cpp) + target_include_directories(guest_mlock_tests PRIVATE ${LIBS_INCLUDE_DIR}) + target_link_libraries(guest_mlock_tests PRIVATE libkernel libc) + configure_windows_unwind(guest_mlock_tests) + add_test(NAME guest_mlock COMMAND guest_mlock_tests) + set_tests_properties(guest_mlock PROPERTIES TIMEOUT 20) + if(WIN32) + set_tests_properties(guest_mlock PROPERTIES ENVIRONMENT_MODIFICATION + "PATH=path_list_prepend:$") + endif() + add_test_executable(guest_msync_tests ${LIBS_INCLUDE_DIR}/tests/GuestMsync.cpp) + target_include_directories(guest_msync_tests PRIVATE ${LIBS_INCLUDE_DIR}) + target_link_libraries(guest_msync_tests PRIVATE libkernel libc) + configure_windows_unwind(guest_msync_tests) + add_test(NAME guest_msync COMMAND guest_msync_tests) + set_tests_properties(guest_msync PROPERTIES TIMEOUT 20) + if(WIN32) + set_tests_properties(guest_msync PROPERTIES ENVIRONMENT_MODIFICATION + "PATH=path_list_prepend:$") + endif() if(WIN32) add_test_executable(guest_directory_buffer_tests ${LIBS_INCLUDE_DIR}/tests/GuestDirectoryBuffer.cpp) target_include_directories(guest_directory_buffer_tests PRIVATE ${LIBS_INCLUDE_DIR}) @@ -481,4 +782,31 @@ if(BUILD_TESTING) set_tests_properties(guest_file_vector PROPERTIES ENVIRONMENT_MODIFICATION "PATH=path_list_prepend:$") endif() + add_test_executable(guest_sandbox_random_word_tests ${LIBS_INCLUDE_DIR}/tests/GuestSandboxRandomWord.cpp) + target_include_directories(guest_sandbox_random_word_tests PRIVATE ${LIBS_INCLUDE_DIR}) + target_link_libraries(guest_sandbox_random_word_tests PRIVATE libkernel libc) + configure_windows_unwind(guest_sandbox_random_word_tests) + add_test(NAME guest_sandbox_random_word COMMAND guest_sandbox_random_word_tests) + set_tests_properties(guest_sandbox_random_word PROPERTIES TIMEOUT 20) + if(WIN32) + set_tests_properties(guest_sandbox_random_word PROPERTIES ENVIRONMENT_MODIFICATION + "PATH=path_list_prepend:$") + add_test_executable(guest_mutex_raise_exception_tests ${LIBS_INCLUDE_DIR}/tests/GuestMutexRaiseException.cpp) + target_include_directories(guest_mutex_raise_exception_tests PRIVATE ${LIBS_INCLUDE_DIR}) + target_link_libraries(guest_mutex_raise_exception_tests PRIVATE libkernel libc) + configure_windows_unwind(guest_mutex_raise_exception_tests) + add_test(NAME guest_mutex_raise_exception COMMAND guest_mutex_raise_exception_tests) + set_tests_properties(guest_mutex_raise_exception PROPERTIES TIMEOUT 20 ENVIRONMENT_MODIFICATION + "PATH=path_list_prepend:$") + endif() + add_test_executable(guest_pthread_rename_tests ${LIBS_INCLUDE_DIR}/tests/GuestPthreadRename.cpp) + target_include_directories(guest_pthread_rename_tests PRIVATE ${LIBS_INCLUDE_DIR}) + target_link_libraries(guest_pthread_rename_tests PRIVATE libkernel libc) + configure_windows_unwind(guest_pthread_rename_tests) + add_test(NAME guest_pthread_rename COMMAND guest_pthread_rename_tests) + set_tests_properties(guest_pthread_rename PROPERTIES TIMEOUT 20) + if(WIN32) + set_tests_properties(guest_pthread_rename PROPERTIES ENVIRONMENT_MODIFICATION + "PATH=path_list_prepend:$") + endif() endif() diff --git a/core/libs/prx/libkernel/DirectMemory/DirectMemory.cpp b/core/libs/prx/libkernel/DirectMemory/DirectMemory.cpp index cd7bf920b..ff3933405 100644 --- a/core/libs/prx/libkernel/DirectMemory/DirectMemory.cpp +++ b/core/libs/prx/libkernel/DirectMemory/DirectMemory.cpp @@ -11,9 +11,12 @@ #include #include #include +#include #include #include #include +#include +#include #include #include @@ -139,6 +142,7 @@ static int mprotect(void* addr, size_t len, int prot) { namespace { constexpr int GuestMapFixedFlag = 0x10; +constexpr int GuestProtGpuReadWrite = 0x30; #if defined(__linux__) void* MapAtOrAbove(std::uintptr_t start, size_t len, int prot, size_t alignment) { @@ -221,10 +225,49 @@ void Trace(const char* format, ...) { va_end(args); } +#if defined(__linux__) +int PhysicalFile() { + static const int file = [] { + const int created = memfd_create("direct memory", MFD_CLOEXEC | MFD_ALLOW_SEALING); + if (created < 0) throw std::system_error(errno, std::generic_category(), "create direct memory backing"); + if (ftruncate(created, static_cast(DIRECT_MEMORY_SIZE)) != 0 || fcntl(created, F_ADD_SEALS, F_SEAL_SHRINK | F_SEAL_GROW) != 0) { + const int error = errno; + ::close(created); + throw std::system_error(error, std::generic_category(), "size direct memory backing"); + } + return created; + }(); + return file; +} + +std::vector g_physicalUsed(DIRECT_MEMORY_SIZE / PS5_PAGE_SIZE); + +void ZeroReusedPhysical(std::uint64_t start, std::size_t bytes) { + const auto first = start / PS5_PAGE_SIZE; + const auto count = bytes / PS5_PAGE_SIZE; + for (std::size_t page = 0; page < count;) { + if (!g_physicalUsed[first + page]) { + g_physicalUsed[first + page] = true; + ++page; + continue; + } + std::size_t run = 0; + while (page + run < count && g_physicalUsed[first + page + run]) ++run; + const auto offset = (first + page) * PS5_PAGE_SIZE; + void* view = ::mmap(nullptr, run * PS5_PAGE_SIZE, PROT_READ | PROT_WRITE, MAP_SHARED, PhysicalFile(), static_cast(offset)); + if (view == MAP_FAILED) throw std::system_error(errno, std::generic_category(), "map reused direct memory"); + std::memset(view, 0, run * PS5_PAGE_SIZE); + ::munmap(view, run * PS5_PAGE_SIZE); + page += run; + } +} +#endif + class PhysicalBacking { public: - explicit PhysicalBacking(std::size_t bytes, int memoryType) : memoryType(memoryType) { + PhysicalBacking(std::size_t bytes, int memoryType, std::uint64_t start) : memoryType(memoryType), bytes(bytes) { #ifdef _WIN32 + static_cast(start); const auto size = static_cast(bytes); section = CreateFileMappingW(INVALID_HANDLE_VALUE, nullptr, PAGE_EXECUTE_READWRITE, static_cast(size >> 32), static_cast(size), nullptr); if (!section) { @@ -234,18 +277,9 @@ public: throw std::system_error(error, std::system_category(), message); } #else - file = memfd_create("direct memory", MFD_CLOEXEC | MFD_ALLOW_SEALING); - if (file < 0) throw std::system_error(errno, std::generic_category(), "create direct memory backing"); - if (ftruncate(file, static_cast(bytes)) != 0) { - const int error = errno; - ::close(file); - throw std::system_error(error, std::generic_category(), "size direct memory backing"); - } - if (fcntl(file, F_ADD_SEALS, F_SEAL_SHRINK | F_SEAL_GROW) != 0) { - const int error = errno; - ::close(file); - throw std::system_error(error, std::generic_category(), "seal direct memory backing"); - } + file = PhysicalFile(); + fileOffset = start; + ZeroReusedPhysical(start, bytes); #endif } @@ -253,13 +287,23 @@ public: #if defined(__linux__) int File() const { return file; } + void* HostWriteData(std::uint64_t offset, std::size_t length) { + if (offset > bytes || length > bytes - offset) return nullptr; + if (hostWriteView == nullptr) { + auto* mapped = ::mmap(nullptr, bytes, PROT_READ | PROT_WRITE, MAP_SHARED, file, static_cast(fileOffset)); + if (mapped == MAP_FAILED) throw std::system_error(errno, std::generic_category(), "Direct memory write view mmap failed"); + hostWriteView = mapped; + } + return static_cast(hostWriteView) + offset; + } + std::uint64_t FileOffset() const { return fileOffset; } #endif ~PhysicalBacking() { #ifdef _WIN32 CloseHandle(section); #else - ::close(file); + if (hostWriteView != nullptr) ::munmap(hostWriteView, bytes); #endif } @@ -270,16 +314,19 @@ public: #ifdef _WIN32 GuestArena::GuestArenaMap_nid_postfix(reinterpret_cast(address), bytes, section, offset, WinProtFromPosix(protection)); #else - if (::mmap(reinterpret_cast(address), bytes, protection, MAP_SHARED | MAP_FIXED, file, static_cast(offset)) == MAP_FAILED) throw std::system_error(errno, std::generic_category(), "map direct memory backing"); + if (::mmap(reinterpret_cast(address), bytes, protection, MAP_SHARED | MAP_FIXED, file, static_cast(fileOffset + offset)) == MAP_FAILED) throw std::system_error(errno, std::generic_category(), "map direct memory backing"); #endif } private: int memoryType; + std::size_t bytes; #ifdef _WIN32 HANDLE section = nullptr; #else int file = -1; + void* hostWriteView = nullptr; + std::uint64_t fileOffset = 0; #endif }; @@ -384,7 +431,39 @@ bool SharedBacking(std::uintptr_t address, std::size_t bytes, int* file, std::ui const int duplicate = fcntl(backing->File(), F_DUPFD_CLOEXEC, 0); if (duplicate < 0) throw std::system_error(errno, std::generic_category(), "duplicate direct memory backing"); *file = duplicate; - *offset = page->second.offset + phys % PS5_PAGE_SIZE; + *offset = backing->FileOffset() + page->second.offset + phys % PS5_PAGE_SIZE; + return true; +} + +bool WriteSharedBacking(std::uintptr_t address, const void* source, std::size_t bytes) { + if (bytes == 0 || bytes > std::numeric_limits::max() - address) return false; + std::lock_guard lock(g_directLock); + const auto next = g_directMappings.upper_bound(address); + if (next == g_directMappings.begin()) return false; + struct WriteSpan { + void* destination; + std::size_t bytes; + }; + std::vector spans; + const auto end = address + bytes; + auto it = std::prev(next); + for (auto cursor = address; cursor < end; ++it) { + if (it == g_directMappings.end() || it->first > cursor || cursor >= it->second.end) return false; + const auto physical = it->second.phys + (cursor - it->first); + const auto page = g_physPages.find(physical - physical % PS5_PAGE_SIZE); + if (page == g_physPages.end() || page->second.backing != it->second.backing) return false; + const auto offset = page->second.offset + physical % PS5_PAGE_SIZE; + const auto length = std::min(end, it->second.end) - cursor; + auto* destination = it->second.backing->HostWriteData(offset, length); + if (destination == nullptr) return false; + spans.push_back({destination, length}); + cursor += length; + } + auto* input = static_cast(source); + for (const auto& span : spans) { + std::memcpy(span.destination, input, span.bytes); + input += span.bytes; + } return true; } #endif @@ -416,7 +495,7 @@ bool RemapFixedIntoRegistered(GuestAllocations::Mutation& mutation, void* addr, ValidateRange(addr, len, PS5_PAGE_SIZE); Trace("remap fixed %p+0x%zx prot=0x%x phys=0x%llx", addr, len, prot, static_cast(physStart)); const auto nativeProtection = LinuxProtFromSce(prot); - mutation.Protect(addr, len, (prot & 3) != 0, (prot & 2) != 0, [&] { + mutation.Protect(addr, len, (prot & 3) != 0, (prot & 2) != 0, (prot & GuestProtGpuReadWrite) != 0, [&] { const auto address = reinterpret_cast(addr); std::lock_guard lock(g_directLock); if (physStart >= 0) { @@ -435,9 +514,24 @@ bool RemapFixedIntoRegistered(GuestAllocations::Mutation& mutation, void* addr, return true; } +#if defined(__linux__) +bool ArenaAvailable() { return GuestArena::GuestArenaAvailable_nid_postfix(); } + +void* ArenaCommit(void* addr, size_t len, int prot) { + return mmap(addr, len, prot, MAP_PRIVATE | MAP_ANONYMOUS | MAP_FIXED, -1, 0); +} + +void ArenaForget(void* addr, size_t len) { + if (GuestArena::GuestArenaContains_nid_postfix(addr, len)) GuestArena::GuestArenaRelease_nid_postfix(addr, len); +} +#endif + void Unmap(void* addr, size_t len) { #if defined(__linux__) - if (munmap(addr, len) != 0) throw std::system_error(errno, std::generic_category(), "munmap failed"); + if (ArenaAvailable() && GuestArena::GuestArenaContains_nid_postfix(addr, len)) + GuestArena::GuestArenaReset_nid_postfix(addr, len); + else if (munmap(addr, len) != 0) throw std::system_error(errno, std::generic_category(), "munmap failed"); + ArenaForget(addr, len); GuestWriteWatch::GuestWriteWatchUnregister_nid_postfix(addr, len); #else if (KernelArena::Get().Contains(addr, len)) munmap(addr, len); @@ -486,6 +580,14 @@ void* MapPlaced(void* addr, size_t len, int prot, int flags, size_t alignment) { } #endif #if defined(__linux__) + if (ArenaAvailable() && !GuestArena::GuestArenaContains_nid_postfix(addr, len)) + throw std::invalid_argument("fixed mapping is outside the guest address space arena"); + if (ArenaAvailable()) { + GuestArena::GuestArenaMarkUsed_nid_postfix(addr, len); + if (void* placed = ArenaCommit(addr, len, prot); placed != MAP_FAILED) return placed; + GuestArena::GuestArenaRelease_nid_postfix(addr, len); + throw std::system_error(errno, std::generic_category(), "Arena mapping failed"); + } const int placement = (flags & GuestMapNoOverwrite) != 0 ? MAP_FIXED_NOREPLACE : MAP_FIXED; #else const int placement = MAP_FIXED; @@ -499,6 +601,13 @@ void* MapPlaced(void* addr, size_t len, int prot, int flags, size_t alignment) { } if (addr) { #if defined(__linux__) + if (ArenaAvailable()) { + void* placed = GuestArena::GuestArenaAllocateAtOrAbove_nid_postfix( + reinterpret_cast(addr), len, alignment); + if (void* committed = ArenaCommit(placed, len, prot); committed != MAP_FAILED) return committed; + ArenaForget(placed, len); + throw std::system_error(errno, std::generic_category(), "Arena mapping failed"); + } return MapAtOrAbove(reinterpret_cast(addr), len, prot, alignment); #else return mmap_aligned(len, prot, alignment, reinterpret_cast(addr)); @@ -506,6 +615,14 @@ void* MapPlaced(void* addr, size_t len, int prot, int flags, size_t alignment) { } #ifdef _WIN32 return mmap_aligned(len, prot, alignment); +#endif +#if defined(__linux__) + if (ArenaAvailable()) { + void* placed = GuestArena::GuestArenaAllocate_nid_postfix(len, alignment); + if (void* committed = ArenaCommit(placed, len, prot); committed != MAP_FAILED) return committed; + ArenaForget(placed, len); + throw std::system_error(errno, std::generic_category(), "Arena mapping failed"); + } #endif if (len > std::numeric_limits::max() - alignment) { throw std::overflow_error("Aligned mapping size overflow"); @@ -664,7 +781,7 @@ int DoMapDirect(void** addr, size_t len, int prot, int flags, int64_t physStart, void* mapped = MapAligned(*addr, len, PROT_NONE, flags, alignment); try { AddMapping(reinterpret_cast(mapped), len, static_cast(physStart), LinuxProtFromSce(prot)); - mutation.Add(mapped, len, (prot & 3) != 0, (prot & 2) != 0); + mutation.Add(mapped, len, (prot & 3) != 0, (prot & 2) != 0, (prot & GuestProtGpuReadWrite) != 0); } catch (...) { EraseMappings(reinterpret_cast(mapped), reinterpret_cast(mapped) + len); Unmap(mapped, len); @@ -677,7 +794,7 @@ int DoMapDirect(void** addr, size_t len, int prot, int flags, int64_t physStart, return 0; } -int DoMapAnon(void** addr, size_t len, int prot, int flags) { +int DoMapAnon(void** addr, size_t len, int prot, int flags, size_t alignment) { ValidateOutput(addr); if (len == 0 || (len & (PS5_PAGE_SIZE - 1)) != 0) return SCE_KERNEL_ERROR_EINVAL; GuestAllocations::Mutation mutation; @@ -688,9 +805,9 @@ int DoMapAnon(void** addr, size_t len, int prot, int flags) { return 0; } ReplaceFixedOverlap(mutation, *addr, len, flags); - void* mapped = MapAligned(*addr, len, LinuxProtFromSce(prot), flags, PS5_PAGE_SIZE); + void* mapped = MapAligned(*addr, len, LinuxProtFromSce(prot), flags, alignment); try { - mutation.Add(mapped, len, (prot & 3) != 0, (prot & 2) != 0); + mutation.Add(mapped, len, (prot & 3) != 0, (prot & 2) != 0, (prot & GuestProtGpuReadWrite) != 0); } catch (...) { Unmap(mapped, len); throw; @@ -702,6 +819,24 @@ int DoMapAnon(void** addr, size_t len, int prot, int flags) { return 0; } +#ifdef _WIN32 +static bool IsGuestModuleImage(const void* base) { + std::wstring path(MAX_PATH, L'\0'); + for (;;) { + const auto length = GetModuleFileNameW(static_cast(const_cast(base)), path.data(), static_cast(path.size())); + if (length == 0) return false; + if (length < path.size()) { + path.resize(length); + break; + } + path.resize(path.size() * 2); + } + constexpr std::wstring_view suffix = L".guest.prx"; + if (path.size() < suffix.size()) return false; + return std::equal(suffix.begin(), suffix.end(), path.end() - static_cast(suffix.size()), [](wchar_t expected, wchar_t actual) { return std::towlower(actual) == expected; }); +} +#endif + int DoMprotect(const void* addr, size_t len, int prot) { Trace("protect %p+0x%zx prot=0x%x", addr, len, prot); const auto address = reinterpret_cast(addr); @@ -718,13 +853,14 @@ int DoMprotect(const void* addr, size_t len, int prot) { MEMORY_BASIC_INFORMATION memory{}; if (VirtualQuery(pointer, &memory, sizeof(memory)) != sizeof(memory)) throw std::runtime_error("Cannot query guest memory protection range"); if (memory.Type == MEM_IMAGE) { - if (memory.AllocationBase != GetModuleHandleW(nullptr)) throw std::invalid_argument("Memory protection of a foreign image is not supported"); - mutation.RegisterMainImage(); + if (memory.AllocationBase == GetModuleHandleW(nullptr)) mutation.RegisterMainImage(); + else if (IsGuestModuleImage(memory.AllocationBase)) mutation.RegisterImage(memory.AllocationBase); + else throw std::invalid_argument("Memory protection of a foreign image is not supported"); } #else mutation.RegisterMainImage(); #endif - mutation.Protect(pointer, bytes, (prot & 3) != 0, (prot & 2) != 0, [&] { + mutation.Protect(pointer, bytes, (prot & 3) != 0, (prot & 2) != 0, (prot & GuestProtGpuReadWrite) != 0, [&] { if (mprotect(const_cast(pointer), bytes, nativeProtection) != 0) throw std::system_error(errno, std::generic_category(), "mprotect failed"); }); RecordProtection(pointer, bytes, prot); @@ -784,7 +920,7 @@ int DoReserveVirtual(void** addr, size_t len, int flags, size_t alignment) { constexpr int GuestMapNoCoalesce = 0x400000; void* mapped = MapAligned(fixed ? *addr : nullptr, len, PROT_NONE, fixed ? GuestMapFixedFlag | (flags & GuestMapNoCoalesce) : 0, alignment); try { - mutation.Add(mapped, len, false, false); + mutation.Add(mapped, len, false, false, false); } catch (...) { Unmap(mapped, len); throw; @@ -803,9 +939,12 @@ void CreateDirectMemoryBacking(int64_t start, size_t len, int memoryType) { } #if defined(__linux__) static std::once_flag registered; - std::call_once(registered, [] { GuestArena::GuestArenaSetSharedBacking_nid_postfix(&SharedBacking); }); + std::call_once(registered, [] { + GuestArena::GuestArenaSetSharedBacking_nid_postfix(&SharedBacking); + GuestArena::GuestArenaSetSharedBackingWriter_nid_no_patch(&WriteSharedBacking); + }); #endif - const auto backing = std::make_shared(len, memoryType); + const auto backing = std::make_shared(len, memoryType, first); std::map pages; for (std::size_t offset = 0; offset < len; offset += PS5_PAGE_SIZE) pages.emplace(first + offset, PhysicalPage{backing, offset}); g_physPages.merge(pages); @@ -854,3 +993,28 @@ bool QueryDirectMapping(std::uintptr_t address, std::uintptr_t* start, std::uint *memoryType = it->second.memoryType; return true; } + +std::vector DirectMemoryViews(int64_t start, size_t len) { + const auto first = static_cast(start); + const auto last = first + len; + std::vector views; + std::lock_guard lock(g_directLock); + for (const auto& [base, mapping] : g_directMappings) { + const auto mappingLast = mapping.phys + (mapping.end - base); + if (mapping.phys >= last || mappingLast <= first) continue; + const auto viewStart = base + (std::max(first, mapping.phys) - mapping.phys); + const auto viewEnd = base + (std::min(last, mappingLast) - mapping.phys); + if (!views.empty() && views.back().address + views.back().bytes == viewStart) views.back().bytes += viewEnd - viewStart; + else views.push_back({viewStart, viewEnd - viewStart}); + } + return views; +} + +void UnmapDirectMemoryViews(const std::vector& views) { + if (views.empty()) return; + GuestAllocations::Mutation mutation; + for (const auto& view : views) { + Trace("unmap released view %p+0x%zx", reinterpret_cast(view.address), view.bytes); + UnmapRegistered(mutation, reinterpret_cast(view.address), view.bytes); + } +} diff --git a/core/libs/prx/libkernel/DirectMemory/DirectMemory.hpp b/core/libs/prx/libkernel/DirectMemory/DirectMemory.hpp index e2310818d..988291463 100644 --- a/core/libs/prx/libkernel/DirectMemory/DirectMemory.hpp +++ b/core/libs/prx/libkernel/DirectMemory/DirectMemory.hpp @@ -3,6 +3,7 @@ #include #include +#include #include "prx/libkernel/KernelErrors.hpp" @@ -15,16 +16,25 @@ bool DirectMemoryCheckedFree(int64_t start, size_t len); void CreateDirectMemoryBacking(int64_t start, size_t len, int memoryType); bool QueryDirectMapping(std::uintptr_t address, std::uintptr_t* start, std::uintptr_t* end, std::uint64_t* offset, int* memoryType); void ForgetDirectMemory(int64_t start, size_t len); + +struct DirectMemoryView { + std::uintptr_t address; + std::size_t bytes; +}; + +std::vector DirectMemoryViews(int64_t start, size_t len); +void UnmapDirectMemoryViews(const std::vector& views); bool DirectMemoryFind(int64_t offset, bool findNext, int64_t* start, int64_t* end, int* memoryType); void DirectMemoryRetype(int64_t start, size_t len, int memoryType); size_t DirectMemoryFreeRun(uint64_t offset, uint64_t limit); int DoMapDirect(void** addr, size_t len, int prot, int flags, int64_t physStart, size_t alignment); -int DoMapAnon(void** addr, size_t len, int prot, int flags); +int DoMapAnon(void** addr, size_t len, int prot, int flags, size_t alignment = PS5_PAGE_SIZE); int DoMprotect(const void* addr, size_t len, int prot); int DoMtypeprotect(const void* addr, size_t len, int type, int prot); int DoMunmap(void* addr, size_t len); int DoReserveVirtual(void** addr, size_t len, int flags, size_t alignment); bool GuestProtection(uintptr_t addr, int* prot); bool GuestReservation(std::uintptr_t addr, std::uintptr_t* start, std::uintptr_t* end); +bool GuestRangeMapped(std::uintptr_t start, std::uintptr_t end); #endif \ No newline at end of file diff --git a/core/libs/prx/libkernel/DirectMemory/Export.cpp b/core/libs/prx/libkernel/DirectMemory/Export.cpp index 8d893ee6c..bee6dd534 100644 --- a/core/libs/prx/libkernel/DirectMemory/Export.cpp +++ b/core/libs/prx/libkernel/DirectMemory/Export.cpp @@ -20,6 +20,7 @@ #include #endif #include +#include #include #include #include @@ -216,7 +217,12 @@ int APS5_VABI sceKernelReleaseFlexibleMemory(void* addr, size_t len) { int APS5_VABI sceKernelReleaseDirectMemory(int64_t start, size_t len) { if (start < 0 || len == 0) return SCE_KERNEL_ERROR_EINVAL; + if ((static_cast(start) & (PS5_PAGE_SIZE - 1)) != 0 || (len & (PS5_PAGE_SIZE - 1)) != 0 || + len > DIRECT_MEMORY_SIZE || static_cast(start) > DIRECT_MEMORY_SIZE - len) + return SCE_KERNEL_ERROR_EINVAL; + const auto views = DirectMemoryViews(start, len); DirectMemoryFree(start, len); + UnmapDirectMemoryViews(views); return 0; } @@ -228,9 +234,6 @@ int APS5_VABI sceKernelVirtualQuery(const void* addr, int flags, VirtualQueryInf if (!info || info_size < sizeof(VirtualQueryInfo)) return SCE_KERNEL_ERROR_EINVAL; memset(info, 0, sizeof(VirtualQueryInfo)); const auto address = reinterpret_cast(addr); - // The mapping that contains the address, or with SCE_KERNEL_VQ_FIND_NEXT (flags bit 0) the first - // mapping at or above it: titles walk their mappings and check that a mapping covers a whole - // allocation, so the answer must be the registered allocation, not a page. constexpr int findNext = 1; std::uintptr_t reservedStart = 0; std::uintptr_t reservedEnd = 0; @@ -239,17 +242,18 @@ int APS5_VABI sceKernelVirtualQuery(const void* addr, int flags, VirtualQueryInf info->end = reservedEnd; return 0; } - const auto lease = GuestAllocations::GuestAllocationsAcquire_nid_postfix(); + const auto lease = GuestAllocations::GuestAllocationsAcquireAll_nid_postfix(); + assert(std::is_sorted(lease.begin(), lease.end(), [](const auto& left, const auto& right) { return left->address < right->address; })); const GuestAllocations::Range* best = nullptr; for (const auto& range : lease) { - const auto begin = range->allocationAddress; - const auto end = begin + range->allocationBytes; + const auto begin = range->address; + const auto end = begin + range->bytes; if (address >= begin && address < end) { best = range.get(); break; } - if ((flags & findNext) != 0 && begin > address && (best == nullptr || begin < best->allocationAddress)) best = range.get(); + if ((flags & findNext) != 0 && begin > address && (best == nullptr || begin < best->address)) best = range.get(); } if (best != nullptr) { - info->start = best->allocationAddress; - info->end = best->allocationAddress + best->allocationBytes; + info->start = best->address; + info->end = best->address + best->bytes; info->protection = (best->readable ? 1 : 0) | (best->writable ? 2 : 0) | (!best->releasable ? 4 : 0); int recorded = 0; if (GuestProtection(std::max(address, info->start), &recorded)) info->protection = recorded; @@ -313,7 +317,10 @@ int APS5_VABI sceKernelVirtualQuery(const void* addr, int flags, VirtualQueryInf int APS5_VABI sceKernelCheckedReleaseDirectMemory(int64_t start, size_t len) { if (start < 0 || (static_cast(start) & (PS5_PAGE_SIZE - 1)) != 0 || (len & (PS5_PAGE_SIZE - 1)) != 0) return SCE_KERNEL_ERROR_EINVAL; if (len == 0) return 0; - return DirectMemoryCheckedFree(start, len) ? 0 : SCE_KERNEL_ERROR_ENOENT; + const auto views = DirectMemoryViews(start, len); + if (!DirectMemoryCheckedFree(start, len)) return SCE_KERNEL_ERROR_ENOENT; + UnmapDirectMemoryViews(views); + return 0; } int APS5_VABI sceKernelMtypeprotect(const void* addr, size_t len, int type, int prot) { @@ -443,8 +450,27 @@ int APS5_VABI sceKernelBatchMap2(KernelBatchMapEntry* entries, int num_entries, } +bool GuestRangeMapped(std::uintptr_t start, std::uintptr_t end) { + for (auto cursor = start; cursor < end;) { + VirtualQueryInfo info{}; + if (sceKernelVirtualQuery(reinterpret_cast(cursor), 0, &info, sizeof(info)) != 0) return false; + if (info.end <= cursor) return false; + cursor = info.end; + } + return true; +} + namespace { +bool PageRange(void* address, std::uint64_t length, std::uintptr_t& start, std::uintptr_t& end) { + constexpr std::uintptr_t pageMask = PS5_PAGE_SIZE - 1; + const auto first = reinterpret_cast(address); + if (length > UINTPTR_MAX - first || first + length > UINTPTR_MAX - pageMask) return false; + start = first & ~pageMask; + end = (first + length + pageMask) & ~pageMask; + return true; +} + #ifdef _WIN32 std::mutex g_workingSetLock; @@ -475,6 +501,53 @@ int LockHostPages(std::uintptr_t start, std::uintptr_t end) { if (GetLastError() != ERROR_WORKING_SET_QUOTA) throw std::system_error(static_cast(GetLastError()), std::system_category(), "VirtualLock failed"); return SCE_KERNEL_ERROR_EAGAIN; } + +std::uintptr_t HostPageSize() { + static const std::uintptr_t size = [] { + SYSTEM_INFO info{}; + GetSystemInfo(&info); + return static_cast(info.dwPageSize); + }(); + return size; +} + +int UnlockHostPages(std::uintptr_t start, std::uintptr_t end) { + for (auto cursor = start; cursor < end;) { + MEMORY_BASIC_INFORMATION region{}; + if (VirtualQuery(reinterpret_cast(cursor), ®ion, sizeof(region)) != sizeof(region) || region.State == MEM_FREE) return SCE_KERNEL_ERROR_ENOMEM; + const auto next = std::min(end, reinterpret_cast(region.BaseAddress) + region.RegionSize); + if (region.State == MEM_RESERVE) { + for (auto reserved = cursor; reserved < next;) { + std::uintptr_t reservationStart = 0; + std::uintptr_t reservationEnd = 0; + int recorded = 0; + if (GuestReservation(reserved, &reservationStart, &reservationEnd)) { + reserved = reservationEnd; + } else if (GuestProtection(reserved, &recorded)) { + reserved = (reserved & ~static_cast(PS5_PAGE_SIZE - 1)) + PS5_PAGE_SIZE; + } else { + return SCE_KERNEL_ERROR_ENOMEM; + } + } + } + cursor = next; + } + for (auto cursor = start; cursor < end;) { + MEMORY_BASIC_INFORMATION region{}; + if (VirtualQuery(reinterpret_cast(cursor), ®ion, sizeof(region)) != sizeof(region)) return SCE_KERNEL_ERROR_ENOMEM; + const auto next = std::min(end, reinterpret_cast(region.BaseAddress) + region.RegionSize); + if (region.State == MEM_COMMIT && (region.Protect & (PAGE_NOACCESS | PAGE_GUARD)) == 0 && !VirtualUnlock(reinterpret_cast(cursor), next - cursor)) { + if (GetLastError() != ERROR_NOT_LOCKED) throw std::system_error(static_cast(GetLastError()), std::system_category(), "VirtualUnlock failed"); + for (auto page = cursor; page < next; page += HostPageSize()) { + if (!VirtualUnlock(reinterpret_cast(page), std::min(HostPageSize(), next - page)) && GetLastError() != ERROR_NOT_LOCKED) { + throw std::system_error(static_cast(GetLastError()), std::system_category(), "VirtualUnlock failed"); + } + } + } + cursor = next; + } + return 0; +} #else int LockHostPages(std::uintptr_t start, std::uintptr_t end) { auto* const address = reinterpret_cast(start); @@ -496,6 +569,14 @@ int LockHostPages(std::uintptr_t start, std::uintptr_t end) { default: throw std::system_error(error, std::generic_category(), "mlock failed"); } } + +int UnlockHostPages(std::uintptr_t start, std::uintptr_t end) { + if (!GuestRangeMapped(start, end)) return SCE_KERNEL_ERROR_ENOMEM; + if (::munlock(reinterpret_cast(start), end - start) == 0) return 0; + const int error = errno; + if (error == ENOMEM) return SCE_KERNEL_ERROR_ENOMEM; + throw std::system_error(error, std::generic_category(), "munlock failed"); +} #endif } @@ -503,13 +584,19 @@ int LockHostPages(std::uintptr_t start, std::uintptr_t end) { extern "C" { int APS5_VABI sceKernelMlock_nid_postfix(void* address, std::uint64_t length) { - constexpr std::uintptr_t pageMask = PS5_PAGE_SIZE - 1; - const auto first = reinterpret_cast(address); - if (length > UINTPTR_MAX - first || first + length > UINTPTR_MAX - pageMask) return SCE_KERNEL_ERROR_EINVAL; - const auto start = first & ~pageMask; - const auto end = (first + length + pageMask) & ~pageMask; + std::uintptr_t start = 0; + std::uintptr_t end = 0; + if (!PageRange(address, length, start, end)) return SCE_KERNEL_ERROR_EINVAL; if (start == end) return 0; return LockHostPages(start, end); } +int APS5_VABI sceKernelMunlock_nid_postfix(void* address, std::uint64_t length) { + std::uintptr_t start = 0; + std::uintptr_t end = 0; + if (!PageRange(address, length, start, end)) return SCE_KERNEL_ERROR_EINVAL; + if (start == end) return 0; + return UnlockHostPages(start, end); +} + } diff --git a/core/libs/prx/libkernel/DirectMemory/PosixMemory.cpp b/core/libs/prx/libkernel/DirectMemory/PosixMemory.cpp index bac216bbc..650734b5e 100644 --- a/core/libs/prx/libkernel/DirectMemory/PosixMemory.cpp +++ b/core/libs/prx/libkernel/DirectMemory/PosixMemory.cpp @@ -1,10 +1,15 @@ #include "DirectMemory.hpp" +#include "SceTypes.hpp" #include "prx/libc/include/general/VabiMacros.hpp" +#include "prx/libc/include/General.hpp" #include #include #include extern "C" int* APS5_VABI __error_nid_postfix(); +extern "C" int APS5_VABI sceKernelMlock_nid_postfix(void* address, std::uint64_t length); +extern "C" int APS5_VABI sceKernelMunlock_nid_postfix(void* address, std::uint64_t length); +extern "C" int APS5_VABI sceKernelVirtualQuery(const void* addr, int flags, VirtualQueryInfo* info, uint64_t info_size); namespace { // FreeBSD/PS5 ABI values, independent of the host's errno and mmap constants. @@ -13,6 +18,13 @@ constexpr int GuestNoMemory = 12; constexpr int GuestNotSupported = 45; constexpr int GuestPrivate = 0x2; constexpr int GuestAnonymous = 0x1000; +constexpr int GuestSyncAsync = 0x1; +constexpr int GuestSyncInvalidate = 0x2; +constexpr int GuestAdviceCore = 9; +constexpr int GuestAdviceProtect = 10; +constexpr std::uintptr_t GuestUserAddressEnd = 0x800000000000; +constexpr int GuestAlignedSuper = 0x1000000; +constexpr std::size_t SuperpageSize = std::size_t{1} << 21; bool RoundLength(std::size_t length, std::size_t& rounded) { constexpr auto mask = PS5_PAGE_SIZE - 1; @@ -25,6 +37,11 @@ bool RoundLength(std::size_t length, std::size_t& rounded) { void SetError(int error) { *__error_nid_postfix() = error; } + +bool Mapped(std::uintptr_t address) { + VirtualQueryInfo info{}; + return sceKernelVirtualQuery(reinterpret_cast(address), 0, &info, sizeof(info)) == 0; +} } extern "C" { @@ -38,7 +55,7 @@ void* APS5_VABI mmap_nid_postfix(void* address, std::size_t length, int protecti std::size_t rounded; if (!RoundLength(length, rounded) || (protection & ~7) != 0) return failed(GuestInvalid); - if (flags != (GuestPrivate | GuestAnonymous)) + if ((flags & ~GuestAlignedSuper) != (GuestPrivate | GuestAnonymous)) return failed(GuestNotSupported); if (descriptor != -1 || offset != 0) return failed(GuestInvalid); @@ -46,7 +63,7 @@ void* APS5_VABI mmap_nid_postfix(void* address, std::size_t length, int protecti (void)address; void* mapped = nullptr; try { - const auto result = DoMapAnon(&mapped, rounded, protection, 0); + const auto result = DoMapAnon(&mapped, rounded, protection, 0, (flags & GuestAlignedSuper) != 0 ? SuperpageSize : PS5_PAGE_SIZE); if (result != 0) return failed(GuestNoMemory); return mapped; } catch (const std::bad_alloc&) { @@ -97,4 +114,45 @@ int APS5_VABI mprotect_nid_postfix(void* address, std::size_t length, int protec } } +int APS5_VABI mlock_nid_postfix(const void* address, std::size_t length) { + const int result = sceKernelMlock_nid_postfix(const_cast(address), length); + if (result == 0) return 0; + SetError(result & 0xffff); + return -1; +} + +int APS5_VABI munlock_nid_postfix(const void* address, std::size_t length) { + const int result = sceKernelMunlock_nid_postfix(const_cast(address), length); + if (result == 0) return 0; + SetError(result & 0xffff); + return -1; +} + +int APS5_VABI msync_nid_postfix(void* address, std::size_t length, int flags) { + const auto failed = [](int error) { + SetError(error); + return -1; + }; + constexpr std::uintptr_t mask = PS5_PAGE_SIZE - 1; + const auto first = reinterpret_cast(address); + const auto start = first & ~mask; + const std::uintptr_t size = (length + (first - start) + mask) & ~mask; + const auto end = start + size; + if (end < start) return failed(GuestInvalid); + if ((flags & (GuestSyncAsync | GuestSyncInvalidate)) == (GuestSyncAsync | GuestSyncInvalidate)) return failed(GuestInvalid); + if (start == end ? !Mapped(start) : !GuestRangeMapped(start, end)) return failed(GuestNoMemory); + if ((flags & GuestSyncInvalidate) != 0) throw std::runtime_error("msync: MS_INVALIDATE is not implemented"); + return 0; +} + +int APS5_VABI madvise_nid_postfix(void* address, std::size_t length, int advice) { + if (advice == GuestAdviceProtect) NotImplemented_nid_no_patch("madvise MADV_PROTECT"); + const auto start = reinterpret_cast(address); + if (advice < 0 || advice > GuestAdviceCore || start > GuestUserAddressEnd || length > GuestUserAddressEnd - start) { + SetError(GuestInvalid); + return -1; + } + return 0; +} + } diff --git a/core/libs/prx/libkernel/File/include/File.hpp b/core/libs/prx/libkernel/File/include/File.hpp index cb3f099ff..5b067ab77 100644 --- a/core/libs/prx/libkernel/File/include/File.hpp +++ b/core/libs/prx/libkernel/File/include/File.hpp @@ -16,6 +16,9 @@ std::int64_t APS5_VABI sceKernelWrite(int d, const void* buf, std::size_t nbytes std::int64_t APS5_VABI sceKernelLseek(int d, std::int64_t offset, int whence); int APS5_VABI sceKernelStat(const char* path, FileStat* sb); int APS5_VABI sceKernelUnlink(const char* path); +#ifdef _WIN32 +std::int64_t NativePositioned_nid_no_patch(int descriptor, void* buf, std::size_t nbytes, std::int64_t offset, bool write); +#endif } diff --git a/core/libs/prx/libkernel/File/include/FileLock.hpp b/core/libs/prx/libkernel/File/include/FileLock.hpp new file mode 100644 index 000000000..0cb00071a --- /dev/null +++ b/core/libs/prx/libkernel/File/include/FileLock.hpp @@ -0,0 +1,11 @@ +#ifndef CORE_LIBS_PRX_LIBKERNEL_FILE_FILELOCK_HPP +#define CORE_LIBS_PRX_LIBKERNEL_FILE_FILELOCK_HPP + +namespace File { + +int Flock(int fd, int operation); +void ForgetFileLock(int fd); + +} + +#endif diff --git a/core/libs/prx/libkernel/File/include/NativeStat.hpp b/core/libs/prx/libkernel/File/include/NativeStat.hpp index f5b8642f5..5715f0211 100644 --- a/core/libs/prx/libkernel/File/include/NativeStat.hpp +++ b/core/libs/prx/libkernel/File/include/NativeStat.hpp @@ -9,6 +9,7 @@ namespace File { void FillFileStat(const std::filesystem::path& nativePath, FileStat* sb); void FillFileStat(int nativeDescriptor, FileStat* sb); bool FillFileStatFromDescriptor(int fd, FileStat* sb); +bool FillLinkStat(const std::filesystem::path& nativePath, FileStat* sb); } diff --git a/core/libs/prx/libkernel/File/include/RandomDevice.hpp b/core/libs/prx/libkernel/File/include/RandomDevice.hpp new file mode 100644 index 000000000..fbe4bd17b --- /dev/null +++ b/core/libs/prx/libkernel/File/include/RandomDevice.hpp @@ -0,0 +1,17 @@ +#ifndef CORE_LIBS_PRX_LIBKERNEL_FILE_RANDOMDEVICE_HPP +#define CORE_LIBS_PRX_LIBKERNEL_FILE_RANDOMDEVICE_HPP + +#include +#include + +namespace File { + +bool IsRandomDevicePath(std::string_view path); +int OpenRandomDevice(); +bool IsRandomDevice(int fd); +bool ReadRandomDevice(int fd, void* buf, std::size_t nbytes); +void ForgetRandomDevice(int fd); + +} + +#endif diff --git a/core/libs/prx/libkernel/File/include/WindowsFileTime.hpp b/core/libs/prx/libkernel/File/include/WindowsFileTime.hpp new file mode 100644 index 000000000..07721951e --- /dev/null +++ b/core/libs/prx/libkernel/File/include/WindowsFileTime.hpp @@ -0,0 +1,54 @@ +#ifndef CORE_LIBS_PRX_LIBKERNEL_FILE_WINDOWSFILETIME_HPP +#define CORE_LIBS_PRX_LIBKERNEL_FILE_WINDOWSFILETIME_HPP + +#include "SceTypes.hpp" +#include +#include +#include +#include + +namespace File { + +class WindowsFileTime { + static constexpr std::int64_t epochSeconds = 11644473600LL; + static constexpr std::int64_t ticksPerSecond = 10000000; + +public: + static KernelTimespec Decode(std::int64_t ticks) { + auto seconds = ticks / ticksPerSecond - epochSeconds; + auto remainder = ticks % ticksPerSecond; + if (remainder < 0) { + --seconds; + remainder += ticksPerSecond; + } + return {seconds, remainder * 100}; + } + + static bool Encode(const KernelTimeval& time, FILETIME& result) { + if (time.tv_usec < 0 || time.tv_usec >= 1000000 || time.tv_sec < -epochSeconds || + time.tv_sec > std::numeric_limits::max() / ticksPerSecond - epochSeconds) return false; + const auto seconds = time.tv_sec + epochSeconds; + const auto fraction = time.tv_usec * 10; + if (seconds > (std::numeric_limits::max() - fraction) / ticksPerSecond) return false; + const auto ticks = static_cast(seconds * ticksPerSecond + fraction); + if (ticks == 0) return false; + result = {static_cast(ticks), static_cast(ticks >> 32)}; + return true; + } + + static int Failure(DWORD error) { + const auto condition = std::system_category().default_error_condition(static_cast(error)); + if (condition == std::errc::no_such_file_or_directory) errno = ENOENT; + else if (condition == std::errc::permission_denied) errno = EACCES; + else if (condition == std::errc::bad_file_descriptor) errno = EBADF; + else if (condition == std::errc::not_enough_memory) errno = ENOMEM; + else if (condition == std::errc::too_many_files_open) errno = EMFILE; + else if (condition == std::errc::invalid_argument) errno = EINVAL; + else errno = EIO; + return -1; + } +}; + +} + +#endif diff --git a/core/libs/prx/libkernel/File/src/FileLock.cpp b/core/libs/prx/libkernel/File/src/FileLock.cpp new file mode 100644 index 000000000..3b2f67122 --- /dev/null +++ b/core/libs/prx/libkernel/File/src/FileLock.cpp @@ -0,0 +1,75 @@ +#include "prx/libkernel/File/include/FileLock.hpp" + +#ifdef _WIN32 + +#include +#include +#include +#include +#include + +namespace { + +constexpr int LockShared = 1; +constexpr int LockExclusive = 2; +constexpr int LockNonblocking = 4; +constexpr int LockUnlock = 8; + +std::mutex g_mutex; +std::map g_locks; + +int HeldLock(int fd) { + std::lock_guard lock(g_mutex); + const auto found = g_locks.find(fd); + return found == g_locks.end() ? 0 : found->second; +} + +void SetHeldLock(int fd, int mode) { + std::lock_guard lock(g_mutex); + if (mode == 0) { + g_locks.erase(fd); + } else { + g_locks[fd] = mode; + } +} + +} + +namespace File { + +int Flock(int fd, int operation) { + const int mode = operation & (LockShared | LockExclusive | LockUnlock); + if (mode != LockShared && mode != LockExclusive && mode != LockUnlock) { + errno = EINVAL; + ::SetLastError(ERROR_INVALID_PARAMETER); + return -1; + } + const auto handle = reinterpret_cast(::_get_osfhandle(fd)); + if (handle == INVALID_HANDLE_VALUE) { + errno = EBADF; + ::SetLastError(ERROR_INVALID_HANDLE); + return -1; + } + const int held = HeldLock(fd); + if (held == mode) return 0; + if (held != 0) { + OVERLAPPED overlapped{}; + if (!::UnlockFileEx(handle, 0, MAXDWORD, MAXDWORD, &overlapped)) return -1; + SetHeldLock(fd, 0); + } + if (mode == LockUnlock) return 0; + DWORD flags = mode == LockExclusive ? LOCKFILE_EXCLUSIVE_LOCK : 0; + if (operation & LockNonblocking) flags |= LOCKFILE_FAIL_IMMEDIATELY; + OVERLAPPED overlapped{}; + if (!::LockFileEx(handle, flags, 0, MAXDWORD, MAXDWORD, &overlapped)) return -1; + SetHeldLock(fd, mode); + return 0; +} + +void ForgetFileLock(int fd) { + SetHeldLock(fd, 0); +} + +} + +#endif diff --git a/core/libs/prx/libkernel/File/src/NativeStat.cpp b/core/libs/prx/libkernel/File/src/NativeStat.cpp index cc1c0af37..1c0ada2a6 100644 --- a/core/libs/prx/libkernel/File/src/NativeStat.cpp +++ b/core/libs/prx/libkernel/File/src/NativeStat.cpp @@ -1,19 +1,52 @@ #include "prx/libkernel/File/include/NativeStat.hpp" #include "prx/libkernel/File/include/DirectoryDescriptor.hpp" +#include "prx/libc/include/General.hpp" +#include #include #include #ifdef _WIN32 +#include "prx/libkernel/File/include/WindowsFileTime.hpp" +#include #include #include -using NativeStat = struct __stat64; +struct NativeStat : __stat64 { + FILE_BASIC_INFO times{}; + BY_HANDLE_FILE_INFORMATION identity{}; + bool hasTimes = false; + bool hasIdentity = false; +}; +static int ReadHandleInfo(HANDLE handle, NativeStat* st) { + if (GetFileType(handle) != FILE_TYPE_DISK) return 0; + if (!GetFileInformationByHandleEx(handle, FileBasicInfo, &st->times, sizeof(st->times))) + return File::WindowsFileTime::Failure(GetLastError()); + st->hasTimes = true; + st->hasIdentity = GetFileInformationByHandle(handle, &st->identity) != 0; + return 0; +} +static std::uint32_t FoldFileIndex(const BY_HANDLE_FILE_INFORMATION& identity) { + const std::uint32_t folded = identity.nFileIndexLow ^ (identity.nFileIndexHigh * 0x9E3779B1u); + return folded == 0 ? 1 : folded; +} static int DoStat(const std::filesystem::path& p, NativeStat* st) { - return _wstat64(p.wstring().c_str(), st); + if (_wstat64(p.c_str(), st) != 0) return -1; + const auto handle = CreateFileW(p.c_str(), FILE_READ_ATTRIBUTES, FILE_SHARE_READ | FILE_SHARE_WRITE | FILE_SHARE_DELETE, + nullptr, OPEN_EXISTING, FILE_FLAG_BACKUP_SEMANTICS, nullptr); + if (handle == INVALID_HANDLE_VALUE) return File::WindowsFileTime::Failure(GetLastError()); + const int result = ReadHandleInfo(handle, st); + CloseHandle(handle); + return result; } static int DoFstat(int fd, NativeStat* st) { if (const auto directory = File::DirectoryDescriptorPath(fd)) return DoStat(*directory, st); - return _fstat64(fd, st); + if (_fstat64(fd, st) != 0) return -1; + return ReadHandleInfo(reinterpret_cast(::_get_osfhandle(fd)), st); +} +static int DoLstat(const std::filesystem::path& p, NativeStat* st) { + std::error_code error; + if (std::filesystem::is_symlink(std::filesystem::symlink_status(p, error))) NotImplemented_nid_no_patch("lstat of a Windows symbolic link"); + return DoStat(p, st); } #else #include @@ -24,6 +57,9 @@ static int DoStat(const std::filesystem::path& p, NativeStat* st) { static int DoFstat(int fd, NativeStat* st) { return ::fstat(fd, st); } +static int DoLstat(const std::filesystem::path& p, NativeStat* st) { + return ::lstat(p.c_str(), st); +} #endif static void CopyNativeStat(const NativeStat& st, FileStat* sb) { @@ -47,6 +83,17 @@ static void CopyNativeStat(const NativeStat& st, FileStat* sb) { sb->st_ctim.tv_nsec = 0; sb->st_birthtim.tv_sec = static_cast(st.st_ctime); sb->st_birthtim.tv_nsec = 0; + if (st.hasTimes) { + sb->st_atim = File::WindowsFileTime::Decode(st.times.LastAccessTime.QuadPart); + sb->st_mtim = File::WindowsFileTime::Decode(st.times.LastWriteTime.QuadPart); + sb->st_ctim = File::WindowsFileTime::Decode(st.times.ChangeTime.QuadPart); + sb->st_birthtim = File::WindowsFileTime::Decode(st.times.CreationTime.QuadPart); + } + if (st.hasIdentity) { + sb->st_dev = st.identity.dwVolumeSerialNumber; + sb->st_ino = FoldFileIndex(st.identity); + sb->st_nlink = static_cast(std::min(st.identity.nNumberOfLinks, 0xffff)); + } #else sb->st_dev = static_cast(st.st_dev); sb->st_ino = static_cast(st.st_ino); @@ -99,4 +146,11 @@ bool FillFileStatFromDescriptor(int fd, FileStat* sb) { return true; } +bool FillLinkStat(const std::filesystem::path& nativePath, FileStat* sb) { + NativeStat st{}; + if (DoLstat(nativePath, &st) != 0) return false; + CopyNativeStat(st, sb); + return true; +} + } diff --git a/core/libs/prx/libkernel/File/src/Open.cpp b/core/libs/prx/libkernel/File/src/Open.cpp index 2e323a8d9..329c2a55d 100644 --- a/core/libs/prx/libkernel/File/src/Open.cpp +++ b/core/libs/prx/libkernel/File/src/Open.cpp @@ -4,6 +4,9 @@ #include "prx/libc/include/GuestArena.hpp" #include "prx/libkernel/File/include/File.hpp" #include "prx/libkernel/File/include/DirectoryDescriptor.hpp" +#include "prx/libkernel/File/include/FileLock.hpp" +#include "prx/libkernel/Socket/include/SocketRuntime.hpp" +#include "prx/libkernel/File/include/RandomDevice.hpp" #include "prx/libkernel/KernelErrors.hpp" #include "SceTypes.hpp" @@ -95,6 +98,8 @@ static int MapFlags(int sceFlags) { } #endif +extern "C" int* APS5_VABI __error_nid_postfix(); + static int SceErrorFromErrno(int error) { constexpr int GuestEio = 5; const int guest = error > 0 && error <= 34 ? error : GuestEio; @@ -105,12 +110,21 @@ extern "C" { int APS5_VABI sceKernelOpen(const char* path, int flags, std::uint16_t mode) { APS5_LOG_OUT("path=%s flags=0x%X nativeFlags=0x%X mode=0%o", path, flags, MapFlags(flags), mode); + if (path != nullptr && File::IsRandomDevicePath(path)) { + if ((flags & SCE_KERNEL_O_ACCMODE) != SCE_KERNEL_O_RDONLY) throw std::runtime_error(std::string(__func__) + ": writing to " + path + " is not implemented"); + const int fd = File::OpenRandomDevice(); + return fd < 0 ? SceErrorFromErrno(errno) : fd; + } auto native = ResolvePath_nid_no_patch(path); int fd = NativeOpen(native, MapFlags(flags), mode); #ifdef _WIN32 if (fd < 0 && errno != ENOENT) { std::error_code error; - if (std::filesystem::is_directory(native, error)) fd = File::OpenDirectoryDescriptor(native); + if (std::filesystem::is_directory(native, error)) { + if ((flags & (SCE_KERNEL_O_CREAT | SCE_KERNEL_O_EXCL)) == (SCE_KERNEL_O_CREAT | SCE_KERNEL_O_EXCL)) errno = EEXIST; + else if ((flags & SCE_KERNEL_O_ACCMODE) != SCE_KERNEL_O_RDONLY || (flags & (SCE_KERNEL_O_CREAT | SCE_KERNEL_O_TRUNC))) errno = EISDIR; + else fd = File::OpenDirectoryDescriptor(native); + } } #endif if (fd < 0) { @@ -122,8 +136,10 @@ int APS5_VABI sceKernelOpen(const char* path, int flags, std::uint16_t mode) { } int APS5_VABI sceKernelClose(int d) { + File::ForgetRandomDevice(d); #ifdef _WIN32 File::ForgetDirectoryDescriptor(d); + File::ForgetFileLock(d); #endif if (NativeClose(d) != 0) { if (errno == EBADF) return SCE_KERNEL_ERROR_EBADF; @@ -133,11 +149,16 @@ int APS5_VABI sceKernelClose(int d) { } std::int64_t APS5_VABI sceKernelRead(int d, void* buf, std::size_t nbytes) { + if (d >= GuestSockets::FirstDescriptor) { + const auto n = GuestSockets::Read(d, buf, nbytes); + return n < 0 ? SceKernelError(*__error_nid_postfix()) : n; + } if (buf == nullptr) { throw std::invalid_argument(std::string(__func__) + ": buf is null"); } const GuestArena::HostWrite destination(buf, nbytes); if (!destination.Open()) errno = EFAULT; + else if (File::ReadRandomDevice(d, buf, nbytes)) return static_cast(nbytes); auto n = destination.Open() ? NativeRead(d, buf, nbytes) : -1; if (n < 0) { throw std::runtime_error(std::string(__func__) + ": read failed, fd=" + std::to_string(d) + ", errno=" + std::to_string(errno)); @@ -146,6 +167,10 @@ std::int64_t APS5_VABI sceKernelRead(int d, void* buf, std::size_t nbytes) { } std::int64_t APS5_VABI sceKernelWrite(int d, const void* buf, std::size_t nbytes) { + if (d >= GuestSockets::FirstDescriptor) { + const auto n = GuestSockets::Write(d, buf, nbytes); + return n < 0 ? SceKernelError(*__error_nid_postfix()) : n; + } if (buf == nullptr) { throw std::invalid_argument(std::string(__func__) + ": buf is null"); } @@ -176,9 +201,14 @@ int APS5_VABI sceKernelStat(const char* path, FileStat* sb) { } const auto native = ResolvePath_nid_no_patch(path); std::error_code error; - if (!std::filesystem::exists(native, error)) { + constexpr int GuestEnotdir = 20; + const auto status = std::filesystem::status(native, error); + if (error == std::errc::not_a_directory) return SceErrorFromErrno(GuestEnotdir); + if (!std::filesystem::exists(status)) { return SceErrorFromErrno(2); } + const std::string_view guestPath(path); + if (!guestPath.empty() && guestPath.back() == '/' && !std::filesystem::is_directory(status)) return SceErrorFromErrno(GuestEnotdir); File::FillFileStat(native, sb); return 0; } @@ -188,6 +218,9 @@ int APS5_VABI sceKernelUnlink(const char* path) { throw std::invalid_argument(std::string(__func__) + ": path is null"); } auto native = ResolvePath_nid_no_patch(path); + constexpr int GuestEperm = 1; + std::error_code error; + if (std::filesystem::is_directory(std::filesystem::symlink_status(native, error))) return SceErrorFromErrno(GuestEperm); if (NativeUnlink(native) != 0) { return SceErrorFromErrno(errno); } diff --git a/core/libs/prx/libkernel/File/src/OpenFileLimit.cpp b/core/libs/prx/libkernel/File/src/OpenFileLimit.cpp new file mode 100644 index 000000000..6d698a585 --- /dev/null +++ b/core/libs/prx/libkernel/File/src/OpenFileLimit.cpp @@ -0,0 +1,21 @@ +#if defined(__linux__) + +#include +#include +#include + +namespace { + +struct OpenFileLimit { + OpenFileLimit() { + rlimit limit{}; + if (getrlimit(RLIMIT_NOFILE, &limit) != 0) throw std::system_error(errno, std::generic_category(), "read open file limit"); + if (limit.rlim_cur >= limit.rlim_max) return; + limit.rlim_cur = limit.rlim_max; + if (setrlimit(RLIMIT_NOFILE, &limit) != 0) throw std::system_error(errno, std::generic_category(), "raise open file limit"); + } +} openFileLimit; + +} + +#endif diff --git a/core/libs/prx/libkernel/File/src/RandomDevice.cpp b/core/libs/prx/libkernel/File/src/RandomDevice.cpp new file mode 100644 index 000000000..3d68811b6 --- /dev/null +++ b/core/libs/prx/libkernel/File/src/RandomDevice.cpp @@ -0,0 +1,64 @@ +#include "prx/libkernel/File/include/RandomDevice.hpp" + +#include +#include +#include +#include +#include + +#ifdef _WIN32 +#include +#include +#else +#include +#include +#endif + +namespace { + +std::mutex g_mutex; +std::set g_descriptors; + +} + +namespace File { + +bool IsRandomDevicePath(std::string_view path) { + return path == "/dev/random" || path == "/dev/urandom"; +} + +int OpenRandomDevice() { +#ifdef _WIN32 + const int fd = ::_open("NUL", _O_RDONLY | _O_BINARY); +#else + const int fd = ::open("/dev/null", O_RDONLY); +#endif + if (fd < 0) return -1; + std::lock_guard lock(g_mutex); + g_descriptors.insert(fd); + return fd; +} + +bool IsRandomDevice(int fd) { + std::lock_guard lock(g_mutex); + return g_descriptors.contains(fd); +} + +bool ReadRandomDevice(int fd, void* buf, std::size_t nbytes) { + std::lock_guard lock(g_mutex); + if (!g_descriptors.contains(fd)) return false; + static std::random_device device; + auto* bytes = static_cast(buf); + for (std::size_t offset = 0; offset < nbytes; offset += sizeof(std::uint32_t)) { + const std::uint32_t value = device(); + std::memcpy(bytes + offset, &value, nbytes - offset < sizeof(value) ? nbytes - offset : sizeof(value)); + } + return true; +} + +void ForgetRandomDevice(int fd) { + std::lock_guard lock(g_mutex); + g_descriptors.erase(fd); +} + +} diff --git a/core/libs/prx/libkernel/File/src/SharedMemory.cpp b/core/libs/prx/libkernel/File/src/SharedMemory.cpp new file mode 100644 index 000000000..94bb3ee39 --- /dev/null +++ b/core/libs/prx/libkernel/File/src/SharedMemory.cpp @@ -0,0 +1,88 @@ +#include "prx/libc/include/General.hpp" +#include "prx/libkernel/File/include/FileFlags.hpp" + +#include +#include +#include + +#ifdef _WIN32 +#include +#include +#include +#include +#include +#include +#include +#else +#include +#endif + +extern "C" int* APS5_VABI __error_nid_postfix(); + +namespace { + +constexpr int GuestNoMemory = 12; +constexpr int GuestInvalid = 22; +constexpr int GuestTooManyFilesInSystem = 23; +constexpr int GuestTooManyFiles = 24; +constexpr int GuestCloseOnExec = 0x00100000; +constexpr int AcceptedFlags = SCE_KERNEL_O_ACCMODE | SCE_KERNEL_O_CREAT | SCE_KERNEL_O_EXCL | SCE_KERNEL_O_TRUNC | GuestCloseOnExec; + +const char* const anonymousObject = reinterpret_cast(1); + +int Fail(int error) { + *__error_nid_postfix() = error; + return -1; +} + +int GuestError(int error) { + switch (error) { + case EMFILE: + return GuestTooManyFiles; + case ENFILE: + return GuestTooManyFilesInSystem; + case ENOMEM: + return GuestNoMemory; + default: + return 0; + } +} + +int CreateAnonymousFile() { +#ifdef _WIN32 + static std::atomic sequence{0}; + const auto directory = std::filesystem::temp_directory_path(); + const auto seed = std::to_string(std::random_device{}()); + for (int attempt = 0; attempt < 64; ++attempt) { + const auto path = directory / ("anyps5-shm-" + seed + "-" + std::to_string(sequence++)); + int descriptor = -1; + const int error = _wsopen_s(&descriptor, path.c_str(), + _O_CREAT | _O_EXCL | _O_RDWR | _O_BINARY | _O_TEMPORARY | _O_SHORT_LIVED | _O_NOINHERIT, + _SH_DENYNO, _S_IREAD | _S_IWRITE); + if (error == 0) return descriptor; + if (error != EEXIST) { + errno = error; + return -1; + } + } + errno = EEXIST; + return -1; +#else + return memfd_create("anyps5 shm_open", MFD_CLOEXEC); +#endif +} + +} + +extern "C" int APS5_VABI shm_open_nid_postfix(const char* path, int flags, int) { + const int accessMode = flags & SCE_KERNEL_O_ACCMODE; + if (accessMode != SCE_KERNEL_O_RDONLY && accessMode != SCE_KERNEL_O_RDWR) return Fail(GuestInvalid); + if ((flags & ~AcceptedFlags) != 0) return Fail(GuestInvalid); + if (path != anonymousObject) NotImplemented_nid_no_patch("shm_open of a named object"); + if (accessMode == SCE_KERNEL_O_RDONLY) return Fail(GuestInvalid); + const int descriptor = CreateAnonymousFile(); + if (descriptor >= 0) return descriptor; + const int error = errno; + if (const int guest = GuestError(error)) return Fail(guest); + throw std::runtime_error("shm_open: creating the host anonymous file failed, errno=" + std::to_string(error)); +} diff --git a/core/libs/prx/libkernel/File/src/Stdio.cpp b/core/libs/prx/libkernel/File/src/Stdio.cpp index 109f729e7..711b9f388 100644 --- a/core/libs/prx/libkernel/File/src/Stdio.cpp +++ b/core/libs/prx/libkernel/File/src/Stdio.cpp @@ -9,6 +9,8 @@ #include "prx/libkernel/File/include/FileFlags.hpp" #include "prx/libkernel/File/include/NativeStat.hpp" #include "prx/libkernel/File/include/DirectoryDescriptor.hpp" +#include "prx/libkernel/File/include/FileLock.hpp" +#include "prx/libkernel/File/include/RandomDevice.hpp" #include "prx/libkernel/KernelErrors.hpp" #include "prx/libkernel/Socket/include/SocketRuntime.hpp" #include @@ -34,7 +36,7 @@ static constexpr int KERNEL_IOV_MAX = 1024; #include #include #include -#include +#include "prx/libkernel/File/include/WindowsFileTime.hpp" static int NativeRmdir(const std::filesystem::path& path) { return ::_wrmdir(path.wstring().c_str()); } @@ -73,31 +75,44 @@ static int NativeFchmod(int descriptor, int mode) { static int NativeFtruncate(int descriptor, std::int64_t length) { return static_cast(::_chsize_s(descriptor, length)); } -static int NativeUtimes(const std::filesystem::path& path, const KernelTimeval* times) { - RecordWrittenPath_nid_no_patch(path); - if (times == nullptr) return ::_wutime(path.wstring().c_str(), nullptr); - struct _utimbuf values{static_cast(times[0].tv_sec), static_cast(times[1].tv_sec)}; - return ::_wutime(path.wstring().c_str(), &values); -} -static int NativeFutimes(int descriptor, const KernelTimeval* times) { - const auto path = NativeDescriptorPath(descriptor); - return path ? NativeUtimes(*path, times) : -1; -} -static int NativeFlock(int descriptor, int operation) { - HANDLE handle = reinterpret_cast(::_get_osfhandle(descriptor)); - if (handle == INVALID_HANDLE_VALUE) { +static int SetTimes(HANDLE handle, const KernelTimeval* times) { + FILETIME access{}, modified{}; + if (times == nullptr) { + GetSystemTimePreciseAsFileTime(&access); + modified = access; + } else if (!File::WindowsFileTime::Encode(times[0], access) || !File::WindowsFileTime::Encode(times[1], modified)) { + errno = EINVAL; return -1; } - OVERLAPPED overlapped{}; - if (operation & 8) { - return ::UnlockFileEx(handle, 0, MAXDWORD, MAXDWORD, &overlapped) ? 0 : -1; - } - DWORD flags = 0; - if (operation & 2) flags |= LOCKFILE_EXCLUSIVE_LOCK; - if (operation & 4) flags |= LOCKFILE_FAIL_IMMEDIATELY; - return ::LockFileEx(handle, flags, 0, MAXDWORD, MAXDWORD, &overlapped) ? 0 : -1; + return SetFileTime(handle, nullptr, &access, &modified) ? 0 : File::WindowsFileTime::Failure(GetLastError()); } -static std::int64_t NativePositioned(int descriptor, void* buf, std::size_t nbytes, std::int64_t offset, bool write) { +static int NativeUtimes(const std::filesystem::path& path, const KernelTimeval* times) { + const auto handle = CreateFileW(path.c_str(), FILE_WRITE_ATTRIBUTES, FILE_SHARE_READ | FILE_SHARE_WRITE | FILE_SHARE_DELETE, + nullptr, OPEN_EXISTING, FILE_FLAG_BACKUP_SEMANTICS, nullptr); + if (handle == INVALID_HANDLE_VALUE) return File::WindowsFileTime::Failure(GetLastError()); + const int result = SetTimes(handle, times); + CloseHandle(handle); + if (result == 0) RecordWrittenPath_nid_no_patch(path); + return result; +} +static int NativeFutimes(int descriptor, const KernelTimeval* times) { + if (const auto directory = File::DirectoryDescriptorPath(descriptor)) return NativeUtimes(*directory, times); + const auto original = reinterpret_cast(::_get_osfhandle(descriptor)); + if (original == INVALID_HANDLE_VALUE) { errno = EBADF; return -1; } + if (GetFileType(original) != FILE_TYPE_DISK) { errno = EINVAL; return -1; } + const auto handle = ReOpenFile(original, FILE_WRITE_ATTRIBUTES, FILE_SHARE_READ | FILE_SHARE_WRITE | FILE_SHARE_DELETE, 0); + if (handle == INVALID_HANDLE_VALUE) return File::WindowsFileTime::Failure(GetLastError()); + const int result = SetTimes(handle, times); + CloseHandle(handle); + if (result == 0) { + if (const auto path = NativeDescriptorPath(descriptor)) RecordWrittenPath_nid_no_patch(*path); + } + return result; +} +static int NativeFlock(int descriptor, int operation) { + return File::Flock(descriptor, operation); +} +std::int64_t NativePositioned_nid_no_patch(int descriptor, void* buf, std::size_t nbytes, std::int64_t offset, bool write) { if (nbytes > static_cast(std::numeric_limits::max())) { throw std::runtime_error("NativePositioned: nbytes exceeds platform limit"); } @@ -123,7 +138,7 @@ static std::int64_t NativePositioned(int descriptor, void* buf, std::size_t nbyt } if (!ok) { if (!write && error == ERROR_HANDLE_EOF) return 0; - errno = EIO; + errno = error == ERROR_ACCESS_DENIED ? EBADF : EIO; return -1; } return done; @@ -139,10 +154,10 @@ static bool NativeIsDisk(int descriptor) { return handle != INVALID_HANDLE_VALUE && ::GetFileType(handle) == FILE_TYPE_DISK; } static std::int64_t NativePread(int descriptor, void* buf, std::size_t nbytes, std::int64_t offset) { - return NativePositioned(descriptor, buf, nbytes, offset, false); + return NativePositioned_nid_no_patch(descriptor, buf, nbytes, offset, false); } static std::int64_t NativePwrite(int descriptor, const void* buf, std::size_t nbytes, std::int64_t offset) { - return NativePositioned(descriptor, const_cast(buf), nbytes, offset, true); + return NativePositioned_nid_no_patch(descriptor, const_cast(buf), nbytes, offset, true); } #else #include @@ -204,6 +219,7 @@ static constexpr int GUEST_EEXIST = 17; static constexpr int GUEST_EINVAL = 22; static constexpr int GUEST_ENAMETOOLONG = 63; static constexpr int GUEST_ENOTDIR = 20; +static constexpr int GUEST_EWOULDBLOCK = 35; static constexpr int GUEST_ENOTEMPTY = 66; static int SceErrorFromErrno(int error) { @@ -256,8 +272,10 @@ int APS5_VABI chmod_nid_postfix(const char* path, int mode) { int APS5_VABI close_nid_postfix(int d) { if (d >= GuestSockets::FirstDescriptor) return GuestSockets::Close(d); + File::ForgetRandomDevice(d); #ifdef _WIN32 File::ForgetDirectoryDescriptor(d); + File::ForgetFileLock(d); return _close(d); #else return ::close(d); @@ -269,11 +287,17 @@ int APS5_VABI _close_nid_postfix(int descriptor) { } int APS5_VABI flock_nid_postfix(int d, int operation) { - if (NativeFlock(d, operation) != 0) { + const int type = operation & 8 ? 8 : operation & 2 ? 2 : operation & 1 ? 1 : 0; + if (type == 0) return PosixFailure(GUEST_EBADF); + if (NativeFlock(d, type | (operation & 4)) != 0) { #ifdef _WIN32 - throw std::runtime_error(std::string(__func__) + ": flock failed, fd=" + std::to_string(d) + ", error=" + std::to_string(::GetLastError())); + const auto error = ::GetLastError(); + if ((operation & 4) && error == ERROR_LOCK_VIOLATION) return PosixFailure(GUEST_EWOULDBLOCK); + throw std::runtime_error(std::string(__func__) + ": flock failed, fd=" + std::to_string(d) + ", error=" + std::to_string(error)); #else - throw std::runtime_error(std::string(__func__) + ": flock failed, fd=" + std::to_string(d) + ", errno=" + std::to_string(errno)); + const int error = errno; + if ((operation & 4) && error == EWOULDBLOCK) return PosixFailure(GUEST_EWOULDBLOCK); + throw std::runtime_error(std::string(__func__) + ": flock failed, fd=" + std::to_string(d) + ", errno=" + std::to_string(error)); #endif } return 0; @@ -289,23 +313,24 @@ int64_t APS5_VABI fstat_nid_disambig1_nid_postfix(int d, FileStat* sb) { int APS5_VABI ftruncate_nid_postfix(int d, int64_t length) { if (length < 0) { - APS5_INVALID_ARG_EX; + return PosixFailure(GUEST_EINVAL); } #ifdef _WIN32 int error = NativeFtruncate(d, length); if (error != 0) { - throw std::runtime_error(std::string(__func__) + ": ftruncate failed, fd=" + std::to_string(d) + ", error=" + std::to_string(error)); + if (error == EACCES) error = GUEST_EINVAL; + return PosixFailure(SceErrorFromErrno(error) & 0xffff); } #else if (NativeFtruncate(d, length) != 0) { - throw std::runtime_error(std::string(__func__) + ": ftruncate failed, fd=" + std::to_string(d) + ", errno=" + std::to_string(errno)); + return PosixFailure(SceErrorFromErrno(errno) & 0xffff); } #endif return 0; } int APS5_VABI sceKernelFtruncate(int d, int64_t length) { - return ftruncate_nid_postfix(d, length); + return ftruncate_nid_postfix(d, length) == 0 ? 0 : SceErrorFromErrno(*__error_nid_postfix()); } int64_t APS5_VABI lseek_nid_postfix(int d, int64_t offset, int whence) { @@ -354,6 +379,7 @@ int64_t APS5_VABI pread_nid_postfix(int d, void* buf, size_t nbytes, int64_t off } const GuestArena::HostWrite destination(buf, nbytes); if (!destination.Open()) errno = EFAULT; + else if (File::ReadRandomDevice(d, buf, nbytes)) return static_cast(nbytes); auto n = destination.Open() ? NativePread(d, buf, nbytes, offset) : -1; if (n < 0) { throw std::runtime_error(std::string(__func__) + ": pread failed, fd=" + std::to_string(d) + ", errno=" + std::to_string(errno)); @@ -376,10 +402,12 @@ int64_t APS5_VABI pwrite_nid_disambig1_nid_postfix(int d, const void* buf, size_ } int64_t APS5_VABI read_nid_postfix(int d, void* buf, uint64_t nbytes) { + if (d >= GuestSockets::FirstDescriptor) return GuestSockets::Read(d, buf, static_cast(nbytes)); return sceKernelRead(d, buf, static_cast(nbytes)); } std::int64_t APS5_VABI _read_nid_postfix(int descriptor, void* buffer, std::size_t count) { + if (descriptor >= GuestSockets::FirstDescriptor) return GuestSockets::Read(descriptor, buffer, count); return sceKernelRead(descriptor, buffer, count); } @@ -387,10 +415,12 @@ int64_t APS5_VABI write_nid_postfix(int d, const char* str, int64_t size) { if (size < 0) { APS5_INVALID_ARG_EX; } + if (d >= GuestSockets::FirstDescriptor) return GuestSockets::Write(d, str, static_cast(size)); return sceKernelWrite(d, str, static_cast(size)); } std::int64_t APS5_VABI _write_nid_postfix(int descriptor, const void* buffer, std::size_t count) { + if (descriptor >= GuestSockets::FirstDescriptor) return GuestSockets::Write(descriptor, buffer, count); return sceKernelWrite(descriptor, buffer, count); } @@ -400,6 +430,13 @@ int APS5_VABI stat_nid_postfix(const char* path, FileStat* sb) { return PosixResult(sceKernelStat(path, sb)); } +int APS5_VABI lstat_nid_postfix(const char* path, FileStat* sb) { + if (sb == nullptr) return PosixFailure(GUEST_EFAULT); + if (const int error = PathError(path)) return PosixFailure(error); + if (!File::FillLinkStat(ResolvePath_nid_no_patch(path), sb)) return PosixResult(SceErrorFromErrno(errno)); + return 0; +} + int APS5_VABI unlink_nid_postfix(const char* path) { if (const int error = PathError(path)) return PosixFailure(error); return PosixResult(sceKernelUnlink(path)); @@ -496,6 +533,14 @@ int APS5_VABI sceKernelGetdents(int fd, char* buf, int nbytes) { #endif +int APS5_VABI getdirentries_nid_postfix(int fd, char* buf, int nbytes, int64_t* basep) { + return PosixResult(sceKernelGetdirentries(fd, buf, nbytes, basep)); +} + +int APS5_VABI getdents_nid_postfix(int fd, char* buf, int nbytes) { + return PosixResult(sceKernelGetdents(fd, buf, nbytes)); +} + int APS5_VABI sceKernelMkdir(const char* path, uint16_t mode) { (void)mode; if (path == nullptr) throw std::invalid_argument("sceKernelMkdir: path is null"); @@ -514,6 +559,15 @@ static int CheckIovecs(const KernelIovec* iov, int iovcnt) { return 0; } +static std::int64_t ReadRandomIovecs(int d, const KernelIovec* iov, int iovcnt) { + std::int64_t total = 0; + for (int i = 0; i < iovcnt; ++i) { + File::ReadRandomDevice(d, iov[i].base, iov[i].length); + total += static_cast(iov[i].length); + } + return total; +} + static bool OpenIovecs(const KernelIovec* iov, int iovcnt, std::deque& destinations) { for (int i = 0; i < iovcnt; ++i) { if (!destinations.emplace_back(iov[i].base, iov[i].length).Open()) return false; @@ -524,11 +578,22 @@ static bool OpenIovecs(const KernelIovec* iov, int iovcnt, std::deque(nbytes); + char empty = 0; + const auto result = NativePread(d, nbytes == 0 && buf == nullptr ? &empty : buf, nbytes, offset); + return result < 0 ? SceErrorFromErrno(errno) : result; } int64_t APS5_VABI sceKernelPwrite(int d, const void* buf, size_t nbytes, int64_t offset) { - return pwrite_nid_disambig1_nid_postfix(d, buf, nbytes, offset); + if (buf == nullptr && nbytes != 0) return SceErrorFromErrno(GUEST_EFAULT); + if (offset < 0) return SceErrorFromErrno(GUEST_EINVAL); + char empty = 0; + const auto result = NativePwrite(d, nbytes == 0 && buf == nullptr ? &empty : buf, nbytes, offset); + return result < 0 ? SceErrorFromErrno(errno) : result; } static std::int64_t TransferIovecs(int d, const KernelIovec* iov, int iovcnt, const std::int64_t* offset, bool write) { @@ -543,9 +608,10 @@ static std::int64_t TransferIovecs(int d, const KernelIovec* iov, int iovcnt, co if (offset != nullptr && total > static_cast(std::numeric_limits::max() - *offset)) return SceErrorFromErrno(GUEST_EINVAL); std::deque destinations; if (!write && !OpenIovecs(iov, iovcnt, destinations)) return SceErrorFromErrno(GUEST_EFAULT); + if (!write && File::IsRandomDevice(d)) return ReadRandomIovecs(d, iov, iovcnt); if (total == 0) { char none = 0; - const auto result = offset != nullptr ? NativePositioned(d, &none, 0, *offset, write) : NativeTransfer(d, &none, 0, write); + const auto result = offset != nullptr ? NativePositioned_nid_no_patch(d, &none, 0, *offset, write) : NativeTransfer(d, &none, 0, write); return result < 0 ? SceErrorFromErrno(errno) : 0; } const bool whole = write || offset != nullptr || NativeIsDisk(d); @@ -554,7 +620,7 @@ static std::int64_t TransferIovecs(int d, const KernelIovec* iov, int iovcnt, co auto* base = static_cast(iov[i].base); for (std::size_t position = 0; position < iov[i].length;) { const auto chunk = std::min(iov[i].length - position, std::numeric_limits::max()); - const auto result = offset != nullptr ? NativePositioned(d, base + position, chunk, *offset + done, write) : NativeTransfer(d, base + position, chunk, write); + const auto result = offset != nullptr ? NativePositioned_nid_no_patch(d, base + position, chunk, *offset + done, write) : NativeTransfer(d, base + position, chunk, write); if (result < 0) return done != 0 ? done : SceErrorFromErrno(errno); done += result; position += static_cast(result); @@ -587,6 +653,7 @@ int64_t APS5_VABI sceKernelPread(int d, void* buf, size_t nbytes, int64_t offset if (offset < 0) return SceErrorFromErrno(GUEST_EINVAL); const GuestArena::HostWrite destination(buf, nbytes); if (!destination.Open()) return SceErrorFromErrno(GUEST_EFAULT); + if (File::ReadRandomDevice(d, buf, nbytes)) return static_cast(nbytes); const auto result = NativePread(d, buf, nbytes, offset); return result < 0 ? SceErrorFromErrno(errno) : result; } @@ -602,6 +669,7 @@ int64_t APS5_VABI sceKernelReadv(int d, const KernelIovec* iov, int iovcnt) { if (const int error = CheckIovecs(iov, iovcnt)) return error; std::deque destinations; if (!OpenIovecs(iov, iovcnt, destinations)) return SceErrorFromErrno(GUEST_EFAULT); + if (File::IsRandomDevice(d)) return ReadRandomIovecs(d, iov, iovcnt); const auto result = static_cast(::readv(d, NativeIovecs(iov), iovcnt)); return result < 0 ? SceErrorFromErrno(errno) : result; } @@ -617,6 +685,7 @@ int64_t APS5_VABI sceKernelPreadv(int d, const KernelIovec* iov, int iovcnt, int if (offset < 0) return SceErrorFromErrno(GUEST_EINVAL); std::deque destinations; if (!OpenIovecs(iov, iovcnt, destinations)) return SceErrorFromErrno(GUEST_EFAULT); + if (File::IsRandomDevice(d)) return ReadRandomIovecs(d, iov, iovcnt); const auto result = static_cast(::preadv(d, NativeIovecs(iov), iovcnt, static_cast(offset))); return result < 0 ? SceErrorFromErrno(errno) : result; } @@ -691,6 +760,11 @@ int APS5_VABI sceKernelTruncate_nid_postfix(const char* path, std::int64_t lengt int APS5_VABI sceKernelUtimes_nid_postfix(const char* path, const KernelTimeval* times) { if (path == nullptr) throw std::invalid_argument("sceKernelUtimes: path is null"); + if (times != nullptr) { + for (int i = 0; i < 2; ++i) { + if (times[i].tv_usec < 0 || times[i].tv_usec >= 1000000) return SCE_KERNEL_ERROR_EINVAL; + } + } const auto native = ResolvePath_nid_no_patch(path); if (NativeUtimes(native, times) != 0) return SceErrorFromErrno(errno); return 0; @@ -717,4 +791,14 @@ int APS5_VABI fsync_nid_postfix(int fd) { return 0; } +int APS5_VABI fdatasync_nid_postfix(int fd) { + if (fd >= GuestSockets::FirstDescriptor) return PosixFailure(GuestSockets::IsOpen(fd) ? GUEST_EINVAL : GUEST_EBADF); +#ifdef _WIN32 + if (::_commit(fd) != 0) return PosixResult(SceErrorFromErrno(errno)); +#else + if (::fdatasync(fd) != 0) return PosixResult(SceErrorFromErrno(errno)); +#endif + return 0; +} + } diff --git a/core/libs/prx/libkernel/File/src/Terminal.cpp b/core/libs/prx/libkernel/File/src/Terminal.cpp new file mode 100644 index 000000000..f013a865f --- /dev/null +++ b/core/libs/prx/libkernel/File/src/Terminal.cpp @@ -0,0 +1,72 @@ +#include "prx/libc/include/general/VabiMacros.hpp" +#include "prx/libkernel/Socket/include/SocketRuntime.hpp" +#include +#include +#ifdef _WIN32 +#include +#include +#else +#include +#include +#endif + +extern "C" int* APS5_VABI __error_nid_postfix(); +#ifdef _WIN32 +extern "C" _invalid_parameter_handler _set_thread_local_invalid_parameter_handler(_invalid_parameter_handler); +#endif + +namespace { +constexpr int BadDescriptor = 9; +constexpr int BadAddress = 14; +constexpr int InvalidArgument = 22; +constexpr int NotTerminal = 25; +constexpr int OperationNotSupported = 45; +constexpr int AddressNotAvailable = 49; +constexpr int SetNow = 0; +constexpr int SetDrain = 1; +constexpr int SetFlush = 2; +constexpr int SetSoft = 0x10; +constexpr int InternetFamily = 2; + +int Fail(int error) { + *__error_nid_postfix() = error; + return -1; +} + +#ifdef _WIN32 +void IgnoreInvalidParameter(const wchar_t*, const wchar_t*, const wchar_t*, unsigned int, std::uintptr_t) {} +bool HostOpen(int descriptor) { + const auto previous = _set_thread_local_invalid_parameter_handler(IgnoreInvalidParameter); + const auto handle = ::_get_osfhandle(descriptor); + _set_thread_local_invalid_parameter_handler(previous); + return reinterpret_cast(handle) != INVALID_HANDLE_VALUE; +} +#else +bool HostOpen(int descriptor) { return ::fcntl(descriptor, F_GETFD) != -1; } +#endif + +int TerminalError(int descriptor) { + if (descriptor >= GuestSockets::FirstDescriptor) { + const int family = GuestSockets::Family(descriptor); + if (family < 0) return BadDescriptor; + return family == InternetFamily ? AddressNotAvailable : OperationNotSupported; + } + if (descriptor < 0 || !HostOpen(descriptor)) return BadDescriptor; + return NotTerminal; +} +} + +extern "C" { +int APS5_VABI tcgetattr_nid_postfix(int descriptor, void* attributes) { + static_cast(attributes); + return Fail(TerminalError(descriptor)); +} + +int APS5_VABI tcsetattr_nid_postfix(int descriptor, int action, const void* attributes) { + if ((action & SetSoft) != 0 && attributes == nullptr) throw std::invalid_argument("tcsetattr: attributes is null"); + const int mode = action & ~SetSoft; + if (mode != SetNow && mode != SetDrain && mode != SetFlush) return Fail(InvalidArgument); + if (attributes == nullptr) return Fail(BadAddress); + return Fail(TerminalError(descriptor)); +} +} diff --git a/core/libs/prx/libkernel/Module/EhFrame.hpp b/core/libs/prx/libkernel/Module/EhFrame.hpp new file mode 100644 index 000000000..e6a852125 --- /dev/null +++ b/core/libs/prx/libkernel/Module/EhFrame.hpp @@ -0,0 +1,117 @@ +#ifndef CORE_LIBS_PRX_LIBKERNEL_MODULE_EHFRAME_HPP +#define CORE_LIBS_PRX_LIBKERNEL_MODULE_EHFRAME_HPP + +#include +#include +#include +#include +#include +#ifdef _WIN32 +#ifndef NOMINMAX +#define NOMINMAX +#endif +#include +#endif + +namespace EhFrame { + +struct Tables { + std::uintptr_t header = 0; + std::uint64_t headerSize = 0; + std::uintptr_t frames = 0; + std::uint64_t framesSize = 0; +}; + +[[noreturn]] inline void Fail(const char* caller, const char* reason) { + throw std::runtime_error(std::string(caller) + ": " + reason); +} + +inline std::uint64_t EncodedSize(std::uint8_t encoding, const char* caller) { + switch (encoding & 0x0fu) { + case 0x00: case 0x04: case 0x0c: return 8; + case 0x02: case 0x0a: return 2; + case 0x03: case 0x0b: return 4; + default: Fail(caller, "unsupported eh_frame_hdr value format"); + } +} + +template +std::uintptr_t EncodedValue(const TContains& contains, std::uintptr_t field, std::uint8_t encoding, const char* caller) { + if ((encoding & 0x80u) != 0) Fail(caller, "unsupported indirect eh_frame_hdr encoding"); + const auto application = encoding & 0x70u; + if (application != 0x00 && application != 0x10) Fail(caller, "eh_frame_hdr encoding other than absptr or pcrel"); + const auto size = EncodedSize(encoding, caller); + if (!contains(field, size)) Fail(caller, "eh_frame_hdr outside the image"); + const auto* bytes = reinterpret_cast(field); + std::int64_t value = 0; + switch (encoding & 0x0fu) { + case 0x02: { std::uint16_t data; std::memcpy(&data, bytes, sizeof(data)); value = data; break; } + case 0x0a: { std::int16_t data; std::memcpy(&data, bytes, sizeof(data)); value = data; break; } + case 0x03: { std::uint32_t data; std::memcpy(&data, bytes, sizeof(data)); value = data; break; } + case 0x0b: { std::int32_t data; std::memcpy(&data, bytes, sizeof(data)); value = data; break; } + default: std::memcpy(&value, bytes, sizeof(value)); break; + } + return application == 0x10 ? field + static_cast(value) : static_cast(value); +} + +template +std::uint64_t HeaderSize(const TContains& contains, std::uintptr_t header, const char* caller) { + const auto* bytes = reinterpret_cast(header); + std::uint64_t size = 4 + EncodedSize(bytes[1], caller); + if (bytes[2] == 0xff) return size; + const auto count = static_cast(EncodedValue(contains, header + size, bytes[2] & 0x0fu, caller)); + size += EncodedSize(bytes[2], caller); + if (bytes[3] == 0xff) return size; + const auto entry = 2 * EncodedSize(bytes[3], caller); + if (count > (std::numeric_limits::max() - size) / entry || !contains(header, size + count * entry)) + Fail(caller, "eh_frame_hdr table outside the image"); + return size + count * entry; +} + +template +std::uint64_t FramesSize(const TContains& contains, std::uintptr_t frames, const char* caller) { + std::uintptr_t position = frames; + for (;;) { + if (!contains(position, 4)) Fail(caller, "eh_frame has no terminator inside the image"); + std::uint32_t length; + std::memcpy(&length, reinterpret_cast(position), sizeof(length)); + if (length == 0) return position - frames; + if (length == 0xffffffffu) Fail(caller, "unsupported eh_frame record with a 64-bit length"); + if (!contains(position + 4, length)) Fail(caller, "eh_frame record outside the image"); + position += 4 + length; + } +} + +template +Tables ReadTables(const TContains& contains, std::uintptr_t header, const char* caller) { + if (!contains(header, 4)) Fail(caller, "eh_frame_hdr outside the image"); + if (reinterpret_cast(header)[0] != 1) Fail(caller, "unsupported eh_frame_hdr version"); + Tables tables; + tables.header = header; + tables.frames = EncodedValue(contains, header + 4, reinterpret_cast(header)[1], caller); + tables.headerSize = HeaderSize(contains, header, caller); + tables.framesSize = FramesSize(contains, tables.frames, caller); + return tables; +} + +#ifdef _WIN32 +inline bool ReadEhmeta(std::uintptr_t base, const IMAGE_NT_HEADERS64& nt, const char* caller, Tables& tables) { + const std::uint64_t imageSize = nt.OptionalHeader.SizeOfImage; + const auto contains = [&](std::uintptr_t begin, std::uint64_t size) { return begin >= base && begin - base <= imageSize && size <= imageSize - (begin - base); }; + const auto* sections = IMAGE_FIRST_SECTION(&nt); + for (unsigned i = 0; i < nt.FileHeader.NumberOfSections; ++i) { + if (std::strncmp(reinterpret_cast(sections[i].Name), ".ehmeta", IMAGE_SIZEOF_SHORT_NAME) != 0) continue; + const auto start = base + sections[i].VirtualAddress; + if (!contains(start, 4)) Fail(caller, ".ehmeta outside the image"); + std::uint32_t headerRva; + std::memcpy(&headerRva, reinterpret_cast(start), sizeof(headerRva)); + tables = ReadTables(contains, base + headerRva, caller); + return true; + } + return false; +} +#endif + +} // namespace EhFrame + +#endif diff --git a/core/libs/prx/libkernel/Module/src/DynamicLoader.cpp b/core/libs/prx/libkernel/Module/src/DynamicLoader.cpp index 509194a50..ffe065082 100644 --- a/core/libs/prx/libkernel/Module/src/DynamicLoader.cpp +++ b/core/libs/prx/libkernel/Module/src/DynamicLoader.cpp @@ -1,4 +1,5 @@ #include "prx/libc/include/General.hpp" +#include "prx/libc/include/GuestAllocations.hpp" #include #include #include @@ -171,12 +172,32 @@ void* APS5_VABI dlsym_nid_postfix(void* handle, const char* name) { } catch (const std::exception& error) { Error(error.what()); return nullptr; } } int APS5_VABI dlclose_nid_postfix(void* handle) { + const auto key = reinterpret_cast(handle); std::shared_ptr module; +#ifdef _WIN32 + bool lastReference = true; +#endif { std::lock_guard lock(modulesMutex); - auto found = modules.find(reinterpret_cast(handle)); + auto found = modules.find(key); + if (found == modules.end()) { Error("dlclose: invalid module handle"); return -1; } + module = found->second; +#ifdef _WIN32 + for (const auto& [other, entry] : modules) { + if (other != key && entry->native == module->native) lastReference = false; + } +#endif + } +#ifdef _WIN32 + if (module->owned && module->native && lastReference) { + GuestAllocations::Mutation mutation; + mutation.UnregisterImage(module->native); + } +#endif + { + std::lock_guard lock(modulesMutex); + auto found = modules.find(key); if (found == modules.end()) { Error("dlclose: invalid module handle"); return -1; } - module = std::move(found->second); modules.erase(found); } // Unload outside the registry lock: module destructors may call loader APIs. diff --git a/core/libs/prx/libkernel/Module/src/Module.cpp b/core/libs/prx/libkernel/Module/src/Module.cpp index 466329d8c..76dd6d63e 100644 --- a/core/libs/prx/libkernel/Module/src/Module.cpp +++ b/core/libs/prx/libkernel/Module/src/Module.cpp @@ -8,6 +8,7 @@ #include "prx/libc/include/specifics/linux/ElfTypes.hpp" #include "prx/libkernel/DirectMemory/DirectMemory.hpp" #include "prx/libkernel/KernelErrors.hpp" +#include "prx/libkernel/Module/EhFrame.hpp" #ifdef _WIN32 #include #include @@ -17,57 +18,62 @@ #include #else #include +#include #endif #ifdef _WIN32 namespace { -std::uint64_t ReadEncoded(const std::uint8_t*& p, std::uint8_t encoding) { - const auto application = encoding & 0x70; - if ((encoding & 0x80) != 0 || (application != 0x00 && application != 0x10)) NotImplemented_nid_no_patch("sceKernelGetModuleInfoForUnwind eh_frame_ptr encoding other than absptr or pcrel"); - std::uint64_t value = 0; - const auto* at = p; - switch (encoding & 0x0f) { - case 0x03: { std::uint32_t v; std::memcpy(&v, p, 4); value = v; p += 4; break; } - case 0x0b: { std::int32_t v; std::memcpy(&v, p, 4); value = static_cast(static_cast(v)); p += 4; break; } - case 0x04: case 0x0c: std::memcpy(&value, p, 8); p += 8; break; - default: NotImplemented_nid_no_patch("sceKernelGetModuleInfoForUnwind eh_frame_ptr value format"); - } - if (application == 0x10) value += reinterpret_cast(at); - return value; -} - void FillGuestUnwindInfo(const std::uint8_t* base, ModuleInfoForUnwind* info) { const auto* dos = reinterpret_cast(base); if (dos->e_magic != IMAGE_DOS_SIGNATURE) throw std::runtime_error("sceKernelGetModuleInfoForUnwind: image without a DOS header"); const auto* nt = reinterpret_cast(base + dos->e_lfanew); if (nt->Signature != IMAGE_NT_SIGNATURE) throw std::runtime_error("sceKernelGetModuleInfoForUnwind: image without an NT header"); - const auto* sections = IMAGE_FIRST_SECTION(nt); - for (unsigned i = 0; i < nt->FileHeader.NumberOfSections; ++i) { - if (std::memcmp(sections[i].Name, ".ehmeta", 8) != 0) continue; - std::uint32_t rva = 0; - std::memcpy(&rva, base + sections[i].VirtualAddress, 4); - const auto* header = base + rva; - if (header[0] != 1) NotImplemented_nid_no_patch("sceKernelGetModuleInfoForUnwind eh_frame_hdr version other than 1"); - const auto* p = header + 4; - const auto frames = ReadEncoded(p, header[1]); - const auto* record = reinterpret_cast(frames); - const auto* end = base + nt->OptionalHeader.SizeOfImage; - if (record < base || record >= end) throw std::runtime_error("sceKernelGetModuleInfoForUnwind: eh_frame_ptr outside the image"); - for (;;) { - if (record + 4 > end) throw std::runtime_error("sceKernelGetModuleInfoForUnwind: eh_frame has no terminator inside the image"); - std::uint32_t length = 0; - std::memcpy(&length, record, 4); - if (length == 0) break; - if (length == 0xffffffffu) NotImplemented_nid_no_patch("sceKernelGetModuleInfoForUnwind eh_frame record with a 64-bit length"); - record += 4 + length; - } - info->eh_frame_hdr_addr = reinterpret_cast(header); - info->eh_frame_addr = frames; - info->eh_frame_size = static_cast(record - reinterpret_cast(frames)); - info->seg0_addr = reinterpret_cast(base); - info->seg0_size = nt->OptionalHeader.SizeOfImage; - return; + EhFrame::Tables tables; + if (!EhFrame::ReadEhmeta(reinterpret_cast(base), *nt, "sceKernelGetModuleInfoForUnwind", tables)) return; + info->eh_frame_hdr_addr = tables.header; + info->eh_frame_addr = tables.frames; + info->eh_frame_size = tables.framesSize; + info->seg0_addr = reinterpret_cast(base); + info->seg0_size = nt->OptionalHeader.SizeOfImage; +} +} +#elif !defined(__APPLE__) +extern "C" int APS5_VABI sceKernelGetModuleInfoFromAddr(std::uint64_t address, int flags, ModuleInfoEx* info); + +namespace { +struct ImageSearch { + std::uint64_t address; + bool relinked; +}; + +int FindRelinkedImage(dl_phdr_info* image, std::size_t, void* data) { + auto& search = *static_cast(data); + bool contains = false; + for (std::uint16_t index = 0; index < image->dlpi_phnum; ++index) { + const auto& header = image->dlpi_phdr[index]; + const auto start = image->dlpi_addr + header.p_vaddr; + if (header.p_type == PT_LOAD && search.address >= start && search.address - start < header.p_memsz) contains = true; } + if (!contains) return 0; + const std::string_view name = image->dlpi_name != nullptr ? image->dlpi_name : ""; + search.relinked = name.empty() || name.ends_with(".guest.prx"); + return 1; +} + +bool IsRelinkedImage(std::uint64_t address) { + ImageSearch search{address, false}; + dl_iterate_phdr(FindRelinkedImage, &search); + return search.relinked; +} + +void FillGuestUnwindInfo(std::uint64_t address, ModuleInfoForUnwind* info) { + ModuleInfoEx module{}; + module.st_size = sizeof(ModuleInfoEx); + if (sceKernelGetModuleInfoFromAddr(address, 2, &module) != 0) + throw std::runtime_error("sceKernelGetModuleInfoForUnwind: failed to query guest module information"); + info->eh_frame_hdr_addr = module.eh_frame_hdr_addr; + info->eh_frame_addr = module.eh_frame_addr; + info->eh_frame_size = module.eh_frame_size; } } #endif @@ -181,20 +187,48 @@ int APS5_VABI sceKernelGetModuleInfoForUnwind(uint64_t addr, int flags, ModuleIn info->eh_frame_size = 0; info->seg0_addr = start; info->seg0_size = end - start; + if (IsRelinkedImage(addr)) FillGuestUnwindInfo(addr, info); return 0; } return SCE_KERNEL_ERROR_ESRCH; #endif } +namespace { + +struct PendingModuleArgs { + std::size_t args = 0; + const void* argp = nullptr; +}; +thread_local PendingModuleArgs pendingModuleArgs; +thread_local int pendingModuleInitResult = 0; + +} + +extern "C" { + +const void* __aps5_get_pending_module_args_nid_no_patch() { + return &pendingModuleArgs; +} + +void __aps5_set_module_init_result_nid_no_patch(int result) { + pendingModuleInitResult = result; +} + +} + KernelModule APS5_VABI sceKernelLoadStartModule(const char* module_file_name, size_t args, const void* argp, uint32_t flags, const KernelLoadModuleOpt* opt, int* res) { - (void)args; - (void)argp; (void)flags; (void)opt; if (res) *res = 0; if (!module_file_name) return static_cast(SCE_KERNEL_ERROR_EFAULT); + pendingModuleArgs = {args, argp}; + pendingModuleInitResult = 0; void* handle = dlopen_nid_postfix(module_file_name, kRtldNow); + const int started = pendingModuleInitResult; + pendingModuleArgs = {}; + pendingModuleInitResult = 0; + if (res) *res = started; if (!handle) return static_cast(SCE_KERNEL_ERROR_ENOENT); return static_cast(reinterpret_cast(handle)); } diff --git a/core/libs/prx/libkernel/Module/src/ModuleInfo.cpp b/core/libs/prx/libkernel/Module/src/ModuleInfo.cpp index 926f25ae5..2a8d8a5ac 100644 --- a/core/libs/prx/libkernel/Module/src/ModuleInfo.cpp +++ b/core/libs/prx/libkernel/Module/src/ModuleInfo.cpp @@ -5,10 +5,21 @@ #include #include #include +#include +#include +#include #include "SceTypes.hpp" #include "prx/libc/include/General.hpp" #include "prx/libkernel/KernelErrors.hpp" -#ifndef _WIN32 +#include "prx/libkernel/Module/EhFrame.hpp" +#ifdef _WIN32 +#ifndef NOMINMAX +#define NOMINMAX +#endif +#include +#include +#include +#else #include #include #include @@ -18,15 +29,45 @@ extern "C" std::int32_t ModuleIdForImage_nid_no_patch(const void* native); namespace { -#ifndef _WIN32 +constexpr char Caller[] = "sceKernelGetModuleInfoFromAddr"; constexpr char GuestModuleSuffix[] = ".guest.prx"; +constexpr std::uint64_t TerminatorSize = 4; constexpr std::int32_t ProtRead = 1; constexpr std::int32_t ProtWrite = 2; constexpr std::int32_t ProtExecute = 4; +template +constexpr bool Extended = std::is_same_v; + +std::uint32_t ToU32(std::uint64_t value, const char* field) { + if (value > std::numeric_limits::max()) + throw std::runtime_error(std::string("module info: ") + field + " exceeds 32 bits"); + return static_cast(value); +} + +std::string ModuleName(std::string fileName) { + if (fileName.size() > sizeof(GuestModuleSuffix) - 1 && fileName.ends_with(GuestModuleSuffix)) + fileName.resize(fileName.size() - (sizeof(GuestModuleSuffix) - 1)); + return fileName; +} + +template +void SetName(TInfo& info, const std::string& name) { + if (name.size() >= sizeof(info.name)) + throw std::runtime_error("module info: module name too long: " + name); + std::memcpy(info.name, name.c_str(), name.size() + 1); +} + +struct GuestImage { + const void* native; + std::uintptr_t address; +}; + +#ifndef _WIN32 +template struct ImageSearch { std::uintptr_t address; - ModuleInfoEx* info; + TInfo* info; bool found; }; @@ -40,63 +81,6 @@ bool Contains(const dl_phdr_info& image, std::uintptr_t begin, std::uint64_t siz return false; } -std::uint32_t ToU32(std::uint64_t value, const char* field) { - if (value > std::numeric_limits::max()) - throw std::runtime_error(std::string("sceKernelGetModuleInfoFromAddr: ") + field + " exceeds 32 bits"); - return static_cast(value); -} - -std::uintptr_t EhFrameAddress(const dl_phdr_info& image, std::uintptr_t header) { - if (!Contains(image, header, 8)) - throw std::runtime_error("sceKernelGetModuleInfoFromAddr: eh_frame_hdr outside the image"); - const auto* bytes = reinterpret_cast(header); - if (bytes[0] != 1) throw std::runtime_error("sceKernelGetModuleInfoFromAddr: unsupported eh_frame_hdr version"); - const std::uintptr_t field = header + 4; - switch (bytes[1]) { - case 0x1b: { - std::int32_t offset; - std::memcpy(&offset, bytes + 4, sizeof(offset)); - return field + static_cast(offset); - } - case 0x03: { - std::uint32_t value; - std::memcpy(&value, bytes + 4, sizeof(value)); - return value; - } - case 0x00: { - std::uint64_t value; - if (!Contains(image, header, 12)) - throw std::runtime_error("sceKernelGetModuleInfoFromAddr: eh_frame_hdr outside the image"); - std::memcpy(&value, bytes + 4, sizeof(value)); - return static_cast(value); - } - default: - throw std::runtime_error("sceKernelGetModuleInfoFromAddr: unsupported eh_frame_ptr encoding"); - } -} - -std::uint64_t EhFrameSize(const dl_phdr_info& image, std::uintptr_t frame) { - std::uintptr_t position = frame; - for (;;) { - if (!Contains(image, position, 4)) - throw std::runtime_error("sceKernelGetModuleInfoFromAddr: eh_frame outside the image"); - std::uint32_t length; - std::memcpy(&length, reinterpret_cast(position), sizeof(length)); - position += 4; - if (length == 0) return position - frame; - std::uint64_t recordLength = length; - if (length == 0xffffffffu) { - if (!Contains(image, position, 8)) - throw std::runtime_error("sceKernelGetModuleInfoFromAddr: eh_frame outside the image"); - std::memcpy(&recordLength, reinterpret_cast(position), sizeof(recordLength)); - position += 8; - } - if (!Contains(image, position, recordLength)) - throw std::runtime_error("sceKernelGetModuleInfoFromAddr: eh_frame record outside the image"); - position += recordLength; - } -} - std::string ImageName(const dl_phdr_info& image) { std::string path = image.dlpi_name ? image.dlpi_name : ""; if (path.empty()) { @@ -105,18 +89,13 @@ std::string ImageName(const dl_phdr_info& image) { if (length < 0) throw std::runtime_error("sceKernelGetModuleInfoFromAddr: cannot resolve the executable path"); path.assign(executable, static_cast(length)); } - std::string name = path.substr(path.find_last_of('/') + 1); - if (name.size() > sizeof(GuestModuleSuffix) - 1 && name.ends_with(GuestModuleSuffix)) - name.resize(name.size() - (sizeof(GuestModuleSuffix) - 1)); - return name; + return ModuleName(path.substr(path.find_last_of('/') + 1)); } -void Fill(const dl_phdr_info& image, ModuleInfoEx& info) { - const auto name = ImageName(image); - if (name.size() >= sizeof(info.name)) - throw std::runtime_error("sceKernelGetModuleInfoFromAddr: module name too long: " + name); - std::memcpy(info.name, name.c_str(), name.size() + 1); - info.tls_index = ToU32(image.dlpi_tls_modid, "TLS module index"); +template +void Fill(const dl_phdr_info& image, TInfo& info) { + SetName(info, ImageName(image)); + if constexpr (Extended) info.tls_index = ToU32(image.dlpi_tls_modid, "TLS module index"); for (std::uint16_t i = 0; i < image.dlpi_phnum; ++i) { const auto& header = image.dlpi_phdr[i]; const std::uintptr_t address = image.dlpi_addr + header.p_vaddr; @@ -125,33 +104,169 @@ void Fill(const dl_phdr_info& image, ModuleInfoEx& info) { segment.address = address; segment.size = ToU32(header.p_memsz, "segment size"); segment.prot = ((header.p_flags & PF_R) ? ProtRead : 0) | ((header.p_flags & PF_W) ? ProtWrite : 0) | ((header.p_flags & PF_X) ? ProtExecute : 0); - } else if (header.p_type == PT_TLS) { - info.tls_init_addr = address; - info.tls_init_size = ToU32(header.p_filesz, "TLS image size"); - info.tls_size = ToU32(header.p_memsz, "TLS size"); - info.tls_align = ToU32(header.p_align, "TLS alignment"); - } else if (header.p_type == PT_GNU_EH_FRAME) { - info.eh_frame_hdr_addr = address; - info.eh_frame_hdr_size = ToU32(header.p_memsz, "eh_frame_hdr size"); - info.eh_frame_addr = EhFrameAddress(image, address); - info.eh_frame_size = ToU32(EhFrameSize(image, info.eh_frame_addr), "eh_frame size"); - } else if (header.p_type == PT_DYNAMIC) { - for (const auto* entry = reinterpret_cast(address); entry->d_tag != DT_NULL; ++entry) { - if (entry->d_tag == DT_INIT) info.init_proc_addr = image.dlpi_addr + entry->d_un.d_ptr; - else if (entry->d_tag == DT_FINI) info.fini_proc_addr = image.dlpi_addr + entry->d_un.d_ptr; + } else if constexpr (Extended) { + if (header.p_type == PT_TLS) { + info.tls_init_addr = address; + info.tls_init_size = ToU32(header.p_filesz, "TLS image size"); + info.tls_size = ToU32(header.p_memsz, "TLS size"); + info.tls_align = ToU32(header.p_align, "TLS alignment"); + } else if (header.p_type == PT_GNU_EH_FRAME) { + const auto contains = [&](std::uintptr_t begin, std::uint64_t size) { return Contains(image, begin, size); }; + const auto tables = EhFrame::ReadTables(contains, address, Caller); + info.eh_frame_hdr_addr = address; + info.eh_frame_hdr_size = ToU32(header.p_memsz, "eh_frame_hdr size"); + info.eh_frame_addr = tables.frames; + info.eh_frame_size = ToU32(tables.framesSize + TerminatorSize, "eh_frame size"); + } else if (header.p_type == PT_DYNAMIC) { + for (const auto* entry = reinterpret_cast(address); entry->d_tag != DT_NULL; ++entry) { + if (entry->d_tag == DT_INIT) info.init_proc_addr = image.dlpi_addr + entry->d_un.d_ptr; + else if (entry->d_tag == DT_FINI) info.fini_proc_addr = image.dlpi_addr + entry->d_un.d_ptr; + } } } } - info.ref_count = 1; + if constexpr (Extended) info.ref_count = 1; } +template int FindImage(dl_phdr_info* image, std::size_t, void* data) { - auto& search = *static_cast(data); + auto& search = *static_cast*>(data); if (!Contains(*image, search.address, 1)) return 0; Fill(*image, *search.info); search.found = true; return 1; } + +int CollectGuestImage(dl_phdr_info* image, std::size_t, void* data) { + const std::string_view name = image->dlpi_name ? image->dlpi_name : ""; + if (!name.empty() && !name.ends_with(".prx")) return 0; + for (std::uint16_t i = 0; i < image->dlpi_phnum; ++i) { + const auto& header = image->dlpi_phdr[i]; + if (header.p_type != PT_LOAD) continue; + static_cast*>(data)->push_back(image->dlpi_addr + header.p_vaddr); + return 0; + } + return 0; +} + +std::vector GuestImages() { + std::vector addresses; + dl_iterate_phdr(CollectGuestImage, &addresses); + std::vector images; + for (const auto address : addresses) { + Dl_info symbol{}; + link_map* native = nullptr; + if (!dladdr1(reinterpret_cast(address), &symbol, reinterpret_cast(&native), RTLD_DL_LINKMAP) || !native) + throw std::runtime_error("sceKernelGetModuleList: no link map for a loaded image"); + images.push_back({native, address}); + } + return images; +} +#else +bool SectionNamed(const IMAGE_SECTION_HEADER& section, const char* name) { + return std::strncmp(reinterpret_cast(section.Name), name, IMAGE_SIZEOF_SHORT_NAME) == 0; +} + +std::int32_t SectionProtection(const IMAGE_SECTION_HEADER& section) { + const auto characteristics = section.Characteristics; + return ((characteristics & IMAGE_SCN_MEM_READ) ? ProtRead : 0) | ((characteristics & IMAGE_SCN_MEM_WRITE) ? ProtWrite : 0) | ((characteristics & IMAGE_SCN_MEM_EXECUTE) ? ProtExecute : 0); +} + +std::wstring ModulePath(HMODULE module) { + std::wstring path(MAX_PATH, L'\0'); + for (;;) { + const auto length = GetModuleFileNameW(module, path.data(), static_cast(path.size())); + if (length == 0) throw std::runtime_error("module info: GetModuleFileNameW failed with error " + std::to_string(GetLastError())); + if (length < path.size()) { + path.resize(length); + return path; + } + path.resize(path.size() * 2); + } +} + +std::string ImageName(HMODULE module) { + const auto path = ModulePath(module); + const auto fileName = std::filesystem::path(path).filename().u8string(); + return ModuleName(std::string(fileName.begin(), fileName.end())); +} + +template +bool Fill(std::uintptr_t address, TInfo& info) { + MEMORY_BASIC_INFORMATION memory{}; + if (VirtualQuery(reinterpret_cast(address), &memory, sizeof(memory)) != sizeof(memory) || memory.Type != MEM_IMAGE) return false; + const auto base = reinterpret_cast(memory.AllocationBase); + const auto* dos = reinterpret_cast(base); + if (dos->e_magic != IMAGE_DOS_SIGNATURE) return false; + const auto* nt = reinterpret_cast(base + static_cast(dos->e_lfanew)); + if (nt->Signature != IMAGE_NT_SIGNATURE || nt->OptionalHeader.Magic != IMAGE_NT_OPTIONAL_HDR64_MAGIC) return false; + SetName(info, ImageName(reinterpret_cast(base))); + const auto* sections = IMAGE_FIRST_SECTION(nt); + const auto sectionCount = nt->FileHeader.NumberOfSections; + bool relinked = false; + for (unsigned i = 0; i < sectionCount; ++i) relinked = relinked || SectionNamed(sections[i], ".elf0"); + EhFrame::Tables tables; + const bool hasEhmeta = EhFrame::ReadEhmeta(base, *nt, Caller, tables); + if constexpr (Extended) { + if (hasEhmeta) { + info.eh_frame_hdr_addr = tables.header; + info.eh_frame_hdr_size = ToU32(tables.headerSize, "eh_frame_hdr size"); + info.eh_frame_addr = tables.frames; + info.eh_frame_size = ToU32(tables.framesSize + TerminatorSize, "eh_frame size"); + } + } + for (unsigned i = 0; i < sectionCount; ++i) { + const auto& section = sections[i]; + const auto start = base + section.VirtualAddress; + if (SectionNamed(section, ".ehmeta")) continue; + if (SectionNamed(section, ".ehfram") && !hasEhmeta) { + if constexpr (Extended) { + const std::uint64_t imageSize = nt->OptionalHeader.SizeOfImage; + const auto contains = [&](std::uintptr_t begin, std::uint64_t size) { return begin >= base && begin - base <= imageSize && size <= imageSize - (begin - base); }; + info.eh_frame_addr = start; + info.eh_frame_size = ToU32(EhFrame::FramesSize(contains, start, Caller) + TerminatorSize, "eh_frame size"); + } + continue; + } + if (relinked && std::strncmp(reinterpret_cast(section.Name), ".elf", 4) != 0) continue; + const auto prot = SectionProtection(section); + if (prot == 0 || info.segment_count >= std::size(info.segments)) continue; + auto& segment = info.segments[info.segment_count++]; + segment.address = start; + segment.size = ToU32(section.Misc.VirtualSize, "segment size"); + segment.prot = prot; + } + if constexpr (Extended) { + info.id = ModuleIdForImage_nid_no_patch(reinterpret_cast(base)); + info.ref_count = 1; + } + return true; +} + +std::vector GuestImages() { + const HANDLE process = GetCurrentProcess(); + std::vector modules(64); + for (;;) { + DWORD needed = 0; + const auto size = static_cast(modules.size() * sizeof(HMODULE)); + if (!EnumProcessModules(process, modules.data(), size, &needed)) + throw std::runtime_error("sceKernelGetModuleList: EnumProcessModules failed with error " + std::to_string(GetLastError())); + modules.resize(needed / sizeof(HMODULE)); + if (needed <= size) break; + } + const HMODULE executable = GetModuleHandleW(nullptr); + constexpr std::wstring_view suffix = L".prx"; + std::vector images; + for (const auto module : modules) { + if (module != executable) { + const auto path = ModulePath(module); + if (path.size() < suffix.size() || CompareStringOrdinal(path.data() + path.size() - suffix.size(), static_cast(suffix.size()), suffix.data(), static_cast(suffix.size()), TRUE) != CSTR_EQUAL) + continue; + } + images.push_back({module, reinterpret_cast(module)}); + } + return images; +} #endif } @@ -163,24 +278,51 @@ int APS5_VABI sceKernelGetModuleInfoFromAddr(std::uint64_t address, int flags, M if (flags != 2) throw std::invalid_argument("sceKernelGetModuleInfoFromAddr: unsupported flags " + std::to_string(flags)); if (info->st_size != sizeof(ModuleInfoEx)) throw std::invalid_argument("sceKernelGetModuleInfoFromAddr: unsupported st_size " + std::to_string(info->st_size)); + ModuleInfoEx result{}; + result.st_size = sizeof(ModuleInfoEx); #ifdef _WIN32 - (void)address; - NotImplemented_nid_no_patch(__func__); - return 0; + if (!Fill(static_cast(address), result)) return SCE_KERNEL_ERROR_ESRCH; #else Dl_info symbol{}; link_map* native = nullptr; if (!dladdr1(reinterpret_cast(address), &symbol, reinterpret_cast(&native), RTLD_DL_LINKMAP) || !native) return SCE_KERNEL_ERROR_ESRCH; - ModuleInfoEx result{}; - result.st_size = sizeof(ModuleInfoEx); - ImageSearch search{static_cast(address), &result, false}; - dl_iterate_phdr(FindImage, &search); + ImageSearch search{static_cast(address), &result, false}; + dl_iterate_phdr(FindImage, &search); if (!search.found) return SCE_KERNEL_ERROR_ESRCH; result.id = ModuleIdForImage_nid_no_patch(native); +#endif *info = result; return 0; +} + +int APS5_VABI sceKernelGetModuleList(KernelModule* handles, std::size_t count, std::size_t* actual) { + if (!handles || !actual) return SCE_KERNEL_ERROR_EFAULT; + const auto images = GuestImages(); + for (std::size_t i = 0; i < images.size() && i < count; ++i) handles[i] = ModuleIdForImage_nid_no_patch(images[i].native); + if (images.size() > count) return SCE_KERNEL_ERROR_ENOMEM; + *actual = images.size(); + return 0; +} + +int APS5_VABI sceKernelGetModuleInfo(KernelModule handle, ModuleInfo* info) { + if (!info) return SCE_KERNEL_ERROR_EFAULT; + if (info->st_size != sizeof(ModuleInfo)) return SCE_KERNEL_ERROR_EINVAL; + for (const auto& image : GuestImages()) { + if (ModuleIdForImage_nid_no_patch(image.native) != handle) continue; + ModuleInfo result{}; + result.st_size = sizeof(ModuleInfo); +#ifdef _WIN32 + if (!Fill(image.address, result)) return SCE_KERNEL_ERROR_ESRCH; +#else + ImageSearch search{image.address, &result, false}; + dl_iterate_phdr(FindImage, &search); + if (!search.found) return SCE_KERNEL_ERROR_ESRCH; #endif + *info = result; + return 0; + } + return SCE_KERNEL_ERROR_ESRCH; } } diff --git a/core/libs/prx/libkernel/Pthread/Posix/Common.hpp b/core/libs/prx/libkernel/Pthread/Posix/Common.hpp index 3b5734120..cf95f707e 100644 --- a/core/libs/prx/libkernel/Pthread/Posix/Common.hpp +++ b/core/libs/prx/libkernel/Pthread/Posix/Common.hpp @@ -10,6 +10,8 @@ extern "C" int APS5_VABI clock_gettime_nid_postfix(int clockId, KernelTimespec* namespace PosixThread { +constexpr int GUEST_ESRCH = 3; +constexpr int GUEST_EFAULT = 14; constexpr int GUEST_EINVAL = 22; constexpr int GUEST_ETIMEDOUT = 60; diff --git a/core/libs/prx/libkernel/Pthread/Posix/Sem.cpp b/core/libs/prx/libkernel/Pthread/Posix/Sem.cpp index 2672b045b..e71072e2a 100644 --- a/core/libs/prx/libkernel/Pthread/Posix/Sem.cpp +++ b/core/libs/prx/libkernel/Pthread/Posix/Sem.cpp @@ -3,6 +3,7 @@ #include "SceTypes.hpp" #include "prx/libc/include/General.hpp" #include "Common.hpp" +#include "../include/Cancel.hpp" #include "prx/libkernel/Time/include/TimedWait.hpp" #include #include @@ -34,12 +35,13 @@ static PosixSemaphore* Get(void* sem) { } static int WaitFor(PosixSemaphore* semaphore, const KernelUseconds* usec) { + ThreadCancel::Check(); std::unique_lock lock(semaphore->lock); const auto ready = [&] { return semaphore->count > 0; }; if (usec) { - if (!semaphore->available.WaitUntil(lock, TimedWait::DeadlineNanos(*usec), ready)) return Fail(PosixThread::GUEST_ETIMEDOUT); + if (!ThreadCancel::WaitUntil(semaphore->available, lock, TimedWait::DeadlineNanos(*usec), ready)) return Fail(PosixThread::GUEST_ETIMEDOUT); } else { - semaphore->available.Wait(lock, ready); + ThreadCancel::WaitUntil(semaphore->available, lock, std::nullopt, ready); } --semaphore->count; return 0; diff --git a/core/libs/prx/libkernel/Pthread/Posix/Thread.cpp b/core/libs/prx/libkernel/Pthread/Posix/Thread.cpp index 66800ba33..226925ada 100644 --- a/core/libs/prx/libkernel/Pthread/Posix/Thread.cpp +++ b/core/libs/prx/libkernel/Pthread/Posix/Thread.cpp @@ -1,7 +1,9 @@ #include #include +#include #include "SceTypes.hpp" #include "../include/ThreadLifecycle.hpp" +#include "../include/Cancel.hpp" #include "prx/libc/include/General.hpp" #include "../include/Pthread.hpp" #include "Common.hpp" @@ -15,8 +17,11 @@ int APS5_VABI scePthreadDetach(Pthread thread); void APS5_VABI scePthreadExit(void* retval); int APS5_VABI scePthreadJoin(Pthread thread, void** retval); int APS5_VABI scePthreadRename(Pthread thread, const char* name); +int APS5_VABI scePthreadGetname(Pthread thread, char* name); Pthread APS5_VABI scePthreadSelf(); +int APS5_VABI scePthreadCancel(Pthread thread); int APS5_VABI scePthreadSetcancelstate(int state, int* old_state); +int APS5_VABI scePthreadSetcanceltype(int type, int* old_type); void APS5_VABI scePthreadTestcancel(); int APS5_VABI scePthreadSetprio(Pthread thread, int prio); int APS5_VABI scePthreadGetprio(Pthread thread, int* prio); @@ -65,9 +70,16 @@ int APS5_VABI pthread_join_nid_postfix(Pthread thread, void** value) { } int APS5_VABI pthread_rename_np_nid_postfix(Pthread thread, const char* name) { + if (thread && name && std::strlen(name) >= 32) return PosixThread::ToErrno(SCE_KERNEL_ERROR_ENAMETOOLONG); return PosixThread::ToErrno(scePthreadRename(thread, name)); } +int APS5_VABI pthread_getname_np_nid_postfix(Pthread thread, char* name) { + if (!thread) return PosixThread::GUEST_ESRCH; + if (!name) return PosixThread::GUEST_EFAULT; + return PosixThread::ToErrno(scePthreadGetname(thread, name)); +} + Pthread APS5_VABI pthread_self_nid_postfix(void) { return scePthreadSelf(); } @@ -76,15 +88,31 @@ int APS5_VABI pthread_equal_nid_postfix(Pthread first, Pthread second) { return first == second; } +int APS5_VABI pthread_getcpuclockid_nid_postfix(Pthread thread, int* clockId) { + constexpr int guestFault = 14; + if (!thread) return PosixThread::GUEST_EINVAL; + if (!clockId) return guestFault; + *clockId = GuestThreadCpuClockId(thread); + return 0; +} + int APS5_VABI sched_yield_nid_postfix(void) { std::this_thread::yield(); return 0; } +int APS5_VABI pthread_cancel_nid_postfix(Pthread thread) { + return PosixThread::ToErrno(scePthreadCancel(thread)); +} + int APS5_VABI pthread_setcancelstate_nid_postfix(int state, int* old_state) { return PosixThread::ToErrno(scePthreadSetcancelstate(state, old_state)); } +int APS5_VABI pthread_setcanceltype_nid_postfix(int type, int* old_type) { + return PosixThread::ToErrno(scePthreadSetcanceltype(type, old_type)); +} + int APS5_VABI pthread_setprio_nid_postfix(Pthread thread, int prio) { return PosixThread::ToErrno(scePthreadSetprio(thread, prio)); } @@ -104,7 +132,9 @@ void APS5_VABI pthread_yield_nid_postfix(void) { } unsigned int APS5_VABI sleep_nid_postfix(unsigned int seconds) { + ThreadCancel::Check(); std::this_thread::sleep_for(std::chrono::seconds(seconds)); + ThreadCancel::Check(); return 0; } diff --git a/core/libs/prx/libkernel/Pthread/include/Cancel.hpp b/core/libs/prx/libkernel/Pthread/include/Cancel.hpp new file mode 100644 index 000000000..26a7c7176 --- /dev/null +++ b/core/libs/prx/libkernel/Pthread/include/Cancel.hpp @@ -0,0 +1,65 @@ +#ifndef CORE_LIBS_PRX_LIBKERNEL_PTHREAD_CANCEL_HPP +#define CORE_LIBS_PRX_LIBKERNEL_PTHREAD_CANCEL_HPP + +#include "SceTypes.hpp" +#include "prx/libkernel/Time/include/TimedWait.hpp" +#include +#include + +inline constexpr int GUEST_CANCEL_ENABLE = 0; +inline constexpr int GUEST_CANCEL_DISABLE = 1; +inline constexpr int GUEST_CANCEL_DEFERRED = 0; +inline constexpr int GUEST_CANCEL_ASYNCHRONOUS = 2; +inline void* const GUEST_CANCELED = reinterpret_cast(std::uintptr_t{1}); + +extern "C" { +void APS5_VABI __pthread_cleanup_push_imp_nid_postfix(void (APS5_VABI* routine)(void*), void* argument, void* info); +void APS5_VABI __pthread_cleanup_pop_imp_nid_postfix(int execute); +} + +namespace ThreadCancel { + +bool Requested(); +void Check(); +void BeginWait(TimedWait::Condition* condition); +void EndWait(); + +template +struct WaitLock { + TLock& inner; + void lock() { + EndWait(); + inner.lock(); + } + void unlock() { inner.unlock(); } +}; + +template +bool Wait(TimedWait::Condition& condition, TLock& lock, std::optional deadlineNanos) { + BeginWait(&condition); + if (Requested()) { + EndWait(); + return true; + } + WaitLock waitLock{lock}; + if (deadlineNanos) return condition.WaitUntil(waitLock, *deadlineNanos); + condition.Wait(waitLock); + return true; +} + +template +bool WaitUntil(TimedWait::Condition& condition, TLock& lock, std::optional deadlineNanos, TPredicate predicate) { + while (!predicate()) { + const bool signaled = Wait(condition, lock, deadlineNanos); + if (Requested()) { + lock.unlock(); + Check(); + } + if (!signaled) return predicate(); + } + return true; +} + +} + +#endif diff --git a/core/libs/prx/libkernel/Pthread/include/Pthread.hpp b/core/libs/prx/libkernel/Pthread/include/Pthread.hpp index 2f3fb16eb..11e6eea38 100644 --- a/core/libs/prx/libkernel/Pthread/include/Pthread.hpp +++ b/core/libs/prx/libkernel/Pthread/include/Pthread.hpp @@ -5,14 +5,20 @@ #include "SceTypes.hpp" #include "prx/libkernel/Time/include/TimedWait.hpp" #include +#include #include #include #include +#include #include #include #include #include +#ifndef _WIN32 +#include +#endif + enum class MutexType : std::uint32_t { ErrorCheck = 1, Recursive = 2, @@ -39,6 +45,12 @@ struct PthreadRwlockattrPrivate { }; struct PthreadRwlockPrivate { + // TODO(technical debt): winpthreads initializes a static rwlock on first use and fails a + // concurrent first lock with EINVAL, which shared_timed_mutex ignores. Initialize it here. + PthreadRwlockPrivate() { + _lock.lock(); + _lock.unlock(); + } std::shared_timed_mutex _lock; std::atomic _writer; }; @@ -62,6 +74,16 @@ struct PthreadSemPrivate { static constexpr KernelCpumask DEFAULT_THREAD_AFFINITY = 0x1FFF; static constexpr int DEFAULT_THREAD_PRIORITY = 700; +static constexpr std::uint32_t CPU_CLOCK_BIT = 0x80000000u; +static constexpr std::uint32_t CPU_CLOCK_PROCESS_BIT = 0x40000000u; +static constexpr std::uint32_t CPU_CLOCK_ID_MASK = ~(CPU_CLOCK_BIT | CPU_CLOCK_PROCESS_BIT); + +inline int GuestCpuFromHost(unsigned hostCpu, KernelCpumask threadAffinity) { + KernelCpumask mask = threadAffinity & DEFAULT_THREAD_AFFINITY; + if (mask == 0) mask = DEFAULT_THREAD_AFFINITY; + for (auto skip = hostCpu % static_cast(std::popcount(mask)); skip != 0; --skip) mask &= mask - 1; + return std::countr_zero(mask); +} struct PthreadAttrPrivate { void* stackAddress = nullptr; @@ -75,19 +97,30 @@ struct PthreadAttrPrivate { int _solosched = 0; }; +inline std::atomic nextThreadId{100000}; + struct PthreadPrivate { + std::int64_t tid = nextThreadId.fetch_add(1, std::memory_order_relaxed); #ifdef _WIN32 void* nativeHandle = nullptr; #else - std::thread _thr; + pthread_t hostThread{}; #endif std::thread::id threadId; std::atomic references{2}; + std::atomic inWait{false}; + std::atomic pendingException{0}; + void* wakeEvent = nullptr; void* stackAddress = nullptr; std::size_t stackSize = 0; std::atomic waitCount{0}; std::atomic affinity{DEFAULT_THREAD_AFFINITY}; std::atomic priority{DEFAULT_THREAD_PRIORITY}; + int cpuClockThread = 0; +#ifndef _WIN32 + clockid_t hostCpuClock{}; + bool hostCpuClockBound = false; +#endif std::mutex nameLock; std::string name; std::atomic _finished; @@ -95,11 +128,19 @@ struct PthreadPrivate { bool _detached; bool _adopted; std::mutex _join_mtx; - std::condition_variable _join_cv; + TimedWait::Condition _join_cv; + std::atomic cancelPending{false}; + std::mutex cancelLock; + TimedWait::Condition* cancelWait = nullptr; - PthreadPrivate() : _finished(false), _retval(nullptr), _detached(false), _adopted(false) {} + PthreadPrivate(); + ~PthreadPrivate(); + PthreadPrivate(const PthreadPrivate&) = delete; + PthreadPrivate& operator=(const PthreadPrivate&) = delete; }; bool GuestThreadStack(std::uintptr_t address, std::uintptr_t* start, std::uintptr_t* end); +int GuestThreadCpuClockId(const PthreadPrivate* thread); +bool GuestThreadCpuNanos(int cpuClockThread, std::uint64_t* nanos); #endif diff --git a/core/libs/prx/libkernel/Pthread/include/WindowsThreadLocal.hpp b/core/libs/prx/libkernel/Pthread/include/WindowsThreadLocal.hpp new file mode 100644 index 000000000..2a42b292f --- /dev/null +++ b/core/libs/prx/libkernel/Pthread/include/WindowsThreadLocal.hpp @@ -0,0 +1,26 @@ +#pragma once + +#ifdef _WIN32 +#include +#include +#include +template +class WindowsThreadLocal { +public: + static T& Get() { + if (slot == FLS_OUT_OF_INDEXES) throw std::runtime_error("Cannot allocate native thread state slot"); + if (auto* value = static_cast(FlsGetValue(slot))) return *value; + auto value = std::make_unique(); + if (!FlsSetValue(slot, value.get())) throw std::runtime_error("Cannot bind native thread state"); + return *value.release(); + } + +private: + static void WINAPI Destroy(void* value) { + FlsSetValue(slot, value); + delete static_cast(value); + FlsSetValue(slot, nullptr); + } + inline static const DWORD slot = FlsAlloc(Destroy); +}; +#endif diff --git a/core/libs/prx/libkernel/Pthread/src/Cond.cpp b/core/libs/prx/libkernel/Pthread/src/Cond.cpp index 9eb2c9449..66887e6a7 100644 --- a/core/libs/prx/libkernel/Pthread/src/Cond.cpp +++ b/core/libs/prx/libkernel/Pthread/src/Cond.cpp @@ -1,6 +1,7 @@ #include "prx/libkernel/Pthread/include/Pthread.hpp" #include "prx/libkernel/Pthread/include/Mutex.hpp" #include "prx/libkernel/Pthread/include/Cond.hpp" +#include "prx/libkernel/Pthread/include/Cancel.hpp" #include "prx/libkernel/Pthread/Posix/Common.hpp" #include "prx/libkernel/Time/include/Time.hpp" #include "prx/libkernel/Time/include/TimedWait.hpp" @@ -61,19 +62,23 @@ int waitUntil(PthreadCond* cond, PthreadMutex* mutex, std::optional_count; m->_count = 0; m->_owner.store(std::thread::id{}, std::memory_order_release); - if (deadlineNanos) timedOut = !c->_cv.WaitUntil(lock, *deadlineNanos); - else c->_cv.Wait(lock); + for (int level = 1; level < previousCount; ++level) + m->_rmtx.unlock(); + timedOut = !ThreadCancel::Wait(c->_cv, lock, deadlineNanos); + for (int level = 1; level < previousCount; ++level) + m->_rmtx.lock(); m->_owner.store(std::this_thread::get_id(), std::memory_order_release); m->_count = previousCount; lock.release(); + ThreadCancel::Check(); return timedOut ? sceTimedOut : 0; } std::unique_lock lock(m->_mtx, std::adopt_lock); m->_owner.store(std::thread::id{}, std::memory_order_release); - if (deadlineNanos) timedOut = !c->_cv.WaitUntil(lock, *deadlineNanos); - else c->_cv.Wait(lock); + timedOut = !ThreadCancel::Wait(c->_cv, lock, deadlineNanos); m->_owner.store(std::this_thread::get_id(), std::memory_order_release); lock.release(); + ThreadCancel::Check(); return timedOut ? sceTimedOut : 0; } diff --git a/core/libs/prx/libkernel/Pthread/src/Mutex.cpp b/core/libs/prx/libkernel/Pthread/src/Mutex.cpp index 67f3ad67b..7f1c2459e 100644 --- a/core/libs/prx/libkernel/Pthread/src/Mutex.cpp +++ b/core/libs/prx/libkernel/Pthread/src/Mutex.cpp @@ -154,14 +154,21 @@ int APS5_VABI scePthreadMutexDestroy(PthreadMutex* mutex) { return 0; } if (*mutex && (*mutex)->_owner.load(std::memory_order_acquire) != std::thread::id{}) - throw std::runtime_error("Cannot destroy a locked mutex"); + return SCE_KERNEL_ERROR_EBUSY; delete *mutex; std::atomic_ref(*mutex).store(destroyedMutex(), std::memory_order_release); return 0; } int APS5_VABI scePthreadMutexLock(PthreadMutex* mutex) { - return acquireMutex(resolveMutex(mutex, true), [](auto& native) { native.lock(); return true; }, 0, false); + return acquireMutex(resolveMutex(mutex, true), [](auto& native) { +#ifdef _WIN32 + while (!native.try_lock()) TimedWait::SleepNanos(1000000); +#else + native.lock(); +#endif + return true; + }, 0, false); } int APS5_VABI scePthreadMutexUnlock(PthreadMutex* mutex) { diff --git a/core/libs/prx/libkernel/Pthread/src/Rwlock.cpp b/core/libs/prx/libkernel/Pthread/src/Rwlock.cpp index 37a7668a6..6d8850f0b 100644 --- a/core/libs/prx/libkernel/Pthread/src/Rwlock.cpp +++ b/core/libs/prx/libkernel/Pthread/src/Rwlock.cpp @@ -42,7 +42,11 @@ int APS5_VABI scePthreadRwlockInit(PthreadRwlock* rwlock, const PthreadRwlockatt int APS5_VABI scePthreadRwlockRdlock(PthreadRwlock* rwlock) { auto* lock = RequireRwlock(rwlock, __func__); if (OwnsWrite(lock)) return SCE_KERNEL_ERROR_EDEADLK; +#ifdef _WIN32 + while (!lock->_lock.try_lock_shared()) TimedWait::SleepNanos(1000000); +#else lock->_lock.lock_shared(); +#endif return SCE_OK; } @@ -62,7 +66,11 @@ int APS5_VABI scePthreadRwlockTimedrdlock(PthreadRwlock* rwlock, KernelUseconds int APS5_VABI scePthreadRwlockWrlock(PthreadRwlock* rwlock) { auto* lock = RequireRwlock(rwlock, __func__); if (OwnsWrite(lock)) return SCE_KERNEL_ERROR_EDEADLK; +#ifdef _WIN32 + while (!lock->_lock.try_lock()) TimedWait::SleepNanos(1000000); +#else lock->_lock.lock(); +#endif lock->_writer.store(std::this_thread::get_id(), std::memory_order_release); return SCE_OK; } diff --git a/core/libs/prx/libkernel/Pthread/src/Sem.cpp b/core/libs/prx/libkernel/Pthread/src/Sem.cpp index 7a1d26c3f..8255d87b9 100644 --- a/core/libs/prx/libkernel/Pthread/src/Sem.cpp +++ b/core/libs/prx/libkernel/Pthread/src/Sem.cpp @@ -1,4 +1,5 @@ #include "../include/Pthread.hpp" +#include "../include/Cancel.hpp" #include "prx/libc/include/General.hpp" #include "prx/libkernel/Time/include/TimedWait.hpp" #include @@ -65,8 +66,9 @@ int APS5_VABI scePthreadSemPost(PthreadSem* sem) { int APS5_VABI scePthreadSemWait(PthreadSem* sem) { auto* current = resolveSem(sem); + ThreadCancel::Check(); std::unique_lock lock(current->_mutex); - current->_cv.Wait(lock, [&] { return current->_count > 0; }); + ThreadCancel::WaitUntil(current->_cv, lock, std::nullopt, [&] { return current->_count > 0; }); --current->_count; return 0; } @@ -82,8 +84,9 @@ int APS5_VABI scePthreadSemTrywait(PthreadSem* sem) { int APS5_VABI scePthreadSemTimedwait(PthreadSem* sem, KernelUseconds usec) { auto* current = resolveSem(sem); + ThreadCancel::Check(); std::unique_lock lock(current->_mutex); - const auto acquired = current->_cv.WaitUntil(lock, TimedWait::DeadlineNanos(usec), [&] { return current->_count > 0; }); + const auto acquired = ThreadCancel::WaitUntil(current->_cv, lock, TimedWait::DeadlineNanos(usec), [&] { return current->_count > 0; }); if (!acquired) return sceTimedOut; --current->_count; diff --git a/core/libs/prx/libkernel/Pthread/src/Thread.cpp b/core/libs/prx/libkernel/Pthread/src/Thread.cpp index b3c368d80..60f902c82 100644 --- a/core/libs/prx/libkernel/Pthread/src/Thread.cpp +++ b/core/libs/prx/libkernel/Pthread/src/Thread.cpp @@ -1,5 +1,7 @@ #include "../include/Pthread.hpp" #include "../include/ThreadLifecycle.hpp" +#include "../include/WindowsThreadLocal.hpp" +#include "../include/Cancel.hpp" #include "prx/libc/include/General.hpp" #include "prx/libkernel/KernelErrors.hpp" #include "prx/libc/include/CpuTopology.hpp" @@ -7,6 +9,7 @@ #include #include #include +#include #include #include #include @@ -17,20 +20,22 @@ #include #ifndef _WIN32 +#include #include #include +#include #endif static constexpr int SCE_OK = 0; static constexpr std::size_t DEFAULT_STACK_SIZE = 1u << 20; +static constexpr std::size_t HOST_STACK_MARGIN = 1u << 20; static constexpr int DETACH_DETACHED = 1; static constexpr std::size_t THREAD_NAME_CAPACITY = 32; #ifdef _WIN32 #include #include -#include #endif struct ThreadArgs { @@ -42,8 +47,29 @@ struct ThreadArgs { static std::atomic threadDtors{nullptr}; static std::atomic threadAtexitCount{nullptr}; static std::atomic threadAtexitReport{nullptr}; -static thread_local PthreadPrivate* currentThread = nullptr; -static thread_local bool threadFinishing = false; +struct CleanupHandler { + CleanupHandler* previous; + void (APS5_VABI* routine)(void*); + void* argument; + int onHeap; +}; + +struct GuestThreadState { + PthreadPrivate* current = nullptr; + bool finishing = false; + bool exiting = false; + int cancelState = GUEST_CANCEL_ENABLE; + int cancelType = GUEST_CANCEL_DEFERRED; + CleanupHandler* cleanupHandlers = nullptr; +}; +static GuestThreadState& CurrentGuestState() { +#ifdef _WIN32 + return WindowsThreadLocal::Get(); +#else + static thread_local GuestThreadState state; + return state; +#endif +} static std::mutex stackLock; static std::map> liveStacks; @@ -79,9 +105,9 @@ static void SetStackFromHost(PthreadPrivate* thread) { } static bool CurrentStack(std::uintptr_t* low, std::uintptr_t* high) { - if (currentThread && currentThread->stackAddress) { - *low = reinterpret_cast(currentThread->stackAddress); - *high = *low + currentThread->stackSize; + if (CurrentGuestState().current && CurrentGuestState().current->stackAddress) { + *low = reinterpret_cast(CurrentGuestState().current->stackAddress); + *high = *low + CurrentGuestState().current->stackSize; return true; } return HostStackLimits(low, high); @@ -119,6 +145,90 @@ bool GuestThreadStack(std::uintptr_t address, std::uintptr_t* start, std::uintpt } return false; } + +struct CpuClockRegistry { + std::mutex lock; + std::map threads; + int nextThread = 100000; +}; + +static CpuClockRegistry& CpuClocks() { + static auto* registry = new CpuClockRegistry(); + return *registry; +} + +PthreadPrivate::PthreadPrivate() : _finished(false), _retval(nullptr), _detached(false), _adopted(false) { + auto& registry = CpuClocks(); + std::lock_guard lock(registry.lock); + if (registry.nextThread > static_cast(CPU_CLOCK_ID_MASK)) + throw std::runtime_error("Guest thread CPU clock identifiers are exhausted"); + cpuClockThread = registry.nextThread++; + registry.threads.emplace(cpuClockThread, this); +} + +PthreadPrivate::~PthreadPrivate() { + auto& registry = CpuClocks(); + std::lock_guard lock(registry.lock); + registry.threads.erase(cpuClockThread); +} + +static void BindHostCpuClock(PthreadPrivate* self) { +#ifdef _WIN32 + (void)self; +#else + clockid_t clock{}; + const int error = pthread_getcpuclockid(pthread_self(), &clock); + if (error != 0) + throw std::system_error(error, std::generic_category(), "Querying the guest thread CPU clock"); + auto& registry = CpuClocks(); + std::lock_guard lock(registry.lock); + self->hostCpuClock = clock; + self->hostCpuClockBound = true; +#endif +} + +int GuestThreadCpuClockId(const PthreadPrivate* thread) { + return static_cast(CPU_CLOCK_BIT | static_cast(thread->cpuClockThread)); +} + +bool GuestThreadCpuNanos(int cpuClockThread, std::uint64_t* nanos) { + auto& registry = CpuClocks(); + std::lock_guard lock(registry.lock); + const auto found = registry.threads.find(cpuClockThread); + if (found == registry.threads.end()) + return false; + PthreadPrivate* thread = found->second; + if (thread->_finished.load(std::memory_order_acquire)) + return false; +#ifdef _WIN32 + if (thread->nativeHandle == nullptr) { + *nanos = 0; + return true; + } + if (WaitForSingleObject(thread->nativeHandle, 0) == WAIT_OBJECT_0) + return false; + FILETIME creation{}; + FILETIME exitTime{}; + FILETIME kernel{}; + FILETIME user{}; + if (!GetThreadTimes(thread->nativeHandle, &creation, &exitTime, &kernel, &user)) + throw std::system_error(GetLastError(), std::system_category(), "Querying guest thread CPU time"); + const auto toTicks = [](const FILETIME& time) { + return (static_cast(time.dwHighDateTime) << 32) | time.dwLowDateTime; + }; + *nanos = (toTicks(user) + toTicks(kernel)) * 100ULL; +#else + if (!thread->hostCpuClockBound) { + *nanos = 0; + return true; + } + struct timespec time{}; + if (clock_gettime(thread->hostCpuClock, &time) != 0) + throw std::system_error(errno, std::generic_category(), "Querying guest thread CPU time"); + *nanos = static_cast(time.tv_sec) * 1000000000ULL + static_cast(time.tv_nsec); +#endif + return true; +} #ifndef _WIN32 static thread_local std::unique_ptr adoptedThread; // pthread_exit force-unwinds through guest frames, whose personality resolves to @@ -126,6 +236,29 @@ static thread_local std::unique_ptr adoptedThread; // and aborts. scePthreadExit instead jumps back to the thread start routine, which, // like _endthreadex on Windows, skips the guest frames without unwinding them. static thread_local std::jmp_buf* threadExitJump = nullptr; + +static void* MeasureStackReserve(void* reserve) { + std::uintptr_t low = 0; + std::uintptr_t high = 0; + if (HostStackLimits(&low, &high)) + *static_cast(reserve) = high - reinterpret_cast(&low); + return nullptr; +} + +static std::size_t HostStackReserve() { + static const std::size_t reserve = [] { + std::size_t measured = 0; + pthread_t probe{}; + if (const int result = pthread_create(&probe, nullptr, MeasureStackReserve, &measured); result != 0) + throw std::system_error(result, std::generic_category(), "Measuring the host thread stack reserve"); + if (const int result = pthread_join(probe, nullptr); result != 0) + throw std::system_error(result, std::generic_category(), "Measuring the host thread stack reserve"); + if (measured == 0) + throw std::runtime_error("Cannot measure the host thread stack reserve"); + return measured; + }(); + return reserve; +} #endif void ThreadLifecycle::SetThreadDtors(thread_dtors_func_t callback) { @@ -153,11 +286,11 @@ void ThreadLifecycle::SetThreadAtexitReport(thread_atexit_report_func_t callback } static void finishThread(PthreadPrivate* self, void* retval) { - if (!self || self != currentThread) + if (!self || self != CurrentGuestState().current) throw std::runtime_error("Finishing an unregistered guest thread"); - if (threadFinishing) + if (CurrentGuestState().finishing) throw std::runtime_error("Guest thread is already finishing"); - threadFinishing = true; + CurrentGuestState().finishing = true; UnregisterStack(self); if (const auto callback = threadDtors.load()) callback(); @@ -166,7 +299,25 @@ static void finishThread(PthreadPrivate* self, void* retval) { self->_retval = retval; self->_finished.store(true, std::memory_order_release); } - self->_join_cv.notify_all(); + self->_join_cv.NotifyAll(); +} + +bool ThreadCancel::Requested() { + return CurrentGuestState().current && !CurrentGuestState().exiting && CurrentGuestState().cancelState == GUEST_CANCEL_ENABLE && CurrentGuestState().current->cancelPending.load(); +} + +void ThreadCancel::Check() { + if (Requested()) scePthreadExit(GUEST_CANCELED); +} + +void ThreadCancel::BeginWait(TimedWait::Condition* condition) { + if (!CurrentGuestState().current) return; + std::lock_guard lock(CurrentGuestState().current->cancelLock); + CurrentGuestState().current->cancelWait = condition; +} + +void ThreadCancel::EndWait() { + BeginWait(nullptr); } static void RunThread(std::unique_ptr args) { @@ -174,13 +325,13 @@ static void RunThread(std::unique_ptr args) { const auto entry = args->entry; void* arg = args->arg; PthreadPrivate* self = args->self; + BindHostCpuClock(self); TimedWait::BindThreadWaitState(&self->waitCount); - if (!self->stackAddress) SetStackFromHost(self); - currentThread = self; + CurrentGuestState().current = self; RegisterStack(self); args.reset(); finishThread(self, entry(arg)); - currentThread = nullptr; + CurrentGuestState().current = nullptr; } static void ReleaseThread(PthreadPrivate* thread) { @@ -189,6 +340,8 @@ static void ReleaseThread(PthreadPrivate* thread) { if (thread->references.fetch_sub(1, std::memory_order_acq_rel) != 1) return; #ifdef _WIN32 + if (thread->wakeEvent) + CloseHandle(static_cast(thread->wakeEvent)); if (!CloseHandle(thread->nativeHandle)) throw std::system_error(GetLastError(), std::system_category(), "Closing guest thread handle"); #endif @@ -225,6 +378,7 @@ static void ApplyJobAffinity(PthreadPrivate& thread, void* handle) { }); CpuTopology::PinTraced(thread.name.c_str(), handle, mask); } +#endif struct NativeThreadArgs { std::unique_ptr guest; @@ -232,6 +386,7 @@ struct NativeThreadArgs { std::promise initialized; }; +#ifdef _WIN32 static unsigned __stdcall StartNativeThread(void* opaque) { std::unique_ptr args(static_cast(opaque)); auto* self = args->guest->self; @@ -249,7 +404,7 @@ static unsigned __stdcall StartNativeThread(void* opaque) { cursor = reinterpret_cast(memory.BaseAddress) + memory.RegionSize; } self->threadId = std::this_thread::get_id(); - currentThread = self; + CurrentGuestState().current = self; if (!self->name.empty()) { const std::wstring description(self->name.begin(), self->name.end()); SetThreadDescription(GetCurrentThread(), description.c_str()); @@ -265,10 +420,46 @@ static unsigned __stdcall StartNativeThread(void* opaque) { auto guest = std::move(args->guest); args.reset(); RunThread(std::move(guest)); - currentThread = nullptr; + CurrentGuestState().current = nullptr; ReleaseThread(self); return 0; } +#else +static void* StartNativeThread(void* opaque) { + std::unique_ptr args(static_cast(opaque)); + auto* self = args->guest->self; + try { + std::uintptr_t low = 0; + std::uintptr_t high = 0; + const auto reserve = HostStackReserve(); + if (!HostStackLimits(&low, &high) || high - low < reserve + self->stackSize) + throw std::runtime_error("Cannot query guest thread stack"); + self->stackAddress = reinterpret_cast(high - reserve - self->stackSize); + self->threadId = std::this_thread::get_id(); + args->initialized.set_value(); + } catch (...) { + args->initialized.set_exception(std::current_exception()); + return nullptr; + } + if (!args->start.get()) + return nullptr; + auto* guest = args->guest.release(); + args.reset(); + struct ThreadGuard { + PthreadPrivate* self; + ~ThreadGuard() { + CurrentGuestState().current = nullptr; + ReleaseThread(self); + } + } guard{self}; + std::jmp_buf exitJump; + if (setjmp(exitJump) == 0) { + threadExitJump = &exitJump; + RunThread(std::unique_ptr(guest)); + } + threadExitJump = nullptr; + return nullptr; +} #endif extern "C" { @@ -291,9 +482,9 @@ int APS5_VABI scePthreadCreate(Pthread* thread, const PthreadAttr* attr, Pthread #ifdef _WIN32 SYSTEM_INFO system{}; GetSystemInfo(&system); - const std::size_t nativeStack = (p->stackSize + system.dwPageSize - 1) / system.dwPageSize * system.dwPageSize; - if (p->stackSize < 16384 || nativeStack > std::numeric_limits::max()) + if (p->stackSize < 16384 || p->stackSize > std::numeric_limits::max() - HOST_STACK_MARGIN - system.dwPageSize) throw std::runtime_error("scePthreadCreate: invalid Windows stack size"); + const std::size_t nativeStack = (p->stackSize + HOST_STACK_MARGIN + system.dwPageSize - 1) / system.dwPageSize * system.dwPageSize; auto native = std::make_unique(NativeThreadArgs{std::move(args), start.get_future(), {}}); auto initialized = native->initialized.get_future(); const auto handle = _beginthreadex(nullptr, static_cast(nativeStack), StartNativeThread, native.get(), 0, nullptr); @@ -315,29 +506,32 @@ int APS5_VABI scePthreadCreate(Pthread* thread, const PthreadAttr* attr, Pthread if (detached) ReleaseThread(published); #else - auto* self = p.get(); - p->_thr = std::thread([self, args = std::move(args), ready = start.get_future()]() mutable { - if (!ready.get()) return; - self->threadId = std::this_thread::get_id(); - struct ThreadGuard { - PthreadPrivate* self; - ~ThreadGuard() { - currentThread = nullptr; - ReleaseThread(self); - } - } guard{self}; - std::jmp_buf exitJump; - if (setjmp(exitJump) == 0) { - threadExitJump = &exitJump; - RunThread(std::move(args)); - } - threadExitJump = nullptr; - }); + const auto reserve = HostStackReserve(); + const auto page = static_cast(sysconf(_SC_PAGESIZE)); + if (p->stackSize < 16384 || p->stackSize > std::numeric_limits::max() - reserve - HOST_STACK_MARGIN - page) + throw std::runtime_error("scePthreadCreate: invalid stack size"); + const std::size_t nativeStack = (p->stackSize + reserve + HOST_STACK_MARGIN + page - 1) / page * page; + auto native = std::make_unique(NativeThreadArgs{std::move(args), start.get_future(), {}}); + auto initialized = native->initialized.get_future(); + pthread_attr_t nativeAttr; + if (const int result = pthread_attr_init(&nativeAttr); result != 0) + throw std::system_error(result, std::generic_category(), "Creating guest thread"); + int created = pthread_attr_setstacksize(&nativeAttr, nativeStack); + if (created == 0) + created = pthread_create(&p->hostThread, &nativeAttr, StartNativeThread, native.get()); + pthread_attr_destroy(&nativeAttr); + if (created != 0) + throw std::system_error(created, std::generic_category(), "Creating guest thread"); + native.release(); try { - if (detached) p->_thr.detach(); + initialized.get(); + if (detached) { + if (const int result = pthread_detach(p->hostThread); result != 0) + throw std::system_error(result, std::generic_category(), "Detaching guest thread"); + } } catch (...) { start.set_value(false); - p->_thr.join(); + pthread_join(p->hostThread, nullptr); throw; } auto* published = p.release(); @@ -352,15 +546,21 @@ int APS5_VABI scePthreadCreate(Pthread* thread, const PthreadAttr* attr, Pthread int APS5_VABI scePthreadJoin(Pthread thread, void** retval) { if (!thread) throw std::runtime_error("scePthreadJoin: null thread"); if (thread->_detached) return SCE_KERNEL_ERROR_EINVAL; - if (thread == currentThread) + if (thread == CurrentGuestState().current) throw std::runtime_error("scePthreadJoin: cannot join current thread"); + ThreadCancel::Check(); + { + std::unique_lock lock(thread->_join_mtx); + ThreadCancel::WaitUntil(thread->_join_cv, lock, std::nullopt, [&] { return thread->_finished.load(); }); + } #ifdef _WIN32 if (WaitForSingleObject(thread->nativeHandle, INFINITE) != WAIT_OBJECT_0) throw std::system_error(GetLastError(), std::system_category(), "Joining guest thread"); if (retval) *retval = thread->_retval; ReleaseThread(thread); #else - if (thread->_thr.joinable()) thread->_thr.join(); + if (const int result = pthread_join(thread->hostThread, nullptr); result != 0) + throw std::system_error(result, std::generic_category(), "Joining guest thread"); if (retval) *retval = thread->_retval; ReleaseThread(thread); #endif @@ -374,18 +574,21 @@ int APS5_VABI scePthreadDetach(Pthread thread) { #ifdef _WIN32 ReleaseThread(thread); #else - if (thread->_thr.joinable()) thread->_thr.detach(); + if (const int result = pthread_detach(thread->hostThread); result != 0) + throw std::system_error(result, std::generic_category(), "Detaching guest thread"); ReleaseThread(thread); #endif return SCE_OK; } void APS5_VABI scePthreadExit(void* retval) { - if (!currentThread) + if (!CurrentGuestState().current) throw std::runtime_error("scePthreadExit: current thread is not registered"); - auto* self = currentThread; + auto* self = CurrentGuestState().current; + CurrentGuestState().exiting = true; + while (CurrentGuestState().cleanupHandlers) __pthread_cleanup_pop_imp_nid_postfix(1); finishThread(self, retval); - currentThread = nullptr; + CurrentGuestState().current = nullptr; #ifdef _WIN32 ReleaseThread(self); _endthreadex(0); @@ -399,7 +602,7 @@ void APS5_VABI scePthreadExit(void* retval) { Pthread APS5_VABI scePthreadSelf() { #ifdef _WIN32 - if (!currentThread) { + if (!CurrentGuestState().current) { auto adopted = std::make_unique(); HANDLE handle = nullptr; if (!DuplicateHandle(GetCurrentProcess(), GetCurrentThread(), GetCurrentProcess(), &handle, 0, FALSE, DUPLICATE_SAME_ACCESS)) @@ -410,19 +613,20 @@ Pthread APS5_VABI scePthreadSelf() { adopted->references.store(1, std::memory_order_relaxed); TimedWait::BindThreadWaitState(&adopted->waitCount); SetStackFromHost(adopted.get()); - currentThread = adopted.release(); + CurrentGuestState().current = adopted.release(); } #else - if (!currentThread) { + if (!CurrentGuestState().current) { adoptedThread = std::make_unique(); adoptedThread->_detached = true; adoptedThread->_adopted = true; + BindHostCpuClock(adoptedThread.get()); adoptedThread->threadId = std::this_thread::get_id(); SetStackFromHost(adoptedThread.get()); - currentThread = adoptedThread.get(); + CurrentGuestState().current = adoptedThread.get(); } #endif - return currentThread; + return CurrentGuestState().current; } void APS5_VABI scePthreadYield() { @@ -430,9 +634,32 @@ void APS5_VABI scePthreadYield() { } int APS5_VABI scePthreadCancel(Pthread thread) { - (void)thread; - NotImplemented_nid_no_patch(__func__); - return 0; + if (!thread) return SCE_KERNEL_ERROR_ESRCH; + thread->cancelPending.store(true); + for (;;) { + { + std::lock_guard lock(thread->cancelLock); + if (!thread->cancelWait) return SCE_OK; + thread->cancelWait->NotifyAll(); + } + std::this_thread::yield(); + } +} + +void APS5_VABI __pthread_cleanup_push_imp_nid_postfix(void (APS5_VABI* routine)(void*), void* argument, void* info) { + auto* handler = static_cast(info); + handler->routine = routine; + handler->argument = argument; + handler->onHeap = 0; + handler->previous = CurrentGuestState().cleanupHandlers; + CurrentGuestState().cleanupHandlers = handler; +} + +void APS5_VABI __pthread_cleanup_pop_imp_nid_postfix(int execute) { + auto* handler = CurrentGuestState().cleanupHandlers; + if (!handler) return; + CurrentGuestState().cleanupHandlers = handler->previous; + if (execute) handler->routine(handler->argument); } int APS5_VABI scePthreadEqual(Pthread thread1, Pthread thread2) { @@ -465,11 +692,7 @@ int APS5_VABI scePthreadGetprio(Pthread thread, int* prio) { } int APS5_VABI scePthreadGetthreadid(void) { -#ifdef _WIN32 - return static_cast(GetCurrentThreadId()); -#else - return static_cast(std::hash{}(std::this_thread::get_id()) & 0x7fffffff); -#endif + return static_cast(scePthreadSelf()->tid); } int APS5_VABI scePthreadRename(Pthread thread, const char* name) { @@ -491,20 +714,23 @@ int APS5_VABI scePthreadSetaffinity(Pthread thread, KernelCpumask mask) { } int APS5_VABI scePthreadSetcancelstate(int state, int* old_state) { - static thread_local int cancelState = 0; - if (old_state) *old_state = cancelState; - cancelState = state; + if (state != GUEST_CANCEL_ENABLE && state != GUEST_CANCEL_DISABLE) return SCE_KERNEL_ERROR_EINVAL; + if (old_state) *old_state = CurrentGuestState().cancelState; + CurrentGuestState().cancelState = state; + if (CurrentGuestState().cancelType == GUEST_CANCEL_ASYNCHRONOUS) ThreadCancel::Check(); return SCE_OK; } int APS5_VABI scePthreadSetcanceltype(int type, int* old_type) { - static thread_local int cancelType = 0; - if (old_type) *old_type = cancelType; - cancelType = type; + if (type != GUEST_CANCEL_DEFERRED && type != GUEST_CANCEL_ASYNCHRONOUS) return SCE_KERNEL_ERROR_EINVAL; + if (old_type) *old_type = CurrentGuestState().cancelType; + CurrentGuestState().cancelType = type; + if (CurrentGuestState().cancelType == GUEST_CANCEL_ASYNCHRONOUS) ThreadCancel::Check(); return SCE_OK; } void APS5_VABI scePthreadTestcancel() { + ThreadCancel::Check(); } int APS5_VABI scePthreadSetprio(Pthread thread, int prio) { diff --git a/core/libs/prx/libkernel/Pthread/src/Tsd.cpp b/core/libs/prx/libkernel/Pthread/src/Tsd.cpp index b737b2f16..f04da09b6 100644 --- a/core/libs/prx/libkernel/Pthread/src/Tsd.cpp +++ b/core/libs/prx/libkernel/Pthread/src/Tsd.cpp @@ -1,4 +1,5 @@ #include "../include/Pthread.hpp" +#include "../include/WindowsThreadLocal.hpp" #include "prx/libc/include/General.hpp" #include #include @@ -54,7 +55,14 @@ struct ThreadValues { } }; -static thread_local ThreadValues g_values; +static ThreadValues& Values() { +#ifdef _WIN32 + return WindowsThreadLocal::Get(); +#else + static thread_local ThreadValues values; + return values; +#endif +} extern "C" { @@ -83,7 +91,7 @@ int APS5_VABI scePthreadKeyDelete(PthreadKey key) { void* APS5_VABI scePthreadGetspecific(PthreadKey key) { if (key < 0 || key >= MAX_KEYS) return nullptr; - const ThreadValue& entry = g_values.values[key]; + const ThreadValue& entry = Values().values[key]; return entry.sequence == g_keys[key].sequence.load(std::memory_order_acquire) ? entry.value : nullptr; } @@ -91,7 +99,7 @@ int APS5_VABI scePthreadSetspecific(PthreadKey key, void* value) { if (key < 0 || key >= MAX_KEYS) return SCE_KERNEL_ERROR_EINVAL; const std::uint64_t sequence = g_keys[key].sequence.load(std::memory_order_acquire); if (sequence == FREE_KEY) return SCE_KERNEL_ERROR_EINVAL; - g_values.values[key] = {value, sequence}; + Values().values[key] = {value, sequence}; return SCE_OK; } diff --git a/core/libs/prx/libkernel/Socket/include/SocketRuntime.hpp b/core/libs/prx/libkernel/Socket/include/SocketRuntime.hpp index 0ef89de10..88599c60f 100644 --- a/core/libs/prx/libkernel/Socket/include/SocketRuntime.hpp +++ b/core/libs/prx/libkernel/Socket/include/SocketRuntime.hpp @@ -1,6 +1,13 @@ #pragma once +#include +#include namespace GuestSockets { constexpr int FirstDescriptor = 0x10000000; int Close(int descriptor); bool IsOpen(int descriptor); +int Family(int descriptor); +std::int64_t Read(int descriptor, void* buffer, std::size_t length); +std::int64_t Write(int descriptor, const void* buffer, std::size_t length); } + +extern "C" bool GuestSocketIsOpen_nid_no_patch(int descriptor); diff --git a/core/libs/prx/libkernel/Socket/src/Udp.cpp b/core/libs/prx/libkernel/Socket/src/Udp.cpp index 5b856cb03..6468a1b17 100644 --- a/core/libs/prx/libkernel/Socket/src/Udp.cpp +++ b/core/libs/prx/libkernel/Socket/src/Udp.cpp @@ -67,6 +67,7 @@ int NativeError() { case WSAECONNREFUSED: return 61; case WSAEINTR: return 4; case WSAEINVAL: return 22; + case WSAESHUTDOWN: return 32; default: return 5; } #else @@ -180,6 +181,15 @@ bool GuestSockets::IsOpen(int descriptor) { std::lock_guard lock(socketsMutex); return sockets.contains(descriptor); } +int GuestSockets::Family(int descriptor) { + std::lock_guard lock(socketsMutex); + const auto found = sockets.find(descriptor); + return found == sockets.end() ? -1 : found->second->family; +} + +extern "C" bool GuestSocketIsOpen_nid_no_patch(int descriptor) { + return GuestSockets::IsOpen(descriptor); +} namespace { #ifdef _WIN32 @@ -731,3 +741,12 @@ int APS5_VABI poll_nid_postfix(GuestPollDescriptor* descriptors, std::uint32_t c return ready; } } + +std::int64_t GuestSockets::Read(int descriptor, void* buffer, std::size_t length) { + if (length == 0) return IsOpen(descriptor) ? 0 : Fail(9); + return recv_nid_postfix(descriptor, buffer, std::min(length, INT_MAX), 0); +} + +std::int64_t GuestSockets::Write(int descriptor, const void* buffer, std::size_t length) { + return send_nid_postfix(descriptor, buffer, length, 0); +} diff --git a/core/libs/prx/libkernel/System/src/Exception.cpp b/core/libs/prx/libkernel/System/src/Exception.cpp index e0ac735c3..718e4119d 100644 --- a/core/libs/prx/libkernel/System/src/Exception.cpp +++ b/core/libs/prx/libkernel/System/src/Exception.cpp @@ -5,6 +5,7 @@ #include "prx/libkernel/KernelErrors.hpp" #include "prx/libkernel/Pthread/include/Pthread.hpp" #include +#include #include #include #include @@ -16,6 +17,7 @@ extern "C" Pthread APS5_VABI scePthreadSelf(); #ifdef _WIN32 extern "C" void Aps5RedirectedEntryStub(); +extern "C" [[noreturn]] void Aps5RedirectedExit(CONTEXT* context, const void* vectors, void (*restore)(CONTEXT*, EXCEPTION_RECORD*)); #endif namespace { @@ -101,13 +103,22 @@ GuestExceptionHandler Handler(int signum) { #ifdef _WIN32 constexpr std::size_t RedZone = 128; constexpr std::size_t HomeArea = 32; +constexpr auto RetryLimit = std::chrono::seconds(1); +constexpr std::size_t VectorBytes = 16 * 32; struct Delivery { GuestExceptionHandler handler; int signum; CONTEXT context; + std::uint8_t vectors[VectorBytes]; + std::uint64_t saveVectors; }; +const bool SaveVectors = [] { + __builtin_cpu_init(); + return __builtin_cpu_supports("avx") != 0; +}(); + void Deliver(GuestExceptionHandler handler, int signum, CONTEXT& context) { GuestUcontext ucontext{}; auto& m = ucontext.mcontext; @@ -159,8 +170,7 @@ void Deliver(GuestExceptionHandler handler, int signum, CONTEXT& context) { [[noreturn]] void RedirectedEntry(Delivery* delivery) { CONTEXT context = delivery->context; Deliver(delivery->handler, delivery->signum, context); - RtlRestoreContext(&context, nullptr); - std::abort(); + Aps5RedirectedExit(&context, delivery->saveVectors != 0 ? delivery->vectors : nullptr, RtlRestoreContext); } bool StackWritable(DWORD64 low, DWORD64 high) { @@ -185,11 +195,28 @@ void CALLBACK WaitingEntry(ULONG_PTR parameter) { static_assert(HomeArea + 8 == 40, "Aps5RedirectedEntryStub finds the delivery 40 bytes above its stack pointer"); static_assert(offsetof(Delivery, context) == 16 && offsetof(CONTEXT, Rax) == 0x78 && offsetof(CONTEXT, Rbp) == 0xa0 && offsetof(CONTEXT, R15) == 0xf0, "Aps5RedirectedEntryStub stores the live registers into the delivery's context"); +static_assert(offsetof(Delivery, vectors) == 1248 && offsetof(Delivery, saveVectors) == 1760, "Aps5RedirectedEntryStub stores ymm0-ymm15 into the delivery when saveVectors is set"); bool Exited(HANDLE native) { return WaitForSingleObject(native, 0) == WAIT_OBJECT_0; } +bool RestoringContext(DWORD64 rip) { + static const std::array stubs = [] { + const HMODULE ntdll = GetModuleHandleW(L"ntdll.dll"); + return std::array{reinterpret_cast(GetProcAddress(ntdll, "NtContinue")), reinterpret_cast(GetProcAddress(ntdll, "NtContinueEx"))}; + }(); + for (const DWORD64 stub : stubs) + if (stub != 0 && rip - stub < 0x20) return true; + return false; +} + +bool InWinpthread(DWORD64 rip) { + static const HMODULE winpthread = GetModuleHandleW(L"libwinpthread-1.dll"); + MEMORY_BASIC_INFORMATION info{}; + return winpthread != nullptr && VirtualQuery(reinterpret_cast(rip), &info, sizeof(info)) == sizeof(info) && info.AllocationBase == winpthread; +} + bool RaiseOn(Pthread thread, GuestExceptionHandler handler, int signum) { if (thread == scePthreadSelf()) { CONTEXT context{}; @@ -198,27 +225,34 @@ bool RaiseOn(Pthread thread, GuestExceptionHandler handler, int signum) { return true; } const auto native = static_cast(thread->nativeHandle); - auto queued = std::make_unique(Delivery{handler, signum, {}}); - if (SuspendThread(native) == static_cast(-1)) { - if (Exited(native)) return false; - throw std::runtime_error("sceKernelRaiseException: cannot suspend the target thread"); - } - if (Exited(native)) { + auto queued = std::make_unique(Delivery{handler, signum, {}, {}, 0}); + alignas(16) Delivery delivery{handler, signum, {}, {}, SaveVectors ? 1u : 0u}; + const auto retryDeadline = std::chrono::steady_clock::now() + RetryLimit; + for (;;) { + if (SuspendThread(native) == static_cast(-1)) { + if (Exited(native)) return false; + throw std::runtime_error("sceKernelRaiseException: cannot suspend the target thread"); + } + if (Exited(native)) { + ResumeThread(native); + return false; + } + delivery.context.ContextFlags = CONTEXT_FULL | CONTEXT_SEGMENTS; + if (!GetThreadContext(native, &delivery.context)) { + ResumeThread(native); + throw std::runtime_error("sceKernelRaiseException: cannot read the target thread context"); + } + if (thread->waitCount.load(std::memory_order_seq_cst) > 0) { + const bool accepted = QueueUserAPC(WaitingEntry, native, reinterpret_cast(queued.get())) != 0; + ResumeThread(native); + if (!accepted) throw std::runtime_error("sceKernelRaiseException: cannot queue delivery to the waiting thread"); + queued.release(); + return true; + } + if (!RestoringContext(delivery.context.Rip) && !InWinpthread(delivery.context.Rip)) break; ResumeThread(native); - return false; - } - if (thread->waitCount.load(std::memory_order_seq_cst) > 0) { - const bool accepted = QueueUserAPC(WaitingEntry, native, reinterpret_cast(queued.get())) != 0; - ResumeThread(native); - if (!accepted) throw std::runtime_error("sceKernelRaiseException: cannot queue delivery to the waiting thread"); - queued.release(); - return true; - } - alignas(16) Delivery delivery{handler, signum, {}}; - delivery.context.ContextFlags = CONTEXT_FULL | CONTEXT_FLOATING_POINT; - if (!GetThreadContext(native, &delivery.context)) { - ResumeThread(native); - throw std::runtime_error("sceKernelRaiseException: cannot read the target thread context"); + if (std::chrono::steady_clock::now() >= retryDeadline) throw std::runtime_error("sceKernelRaiseException: the target thread stayed inside NtContinue or winpthreads for 1 s"); + SwitchToThread(); } const DWORD64 slot = (delivery.context.Rsp - RedZone - sizeof(Delivery)) & ~static_cast(15); if (!StackWritable(slot - HomeArea - 8, delivery.context.Rsp - RedZone)) { @@ -262,8 +296,50 @@ asm(".text\n" " movq %r13, 280(%rsp)\n" " movq %r14, 288(%rsp)\n" " movq %r15, 296(%rsp)\n" + " cmpq $0, 1800(%rsp)\n" + " je 1f\n" + " vmovdqu %ymm0, 1288(%rsp)\n" + " vmovdqu %ymm1, 1320(%rsp)\n" + " vmovdqu %ymm2, 1352(%rsp)\n" + " vmovdqu %ymm3, 1384(%rsp)\n" + " vmovdqu %ymm4, 1416(%rsp)\n" + " vmovdqu %ymm5, 1448(%rsp)\n" + " vmovdqu %ymm6, 1480(%rsp)\n" + " vmovdqu %ymm7, 1512(%rsp)\n" + " vmovdqu %ymm8, 1544(%rsp)\n" + " vmovdqu %ymm9, 1576(%rsp)\n" + " vmovdqu %ymm10, 1608(%rsp)\n" + " vmovdqu %ymm11, 1640(%rsp)\n" + " vmovdqu %ymm12, 1672(%rsp)\n" + " vmovdqu %ymm13, 1704(%rsp)\n" + " vmovdqu %ymm14, 1736(%rsp)\n" + " vmovdqu %ymm15, 1768(%rsp)\n" + "1:\n" " leaq 40(%rsp), %rcx\n" - " jmp Aps5RedirectedEntry\n"); + " jmp Aps5RedirectedEntry\n" + ".globl Aps5RedirectedExit\n" + "Aps5RedirectedExit:\n" + " testq %rdx, %rdx\n" + " je 1f\n" + " vmovdqu 0(%rdx), %ymm0\n" + " vmovdqu 32(%rdx), %ymm1\n" + " vmovdqu 64(%rdx), %ymm2\n" + " vmovdqu 96(%rdx), %ymm3\n" + " vmovdqu 128(%rdx), %ymm4\n" + " vmovdqu 160(%rdx), %ymm5\n" + " vmovdqu 192(%rdx), %ymm6\n" + " vmovdqu 224(%rdx), %ymm7\n" + " vmovdqu 256(%rdx), %ymm8\n" + " vmovdqu 288(%rdx), %ymm9\n" + " vmovdqu 320(%rdx), %ymm10\n" + " vmovdqu 352(%rdx), %ymm11\n" + " vmovdqu 384(%rdx), %ymm12\n" + " vmovdqu 416(%rdx), %ymm13\n" + " vmovdqu 448(%rdx), %ymm14\n" + " vmovdqu 480(%rdx), %ymm15\n" + "1:\n" + " xorl %edx, %edx\n" + " jmp *%r8\n"); #endif extern "C" { diff --git a/core/libs/prx/libkernel/System/src/Process.cpp b/core/libs/prx/libkernel/System/src/Process.cpp index 13e447cd1..4df88ebd6 100644 --- a/core/libs/prx/libkernel/System/src/Process.cpp +++ b/core/libs/prx/libkernel/System/src/Process.cpp @@ -3,6 +3,7 @@ #define NOMINMAX #endif #include +#include #else #include #include @@ -13,6 +14,7 @@ #include "prx/libc/include/ApplicationHeap.hpp" #include "prx/libc/include/Shutdown.hpp" #include "prx/libkernel/DirectMemory/DirectMemory.hpp" +#include "prx/libkernel/Pthread/include/Pthread.hpp" #include #include #include @@ -33,24 +35,42 @@ #include #endif +extern "C" int* APS5_VABI __error_nid_postfix(); + namespace { constexpr int sceInvalidArgument = static_cast(0x80020016u); +constexpr int errnoNoChild = 10; +constexpr int errnoInvalidArgument = 22; +constexpr unsigned freebsdWaitOptions = 0x8000003Fu; std::atomic gpoBits{0}; constexpr std::array openPsId{'A', 'n', 'y', 'P', 'S', '5', 'O', 'p', 'e', 'n', 'P', 's', 'I', 'd', 0, 1}; +#ifdef _WIN32 +std::string toUtf8(const wchar_t* value) { + const int size = WideCharToMultiByte(CP_UTF8, WC_ERR_INVALID_CHARS, value, -1, nullptr, 0, nullptr, nullptr); + if (size <= 0) + throw std::system_error(GetLastError(), std::system_category(), "Converting a process argument to UTF-8"); + std::string text(static_cast(size), '\0'); + if (WideCharToMultiByte(CP_UTF8, WC_ERR_INVALID_CHARS, value, -1, text.data(), size, nullptr, nullptr) != size) + throw std::system_error(GetLastError(), std::system_category(), "Converting a process argument to UTF-8"); + text.pop_back(); + return text; +} +#endif + class ProcessArguments { public: ProcessArguments() { #ifdef _WIN32 - std::array path{}; - const auto size = GetModuleFileNameA(nullptr, path.data(), static_cast(path.size())); - if (size == 0) - throw std::system_error(GetLastError(), std::system_category(), "Reading executable path"); - if (size >= path.size()) - throw std::runtime_error("Executable path exceeds the guest argument buffer"); - arguments.emplace_back(path.data(), size); + int count = 0; + const std::unique_ptr values(CommandLineToArgvW(GetCommandLineW(), &count), + [](LPWSTR* parsed) { LocalFree(parsed); }); + if (!values) + throw std::system_error(GetLastError(), std::system_category(), "Reading process arguments"); + for (int index = 0; index < count; ++index) + arguments.push_back(toUtf8(values.get()[index])); #else std::ifstream stream("/proc/self/cmdline", std::ios::binary); if (!stream) @@ -85,11 +105,46 @@ ProcessArguments& getProcessArguments() { } void validateSchedulingPolicy(int policy) { - if (policy != 1 && policy != 3) + if (policy < 1 || policy > 3) throw std::invalid_argument("Unsupported guest scheduling policy"); } +constexpr int guestFault = 14; +constexpr int guestInvalid = 22; +constexpr int guestRlimitData = 2; +constexpr int guestRlimitCount = 15; +constexpr std::int64_t guestRlimitInfinity = std::numeric_limits::max(); + +#ifdef _WIN32 +std::int64_t hostMemoryLimit() { + BOOL inJob = FALSE; + if (!IsProcessInJob(GetCurrentProcess(), nullptr, &inJob)) + throw std::system_error(GetLastError(), std::system_category(), "getrlimit: IsProcessInJob failed"); + if (!inJob) + return guestRlimitInfinity; + JOBOBJECT_EXTENDED_LIMIT_INFORMATION limits{}; + if (!QueryInformationJobObject(nullptr, JobObjectExtendedLimitInformation, &limits, sizeof(limits), nullptr)) + throw std::system_error(GetLastError(), std::system_category(), "getrlimit: QueryInformationJobObject failed"); + if ((limits.BasicLimitInformation.LimitFlags & JOB_OBJECT_LIMIT_PROCESS_MEMORY) == 0 || limits.ProcessMemoryLimit > static_cast(guestRlimitInfinity)) + return guestRlimitInfinity; + return static_cast(limits.ProcessMemoryLimit); } +#else +std::int64_t toGuestLimit(rlim_t value) { + if (value == RLIM_INFINITY || value > static_cast(guestRlimitInfinity)) + return guestRlimitInfinity; + return static_cast(value); +} +#endif + +} + +extern "C" int* APS5_VABI __error_nid_postfix(); + +struct GuestResourceLimit { + std::int64_t rlim_cur; + std::int64_t rlim_max; +}; struct GuestResourceUsage { KernelTimeval ru_utime; @@ -110,10 +165,14 @@ struct GuestResourceUsage { std::int64_t ru_nivcsw; }; +extern "C" Pthread APS5_VABI scePthreadSelf(); + extern "C" { // unknown data const char* __progname_nid_postfix = "eboot.bin"; +static char* emptyEnvironment[1]; +char** environ_nid_postfix = emptyEnvironment; int APS5_VABI getargc_nid_postfix(void) { return getProcessArguments().GetCount(); @@ -138,12 +197,44 @@ int APS5_VABI getpid_nid_postfix(void) { return static_cast(pid); } +int APS5_VABI getuid_nid_postfix(void) { + return 0; +} + +int APS5_VABI geteuid_nid_postfix(void) { + return 0; +} + +int APS5_VABI getgid_nid_postfix(void) { + return 0; +} + +int APS5_VABI getegid_nid_postfix(void) { + return 0; +} + +int APS5_VABI issetugid_nid_postfix(void) { + return 0; +} + void APS5_VABI exit_nid_postfix(int code) { LibcExit_nid_no_patch(code); } [[noreturn]] void APS5_VABI _exit_nid_postfix(int status) { - std::_Exit(status); + LibcTerminate_nid_no_patch(status); +} + +int APS5_VABI system_nid_postfix(const char* command) { + constexpr int shellNotExecuted = 127 << 8; + return command == nullptr ? 1 : shellNotExecuted; +} + +int APS5_VABI waitpid_nid_postfix(int pid, int* status, int options) { + (void)pid; + (void)status; + *__error_nid_postfix() = (static_cast(options) & ~freebsdWaitOptions) != 0 ? errnoInvalidArgument : errnoNoChild; + return -1; } int APS5_VABI sceKernelGetCurrentCpu(void) { @@ -157,13 +248,13 @@ int APS5_VABI sceKernelGetCurrentCpu(void) { throw std::system_error(GetLastError(), std::system_category(), "Reading processor group size"); index += count; } - return static_cast(index); #else const int cpu = ::sched_getcpu(); if (cpu < 0) throw std::system_error(errno, std::generic_category(), "Reading current processor"); - return cpu; + const auto index = static_cast(cpu); #endif + return GuestCpuFromHost(index, scePthreadSelf()->affinity.load(std::memory_order_relaxed)); } std::uint64_t APS5_VABI sceKernelGetGPI(void) { @@ -278,6 +369,31 @@ int APS5_VABI getrusage_nid_postfix(int who, GuestResourceUsage* usage) { return 0; } +int APS5_VABI getrlimit_nid_postfix(int resource, GuestResourceLimit* limit) { + if (resource < 0 || resource >= guestRlimitCount) { + *__error_nid_postfix() = guestInvalid; + return -1; + } + if (limit == nullptr) { + *__error_nid_postfix() = guestFault; + return -1; + } + if (resource != guestRlimitData) + throw std::runtime_error("getrlimit: unsupported resource " + std::to_string(resource)); +#ifdef _WIN32 + const std::int64_t memory = hostMemoryLimit(); + limit->rlim_cur = memory; + limit->rlim_max = memory; +#else + rlimit native{}; + if (::getrlimit(RLIMIT_DATA, &native) != 0) + throw std::system_error(errno, std::generic_category(), "getrlimit: getrlimit failed"); + limit->rlim_cur = toGuestLimit(native.rlim_cur); + limit->rlim_max = toGuestLimit(native.rlim_max); +#endif + return 0; +} + int APS5_VABI sceKernelIsTrinityMode(void) { return 0; } @@ -290,4 +406,16 @@ int APS5_VABI sceKernelGetOperationMode(int* mode, int* submode) { return 0; } +int APS5_VABI seteuid_nid_postfix(std::uint32_t euid) { + if (euid != 0) + throw std::runtime_error("seteuid: unsupported user id " + std::to_string(euid)); + return 0; +} + +int APS5_VABI setegid_nid_postfix(std::uint32_t egid) { + if (egid != 0) + throw std::runtime_error("setegid: unsupported group id " + std::to_string(egid)); + return 0; +} + } diff --git a/core/libs/prx/libkernel/System/src/Sandbox.cpp b/core/libs/prx/libkernel/System/src/Sandbox.cpp new file mode 100644 index 000000000..dd3918c68 --- /dev/null +++ b/core/libs/prx/libkernel/System/src/Sandbox.cpp @@ -0,0 +1,6 @@ +#include "prx/libc/include/general/VabiMacros.hpp" + +extern "C" const char* APS5_VABI sceKernelGetFsSandboxRandomWord() { + static constexpr char word[] = "gkbHtXc3Wq"; + return word; +} diff --git a/core/libs/prx/libkernel/System/src/SignalSets.cpp b/core/libs/prx/libkernel/System/src/SignalSets.cpp new file mode 100644 index 000000000..f2831d86c --- /dev/null +++ b/core/libs/prx/libkernel/System/src/SignalSets.cpp @@ -0,0 +1,62 @@ +#include "prx/libc/include/general/VabiMacros.hpp" +#include +#include +#include +#include + +extern "C" int* APS5_VABI __error_nid_postfix(); + +struct GuestSignalSet { + std::uint32_t bits[4]; +}; +static_assert(sizeof(GuestSignalSet) == 16); + +namespace { +constexpr int MaximumSignal = 128; +constexpr int InvalidArgument = 22; + +bool ValidSignal(int signal) { return signal > 0 && signal <= MaximumSignal; } +std::size_t WordIndex(int signal) { return static_cast(signal - 1) >> 5; } +std::uint32_t Bit(int signal) { return 1u << ((signal - 1) & 31); } +int RejectSignal() { + *__error_nid_postfix() = InvalidArgument; + return -1; +} +void RequireSet(const GuestSignalSet* set, const char* function) { + if (set == nullptr) throw std::invalid_argument(std::string(function) + ": set is null"); +} +} + +extern "C" { +int APS5_VABI sigemptyset_nid_postfix(GuestSignalSet* set) { + RequireSet(set, "sigemptyset"); + for (auto& word : set->bits) word = 0; + return 0; +} + +int APS5_VABI sigfillset_nid_postfix(GuestSignalSet* set) { + RequireSet(set, "sigfillset"); + for (auto& word : set->bits) word = ~0u; + return 0; +} + +int APS5_VABI sigaddset_nid_postfix(GuestSignalSet* set, int signal) { + RequireSet(set, "sigaddset"); + if (!ValidSignal(signal)) return RejectSignal(); + set->bits[WordIndex(signal)] |= Bit(signal); + return 0; +} + +int APS5_VABI sigdelset_nid_postfix(GuestSignalSet* set, int signal) { + RequireSet(set, "sigdelset"); + if (!ValidSignal(signal)) return RejectSignal(); + set->bits[WordIndex(signal)] &= ~Bit(signal); + return 0; +} + +int APS5_VABI sigismember_nid_postfix(const GuestSignalSet* set, int signal) { + RequireSet(set, "sigismember"); + if (!ValidSignal(signal)) return RejectSignal(); + return (set->bits[WordIndex(signal)] & Bit(signal)) != 0 ? 1 : 0; +} +} diff --git a/core/libs/prx/libkernel/System/src/Signals.cpp b/core/libs/prx/libkernel/System/src/Signals.cpp index 7c1a340eb..f1ee8327e 100644 --- a/core/libs/prx/libkernel/System/src/Signals.cpp +++ b/core/libs/prx/libkernel/System/src/Signals.cpp @@ -2,17 +2,38 @@ #include "prx/libc/include/general/VabiMacros.hpp" #include #include +#include #include #include +#include extern "C" int* APS5_VABI __error_nid_postfix(); +extern "C" int APS5_VABI getpid_nid_postfix(void); + +struct GuestSignalSet { + std::uint32_t bits[4]; +}; + +struct GuestSigaction { + std::uintptr_t handler; + int flags; + GuestSignalSet mask; +}; +static_assert(sizeof(GuestSigaction) == 32 && offsetof(GuestSigaction, flags) == 8 && offsetof(GuestSigaction, mask) == 12); + namespace { using GuestHandler = void (APS5_VABI *)(int); -std::atomic handlers[32]{}; +constexpr int MaxSignal = 128; +constexpr int GuestSigkill = 9; +constexpr int GuestSigstop = 17; +constexpr int SaRestart = 0x2; +constexpr int SaSiginfo = 0x40; +std::atomic handlers[MaxSignal + 1]{}; static_assert(std::atomic::is_always_lock_free); std::atomic blockedMask{0}; static_assert(std::atomic::is_always_lock_free); std::mutex registration; +GuestSigaction dispositions[MaxSignal + 1]{}; int NativeSignal(int guest) { switch (guest) { case 2: return SIGINT; @@ -33,15 +54,37 @@ void Dispatch(int native) { // Preserve the guest's persistent registration across CRT delivery. std::signal(native, Dispatch); #endif - if ((blockedMask.load() & (1u << guest)) != 0) return; + if ((blockedMask.load() & (1u << (guest - 1))) != 0) return; const auto callback = handlers[guest].load(); if (reinterpret_cast(callback) > 1) callback(guest); } +bool Delivered(const GuestSigaction& action) { + return action.handler > 1 && (action.flags & SaSiginfo) == 0; +} +bool Install(int guest, const GuestSigaction& action) { + const int native = NativeSignal(guest); + if (!native) return true; + const auto previous = handlers[guest].exchange(Delivered(action) ? reinterpret_cast(action.handler) : nullptr); + auto hostHandler = action.handler == 1 ? SIG_IGN : Delivered(action) ? Dispatch : SIG_DFL; + if (std::signal(native, hostHandler) == SIG_ERR) { + handlers[guest].store(previous); + return false; + } + return true; +} } -struct GuestSignalSet { - std::uint32_t bits[4]; +struct GuestStack { + void* sp; + std::size_t size; + int flags; }; +static_assert(sizeof(GuestStack) == 24 && offsetof(GuestStack, flags) == 16); +namespace { +constexpr int SsDisable = 4; +constexpr std::size_t MinSignalStackSize = 2048; +thread_local GuestStack alternateStack{nullptr, 0, SsDisable}; +} extern "C" { GuestHandler APS5_VABI signal_nid_postfix(int guest, GuestHandler handler) { @@ -49,23 +92,53 @@ GuestHandler APS5_VABI signal_nid_postfix(int guest, GuestHandler handler) { const int native = NativeSignal(guest); if (!native || handler == invalid) { *__error_nid_postfix() = 22; return invalid; } std::lock_guard lock(registration); - const auto previous = handlers[guest].exchange(handler); - const auto address = reinterpret_cast(handler); - auto hostHandler = address == 0 ? SIG_DFL : address == 1 ? SIG_IGN : Dispatch; - if (std::signal(native, hostHandler) == SIG_ERR) { - handlers[guest].store(previous); - *__error_nid_postfix() = 22; - return invalid; - } + const GuestSigaction action{reinterpret_cast(handler), SaRestart, {}}; + if (!Install(guest, action)) { *__error_nid_postfix() = 22; return invalid; } + const auto previous = reinterpret_cast(dispositions[guest].handler); + dispositions[guest] = action; return previous; } +int APS5_VABI sigaction_nid_postfix(int guest, const GuestSigaction* action, GuestSigaction* previous) { + if (guest < 1 || guest > MaxSignal) { *__error_nid_postfix() = 22; return -1; } + if (action && (guest == GuestSigkill || guest == GuestSigstop) && action->handler != 0) { *__error_nid_postfix() = 22; return -1; } + std::lock_guard lock(registration); + const GuestSigaction old = dispositions[guest]; + if (action) { + if (!Install(guest, *action)) { *__error_nid_postfix() = 22; return -1; } + dispositions[guest] = *action; + } + if (previous) *previous = old; + return 0; +} int APS5_VABI raise_nid_postfix(int guest) { const int native = NativeSignal(guest); if (!native) { *__error_nid_postfix() = 22; return -1; } + { + std::lock_guard lock(registration); + if (dispositions[guest].handler > 1 && !Delivered(dispositions[guest])) throw std::runtime_error("raise: SA_SIGINFO handlers are not delivered"); + } const int result = std::raise(native); if (result) *__error_nid_postfix() = 22; return result ? -1 : 0; } +int APS5_VABI kill_nid_postfix(int pid, int guest) { + if (guest < 0 || guest > MaxSignal) { *__error_nid_postfix() = 22; return -1; } + const int self = getpid_nid_postfix(); + if (pid != self && pid != 0 && pid != -self) { *__error_nid_postfix() = 3; return -1; } + return guest == 0 ? 0 : raise_nid_postfix(guest); +} +int APS5_VABI sigaltstack_nid_postfix(const GuestStack* stack, GuestStack* previous) { + GuestStack replacement = alternateStack; + if (stack != nullptr) { + if ((stack->flags & ~SsDisable) != 0) { *__error_nid_postfix() = 22; return -1; } + if (stack->flags & SsDisable) replacement.flags = SsDisable; + else if (stack->size < MinSignalStackSize) { *__error_nid_postfix() = 12; return -1; } + else replacement = {stack->sp, stack->size, 0}; + } + if (previous != nullptr) *previous = alternateStack; + alternateStack = replacement; + return 0; +} int APS5_VABI _sigprocmask_nid_postfix(int how, const GuestSignalSet* set, GuestSignalSet* previousSet) { std::lock_guard lock(registration); if (previousSet != nullptr) { diff --git a/core/libs/prx/libkernel/System/src/SystemConfiguration.cpp b/core/libs/prx/libkernel/System/src/SystemConfiguration.cpp index b5a5e777b..3b245aa97 100644 --- a/core/libs/prx/libkernel/System/src/SystemConfiguration.cpp +++ b/core/libs/prx/libkernel/System/src/SystemConfiguration.cpp @@ -1,6 +1,11 @@ #include "prx/libc/include/general/VabiMacros.hpp" #include "prx/libkernel/DirectMemory/DirectMemory.hpp" +#include +#include #include +#include +#include +#include #ifdef _WIN32 #define WIN32_LEAN_AND_MEAN #include @@ -10,6 +15,25 @@ extern "C" int* APS5_VABI __error_nid_postfix(); +namespace { + +int CopyOut(const void* value, std::size_t size, void* oldValue, std::size_t* oldLength) { + if (oldValue == nullptr) { + if (oldLength != nullptr) *oldLength = size; + return 0; + } + const std::size_t copied = std::min(oldLength != nullptr ? *oldLength : 0, size); + std::memcpy(oldValue, value, copied); + if (oldLength != nullptr) *oldLength = copied; + if (copied < size) { + *__error_nid_postfix() = 12; + return -1; + } + return 0; +} + +} + extern "C" { // Guest long is 64 bits, including when the Windows host long is 32 bits. std::int64_t APS5_VABI sysconf_nid_postfix(int name) { @@ -48,4 +72,38 @@ std::int64_t APS5_VABI sysconf_nid_postfix(int name) { *__error_nid_postfix() = saved; return result; } +int APS5_VABI sysctl_nid_postfix(const int* name, std::uint32_t nameLength, void* oldValue, std::size_t* oldLength, const void* newValue, std::size_t newLength) { + (void)newLength; + if (nameLength < 2 || nameLength > 24) { + *__error_nid_postfix() = 22; + return -1; + } + if (name == nullptr) { + *__error_nid_postfix() = 14; + return -1; + } + if (nameLength != 2 || name[0] != 6 || name[1] != 3) { + std::string mib; + for (std::uint32_t i = 0; i < nameLength; ++i) mib += (i == 0 ? "" : ".") + std::to_string(name[i]); + throw std::runtime_error(std::string(__func__) + ": unsupported MIB " + mib); + } + if (newValue != nullptr) { + *__error_nid_postfix() = 1; + return -1; + } + const std::int64_t processors = sysconf_nid_postfix(58); + if (processors <= 0) return -1; + const int value = static_cast(processors); + return CopyOut(&value, sizeof(value), oldValue, oldLength); +} + +int APS5_VABI sysctlbyname_nid_postfix(const char* name, void* oldValue, std::size_t* oldLength, const void* newValue, std::size_t newLength) { + if (name == nullptr) { + *__error_nid_postfix() = 14; + return -1; + } + if (std::strcmp(name, "hw.ncpu") != 0) throw std::runtime_error(std::string(__func__) + ": unsupported name " + name); + static constexpr int mib[] = {6, 3}; + return sysctl_nid_postfix(mib, 2, oldValue, oldLength, newValue, newLength); +} } diff --git a/core/libs/prx/libkernel/Time/Time.cpp b/core/libs/prx/libkernel/Time/Time.cpp index 395f85829..70c5da380 100644 --- a/core/libs/prx/libkernel/Time/Time.cpp +++ b/core/libs/prx/libkernel/Time/Time.cpp @@ -1,8 +1,10 @@ #include "prx/libkernel/Time/include/Time.hpp" #include "prx/libkernel/Time/include/TimedWait.hpp" +#include "prx/libkernel/Pthread/include/Cancel.hpp" #include "prx/libc/include/General.hpp" #include "prx/libkernel/KernelErrors.hpp" +#include "prx/libkernel/Pthread/include/Pthread.hpp" #include #include #include @@ -29,6 +31,18 @@ extern "C" int* APS5_VABI __error_nid_postfix(); +static constexpr int GUEST_EINVAL = 22; +static constexpr int GUEST_CLOCK_THREAD_CPUTIME_ID = 14; + +static bool IsCpuClock(int clockId) { + const auto bits = static_cast(clockId); + if ((bits & CPU_CLOCK_BIT) == 0) + return false; + if ((bits & CPU_CLOCK_PROCESS_BIT) != 0) + throw std::runtime_error("clock: unsupported process CPU clock_id " + std::to_string(clockId)); + return true; +} + static std::uint64_t RawMonotonicNanos() { #ifdef _WIN32 return TimedWait::NowNanos(); @@ -147,11 +161,13 @@ static const bool g_timerSelfTest = [] { extern "C" { std::uint64_t APS5_VABI sceKernelGetProcessTime() { - return (GetMonotonicNanos() - GetStartNanos()) / 1000ULL; + const std::uint64_t start = GetStartNanos(); + return (GetMonotonicNanos() - start) / 1000ULL; } std::uint64_t APS5_VABI sceKernelGetProcessTimeCounter() { - return GetMonotonicNanos() - GetStartNanos(); + const std::uint64_t start = GetStartNanos(); + return GetMonotonicNanos() - start; } std::uint64_t APS5_VABI sceKernelGetProcessTimeCounterFrequency() { @@ -221,9 +237,15 @@ void KernelTraceWait_nid_postfix(const char* kind, const void* caller, std::uint for (auto& [name, s] : sites) s = Site{}; } +static void CancellableSleep(std::uint64_t nanos) { + ThreadCancel::Check(); + TimedWait::SleepNanos(nanos); + ThreadCancel::Check(); +} + int APS5_VABI sceKernelUsleep_nid_postfix(KernelUseconds microseconds) { TraceSleep(__builtin_return_address(0), microseconds); - TimedWait::SleepNanos(static_cast(microseconds) * 1000ULL); + CancellableSleep(static_cast(microseconds) * 1000ULL); return 0; } @@ -233,8 +255,8 @@ static int SleepForRequest(const KernelTimespec* rqtp, KernelTimespec* rmtp) { if (rqtp == nullptr) return guestFault; if (rqtp->tv_nsec < 0 || rqtp->tv_nsec >= 1000000000LL) return guestInvalid; if (rqtp->tv_sec >= 0) { - TimedWait::SleepNanos(static_cast(rqtp->tv_sec) * 1000000000ULL + - static_cast(rqtp->tv_nsec)); + CancellableSleep(static_cast(rqtp->tv_sec) * 1000000000ULL + + static_cast(rqtp->tv_nsec)); } if (rmtp != nullptr) { rmtp->tv_sec = 0; @@ -260,7 +282,7 @@ int APS5_VABI _nanosleep_nid_postfix(const KernelTimespec* rqtp, KernelTimespec* } int APS5_VABI usleep_nid_postfix(KernelUseconds microseconds) { - TimedWait::SleepNanos(static_cast(microseconds) * 1000ULL); + CancellableSleep(static_cast(microseconds) * 1000ULL); return 0; } @@ -329,6 +351,16 @@ int APS5_VABI clock_gettime_nid_postfix(int clockId, KernelTimespec* tp) { if (tp == nullptr) { APS5_INVALID_ARG_EX; } + if (IsCpuClock(clockId)) { + std::uint64_t nanos = 0; + if (!GuestThreadCpuNanos(static_cast(static_cast(clockId) & CPU_CLOCK_ID_MASK), &nanos)) { + *__error_nid_postfix() = GUEST_EINVAL; + return -1; + } + tp->tv_sec = static_cast(nanos / 1000000000ULL); + tp->tv_nsec = static_cast(nanos % 1000000000ULL); + return 0; + } if (clockId == 1 || clockId == 2) { const std::uint64_t nanos = ProcessCpuNanos(clockId == 2); tp->tv_sec = static_cast(nanos / 1000000000ULL); @@ -414,9 +446,8 @@ int APS5_VABI clock_gettime_nid_postfix(int clockId, KernelTimespec* tp) { } int APS5_VABI gettimeofday_nid_postfix(KernelTimeval* tv, KernelTimezone* tz) { - if (tv == nullptr) { - APS5_INVALID_ARG_EX; - } + KernelTimeval ignoredTime{}; + if (tv == nullptr) tv = &ignoredTime; #ifdef _WIN32 FILETIME ft{}; GetSystemTimePreciseAsFileTime(&ft); @@ -440,9 +471,10 @@ int APS5_VABI gettimeofday_nid_postfix(KernelTimeval* tv, KernelTimezone* tz) { } int APS5_VABI clock_getres_nid_postfix(int clockId, KernelTimespec* res) { - if (res == nullptr) { - APS5_INVALID_ARG_EX; - } + KernelTimespec ignoredResolution{}; + if (res == nullptr) res = &ignoredResolution; + if (IsCpuClock(clockId)) + return clock_getres_nid_postfix(GUEST_CLOCK_THREAD_CPUTIME_ID, res); if (clockId == 1 || clockId == 2) { const std::uint64_t nanos = ProcessCpuResolutionNanos(); res->tv_sec = static_cast(nanos / 1000000000ULL); @@ -539,10 +571,10 @@ int APS5_VABI sceKernelClockGettime(KernelClockid clock_id, KernelTimespec* tp) return clock_gettime_nid_postfix(static_cast(clock_id), tp); } -int APS5_VABI sceKernelConvertLocaltimeToUtc(int64_t local_time, int64_t reserved, int64_t* utc_time, KernelTimezone* timezone, int32_t* dst_seconds) { +int APS5_VABI sceKernelConvertLocaltimeToUtc(int64_t local_time, int64_t reserved, int64_t* utc_time, KernelTimesec* timezone, int32_t* dst_seconds) { (void)reserved; if (utc_time != nullptr) *utc_time = local_time; - if (timezone != nullptr) *timezone = {0, 0}; + if (timezone != nullptr) *timezone = {local_time, 0u, 0u}; if (dst_seconds != nullptr) *dst_seconds = 0; return 0; } @@ -584,7 +616,7 @@ uint64_t APS5_VABI sceKernelGetTscFrequency(void) { } unsigned int APS5_VABI sceKernelSleep(unsigned int seconds) { - TimedWait::SleepNanos(static_cast(seconds) * 1000000000ULL); + CancellableSleep(static_cast(seconds) * 1000000000ULL); return 0; } diff --git a/core/libs/prx/libkernel/Time/TimedWait.cpp b/core/libs/prx/libkernel/Time/TimedWait.cpp index f5949d6da..6d57117e4 100644 --- a/core/libs/prx/libkernel/Time/TimedWait.cpp +++ b/core/libs/prx/libkernel/Time/TimedWait.cpp @@ -1,4 +1,5 @@ #include "prx/libkernel/Time/include/TimedWait.hpp" +#include "prx/libkernel/Pthread/include/WindowsThreadLocal.hpp" #include #include @@ -17,8 +18,18 @@ namespace { constexpr std::uint64_t YIELD_BELOW_NANOS = 50000ULL; constexpr std::uint64_t LEAD_NANOS = 500000ULL; -thread_local std::atomic ownWaitState{0}; -thread_local std::atomic* waitState = &ownWaitState; +struct ThreadWaitState { + std::atomic own{0}; + std::atomic* bound = &own; +}; +ThreadWaitState& WaitState() { +#ifdef _WIN32 + return WindowsThreadLocal::Get(); +#else + static thread_local ThreadWaitState state; + return state; +#endif +} #ifdef _WIN32 @@ -31,35 +42,40 @@ void DrainApcs() { } DWORD AlertableWait(DWORD count, const HANDLE* handles, DWORD milliseconds) { - waitState->fetch_add(1, std::memory_order_seq_cst); + WaitState().bound->fetch_add(1, std::memory_order_seq_cst); DWORD result; do { result = WaitForMultipleObjectsEx(count, handles, FALSE, milliseconds, TRUE); } while (result == WAIT_IO_COMPLETION); - waitState->fetch_sub(1, std::memory_order_seq_cst); + WaitState().bound->fetch_sub(1, std::memory_order_seq_cst); DrainApcs(); return result; } void AlertableSleep(DWORD milliseconds) { - waitState->fetch_add(1, std::memory_order_seq_cst); + WaitState().bound->fetch_add(1, std::memory_order_seq_cst); SleepEx(milliseconds, TRUE); - waitState->fetch_sub(1, std::memory_order_seq_cst); + WaitState().bound->fetch_sub(1, std::memory_order_seq_cst); DrainApcs(); } -void WINAPI DestroyWaiter(void* value); - -DWORD WaiterSlotIndex() { - static const DWORD index = FlsAlloc(DestroyWaiter); - return index; -} +HANDLE ThreadTimer(unsigned kind); HANDLE CreateHighResolutionTimer() { return CreateWaitableTimerExW(nullptr, nullptr, CREATE_WAITABLE_TIMER_HIGH_RESOLUTION, TIMER_ALL_ACCESS); } bool TimerWait(HANDLE timer, std::uint64_t nanos) { + thread_local int depth = 0; + struct Nesting { + HANDLE own; + explicit Nesting(bool nested) : own(nested ? CreateHighResolutionTimer() : nullptr) { ++depth; } + ~Nesting() { + --depth; + if (own) CloseHandle(own); + } + } nesting(depth != 0); + if (depth > 1) timer = nesting.own; if (!timer) return false; LARGE_INTEGER due{}; due.QuadPart = -static_cast(nanos / 100ULL); @@ -78,7 +94,7 @@ void SleepNanosCoarse(std::uint64_t nanos) { SwitchToThread(); return; } - thread_local HANDLE timer = CreateHighResolutionTimer(); + const HANDLE timer = ThreadTimer(0); if (TimerWait(timer, nanos)) return; AlertableSleep(static_cast((nanos + 999999ULL) / 1000000ULL)); } @@ -92,6 +108,7 @@ void SleepNanosCoarse(std::uint64_t nanos) { struct Waiter { HANDLE event = CreateEventW(nullptr, FALSE, FALSE, nullptr); HANDLE timer = CreateHighResolutionTimer(); + HANDLE sleepTimers[3]{}; Waiter* previous = nullptr; Waiter* next = nullptr; Waiter* nested = nullptr; @@ -102,23 +119,20 @@ struct Waiter { delete nested; if (event) CloseHandle(event); if (timer) CloseHandle(timer); + for (const auto sleepTimer : sleepTimers) if (sleepTimer) CloseHandle(sleepTimer); } }; namespace { -void WINAPI DestroyWaiter(void* value) { - delete static_cast(value); +Waiter* ThisThreadWaiter() { + return &WindowsThreadLocal::Get(); } -Waiter* ThisThreadWaiter() { - thread_local Waiter* waiter = nullptr; - if (!waiter) { - waiter = new Waiter(); - const DWORD slot = WaiterSlotIndex(); - if (slot != FLS_OUT_OF_INDEXES) FlsSetValue(slot, waiter); - } - return waiter; +HANDLE ThreadTimer(unsigned kind) { + auto& timer = ThisThreadWaiter()->sleepTimers[kind]; + if (!timer) timer = CreateHighResolutionTimer(); + return timer; } Waiter* AcquireWaiter() { @@ -238,7 +252,8 @@ void Condition::NotifyAll() { #endif void BindThreadWaitState(std::atomic* state) { - waitState = state != nullptr ? state : &ownWaitState; + auto& local = WaitState(); + local.bound = state != nullptr ? state : &local.own; } bool Coarse() { @@ -283,7 +298,7 @@ void SleepUntil(std::uint64_t deadlineNanos) { #ifdef _WIN32 std::uint64_t now = NowNanos(); if (deadlineNanos > now + LEAD_NANOS) { - thread_local HANDLE timer = CreateHighResolutionTimer(); + const HANDLE timer = ThreadTimer(1); if (!TimerWait(timer, deadlineNanos - now - LEAD_NANOS)) AlertableSleep(WholeMilliseconds(deadlineNanos - now - LEAD_NANOS)); now = NowNanos(); } @@ -301,7 +316,7 @@ void PollSleepUntil(std::uint64_t deadlineNanos) { #ifdef _WIN32 const std::uint64_t now = NowNanos(); if (deadlineNanos <= now) return; - thread_local HANDLE timer = CreateHighResolutionTimer(); + const HANDLE timer = ThreadTimer(2); if (!TimerWait(timer, deadlineNanos - now)) AlertableSleep(WholeMilliseconds(deadlineNanos - now)); #else SleepUntil(deadlineNanos); diff --git a/core/libs/prx/libkernel/Unimplemented.cpp b/core/libs/prx/libkernel/Unimplemented.cpp index 18df01488..9fa34427b 100644 --- a/core/libs/prx/libkernel/Unimplemented.cpp +++ b/core/libs/prx/libkernel/Unimplemented.cpp @@ -15,11 +15,105 @@ int APS5_VABI __tls_get_addr_nid_postfix(void) { return 0; } -// Canonical lib is libScePosix (dead import of Cyberpunk 2077): 35 of its -// 36 sibling imports resolve to libkernel, and libScePosix is not in the -// game's NEEDED list so only a NEEDED module can satisfy the loader here. -int APS5_VABI pthread_cancel_nid_postfix(void) { - NotImplemented_nid_no_patch("0D4-FVvEikw"); +APS5_EXPORT("AC1FtjqMCL0", sceKernelUnknown03); +int APS5_VABI sceKernelUnknown03(void) { + NotImplemented_nid_no_patch("AC1FtjqMCL0"); + return 0; +} + +APS5_EXPORT("0iEKNAvT600", sceKernelUnknown04); +int APS5_VABI sceKernelUnknown04(void) { + NotImplemented_nid_no_patch("0iEKNAvT600"); + return 0; +} + +APS5_EXPORT("2cELEBPdYQ0", sceKernelUnknown05); +int APS5_VABI sceKernelUnknown05(void) { + NotImplemented_nid_no_patch("2cELEBPdYQ0"); + return 0; +} + +APS5_EXPORT("5xYxiOtOccA", sceKernelUnknown06); +int APS5_VABI sceKernelUnknown06(void) { + NotImplemented_nid_no_patch("5xYxiOtOccA"); + return 0; +} + +APS5_EXPORT("6W+shIH315Q", sceKernelUnknown07); +int APS5_VABI sceKernelUnknown07(void) { + NotImplemented_nid_no_patch("6W+shIH315Q"); + return 0; +} + +APS5_EXPORT("7ilObq815O4", sceKernelUnknown08); +int APS5_VABI sceKernelUnknown08(void) { + NotImplemented_nid_no_patch("7ilObq815O4"); + return 0; +} + +APS5_EXPORT("7iq7Hfs8HBk", sceKernelUnknown09); +int APS5_VABI sceKernelUnknown09(void) { + NotImplemented_nid_no_patch("7iq7Hfs8HBk"); + return 0; +} + +APS5_EXPORT("AMJZhpC+siY", sceKernelUnknown10); +int APS5_VABI sceKernelUnknown10(void) { + NotImplemented_nid_no_patch("AMJZhpC+siY"); + return 0; +} + +APS5_EXPORT("E9XkHowt+dY", sceKernelUnknown11); +int APS5_VABI sceKernelUnknown11(void) { + NotImplemented_nid_no_patch("E9XkHowt+dY"); + return 0; +} + +APS5_EXPORT("GADjreszOgM", sceKernelUnknown12); +int APS5_VABI sceKernelUnknown12(void) { + NotImplemented_nid_no_patch("GADjreszOgM"); + return 0; +} + +APS5_EXPORT("Giyg2xGrXDU", sceKernelUnknown13); +int APS5_VABI sceKernelUnknown13(void) { + NotImplemented_nid_no_patch("Giyg2xGrXDU"); + return 0; +} + +APS5_EXPORT("KC3y21wrqzc", sceKernelUnknown14); +int APS5_VABI sceKernelUnknown14(void) { + NotImplemented_nid_no_patch("KC3y21wrqzc"); + return 0; +} + +APS5_EXPORT("Qb8wS4GZUsw", sceKernelUnknown15); +int APS5_VABI sceKernelUnknown15(void) { + NotImplemented_nid_no_patch("Qb8wS4GZUsw"); + return 0; +} + +APS5_EXPORT("aAQZSjb2fho", sceKernelUnknown16); +int APS5_VABI sceKernelUnknown16(void) { + NotImplemented_nid_no_patch("aAQZSjb2fho"); + return 0; +} + +APS5_EXPORT("pnz21I6VjDs", sceKernelUnknown17); +int APS5_VABI sceKernelUnknown17(void) { + NotImplemented_nid_no_patch("pnz21I6VjDs"); + return 0; +} + +APS5_EXPORT("qk043QPsrZU", sceKernelUnknown18); +int APS5_VABI sceKernelUnknown18(void) { + NotImplemented_nid_no_patch("qk043QPsrZU"); + return 0; +} + +APS5_EXPORT("uDGXwUKqi8s", sceKernelUnknown19); +int APS5_VABI sceKernelUnknown19(void) { + NotImplemented_nid_no_patch("uDGXwUKqi8s"); return 0; } } diff --git a/core/libs/prx/libkernel_sys/Export.cpp b/core/libs/prx/libkernel_sys/Export.cpp index fb34206c2..923ebaa27 100644 --- a/core/libs/prx/libkernel_sys/Export.cpp +++ b/core/libs/prx/libkernel_sys/Export.cpp @@ -1,10 +1,51 @@ +#include #include #include +#include +#include +#include +#include +#include #include "SceTypes.hpp" #include "prx/libc/include/General.hpp" +extern "C" int* APS5_VABI __error_nid_postfix(); + +namespace { +constexpr int GuestEnoent = 2; +constexpr int GuestEio = 5; +constexpr int GuestEfault = 14; +constexpr int GuestEinval = 22; + +std::int64_t Fail(int error) { + *__error_nid_postfix() = error; + return -1; +} + +int GuestError(const std::error_code& error) { + const int value = error.default_error_condition().value(); + return value > 0 && value <= 34 ? value : GuestEio; +} +} + extern "C" { -APS5_DUMMY_FUN +std::int64_t APS5_VABI readlink_nid_postfix(const char* path, char* buffer, std::size_t size) { + if (path == nullptr) return Fail(GuestEfault); + if (*path == '\0') return Fail(GuestEnoent); + if (size > static_cast(std::numeric_limits::max())) return Fail(GuestEinval); + const auto native = ResolvePath_nid_no_patch(path); + std::error_code error; + const auto status = std::filesystem::symlink_status(native, error); + if (error) return Fail(GuestError(error)); + if (status.type() == std::filesystem::file_type::not_found) return Fail(GuestEnoent); + if (!std::filesystem::is_symlink(status)) return Fail(GuestEinval); + const auto target = std::filesystem::read_symlink(native, error).generic_string(); + if (error) return Fail(GuestError(error)); + const auto count = std::min(size, target.size()); + if (count != 0 && buffer == nullptr) return Fail(GuestEfault); + if (count != 0) std::memcpy(buffer, target.data(), count); + return static_cast(count); +} } diff --git a/core/libs/tests/ApplicationHeap.cpp b/core/libs/tests/ApplicationHeap.cpp index 7ea5cec5b..6cd531139 100644 --- a/core/libs/tests/ApplicationHeap.cpp +++ b/core/libs/tests/ApplicationHeap.cpp @@ -21,6 +21,7 @@ void APS5_VABI _ZdlPvmSt11align_val_t_nid_postfix(void*, std::size_t, std::size_ void* ApplicationHeapRealign_nid_no_patch(void*, std::size_t, std::size_t); char* APS5_VABI strdup_nid_postfix(const char*); char* APS5_VABI strndup_nid_postfix(const char*, std::size_t); +char* APS5_VABI getcwd_nid_postfix(char*, std::size_t); int* APS5_VABI __error_nid_postfix(); int APS5_VABI atexit_nid_postfix(void (APS5_VABI*)()); } @@ -156,6 +157,9 @@ int main(int argc, char** argv) { require(ApplicationHeapPosixAlign_nid_no_patch(&aligned, 256, 99) == 0); require(reinterpret_cast(aligned) % 256 == 0); ApplicationHeapFree_nid_no_patch(aligned); + void* refused = nullptr; + require(ApplicationHeapPosixAlign_nid_no_patch(&refused, 256, SIZE_MAX) == 12 && refused == nullptr); + require(posix_memalign_nid_postfix(&refused, 4096, SIZE_MAX - 4096) == 12 && refused == nullptr); ApplicationHeapFree_nid_no_patch(nullptr); reject([] { ApplicationHeapCalloc_nid_no_patch(SIZE_MAX, 2); }); reject([] { ApplicationHeapAlign_nid_no_patch(3, 16); }); @@ -260,6 +264,12 @@ int main(int argc, char** argv) { copy = strndup_nid_postfix(text, 5); require(lastSize == 6 && std::strcmp(copy, "guest") == 0); ApplicationHeapFree_nid_no_patch(copy); + char* directory = getcwd_nid_postfix(nullptr, 0); + require(directory == reinterpret_cast(storage.data()) && directory[0] == '/' && lastSize == std::strlen(directory) + 1); + ApplicationHeapFree_nid_no_patch(directory); + directory = getcwd_nid_postfix(nullptr, 200); + require(directory == reinterpret_cast(storage.data()) && lastSize == 200); + ApplicationHeapFree_nid_no_patch(directory); fail = true; *__error_nid_postfix() = 0; require(strdup_nid_postfix(text) == nullptr && *__error_nid_postfix() == 12); @@ -267,5 +277,7 @@ int main(int argc, char** argv) { require(strdup_nid_postfix("") == nullptr && *__error_nid_postfix() == 12); *__error_nid_postfix() = 0; require(strndup_nid_postfix(text, 5) == nullptr && *__error_nid_postfix() == 12); + *__error_nid_postfix() = 0; + require(getcwd_nid_postfix(nullptr, 0) == nullptr && *__error_nid_postfix() == 12); fail = false; } diff --git a/core/libs/tests/ApplicationHeapStats.cpp b/core/libs/tests/ApplicationHeapStats.cpp new file mode 100644 index 000000000..42d97d01d --- /dev/null +++ b/core/libs/tests/ApplicationHeapStats.cpp @@ -0,0 +1,102 @@ +#include "prx/libc/include/ApplicationHeap.hpp" +#include "prx/libc/include/general/VabiMacros.hpp" +#include +#include +#include +#include +#include + +extern "C" { +int APS5_VABI malloc_stats_fast_nid_postfix(void*); +} + +namespace { + +alignas(64) std::array storage{}; +unsigned statsCalls = 0; +void* lastStats = nullptr; +bool statsAllocate = false; + +void require(bool condition) { + if (!condition) throw std::runtime_error("application heap stats test failed"); +} + +template +void reject(TAction action) { + bool rejected = false; + try { action(); } catch (const std::exception&) { rejected = true; } + require(rejected); +} + +void* APS5_VABI allocate(std::size_t) { return storage.data(); } +void APS5_VABI release(void*) {} +void* APS5_VABI reallocate(void*, std::size_t) { return storage.data(); } +void* APS5_VABI allocateZeroed(std::size_t, std::size_t) { return storage.data(); } +void* APS5_VABI align(std::size_t, std::size_t) { return storage.data(); } +void* APS5_VABI realign(void*, std::size_t, std::size_t) { return storage.data(); } +int APS5_VABI posixAlign(void** pointer, std::size_t, std::size_t) { + *pointer = storage.data(); + return 0; +} + +int APS5_VABI statsFast(void* stats) { + ++statsCalls; + lastStats = stats; + if (statsAllocate) ApplicationHeapAllocate_nid_no_patch(16); + return 0x2a; +} + +template +void write(std::array& data, std::size_t offset, TValue value) { + require(offset <= data.size() && sizeof(value) <= data.size() - offset); + std::memcpy(data.data() + offset, &value, sizeof(value)); +} + +} + +int main(int argc, char** argv) { + std::array stats{}; + reject([&] { malloc_stats_fast_nid_postfix(stats.data()); }); + std::array process{}; + std::array libc{}; + std::array replacement{}; + write(process, 0, std::uint64_t{0x40}); + write(process, 8, std::uint32_t{0x4942524f}); + write(process, 0x38, libc.data()); + write(libc, 0, std::uint64_t{0x38}); + write(libc, 0x30, replacement.data()); + write(replacement, 0, std::uint64_t{0x78}); + write(replacement, 8, std::uint64_t{2}); + write(replacement, 0x20, &allocate); + write(replacement, 0x28, &release); + write(replacement, 0x30, &allocateZeroed); + write(replacement, 0x38, &reallocate); + write(replacement, 0x40, &align); + write(replacement, 0x48, &realign); + write(replacement, 0x50, &posixAlign); + write(replacement, 0x60, &statsFast); + if (argc > 1 && std::strcmp(argv[1], "default") == 0) { + std::memset(replacement.data() + 0x20, 0, 10 * sizeof(void*)); + ApplicationHeapInitialize_nid_no_patch(process.data()); + require(ApplicationHeapAllocate_nid_no_patch(16) != nullptr); + reject([&] { malloc_stats_fast_nid_postfix(stats.data()); }); + return 0; + } + if (argc > 1 && std::strcmp(argv[1], "missing") == 0) { + write(replacement, 0x60, static_cast(nullptr)); + ApplicationHeapInitialize_nid_no_patch(process.data()); + require(ApplicationHeapAllocate_nid_no_patch(16) == storage.data()); + reject([&] { malloc_stats_fast_nid_postfix(stats.data()); }); + require(statsCalls == 0); + return 0; + } + ApplicationHeapInitialize_nid_no_patch(process.data()); + require(malloc_stats_fast_nid_postfix(stats.data()) == 0x2a && statsCalls == 1 && lastStats == stats.data()); + require(malloc_stats_fast_nid_postfix(nullptr) == 0x2a && statsCalls == 2 && lastStats == nullptr); + statsAllocate = true; + reject([&] { malloc_stats_fast_nid_postfix(stats.data()); }); + statsAllocate = false; + require(statsCalls == 3); + require(malloc_stats_fast_nid_postfix(stats.data()) == 0x2a && statsCalls == 4); + require(ApplicationHeapAllocate_nid_no_patch(16) == storage.data()); +} diff --git a/core/libs/tests/BitfieldMask.cpp b/core/libs/tests/BitfieldMask.cpp new file mode 100644 index 000000000..fd5e851a1 --- /dev/null +++ b/core/libs/tests/BitfieldMask.cpp @@ -0,0 +1,83 @@ +#include "Translation/TranslationContext.hpp" +#include "Optimization/ConstantFolder.hpp" +#include +#include +#include +#include +#include + +using namespace ShaderRecompiler; + +namespace { + +std::uint32_t expectedMask(std::uint32_t width, std::uint32_t offset) { + width &= 31u; + offset &= 31u; + std::uint32_t mask = 0u; + for (std::uint32_t bit = offset; bit < 32u && bit - offset < width; ++bit) { + mask |= 1u << bit; + } + return mask; +} + +void check(RdnaOpcode opcode, std::uint32_t width, std::uint32_t offset) { + const bool scalar = opcode == RdnaOpcode::SBfmB32; + IrProgram program; + auto& block = program.CreateBlock(); + program.SetEntryBlock(block); + program.BlockOrder().push_back(&block); + RdnaInstruction instruction{}; + instruction.op = opcode; + instruction.family = scalar ? RdnaInstructionFamily::SOP2 : RdnaInstructionFamily::VOP3; + instruction.sourceCount = 2u; + instruction.source0.kind = RdnaOperandKind::LiteralConstant; + instruction.source0.value = width; + instruction.source1.kind = RdnaOperandKind::LiteralConstant; + instruction.source1.value = offset; + instruction.destination.kind = scalar ? RdnaOperandKind::ScalarRegister : RdnaOperandKind::VectorRegister; + instruction.destination.reg = 10u; + TranslationContext context(program, block, 256u); + context.TranslateInstruction(instruction); + ConstantFolder{}.Fold(program); + const IrValue* result = nullptr; + for (const auto* value : block.Instructions()) { + if (value->Opcode() == IrOpcode::SetScc) { + throw std::runtime_error("bitfield mask changed SCC"); + } + if (value->Opcode() != (scalar ? IrOpcode::SetScalarRegister : IrOpcode::SetVectorRegister)) continue; + result = value->Argument(1u)->Resolve(); + } + if (result == nullptr) throw std::runtime_error("bitfield mask did not write its destination"); + if (!scalar) { + if (result->Opcode() != IrOpcode::SelectU32) throw std::runtime_error("bitfield mask lost EXEC masking"); + result = result->Argument(1u)->Resolve(); + } + if (!result->HasImmediate() || result->ImmediateU32() != expectedMask(width, offset)) { + throw std::runtime_error(std::string(scalar ? "s_bfm_b32" : "v_bfm_b32") + + " width=" + std::to_string(width) + " offset=" + std::to_string(offset) + " did not fold to the expected mask"); + } +} + +} + +int main() { + try { + const std::array upperBits{0u, 32u, 0x80000000u, 0xffffffe0u}; + for (const auto opcode : {RdnaOpcode::SBfmB32, RdnaOpcode::VBfmB32}) { + for (std::uint32_t width = 0u; width < 32u; ++width) { + for (std::uint32_t offset = 0u; offset < 32u; ++offset) { + for (const auto widthBits : upperBits) { + for (const auto offsetBits : upperBits) { + check(opcode, width | widthBits, offset | offsetBits); + } + } + } + } + } + std::puts("32768 bitfield mask translation cases passed"); + return 0; + } catch (const std::exception& error) { + std::fprintf(stderr, "%s\n", error.what()); + return 1; + } +} diff --git a/core/libs/tests/ExceptionRuntime.cpp b/core/libs/tests/ExceptionRuntime.cpp index a6d0715d8..c55a2cb15 100644 --- a/core/libs/tests/ExceptionRuntime.cpp +++ b/core/libs/tests/ExceptionRuntime.cpp @@ -36,6 +36,10 @@ thread_local int destroyed = 0; struct Guard { ~Guard() { assert(__cxa_uncaught_exceptions_nid_postfix() > 0); ++destroyed; } }; +thread_local int counted = 0; +struct Counted { + ~Counted() { ++counted; } +}; struct Base { virtual ~Base() = default; int value = 7; }; struct Other { virtual ~Other() = default; int padding = 9; }; struct Derived : Other, Base {}; @@ -188,6 +192,11 @@ int main() { try { __cxa_rethrow_primary_exception_nid_postfix(retained); assert(false); } catch (int value) { assert(value == 27); } __cxa_decrement_exception_refcount_nid_postfix(retained); + { + auto made = std::make_exception_ptr(Counted()); + try { std::rethrow_exception(made); assert(false); } catch (const Counted&) {} + } + assert(counted == 2); Derived object; Base* pointer = &object; try { throw &object; } catch (Base* value) { assert(value == pointer); } diff --git a/core/libs/tests/FileStreams.cpp b/core/libs/tests/FileStreams.cpp index c2778f977..aa4b41128 100644 --- a/core/libs/tests/FileStreams.cpp +++ b/core/libs/tests/FileStreams.cpp @@ -1,18 +1,24 @@ #include "prx/libc/include/FileStream.hpp" +#include "prx/libc/include/general/VabiMacros.hpp" #include +#include +#include #include +#include #include #include +#include extern "C" { -FileStream* fopen_nid_postfix(const char* filename, const char* mode); -int fclose_nid_postfix(FileStream* stream); -std::size_t fread_nid_postfix(void* buffer, std::size_t size, std::size_t count, FileStream* stream); -std::size_t fwrite_nid_postfix(const void* buffer, std::size_t size, std::size_t count, FileStream* stream); -int fseek_nid_postfix(FileStream* stream, long offset, int origin); -long ftell_nid_postfix(FileStream* stream); -int fputs_nid_postfix(const char* str, FileStream* stream); -int fflush_nid_postfix(FileStream* stream); +FileStream* APS5_VABI fopen_nid_postfix(const char* filename, const char* mode); +FileStream* APS5_VABI freopen_nid_postfix(const char* filename, const char* mode, FileStream* stream); +int APS5_VABI fclose_nid_postfix(FileStream* stream); +std::size_t APS5_VABI fread_nid_postfix(void* buffer, std::size_t size, std::size_t count, FileStream* stream); +std::size_t APS5_VABI fwrite_nid_postfix(const void* buffer, std::size_t size, std::size_t count, FileStream* stream); +int APS5_VABI fseek_nid_postfix(FileStream* stream, std::int64_t offset, int origin); +std::int64_t APS5_VABI ftell_nid_postfix(FileStream* stream); +int APS5_VABI fputs_nid_postfix(const char* str, FileStream* stream); +int APS5_VABI fflush_nid_postfix(FileStream* stream); } static void Require(bool condition) { @@ -51,10 +57,10 @@ int main(int argc, char** argv) { Require(std::strcmp(buffer.data(), payload) == 0); Require(fread_nid_postfix(buffer.data(), 1, buffer.size(), stream) == 0); ExpectException([&] { fputs_nid_postfix(nullptr, stream); }); - ExpectException([&] { fseek_nid_postfix(stream, 0, -1); }); - Require(fclose_nid_postfix(stream) == 0); - stream = fopen_nid_postfix(argv[1], "rb"); - ExpectException([&] { fwrite_nid_postfix(payload, 1, sizeof(payload), stream); }); + ExpectException([&] { fwrite_nid_postfix(nullptr, 1, sizeof(payload), stream); }); + errno = 0; + Require(fseek_nid_postfix(stream, 0, -1) == -1); + Require(errno == EINVAL); Require(fclose_nid_postfix(stream) == 0); ExpectException([] { fputs_nid_postfix("invalid stream", nullptr); }); ExpectException([] { fopen_nid_postfix(nullptr, "r"); }); @@ -62,6 +68,28 @@ int main(int argc, char** argv) { closed.Close(); ExpectException([&] { fflush_nid_postfix(&closed); }); Require(std::remove(argv[1]) == 0); - ExpectException([&] { fopen_nid_postfix(argv[1], "rb"); }); + errno = 0; + Require(fopen_nid_postfix(argv[1], "rb") == nullptr); + Require(errno == ENOENT); + const std::u8string unicodeName = u8"セーブé.tmp"; + const std::string guestName(unicodeName.begin(), unicodeName.end()); + const std::filesystem::path hostName(unicodeName); + constexpr char unicodePayload[] = "unicode"; + stream = fopen_nid_postfix(guestName.c_str(), "wb"); + Require(stream != nullptr); + Require(fwrite_nid_postfix(unicodePayload, 1, sizeof(unicodePayload), stream) == sizeof(unicodePayload)); + Require(fclose_nid_postfix(stream) == 0); + Require(std::filesystem::exists(hostName)); + Require(std::filesystem::file_size(hostName) == sizeof(unicodePayload)); + stream = fopen_nid_postfix(guestName.c_str(), "rb"); + Require(stream != nullptr); + std::array unicodeBuffer{}; + Require(fread_nid_postfix(unicodeBuffer.data(), 1, unicodeBuffer.size(), stream) == unicodeBuffer.size()); + Require(std::strcmp(unicodeBuffer.data(), unicodePayload) == 0); + stream = freopen_nid_postfix(guestName.c_str(), "wb", stream); + Require(stream != nullptr); + Require(fclose_nid_postfix(stream) == 0); + Require(std::filesystem::file_size(hostName) == 0); + std::filesystem::remove(hostName); std::cout << "PASS: stream objects, file operations, EOF and error handling\n"; } diff --git a/core/libs/tests/Formatting.cpp b/core/libs/tests/Formatting.cpp index 6cccc4a31..796def8e3 100644 --- a/core/libs/tests/Formatting.cpp +++ b/core/libs/tests/Formatting.cpp @@ -1,4 +1,5 @@ #include "SceTypes.hpp" +#include #include #include #include @@ -39,7 +40,6 @@ static int APS5_VABI PrintList(const char* format, ...) { return result; } -#ifdef _WIN32 static bool CheckWidePrecision() { const char16_t input[] = u"A\u00e9\u20ac\U0001f600Z"; const char* expected[] = {"", "A", "A", "A\xc3\xa9", "A\xc3\xa9", "A\xc3\xa9", "A\xc3\xa9\xe2\x82\xac", @@ -65,9 +65,23 @@ static bool CheckWidePrecision() { Require(snprintf_nid_postfix(output, sizeof(output), "%.2ls", bounded) == 2 && std::strcmp(output, "AB") == 0); Require(snprintf_nid_postfix(nullptr, 0, "%.1ls", u"\u00e9") == 0); Require(snprintf_nid_postfix(output, sizeof(output), "%.1s", "\xc3\xa9") == 1 && static_cast(output[0]) == 0xc3 && output[1] == 0); + Require(FormatList(output, sizeof(output), "%S:%lc:%C:%d", u"\u00e9", 0x20ac, 0x41, 7) == 10 && std::strcmp(output, "\xc3\xa9:\xe2\x82\xac:A:7") == 0); + Require(snprintf_nid_postfix(output, sizeof(output), "%.*S", 3, u"\u00e9\u20ac") == 2 && std::strcmp(output, "\xc3\xa9") == 0); + Require(snprintf_nid_postfix(output, sizeof(output), "[%ls]", static_cast(nullptr)) == 8 && std::strcmp(output, "[(null)]") == 0); + Require(snprintf_nid_postfix(output, sizeof(output), "a%lcb%C", 0, 0) == 4 && std::memcmp(output, "a\0b\0", 5) == 0); +#ifndef _WIN32 + Require(FormatList(output, sizeof(output), "%%ls:%2$d:%1$d", 3, 7) == 7 && std::strcmp(output, "%ls:7:3") == 0); +#endif + for (std::size_t capacity = 0; capacity <= 6; ++capacity) { + std::memset(output, '!', sizeof(output)); + int count = -1; + Require(FormatList(capacity ? output : nullptr, capacity, "%ls%n", u"A\u00e9Z", &count) == 4 && count == 4); + const auto copied = capacity == 0 ? 0 : std::min(capacity - 1, 4); + Require(std::memcmp(output, "A\xc3\xa9Z", copied) == 0 && output[capacity] == '!'); + if (capacity) Require(output[copied] == 0); + } return correct; } -#endif __attribute__((noinline)) static void APS5_VABI RunChecks() { char buffer[1024]; @@ -126,8 +140,4 @@ __attribute__((noinline)) static void APS5_VABI RunChecks() { std::puts("Formatting checks passed: 10000 iterations"); } -#ifdef _WIN32 int main() { RunChecks(); return CheckWidePrecision() ? 0 : 1; } -#else -int main() { RunChecks(); } -#endif diff --git a/core/libs/tests/GuestAgcDriverResources.cpp b/core/libs/tests/GuestAgcDriverResources.cpp index eb97484c7..df3981cfb 100644 --- a/core/libs/tests/GuestAgcDriverResources.cpp +++ b/core/libs/tests/GuestAgcDriverResources.cpp @@ -1,12 +1,15 @@ #include "SceTypes.hpp" +#include #include #include #include +#include #include extern "C" { int APS5_VABI sceAgcDriverRegisterOwner(std::uint32_t*, const char*); int APS5_VABI sceAgcDriverRegisterResource(std::uint32_t*, std::uint32_t, const void*, std::size_t, const char*, std::uint32_t, std::uint64_t); +int APS5_VABI sceAgcDriverRegisterMultipleResources(std::uint32_t*, std::uint32_t, const void*, std::uint32_t); int APS5_VABI sceAgcDriverRegisterGdsResource(std::uint32_t*, std::uint32_t, std::uint32_t, std::uint32_t, const char*, std::uint32_t, std::uint64_t); int APS5_VABI sceAgcDriverUnregisterAllResourcesForOwner(std::uint32_t); int APS5_VABI sceAgcDriverRegisterWorkloadStream(std::uint32_t, const void*); @@ -29,6 +32,17 @@ bool APS5_VABI sceAgcDriverIsSubmitValidationEnabled(void); static constexpr int Unavailable = static_cast(0x8A6C9018); static void Require(bool value) { if (!value) std::abort(); } +struct ResourceDescriptor { + const void* memory; + std::size_t size; + const char* name; + std::uint32_t type; + std::uint32_t reserved; + std::uint64_t userData; +}; + +static_assert(sizeof(ResourceDescriptor) == 40); + int main() { std::uint32_t owner = 7u; std::uint32_t resource = 9u; @@ -37,6 +51,22 @@ int main() { Require(sceAgcDriverRegisterResource(&resource, owner, memory, sizeof(memory), "resource", 1u, 2u) == Unavailable); Require(sceAgcDriverRegisterGdsResource(&resource, owner, 0u, 64u, "gds", 1u, 2u) == Unavailable); Require(owner == 7u && resource == 9u); + std::array bulkMemory{0x11223344u, 0x55667788u, 0x99aabbccu, 0xddeeff00u}; + const auto originalBulkMemory = bulkMemory; + std::array descriptors{{ + {bulkMemory.data(), sizeof(bulkMemory), "default", 3u, 0u, 0u}, + {bulkMemory.data(), sizeof(bulkMemory), "second", 5u, 0u, 23u}, + }}; + const auto originalDescriptors = descriptors; + std::uint32_t bulkHandle = 0xffffffffu; + Require(sceAgcDriverRegisterMultipleResources(&bulkHandle, 0u, descriptors.data(), 1u) == Unavailable); + Require(bulkHandle == 0xffffffffu); + bulkHandle = 17u; + Require(sceAgcDriverRegisterMultipleResources(&bulkHandle, owner, descriptors.data(), 2u) == Unavailable); + Require(sceAgcDriverRegisterMultipleResources(&bulkHandle, owner, nullptr, 0u) == Unavailable); + Require(bulkHandle == 17u && owner == 7u); + Require(std::memcmp(descriptors.data(), originalDescriptors.data(), sizeof(descriptors)) == 0); + Require(bulkMemory == originalBulkMemory); Require(sceAgcDriverGetDefaultOwner(&owner) == Unavailable && owner == 7u); std::uint32_t maxNameLength = 8u; Require(sceAgcDriverGetResourceRegistrationMaxNameLength(&maxNameLength) == 0 && maxNameLength == 0xfcu); diff --git a/core/libs/tests/GuestAio.cpp b/core/libs/tests/GuestAio.cpp index 84d3f22cd..0bbc3a952 100644 --- a/core/libs/tests/GuestAio.cpp +++ b/core/libs/tests/GuestAio.cpp @@ -10,6 +10,7 @@ #include #include #include +#include #include #include #include @@ -17,6 +18,7 @@ extern "C" { int APS5_VABI sceKernelOpen(const char* path, int flags, std::uint16_t mode); int APS5_VABI sceKernelClose(int d); +std::int64_t APS5_VABI sceKernelLseek(int d, std::int64_t offset, int whence); int APS5_VABI sceKernelAioDeleteRequest(std::int32_t id, std::int32_t* ret); int APS5_VABI sceKernelAioInitializeImpl(void* param, std::int32_t size); void APS5_VABI sceKernelAioInitializeParam(void* param); @@ -27,6 +29,7 @@ int APS5_VABI sceKernelAioWaitRequest(std::int32_t id, std::int32_t* state, std: int APS5_VABI sceKernelAioSubmitReadCommandsMultiple(KernelAioRwRequest* req, std::int32_t size, std::int32_t prio, std::int32_t* id); int APS5_VABI sceKernelAioSubmitWriteCommandsMultiple(KernelAioRwRequest* req, std::int32_t size, std::int32_t prio, std::int32_t* id); int APS5_VABI sceKernelAioWaitRequests(std::int32_t* id, std::int32_t num, std::int32_t* state, std::uint32_t mode, std::uint32_t* usec); +int APS5_VABI sceKernelAioPollRequests(std::int32_t* id, std::int32_t num, std::int32_t* state); int APS5_VABI sceKernelAioCancelRequest(std::int32_t id, std::int32_t* state); int APS5_VABI sceKernelAioCancelRequests(std::int32_t* id, std::int32_t num, std::int32_t* state); int APS5_VABI sceKernelAioDeleteRequests(std::int32_t* id, std::int32_t num, std::int32_t* ret); @@ -68,6 +71,29 @@ int main() { Check(secondResult.state == 3 && secondResult.return_value == 16); Check(std::memcmp(first.data(), seed.data(), 16) == 0); Check(std::memcmp(second.data(), seed.data() + 32, 16) == 0); + KernelAioResult invalidReadResult{-1, 0}; + KernelAioRwRequest invalidReadRequest{-1, first.size(), first.data(), &invalidReadResult, fd}; + std::int32_t invalidReadId = 0; + Check(sceKernelAioSubmitReadCommands(&invalidReadRequest, 1, 0, &invalidReadId) == 0); + std::int32_t invalidReadState = 0; + Check(sceKernelAioWaitRequest(invalidReadId, &invalidReadState, nullptr) == 0); + Check(invalidReadState == 4); + Check(invalidReadResult.state == 4 && invalidReadResult.return_value == SCE_KERNEL_ERROR_EINVAL); + std::array aggregateReadBuffer{}; + KernelAioResult aggregateReadSuccessResult{-1, 0}; + KernelAioResult aggregateReadErrorResult{-1, 0}; + KernelAioRwRequest aggregateReads[2] = { + {4, aggregateReadBuffer.size(), aggregateReadBuffer.data(), &aggregateReadSuccessResult, fd}, + {-1, aggregateReadBuffer.size(), aggregateReadBuffer.data(), &aggregateReadErrorResult, fd}, + }; + std::int32_t aggregateReadId = 0; + Check(sceKernelAioSubmitReadCommands(aggregateReads, 2, 0, &aggregateReadId) == 0); + std::int32_t aggregateReadState = 0; + Check(sceKernelAioWaitRequest(aggregateReadId, &aggregateReadState, nullptr) == 0); + Check(aggregateReadState == 4); + Check(aggregateReadSuccessResult.state == 3 && aggregateReadSuccessResult.return_value == 4); + Check(aggregateReadErrorResult.state == 4 && aggregateReadErrorResult.return_value == SCE_KERNEL_ERROR_EINVAL); + Check(std::memcmp(aggregateReadBuffer.data(), seed.data() + 4, aggregateReadBuffer.size()) == 0); std::int32_t polled = 0; Check(sceKernelAioPollRequest(id, &polled) == 0); Check(polled == 3); @@ -85,9 +111,37 @@ int main() { std::int32_t writeState = 0; Check(sceKernelAioWaitRequest(writeId, &writeState, nullptr) == 0); Check(writeState == 3 && writeResult.state == 3 && writeResult.return_value == 5); + KernelAioResult invalidWriteResult{-1, 0}; + KernelAioRwRequest invalidWriteRequest{-1, written.size(), written.data(), &invalidWriteResult, writeFd}; + std::int32_t invalidWriteId = 0; + Check(sceKernelAioSubmitWriteCommands(&invalidWriteRequest, 1, 0, &invalidWriteId) == 0); + std::int32_t invalidWriteState = 0; + Check(sceKernelAioWaitRequest(invalidWriteId, &invalidWriteState, nullptr) == 0); + Check(invalidWriteState == 4); + Check(invalidWriteResult.state == 4 && invalidWriteResult.return_value == SCE_KERNEL_ERROR_EINVAL); + std::array aggregateWriteBuffer = {'G', 'O', 'O', 'D'}; + KernelAioResult aggregateWriteSuccessResult{-1, 0}; + KernelAioResult aggregateWriteErrorResult{-1, 0}; + KernelAioRwRequest aggregateWrites[2] = { + {8, aggregateWriteBuffer.size(), aggregateWriteBuffer.data(), &aggregateWriteSuccessResult, writeFd}, + {-1, aggregateWriteBuffer.size(), aggregateWriteBuffer.data(), &aggregateWriteErrorResult, writeFd}, + }; + std::int32_t aggregateWriteId = 0; + Check(sceKernelAioSubmitWriteCommands(aggregateWrites, 2, 0, &aggregateWriteId) == 0); + std::int32_t aggregateWriteState = 0; + Check(sceKernelAioWaitRequest(aggregateWriteId, &aggregateWriteState, nullptr) == 0); + Check(aggregateWriteState == 4); + Check(aggregateWriteSuccessResult.state == 3 && aggregateWriteSuccessResult.return_value == 4); + Check(aggregateWriteErrorResult.state == 4 && aggregateWriteErrorResult.return_value == SCE_KERNEL_ERROR_EINVAL); Check(sceKernelClose(writeFd) == 0); { std::ifstream stream(file, std::ios::binary); std::string contents; std::getline(stream, contents); Check(contents.rfind("HELLO", 0) == 0); } + Check(std::filesystem::file_size(file) == seed.size()); + { std::ifstream stream(file, std::ios::binary); + std::string contents{std::istreambuf_iterator(stream), std::istreambuf_iterator()}; + Check(contents.size() == seed.size()); + Check(contents.substr(8, aggregateWriteBuffer.size()) == "GOOD"); + Check(contents.substr(12, 4) == seed.substr(12, 4)); } Check(sceKernelAioSubmitReadCommands(nullptr, 1, 0, &id) == SCE_KERNEL_ERROR_EFAULT); Check(sceKernelAioSubmitReadCommands(requests, 0, 0, &id) == SCE_KERNEL_ERROR_EINVAL); Check(sceKernelAioSubmitReadCommands(requests, 1, 0, nullptr) == SCE_KERNEL_ERROR_EFAULT); @@ -98,6 +152,37 @@ int main() { Check(sceKernelAioDeleteRequest(id, nullptr) == SCE_KERNEL_ERROR_EFAULT); const int batchFd = sceKernelOpen(file.string().c_str(), SCE_KERNEL_O_RDWR, 0); Check(batchFd >= 0); + Check(sceKernelLseek(batchFd, 2, 0) == 2); + std::array batchReadBuffer{}; + KernelAioResult batchReadSuccessResult{-1, 0}; + KernelAioResult batchReadErrorResult{-1, 0}; + KernelAioRwRequest batchReads[2] = { + {0, batchReadBuffer.size(), batchReadBuffer.data(), &batchReadSuccessResult, batchFd}, + {-1, batchReadBuffer.size(), batchReadBuffer.data(), &batchReadErrorResult, batchFd}, + }; + std::int32_t batchReadIds[2] = {0, 0}; + Check(sceKernelAioSubmitReadCommandsMultiple(batchReads, 2, 0, batchReadIds) == 0); + Check(batchReadIds[0] > 0 && batchReadIds[1] > 0 && batchReadIds[0] != batchReadIds[1]); + Check(batchReadSuccessResult.state == 3 && batchReadSuccessResult.return_value == 4); + Check(batchReadErrorResult.state == 4 && batchReadErrorResult.return_value == SCE_KERNEL_ERROR_EINVAL); + std::int32_t batchReadStates[2] = {0, 0}; + Check(sceKernelAioWaitRequests(batchReadIds, 2, batchReadStates, 1, nullptr) == 0); + Check(batchReadStates[0] == 3 && batchReadStates[1] == 4); + std::array batchWriteData = {'A', 'B', 'C', 'D'}; + KernelAioResult batchWriteSuccessResult{-1, 0}; + KernelAioResult batchWriteErrorResult{-1, 0}; + KernelAioRwRequest batchWrites[2] = { + {12, batchWriteData.size(), batchWriteData.data(), &batchWriteSuccessResult, batchFd}, + {-1, batchWriteData.size(), batchWriteData.data(), &batchWriteErrorResult, batchFd}, + }; + std::int32_t batchWriteIds[2] = {0, 0}; + Check(sceKernelAioSubmitWriteCommandsMultiple(batchWrites, 2, 0, batchWriteIds) == 0); + Check(batchWriteIds[0] > 0 && batchWriteIds[1] > 0 && batchWriteIds[0] != batchWriteIds[1]); + Check(batchWriteSuccessResult.state == 3 && batchWriteSuccessResult.return_value == 4); + Check(batchWriteErrorResult.state == 4 && batchWriteErrorResult.return_value == SCE_KERNEL_ERROR_EINVAL); + std::int32_t batchWriteStates[2] = {0, 0}; + Check(sceKernelAioWaitRequests(batchWriteIds, 2, batchWriteStates, 1, nullptr) == 0); + Check(batchWriteStates[0] == 3 && batchWriteStates[1] == 4); std::array patch = {'W', 'X', 'Y', 'Z'}; KernelAioResult patchResult{-1, 0}; KernelAioResult badResult{-1, 0}; @@ -107,12 +192,16 @@ int main() { }; std::int32_t writeIds[2] = {0, 0}; Check(sceKernelAioSubmitWriteCommandsMultiple(writeBatch, 2, 0, writeIds) == 0); + Check(sceKernelLseek(batchFd, 0, 1) == 2); Check(writeIds[0] > 0 && writeIds[1] > 0 && writeIds[0] != writeIds[1]); Check(patchResult.state == 3 && patchResult.return_value == 4); Check(badResult.state == 4 && badResult.return_value == SCE_KERNEL_ERROR_EBADF); std::int32_t writeStates[2] = {0, 0}; Check(sceKernelAioWaitRequests(writeIds, 2, writeStates, 1, nullptr) == 0); Check(writeStates[0] == 3 && writeStates[1] == 4); + std::int32_t polledStates[2] = {0, 0}; + Check(sceKernelAioPollRequests(writeIds, 2, polledStates) == 0); + Check(polledStates[0] == 3 && polledStates[1] == 4); std::array readBack{}; std::array tail{}; KernelAioResult readBackResult{-1, 0}; @@ -145,6 +234,8 @@ int main() { Check(cancelStates[0] == 2 && cancelStates[1] == 4); Check(sceKernelAioPollRequest(readIds[1], &polled) == 0); Check(polled == 4); + Check(sceKernelAioPollRequests(readIds, 2, polledStates) == 0); + Check(polledStates[0] == 4 && polledStates[1] == 4); std::int32_t deletedRets[2] = {-1, -1}; Check(sceKernelAioDeleteRequests(writeIds, 2, deletedRets) == 0); Check(deletedRets[0] == 0 && deletedRets[1] == 0); @@ -164,6 +255,13 @@ int main() { Check(sceKernelAioWaitRequests(readIds, -1, readStates, 1, nullptr) == SCE_KERNEL_ERROR_EINVAL); Check(sceKernelAioWaitRequests(badIds, 2, untouched, 1, nullptr) == SCE_KERNEL_ERROR_EINVAL); Check(untouched[0] == 7 && untouched[1] == 7); + Check(sceKernelAioPollRequests(readIds, 2, nullptr) == SCE_KERNEL_ERROR_EFAULT); + Check(sceKernelAioPollRequests(nullptr, 2, readStates) == SCE_KERNEL_ERROR_EFAULT); + Check(sceKernelAioPollRequests(readIds, -1, readStates) == SCE_KERNEL_ERROR_EINVAL); + Check(sceKernelAioPollRequests(badIds, 2, untouched) == SCE_KERNEL_ERROR_EINVAL); + Check(untouched[0] == 7 && untouched[1] == 7); + Check(sceKernelAioPollRequests(readIds, 0, untouched) == 0); + Check(untouched[0] == 7 && untouched[1] == 7); bool threw = false; try { sceKernelAioWaitRequests(readIds, 2, readStates, 3, nullptr); } catch (const std::runtime_error&) { threw = true; } Check(threw); @@ -173,6 +271,9 @@ int main() { manyIds.fill(readIds[0]); try { sceKernelAioDeleteRequests(manyIds.data(), 129, manyStates.data()); } catch (const std::runtime_error&) { threw = true; } Check(threw); + threw = false; + try { sceKernelAioPollRequests(manyIds.data(), 129, manyStates.data()); } catch (const std::runtime_error&) { threw = true; } + Check(threw); Check(sceKernelAioCancelRequest(9999, &cancelled) == SCE_KERNEL_ERROR_EINVAL); Check(sceKernelAioCancelRequest(readIds[0], nullptr) == SCE_KERNEL_ERROR_EFAULT); Check(sceKernelAioCancelRequests(badIds, 2, untouched) == SCE_KERNEL_ERROR_EINVAL); diff --git a/core/libs/tests/GuestAjm.cpp b/core/libs/tests/GuestAjm.cpp index 0c88f35d7..c8ab3c3fa 100644 --- a/core/libs/tests/GuestAjm.cpp +++ b/core/libs/tests/GuestAjm.cpp @@ -20,6 +20,7 @@ int APS5_VABI sceAjmBatchJobInitialize(AjmBatchInfo*, std::uint32_t, const void* int APS5_VABI sceAjmBatchJobDecode(AjmBatchInfo*, std::uint32_t, const void*, std::size_t, void*, std::size_t, void*); int APS5_VABI sceAjmBatchJobDecodeSingle(AjmBatchInfo*, std::uint32_t, const void*, std::size_t, void*, std::size_t, void*); int APS5_VABI sceAjmBatchJobRun(AjmBatchInfo*, std::uint32_t, std::uint64_t, const void*, std::size_t, void*, std::size_t, void*, std::size_t); +int APS5_VABI sceAjmBatchJobRunSplit(AjmBatchInfo*, std::uint32_t, std::uint64_t, const AjmBuffer*, std::size_t, const AjmBuffer*, std::size_t, void*, std::size_t); int APS5_VABI sceAjmBatchJobSetGaplessDecode(AjmBatchInfo*, std::uint32_t, const void*, int, void*); int APS5_VABI sceAjmBatchJobControl(AjmBatchInfo*, std::uint32_t, std::uint64_t, const void*, std::size_t, void*, std::size_t); int APS5_VABI sceAjmBatchJobGetGaplessDecode(AjmBatchInfo*, std::uint32_t, void*); @@ -610,7 +611,7 @@ void TestMultichannelAt9(std::uint32_t context) { const std::uint8_t validationBitSet[4] = {0x30, 0x73, 0xE1, 0x7E}; Require(sceAjmDecAt9ParseConfigData(validationBitSet, &parsed) == invalidParameter); - const SyntheticAt9Channel channels[2] = {{0x00, 0x07, 27}, {0x01, 0x0E, 22}}; + const SyntheticAt9Channel channels[2] = {{0x00, 0xA5, 27}, {0x01, 0x0E, 22}}; const std::uint8_t mono[4] = {0xFE, 0x70, 0x0B, 0xF0}; std::vector reference[2]; for (std::size_t channel = 0; channel < 2; ++channel) { @@ -955,6 +956,61 @@ void TestResampleAt9(std::uint32_t context) { for (const std::uint32_t instance : {plain, faster, slower}) Require(sceAjmInstanceDestroy(context, instance) == 0); } +struct At9Run { + DecodeSideband stream; + std::uint32_t frames; + std::uint32_t reserved; +}; + +At9Run RunAt9Job(std::uint32_t context, std::uint32_t instance, std::uint64_t flags, const std::uint8_t* input, std::size_t inputSize) { + constexpr std::uint64_t runMultipleFrames = 1ull << 12; + constexpr std::uint64_t sidebandStream = 1ull << 47; + std::vector batch(4096); + std::vector pcm(1024); + At9Run decoded{{-1, -1, 0, 0, 0}, 0, 0}; + AjmBatchInfo info{}; + Require(sceAjmBatchInitialize(batch.data(), batch.size(), &info) == 0); + const std::size_t sidebandSize = (flags & runMultipleFrames) != 0 ? sizeof(decoded) : sizeof(decoded.stream); + Require(sceAjmBatchJobRun(&info, instance, sidebandStream | flags, input, inputSize, pcm.data(), pcm.size() * sizeof(std::int16_t), &decoded, sidebandSize) == 0); + Submit(context, info); + Require(decoded.stream.result == 0); + return decoded; +} + +void TestAt9GaplessSegments(std::uint32_t context) { + constexpr std::uint64_t runMultipleFrames = 1ull << 12; + std::vector stream(1024, 0); + std::memcpy(stream.data(), AT9_MONO_SILENT_SUPERFRAME, sizeof(AT9_MONO_SILENT_SUPERFRAME)); + std::uint32_t instance = 0; + Require(sceAjmInstanceCreate(context, 1, 0, &instance) == 0); + const At9Control start{{512, 0, 0}, {0xFE, 0x70, 0x1F, 0xF0}, 0}; + std::int32_t result[2] = {-1, -1}; + RunControl(context, instance, CONTROL_START, &start, sizeof(start), result); + Require(result[0] == 0); + const auto first = RunAt9Job(context, instance, runMultipleFrames, stream.data(), stream.size()); + Require(first.stream.outputWritten == 512 * 2 && first.frames == 2); + Require(first.stream.inputConsumed > 0 && static_cast(first.stream.inputConsumed) < stream.size()); + const auto consumed = static_cast(first.stream.inputConsumed); + const auto second = RunAt9Job(context, instance, runMultipleFrames, stream.data() + consumed, stream.size() - consumed); + Require(second.stream.outputWritten == 512 * 2 && second.frames == 2); + Require(consumed + static_cast(second.stream.inputConsumed) == stream.size()); + Require(sceAjmInstanceDestroy(context, instance) == 0); +} + +void TestAt9RunDecodesWholeInput(std::uint32_t context) { + std::vector stream(1024, 0); + std::memcpy(stream.data(), AT9_MONO_SILENT_SUPERFRAME, sizeof(AT9_MONO_SILENT_SUPERFRAME)); + std::uint32_t instance = 0; + Require(sceAjmInstanceCreate(context, 1, 0, &instance) == 0); + const At9Control start{{0, 0, 0}, {0xFE, 0x70, 0x1F, 0xF0}, 0}; + std::int32_t result[2] = {-1, -1}; + RunControl(context, instance, CONTROL_START, &start, sizeof(start), result); + Require(result[0] == 0); + const auto run = RunAt9Job(context, instance, 0, stream.data(), stream.size()); + Require(static_cast(run.stream.inputConsumed) == stream.size() && run.stream.outputWritten == 1024 * 2); + Require(sceAjmInstanceDestroy(context, instance) == 0); +} + } int main() { @@ -971,6 +1027,16 @@ int main() { const std::uint8_t badHeader[4] = {0xFD, 0x72, 0x1F, 0xF0}; Require(sceAjmDecAt9ParseConfigData(badHeader, &info) == invalidParameter); Require(sceAjmDecAt9ParseConfigData(nullptr, &info) == invalidParameter); + std::vector batch(4096); + AjmBatchInfo batchInfo{}; + Require(sceAjmBatchInitialize(batch.data(), batch.size(), &batchInfo) == 0); + bool nullBuffersThrow = false; + try { + sceAjmBatchJobRunSplit(&batchInfo, 0, 0, nullptr, 1, nullptr, 0, nullptr, 0); + } catch (const std::runtime_error&) { + nullBuffersThrow = true; + } + Require(nullBuffersThrow && batchInfo.offset == 0); TestMp3ParseFrame(); TestMp3ParseOfl(); TestMp3(context); @@ -988,5 +1054,7 @@ int main() { TestResampleOpus(context); TestResampleMp3(context); TestResampleAt9(context); + TestAt9GaplessSegments(context); + TestAt9RunDecodesWholeInput(context); Require(sceAjmFinalize(context) == 0); } diff --git a/core/libs/tests/GuestAjmTeardown.cpp b/core/libs/tests/GuestAjmTeardown.cpp new file mode 100644 index 000000000..c62f9ffff --- /dev/null +++ b/core/libs/tests/GuestAjmTeardown.cpp @@ -0,0 +1,81 @@ +#include "SceTypes.hpp" +#include "prx/libc/include/general/VabiMacros.hpp" +#include +#include +#include +#include + +namespace { + +using Initialize = int (APS5_VABI*)(std::int64_t, std::uint32_t*); +using InstanceCreate = int (APS5_VABI*)(std::uint32_t, std::uint32_t, std::uint64_t, std::uint32_t*); +using InstanceDestroy = int (APS5_VABI*)(std::uint32_t, std::uint32_t); +using BatchInitialize = int (APS5_VABI*)(void*, std::size_t, AjmBatchInfo*); +using BatchJobClearContext = int (APS5_VABI*)(AjmBatchInfo*, std::uint32_t, void*); +using BatchStart = int (APS5_VABI*)(std::uint32_t, const AjmBatchInfo*, int, AjmBatchError*, std::uint32_t*); +using BatchWait = int (APS5_VABI*)(std::uint32_t, std::uint32_t, std::uint32_t, AjmBatchError*); + +struct Functions { + Initialize initialize; + InstanceCreate instanceCreate; + InstanceDestroy instanceDestroy; + BatchInitialize batchInitialize; + BatchJobClearContext batchJobClearContext; + BatchStart batchStart; + BatchWait batchWait; +}; + +struct SidebandResult { + std::int32_t result; + std::int32_t internalResult; +}; + +Functions ajm{}; +std::uint32_t context = 0; +std::uint32_t instance = 0; + +[[noreturn]] void Fail(const char* message) { + std::fprintf(stderr, "%s\n", message); + std::fflush(stderr); + std::_Exit(1); +} + +template +T Resolve(void* library, const char* name) { + auto* symbol = dlsym(library, name); + if (symbol == nullptr) Fail(name); + return reinterpret_cast(symbol); +} + +void UseAfterStaticTeardown() { + alignas(16) std::uint8_t buffer[0x100]{}; + AjmBatchInfo info{}; + SidebandResult sideband{-1, -1}; + if (ajm.batchInitialize(buffer, sizeof(buffer), &info) != 0 || ajm.batchJobClearContext(&info, instance, &sideband) != 0) Fail("a batch cannot be built after static teardown"); + std::uint32_t batch = 0; + AjmBatchError error{}; + if (ajm.batchStart(context, &info, 0, &error, &batch) != 0) Fail("sceAjmBatchStart failed after static teardown"); + if (sideband.result != 0) Fail("an instance created before exit is unknown after static teardown"); + if (ajm.batchWait(context, batch, 0, &error) != 0) Fail("sceAjmBatchWait does not find a batch started after static teardown"); + if (ajm.instanceDestroy(context, instance) != 0) Fail("sceAjmInstanceDestroy does not find the instance after static teardown"); +} + +} + +int main(int argc, char** argv) { + if (argc != 2) Fail("usage: guest_ajm_teardown_tests "); + if (std::atexit(UseAfterStaticTeardown) != 0) Fail("atexit failed"); + auto* library = dlopen(argv[1], RTLD_NOW); + if (library == nullptr) Fail(dlerror()); + ajm = { + Resolve(library, "sceAjmInitialize"), + Resolve(library, "sceAjmInstanceCreate"), + Resolve(library, "sceAjmInstanceDestroy"), + Resolve(library, "sceAjmBatchInitialize"), + Resolve(library, "sceAjmBatchJobClearContext"), + Resolve(library, "sceAjmBatchStart"), + Resolve(library, "sceAjmBatchWait"), + }; + if (ajm.initialize(0, &context) != 0 || ajm.instanceCreate(context, 0, 0, &instance) != 0) Fail("AJM setup failed"); + return 0; +} diff --git a/core/libs/tests/GuestAmpr.cpp b/core/libs/tests/GuestAmpr.cpp index 21fa65958..7f45f4ff3 100644 --- a/core/libs/tests/GuestAmpr.cpp +++ b/core/libs/tests/GuestAmpr.cpp @@ -16,6 +16,7 @@ extern "C" { int APS5_VABI sceAmprCommandBufferConstructor(Apr::CommandBufferObject*); int APS5_VABI sceAmprAprCommandBufferConstructor(Apr::CommandBufferObject*, std::uint64_t*, std::uint64_t*); int APS5_VABI sceAmprCommandBufferSetBuffer(Apr::CommandBufferObject*, void*, std::uint32_t); +int APS5_VABI sceAmprCommandBufferReset(Apr::CommandBufferObject*); void* APS5_VABI sceAmprCommandBufferClearBuffer(Apr::CommandBufferObject*); std::uint32_t APS5_VABI sceAmprCommandBufferGetCurrentOffset(const Apr::CommandBufferObject*); std::uint32_t APS5_VABI sceAmprCommandBufferGetNumCommands(const Apr::CommandBufferObject*); @@ -514,6 +515,43 @@ bool MatchesFile(const std::uint8_t* data, std::size_t offset, std::size_t bytes return true; } +void TestZeroFilledBuffer() { + const char* path = "ampr_zero_filled.bin"; + { + std::ofstream file(path, std::ios::binary); + for (std::size_t offset = 0; offset < 256; ++offset) file.put(static_cast(FileByte(offset))); + } + std::uint32_t fileId = 0; + std::uint32_t failed = 0; + Require(sceKernelAprResolveFilepathsToIds(&path, 1, &fileId, &failed) == 0); + KernelEqueue eq = 0; + Require(sceKernelCreateEqueue(&eq, "ampr") == 0); + Require(sceKernelAddAmprEvent(eq, 0, nullptr) == 0); + + alignas(8) std::array memory{}; + Apr::CommandBufferObject buffer{}; + std::uint64_t gatherState = 0; + std::uint64_t scatterState = 0; + Require(sceAmprCommandBufferSetBuffer(&buffer, memory.data(), static_cast(memory.size())) == 0); + Require(sceAmprCommandBufferReset(&buffer) == 0); + std::array data{}; + std::uint64_t done = 0; + Require(sceAmprAprCommandBufferReadFile(&buffer, &gatherState, &scatterState, fileId, data.data(), data.size(), 64) == 0); + Require(sceAmprCommandBufferWriteAddressOnCompletion(&buffer, &done, 0x1234567) == 0); + Require(sceAmprCommandBufferWriteKernelEventQueue_04_00(&buffer, static_cast(eq), 0, 0, 0) == 0); + Require(sceKernelAprSubmitCommandBuffer(&buffer, 1) == 0); + Require(MatchesFile(data.data(), 64, data.size()) && done == 0x1234567); + + KernelEvent event{}; + int count = 0; + const KernelUseconds poll = 0; + Require(sceKernelWaitEqueue(eq, &event, 1, &count, &poll) == 0 && count == 1); + Require(sceKernelGetEventFilter(&event) == -25 && sceKernelGetEventId(&event) == 0); + Require(sceKernelDeleteAmprEvent(eq, 0) == 0); + Require(sceKernelDeleteEqueue(eq) == 0); + std::remove(path); +} + void TestGatherScatter() { const char* path = "ampr_gather_scatter.bin"; { @@ -949,6 +987,7 @@ int main() { TestVersionedCounters(); TestClearBuffer(); TestConstructed(); + TestZeroFilledBuffer(); TestGatherScatter(); TestAmm(); TestAmmRemapAndProtect(); diff --git a/core/libs/tests/GuestAudio3dPort.cpp b/core/libs/tests/GuestAudio3dPort.cpp index 30b8c7d8a..fa5d8f243 100644 --- a/core/libs/tests/GuestAudio3dPort.cpp +++ b/core/libs/tests/GuestAudio3dPort.cpp @@ -16,8 +16,10 @@ int APS5_VABI sceAudio3dPortSetAttribute(std::uint32_t port_id, std::uint32_t at int APS5_VABI sceAudio3dPortGetQueueLevel(std::uint32_t port_id, std::uint32_t* queue_level, std::uint32_t* queue_available); int APS5_VABI sceAudio3dPortAdvance(std::uint32_t port_id); int APS5_VABI sceAudio3dPortPush(std::uint32_t port_id, std::uint32_t blocking); +int APS5_VABI sceAudio3dPortFlush(std::uint32_t port_id); int APS5_VABI sceAudio3dObjectReserve(std::uint32_t port_id, std::uint32_t* object_id); int APS5_VABI sceAudio3dObjectUnreserve(std::uint32_t port_id, std::uint32_t object_id); +int APS5_VABI sceAudio3dObjectSetAttributes(std::uint32_t port_id, std::uint32_t object_id, std::uint64_t num_attributes, const Audio3dAttribute* attribute_array); } namespace { @@ -212,6 +214,41 @@ void CheckQueue() { Require(sceAudio3dPortClose(0) == 0, "close"); } +void CheckFlush() { + Require(sceAudio3dPortFlush(0) == INVALID_PORT, "flush needs an open port"); + Audio3dOpenParameters parameters = Defaults(); + std::uint32_t id = 7; + Require(Open(parameters, &id) == 0 && id == 0, "open"); + Require(sceAudio3dPortAdvance(0) == 0, "advance"); + RequireLevel(1, 1, "one frame queued"); + Require(sceAudio3dPortFlush(1) == INVALID_PORT, "flush of port 1"); + Require(sceAudio3dPortFlush(0) == 0, "flush"); + RequireLevel(0, 2, "flush drops the queued frames"); + Require(sceAudio3dPortClose(0) == 0, "close"); +} + +void CheckObjectAttributes() { + float value = 0.5f; + Audio3dAttribute attribute{0x10001, 0, &value, sizeof(value)}; + Require(sceAudio3dObjectSetAttributes(0, 1, 1, &attribute) == INVALID_PORT, "attributes need an open port"); + Audio3dOpenParameters parameters = Defaults(); + std::uint32_t id = 7; + Require(Open(parameters, &id) == 0 && id == 0, "open"); + std::uint32_t object = 0; + Require(sceAudio3dObjectReserve(0, &object) == 0 && object == 1, "reserve"); + Require(sceAudio3dObjectSetAttributes(1, 1, 1, &attribute) == INVALID_PORT, "attributes on port 1"); + Require(sceAudio3dObjectSetAttributes(0, 1, 0, &attribute) == INVALID_PARAMETER, "zero attributes"); + Require(sceAudio3dObjectSetAttributes(0, 1, 1, nullptr) == INVALID_PARAMETER, "null array"); + Require(sceAudio3dObjectSetAttributes(0, 2, 1, &attribute) == INVALID_OBJECT, "unreserved object"); + Require(sceAudio3dObjectSetAttributes(0, 1, 1, &attribute) == 0, "set one attribute"); + Audio3dAttribute reset{0x20000, 0, nullptr, 0}; + Require(sceAudio3dObjectSetAttributes(0, 1, 1, &reset) == 0, "reset state takes a null value"); + Audio3dAttribute missing{0x10001, 0, nullptr, 0}; + Require(sceAudio3dObjectSetAttributes(0, 1, 1, &missing) == INVALID_PARAMETER, "a value is required"); + Require(sceAudio3dObjectUnreserve(0, 1) == 0, "unreserve"); + Require(sceAudio3dPortClose(0) == 0, "close"); +} + void CheckTerminate() { std::uint32_t id = 7; const Audio3dOpenParameters parameters = Defaults(); @@ -236,5 +273,7 @@ int main() { CheckOpenClose(); CheckObjects(); CheckQueue(); + CheckFlush(); + CheckObjectAttributes(); CheckTerminate(); } diff --git a/core/libs/tests/GuestAudiodec.cpp b/core/libs/tests/GuestAudiodec.cpp index fb9a4b3e5..2e6af3008 100644 --- a/core/libs/tests/GuestAudiodec.cpp +++ b/core/libs/tests/GuestAudiodec.cpp @@ -48,6 +48,74 @@ const std::uint8_t AAC_ADTS[] = { 0x33, 0x50, 0x8E, 0x34, 0xA2, 0x64, 0xA6, 0xBB, 0xDF, 0xD5, 0xAA, 0xE4, 0xB4, 0x97, 0x1D, 0x72, 0x48, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x1C, }; +const std::uint8_t AAC_RAW_51[] = { + 0x00, 0xDE, 0x34, 0xAC, 0xF1, 0x1D, 0x0C, 0x05, 0x89, 0xA2, 0x61, 0x40, + 0xB4, 0x4A, 0x37, 0x8D, 0xFF, 0x7F, 0xFD, 0x3F, 0xCF, 0xFB, 0x79, 0xE7, + 0xE3, 0xCE, 0x4C, 0x8B, 0x5D, 0xD5, 0xD5, 0x3F, 0xE9, 0x47, 0x6A, 0x5D, + 0x69, 0xBA, 0xB1, 0x9A, 0x78, 0xAA, 0x3C, 0xD4, 0xF7, 0xA6, 0xD3, 0x13, + 0xCC, 0x21, 0x93, 0x2B, 0x3E, 0x67, 0xDE, 0x74, 0x1A, 0xA3, 0xE7, 0x31, + 0xE2, 0x47, 0xE8, 0x68, 0xFB, 0xD5, 0xF6, 0xB5, 0xFC, 0x57, 0xDE, 0x9E, + 0x43, 0x6F, 0x3E, 0xF5, 0xF2, 0x1F, 0xBC, 0xFB, 0xC5, 0xBC, 0x53, 0x16, + 0xE4, 0x57, 0xAC, 0x64, 0x14, 0xE6, 0x88, 0xAF, 0x66, 0xBA, 0x8C, 0x7A, + 0x86, 0xF5, 0x8D, 0xC4, 0xA2, 0x36, 0x89, 0xE3, 0x21, 0xF9, 0xA5, 0x8A, + 0xAF, 0xF0, 0xE5, 0x91, 0x2C, 0x15, 0xA2, 0x67, 0xF4, 0x97, 0xDA, 0x40, + 0x0A, 0x00, 0x8A, 0x45, 0x7C, 0x92, 0xEA, 0x8F, 0x87, 0x56, 0xC2, 0x40, + 0xE1, 0xA2, 0x23, 0x1E, 0x89, 0xE7, 0xBA, 0x61, 0xA3, 0xD5, 0xB9, 0x77, + 0x27, 0x13, 0x73, 0x49, 0xE4, 0x98, 0x83, 0x3A, 0x25, 0x86, 0x3A, 0x10, + 0x3D, 0x5C, 0xC8, 0xC0, 0x8F, 0xDB, 0x58, 0x50, 0xB0, 0xFB, 0x3F, 0x33, + 0xAD, 0xE1, 0xD7, 0xC2, 0xF7, 0xB3, 0x98, 0xDF, 0x39, 0x87, 0xE2, 0x97, + 0x34, 0xB4, 0x2A, 0x3A, 0xA0, 0x64, 0x0A, 0x80, 0x76, 0x84, 0xA6, 0x1D, + 0x43, 0xF4, 0x0E, 0x12, 0x3E, 0xF2, 0xE2, 0x6A, 0x1F, 0x30, 0xA1, 0xD5, + 0xE8, 0x0A, 0x22, 0x39, 0x9E, 0x44, 0xF3, 0xD1, 0x81, 0x6F, 0x49, 0x5A, + 0x9C, 0x1F, 0x54, 0xB9, 0x2B, 0xB4, 0x73, 0xA3, 0x8D, 0x74, 0x6F, 0x87, + 0x2D, 0x51, 0xCE, 0x3A, 0x39, 0xED, 0x38, 0x49, 0x55, 0x7C, 0x7D, 0xBE, + 0xED, 0x65, 0x29, 0xAE, 0x3E, 0xBE, 0x70, 0xD1, 0x08, 0xD2, 0x1B, 0xEC, + 0xF0, 0x66, 0x56, 0x8A, 0x03, 0xA9, 0x80, 0xE9, 0x5F, 0x2F, 0xFA, 0x7F, + 0xF1, 0xFE, 0xFF, 0xED, 0xE7, 0x9F, 0x8F, 0x3C, 0xEB, 0x37, 0xAC, 0xEA, + 0xD9, 0x75, 0x02, 0xD4, 0xDC, 0xDD, 0x45, 0xED, 0x9C, 0x2A, 0x83, 0x34, + 0xF1, 0x54, 0xD2, 0x3F, 0xBD, 0x36, 0x98, 0x69, 0x87, 0x04, 0xAA, 0x5F, + 0x99, 0xF7, 0x9D, 0x06, 0xA8, 0xF9, 0xCC, 0x78, 0x91, 0xFA, 0x1A, 0x3E, + 0xF5, 0x7C, 0xD8, 0xF7, 0x9F, 0x7A, 0x79, 0x0D, 0xD5, 0x45, 0x5C, 0x55, + 0x2A, 0x4A, 0xA0, 0x66, 0x59, 0xC6, 0x82, 0x89, 0x66, 0x0C, 0x65, 0xF5, + 0xE1, 0xEA, 0xEE, 0xD9, 0x9E, 0x5F, 0x5B, 0xB4, 0xE0, 0x93, 0x0F, 0x11, + 0xA2, 0x49, 0xC3, 0x2F, 0xC7, 0xA1, 0x6E, 0x1A, 0x81, 0xFF, 0x57, 0x7D, + 0x6F, 0xC3, 0xAB, 0x61, 0x20, 0x70, 0xD1, 0x11, 0x8F, 0x44, 0xEB, 0x3C, + 0xF4, 0x60, 0x59, 0xA4, 0xAD, 0x4E, 0x0F, 0x54, 0xB4, 0x5C, 0xDF, 0x67, + 0x88, 0x68, 0xA0, 0x3A, 0x9F, 0x0E, 0x95, 0xF2, 0xFF, 0xB7, 0xFE, 0x3F, + 0xDF, 0xFD, 0xBC, 0xF3, 0xF1, 0xE7, 0xBE, 0xB2, 0xA6, 0xFA, 0xBA, 0x49, + 0x02, 0xD5, 0x6E, 0x6E, 0xA2, 0x61, 0xC2, 0xA8, 0x33, 0x4F, 0x15, 0x4D, + 0x23, 0xFB, 0xD3, 0x69, 0x86, 0x6C, 0x70, 0x4A, 0xA5, 0xF9, 0x9F, 0x79, + 0xD0, 0x6A, 0x8F, 0x9C, 0xC7, 0x89, 0x1F, 0xA1, 0xA3, 0xEF, 0x57, 0xCD, + 0x2D, 0xE7, 0xEF, 0x4F, 0x21, 0xB7, 0x9F, 0x7A, 0xF9, 0x0F, 0xDE, 0x3B, + 0xC5, 0xBC, 0x50, 0x0B, 0x33, 0xEB, 0x19, 0x05, 0x39, 0xA2, 0x2B, 0xD9, + 0xAE, 0xA3, 0x1D, 0x81, 0xCC, 0x1B, 0xCC, 0x9C, 0xA3, 0x5F, 0x03, 0xE4, + 0x96, 0xE6, 0xF8, 0xAC, 0xC0, 0x98, 0x22, 0xA8, 0x3E, 0x29, 0x7A, 0xC8, + 0x30, 0xEA, 0xD8, 0x48, 0x1C, 0x34, 0x44, 0x63, 0xD1, 0x39, 0x18, 0xD2, + 0x1B, 0xEC, 0xF1, 0x0D, 0x14, 0x07, 0x52, 0xE1, 0xD2, 0xBE, 0x5F, 0xF6, + 0xFF, 0xC7, 0xFB, 0xFF, 0xB7, 0x9E, 0x7E, 0x3C, 0xF7, 0xD6, 0x54, 0xDF, + 0x57, 0x49, 0x20, 0x5A, 0xAD, 0xCD, 0xD4, 0x4C, 0x6F, 0x85, 0x50, 0x66, + 0x9E, 0x2A, 0x9A, 0x47, 0xF7, 0xA6, 0xD3, 0x0C, 0xD8, 0xE0, 0x95, 0x4B, + 0xF3, 0x3E, 0xF3, 0xA0, 0xD5, 0x1F, 0x39, 0x8F, 0x12, 0x3F, 0x43, 0x47, + 0xDE, 0xAF, 0x9A, 0x5B, 0xCF, 0xDE, 0x9E, 0x43, 0x6F, 0x3E, 0xF5, 0xF2, + 0x1F, 0xBC, 0x77, 0x8B, 0x78, 0xA0, 0x16, 0x67, 0xD6, 0x32, 0x0A, 0x73, + 0x44, 0x57, 0xB3, 0x5D, 0x46, 0x3B, 0x03, 0x98, 0x37, 0x99, 0x39, 0x46, + 0xBE, 0x07, 0xC9, 0x2D, 0xCD, 0xF1, 0x59, 0x81, 0x30, 0x45, 0x50, 0x7C, + 0x52, 0xF5, 0x90, 0x61, 0xD5, 0xB0, 0x90, 0x38, 0x68, 0x88, 0xC7, 0xA2, + 0x7B, 0x9D, 0x3C, 0x6A, 0xE3, 0x56, 0xE9, 0x8F, 0x73, 0x3A, 0xA6, 0xFB, + 0x3C, 0x43, 0x45, 0x01, 0xD4, 0xF8, 0x74, 0xAF, 0x97, 0xFD, 0xBF, 0xF1, + 0xFE, 0xFF, 0xED, 0xE7, 0x9F, 0x8F, 0x3D, 0xF5, 0x95, 0x37, 0xD5, 0xD2, + 0x48, 0x16, 0xAB, 0x73, 0x75, 0x13, 0x0E, 0x15, 0x41, 0x9A, 0x78, 0xAA, + 0x69, 0x1F, 0xDE, 0x9B, 0x4C, 0x33, 0x63, 0x82, 0x55, 0x2F, 0xCC, 0xFB, + 0xCE, 0x83, 0x54, 0x7C, 0xE6, 0x3C, 0x48, 0xFD, 0x0D, 0x1F, 0x7A, 0xBE, + 0x69, 0x6F, 0x3F, 0x7A, 0x79, 0x0D, 0xBC, 0xFB, 0xD7, 0xC8, 0x7E, 0xF1, + 0xDE, 0x2D, 0xE2, 0x80, 0x59, 0x9F, 0x58, 0xC8, 0x29, 0xCD, 0x11, 0x5E, + 0xCD, 0x75, 0x18, 0xEC, 0x0E, 0x60, 0xDE, 0x64, 0xE5, 0x1A, 0xF8, 0x1F, + 0x24, 0xB7, 0x37, 0xC5, 0x66, 0x04, 0xC1, 0x15, 0x41, 0xF1, 0x4B, 0xD6, + 0x41, 0x87, 0x56, 0xC2, 0x40, 0xE1, 0xA2, 0x23, 0x1E, 0x89, 0xD8, 0x3B, + 0x00, 0x6B, 0x1B, 0xED, 0x80, 0x8B, 0x8E, 0x4A, 0x30, 0x78, 0x0D, 0xAE, + 0x19, 0x60, 0x33, 0x3A, 0x38, +}; + const std::uint8_t MP3_MONO[] = { 0xFF, 0xFB, 0x14, 0xC4, 0x00, 0x00, 0x03, 0xB0, 0x21, 0x5E, 0xF4, 0x30, 0x80, 0x30, 0x98, 0x88, 0xA9, 0x83, 0x34, 0x70, 0x00, 0x18, 0x89, 0xCB, 0x20, 0x00, 0x3B, 0xBB, 0xB9, 0x9F, 0xD7, 0x0E, 0x06, 0x06, 0x06, 0x2C, 0x3E, 0xEF, 0x83, 0xE7, 0xF1, 0x00, 0x63, 0x83, 0xFD, 0x4E, 0xFC, 0xFF, @@ -115,12 +183,12 @@ std::size_t AdtsLength(const std::uint8_t* frame) { return static_cast((frame[3] & 3) << 11 | frame[4] << 3 | frame[5] >> 5); } -void TestAac(bool adts) { - AudiodecParamM4aac param{sizeof(param), 1, adts ? 1u : 2u, 3, 2, 0}; +void TestAac(bool adts, std::uint32_t maxChannels = 2) { + AudiodecParamM4aac param{sizeof(param), 1, adts ? 1u : 2u, 3, maxChannels, 0}; AudiodecM4aacInfo info{sizeof(info), 0, 0, 0, 0}; AudiodecCtrl ctrl{¶m, &info, nullptr, nullptr}; const std::int32_t handle = sceAudiodecCreateDecoder(&ctrl, 3); - Require(handle > 0 && info.ui_sampling_freq == 48000 && info.ui_number_of_channels == 2); + Require(handle > 0 && info.ui_sampling_freq == 48000 && info.ui_number_of_channels == maxChannels); Streams streams; int crossings = 0; for (std::size_t offset = 0, frame = 0; offset < sizeof(AAC_ADTS); offset += AdtsLength(AAC_ADTS + offset), ++frame) { @@ -143,6 +211,20 @@ void TestAac(bool adts) { Require(sceAudiodecDeleteDecoder(handle) == 0); } +void TestAacRaw51(std::uint32_t maxChannels) { + AudiodecParamM4aac param{sizeof(param), 1, 2, 3, maxChannels, 0}; + AudiodecM4aacInfo info{sizeof(info), 0, 0, 0, 0}; + AudiodecCtrl ctrl{¶m, &info, nullptr, nullptr}; + const std::int32_t handle = sceAudiodecCreateDecoder(&ctrl, 3); + Require(handle > 0 && info.ui_number_of_channels == maxChannels); + Streams streams; + Require(Decode(handle, ctrl, streams, AAC_RAW_51, sizeof(AAC_RAW_51)) == 0); + Require(streams.pcm.ui_pcm_size == 1024 * 6 * 2); + Require(info.ui_sampling_freq == 48000 && info.ui_number_of_channels == 6 && info.i_result == 0); + Require(sceAudiodecClearContext(handle) == 0); + Require(sceAudiodecDeleteDecoder(handle) == 0); +} + void TestMp3() { AudiodecParamMp3 param{sizeof(param), 2}; AudiodecMp3Info info{}; @@ -229,6 +311,10 @@ int main() { TestAac(true); TestAac(false); + TestAac(false, 6); + TestAac(false, 8); + TestAacRaw51(6); + TestAacRaw51(8); TestMp3(); TestAt9(); for (std::uint32_t type = 1; type <= 3; ++type) Require(sceAudiodecTermLibrary(type) == 0); diff --git a/core/libs/tests/GuestAvPlayer.cpp b/core/libs/tests/GuestAvPlayer.cpp index b38ee2895..2fb32a5db 100644 --- a/core/libs/tests/GuestAvPlayer.cpp +++ b/core/libs/tests/GuestAvPlayer.cpp @@ -2,6 +2,7 @@ #include #include #include +#include #include #include #include @@ -993,6 +994,74 @@ void TestHandedOutFramesStayIntact() { Check(sceAvPlayerClose(player) == 0, "close failed"); } +void TestPs5ExtendedInitLayout() { + constexpr std::size_t Ps5Size = 0x230; + constexpr std::size_t Ps5AutoStart = 0x74; + constexpr std::size_t Ps5ThreadParameters = 0x78; + constexpr std::size_t Ps5VideoFrameBuffers = 0x228; + constexpr std::int32_t Buffers = 3; + constexpr int DecodeAheadBuffers = 4; + static_assert(offsetof(AvPlayerInitDataEx, audio_decoder_priority) == Ps5AutoStart); + const auto run = [&](bool autoStart) { + Events events; + const AvPlayerInitData common = InitData(&events); + AvPlayerInitDataEx head{}; + head.this_size = Ps5Size; + head.memory_replacement = common.memory_replacement; + head.event_replacement = common.event_replacement; + head.default_language = common.default_language; + alignas(AvPlayerInitDataEx) std::array raw{}; + std::memcpy(raw.data(), &head, Ps5AutoStart); + raw[Ps5AutoStart] = autoStart ? 1 : 0; + if (!autoStart) std::fill(raw.begin() + static_cast(Ps5ThreadParameters), raw.end(), std::uint8_t{1}); + std::memcpy(raw.data() + Ps5VideoFrameBuffers, &Buffers, sizeof(Buffers)); + AvPlayerInternal* player = nullptr; + Check(sceAvPlayerInitEx(reinterpret_cast(raw.data()), &player) == 0 && player != nullptr, "PS5 extended init failed"); + Check(sceAvPlayerAddSource(player, "/app0/avplayer.mp4") == 0, "add source after the PS5 extended init failed"); + if (autoStart) { + Check(WaitFor([&] { return events.Seen(EventPlay); }), "auto start at 0x74 of the PS5 extended init did not start playback"); + } else { + Check(WaitFor([&] { return events.Seen(EventReady); }), "ready event missing after the PS5 extended init"); + std::this_thread::sleep_for(std::chrono::milliseconds(200)); + Check(!events.Seen(EventPlay), "playback started although auto start at 0x74 of the PS5 extended init is 0"); + Check(sceAvPlayerEnableStream(player, VideoStream) == 0, "enable video after the PS5 extended init failed"); + Check(sceAvPlayerStart(player) == 0, "start after the PS5 extended init failed"); + } + Check(WaitFor([&] { return TextureCount() == Buffers + DecodeAheadBuffers; }), "video frame buffer count at 0x228 of the PS5 extended init not used"); + Check(sceAvPlayerClose(player) == 0, "close after the PS5 extended init failed"); + }; + run(false); + run(true); +} + +void TestUnsyncedVideoKeepsUpWithAudio() { + AvPlayerInitData init = InitData(nullptr); + auto* player = sceAvPlayerInit(&init); + Check(player != nullptr, "init failed"); + Check(sceAvPlayerSetAvSyncMode(player, 1) == 0, "sync mode rejected"); + Check(sceAvPlayerAddSource(player, "/app0/avplayer.mp4") == 0, "add source failed"); + Check(sceAvPlayerEnableStream(player, VideoStream) == 0, "enable video failed"); + Check(sceAvPlayerEnableStream(player, EnglishAudioStream) == 0, "enable audio failed"); + Check(sceAvPlayerStart(player) == 0, "start failed"); + AvPlayerFrameInfo sound{}; + Check(WaitFor([&] { return sceAvPlayerGetAudioData(player, &sound) != 0; }), "no audio"); + const auto start = std::chrono::steady_clock::now(); + const auto first = sound.timestamp; + std::uint64_t worst = 0; + while (sceAvPlayerIsActive(player) && std::chrono::steady_clock::now() - start < std::chrono::milliseconds(900)) { + const auto elapsed = static_cast(std::chrono::duration_cast(std::chrono::steady_clock::now() - start).count()); + while (sound.timestamp <= first + elapsed && sceAvPlayerGetAudioData(player, &sound)) CheckEnglishAudio(sound); + AvPlayerFrameInfoEx frame{}; + if (sceAvPlayerGetVideoDataEx(player, &frame)) { + CheckVideoFrame(frame); + if (sound.timestamp > frame.timestamp) worst = std::max(worst, sound.timestamp - frame.timestamp); + } + std::this_thread::sleep_for(std::chrono::milliseconds(66)); + } + Check(worst <= 150, "video fell " + std::to_string(worst) + " ms behind the audio"); + Check(sceAvPlayerClose(player) == 0, "close failed"); +} + } int main() { @@ -1005,6 +1074,8 @@ int main() { TestOptionalVideoBuffersRespectMemoryLimit(); TestFileReplacementAutoStart(); TestHandedOutFramesStayIntact(); + TestPs5ExtendedInitLayout(); + TestUnsyncedVideoKeepsUpWithAudio(); std::puts("AvPlayer tests passed"); return 0; } catch (const std::exception& error) { diff --git a/core/libs/tests/GuestCondRecursiveMutex.cpp b/core/libs/tests/GuestCondRecursiveMutex.cpp new file mode 100644 index 000000000..92f716607 --- /dev/null +++ b/core/libs/tests/GuestCondRecursiveMutex.cpp @@ -0,0 +1,101 @@ +#include "SceTypes.hpp" +#include + +extern "C" { +int APS5_VABI scePthreadCreate(Pthread* thread, const PthreadAttr* attr, PthreadEntry entry, void* arg, const char* name); +int APS5_VABI scePthreadJoin(Pthread thread, void** retval); +int APS5_VABI scePthreadMutexattrInit(PthreadMutexattr* attr); +int APS5_VABI scePthreadMutexattrDestroy(PthreadMutexattr* attr); +int APS5_VABI scePthreadMutexattrSettype(PthreadMutexattr* attr, int type); +int APS5_VABI scePthreadMutexInit(PthreadMutex* mutex, const PthreadMutexattr* attr, const char* name); +int APS5_VABI scePthreadMutexDestroy(PthreadMutex* mutex); +int APS5_VABI scePthreadMutexLock(PthreadMutex* mutex); +int APS5_VABI scePthreadMutexUnlock(PthreadMutex* mutex); +int APS5_VABI scePthreadCondInit(PthreadCond* cond, const PthreadCondattr* attr, const char* name); +int APS5_VABI scePthreadCondDestroy(PthreadCond* cond); +int APS5_VABI scePthreadCondSignal(PthreadCond* cond); +int APS5_VABI scePthreadCondWait(PthreadCond* cond, PthreadMutex* mutex); +int APS5_VABI scePthreadCondTimedwait(PthreadCond* cond, PthreadMutex* mutex, unsigned int usec); +int APS5_VABI pthread_cond_wait_nid_postfix(PthreadCond* cond, PthreadMutex* mutex); +int APS5_VABI pthread_cond_timedwait_nid_postfix(PthreadCond* cond, PthreadMutex* mutex, const KernelTimespec* abstime); +int APS5_VABI clock_gettime_nid_postfix(int clockId, KernelTimespec* tp); +} + +static constexpr int SCE_OK = 0; +static constexpr int SCE_KERNEL_ERROR_EPERM = static_cast(0x80020001); +static constexpr int SCE_KERNEL_ERROR_ETIMEDOUT = static_cast(0x8002003C); +static constexpr int POSIX_ETIMEDOUT = 60; +static constexpr int MUTEX_TYPE_RECURSIVE = 2; + +static void Require(bool value) { if (!value) std::abort(); } + +struct Context { + PthreadMutex mutex = nullptr; + PthreadCond cond = nullptr; + bool signalled = false; +}; + +static void* APS5_VABI Signaller(void* arg) { + auto& context = *static_cast(arg); + Require(scePthreadMutexLock(&context.mutex) == SCE_OK); + context.signalled = true; + Require(scePthreadCondSignal(&context.cond) == SCE_OK); + Require(scePthreadMutexUnlock(&context.mutex) == SCE_OK); + return nullptr; +} + +static int SceWait(Context& context) { + return scePthreadCondWait(&context.cond, &context.mutex); +} + +static int SceTimedwait(Context& context) { + const int result = scePthreadCondTimedwait(&context.cond, &context.mutex, 1000); + return result == SCE_KERNEL_ERROR_ETIMEDOUT ? SCE_OK : result; +} + +static int PosixWait(Context& context) { + return pthread_cond_wait_nid_postfix(&context.cond, &context.mutex); +} + +static int PosixTimedwait(Context& context) { + KernelTimespec deadline{}; + Require(clock_gettime_nid_postfix(0, &deadline) == 0); + deadline.tv_nsec += 1000000; + if (deadline.tv_nsec >= 1000000000) { + deadline.tv_nsec -= 1000000000; + ++deadline.tv_sec; + } + const int result = pthread_cond_timedwait_nid_postfix(&context.cond, &context.mutex, &deadline); + return result == POSIX_ETIMEDOUT ? SCE_OK : result; +} + +static void WaitWithRecursiveMutex(int (*wait)(Context&)) { + Context context; + PthreadMutexattr attr = nullptr; + Require(scePthreadMutexattrInit(&attr) == SCE_OK); + Require(scePthreadMutexattrSettype(&attr, MUTEX_TYPE_RECURSIVE) == SCE_OK); + Require(scePthreadMutexInit(&context.mutex, &attr, nullptr) == SCE_OK); + Require(scePthreadMutexattrDestroy(&attr) == SCE_OK); + Require(scePthreadCondInit(&context.cond, nullptr, nullptr) == SCE_OK); + + Require(scePthreadMutexLock(&context.mutex) == SCE_OK); + Require(scePthreadMutexLock(&context.mutex) == SCE_OK); + Pthread thread = nullptr; + Require(scePthreadCreate(&thread, nullptr, Signaller, &context, nullptr) == SCE_OK); + while (!context.signalled) + Require(wait(context) == SCE_OK); + Require(scePthreadMutexUnlock(&context.mutex) == SCE_OK); + Require(scePthreadMutexUnlock(&context.mutex) == SCE_OK); + Require(scePthreadMutexUnlock(&context.mutex) == SCE_KERNEL_ERROR_EPERM); + Require(scePthreadJoin(thread, nullptr) == SCE_OK); + + Require(scePthreadCondDestroy(&context.cond) == SCE_OK); + Require(scePthreadMutexDestroy(&context.mutex) == SCE_OK); +} + +int main() { + WaitWithRecursiveMutex(SceWait); + WaitWithRecursiveMutex(SceTimedwait); + WaitWithRecursiveMutex(PosixWait); + WaitWithRecursiveMutex(PosixTimedwait); +} diff --git a/core/libs/tests/GuestCredentials.cpp b/core/libs/tests/GuestCredentials.cpp new file mode 100644 index 000000000..aa2c8544f --- /dev/null +++ b/core/libs/tests/GuestCredentials.cpp @@ -0,0 +1,31 @@ +#include "prx/libc/include/general/VabiMacros.hpp" +#include +#include +#include +extern "C" { +int APS5_VABI seteuid_nid_postfix(std::uint32_t); +int APS5_VABI setegid_nid_postfix(std::uint32_t); +int* APS5_VABI __error_nid_postfix(); +} +using SetId = int (APS5_VABI *)(std::uint32_t); +static void Require(bool value) { if (!value) std::abort(); } +static bool Throws(SetId function, std::uint32_t id) { + try { + function(id); + } catch (const std::runtime_error&) { + return true; + } + return false; +} +int main() { + *__error_nid_postfix() = 13; + Require(seteuid_nid_postfix(0) == 0); + Require(setegid_nid_postfix(0) == 0); + Require(*__error_nid_postfix() == 13); + Require(Throws(seteuid_nid_postfix, 1000)); + Require(Throws(setegid_nid_postfix, 1000)); + Require(Throws(seteuid_nid_postfix, 0xFFFFFFFFu)); + Require(Throws(setegid_nid_postfix, 0xFFFFFFFFu)); + Require(seteuid_nid_postfix(0) == 0); + Require(setegid_nid_postfix(0) == 0); +} diff --git a/core/libs/tests/GuestCurrentCpu.cpp b/core/libs/tests/GuestCurrentCpu.cpp new file mode 100644 index 000000000..b7d7d75cc --- /dev/null +++ b/core/libs/tests/GuestCurrentCpu.cpp @@ -0,0 +1,66 @@ +#include "SceTypes.hpp" +#include "prx/libkernel/Pthread/include/Pthread.hpp" +#include +#include +#include + +extern "C" { +int APS5_VABI sceKernelGetCurrentCpu(void); +KernelCpumask APS5_VABI sceKernelGetAvailableCpumask(void); +Pthread APS5_VABI scePthreadSelf(); +int APS5_VABI scePthreadCreate(Pthread* thread, const PthreadAttr* attr, PthreadEntry entry, void* arg, const char* name); +int APS5_VABI scePthreadJoin(Pthread thread, void** retval); +int APS5_VABI scePthreadSetaffinity(Pthread thread, KernelCpumask mask); +} + +static constexpr int SCE_OK = 0; +static constexpr unsigned MAX_HOST_CPUS = 256; +static constexpr int CALLS_PER_THREAD = 64; + +static void Require(bool value) { if (!value) std::abort(); } + +static bool InMask(int cpu, KernelCpumask mask) { + return cpu >= 0 && cpu < 64 && (mask >> cpu & 1) != 0; +} + +struct PinnedRun { + KernelCpumask affinity; + bool inside; +}; + +static void* APS5_VABI ReportWhilePinned(void* arg) { + auto* run = static_cast(arg); + run->inside = scePthreadSetaffinity(scePthreadSelf(), run->affinity) == SCE_OK; + for (int call = 0; call < CALLS_PER_THREAD && run->inside; ++call) + run->inside = InMask(sceKernelGetCurrentCpu(), run->affinity); + return nullptr; +} + +int main() { + const KernelCpumask available = sceKernelGetAvailableCpumask(); + + const std::array affinities{available, 0x1FFB, 0x3, 0x3F, 0x1000, 0x1, 0xFFFF, 0x10000}; + for (const KernelCpumask affinity : affinities) { + const KernelCpumask expected = (affinity & available) != 0 ? affinity & available : available; + KernelCpumask reached = 0; + for (unsigned hostCpu = 0; hostCpu < MAX_HOST_CPUS; ++hostCpu) { + const int cpu = GuestCpuFromHost(hostCpu, affinity); + Require(InMask(cpu, expected)); + Require(GuestCpuFromHost(hostCpu, affinity) == cpu); + reached |= KernelCpumask{1} << cpu; + if (hostCpu + 1 == static_cast(std::popcount(expected))) Require(reached == expected); + } + Require(reached == expected); + } + + for (int call = 0; call < CALLS_PER_THREAD; ++call) Require(InMask(sceKernelGetCurrentCpu(), available)); + + for (KernelCpumask cpu = 1; cpu != 0; cpu <<= 1) { + if ((available & cpu) == 0) continue; + PinnedRun run{cpu, false}; + Pthread thread = nullptr; + Require(scePthreadCreate(&thread, nullptr, ReportWhilePinned, &run, nullptr) == SCE_OK); + Require(scePthreadJoin(thread, nullptr) == SCE_OK); + Require(run.inside); + } +} diff --git a/core/libs/tests/GuestDirectory.cpp b/core/libs/tests/GuestDirectory.cpp index 6d3288f0d..4ada7880c 100644 --- a/core/libs/tests/GuestDirectory.cpp +++ b/core/libs/tests/GuestDirectory.cpp @@ -46,6 +46,21 @@ int main() { Require(*__error_nid_postfix() == 20); Require(opendir_nid_postfix("") == nullptr && *__error_nid_postfix() == 2); Require(closedir_nid_postfix(nullptr) == -1 && *__error_nid_postfix() == 9); + const std::u8string folder = u8"セーブ"; + const std::u8string file = u8"été.txt"; + Require(std::filesystem::create_directory(root / std::filesystem::path(folder))); + { std::ofstream stream(root / std::filesystem::path(folder) / std::filesystem::path(file)); stream << "x"; } + const std::string guestFolder = root.string() + "/" + std::string(folder.begin(), folder.end()); + void* unicode = opendir_nid_postfix(guestFolder.c_str()); + Require(unicode != nullptr); + std::map names; + while (auto* entry = readdir_nid_postfix(unicode)) { + Require(entry->nameLength == std::strlen(entry->name)); + names[entry->name] = entry->type; + } + Require(names.at(std::string(file.begin(), file.end())) == 8 && names.size() == 3); + Require(closedir_nid_postfix(unicode) == 0); + std::filesystem::remove_all(root / std::filesystem::path(folder)); std::filesystem::remove(root / "sample.txt"); std::filesystem::remove(root / "subdirectory"); std::filesystem::remove(root); diff --git a/core/libs/tests/GuestDirectoryEntries.cpp b/core/libs/tests/GuestDirectoryEntries.cpp index 912247d44..35bf2413c 100644 --- a/core/libs/tests/GuestDirectoryEntries.cpp +++ b/core/libs/tests/GuestDirectoryEntries.cpp @@ -18,6 +18,9 @@ std::int64_t APS5_VABI sceKernelRead(int, void*, std::size_t); int APS5_VABI sceKernelLseek(int, std::int64_t, int); int APS5_VABI sceKernelGetdents(int, char*, int); int APS5_VABI sceKernelGetdirentries(int, char*, int, std::int64_t*); +int APS5_VABI getdents_nid_postfix(int, char*, int); +int APS5_VABI getdirentries_nid_postfix(int, char*, int, std::int64_t*); +int* APS5_VABI __error_nid_postfix(); std::int64_t APS5_VABI sceKernelPread(int, void*, std::size_t, std::int64_t); std::int64_t APS5_VABI sceKernelPwrite(int, const void*, std::size_t, std::int64_t); } @@ -100,7 +103,32 @@ int main() { const int readAll = sceKernelGetdents(directory, large.data(), static_cast(large.size())); Require(readAll > 0 && Collect(large.data(), readAll, all) == 6); Require(sceKernelGetdents(directory, large.data(), static_cast(large.size())) == 0); + + Require(sceKernelLseek(directory, 0, 0) == 0); + Require(getdents_nid_postfix(directory, tooSmall.data(), static_cast(tooSmall.size())) == -1 && *__error_nid_postfix() == 22); + *__error_nid_postfix() = 0; + Require(getdents_nid_postfix(directory, nullptr, 64) == -1 && *__error_nid_postfix() == 14); + *__error_nid_postfix() = 0; + Require(getdirentries_nid_postfix(directory, small.data(), 0, nullptr) == -1 && *__error_nid_postfix() == 22); + std::int64_t posixBase = -1; + *__error_nid_postfix() = 1234; + const int firstPart = getdirentries_nid_postfix(directory, small.data(), static_cast(small.size()), &posixBase); + Require(firstPart > 0 && posixBase == 0 && *__error_nid_postfix() == 1234); + const int secondPart = getdirentries_nid_postfix(directory, small.data(), static_cast(small.size()), &posixBase); + Require(secondPart > 0 && posixBase > 0 && *__error_nid_postfix() == 1234); + Require(sceKernelLseek(directory, 0, 0) == 0); + std::map posix; + const int posixRead = getdirentries_nid_postfix(directory, large.data(), static_cast(large.size()), &posixBase); + Require(posixRead == readAll && posixBase == 0 && Collect(large.data(), posixRead, posix) == 6); + Require(getdents_nid_postfix(directory, large.data(), static_cast(large.size())) == 0); + Require(sceKernelLseek(directory, 0, 0) == 0); + std::map dents; + const int dentsRead = getdents_nid_postfix(directory, large.data(), static_cast(large.size())); + Require(dentsRead == readAll && Collect(large.data(), dentsRead, dents) == 6); Require(sceKernelClose(directory) == 0); + Require(getdents_nid_postfix(directory, large.data(), static_cast(large.size())) == -1 && *__error_nid_postfix() == 9); + *__error_nid_postfix() = 0; + Require(getdirentries_nid_postfix(directory, large.data(), static_cast(large.size()), nullptr) == -1 && *__error_nid_postfix() == 9); const int file = sceKernelOpen((root / "data.bin").string().c_str(), SCE_KERNEL_O_RDWR, 0); Require(file >= 0); diff --git a/core/libs/tests/GuestDownloadDataMount.cpp b/core/libs/tests/GuestDownloadDataMount.cpp new file mode 100644 index 000000000..e588f786d --- /dev/null +++ b/core/libs/tests/GuestDownloadDataMount.cpp @@ -0,0 +1,26 @@ +#include "prx/libkernel/AppMetadata/include/AppMetadata.hpp" +#include +#include +#include +#include +#include + +static void Require(bool value) { if (!value) std::abort(); } + +int main(int argc, char** argv) { + Require(argc == 2); + const std::string_view mode = argv[1]; + Require(std::filesystem::is_directory("download0") == (mode == "declared")); + if (mode == "invalid") { + bool rejected = false; + try { + GetAppTitleId_nid_postfix(); + } catch (const std::runtime_error&) { + rejected = true; + } + Require(rejected); + return 0; + } + Require(std::strcmp(GetAppTitleId_nid_postfix().value, "PPSA00000") == 0); + std::filesystem::remove_all("download0"); +} diff --git a/core/libs/tests/GuestExitOrder.cpp b/core/libs/tests/GuestExitOrder.cpp new file mode 100644 index 000000000..067bea90f --- /dev/null +++ b/core/libs/tests/GuestExitOrder.cpp @@ -0,0 +1,53 @@ +#include "prx/libc/include/Shutdown.hpp" +#include "prx/libc/include/general/VabiMacros.hpp" +#include +#include +#include +#include + +extern "C" int APS5_VABI __cxa_atexit_nid_postfix(void (APS5_VABI *func)(void*), void* arg, void* dsoHandle); + +namespace { + +int events[2]; + +void Record(char value) { + static_cast(!write(events[1], &value, 1)); +} + +void APS5_VABI GuestHandler(void*) { Record('G'); } + +void HostHandler() { Record('H'); } + +bool GuestHandlersRunFirst() { + if (pipe(events) != 0) return false; + const pid_t child = fork(); + if (child < 0) return false; + if (child == 0) { + close(events[0]); + __cxa_atexit_nid_postfix(GuestHandler, nullptr, nullptr); + std::atexit(HostHandler); + LibcExit_nid_no_patch(0); + } + close(events[1]); + char order[3] = {}; + ssize_t length = 0; + while (length < 2) { + const ssize_t count = read(events[0], order + length, 2 - length); + if (count <= 0) break; + length += count; + } + close(events[0]); + int status = 0; + return waitpid(child, &status, 0) == child && WIFEXITED(status) && WEXITSTATUS(status) == 0 && length == 2 && order[0] == 'G' && order[1] == 'H'; +} + +} + +int main() { + if (!GuestHandlersRunFirst()) { + std::fputs("guest exit handlers must run before the host exit handlers\n", stderr); + return 1; + } + return 0; +} diff --git a/core/libs/tests/GuestFileTimestamps.cpp b/core/libs/tests/GuestFileTimestamps.cpp new file mode 100644 index 000000000..48a175895 --- /dev/null +++ b/core/libs/tests/GuestFileTimestamps.cpp @@ -0,0 +1,149 @@ +#include "SceTypes.hpp" +#include +#include +#include +#include +#include +#include +#ifdef _WIN32 +#include +#include +#endif + +extern "C" { +int APS5_VABI utimes_nid_postfix(const char*, const KernelTimeval*); +int APS5_VABI futimes_nid_postfix(int, const KernelTimeval*); +int APS5_VABI sceKernelUtimes_nid_postfix(const char*, const KernelTimeval*); +int APS5_VABI stat_nid_postfix(const char*, FileStat*); +int APS5_VABI sceKernelFstat(int, FileStat*); +std::int64_t APS5_VABI fstat_nid_disambig1_nid_postfix(int, FileStat*); +int APS5_VABI open_nid_postfix(const char*, int, int); +int APS5_VABI close_nid_postfix(int); +int APS5_VABI pipe_nid_postfix(int*); +int* APS5_VABI __error_nid_postfix(); +} + +static void Require(bool value, const char* message) { + if (!value) { + std::fprintf(stderr, "%s (guest errno %d)\n", message, *__error_nid_postfix()); + std::exit(1); + } +} + +static bool Same(const KernelTimespec& actual, const KernelTimeval& expected) { + return actual.tv_sec == expected.tv_sec && actual.tv_nsec == expected.tv_usec * 1000; +} + +static void CheckTimes(const std::string& path, int fd, const KernelTimeval* expected) { + FileStat byPath{}, byFd{}, byKernel{}; + Require(stat_nid_postfix(path.c_str(), &byPath) == 0, "stat"); + Require(fstat_nid_disambig1_nid_postfix(fd, &byFd) == 0, "fstat"); + Require(sceKernelFstat(fd, &byKernel) == 0, "sceKernelFstat"); + for (const auto* status : {&byPath, &byFd, &byKernel}) { + Require(Same(status->st_atim, expected[0]), "access time preserves microseconds"); + Require(Same(status->st_mtim, expected[1]), "modification time preserves microseconds"); + } +#ifdef _WIN32 + FILETIME access{}, modified{}; + Require(GetFileTime(reinterpret_cast(_get_osfhandle(fd)), nullptr, &access, &modified), "native GetFileTime"); + const auto ticks = [](FILETIME time) { + return (static_cast(time.dwHighDateTime) << 32) | time.dwLowDateTime; + }; + Require(ticks(access) == static_cast(expected[0].tv_sec + 11644473600LL) * 10000000 + expected[0].tv_usec * 10, + "native access time preserves microseconds"); + Require(ticks(modified) == static_cast(expected[1].tv_sec + 11644473600LL) * 10000000 + expected[1].tv_usec * 10, + "native modification time preserves microseconds"); +#endif +} + +int main() { + const auto root = std::filesystem::path("anyps5-file-times-" + std::to_string(std::chrono::steady_clock::now().time_since_epoch().count())); + Require(std::filesystem::create_directory(root), "create fixture directory"); + const auto path = (root / "file.txt").string(); + { std::ofstream file(path); file << "timestamp fixture"; } + const int fd = open_nid_postfix(path.c_str(), 0, 0); + Require(fd >= 0, "open read-only descriptor"); + + const KernelTimeval first[2]{{1000000000, 123456}, {1000000001, 500000}}; + Require(utimes_nid_postfix(path.c_str(), first) == 0, "utimes fractional timestamp"); + CheckTimes(path, fd, first); + const KernelTimeval second[2]{{1000000002, 1}, {1000000003, 999999}}; + Require(futimes_nid_postfix(fd, second) == 0, "futimes on read-only descriptor"); + CheckTimes(path, fd, second); + Require(sceKernelUtimes_nid_postfix(path.c_str(), first) == 0, "sceKernelUtimes fractional timestamp"); + CheckTimes(path, fd, first); + + for (int index = 0; index < 2; ++index) { + for (const auto micros : {-1LL, 1000000LL}) { + KernelTimeval invalid[2]{first[0], first[1]}; + invalid[index].tv_usec = micros; + Require(utimes_nid_postfix(path.c_str(), invalid) == -1 && *__error_nid_postfix() == 22, "utimes rejects invalid microseconds"); + Require(futimes_nid_postfix(fd, invalid) == -1 && *__error_nid_postfix() == 22, "futimes rejects invalid microseconds"); + Require(sceKernelUtimes_nid_postfix(path.c_str(), invalid) == static_cast(0x80020016u), "sceKernelUtimes rejects invalid microseconds"); + CheckTimes(path, fd, first); + } + } + +#ifdef _WIN32 + const auto handle = CreateFileW(std::filesystem::path(path).c_str(), FILE_WRITE_ATTRIBUTES | FILE_READ_ATTRIBUTES, + FILE_SHARE_READ | FILE_SHARE_WRITE | FILE_SHARE_DELETE, nullptr, OPEN_EXISTING, 0, nullptr); + Require(handle != INVALID_HANDLE_VALUE, "open native timestamp handle"); + constexpr std::uint64_t ticks = 116444736000000000ULL + 1000000000ULL * 10000000 + 1234567; + const FILETIME precise{static_cast(ticks), static_cast(ticks >> 32)}; + Require(SetFileTime(handle, &precise, &precise, &precise), "set native 100-nanosecond timestamps"); + FILE_BASIC_INFO native{}; + Require(GetFileInformationByHandleEx(handle, FileBasicInfo, &native, sizeof(native)), "read native basic info"); + FileStat status{}; + Require(stat_nid_postfix(path.c_str(), &status) == 0, "stat native timestamps"); + Require(status.st_birthtim.tv_sec == 1000000000 && status.st_birthtim.tv_nsec == 123456700, "creation time retains native precision"); + Require(status.st_atim.tv_sec == 1000000000 && status.st_atim.tv_nsec == 123456700, "access time retains native precision"); + Require(status.st_mtim.tv_sec == 1000000000 && status.st_mtim.tv_nsec == 123456700, "modification time retains native precision"); + Require(status.st_ctim.tv_sec == native.ChangeTime.QuadPart / 10000000 - 11644473600LL && + status.st_ctim.tv_nsec == native.ChangeTime.QuadPart % 10000000 * 100, "change time uses native metadata change time"); + Require(CloseHandle(handle), "close native timestamp handle"); + + const KernelTimeval preEpoch[2]{{-1, 500000}, {0, 1}}; + Require(utimes_nid_postfix(path.c_str(), preEpoch) == 0, "timestamps around Unix epoch"); + CheckTimes(path, fd, preEpoch); + Require(utimes_nid_postfix(path.c_str(), first) == 0, "restore valid times"); + for (const auto seconds : {std::numeric_limits::min(), std::numeric_limits::max(), -11644473601LL, -11644473600LL}) { + const KernelTimeval invalid[2]{first[0], {seconds, 0}}; + Require(utimes_nid_postfix(path.c_str(), invalid) == -1 && *__error_nid_postfix() == 22, "reject unrepresentable Windows timestamp"); + Require(futimes_nid_postfix(fd, invalid) == -1 && *__error_nid_postfix() == 22, "futimes rejects unrepresentable Windows timestamp"); + CheckTimes(path, fd, first); + } +#endif + + const auto before = std::chrono::system_clock::now(); + Require(futimes_nid_postfix(fd, nullptr) == 0, "futimes current time"); + const auto after = std::chrono::system_clock::now(); + FileStat now{}; + Require(stat_nid_postfix(path.c_str(), &now) == 0, "stat current time"); + const auto recorded = std::chrono::seconds(now.st_mtim.tv_sec) + std::chrono::nanoseconds(now.st_mtim.tv_nsec); + Require(recorded >= before.time_since_epoch() - std::chrono::seconds(1) && + recorded <= after.time_since_epoch() + std::chrono::seconds(1), "null times use current clock"); + Require(utimes_nid_postfix(path.c_str(), nullptr) == 0, "utimes current time"); + Require(close_nid_postfix(fd) == 0, "close fixture descriptor"); + + const auto directory = (root / "directory").string(); + Require(std::filesystem::create_directory(directory), "create timestamp directory"); + const int directoryFd = open_nid_postfix(directory.c_str(), 0, 0); + Require(directoryFd >= 0, "open directory descriptor"); + Require(utimes_nid_postfix(directory.c_str(), first) == 0, "directory utimes"); + FileStat directoryStatus{}; + Require(stat_nid_postfix(directory.c_str(), &directoryStatus) == 0 && Same(directoryStatus.st_mtim, first[1]), "directory stat precision"); + Require(futimes_nid_postfix(directoryFd, second) == 0, "directory futimes"); + Require(sceKernelFstat(directoryFd, &directoryStatus) == 0 && Same(directoryStatus.st_mtim, second[1]), "directory fstat precision"); + Require(close_nid_postfix(directoryFd) == 0, "close directory descriptor"); + + Require(utimes_nid_postfix((root / "missing").string().c_str(), first) == -1 && *__error_nid_postfix() == 2, "missing path reports ENOENT"); + int pipe[2]; + Require(pipe_nid_postfix(pipe) == 0, "create pipe"); + FileStat pipeStatus{}; + Require(sceKernelFstat(pipe[0], &pipeStatus) == 0, "pipe stat still works"); +#ifdef _WIN32 + Require(futimes_nid_postfix(pipe[0], first) == -1 && *__error_nid_postfix() == 22, "pipe timestamp update fails"); +#endif + Require(close_nid_postfix(pipe[0]) == 0 && close_nid_postfix(pipe[1]) == 0, "close pipe"); + std::filesystem::remove_all(root); +} diff --git a/core/libs/tests/GuestFileVector.cpp b/core/libs/tests/GuestFileVector.cpp index 1851fb20b..b73f03fb0 100644 --- a/core/libs/tests/GuestFileVector.cpp +++ b/core/libs/tests/GuestFileVector.cpp @@ -30,11 +30,15 @@ extern "C" { int APS5_VABI sceKernelOpen(const char*, int, std::uint16_t); int APS5_VABI sceKernelClose(int); std::int64_t APS5_VABI sceKernelRead(int, void*, std::size_t); +std::int64_t APS5_VABI sceKernelPread(int, void*, std::size_t, std::int64_t); int APS5_VABI sceKernelLseek(int, std::int64_t, int); std::int64_t APS5_VABI sceKernelReadv(int, const GuestIovec*, int); std::int64_t APS5_VABI sceKernelWritev(int, const GuestIovec*, int); std::int64_t APS5_VABI sceKernelPreadv(int, const GuestIovec*, int, std::int64_t); std::int64_t APS5_VABI sceKernelPwritev(int, const GuestIovec*, int, std::int64_t); +std::int64_t APS5_VABI sceKernelPread(int, void*, std::size_t, std::int64_t); +std::int64_t APS5_VABI sceKernelPwrite(int, const void*, std::size_t, std::int64_t); +int APS5_VABI sceKernelFtruncate(int, std::int64_t); } static void Check(bool value, int line) { @@ -119,11 +123,36 @@ int main() { Require(sceKernelPwritev(file, writes, 2, -1) == ErrorEinval); Require(Contents(path) == "01ABCDE7ABCDE"); + char byte = 0; + Require(sceKernelPread(file, &byte, 1, 2) == 1 && byte == 'A'); + Require(sceKernelPread(file, nullptr, 0, 0) == 0); + Require(sceKernelPwrite(file, nullptr, 0, 0) == 0); + Require(sceKernelPread(file, nullptr, 1, 0) == ErrorEfault); + Require(sceKernelPwrite(file, nullptr, 1, 0) == ErrorEfault); + Require(sceKernelPread(file, &byte, 1, -1) == ErrorEinval); + Require(sceKernelPwrite(file, &byte, 1, -1) == ErrorEinval); + Require(sceKernelFtruncate(file, -1) == ErrorEinval); + const int readOnly = sceKernelOpen(path.string().c_str(), SCE_KERNEL_O_RDONLY, 0); + Require(readOnly >= 0); + Require(sceKernelPwrite(readOnly, "X", 1, 0) == ErrorEbadf); + Require(sceKernelFtruncate(readOnly, 0) == ErrorEinval); + Require(sceKernelClose(readOnly) == 0); + const int writeOnly = sceKernelOpen(path.string().c_str(), SCE_KERNEL_O_WRONLY, 0); + Require(writeOnly >= 0); + Require(sceKernelPread(writeOnly, &byte, 1, 0) == ErrorEbadf); + Require(sceKernelClose(writeOnly) == 0); + Require(Contents(path) == "01ABCDE7ABCDE"); + Require(sceKernelFtruncate(file, 5) == 0); + Require(Contents(path) == "01ABC"); + Require(sceKernelClose(file) == 0); Require(sceKernelReadv(file, reads, 2) == ErrorEbadf); Require(sceKernelWritev(file, writes, 2) == ErrorEbadf); Require(sceKernelPreadv(file, reads, 2, 0) == ErrorEbadf); Require(sceKernelPwritev(file, writes, 2, 0) == ErrorEbadf); + Require(sceKernelPread(file, &byte, 1, 0) == ErrorEbadf); + Require(sceKernelPwrite(file, &byte, 1, 0) == ErrorEbadf); + Require(sceKernelFtruncate(file, 0) == ErrorEbadf); int ends[2] = {}; Require(MakePipe(ends) == 0); @@ -136,6 +165,16 @@ int main() { Require(sceKernelPwritev(ends[1], message, 1, 0) == ErrorEspipe); Require(ClosePipe(ends[0]) == 0 && ClosePipe(ends[1]) == 0); + const int random = sceKernelOpen("/dev/urandom", 0, 0); + Require(random >= 0); + unsigned char noise[48]{}; + GuestIovec pieces[2] = {{noise, 16}, {noise + 16, 16}}; + Require(sceKernelReadv(random, pieces, 2) == 32); + Require(sceKernelPreadv(random, pieces, 1, 1000) == 16); + Require(sceKernelPread(random, noise + 32, 16, 1000) == 16); + Require(std::memcmp(noise, noise + 16, 16) != 0 && std::memcmp(noise + 16, noise + 32, 16) != 0); + Require(sceKernelClose(random) == 0); + std::filesystem::remove_all(root); return 0; } diff --git a/core/libs/tests/GuestFilesystem.cpp b/core/libs/tests/GuestFilesystem.cpp index bd6a113c6..bd0f598b7 100644 --- a/core/libs/tests/GuestFilesystem.cpp +++ b/core/libs/tests/GuestFilesystem.cpp @@ -20,6 +20,7 @@ int APS5_VABI fchmod_nid_postfix(int, int); int APS5_VABI futimes_nid_postfix(int, const KernelTimeval*); int APS5_VABI socket_nid_postfix(int, int, int); int APS5_VABI sceKernelFsync(int); +int APS5_VABI fdatasync_nid_postfix(int); int APS5_VABI sceKernelWriteThrottlingStatus(std::uint64_t*); int APS5_VABI sceKernelFtruncate(int, long long); int APS5_VABI sceKernelTruncate_nid_postfix(const char*, long long); @@ -27,13 +28,16 @@ int APS5_VABI sceKernelUtimes_nid_postfix(const char*, const void*); int APS5_VABI open_nid_postfix(const char*, int, int); int APS5_VABI _open_nid_postfix(const char*, int, ...); int APS5_VABI close_nid_postfix(int); +int APS5_VABI flock_nid_postfix(int, int); int APS5_VABI stat_nid_postfix(const char*, FileStat*); +int APS5_VABI lstat_nid_postfix(const char*, FileStat*); int APS5_VABI unlink_nid_postfix(const char*); int APS5_VABI rmdir_nid_postfix(const char*); int APS5_VABI mkdir_nid_postfix(const char*, unsigned short); int APS5_VABI sceKernelOpen(const char*, int, unsigned short); int APS5_VABI sceKernelClose(int); int APS5_VABI sceKernelStat(const char*, FileStat*); +int APS5_VABI sceKernelFstat(int, FileStat*); int APS5_VABI sceKernelUnlink(const char*); int APS5_VABI sceKernelRmdir(const char*); int* APS5_VABI __error_nid_postfix(); @@ -101,6 +105,10 @@ int main() { Require(remove_nid_postfix((file / "invalid").string().c_str()) == -1); Require(remove_nid_postfix("") == -1 && *__error_nid_postfix() == 2); Require(remove_nid_postfix(nullptr) == -1 && *__error_nid_postfix() == 14); + const auto readOnly = root / "read-only.txt"; + { std::ofstream stream(readOnly); stream << "removable"; } + Require(sceKernelChmod_nid_postfix(readOnly.string().c_str(), 0400) == 0); + Require(remove_nid_postfix(readOnly.string().c_str()) == 0 && !std::filesystem::exists(readOnly)); const auto renamed = root / "renamed.txt"; { std::ofstream stream(renamed); stream << "old contents"; } Require(rename_nid_postfix(file.string().c_str(), renamed.string().c_str()) == 0); @@ -131,6 +139,7 @@ int main() { const int descriptor = ::fileno(native); #endif Require(descriptor >= 0 && sceKernelFsync(descriptor) == 0); + Require(fdatasync_nid_postfix(descriptor) == 0); const auto ownerWrite = [&] { return (std::filesystem::status(sized).permissions() & std::filesystem::perms::owner_write) != std::filesystem::perms::none; }; @@ -153,15 +162,18 @@ int main() { Require(sceKernelFchmod(descriptor, 0600) == static_cast(0x80020009u)); Require(fchmod_nid_postfix(descriptor, 0600) == -1 && *__error_nid_postfix() == 9); Require(futimes_nid_postfix(descriptor, nullptr) == -1 && *__error_nid_postfix() == 9); + Require(fdatasync_nid_postfix(descriptor) == -1 && *__error_nid_postfix() == 9); #endif const int socket = socket_nid_postfix(2, 2, 0); Require(socket >= 0); Require(sceKernelFchmod(socket, 0600) == static_cast(0x80020016u)); Require(fchmod_nid_postfix(socket, 0600) == -1 && *__error_nid_postfix() == 22); Require(futimes_nid_postfix(socket, nullptr) == -1 && *__error_nid_postfix() == 22); + Require(fdatasync_nid_postfix(socket) == -1 && *__error_nid_postfix() == 22); Require(close_nid_postfix(socket) == 0); Require(fchmod_nid_postfix(socket, 0600) == -1 && *__error_nid_postfix() == 9); Require(futimes_nid_postfix(socket, nullptr) == -1 && *__error_nid_postfix() == 9); + Require(fdatasync_nid_postfix(socket) == -1 && *__error_nid_postfix() == 9); Require(remove_nid_postfix(sized.string().c_str()) == 0); const auto present = root / "present.txt"; const auto presentName = present.string(); @@ -173,17 +185,111 @@ int main() { Require(stat_nid_postfix(missingName.c_str(), &status) == -1 && *__error_nid_postfix() == 2); Require(sceKernelStat(missingName.c_str(), &status) == static_cast(0x80020002u)); Require(stat_nid_postfix("", &status) == -1 && *__error_nid_postfix() == 2); + Require(stat_nid_postfix((presentName + "/").c_str(), &status) == -1 && *__error_nid_postfix() == 20); + Require(sceKernelStat((presentName + "/").c_str(), &status) == static_cast(0x80020014u)); + FileStat dirStatus{}; + Require(stat_nid_postfix((rootName + "/").c_str(), &dirStatus) == 0 && (dirStatus.st_mode & 0170000) == 0040000); Require(stat_nid_postfix(nullptr, &status) == -1 && *__error_nid_postfix() == 14); Require(stat_nid_postfix(presentName.c_str(), nullptr) == -1 && *__error_nid_postfix() == 14); + FileStat linkStatus{}; + Require(lstat_nid_postfix(presentName.c_str(), &linkStatus) == 0 && linkStatus.st_size == 5); + Require((linkStatus.st_mode & 0170000) == 0100000 && linkStatus.st_ino == status.st_ino); + Require(lstat_nid_postfix(rootName.c_str(), &linkStatus) == 0 && (linkStatus.st_mode & 0170000) == 0040000); + Require(lstat_nid_postfix(missingName.c_str(), &linkStatus) == -1 && *__error_nid_postfix() == 2); + Require(lstat_nid_postfix("", &linkStatus) == -1 && *__error_nid_postfix() == 2); + Require(lstat_nid_postfix(nullptr, &linkStatus) == -1 && *__error_nid_postfix() == 14); + Require(lstat_nid_postfix(presentName.c_str(), nullptr) == -1 && *__error_nid_postfix() == 14); +#ifndef _WIN32 + const auto link = root / "link"; + std::filesystem::create_symlink("present.txt", link); + Require(lstat_nid_postfix(link.string().c_str(), &linkStatus) == 0); + Require((linkStatus.st_mode & 0170000) == 0120000 && linkStatus.st_size == 11); + Require(stat_nid_postfix(link.string().c_str(), &status) == 0 && (status.st_mode & 0170000) == 0100000); + const auto dangling = root / "dangling"; + std::filesystem::create_symlink("missing.txt", dangling); + Require(lstat_nid_postfix(dangling.string().c_str(), &linkStatus) == 0 && (linkStatus.st_mode & 0170000) == 0120000); + Require(stat_nid_postfix(dangling.string().c_str(), &status) == -1 && *__error_nid_postfix() == 2); + Require(lstat_nid_postfix((root / "present.txt" / "child").string().c_str(), &linkStatus) == -1 && *__error_nid_postfix() == 20); + Require(unlink_nid_postfix(link.string().c_str()) == 0 && unlink_nid_postfix(dangling.string().c_str()) == 0); +#endif + FileStat identity{}; + FileStat byDescriptor{}; + const int identityDescriptor = open_nid_postfix(presentName.c_str(), 0, 0); + Require(identityDescriptor >= 0 && stat_nid_postfix(presentName.c_str(), &identity) == 0); + Require(sceKernelFstat(identityDescriptor, &byDescriptor) == 0 && close_nid_postfix(identityDescriptor) == 0); + Require(identity.st_ino != 0 && byDescriptor.st_dev == identity.st_dev && byDescriptor.st_ino == identity.st_ino); + const auto sibling = root / "sibling.txt"; + { std::ofstream stream(sibling); stream << "other"; } + FileStat siblingStatus{}; + Require(stat_nid_postfix(sibling.string().c_str(), &siblingStatus) == 0); + Require(siblingStatus.st_dev == identity.st_dev && siblingStatus.st_ino != identity.st_ino); + Require(unlink_nid_postfix(sibling.string().c_str()) == 0); + const auto hardLink = root / "hardlink.txt"; + std::error_code linkError; + std::filesystem::create_hard_link(present, hardLink, linkError); + if (!linkError) { + FileStat linked{}; + Require(stat_nid_postfix(hardLink.string().c_str(), &linked) == 0); + Require(linked.st_dev == identity.st_dev && linked.st_ino == identity.st_ino && linked.st_nlink == 2); + Require(unlink_nid_postfix(hardLink.string().c_str()) == 0); + } const int opened = open_nid_postfix(presentName.c_str(), 0, 0); Require(opened >= 0 && close_nid_postfix(opened) == 0); const int reopened = _open_nid_postfix(presentName.c_str(), 0); Require(reopened >= 0 && close_nid_postfix(reopened) == 0); + const int locked = open_nid_postfix(presentName.c_str(), 0, 0); + const int other = open_nid_postfix(presentName.c_str(), 0, 0); + Require(locked >= 0 && other >= 0); + Require(flock_nid_postfix(locked, 8) == 0); + Require(flock_nid_postfix(locked, 2 | 4) == 0); + Require(flock_nid_postfix(locked, 2 | 4) == 0); + Require(flock_nid_postfix(locked, 1 | 4) == 0); + Require(flock_nid_postfix(other, 1 | 4) == 0); + Require(flock_nid_postfix(other, 8) == 0); + Require(flock_nid_postfix(locked, 1 | 4) == 0); + Require(flock_nid_postfix(locked, 2 | 4) == 0); + Require(flock_nid_postfix(locked, 8) == 0); + Require(flock_nid_postfix(locked, 8) == 0); + Require(flock_nid_postfix(other, 2 | 4) == 0); + Require(flock_nid_postfix(other, 8) == 0); + Require(flock_nid_postfix(locked, 2 | 4) == 0); + Require(close_nid_postfix(locked) == 0); + const int recycled = open_nid_postfix(presentName.c_str(), 0, 0); + Require(recycled >= 0 && flock_nid_postfix(recycled, 8) == 0); + Require(flock_nid_postfix(recycled, 2 | 4) == 0 && sceKernelClose(recycled) == 0); + const int kernelRecycled = sceKernelOpen(presentName.c_str(), 0, 0); + Require(kernelRecycled >= 0 && flock_nid_postfix(kernelRecycled, 8) == 0); + Require(flock_nid_postfix(other, 0) == -1 && *__error_nid_postfix() == 9); + Require(flock_nid_postfix(other, 4 | 0x10) == -1 && *__error_nid_postfix() == 9); + Require(flock_nid_postfix(kernelRecycled, 2 | 4 | 0x10) == 0); + Require(flock_nid_postfix(kernelRecycled, 8 | 2) == 0); + Require(flock_nid_postfix(other, 2 | 4) == 0 && flock_nid_postfix(other, 8) == 0); + Require(sceKernelClose(kernelRecycled) == 0 && close_nid_postfix(other) == 0); + const int holder = open_nid_postfix(presentName.c_str(), 0, 0); + const int contender = open_nid_postfix(presentName.c_str(), 0, 0); + Require(holder >= 0 && contender >= 0 && flock_nid_postfix(holder, 2 | 4) == 0); + Require(flock_nid_postfix(contender, 2 | 4) == -1 && *__error_nid_postfix() == 35); + Require(flock_nid_postfix(contender, 1 | 4) == -1 && *__error_nid_postfix() == 35); + Require(flock_nid_postfix(holder, 1 | 4) == 0 && flock_nid_postfix(contender, 1 | 4) == 0); + Require(flock_nid_postfix(contender, 2 | 4) == -1 && *__error_nid_postfix() == 35); + Require(flock_nid_postfix(holder, 8) == 0 && flock_nid_postfix(contender, 2 | 4) == 0); + Require(flock_nid_postfix(contender, 8) == 0); + Require(flock_nid_postfix(holder, 1 | 2 | 4 | 0x10) == 0); + Require(flock_nid_postfix(contender, 1 | 4) == -1 && *__error_nid_postfix() == 35); + Require(flock_nid_postfix(holder, 8) == 0); + Require(close_nid_postfix(contender) == 0 && close_nid_postfix(holder) == 0); Require(open_nid_postfix(missingName.c_str(), 0, 0) == -1 && *__error_nid_postfix() == 2); Require(_open_nid_postfix(missingName.c_str(), 0) == -1 && *__error_nid_postfix() == 2); Require(sceKernelOpen(missingName.c_str(), 0, 0) == static_cast(0x80020002u)); Require(open_nid_postfix(presentName.c_str(), 0x0a02, 0644) == -1 && *__error_nid_postfix() == 17); Require(_open_nid_postfix(presentName.c_str(), 0x0a02, 0644) == -1 && *__error_nid_postfix() == 17); + Require(open_nid_postfix(rootName.c_str(), 0x0a02, 0644) == -1 && *__error_nid_postfix() == 17); + Require(open_nid_postfix(rootName.c_str(), 0x0a00, 0644) == -1 && *__error_nid_postfix() == 17); + Require(open_nid_postfix(rootName.c_str(), 0x0200, 0644) == -1 && *__error_nid_postfix() == 21); + Require(open_nid_postfix(rootName.c_str(), 0x0002, 0) == -1 && *__error_nid_postfix() == 21); + Require(open_nid_postfix(rootName.c_str(), 0x0001, 0) == -1 && *__error_nid_postfix() == 21); + const int directoryDescriptor = open_nid_postfix(rootName.c_str(), 0, 0); + Require(directoryDescriptor >= 0 && close_nid_postfix(directoryDescriptor) == 0); Require(open_nid_postfix("", 0, 0) == -1 && *__error_nid_postfix() == 2); Require(_open_nid_postfix("", 0) == -1 && *__error_nid_postfix() == 2); Require(open_nid_postfix(nullptr, 0, 0) == -1 && *__error_nid_postfix() == 14); @@ -198,10 +304,22 @@ int main() { Require(sceKernelUnlink(missingName.c_str()) == static_cast(0x80020002u)); Require(unlink_nid_postfix("") == -1 && *__error_nid_postfix() == 2); Require(unlink_nid_postfix(nullptr) == -1 && *__error_nid_postfix() == 14); + Require(unlink_nid_postfix(rootName.c_str()) == -1 && *__error_nid_postfix() == 1 && std::filesystem::is_directory(root)); + Require(sceKernelUnlink(rootName.c_str()) == static_cast(0x80020001u)); const int closable = sceKernelOpen(presentName.c_str(), 0, 0); Require(closable >= 0 && sceKernelClose(closable) == 0); Require(sceKernelClose(closable) == static_cast(0x80020009u)); Require(sceKernelClose(-1) == static_cast(0x80020009u)); + for (const char* device : {"/dev/urandom", "/dev/random"}) { + const int random = open_nid_postfix(device, 0, 0); + Require(random >= 0); + unsigned char first[64]{}; + unsigned char second[64]{}; + Require(read_nid_postfix(random, first, sizeof(first)) == sizeof(first)); + Require(read_nid_postfix(random, second, 3) == 3); + Require(std::memcmp(first, second, sizeof(first)) != 0 && second[3] == 0); + Require(close_nid_postfix(random) == 0); + } Require(unlink_nid_postfix(presentName.c_str()) == 0 && !std::filesystem::exists(present)); const auto empty = root / "empty"; Require(std::filesystem::create_directory(empty)); diff --git a/core/libs/tests/GuestFont.cpp b/core/libs/tests/GuestFont.cpp index 97e814da7..eb30bd2a2 100644 --- a/core/libs/tests/GuestFont.cpp +++ b/core/libs/tests/GuestFont.cpp @@ -3,8 +3,10 @@ #include #include #include +#include #include #include +#include #include #include @@ -51,6 +53,14 @@ static void Check(bool value, int line) { } #define Require(value) Check((value), __LINE__) +static void SetFontDirectory(const std::filesystem::path& directory) { +#ifdef _WIN32 + Require(_putenv_s("ANYPS5_SYSTEM_FONTS", directory.string().c_str()) == 0); +#else + Require(::setenv("ANYPS5_SYSTEM_FONTS", directory.string().c_str(), 1) == 0); +#endif +} + static int allocations = 0; static void* APS5_VABI Allocate(void*, std::uint32_t size) { ++allocations; @@ -118,6 +128,10 @@ static bool KerningIs(const FontKerning& kerning, float offsetX) { int main() { constexpr std::uint32_t SystemFontSet = 0x18070043u; + const std::filesystem::path fontRoot = std::filesystem::temp_directory_path() / ("anyps5_guest_font-" + std::to_string(std::random_device{}())); + std::filesystem::remove_all(fontRoot); + std::filesystem::create_directories(fontRoot); + SetFontDirectory(fontRoot); const FontMemoryInterface iface{Allocate, Release, nullptr, nullptr, nullptr, nullptr}; FontMemory memory{}; Require(sceFontMemoryInit(&memory, nullptr, 0, &iface, nullptr, nullptr, nullptr) == SCE_FONT_OK); @@ -288,4 +302,5 @@ int main() { Require(allocations == 0); Require(sceFontMemoryTerm(&memory) == SCE_FONT_OK); Require(sceFontMemoryTerm(&memory) == SCE_FONT_ERROR_INVALID_MEMORY); + std::filesystem::remove_all(fontRoot); } diff --git a/core/libs/tests/GuestFreebsdClocks.cpp b/core/libs/tests/GuestFreebsdClocks.cpp index 75a69daa6..be5f30434 100644 --- a/core/libs/tests/GuestFreebsdClocks.cpp +++ b/core/libs/tests/GuestFreebsdClocks.cpp @@ -1,4 +1,5 @@ #include "SceTypes.hpp" +#include #include #include @@ -10,6 +11,8 @@ int APS5_VABI sceKernelClockGetres(KernelClockid clockId, KernelTimespec* tp); int APS5_VABI sceKernelUsleep_nid_postfix(KernelUseconds microseconds); int APS5_VABI gettimeofday_nid_postfix(KernelTimeval* tv, KernelTimezone* tz); std::int64_t APS5_VABI _Xtime_get_ticks_nid_postfix(); +int APS5_VABI sceKernelConvertLocaltimeToUtc(std::int64_t, std::int64_t, std::int64_t*, KernelTimesec*, std::int32_t*); +int APS5_VABI sceKernelConvertUtcToLocaltime(std::int64_t, std::int64_t*, KernelTimesec*, std::uint64_t*); } static constexpr int SCE_OK = 0; @@ -137,8 +140,41 @@ static void XtimeTicksAreGettimeofdayMicroseconds() { } } +static void CalendarConversionsInitializeTheirState() { + constexpr std::uint64_t sentinel = 0xa5a5a5a5a5a5a5a5ull; + struct GuardedState { + std::uint64_t before; + KernelTimesec state; + std::uint64_t after; + }; + for (const auto seconds : std::array{-86400, 0, 1767225600, 2147483648}) { + GuardedState local{sentinel, {-1, 0xa5a5a5a5u, 0xa5a5a5a5u}, sentinel}; + std::int64_t utc = -1; + std::int32_t dst = -1; + Require(sceKernelConvertLocaltimeToUtc(seconds, 0, &utc, &local.state, &dst) == SCE_OK); + Require(utc == seconds && dst == 0); + Require(local.state.t == seconds && local.state.west_sec == 0 && local.state.dst_sec == 0); + Require(local.before == sentinel && local.after == sentinel); + GuardedState universal{sentinel, {-1, 0xa5a5a5a5u, 0xa5a5a5a5u}, sentinel}; + std::int64_t converted = -1; + std::uint64_t universalDst = sentinel; + Require(sceKernelConvertUtcToLocaltime(utc, &converted, &universal.state, &universalDst) == SCE_OK); + Require(converted == seconds && universalDst == 0); + Require(universal.state.t == seconds && universal.state.west_sec == 0 && universal.state.dst_sec == 0); + Require(universal.before == sentinel && universal.after == sentinel); + } + Require(sceKernelConvertLocaltimeToUtc(0, 0, nullptr, nullptr, nullptr) == SCE_OK); + Require(sceKernelConvertUtcToLocaltime(0, nullptr, nullptr, nullptr) == SCE_OK); +} + int main() { + CalendarConversionsInitializeTheirState(); SecondClockReportsWholeSeconds(); + KernelTimezone zone{-1, -1}; + Require(gettimeofday_nid_postfix(nullptr, nullptr) == SCE_OK); + Require(gettimeofday_nid_postfix(nullptr, &zone) == SCE_OK && zone.tz_minuteswest == 0 && zone.tz_dsttime == 0); + const int nullResolutionClocks[] = {GUEST_CLOCK_REALTIME, GUEST_CLOCK_VIRTUAL, GUEST_CLOCK_MONOTONIC, GUEST_CLOCK_SECOND, GUEST_CLOCK_PROCESS_CPUTIME_ID}; + for (const int clockId : nullResolutionClocks) Require(clock_getres_nid_postfix(clockId, nullptr) == SCE_OK); KernelTimespec time{-1, -1}; Require(sceKernelClockGettime(GUEST_CLOCK_VIRTUAL, &time) == SCE_OK); diff --git a/core/libs/tests/GuestGetrlimit.cpp b/core/libs/tests/GuestGetrlimit.cpp new file mode 100644 index 000000000..598ae8ed9 --- /dev/null +++ b/core/libs/tests/GuestGetrlimit.cpp @@ -0,0 +1,90 @@ +#include "prx/libc/include/general/VabiMacros.hpp" +#include +#include +#include +#include +#ifndef _WIN32 +#include +#endif + +struct ResourceLimit { + std::int64_t rlim_cur; + std::int64_t rlim_max; +}; + +extern "C" int APS5_VABI getrlimit_nid_postfix(int resource, ResourceLimit* limit); +extern "C" int* APS5_VABI __error_nid_postfix(); + +static constexpr int GUEST_EFAULT = 14; +static constexpr int GUEST_EINVAL = 22; +static constexpr int GUEST_RLIMIT_DATA = 2; +static constexpr int GUEST_RLIMIT_NOFILE = 8; +static constexpr int GUEST_RLIMIT_AS = 10; +static constexpr int GUEST_RLIM_NLIMITS = 15; +static constexpr std::int64_t GUEST_RLIM_INFINITY = std::numeric_limits::max(); + +static void Require(bool value) { if (!value) std::abort(); } + +static ResourceLimit GuestLimit(int resource) { + ResourceLimit limit{-1, -1}; + Require(getrlimit_nid_postfix(resource, &limit) == 0); + Require(limit.rlim_cur >= 0 && limit.rlim_cur <= limit.rlim_max); + return limit; +} + +static void RequireError(int resource, ResourceLimit* limit, int error) { + *__error_nid_postfix() = 0; + Require(getrlimit_nid_postfix(resource, limit) == -1); + Require(*__error_nid_postfix() == error); +} + +#ifndef _WIN32 +static std::int64_t Guest(rlim_t value) { + return value == RLIM_INFINITY ? GUEST_RLIM_INFINITY : static_cast(value); +} + +static void RequireHostLimit(int resource, int hostResource) { + rlimit host{}; + Require(getrlimit(hostResource, &host) == 0); + const ResourceLimit guest = GuestLimit(resource); + Require(guest.rlim_cur == Guest(host.rlim_cur)); + Require(guest.rlim_max == Guest(host.rlim_max)); +} + +static void LoweredHostLimitIsReported() { + rlimit host{}; + Require(getrlimit(RLIMIT_DATA, &host) == 0); + const rlim_t lowered = rlim_t{1} << 40; + if (host.rlim_max != RLIM_INFINITY && host.rlim_max < lowered) return; + host.rlim_cur = lowered; + Require(setrlimit(RLIMIT_DATA, &host) == 0); + Require(GuestLimit(GUEST_RLIMIT_DATA).rlim_cur == static_cast(lowered)); +} +#endif + +static void RequireUnsupported(int resource) { + ResourceLimit limit{}; + bool rejected = false; + try { + getrlimit_nid_postfix(resource, &limit); + } catch (const std::runtime_error&) { + rejected = true; + } + Require(rejected); +} + +int main() { +#ifdef _WIN32 + GuestLimit(GUEST_RLIMIT_DATA); +#else + RequireHostLimit(GUEST_RLIMIT_DATA, RLIMIT_DATA); + LoweredHostLimitIsReported(); +#endif + ResourceLimit limit{}; + RequireError(GUEST_RLIM_NLIMITS, &limit, GUEST_EINVAL); + RequireError(-1, &limit, GUEST_EINVAL); + RequireError(GUEST_RLIM_NLIMITS, nullptr, GUEST_EINVAL); + RequireError(GUEST_RLIMIT_DATA, nullptr, GUEST_EFAULT); + RequireUnsupported(GUEST_RLIMIT_NOFILE); + RequireUnsupported(GUEST_RLIMIT_AS); +} diff --git a/core/libs/tests/GuestImeParameters.cpp b/core/libs/tests/GuestImeParameters.cpp index f7711197d..23f590a38 100644 --- a/core/libs/tests/GuestImeParameters.cpp +++ b/core/libs/tests/GuestImeParameters.cpp @@ -8,6 +8,7 @@ extern "C" { void APS5_VABI sceImeParamInit(Param* param); int APS5_VABI sceImeGetPanelSize(const Param* param, uint32_t* width, uint32_t* height); int APS5_VABI sceImeClose_nid_postfix(void); +int APS5_VABI sceImeGetPanelPositionAndForm(PositionAndForm* form); int APS5_VABI sceImeSetCaret(const Caret* caret); int APS5_VABI sceImeSetText(const char16_t* text, uint32_t length); int APS5_VABI sceImeSetTextGeometry(TextAreaMode mode, const TextGeometry* geometry); @@ -119,6 +120,12 @@ static void CheckClosedPanel() { Require(sceImeSetText(text, 4) == notOpened, "text needs an open panel"); Require(sceImeSetTextGeometry(TextAreaMode::Edit, &geometry) == notOpened, "geometry needs an open panel"); Require(sceImeClose_nid_postfix() == notOpened, "closing needs an open panel"); + PositionAndForm form; + std::memset(&form, 0xa5, sizeof(form)); + Require(sceImeGetPanelPositionAndForm(&form) == notOpened, "panel position needs an open panel"); + Require(sceImeGetPanelPositionAndForm(nullptr) == notOpened, "panel position checks the panel first"); + const auto* formBytes = reinterpret_cast(&form); + for (size_t i = 0; i < sizeof(form); ++i) Require(formBytes[i] == 0xa5, "panel position written without an open panel"); } static void CheckKeyboardResourceIds() { diff --git a/core/libs/tests/GuestKernelErrors.cpp b/core/libs/tests/GuestKernelErrors.cpp index 9a16733f3..ddb8828a5 100644 --- a/core/libs/tests/GuestKernelErrors.cpp +++ b/core/libs/tests/GuestKernelErrors.cpp @@ -99,6 +99,7 @@ int main() { Require(scePthreadMutexattrDestroy(&attr) == SCE_OK); Require(scePthreadMutexLock(&mutex) == SCE_OK); Require(scePthreadMutexLock(&mutex) == SCE_KERNEL_ERROR_EDEADLK); + Require(scePthreadMutexDestroy(&mutex) == SCE_KERNEL_ERROR_EBUSY); Require(scePthreadMutexUnlock(&mutex) == SCE_OK); Require(scePthreadMutexDestroy(&mutex) == SCE_OK); diff --git a/core/libs/tests/GuestKernelRename.cpp b/core/libs/tests/GuestKernelRename.cpp new file mode 100644 index 000000000..6b74a12f9 --- /dev/null +++ b/core/libs/tests/GuestKernelRename.cpp @@ -0,0 +1,65 @@ +#include "prx/libc/include/general/VabiMacros.hpp" +#include +#include +#include +#include +#include +#include +#include +#ifdef _WIN32 +#include +#else +#include +#endif +extern "C" { +int* APS5_VABI __error_nid_postfix(); +} +using RenameFunction = int (APS5_VABI*)(const char*, const char*); +static RenameFunction LoadKernelRename() { +#ifdef _WIN32 + const HMODULE module = LoadLibraryA(KERNEL_PRX_PATH); + if (module == nullptr) return nullptr; + const auto function = reinterpret_cast(reinterpret_cast(GetProcAddress(module, "rename_nid_postfix"))); + HMODULE owner = nullptr; + if (function == nullptr || !GetModuleHandleExA(GET_MODULE_HANDLE_EX_FLAG_FROM_ADDRESS | GET_MODULE_HANDLE_EX_FLAG_UNCHANGED_REFCOUNT, + reinterpret_cast(function), &owner) || owner != module) return nullptr; + return function; +#else + void* module = dlopen(KERNEL_PRX_PATH, RTLD_NOW | RTLD_LOCAL); + if (module == nullptr) return nullptr; + void* function = dlsym(module, "rename_nid_postfix"); + Dl_info owner{}; + void* self = dlsym(module, "sceKernelRename"); + Dl_info expected{}; + if (function == nullptr || self == nullptr || !dladdr(function, &owner) || !dladdr(self, &expected) || + owner.dli_fname == nullptr || expected.dli_fname == nullptr || std::strcmp(owner.dli_fname, expected.dli_fname) != 0) return nullptr; + return reinterpret_cast(function); +#endif +} +static void Check(bool value, int line) { + if (!value) { + std::fprintf(stderr, "Kernel rename check failed at line %d (guest errno %d)\n", line, *__error_nid_postfix()); + std::abort(); + } +} +#define Require(value) Check((value), __LINE__) +int main() { + const RenameFunction rename_nid_postfix = LoadKernelRename(); + if (rename_nid_postfix == nullptr) { std::fputs("rename_nid_postfix is not defined by libkernel\n", stderr); return 1; } + const auto root = std::filesystem::path("anyps5-kernel-rename-test-" + + std::to_string(std::chrono::steady_clock::now().time_since_epoch().count())); + Require(std::filesystem::create_directory(root)); + const auto file = root / "file.txt"; + const auto renamed = root / "renamed.txt"; + { std::ofstream stream(file); stream << "retained"; } + Require(rename_nid_postfix(file.string().c_str(), renamed.string().c_str()) == 0); + Require(!std::filesystem::exists(file) && std::filesystem::exists(renamed)); + Require(rename_nid_postfix(renamed.string().c_str(), renamed.string().c_str()) == 0); + Require(rename_nid_postfix(file.string().c_str(), renamed.string().c_str()) == -1 && *__error_nid_postfix() == 2); + Require(rename_nid_postfix(nullptr, renamed.string().c_str()) == -1 && *__error_nid_postfix() == 14); + Require(rename_nid_postfix(renamed.string().c_str(), nullptr) == -1 && *__error_nid_postfix() == 14); + Require(rename_nid_postfix("", renamed.string().c_str()) == -1 && *__error_nid_postfix() == 2); + Require(rename_nid_postfix(renamed.string().c_str(), "") == -1 && *__error_nid_postfix() == 2); + std::filesystem::remove_all(root); + return 0; +} diff --git a/core/libs/tests/GuestKernelSync.cpp b/core/libs/tests/GuestKernelSync.cpp index bb227cb5e..526218a99 100644 --- a/core/libs/tests/GuestKernelSync.cpp +++ b/core/libs/tests/GuestKernelSync.cpp @@ -6,7 +6,9 @@ #include #include #ifdef _WIN32 +#ifndef NOMINMAX #define NOMINMAX +#endif #define WIN32_LEAN_AND_MEAN #include #endif diff --git a/core/libs/tests/GuestKernelSys.cpp b/core/libs/tests/GuestKernelSys.cpp new file mode 100644 index 000000000..6b523f471 --- /dev/null +++ b/core/libs/tests/GuestKernelSys.cpp @@ -0,0 +1,58 @@ +#include "prx/libc/include/general/VabiMacros.hpp" +#include +#include +#include +#include +#include +#include +#include +#include +#include + +extern "C" { +std::int64_t APS5_VABI readlink_nid_postfix(const char*, char*, std::size_t); +int* APS5_VABI __error_nid_postfix(); +} + +void Require(bool condition, std::source_location location = std::source_location::current()) { + if (!condition) { + std::fprintf(stderr, "Kernel sys check failed at line %u (guest errno %d)\n", location.line(), *__error_nid_postfix()); + std::abort(); + } +} + +int Error() { + return *__error_nid_postfix(); +} + +int main() { + const auto root = std::filesystem::path("anyps5-kernel-sys-test-" + + std::to_string(std::chrono::steady_clock::now().time_since_epoch().count())); + Require(std::filesystem::create_directory(root)); + const auto file = (root / "file.txt").string(); + std::ofstream(file).put('x'); + char buffer[64]; + Require(readlink_nid_postfix(file.c_str(), buffer, sizeof(buffer)) == -1 && Error() == 22); + Require(readlink_nid_postfix(root.string().c_str(), buffer, sizeof(buffer)) == -1 && Error() == 22); + Require(readlink_nid_postfix((root / "missing").string().c_str(), buffer, sizeof(buffer)) == -1 && Error() == 2); + Require(readlink_nid_postfix("", buffer, sizeof(buffer)) == -1 && Error() == 2); + Require(readlink_nid_postfix(nullptr, buffer, sizeof(buffer)) == -1 && Error() == 14); + std::error_code error; + const auto link = (root / "link").string(); + std::filesystem::create_symlink("sub/target.txt", link, error); + if (!error) { + std::memset(buffer, '#', sizeof(buffer)); + Require(readlink_nid_postfix(link.c_str(), buffer, sizeof(buffer)) == 14); + Require(std::memcmp(buffer, "sub/target.txt", 14) == 0 && buffer[14] == '#'); + std::memset(buffer, '#', sizeof(buffer)); + Require(readlink_nid_postfix(link.c_str(), buffer, 3) == 3); + Require(std::memcmp(buffer, "sub", 3) == 0 && buffer[3] == '#'); + Require(readlink_nid_postfix(link.c_str(), nullptr, 0) == 0); + Require(readlink_nid_postfix(link.c_str(), nullptr, sizeof(buffer)) == -1 && Error() == 14); + Require(readlink_nid_postfix((root / "LINK").string().c_str(), buffer, sizeof(buffer)) == 14); + } else { + std::fprintf(stderr, "symbolic links unavailable: the link cases not tested\n"); + } + std::filesystem::remove_all(root); + return 0; +} diff --git a/core/libs/tests/GuestKill.cpp b/core/libs/tests/GuestKill.cpp new file mode 100644 index 000000000..4a2d75547 --- /dev/null +++ b/core/libs/tests/GuestKill.cpp @@ -0,0 +1,46 @@ +#include "prx/libc/include/general/VabiMacros.hpp" +#include +#include +#include +#include +using Handler = void (APS5_VABI *)(int); +extern "C" { +Handler APS5_VABI signal_nid_postfix(int, Handler); +int APS5_VABI kill_nid_postfix(int, int); +int APS5_VABI getpid_nid_postfix(void); +int* APS5_VABI __error_nid_postfix(); +} +volatile std::sig_atomic_t received = 0; +void APS5_VABI Callback(int value) { received = value; } +static void Require(bool value) { if (!value) std::abort(); } +static void Fails(int pid, int guest, int error) { + *__error_nid_postfix() = 0; + Require(kill_nid_postfix(pid, guest) == -1 && *__error_nid_postfix() == error); +} +int main() { + const int self = getpid_nid_postfix(); + const auto invalid = reinterpret_cast(static_cast(-1)); + const auto ignore = reinterpret_cast(std::uintptr_t{1}); + for (int target : {self, 0, -self}) Require(kill_nid_postfix(target, 0) == 0); + for (int target : {self + 1, -1, -(self + 1)}) { + Fails(target, 0, 3); + Fails(target, 15, 3); + } + for (int guest : {-1, 129}) { + Fails(self, guest, 22); + Fails(self + 1, guest, 22); + } + Require(signal_nid_postfix(15, Callback) != invalid); + for (int target : {self, 0, -self}) { + received = 0; + Require(kill_nid_postfix(target, 0) == 0 && received == 0); + Require(kill_nid_postfix(target, 15) == 0 && received == 15); + } + received = 0; + Fails(self + 1, 15, 3); + Require(received == 0); + Require(signal_nid_postfix(15, ignore) == Callback); + Require(kill_nid_postfix(self, 15) == 0 && received == 0); + Require(signal_nid_postfix(15, nullptr) == ignore); + Fails(self, 100, 22); +} diff --git a/core/libs/tests/GuestMath.cpp b/core/libs/tests/GuestMath.cpp index ffdc06dff..ee0b4d20f 100644 --- a/core/libs/tests/GuestMath.cpp +++ b/core/libs/tests/GuestMath.cpp @@ -15,11 +15,17 @@ long double APS5_VABI strtold_nid_postfix(const char*, char**); std::int64_t APS5_VABI strtol_nid_postfix(const char*, char**, int); std::uint64_t APS5_VABI strtoul_nid_postfix(const char*, char**, int); std::intmax_t APS5_VABI strtoimax_nid_postfix(const char*, char**, int); +long long APS5_VABI strtoll_nid_postfix(const char*, char**, int); +unsigned long long APS5_VABI strtoull_nid_postfix(const char*, char**, int); +std::uintmax_t APS5_VABI strtoumax_nid_postfix(const char*, char**, int); +unsigned long long APS5_VABI _Stoull_nid_postfix(const char*, char**, int); +std::uint64_t APS5_VABI _Stoul_nid_postfix(const char*, char**, int); int* APS5_VABI __error_nid_postfix(); struct LibcFloatConstant { std::uint32_t bits[4]; }; extern LibcFloatConstant _FInf_nid_postfix; extern LibcFloatConstant _FNan_nid_postfix; short APS5_VABI _FDtest_nid_postfix(const float*); +int APS5_VABI __fpclassifyf_nid_postfix(float); float APS5_VABI fmodf_nid_postfix(float, float); float APS5_VABI asinf_nid_postfix(float); float APS5_VABI acosf_nid_postfix(float); @@ -44,6 +50,7 @@ int APS5_VABI __isnormal_nid_postfix(double); int APS5_VABI __isnormalf_nid_postfix(float); int APS5_VABI __isinff_nid_postfix(float); std::lldiv_t APS5_VABI lldiv_nid_postfix(long long, long long); +std::lldiv_t APS5_VABI ldiv_nid_postfix(std::int64_t, std::int64_t); } static void Require(bool value) { if (!value) std::abort(); } @@ -53,8 +60,12 @@ static void CheckIntegerConversions() { const auto result = lldiv_nid_postfix(numerator, denominator); Require(result.quot == numerator / denominator && result.rem == numerator % denominator); Require(result.quot * denominator + result.rem == numerator); + const auto wide = ldiv_nid_postfix(numerator, denominator); + Require(wide.quot == result.quot && wide.rem == result.rem); } } + const auto extreme = ldiv_nid_postfix(INT64_MIN, 2); + Require(extreme.quot == INT64_MIN / 2 && extreme.rem == 0); struct SignedCase { const char* text; int base; @@ -77,6 +88,11 @@ static void CheckIntegerConversions() { {"0x8000000000000000!", 16, INT64_MAX, 18, 34}, {"0100000000000!", 0, INT64_C(8589934592), 13, 0}, {"100000000000000000000000000000000!", 2, INT64_C(4294967296), 33, 0}, + {"0b101", 0, 0, 1, 0}, + {"0b101", 2, 0, 1, 0}, + {" -0B1!", 0, 0, 3, 0}, + {"0b2", 0, 0, 1, 0}, + {"0b101", 16, 0xb101, 5, 0}, {"z!", 36, 35, 1, 0}, {"", 10, 0, 0, 0}, {" \t+!", 10, 0, 0, 0}, @@ -118,6 +134,10 @@ static void CheckIntegerConversions() { {"0x10000000000000000!", 16, UINT64_MAX, 19, 34}, {"0100000000000!", 0, UINT64_C(8589934592), 13, 0}, {"100000000000000000000000000000000!", 2, UINT64_C(4294967296), 33, 0}, + {"0b101", 0, 0, 1, 0}, + {"0B11", 2, 0, 1, 0}, + {" +0b1!", 2, 0, 3, 0}, + {"0b101", 16, 0xb101, 5, 0}, {"z!", 36, 35, 1, 0}, {"", 10, 0, 0, 0}, {" \t-!", 10, 0, 0, 0}, @@ -132,11 +152,24 @@ static void CheckIntegerConversions() { std::abort(); } } + for (const int base : {0, 2}) { + const char text[] = " -0B11"; + char* end = nullptr; + Require(strtoll_nid_postfix(text, &end, base) == 0 && end == text + 3); + Require(strtoull_nid_postfix(text, &end, base) == 0 && end == text + 3); + Require(strtoumax_nid_postfix(text, &end, base) == 0 && end == text + 3); + Require(_Stoull_nid_postfix(text, &end, base) == 0 && end == text + 3); + Require(_Stoul_nid_postfix(text, &end, base) == 0 && end == text + 3); + } *__error_nid_postfix() = 13; Require(strtol_nid_postfix("-4294967296", nullptr, 10) == -INT64_C(4294967296)); Require(*__error_nid_postfix() == 13); Require(strtoul_nid_postfix("4294967296", nullptr, 10) == UINT64_C(4294967296)); Require(*__error_nid_postfix() == 13); + Require(_Stoul_nid_postfix("4294967296", nullptr, 10) == UINT64_C(4294967296)); + Require(*__error_nid_postfix() == 13); + Require(_Stoul_nid_postfix("18446744073709551615", nullptr, 10) == UINT64_MAX); + Require(_Stoul_nid_postfix("-1", nullptr, 10) == UINT64_MAX && *__error_nid_postfix() == 13); *__error_nid_postfix() = 0; } @@ -155,6 +188,11 @@ static void CheckFloatClassification() { std::fprintf(stderr, "Guest _FDtest failed for %08x\n", test.bits); std::abort(); } + const int fpclass = test.code == 0 ? 0x10 : test.code == -2 ? 0x08 : test.code == -1 ? 0x04 : test.code == 1 ? 0x01 : 0x02; + if (__fpclassifyf_nid_postfix(value) != fpclass) { + std::fprintf(stderr, "Guest __fpclassifyf failed for %08x\n", test.bits); + std::abort(); + } } Require(_FDtest_nid_postfix(reinterpret_cast(&_FInf_nid_postfix)) == 1); Require(_FDtest_nid_postfix(reinterpret_cast(&_FNan_nid_postfix)) == 2); diff --git a/core/libs/tests/GuestMemory.cpp b/core/libs/tests/GuestMemory.cpp index 892cefb40..caed0eb13 100644 --- a/core/libs/tests/GuestMemory.cpp +++ b/core/libs/tests/GuestMemory.cpp @@ -5,7 +5,10 @@ #include "prx/libc/include/GuestWriteWatch.hpp" #include "prx/libkernel/File/include/FileFlags.hpp" #include "prx/libkernel/KernelErrors.hpp" +#include "prx/libkernel/DirectMemory/DirectMemory.hpp" +#include #include +#include #include "SceTypes.hpp" #include #include @@ -18,11 +21,13 @@ #include #include #include +#include #include #include #include #if defined(__linux__) #include +#include #include #include #include @@ -37,6 +42,7 @@ extern "C" { void* APS5_VABI mmap_nid_postfix(void*, std::size_t, int, int, int, std::int64_t) noexcept; int APS5_VABI munmap_nid_postfix(void*, std::size_t) noexcept; int APS5_VABI mprotect_nid_postfix(void*, std::size_t, int) noexcept; +int APS5_VABI madvise_nid_postfix(void*, std::size_t, int); int* APS5_VABI __error_nid_postfix(); int APS5_VABI sceKernelMapNamedFlexibleMemory(void**, std::size_t, int, int, const char*); int APS5_VABI sceKernelMapNamedFlexibleMemoryInternal(void**, std::size_t, int, int, const char*); @@ -67,6 +73,9 @@ int APS5_VABI sceKernelAioDeleteRequest(std::int32_t, std::int32_t*); int APS5_VABI sceKernelMlock_nid_postfix(void*, std::uint64_t); int APS5_VABI sceKernelGetDirectMemoryType(std::int64_t, int*, std::int64_t*, std::int64_t*); int APS5_VABI sceKernelBatchMap2(KernelBatchMapEntry*, int, int*, int); +void* APS5_VABI dlopen_nid_postfix(const char*, int); +void* APS5_VABI dlsym_nid_postfix(void*, const char*); +int APS5_VABI dlclose_nid_postfix(void*); } static void Require(bool condition, std::source_location location = std::source_location::current()) { @@ -278,13 +287,30 @@ static void CheckReleaseDirectMemoryClearsMappings() { Require(sceKernelVirtualQuery(mapped, 0, &split, sizeof(split)) == 0); Require(split.is_direct && split.offset == static_cast(phys)); VirtualQueryInfo dropped{}; - Require(sceKernelVirtualQuery(static_cast(mapped) + page, 0, &dropped, sizeof(dropped)) == 0); - Require(!dropped.is_direct && dropped.offset == 0); + const int droppedResult = sceKernelVirtualQuery(static_cast(mapped) + page, 0, &dropped, sizeof(dropped)); + Require(droppedResult != 0 || !dropped.is_direct); + void* alias = nullptr; + Require(sceKernelMapDirectMemory(&alias, page, 3, 0, phys, 0) == 0); Require(sceKernelReleaseDirectMemory(phys, page) == 0); - VirtualQueryInfo cleared{}; - Require(sceKernelVirtualQuery(mapped, 0, &cleared, sizeof(cleared)) == 0); - Require(!cleared.is_direct && cleared.offset == 0); - Require(sceKernelMunmap(mapped, page * 2) == 0); + for (void* view : {mapped, alias}) { + VirtualQueryInfo cleared{}; + Require(sceKernelVirtualQuery(view, 0, &cleared, sizeof(cleared)) != 0); + VirtualQueryInfo next{}; + if (sceKernelVirtualQuery(view, 1, &next, sizeof(next)) == 0) Require(next.start != reinterpret_cast(view)); + } +} + +static void CheckReleaseDirectMemoryRejectsInvalidRanges() { + constexpr std::size_t page = 0x4000; + Require(sceKernelReleaseDirectMemory(-1, page) == SCE_KERNEL_ERROR_EINVAL); + Require(sceKernelReleaseDirectMemory(0, 0) == SCE_KERNEL_ERROR_EINVAL); + Require(sceKernelReleaseDirectMemory(0, 1) == SCE_KERNEL_ERROR_EINVAL); + Require(sceKernelReleaseDirectMemory(0, page - 1) == SCE_KERNEL_ERROR_EINVAL); + Require(sceKernelReleaseDirectMemory(1, page) == SCE_KERNEL_ERROR_EINVAL); + Require(sceKernelReleaseDirectMemory(0, std::numeric_limits::max()) == SCE_KERNEL_ERROR_EINVAL); + Require(sceKernelReleaseDirectMemory(0, DIRECT_MEMORY_SIZE + page) == SCE_KERNEL_ERROR_EINVAL); + Require(sceKernelReleaseDirectMemory(DIRECT_MEMORY_SIZE - page, page * 2) == SCE_KERNEL_ERROR_EINVAL); + Require(sceKernelReleaseDirectMemory(DIRECT_MEMORY_SIZE, page) == SCE_KERNEL_ERROR_EINVAL); } static void CheckFixedMappingReplacesPartialOverlap() { @@ -398,6 +424,48 @@ static void CheckDirectMemoryGpuProtBits() { Require(sceKernelReleaseDirectMemory(phys, page) == 0); } +static bool GpuMapped(const void* pointer) { + GuestAllocations::Mutation mutation; + return mutation.Find(pointer).gpu; +} + +static void CheckGpuAccessFollowsProtection() { + constexpr std::size_t page = 0x4000; + std::int64_t phys = 0; + Require(sceKernelAllocateDirectMemory(0, 0x7fffffffffll, page * 2, 0, 0, &phys) == 0); + void* cpu = nullptr; + Require(sceKernelMapDirectMemory(&cpu, page, 3, 0, phys, 0) == 0); + Require(!GpuMapped(cpu)); + void* gpu = nullptr; + Require(sceKernelMapDirectMemory(&gpu, page, 0x32, 0, phys + static_cast(page), 0) == 0); + Require(GpuMapped(gpu)); + Require(sceKernelMprotect(cpu, page, 0x13) == 0); + Require(GpuMapped(cpu)); + Require(sceKernelMprotect(gpu, page, 3) == 0); + Require(!GpuMapped(gpu)); + static int imageProbe = 0; + const auto probe = reinterpret_cast(&imageProbe); + { + GuestAllocations::Mutation mutation; + mutation.RegisterMainImage(); + } + { + const auto lease = GuestAllocations::GuestAllocationsAcquire_nid_postfix(); + const auto image = std::find_if(lease.begin(), lease.end(), [&](const auto& range) { return probe >= range->address && probe - range->address < range->bytes; }); + Require(image != lease.end() && !(*image)->releasable && !(*image)->gpu); + } + Require(sceKernelMunmap(cpu, page) == 0); + Require(sceKernelMunmap(gpu, page) == 0); + Require(sceKernelReleaseDirectMemory(phys, page * 2) == 0); + void* flexible = nullptr; + Require(sceKernelMapNamedFlexibleMemory(&flexible, page, 3, 0, "cpu") == 0); + Require(!GpuMapped(flexible)); + Require(sceKernelMunmap(flexible, page) == 0); + Require(sceKernelMapNamedFlexibleMemory(&flexible, page, 0x33, 0, "gpu") == 0); + Require(GpuMapped(flexible)); + Require(sceKernelMunmap(flexible, page) == 0); +} + static void CheckFixedVirtualReservation() { constexpr std::size_t page = 0x4000; void* probe = nullptr; @@ -493,13 +561,18 @@ static void CheckNoOverwriteRefusesLiveMapping() { } #ifdef _WIN32 +constexpr std::uint32_t ReadWriteProtection = PAGE_READWRITE; +#else +constexpr std::uint32_t ReadWriteProtection = 0x04; +#endif + static void CheckNoOverwriteRejectsHostOccupiedMapping() { constexpr std::size_t page = 0x4000; void* reservation = nullptr; Require(sceKernelReserveVirtualRange(&reservation, page * 4, 0, 0) == 0); Require(sceKernelMunmap(reservation, page * 4) == 0); void* target = static_cast(reservation) + page; - GuestArena::GuestArenaCommit_nid_postfix(target, page, PAGE_READWRITE, page); + GuestArena::GuestArenaCommit_nid_postfix(target, page, ReadWriteProtection, page); std::int64_t phys = 0; Require(sceKernelAllocateDirectMemory(0, 0x7fffffffffll, page, 0, 0, &phys) == 0); void* fixed = target; @@ -514,6 +587,171 @@ static void CheckNoOverwriteRejectsHostOccupiedMapping() { Require(sceKernelReleaseDirectMemory(phys, page) == 0); } +#ifdef _WIN32 +static DWORD ImageProtection(const void* pointer) { + MEMORY_BASIC_INFORMATION memory{}; + Require(VirtualQuery(pointer, &memory, sizeof(memory)) == sizeof(memory)); + return memory.Protect & 0xffu; +} + +static unsigned char* GuestModulePage(void* module, std::size_t page) { + auto* data = static_cast(dlsym_nid_postfix(module, "guestMemoryModuleData")); + Require(data != nullptr); + auto* target = reinterpret_cast((reinterpret_cast(data) + page - 1) & ~(page - 1)); + Require(target + page <= data + 0x10000); + return target; +} + +static bool RegisteredGuestRange(const void* pointer, std::size_t bytes) { + GuestAllocations::Mutation mutation; + return mutation.Overlaps(pointer, bytes); +} + +static void CheckGuestModuleImageProtection() { + constexpr std::size_t page = 0x4000; + const auto path = std::filesystem::relative(GUEST_MEMORY_MODULE).generic_string(); + void* module = dlopen_nid_postfix(path.c_str(), 2); + Require(module != nullptr); + auto* target = GuestModulePage(module, page); + Require(!RegisteredGuestRange(target, page)); + Require(sceKernelMprotect(target, page, 1) == 0); + Require(RegisteredGuestRange(target, page)); + Require(ImageProtection(target) == PAGE_READONLY); + Require(sceKernelMprotect(target, page, 3) == 0); + Require(ImageProtection(target) == PAGE_READWRITE || ImageProtection(target) == PAGE_WRITECOPY); + target[1] = 7; + Require(target[1] == 7); + Require(sceKernelMprotect(target, page, 1) == 0); + Require(ImageProtection(target) == PAGE_READONLY); + + { + auto held = GuestAllocations::GuestAllocationsAcquire_nid_postfix(); + bool failed = false; + try { + dlclose_nid_postfix(module); + } catch (const std::runtime_error&) { + failed = true; + } + Require(failed); + Require(GuestModulePage(module, page) == target); + Require(RegisteredGuestRange(target, page)); + Require(ImageProtection(target) == PAGE_READONLY); + } + + auto lease = GuestAllocations::GuestAllocationsAcquire_nid_postfix(); + std::atomic closed{false}; + std::thread closer([&] { + Require(dlclose_nid_postfix(module) == 0); + closed = true; + }); + std::this_thread::sleep_for(std::chrono::milliseconds(200)); + Require(!closed); + Require(ImageProtection(target) == PAGE_READONLY); + lease.clear(); + closer.join(); + Require(!RegisteredGuestRange(target, page)); + { + GuestAllocations::Mutation mutation; + mutation.Add(target, page, true, true, false); + mutation.Remove(target); + } + + module = dlopen_nid_postfix(path.c_str(), 2); + Require(module != nullptr); + target = GuestModulePage(module, page); + Require(!RegisteredGuestRange(target, page)); + Require(sceKernelMprotect(target, page, 1) == 0); + Require(ImageProtection(target) == PAGE_READONLY); + Require(RegisteredGuestRange(target, page)); + Require(dlclose_nid_postfix(module) == 0); + Require(!RegisteredGuestRange(target, page)); + + bool rejected = false; + try { + sceKernelMprotect(GetModuleHandleA("kernel32.dll"), page, 1); + } catch (const std::invalid_argument&) { + rejected = true; + } + Require(rejected); +} +#endif + +static void CheckUnhintedKernelMappingsLandInTheWindow() { + constexpr std::size_t page = 0x4000; + constexpr std::uintptr_t windowStart = 0x200000000ull; + constexpr std::uintptr_t windowEnd = 0xFC00000000ull; + std::int64_t phys = 0; + Require(sceKernelAllocateDirectMemory(0, 0x7fffffffffll, page, 0, 0, &phys) == 0); + void* direct = nullptr; + Require(sceKernelMapDirectMemory(&direct, page, 3, 0, phys, 0) == 0); + Require(direct != nullptr); + const auto directAddress = reinterpret_cast(direct); + Require(directAddress >= windowStart); + Require(directAddress + page <= windowEnd); + void* flexible = nullptr; + Require(sceKernelMapNamedFlexibleMemory(&flexible, page, 3, 0, "arena-window-test") == 0); + Require(flexible != nullptr); + const auto flexibleAddress = reinterpret_cast(flexible); + Require(flexibleAddress >= windowStart); + Require(flexibleAddress + page <= windowEnd); + Require(sceKernelMunmap(direct, page) == 0); + Require(sceKernelMunmap(flexible, page) == 0); + Require(sceKernelReleaseDirectMemory(phys, page) == 0); +} + +#if defined(__linux__) +static void CheckUnmappedArenaRangeStaysReserved() { + constexpr std::size_t page = 0x4000; + void* address = nullptr; + Require(sceKernelMapNamedFlexibleMemory(&address, page, 3, 0, "reserved-after-unmap") == 0); + Require(address != nullptr); + Require(sceKernelMunmap(address, page) == 0); + void* hostile = mmap(address, page, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS | MAP_FIXED_NOREPLACE, -1, 0); + Require(hostile == MAP_FAILED && errno == EEXIST); + void* again = nullptr; + Require(sceKernelMapNamedFlexibleMemory(&again, page, 3, 0, "reserved-after-unmap-2") == 0); + Require(again != nullptr); + Require(sceKernelMunmap(again, page) == 0); +} +#endif + +static void CheckArenaPlacesUnhintedMappingsInsideTheWindow() { + constexpr std::size_t page = 0x4000; + std::uintptr_t base = 0; + std::size_t size = 0; + GuestArena::GuestArenaRange_nid_postfix(&base, &size); + Require(size != 0); + auto* placed = GuestArena::GuestArenaAllocate_nid_postfix(page, page); + Require(placed != nullptr); + const auto address = reinterpret_cast(placed); + Require(address >= base); + Require(address + page <= base + size); + GuestArena::GuestArenaRelease_nid_postfix(placed, page); +} + +static void CheckArenaReusesAFreedRange() { + constexpr std::size_t page = 0x4000; + auto* first = GuestArena::GuestArenaAllocate_nid_postfix(page, page); + Require(first != nullptr); + GuestArena::GuestArenaRelease_nid_postfix(first, page); + auto* second = GuestArena::GuestArenaAllocate_nid_postfix(page, page); + Require(second == first); + GuestArena::GuestArenaRelease_nid_postfix(second, page); +} + +static void CheckArenaHonoursHintsWithoutGoingBelowThem() { + constexpr std::size_t page = 0x4000; + std::uintptr_t base = 0; + std::size_t size = 0; + GuestArena::GuestArenaRange_nid_postfix(&base, &size); + const auto hint = base + page; + auto* placed = GuestArena::GuestArenaAllocateAtOrAbove_nid_postfix(hint, page, page); + Require(placed != nullptr); + Require(reinterpret_cast(placed) >= hint); + Require(GuestArena::GuestArenaContains_nid_postfix(placed, page)); + GuestArena::GuestArenaRelease_nid_postfix(placed, page); +} + static void CheckFixedMappingsReachTheApplicationAreaEnd() { constexpr std::size_t page = 0x4000; constexpr std::uintptr_t applicationAreaEnd = 0xFC00000000ull; @@ -545,7 +783,6 @@ static void CheckFixedMappingsReachTheApplicationAreaEnd() { Require(sceKernelMunmap(mapped, page * 2) == 0); Require(sceKernelReleaseDirectMemory(phys, page * 2) == 0); } -#endif #if defined(__linux__) static std::size_t LockedKilobytes() { @@ -664,6 +901,24 @@ static void CheckSharedDirectMemoryLifecycle() { Require(sceKernelReleaseDirectMemory(phys, page * 3) == 0); } +static void CheckHeapAlignment() { + std::vector> blocks; + for (std::size_t bytes = 1; bytes <= 600; ++bytes) blocks.emplace_back(static_cast(GuestHeap::GuestHeapAllocate_nid_postfix(bytes)), bytes); + for (const std::size_t bytes : {std::size_t{4000}, std::size_t{70000}, std::size_t{0x30000}}) blocks.emplace_back(static_cast(GuestHeap::GuestHeapAllocate_nid_postfix(bytes)), bytes); + auto* grown = static_cast(GuestHeap::GuestHeapReallocate_nid_postfix(GuestHeap::GuestHeapAllocate_nid_postfix(8), 333)); + blocks.emplace_back(grown, 333); + for (std::size_t index = 0; index < blocks.size(); ++index) { + const auto [pointer, bytes] = blocks[index]; + Require((reinterpret_cast(pointer) & 31u) == 0); + std::memset(pointer, static_cast(index & 0xffu), bytes); + } + for (std::size_t index = 0; index < blocks.size(); ++index) { + const auto [pointer, bytes] = blocks[index]; + for (std::size_t at = 0; at < bytes; ++at) Require(pointer[at] == static_cast(index & 0xffu)); + GuestHeap::GuestHeapFree_nid_postfix(pointer); + } +} + static void CheckHeapAfterMappingReuse() { constexpr std::size_t bytes = 0x30000; auto* pointer = static_cast(GuestHeap::GuestHeapAllocate_nid_postfix(bytes)); @@ -730,6 +985,39 @@ static void CheckSharedWriteTracking() { #endif } +static void CheckFailedCollectKeepsWrites() { +#ifdef _WIN32 + constexpr std::size_t page = 0x4000; + const auto collect = [](void* address, std::size_t bytes, std::size_t& count) { + std::array pages{}; + count = pages.size(); + return GuestArena::GuestArenaCollectWrites_nid_postfix(reinterpret_cast(address), bytes, pages.data(), &count, true); + }; + std::size_t count = 0; + auto* block = static_cast(GuestArena::GuestArenaAllocate_nid_postfix(page * 3, page)); + GuestArena::GuestArenaCommit_nid_postfix(block, page, PAGE_READWRITE, page); + GuestArena::GuestArenaCommit_nid_postfix(block + page * 2, page, PAGE_READWRITE, page); + Require(collect(block, page, count) && collect(block + page * 2, page, count)); + static_cast(block)[8] = 1; + Require(!collect(block, page * 3, count)); + Require(collect(block, page, count) && count == 1); + GuestArena::GuestArenaRelease_nid_postfix(block, page * 3); + + std::int64_t phys = 0; + Require(sceKernelAllocateDirectMemory(0, 0x7fffffffffll, page * 2, 0, 0, &phys) == 0); + void* mapped = nullptr; + Require(sceKernelMapDirectMemory(&mapped, page * 2, 3, 0, phys, 0) == 0); + auto* shared = static_cast(mapped); + Require(collect(mapped, page * 2, count)); + shared[8] = 2; + Require(sceKernelMprotect(const_cast(shared + page), page, 0) == 0); + Require(!collect(mapped, page * 2, count)); + Require(collect(mapped, page, count) && count == 4); + Require(sceKernelMunmap(mapped, page * 2) == 0); + Require(sceKernelReleaseDirectMemory(phys, page * 2) == 0); +#endif +} + static void CheckPinnedSharedPages() { #ifdef _WIN32 constexpr std::size_t page = 0x4000; @@ -820,13 +1108,7 @@ static void CheckReadsIntoSharedWriteTracking() { std::int32_t deleted = -1; Require(sceKernelAioDeleteRequest(id, &deleted) == 0); Require(sceKernelMprotect(mapped, page * 2, 1) == 0); - bool refused = false; - try { - sceKernelPread(fd, mapped, page, 0); - } catch (const std::exception&) { - refused = true; - } - Require(refused); + Require(sceKernelPread(fd, mapped, page, 0) == static_cast(SCE_KERNEL_ERROR_EFAULT)); KernelAioResult refusedResult{-1, 0}; KernelAioRwRequest refusedRequest{0, page, bytes, &refusedResult, fd}; Require(sceKernelAioSubmitReadCommands(&refusedRequest, 1, 0, &id) == 0); @@ -864,6 +1146,62 @@ static bool Written(const void* base, std::size_t bytes, PageRuns expected) { return false; } +static bool CollectArmedRuns(const void* base, std::size_t offset, std::size_t bytes, PageRuns& runs) { + runs.clear(); + const auto address = reinterpret_cast(base); + std::pair context{address, &runs}; + return GuestWriteWatch::GuestWriteWatchCollectArmed_nid_postfix(address + offset, bytes, [](void* context, std::uintptr_t begin, std::uintptr_t end) { + auto& [origin, into] = *static_cast*>(context); + if (!into->empty() && into->back().second == (begin - origin) / 4096) into->back().second = (end - origin) / 4096; + else into->emplace_back((begin - origin) / 4096, (end - origin) / 4096); + }, &context); +} + +static void CheckWriteWatchArmedCollect() { + if (!GuestWriteWatch::GuestWriteWatchAvailable_nid_postfix()) return; + constexpr std::size_t length = 0x100000; + constexpr std::size_t small = 4096; + void* mapping = nullptr; + Require(sceKernelMapFlexibleMemory(&mapping, length, 3, 0) == 0); + auto* bytes = static_cast(mapping); + PageRuns runs; + Require(CollectRuns(mapping, 0, length / 2, runs) && runs == PageRuns{{0, length / small / 2}}); + bytes[5 * small] = 1; + bytes[200 * small] = 1; + Require(CollectArmedRuns(mapping, 0, length, runs) && runs == PageRuns{{5, 6}}); + Require(Written(mapping, length, {{length / small / 2, length / small}})); + Require(Written(mapping, length, {})); + Require(sceKernelMunmap(mapping, length) == 0); +} + +static bool CollectFreshRuns(const void* base, std::size_t offset, std::size_t bytes, PageRuns& runs) { + runs.clear(); + const auto address = reinterpret_cast(base); + std::pair context{address, &runs}; + return GuestWriteWatch::GuestWriteWatchCollectFresh_nid_postfix(address + offset, bytes, [](void* context, std::uintptr_t begin, std::uintptr_t end) { + auto& [origin, into] = *static_cast*>(context); + if (!into->empty() && into->back().second == (begin - origin) / 4096) into->back().second = (end - origin) / 4096; + else into->emplace_back((begin - origin) / 4096, (end - origin) / 4096); + }, &context); +} + +static void CheckWriteWatchFreshCollect() { + if (!GuestWriteWatch::GuestWriteWatchAvailable_nid_postfix()) return; + constexpr std::size_t length = 0x100000; + constexpr std::size_t small = 4096; + void* mapping = nullptr; + Require(sceKernelMapFlexibleMemory(&mapping, length, 3, 0) == 0); + auto* bytes = static_cast(mapping); + PageRuns runs; + Require(CollectRuns(mapping, 0, length / 2, runs) && runs == PageRuns{{0, length / small / 2}}); + bytes[5 * small] = 1; + Require(CollectFreshRuns(mapping, 0, length, runs) && runs == PageRuns{{length / small / 2, length / small}}); + Require(CollectFreshRuns(mapping, 0, length, runs) && runs.empty()); + bytes[200 * small] = 1; + Require(Written(mapping, length, {{5, 6}, {200, 201}})); + Require(sceKernelMunmap(mapping, length) == 0); +} + static void CheckWriteWatch() { if (!GuestWriteWatch::GuestWriteWatchAvailable_nid_postfix()) { std::puts("write watch unavailable: not tested"); @@ -975,15 +1313,16 @@ static void CheckDirectMemorySharedBacking() { static_cast(first)[page + 5] = 0x5a; int file = -1; std::uint64_t offset = 0; + const auto base = static_cast(phys); Require(GuestArena::GuestArenaSharedBacking_nid_postfix(address + page, page, &file, &offset)); - Require(offset == page * 2); + Require(offset == base + page * 2); unsigned char byte = 0; Require(pread(file, &byte, 1, static_cast(offset + 5)) == 1 && byte == 0x5a); const int seals = fcntl(file, F_GET_SEALS); Require(seals >= 0 && (seals & F_SEAL_SHRINK) != 0 && (seals & F_SEAL_GROW) != 0); Require((fcntl(file, F_GETFD) & FD_CLOEXEC) != 0); close(file); - Require(GuestArena::GuestArenaSharedBacking_nid_postfix(address, page * 2, &file, &offset) && offset == page); + Require(GuestArena::GuestArenaSharedBacking_nid_postfix(address, page * 2, &file, &offset) && offset == base + page); close(file); Require(GuestArena::GuestArenaSharedBacking_nid_postfix(address + page + 0x1000, 0x1000, &file, &offset) && offset == page * 2 + 0x1000); close(file); @@ -997,11 +1336,11 @@ static void CheckDirectMemorySharedBacking() { Require(sceKernelMapDirectMemory(&low, page, 3, 0x10, phys, 0) == 0); void* high = static_cast(flexible) + page; Require(sceKernelMapDirectMemory(&high, page, 3, 0x10, phys + page, 0) == 0); - Require(GuestArena::GuestArenaSharedBacking_nid_postfix(flexibleAddress, page * 2, &file, &offset) && offset == 0); + Require(GuestArena::GuestArenaSharedBacking_nid_postfix(flexibleAddress, page * 2, &file, &offset) && offset == base); close(file); Require(sceKernelMapDirectMemory(&high, page, 3, 0x10, phys + page * 2, 0) == 0); Require(!GuestArena::GuestArenaSharedBacking_nid_postfix(flexibleAddress, page * 2, &file, &offset)); - Require(GuestArena::GuestArenaSharedBacking_nid_postfix(flexibleAddress + page, page, &file, &offset) && offset == page * 2); + Require(GuestArena::GuestArenaSharedBacking_nid_postfix(flexibleAddress + page, page, &file, &offset) && offset == base + page * 2); close(file); Require(sceKernelMunmap(high, page) == 0); Require(!GuestArena::GuestArenaSharedBacking_nid_postfix(flexibleAddress + page, page, &file, &offset)); @@ -1009,6 +1348,38 @@ static void CheckDirectMemorySharedBacking() { Require(sceKernelMunmap(first, page * 2) == 0); Require(!GuestArena::GuestArenaSharedBacking_nid_postfix(address, page, &file, &offset)); Require(sceKernelReleaseDirectMemory(phys, page * 3) == 0); + + std::int64_t left = 0; + std::int64_t right = 0; + Require(sceKernelAllocateDirectMemory(0, 0x7fffffffffll, page, 0, 0, &left) == 0); + Require(sceKernelAllocateDirectMemory(0, 0x7fffffffffll, page, 0, 0, &right) == 0); + void* leftView = nullptr; + void* rightView = nullptr; + Require(sceKernelMapDirectMemory(&leftView, page, 3, 0, left, 0) == 0); + Require(sceKernelMapDirectMemory(&rightView, page, 3, 0, right, 0) == 0); + int leftFile = -1; + int rightFile = -1; + std::uint64_t leftOffset = 0; + std::uint64_t rightOffset = 0; + Require(GuestArena::GuestArenaSharedBacking_nid_postfix(reinterpret_cast(leftView), page, &leftFile, &leftOffset) && leftOffset == static_cast(left)); + Require(GuestArena::GuestArenaSharedBacking_nid_postfix(reinterpret_cast(rightView), page, &rightFile, &rightOffset) && rightOffset == static_cast(right)); + struct stat leftInfo {}; + struct stat rightInfo {}; + Require(fstat(leftFile, &leftInfo) == 0 && fstat(rightFile, &rightInfo) == 0 && leftInfo.st_ino == rightInfo.st_ino && leftInfo.st_dev == rightInfo.st_dev); + close(leftFile); + close(rightFile); + std::memset(leftView, 0x77, page); + Require(sceKernelMunmap(leftView, page) == 0); + Require(sceKernelReleaseDirectMemory(left, page) == 0); + std::int64_t again = 0; + Require(sceKernelAllocateDirectMemory(left, left + static_cast(page), page, 0, 0, &again) == 0 && again == left); + void* againView = nullptr; + Require(sceKernelMapDirectMemory(&againView, page, 3, 0, again, 0) == 0); + for (std::size_t i = 0; i < page; ++i) Require(static_cast(againView)[i] == 0); + Require(sceKernelMunmap(againView, page) == 0); + Require(sceKernelMunmap(rightView, page) == 0); + Require(sceKernelReleaseDirectMemory(again, page) == 0); + Require(sceKernelReleaseDirectMemory(right, page) == 0); } static void CheckDirectMemoryWriteWatch() { @@ -1062,8 +1433,88 @@ static void CheckDirectMemoryWriteWatch() { } #endif +static void CheckVirtualQuerySplitFlexibleRanges() { + constexpr std::size_t page = 0x4000; + void* mapped = nullptr; + Require(sceKernelMapFlexibleMemory(&mapped, page * 3, 3, 0) == 0); + auto* bytes = static_cast(mapped); + bytes[0] = 17; + bytes[page * 2] = 29; + Require(sceKernelMprotect(bytes + page, page, 1) == 0); + for (std::size_t index = 0; index < 3; ++index) { + VirtualQueryInfo info{}; + Require(sceKernelVirtualQuery(bytes + page * index, 0, &info, sizeof(info)) == 0); + Require(info.start == reinterpret_cast(bytes + page * index)); + Require(info.end == info.start + page); + Require(info.protection == (index == 1 ? 1 : 3)); + Require(info.is_committed && info.is_flexible && !info.is_direct); + } + Require(sceKernelMunmap(bytes + page, page) == 0); + for (std::size_t index : {std::size_t{0}, std::size_t{2}}) { + VirtualQueryInfo info{}; + Require(sceKernelVirtualQuery(bytes + page * index, 0, &info, sizeof(info)) == 0); + Require(info.start == reinterpret_cast(bytes + page * index)); + Require(info.end == info.start + page); + } + VirtualQueryInfo next{}; + Require(sceKernelVirtualQuery(bytes + page, 1, &next, sizeof(next)) == 0); + Require(next.start == reinterpret_cast(bytes + page * 2)); + Require(next.end == next.start + page && next.protection == 3); + Require(bytes[0] == 17 && bytes[page * 2] == 29); + Require(sceKernelMunmap(bytes, page) == 0); + Require(sceKernelMunmap(bytes + page * 2, page) == 0); +} + +static void CheckVirtualQueryForNonReadableGuestRange() { + constexpr std::size_t page = 0x4000; + auto* mapped = static_cast(mmap_nid_postfix(nullptr, page, 3, 0x1002, -1, 0)); + const auto failed = reinterpret_cast(static_cast(-1)); + Require(mapped != failed); + Require(mprotect_nid_postfix(mapped, page, 0) == 0); + VirtualQueryInfo info{}; + Require(sceKernelVirtualQuery(mapped, 0, &info, sizeof(info)) == 0); + Require(info.start == reinterpret_cast(mapped)); + Require(info.end == info.start + page && info.protection == 0); + Require(sceKernelMunmap(mapped, page) == 0); +} + +static void CheckVirtualQueryPartialMunmap() { + constexpr std::size_t page = 0x4000; + const auto failed = reinterpret_cast(static_cast(-1)); + void* raw = mmap_nid_postfix(nullptr, page * 3, 3, 0x1002, -1, 0); + Require(raw != failed); + auto* memory = static_cast(raw); + VirtualQueryInfo before{}; + Require(sceKernelVirtualQuery(memory, 0, &before, sizeof(before)) == 0); + Require(before.start == reinterpret_cast(memory)); + Require(before.end == reinterpret_cast(memory) + page * 3); + Require(munmap_nid_postfix(memory + page, page) == 0); + VirtualQueryInfo first{}; + Require(sceKernelVirtualQuery(memory, 0, &first, sizeof(first)) == 0); + Require(first.start == reinterpret_cast(memory)); + Require(first.end == reinterpret_cast(memory) + page); + VirtualQueryInfo middle{}; + Require(sceKernelVirtualQuery(memory + page, 0, &middle, sizeof(middle)) == SCE_KERNEL_ERROR_EACCES); + VirtualQueryInfo next{}; + Require(sceKernelVirtualQuery(memory + page, 1, &next, sizeof(next)) == 0); + Require(next.start == reinterpret_cast(memory) + page * 2); + Require(next.end == reinterpret_cast(memory) + page * 3); + VirtualQueryInfo tail{}; + Require(sceKernelVirtualQuery(memory + page * 2, 0, &tail, sizeof(tail)) == 0); + Require(tail.start == reinterpret_cast(memory) + page * 2); + Require(tail.end == reinterpret_cast(memory) + page * 3); + Require(munmap_nid_postfix(memory, page) == 0); + Require(munmap_nid_postfix(memory + page * 2, page) == 0); +} + int main() { +#ifdef _WIN32 + _putenv_s("APS5_PIN_WAIT_MS", "1000"); +#endif + CheckVirtualQuerySplitFlexibleRanges(); + CheckVirtualQueryForNonReadableGuestRange(); CheckReleaseFlexibleMemory(); + CheckVirtualQueryPartialMunmap(); CheckNamedAndHintedMappings(); CheckInternalNamedFlexibleMapping(); CheckBatchMapStopsAtInvalidEntry(); @@ -1071,8 +1522,10 @@ int main() { CheckAudioCoprocessorProtection(); CheckDirectMemoryFollowsPhysicalPages(); CheckReleaseDirectMemoryClearsMappings(); + CheckReleaseDirectMemoryRejectsInvalidRanges(); CheckFixedMappingReplacesPartialOverlap(); CheckDirectMemoryGpuProtBits(); + CheckGpuAccessFollowsProtection(); CheckFixedVirtualReservation(); CheckReservedRangeIsNotCommitted(); CheckNoOverwriteRefusesLiveMapping(); @@ -1080,16 +1533,28 @@ int main() { CheckSharedDirectMemoryLifecycle(); CheckGetDirectMemoryType(); CheckMtypeprotect(); + CheckHeapAlignment(); CheckHeapAfterMappingReuse(); -#ifdef _WIN32 CheckNoOverwriteRejectsHostOccupiedMapping(); CheckFixedMappingsReachTheApplicationAreaEnd(); +#ifdef _WIN32 + CheckGuestModuleImageProtection(); #endif + CheckUnhintedKernelMappingsLandInTheWindow(); +#if defined(__linux__) + CheckUnmappedArenaRangeStaysReserved(); +#endif + CheckArenaPlacesUnhintedMappingsInsideTheWindow(); + CheckArenaReusesAFreedRange(); + CheckArenaHonoursHintsWithoutGoingBelowThem(); CheckSharedWriteTracking(); CheckReadsIntoSharedWriteTracking(); CheckPinnedSharedPages(); + CheckFailedCollectKeepsWrites(); #if defined(__linux__) CheckWriteWatch(); + CheckWriteWatchArmedCollect(); + CheckWriteWatchFreshCollect(); CheckDirectMemoryWriteWatch(); CheckDirectMemoryBackingNeedsNoFilesystem(); CheckDirectMemorySharedBacking(); @@ -1136,6 +1601,19 @@ int main() { GuestAllocations::Mutation mutation; Require(mutation.Find(memory).writable); } + for (int advice = 0; advice <= 9; ++advice) Require(madvise_nid_postfix(memory, page * 3, advice) == 0); + Require(memory[0] == 42 && memory[page * 2] == 73); + Require(madvise_nid_postfix(memory + 1, 0, 4) == 0); + Require(madvise_nid_postfix(memory, page, 11) == -1 && *__error_nid_postfix() == 22); + *__error_nid_postfix() = 0; + Require(madvise_nid_postfix(memory, page, -1) == -1 && *__error_nid_postfix() == 22); + *__error_nid_postfix() = 0; + Require(madvise_nid_postfix(memory, 0, 11) == -1 && *__error_nid_postfix() == 22); + *__error_nid_postfix() = 0; + Require(madvise_nid_postfix(memory, std::numeric_limits::max(), 0) == -1 && *__error_nid_postfix() == 22); + *__error_nid_postfix() = 0; + Require(madvise_nid_postfix(reinterpret_cast(0x800000000000), 1, 0) == -1 && *__error_nid_postfix() == 22); + Require(madvise_nid_postfix(reinterpret_cast(0x7fffffffc000), 0x4000, 4) == 0); Require(munmap_nid_postfix(memory + 1, page) == -1 && *__error_nid_postfix() == 22); Require(munmap_nid_postfix(memory, 0) == -1 && *__error_nid_postfix() == 22); Require(memory[0] == 42); @@ -1145,6 +1623,7 @@ int main() { Require(memory[page * 2] == 73); Require(munmap_nid_postfix(memory + page * 2, page) == 0); Require(munmap_nid_postfix(memory, page) == -1); + Require(madvise_nid_postfix(memory, page, 4) == 0); *__error_nid_postfix() = 0; Require(mprotect_nid_postfix(memory, page, 1) == -1 && *__error_nid_postfix() == 22); *__error_nid_postfix() = 0; @@ -1161,4 +1640,17 @@ int main() { } Require(munmap_nid_postfix(mapped, 1) == 0); } + + reject(page, 3, 0x2001002, -1, 0, 45); + constexpr std::size_t superpage = std::size_t{1} << 21; + auto* aligned = static_cast(mmap_nid_postfix(nullptr, superpage + page, 3, 0x1001002, -1, 0)); + Require(aligned != failed && (reinterpret_cast(aligned) & (superpage - 1)) == 0); + aligned[0] = 1; + aligned[superpage + page - 1] = 2; + { + GuestAllocations::Mutation mutation; + const auto range = mutation.Find(aligned); + Require(range.bytes == superpage + page && range.readable && range.writable); + } + Require(munmap_nid_postfix(aligned, superpage + page) == 0); } diff --git a/core/libs/tests/GuestMemoryModule.cpp b/core/libs/tests/GuestMemoryModule.cpp new file mode 100644 index 000000000..ed8642ed6 --- /dev/null +++ b/core/libs/tests/GuestMemoryModule.cpp @@ -0,0 +1 @@ +extern "C" unsigned char guestMemoryModuleData[0x10000] = {1}; diff --git a/core/libs/tests/GuestMessageDestructorAbi.S b/core/libs/tests/GuestMessageDestructorAbi.S new file mode 100644 index 000000000..57ac11e82 --- /dev/null +++ b/core/libs/tests/GuestMessageDestructorAbi.S @@ -0,0 +1,194 @@ +.cfi_sections .eh_frame +.text +.macro PROLOGUE + pushq %rbp + .cfi_def_cfa_offset 16 + .cfi_offset %rbp, -16 + movq %rsp, %rbp + .cfi_def_cfa_register %rbp + subq $16, %rsp +.endm +.macro WHAT + movq %rax, %rdi + movq (%rax), %r11 + xorl %ecx, %ecx + call *16(%r11) + movq %rax, %rdi + call FixtureCheckWhat +.endm +.globl GuestThrow +GuestThrow: + .cfi_startproc + PROLOGUE + movl $16, %edi + call __cxa_allocate_exception_nid_postfix + movq %rax, FixtureLastObject(%rip) + leaq FixtureVtable+16(%rip), %rdx + movq %rdx, (%rax) + leaq FixtureMessage(%rip), %rdx + movq %rdx, 8(%rax) + movq %rax, %rdi + leaq FixtureErrorType(%rip), %rsi + leaq FixtureDestroy(%rip), %rdx + call __cxa_throw_nid_postfix + ud2 + .cfi_endproc + +.globl GuestInner +GuestInner: + .cfi_startproc + .cfi_personality 0x00, __gxx_personality_v0_nid_postfix + .cfi_lsda 0x00, .Linner_lsda + PROLOGUE +.Linner_call: + call GuestThrow +.Linner_call_end: + ud2 +.Linner_catch: + cmpq $1, %rdx + jne .Linner_bad + movq %rax, %rdi + call __cxa_begin_catch_nid_postfix + incl FixtureCaught(%rip) + WHAT +.Linner_rethrow: + call __cxa_rethrow_nid_postfix +.Linner_rethrow_end: + ud2 +.Linner_cleanup: + movq %rax, -8(%rbp) + call __cxa_end_catch_nid_postfix + incl FixtureGuardDestroyed(%rip) + movq -8(%rbp), %rdi + call _Unwind_Resume_nid_postfix + ud2 +.Linner_bad: + movl $31, %edi + call FixtureFail + ud2 + .cfi_endproc + +.globl GuestOuter +GuestOuter: + .cfi_startproc + .cfi_personality 0x00, __gxx_personality_v0_nid_postfix + .cfi_lsda 0x00, .Louter_lsda + PROLOGUE +.Louter_call: + call GuestInner +.Louter_call_end: + ud2 +.Louter_catch: + cmpq $2, %rdx + jne .Louter_bad + movq %rax, %rdi + call __cxa_begin_catch_nid_postfix + cmpq FixtureLastObject(%rip), %rax + jne .Louter_bad + incl FixtureCaught(%rip) + WHAT + call __cxa_end_catch_nid_postfix + xorl %eax, %eax + leave + .cfi_def_cfa %rsp, 8 + ret +.Louter_bad: + movl $32, %edi + call FixtureFail + ud2 + .cfi_endproc + +.globl GuestStdCapture +GuestStdCapture: + .cfi_startproc + .cfi_personality 0x00, __gxx_personality_v0_nid_postfix + .cfi_lsda 0x00, .Lstd_lsda + PROLOGUE + movq FixtureArgument(%rip), %rdi +.Lstd_call: + call *FixtureThrow(%rip) +.Lstd_call_end: + ud2 +.Lstd_catch: + cmpq $1, %rdx + jne .Lstd_bad + movq %rax, %rdi + call __cxa_begin_catch_nid_postfix + movq %rax, %rdi + call FixtureInspect + call __cxa_end_catch_nid_postfix + leave + .cfi_def_cfa %rsp, 8 + ret +.Lstd_bad: + movl $33, %edi + call FixtureFail + ud2 + .cfi_endproc + +.globl ProbeWhat +ProbeWhat: +.globl ProbeDestroy +ProbeDestroy: + .cfi_startproc + PROLOGUE + subq $32, %rsp + movq %rdi, %r11 + movq %rsi, %rdi + movq %rdx, %rcx + call *%r11 + leave + .cfi_def_cfa %rsp, 8 + ret + .cfi_endproc +.section .rdata,"dr" +.Linner_lsda: + .byte 0xff, 0x00 + .uleb128 .Linner_types_end-.Linner_types_offset +.Linner_types_offset: + .byte 0x01 + .uleb128 .Linner_calls_end-.Linner_calls +.Linner_calls: + .uleb128 .Linner_call-GuestInner + .uleb128 .Linner_call_end-.Linner_call + .uleb128 .Linner_catch-GuestInner + .uleb128 1 + .uleb128 .Linner_rethrow-GuestInner + .uleb128 .Linner_rethrow_end-.Linner_rethrow + .uleb128 .Linner_cleanup-GuestInner + .uleb128 0 +.Linner_calls_end: + .byte 1, 0 + .quad FixtureErrorType +.Linner_types_end: +.Louter_lsda: + .byte 0xff, 0x00 + .uleb128 .Louter_types_end-.Louter_types_offset +.Louter_types_offset: + .byte 0x01 + .uleb128 .Louter_calls_end-.Louter_calls +.Louter_calls: + .uleb128 .Louter_call-GuestOuter + .uleb128 .Louter_call_end-.Louter_call + .uleb128 .Louter_catch-GuestOuter + .uleb128 1 +.Louter_calls_end: + .byte 1, 1, 2, 0 + .quad FixtureBaseType + .quad FixtureOtherType +.Louter_types_end: +.Lstd_lsda: + .byte 0xff, 0x80 + .uleb128 .Lstd_types_end-.Lstd_types_offset +.Lstd_types_offset: + .byte 0x01 + .uleb128 .Lstd_calls_end-.Lstd_calls +.Lstd_calls: + .uleb128 .Lstd_call-GuestStdCapture + .uleb128 .Lstd_call_end-.Lstd_call + .uleb128 .Lstd_catch-GuestStdCapture + .uleb128 1 +.Lstd_calls_end: + .byte 1, 0 + .quad FixtureStdOutType +.Lstd_types_end: diff --git a/core/libs/tests/GuestMessageDestructorAbi.cpp b/core/libs/tests/GuestMessageDestructorAbi.cpp new file mode 100644 index 000000000..edf1e8cd4 --- /dev/null +++ b/core/libs/tests/GuestMessageDestructorAbi.cpp @@ -0,0 +1,230 @@ +#include "prx/libc/include/exceptions/Runtime.hpp" +#include +#include +#include +#include +#include + +using GuestWhat = const char* (APS5_VABI *)(const void*); +using GuestDestroy = void (APS5_VABI *)(void*); +using HostDestroy = void (*)(void*); +using GuestThrowFunction = void (APS5_VABI *)(std::uintptr_t); +struct TypeRecord { const void* vtable; const char* name; const TypeRecord* base; }; +struct Object { const void* vtable; const char* message; }; +struct Table { std::ptrdiff_t offset; const TypeRecord* type; GuestDestroy destroy; GuestDestroy deleteObject; GuestWhat what; }; +struct Message { std::size_t length; std::size_t capacity; std::atomic references; }; +static_assert(sizeof(Object) == 16); + +extern "C" { +extern const unsigned char _ZTVN10__cxxabiv117__class_type_infoE_nid_postfix[]; +extern const unsigned char _ZTVN10__cxxabiv120__si_class_type_infoE_nid_postfix[]; +extern const unsigned char _ZTISt12out_of_range_nid_postfix[]; +extern const unsigned char _ZTISt11regex_error_nid_postfix[]; +void APS5_VABI _ZNSt12out_of_rangeC1EPKc_nid_postfix(Object*, const char*); +void APS5_VABI _ZNSt12out_of_rangeC1ERKS__nid_postfix(Object*, const Object*); +void APS5_VABI _ZNSt12out_of_rangeD1Ev_nid_postfix(Object*); +[[noreturn]] void APS5_VABI _ZSt14_Xout_of_rangePKc_nid_postfix(const char*); +[[noreturn]] void APS5_VABI _ZSt13_Xregex_errorNSt15regex_constants10error_typeE_nid_postfix(std::regex_constants::error_type); +int APS5_VABI GuestOuter(); +void APS5_VABI GuestStdCapture(); +void APS5_VABI ProbeDestroy(GuestDestroy, void*, void*); +const char* APS5_VABI ProbeWhat(GuestWhat, const void*, const void*); +void APS5_VABI _ZNSt8bad_castC1Ev_nid_postfix(Object*); +void APS5_VABI _ZNSt9bad_allocC1Ev_nid_postfix(Object*); + +TypeRecord FixtureBaseType{_ZTVN10__cxxabiv117__class_type_infoE_nid_postfix + 16, "11FixtureBase", nullptr}; +TypeRecord FixtureErrorType{_ZTVN10__cxxabiv120__si_class_type_infoE_nid_postfix + 16, "12FixtureError", &FixtureBaseType}; +TypeRecord FixtureOtherType{_ZTVN10__cxxabiv117__class_type_infoE_nid_postfix + 16, "12FixtureOther", nullptr}; +const void* FixtureStdOutType = _ZTISt12out_of_range_nid_postfix; +GuestThrowFunction FixtureThrow = reinterpret_cast(_ZSt14_Xout_of_rangePKc_nid_postfix); +std::uintptr_t FixtureArgument; +extern const char FixtureMessage[] = "own System-V guest message"; +void* FixtureLastObject = nullptr; +unsigned FixtureDestroyed = 0; +unsigned FixtureGuardDestroyed = 0; +unsigned FixtureCaught = 0; +unsigned FixtureWhatChecked = 0; + +[[noreturn]] void APS5_VABI FixtureFail(unsigned code) { + std::fprintf(stderr, "guest exception failure %u\n", code); + ExitProcess(code); +} +const char* APS5_VABI FixtureWhat(const void* object) { return static_cast(object)->message; } +void APS5_VABI FixtureDestroy(void* object) { + if (object != FixtureLastObject) FixtureFail(11); + ++FixtureDestroyed; +} +Table FixtureVtable{0, &FixtureErrorType, FixtureDestroy, FixtureDestroy, FixtureWhat}; +void APS5_VABI FixtureCheckWhat(const char* message) { + if (!message || std::strcmp(message, FixtureMessage)) FixtureFail(12); + ++FixtureWhatChecked; +} +} + +static Object decoy{}, retained{}; +static GuestDestroy registeredDestructor; +static void* expectedObject; +static void* poison; +static unsigned callbacks; +static bool regularRelease; +static bool keepMessage; +static const char* expectedMessage = FixtureMessage; + +using NativeFree = void (*)(void*); +static NativeFree originalFree; +static std::uintptr_t* freeSlot; +static void* watchedObject; +static void* watchedMessage; +static void* foreignObject; +static void* foreignMessage; +static unsigned objectFrees, messageFrees; + +static void TrackedFree(void* pointer) { + if (pointer && (pointer == foreignObject || pointer == foreignMessage)) FixtureFail(50); + if (pointer && pointer == watchedObject && ++objectFrees != 1) FixtureFail(51); + if (pointer && pointer == watchedMessage && ++messageFrees != 1) FixtureFail(52); + originalFree(pointer); +} + +static void ReplaceFree(std::uintptr_t address) { + DWORD protection; + if (!VirtualProtect(freeSlot, sizeof(*freeSlot), PAGE_READWRITE, &protection)) FixtureFail(53); + *freeSlot = address; + DWORD ignored; + if (!VirtualProtect(freeSlot, sizeof(*freeSlot), protection, &ignored)) FixtureFail(54); +} + +static void TrackFree() { + auto* image = reinterpret_cast(GetModuleHandleW(L"libc.prx")); + if (!image) FixtureFail(55); + auto* dos = reinterpret_cast(image); + auto* nt = reinterpret_cast(image + dos->e_lfanew); + auto* imports = reinterpret_cast(image + + nt->OptionalHeader.DataDirectory[IMAGE_DIRECTORY_ENTRY_IMPORT].VirtualAddress); + for (; imports->Name; ++imports) { + if (!imports->OriginalFirstThunk) continue; + auto* names = reinterpret_cast(image + imports->OriginalFirstThunk); + auto* slots = reinterpret_cast(image + imports->FirstThunk); + for (; names->u1.AddressOfData; ++names, ++slots) { + if (IMAGE_SNAP_BY_ORDINAL64(names->u1.Ordinal)) continue; + auto* name = reinterpret_cast(image + names->u1.AddressOfData); + if (std::strcmp(name->Name, "free")) continue; + freeSlot = reinterpret_cast(&slots->u1.Function); + originalFree = reinterpret_cast(*freeSlot); + ReplaceFree(reinterpret_cast(TrackedFree)); + return; + } + } + FixtureFail(56); +} + +static int TestVtable(const char* mode) { + const bool plain = std::strstr(mode, "plain") != nullptr; + const bool deleting = std::strstr(mode, "delete") != nullptr; + const bool destroying = deleting || std::strstr(mode, "destroy") != nullptr; + auto* actual = static_cast(std::malloc(sizeof(Object))); + auto* other = static_cast(std::malloc(sizeof(Object))); + if (!actual || !other) FixtureFail(57); + actual->message = other->message = nullptr; + if (plain) { + _ZNSt9bad_allocC1Ev_nid_postfix(actual); + _ZNSt8bad_castC1Ev_nid_postfix(other); + } else { + _ZNSt12out_of_rangeC1EPKc_nid_postfix(actual, "actual virtual message"); + _ZNSt12out_of_rangeC1EPKc_nid_postfix(other, "unrelated virtual message"); + } + const Object originalOther = *other; + watchedObject = actual; + watchedMessage = plain ? nullptr : reinterpret_cast(const_cast(actual->message)) - 1; + foreignObject = other; + foreignMessage = plain ? nullptr : reinterpret_cast(const_cast(other->message)) - 1; + TrackFree(); + auto* unused = std::strstr(mode, "zero") ? nullptr : other; + GuestDestroy destroy, deleteObject; + GuestWhat what; + auto* slots = static_cast(actual->vtable); + std::memcpy(&destroy, slots, sizeof(destroy)); + std::memcpy(&deleteObject, slots + sizeof(void*), sizeof(deleteObject)); + std::memcpy(&what, slots + 2 * sizeof(void*), sizeof(what)); + if (destroying) { + ProbeDestroy(deleting ? deleteObject : destroy, actual, unused); + if (objectFrees != (deleting ? 1u : 0u) || messageFrees != (plain ? 0u : 1u)) FixtureFail(58); + if (!deleting && actual->message) FixtureFail(59); + } else { + const char* result = ProbeWhat(what, actual, unused); + if (!result || std::strcmp(result, plain ? "std::bad_alloc" : "actual virtual message") || + objectFrees || messageFrees) FixtureFail(60); + } + if (other->vtable != originalOther.vtable || other->message != originalOther.message || + (!plain && std::strcmp(other->message, "unrelated virtual message"))) FixtureFail(61); + ReplaceFree(reinterpret_cast(originalFree)); + if (!plain) { + if (!deleting) _ZNSt12out_of_rangeD1Ev_nid_postfix(actual); + _ZNSt12out_of_rangeD1Ev_nid_postfix(other); + } + if (!deleting) std::free(actual); + std::free(other); + return 0; +} + +static Message* MessageHeader(const Object& object) { + return reinterpret_cast(const_cast(object.message)) - 1; +} + +extern "C" void APS5_VABI ObserveDestroy(void* pointer) { + if (pointer != expectedObject || ++callbacks != 1) FixtureFail(40); + ProbeDestroy(registeredDestructor, pointer, poison); + if (static_cast(pointer)->message != nullptr || + !decoy.message || std::strcmp(decoy.message, "unrelated message") || + (keepMessage && MessageHeader(retained)->references.load() != 0)) FixtureFail(41); +} + +extern "C" void APS5_VABI FixtureInspect(void* pointer) { + auto* header = LibcException::FromObject(pointer); + if (header->_pad != 0 || !header->destructor) FixtureFail(42); + expectedObject = pointer; + GuestWhat what; + std::memcpy(&what, static_cast(static_cast(pointer)->vtable) + + 2 * sizeof(void*), sizeof(what)); + const char* message = ProbeWhat(what, pointer, poison); + if (!message || std::strcmp(message, expectedMessage)) FixtureFail(62); + if (keepMessage) { + _ZNSt12out_of_rangeC1ERKS__nid_postfix(&retained, static_cast(pointer)); + if (MessageHeader(retained)->references.load() != 1) FixtureFail(43); + } + registeredDestructor = reinterpret_cast(header->destructor); + if (!regularRelease) header->destructor = reinterpret_cast(ObserveDestroy); +} + +int main(int argc, char** argv) { + SetErrorMode(SEM_FAILCRITICALERRORS | SEM_NOGPFAULTERRORBOX); + if (argc != 2) return 2; + if (std::strncmp(argv[1], "vtable-", 7) == 0) return TestVtable(argv[1]); + if (std::strcmp(argv[1], "control") == 0) { + if (GuestOuter() != 0 || FixtureDestroyed != 1 || FixtureGuardDestroyed != 1 || + FixtureCaught != 2 || FixtureWhatChecked != 2) return 10; + return 0; + } + const bool regex = std::strstr(argv[1], "regex") != nullptr; + regularRelease = std::strcmp(argv[1], "release") == 0; + keepMessage = std::strstr(argv[1], "unshared") == nullptr; + FixtureArgument = reinterpret_cast(FixtureMessage); + if (regex) { + expectedMessage = "regular expression error"; + FixtureStdOutType = _ZTISt11regex_error_nid_postfix; + FixtureThrow = reinterpret_cast(_ZSt13_Xregex_errorNSt15regex_constants10error_typeE_nid_postfix); + FixtureArgument = static_cast(std::regex_constants::error_collate); + } + _ZNSt12out_of_rangeC1EPKc_nid_postfix(&decoy, "unrelated message"); + poison = std::strstr(argv[1], "zero") ? nullptr : &decoy; + GuestStdCapture(); + if (callbacks != (regularRelease ? 0u : 1u) || + (keepMessage && (MessageHeader(retained)->references.load() != 0 || + std::strcmp(retained.message, regex ? "regular expression error" : FixtureMessage))) || + !decoy.message || std::strcmp(decoy.message, "unrelated message")) return 44; + _ZNSt12out_of_rangeD1Ev_nid_postfix(&retained); + _ZNSt12out_of_rangeD1Ev_nid_postfix(&decoy); + if (retained.message || decoy.message) return 45; + std::puts("guest message callback, retained ownership and release: PASS"); + return 0; +} diff --git a/core/libs/tests/GuestMlock.cpp b/core/libs/tests/GuestMlock.cpp new file mode 100644 index 000000000..044cb6828 --- /dev/null +++ b/core/libs/tests/GuestMlock.cpp @@ -0,0 +1,118 @@ +#include "prx/libc/include/general/VabiMacros.hpp" +#include +#include +#include +#include +#ifdef _WIN32 +#ifndef NOMINMAX +#define NOMINMAX +#endif +#include +#endif +#if defined(__linux__) +#include +#include +#endif + +extern "C" { +int APS5_VABI sceKernelMapFlexibleMemory(void**, std::size_t, int, int); +int APS5_VABI sceKernelReserveVirtualRange(void**, std::size_t, int, std::size_t); +int APS5_VABI sceKernelMunmap(void*, std::size_t); +int APS5_VABI mlock_nid_postfix(const void*, std::size_t); +int APS5_VABI munlock_nid_postfix(const void*, std::size_t); +int* APS5_VABI __error_nid_postfix(); +} + +static void Require(bool value) { if (!value) std::abort(); } + +#if defined(__linux__) +static std::size_t LockedKilobytes() { + std::ifstream status("/proc/self/status"); + std::string line; + while (std::getline(status, line)) { + if (line.rfind("VmLck:", 0) == 0) return std::strtoull(line.c_str() + 6, nullptr, 10); + } + return 0; +} +#endif + +int main() { + constexpr int untouched = 12345; + constexpr std::size_t page = 0x4000; +#ifdef _WIN32 + constexpr std::size_t length = 0x400000; +#else + constexpr std::size_t length = 0x10000; +#endif + void* mapped = nullptr; + Require(sceKernelMapFlexibleMemory(&mapped, length, 3, 0) == 0); + auto* bytes = static_cast(mapped); + + *__error_nid_postfix() = untouched; + Require(mlock_nid_postfix(mapped, 0) == 0); + Require(*__error_nid_postfix() == untouched); +#if defined(__linux__) + const auto lockedBefore = LockedKilobytes(); +#endif + Require(mlock_nid_postfix(bytes + 1, length - page) == 0); +#if defined(__linux__) + Require(LockedKilobytes() - lockedBefore == length / 1024); +#endif + Require(mlock_nid_postfix(mapped, length) == 0); + Require(mlock_nid_postfix(mapped, length) == 0); + bytes[length - 1] = 7; + + *__error_nid_postfix() = untouched; + Require(munlock_nid_postfix(mapped, 0) == 0); + Require(*__error_nid_postfix() == untouched); + Require(munlock_nid_postfix(bytes + 1, length - page) == 0); +#if defined(__linux__) + Require(LockedKilobytes() == lockedBefore); +#elif defined(_WIN32) + Require(!VirtualUnlock(mapped, length) && GetLastError() == ERROR_NOT_LOCKED); +#endif + *__error_nid_postfix() = untouched; + Require(munlock_nid_postfix(mapped, length) == 0); + Require(*__error_nid_postfix() == untouched); + bytes[0] = 7; + Require(mlock_nid_postfix(mapped, page) == 0); +#if defined(__linux__) + Require(LockedKilobytes() - lockedBefore == page / 1024); +#endif + Require(munlock_nid_postfix(mapped, length) == 0); +#if defined(__linux__) + Require(LockedKilobytes() == lockedBefore); +#elif defined(_WIN32) + Require(!VirtualUnlock(mapped, page) && GetLastError() == ERROR_NOT_LOCKED); +#endif + Require(mlock_nid_postfix(mapped, page) == 0); + Require(munlock_nid_postfix(mapped, page) == 0); +#if defined(__linux__) + Require(LockedKilobytes() == lockedBefore); +#elif defined(_WIN32) + Require(!VirtualUnlock(mapped, page) && GetLastError() == ERROR_NOT_LOCKED); +#endif + + *__error_nid_postfix() = 0; + Require(mlock_nid_postfix(reinterpret_cast(std::numeric_limits::max() - page + 1), page * 2) == -1); + Require(*__error_nid_postfix() == 22); + *__error_nid_postfix() = 0; + Require(munlock_nid_postfix(reinterpret_cast(std::numeric_limits::max() - page + 1), page * 2) == -1); + Require(*__error_nid_postfix() == 22); + + void* reserved = nullptr; + Require(sceKernelReserveVirtualRange(&reserved, page, 0, 0) == 0); + *__error_nid_postfix() = 0; + Require(mlock_nid_postfix(reserved, page) == -1 && *__error_nid_postfix() == 12); + *__error_nid_postfix() = untouched; + Require(munlock_nid_postfix(reserved, page) == 0); + Require(*__error_nid_postfix() == untouched); + Require(sceKernelMunmap(reserved, page) == 0); + + Require(sceKernelMunmap(mapped, length) == 0); + *__error_nid_postfix() = 0; + Require(mlock_nid_postfix(mapped, page) == -1 && *__error_nid_postfix() == 12); + *__error_nid_postfix() = 0; + Require(munlock_nid_postfix(mapped, page) == -1 && *__error_nid_postfix() == 12); + return 0; +} diff --git a/core/libs/tests/GuestModuleInfo.cpp b/core/libs/tests/GuestModuleInfo.cpp index 5921931c9..9cfc631e0 100644 --- a/core/libs/tests/GuestModuleInfo.cpp +++ b/core/libs/tests/GuestModuleInfo.cpp @@ -1,15 +1,21 @@ #include "SceTypes.hpp" #include "prx/libkernel/KernelErrors.hpp" +#include +#include #include #include #include +#include #include #include +#include extern "C" { void* APS5_VABI dlopen_nid_postfix(const char*, int); void* APS5_VABI dlsym_nid_postfix(void*, const char*); int APS5_VABI dlclose_nid_postfix(void*); int APS5_VABI sceKernelGetModuleInfoFromAddr(std::uint64_t, int, ModuleInfoEx*); +int APS5_VABI sceKernelGetModuleList(KernelModule*, std::size_t, std::size_t*); +int APS5_VABI sceKernelGetModuleInfo(KernelModule, ModuleInfo*); } static void Require(bool value) { if (!value) std::abort(); } template @@ -45,13 +51,60 @@ int main(int argc, char** argv) { executable = (segment.prot & 5) == 5; } Require(executable); - Require(info.eh_frame_hdr_addr != 0 && info.eh_frame_hdr_size != 0 && info.eh_frame_addr != 0 && info.eh_frame_size != 0); + Require(info.eh_frame_addr != 0 && info.eh_frame_size != 0); +#ifndef _WIN32 + Require(info.eh_frame_hdr_addr != 0 && info.eh_frame_hdr_size != 0); +#endif Require(info.init_proc_addr == 0 || info.init_proc_addr >= info.segments[0].address); const auto self = Query(reinterpret_cast(&Require), 0); Require(self.id != info.id && Query(reinterpret_cast(&Query), 0).id == self.id); void* second = dlopen_nid_postfix(argv[1], 2); Require(second && Query(add, 0).id == info.id); Require(dlclose_nid_postfix(second) == 0); + std::vector handles(512, -1); + std::size_t count = 0; + Require(sceKernelGetModuleList(handles.data(), handles.size(), &count) == 0); + Require(count >= 2 && count < handles.size() && handles[count] == -1); + handles.resize(count); + Require(handles.front() == self.id); + const auto kernel = Query(reinterpret_cast(&sceKernelGetModuleList), 0).id; + Require(std::count(handles.begin(), handles.end(), kernel) == 1); + Require(std::count(handles.begin(), handles.end(), info.id) == 0); + auto sorted = handles; + std::sort(sorted.begin(), sorted.end()); + Require(std::adjacent_find(sorted.begin(), sorted.end()) == sorted.end()); + std::vector shorter(count, -1); + std::size_t untouched = 99; + Require(sceKernelGetModuleList(shorter.data(), count - 1, &untouched) == SCE_KERNEL_ERROR_ENOMEM); + Require(untouched == 99 && shorter.back() == -1 && std::equal(handles.begin(), handles.end() - 1, shorter.begin())); + Require(sceKernelGetModuleList(nullptr, 0, &untouched) == SCE_KERNEL_ERROR_EFAULT); + Require(sceKernelGetModuleList(handles.data(), handles.size(), nullptr) == SCE_KERNEL_ERROR_EFAULT); + const auto kernelEx = Query(reinterpret_cast(&sceKernelGetModuleInfo), 0); + ModuleInfo kernelInfo{}; + kernelInfo.st_size = sizeof(ModuleInfo); + Require(sceKernelGetModuleInfo(kernel, &kernelInfo) == 0); + Require(kernelInfo.st_size == sizeof(ModuleInfo) && std::string(kernelInfo.name) == kernelEx.name); + Require(kernelInfo.segment_count == kernelEx.segment_count && kernelInfo.segment_count > 0); + for (std::uint32_t i = 0; i < kernelInfo.segment_count; ++i) + Require(kernelInfo.segments[i].address == kernelEx.segments[i].address && kernelInfo.segments[i].size == kernelEx.segments[i].size && kernelInfo.segments[i].prot == kernelEx.segments[i].prot); + Require(std::all_of(std::begin(kernelInfo.fingerprint), std::end(kernelInfo.fingerprint), [](std::uint8_t byte) { return byte == 0; })); + for (const auto handle : handles) { + ModuleInfo listed{}; + listed.st_size = sizeof(ModuleInfo); + Require(sceKernelGetModuleInfo(handle, &listed) == 0 && listed.name[0] != '\0'); + if (handle == self.id) Require(std::string(listed.name) == self.name); + } + for (const std::uint64_t size : {0x158, 0x1a8, 0x1b0, 0x200, 0}) { + ModuleInfo sized{}; + sized.st_size = size; + sized.name[0] = 'x'; + Require(sceKernelGetModuleInfo(kernel, &sized) == SCE_KERNEL_ERROR_EINVAL && sized.st_size == size && sized.name[0] == 'x'); + } + Require(sceKernelGetModuleInfo(kernel, nullptr) == SCE_KERNEL_ERROR_EFAULT); + ModuleInfo unknown{}; + unknown.st_size = sizeof(ModuleInfo); + Require(sceKernelGetModuleInfo(info.id, &unknown) == SCE_KERNEL_ERROR_ESRCH); + Require(sceKernelGetModuleInfo(-1, &unknown) == SCE_KERNEL_ERROR_ESRCH && unknown.name[0] == '\0'); int local = 0; Query(&local, SCE_KERNEL_ERROR_ESRCH); Require(sceKernelGetModuleInfoFromAddr(reinterpret_cast(add), 2, nullptr) == SCE_KERNEL_ERROR_EFAULT); diff --git a/core/libs/tests/GuestMsync.cpp b/core/libs/tests/GuestMsync.cpp new file mode 100644 index 000000000..451dd4458 --- /dev/null +++ b/core/libs/tests/GuestMsync.cpp @@ -0,0 +1,85 @@ +#include "prx/libc/include/general/VabiMacros.hpp" +#include +#include +#include +#include +#include +#include + +extern "C" { +void* APS5_VABI mmap_nid_postfix(void*, std::size_t, int, int, int, std::int64_t) noexcept; +int APS5_VABI munmap_nid_postfix(void*, std::size_t) noexcept; +int APS5_VABI msync_nid_postfix(void*, std::size_t, int); +int APS5_VABI sceKernelReserveVirtualRange(void**, std::size_t, int, std::size_t); +int* APS5_VABI __error_nid_postfix(); +} + +namespace { +constexpr std::size_t Page = 0x4000; +constexpr int ReadWrite = 3; +constexpr int PrivateAnonymous = 0x1002; +constexpr int Sync = 0x0; +constexpr int Async = 0x1; +constexpr int Invalidate = 0x2; +constexpr int Invalid = 22; +constexpr int NoMemory = 12; + +void Require(bool condition, std::source_location location = std::source_location::current()) { + if (condition) return; + std::fprintf(stderr, "msync check failed at line %u\n", static_cast(location.line())); + std::abort(); +} + +bool Succeeds(void* address, std::size_t length, int flags) { + *__error_nid_postfix() = 77; + return msync_nid_postfix(address, length, flags) == 0 && *__error_nid_postfix() == 77; +} + +bool Fails(void* address, std::size_t length, int flags, int error) { + *__error_nid_postfix() = 0; + return msync_nid_postfix(address, length, flags) == -1 && *__error_nid_postfix() == error; +} + +bool Throws(void* address, std::size_t length, int flags) { + try { + msync_nid_postfix(address, length, flags); + } catch (const std::runtime_error&) { + return true; + } + return false; +} + +char* Map(std::size_t pages) { + void* mapped = mmap_nid_postfix(nullptr, pages * Page, ReadWrite, PrivateAnonymous, -1, 0); + Require(mapped != reinterpret_cast(static_cast(-1))); + return static_cast(mapped); +} +} + +int main() { + char* mapped = Map(3); + mapped[0] = 1; + Require(Succeeds(mapped, 3 * Page, Sync)); + Require(Succeeds(mapped, 3 * Page, Async)); + Require(Succeeds(mapped + 100, 10, Sync)); + Require(Succeeds(mapped + Page - 1, 2, Sync)); + Require(Succeeds(mapped + Page, 0, Sync)); + Require(Succeeds(mapped, Page, 0x4)); + + Require(Fails(mapped, Page, Async | Invalidate, Invalid)); + const auto top = std::numeric_limits::max() & ~(Page - 1); + Require(Fails(reinterpret_cast(top), 2 * Page, Sync, Invalid)); + Require(Throws(mapped, Page, Invalidate)); + + Require(munmap_nid_postfix(mapped, 3 * Page) == 0); + Require(Fails(mapped, Page, Sync, NoMemory)); + Require(Fails(mapped + Page, 3 * Page, Async, NoMemory)); + Require(Fails(mapped + Page, 0, Sync, NoMemory)); + Require(Fails(mapped, Page, Invalidate, NoMemory)); + + void* reserved = nullptr; + Require(sceKernelReserveVirtualRange(&reserved, 2 * Page, 0, 0) == 0); + Require(Succeeds(reserved, 2 * Page, Sync)); + Require(Succeeds(reserved, 0, Async)); + return 0; +} diff --git a/core/libs/tests/GuestMutexRaiseException.cpp b/core/libs/tests/GuestMutexRaiseException.cpp new file mode 100644 index 000000000..ecdd471b4 --- /dev/null +++ b/core/libs/tests/GuestMutexRaiseException.cpp @@ -0,0 +1,52 @@ +#include "SceTypes.hpp" +#include +#include +#include +#include +#include + +extern "C" { +int APS5_VABI scePthreadCreate(Pthread*, const PthreadAttr*, PthreadEntry, void*, const char*); +int APS5_VABI scePthreadJoin(Pthread, void**); +int APS5_VABI scePthreadMutexInit(PthreadMutex*, const PthreadMutexattr*, const char*); +int APS5_VABI scePthreadMutexDestroy(PthreadMutex*); +int APS5_VABI scePthreadMutexLock(PthreadMutex*); +int APS5_VABI scePthreadMutexUnlock(PthreadMutex*); +int APS5_VABI sceKernelInstallExceptionHandler(int, void*); +int APS5_VABI sceKernelRemoveExceptionHandler(int); +int APS5_VABI sceKernelRaiseException(Pthread, int); +} + +static constexpr int SigUsr1 = 30; +static void Require(bool value) { if (!value) std::abort(); } +static std::atomic deliveries{0}; +static void APS5_VABI Handler(int, void*) { deliveries.fetch_add(1); } + +struct Context { PthreadMutex mutex = nullptr; std::atomic started{false}; std::atomic acquired{false}; }; +static void* APS5_VABI Worker(void* opaque) { + auto& context = *static_cast(opaque); + context.started.store(true); + Require(scePthreadMutexLock(&context.mutex) == 0); + context.acquired.store(true); + Require(scePthreadMutexUnlock(&context.mutex) == 0); + return nullptr; +} + +int main() { + Require(sceKernelInstallExceptionHandler(SigUsr1, reinterpret_cast(&Handler)) == 0); + Context context; + Require(scePthreadMutexInit(&context.mutex, nullptr, nullptr) == 0); + Require(scePthreadMutexLock(&context.mutex) == 0); + Pthread worker = nullptr; + Require(scePthreadCreate(&worker, nullptr, Worker, &context, "mutex wait") == 0); + while (!context.started.load()) std::this_thread::yield(); + std::this_thread::sleep_for(std::chrono::milliseconds(10)); + Require(sceKernelRaiseException(worker, SigUsr1) == 0); + for (int i = 0; i < 5000 && deliveries.load() == 0; ++i) std::this_thread::sleep_for(std::chrono::milliseconds(1)); + Require(deliveries.load() == 1); + Require(scePthreadMutexUnlock(&context.mutex) == 0); + Require(scePthreadJoin(worker, nullptr) == 0); + Require(context.acquired.load()); + Require(scePthreadMutexDestroy(&context.mutex) == 0); + Require(sceKernelRemoveExceptionHandler(SigUsr1) == 0); +} \ No newline at end of file diff --git a/core/libs/tests/GuestNewHandler.cpp b/core/libs/tests/GuestNewHandler.cpp new file mode 100644 index 000000000..898358f9e --- /dev/null +++ b/core/libs/tests/GuestNewHandler.cpp @@ -0,0 +1,147 @@ +#include "prx/libc/include/ApplicationHeap.hpp" +#include "prx/libc/include/general/VabiMacros.hpp" +#include +#include +#include +#include +#include + +extern "C" { +using Handler = void (APS5_VABI*)(); +Handler APS5_VABI _ZSt15set_new_handlerPFvvE_nid_postfix(Handler); +Handler APS5_VABI _ZSt15get_new_handlerv_nid_postfix(); +void* APS5_VABI _Znwm_nid_postfix(std::size_t); +void* APS5_VABI _Znam_nid_postfix(std::size_t); +void* APS5_VABI _ZnwmRKSt9nothrow_t_nid_postfix(std::size_t, const void*) noexcept; +void* APS5_VABI _ZnamRKSt9nothrow_t_nid_postfix(std::size_t, const void*) noexcept; +} + +namespace { + +alignas(16) std::array storage{}; +unsigned failuresLeft = 0; +unsigned allocations = 0; +unsigned handlerCalls = 0; +std::size_t lastSize = 0; + +void Require(bool condition, int line) { + if (!condition) { + std::fprintf(stderr, "Guest new handler check failed at line %d\n", line); + std::abort(); + } +} + +#define REQUIRE(condition) Require((condition), __LINE__) + +void* APS5_VABI allocate(std::size_t bytes) { + ++allocations; + lastSize = bytes; + if (failuresLeft != 0) { + --failuresLeft; + return nullptr; + } + return storage.data(); +} + +void APS5_VABI release(void*) {} +void* APS5_VABI allocateZeroed(std::size_t count, std::size_t bytes) { return allocate(count * bytes); } +void* APS5_VABI reallocate(void*, std::size_t bytes) { return allocate(bytes); } +void* APS5_VABI align(std::size_t, std::size_t bytes) { return allocate(bytes); } +void* APS5_VABI realign(void*, std::size_t bytes, std::size_t) { return allocate(bytes); } +int APS5_VABI posixAlign(void** pointer, std::size_t, std::size_t bytes) { + *pointer = allocate(bytes); + return *pointer == nullptr ? 12 : 0; +} + +void APS5_VABI countingHandler() { ++handlerCalls; } +void APS5_VABI uninstallingHandler() { + if (++handlerCalls == 3) _ZSt15set_new_handlerPFvvE_nid_postfix(nullptr); +} +void APS5_VABI throwingHandler() { + ++handlerCalls; + throw std::bad_alloc(); +} +void APS5_VABI allocatingHandler() { + ++handlerCalls; + failuresLeft = 0; + REQUIRE(_Znwm_nid_postfix(8) == storage.data()); +} + +void Reset(unsigned failures) { + failuresLeft = failures; + allocations = 0; + handlerCalls = 0; +} + +template +bool Throws(TAction action) { + try { + action(); + } catch (const std::bad_alloc&) { + return true; + } + return false; +} + +} + +int main() { + std::array api{}; + api[0] = reinterpret_cast(&allocate); + api[1] = reinterpret_cast(&release); + api[2] = reinterpret_cast(&allocateZeroed); + api[3] = reinterpret_cast(&reallocate); + api[4] = reinterpret_cast(&align); + api[5] = reinterpret_cast(&realign); + api[6] = reinterpret_cast(&posixAlign); + ApplicationHeapRegister_nid_no_patch(api.data()); + + REQUIRE(_ZSt15get_new_handlerv_nid_postfix() == nullptr); + + Reset(1); + REQUIRE(Throws([] { _Znwm_nid_postfix(8); }) && handlerCalls == 0 && allocations == 1); + Reset(1); + REQUIRE(Throws([] { _Znam_nid_postfix(8); }) && handlerCalls == 0 && allocations == 1); + Reset(1); + REQUIRE(_ZnwmRKSt9nothrow_t_nid_postfix(8, nullptr) == nullptr && handlerCalls == 0); + Reset(1); + REQUIRE(_ZnamRKSt9nothrow_t_nid_postfix(8, nullptr) == nullptr && handlerCalls == 0); + + REQUIRE(_ZSt15set_new_handlerPFvvE_nid_postfix(&countingHandler) == nullptr); + REQUIRE(_ZSt15get_new_handlerv_nid_postfix() == &countingHandler); + Reset(2); + REQUIRE(_Znwm_nid_postfix(8) == storage.data() && handlerCalls == 2 && allocations == 3); + Reset(3); + REQUIRE(_Znam_nid_postfix(8) == storage.data() && handlerCalls == 3 && allocations == 4); + Reset(1); + REQUIRE(_ZnwmRKSt9nothrow_t_nid_postfix(8, nullptr) == storage.data() && handlerCalls == 1 && allocations == 2); + Reset(1); + REQUIRE(_ZnamRKSt9nothrow_t_nid_postfix(8, nullptr) == storage.data() && handlerCalls == 1 && allocations == 2); + Reset(0); + REQUIRE(_Znwm_nid_postfix(8) == storage.data() && handlerCalls == 0 && allocations == 1); + Reset(1); + REQUIRE(_Znwm_nid_postfix(0) == storage.data() && lastSize == 1 && handlerCalls == 1); + + REQUIRE(_ZSt15set_new_handlerPFvvE_nid_postfix(&uninstallingHandler) == &countingHandler); + Reset(100); + REQUIRE(Throws([] { _Znwm_nid_postfix(8); })); + REQUIRE(handlerCalls == 3 && allocations == 4 && _ZSt15get_new_handlerv_nid_postfix() == nullptr); + + REQUIRE(_ZSt15set_new_handlerPFvvE_nid_postfix(&throwingHandler) == nullptr); + Reset(1); + REQUIRE(Throws([] { _Znwm_nid_postfix(8); }) && handlerCalls == 1 && allocations == 1); + Reset(1); + REQUIRE(_ZnwmRKSt9nothrow_t_nid_postfix(8, nullptr) == nullptr && handlerCalls == 1 && allocations == 1); + Reset(1); + REQUIRE(_ZnamRKSt9nothrow_t_nid_postfix(8, nullptr) == nullptr && handlerCalls == 1 && allocations == 1); + + REQUIRE(_ZSt15set_new_handlerPFvvE_nid_postfix(&allocatingHandler) == &throwingHandler); + Reset(1); + REQUIRE(_Znwm_nid_postfix(16) == storage.data() && handlerCalls == 1); + + REQUIRE(_ZSt15set_new_handlerPFvvE_nid_postfix(nullptr) == &allocatingHandler); + REQUIRE(_ZSt15get_new_handlerv_nid_postfix() == nullptr); + + std::puts("Guest new handler checks passed"); + return 0; +} diff --git a/core/libs/tests/GuestNpReachability.cpp b/core/libs/tests/GuestNpReachability.cpp index 541aa0d3e..4e0c1f226 100644 --- a/core/libs/tests/GuestNpReachability.cpp +++ b/core/libs/tests/GuestNpReachability.cpp @@ -6,11 +6,13 @@ int APS5_VABI sceNpRegisterNpReachabilityStateCallback(void*, void*); int APS5_VABI sceNpUnregisterNpReachabilityStateCallback(); int APS5_VABI sceNpGetNpReachabilityState(int, uint32_t*); int APS5_VABI sceNpCheckCallback(); +int APS5_VABI sceNpCheckNpAvailability(int, const NpOnlineId*); } static constexpr int ErrorArgument = static_cast(0x80550003); static constexpr int ErrorAlreadyRegistered = static_cast(0x80550008); static constexpr int ErrorNotRegistered = static_cast(0x80550009); +static constexpr int ErrorUserNotFound = static_cast(0x80550007); static void Require(bool value) { if (!value) std::abort(); } static void APS5_VABI Callback(int, uint32_t, void*) { std::abort(); } @@ -29,4 +31,8 @@ int main() { Require(sceNpUnregisterNpReachabilityStateCallback() == ErrorNotRegistered); Require(sceNpRegisterNpReachabilityStateCallback(callback, nullptr) == 0); Require(sceNpUnregisterNpReachabilityStateCallback() == 0); + NpOnlineId onlineId{}; + onlineId.data[0] = 'p'; + Require(sceNpCheckNpAvailability(1, &onlineId) == ErrorUserNotFound); + Require(sceNpCheckNpAvailability(1, nullptr) == ErrorArgument); } diff --git a/core/libs/tests/GuestOfflineImports.cpp b/core/libs/tests/GuestOfflineImports.cpp index 99d3178f0..536bf001d 100644 --- a/core/libs/tests/GuestOfflineImports.cpp +++ b/core/libs/tests/GuestOfflineImports.cpp @@ -18,6 +18,7 @@ int APS5_VABI sceNpEntitlementAccessPollUnifiedEntitlementInfoList(); int APS5_VABI sceNpEntitlementAccessRequestServiceEntitlementInfoList(); int APS5_VABI sceNpEntitlementAccessPollServiceEntitlementInfoList(); int APS5_VABI sceRudpInit_nid_postfix(void*, int); +int APS5_VABI sceRudpActivate(); int APS5_VABI sceRudpGetStatus(void*, std::size_t); int APS5_VABI sceRudpTerminate(); } @@ -67,23 +68,18 @@ int main() { Require(sceNpEntitlementAccessRequestServiceEntitlementInfoList() == signedOut); Require(sceNpEntitlementAccessPollServiceEntitlementInfoList() == signedOut); + Require(sceRudpActivate() == 0); std::array status; status.fill(0x5a); const auto originalStatus = status; - auto unsupported = [](void* data, std::size_t size) { - try { - sceRudpGetStatus(data, size); - } catch (const std::runtime_error& error) { - return std::string_view(error.what()) == "sceRudpGetStatus not implemented"; - } - return false; - }; - Require(unsupported(status.data(), status.size())); + constexpr int rudpNotInitialized = static_cast(0x80770001); + Require(sceRudpGetStatus(status.data(), status.size()) == rudpNotInitialized); Require(status == originalStatus); Require(sceRudpInit_nid_postfix(nullptr, 0) == 0); - Require(unsupported(status.data(), status.size())); - Require(status == originalStatus); - Require(unsupported(nullptr, 0)); + Require(sceRudpActivate() == 0); + Require(sceRudpGetStatus(status.data(), status.size()) == 0); + for (unsigned char byte : status) Require(byte == 0); + Require(sceRudpGetStatus(nullptr, 0) == 0); Require(sceRudpTerminate() == 0); return 0; } diff --git a/core/libs/tests/GuestOpusDec.cpp b/core/libs/tests/GuestOpusDec.cpp new file mode 100644 index 000000000..ef1bf7304 --- /dev/null +++ b/core/libs/tests/GuestOpusDec.cpp @@ -0,0 +1,114 @@ +#include "prx/libc/include/general/VabiMacros.hpp" +#include +#include +#include +#include +#include +#include + +extern "C" { +int APS5_VABI sceOpusDecInitialize(std::uint32_t*); +int APS5_VABI sceOpusDecTerminate(std::uint32_t*); +int APS5_VABI sceOpusDecGetSize(int); +int APS5_VABI sceOpusDecCreateEx(std::uint32_t*, void*, int, int); +int APS5_VABI sceOpusDecDecode(void*, const std::uint8_t*, int, std::int16_t*, int); +int APS5_VABI sceOpusDecDestroy(void*); +} + +namespace { + +void Require(bool value) { if (!value) std::abort(); } + +template +void RequireThrows(TFunction function) { + try { function(); } catch (const std::runtime_error&) { return; } + std::abort(); +} + +const std::uint8_t OPUS_STEREO[] = { + 0x46, 0x00, 0x7C, 0x87, 0xFC, 0xB1, 0x1D, 0xC0, 0xE3, 0x07, 0xD5, 0x9C, 0x4D, 0x91, 0x62, 0x6B, 0x73, 0x86, 0x05, 0xE8, 0xDB, 0xC6, 0x23, 0x6D, + 0x5E, 0x6F, 0x73, 0xF8, 0xF2, 0x47, 0xD9, 0x5E, 0xEA, 0xB3, 0xD2, 0x74, 0x6C, 0xE0, 0xCE, 0xE2, 0x3C, 0xFA, 0x59, 0x4A, 0xBA, 0x8F, 0x76, 0x0F, + 0x15, 0xE1, 0x3E, 0x60, 0xDF, 0x80, 0xC5, 0x44, 0xB1, 0x1B, 0xEF, 0x43, 0x03, 0x4F, 0xF2, 0xDE, 0xE3, 0xAD, 0x8B, 0x20, 0xFF, 0x68, 0x0C, 0x0A, + 0x3F, 0x00, 0x7C, 0x87, 0xFD, 0x45, 0xBD, 0x12, 0x00, 0xA5, 0xB1, 0xA0, 0x0A, 0x62, 0x24, 0x6F, 0x63, 0x70, 0xA5, 0x35, 0x13, 0x03, 0x74, 0x0D, + 0x83, 0xC9, 0x74, 0x1B, 0x79, 0xED, 0xA3, 0x09, 0x83, 0xA5, 0x02, 0xC4, 0x60, 0x49, 0xC9, 0xB6, 0x9A, 0x60, 0xAD, 0x6C, 0x77, 0x84, 0xE5, 0xC2, + 0xCE, 0x2E, 0x71, 0x07, 0x7E, 0x40, 0xCB, 0xDC, 0x13, 0xDF, 0xCF, 0x84, 0x5D, 0x55, 0x3B, 0x6B, 0x6F, 0x44, 0x00, 0x7C, 0x88, 0x01, 0xE8, 0xA1, + 0x2A, 0x12, 0x7D, 0x5E, 0xF6, 0x74, 0x78, 0xB1, 0x27, 0x2B, 0x8C, 0xF0, 0x7F, 0x95, 0x51, 0x71, 0x28, 0x18, 0xEA, 0x66, 0xEB, 0xCA, 0xAC, 0xDF, + 0x6C, 0x9C, 0xFD, 0xED, 0x88, 0x8E, 0x66, 0xD7, 0xDA, 0x36, 0xD7, 0xEB, 0x5F, 0xA6, 0x05, 0x72, 0xDA, 0x52, 0x14, 0x7F, 0xF5, 0x84, 0x54, 0xA1, + 0xA9, 0xF1, 0xAA, 0xA8, 0x73, 0x9A, 0xCC, 0x91, 0x83, 0x92, 0xD0, 0x7F, 0x3B, 0x6B, 0x65, 0x43, 0x00, 0x7C, 0x88, 0x01, 0xE8, 0xA1, 0x2A, 0x12, + 0x7D, 0x5F, 0x04, 0xF8, 0xF5, 0x1F, 0xA0, 0x85, 0x89, 0xED, 0xBA, 0x7C, 0x5F, 0x63, 0x2A, 0x9E, 0x05, 0xE8, 0x63, 0xD1, 0x73, 0x0C, 0xAF, 0xC8, + 0xC9, 0x22, 0xF7, 0x11, 0x1D, 0x8B, 0xA9, 0x9A, 0x90, 0x77, 0xF2, 0x86, 0x49, 0x15, 0x6E, 0xD6, 0x34, 0x1F, 0x50, 0x15, 0xEC, 0x85, 0xC7, 0xF5, + 0xA7, 0xFA, 0x99, 0xF9, 0x47, 0x32, 0x07, 0xAF, 0x24, 0xBF, 0x14, 0xC5, +}; + +} // namespace + +int main() { + std::uint32_t context = 0, otherContext = 0; + Require(sceOpusDecInitialize(&context) == 0); + Require(sceOpusDecInitialize(&otherContext) == 0 && context != otherContext); + RequireThrows([&] { sceOpusDecInitialize(&context); }); + Require(sceOpusDecGetSize(1) == 640 && sceOpusDecGetSize(2) == 640); + std::array state{}, otherState{}, monoState{}; + Require(sceOpusDecCreateEx(&context, state.data(), 48000, 2) == 0); + Require(sceOpusDecCreateEx(&otherContext, otherState.data(), 48000, 2) == 0); + Require(sceOpusDecCreateEx(&context, monoState.data(), 48000, 1) == 0); + RequireThrows([&] { sceOpusDecTerminate(&context); }); + RequireThrows([&] { sceOpusDecCreateEx(&context, state.data(), 48000, 2); }); + RequireThrows([&] { sceOpusDecGetSize(3); }); + std::array unsupported{}; + for (int rate : {8000, 12000, 16000, 24000}) + RequireThrows([&] { sceOpusDecCreateEx(&context, unsupported.data(), rate, 2); }); + std::array pcm{}, reference{}; + pcm.fill(12345); + RequireThrows([&] { sceOpusDecDecode(state.data(), OPUS_STEREO + 2, 70, pcm.data() + 1, 3839); }); + Require(std::all_of(pcm.begin(), pcm.end(), [](auto value) { return value == 12345; })); + const std::uint8_t malformed[] = {0x7f}; + RequireThrows([&] { sceOpusDecDecode(state.data(), malformed, 1, pcm.data() + 1, 3840); }); + RequireThrows([&] { sceOpusDecDecode(state.data(), nullptr, 0, pcm.data() + 1, 3840); }); + Require(std::all_of(pcm.begin(), pcm.end(), [](auto value) { return value == 12345; })); + std::size_t offset = 0; + for (int frame = 0; frame < 3; ++frame) { + const int bytes = OPUS_STEREO[offset] | (OPUS_STEREO[offset + 1] << 8); + const auto* packet = OPUS_STEREO + offset + 2; + Require(sceOpusDecDecode(state.data(), packet, bytes, pcm.data() + 1, 3841) == 3840); + Require(pcm.front() == 12345 && pcm.back() == 12345); + Require(sceOpusDecDecode(otherState.data(), packet, bytes, reference.data() + 1, 3840) == 3840); + Require(std::equal(pcm.begin() + 1, pcm.end() - 1, reference.begin() + 1)); + if (frame == 1) { + Require(sceOpusDecCreateEx(&context, unsupported.data(), 48000, 2) == 0); + std::array fresh{}; + Require(sceOpusDecDecode(unsupported.data(), packet, bytes, fresh.data(), 3840) == 3840); + Require(!std::equal(fresh.begin(), fresh.end(), pcm.begin() + 1)); + Require(sceOpusDecDestroy(unsupported.data()) == 0); + } + if (frame > 0) { + int peak = 0; + for (auto it = pcm.begin() + 1; it != pcm.end() - 1; ++it) peak = std::max(peak, std::abs(static_cast(*it))); + Require(peak > 900 && peak < 1200); + } + std::array mono{}; + mono.front() = mono.back() = 12345; + Require(sceOpusDecDecode(monoState.data(), packet, bytes, mono.data() + 1, 1921) == 1920); + Require(mono.front() == 12345 && mono.back() == 12345); + Require(std::any_of(mono.begin() + 1, mono.end() - 1, [](auto value) { return value != 0; })); + offset += bytes + 2; + } + Require(sceOpusDecDestroy(state.data()) == 0); + RequireThrows([&] { sceOpusDecDestroy(state.data()); }); + RequireThrows([&] { sceOpusDecDecode(state.data(), OPUS_STEREO + 2, 70, pcm.data(), 3840); }); + Require(sceOpusDecCreateEx(&context, state.data(), 48000, 2) == 0); + Require(sceOpusDecDecode(state.data(), OPUS_STEREO + 2, 70, pcm.data(), 3840) == 3840); + std::array rejected; + rejected.fill(12345); + const std::uint8_t invalidFraming[] = {0x7d, 1}; + RequireThrows([&] { sceOpusDecDecode(state.data(), invalidFraming, 2, rejected.data() + 1, 7680); }); + Require(std::all_of(rejected.begin(), rejected.end(), [](auto value) { return value == 12345; })); + Require(sceOpusDecDestroy(state.data()) == 0); + Require(sceOpusDecDestroy(monoState.data()) == 0); + Require(sceOpusDecTerminate(&context) == 0); + RequireThrows([&] { sceOpusDecTerminate(&context); }); + RequireThrows([&] { sceOpusDecCreateEx(&context, state.data(), 48000, 2); }); + Require(sceOpusDecDecode(otherState.data(), OPUS_STEREO + offset + 2, OPUS_STEREO[offset], reference.data(), 3840) == 3840); + Require(sceOpusDecDestroy(otherState.data()) == 0); + Require(sceOpusDecTerminate(&otherContext) == 0); +} diff --git a/core/libs/tests/GuestPad.cpp b/core/libs/tests/GuestPad.cpp index e8696ac01..213c3cc88 100644 --- a/core/libs/tests/GuestPad.cpp +++ b/core/libs/tests/GuestPad.cpp @@ -19,6 +19,7 @@ int APS5_VABI scePadReadState(int, PadData*); int APS5_VABI scePadSetTiltCorrectionState(int, bool); int APS5_VABI scePadResetOrientation(int); int APS5_VABI scePadSetAngularVelocityDeadbandState(int, bool); +int APS5_VABI scePadSetAngularVelocityBiasCorrectionState(int, bool); int APS5_VABI scePadIsRemoteController(int, bool*); } @@ -112,4 +113,6 @@ int main() { Require(scePadSetVibrationTriggerEffectWeakWhileEmbeddedMicInUse(true) == 0); Require(scePadSetAngularVelocityDeadbandState(handle, false) == 0); Require(scePadSetAngularVelocityDeadbandState(handle + 1, false) == PAD_ERROR_INVALID_HANDLE); + Require(scePadSetAngularVelocityBiasCorrectionState(handle, false) == PAD_OK); + Require(scePadSetAngularVelocityBiasCorrectionState(handle + 1, false) == PAD_ERROR_INVALID_HANDLE); } diff --git a/core/libs/tests/GuestPngDec.cpp b/core/libs/tests/GuestPngDec.cpp index d75515f8c..6bb53f9f6 100644 --- a/core/libs/tests/GuestPngDec.cpp +++ b/core/libs/tests/GuestPngDec.cpp @@ -105,6 +105,13 @@ int main() { const PngDecParseParam parsePalette{PALETTE_TRNS, sizeof(PALETTE_TRNS), 0}; Require(scePngDecParseHeader(&parsePalette, &info) == 0); Require(info.color_space == 4 && info.bit_depth == 2 && info.image_flag == 2); + for (const std::uint32_t size : {69u, 71u}) { + const PngDecParseParam truncatedPalette{PALETTE_TRNS, size, 0}; + Require(scePngDecParseHeader(&truncatedPalette, &info) == 0); + Require(info.color_space == 4 && info.bit_depth == 2 && info.image_flag == 0); + } + const PngDecParseParam completeTransparency{PALETTE_TRNS, 73, 0}; + Require(scePngDecParseHeader(&completeTransparency, &info) == 0 && info.image_flag == 2); Require(scePngDecParseHeader(nullptr, &info) == invalidParam); Require(scePngDecParseHeader(&parse, nullptr) == invalidAddr); const PngDecParseParam parseNull{nullptr, 16, 0}; diff --git a/core/libs/tests/GuestPosixFilesystem.cpp b/core/libs/tests/GuestPosixFilesystem.cpp new file mode 100644 index 000000000..540ca69d5 --- /dev/null +++ b/core/libs/tests/GuestPosixFilesystem.cpp @@ -0,0 +1,140 @@ +#include "prx/libc/include/General.hpp" +#include "prx/libkernel/Socket/include/SocketRuntime.hpp" +#include +#include +#include +#include +#include +#include +#include +#include +#ifdef _WIN32 +#include +#include +#include +#else +#include +#include +#include +#endif + +extern "C" { +int APS5_VABI mkstemp_nid_postfix(char*); +int APS5_VABI isatty_nid_postfix(int); +int* APS5_VABI __error_nid_postfix(); +int APS5_VABI close_nid_postfix(int); +int APS5_VABI pipe_nid_postfix(int*); +int APS5_VABI socket_nid_postfix(int, int, int); +std::int64_t APS5_VABI read_nid_postfix(int, void*, std::size_t); +std::int64_t APS5_VABI write_nid_postfix(int, const void*, std::size_t); +std::int64_t APS5_VABI lseek_nid_postfix(int, std::int64_t, int); +} + +static void Require(bool condition, std::source_location location = std::source_location::current()) { + if (!condition) { + std::fprintf(stderr, "POSIX filesystem check failed at line %u (guest errno %d)\n", location.line(), *__error_nid_postfix()); + std::abort(); + } +} + +int main() { + std::random_device random; + const auto root = std::filesystem::temp_directory_path() / + ("anyps5-posix-test-" + std::to_string(random()) + "-" + std::to_string(random())); + Require(std::filesystem::create_directory(root)); + AddPathAlias_nid_no_patch("/anyps5-posix-test", root.string().c_str()); + const std::string prefix = "/anyps5-posix-test/file-"; + std::string pattern = prefix + "XXXXXXXXX"; + const int descriptor = mkstemp_nid_postfix(pattern.data()); + Require(descriptor >= 0); + Require(pattern.starts_with(prefix) && pattern.size() == prefix.size() + 9); + Require(pattern.substr(prefix.size()).find_first_not_of("0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz") == std::string::npos); + Require(std::filesystem::file_size(ResolvePath_nid_no_patch(pattern.c_str())) == 0); +#ifndef _WIN32 + struct stat status{}; + Require(::fstat(descriptor, &status) == 0 && (status.st_mode & 0777) == 0600); +#endif + const char payload[] = {'a', '\0', '\r', '\n', '\x1a', 'z'}; + Require(write_nid_postfix(descriptor, payload, sizeof(payload)) == sizeof(payload)); + Require(lseek_nid_postfix(descriptor, 0, SEEK_SET) == 0); + std::array received{}; + Require(read_nid_postfix(descriptor, received.data(), received.size()) == static_cast(received.size())); + Require(std::memcmp(payload, received.data(), sizeof(payload)) == 0); + Require(isatty_nid_postfix(descriptor) == 0 && *__error_nid_postfix() == 25); + Require(close_nid_postfix(descriptor) == 0); + Require(isatty_nid_postfix(descriptor) == 0 && *__error_nid_postfix() == 9); + std::string second = prefix + "XXXXXXXXX"; + const int other = mkstemp_nid_postfix(second.data()); + Require(other >= 0 && second != pattern); + Require(close_nid_postfix(other) == 0); + const std::string candidates = "0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz"; + for (const char character : candidates) { + std::ofstream stream(root / (std::string("occupied-") + character)); + stream << "preserved"; + Require(stream.good()); + } + std::string occupied = "/anyps5-posix-test/occupied-X"; + Require(mkstemp_nid_postfix(occupied.data()) == -1 && *__error_nid_postfix() == 17); + for (const char character : candidates) { + std::ifstream stream(root / (std::string("occupied-") + character)); + std::string contents; + stream >> contents; + Require(contents == "preserved"); + } + Require(std::filesystem::remove(root / "occupied-7")); + occupied = "/anyps5-posix-test/occupied-X"; + const int collision = mkstemp_nid_postfix(occupied.data()); + Require(collision >= 0 && occupied == "/anyps5-posix-test/occupied-7"); + Require(close_nid_postfix(collision) == 0); + std::string fixed = "/anyps5-posix-test/fixed"; + const int unchanged = mkstemp_nid_postfix(fixed.data()); + Require(unchanged >= 0 && fixed == "/anyps5-posix-test/fixed"); + Require(close_nid_postfix(unchanged) == 0); + Require(mkstemp_nid_postfix(fixed.data()) == -1 && *__error_nid_postfix() == 17); + std::string missing = "/anyps5-posix-test/missing/file-XXXXXX"; + Require(mkstemp_nid_postfix(missing.data()) == -1 && *__error_nid_postfix() == 2); + std::string notDirectory = fixed + "/file-XXXXXX"; + Require(mkstemp_nid_postfix(notDirectory.data()) == -1); + Require(*__error_nid_postfix() == 20 || *__error_nid_postfix() == 2); + std::string empty; + Require(mkstemp_nid_postfix(empty.data()) == -1 && *__error_nid_postfix() == 22); + Require(mkstemp_nid_postfix(nullptr) == -1 && *__error_nid_postfix() == 14); + std::string tooLong(1024, 'X'); + Require(mkstemp_nid_postfix(tooLong.data()) == -1 && *__error_nid_postfix() == 63); + Require(tooLong == std::string(1024, 'X')); + Require(isatty_nid_postfix(-1) == 0 && *__error_nid_postfix() == 9); + Require(isatty_nid_postfix(100000) == 0 && *__error_nid_postfix() == 9); + int pipes[2]{}; + Require(pipe_nid_postfix(pipes) == 0); + for (const int pipe : pipes) { + Require(isatty_nid_postfix(pipe) == 0 && *__error_nid_postfix() == 25); + Require(close_nid_postfix(pipe) == 0); + } + const int socket = socket_nid_postfix(2, 2, 0); + Require(socket >= GuestSockets::FirstDescriptor); + Require(isatty_nid_postfix(socket) == 0 && *__error_nid_postfix() == 25); + Require(close_nid_postfix(socket) == 0); + Require(isatty_nid_postfix(socket) == 0 && *__error_nid_postfix() == 9); +#ifdef _WIN32 + const int nullDevice = ::_open("NUL", _O_RDWR); + Require(nullDevice >= 0); + Require(isatty_nid_postfix(nullDevice) == 0 && *__error_nid_postfix() == 25); + Require(::_close(nullDevice) == 0); + const int console = ::_open("CONIN$", _O_RDONLY); + if (console >= 0) { + *__error_nid_postfix() = 123; + Require(isatty_nid_postfix(console) == 1 && *__error_nid_postfix() == 123); + Require(::_close(console) == 0); + } +#else + const int terminal = ::posix_openpt(O_RDWR | O_NOCTTY); + Require(terminal >= 0 && ::grantpt(terminal) == 0 && ::unlockpt(terminal) == 0); + const int slave = ::open(::ptsname(terminal), O_RDWR | O_NOCTTY); + Require(slave >= 0); + *__error_nid_postfix() = 123; + Require(isatty_nid_postfix(slave) == 1 && *__error_nid_postfix() == 123); + Require(::close(slave) == 0 && ::close(terminal) == 0); +#endif + RemovePathAlias_nid_no_patch("/anyps5-posix-test"); + Require(std::filesystem::remove_all(root) > 0); +} diff --git a/core/libs/tests/GuestProcessArguments.cpp b/core/libs/tests/GuestProcessArguments.cpp new file mode 100644 index 000000000..fa0f5584b --- /dev/null +++ b/core/libs/tests/GuestProcessArguments.cpp @@ -0,0 +1,29 @@ +#include "prx/libc/include/general/VabiMacros.hpp" +#include +#include +#include + +extern "C" int APS5_VABI getargc_nid_postfix(void); +extern "C" const char** APS5_VABI getargv_nid_postfix(void); + +static void Require(bool condition, const char* message) { + if (!condition) { + std::fprintf(stderr, "%s\n", message); + std::abort(); + } +} + +int main() { + const char* const expected[] = {"plain", "two words", "ü日"}; + constexpr int expectedCount = 1 + static_cast(sizeof(expected) / sizeof(expected[0])); + const int count = getargc_nid_postfix(); + const char** values = getargv_nid_postfix(); + Require(count == expectedCount, "getargc does not count every command-line argument"); + Require(values != nullptr && values[0] != nullptr && values[0][0] != '\0', "getargv has no program name"); + for (int index = 1; index < expectedCount; ++index) { + Require(values[index] != nullptr && std::strcmp(values[index], expected[index - 1]) == 0, + "getargv does not return a command-line argument as UTF-8"); + } + Require(values[count] == nullptr, "getargv is not terminated by a null pointer"); + return 0; +} diff --git a/core/libs/tests/GuestProcessTime.cpp b/core/libs/tests/GuestProcessTime.cpp new file mode 100644 index 000000000..b24cf8249 --- /dev/null +++ b/core/libs/tests/GuestProcessTime.cpp @@ -0,0 +1,24 @@ +#include "prx/libkernel/Time/include/Time.hpp" +#include +#include +#include +#include + +static void Require(bool condition, const char* message) { + if (!condition) { + std::fprintf(stderr, "%s\n", message); + std::abort(); + } +} + +int main(int argc, char** argv) { + constexpr std::uint64_t minuteMicros = 60ULL * 1000 * 1000; + if (argc > 1 && std::strcmp(argv[1], "counter") == 0) { + Require(sceKernelGetProcessTimeCounter() < minuteMicros * 1000, "first process time counter wrapped"); + return 0; + } + const std::uint64_t first = sceKernelGetProcessTime(); + Require(first < minuteMicros, "first process time wrapped"); + Require(sceKernelGetProcessTime() >= first, "process time went backwards"); + return 0; +} diff --git a/core/libs/tests/GuestPthreadCancel.cpp b/core/libs/tests/GuestPthreadCancel.cpp new file mode 100644 index 000000000..311fb8a5a --- /dev/null +++ b/core/libs/tests/GuestPthreadCancel.cpp @@ -0,0 +1,236 @@ +#include "SceTypes.hpp" +#include +#include +#include +#include +#include + +extern "C" { +int APS5_VABI scePthreadCreate(Pthread* thread, const PthreadAttr* attr, PthreadEntry entry, void* arg, const char* name); +int APS5_VABI scePthreadJoin(Pthread thread, void** retval); +void APS5_VABI scePthreadExit(void* retval); +int APS5_VABI scePthreadCancel(Pthread thread); +int APS5_VABI scePthreadSetcancelstate(int state, int* old_state); +int APS5_VABI scePthreadSetcanceltype(int type, int* old_type); +void APS5_VABI scePthreadTestcancel(); +int APS5_VABI scePthreadMutexInit(PthreadMutex* mutex, const PthreadMutexattr* attr, const char* name); +int APS5_VABI scePthreadMutexDestroy(PthreadMutex* mutex); +int APS5_VABI scePthreadMutexLock(PthreadMutex* mutex); +int APS5_VABI scePthreadMutexUnlock(PthreadMutex* mutex); +int APS5_VABI scePthreadCondInit(PthreadCond* cond, const PthreadCondattr* attr, const char* name); +int APS5_VABI scePthreadCondDestroy(PthreadCond* cond); +int APS5_VABI scePthreadCondWait(PthreadCond* cond, PthreadMutex* mutex); +int APS5_VABI scePthreadSemInit(PthreadSem* sem, int flag, unsigned int value, const char* name); +int APS5_VABI scePthreadSemDestroy(PthreadSem* sem); +int APS5_VABI scePthreadSemWait(PthreadSem* sem); +int APS5_VABI sceKernelUsleep_nid_postfix(KernelUseconds microseconds); +int APS5_VABI pthread_cancel_nid_postfix(Pthread thread); +int APS5_VABI pthread_setcanceltype_nid_postfix(int type, int* old_type); +int APS5_VABI sem_init_nid_postfix(void* sem, int pshared, unsigned int value); +int APS5_VABI sem_destroy_nid_postfix(void* sem); +int APS5_VABI sem_wait_nid_postfix(void* sem); +void APS5_VABI __pthread_cleanup_push_imp_nid_postfix(void (APS5_VABI* routine)(void*), void* argument, void* info); +void APS5_VABI __pthread_cleanup_pop_imp_nid_postfix(int execute); +} + +static constexpr int SCE_OK = 0; +static constexpr int SCE_KERNEL_ERROR_ESRCH = static_cast(0x80020003); +static constexpr int SCE_KERNEL_ERROR_EINVAL = static_cast(0x80020016); +static constexpr int GUEST_ESRCH = 3; +static constexpr int GUEST_EINVAL = 22; +static constexpr int CANCEL_ENABLE = 0; +static constexpr int CANCEL_DISABLE = 1; +static constexpr int CANCEL_DEFERRED = 0; +static constexpr int CANCEL_ASYNCHRONOUS = 2; +static void* const CANCELED = reinterpret_cast(std::uintptr_t{1}); + +static void Require(bool value) { if (!value) std::abort(); } + +struct Shared { + PthreadMutex mutex = nullptr; + PthreadCond cond = nullptr; + PthreadSem sem = nullptr; + void* posixSem = nullptr; + Pthread target = nullptr; + std::atomic ready{false}; + std::atomic release{false}; + std::atomic returned{false}; + std::atomic survived{false}; + std::atomic cleanups{0}; + int order[2] = {0, 0}; +}; + +static Shared shared; + +static void APS5_VABI UnlockMutex(void*) { + Require(scePthreadMutexUnlock(&shared.mutex) == SCE_OK); + shared.cleanups.fetch_add(1); +} + +static void APS5_VABI RecordOrder(void* arg) { + shared.order[shared.cleanups.fetch_add(1)] = static_cast(reinterpret_cast(arg)); +} + +static void* APS5_VABI CondWaiter(void*) { + std::uintptr_t info[8]; + Require(scePthreadMutexLock(&shared.mutex) == SCE_OK); + __pthread_cleanup_push_imp_nid_postfix(UnlockMutex, nullptr, info); + shared.ready.store(true); + for (;;) Require(scePthreadCondWait(&shared.cond, &shared.mutex) == SCE_OK); +} + +static void* APS5_VABI SemWaiter(void*) { + shared.ready.store(true); + scePthreadSemWait(&shared.sem); + shared.returned.store(true); + return nullptr; +} + +static void* APS5_VABI PosixSemWaiter(void*) { + shared.ready.store(true); + sem_wait_nid_postfix(&shared.posixSem); + shared.returned.store(true); + return nullptr; +} + +static void* APS5_VABI Idler(void*) { + while (!shared.release.load()) std::this_thread::sleep_for(std::chrono::milliseconds(1)); + return nullptr; +} + +static void* APS5_VABI Joiner(void*) { + shared.ready.store(true); + scePthreadJoin(shared.target, nullptr); + shared.returned.store(true); + return nullptr; +} + +static void* APS5_VABI Sleeper(void*) { + shared.ready.store(true); + for (;;) sceKernelUsleep_nid_postfix(1000); +} + +static void* APS5_VABI Disabled(void*) { + int old = -1; + Require(scePthreadSetcancelstate(CANCEL_DISABLE, &old) == SCE_OK && old == CANCEL_ENABLE); + shared.ready.store(true); + while (!shared.release.load()) { + sceKernelUsleep_nid_postfix(1000); + scePthreadTestcancel(); + } + shared.survived.store(true); + Require(scePthreadSetcancelstate(CANCEL_ENABLE, &old) == SCE_OK && old == CANCEL_DISABLE); + scePthreadTestcancel(); + shared.returned.store(true); + return nullptr; +} + +static void* APS5_VABI Asynchronous(void*) { + int old = -1; + Require(scePthreadSetcancelstate(CANCEL_DISABLE, nullptr) == SCE_OK); + Require(scePthreadSetcanceltype(CANCEL_ASYNCHRONOUS, &old) == SCE_OK && old == CANCEL_DEFERRED); + shared.ready.store(true); + while (!shared.release.load()) std::this_thread::sleep_for(std::chrono::milliseconds(1)); + scePthreadSetcancelstate(CANCEL_ENABLE, nullptr); + shared.returned.store(true); + return nullptr; +} + +static void* APS5_VABI Handlers(void*) { + std::uintptr_t first[8]; + std::uintptr_t second[8]; + __pthread_cleanup_push_imp_nid_postfix(RecordOrder, reinterpret_cast(std::intptr_t{1}), first); + __pthread_cleanup_push_imp_nid_postfix(RecordOrder, reinterpret_cast(std::intptr_t{2}), second); + __pthread_cleanup_pop_imp_nid_postfix(0); + __pthread_cleanup_push_imp_nid_postfix(RecordOrder, reinterpret_cast(std::intptr_t{3}), second); + __pthread_cleanup_pop_imp_nid_postfix(1); + scePthreadExit(reinterpret_cast(std::intptr_t{7})); + return nullptr; +} + +static Pthread Start(PthreadEntry entry) { + shared.ready.store(false); + shared.release.store(false); + shared.returned.store(false); + shared.cleanups.store(0); + Pthread thread = nullptr; + Require(scePthreadCreate(&thread, nullptr, entry, nullptr, nullptr) == SCE_OK); + while (!shared.ready.load()) std::this_thread::yield(); + std::this_thread::sleep_for(std::chrono::milliseconds(20)); + return thread; +} + +static void JoinCanceled(Pthread thread) { + void* result = nullptr; + Require(scePthreadJoin(thread, &result) == SCE_OK); + Require(result == CANCELED); + Require(!shared.returned.load()); +} + +int main() { + Require(scePthreadCancel(nullptr) == SCE_KERNEL_ERROR_ESRCH); + Require(pthread_cancel_nid_postfix(nullptr) == GUEST_ESRCH); + Require(scePthreadSetcancelstate(7, nullptr) == SCE_KERNEL_ERROR_EINVAL); + Require(scePthreadSetcanceltype(1, nullptr) == SCE_KERNEL_ERROR_EINVAL); + Require(pthread_setcanceltype_nid_postfix(1, nullptr) == GUEST_EINVAL); + + Require(scePthreadMutexInit(&shared.mutex, nullptr, nullptr) == SCE_OK); + Require(scePthreadCondInit(&shared.cond, nullptr, nullptr) == SCE_OK); + Require(scePthreadSemInit(&shared.sem, 0, 0, nullptr) == SCE_OK); + Require(sem_init_nid_postfix(&shared.posixSem, 0, 0) == 0); + + Pthread thread = Start(CondWaiter); + Require(scePthreadMutexLock(&shared.mutex) == SCE_OK); + Require(scePthreadCancel(thread) == SCE_OK); + Require(shared.cleanups.load() == 0); + Require(scePthreadMutexUnlock(&shared.mutex) == SCE_OK); + JoinCanceled(thread); + Require(shared.cleanups.load() == 1); + Require(scePthreadMutexLock(&shared.mutex) == SCE_OK); + Require(scePthreadMutexUnlock(&shared.mutex) == SCE_OK); + + thread = Start(SemWaiter); + Require(scePthreadCancel(thread) == SCE_OK); + JoinCanceled(thread); + + thread = Start(PosixSemWaiter); + Require(pthread_cancel_nid_postfix(thread) == 0); + JoinCanceled(thread); + + Require(scePthreadCreate(&shared.target, nullptr, Idler, nullptr, nullptr) == SCE_OK); + thread = Start(Joiner); + Require(scePthreadCancel(thread) == SCE_OK); + JoinCanceled(thread); + shared.release.store(true); + Require(scePthreadJoin(shared.target, nullptr) == SCE_OK); + + thread = Start(Sleeper); + Require(scePthreadCancel(thread) == SCE_OK); + JoinCanceled(thread); + + shared.survived.store(false); + thread = Start(Disabled); + Require(scePthreadCancel(thread) == SCE_OK); + std::this_thread::sleep_for(std::chrono::milliseconds(20)); + shared.release.store(true); + JoinCanceled(thread); + Require(shared.survived.load()); + + thread = Start(Asynchronous); + Require(scePthreadCancel(thread) == SCE_OK); + shared.release.store(true); + JoinCanceled(thread); + + shared.ready.store(true); + shared.cleanups.store(0); + void* result = nullptr; + Require(scePthreadCreate(&thread, nullptr, Handlers, nullptr, nullptr) == SCE_OK); + Require(scePthreadJoin(thread, &result) == SCE_OK); + Require(reinterpret_cast(result) == 7); + Require(shared.cleanups.load() == 2 && shared.order[0] == 3 && shared.order[1] == 1); + + Require(sem_destroy_nid_postfix(&shared.posixSem) == 0); + Require(scePthreadSemDestroy(&shared.sem) == SCE_OK); + Require(scePthreadCondDestroy(&shared.cond) == SCE_OK); + Require(scePthreadMutexDestroy(&shared.mutex) == SCE_OK); +} diff --git a/core/libs/tests/GuestPthreadGetname.cpp b/core/libs/tests/GuestPthreadGetname.cpp new file mode 100644 index 000000000..494309390 --- /dev/null +++ b/core/libs/tests/GuestPthreadGetname.cpp @@ -0,0 +1,51 @@ +#include "SceTypes.hpp" +#include +#include +#include +#include + +extern "C" { +Pthread APS5_VABI pthread_self_nid_postfix(void); +int APS5_VABI pthread_rename_np_nid_postfix(Pthread thread, const char* name); +int APS5_VABI pthread_getname_np_nid_postfix(Pthread thread, char* name); +int APS5_VABI scePthreadCreate(Pthread* thread, const PthreadAttr* attr, PthreadEntry entry, void* arg, const char* name); +int APS5_VABI scePthreadJoin(Pthread thread, void** retval); +} + +static constexpr int GUEST_ESRCH = 3; +static constexpr int GUEST_EFAULT = 14; + +static void Require(bool value) { if (!value) std::abort(); } + +static std::array Read(Pthread thread, int expected) { + std::array buffer{}; + buffer.fill(0xAA); + Require(pthread_getname_np_nid_postfix(thread, reinterpret_cast(buffer.data())) == expected); + return buffer; +} + +static void RequireName(Pthread thread, const std::string& name) { + const auto buffer = Read(thread, 0); + Require(std::memcmp(buffer.data(), name.c_str(), name.size() + 1) == 0); + for (std::size_t index = name.size() + 1; index < buffer.size(); ++index) Require(buffer[index] == 0xAA); +} + +static void* APS5_VABI Unnamed(void*) { + RequireName(pthread_self_nid_postfix(), ""); + return nullptr; +} + +int main() { + const Pthread self = pthread_self_nid_postfix(); + for (const char* name : {"", "A", "AnyPS5Probe", "0123456789012345678901234567890"}) { + Require(pthread_rename_np_nid_postfix(self, name) == 0); + RequireName(self, name); + } + Pthread worker = nullptr; + Require(scePthreadCreate(&worker, nullptr, Unnamed, nullptr, nullptr) == 0); + Require(scePthreadJoin(worker, nullptr) == 0); + + const auto untouched = Read(nullptr, GUEST_ESRCH); + for (unsigned char byte : untouched) Require(byte == 0xAA); + Require(pthread_getname_np_nid_postfix(self, nullptr) == GUEST_EFAULT); +} diff --git a/core/libs/tests/GuestPthreadRename.cpp b/core/libs/tests/GuestPthreadRename.cpp new file mode 100644 index 000000000..e3fdbd18b --- /dev/null +++ b/core/libs/tests/GuestPthreadRename.cpp @@ -0,0 +1,66 @@ +#include "SceTypes.hpp" +#include +#include +#include +#include +#include + +extern "C" { +int APS5_VABI scePthreadCreate(Pthread* thread, const PthreadAttr* attr, PthreadEntry entry, void* arg, const char* name); +int APS5_VABI scePthreadJoin(Pthread thread, void** retval); +Pthread APS5_VABI scePthreadSelf(); +int APS5_VABI scePthreadGetname(Pthread thread, char* name); +int APS5_VABI pthread_rename_np_nid_postfix(Pthread thread, const char* name); +int* APS5_VABI __error_nid_postfix(); +} + +static void Require(bool value) { if (!value) std::abort(); } + +static void CheckStoredName(Pthread thread, const std::string& expected) { + std::array buffer; + buffer.fill(0xaa); + Require(scePthreadGetname(thread, reinterpret_cast(buffer.data())) == 0); + Require(std::memcmp(buffer.data(), expected.c_str(), expected.size() + 1) == 0); + for (std::size_t i = expected.size() + 1; i < buffer.size(); ++i) Require(buffer[i] == 0xaa); +} + +static void CheckThreadRename(Pthread thread) { + const int savedError = *__error_nid_postfix(); + const std::string names[] = {"", "A", "AnyPS5Probe", "0123456789012345678901234567890"}; + for (const auto& name : names) { + *__error_nid_postfix() = 77; + Require(pthread_rename_np_nid_postfix(thread, name.c_str()) == 0); + Require(*__error_nid_postfix() == 77); + CheckStoredName(thread, name); + } + for (const std::size_t length : {32u, 40u, 1024u}) { + const std::string name(length, 'x'); + *__error_nid_postfix() = 77; + const int error = pthread_rename_np_nid_postfix(thread, name.c_str()); + if (error != 63) { + std::fprintf(stderr, "pthread_rename_np: %zu-byte name returned %d; expected ENAMETOOLONG=63\n", length, error); + std::abort(); + } + Require(*__error_nid_postfix() == 77); + CheckStoredName(thread, names[3]); + } + Require(pthread_rename_np_nid_postfix(thread, "renamed") == 0); + CheckStoredName(thread, "renamed"); + *__error_nid_postfix() = savedError; +} + +static void* APS5_VABI Worker(void*) { + const Pthread self = scePthreadSelf(); + Require(self != nullptr); + CheckThreadRename(self); + return nullptr; +} + +int main() { + const Pthread self = scePthreadSelf(); + Require(self != nullptr); + CheckThreadRename(self); + Pthread thread = nullptr; + Require(scePthreadCreate(&thread, nullptr, Worker, nullptr, "rename-test") == 0); + Require(scePthreadJoin(thread, nullptr) == 0); +} diff --git a/core/libs/tests/GuestPthreadRwlock.cpp b/core/libs/tests/GuestPthreadRwlock.cpp index 915d7d1ab..00f15cf97 100644 --- a/core/libs/tests/GuestPthreadRwlock.cpp +++ b/core/libs/tests/GuestPthreadRwlock.cpp @@ -75,7 +75,43 @@ static void ExpectTimeout(TimedLock lock, PthreadRwlock* rwlock) { Require(lock(rwlock, &negative) == GUEST_EINVAL); } +struct FirstLockRace { + PthreadRwlock rwlock = nullptr; + std::atomic ready{0}; + std::atomic inside{0}; + std::atomic overlaps{0}; +}; + +static void* APS5_VABI FirstWrlock(void* arg) { + auto& race = *static_cast(arg); + race.ready.fetch_add(1); + while (race.ready.load() < 2) {} + Require(pthread_rwlock_wrlock_nid_postfix(&race.rwlock) == 0); + if (race.inside.fetch_add(1) != 0) race.overlaps.fetch_add(1); + for (int spin = 0; spin < 2000; ++spin) std::atomic_signal_fence(std::memory_order_seq_cst); + race.inside.fetch_sub(1); + Require(pthread_rwlock_unlock_nid_postfix(&race.rwlock) == 0); + return nullptr; +} + +static void CheckConcurrentFirstWrlockExcludes() { + int overlaps = 0; + for (int round = 0; round < 500; ++round) { + FirstLockRace race; + Pthread first = nullptr; + Pthread second = nullptr; + Require(scePthreadCreate(&first, nullptr, FirstWrlock, &race, nullptr) == SCE_OK); + Require(scePthreadCreate(&second, nullptr, FirstWrlock, &race, nullptr) == SCE_OK); + Require(scePthreadJoin(first, nullptr) == SCE_OK); + Require(scePthreadJoin(second, nullptr) == SCE_OK); + overlaps += race.overlaps.load(); + Require(pthread_rwlock_destroy_nid_postfix(&race.rwlock) == 0); + } + Require(overlaps == 0); +} + int main() { + CheckConcurrentFirstWrlockExcludes(); const KernelTimespec invalid{0, NANOS_PER_SECOND}; PthreadRwlock fresh[4] = {}; Require(pthread_rwlock_tryrdlock_nid_postfix(&fresh[0]) == 0); diff --git a/core/libs/tests/GuestPthreadStack.cpp b/core/libs/tests/GuestPthreadStack.cpp new file mode 100644 index 000000000..21045925e --- /dev/null +++ b/core/libs/tests/GuestPthreadStack.cpp @@ -0,0 +1,79 @@ +#include "SceTypes.hpp" +#include +#include +#include + +extern "C" { +int APS5_VABI scePthreadCreate(Pthread* thread, const PthreadAttr* attr, PthreadEntry entry, void* arg, const char* name); +int APS5_VABI scePthreadJoin(Pthread thread, void** retval); +int APS5_VABI scePthreadAttrInit(PthreadAttr* attr); +int APS5_VABI scePthreadAttrDestroy(PthreadAttr* attr); +int APS5_VABI scePthreadAttrGet(Pthread thread, PthreadAttr* attr); +int APS5_VABI scePthreadAttrSetstacksize(PthreadAttr* attr, std::size_t stackSize); +int APS5_VABI scePthreadAttrGetstack(const PthreadAttr* attr, void** stackAddress, std::size_t* stackSize); +Pthread APS5_VABI scePthreadSelf(); +} + +static constexpr int SCE_OK = 0; +static constexpr std::size_t FRAME_SIZE = 4096; +static constexpr std::size_t FRAME_MARGIN = 2 * FRAME_SIZE; +static constexpr std::size_t HOST_CALL_STACK = 256 * 1024; +static constexpr std::size_t STACK_SIZES[] = {16384, 65536, 1u << 20, 16u << 20}; + +thread_local unsigned char threadLocalBlock[512 * 1024]; + +static void Require(bool value) { if (!value) std::abort(); } + +static std::uintptr_t Descend(std::uintptr_t floor); +static std::uintptr_t (*volatile descend)(std::uintptr_t) = Descend; + +static std::uintptr_t Descend(std::uintptr_t floor) { + volatile unsigned char frame[FRAME_SIZE]; + frame[0] = 0x5a; + frame[FRAME_SIZE - 1] = 0xa5; + const auto here = reinterpret_cast(&frame[0]); + const auto deepest = here < floor + FRAME_MARGIN ? here : descend(floor); + Require(frame[0] == 0x5a && frame[FRAME_SIZE - 1] == 0xa5); + return deepest; +} + +static void* APS5_VABI Worker(void* arg) { + const auto requested = *static_cast(arg); + threadLocalBlock[0] = 1; + threadLocalBlock[sizeof(threadLocalBlock) - 1] = 2; + + PthreadAttr attr = nullptr; + Require(scePthreadAttrInit(&attr) == SCE_OK); + Require(scePthreadAttrGet(scePthreadSelf(), &attr) == SCE_OK); + void* address = nullptr; + std::size_t size = 0; + Require(scePthreadAttrGetstack(&attr, &address, &size) == SCE_OK); + Require(scePthreadAttrDestroy(&attr) == SCE_OK); + Require(size == requested); + + const auto begin = reinterpret_cast(address); + const auto local = reinterpret_cast(&size); + Require(local > begin && local < begin + size); + Require(local - begin + FRAME_SIZE >= requested); + const auto deepest = Descend(begin); + Require(deepest >= begin && deepest < begin + FRAME_MARGIN); + const auto hostDeepest = descend(begin - HOST_CALL_STACK); + Require(hostDeepest < begin - HOST_CALL_STACK + FRAME_MARGIN); + + Require(threadLocalBlock[0] == 1 && threadLocalBlock[sizeof(threadLocalBlock) - 1] == 2); + return arg; +} + +int main() { + for (auto stackSize : STACK_SIZES) { + PthreadAttr attr = nullptr; + Require(scePthreadAttrInit(&attr) == SCE_OK); + Require(scePthreadAttrSetstacksize(&attr, stackSize) == SCE_OK); + Pthread thread = nullptr; + Require(scePthreadCreate(&thread, &attr, Worker, &stackSize, nullptr) == SCE_OK); + Require(scePthreadAttrDestroy(&attr) == SCE_OK); + void* result = nullptr; + Require(scePthreadJoin(thread, &result) == SCE_OK); + Require(result == &stackSize); + } +} diff --git a/core/libs/tests/GuestRaiseException.cpp b/core/libs/tests/GuestRaiseException.cpp index 7af034edf..59d2bc455 100644 --- a/core/libs/tests/GuestRaiseException.cpp +++ b/core/libs/tests/GuestRaiseException.cpp @@ -7,6 +7,7 @@ #include #include #include +#include extern "C" { int APS5_VABI sceKernelInstallExceptionHandler(int signum, void* handler); @@ -34,6 +35,8 @@ static std::atomic calls{0}; static std::atomic handlerThread; static std::atomic handlerRsp{0}; static std::atomic handlerFrame{0}; +static std::atomic handlerSelf{nullptr}; +static std::atomic clobberVectors{false}; static void APS5_VABI Handler(int signum, void* context) { Require(signum == SIGUSR1); @@ -43,6 +46,8 @@ static void APS5_VABI Handler(int signum, void* context) { handlerFrame.store(reinterpret_cast(&local)); handlerRsp.store(rsp); handlerThread.store(std::this_thread::get_id()); + handlerSelf.store(scePthreadSelf()); + if (clobberVectors.load()) asm volatile("vzeroall" ::: "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", "xmm6", "xmm7", "xmm8", "xmm9", "xmm10", "xmm11", "xmm12", "xmm13", "xmm14", "xmm15"); calls.fetch_add(1); } @@ -104,6 +109,111 @@ static void* APS5_VABI Leaving(void* arg) { return nullptr; } +static constexpr DWORD ContinuedCode = 0xe0000001u; +static constexpr int ContinuingRounds = 50; +static std::atomic continued{0}; + +static LONG CALLBACK ContinueRaised(EXCEPTION_POINTERS* info) { + if (info->ExceptionRecord->ExceptionCode != ContinuedCode) return EXCEPTION_CONTINUE_SEARCH; + continued.fetch_add(1); + return EXCEPTION_CONTINUE_EXECUTION; +} + +static constexpr int SelfingRounds = 100; +static std::atomic selfSink{nullptr}; + +static void* APS5_VABI Selfing(void* arg) { + auto& worker = *static_cast(arg); + worker.id = std::this_thread::get_id(); + worker.started.store(true); + while (!worker.stop.load()) selfSink.store(scePthreadSelf()); + return nullptr; +} + +static void* APS5_VABI Continuing(void* arg) { + auto& worker = *static_cast(arg); + worker.id = std::this_thread::get_id(); + worker.started.store(true); + while (!worker.stop.load()) RaiseException(ContinuedCode, 0, 0, nullptr); + return nullptr; +} + +using NtContinueFunction = LONG(NTAPI*)(CONTEXT*, BOOLEAN); +alignas(16) static CONTEXT loopContext; +alignas(16) static CONTEXT leaveContext; +static std::atomic loopingId{0}; + +static void* APS5_VABI Looping(void* arg) { + auto& worker = *static_cast(arg); + const auto ntContinue = reinterpret_cast(reinterpret_cast(GetProcAddress(GetModuleHandleW(L"ntdll.dll"), "NtContinue"))); + volatile bool left = false; + RtlCaptureContext(&leaveContext); + if (left) return nullptr; + left = true; + loopContext = leaveContext; + loopContext.Rip = reinterpret_cast(ntContinue); + loopContext.Rcx = reinterpret_cast(&loopContext); + loopContext.Rdx = 0; + loopingId.store(GetCurrentThreadId()); + worker.started.store(true); + ntContinue(&loopContext, FALSE); + return nullptr; +} + +static constexpr int VectorRounds = 100; +static constexpr std::size_t VectorBytes = 16 * 32; +alignas(32) static std::uint8_t vectorPattern[VectorBytes]; +alignas(32) static std::uint8_t vectorResult[VectorBytes]; +static volatile std::uint8_t vectorStop = 0; + +static void* APS5_VABI Vectors(void* arg) { + auto& worker = *static_cast(arg); + worker.id = std::this_thread::get_id(); + worker.started.store(true); + asm volatile( + "vmovdqu 0(%[pattern]), %%ymm0\n" + "vmovdqu 32(%[pattern]), %%ymm1\n" + "vmovdqu 64(%[pattern]), %%ymm2\n" + "vmovdqu 96(%[pattern]), %%ymm3\n" + "vmovdqu 128(%[pattern]), %%ymm4\n" + "vmovdqu 160(%[pattern]), %%ymm5\n" + "vmovdqu 192(%[pattern]), %%ymm6\n" + "vmovdqu 224(%[pattern]), %%ymm7\n" + "vmovdqu 256(%[pattern]), %%ymm8\n" + "vmovdqu 288(%[pattern]), %%ymm9\n" + "vmovdqu 320(%[pattern]), %%ymm10\n" + "vmovdqu 352(%[pattern]), %%ymm11\n" + "vmovdqu 384(%[pattern]), %%ymm12\n" + "vmovdqu 416(%[pattern]), %%ymm13\n" + "vmovdqu 448(%[pattern]), %%ymm14\n" + "vmovdqu 480(%[pattern]), %%ymm15\n" + "1:\n" + "pause\n" + "cmpb $0, (%[stop])\n" + "je 1b\n" + "vmovdqu %%ymm0, 0(%[result])\n" + "vmovdqu %%ymm1, 32(%[result])\n" + "vmovdqu %%ymm2, 64(%[result])\n" + "vmovdqu %%ymm3, 96(%[result])\n" + "vmovdqu %%ymm4, 128(%[result])\n" + "vmovdqu %%ymm5, 160(%[result])\n" + "vmovdqu %%ymm6, 192(%[result])\n" + "vmovdqu %%ymm7, 224(%[result])\n" + "vmovdqu %%ymm8, 256(%[result])\n" + "vmovdqu %%ymm9, 288(%[result])\n" + "vmovdqu %%ymm10, 320(%[result])\n" + "vmovdqu %%ymm11, 352(%[result])\n" + "vmovdqu %%ymm12, 384(%[result])\n" + "vmovdqu %%ymm13, 416(%[result])\n" + "vmovdqu %%ymm14, 448(%[result])\n" + "vmovdqu %%ymm15, 480(%[result])\n" + "vzeroupper\n" + : + : [pattern] "r"(vectorPattern), [result] "r"(vectorResult), [stop] "r"(&vectorStop) + : "memory", "cc", "xmm0", "xmm1", "xmm2", "xmm3", "xmm4", "xmm5", "xmm6", "xmm7", "xmm8", "xmm9", "xmm10", "xmm11", "xmm12", "xmm13", "xmm14", "xmm15"); + return nullptr; +} + static std::atomic finishedReturned{false}; static void* APS5_VABI Finished(void*) { @@ -281,6 +391,77 @@ int main() { Require(scePthreadJoin(leavingThread, nullptr) == 0); Require(sceKernelDeleteSema(leaving.sem) == 0); + void* const vectored = AddVectoredExceptionHandler(1, ContinueRaised); + Require(vectored != nullptr); + Worker continuing; + Pthread continuingThread = nullptr; + Require(scePthreadCreate(&continuingThread, nullptr, Continuing, &continuing, "continuing") == 0); + while (!continuing.started.load() || continued.load() == 0) std::this_thread::yield(); + for (int raised = 0; raised < ContinuingRounds; ++raised) { + Require(sceKernelRaiseException(continuingThread, SIGUSR1) == 0); + ExpectDelivery(1 + 2 * Repeats + HostRounds + LeavingRounds + raised, continuing.id); + } + continuing.stop.store(true); + Require(scePthreadJoin(continuingThread, nullptr) == 0); + Require(RemoveVectoredExceptionHandler(vectored) != 0); + + Worker selfing; + Pthread selfingThread = nullptr; + Require(scePthreadCreate(&selfingThread, nullptr, Selfing, &selfing, "selfing") == 0); + while (!selfing.started.load()) std::this_thread::yield(); + for (int raised = 0; raised < SelfingRounds; ++raised) { + Require(sceKernelRaiseException(selfingThread, SIGUSR1) == 0); + ExpectDelivery(1 + 2 * Repeats + HostRounds + LeavingRounds + ContinuingRounds + raised, selfing.id); + Require(handlerSelf.load() == selfingThread); + } + selfing.stop.store(true); + Require(scePthreadJoin(selfingThread, nullptr) == 0); + + Worker looping; + Pthread loopingThread = nullptr; + Require(scePthreadCreate(&loopingThread, nullptr, Looping, &looping, "looping") == 0); + while (!looping.started.load()) std::this_thread::yield(); + std::this_thread::sleep_for(std::chrono::milliseconds(20)); + const int beforeLooping = calls.load(); + const auto raisedAt = std::chrono::steady_clock::now(); + bool refused = false; + try { + sceKernelRaiseException(loopingThread, SIGUSR1); + } catch (const std::runtime_error&) { + refused = true; + } + const auto retried = std::chrono::steady_clock::now() - raisedAt; + Require(refused && retried >= std::chrono::milliseconds(900) && retried < std::chrono::seconds(10)); + Require(calls.load() == beforeLooping); + const HANDLE loopingHandle = OpenThread(THREAD_SUSPEND_RESUME | THREAD_GET_CONTEXT, FALSE, loopingId.load()); + Require(loopingHandle != nullptr); + Require(SuspendThread(loopingHandle) != static_cast(-1)); + CONTEXT stopped{}; + stopped.ContextFlags = CONTEXT_CONTROL; + Require(GetThreadContext(loopingHandle, &stopped) != 0); + loopContext = leaveContext; + Require(ResumeThread(loopingHandle) != static_cast(-1)); + CloseHandle(loopingHandle); + Require(scePthreadJoin(loopingThread, nullptr) == 0); + + if (__builtin_cpu_supports("avx")) { + for (std::size_t i = 0; i < VectorBytes; ++i) vectorPattern[i] = static_cast(i * 7 + 1); + clobberVectors.store(true); + Worker vectors; + Pthread vectorsThread = nullptr; + Require(scePthreadCreate(&vectorsThread, nullptr, Vectors, &vectors, "vectors") == 0); + while (!vectors.started.load()) std::this_thread::yield(); + std::this_thread::sleep_for(std::chrono::milliseconds(20)); + for (int raised = 0; raised < VectorRounds; ++raised) { + Require(sceKernelRaiseException(vectorsThread, SIGUSR1) == 0); + ExpectDelivery(1 + 2 * Repeats + HostRounds + LeavingRounds + ContinuingRounds + SelfingRounds + raised, vectors.id); + } + clobberVectors.store(false); + vectorStop = 1; + Require(scePthreadJoin(vectorsThread, nullptr) == 0); + Require(std::memcmp(vectorPattern, vectorResult, VectorBytes) == 0); + } + Pthread finishedThread = nullptr; Require(scePthreadCreate(&finishedThread, nullptr, Finished, nullptr, "finished") == 0); while (!finishedReturned.load()) std::this_thread::yield(); diff --git a/core/libs/tests/GuestRaiseNestedSleep.cpp b/core/libs/tests/GuestRaiseNestedSleep.cpp new file mode 100644 index 000000000..839f831ec --- /dev/null +++ b/core/libs/tests/GuestRaiseNestedSleep.cpp @@ -0,0 +1,70 @@ +#include "SceTypes.hpp" +#include +#include +#include +#include + +extern "C" { +int APS5_VABI sceKernelInstallExceptionHandler(int signum, void* handler); +int APS5_VABI sceKernelRemoveExceptionHandler(int signum); +int APS5_VABI sceKernelRaiseException(Pthread thread, int signum); +int APS5_VABI scePthreadCreate(Pthread* thread, const PthreadAttr* attr, PthreadEntry entry, void* arg, const char* name); +int APS5_VABI scePthreadJoin(Pthread thread, void** retval); +int APS5_VABI sceKernelUsleep_nid_postfix(KernelUseconds microseconds); +} + +static constexpr int SIGUSR1 = 30; +static constexpr int SCE_OK = 0; +static constexpr KernelUseconds OuterSleep = 1000000; +static constexpr KernelUseconds HandlerSleep = 20000; + +using Clock = std::chrono::steady_clock; + +static void Require(bool value) { if (!value) std::abort(); } + +static std::atomic handled{0}; +static std::atomic started{false}; +static std::atomic returned{false}; +static Clock::time_point sleepStart; +static Clock::time_point sleepEnd; + +static void APS5_VABI Handler(int signum, void*) { + Require(signum == SIGUSR1); + Require(sceKernelUsleep_nid_postfix(HandlerSleep) == SCE_OK); + handled.fetch_add(1); +} + +static void* APS5_VABI Sleeper(void*) { + sleepStart = Clock::now(); + started.store(true); + Require(sceKernelUsleep_nid_postfix(OuterSleep) == SCE_OK); + sleepEnd = Clock::now(); + returned.store(true); + return nullptr; +} + +template +static bool WaitFor(TCondition condition, std::chrono::milliseconds limit) { + const auto deadline = Clock::now() + limit; + while (!condition()) { + if (Clock::now() >= deadline) return false; + std::this_thread::sleep_for(std::chrono::milliseconds(1)); + } + return true; +} + +int main() { + Require(sceKernelInstallExceptionHandler(SIGUSR1, reinterpret_cast(&Handler)) == 0); + Pthread thread = nullptr; + Require(scePthreadCreate(&thread, nullptr, Sleeper, nullptr, "sleeper") == 0); + Require(WaitFor([] { return started.load(); }, std::chrono::milliseconds(5000))); + std::this_thread::sleep_for(std::chrono::milliseconds(50)); + Require(sceKernelRaiseException(thread, SIGUSR1) == 0); + Require(WaitFor([] { return handled.load() == 1; }, std::chrono::milliseconds(5000))); + Require(WaitFor([] { return returned.load(); }, std::chrono::milliseconds(5000))); + const auto slept = std::chrono::duration_cast(sleepEnd - sleepStart).count(); + Require(slept >= OuterSleep && slept < OuterSleep + 1000000); + Require(scePthreadJoin(thread, nullptr) == 0); + Require(sceKernelRemoveExceptionHandler(SIGUSR1) == 0); + return 0; +} diff --git a/core/libs/tests/GuestResolver.cpp b/core/libs/tests/GuestResolver.cpp index d65089c55..33bbed3d4 100644 --- a/core/libs/tests/GuestResolver.cpp +++ b/core/libs/tests/GuestResolver.cpp @@ -1,3 +1,9 @@ +#ifdef _WIN32 +#define WIN32_LEAN_AND_MEAN +#include +#else +#include +#endif #include "prx/libc/include/general/VabiMacros.hpp" #include "prx/libScePosixForWebKit/GuestResolver.hpp" #include @@ -6,8 +12,13 @@ #include #include #include +#include +#include +#include extern "C" { +GuestResolver::HostEntry* APS5_VABI gethostbyname_nid_postfix(const char*); +int* APS5_VABI sceNetErrnoLoc(); int APS5_VABI getaddrinfo_nid_postfix(const char*, const char*, const GuestResolver::AddressInfo*, GuestResolver::AddressInfo**); void APS5_VABI freeaddrinfo_nid_postfix(GuestResolver::AddressInfo*); int APS5_VABI getnameinfo_nid_postfix(const void*, std::uint32_t, char*, std::uint32_t, char*, std::uint32_t, int); @@ -20,6 +31,77 @@ void Require(bool condition, std::source_location location = std::source_locatio } } int main() { + auto* legacy = gethostbyname_nid_postfix("127.0.0.1"); + Require(legacy && legacy->name && std::strcmp(legacy->name, "127.0.0.1") == 0); + Require(legacy->family == 2 && legacy->length == 4); + const std::array loopback{127, 0, 0, 1}; + Require(legacy->addresses && legacy->addresses[0]); + Require(std::memcmp(legacy->addresses[0], loopback.data(), loopback.size()) == 0); + Require(legacy->addresses[1] == nullptr && legacy->aliases && legacy->aliases[0] == nullptr); + struct NumericCase { + const char* name; + std::array address; + }; + const NumericCase numericCases[]{ + {"127.1", {127, 0, 0, 1}}, + {"127.0.1", {127, 0, 0, 1}}, + {"0x7f.1", {127, 0, 0, 1}}, + {"0X7F.0x1", {127, 0, 0, 1}}, + {"0177.01", {127, 0, 0, 1}}, + {"0x7f000001", {127, 0, 0, 1}}, + {"2130706433", {127, 0, 0, 1}}, + {"127.1 trailing", {127, 0, 0, 1}}, + {"127.0.0.1\tignored", {127, 0, 0, 1}}, + {"4294967297", {0, 0, 0, 1}}, + {"18446744073709551617", {0, 0, 0, 1}}, + {"0x.1", {0, 0, 0, 1}}, + {"255.255.255.255", {255, 255, 255, 255}}, + {"0", {0, 0, 0, 0}}, + {"1.16777215", {1, 255, 255, 255}}, + {"1.2.65535", {1, 2, 255, 255}} + }; + for (const auto& test : numericCases) { + *sceNetErrnoLoc() = 77; + legacy = gethostbyname_nid_postfix(test.name); + Require(legacy && std::strcmp(legacy->name, test.name) == 0); + Require(legacy->family == 2 && legacy->length == 4); + Require(legacy->aliases && !legacy->aliases[0]); + Require(legacy->addresses && legacy->addresses[0] && !legacy->addresses[1]); + Require(std::memcmp(legacy->addresses[0], test.address.data(), test.address.size()) == 0); + Require(*sceNetErrnoLoc() == 0); + } + legacy = gethostbyname_nid_postfix("localhost"); + const auto* native = ::gethostbyname("localhost"); + Require(legacy && native && std::strcmp(legacy->name, native->h_name) == 0); + Require(legacy->family == 2 && legacy->length == native->h_length); + std::size_t index = 0; + while (native->h_aliases[index]) { + Require(legacy->aliases[index] && std::strcmp(legacy->aliases[index], native->h_aliases[index]) == 0); + ++index; + } + Require(legacy->aliases[index] == nullptr); + index = 0; + while (native->h_addr_list[index]) { + Require(legacy->addresses[index] && std::memcmp(legacy->addresses[index], native->h_addr_list[index], legacy->length) == 0); + ++index; + } + Require(legacy->addresses[index] == nullptr); + std::barrier ready(2); + std::thread worker([&] { + auto* entry = gethostbyname_nid_postfix("127.0.0.2"); + ready.arrive_and_wait(); + const std::array expected{127, 0, 0, 2}; + Require(entry && std::strcmp(entry->name, "127.0.0.2") == 0); + Require(std::memcmp(entry->addresses[0], expected.data(), expected.size()) == 0); + }); + legacy = gethostbyname_nid_postfix("127.0.0.1"); + ready.arrive_and_wait(); + Require(std::strcmp(legacy->name, "127.0.0.1") == 0); + Require(std::memcmp(legacy->addresses[0], loopback.data(), loopback.size()) == 0); + worker.join(); + *sceNetErrnoLoc() = 77; + Require(gethostbyname_nid_postfix(std::string(1024, 'a').c_str()) == nullptr); + Require(*sceNetErrnoLoc() >= 1 && *sceNetErrnoLoc() <= 4); std::array v4{16, 2, 0x6d, 0x06, 127, 0, 0, 1}; char host[128]{}, service[32]{}; Require(getnameinfo_nid_postfix(v4.data(), v4.size(), host, sizeof(host), service, sizeof(service), 10) == 0); diff --git a/core/libs/tests/GuestRtc.cpp b/core/libs/tests/GuestRtc.cpp index bb64f3ece..bf3528b2c 100644 --- a/core/libs/tests/GuestRtc.cpp +++ b/core/libs/tests/GuestRtc.cpp @@ -402,4 +402,29 @@ int main() { Require(sceRtcFormatRFC2822LocalTime(rfc2822, nullptr) == invalidPointer); Require(sceRtcFormatRFC3339LocalTime(nullptr, &tick) == invalidPointer); Require(sceRtcFormatRFC3339LocalTime(text, nullptr) == invalidPointer); + for (const char* zone : {"UTC0", "XXX-5:30", "XXX+3"}) { + SetTimeZone(zone); + for (std::uint64_t invalidTick : {maxTick + 1, std::uint64_t{1} << 63u, std::numeric_limits::max()}) { + source.tick = invalidTick; + result.tick = 123; + Require(sceRtcConvertUtcToLocalTime(&source, &result) == invalidValue && result.tick == 123); + Require(sceRtcConvertLocalTimeToUtc(&source, &result) == invalidValue && result.tick == 123); + Require(sceRtcConvertUtcToLocalTime(&source, &source) == invalidValue && source.tick == invalidTick); + Require(sceRtcConvertLocalTimeToUtc(&source, &source) == invalidValue && source.tick == invalidTick); + std::memset(text, 'x', sizeof(text)); + char originalText[sizeof(text)]; + std::memcpy(originalText, text, sizeof(text)); + Require(sceRtcFormatRFC3339LocalTime(text, &source) == invalidValue); + Require(std::memcmp(text, originalText, sizeof(text)) == 0); + std::memset(rfc2822, 'x', sizeof(rfc2822)); + char originalRfc2822[sizeof(rfc2822)]; + std::memcpy(originalRfc2822, rfc2822, sizeof(rfc2822)); + Require(sceRtcFormatRFC2822LocalTime(rfc2822, &source) == invalidValue); + Require(std::memcmp(rfc2822, originalRfc2822, sizeof(rfc2822)) == 0); + } + source.tick = leapDayTick; + Require(sceRtcConvertUtcToLocalTime(&source, &result) == 0); + Require(sceRtcConvertLocalTimeToUtc(&result, &result) == 0 && result.tick == source.tick); + } + SetTimeZone("UTC0"); } diff --git a/core/libs/tests/GuestRuneLocale.cpp b/core/libs/tests/GuestRuneLocale.cpp new file mode 100644 index 000000000..86d5de99b --- /dev/null +++ b/core/libs/tests/GuestRuneLocale.cpp @@ -0,0 +1,129 @@ +#include "prx/libc/include/general/VabiMacros.hpp" +#include +#include +#include +#include +#include + +extern "C" { +extern const unsigned char _DefaultRuneLocale_nid_postfix[4224]; +extern const unsigned char* _CurrentRuneLocale_nid_postfix; +extern int __mb_sb_limit_nid_postfix; +int APS5_VABI isalpha_nid_postfix(int); +int APS5_VABI iscntrl_nid_postfix(int); +int APS5_VABI isdigit_nid_postfix(int); +int APS5_VABI isgraph_nid_postfix(int); +int APS5_VABI islower_nid_postfix(int); +int APS5_VABI isprint_nid_postfix(int); +int APS5_VABI ispunct_nid_postfix(int); +int APS5_VABI isspace_nid_postfix(int); +int APS5_VABI isupper_nid_postfix(int); +int APS5_VABI isxdigit_nid_postfix(int); +int APS5_VABI isblank_nid_postfix(int); +int APS5_VABI tolower_nid_postfix(int); +int APS5_VABI toupper_nid_postfix(int); +int APS5_VABI iswalnum_nid_postfix(std::uint32_t); +int APS5_VABI iswalpha_nid_postfix(std::uint32_t); +int APS5_VABI iswblank_nid_postfix(std::uint32_t); +int APS5_VABI iswcntrl_nid_postfix(std::uint32_t); +int APS5_VABI iswdigit_nid_postfix(std::uint32_t); +int APS5_VABI iswgraph_nid_postfix(std::uint32_t); +int APS5_VABI iswlower_nid_postfix(std::uint32_t); +int APS5_VABI iswprint_nid_postfix(std::uint32_t); +int APS5_VABI iswpunct_nid_postfix(std::uint32_t); +int APS5_VABI iswspace_nid_postfix(std::uint32_t); +int APS5_VABI iswupper_nid_postfix(std::uint32_t); +int APS5_VABI iswxdigit_nid_postfix(std::uint32_t); +int APS5_VABI iswctype_nid_postfix(std::uint32_t, std::uint64_t); +std::uint64_t APS5_VABI wctype_nid_postfix(const char*); +std::uint32_t APS5_VABI towlower_nid_postfix(std::uint32_t); +std::uint32_t APS5_VABI towupper_nid_postfix(std::uint32_t); +std::uint32_t APS5_VABI btowc_nid_postfix(int); +int APS5_VABI wctob_nid_postfix(std::uint32_t); +} + +void Require(bool condition, std::source_location location = std::source_location::current()) { + if (!condition) { + std::fprintf(stderr, "Rune locale check failed at line %u\n", location.line()); + std::abort(); + } +} + +template +T Field(const unsigned char* locale, std::size_t offset) { + T value; + std::memcpy(&value, locale + offset, sizeof(value)); + return value; +} + +int main() { + const unsigned char* locale = _DefaultRuneLocale_nid_postfix; + Require(_CurrentRuneLocale_nid_postfix == locale); + Require(__mb_sb_limit_nid_postfix == 256); + Require(std::memcmp(locale, "RuneMagi", 8) == 0); + Require(std::strcmp(reinterpret_cast(locale + 8), "NONE") == 0); + Require(Field(locale, 40) == nullptr && Field(locale, 48) == nullptr); + Require(Field(locale, 56) == 0xfffd); + for (std::size_t offset = 4160; offset < 4224; ++offset) Require(locale[offset] == 0); + const struct { + std::uint64_t bit; + int (APS5_VABI* classify)(int); + } classes[] = { + {0x100, isalpha_nid_postfix}, {0x200, iscntrl_nid_postfix}, {0x400, isdigit_nid_postfix}, + {0x800, isgraph_nid_postfix}, {0x1000, islower_nid_postfix}, {0x2000, ispunct_nid_postfix}, + {0x4000, isspace_nid_postfix}, {0x8000, isupper_nid_postfix}, {0x10000, isxdigit_nid_postfix}, + {0x20000, isblank_nid_postfix}, {0x40000, isprint_nid_postfix}, + }; + for (int c = 0; c < 256; ++c) { + const auto type = Field(locale, 64 + 8 * c); + for (const auto& entry : classes) Require(((type & entry.bit) != 0) == (entry.classify(c) != 0)); + Require(((type & 0x400000) != 0) == (isdigit_nid_postfix(c) != 0)); + Require((type & ~std::uint64_t{0x47ffff}) == 0); + if (isxdigit_nid_postfix(c)) { + const int value = c <= '9' ? c - '0' : (c | 0x20) - 'a' + 10; + Require(static_cast(type & 0xff) == value); + } else { + Require((type & 0xff) == 0); + } + Require(Field(locale, 2112 + 4 * c) == tolower_nid_postfix(c)); + Require(Field(locale, 3136 + 4 * c) == toupper_nid_postfix(c)); + } + Require(Field(locale, 64 + 8 * 0x80) == 0 && Field(locale, 64 + 8 * 0xff) == 0); + + const struct { + const char* name; + std::uint64_t mask; + int (APS5_VABI* wide)(std::uint32_t); + int (APS5_VABI* narrow)(int); + } wide[] = { + {"alnum", 0x400100, iswalnum_nid_postfix, nullptr}, {"alpha", 0x100, iswalpha_nid_postfix, isalpha_nid_postfix}, + {"blank", 0x20000, iswblank_nid_postfix, isblank_nid_postfix}, {"cntrl", 0x200, iswcntrl_nid_postfix, iscntrl_nid_postfix}, + {"digit", 0x400, iswdigit_nid_postfix, isdigit_nid_postfix}, {"graph", 0x800, iswgraph_nid_postfix, isgraph_nid_postfix}, + {"lower", 0x1000, iswlower_nid_postfix, islower_nid_postfix}, {"print", 0x40000, iswprint_nid_postfix, isprint_nid_postfix}, + {"punct", 0x2000, iswpunct_nid_postfix, ispunct_nid_postfix}, {"space", 0x4000, iswspace_nid_postfix, isspace_nid_postfix}, + {"upper", 0x8000, iswupper_nid_postfix, isupper_nid_postfix}, {"xdigit", 0x10000, iswxdigit_nid_postfix, isxdigit_nid_postfix}, + }; + for (const auto& entry : wide) { + Require(wctype_nid_postfix(entry.name) == entry.mask); + for (std::uint32_t c = 0; c < 300; ++c) { + const int expected = c < 256 && (entry.narrow ? entry.narrow(static_cast(c)) : isalpha_nid_postfix(static_cast(c)) || isdigit_nid_postfix(static_cast(c))); + Require((entry.wide(c) != 0) == (expected != 0)); + Require((iswctype_nid_postfix(c, entry.mask) != 0) == (expected != 0)); + } + Require(entry.wide(0xffffffff) == 0 && iswctype_nid_postfix(0xffffffff, entry.mask) == 0); + } + Require(wctype_nid_postfix("ideogram") == 0x80000 && wctype_nid_postfix("special") == 0x100000); + Require(wctype_nid_postfix("phonogram") == 0x200000 && wctype_nid_postfix("number") == 0x400000); + Require(wctype_nid_postfix("rune") == 0xffffff00 && wctype_nid_postfix("") == 0 && wctype_nid_postfix("Alpha") == 0); + Require(iswctype_nid_postfix('A', 0) == 0 && iswctype_nid_postfix('7', wctype_nid_postfix("number")) != 0); + for (std::uint32_t c = 0; c < 300; ++c) { + Require(towlower_nid_postfix(c) == (c < 256 ? static_cast(tolower_nid_postfix(static_cast(c))) : c)); + Require(towupper_nid_postfix(c) == (c < 256 ? static_cast(toupper_nid_postfix(static_cast(c))) : c)); + } + Require(towlower_nid_postfix(0xffffffff) == 0xffffffff && towupper_nid_postfix(0xffffffff) == 0xffffffff); + Require(btowc_nid_postfix(-1) == 0xffffffff && btowc_nid_postfix(0) == 0); + Require(btowc_nid_postfix('A') == 'A' && btowc_nid_postfix(0xe9) == 0xe9 && btowc_nid_postfix(-23) == 0xe9); + Require(wctob_nid_postfix('A') == 'A' && wctob_nid_postfix(0xe9) == 0xe9 && wctob_nid_postfix(0) == 0); + Require(wctob_nid_postfix(0x100) == -1 && wctob_nid_postfix(0x20ac) == -1 && wctob_nid_postfix(0xffffffff) == -1); + return 0; +} diff --git a/core/libs/tests/GuestSandboxRandomWord.cpp b/core/libs/tests/GuestSandboxRandomWord.cpp new file mode 100644 index 000000000..15eb6eb24 --- /dev/null +++ b/core/libs/tests/GuestSandboxRandomWord.cpp @@ -0,0 +1,34 @@ +#include "prx/libc/include/general/VabiMacros.hpp" +#include +#include +#include +#include +#include + +extern "C" { +const char* APS5_VABI sceKernelGetFsSandboxRandomWord(); +int APS5_VABI open_nid_postfix(const char*, int, int); +int* APS5_VABI __error_nid_postfix(); +} + +static void Require(bool value) { if (!value) std::abort(); } + +int main() { + const char* word = sceKernelGetFsSandboxRandomWord(); + Require(word != nullptr); + const std::size_t length = std::strlen(word); + Require(length >= 1); + for (std::size_t index = 0; index < length; ++index) + Require(std::isalnum(static_cast(word[index]))); + + Require(sceKernelGetFsSandboxRandomWord() == word); + const char* fromThread = nullptr; + std::thread([&] { fromThread = sceKernelGetFsSandboxRandomWord(); }).join(); + Require(fromThread == word); + + const std::string path = std::string("/") + word + "/common/lib/libc.sprx"; + *__error_nid_postfix() = 0; + Require(open_nid_postfix(path.c_str(), 0, 0) == -1); + Require(*__error_nid_postfix() == 2); + return 0; +} diff --git a/core/libs/tests/GuestSaveDataCancel.cpp b/core/libs/tests/GuestSaveDataCancel.cpp new file mode 100644 index 000000000..9b8087c03 --- /dev/null +++ b/core/libs/tests/GuestSaveDataCancel.cpp @@ -0,0 +1,18 @@ +#include "prx/libc/include/general/VabiMacros.hpp" +#include +#include +#include + +extern "C" int APS5_VABI sceSaveDataCancel(); + +int main() { + try { + sceSaveDataCancel(); + } catch (const std::runtime_error& error) { + if (std::string(error.what()) == "sceSaveDataCancel not implemented") return 0; + std::fprintf(stderr, "sceSaveDataCancel threw: %s\n", error.what()); + return 1; + } + std::fprintf(stderr, "sceSaveDataCancel returned instead of reporting that it is not implemented\n"); + return 1; +} diff --git a/core/libs/tests/GuestSceNet.cpp b/core/libs/tests/GuestSceNet.cpp index 4432ef709..cce92d275 100644 --- a/core/libs/tests/GuestSceNet.cpp +++ b/core/libs/tests/GuestSceNet.cpp @@ -4,8 +4,10 @@ #include #include #include +#include #include #include +#include #include #include @@ -24,12 +26,14 @@ std::int64_t APS5_VABI sceNetRecv(int, void*, std::size_t, int); std::int64_t APS5_VABI sceNetSendto(int, const void*, std::size_t, int, const void*, std::uint32_t); std::int64_t APS5_VABI sceNetRecvfrom(int, void*, std::size_t, int, void*, std::uint32_t*); int APS5_VABI sceNetSocketClose(int); +int APS5_VABI sceNetShutdown(int, int); int APS5_VABI sceNetSetsockopt(int, int, int, const void*, std::uint32_t); int* APS5_VABI sceNetErrnoLoc(void); int APS5_VABI sceNetEpollCreate(const char*, int); int APS5_VABI sceNetEpollControl(int, int, int, const NetEpollEvent*); int APS5_VABI sceNetEpollWait(int, NetEpollEvent*, int, int); int APS5_VABI sceNetEpollDestroy(int); +int APS5_VABI sceNetEpollAbort(int, int); extern const std::uint32_t sce_net_in6addr_any[4]; int APS5_VABI sceNetResolverCreate(const char*, int, int); int APS5_VABI sceNetResolverStartNtoa(int, const char*, void*, int, int, int); @@ -92,6 +96,44 @@ static void CheckPoolStats() { Require(Failed(sceNetGetMemoryPoolStats(pool, &stats), 9)); } +static void CheckEmptyEpoll() { + const int epoll = sceNetEpollCreate("empty-epoll", 0); + Require(epoll >= 0); + NetEpollEvent event{}; + Require(sceNetEpollWait(epoll, &event, 1, 0) == 0); + std::array elapsed{}; + for (auto& sample : elapsed) { + const auto start = std::chrono::steady_clock::now(); + Require(sceNetEpollWait(epoll, &event, 1, 1000) == 0); + sample = std::chrono::steady_clock::now() - start; + Require(sample >= std::chrono::microseconds(1000)); + } +#ifdef _WIN32 + if (std::getenv("APS5_NO_TIMER_RESOLUTION") == nullptr) { + std::sort(elapsed.begin(), elapsed.end()); + Require(elapsed[elapsed.size() / 2] < std::chrono::milliseconds(8)); + } +#endif + Require(sceNetEpollAbort(epoll, 0) == 0); + Require(Failed(sceNetEpollWait(epoll, &event, 1, 1000), 53)); + Require(sceNetEpollWait(epoll, &event, 1, 0) == 0); + Require(sceNetEpollDestroy(epoll) == 0); + + for (const bool abort : {true, false}) { + const int waiting_epoll = sceNetEpollCreate("waiting-epoll", 0); + Require(waiting_epoll >= 0); + auto waiter = std::async(std::launch::async, [waiting_epoll, abort] { + NetEpollEvent ready{}; + Require(Failed(sceNetEpollWait(waiting_epoll, &ready, 1, -1), abort ? 53 : 9)); + }); + std::this_thread::sleep_for(std::chrono::milliseconds(20)); + Require((abort ? sceNetEpollAbort(waiting_epoll, 0) : sceNetEpollDestroy(waiting_epoll)) == 0); + Require(waiter.wait_for(std::chrono::seconds(1)) == std::future_status::ready); + waiter.get(); + if (abort) Require(sceNetEpollDestroy(waiting_epoll) == 0); + } +} + static void CheckMessages(int receiver, int sender, const std::array& address) { char head[] = "scatter "; char tail[] = "gather"; @@ -169,6 +211,8 @@ static void CheckAddressText(int family, const char* text) { Require(sceNetInetNtop(family, address.data(), nullptr, output.size()) == nullptr && *sceNetErrnoLoc() == 22); } +static bool ipv6Unavailable = false; + static void CheckUnspecifiedIpv6() { std::array unspecified{}; Require(sceNetInetPton(28, "::", unspecified.data()) == 1); @@ -178,6 +222,10 @@ static void CheckUnspecifiedIpv6() { Require(std::strcmp(text, "::") == 0 && text[3] == 'x'); const int receiver = sceNetSocket("ipv6-any", 28, 2, 17); + if (receiver < 0 && *sceNetErrnoLoc() == 47) { + ipv6Unavailable = true; + return; + } const int sender = sceNetSocket("ipv6-loopback", 28, 2, 17); Require(receiver >= 0 && sender >= 0); std::array address{28, 28}; @@ -210,6 +258,7 @@ int main() { } Require(sceNetInit_nid_postfix() == 0); CheckPoolStats(); + CheckEmptyEpoll(); CheckAddressText(2, "127.0.0.1"); CheckAddressText(2, "255.255.255.255"); CheckAddressText(28, "::1"); @@ -250,13 +299,17 @@ int main() { registration.events = 1; registration.ident = static_cast(accepted); registration.data.u32 = 77; + NetEpollEvent ready{}; + auto waiter = std::async(std::launch::async, [&] { return sceNetEpollWait(epoll, &ready, 1, -1); }); + std::this_thread::sleep_for(std::chrono::milliseconds(20)); Require(sceNetEpollControl(epoll, 1, accepted, ®istration) == 0); const char request[] = "guest tcp loopback"; char response[sizeof(request)]{}; Require(sceNetSend(client, request, sizeof(request), 0) == sizeof(request)); - NetEpollEvent ready{}; - const int ready_count = sceNetEpollWait(epoll, &ready, 1, 1000000); - Require(ready_count == 1 && (ready.events & 1) && ready.ident == static_cast(accepted)); + Require(waiter.wait_for(std::chrono::seconds(1)) == std::future_status::ready); + const int ready_count = waiter.get(); + Require(ready_count == 1 && (ready.events & 1) && ready.ident == static_cast(accepted) && ready.data.u32 == 77); + Require(sceNetEpollWait(epoll, &ready, 1, 1000000) == 1); Require(sceNetRecv(accepted, response, sizeof(response), 0) == sizeof(response)); Require(std::strcmp(request, response) == 0); char reply[] = "stream reply"; @@ -269,6 +322,11 @@ int main() { NetMsghdr reply_receive{reply_name.data(), 16, reply_in, 1, nullptr, 0, -1}; Require(sceNetRecvmsg(client, &reply_receive, 0) == sizeof(reply)); Require(std::strcmp(reply, reply_result) == 0 && reply_receive.flags == 0 && reply_receive.name_length == 0); + Require(sceNetSend(client, request, sizeof(request), 0) == sizeof(request)); + ready = {}; + Require(sceNetEpollWait(epoll, &ready, 1, 1000000) == 1 && (ready.events & 1) && ready.ident == static_cast(accepted)); + Require(sceNetShutdown(accepted, 1) == 0); + Require(Failed(sceNetSend(accepted, request, sizeof(request), 0), 32)); Require(sceNetEpollDestroy(epoll) == 0); Require(sceNetSocketClose(accepted) == 0); bool send_failed = false; @@ -371,4 +429,8 @@ int main() { int state = -1; Require(sceNetCtlGetState(&state) == 0); Require(state == 0 || state == 3); + if (ipv6Unavailable) { + std::puts("skipped the IPv6 socket checks, the host has no IPv6"); + return 77; + } } diff --git a/core/libs/tests/GuestSchedPriority.cpp b/core/libs/tests/GuestSchedPriority.cpp new file mode 100644 index 000000000..7f135f5ef --- /dev/null +++ b/core/libs/tests/GuestSchedPriority.cpp @@ -0,0 +1,19 @@ +#include "prx/libc/include/general/VabiMacros.hpp" +#include +#include +extern "C" { +int APS5_VABI sched_get_priority_max_nid_postfix(int policy); +int APS5_VABI sched_get_priority_min_nid_postfix(int policy); +} +static void Require(bool value) { if (!value) std::abort(); } +static bool Rejects(int policy) { + try { sched_get_priority_max_nid_postfix(policy); } catch (const std::invalid_argument&) { return true; } + return false; +} +int main() { + for (int policy : {1, 2, 3}) { + Require(sched_get_priority_max_nid_postfix(policy) == 256); + Require(sched_get_priority_min_nid_postfix(policy) == 767); + } + Require(Rejects(0) && Rejects(4)); +} diff --git a/core/libs/tests/GuestShaderLink.cpp b/core/libs/tests/GuestShaderLink.cpp index 20ba2f0cd..775f2fe1f 100644 --- a/core/libs/tests/GuestShaderLink.cpp +++ b/core/libs/tests/GuestShaderLink.cpp @@ -16,7 +16,9 @@ static unsigned links = 0; static const Shader* mappedPixel = nullptr; extern "C" void AgcDriverResolveGraphicsStagesAbi_nid_postfix(std::span stages, std::span context, std::span primitive) { - Require(!stages.empty() && stages[0] != nullptr && context.size() == 2u && primitive.size() == 3u); + Require(!stages.empty() && stages[0] != nullptr && primitive.size() == 3u); + Require((stages.size() == 1u && context.size() == 2u) || + (stages.size() == 2u && stages[1] == mappedPixel && context.size() == 34u)); ++preparations; } @@ -50,7 +52,7 @@ int main() { } Require(hasRenderTarget && hasRasterizer); ShaderSpecialRegs special{}; - special.vgt_shader_stages_en = {VGT_SHADER_STAGES_EN, VGT_SHADER_STAGES_NGG_BIT}; + special.vgt_shader_stages_en = {VGT_SHADER_STAGES_EN, 0x2000u}; special.vgt_gs_out_prim_type = {VGT_GS_OUT_PRIM_TYPE, 0}; special.ge_cntl = {GE_CNTL, 0x123}; special.ge_user_vgpr_en = {GE_USER_VGPR_EN, 7}; @@ -64,7 +66,7 @@ int main() { context.back() = {0xdeadbeef, 0xcafebabe}; primitive.back() = context.back(); Require(sceAgcLinkShaders(context.data(), primitive.data(), nullptr, &vertex, &pixel, 4) == 0); - Require(context[0].offset == VGT_SHADER_STAGES_EN && context[0].value == VGT_SHADER_STAGES_NGG_BIT); + Require(context[0].offset == VGT_SHADER_STAGES_EN && context[0].value == 0x2000u); Require(context[1].offset == VGT_GS_OUT_PRIM_TYPE && context[1].value == 2); for (unsigned i = 0; i < 32; ++i) Require(context[i + 2].offset == SPI_PS_INPUT_CNTL_0 + i && context[i + 2].value == i); @@ -104,5 +106,5 @@ int main() { input.default_value = 2; Require(sceAgcLinkShaders(context.data(), primitive.data(), nullptr, &vertex, &pixel, 4) == 0); Require(context[2].value == 0x220); - Require(preparations == 4u && mappings == 3u && links == 7u); + Require(preparations == 7u && mappings == 3u && links == 7u); } diff --git a/core/libs/tests/GuestShare.cpp b/core/libs/tests/GuestShare.cpp index 6e062efd8..400e2f5a7 100644 --- a/core/libs/tests/GuestShare.cpp +++ b/core/libs/tests/GuestShare.cpp @@ -11,6 +11,7 @@ int APS5_VABI sceShareCaptureVideoClip(const void* param, std::int32_t* req_id); int APS5_VABI sceShareGetCurrentStatus(std::uint32_t feature_flag, void* status); int APS5_VABI sceShareOpenMenuForContent(const void* content_id); int APS5_VABI sceShareGetRunningStatus(std::uint32_t* status); +int APS5_VABI sceShareSetContentParamForApplicationTitle(const char* application_title); } namespace { @@ -64,5 +65,7 @@ int main() { Require(sceShareGetRunningStatus(running) == 0); Require(running[0] == 0 && running[1] == 0x5a5a5a5au); Require(sceShareGetRunningStatus(nullptr) == invalidParam); + Require(sceShareSetContentParamForApplicationTitle("title") == 0); + Require(sceShareSetContentParamForApplicationTitle(nullptr) == invalidParam); return 0; } diff --git a/core/libs/tests/GuestShmOpen.cpp b/core/libs/tests/GuestShmOpen.cpp new file mode 100644 index 000000000..112ae3c90 --- /dev/null +++ b/core/libs/tests/GuestShmOpen.cpp @@ -0,0 +1,57 @@ +#include "prx/libc/include/general/VabiMacros.hpp" +#include +#include +#include +#include +#include +extern "C" { +int APS5_VABI shm_open_nid_postfix(const char*, int, int); +int APS5_VABI ftruncate_nid_postfix(int, std::int64_t); +std::int64_t APS5_VABI pread_nid_postfix(int, void*, std::size_t, std::int64_t); +std::int64_t APS5_VABI pwrite_nid_disambig1_nid_postfix(int, const void*, std::size_t, std::int64_t); +int APS5_VABI close_nid_postfix(int); +int* APS5_VABI __error_nid_postfix(); +} +static void Require(bool value) { if (!value) std::abort(); } +static const char* const anonymous = reinterpret_cast(1); +static void RequireInvalid(const char* path, int flags) { + *__error_nid_postfix() = 0; + Require(shm_open_nid_postfix(path, flags, 0600) == -1); + Require(*__error_nid_postfix() == 22); +} +int main() { + *__error_nid_postfix() = 13; + const int first = shm_open_nid_postfix(anonymous, 0x100202, 0600); + const int second = shm_open_nid_postfix(anonymous, 0x2, 0); + const int third = shm_open_nid_postfix(anonymous, 0x100e02, 0400); + Require(first >= 0 && second >= 0 && third >= 0); + Require(first != second && second != third && first != third); + Require(*__error_nid_postfix() == 13); + Require(ftruncate_nid_postfix(first, 0x4000) == 0); + const char text[] = "anonymous"; + Require(pwrite_nid_disambig1_nid_postfix(first, text, sizeof(text), 0x1000) == static_cast(sizeof(text))); + char back[sizeof(text)] = {}; + Require(pread_nid_postfix(first, back, sizeof(back), 0x1000) == static_cast(sizeof(back))); + Require(std::memcmp(back, text, sizeof(text)) == 0); + char last = 1; + Require(pread_nid_postfix(first, &last, 1, 0x3fff) == 1 && last == 0); + Require(pread_nid_postfix(first, &last, 1, 0x4000) == 0); + Require(pread_nid_postfix(second, back, sizeof(back), 0) == 0); + Require(pwrite_nid_disambig1_nid_postfix(third, text, sizeof(text), 0) == static_cast(sizeof(text))); + Require(close_nid_postfix(first) == 0 && close_nid_postfix(second) == 0 && close_nid_postfix(third) == 0); + RequireInvalid(anonymous, 0x0); + RequireInvalid(anonymous, 0x200); + RequireInvalid(anonymous, 0x1); + RequireInvalid(anonymous, 0x3); + RequireInvalid(anonymous, 0x2 | 0x8); + RequireInvalid(anonymous, 0x2 | 0x4); + RequireInvalid("/anyps5-shm", 0x1); + RequireInvalid("/anyps5-shm", 0x202 | 0x8); + bool threw = false; + try { + shm_open_nid_postfix("/anyps5-shm", 0x202, 0600); + } catch (const std::runtime_error&) { + threw = true; + } + Require(threw); +} diff --git a/core/libs/tests/GuestSigaltstack.cpp b/core/libs/tests/GuestSigaltstack.cpp new file mode 100644 index 000000000..6260846fd --- /dev/null +++ b/core/libs/tests/GuestSigaltstack.cpp @@ -0,0 +1,63 @@ +#include "prx/libc/include/general/VabiMacros.hpp" +#include +#include +#include +#include +struct GuestStack { + void* sp; + std::size_t size; + int flags; +}; +extern "C" { +int APS5_VABI sigaltstack_nid_postfix(const GuestStack*, GuestStack*); +int* APS5_VABI __error_nid_postfix(); +} +static void Require(bool value) { if (!value) std::abort(); } +static GuestStack Query() { + GuestStack current; + std::memset(¤t, 0xa5, sizeof(current)); + Require(sigaltstack_nid_postfix(nullptr, ¤t) == 0); + return current; +} +static void Fails(const GuestStack& stack, int error) { + GuestStack untouched; + std::memset(&untouched, 0xa5, sizeof(untouched)); + GuestStack previous = untouched; + *__error_nid_postfix() = 0; + Require(sigaltstack_nid_postfix(&stack, &previous) == -1 && *__error_nid_postfix() == error); + Require(std::memcmp(&previous, &untouched, sizeof(previous)) == 0); +} +int main() { + alignas(16) static char area[4096]; + GuestStack current = Query(); + Require(current.sp == nullptr && current.size == 0 && current.flags == 4); + Require(sigaltstack_nid_postfix(nullptr, nullptr) == 0); + + Fails({area, sizeof(area), 1}, 22); + Fails({area, sizeof(area), 4 | 2}, 22); + Fails({area, 2047, 0}, 12); + current = Query(); + Require(current.sp == nullptr && current.size == 0 && current.flags == 4); + + GuestStack previous{}; + const GuestStack minimum{area, 2048, 0}; + Require(sigaltstack_nid_postfix(&minimum, &previous) == 0); + Require(previous.sp == nullptr && previous.flags == 4); + const GuestStack installed{area, sizeof(area), 0}; + Require(sigaltstack_nid_postfix(&installed, &previous) == 0); + Require(previous.sp == area && previous.size == 2048 && previous.flags == 0); + current = Query(); + Require(current.sp == area && current.size == sizeof(area) && current.flags == 0); + + std::thread([] { + const GuestStack fresh = Query(); + Require(fresh.sp == nullptr && fresh.size == 0 && fresh.flags == 4); + }).join(); + + const GuestStack disable{nullptr, 0, 4}; + Require(sigaltstack_nid_postfix(&disable, &previous) == 0); + Require(previous.sp == area && previous.size == sizeof(area) && previous.flags == 0); + current = Query(); + Require(current.sp == area && current.size == sizeof(area) && current.flags == 4); + Fails({area, 16, 0}, 12); +} diff --git a/core/libs/tests/GuestSignalSets.cpp b/core/libs/tests/GuestSignalSets.cpp new file mode 100644 index 000000000..f9df0f153 --- /dev/null +++ b/core/libs/tests/GuestSignalSets.cpp @@ -0,0 +1,104 @@ +#include "prx/libc/include/general/VabiMacros.hpp" +#include +#include +#include +#include + +struct GuestSignalSet { + std::uint32_t bits[4]; +}; + +extern "C" { +int APS5_VABI sigemptyset_nid_postfix(GuestSignalSet*); +int APS5_VABI sigfillset_nid_postfix(GuestSignalSet*); +int APS5_VABI sigaddset_nid_postfix(GuestSignalSet*, int); +int APS5_VABI sigdelset_nid_postfix(GuestSignalSet*, int); +int APS5_VABI sigismember_nid_postfix(const GuestSignalSet*, int); +int* APS5_VABI __error_nid_postfix(); +} + +static void Require(bool value) { if (!value) std::abort(); } + +static bool Equals(const GuestSignalSet& set, std::uint32_t w0, std::uint32_t w1, std::uint32_t w2, std::uint32_t w3) { + return set.bits[0] == w0 && set.bits[1] == w1 && set.bits[2] == w2 && set.bits[3] == w3; +} + +int main() { + static constexpr int Untouched = 5; + GuestSignalSet set{{0x12345678u, 0x9abcdef0u, 0xffffffffu, 1u}}; + *__error_nid_postfix() = Untouched; + Require(sigemptyset_nid_postfix(&set) == 0); + Require(Equals(set, 0, 0, 0, 0)); + Require(*__error_nid_postfix() == Untouched); + + Require(sigfillset_nid_postfix(&set) == 0); + Require(Equals(set, 0xffffffffu, 0xffffffffu, 0xffffffffu, 0xffffffffu)); + for (int signal = 1; signal <= 128; ++signal) Require(sigismember_nid_postfix(&set, signal) == 1); + Require(*__error_nid_postfix() == Untouched); + + Require(sigemptyset_nid_postfix(&set) == 0); + Require(sigaddset_nid_postfix(&set, 1) == 0); + Require(Equals(set, 1u, 0, 0, 0)); + Require(sigaddset_nid_postfix(&set, 32) == 0); + Require(Equals(set, 0x80000001u, 0, 0, 0)); + Require(sigaddset_nid_postfix(&set, 33) == 0); + Require(Equals(set, 0x80000001u, 1u, 0, 0)); + Require(sigaddset_nid_postfix(&set, 65) == 0); + Require(Equals(set, 0x80000001u, 1u, 1u, 0)); + Require(sigaddset_nid_postfix(&set, 96) == 0); + Require(Equals(set, 0x80000001u, 1u, 0x80000001u, 0)); + Require(sigaddset_nid_postfix(&set, 128) == 0); + Require(Equals(set, 0x80000001u, 1u, 0x80000001u, 0x80000000u)); + Require(sigaddset_nid_postfix(&set, 33) == 0); + Require(Equals(set, 0x80000001u, 1u, 0x80000001u, 0x80000000u)); + Require(*__error_nid_postfix() == Untouched); + + Require(sigismember_nid_postfix(&set, 1) == 1); + Require(sigismember_nid_postfix(&set, 2) == 0); + Require(sigismember_nid_postfix(&set, 32) == 1); + Require(sigismember_nid_postfix(&set, 33) == 1); + Require(sigismember_nid_postfix(&set, 64) == 0); + Require(sigismember_nid_postfix(&set, 65) == 1); + Require(sigismember_nid_postfix(&set, 96) == 1); + Require(sigismember_nid_postfix(&set, 97) == 0); + Require(sigismember_nid_postfix(&set, 128) == 1); + Require(*__error_nid_postfix() == Untouched); + + Require(sigdelset_nid_postfix(&set, 1) == 0); + Require(Equals(set, 0x80000000u, 1u, 0x80000001u, 0x80000000u)); + Require(sigdelset_nid_postfix(&set, 32) == 0); + Require(Equals(set, 0, 1u, 0x80000001u, 0x80000000u)); + Require(sigdelset_nid_postfix(&set, 33) == 0); + Require(Equals(set, 0, 0, 0x80000001u, 0x80000000u)); + Require(sigdelset_nid_postfix(&set, 96) == 0); + Require(Equals(set, 0, 0, 1u, 0x80000000u)); + Require(sigdelset_nid_postfix(&set, 128) == 0); + Require(Equals(set, 0, 0, 1u, 0)); + Require(sigdelset_nid_postfix(&set, 2) == 0); + Require(Equals(set, 0, 0, 1u, 0)); + Require(sigismember_nid_postfix(&set, 128) == 0); + Require(*__error_nid_postfix() == Untouched); + + for (const int invalid : {0, -1, 129, 1000, INT_MIN, INT_MAX}) { + *__error_nid_postfix() = 0; + Require(sigaddset_nid_postfix(&set, invalid) == -1 && *__error_nid_postfix() == 22); + *__error_nid_postfix() = 0; + Require(sigdelset_nid_postfix(&set, invalid) == -1 && *__error_nid_postfix() == 22); + *__error_nid_postfix() = 0; + Require(sigismember_nid_postfix(&set, invalid) == -1 && *__error_nid_postfix() == 22); + Require(Equals(set, 0, 0, 1u, 0)); + } + + Require(sigfillset_nid_postfix(&set) == 0); + for (const int invalid : {0, -1, 129, 1000, INT_MIN, INT_MAX}) { + Require(sigaddset_nid_postfix(&set, invalid) == -1 && *__error_nid_postfix() == 22); + Require(sigdelset_nid_postfix(&set, invalid) == -1 && *__error_nid_postfix() == 22); + Require(sigismember_nid_postfix(&set, invalid) == -1 && *__error_nid_postfix() == 22); + Require(Equals(set, 0xffffffffu, 0xffffffffu, 0xffffffffu, 0xffffffffu)); + } + + const GuestSignalSet empty{{0, 0, 0, 0}}; + Require(sigismember_nid_postfix(&empty, 1) == 0); + Require(sigismember_nid_postfix(&empty, 128) == 0); + return 0; +} diff --git a/core/libs/tests/GuestSignals.cpp b/core/libs/tests/GuestSignals.cpp index a15423d10..dc5bf1a96 100644 --- a/core/libs/tests/GuestSignals.cpp +++ b/core/libs/tests/GuestSignals.cpp @@ -2,10 +2,13 @@ #include #include #include +#include +#include using Handler = void (APS5_VABI *)(int); extern "C" { Handler APS5_VABI signal_nid_postfix(int, Handler); int APS5_VABI raise_nid_postfix(int); +int APS5_VABI sigaction_nid_postfix(int, const void*, void*); int APS5_VABI sigprocmask_nid_postfix(int, const void*, void*); int* APS5_VABI __error_nid_postfix(); int APS5_VABI _is_signal_return_nid_postfix(std::uint64_t); @@ -13,8 +16,18 @@ int APS5_VABI _is_signal_return_nid_postfix(std::uint64_t); struct GuestSignalSet { std::uint32_t bits[4]; }; +struct GuestSigaction { + std::uintptr_t handler; + int flags; + GuestSignalSet mask; +}; +static_assert(sizeof(GuestSigaction) == 32); +static bool Same(const GuestSigaction& left, const GuestSigaction& right) { + return left.handler == right.handler && left.flags == right.flags && std::memcmp(&left.mask, &right.mask, sizeof(left.mask)) == 0; +} volatile std::sig_atomic_t received = 0; volatile std::uintptr_t handlerReturn = 0; +void APS5_VABI InfoCallback(int, void*, void*) {} void APS5_VABI Callback(int value) { received = value; handlerReturn = reinterpret_cast(__builtin_return_address(0)); @@ -46,4 +59,62 @@ int main() { Require(sigprocmask_nid_postfix(2, &blocked, nullptr) == 0); Require(sigprocmask_nid_postfix(1, nullptr, &previous) == 0); Require(previous.bits[0] == 0); + + const auto callback = reinterpret_cast(Callback); + const auto infoCallback = reinterpret_cast(InfoCallback); + GuestSigaction current{}; + std::memset(¤t, 0xa5, sizeof(current)); + Require(sigaction_nid_postfix(15, nullptr, ¤t) == 0); + Require(current.handler == 0 && current.flags == 0x2 && current.mask.bits[0] == 0 && current.mask.bits[3] == 0); + const GuestSigaction term{callback, 0, {{0x4000, 0, 0, 0x80000000u}}}; + Require(sigaction_nid_postfix(15, &term, ¤t) == 0 && current.handler == 0); + received = 0; + Require(raise_nid_postfix(15) == 0 && received == 15); + Require(sigaction_nid_postfix(15, nullptr, ¤t) == 0); + Require(Same(current, term)); + const GuestSigaction defaults{0, 0, {}}; + Require(sigaction_nid_postfix(15, &defaults, nullptr) == 0); + Require(signal_nid_postfix(15, Callback) == nullptr); + Require(sigaction_nid_postfix(15, nullptr, ¤t) == 0 && current.handler == callback && current.flags == 0x2); + Require(signal_nid_postfix(15, nullptr) == Callback); + + const GuestSigaction crash{infoCallback, 0x40 | 0x10, {}}; + for (int recorded : {10, 1, 64, 128}) { + Require(sigaction_nid_postfix(recorded, &crash, nullptr) == 0); + Require(sigaction_nid_postfix(recorded, nullptr, ¤t) == 0); + Require(Same(current, crash)); + Require(sigaction_nid_postfix(recorded, &defaults, nullptr) == 0); + } + Require(sigaction_nid_postfix(6, &crash, nullptr) == 0); + bool rejected = false; + try { + raise_nid_postfix(6); + } catch (const std::runtime_error&) { + rejected = true; + } + Require(rejected); + Require(sigaction_nid_postfix(6, &defaults, ¤t) == 0 && current.handler == infoCallback && current.flags == 0x50); + + *__error_nid_postfix() = 0; + Require(sigaction_nid_postfix(0, nullptr, ¤t) == -1 && *__error_nid_postfix() == 22); + *__error_nid_postfix() = 0; + Require(sigaction_nid_postfix(129, nullptr, ¤t) == -1 && *__error_nid_postfix() == 22); + for (int fixed : {9, 17}) { + *__error_nid_postfix() = 0; + Require(sigaction_nid_postfix(fixed, &term, ¤t) == -1 && *__error_nid_postfix() == 22); + Require(sigaction_nid_postfix(fixed, &defaults, ¤t) == 0 && current.handler == 0); + } + Require(sigaction_nid_postfix(15, nullptr, nullptr) == 0); + Require(signal_nid_postfix(15, Callback) != invalid); + GuestSignalSet termMask{{1u << 14, 0, 0, 0}}; + received = 0; + Require(sigprocmask_nid_postfix(1, &termMask, nullptr) == 0); + Require(raise_nid_postfix(15) == 0 && received == 0); + Require(sigprocmask_nid_postfix(2, &termMask, nullptr) == 0); + Require(raise_nid_postfix(15) == 0 && received == 15); + GuestSignalSet urgMask{{1u << 15, 0, 0, 0}}; + received = 0; + Require(sigprocmask_nid_postfix(1, &urgMask, nullptr) == 0); + Require(raise_nid_postfix(15) == 0 && received == 15); + Require(sigprocmask_nid_postfix(2, &urgMask, nullptr) == 0); } diff --git a/core/libs/tests/GuestSocketReadWrite.cpp b/core/libs/tests/GuestSocketReadWrite.cpp new file mode 100644 index 000000000..ad6eae0f8 --- /dev/null +++ b/core/libs/tests/GuestSocketReadWrite.cpp @@ -0,0 +1,114 @@ +#include "prx/libc/include/general/VabiMacros.hpp" +#include +#include +#include +#include +#include +#include +#ifndef _WIN32 +#include +#endif + +struct PollDescriptor { + int descriptor; + short events; + short revents; +}; + +extern "C" { +int APS5_VABI socket_nid_postfix(int, int, int); +int APS5_VABI bind_nid_postfix(int, const void*, std::uint32_t); +int APS5_VABI listen_nid_postfix(int, int); +int APS5_VABI getsockname_nid_postfix(int, void*, std::uint32_t*); +int APS5_VABI connect_nid_postfix(int, const void*, std::uint32_t); +int APS5_VABI accept_nid_postfix(int, void*, std::uint32_t*); +int APS5_VABI fcntl_nid_postfix(int, int, ...); +int APS5_VABI poll_nid_postfix(PollDescriptor*, std::uint32_t, int); +int APS5_VABI close_nid_postfix(int); +std::int64_t APS5_VABI read_nid_postfix(int, void*, std::uint64_t); +std::int64_t APS5_VABI write_nid_postfix(int, const char*, std::int64_t); +std::int64_t APS5_VABI _read_nid_postfix(int, void*, std::size_t); +std::int64_t APS5_VABI _write_nid_postfix(int, const void*, std::size_t); +std::int64_t APS5_VABI sceKernelRead(int, void*, std::size_t); +std::int64_t APS5_VABI sceKernelWrite(int, const void*, std::size_t); +int* APS5_VABI __error_nid_postfix(); +} + +static void Check(bool value, int line) { + if (!value) { + std::fprintf(stderr, "socket read/write check failed at line %d, errno %d\n", line, *__error_nid_postfix()); + std::abort(); + } +} +#define Require(value) Check((value), __LINE__) + +constexpr short In = 0x1; +constexpr short Hup = 0x10; +constexpr int KernelAgain = static_cast(0x80020023u); +constexpr int KernelBadDescriptor = static_cast(0x80020009u); + +static bool Readable(int descriptor, short accepted = In) { + PollDescriptor waited{descriptor, In, 0}; + return poll_nid_postfix(&waited, 1, 2000) == 1 && (waited.revents & accepted) != 0; +} + +int main() { +#ifndef _WIN32 + std::signal(SIGPIPE, SIG_IGN); +#endif + const int listener = socket_nid_postfix(2, 1, 0); + Require(listener >= 0); + std::array address{16, 2, 0, 0, 127, 0, 0, 1}; + Require(bind_nid_postfix(listener, address.data(), address.size()) == 0); + Require(listen_nid_postfix(listener, 4) == 0); + std::uint32_t addressSize = address.size(); + Require(getsockname_nid_postfix(listener, address.data(), &addressSize) == 0); + const int client = socket_nid_postfix(2, 1, 0); + Require(client >= 0 && connect_nid_postfix(client, address.data(), address.size()) == 0); + const int server = accept_nid_postfix(listener, nullptr, nullptr); + Require(server >= 0); + + const char request[] = "written to a socket"; + char received[32]{}; + Require(write_nid_postfix(client, request, sizeof(request)) == sizeof(request)); + Require(Readable(server) && read_nid_postfix(server, received, sizeof(received)) == sizeof(request)); + Require(std::strcmp(received, request) == 0); + + const char reply[] = "and back"; + std::memset(received, 0, sizeof(received)); + Require(_write_nid_postfix(server, reply, sizeof(reply)) == sizeof(reply)); + Require(Readable(client) && _read_nid_postfix(client, received, sizeof(received)) == sizeof(reply)); + Require(std::strcmp(received, reply) == 0); + + const char kernel[] = "through sceKernelWrite"; + std::memset(received, 0, sizeof(received)); + Require(sceKernelWrite(client, kernel, sizeof(kernel)) == sizeof(kernel)); + Require(Readable(server) && sceKernelRead(server, received, sizeof(received)) == sizeof(kernel)); + Require(std::strcmp(received, kernel) == 0); + + Require(fcntl_nid_postfix(server, 4, 4) == 0); + *__error_nid_postfix() = 0; + Require(read_nid_postfix(server, received, sizeof(received)) == -1 && *__error_nid_postfix() == 35); + Require(sceKernelRead(server, received, sizeof(received)) == KernelAgain); + + Require(read_nid_postfix(server, received, 0) == 0 && sceKernelRead(server, received, 0) == 0); + Require(close_nid_postfix(client) == 0); + Require(Readable(server, In | Hup) && read_nid_postfix(server, received, sizeof(received)) == 0); + Require(sceKernelRead(server, received, sizeof(received)) == 0); +#ifndef _WIN32 + std::int64_t written = 0; + for (int i = 0; i < 100 && written >= 0; ++i) written = write_nid_postfix(server, request, sizeof(request)); + Require(written == -1 && *__error_nid_postfix() == 32); +#endif + Require(close_nid_postfix(server) == 0); + *__error_nid_postfix() = 0; + Require(read_nid_postfix(server, received, sizeof(received)) == -1 && *__error_nid_postfix() == 9); + Require(read_nid_postfix(server, received, 0) == -1 && *__error_nid_postfix() == 9); + Require(write_nid_postfix(server, request, sizeof(request)) == -1 && *__error_nid_postfix() == 9); + Require(_write_nid_postfix(server, request, sizeof(request)) == -1 && *__error_nid_postfix() == 9); + Require(_read_nid_postfix(server, received, sizeof(received)) == -1 && *__error_nid_postfix() == 9); + Require(sceKernelRead(server, received, sizeof(received)) == KernelBadDescriptor); + Require(sceKernelWrite(server, request, sizeof(request)) == KernelBadDescriptor); + Require(close_nid_postfix(listener) == 0); + return 0; +} diff --git a/core/libs/tests/GuestStandardStreams.cpp b/core/libs/tests/GuestStandardStreams.cpp index 99f344536..2351dc61f 100644 --- a/core/libs/tests/GuestStandardStreams.cpp +++ b/core/libs/tests/GuestStandardStreams.cpp @@ -95,7 +95,7 @@ static bool CheckBinaryModes() { const auto filename = directory + "/bytes"; const std::string original("A\r\n\x1a" "B\0C", 7); const std::string written("D\n\x1a" "E\0F", 6); - const char* modes[] = {"r", "r+", "w", "w+", "a", "a+", "rb", "rb+", "r+b", "wb", "wb+", "w+b", "ab", "ab+", "a+b"}; + const char* modes[] = {"r", "r+", "w", "w+", "a", "a+", "rb", "rb+", "r+b", "wb", "wb+", "w+b", "ab", "ab+", "a+b", "rt", "r+t", "wt", "w+e", "ae", "rbv"}; bool correct = true; for (const auto* mode : modes) { for (const bool reopen : {false, true}) { @@ -124,6 +124,20 @@ static bool CheckBinaryModes() { correct &= CheckFileBytes(filename, expected, mode, reopen); } } + for (const auto* mode : {"", "q", "tr", "rx", "rbx"}) { + FileStream redirected(std::tmpfile()); + Require(fopen_nid_postfix(filename.c_str(), mode) == nullptr && *__error_nid_postfix() == 22); + Require(freopen_nid_postfix(filename.c_str(), mode, &redirected) == nullptr && *__error_nid_postfix() == 22); + } + Require(fopen_nid_postfix(filename.c_str(), "wx") == nullptr && *__error_nid_postfix() == 17); + FileStream* readOnly = fopen_nid_postfix(filename.c_str(), "rt+"); + Require(readOnly != nullptr && fputc_nid_postfix('X', readOnly) == EOF); + Require(fclose_nid_postfix(readOnly) == 0); + Require(std::filesystem::remove(filename)); + FileStream* created = fopen_nid_postfix(filename.c_str(), "wx"); + Require(created != nullptr && fputc_nid_postfix('Y', created) == 'Y'); + Require(fclose_nid_postfix(created) == 0); + correct &= CheckFileBytes(filename, "Y", "wx", false); Require(std::filesystem::remove(filename)); Require(std::filesystem::remove(directory)); return correct; diff --git a/core/libs/tests/GuestStrings.cpp b/core/libs/tests/GuestStrings.cpp index 019e4425c..c7b048a8a 100644 --- a/core/libs/tests/GuestStrings.cpp +++ b/core/libs/tests/GuestStrings.cpp @@ -1,4 +1,6 @@ #include "prx/libc/include/general/VabiMacros.hpp" +#include +#include #include #include #include @@ -6,6 +8,7 @@ extern "C" { char* APS5_VABI basename_nid_postfix(const char*); +char* APS5_VABI __inet_ntoa_nid_postfix(unsigned int); int* APS5_VABI __error_nid_postfix(); std::size_t APS5_VABI strnlen_nid_postfix(const char*, std::size_t); std::size_t APS5_VABI strnlen_s_nid_postfix(const char*, std::size_t); @@ -13,6 +16,7 @@ char* APS5_VABI strncat_nid_postfix(char*, const char*, std::size_t); char* APS5_VABI strpbrk_nid_postfix(const char*, const char*); std::size_t APS5_VABI strcspn_nid_postfix(const char*, const char*); std::size_t APS5_VABI strlcat_nid_postfix(char*, const char*, std::size_t); +char* APS5_VABI stpcpy_nid_postfix(char*, const char*); char* APS5_VABI strtok_r_nid_postfix(char*, const char*, char**); char* APS5_VABI strtok_nid_postfix(char*, const char*); char* APS5_VABI strcasestr_nid_postfix(const char*, const char*); @@ -25,6 +29,8 @@ int APS5_VABI memset_s_nid_postfix(void*, std::size_t, int, std::size_t); char* APS5_VABI strnstr_nid_postfix(const char*, const char*, std::size_t); int APS5_VABI snprintf_s_nid_postfix(char*, std::size_t, const char*, ...); int APS5_VABI sscanf_s_nid_postfix(const char*, const char*, ...); +int APS5_VABI __inet_aton_nid_postfix(const char*, void*); +std::uint32_t APS5_VABI __inet_addr_nid_postfix(const char*); } static void Require(bool condition) { @@ -117,6 +123,21 @@ int main() { Require(std::strcmp(basename_nid_postfix("one\\two"), "one\\two") == 0); const std::string longName(1024, 'x'); Require(basename_nid_postfix(longName.c_str()) == nullptr && *__error_nid_postfix() == 63); + const unsigned char loopbackBytes[4] = {127, 0, 0, 1}; + unsigned int loopback; + std::memcpy(&loopback, loopbackBytes, sizeof(loopback)); + char* const dotted = __inet_ntoa_nid_postfix(loopback); + Require(std::strcmp(dotted, "127.0.0.1") == 0); + Require(__inet_ntoa_nid_postfix(0xffffffffu) == dotted && std::strcmp(dotted, "255.255.255.255") == 0); + Require(std::strcmp(__inet_ntoa_nid_postfix(0), "0.0.0.0") == 0); + const unsigned char mixedBytes[4] = {10, 200, 3, 45}; + unsigned int mixed; + std::memcpy(&mixed, mixedBytes, sizeof(mixed)); + Require(std::strcmp(__inet_ntoa_nid_postfix(mixed), "10.200.3.45") == 0); + const unsigned char paddingBytes[4] = {100, 9, 99, 0}; + unsigned int padding; + std::memcpy(&padding, paddingBytes, sizeof(padding)); + Require(std::strcmp(__inet_ntoa_nid_postfix(padding), "100.9.99.0") == 0); const char bounded[] = {'a', 'b', 'c'}; Require(strnlen_nid_postfix(bounded, 0) == 0); Require(strnlen_nid_postfix(bounded, sizeof(bounded)) == 3); @@ -132,11 +153,53 @@ int main() { Require(strlcat_nid_postfix(buffer, "xyz", 0) == 3); buffer[0] = '\0'; Require(strlcat_nid_postfix(buffer, "x", 1) == 1 && buffer[0] == '\0'); + char chained[8] = "zzzzzzz"; + char* end = stpcpy_nid_postfix(chained, "ab"); + Require(end == chained + 2 && *end == '\0' && chained[3] == 'z'); + end = stpcpy_nid_postfix(end, "cd"); + Require(end == chained + 4 && std::strcmp(chained, "abcd") == 0); + Require(stpcpy_nid_postfix(end, "") == end && chained[5] == 'z'); Require(strncat_nid_postfix(buffer, "xyz", 2) == buffer); Require(std::strcmp(buffer, "xy") == 0); Require(strpbrk_nid_postfix(buffer, "ay") == buffer + 1); Require(strpbrk_nid_postfix(buffer, "") == nullptr); Require(strcspn_nid_postfix(buffer, "y") == 1); + const auto inetAton = [](const char* text, const char* expected) { + unsigned char address[4] = {0xA5, 0xA5, 0xA5, 0xA5}; + if (__inet_aton_nid_postfix(text, address) != 1) return false; + char formatted[16]; + std::snprintf(formatted, sizeof(formatted), "%u.%u.%u.%u", address[0], address[1], address[2], address[3]); + return std::strcmp(formatted, expected) == 0; + }; + Require(inetAton("192.0.2.42", "192.0.2.42")); + Require(inetAton("10.1.2", "10.1.0.2")); + Require(inetAton("127.1", "127.0.0.1")); + Require(inetAton("3232235777", "192.168.1.1")); + Require(inetAton("0x7f.0.0.0x1", "127.0.0.1")); + Require(inetAton("0377.0.0.010", "255.0.0.8")); + Require(inetAton("1.2.3.4 trailing", "1.2.3.4")); + Require(inetAton("1.2.3.4\n", "1.2.3.4")); + Require(__inet_aton_nid_postfix("1.2.3.4", nullptr) == 1); + for (const char* invalid : {"", " 1.2.3.4", "1.2.3.4.5", "256.1.1.1", "1.2.3.256", "1.2.65536", "08", "1..2", "a.b.c.d", + "1.2.3.4x", "0x", "1.2.3.", "-1"}) { + unsigned char address[4] = {0xA5, 0xA5, 0xA5, 0xA5}; + Require(__inet_aton_nid_postfix(invalid, address) == 0); + Require(address[0] == 0xA5 && address[3] == 0xA5); + Require(__inet_addr_nid_postfix(invalid) == 0xffffffff); + } + const auto inetAddr = [](const char* text, std::array expected) { + const std::uint32_t value = __inet_addr_nid_postfix(text); + std::array bytes{}; + std::memcpy(bytes.data(), &value, sizeof(value)); + return bytes == expected; + }; + Require(inetAddr("192.0.2.42", {192, 0, 2, 42})); + Require(inetAddr("127.1", {127, 0, 0, 1})); + Require(inetAddr("0x7f.0.0.0x1", {127, 0, 0, 1})); + Require(inetAddr("0.0.0.0", {0, 0, 0, 0})); + Require(inetAddr("255.255.255.255", {255, 255, 255, 255})); + Require(inetAddr("4294967296", {0, 0, 0, 0})); + Require(inetAddr("18446744073709551617", {0, 0, 0, 1})); char first[] = ",a,,b,"; char second[] = "x:y"; char* firstState = nullptr; diff --git a/core/libs/tests/GuestSyslog.cpp b/core/libs/tests/GuestSyslog.cpp new file mode 100644 index 000000000..a4634c5d6 --- /dev/null +++ b/core/libs/tests/GuestSyslog.cpp @@ -0,0 +1,51 @@ +#include "prx/libc/include/general/VabiMacros.hpp" +#include +#include +#include +#ifdef _WIN32 +#include +#else +#include +#endif + +extern "C" { +void APS5_VABI syslog_nid_postfix(int, const char*, ...); +int* APS5_VABI __error_nid_postfix(); +} +static void Require(bool value) { if (!value) std::abort(); } +int main() { + auto* captured = std::tmpfile(); + Require(captured != nullptr); + std::fflush(stderr); +#ifdef _WIN32 + const int saved = ::_dup(::_fileno(stderr)); + Require(saved >= 0 && ::_dup2(::_fileno(captured), ::_fileno(stderr)) == 0); +#else + const int saved = ::dup(::fileno(stderr)); + Require(saved >= 0 && ::dup2(::fileno(captured), ::fileno(stderr)) == ::fileno(stderr)); +#endif + *__error_nid_postfix() = 2; + syslog_nid_postfix(6, "station %s at %d kbps", "radio", 128); + const int afterFirst = *__error_nid_postfix(); + syslog_nid_postfix(16 | 3, "open: %m, %%m kept, %d%%\n", 50); + syslog_nid_postfix(0x10000 | 4, "masked"); + syslog_nid_postfix(7, ""); + std::fflush(stderr); +#ifdef _WIN32 + Require(::_dup2(saved, ::_fileno(stderr)) == 0 && ::_close(saved) == 0); +#else + Require(::dup2(saved, ::fileno(stderr)) == ::fileno(stderr) && ::close(saved) == 0); +#endif + Require(afterFirst == 2 && *__error_nid_postfix() == 2); + std::rewind(captured); + char text[512]{}; + const auto length = std::fread(text, 1, sizeof(text) - 1, captured); + const char* expected = + "[syslog:14] station radio at 128 kbps\n" + "[syslog:19] open: No such file or directory, %m kept, 50%\n" + "[syslog:35] syslog: unknown facility/priority: 10004\n" + "[syslog:12] masked\n" + "[syslog:15] \n"; + Require(length == std::strlen(expected) && std::strcmp(text, expected) == 0); + Require(std::fclose(captured) == 0); +} diff --git a/core/libs/tests/GuestSysmodule.cpp b/core/libs/tests/GuestSysmodule.cpp index bf6dbaa00..f3ffdecb0 100644 --- a/core/libs/tests/GuestSysmodule.cpp +++ b/core/libs/tests/GuestSysmodule.cpp @@ -3,6 +3,9 @@ #include extern "C" { +int APS5_VABI sceSysmoduleIsLoaded(std::uint16_t id); +int APS5_VABI sceSysmoduleLoadModule(std::uint16_t id); +int APS5_VABI sceSysmoduleUnloadModule(std::uint16_t id); int APS5_VABI sceSysmoduleLoadModuleInternal(std::uint32_t id); int APS5_VABI sceSysmoduleUnloadModuleInternal(std::uint32_t id); } @@ -11,15 +14,27 @@ namespace { void Require(bool value) { if (!value) std::abort(); } -constexpr std::uint32_t kFiberModuleId = 0x00000006; -constexpr int kModuleNotLoaded = static_cast(0x80A90003); +constexpr std::uint16_t kFiberModuleId = 0x0006; +constexpr std::uint16_t kUltModuleId = 0x0007; +constexpr int kModuleUnloaded = static_cast(0x805A1001); } int main() { + Require(sceSysmoduleIsLoaded(kFiberModuleId) == kModuleUnloaded); Require(sceSysmoduleLoadModuleInternal(kFiberModuleId) == 0); + Require(sceSysmoduleIsLoaded(kFiberModuleId) == 0); Require(sceSysmoduleLoadModuleInternal(kFiberModuleId) == 0); Require(sceSysmoduleUnloadModuleInternal(kFiberModuleId) == 0); + Require(sceSysmoduleIsLoaded(kFiberModuleId) == 0); Require(sceSysmoduleUnloadModuleInternal(kFiberModuleId) == 0); - Require(sceSysmoduleUnloadModuleInternal(kFiberModuleId) == kModuleNotLoaded); + Require(sceSysmoduleIsLoaded(kFiberModuleId) == kModuleUnloaded); + Require(sceSysmoduleUnloadModuleInternal(kFiberModuleId) == kModuleUnloaded); + + Require(sceSysmoduleIsLoaded(kUltModuleId) == kModuleUnloaded); + Require(sceSysmoduleUnloadModule(kUltModuleId) == kModuleUnloaded); + Require(sceSysmoduleLoadModule(kUltModuleId) == 0); + Require(sceSysmoduleIsLoaded(kUltModuleId) == 0); + Require(sceSysmoduleUnloadModule(kUltModuleId) == 0); + Require(sceSysmoduleIsLoaded(kUltModuleId) == kModuleUnloaded); } diff --git a/core/libs/tests/GuestSystemCommand.cpp b/core/libs/tests/GuestSystemCommand.cpp new file mode 100644 index 000000000..23290b082 --- /dev/null +++ b/core/libs/tests/GuestSystemCommand.cpp @@ -0,0 +1,17 @@ +#include "prx/libc/include/general/VabiMacros.hpp" +#include +extern "C" { +int APS5_VABI system_nid_postfix(const char*); +int* APS5_VABI __error_nid_postfix(); +} +static void Require(bool value) { if (!value) std::abort(); } +int main() { + *__error_nid_postfix() = 13; + Require(system_nid_postfix(nullptr) == 1); + const int status = system_nid_postfix("clrxdisasm --version > /dev/null 2>&1"); + Require((status & 0x7f) == 0); + Require(((status >> 8) & 0xff) == 127); + Require(system_nid_postfix("exit 0") == status); + Require(system_nid_postfix("") == status); + Require(*__error_nid_postfix() == 13); +} diff --git a/core/libs/tests/GuestSystemConfiguration.cpp b/core/libs/tests/GuestSystemConfiguration.cpp index be698e8b5..aed9e28e3 100644 --- a/core/libs/tests/GuestSystemConfiguration.cpp +++ b/core/libs/tests/GuestSystemConfiguration.cpp @@ -1,13 +1,59 @@ #include "prx/libc/include/general/VabiMacros.hpp" +#include #include #include +#include extern "C" { std::int64_t APS5_VABI sysconf_nid_postfix(int); int APS5_VABI getpagesize_nid_postfix(); int* APS5_VABI __error_nid_postfix(); +int APS5_VABI sysctl_nid_postfix(const int*, std::uint32_t, void*, std::size_t*, const void*, std::size_t); +int APS5_VABI sysctlbyname_nid_postfix(const char*, void*, std::size_t*, const void*, std::size_t); +extern char** environ_nid_postfix; } static void Require(bool value) { if (!value) std::abort(); } +static void CheckProcessorCountSysctl() { + const int mib[] = {6, 3}; + const int processors = static_cast(sysconf_nid_postfix(58)); + *__error_nid_postfix() = 13; + std::size_t length = 0; + Require(sysctl_nid_postfix(mib, 2, nullptr, &length, nullptr, 0) == 0 && length == sizeof(int)); + int value = 0; + length = sizeof(value); + Require(sysctl_nid_postfix(mib, 2, &value, &length, nullptr, 0) == 0 && value == processors && length == sizeof(int)); + value = 0; + length = 16; + Require(sysctlbyname_nid_postfix("hw.ncpu", &value, &length, nullptr, 0) == 0 && value == processors && length == sizeof(int)); + Require(*__error_nid_postfix() == 13); + unsigned char bytes[4] = {0xaa, 0xaa, 0xaa, 0xaa}; + length = 2; + Require(sysctlbyname_nid_postfix("hw.ncpu", bytes, &length, nullptr, 0) == -1 && *__error_nid_postfix() == 12); + Require(length == 2 && bytes[2] == 0xaa && bytes[3] == 0xaa); + Require(sysctl_nid_postfix(mib, 2, bytes, nullptr, nullptr, 0) == -1 && *__error_nid_postfix() == 12); + length = sizeof(value); + Require(sysctl_nid_postfix(mib, 2, &value, &length, &value, sizeof(value)) == -1 && *__error_nid_postfix() == 1); + Require(sysctl_nid_postfix(mib, 1, &value, &length, nullptr, 0) == -1 && *__error_nid_postfix() == 22); + Require(sysctl_nid_postfix(mib, 25, &value, &length, nullptr, 0) == -1 && *__error_nid_postfix() == 22); + Require(sysctl_nid_postfix(nullptr, 2, &value, &length, nullptr, 0) == -1 && *__error_nid_postfix() == 14); + Require(sysctlbyname_nid_postfix(nullptr, &value, &length, nullptr, 0) == -1 && *__error_nid_postfix() == 14); + bool threw = false; + try { + sysctlbyname_nid_postfix("kern.osreldate", &value, &length, nullptr, 0); + } catch (const std::runtime_error&) { + threw = true; + } + Require(threw); + threw = false; + const int unknown[] = {6, 5}; + try { + sysctl_nid_postfix(unknown, 2, &value, &length, nullptr, 0); + } catch (const std::runtime_error&) { + threw = true; + } + Require(threw); +} int main() { + CheckProcessorCountSysctl(); *__error_nid_postfix() = 13; Require(sysconf_nid_postfix(47) == 0x4000); Require(getpagesize_nid_postfix() == sysconf_nid_postfix(47)); @@ -18,4 +64,5 @@ int main() { Require(sysconf_nid_postfix(-1) == -1); // verifies full-width signed return Require(*__error_nid_postfix() == 22); Require(sysconf_nid_postfix(0x7fffffff) == -1); + Require(environ_nid_postfix && !environ_nid_postfix[0]); } diff --git a/core/libs/tests/GuestSystemExit.cpp b/core/libs/tests/GuestSystemExit.cpp index 3a22d960f..6dc172ac0 100644 --- a/core/libs/tests/GuestSystemExit.cpp +++ b/core/libs/tests/GuestSystemExit.cpp @@ -5,6 +5,9 @@ #include #include #include +#ifdef _WIN32 +#include +#endif extern "C" int APS5_VABI sceSystemServiceLoadExec(const char*, const char* const*); extern "C" void APS5_VABI _Exit_nid_postfix(int); extern "C" void APS5_VABI catchReturnFromMain_nid_postfix(int); @@ -27,11 +30,18 @@ void APS5_VABI QuickFirst() { } void Require(bool value) { if (!value) std::abort(); } void UnexpectedCleanup() { std::_Exit(3); } +#ifdef _WIN32 +void WINAPI UnexpectedDetach(PVOID) { TerminateProcess(GetCurrentProcess(), 4); } +#endif } int main(int argc, char** argv) { if (argc > 1 && std::strcmp(argv[1], "--immediate") == 0) { LibcRegisterShutdown_nid_postfix(UnexpectedCleanup); Require(std::atexit(UnexpectedCleanup) == 0); +#ifdef _WIN32 + auto slot = FlsAlloc(UnexpectedDetach); + Require(slot != FLS_OUT_OF_INDEXES && FlsSetValue(slot, &slot)); +#endif _Exit_nid_postfix(0); return 2; } diff --git a/core/libs/tests/GuestSystemFont.cpp b/core/libs/tests/GuestSystemFont.cpp index 40a1c5588..ec2f80785 100644 --- a/core/libs/tests/GuestSystemFont.cpp +++ b/core/libs/tests/GuestSystemFont.cpp @@ -173,6 +173,9 @@ int main() { Require(sceFontSetScalePixel(font, 100.0f, 100.0f) == SCE_FONT_OK); Require(AdvanceOf(font, 'A') == 70.0f); FontGlyphMetrics metrics{}; + Require(sceFontGetCharGlyphMetrics(font, 'A', &metrics) == SCE_FONT_OK); + Require(metrics.width == 50.0f && metrics.height == 70.0f && metrics.Horizontal.bearingX == 10.0f && metrics.Horizontal.bearingY == 70.0f); + Require(metrics.Vertical.bearingX == -25.0f && metrics.Vertical.bearingY == 15.0f && metrics.Vertical.advance == 100.0f); Require(sceFontGetCharGlyphMetrics(font, 'B', &metrics) == SCE_FONT_ERROR_NO_SUPPORT_GLYPH); FontHorizontalLayout layout{}; Require(sceFontGetHorizontalLayout(font, &layout) == SCE_FONT_OK && layout.baselineOffset > 0.0f && layout.lineAdvance >= layout.baselineOffset); diff --git a/core/libs/tests/GuestSystemService.cpp b/core/libs/tests/GuestSystemService.cpp index f013e0f5a..84d4454e9 100644 --- a/core/libs/tests/GuestSystemService.cpp +++ b/core/libs/tests/GuestSystemService.cpp @@ -10,6 +10,8 @@ extern "C" int APS5_VABI sceSystemServiceParamGetString(int paramId, char* buf, namespace { +constexpr int browserUnavailable = static_cast(0x8002002Du); + void Require(bool value) { if (!value) std::abort(); } bool ParamGetStringThrows(int paramId, char* buf, std::size_t bufSize) { @@ -29,6 +31,7 @@ extern "C" int APS5_VABI sceSystemServiceDisableMusicPlayer(void); extern "C" int APS5_VABI sceSystemServiceReenableMusicPlayer(void); extern "C" int APS5_VABI sceSystemServiceDisableMediaPlay(void); extern "C" int APS5_VABI sceSystemServiceReenableMediaPlay(void); +extern "C" int APS5_VABI sceSystemServiceLaunchWebBrowser(const char* uri, void* param); int main() { Require(sceSystemServicePowerTick() == SYSTEM_SERVICE_OK); @@ -44,6 +47,13 @@ int main() { Require(sceSystemServiceDisableMediaPlay() == SYSTEM_SERVICE_OK); Require(sceSystemServiceReenableMediaPlay() == SYSTEM_SERVICE_OK); Require(sceSystemServiceReenableMediaPlay() == SYSTEM_SERVICE_OK); + Require(sceSystemServiceLaunchWebBrowser("http://127.0.0.1:8780/video?v=0", nullptr) == browserUnavailable); + unsigned char browserParam[64]; + std::memset(browserParam, 0x5a, sizeof(browserParam)); + Require(sceSystemServiceLaunchWebBrowser("https://example.com/", browserParam) == browserUnavailable); + for (unsigned char byte : browserParam) Require(byte == 0x5a); + Require(sceSystemServiceLaunchWebBrowser("", nullptr) == browserUnavailable); + Require(sceSystemServiceLaunchWebBrowser(nullptr, nullptr) == browserUnavailable); Require(sceSystemServiceGetHdrToneMapLuminance(nullptr) == SYSTEM_SERVICE_ERROR_PARAMETER); SystemServiceHdrToneMapLuminance luminance{-1.0f, -1.0f, -1.0f}; Require(sceSystemServiceGetHdrToneMapLuminance(&luminance) == SYSTEM_SERVICE_OK); diff --git a/core/libs/tests/GuestTcp.cpp b/core/libs/tests/GuestTcp.cpp index f82ec5ab4..d1f70d97d 100644 --- a/core/libs/tests/GuestTcp.cpp +++ b/core/libs/tests/GuestTcp.cpp @@ -14,6 +14,7 @@ int APS5_VABI accept_nid_postfix(int, void*, std::uint32_t*); std::int64_t APS5_VABI send_nid_postfix(int, const void*, std::uint64_t, int); std::int64_t APS5_VABI recv_nid_postfix(int, void*, std::uint64_t, int); int APS5_VABI getpeername_nid_postfix(int, void*, std::uint32_t*); +int APS5_VABI shutdown_nid_postfix(int, int); int APS5_VABI close_nid_postfix(int); int* APS5_VABI __error_nid_postfix(); } @@ -53,6 +54,8 @@ int main() { Require(send_nid_postfix(client, request, sizeof(request), 0x20000) == sizeof(request)); Require(recv_nid_postfix(accepted, received, sizeof(received), 0) == sizeof(received)); Require(std::strcmp(request, received) == 0); + Require(shutdown_nid_postfix(client, 1) == 0); + Require(send_nid_postfix(client, request, sizeof(request), 0x20000) == -1 && *__error_nid_postfix() == 32); Require(close_nid_postfix(accepted) == 0); #ifndef _WIN32 diff --git a/core/libs/tests/GuestTerminal.cpp b/core/libs/tests/GuestTerminal.cpp new file mode 100644 index 000000000..19653acc3 --- /dev/null +++ b/core/libs/tests/GuestTerminal.cpp @@ -0,0 +1,97 @@ +#include "prx/libc/include/general/VabiMacros.hpp" +#include +#include +#include +#include +#include +#include +#include + +extern "C" { +int APS5_VABI tcgetattr_nid_postfix(int, void*); +int APS5_VABI tcsetattr_nid_postfix(int, int, const void*); +int APS5_VABI open_nid_postfix(const char*, int, int); +int APS5_VABI pipe_nid_postfix(int*); +int APS5_VABI socket_nid_postfix(int, int, int); +int APS5_VABI close_nid_postfix(int); +int* APS5_VABI __error_nid_postfix(); +} + +static void Check(bool value, int line) { + if (!value) { + std::fprintf(stderr, "terminal check failed at line %d, errno %d\n", line, *__error_nid_postfix()); + std::abort(); + } +} +#define Require(value) Check((value), __LINE__) + +constexpr int Now = 0; +constexpr int Drain = 1; +constexpr int Flush = 2; +constexpr int Soft = 0x10; +constexpr int NotTerminal = 25; +constexpr int BadDescriptor = 9; +constexpr int InvalidArgument = 22; +constexpr int BadAddress = 14; +constexpr int AddressNotAvailable = 49; + +static bool Fails(int result, int error) { return result == -1 && *__error_nid_postfix() == error; } + +static void RequireNotTerminal(int descriptor) { + unsigned char attributes[44]; + std::memset(attributes, 0xaa, sizeof(attributes)); + unsigned char untouched[44]; + std::memset(untouched, 0xaa, sizeof(untouched)); + *__error_nid_postfix() = 0; + Require(Fails(tcgetattr_nid_postfix(descriptor, attributes), NotTerminal)); + Require(std::memcmp(attributes, untouched, sizeof(attributes)) == 0); + for (const int action : {Now, Drain, Flush, Now | Soft, Flush | Soft}) { + *__error_nid_postfix() = 0; + Require(Fails(tcsetattr_nid_postfix(descriptor, action, attributes), NotTerminal)); + } +} + +int main() { + const auto name = "anyps5-terminal-test-" + std::to_string(std::chrono::steady_clock::now().time_since_epoch().count()); + const int file = open_nid_postfix(name.c_str(), 0x202, 0644); + Require(file >= 0); + RequireNotTerminal(file); + RequireNotTerminal(0); + RequireNotTerminal(1); + RequireNotTerminal(2); + + int ends[2] = {-1, -1}; + Require(pipe_nid_postfix(ends) == 0); + RequireNotTerminal(ends[0]); + RequireNotTerminal(ends[1]); + + const int socket = socket_nid_postfix(2, 2, 0); + Require(socket >= 0); + unsigned char untouched[44]; + std::memset(untouched, 0xaa, sizeof(untouched)); + unsigned char probe[44]; + std::memset(probe, 0xaa, sizeof(probe)); + Require(Fails(tcgetattr_nid_postfix(socket, probe), AddressNotAvailable)); + Require(std::memcmp(probe, untouched, sizeof(probe)) == 0); + Require(Fails(tcsetattr_nid_postfix(socket, Flush, probe), AddressNotAvailable)); + + unsigned char attributes[44]{}; + Require(Fails(tcsetattr_nid_postfix(file, 3, attributes), InvalidArgument)); + Require(Fails(tcsetattr_nid_postfix(file, -1, attributes), InvalidArgument)); + Require(Fails(tcsetattr_nid_postfix(file, Soft | 3, attributes), InvalidArgument)); + Require(Fails(tcsetattr_nid_postfix(-1, 3, attributes), InvalidArgument)); + Require(Fails(tcsetattr_nid_postfix(file, Now, nullptr), BadAddress)); + Require(Fails(tcsetattr_nid_postfix(-1, Now, nullptr), BadAddress)); + Require(Fails(tcgetattr_nid_postfix(-1, attributes), BadDescriptor)); + Require(Fails(tcsetattr_nid_postfix(-1, Now, attributes), BadDescriptor)); + + Require(close_nid_postfix(socket) == 0); + Require(Fails(tcgetattr_nid_postfix(socket, attributes), BadDescriptor)); + Require(Fails(tcsetattr_nid_postfix(socket, Now, attributes), BadDescriptor)); + Require(close_nid_postfix(ends[0]) == 0 && close_nid_postfix(ends[1]) == 0); + Require(close_nid_postfix(file) == 0); + Require(Fails(tcgetattr_nid_postfix(file, attributes), BadDescriptor)); + Require(Fails(tcsetattr_nid_postfix(file, Flush, attributes), BadDescriptor)); + std::filesystem::remove(name); + return 0; +} diff --git a/core/libs/tests/GuestThreadCpuClock.cpp b/core/libs/tests/GuestThreadCpuClock.cpp new file mode 100644 index 000000000..058ed0969 --- /dev/null +++ b/core/libs/tests/GuestThreadCpuClock.cpp @@ -0,0 +1,111 @@ +#include "SceTypes.hpp" +#include +#include +#include +#include +#include + +extern "C" { +int APS5_VABI pthread_getcpuclockid_nid_postfix(Pthread thread, int* clockId); +Pthread APS5_VABI pthread_self_nid_postfix(void); +int APS5_VABI clock_gettime_nid_postfix(int clockId, KernelTimespec* tp); +int APS5_VABI clock_getres_nid_postfix(int clockId, KernelTimespec* res); +int APS5_VABI scePthreadCreate(Pthread* thread, const PthreadAttr* attr, PthreadEntry entry, void* arg, const char* name); +int APS5_VABI scePthreadJoin(Pthread thread, void** retval); +int* APS5_VABI __error_nid_postfix(); +} + +static constexpr int GUEST_EFAULT = 14; +static constexpr int GUEST_EINVAL = 22; +static constexpr int GUEST_CLOCK_MONOTONIC = 4; +static constexpr int GUEST_CLOCK_THREAD_CPUTIME_ID = 14; +static constexpr std::int64_t NANOS_PER_SECOND = 1000000000LL; +static constexpr std::int64_t NANOS_PER_MILLISECOND = 1000000LL; +static constexpr std::int64_t BURN_NANOS = 200 * NANOS_PER_MILLISECOND; +static constexpr std::int64_t TICK_MARGIN_NANOS = 50 * NANOS_PER_MILLISECOND; +static constexpr std::int64_t GIVE_UP_NANOS = 10 * NANOS_PER_SECOND; + +static void Require(bool value) { if (!value) std::abort(); } + +static std::int64_t Nanos(int clockId) { + KernelTimespec time{-1, -1}; + Require(clock_gettime_nid_postfix(clockId, &time) == 0); + Require(time.tv_sec >= 0); + Require(time.tv_nsec >= 0 && time.tv_nsec < NANOS_PER_SECOND); + return time.tv_sec * NANOS_PER_SECOND + time.tv_nsec; +} + +static int CpuClock(Pthread thread) { + int clockId = 0; + Require(pthread_getcpuclockid_nid_postfix(thread, &clockId) == 0); + return clockId; +} + +static void BurnOwnCpu(std::int64_t nanos) { + const std::int64_t wallStart = Nanos(GUEST_CLOCK_MONOTONIC); + const std::int64_t cpuStart = Nanos(GUEST_CLOCK_THREAD_CPUTIME_ID); + while (Nanos(GUEST_CLOCK_THREAD_CPUTIME_ID) - cpuStart < nanos) + Require(Nanos(GUEST_CLOCK_MONOTONIC) - wallStart < GIVE_UP_NANOS); +} + +static std::atomic workerClock{0}; +static std::atomic workerBurned{false}; +static std::atomic releaseWorker{false}; + +static void* APS5_VABI Worker(void*) { + workerClock.store(CpuClock(pthread_self_nid_postfix())); + BurnOwnCpu(BURN_NANOS); + workerBurned.store(true); + while (!releaseWorker.load()) + std::this_thread::sleep_for(std::chrono::milliseconds(5)); + return nullptr; +} + +static void OwnClockFollowsThreadCpuTime() { + const int own = CpuClock(pthread_self_nid_postfix()); + Require((static_cast(own) & 0xC0000000u) == 0x80000000u); + Require(CpuClock(pthread_self_nid_postfix()) == own); + const std::int64_t start = Nanos(own); + BurnOwnCpu(BURN_NANOS); + const std::int64_t before = Nanos(own); + const std::int64_t reference = Nanos(GUEST_CLOCK_THREAD_CPUTIME_ID); + const std::int64_t after = Nanos(own); + Require(before - start >= BURN_NANOS - TICK_MARGIN_NANOS); + Require(before <= reference + TICK_MARGIN_NANOS && reference <= after + TICK_MARGIN_NANOS); + KernelTimespec ownResolution{-1, -1}; + KernelTimespec threadResolution{-1, -1}; + Require(clock_getres_nid_postfix(own, &ownResolution) == 0); + Require(clock_getres_nid_postfix(GUEST_CLOCK_THREAD_CPUTIME_ID, &threadResolution) == 0); + Require(ownResolution.tv_sec == threadResolution.tv_sec && ownResolution.tv_nsec == threadResolution.tv_nsec); +} + +static void OtherThreadClockReadsThatThread() { + Pthread worker = nullptr; + Require(scePthreadCreate(&worker, nullptr, Worker, nullptr, "cpu clock worker") == 0); + const std::int64_t wallStart = Nanos(GUEST_CLOCK_MONOTONIC); + while (!workerBurned.load()) { + Require(Nanos(GUEST_CLOCK_MONOTONIC) - wallStart < GIVE_UP_NANOS); + std::this_thread::sleep_for(std::chrono::milliseconds(5)); + } + const int clockId = workerClock.load(); + Require(clockId == CpuClock(worker)); + Require(clockId != CpuClock(pthread_self_nid_postfix())); + const std::int64_t burned = Nanos(clockId); + Require(burned >= BURN_NANOS - TICK_MARGIN_NANOS); + BurnOwnCpu(BURN_NANOS); + Require(Nanos(clockId) - burned <= TICK_MARGIN_NANOS); + releaseWorker.store(true); + Require(scePthreadJoin(worker, nullptr) == 0); + KernelTimespec time{-1, -1}; + *__error_nid_postfix() = 0; + Require(clock_gettime_nid_postfix(clockId, &time) == -1); + Require(*__error_nid_postfix() == GUEST_EINVAL); +} + +int main() { + int clockId = 0; + Require(pthread_getcpuclockid_nid_postfix(nullptr, &clockId) == GUEST_EINVAL); + Require(pthread_getcpuclockid_nid_postfix(pthread_self_nid_postfix(), nullptr) == GUEST_EFAULT); + OwnClockFollowsThreadCpuTime(); + OtherThreadClockReadsThatThread(); +} diff --git a/core/libs/tests/GuestThreadSelf.cpp b/core/libs/tests/GuestThreadSelf.cpp index 1358c5045..16c542ab2 100644 --- a/core/libs/tests/GuestThreadSelf.cpp +++ b/core/libs/tests/GuestThreadSelf.cpp @@ -1,6 +1,7 @@ #include "SceTypes.hpp" #include #include +#include extern "C" { int APS5_VABI scePthreadCreate(Pthread* thread, const PthreadAttr* attr, PthreadEntry entry, void* arg, const char* name); @@ -8,6 +9,7 @@ int APS5_VABI scePthreadJoin(Pthread thread, void** retval); int APS5_VABI scePthreadDetach(Pthread thread); void APS5_VABI scePthreadExit(void* retval); Pthread APS5_VABI scePthreadSelf(); +int APS5_VABI scePthreadGetthreadid(void); void APS5_VABI scePthreadTestcancel(); void APS5_VABI pthread_testcancel_nid_postfix(void); int APS5_VABI scePthreadSetcancelstate(int state, int* old_state); @@ -48,9 +50,17 @@ static constexpr std::intptr_t WorkerRetval = 0x1234; static void Require(bool value) { if (!value) std::abort(); } +static std::int64_t ThreadIdField(Pthread thread) { + std::int64_t tid = 0; + std::memcpy(&tid, static_cast(thread), sizeof(tid)); + return tid; +} + struct WorkerContext { Pthread thread = nullptr; Pthread selfFromWorker = nullptr; + std::int64_t workerTid = 0; + int workerThreadId = 0; bool workerStackReported = false; PthreadMutex* mutex = nullptr; int unlockResult = 0; @@ -60,6 +70,8 @@ struct WorkerContext { static void* APS5_VABI Worker(void* arg) { auto& context = *static_cast(arg); context.selfFromWorker = scePthreadSelf(); + context.workerTid = ThreadIdField(context.selfFromWorker); + context.workerThreadId = scePthreadGetthreadid(); int local = 0; context.workerStackReported = StackContains(context.selfFromWorker, &local); context.unlockResult = scePthreadMutexUnlock(context.mutex); @@ -78,6 +90,9 @@ int main() { const Pthread mainSelf = scePthreadSelf(); Require(mainSelf != nullptr); Require(scePthreadSelf() == mainSelf); + const std::int64_t mainTid = ThreadIdField(mainSelf); + Require(mainTid != 0); + Require(scePthreadGetthreadid() == static_cast(mainTid)); int local = 0; Require(StackContains(mainSelf, &local)); Require(scePthreadJoin(mainSelf, nullptr) == SCE_KERNEL_ERROR_EINVAL); @@ -104,6 +119,8 @@ int main() { Require(context.selfFromWorker != nullptr); Require(context.selfFromWorker == context.thread); Require(context.selfFromWorker != mainSelf); + Require(context.workerTid != 0 && context.workerTid != mainTid); + Require(context.workerThreadId == static_cast(context.workerTid)); Require(context.unlockResult == SCE_KERNEL_ERROR_EPERM); Require(context.workerStackReported); Require(context.testcancelReturned); diff --git a/core/libs/tests/GuestTimeConversion.cpp b/core/libs/tests/GuestTimeConversion.cpp index 005c15451..6cd199d70 100644 --- a/core/libs/tests/GuestTimeConversion.cpp +++ b/core/libs/tests/GuestTimeConversion.cpp @@ -1,4 +1,5 @@ #include "prx/libc/include/general/VabiMacros.hpp" +#include "prx/libc/include/GuestTime.hpp" #include #include #include @@ -6,21 +7,23 @@ #include #include #include -#include +#include #include #include extern "C" { -std::tm* APS5_VABI libc_gmtime_nid_postfix(const std::int64_t*); -std::tm* APS5_VABI gmtime_nid_postfix(const std::int64_t*); -std::tm* APS5_VABI libc_localtime_nid_postfix(const std::int64_t*); -std::tm* APS5_VABI localtime_nid_postfix(const std::int64_t*); -std::tm* APS5_VABI gmtime_s_nid_postfix(const std::int64_t*, std::tm*); -std::tm* APS5_VABI localtime_s_nid_postfix(const std::int64_t*, std::tm*); +GuestTm* APS5_VABI libc_gmtime_nid_postfix(const std::int64_t*); +GuestTm* APS5_VABI gmtime_nid_postfix(const std::int64_t*); +GuestTm* APS5_VABI libc_localtime_nid_postfix(const std::int64_t*); +GuestTm* APS5_VABI localtime_nid_postfix(const std::int64_t*); +GuestTm* APS5_VABI gmtime_s_nid_postfix(const std::int64_t*, GuestTm*); +GuestTm* APS5_VABI localtime_s_nid_postfix(const std::int64_t*, GuestTm*); +std::int64_t APS5_VABI mktime_nid_postfix(GuestTm*); +std::size_t APS5_VABI strftime_nid_postfix(char*, std::size_t, const char*, const GuestTm*); } -using Converter = std::tm* (APS5_VABI *)(const std::int64_t*); -using BufferedConverter = std::tm* (APS5_VABI *)(const std::int64_t*, std::tm*); +using Converter = GuestTm* (APS5_VABI *)(const std::int64_t*); +using BufferedConverter = GuestTm* (APS5_VABI *)(const std::int64_t*, GuestTm*); static void Require(bool value, const char* message) { if (!value) { @@ -29,7 +32,7 @@ static void Require(bool value, const char* message) { } } -static bool Equal(const std::tm& left, const std::tm& right) { +static bool Equal(const GuestTm& left, const GuestTm& right) { return left.tm_sec == right.tm_sec && left.tm_min == right.tm_min && left.tm_hour == right.tm_hour && left.tm_mday == right.tm_mday && left.tm_mon == right.tm_mon && left.tm_year == right.tm_year && left.tm_wday == right.tm_wday && left.tm_yday == right.tm_yday && left.tm_isdst == right.tm_isdst; @@ -37,7 +40,7 @@ static bool Equal(const std::tm& left, const std::tm& right) { static void CheckConcurrent(Converter convert, BufferedConverter buffered, const char* name) { constexpr std::array timers{0, 86400, 946684800, 1078012800, 1609459200, 1709164800, 1893456000, 2145916799}; - std::array expected{}; + std::array expected{}; for (std::size_t index = 0; index < timers.size(); ++index) Require(buffered(&timers[index], &expected[index]) == &expected[index], "Buffered conversion failed"); std::barrier start(static_cast(timers.size())); @@ -61,6 +64,30 @@ static void CheckConcurrent(Converter convert, BufferedConverter buffered, const Require(convert(&invalid) == nullptr, "Unrepresentable time was accepted"); } +struct UtcCase { + std::int64_t timer; + int year; + int mon; + int mday; + int hour; + int min; + int sec; + int wday; + int yday; +}; + +static void CheckUtc(const UtcCase& expected) { + GuestTm utc{}; + std::memset(&utc, 0xAA, sizeof(utc)); + Require(gmtime_s_nid_postfix(&expected.timer, &utc) == &utc, "UTC conversion failed"); + Require(utc.tm_year == expected.year && utc.tm_mon == expected.mon && utc.tm_mday == expected.mday + && utc.tm_hour == expected.hour && utc.tm_min == expected.min && utc.tm_sec == expected.sec + && utc.tm_wday == expected.wday && utc.tm_yday == expected.yday && utc.tm_isdst == 0, + "UTC calendar fields are wrong"); + Require(utc.tm_gmtoff == 0, "UTC conversion did not reset tm_gmtoff"); + Require(utc.tm_zone != nullptr && std::strcmp(utc.tm_zone, "UTC") == 0, "UTC conversion did not set tm_zone"); +} + int main() { #ifdef _WIN32 _putenv_s("TZ", "UTC-2"); @@ -69,12 +96,43 @@ int main() { setenv("TZ", "UTC-2", 1); tzset(); #endif + constexpr std::array utcCases{{ + {0, 70, 0, 1, 0, 0, 0, 4, 0}, + {-1, 69, 11, 31, 23, 59, 59, 3, 364}, + {-31536000, 69, 0, 1, 0, 0, 0, 3, 0}, + {951782400, 100, 1, 29, 0, 0, 0, 2, 59}, + {32535216000, 1101, 0, 1, 0, 0, 0, 4, 0}, + {253402300799, 8099, 11, 31, 23, 59, 59, 5, 364}, + {-62135596800, -1899, 0, 1, 0, 0, 0, 1, 0}, + }}; + for (const auto& utcCase : utcCases) CheckUtc(utcCase); + const std::int64_t underflow = std::numeric_limits::min(); + GuestTm scratch{}; + Require(gmtime_s_nid_postfix(&underflow, &scratch) == nullptr, "Unrepresentable negative time was accepted"); + const std::int64_t epoch = 0; - std::tm utc{}, local{}; - Require(gmtime_s_nid_postfix(&epoch, &utc) == &utc && utc.tm_year == 70 && utc.tm_mon == 0 - && utc.tm_mday == 1 && utc.tm_hour == 0, "UTC epoch conversion failed"); + GuestTm local{}; + std::memset(&local, 0xAA, sizeof(local)); Require(localtime_s_nid_postfix(&epoch, &local) == &local && local.tm_year == 70 && local.tm_mon == 0 && local.tm_mday == 1 && local.tm_hour == 2, "Local epoch conversion failed"); + Require(local.tm_gmtoff == 7200, "Local conversion did not set tm_gmtoff"); + Require(local.tm_zone != nullptr && std::strlen(local.tm_zone) > 0, "Local conversion did not set tm_zone"); + + GuestTm roundTrip = local; + roundTrip.tm_wday = -1; + roundTrip.tm_yday = -1; + roundTrip.tm_gmtoff = -1; + roundTrip.tm_zone = nullptr; + Require(mktime_nid_postfix(&roundTrip) == 0, "mktime did not invert localtime"); + Require(roundTrip.tm_wday == 4 && roundTrip.tm_yday == 0, "mktime did not normalize the calendar fields"); + Require(roundTrip.tm_gmtoff == 7200 && roundTrip.tm_zone != nullptr, "mktime did not set the zone fields"); + + GuestTm utc{}; + Require(gmtime_s_nid_postfix(&epoch, &utc) == &utc, "UTC epoch conversion failed"); + char formatted[64]{}; + Require(strftime_nid_postfix(formatted, sizeof(formatted), "%Y-%m-%d %H:%M:%S", &utc) == 19 + && std::strcmp(formatted, "1970-01-01 00:00:00") == 0, "strftime did not read the guest tm"); + CheckConcurrent(libc_gmtime_nid_postfix, gmtime_s_nid_postfix, "Concurrent libc_gmtime returned another thread's date"); CheckConcurrent(gmtime_nid_postfix, gmtime_s_nid_postfix, "Concurrent gmtime returned another thread's date"); CheckConcurrent(libc_localtime_nid_postfix, localtime_s_nid_postfix, "Concurrent libc_localtime returned another thread's date"); diff --git a/core/libs/tests/GuestUltThread.cpp b/core/libs/tests/GuestUltThread.cpp index e46308a40..13ac01a59 100644 --- a/core/libs/tests/GuestUltThread.cpp +++ b/core/libs/tests/GuestUltThread.cpp @@ -13,6 +13,11 @@ std::uint64_t APS5_VABI sceUltUlthreadRuntimeGetWorkAreaSize(std::uint32_t, std: int APS5_VABI sceUltUlthreadRuntimeCreate(void*, const char*, std::uint32_t, std::uint32_t, void*, const void*, std::uint32_t); int APS5_VABI sceUltUlthreadCreate(void*, const char*, UltUlthreadEntry, std::uint64_t, void*, std::uint64_t, void*, const void*, std::uint32_t); int APS5_VABI sceUltUlthreadJoin(void*, std::int32_t*); +int APS5_VABI sceUltUlthreadTryJoin(void*, std::int32_t*); +int APS5_VABI sceUltUlthreadRuntimeDestroy(void*); +int APS5_VABI _sceUltUlthreadRuntimeOptParamInitialize(void*, std::uint32_t); +int APS5_VABI _sceUltUlthreadRuntimeCreate(void*, const char*, std::uint32_t, std::uint32_t, void*, const void*, std::uint32_t); +int APS5_VABI _sceUltUlthreadCreate(void*, const char*, UltUlthreadEntry, std::uint64_t, void*, std::uint64_t, void*, const void*, std::uint32_t); } static constexpr int Ok = 0; @@ -85,10 +90,55 @@ static void Validation() { CHECK(sceUltUlthreadRuntimeGetWorkAreaSize(1, 1) == 256u + 16u * 1024u); } +static void TryJoin() { + Runtime runtime; + Thread thread; + std::uint8_t optParam[128]; + std::memset(optParam, 0xff, sizeof(optParam)); + CHECK(_sceUltUlthreadRuntimeOptParamInitialize(nullptr, 0) == Null); + CHECK(_sceUltUlthreadRuntimeOptParamInitialize(optParam, 0) == Ok && optParam[0] == 0); + CHECK(_sceUltUlthreadRuntimeCreate(&runtime, "runtime", 1, 1, nullptr, nullptr, 0) == Ok); + CHECK(_sceUltUlthreadCreate(&thread, "parked", Parked, 0x3C, nullptr, 0, &runtime, nullptr, 0) == Ok); + while (!gStarted.load(std::memory_order_acquire)) std::this_thread::yield(); + + std::int32_t status = 0; + const int running = sceUltUlthreadTryJoin(&thread, &status); + gRelease.store(true, std::memory_order_release); + int joined; + while ((joined = sceUltUlthreadTryJoin(&thread, &status)) == Busy) std::this_thread::yield(); + + CHECK(running == Busy); + CHECK(joined == Ok && status == 0x3C); + CHECK(sceUltUlthreadTryJoin(&thread, &status) == State); + CHECK(sceUltUlthreadTryJoin(nullptr, &status) == Null); + CHECK(sceUltUlthreadRuntimeDestroy(&runtime) == Ok); +} + +static void Destroy() { + Runtime runtime; + Runtime other; + Thread thread; + CHECK(sceUltUlthreadRuntimeDestroy(nullptr) == Null); + CHECK(sceUltUlthreadRuntimeDestroy(&runtime) == State); + CHECK(_sceUltUlthreadRuntimeCreate(&runtime, "runtime", 1, 1, nullptr, nullptr, 0) == Ok); + CHECK(_sceUltUlthreadRuntimeCreate(&other, "other", 1, 1, nullptr, nullptr, 0) == Ok); + CHECK(_sceUltUlthreadCreate(&thread, "thread", Immediate, 0x11, nullptr, 0, &runtime, nullptr, 0) == Ok); + CHECK(sceUltUlthreadRuntimeDestroy(&other) == Ok); + CHECK(sceUltUlthreadRuntimeDestroy(&runtime) == Busy); + std::int32_t status = 0; + CHECK(sceUltUlthreadJoin(&thread, &status) == Ok && status == 0x11); + CHECK(sceUltUlthreadRuntimeDestroy(&runtime) == Ok); + CHECK(sceUltUlthreadRuntimeDestroy(&runtime) == State); + CHECK(_sceUltUlthreadCreate(&thread, "thread", Immediate, 0, nullptr, 0, &runtime, nullptr, 0) == State); + CHECK(sceUltFinalize() == Ok); +} + int main(int argc, char** argv) { CHECK(argc == 2); CHECK(sceUltInitialize() == Ok); if (std::strcmp(argv[1], "outstanding") == 0) Outstanding(); else if (std::strcmp(argv[1], "validation") == 0) Validation(); + else if (std::strcmp(argv[1], "tryjoin") == 0) TryJoin(); + else if (std::strcmp(argv[1], "destroy") == 0) Destroy(); else CHECK(false); } diff --git a/core/libs/tests/GuestUnwindModuleInfoFailureLinux.cpp b/core/libs/tests/GuestUnwindModuleInfoFailureLinux.cpp new file mode 100644 index 000000000..222fa1766 --- /dev/null +++ b/core/libs/tests/GuestUnwindModuleInfoFailureLinux.cpp @@ -0,0 +1,44 @@ +#include "SceTypes.hpp" +#include "prx/libkernel/KernelErrors.hpp" +#include +#include +#include +#include + +extern "C" int APS5_VABI sceKernelGetModuleInfoForUnwind(std::uint64_t address, int flags, ModuleInfoForUnwind* info); + +static std::uint64_t expectedAddress = 0; +static int calls = 0; + +static void Require(bool value, const char* message) { + if (!value) throw std::runtime_error(message); +} + +extern "C" int APS5_VABI sceKernelGetModuleInfoFromAddr(std::uint64_t address, int flags, ModuleInfoEx* info) { + Require(address == expectedAddress, "module lookup received the wrong address"); + Require(flags == 2, "module lookup received the wrong flags"); + Require(info != nullptr && info->st_size == sizeof(ModuleInfoEx), "module lookup received the wrong structure size"); + ++calls; + return SCE_KERNEL_ERROR_ESRCH; +} + +int main() { + try { + expectedAddress = reinterpret_cast(&Require); + ModuleInfoForUnwind info{}; + bool threw = false; + try { + sceKernelGetModuleInfoForUnwind(expectedAddress, 1, &info); + } catch (const std::runtime_error& error) { + threw = std::string_view(error.what()) == "sceKernelGetModuleInfoForUnwind: failed to query guest module information"; + } + Require(calls == 1, "guest module lookup was not intercepted exactly once"); + Require(threw, "failed guest module lookup must throw instead of reporting success"); + Require(sceKernelGetModuleInfoForUnwind(reinterpret_cast(&sceKernelGetModuleInfoForUnwind), 1, &info) == 0, "host query failed"); + Require(calls == 1, "host query attempted a guest module lookup"); + Require(info.eh_frame_hdr_addr == 0 && info.eh_frame_addr == 0 && info.eh_frame_size == 0, "host query reported guest unwind tables"); + } catch (const std::exception& error) { + std::fprintf(stderr, "%s\n", error.what()); + return 1; + } +} diff --git a/core/libs/tests/GuestUnwindModuleInfoLinux.cpp b/core/libs/tests/GuestUnwindModuleInfoLinux.cpp new file mode 100644 index 000000000..c7c126b26 --- /dev/null +++ b/core/libs/tests/GuestUnwindModuleInfoLinux.cpp @@ -0,0 +1,136 @@ +#include "SceTypes.hpp" +#include +#include +#include +#include +#include + +extern "C" { +int APS5_VABI sceKernelGetModuleInfoForUnwind(uint64_t addr, int flags, ModuleInfoForUnwind* info); +void* APS5_VABI dlopen_nid_postfix(const char* path, int flags); +void* APS5_VABI dlsym_nid_postfix(void* handle, const char* name); +int APS5_VABI dlclose_nid_postfix(void* handle); +} + +static void Require(bool value, const char* message) { + if (!value) throw std::runtime_error(message); +} + +struct TableSearch { + std::uintptr_t address; + std::uintptr_t header = 0; + std::uintptr_t frames = 0; + std::uint64_t size = 0; + bool found = false; +}; + +static std::uint64_t ReadableBytes(const dl_phdr_info& image, std::uintptr_t address) { + for (std::uint16_t i = 0; i < image.dlpi_phnum; ++i) { + const auto& segment = image.dlpi_phdr[i]; + const std::uintptr_t start = image.dlpi_addr + segment.p_vaddr; + if (segment.p_type == PT_LOAD && (segment.p_flags & PF_R) != 0 && address >= start && address - start < segment.p_memsz) + return segment.p_memsz - (address - start); + } + return 0; +} + +static int FindTables(dl_phdr_info* image, std::size_t, void* data) { + auto& search = *static_cast(data); + bool contains = false; + std::uintptr_t headerAddress = 0; + for (std::uint16_t i = 0; i < image->dlpi_phnum; ++i) { + const auto& segment = image->dlpi_phdr[i]; + const std::uintptr_t start = image->dlpi_addr + segment.p_vaddr; + if (segment.p_type == PT_LOAD && search.address >= start && search.address - start < segment.p_memsz) contains = true; + if (segment.p_type == PT_GNU_EH_FRAME) headerAddress = start; + } + if (!contains) return 0; + search.found = true; + search.header = headerAddress; + if (headerAddress == 0) return 1; + Require(ReadableBytes(*image, headerAddress) >= 8, "fixture EH header is outside readable memory"); + const auto* header = reinterpret_cast(headerAddress); + Require(header[0] == 1 && header[1] == 0x1b, "unexpected fixture EH header encoding"); + std::int32_t offset = 0; + std::memcpy(&offset, header + 4, sizeof(offset)); + search.frames = headerAddress + 4 + static_cast(offset); + const auto* frames = reinterpret_cast(search.frames); + const auto available = ReadableBytes(*image, search.frames); + std::uint64_t consumed = 0; + for (;;) { + Require(available - consumed >= 4, "fixture EH table has no mapped terminator"); + std::uint32_t length = 0; + std::memcpy(&length, frames + consumed, sizeof(length)); + consumed += 4; + if (length == 0) break; + std::uint64_t payloadSize = length; + if (length == 0xffffffffu) { + Require(available - consumed >= 8, "fixture extended EH length is outside readable memory"); + std::memcpy(&payloadSize, frames + consumed, sizeof(payloadSize)); + consumed += 8; + } + Require(payloadSize <= available - consumed, "fixture EH record is outside readable memory"); + consumed += payloadSize; + } + Require(consumed > 4, "fixture EH table is empty"); + search.size = consumed; + return 1; +} + +static TableSearch Tables(const void* address) { + TableSearch search{reinterpret_cast(address)}; + dl_iterate_phdr(FindTables, &search); + Require(search.found, "fixture was not found in loaded images"); + return search; +} + +static ModuleInfoForUnwind Query(const void* address) { + ModuleInfoForUnwind info{}; + Require(sceKernelGetModuleInfoForUnwind(reinterpret_cast(address), 1, &info) == 0, "unwind query failed"); + Require(info.st_size == sizeof(ModuleInfoForUnwind), "unwind info has the wrong size"); + Require(reinterpret_cast(address) - info.seg0_addr < info.seg0_size, "unwind segment does not contain the queried address"); + return info; +} + +static void RequireTables(const void* address) { + const auto tables = Tables(address); + Require(tables.header != 0, "fixture has no EH header"); + const auto info = Query(address); + Require(info.eh_frame_hdr_addr == tables.header, "wrong EH header address"); + Require(info.eh_frame_addr == tables.frames, "wrong EH frame address"); + Require(info.eh_frame_size == tables.size, "EH frame size must include the four-byte terminator"); +} + +static void RequireNoTables(const void* address) { + const auto info = Query(address); + Require(info.eh_frame_hdr_addr == 0 && info.eh_frame_addr == 0 && info.eh_frame_size == 0, "unexpected unwind tables"); +} + +static void CheckModule(const char* path, bool hasHeader) { + void* module = dlopen_nid_postfix(path, 2); + Require(module != nullptr, "failed to load guest fixture"); + const void* add = dlsym_nid_postfix(module, "GuestModuleAdd_nid_postfix"); + Require(add != nullptr, "guest fixture export was not found"); + if (hasHeader) RequireTables(add); + else { + Require(Tables(add).header == 0, "no-header fixture unexpectedly has an EH header"); + RequireNoTables(add); + } + Require(dlclose_nid_postfix(module) == 0, "failed to close guest fixture"); +} + +int main(int argc, char** argv) { + try { + Require(argc == 3, "expected guest fixtures with and without an EH header"); + RequireTables(reinterpret_cast(&Query)); + CheckModule(argv[1], true); + CheckModule(argv[2], false); + const auto* host = reinterpret_cast(&sceKernelGetModuleInfoForUnwind); + Require(Tables(host).header != 0, "host fixture has no EH tables to suppress"); + RequireNoTables(host); + RequireNoTables(reinterpret_cast(&dl_iterate_phdr)); + } catch (const std::exception& error) { + std::fprintf(stderr, "%s\n", error.what()); + return 1; + } +} diff --git a/core/libs/tests/GuestVdecsw.cpp b/core/libs/tests/GuestVdecsw.cpp new file mode 100644 index 000000000..99a11fe96 --- /dev/null +++ b/core/libs/tests/GuestVdecsw.cpp @@ -0,0 +1,248 @@ +#include +#include +#include +#include +#include +#include +#include +#include "prx/libc/include/General.hpp" +#include "H264Fixture.hpp" + +struct ComputeMemoryInfo { + std::uint64_t thisSize; + std::uint64_t cpuGpuMemorySize; + void* cpuGpuMemory; +}; + +struct ComputeConfigInfo { + std::uint64_t thisSize; + std::uint16_t computePipeId; + std::uint16_t computeQueueId; + bool checkMemoryType; +}; + +struct DecoderConfigInfo { + std::uint64_t thisSize; + std::uint32_t resourceType; + std::uint32_t codecType; + std::uint32_t profile; + std::uint32_t maxLevel; + std::int32_t maxFrameWidth; + std::int32_t maxFrameHeight; + std::int32_t maxDpbFrameCount; + std::uint32_t decodePipelineDepth; + std::uint64_t computeQueue; + std::uint64_t cpuAffinityMask; + std::int32_t cpuThreadPriority; + bool optimizeProgressiveVideo; + std::uint8_t reserved[19]; +}; + +struct DecoderMemoryInfo { + std::uint64_t thisSize; + std::uint64_t cpuMemorySize; + void* cpuMemory; + std::uint64_t gpuMemorySize; + void* gpuMemory; + std::uint64_t cpuGpuMemorySize; + void* cpuGpuMemory; + std::uint64_t maxFrameBufferSize; + std::uint32_t frameBufferAlignment; +}; + +struct InputData { + std::uint64_t thisSize; + const std::uint8_t* auData; + std::uint64_t auSize; + std::uint64_t ptsData; + std::uint64_t dtsData; + std::uint64_t attachedData; +}; + +struct InputResult { + std::uint64_t thisSize; + const void* decodedAu; + std::uint32_t outputFrameCount; + std::uint32_t reserved; +}; + +struct FrameBuffer { + std::uint64_t thisSize; + void* frameBuffer; + std::uint64_t frameBufferSize; +}; + +struct OutputInfo { + std::uint64_t thisSize; + bool isValid; + bool isLastFrame; + bool isErrorFrame; + std::uint8_t pictureCount; + std::uint32_t codecType; + std::uint32_t frameWidth; + std::uint32_t framePitch; + std::uint32_t frameHeight; + bool isDiscardedFrame; + void* frameBuffer; + std::uint64_t frameBufferSize; + std::uint32_t frameFormat; + std::uint32_t framePitchInBytes; +}; + +struct AvcPictureInfo { + std::uint64_t thisSize; + bool isValid; + std::uint64_t ptsData; + std::uint64_t dtsData; + std::uint64_t attachedData; + std::uint8_t idrPictureFlag; + std::uint8_t profileIdc; + std::uint8_t levelIdc; + std::uint32_t picWidthInMbsMinus1; + std::uint32_t picHeightInMapUnitsMinus1; + std::uint8_t frameMbsOnlyFlag; + std::uint8_t frameCroppingFlag; + std::uint32_t frameCropLeftOffset; + std::uint32_t frameCropRightOffset; + std::uint32_t frameCropTopOffset; + std::uint32_t frameCropBottomOffset; + std::uint8_t vui[44]; +}; + +static_assert(sizeof(DecoderConfigInfo) == 0x50 && sizeof(OutputInfo) == 0x38 && offsetof(OutputInfo, isLastFrame) == 9 && offsetof(OutputInfo, pictureCount) == 0xb && sizeof(AvcPictureInfo) == 0x78); + +extern "C" { +int APS5_VABI sceVdecswQueryComputeMemoryInfo(ComputeMemoryInfo*); +int APS5_VABI sceVdecswAllocateComputeQueue(const ComputeConfigInfo*, const ComputeMemoryInfo*, std::uint64_t*); +int APS5_VABI sceVdecswReleaseComputeQueue(std::uint64_t); +int APS5_VABI sceVdecswQueryDecoderMemoryInfo(const DecoderConfigInfo*, DecoderMemoryInfo*); +int APS5_VABI sceVdecswCreateDecoder(const DecoderConfigInfo*, const DecoderMemoryInfo*, std::uint64_t*); +int APS5_VABI sceVdecswDeleteDecoder(std::uint64_t); +int APS5_VABI sceVdecswSetDecodeInput(std::uint64_t, const InputData*); +int APS5_VABI sceVdecswTrySyncDecodeInput(std::uint64_t, InputResult*); +int APS5_VABI sceVdecswSetDecodeOutput(std::uint64_t, const FrameBuffer*); +int APS5_VABI sceVdecswTrySyncDecodeOutput(std::uint64_t, OutputInfo*); +int APS5_VABI sceVdecswGetAvcPictureInfo(const OutputInfo*, AvcPictureInfo*, AvcPictureInfo*); +int APS5_VABI sceVdecswFinalizeDecodeSequence(std::uint64_t); +int APS5_VABI sceVdecswResetDecoder(std::uint64_t); +} + +namespace { + +using namespace H264Fixture; + +constexpr int OutputPending = static_cast(0x81510115u); +constexpr int InputQueueEmpty = static_cast(0x81510116u); + +void check(bool condition, const std::string& message) { + if (!condition) throw std::runtime_error(message); +} + +struct Session { + std::uint64_t queue = 0; + std::uint64_t decoder = 0; + DecoderMemoryInfo memory{sizeof(DecoderMemoryInfo)}; + + Session() { + ComputeMemoryInfo compute{sizeof(ComputeMemoryInfo)}; + check(sceVdecswQueryComputeMemoryInfo(&compute) == 0 && compute.cpuGpuMemorySize != 0, "compute memory query failed"); + const ComputeConfigInfo computeConfig{sizeof(ComputeConfigInfo), 3, 3, true}; + check(sceVdecswAllocateComputeQueue(&computeConfig, &compute, &queue) == 0, "compute queue allocation failed"); + DecoderConfigInfo config{sizeof(DecoderConfigInfo), 1, 1, 100, 42, static_cast(Width), static_cast(Height), 4, 3, queue, 0x3f, 0, true, {}}; + check(sceVdecswQueryDecoderMemoryInfo(&config, &memory) == 0 && memory.maxFrameBufferSize != 0, "decoder memory query failed"); + check(sceVdecswCreateDecoder(&config, &memory, &decoder) == 0, "decoder creation failed"); + } + + ~Session() { + sceVdecswDeleteDecoder(decoder); + sceVdecswReleaseComputeQueue(queue); + } +}; + +void testDecode() { + Session session; + const auto units = accessUnits(false); + check(units.size() == PictureHashes.size(), "unexpected access unit count"); + std::array, 2> buffers{std::vector(session.memory.maxFrameBufferSize), std::vector(session.memory.maxFrameBufferSize)}; + InputResult result{sizeof(InputResult)}; + check(sceVdecswTrySyncDecodeInput(session.decoder, &result) == InputQueueEmpty && result.decodedAu == nullptr, "an idle decoder did not report an empty input queue"); + std::size_t pictures = 0; + bool outputSet = false; + bool ended = false; + const auto drain = [&] { + for (;;) { + if (!outputSet) { + const FrameBuffer frame{sizeof(FrameBuffer), buffers[pictures % 2].data(), buffers[pictures % 2].size()}; + check(sceVdecswSetDecodeOutput(session.decoder, &frame) == 0, "output buffer was rejected"); + outputSet = true; + } + OutputInfo output{sizeof(OutputInfo)}; + const auto status = sceVdecswTrySyncDecodeOutput(session.decoder, &output); + if (status == OutputPending) return; + check(status == 0, "output sync failed"); + outputSet = false; + if (!output.isValid) { + check(output.isLastFrame && output.pictureCount == 0, "an output without a picture is not the end of the sequence"); + ended = true; + return; + } + check(pictures < PictureHashes.size(), "more pictures than access units"); + check(output.isLastFrame == (pictures + 1 == PictureHashes.size()), "only the last picture of the finalized sequence is marked last"); + check(!output.isErrorFrame && !output.isDiscardedFrame && output.pictureCount == 1 && output.frameWidth == Width && output.frameHeight == Height && output.frameBuffer == buffers[pictures % 2].data(), "unexpected picture geometry"); + AvcPictureInfo info{sizeof(AvcPictureInfo)}; + check(sceVdecswGetAvcPictureInfo(&output, &info, nullptr) == 0 && info.isValid, "picture info missing"); + const auto unit = DisplayOrderUnits[pictures]; + check(info.ptsData == 1000 + unit && info.dtsData == unit && info.attachedData == 0xa0 + unit && info.idrPictureFlag == (pictures == 0 ? 1 : 0) && info.profileIdc == 100, "pictures out of display order"); + check(info.picWidthInMbsMinus1 == Width / 16 - 1 && info.picHeightInMapUnitsMinus1 == (Height + 15) / 16 - 1 && info.frameMbsOnlyFlag == 1 && info.frameCroppingFlag == 1 && info.frameCropRightOffset == 0 && info.frameCropBottomOffset == ((Height + 15) / 16 * 16 - Height) / 2, "picture geometry in the picture info is wrong"); + check(hashNv12(static_cast(output.frameBuffer), output.framePitch) == PictureHashes[pictures], "picture " + std::to_string(pictures) + " differs from the reference"); + ++pictures; + } + }; + std::uint32_t produced = 0; + for (std::size_t unit = 0; unit < units.size(); ++unit) { + const InputData input{sizeof(InputData), units[unit].data(), units[unit].size(), 1000 + unit, unit, 0xa0 + unit}; + check(sceVdecswSetDecodeInput(session.decoder, &input) == 0, "decode input was rejected"); + InputResult consumed{sizeof(InputResult)}; + check(sceVdecswTrySyncDecodeInput(session.decoder, &consumed) == 0 && consumed.decodedAu == units[unit].data(), "an access unit was not consumed"); + produced += consumed.outputFrameCount; + drain(); + } + check(pictures < PictureHashes.size(), "the last picture was returned before the sequence was finalized"); + check(sceVdecswFinalizeDecodeSequence(session.decoder) == 0, "finalize failed"); + while (!ended) drain(); + check(pictures == PictureHashes.size(), "missing pictures after finalizing the sequence"); + check(produced <= PictureHashes.size(), "more pictures counted than decoded"); + check(sceVdecswTrySyncDecodeInput(session.decoder, &result) == InputQueueEmpty, "a finalized decoder did not report an empty input queue"); +} + +void testWithoutOutput() { + Session session; + const auto units = accessUnits(false); + const InputData input{sizeof(InputData), units[0].data(), units[0].size(), 1000, 0, 0xa0}; + check(sceVdecswSetDecodeInput(session.decoder, &input) == 0, "decode input was rejected"); + InputResult result{sizeof(InputResult)}; + check(sceVdecswTrySyncDecodeInput(session.decoder, &result) == 0 && result.decodedAu == units[0].data(), "an input was not decoded before an output buffer was set"); + std::vector buffer(session.memory.maxFrameBufferSize); + const FrameBuffer frame{sizeof(FrameBuffer), buffer.data(), buffer.size()}; + check(sceVdecswSetDecodeOutput(session.decoder, &frame) == 0, "output buffer was rejected"); + OutputInfo output{sizeof(OutputInfo)}; + check(sceVdecswTrySyncDecodeOutput(session.decoder, &output) == OutputPending, "the only picture of an open sequence was returned before it was known to be the last"); + check(sceVdecswFinalizeDecodeSequence(session.decoder) == 0, "finalize failed"); + check(sceVdecswTrySyncDecodeOutput(session.decoder, &output) == 0 && output.isValid && output.isLastFrame, "the only picture of a finalized sequence is not marked last"); + check(sceVdecswResetDecoder(session.decoder) == 0, "reset failed"); + check(sceVdecswTrySyncDecodeInput(session.decoder, &result) == InputQueueEmpty, "a reset decoder kept its inputs"); +} + +} + +int main() { + try { + testDecode(); + testWithoutOutput(); + std::puts("Vdecsw tests passed"); + return 0; + } catch (const std::exception& error) { + std::fprintf(stderr, "%s\n", error.what()); + return 1; + } +} diff --git a/core/libs/tests/GuestVideoOutOutputMode.cpp b/core/libs/tests/GuestVideoOutOutputMode.cpp new file mode 100644 index 000000000..dd8dbb31d --- /dev/null +++ b/core/libs/tests/GuestVideoOutOutputMode.cpp @@ -0,0 +1,88 @@ +#include "SceTypes.hpp" +#include "prx/libc/include/Shutdown.hpp" +#include +#include +#include +#include +#include +#include + +extern "C" { +int APS5_VABI sceVideoOutOpen(int userId, int busType, int index, const void* param); +int APS5_VABI sceVideoOutClose(int handle); +int APS5_VABI sceVideoOutInitializeOutputOptions(VideoOutOutputOptions* options); +int APS5_VABI sceVideoOutIsOutputSupported(int handle, std::uint64_t mode, const VideoOutOutputOptions* options, void* reservedPtr, std::uint64_t reserved); +int APS5_VABI sceVideoOutConfigureOutput(int handle, std::uint64_t mode, const VideoOutOutputOptions* options, void* reservedPtr, std::uint64_t reserved); +} + +static constexpr int SYSTEM_USER = 255; +static constexpr int MAIN_BUS = 0; +static constexpr int INVALID_VALUE = static_cast(0x80290001); +static constexpr int INVALID_ADDRESS = static_cast(0x80290002); +static constexpr int INVALID_HANDLE = static_cast(0x8029000B); +static constexpr int UNSUPPORTED_OUTPUT_MODE = static_cast(0x80290016); +static constexpr int UNKNOWN_OUTPUT_MODE = static_cast(0x8029001E); + +static void Require(bool value) { if (!value) std::abort(); } + +int main() { + std::array raw{}; + raw.fill(0xAA); + VideoOutOutputOptions options{}; + std::memcpy(&options, raw.data(), raw.size()); + Require(sceVideoOutInitializeOutputOptions(&options) == 0); + std::memcpy(raw.data(), &options, raw.size()); + for (std::size_t index = 0; index < raw.size(); ++index) Require(raw[index] == (index == 2 ? 0xFF : 0)); + Require(sceVideoOutInitializeOutputOptions(nullptr) == INVALID_ADDRESS); + + int handle = 0; + try { + for (int unopened : {-1, 0, 1, 0x7FFFFFFF}) { + Require(sceVideoOutIsOutputSupported(unopened, 1, &options, nullptr, 0) == INVALID_HANDLE); + Require(sceVideoOutIsOutputSupported(unopened, 2, nullptr, &options, 1) == INVALID_HANDLE); + } + handle = sceVideoOutOpen(SYSTEM_USER, MAIN_BUS, 0, nullptr); + } catch (const std::runtime_error& error) { + if (std::getenv("ANYPS5_REQUIRE_DISPLAY") != nullptr) throw; + std::printf("skipped, no display or Vulkan device: %s\n", error.what()); + return 77; + } + Require(handle > 0); + Require(sceVideoOutIsOutputSupported(handle, 1, &options, nullptr, 0) == 1); + Require(sceVideoOutIsOutputSupported(handle, 1, nullptr, nullptr, 0) == 1); + for (std::uint64_t mode : {0x4ull, 0x7ull, 0x8ull, 0xCull, 0xDull, 0xEull, 0xFull, 0x10ull, 0x11ull, 0x13ull}) { + Require(sceVideoOutIsOutputSupported(handle, mode, &options, nullptr, 0) == UNSUPPORTED_OUTPUT_MODE); + Require(sceVideoOutConfigureOutput(handle, mode, &options, nullptr, 0) == UNSUPPORTED_OUTPUT_MODE); + } + for (std::uint64_t mode : {0x0ull, 0x2ull, 0x3ull, 0x5ull, 0x6ull, 0x9ull, 0xAull, 0xBull, 0x14ull, 0x20ull, 0x3Full, 0x100ull, 0xD000000Aull, 0x8000000000000001ull, ~0ull}) { + Require(sceVideoOutIsOutputSupported(handle, mode, &options, nullptr, 0) == UNKNOWN_OUTPUT_MODE); + } + bool thrown = false; + try { + sceVideoOutIsOutputSupported(handle, 0x12, &options, nullptr, 0); + } catch (const std::runtime_error&) { + thrown = true; + } + Require(thrown); + + Require(sceVideoOutIsOutputSupported(handle, 1, &options, &options, 0) == INVALID_VALUE); + Require(sceVideoOutIsOutputSupported(handle, 1, &options, nullptr, 1) == INVALID_VALUE); + Require(sceVideoOutIsOutputSupported(handle, 2, &options, nullptr, 1) == INVALID_VALUE); + VideoOutOutputOptions zeroed{}; + Require(sceVideoOutIsOutputSupported(handle, 1, &zeroed, nullptr, 0) == INVALID_VALUE); + Require(sceVideoOutIsOutputSupported(handle, 2, &zeroed, nullptr, 0) == UNKNOWN_OUTPUT_MODE); + Require(sceVideoOutIsOutputSupported(handle, 4, &zeroed, nullptr, 0) == UNSUPPORTED_OUTPUT_MODE); + for (std::size_t word = 0; word < 16; ++word) { + VideoOutOutputOptions changed = options; + changed.internalData[word] ^= 1; + Require(sceVideoOutIsOutputSupported(handle, 1, &changed, nullptr, 0) == (word == 3 ? 1 : INVALID_VALUE)); + } + VideoOutOutputOptions freeWord = options; + freeWord.internalData[3] = 0xFFFFFFFF; + Require(sceVideoOutIsOutputSupported(handle, 1, &freeWord, nullptr, 0) == 1); + Require(sceVideoOutConfigureOutput(handle, 1, &options, nullptr, 0) == 0); + + Require(sceVideoOutClose(handle) == 0); + Require(sceVideoOutIsOutputSupported(handle, 1, &options, nullptr, 0) == INVALID_HANDLE); + LibcRunShutdown_nid_postfix(); +} diff --git a/core/libs/tests/GuestVideoOutResolutionStatus.cpp b/core/libs/tests/GuestVideoOutResolutionStatus.cpp new file mode 100644 index 000000000..0e81b766c --- /dev/null +++ b/core/libs/tests/GuestVideoOutResolutionStatus.cpp @@ -0,0 +1,61 @@ +#include "SceTypes.hpp" +#include "prx/libc/include/Shutdown.hpp" +#include +#include +#include +#include + +extern "C" { +int APS5_VABI sceVideoOutOpen(int userId, int busType, int index, const void* param); +int APS5_VABI sceVideoOutClose(int handle); +int APS5_VABI sceVideoOutGetOutputStatus(int handle, VideoOutOutputStatus* status); +int APS5_VABI sceVideoOutGetResolutionStatus(int handle, VideoOutResolutionStatus* status); +} + +static constexpr int SYSTEM_USER = 255; +static constexpr int MAIN_BUS = 0; +static constexpr int NEVER_OPENED_HANDLE = 2; + +static void Require(bool value) { if (!value) std::abort(); } + +static bool Rejects(int handle, VideoOutResolutionStatus* status) { + try { + sceVideoOutGetResolutionStatus(handle, status); + } catch (const std::runtime_error&) { + return true; + } + return false; +} + +int main() { + int handle = 0; + try { + handle = sceVideoOutOpen(SYSTEM_USER, MAIN_BUS, 0, nullptr); + } catch (const std::runtime_error& error) { + if (std::getenv("ANYPS5_REQUIRE_DISPLAY") != nullptr) throw; + std::printf("skipped, no display or Vulkan device: %s\n", error.what()); + return 77; + } + Require(handle > 0); + + VideoOutResolutionStatus status; + std::memset(&status, 0xff, sizeof(status)); + Require(sceVideoOutGetResolutionStatus(handle, &status) == 0); + Require(status.fullWidth == 1920 && status.fullHeight == 1080); + Require(status.paneWidth == status.fullWidth && status.paneHeight == status.fullHeight); + Require(status.screenSizeInInch == 0.0f && status.flags == 0 && status.reserved0 == 0); + Require(status.reserved1[0] == 0 && status.reserved1[1] == 0 && status.reserved1[2] == 0); + + VideoOutOutputStatus output{}; + Require(sceVideoOutGetOutputStatus(handle, &output) == 0); + Require(status.refreshRate == output.refreshRate); + + Require(Rejects(handle, nullptr)); + Require(Rejects(0, &status)); + Require(Rejects(-1, &status)); + Require(Rejects(NEVER_OPENED_HANDLE, &status)); + + Require(sceVideoOutClose(handle) == 0); + Require(Rejects(handle, &status)); + LibcRunShutdown_nid_postfix(); +} diff --git a/core/libs/tests/GuestVideodec2.cpp b/core/libs/tests/GuestVideodec2.cpp index a0fac8c59..29649d183 100644 --- a/core/libs/tests/GuestVideodec2.cpp +++ b/core/libs/tests/GuestVideodec2.cpp @@ -6,6 +6,7 @@ #include #include #include "prx/libc/include/General.hpp" +#include "H264Fixture.hpp" struct DecoderConfigInfo { std::uint64_t thisSize; @@ -88,111 +89,8 @@ int APS5_VABI sceVideodec2GetAvcPictureInfo_nid_postfix(const OutputInfo*, AvcPi namespace { -constexpr std::uint32_t Width = 64; -constexpr std::uint32_t Height = 40; +using namespace H264Fixture; -constexpr std::array Stream{ - 0x00, 0x00, 0x00, 0x01, 0x09, 0x10, 0x00, 0x00, 0x00, 0x01, 0x67, 0x64, 0x00, 0x0a, 0xac, 0xd9, - 0x44, 0x7f, 0x97, 0x01, 0x10, 0x00, 0x00, 0x03, 0x00, 0x10, 0x00, 0x00, 0x03, 0x03, 0xc0, 0xf1, - 0x22, 0x59, 0x60, 0x00, 0x00, 0x00, 0x01, 0x68, 0xeb, 0xe3, 0xcb, 0x22, 0xc0, 0x00, 0x00, 0x01, - 0x06, 0x05, 0xff, 0xff, 0xa5, 0xdc, 0x45, 0xe9, 0xbd, 0xe6, 0xd9, 0x48, 0xb7, 0x96, 0x2c, 0xd8, - 0x20, 0xd9, 0x23, 0xee, 0xef, 0x78, 0x32, 0x36, 0x34, 0x20, 0x2d, 0x20, 0x63, 0x6f, 0x72, 0x65, - 0x20, 0x31, 0x36, 0x35, 0x20, 0x72, 0x33, 0x32, 0x32, 0x32, 0x20, 0x62, 0x33, 0x35, 0x36, 0x30, - 0x35, 0x61, 0x20, 0x2d, 0x20, 0x48, 0x2e, 0x32, 0x36, 0x34, 0x2f, 0x4d, 0x50, 0x45, 0x47, 0x2d, - 0x34, 0x20, 0x41, 0x56, 0x43, 0x20, 0x63, 0x6f, 0x64, 0x65, 0x63, 0x20, 0x2d, 0x20, 0x43, 0x6f, - 0x70, 0x79, 0x6c, 0x65, 0x66, 0x74, 0x20, 0x32, 0x30, 0x30, 0x33, 0x2d, 0x32, 0x30, 0x32, 0x35, - 0x20, 0x2d, 0x20, 0x68, 0x74, 0x74, 0x70, 0x3a, 0x2f, 0x2f, 0x77, 0x77, 0x77, 0x2e, 0x76, 0x69, - 0x64, 0x65, 0x6f, 0x6c, 0x61, 0x6e, 0x2e, 0x6f, 0x72, 0x67, 0x2f, 0x78, 0x32, 0x36, 0x34, 0x2e, - 0x68, 0x74, 0x6d, 0x6c, 0x20, 0x2d, 0x20, 0x6f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x3a, 0x20, - 0x63, 0x61, 0x62, 0x61, 0x63, 0x3d, 0x31, 0x20, 0x72, 0x65, 0x66, 0x3d, 0x33, 0x20, 0x64, 0x65, - 0x62, 0x6c, 0x6f, 0x63, 0x6b, 0x3d, 0x31, 0x3a, 0x30, 0x3a, 0x30, 0x20, 0x61, 0x6e, 0x61, 0x6c, - 0x79, 0x73, 0x65, 0x3d, 0x30, 0x78, 0x33, 0x3a, 0x30, 0x78, 0x31, 0x31, 0x33, 0x20, 0x6d, 0x65, - 0x3d, 0x68, 0x65, 0x78, 0x20, 0x73, 0x75, 0x62, 0x6d, 0x65, 0x3d, 0x37, 0x20, 0x70, 0x73, 0x79, - 0x3d, 0x31, 0x20, 0x70, 0x73, 0x79, 0x5f, 0x72, 0x64, 0x3d, 0x31, 0x2e, 0x30, 0x30, 0x3a, 0x30, - 0x2e, 0x30, 0x30, 0x20, 0x6d, 0x69, 0x78, 0x65, 0x64, 0x5f, 0x72, 0x65, 0x66, 0x3d, 0x31, 0x20, - 0x6d, 0x65, 0x5f, 0x72, 0x61, 0x6e, 0x67, 0x65, 0x3d, 0x31, 0x36, 0x20, 0x63, 0x68, 0x72, 0x6f, - 0x6d, 0x61, 0x5f, 0x6d, 0x65, 0x3d, 0x31, 0x20, 0x74, 0x72, 0x65, 0x6c, 0x6c, 0x69, 0x73, 0x3d, - 0x31, 0x20, 0x38, 0x78, 0x38, 0x64, 0x63, 0x74, 0x3d, 0x31, 0x20, 0x63, 0x71, 0x6d, 0x3d, 0x30, - 0x20, 0x64, 0x65, 0x61, 0x64, 0x7a, 0x6f, 0x6e, 0x65, 0x3d, 0x32, 0x31, 0x2c, 0x31, 0x31, 0x20, - 0x66, 0x61, 0x73, 0x74, 0x5f, 0x70, 0x73, 0x6b, 0x69, 0x70, 0x3d, 0x31, 0x20, 0x63, 0x68, 0x72, - 0x6f, 0x6d, 0x61, 0x5f, 0x71, 0x70, 0x5f, 0x6f, 0x66, 0x66, 0x73, 0x65, 0x74, 0x3d, 0x2d, 0x32, - 0x20, 0x74, 0x68, 0x72, 0x65, 0x61, 0x64, 0x73, 0x3d, 0x31, 0x20, 0x6c, 0x6f, 0x6f, 0x6b, 0x61, - 0x68, 0x65, 0x61, 0x64, 0x5f, 0x74, 0x68, 0x72, 0x65, 0x61, 0x64, 0x73, 0x3d, 0x31, 0x20, 0x73, - 0x6c, 0x69, 0x63, 0x65, 0x64, 0x5f, 0x74, 0x68, 0x72, 0x65, 0x61, 0x64, 0x73, 0x3d, 0x30, 0x20, - 0x6e, 0x72, 0x3d, 0x30, 0x20, 0x64, 0x65, 0x63, 0x69, 0x6d, 0x61, 0x74, 0x65, 0x3d, 0x31, 0x20, - 0x69, 0x6e, 0x74, 0x65, 0x72, 0x6c, 0x61, 0x63, 0x65, 0x64, 0x3d, 0x30, 0x20, 0x62, 0x6c, 0x75, - 0x72, 0x61, 0x79, 0x5f, 0x63, 0x6f, 0x6d, 0x70, 0x61, 0x74, 0x3d, 0x30, 0x20, 0x63, 0x6f, 0x6e, - 0x73, 0x74, 0x72, 0x61, 0x69, 0x6e, 0x65, 0x64, 0x5f, 0x69, 0x6e, 0x74, 0x72, 0x61, 0x3d, 0x30, - 0x20, 0x62, 0x66, 0x72, 0x61, 0x6d, 0x65, 0x73, 0x3d, 0x32, 0x20, 0x62, 0x5f, 0x70, 0x79, 0x72, - 0x61, 0x6d, 0x69, 0x64, 0x3d, 0x32, 0x20, 0x62, 0x5f, 0x61, 0x64, 0x61, 0x70, 0x74, 0x3d, 0x30, - 0x20, 0x62, 0x5f, 0x62, 0x69, 0x61, 0x73, 0x3d, 0x30, 0x20, 0x64, 0x69, 0x72, 0x65, 0x63, 0x74, - 0x3d, 0x31, 0x20, 0x77, 0x65, 0x69, 0x67, 0x68, 0x74, 0x62, 0x3d, 0x31, 0x20, 0x6f, 0x70, 0x65, - 0x6e, 0x5f, 0x67, 0x6f, 0x70, 0x3d, 0x30, 0x20, 0x77, 0x65, 0x69, 0x67, 0x68, 0x74, 0x70, 0x3d, - 0x32, 0x20, 0x6b, 0x65, 0x79, 0x69, 0x6e, 0x74, 0x3d, 0x36, 0x20, 0x6b, 0x65, 0x79, 0x69, 0x6e, - 0x74, 0x5f, 0x6d, 0x69, 0x6e, 0x3d, 0x31, 0x20, 0x73, 0x63, 0x65, 0x6e, 0x65, 0x63, 0x75, 0x74, - 0x3d, 0x30, 0x20, 0x69, 0x6e, 0x74, 0x72, 0x61, 0x5f, 0x72, 0x65, 0x66, 0x72, 0x65, 0x73, 0x68, - 0x3d, 0x30, 0x20, 0x72, 0x63, 0x5f, 0x6c, 0x6f, 0x6f, 0x6b, 0x61, 0x68, 0x65, 0x61, 0x64, 0x3d, - 0x36, 0x20, 0x72, 0x63, 0x3d, 0x63, 0x72, 0x66, 0x20, 0x6d, 0x62, 0x74, 0x72, 0x65, 0x65, 0x3d, - 0x31, 0x20, 0x63, 0x72, 0x66, 0x3d, 0x32, 0x33, 0x2e, 0x30, 0x20, 0x71, 0x63, 0x6f, 0x6d, 0x70, - 0x3d, 0x30, 0x2e, 0x36, 0x30, 0x20, 0x71, 0x70, 0x6d, 0x69, 0x6e, 0x3d, 0x30, 0x20, 0x71, 0x70, - 0x6d, 0x61, 0x78, 0x3d, 0x36, 0x39, 0x20, 0x71, 0x70, 0x73, 0x74, 0x65, 0x70, 0x3d, 0x34, 0x20, - 0x69, 0x70, 0x5f, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x3d, 0x31, 0x2e, 0x34, 0x30, 0x20, 0x61, 0x71, - 0x3d, 0x31, 0x3a, 0x31, 0x2e, 0x30, 0x30, 0x00, 0x80, 0x00, 0x00, 0x01, 0x65, 0x88, 0x84, 0x00, - 0x21, 0xff, 0x5b, 0xff, 0x78, 0x50, 0x63, 0x8c, 0x32, 0x61, 0x08, 0xb9, 0xb3, 0xab, 0x26, 0xb5, - 0xe1, 0xe9, 0x5b, 0xb7, 0x7b, 0xfd, 0x6b, 0x41, 0x3a, 0x52, 0x12, 0x59, 0x95, 0x0c, 0xa4, 0x3b, - 0xc7, 0x28, 0x9a, 0x87, 0xd9, 0x38, 0xc9, 0xcb, 0x8f, 0x05, 0x47, 0xff, 0xd1, 0x1e, 0xd3, 0x68, - 0xb5, 0x05, 0x8c, 0xb4, 0xf3, 0x18, 0x74, 0x33, 0x57, 0x68, 0xf6, 0xf0, 0xc7, 0xcb, 0xbf, 0x7b, - 0x3d, 0x43, 0x0f, 0xa7, 0x42, 0x5a, 0x78, 0x5b, 0x61, 0x0e, 0xc6, 0x5c, 0xb2, 0xdf, 0xa0, 0xe0, - 0x96, 0x11, 0xdd, 0x02, 0xec, 0x17, 0x80, 0x72, 0x4f, 0x35, 0x05, 0x1e, 0x8b, 0xa8, 0xfc, 0x38, - 0x28, 0xa8, 0xc7, 0x19, 0x5a, 0xa0, 0x8b, 0x96, 0x47, 0x6c, 0xf3, 0x37, 0x65, 0x02, 0x13, 0x8a, - 0x97, 0xbc, 0x86, 0xf8, 0x31, 0xc2, 0xab, 0x7f, 0xe1, 0xa4, 0x68, 0x21, 0xc5, 0xe1, 0x56, 0xca, - 0x96, 0xd7, 0xa3, 0x7f, 0x77, 0x7b, 0xed, 0x28, 0x08, 0xd4, 0xe3, 0x98, 0xdc, 0xc1, 0x2f, 0x0f, - 0x0c, 0x37, 0x06, 0x36, 0xd4, 0x10, 0xdb, 0x7e, 0x99, 0xd4, 0xb1, 0x67, 0xda, 0xbd, 0x28, 0x58, - 0xe6, 0x0e, 0x70, 0x33, 0xe4, 0xb0, 0xb8, 0xd2, 0xb4, 0x60, 0x16, 0x41, 0x1d, 0xda, 0xc5, 0x47, - 0xad, 0xd3, 0x55, 0x30, 0xdf, 0xca, 0xab, 0xb9, 0x91, 0xca, 0xca, 0xe6, 0xfe, 0xb4, 0x9e, 0x55, - 0x18, 0xeb, 0x74, 0x83, 0x01, 0x95, 0xb3, 0x0c, 0x92, 0x4d, 0xb8, 0xfd, 0xa6, 0x28, 0x77, 0xdb, - 0xb9, 0xbc, 0xf2, 0x2c, 0xde, 0xe2, 0x1f, 0x55, 0x39, 0x80, 0x05, 0x67, 0xf1, 0x8c, 0x46, 0xf3, - 0xf8, 0x61, 0x00, 0x00, 0x00, 0x01, 0x09, 0x30, 0x00, 0x00, 0x01, 0x41, 0x9a, 0x23, 0x6c, 0x47, - 0x7f, 0xd6, 0x70, 0xed, 0x07, 0x2d, 0x6f, 0xf0, 0xd9, 0x1c, 0xc5, 0x57, 0x33, 0x9c, 0x0b, 0xe3, - 0x29, 0x4d, 0xc8, 0x18, 0xba, 0x21, 0x49, 0x77, 0xb9, 0xc5, 0x9c, 0xab, 0xcb, 0xca, 0xe9, 0xab, - 0xf7, 0xad, 0xfb, 0x45, 0xda, 0x68, 0x98, 0x7e, 0xff, 0xbd, 0x7e, 0x91, 0xb8, 0x93, 0xb5, 0x78, - 0x93, 0x98, 0x6e, 0x6e, 0xdf, 0x1f, 0x8f, 0x3c, 0x4e, 0xc9, 0x78, 0x17, 0xbf, 0xf4, 0xfb, 0x47, - 0x64, 0x2f, 0xb3, 0xf8, 0xb6, 0xdc, 0x1a, 0xe9, 0xb7, 0x7c, 0xfc, 0x6a, 0x5d, 0xb5, 0x95, 0xb6, - 0x47, 0x6b, 0xfe, 0x5d, 0xf6, 0x8e, 0x6e, 0x85, 0x65, 0x2a, 0x2d, 0x05, 0x21, 0x45, 0x13, 0x0d, - 0x0c, 0xf3, 0xea, 0x6f, 0x58, 0x28, 0x7a, 0x66, 0x54, 0xfd, 0x7e, 0x27, 0xc6, 0x6a, 0xf0, 0x00, - 0x00, 0x00, 0x01, 0x09, 0x50, 0x00, 0x00, 0x01, 0x41, 0x9e, 0x41, 0x78, 0x84, 0x3f, 0xe3, 0x5a, - 0x3c, 0x1e, 0x34, 0x44, 0x68, 0x14, 0xb9, 0xbc, 0x32, 0x4a, 0xc3, 0x1b, 0x54, 0xd4, 0x00, 0x3c, - 0x17, 0xa1, 0xbd, 0xc0, 0x86, 0xe1, 0xbf, 0x7d, 0xf4, 0x08, 0x3c, 0xa3, 0xeb, 0xe0, 0xfd, 0x44, - 0x09, 0x7d, 0x95, 0x20, 0xf4, 0x50, 0x4b, 0x4e, 0xf3, 0x78, 0xa5, 0x9f, 0x14, 0x23, 0x7f, 0xc1, - 0x4e, 0x94, 0xf1, 0x1b, 0xa1, 0x94, 0x86, 0x6d, 0x14, 0x53, 0xa8, 0xe6, 0xa4, 0xd4, 0x28, 0xcd, - 0x86, 0xb1, 0xcb, 0x9f, 0x67, 0x72, 0xc5, 0xf8, 0x99, 0x9a, 0xf3, 0xba, 0x47, 0xdf, 0x41, 0x00, - 0x00, 0x00, 0x01, 0x09, 0x50, 0x00, 0x00, 0x01, 0x01, 0x9e, 0x62, 0x6a, 0x42, 0x5f, 0xdf, 0xac, - 0xe9, 0x76, 0x6c, 0x13, 0x5c, 0xa1, 0x08, 0xbf, 0xd4, 0xd8, 0x03, 0xef, 0xd2, 0x0d, 0xc1, 0xde, - 0x35, 0x3b, 0x9e, 0x3d, 0xe1, 0xf6, 0xd8, 0xe7, 0xae, 0xdf, 0x9f, 0x20, 0x19, 0xbc, 0x50, 0xa9, - 0x69, 0x82, 0x26, 0x04, 0xaf, 0x83, 0x44, 0x79, 0x1b, 0x74, 0x18, 0x2b, 0x00, 0xb8, 0x62, 0xa3, - 0xd1, 0x84, 0x8b, 0x61, 0x87, 0x1a, 0x4f, 0x5b, 0xef, 0x16, 0xeb, 0xb3, 0x1a, 0x06, 0xfd, 0x92, - 0x87, 0x06, 0xc3, 0xec, 0xda, 0xde, 0xec, 0x3d, 0x7d, 0x8a, 0x06, 0x91, 0xc8, 0xdf, 0x59, 0xa7, - 0x58, 0xef, 0x7a, 0x3c, 0x27, 0x1d, 0x00, 0x00, 0x00, 0x01, 0x09, 0x30, 0x00, 0x00, 0x01, 0x41, - 0x9a, 0x65, 0x49, 0xa8, 0x41, 0x68, 0x99, 0x4c, 0x14, 0xf0, 0xc7, 0xde, 0x88, 0x44, 0x37, 0xa7, - 0x7f, 0x10, 0x1e, 0xf9, 0x41, 0xb5, 0x7c, 0x9b, 0xf7, 0x1d, 0x61, 0x0e, 0xe1, 0xec, 0xbc, 0x75, - 0x08, 0x53, 0x8f, 0x68, 0x84, 0x84, 0x93, 0x11, 0x08, 0xf6, 0xb2, 0xe5, 0x6f, 0x30, 0x16, 0xd0, - 0x9d, 0xf2, 0x87, 0x44, 0x40, 0x82, 0x85, 0xcf, 0x16, 0x73, 0x64, 0x1f, 0xb0, 0x22, 0x75, 0xc8, - 0xd7, 0xd2, 0x61, 0xdc, 0x1e, 0x9c, 0xa0, 0xeb, 0x88, 0x7d, 0x1a, 0x55, 0xe4, 0x66, 0x49, 0x5e, - 0xa8, 0xff, 0xc2, 0xb5, 0xb0, 0x46, 0x2a, 0xb5, 0xcd, 0xed, 0x7c, 0xf9, 0xd8, 0xe7, 0x00, 0x00, - 0x00, 0x01, 0x09, 0x50, 0x00, 0x00, 0x01, 0x01, 0x9e, 0x84, 0x6a, 0x42, 0x5f, 0xf9, 0x0f, 0xf0, - 0x52, 0xca, 0xb7, 0x04, 0x3a, 0xd3, 0x0a, 0xb3, 0xaa, 0x47, 0xf0, 0x8c, 0x9f, 0x4b, 0x6f, 0x0e, - 0x38, 0x62, 0x53, 0x04, 0x7d, 0x8f, 0x80, 0xca, 0x62, 0x93, 0xfd, 0xbb, 0x97, 0xef, 0x71, 0xf8, - 0xbc, 0x20, 0x23, 0xd1, 0x9d, 0x50, 0xee, 0xd4, 0x9a, 0x13, 0xb6, 0xfb, 0x78, 0x71, 0x43, 0x78, - 0x65, 0xd6, 0x24, 0xe4, 0xca, 0x6f, 0xb4, 0x42, 0x4d, 0x64, 0xd5, 0xfc, 0x12, 0x29, 0x20, 0x52, - 0xd8, 0x82, 0x78, 0x3d, 0x2f, -}; - -constexpr std::array PictureHashes{ - 0x5871fdc85dfa274eull, 0x26ce4afb974cffc4ull, 0xd2e739633733d5baull, - 0x54feb3d290e028fdull, 0xfa1342a4ba1819c8ull, 0x870125dd7f079a4cull, -}; - -constexpr std::array DisplayOrderUnits{0, 2, 3, 1, 5, 4}; constexpr std::uint32_t CodecHevc = 974921; constexpr std::uint32_t CodecVp9 = 2382845; @@ -370,48 +268,6 @@ void check(bool condition, const std::string& message) { if (!condition) throw std::runtime_error(message); } -std::uint64_t hashNv12(const std::uint8_t* buffer, std::uint32_t pitch) { - std::uint64_t hash = 0xcbf29ce484222325ull; - const auto mix = [&](const std::uint8_t* row) { - for (std::uint32_t column = 0; column < Width; ++column) hash = (hash ^ row[column]) * 0x100000001b3ull; - }; - for (std::uint32_t row = 0; row < Height; ++row) mix(buffer + static_cast(row) * pitch); - const auto* chroma = buffer + static_cast(pitch) * Height; - for (std::uint32_t row = 0; row < Height / 2; ++row) mix(chroma + static_cast(row) * pitch); - return hash; -} - -std::vector> accessUnits(bool lengthPrefixed) { - std::vector starts; - for (std::size_t index = 0; index + 4 < Stream.size(); ++index) { - if (Stream[index] == 0 && Stream[index + 1] == 0 && Stream[index + 2] == 0 && Stream[index + 3] == 1 && (Stream[index + 4] & 0x1fu) == 9) starts.push_back(index); - } - starts.push_back(Stream.size()); - std::vector> units; - for (std::size_t unit = 0; unit + 1 < starts.size(); ++unit) { - if (!lengthPrefixed) { - units.emplace_back(Stream.begin() + static_cast(starts[unit]), Stream.begin() + static_cast(starts[unit + 1])); - continue; - } - std::vector nals; - for (std::size_t index = starts[unit]; index + 3 < starts[unit + 1]; ++index) { - if (Stream[index] == 0 && Stream[index + 1] == 0 && Stream[index + 2] == 1) { - nals.push_back(index + 3); - index += 2; - } - } - std::vector converted; - for (std::size_t nal = 0; nal < nals.size(); ++nal) { - auto end = nal + 1 < nals.size() ? nals[nal + 1] - 3 : starts[unit + 1]; - while (end > nals[nal] && Stream[end - 1] == 0) --end; - const auto length = end - nals[nal]; - for (int shift = 24; shift >= 0; shift -= 8) converted.push_back(static_cast(length >> shift)); - converted.insert(converted.end(), Stream.begin() + static_cast(nals[nal]), Stream.begin() + static_cast(end)); - } - units.push_back(std::move(converted)); - } - return units; -} template void checkThrows(TAction action) { diff --git a/core/libs/tests/GuestWaitpid.cpp b/core/libs/tests/GuestWaitpid.cpp new file mode 100644 index 000000000..235c57d91 --- /dev/null +++ b/core/libs/tests/GuestWaitpid.cpp @@ -0,0 +1,31 @@ +#include "prx/libc/include/general/VabiMacros.hpp" +#include +#include +extern "C" { +int APS5_VABI waitpid_nid_postfix(int, int*, int); +int APS5_VABI getpid_nid_postfix(void); +int* APS5_VABI __error_nid_postfix(); +} +constexpr int noChild = 10; +constexpr int invalidArgument = 22; +constexpr int untouched = 0x5a5a5a5a; +static void Require(bool value) { if (!value) std::abort(); } +static void RequireFailure(int pid, int options, int error) { + int status = untouched; + *__error_nid_postfix() = 0; + Require(waitpid_nid_postfix(pid, &status, options) == -1); + Require(*__error_nid_postfix() == error); + Require(status == untouched); + *__error_nid_postfix() = 0; + Require(waitpid_nid_postfix(pid, nullptr, options) == -1); + Require(*__error_nid_postfix() == error); +} +int main() { + const int self = getpid_nid_postfix(); + for (int pid : {-1, 0, 1, self, self + 1, -self}) { + for (int options : {0, 1, 2, 3, 4, 8, 16, 32, 0x2c, 0x3f, static_cast(0x80000000u), static_cast(0x80000001u), static_cast(0x8000003fu)}) + RequireFailure(pid, options, noChild); + for (int options : {0x40, 0x41, 0x80, 0x100, 0x40000000, static_cast(0x80000040u), static_cast(0xffffffffu)}) + RequireFailure(pid, options, invalidArgument); + } +} diff --git a/core/libs/tests/GuestWcsrtombs.cpp b/core/libs/tests/GuestWcsrtombs.cpp new file mode 100644 index 000000000..745e80990 --- /dev/null +++ b/core/libs/tests/GuestWcsrtombs.cpp @@ -0,0 +1,75 @@ +#include "prx/libc/include/general/VabiMacros.hpp" +#include +#include +#include +#include +#include + +extern "C" std::size_t APS5_VABI wcsrtombs_nid_postfix(char*, const std::uint16_t**, std::size_t, void*); +extern "C" int* APS5_VABI __error_nid_postfix(); + +static void Require(bool value, const char* message) { + if (!value) { + std::fprintf(stderr, "%s\n", message); + std::exit(1); + } +} + +int main() { + const std::uint16_t text[] = {'a', 'b', 'c', 0}; + const std::uint16_t empty[] = {0}; + const std::uint16_t invalid[] = {'a', 0x100, 'b', 0}; + const std::uint16_t bytes[] = {0x7f, 0x80, 0xe9, 0xff, 0}; + constexpr auto Failed = static_cast(-1); + std::uint64_t state[2]{}; + char out[8]; + const auto* source = text; + std::memset(out, 'x', sizeof(out)); + *__error_nid_postfix() = 7; + Require(wcsrtombs_nid_postfix(out, &source, sizeof(out), state) == 3, "full conversion length"); + Require(source == nullptr && std::memcmp(out, "abc\0xxxx", 8) == 0, "full conversion and terminator"); + Require(*__error_nid_postfix() == 7, "successful conversion preserves errno"); + + source = text; + std::memset(out, 'x', sizeof(out)); + Require(wcsrtombs_nid_postfix(out, &source, 0, nullptr) == 0, "zero capacity length"); + Require(source == text && out[0] == 'x', "zero capacity leaves input and output alone"); + Require(wcsrtombs_nid_postfix(out, &source, 2, nullptr) == 2, "partial conversion length"); + Require(source == text + 2 && std::memcmp(out, "abxxxxxx", 8) == 0, "partial conversion has no terminator"); + Require(wcsrtombs_nid_postfix(out + 2, &source, 1, nullptr) == 1, "exact capacity length"); + Require(source == text + 3 && std::memcmp(out, "abcxxxxx", 8) == 0, "exact capacity leaves terminator pending"); + Require(wcsrtombs_nid_postfix(out + 3, &source, 1, nullptr) == 0, "resume at terminator"); + Require(source == nullptr && std::memcmp(out, "abc\0xxxx", 8) == 0, "resumed conversion terminates"); + + source = empty; + std::memset(out, 'x', sizeof(out)); + Require(wcsrtombs_nid_postfix(out, &source, 1, state) == 0, "empty string length"); + Require(source == nullptr && out[0] == 0 && out[1] == 'x', "empty string terminator"); + source = bytes; + Require(wcsrtombs_nid_postfix(out, &source, sizeof(out), state) == 4, "guest single-byte conversion length"); + Require(source == nullptr && std::memcmp(out, "\x7f\x80\xe9\xff\0", 5) == 0, "guest high bytes"); + + source = text; + Require(wcsrtombs_nid_postfix(nullptr, &source, 0, state) == 3 && source == text, "length query ignores capacity and preserves source"); + Require(wcsrtombs_nid_postfix(nullptr, &source, 1, nullptr) == 3 && source == text, "length query with implicit state"); + source = empty; + Require(wcsrtombs_nid_postfix(nullptr, &source, 0, state) == 0 && source == empty, "empty length query"); + + source = invalid; + std::memset(out, 'x', sizeof(out)); + *__error_nid_postfix() = 0; + Require(wcsrtombs_nid_postfix(out, &source, sizeof(out), state) == Failed, "invalid character fails conversion"); + Require(*__error_nid_postfix() == 86, "invalid character uses guest EILSEQ"); + Require(source == invalid + 1 && std::memcmp(out, "axxxxxxx", 8) == 0, "error preserves prefix and identifies invalid character"); + *__error_nid_postfix() = 7; + Require(wcsrtombs_nid_postfix(out, &source, 0, state) == 0 && source == invalid + 1, "zero capacity does not convert invalid character"); + Require(*__error_nid_postfix() == 7, "zero capacity preserves errno"); + + source = invalid; + Require(wcsrtombs_nid_postfix(nullptr, &source, 0, state) == Failed, "length query rejects invalid character"); + Require(*__error_nid_postfix() == 86 && source == invalid, "failed length query preserves source and reports guest EILSEQ"); + source = invalid; + *__error_nid_postfix() = 7; + Require(wcsrtombs_nid_postfix(out, &source, 1, state) == 1 && source == invalid + 1, "capacity stops before invalid character"); + Require(*__error_nid_postfix() == 7, "unconverted invalid character does not set errno"); +} diff --git a/core/libs/tests/GuestWideFormatting.cpp b/core/libs/tests/GuestWideFormatting.cpp index c758ba10b..facf372a6 100644 --- a/core/libs/tests/GuestWideFormatting.cpp +++ b/core/libs/tests/GuestWideFormatting.cpp @@ -8,6 +8,7 @@ extern "C" int APS5_VABI vswprintf_nid_postfix(char16_t*, std::size_t, const char16_t*, VaList*); extern "C" int APS5_VABI snwprintf_s_nid_postfix(char16_t*, std::size_t, const char16_t*, ...); +extern "C" int APS5_VABI swprintf_nid_postfix(char16_t*, std::size_t, const char16_t*, ...); static int APS5_VABI Format(char16_t* buffer, std::size_t size, const char16_t* format, ...) { #ifdef _WIN32 @@ -111,9 +112,34 @@ static void CheckCount() { Require(Format(buffer, 16, u"a%n", nullCount) < 0, "vswprintf null %n argument"); } +static void CheckSwprintf() { + char16_t buffer[8]; + for (auto& unit : buffer) unit = u'x'; + Require(swprintf_nid_postfix(buffer, 8, u"%d-%ls", 42, u"ab") == 5, "swprintf length"); + Require(buffer == std::u16string(u"42-ab") && buffer[6] == u'x', "swprintf output"); + Require(swprintf_nid_postfix(buffer, 8, u"中%ls%s", u"æ–‡", "\xc3\xa9") == 3, "swprintf non-ASCII length"); + Require(buffer == std::u16string(u"中文é"), "swprintf non-ASCII output"); + Require(swprintf_nid_postfix(buffer, 8, u"%s", "\xf0\x9f\x98\x80") == 2, "swprintf surrogate pair length"); + Require(buffer == std::u16string(u"\U0001f600"), "swprintf surrogate pair output"); + Require(swprintf_nid_postfix(buffer, 8, u"") == 0 && buffer[0] == 0, "swprintf empty format"); + for (auto& unit : buffer) unit = u'x'; + Require(swprintf_nid_postfix(buffer, 3, u"%ls", u"abcd") < 0, "swprintf truncation result"); + Require(buffer == std::u16string(u"ab") && buffer[3] == u'x', "swprintf truncation output"); + Require(swprintf_nid_postfix(buffer, 3, u"abc") < 0, "swprintf terminator does not fit"); + buffer[0] = u'x'; + Require(swprintf_nid_postfix(buffer, 0, u"a") < 0 && buffer[0] == u'x', "swprintf zero size"); + Require(swprintf_nid_postfix(nullptr, 8, u"a") < 0, "swprintf null buffer"); + Require(swprintf_nid_postfix(buffer, 5, u"%ls", u"abcd") == 4 && buffer == std::u16string(u"abcd"), "swprintf exact fit"); + char16_t wide[16]; + Require(swprintf_nid_postfix(wide, 16, u"%5ls|%-3c|%lc", u"wide", 'z', 0x20ac) == 11, "swprintf padded length"); + Require(wide == std::u16string(u" wide|z |€"), "swprintf padded output"); + Require(swprintf_nid_postfix(wide, 16, u"%q") < 0 && wide[0] == 0, "swprintf invalid conversion"); +} + int main() { CheckBounded(); CheckCount(); + CheckSwprintf(); Check(u"%.2s", "\xc3\xa9\xc3\xa8", u"\u00e9\u00e8"); Check(u"%.1s", "\xc3\xa9\xc3\xa8", u"\u00e9"); Check(u"%.0s", "\xc3\xa9", u""); diff --git a/core/libs/tests/GuestWideStrings.cpp b/core/libs/tests/GuestWideStrings.cpp index e08349ab4..2e01b4d33 100644 --- a/core/libs/tests/GuestWideStrings.cpp +++ b/core/libs/tests/GuestWideStrings.cpp @@ -2,6 +2,7 @@ #include #include #include +#include extern "C" { const char16_t* APS5_VABI wmemchr_nid_postfix(const char16_t* s, char16_t c, std::size_t n); @@ -125,6 +126,14 @@ int main() { require(wcstoull_nid_postfix(u"18446744073709551615", nullptr, 10) == ULLONG_MAX); const char16_t* letters = u"abc"; require(wcstol_nid_postfix(letters, &end, 10) == 0 && end == letters); + const char16_t* binary = u" -0b101"; + for (const int base : {0, 2}) { + require(wcstol_nid_postfix(binary, &end, base) == 0 && end == binary + 3); + require(wcstoll_nid_postfix(binary, &end, base) == 0 && end == binary + 3); + require(wcstoul_nid_postfix(binary, &end, base) == 0 && end == binary + 3); + require(wcstoull_nid_postfix(binary, &end, base) == 0 && end == binary + 3); + } + require(wcstoll_nid_postfix(binary + 2, &end, 16) == 0xb101 && end == binary + 7); const char16_t* scientific = u"3.5e2!"; require(wcstod_nid_postfix(scientific, &end) == 350.0 && end == scientific + 5); diff --git a/core/libs/tests/GuestWorkingDirectory.cpp b/core/libs/tests/GuestWorkingDirectory.cpp index feeb9a9f4..df874768e 100644 --- a/core/libs/tests/GuestWorkingDirectory.cpp +++ b/core/libs/tests/GuestWorkingDirectory.cpp @@ -1,5 +1,7 @@ +#include "prx/libc/include/ApplicationHeap.hpp" #include "prx/libc/include/General.hpp" #include "prx/libc/include/GuestHeap.hpp" +#include #include #include #include @@ -10,7 +12,15 @@ char* APS5_VABI getcwd_nid_postfix(char*, std::size_t); int* APS5_VABI __error_nid_postfix(); } static void Require(bool value) { if (!value) std::abort(); } +static int allocations = 0; +static int releases = 0; +static void* APS5_VABI Allocate(std::size_t bytes) { ++allocations; return GuestHeap::GuestHeapAllocate_nid_postfix(bytes); } +static void APS5_VABI Release(void* pointer) { ++releases; GuestHeap::GuestHeapFree_nid_postfix(pointer); } +static void* APS5_VABI Unused() { std::abort(); } int main() { + const std::array api{reinterpret_cast(&Allocate), reinterpret_cast(&Release), reinterpret_cast(&Unused), + reinterpret_cast(&Unused), reinterpret_cast(&Unused), reinterpret_cast(&Unused), reinterpret_cast(&Unused)}; + ApplicationHeapRegister_nid_no_patch(api.data()); const auto host = std::filesystem::canonical(std::filesystem::current_path()); char path[1024]; Require(getcwd_nid_postfix(path, sizeof(path)) == path && std::strcmp(path, "/") == 0); @@ -26,9 +36,11 @@ int main() { char tiny[] = "xyz"; Require(getcwd_nid_postfix(tiny, 2) == nullptr && *__error_nid_postfix() == 34); Require(std::strcmp(tiny, "xyz") == 0); + Require(getcwd_nid_postfix(path, 0) == nullptr && *__error_nid_postfix() == 22); char* allocated = getcwd_nid_postfix(nullptr, 0); - Require(allocated && std::strcmp(allocated, path) == 0); - GuestHeap::GuestHeapFree_nid_postfix(allocated); + Require(allocated && std::strcmp(allocated, path) == 0 && allocations == 1); + ApplicationHeapFree_nid_no_patch(allocated); + Require(releases == 1); Require(chdir_nid_postfix("sample.txt") == -1 && *__error_nid_postfix() == 20); Require(chdir_nid_postfix("missing") == -1 && *__error_nid_postfix() == 2); Require(chdir_nid_postfix("..") == 0); diff --git a/core/libs/tests/GuestZlib.cpp b/core/libs/tests/GuestZlib.cpp new file mode 100644 index 000000000..1245c9169 --- /dev/null +++ b/core/libs/tests/GuestZlib.cpp @@ -0,0 +1,77 @@ +#include "prx/libc/include/general/VabiMacros.hpp" +#include +#include +#include +#include +#include +#include + +extern "C" { +std::int32_t APS5_VABI sceZlibInitialize(const void*, std::uint64_t); +std::int32_t APS5_VABI sceZlibFinalize(); +std::int32_t APS5_VABI sceZlibInflate(const void*, std::uint32_t, void*, std::uint32_t, std::uint64_t*); +std::int32_t APS5_VABI sceZlibWaitForDone(std::uint64_t*, std::uint32_t*); +std::int32_t APS5_VABI sceZlibGetResult(std::uint64_t, std::uint32_t*, std::int32_t*); +} + +static void Require(bool value) { if (!value) std::abort(); } + +template +static void RequireThrows(TAction action) { + try { action(); } catch (const std::runtime_error&) { return; } + std::abort(); +} + +int main() { + constexpr std::array compressed{0x78, 0x9c, 0xed, 0xca, 0xb1, 0xd, 0x80, 0x20, 0x10, 0x40, 0xd1, 0x9e, 0x29, 0x6e, 0x2, 0x3a, 0x6, 0x60, 0x3, 0x12, 0x26, 0x10, 0x30, 0x91, 0x88, 0x47, 0x71, 0xa1, 0xd0, 0xe9, 0x1d, 0x82, 0xf6, 0xbf, 0xfa, 0x45, 0x7d, 0x53, 0xe, 0xf2, 0x8d, 0x5e, 0xa4, 0x5e, 0x67, 0xbd, 0x6d, 0x3d, 0x72, 0x68, 0x93, 0x32, 0x97, 0x36, 0xf3, 0x2e, 0x12, 0x8, 0x4, 0x2, 0x81, 0x40, 0x20, 0x10, 0x8, 0x4, 0x2, 0x81, 0x40, 0x20, 0xec, 0x85, 0x1f, 0xf5, 0xf3, 0xbd, 0x4c}; + constexpr char text[] = "AnyPS5 zlib checksum and bounds.\n"; + constexpr std::uint32_t expectedLength = (sizeof(text) - 1) * 128; + alignas(2048) std::array storage{}; + auto* output = storage.data() + 2048; + std::uint64_t id = 0; + std::uint32_t produced = 0; + std::int32_t status = 0; + Require(sceZlibFinalize() == static_cast(0x81120032)); + Require(sceZlibInflate(nullptr, 0, nullptr, 0, nullptr) == static_cast(0x81120032)); + RequireThrows([&] { sceZlibInitialize(nullptr, 1); }); + Require(sceZlibInitialize(nullptr, 0) == 0); + Require(sceZlibInitialize(nullptr, 0) == static_cast(0x81120033)); + RequireThrows([&] { sceZlibInflate(compressed.data(), compressed.size(), output + 1, expectedLength, &id); }); + RequireThrows([&] { sceZlibInflate(compressed.data(), compressed.size(), output, 65537, &id); }); + RequireThrows([&] { sceZlibInflate(nullptr, compressed.size(), output, expectedLength, &id); }); + std::uint64_t previous = 0; + auto decode = [&](const void* input, std::uint32_t inputLength, std::uint32_t capacity, std::int32_t expectedStatus) { + storage.fill(0xA5); + Require(sceZlibInflate(input, inputLength, output, capacity, &id) == 0); + Require(id > previous); + previous = id; + RequireThrows([&] { sceZlibInflate(input, inputLength, output, capacity, &id); }); + RequireThrows([&] { sceZlibFinalize(); }); + std::uint32_t timeout = 0; + RequireThrows([&] { sceZlibWaitForDone(&id, &timeout); }); + std::uint64_t completed; + Require(sceZlibWaitForDone(&completed, nullptr) == 0); + Require(completed == id); + RequireThrows([&] { sceZlibGetResult(id + 1, &produced, &status); }); + Require(sceZlibGetResult(id, &produced, &status) == 0); + Require(status == expectedStatus && produced <= capacity); + RequireThrows([&] { sceZlibGetResult(id, &produced, &status); }); + Require(std::all_of(storage.begin(), storage.begin() + 2048, [](auto byte) { return byte == 0xA5; })); + Require(std::all_of(storage.begin() + 2048 + capacity, storage.end(), [](auto byte) { return byte == 0xA5; })); + }; + decode(compressed.data(), compressed.size(), expectedLength, 0); + Require(produced == expectedLength); + for (std::uint32_t offset = 0; offset < produced; offset += sizeof(text) - 1) + Require(std::memcmp(output + offset, text, sizeof(text) - 1) == 0); + decode(compressed.data(), compressed.size(), expectedLength - 1, static_cast(0x8112001C)); + auto corrupt = compressed; + corrupt.back() ^= 1; + decode(corrupt.data(), corrupt.size(), expectedLength, static_cast(0x811200FF)); + decode(compressed.data(), compressed.size() - 1, 65536, static_cast(0x811200FF)); + Require(sceZlibFinalize() == 0); + Require(sceZlibGetResult(id, &produced, &status) == static_cast(0x81120032)); + std::array scratch{}; + Require(sceZlibInitialize(scratch.data(), scratch.size()) == 0); + decode(compressed.data(), compressed.size(), 65536, 0); + Require(sceZlibFinalize() == 0); +} diff --git a/core/libs/tests/H264Fixture.hpp b/core/libs/tests/H264Fixture.hpp new file mode 100644 index 000000000..f63718b67 --- /dev/null +++ b/core/libs/tests/H264Fixture.hpp @@ -0,0 +1,159 @@ +#pragma once + +#include +#include +#include +#include + +namespace H264Fixture { + +inline constexpr std::uint32_t Width = 64; +inline constexpr std::uint32_t Height = 40; + +inline constexpr std::array Stream{ + 0x00, 0x00, 0x00, 0x01, 0x09, 0x10, 0x00, 0x00, 0x00, 0x01, 0x67, 0x64, 0x00, 0x0a, 0xac, 0xd9, + 0x44, 0x7f, 0x97, 0x01, 0x10, 0x00, 0x00, 0x03, 0x00, 0x10, 0x00, 0x00, 0x03, 0x03, 0xc0, 0xf1, + 0x22, 0x59, 0x60, 0x00, 0x00, 0x00, 0x01, 0x68, 0xeb, 0xe3, 0xcb, 0x22, 0xc0, 0x00, 0x00, 0x01, + 0x06, 0x05, 0xff, 0xff, 0xa5, 0xdc, 0x45, 0xe9, 0xbd, 0xe6, 0xd9, 0x48, 0xb7, 0x96, 0x2c, 0xd8, + 0x20, 0xd9, 0x23, 0xee, 0xef, 0x78, 0x32, 0x36, 0x34, 0x20, 0x2d, 0x20, 0x63, 0x6f, 0x72, 0x65, + 0x20, 0x31, 0x36, 0x35, 0x20, 0x72, 0x33, 0x32, 0x32, 0x32, 0x20, 0x62, 0x33, 0x35, 0x36, 0x30, + 0x35, 0x61, 0x20, 0x2d, 0x20, 0x48, 0x2e, 0x32, 0x36, 0x34, 0x2f, 0x4d, 0x50, 0x45, 0x47, 0x2d, + 0x34, 0x20, 0x41, 0x56, 0x43, 0x20, 0x63, 0x6f, 0x64, 0x65, 0x63, 0x20, 0x2d, 0x20, 0x43, 0x6f, + 0x70, 0x79, 0x6c, 0x65, 0x66, 0x74, 0x20, 0x32, 0x30, 0x30, 0x33, 0x2d, 0x32, 0x30, 0x32, 0x35, + 0x20, 0x2d, 0x20, 0x68, 0x74, 0x74, 0x70, 0x3a, 0x2f, 0x2f, 0x77, 0x77, 0x77, 0x2e, 0x76, 0x69, + 0x64, 0x65, 0x6f, 0x6c, 0x61, 0x6e, 0x2e, 0x6f, 0x72, 0x67, 0x2f, 0x78, 0x32, 0x36, 0x34, 0x2e, + 0x68, 0x74, 0x6d, 0x6c, 0x20, 0x2d, 0x20, 0x6f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x3a, 0x20, + 0x63, 0x61, 0x62, 0x61, 0x63, 0x3d, 0x31, 0x20, 0x72, 0x65, 0x66, 0x3d, 0x33, 0x20, 0x64, 0x65, + 0x62, 0x6c, 0x6f, 0x63, 0x6b, 0x3d, 0x31, 0x3a, 0x30, 0x3a, 0x30, 0x20, 0x61, 0x6e, 0x61, 0x6c, + 0x79, 0x73, 0x65, 0x3d, 0x30, 0x78, 0x33, 0x3a, 0x30, 0x78, 0x31, 0x31, 0x33, 0x20, 0x6d, 0x65, + 0x3d, 0x68, 0x65, 0x78, 0x20, 0x73, 0x75, 0x62, 0x6d, 0x65, 0x3d, 0x37, 0x20, 0x70, 0x73, 0x79, + 0x3d, 0x31, 0x20, 0x70, 0x73, 0x79, 0x5f, 0x72, 0x64, 0x3d, 0x31, 0x2e, 0x30, 0x30, 0x3a, 0x30, + 0x2e, 0x30, 0x30, 0x20, 0x6d, 0x69, 0x78, 0x65, 0x64, 0x5f, 0x72, 0x65, 0x66, 0x3d, 0x31, 0x20, + 0x6d, 0x65, 0x5f, 0x72, 0x61, 0x6e, 0x67, 0x65, 0x3d, 0x31, 0x36, 0x20, 0x63, 0x68, 0x72, 0x6f, + 0x6d, 0x61, 0x5f, 0x6d, 0x65, 0x3d, 0x31, 0x20, 0x74, 0x72, 0x65, 0x6c, 0x6c, 0x69, 0x73, 0x3d, + 0x31, 0x20, 0x38, 0x78, 0x38, 0x64, 0x63, 0x74, 0x3d, 0x31, 0x20, 0x63, 0x71, 0x6d, 0x3d, 0x30, + 0x20, 0x64, 0x65, 0x61, 0x64, 0x7a, 0x6f, 0x6e, 0x65, 0x3d, 0x32, 0x31, 0x2c, 0x31, 0x31, 0x20, + 0x66, 0x61, 0x73, 0x74, 0x5f, 0x70, 0x73, 0x6b, 0x69, 0x70, 0x3d, 0x31, 0x20, 0x63, 0x68, 0x72, + 0x6f, 0x6d, 0x61, 0x5f, 0x71, 0x70, 0x5f, 0x6f, 0x66, 0x66, 0x73, 0x65, 0x74, 0x3d, 0x2d, 0x32, + 0x20, 0x74, 0x68, 0x72, 0x65, 0x61, 0x64, 0x73, 0x3d, 0x31, 0x20, 0x6c, 0x6f, 0x6f, 0x6b, 0x61, + 0x68, 0x65, 0x61, 0x64, 0x5f, 0x74, 0x68, 0x72, 0x65, 0x61, 0x64, 0x73, 0x3d, 0x31, 0x20, 0x73, + 0x6c, 0x69, 0x63, 0x65, 0x64, 0x5f, 0x74, 0x68, 0x72, 0x65, 0x61, 0x64, 0x73, 0x3d, 0x30, 0x20, + 0x6e, 0x72, 0x3d, 0x30, 0x20, 0x64, 0x65, 0x63, 0x69, 0x6d, 0x61, 0x74, 0x65, 0x3d, 0x31, 0x20, + 0x69, 0x6e, 0x74, 0x65, 0x72, 0x6c, 0x61, 0x63, 0x65, 0x64, 0x3d, 0x30, 0x20, 0x62, 0x6c, 0x75, + 0x72, 0x61, 0x79, 0x5f, 0x63, 0x6f, 0x6d, 0x70, 0x61, 0x74, 0x3d, 0x30, 0x20, 0x63, 0x6f, 0x6e, + 0x73, 0x74, 0x72, 0x61, 0x69, 0x6e, 0x65, 0x64, 0x5f, 0x69, 0x6e, 0x74, 0x72, 0x61, 0x3d, 0x30, + 0x20, 0x62, 0x66, 0x72, 0x61, 0x6d, 0x65, 0x73, 0x3d, 0x32, 0x20, 0x62, 0x5f, 0x70, 0x79, 0x72, + 0x61, 0x6d, 0x69, 0x64, 0x3d, 0x32, 0x20, 0x62, 0x5f, 0x61, 0x64, 0x61, 0x70, 0x74, 0x3d, 0x30, + 0x20, 0x62, 0x5f, 0x62, 0x69, 0x61, 0x73, 0x3d, 0x30, 0x20, 0x64, 0x69, 0x72, 0x65, 0x63, 0x74, + 0x3d, 0x31, 0x20, 0x77, 0x65, 0x69, 0x67, 0x68, 0x74, 0x62, 0x3d, 0x31, 0x20, 0x6f, 0x70, 0x65, + 0x6e, 0x5f, 0x67, 0x6f, 0x70, 0x3d, 0x30, 0x20, 0x77, 0x65, 0x69, 0x67, 0x68, 0x74, 0x70, 0x3d, + 0x32, 0x20, 0x6b, 0x65, 0x79, 0x69, 0x6e, 0x74, 0x3d, 0x36, 0x20, 0x6b, 0x65, 0x79, 0x69, 0x6e, + 0x74, 0x5f, 0x6d, 0x69, 0x6e, 0x3d, 0x31, 0x20, 0x73, 0x63, 0x65, 0x6e, 0x65, 0x63, 0x75, 0x74, + 0x3d, 0x30, 0x20, 0x69, 0x6e, 0x74, 0x72, 0x61, 0x5f, 0x72, 0x65, 0x66, 0x72, 0x65, 0x73, 0x68, + 0x3d, 0x30, 0x20, 0x72, 0x63, 0x5f, 0x6c, 0x6f, 0x6f, 0x6b, 0x61, 0x68, 0x65, 0x61, 0x64, 0x3d, + 0x36, 0x20, 0x72, 0x63, 0x3d, 0x63, 0x72, 0x66, 0x20, 0x6d, 0x62, 0x74, 0x72, 0x65, 0x65, 0x3d, + 0x31, 0x20, 0x63, 0x72, 0x66, 0x3d, 0x32, 0x33, 0x2e, 0x30, 0x20, 0x71, 0x63, 0x6f, 0x6d, 0x70, + 0x3d, 0x30, 0x2e, 0x36, 0x30, 0x20, 0x71, 0x70, 0x6d, 0x69, 0x6e, 0x3d, 0x30, 0x20, 0x71, 0x70, + 0x6d, 0x61, 0x78, 0x3d, 0x36, 0x39, 0x20, 0x71, 0x70, 0x73, 0x74, 0x65, 0x70, 0x3d, 0x34, 0x20, + 0x69, 0x70, 0x5f, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x3d, 0x31, 0x2e, 0x34, 0x30, 0x20, 0x61, 0x71, + 0x3d, 0x31, 0x3a, 0x31, 0x2e, 0x30, 0x30, 0x00, 0x80, 0x00, 0x00, 0x01, 0x65, 0x88, 0x84, 0x00, + 0x21, 0xff, 0x5b, 0xff, 0x78, 0x50, 0x63, 0x8c, 0x32, 0x61, 0x08, 0xb9, 0xb3, 0xab, 0x26, 0xb5, + 0xe1, 0xe9, 0x5b, 0xb7, 0x7b, 0xfd, 0x6b, 0x41, 0x3a, 0x52, 0x12, 0x59, 0x95, 0x0c, 0xa4, 0x3b, + 0xc7, 0x28, 0x9a, 0x87, 0xd9, 0x38, 0xc9, 0xcb, 0x8f, 0x05, 0x47, 0xff, 0xd1, 0x1e, 0xd3, 0x68, + 0xb5, 0x05, 0x8c, 0xb4, 0xf3, 0x18, 0x74, 0x33, 0x57, 0x68, 0xf6, 0xf0, 0xc7, 0xcb, 0xbf, 0x7b, + 0x3d, 0x43, 0x0f, 0xa7, 0x42, 0x5a, 0x78, 0x5b, 0x61, 0x0e, 0xc6, 0x5c, 0xb2, 0xdf, 0xa0, 0xe0, + 0x96, 0x11, 0xdd, 0x02, 0xec, 0x17, 0x80, 0x72, 0x4f, 0x35, 0x05, 0x1e, 0x8b, 0xa8, 0xfc, 0x38, + 0x28, 0xa8, 0xc7, 0x19, 0x5a, 0xa0, 0x8b, 0x96, 0x47, 0x6c, 0xf3, 0x37, 0x65, 0x02, 0x13, 0x8a, + 0x97, 0xbc, 0x86, 0xf8, 0x31, 0xc2, 0xab, 0x7f, 0xe1, 0xa4, 0x68, 0x21, 0xc5, 0xe1, 0x56, 0xca, + 0x96, 0xd7, 0xa3, 0x7f, 0x77, 0x7b, 0xed, 0x28, 0x08, 0xd4, 0xe3, 0x98, 0xdc, 0xc1, 0x2f, 0x0f, + 0x0c, 0x37, 0x06, 0x36, 0xd4, 0x10, 0xdb, 0x7e, 0x99, 0xd4, 0xb1, 0x67, 0xda, 0xbd, 0x28, 0x58, + 0xe6, 0x0e, 0x70, 0x33, 0xe4, 0xb0, 0xb8, 0xd2, 0xb4, 0x60, 0x16, 0x41, 0x1d, 0xda, 0xc5, 0x47, + 0xad, 0xd3, 0x55, 0x30, 0xdf, 0xca, 0xab, 0xb9, 0x91, 0xca, 0xca, 0xe6, 0xfe, 0xb4, 0x9e, 0x55, + 0x18, 0xeb, 0x74, 0x83, 0x01, 0x95, 0xb3, 0x0c, 0x92, 0x4d, 0xb8, 0xfd, 0xa6, 0x28, 0x77, 0xdb, + 0xb9, 0xbc, 0xf2, 0x2c, 0xde, 0xe2, 0x1f, 0x55, 0x39, 0x80, 0x05, 0x67, 0xf1, 0x8c, 0x46, 0xf3, + 0xf8, 0x61, 0x00, 0x00, 0x00, 0x01, 0x09, 0x30, 0x00, 0x00, 0x01, 0x41, 0x9a, 0x23, 0x6c, 0x47, + 0x7f, 0xd6, 0x70, 0xed, 0x07, 0x2d, 0x6f, 0xf0, 0xd9, 0x1c, 0xc5, 0x57, 0x33, 0x9c, 0x0b, 0xe3, + 0x29, 0x4d, 0xc8, 0x18, 0xba, 0x21, 0x49, 0x77, 0xb9, 0xc5, 0x9c, 0xab, 0xcb, 0xca, 0xe9, 0xab, + 0xf7, 0xad, 0xfb, 0x45, 0xda, 0x68, 0x98, 0x7e, 0xff, 0xbd, 0x7e, 0x91, 0xb8, 0x93, 0xb5, 0x78, + 0x93, 0x98, 0x6e, 0x6e, 0xdf, 0x1f, 0x8f, 0x3c, 0x4e, 0xc9, 0x78, 0x17, 0xbf, 0xf4, 0xfb, 0x47, + 0x64, 0x2f, 0xb3, 0xf8, 0xb6, 0xdc, 0x1a, 0xe9, 0xb7, 0x7c, 0xfc, 0x6a, 0x5d, 0xb5, 0x95, 0xb6, + 0x47, 0x6b, 0xfe, 0x5d, 0xf6, 0x8e, 0x6e, 0x85, 0x65, 0x2a, 0x2d, 0x05, 0x21, 0x45, 0x13, 0x0d, + 0x0c, 0xf3, 0xea, 0x6f, 0x58, 0x28, 0x7a, 0x66, 0x54, 0xfd, 0x7e, 0x27, 0xc6, 0x6a, 0xf0, 0x00, + 0x00, 0x00, 0x01, 0x09, 0x50, 0x00, 0x00, 0x01, 0x41, 0x9e, 0x41, 0x78, 0x84, 0x3f, 0xe3, 0x5a, + 0x3c, 0x1e, 0x34, 0x44, 0x68, 0x14, 0xb9, 0xbc, 0x32, 0x4a, 0xc3, 0x1b, 0x54, 0xd4, 0x00, 0x3c, + 0x17, 0xa1, 0xbd, 0xc0, 0x86, 0xe1, 0xbf, 0x7d, 0xf4, 0x08, 0x3c, 0xa3, 0xeb, 0xe0, 0xfd, 0x44, + 0x09, 0x7d, 0x95, 0x20, 0xf4, 0x50, 0x4b, 0x4e, 0xf3, 0x78, 0xa5, 0x9f, 0x14, 0x23, 0x7f, 0xc1, + 0x4e, 0x94, 0xf1, 0x1b, 0xa1, 0x94, 0x86, 0x6d, 0x14, 0x53, 0xa8, 0xe6, 0xa4, 0xd4, 0x28, 0xcd, + 0x86, 0xb1, 0xcb, 0x9f, 0x67, 0x72, 0xc5, 0xf8, 0x99, 0x9a, 0xf3, 0xba, 0x47, 0xdf, 0x41, 0x00, + 0x00, 0x00, 0x01, 0x09, 0x50, 0x00, 0x00, 0x01, 0x01, 0x9e, 0x62, 0x6a, 0x42, 0x5f, 0xdf, 0xac, + 0xe9, 0x76, 0x6c, 0x13, 0x5c, 0xa1, 0x08, 0xbf, 0xd4, 0xd8, 0x03, 0xef, 0xd2, 0x0d, 0xc1, 0xde, + 0x35, 0x3b, 0x9e, 0x3d, 0xe1, 0xf6, 0xd8, 0xe7, 0xae, 0xdf, 0x9f, 0x20, 0x19, 0xbc, 0x50, 0xa9, + 0x69, 0x82, 0x26, 0x04, 0xaf, 0x83, 0x44, 0x79, 0x1b, 0x74, 0x18, 0x2b, 0x00, 0xb8, 0x62, 0xa3, + 0xd1, 0x84, 0x8b, 0x61, 0x87, 0x1a, 0x4f, 0x5b, 0xef, 0x16, 0xeb, 0xb3, 0x1a, 0x06, 0xfd, 0x92, + 0x87, 0x06, 0xc3, 0xec, 0xda, 0xde, 0xec, 0x3d, 0x7d, 0x8a, 0x06, 0x91, 0xc8, 0xdf, 0x59, 0xa7, + 0x58, 0xef, 0x7a, 0x3c, 0x27, 0x1d, 0x00, 0x00, 0x00, 0x01, 0x09, 0x30, 0x00, 0x00, 0x01, 0x41, + 0x9a, 0x65, 0x49, 0xa8, 0x41, 0x68, 0x99, 0x4c, 0x14, 0xf0, 0xc7, 0xde, 0x88, 0x44, 0x37, 0xa7, + 0x7f, 0x10, 0x1e, 0xf9, 0x41, 0xb5, 0x7c, 0x9b, 0xf7, 0x1d, 0x61, 0x0e, 0xe1, 0xec, 0xbc, 0x75, + 0x08, 0x53, 0x8f, 0x68, 0x84, 0x84, 0x93, 0x11, 0x08, 0xf6, 0xb2, 0xe5, 0x6f, 0x30, 0x16, 0xd0, + 0x9d, 0xf2, 0x87, 0x44, 0x40, 0x82, 0x85, 0xcf, 0x16, 0x73, 0x64, 0x1f, 0xb0, 0x22, 0x75, 0xc8, + 0xd7, 0xd2, 0x61, 0xdc, 0x1e, 0x9c, 0xa0, 0xeb, 0x88, 0x7d, 0x1a, 0x55, 0xe4, 0x66, 0x49, 0x5e, + 0xa8, 0xff, 0xc2, 0xb5, 0xb0, 0x46, 0x2a, 0xb5, 0xcd, 0xed, 0x7c, 0xf9, 0xd8, 0xe7, 0x00, 0x00, + 0x00, 0x01, 0x09, 0x50, 0x00, 0x00, 0x01, 0x01, 0x9e, 0x84, 0x6a, 0x42, 0x5f, 0xf9, 0x0f, 0xf0, + 0x52, 0xca, 0xb7, 0x04, 0x3a, 0xd3, 0x0a, 0xb3, 0xaa, 0x47, 0xf0, 0x8c, 0x9f, 0x4b, 0x6f, 0x0e, + 0x38, 0x62, 0x53, 0x04, 0x7d, 0x8f, 0x80, 0xca, 0x62, 0x93, 0xfd, 0xbb, 0x97, 0xef, 0x71, 0xf8, + 0xbc, 0x20, 0x23, 0xd1, 0x9d, 0x50, 0xee, 0xd4, 0x9a, 0x13, 0xb6, 0xfb, 0x78, 0x71, 0x43, 0x78, + 0x65, 0xd6, 0x24, 0xe4, 0xca, 0x6f, 0xb4, 0x42, 0x4d, 0x64, 0xd5, 0xfc, 0x12, 0x29, 0x20, 0x52, + 0xd8, 0x82, 0x78, 0x3d, 0x2f, +}; + +inline constexpr std::array PictureHashes{ + 0x5871fdc85dfa274eull, 0x26ce4afb974cffc4ull, 0xd2e739633733d5baull, + 0x54feb3d290e028fdull, 0xfa1342a4ba1819c8ull, 0x870125dd7f079a4cull, +}; + +inline constexpr std::array DisplayOrderUnits{0, 2, 3, 1, 5, 4}; + +inline std::uint64_t hashNv12(const std::uint8_t* buffer, std::uint32_t pitch) { + std::uint64_t hash = 0xcbf29ce484222325ull; + const auto mix = [&](const std::uint8_t* row) { + for (std::uint32_t column = 0; column < Width; ++column) hash = (hash ^ row[column]) * 0x100000001b3ull; + }; + for (std::uint32_t row = 0; row < Height; ++row) mix(buffer + static_cast(row) * pitch); + const auto* chroma = buffer + static_cast(pitch) * Height; + for (std::uint32_t row = 0; row < Height / 2; ++row) mix(chroma + static_cast(row) * pitch); + return hash; +} + +inline std::vector> accessUnits(bool lengthPrefixed) { + std::vector starts; + for (std::size_t index = 0; index + 4 < Stream.size(); ++index) { + if (Stream[index] == 0 && Stream[index + 1] == 0 && Stream[index + 2] == 0 && Stream[index + 3] == 1 && (Stream[index + 4] & 0x1fu) == 9) starts.push_back(index); + } + starts.push_back(Stream.size()); + std::vector> units; + for (std::size_t unit = 0; unit + 1 < starts.size(); ++unit) { + if (!lengthPrefixed) { + units.emplace_back(Stream.begin() + static_cast(starts[unit]), Stream.begin() + static_cast(starts[unit + 1])); + continue; + } + std::vector nals; + for (std::size_t index = starts[unit]; index + 3 < starts[unit + 1]; ++index) { + if (Stream[index] == 0 && Stream[index + 1] == 0 && Stream[index + 2] == 1) { + nals.push_back(index + 3); + index += 2; + } + } + std::vector converted; + for (std::size_t nal = 0; nal < nals.size(); ++nal) { + auto end = nal + 1 < nals.size() ? nals[nal + 1] - 3 : starts[unit + 1]; + while (end > nals[nal] && Stream[end - 1] == 0) --end; + const auto length = end - nals[nal]; + for (int shift = 24; shift >= 0; shift -= 8) converted.push_back(static_cast(length >> shift)); + converted.insert(converted.end(), Stream.begin() + static_cast(nals[nal]), Stream.begin() + static_cast(end)); + } + units.push_back(std::move(converted)); + } + return units; +} + +} diff --git a/core/libs/tests/HostThreadLocal.cpp b/core/libs/tests/HostThreadLocal.cpp index 0a3639bd2..5419c1a04 100644 --- a/core/libs/tests/HostThreadLocal.cpp +++ b/core/libs/tests/HostThreadLocal.cpp @@ -1,5 +1,6 @@ #include #include +#include #include #include #ifdef _WIN32 @@ -30,6 +31,8 @@ int main() { worker = std::async(std::launch::async, [] { TouchHostThreadLocal(); TouchHostThreadLocal(); }); } for (auto& worker : workers) worker.get(); + const auto deadline = std::chrono::steady_clock::now() + std::chrono::seconds(5); + while (DestroyedHostThreadLocals() < before + 8 && std::chrono::steady_clock::now() < deadline) std::this_thread::sleep_for(std::chrono::milliseconds(1)); if (DestroyedHostThreadLocals() != before + 8) std::abort(); } diff --git a/core/libs/tests/ImageSampleGather.cpp b/core/libs/tests/ImageSampleGather.cpp index 5332d836a..95d335262 100644 --- a/core/libs/tests/ImageSampleGather.cpp +++ b/core/libs/tests/ImageSampleGather.cpp @@ -2,6 +2,7 @@ #include "RdnaDecoder/RdnaImageOpDecoder.hpp" #include #include +#include using namespace ShaderRecompiler; static void Require(bool value) { if (!value) throw std::runtime_error("image sample/gather regression"); } @@ -37,6 +38,16 @@ static void CheckPckRejectsZeroMask(std::uint32_t encoding) { } Require(rejected); } +static void CheckUnsupportedOpcodeDiagnostic(std::uint32_t encoding, std::string_view expected) { + const std::array code{(0x3cu << 26u) | ((encoding & 0x7fu) << 18u) | (encoding >> 7u), 0u}; + try { + DecodeRdnaMimg(0u, code, 0u); + } catch (const std::runtime_error& error) { + Require(std::string_view(error.what()).find(expected) != std::string_view::npos); + return; + } + Require(false); +} int main() { Check(0x26u, RdnaOpcode::ImageSampleBCl, 0xfu); Check(0x36u, RdnaOpcode::ImageSampleBClO, 0xfu); @@ -88,4 +99,6 @@ int main() { CheckPck(0x63u, RdnaOpcode::ImageGather8hPck, 0xFu, 4u); CheckPckRejectsZeroMask(0x62u); CheckPckRejectsZeroMask(0x63u); + CheckUnsupportedOpcodeDiagnostic(0x06u, "MIMG opcode 0x06"); + CheckUnsupportedOpcodeDiagnostic(0xffu, "MIMG opcode 0xff"); } diff --git a/core/libs/tests/PatchedLibraries.cpp b/core/libs/tests/PatchedLibraries.cpp index ec0b43273..67e0cad55 100644 --- a/core/libs/tests/PatchedLibraries.cpp +++ b/core/libs/tests/PatchedLibraries.cpp @@ -12,6 +12,9 @@ int main(int argc, char** argv) { std::fprintf(stderr, "usage: %s \n", argv[0]); return 2; } +#ifdef _WIN32 + SetErrorMode(SEM_FAILCRITICALERRORS); +#endif int loaded = 0; int failed = 0; for (const auto& entry : std::filesystem::directory_iterator(argv[1])) { diff --git a/core/libs/tests/PixelAncillary.cpp b/core/libs/tests/PixelAncillary.cpp index 2d6f10dad..b7be6fc81 100644 --- a/core/libs/tests/PixelAncillary.cpp +++ b/core/libs/tests/PixelAncillary.cpp @@ -2,25 +2,54 @@ #include "Optimization/ConstantFolder.hpp" #include "Optimization/DeadCodeEliminator.hpp" #include "Optimization/ShaderInfoCollector.hpp" +#include #include #include #include using namespace ShaderRecompiler; static void Require(bool value) { if (!value) throw std::runtime_error("packed pixel ancillary regression"); } -static IrValue& Build(IrProgram& program, IrOpcode opcode, std::uint32_t offset, std::uint32_t count) { + +constexpr std::uint32_t UnmodelledBits = 0xE000F0FFu; +constexpr std::uint32_t SelectOther = 0x5A3C9E17u; + +struct Row { + std::uint32_t sample; + std::uint32_t layer; + bool frontFacing; +}; +constexpr std::array Rows {{{0u, 0u, false}, {0u, 0u, true}, {15u, 0x1FFFu, false}, {15u, 0x1FFFu, true}, {0u, 0x1FFFu, true}, {15u, 0u, false}}}; + +static IrBlock& Begin(IrProgram& program) { program.Resources().stage = IrShaderStage::Pixel; program.Resources().resourceTrackingComplete = true; auto& block = program.CreateBlock(); program.SetEntryBlock(block); program.BlockOrder().push_back(&block); - IrBuilder builder(program); - builder.SetInsertionPoint(block); - auto& ancillary = builder.Emit(IrOpcode::GetBuiltin, IrType::U32, {&builder.Constant(static_cast(StageInputKind::PackedAncillary)), &builder.Constant(0u)}); - auto& user = opcode == IrOpcode::BitwiseOr32 ? builder.Emit(opcode, IrType::U32, {&ancillary, &builder.Constant(offset)}) - : builder.Emit(opcode, IrType::U32, {&ancillary, &builder.Constant(offset), &builder.Constant(count)}); + return block; +} +static IrValue& AncillaryBuiltin(IrBuilder& builder) { + return builder.Emit(IrOpcode::GetBuiltin, IrType::U32, {&builder.Constant(static_cast(StageInputKind::PackedAncillary)), &builder.Constant(0u)}); +} +static IrValue& FrontFacingBuiltin(IrBuilder& builder) { + return builder.Emit(IrOpcode::GetBuiltin, IrType::U1, {&builder.Constant(static_cast(StageInputKind::FrontFacing)), &builder.Constant(0u)}); +} +static void End(IrBuilder& builder, IrValue& user) { static_cast(builder.Emit(IrOpcode::ReferenceU32, IrType::Void, {&user})); static_cast(builder.Emit(IrOpcode::Return, IrType::Void, {})); +} +static IrValue& Build(IrProgram& program, IrOpcode opcode, std::uint32_t offset, std::uint32_t count, bool throughSelect = false) { + auto& block = Begin(program); + IrBuilder builder(program); + builder.SetInsertionPoint(block); + IrValue& ancillary = AncillaryBuiltin(builder); + IrValue* source = &ancillary; + if (throughSelect) { + source = &builder.Emit(IrOpcode::SelectU32, IrType::U32, {&FrontFacingBuiltin(builder), &ancillary, &builder.Constant(SelectOther)}); + } + IrValue& user = opcode == IrOpcode::BitwiseOr32 ? builder.Emit(opcode, IrType::U32, {source, &builder.Constant(offset)}) + : builder.Emit(opcode, IrType::U32, {source, &builder.Constant(offset), &builder.Constant(count)}); + End(builder, user); return user; } static void Lower(IrProgram& program) { @@ -30,6 +59,59 @@ static void Lower(IrProgram& program) { const ShaderPixelInputInfo pixel {}; ShaderInfoCollector().Collect(program, ShaderStageInputInfo {nullptr, &pixel, nullptr}); } +static void ExtractVector(std::uint32_t offset, std::uint32_t count, StageInputKind kind, std::uint32_t fieldOffset) { + IrProgram program; + program.Resources().stage = IrShaderStage::Pixel; + program.Resources().resourceTrackingComplete = true; + auto& block = program.CreateBlock(); + program.SetEntryBlock(block); + program.BlockOrder().push_back(&block); + IrBuilder builder(program); + builder.SetInsertionPoint(block); + auto& ancillary = builder.Emit(IrOpcode::GetBuiltin, IrType::U32, {&builder.Constant(static_cast(StageInputKind::PackedAncillary)), &builder.Constant(0u)}); + auto& maskedOffset = builder.BitwiseAnd(builder.Constant(offset), builder.Constant(31u)); + auto& maskedCount = builder.BitwiseAnd(builder.Constant(count), builder.Constant(31u)); + auto& available = builder.ISub(builder.Constant(32u), maskedOffset); + auto& clampedCount = builder.Emit(IrOpcode::UMin32, IrType::U32, {&maskedCount, &available}); + auto& user = builder.Emit(IrOpcode::BitFieldUExtract, IrType::U32, {&ancillary, &maskedOffset, &clampedCount}); + static_cast(builder.Emit(IrOpcode::ReferenceU32, IrType::Void, {&user})); + static_cast(builder.Emit(IrOpcode::Return, IrType::Void, {})); + Lower(program); + const IrValue* field = user.Argument(0)->Resolve(); + Require(field->Opcode() == IrOpcode::GetBuiltin); + Require(static_cast(field->Argument(0)->Resolve()->ImmediateU32()) == kind); + Require(user.Argument(1)->Resolve()->ImmediateU32() == fieldOffset); + Require(user.Argument(2)->Resolve()->ImmediateU32() == count); +} + +static void ExtractVectorMaskedCopy(std::uint32_t offset, std::uint32_t count, StageInputKind kind, std::uint32_t fieldOffset) { + IrProgram program; + program.Resources().stage = IrShaderStage::Pixel; + program.Resources().resourceTrackingComplete = true; + auto& block = program.CreateBlock(); + program.SetEntryBlock(block); + program.BlockOrder().push_back(&block); + IrBuilder builder(program); + builder.SetInsertionPoint(block); + auto& ancillary = builder.Emit(IrOpcode::GetBuiltin, IrType::U32, {&builder.Constant(static_cast(StageInputKind::PackedAncillary)), &builder.Constant(0u)}); + auto& maskedOffset = builder.BitwiseAnd(builder.Constant(offset), builder.Constant(31u)); + auto& maskedCount = builder.BitwiseAnd(builder.Constant(count), builder.Constant(31u)); + auto& available = builder.ISub(builder.Constant(32u), maskedOffset); + auto& clampedCount = builder.Emit(IrOpcode::UMin32, IrType::U32, {&maskedCount, &available}); + auto& user = builder.Emit(IrOpcode::BitFieldUExtract, IrType::U32, {&ancillary, &maskedOffset, &clampedCount}); + static_cast(builder.Emit(IrOpcode::ReferenceU32, IrType::Void, {&user})); + auto& lane = builder.Emit(IrOpcode::LaneId, IrType::U32, {}); + auto& active = builder.Emit(IrOpcode::ULessThan32, IrType::Bool, {&lane, &builder.Constant(16u)}); + auto& copy = builder.Emit(IrOpcode::SelectU32, IrType::U32, {&active, &ancillary, &builder.Constant(0u)}); + static_cast(builder.Emit(IrOpcode::ReferenceU32, IrType::Void, {©})); + static_cast(builder.Emit(IrOpcode::Return, IrType::Void, {})); + ConstantFolder().Fold(program); + const IrValue* field = user.Argument(0)->Resolve(); + Require(field->Opcode() == IrOpcode::GetBuiltin); + Require(static_cast(field->Argument(0)->Resolve()->ImmediateU32()) == kind); + Require(user.Argument(1)->Resolve()->ImmediateU32() == fieldOffset); + Require(user.Argument(2)->Resolve()->ImmediateU32() == count); +} static void Extract(IrOpcode opcode, std::uint32_t offset, std::uint32_t count, StageInputKind kind, std::uint32_t fieldOffset) { IrProgram program; auto& user = Build(program, opcode, offset, count); @@ -40,9 +122,129 @@ static void Extract(IrOpcode opcode, std::uint32_t offset, std::uint32_t count, Require(user.Argument(1)->Resolve()->ImmediateU32() == fieldOffset); Require(user.Argument(2)->Resolve()->ImmediateU32() == count); } -static void Refused(IrOpcode opcode, std::uint32_t offset, std::uint32_t count) { +static void Refused(IrOpcode opcode, std::uint32_t offset, std::uint32_t count, bool throughSelect = false) { IrProgram program; - static_cast(Build(program, opcode, offset, count)); + static_cast(Build(program, opcode, offset, count, throughSelect)); + try { + Lower(program); + } catch (const std::runtime_error& error) { + Require(std::string(error.what()).find("unsupported live use") != std::string::npos); + return; + } + Require(false); +} +static std::uint32_t BitField(std::uint32_t word, std::uint32_t offset, std::uint32_t count, bool isSigned) { + const std::uint32_t mask = count == 32u ? ~0u : (1u << count) - 1u; + std::uint32_t bits = (word >> offset) & mask; + if (isSigned && ((bits >> (count - 1u)) & 1u) != 0u) { + bits |= ~mask; + } + return bits; +} +static std::uint32_t Evaluate(const IrValue* value, const Row& row) { + value = value->Resolve(); + if (value->HasImmediate()) { + return value->ImmediateU32(); + } + switch (value->Opcode()) { + case IrOpcode::GetBuiltin: + switch (static_cast(Evaluate(value->Argument(0), row))) { + case StageInputKind::SampleId: return row.sample; + case StageInputKind::Layer: return row.layer; + case StageInputKind::FrontFacing: return row.frontFacing ? 1u : 0u; + default: break; + } + break; + case IrOpcode::ShiftLeftLogical32: return Evaluate(value->Argument(0), row) << Evaluate(value->Argument(1), row); + case IrOpcode::BitwiseOr32: return Evaluate(value->Argument(0), row) | Evaluate(value->Argument(1), row); + case IrOpcode::BitFieldUExtract: + case IrOpcode::BitFieldSExtract: + return BitField(Evaluate(value->Argument(0), row), Evaluate(value->Argument(1), row), Evaluate(value->Argument(2), row), value->Opcode() == IrOpcode::BitFieldSExtract); + case IrOpcode::SelectU32: return Evaluate(value->Argument(0), row) != 0u ? Evaluate(value->Argument(1), row) : Evaluate(value->Argument(2), row); + case IrOpcode::Phi: return Evaluate(value->Argument(row.frontFacing ? 0u : 1u), row); + default: break; + } + throw std::runtime_error("packed pixel ancillary evaluation reached an unsupported value"); +} +static std::uint32_t PackedWord(const Row& row) { + return UnmodelledBits | (row.sample << 8u) | (row.layer << 16u); +} +static void Values(IrOpcode opcode, std::uint32_t offset, std::uint32_t count, bool throughSelect) { + IrProgram program; + auto& user = Build(program, opcode, offset, count, throughSelect); + Lower(program); + for (const Row& row : Rows) { + const std::uint32_t word = throughSelect && !row.frontFacing ? SelectOther : PackedWord(row); + Require(Evaluate(&user, row) == BitField(word, offset, count, opcode == IrOpcode::BitFieldSExtract)); + } +} +static void LateFold() { + IrProgram program; + auto& block = Begin(program); + IrBuilder builder(program); + builder.SetInsertionPoint(block); + IrValue& ancillary = AncillaryBuiltin(builder); + IrValue& offset = builder.Emit(IrOpcode::BitwiseAnd32, IrType::U32, {&builder.Constant(16u), &builder.Constant(31u)}); + IrValue& count = builder.Emit(IrOpcode::UMin32, IrType::U32, {&builder.Constant(11u), &builder.Constant(13u)}); + IrValue& user = builder.Emit(IrOpcode::BitFieldUExtract, IrType::U32, {&ancillary, &offset, &count}); + End(builder, user); + Lower(program); + const IrValue* field = user.Argument(0)->Resolve(); + Require(field->Opcode() == IrOpcode::GetBuiltin); + Require(static_cast(field->Argument(0)->Resolve()->ImmediateU32()) == StageInputKind::Layer); + Require(user.Argument(1)->Resolve()->ImmediateU32() == 0u); + Require(user.Argument(2)->Resolve()->ImmediateU32() == 11u); + for (const Row& row : Rows) { + Require(Evaluate(&user, row) == BitField(PackedWord(row), 16u, 11u, false)); + } +} +static IrValue& Merged(IrProgram& program, IrOpcode opcode, std::uint32_t offset, std::uint32_t count, bool loop) { + auto& entry = Begin(program); + auto& carry = program.CreateBlock(); + auto& merge = program.CreateBlock(); + program.BlockOrder().push_back(&carry); + program.BlockOrder().push_back(&merge); + IrBuilder builder(program); + builder.SetInsertionPoint(entry); + IrValue& ancillary = AncillaryBuiltin(builder); + IrValue& frontFacing = FrontFacingBuiltin(builder); + IrValue& phi = program.CreateValue(IrOpcode::Phi, IrType::U32); + IrValue* other = &builder.Constant(SelectOther); + if (loop) { + entry.AddBranch(&carry); + carry.AddBranch(&carry); + carry.AddBranch(&merge); + carry.AppendInstruction(&phi); + builder.SetInsertionPoint(carry); + other = &builder.Emit(IrOpcode::SelectU32, IrType::U32, {&frontFacing, &phi, &builder.Constant(SelectOther)}); + phi.AddPhiOperand(&entry, &ancillary); + phi.AddPhiOperand(&carry, other); + } else { + entry.AddBranch(&carry); + entry.AddBranch(&merge); + carry.AddBranch(&merge); + merge.AppendInstruction(&phi); + phi.AddPhiOperand(&entry, &ancillary); + phi.AddPhiOperand(&carry, other); + } + builder.SetInsertionPoint(merge); + IrValue& user = opcode == IrOpcode::BitwiseOr32 ? builder.Emit(opcode, IrType::U32, {&phi, &builder.Constant(offset)}) + : builder.Emit(opcode, IrType::U32, {&phi, &builder.Constant(offset), &builder.Constant(count)}); + End(builder, user); + return user; +} +static void MergedValues(IrOpcode opcode, std::uint32_t offset, std::uint32_t count, bool loop) { + IrProgram program; + auto& user = Merged(program, opcode, offset, count, loop); + Lower(program); + for (const Row& row : Rows) { + const std::uint32_t word = row.frontFacing ? PackedWord(row) : SelectOther; + Require(Evaluate(&user, row) == BitField(word, offset, count, opcode == IrOpcode::BitFieldSExtract)); + } +} +static void MergedRefused(IrOpcode opcode, std::uint32_t offset, std::uint32_t count, bool loop) { + IrProgram program; + static_cast(Merged(program, opcode, offset, count, loop)); try { Lower(program); } catch (const std::runtime_error& error) { @@ -56,9 +258,38 @@ int main() { Extract(IrOpcode::BitFieldUExtract, 9u, 2u, StageInputKind::SampleId, 1u); Extract(IrOpcode::BitFieldUExtract, 16u, 13u, StageInputKind::Layer, 0u); Extract(IrOpcode::BitFieldSExtract, 20u, 9u, StageInputKind::Layer, 4u); + Values(IrOpcode::BitFieldUExtract, 8u, 4u, false); + Values(IrOpcode::BitFieldSExtract, 8u, 4u, false); + Values(IrOpcode::BitFieldUExtract, 9u, 2u, false); + Values(IrOpcode::BitFieldUExtract, 16u, 13u, false); + Values(IrOpcode::BitFieldSExtract, 20u, 9u, false); + Values(IrOpcode::BitFieldUExtract, 28u, 1u, false); + LateFold(); + Values(IrOpcode::BitFieldUExtract, 8u, 4u, true); + Values(IrOpcode::BitFieldUExtract, 16u, 13u, true); + Values(IrOpcode::BitFieldSExtract, 20u, 9u, true); + ExtractVector(16u, 11u, StageInputKind::Layer, 0u); + ExtractVectorMaskedCopy(16u, 11u, StageInputKind::Layer, 0u); + ExtractVectorMaskedCopy(8u, 4u, StageInputKind::SampleId, 0u); + ExtractVector(8u, 4u, StageInputKind::SampleId, 0u); Refused(IrOpcode::BitwiseOr32, 1u, 0u); + Refused(IrOpcode::BitwiseOr32, 1u, 0u, true); Refused(IrOpcode::BitFieldUExtract, 2u, 4u); Refused(IrOpcode::BitFieldUExtract, 10u, 4u); Refused(IrOpcode::BitFieldUExtract, 13u, 2u); Refused(IrOpcode::BitFieldUExtract, 16u, 14u); + Refused(IrOpcode::BitFieldUExtract, 0u, 2u); + Refused(IrOpcode::BitFieldUExtract, 7u, 1u); + Refused(IrOpcode::BitFieldUExtract, 12u, 4u); + Refused(IrOpcode::BitFieldUExtract, 29u, 1u); + Refused(IrOpcode::BitFieldUExtract, 8u, 0u); + Refused(IrOpcode::BitFieldUExtract, 0u, 2u, true); + Refused(IrOpcode::BitFieldSExtract, 12u, 4u, true); + MergedValues(IrOpcode::BitFieldUExtract, 8u, 4u, false); + MergedValues(IrOpcode::BitFieldUExtract, 16u, 13u, false); + MergedValues(IrOpcode::BitFieldSExtract, 20u, 9u, false); + MergedValues(IrOpcode::BitFieldUExtract, 16u, 11u, true); + MergedRefused(IrOpcode::BitwiseOr32, 1u, 0u, false); + MergedRefused(IrOpcode::BitFieldUExtract, 12u, 4u, false); + MergedRefused(IrOpcode::BitFieldUExtract, 0u, 2u, true); } diff --git a/core/libs/tests/PreciseWait.cpp b/core/libs/tests/PreciseWait.cpp new file mode 100644 index 000000000..4089cb4ab --- /dev/null +++ b/core/libs/tests/PreciseWait.cpp @@ -0,0 +1,14 @@ +#include "prx/libc/include/PreciseWait.hpp" +#include +#include + +int main() { + const auto start = std::chrono::steady_clock::now(); + PreciseSleepUs(50); + const auto elapsed = std::chrono::steady_clock::now() - start; + if (elapsed < std::chrono::microseconds(50)) { + std::cerr << "PreciseSleepUs returned before its requested duration\n"; + return 1; + } + return 0; +} diff --git a/core/libs/tests/PthreadStart.cpp b/core/libs/tests/PthreadStart.cpp index 4e3f8fb23..8750d5eb5 100644 --- a/core/libs/tests/PthreadStart.cpp +++ b/core/libs/tests/PthreadStart.cpp @@ -1,4 +1,4 @@ -#include "prx/libkernel/Pthread/Pthread.hpp" +#include "prx/libkernel/Pthread/include/Pthread.hpp" #include #include #include @@ -41,8 +41,7 @@ static void* APS5_VABI CheckThread(void* arg) { if (context.thread->threadId != std::this_thread::get_id() || !context.thread->nativeHandle) throw std::runtime_error("Native thread was not initialized"); #else - if (context.thread->_thr.get_id() != std::this_thread::get_id()) throw std::runtime_error("Thread object was not initialized"); - if (!context.thread->_thr.joinable()) throw std::runtime_error("Thread object is not joinable"); + if (!pthread_equal(context.thread->hostThread, pthread_self())) throw std::runtime_error("Thread object was not initialized"); #endif PthreadAttr attr = nullptr; if (scePthreadAttrInit(&attr) != 0) throw std::runtime_error("Attribute initialization failed"); diff --git a/core/libs/tests/RegisterContext.cpp b/core/libs/tests/RegisterContext.cpp new file mode 100644 index 000000000..28c22057c --- /dev/null +++ b/core/libs/tests/RegisterContext.cpp @@ -0,0 +1,156 @@ +#include "prx/libc/include/exceptions/Unwind.hpp" +#include "prx/libc/src/specifics/x86_64/RegisterContext.cpp" + +#include +#include +#include +#include +#include + +#if !defined(__x86_64__) || (!defined(__linux__) && !defined(__APPLE__)) +#error Register context probes require an x86-64 System V host +#endif + +static_assert(sizeof(_Unwind_Context::registers) == 17 * sizeof(std::uintptr_t)); +static_assert(offsetof(_Unwind_Context, cfa) == 17 * sizeof(std::uintptr_t)); + +extern "C" void CaptureProbe(std::uintptr_t*); +extern "C" void RestoreProbe(std::uintptr_t*, std::uintptr_t*); + +#ifdef __APPLE__ +#define CONTEXT_SYMBOL(name) "_" #name +#else +#define CONTEXT_SYMBOL(name) #name +#endif + +asm( +".text\n" +".globl " CONTEXT_SYMBOL(CaptureProbe) "\n" +CONTEXT_SYMBOL(CaptureProbe) ":\n" +"pushq %rbp\npushq %rbx\npushq %r12\npushq %r13\npushq %r14\npushq %r15\n" +"subq $24,%rsp\n" +"movq %rsp,136(%rdi)\n" +"leaq 1f(%rip),%rax\nmovq %rax,144(%rdi)\n" +"movabsq $0x1234567800000100,%rax\n" +"movabsq $0x1234567800000101,%rdx\n" +"movabsq $0x1234567800000102,%rcx\n" +"movabsq $0x1234567800000103,%rbx\n" +"movabsq $0x1234567800000104,%rsi\n" +"movabsq $0x1234567800000106,%rbp\n" +"movabsq $0x1234567800000108,%r8\n" +"movabsq $0x1234567800000109,%r9\n" +"movabsq $0x123456780000010a,%r10\n" +"movabsq $0x123456780000010b,%r11\n" +"movabsq $0x123456780000010c,%r12\n" +"movabsq $0x123456780000010d,%r13\n" +"movabsq $0x123456780000010e,%r14\n" +"movabsq $0x123456780000010f,%r15\n" +"call " CONTEXT_SYMBOL(LibcCaptureRegisters) "\n" +"1:\n" +"addq $24,%rsp\n" +"popq %r15\npopq %r14\npopq %r13\npopq %r12\npopq %rbx\npopq %rbp\nret\n" +".globl " CONTEXT_SYMBOL(RestoreProbe) "\n" +CONTEXT_SYMBOL(RestoreProbe) ":\n" +"pushq %rbp\npushq %rbx\npushq %r12\npushq %r13\npushq %r14\npushq %r15\n" +"subq $24,%rsp\n" +"movq %rsi,0(%rsp)\nmovq %rsp,56(%rdi)\n" +"leaq 2f(%rip),%rax\nmovq %rax,128(%rdi)\n" +"call LRegisterContextRestoreNested\nud2\n" +"2:\n" +"movq 0(%rsp),%r10\n" +"movq %rax,0(%r10)\nmovq %rdx,8(%r10)\nmovq %rcx,16(%r10)\nmovq %rbx,24(%r10)\n" +"movq %rsi,32(%r10)\nmovq %rdi,40(%r10)\nmovq %rbp,48(%r10)\nmovq %rsp,56(%r10)\n" +"movq %r8,64(%r10)\nmovq %r9,72(%r10)\n" +"movq %r12,96(%r10)\nmovq %r13,104(%r10)\nmovq %r14,112(%r10)\nmovq %r15,120(%r10)\n" +"leaq 2b(%rip),%r11\nmovq %r11,128(%r10)\n" +"addq $24,%rsp\n" +"popq %r15\npopq %r14\npopq %r13\npopq %r12\npopq %rbx\npopq %rbp\nret\n" +"LRegisterContextRestoreNested:\n" +"subq $200,%rsp\n" +"xorq %rbx,%rbx\nxorq %rbp,%rbp\nxorq %r12,%r12\nxorq %r13,%r13\nxorq %r14,%r14\nxorq %r15,%r15\n" +"call " CONTEXT_SYMBOL(LibcRestoreRegisters) "\nud2\n" +); + +#undef CONTEXT_SYMBOL + +namespace { + +constexpr std::uintptr_t Guard = 0xa1b2c3d4e5f60718; + +struct Capture { + std::uintptr_t before = Guard; + std::array registers{}; + std::uintptr_t expectedStack = 0; + std::uintptr_t expectedInstruction = 0; + std::uintptr_t after = Guard; +}; + +static_assert(offsetof(Capture, expectedStack) - offsetof(Capture, registers) == 136); +static_assert(offsetof(Capture, expectedInstruction) - offsetof(Capture, registers) == 144); + +struct Registers { + std::uintptr_t before = Guard; + std::array values{}; + std::uintptr_t after = Guard; +}; + +void Require(bool condition, const char* message) { + if (!condition) throw std::runtime_error(message); +} + +void CheckCapture() { + Capture captured; + CaptureProbe(captured.registers.data()); + Require(captured.before == Guard && captured.after == Guard, "capture wrote outside its register context"); + for (std::size_t i = 0; i < 16; ++i) { + const auto expected = i == 5 ? reinterpret_cast(captured.registers.data()) + : i == 7 ? captured.expectedStack : 0x1234567800000100 + i; + Require(captured.registers[i] == expected, "capture saved the wrong general register"); + } + Require(captured.registers[16] == captured.expectedInstruction, "capture saved the wrong return instruction"); + Require(captured.registers[7] % 16 == 0, "capture saved the callee stack instead of the caller stack"); +} + +void CheckRestore(std::uintptr_t seed) { + Registers restored; + Registers observed; + for (std::size_t i = 0; i < restored.values.size(); ++i) { + restored.values[i] = (0xfedcba9800000000 | (seed << 8) | i); + } + RestoreProbe(restored.values.data(), observed.values.data()); + Require(restored.before == Guard && restored.after == Guard, "restore overwrote its source context"); + Require(observed.before == Guard && observed.after == Guard, "restore probe overwrote its destination context"); + for (std::size_t i = 0; i < restored.values.size(); ++i) { + if (i == 10 || i == 11) continue; + Require(observed.values[i] == restored.values[i], "restore did not install the landing pad context"); + } +} + +void CheckThread(std::uintptr_t seed) { + for (std::uintptr_t i = 0; i < 256; ++i) { + CheckCapture(); + CheckRestore(seed + i); + } +} + +} + +int main(int argc, char** argv) { + try { + Require(argc == 2, "expected capture, restore or threads"); + const std::string_view mode(argv[1]); + if (mode == "capture") CheckCapture(); + else if (mode == "restore") { + for (std::uintptr_t seed = 0; seed < 256; ++seed) CheckRestore(seed); + } else if (mode == "threads") { + std::array threads; + for (std::size_t i = 0; i < threads.size(); ++i) threads[i] = std::thread(CheckThread, i * 256); + for (auto& thread : threads) thread.join(); + } else throw std::runtime_error("unknown register context probe"); + std::printf("register context %s passed\n", argv[1]); + return 0; + } catch (const std::exception& error) { + std::fprintf(stderr, "%s\n", error.what()); + return 1; + } +} diff --git a/core/libs/tests/SaveDataMemoryGrowth.cpp b/core/libs/tests/SaveDataMemoryGrowth.cpp index 924c4d4c6..95d5225a3 100644 --- a/core/libs/tests/SaveDataMemoryGrowth.cpp +++ b/core/libs/tests/SaveDataMemoryGrowth.cpp @@ -104,6 +104,15 @@ int main(int argc, char**) { Require(sceSaveDataSetupSaveDataMemory2(&setup, &result) == 0); Require(result.existed_memory_size == 0); Require(Read("_sd_mem/u7531/slot1.bin") == std::vector(2, 0)); + setup.slot_id = 0; + setup.memory_size = 32u * 1024u * 1024u; + Require(sceSaveDataSetupSaveDataMemory2(&setup, &result) == 0); + Require(result.existed_memory_size == expected.size()); + expected.resize(setup.memory_size, 0); + Require(Read(path) == expected); + ++setup.memory_size; + Require(sceSaveDataSetupSaveDataMemory2(&setup, &result) == static_cast(0x809F0000u)); + Require(Read(path) == expected); } Require(sceSaveDataTerminate() == 0); std::filesystem::current_path(previousDirectory); diff --git a/core/libs/tests/VectorConstantFactorTininess.cpp b/core/libs/tests/VectorConstantFactorTininess.cpp new file mode 100644 index 000000000..27edb18e2 --- /dev/null +++ b/core/libs/tests/VectorConstantFactorTininess.cpp @@ -0,0 +1,94 @@ +#include "Translation/TranslationContext.hpp" +#include "RdnaDecoder/RdnaVectorOpDecoder.hpp" +#include +#include +#include +#include +#include +#include + +using namespace ShaderRecompiler; + +namespace { + +enum class Tininess { + None, + ExponentsOnly, + Full, +}; + +void Require(bool value, const std::string& message) { + if (!value) throw std::runtime_error("f32 constant factor: " + message); +} + +const IrBlock& Translate(IrProgram& program, const std::string& name, const std::vector& words, RdnaOpcode opcode, std::uint32_t floatMode) { + const RdnaInstruction instruction = DecodeRdnaVectorOp(std::span(words), 0u); + Require(instruction.op == opcode, name + " decodes to another opcode"); + Require(instruction.wordCount == words.size(), name + " has the wrong length"); + auto& block = program.CreateBlock(); + program.SetEntryBlock(block); + TranslationContext context(program, block, 256); + context.SetFloatMode(ShaderFloatMode{floatMode, true, false, false}); + context.TranslateInstruction(instruction); + return block; +} + +void Check(const std::string& name, const std::vector& words, RdnaOpcode opcode, Tininess expected, std::uint32_t floatMode = 0xc0u) { + IrProgram program; + const auto& block = Translate(program, name, words, opcode, floatMode); + std::uint32_t multiplyHigh = 0u; + std::uint32_t tinyCompares = 0u; + for (const auto* value : block.Instructions()) { + if (value->Opcode() == IrOpcode::UMulHi) ++multiplyHigh; + if (value->Opcode() == IrOpcode::ULessThan32 && value->Argument(1)->HasImmediate() && value->Argument(1)->ImmediateU32() == 128u) ++tinyCompares; + } + const std::uint32_t expectedCompares = expected == Tininess::None ? 0u : 1u; + const std::uint32_t expectedMultiplyHigh = expected == Tininess::Full ? 1u : 0u; + Require(tinyCompares == expectedCompares, name + " emits " + std::to_string(tinyCompares) + " tiny product checks, expected " + std::to_string(expectedCompares)); + Require(multiplyHigh == expectedMultiplyHigh, name + " emits " + std::to_string(multiplyHigh) + " significand products, expected " + std::to_string(expectedMultiplyHigh)); +} + +void CheckFoldedSource(const std::string& name, const std::vector& words, RdnaOpcode opcode, IrOpcode product, std::uint32_t index, std::uint32_t expected) { + IrProgram program; + const auto& block = Translate(program, name, words, opcode, 0xc0u); + for (const auto* value : block.Instructions()) { + if (value->Opcode() != product) continue; + const IrValue* source = value->Argument(index); + Require(source->Opcode() == IrOpcode::BitCastF32U32 && source->Argument(0)->HasImmediate(), name + " source " + std::to_string(index) + " is not a constant"); + Require(source->Argument(0)->ImmediateU32() == expected, name + " source " + std::to_string(index) + " is folded to the wrong constant"); + return; + } + Require(false, name + " emits no product"); +} + +} + +int main() { + try { + Check("v_mul_f32 v5, v6, v7", {0x100a0f06u}, RdnaOpcode::VMulF32, Tininess::Full); + Check("v_mul_f32 v5, 2.0, v7", {0x100a0ef4u}, RdnaOpcode::VMulF32, Tininess::None); + Check("v_mul_f32 v5, -4.0, v7", {0x100a0ef7u}, RdnaOpcode::VMulF32, Tininess::None); + Check("v_mul_f32 v5, 1.0, v7", {0x100a0ef2u}, RdnaOpcode::VMulF32, Tininess::None); + Check("v_mul_f32 v5, 0x7f800000, v7", {0x100a0effu, 0x7f800000u}, RdnaOpcode::VMulF32, Tininess::None); + Check("v_mul_f32 v5, 0x00400000, v7", {0x100a0effu, 0x00400000u}, RdnaOpcode::VMulF32, Tininess::None); + Check("v_mul_f32 v5, 1, v7", {0x100a0e81u}, RdnaOpcode::VMulF32, Tininess::None); + Check("v_mul_f32 v5, 0.5, v7", {0x100a0ef0u}, RdnaOpcode::VMulF32, Tininess::ExponentsOnly); + Check("v_mul_f32 v5, 0x3e800000, v7", {0x100a0effu, 0x3e800000u}, RdnaOpcode::VMulF32, Tininess::ExponentsOnly); + Check("v_mul_f32 v5, 0x3f7fffff, v7", {0x100a0effu, 0x3f7fffffu}, RdnaOpcode::VMulF32, Tininess::Full); + Check("v_mul_f32 v5, 0.15915494, v7", {0x100a0ef8u}, RdnaOpcode::VMulF32, Tininess::Full); + Check("v_mul_legacy_f32 v5, 4.0, v7", {0x0e0a0ef6u}, RdnaOpcode::VMulLegacyF32, Tininess::None); + Check("v_fmamk_f32 v5, v6, 0x40400000, v7", {0x580a0f06u, 0x40400000u}, RdnaOpcode::VMadmkF32, Tininess::None); + Check("v_fmaak_f32 v5, v6, v7, 0x40400000", {0x5a0a0f06u, 0x40400000u}, RdnaOpcode::VMadakF32, Tininess::Full); + Check("v_fma_f32 v5, v6, -|2.0|, v7", {0xd54b0205u, 0x441de906u}, RdnaOpcode::VFmaF32, Tininess::None); + Check("v_fma_f32 v5, v6, -|0.5|, v7", {0xd54b0205u, 0x441de106u}, RdnaOpcode::VFmaF32, Tininess::ExponentsOnly); + Check("v_mul_f32 v5, 2.0, v7 with f32 denormals kept", {0x100a0ef4u}, RdnaOpcode::VMulF32, Tininess::None, 0xf0u); + Check("v_mul_f32 v5, v6, v7 with f32 denormals kept", {0x100a0f06u}, RdnaOpcode::VMulF32, Tininess::None, 0xf0u); + CheckFoldedSource("v_fma_f32 v5, v6, -|2.0|, v7", {0xd54b0205u, 0x441de906u}, RdnaOpcode::VFmaF32, IrOpcode::FPFma32, 1u, 0xc0000000u); + CheckFoldedSource("v_mul_f32 v5, 0x00400000, v7", {0x100a0effu, 0x00400000u}, RdnaOpcode::VMulF32, IrOpcode::FPMul32, 0u, 0x00000000u); + CheckFoldedSource("v_mul_f32 v5, 0x80400000, v7", {0x100a0effu, 0x80400000u}, RdnaOpcode::VMulF32, IrOpcode::FPMul32, 0u, 0x80000000u); + return 0; + } catch (const std::exception& error) { + std::cerr << error.what() << '\n'; + return 1; + } +} diff --git a/core/libs/tests/VectorFusedMultiplyAdd.cpp b/core/libs/tests/VectorFusedMultiplyAdd.cpp index 3315e155a..f17e7f083 100644 --- a/core/libs/tests/VectorFusedMultiplyAdd.cpp +++ b/core/libs/tests/VectorFusedMultiplyAdd.cpp @@ -90,6 +90,42 @@ void CheckMix(const std::string& name, const std::array& words Require(fused == 1u, name + " emits " + std::to_string(fused) + " fused multiply-adds"); } +void CheckMixHalf(const std::string& name, const std::array& words, RdnaOpcode opcode, std::uint32_t floatMode) { + const RdnaInstruction instruction = DecodeRdnaVectorOp(std::span(words), 0u); + Require(instruction.family == RdnaInstructionFamily::VOP3P, name + " is not decoded as VOP3P"); + Require(instruction.op == opcode, name + " decodes to another opcode"); + + IrProgram program; + auto& block = program.CreateBlock(); + program.SetEntryBlock(block); + TranslationContext context(program, block, 256); + context.SetFloatMode(ShaderFloatMode{floatMode, false, false, false}); + context.TranslateInstruction(instruction); + + std::uint32_t exact = 0u; + for (const auto* value : block.Instructions()) { + Require(value->Opcode() != IrOpcode::FPFma32 && value->Opcode() != IrOpcode::FPMad32, name + " rounds to f32 before rounding to f16"); + if (value->Opcode() == IrOpcode::FPInterpolateF16) ++exact; + } + Require(exact == 1u, name + " emits " + std::to_string(exact) + " product-sums rounded once to f16"); +} + +void RejectMixHalf(const std::string& name, const std::array& words, std::uint32_t floatMode) { + const RdnaInstruction instruction = DecodeRdnaVectorOp(std::span(words), 0u); + IrProgram program; + auto& block = program.CreateBlock(); + program.SetEntryBlock(block); + TranslationContext context(program, block, 256); + context.SetFloatMode(ShaderFloatMode{floatMode, false, false, false}); + try { + context.TranslateInstruction(instruction); + } catch (const std::runtime_error& error) { + Require(std::string(error.what()).find("FLOAT_MODE " + std::to_string(floatMode)) != std::string::npos, name + " fails with another error: " + error.what()); + return; + } + Require(false, name + " is translated in FLOAT_MODE " + std::to_string(floatMode)); +} + } int main() { @@ -105,6 +141,12 @@ int main() { Check("v_fmaak_f32", 0x2du, 2u, RdnaOpcode::VMadakF32, IrOpcode::FPFma32, addLiteral); CheckMix("v_fma_mix_f32", {0xcc200005u, 0x04220f06u}); CheckMix("v_fma_mix_f32 op_sel_hi:[1,0,1]", {0xcc204005u, 0x0c220f06u}); + CheckMixHalf("v_fma_mixlo_f16", {0xcc210005u, 0x04220f06u}, RdnaOpcode::VMadMixloF16, 0xc0u); + CheckMixHalf("v_fma_mixhi_f16", {0xcc220005u, 0x04220f06u}, RdnaOpcode::VMadMixhiF16, 0xc0u); + CheckMixHalf("v_fma_mixlo_f16 op_sel_hi:[1,1,1]", {0xcc214005u, 0x1c220f06u}, RdnaOpcode::VMadMixloF16, 0xc0u); + CheckMixHalf("v_fma_mixlo_f16 with f32 round toward zero", {0xcc210005u, 0x04220f06u}, RdnaOpcode::VMadMixloF16, 0xf3u); + RejectMixHalf("v_fma_mixlo_f16 with f16 denormals flushed", {0xcc210005u, 0x04220f06u}, 0x30u); + RejectMixHalf("v_fma_mixhi_f16 with f16 round toward -inf", {0xcc220005u, 0x04220f06u}, 0xf8u); return 0; } catch (const std::exception& error) { std::cerr << error.what() << '\n'; diff --git a/core/libs/tests/VectorMadLegacy.cpp b/core/libs/tests/VectorMadLegacy.cpp index cf2677161..aaf379bc7 100644 --- a/core/libs/tests/VectorMadLegacy.cpp +++ b/core/libs/tests/VectorMadLegacy.cpp @@ -21,6 +21,6 @@ static void Check(std::uint32_t encoding, RdnaOpcode opcode, IrOpcode expected) Require(found); } int main() { - Check(0x140u, RdnaOpcode::VMadLegacyF32, IrOpcode::FPFma32); + Check(0x140u, RdnaOpcode::VMadLegacyF32, IrOpcode::FPMul32); Check(0x150u, RdnaOpcode::VMullitF32, IrOpcode::FPMul32); } diff --git a/core/libs/tests/WindowsExceptions.cpp b/core/libs/tests/WindowsExceptions.cpp index 2e13adb43..1a265385c 100644 --- a/core/libs/tests/WindowsExceptions.cpp +++ b/core/libs/tests/WindowsExceptions.cpp @@ -4,6 +4,7 @@ #include #include #include +#include #include #include #include @@ -105,9 +106,61 @@ static void testExceptionPointer() { if (std::current_exception()) throw std::runtime_error("stale current exception"); } +static void testFutureException() { + std::atomic count{0}; + { + std::promise promise; + auto future = promise.get_future(); + try { + throw TrackedError(&count); + } catch (const TrackedError&) { + promise.set_exception(std::current_exception()); + } + try { + future.get(); + throw std::runtime_error("future did not rethrow"); + } catch (const TrackedError& error) { + if (std::strcmp(error.what(), "retained error")) throw std::runtime_error("future rethrew another error"); + } + } + if (count != 1) throw std::runtime_error("future result destroyed its exception an incorrect number of times"); +} + +static void testMadeExceptionPointer() { + std::atomic count{0}; + { + auto made = std::make_exception_ptr(TrackedError(&count)); + try { + std::rethrow_exception(made); + } catch (const TrackedError& error) { + if (std::strcmp(error.what(), "retained error")) throw std::runtime_error("made exception lost its message"); + } + } + if (count != 2) throw std::runtime_error("made exception destroyed an incorrect number of times"); +} + +struct UnwindProbe { + int& seen; + ~UnwindProbe() { seen = std::uncaught_exceptions(); } +}; + +static void testUncaughtExceptions() { + if (std::uncaught_exceptions() != 0) throw std::runtime_error("uncaught exception outside unwinding"); + int seen = -1; + try { + UnwindProbe probe{seen}; + throw std::runtime_error("probe"); + } catch (const std::runtime_error&) { + } + if (seen != 1) throw std::runtime_error("unwinding did not count its exception"); +} + int main() { TestTypeInfoVtables(); testExceptionPointer(); + testFutureException(); + testMadeExceptionPointer(); + testUncaughtExceptions(); try { Rethrow(); return 1; diff --git a/core/libs/tests/WriteWatch.cpp b/core/libs/tests/WriteWatch.cpp new file mode 100644 index 000000000..84b3029c5 --- /dev/null +++ b/core/libs/tests/WriteWatch.cpp @@ -0,0 +1,193 @@ +#include "prx/libc/include/GuestWriteWatch.hpp" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include + +#if !defined(__APPLE__) +#error The page protection write watch is the macOS backend +#endif + +using namespace GuestWriteWatch; + +namespace { + +void Require(bool condition, const char* message) { + if (!condition) throw std::runtime_error(message); +} + +struct Region { + explicit Region(std::size_t pageCount) : page(static_cast(::getpagesize())), bytes(pageCount * page) { + base = static_cast(::mmap(nullptr, bytes, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANON, -1, 0)); + Require(base != MAP_FAILED, "cannot map the test region"); + } + ~Region() { + GuestWriteWatchUnregister_nid_postfix(base, bytes); + ::munmap(base, bytes); + } + std::uintptr_t Address(std::size_t index) const { + return reinterpret_cast(base) + index * page; + } + std::set Collect() { + std::set pages; + struct Sink { + Region* region; + std::set* pages; + } sink{this, &pages}; + const bool complete = GuestWriteWatchCollect_nid_postfix(Address(0), bytes, [](void* context, std::uintptr_t begin, std::uintptr_t end) { + auto& target = *static_cast(context); + for (auto at = begin; at < end; at += target.region->page) target.pages->insert((at - target.region->Address(0)) / target.region->page); + }, &sink); + Require(complete, "collection of a watched range is incomplete"); + return pages; + } + std::size_t page; + std::size_t bytes; + std::uint8_t* base = nullptr; +}; + +std::set All(std::size_t count) { + std::set pages; + for (std::size_t index = 0; index < count; ++index) pages.insert(index); + return pages; +} + +void CheckCollect() { + Region region(8); + Require(GuestWriteWatchAvailable_nid_postfix(), "write watch is unavailable"); + Require(!GuestWriteWatchCovers_nid_postfix(region.Address(0), region.bytes), "unregistered range is covered"); + GuestWriteWatchRegister_nid_postfix(region.base, region.bytes); + Require(GuestWriteWatchCovers_nid_postfix(region.Address(0), region.bytes), "registered range is not covered"); + Require(region.Collect() == All(8), "a new range is not reported in full"); + Require(region.Collect().empty(), "an untouched range reports writes"); + region.base[3 * region.page + 17] = 1; + region.base[5 * region.page] = 2; + region.base[5 * region.page + 9] = 3; + Require(region.Collect() == std::set({3, 5}), "written pages are not reported exactly"); + Require(region.Collect().empty(), "collected writes are reported again"); + Require(region.base[3 * region.page + 17] == 1 && region.base[5 * region.page + 9] == 3, "watched writes were lost"); + std::memset(region.base + region.page - 4, 7, 8); + Require(region.Collect() == std::set({0, 1}), "a write across a page boundary is not reported on both pages"); +} + +void CheckRegisterAgain() { + Region region(4); + GuestWriteWatchRegister_nid_postfix(region.base, region.bytes); + Require(region.Collect() == All(4), "a new range is not reported in full"); + GuestWriteWatchRegister_nid_postfix(region.base, region.bytes); + Require(region.Collect() == All(4), "a range registered again is not reported in full"); + region.base[2 * region.page] = 1; + Require(region.Collect() == std::set({2}), "a range registered while protected stops reporting writes"); + GuestWriteWatchRegister_nid_postfix(region.base + region.page, region.page); + Require(region.Collect() == std::set({1}), "registering part of a range does not report that part alone"); + region.base[0] = 1; + region.base[3 * region.page] = 1; + Require(region.Collect() == std::set({0, 3}), "the rest of a split range stops reporting writes"); +} + +void CheckHostWrite() { + Region region(4); + GuestWriteWatchRegister_nid_postfix(region.base, region.bytes); + Require(region.Collect() == All(4), "a new range is not reported in full"); + int descriptors[2]; + Require(::pipe(descriptors) == 0, "cannot create a pipe"); + const std::array payload{'w', 'a', 't', 'c', 'h', 'e', 'd', '!'}; + Require(::write(descriptors[1], payload.data(), payload.size()) == static_cast(payload.size()), "cannot fill the pipe"); + errno = 0; + Require(::read(descriptors[0], region.base + region.page, payload.size()) == -1 && errno == EFAULT, "a system call wrote a protected page without a host write"); + GuestWriteWatchBeginHostWrite_nid_postfix(region.base + region.page, payload.size()); + Require(region.Collect() == std::set({1}), "a page under a host write is not reported"); + Require(::read(descriptors[0], region.base + region.page, payload.size()) == static_cast(payload.size()), "a collection protected a page during a host write"); + GuestWriteWatchEndHostWrite_nid_postfix(region.base + region.page, payload.size()); + Require(std::memcmp(region.base + region.page, payload.data(), payload.size()) == 0, "the host write did not reach the page"); + Require(region.Collect() == std::set({1}), "a host write is not reported after it ends"); + Require(region.Collect().empty(), "a finished host write is reported again"); + ::close(descriptors[0]); + ::close(descriptors[1]); +} + +void CheckProtection() { + Region region(6); + std::uintptr_t runEnd = 0; + Require(GuestWriteWatchProtection_nid_postfix(region.Address(0), region.Address(6), &runEnd) == -1 && runEnd == region.Address(6), "an unwatched range reports a protection"); + GuestWriteWatchRegister_nid_postfix(region.base + 2 * region.page, 3 * region.page); + Require(GuestWriteWatchProtection_nid_postfix(region.Address(0), region.Address(6), &runEnd) == -1 && runEnd == region.Address(2), "the run before a watched range does not end at it"); + static_cast(GuestWriteWatchCollect_nid_postfix(region.Address(2), 3 * region.page, [](void*, std::uintptr_t, std::uintptr_t) {}, nullptr)); + Require(GuestWriteWatchProtection_nid_postfix(region.Address(2), region.Address(6), &runEnd) == (PROT_READ | PROT_WRITE) && runEnd == region.Address(5), "a protected page does not report the protection it was given"); + Require(::mprotect(region.base + 3 * region.page, region.page, PROT_READ) == 0, "cannot change the protection"); + GuestWriteWatchProtectionChanged_nid_postfix(region.base + 3 * region.page, region.page, PROT_READ); + Require(GuestWriteWatchProtection_nid_postfix(region.Address(2), region.Address(6), &runEnd) == (PROT_READ | PROT_WRITE) && runEnd == region.Address(3), "a run does not end where the protection changes"); + Require(GuestWriteWatchProtection_nid_postfix(region.Address(3), region.Address(6), &runEnd) == PROT_READ && runEnd == region.Address(4), "a changed protection is not reported"); + std::set pages; + static_cast(GuestWriteWatchCollect_nid_postfix(region.Address(2), 3 * region.page, [](void* context, std::uintptr_t begin, std::uintptr_t) { static_cast*>(context)->insert(begin); }, &pages)); + Require(pages == std::set({region.Address(3)}), "a protection change is not reported as a write"); + Require(::mprotect(region.base + 3 * region.page, region.page, PROT_READ | PROT_WRITE) == 0, "cannot restore the protection"); + GuestWriteWatchProtectionChanged_nid_postfix(region.base + 3 * region.page, region.page, PROT_READ | PROT_WRITE); + region.base[3 * region.page] = 1; + GuestWriteWatchUnregister_nid_postfix(region.base + 2 * region.page, 3 * region.page); + region.base[2 * region.page] = 1; + region.base[4 * region.page] = 1; + Require(!GuestWriteWatchCovers_nid_postfix(region.Address(2), region.page), "an unregistered range is covered"); +} + +void CheckThreads() { + constexpr std::size_t Threads = 4; + constexpr std::size_t PagesPerThread = 64; + constexpr int Rounds = 200; + Region region(Threads * PagesPerThread); + GuestWriteWatchRegister_nid_postfix(region.base, region.bytes); + Require(region.Collect() == All(Threads * PagesPerThread), "a new range is not reported in full"); + std::atomic start{false}; + std::atomic finished{0}; + std::array writers; + for (std::size_t thread = 0; thread < Threads; ++thread) { + writers[thread] = std::thread([&, thread] { + while (!start.load()) std::this_thread::yield(); + for (int round = 1; round <= Rounds; ++round) + for (std::size_t index = 0; index < PagesPerThread; ++index) region.base[(thread * PagesPerThread + index) * region.page] = static_cast(round); + finished.fetch_add(1); + }); + } + std::vector seen(Threads * PagesPerThread, 0); + const auto take = [&] { + for (const auto index : region.Collect()) seen[index] = region.base[index * region.page]; + }; + start.store(true); + while (finished.load() != Threads) take(); + for (auto& writer : writers) writer.join(); + take(); + for (std::size_t index = 0; index < seen.size(); ++index) + Require(seen[index] == static_cast(Rounds), "a write that raced a collection was not reported"); + Require(region.Collect().empty(), "writes are reported after the writers stopped"); +} + +} + +int main(int argc, char** argv) { + try { + Require(argc == 2, "expected collect, register, host, protection or threads"); + const std::string_view mode(argv[1]); + if (mode == "collect") CheckCollect(); + else if (mode == "register") CheckRegisterAgain(); + else if (mode == "host") CheckHostWrite(); + else if (mode == "protection") CheckProtection(); + else if (mode == "threads") CheckThreads(); + else throw std::runtime_error("unknown write watch test"); + std::printf("write watch %s passed\n", argv[1]); + return 0; + } catch (const std::exception& error) { + std::fprintf(stderr, "%s\n", error.what()); + return 1; + } +} diff --git a/core/libs/tests/register_context/CMakeLists.txt b/core/libs/tests/register_context/CMakeLists.txt new file mode 100644 index 000000000..eedbc80e3 --- /dev/null +++ b/core/libs/tests/register_context/CMakeLists.txt @@ -0,0 +1,21 @@ +cmake_minimum_required(VERSION 3.20) + +if(CMAKE_SOURCE_DIR STREQUAL CMAKE_CURRENT_SOURCE_DIR) + project(RegisterContextTests LANGUAGES CXX) + enable_testing() + set(ANYPS5_TEST_OUTPUT_DIRECTORY "${CMAKE_BINARY_DIR}/tests") +endif() + +find_package(Threads REQUIRED) +add_executable(register_context_tests ../RegisterContext.cpp) +target_include_directories(register_context_tests PRIVATE ../..) +target_compile_features(register_context_tests PRIVATE cxx_std_20) +target_link_libraries(register_context_tests PRIVATE Threads::Threads) +set_target_properties(register_context_tests PROPERTIES + CXX_EXTENSIONS OFF + RUNTIME_OUTPUT_DIRECTORY "$<1:${ANYPS5_TEST_OUTPUT_DIRECTORY}>") + +foreach(mode capture restore threads) + add_test(NAME register_context_${mode} COMMAND register_context_tests ${mode}) + set_tests_properties(register_context_${mode} PROPERTIES TIMEOUT 15) +endforeach() diff --git a/core/libs/tests/write_watch/CMakeLists.txt b/core/libs/tests/write_watch/CMakeLists.txt new file mode 100644 index 000000000..5d4a7a58f --- /dev/null +++ b/core/libs/tests/write_watch/CMakeLists.txt @@ -0,0 +1,21 @@ +cmake_minimum_required(VERSION 3.20) + +if(CMAKE_SOURCE_DIR STREQUAL CMAKE_CURRENT_SOURCE_DIR) + project(WriteWatchTests LANGUAGES CXX) + enable_testing() + set(ANYPS5_TEST_OUTPUT_DIRECTORY "${CMAKE_BINARY_DIR}/tests") +endif() + +find_package(Threads REQUIRED) +add_executable(write_watch_tests ../WriteWatch.cpp ../../prx/libc/src/GuestWriteWatch.cpp) +target_include_directories(write_watch_tests PRIVATE ../..) +target_compile_features(write_watch_tests PRIVATE cxx_std_20) +target_link_libraries(write_watch_tests PRIVATE Threads::Threads) +set_target_properties(write_watch_tests PROPERTIES + CXX_EXTENSIONS OFF + RUNTIME_OUTPUT_DIRECTORY "$<1:${ANYPS5_TEST_OUTPUT_DIRECTORY}>") + +foreach(mode collect register host protection threads) + add_test(NAME write_watch_${mode} COMMAND write_watch_tests ${mode}) + set_tests_properties(write_watch_${mode} PROPERTIES TIMEOUT 30) +endforeach() diff --git a/core/relinker/CMakeLists.txt b/core/relinker/CMakeLists.txt index f1d7705e2..288dddef6 100644 --- a/core/relinker/CMakeLists.txt +++ b/core/relinker/CMakeLists.txt @@ -21,6 +21,9 @@ add_executable(relinker ${CMAKE_CURRENT_SOURCE_DIR}/elfpatcher/src/windows/WindowsDependencyStubBuilder.cpp ${CMAKE_CURRENT_SOURCE_DIR}/elfpatcher/src/windows/WindowsStubEmitter.cpp ${CMAKE_CURRENT_SOURCE_DIR}/elfpatcher/src/windows/WindowsPeWriter.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/elfpatcher/src/macos/MacOsMachOPatcher.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/elfpatcher/src/macos/MacOsImage.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/elfpatcher/src/macos/MacOsGuestModuleWriter.cpp ${CMAKE_CURRENT_SOURCE_DIR}/elfpatcher/src/general/SegmentFilter.cpp ${CMAKE_CURRENT_SOURCE_DIR}/elfpatcher/src/general/EntryStubBuilder.cpp ${CMAKE_CURRENT_SOURCE_DIR}/elfpatcher/src/general/ProgramHeaderLayoutBuilder.cpp @@ -192,9 +195,20 @@ if(BUILD_TESTING) ${CMAKE_CURRENT_SOURCE_DIR}/io/include ) add_test(NAME amd64_only_converter COMMAND amd64_only_converter_tests) + add_test_executable(decoder_length_tests + ${CMAKE_CURRENT_SOURCE_DIR}/codegen/tests/DecoderLengthTests.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/codegen/src/x86/X64InstructionDecoder.cpp + ) + target_include_directories(decoder_length_tests PRIVATE + ${CMAKE_CURRENT_SOURCE_DIR}/domain/include + ${CMAKE_CURRENT_SOURCE_DIR}/codegen/include + ) + add_test(NAME decoder_length COMMAND decoder_length_tests) find_package(Python3 QUIET COMPONENTS Interpreter) if(Python3_Interpreter_FOUND) + add_test(NAME cli_help COMMAND ${Python3_EXECUTABLE} + ${CMAKE_CURRENT_SOURCE_DIR}/cli/tests/test_help.py $) add_test(NAME elf_offsets COMMAND ${Python3_EXECUTABLE} ${CMAKE_CURRENT_SOURCE_DIR}/relinker/tests/test_elf_offsets.py $) add_test(NAME dynamic_segment COMMAND ${Python3_EXECUTABLE} @@ -205,8 +219,18 @@ if(BUILD_TESTING) ${CMAKE_CURRENT_SOURCE_DIR}/relinker/tests/test_optional_plt.py $) add_test(NAME empty_tls COMMAND ${Python3_EXECUTABLE} ${CMAKE_CURRENT_SOURCE_DIR}/relinker/tests/test_empty_tls.py $) - add_test(NAME tls_function_coverage COMMAND ${Python3_EXECUTABLE} - ${CMAKE_CURRENT_SOURCE_DIR}/relinker/tests/test_tls_function_coverage.py $) + set(tlsCoverageGroups metadata padding register_loads displacement_bounds alu_execution register_address_loads rejected) + foreach(register IN ITEMS 0 1 2 3 5 6 7 8 9 10 11 12 13 14 15) + list(APPEND tlsCoverageGroups displacement_r${register}) + endforeach() + foreach(opcode IN ITEMS add or adc sbb and sub xor cmp) + list(APPEND tlsCoverageGroups alu_${opcode}) + endforeach() + foreach(group IN LISTS tlsCoverageGroups) + add_test(NAME tls_function_coverage_${group} COMMAND ${Python3_EXECUTABLE} + ${CMAKE_CURRENT_SOURCE_DIR}/relinker/tests/test_tls_function_coverage.py $ ${group}) + set_tests_properties(tls_function_coverage_${group} PROPERTIES TIMEOUT 30 LABELS tls_function_coverage) + endforeach() add_test(NAME tls_diagnostics COMMAND ${Python3_EXECUTABLE} ${CMAKE_CURRENT_SOURCE_DIR}/relinker/tests/test_tls_diagnostics.py $) add_test(NAME linux_load_alignment COMMAND ${Python3_EXECUTABLE} @@ -215,14 +239,22 @@ if(BUILD_TESTING) ${CMAKE_CURRENT_SOURCE_DIR}/relinker/tests/test_linux_eh_frame_header.py $) add_test(NAME linux_dynamic_strings COMMAND ${Python3_EXECUTABLE} ${CMAKE_CURRENT_SOURCE_DIR}/relinker/tests/test_linux_dynamic_strings.py $) + add_test(NAME guest_runpath COMMAND ${Python3_EXECUTABLE} + ${CMAKE_CURRENT_SOURCE_DIR}/relinker/tests/test_guest_runpath.py $) add_test(NAME executable_segments COMMAND ${Python3_EXECUTABLE} ${CMAKE_CURRENT_SOURCE_DIR}/relinker/tests/test_executable_segments.py $) add_test(NAME linux_entry_argv COMMAND ${Python3_EXECUTABLE} ${CMAKE_CURRENT_SOURCE_DIR}/relinker/tests/test_linux_entry_argv.py $) + add_test(NAME linux_program_header_order COMMAND ${Python3_EXECUTABLE} + ${CMAKE_CURRENT_SOURCE_DIR}/relinker/tests/test_linux_program_header_order.py $) add_test(NAME windows_gui COMMAND ${Python3_EXECUTABLE} ${CMAKE_CURRENT_SOURCE_DIR}/relinker/tests/test_windows_gui.py $) + add_test(NAME windows_working_directory COMMAND ${Python3_EXECUTABLE} + ${CMAKE_CURRENT_SOURCE_DIR}/relinker/tests/test_windows_working_directory.py $) add_test(NAME windows_address_space COMMAND ${Python3_EXECUTABLE} ${CMAKE_CURRENT_SOURCE_DIR}/relinker/tests/test_windows_address_space.py $) + add_test(NAME windows_bss COMMAND ${Python3_EXECUTABLE} + ${CMAKE_CURRENT_SOURCE_DIR}/relinker/tests/test_windows_bss.py $) add_test(NAME windows_icon COMMAND ${Python3_EXECUTABLE} ${CMAKE_CURRENT_SOURCE_DIR}/relinker/tests/test_windows_icon.py $) add_test(NAME guest_intel_trampolines COMMAND ${Python3_EXECUTABLE} @@ -233,24 +265,56 @@ if(BUILD_TESTING) ${CMAKE_CURRENT_SOURCE_DIR}/relinker/tests/test_guest_needed_modules.py $) add_test(NAME guest_module_directories COMMAND ${Python3_EXECUTABLE} ${CMAKE_CURRENT_SOURCE_DIR}/relinker/tests/test_guest_module_directories.py $) + add_test(NAME sce_module_path COMMAND ${Python3_EXECUTABLE} + ${CMAKE_CURRENT_SOURCE_DIR}/relinker/tests/test_sce_module_path.py $) + add_test(NAME linux_guest_stack COMMAND ${Python3_EXECUTABLE} + ${CMAKE_CURRENT_SOURCE_DIR}/relinker/tests/test_linux_guest_stack.py $) add_test(NAME guest_symbol_names COMMAND ${Python3_EXECUTABLE} ${CMAKE_CURRENT_SOURCE_DIR}/relinker/tests/test_guest_symbol_names.py $) + add_test(NAME guest_registry COMMAND ${Python3_EXECUTABLE} + ${CMAKE_CURRENT_SOURCE_DIR}/relinker/tests/test_guest_registry.py $) + add_test(NAME macos_output COMMAND ${Python3_EXECUTABLE} + ${CMAKE_CURRENT_SOURCE_DIR}/relinker/tests/test_macos_output.py $) add_test(NAME guest_host_libc COMMAND ${Python3_EXECUTABLE} ${CMAKE_CURRENT_SOURCE_DIR}/relinker/tests/test_guest_host_libc.py $) + add_library(guest_module_args_host SHARED elfpatcher/tests/GuestModuleArgsHost.cpp) + target_include_directories(guest_module_args_host PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/../libs) + set_target_properties(guest_module_args_host PROPERTIES + PREFIX "" SUFFIX ".prx" WINDOWS_EXPORT_ALL_SYMBOLS ON) + add_test(NAME guest_module_args COMMAND ${Python3_EXECUTABLE} + ${CMAKE_CURRENT_SOURCE_DIR}/relinker/tests/test_guest_module_args.py + $ $) add_test(NAME guest_absolute_symbols COMMAND ${Python3_EXECUTABLE} ${CMAKE_CURRENT_SOURCE_DIR}/relinker/tests/test_guest_absolute_symbols.py $) + add_test(NAME guest_module_identity COMMAND ${Python3_EXECUTABLE} + ${CMAKE_CURRENT_SOURCE_DIR}/relinker/tests/test_guest_module_identity.py $) + add_test(NAME guest_needed_nested_modules COMMAND ${Python3_EXECUTABLE} + ${CMAKE_CURRENT_SOURCE_DIR}/relinker/tests/test_guest_needed_nested_modules.py $) add_test(NAME windows_import_modules COMMAND ${Python3_EXECUTABLE} ${CMAKE_CURRENT_SOURCE_DIR}/relinker/tests/test_windows_import_modules.py $) add_test(NAME elf_header COMMAND ${Python3_EXECUTABLE} ${CMAKE_CURRENT_SOURCE_DIR}/relinker/tests/test_elf_header.py $) add_test(NAME input_magic COMMAND ${Python3_EXECUTABLE} ${CMAKE_CURRENT_SOURCE_DIR}/relinker/tests/test_input_magic.py $) + add_test(NAME input_preservation COMMAND ${Python3_EXECUTABLE} + ${CMAKE_CURRENT_SOURCE_DIR}/relinker/tests/test_input_preservation.py $) + add_test(NAME input_file COMMAND ${Python3_EXECUTABLE} + ${CMAKE_CURRENT_SOURCE_DIR}/relinker/tests/test_input_file.py $) add_test(NAME string_table_bounds COMMAND ${Python3_EXECUTABLE} ${CMAKE_CURRENT_SOURCE_DIR}/relinker/tests/test_string_table_bounds.py $) add_test(NAME call_registry COMMAND ${Python3_EXECUTABLE} ${CMAKE_CURRENT_SOURCE_DIR}/relinker/tests/test_call_registry.py $) add_test(NAME jmprel_table_bounds COMMAND ${Python3_EXECUTABLE} ${CMAKE_CURRENT_SOURCE_DIR}/relinker/tests/test_jmprel_table_bounds.py $) + if(WIN32) + add_test(NAME windows_diagnostic_fixture_isolation COMMAND ${Python3_EXECUTABLE} + ${CMAKE_CURRENT_SOURCE_DIR}/elfpatcher/tests/test_windows_diagnostic_fixture_isolation.py $) + set_tests_properties(windows_diagnostic_fixture_isolation PROPERTIES TIMEOUT 30) + endif() + add_test(NAME entry_point_bounds COMMAND ${Python3_EXECUTABLE} + ${CMAKE_CURRENT_SOURCE_DIR}/relinker/tests/test_entry_point_bounds.py $) + add_test(NAME init_array_roots COMMAND ${Python3_EXECUTABLE} + ${CMAKE_CURRENT_SOURCE_DIR}/relinker/tests/test_init_array_roots.py $) else() message(STATUS "Python3 interpreter not found; skipping Python relinker tests") endif() @@ -267,6 +331,16 @@ if(BUILD_TESTING) ) add_test(NAME windows_icon_resource COMMAND windows_icon_resource_tests) + add_test_executable(entry_stub_tests + ${CMAKE_CURRENT_SOURCE_DIR}/elfpatcher/tests/EntryStubTests.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/elfpatcher/src/general/EntryStubBuilder.cpp + ) + target_include_directories(entry_stub_tests PRIVATE + ${CMAKE_CURRENT_SOURCE_DIR}/elfpatcher/include + ${CMAKE_CURRENT_SOURCE_DIR}/domain/include + ) + add_test(NAME entry_stub COMMAND entry_stub_tests) + add_test_executable(strict_nid_filter_tests ${CMAKE_CURRENT_SOURCE_DIR}/relinker/tests/StrictNidFilterTests.cpp ${CMAKE_CURRENT_SOURCE_DIR}/relinker/src/analysis/UnusedNidFilter/StrictNidFilter.cpp @@ -306,5 +380,15 @@ if(BUILD_TESTING) ${CMAKE_CURRENT_SOURCE_DIR}/relinker/tests/GuestIntelTrampolinesTests.cpp ) add_test(NAME guest_intel_trampolines_execution COMMAND guest_intel_trampolines_tests $) + + add_test_executable(dynamic_table_bounds_tests + ${CMAKE_CURRENT_SOURCE_DIR}/relinker/tests/DynamicTableBoundsTests.cpp + ) + add_test(NAME dynamic_table_bounds COMMAND dynamic_table_bounds_tests $) + + add_test_executable(reloc_table_bounds_tests + ${CMAKE_CURRENT_SOURCE_DIR}/relinker/tests/RelocTableBoundsTests.cpp + ) + add_test(NAME reloc_table_bounds COMMAND reloc_table_bounds_tests $) endif() endif() diff --git a/core/relinker/cli/include/Cli.hpp b/core/relinker/cli/include/Cli.hpp index 15005aa2a..9e44a1d9a 100644 --- a/core/relinker/cli/include/Cli.hpp +++ b/core/relinker/cli/include/Cli.hpp @@ -8,11 +8,13 @@ namespace Cli { struct Args { + bool showHelp = false; bool skipSyscallCheck = false; bool skipSceModule = false; bool toIntel = false; bool writeRegistry = false; bool toWindows = false; + bool toMacos = false; bool lazyBinding = false; bool autorun = false; bool windowsDiagnostics = false; @@ -20,12 +22,15 @@ struct Args { std::uint32_t unusedFilterLevel = 0; std::string inputPath; std::string outputPath; + std::string sceModulePath; std::string runPath = "$ORIGIN/libs"; std::set excludedSceModules; }; Args ParseArgs(int argc, char* argv[]); +const char* Usage(); + int Autorun(const std::string& absPath, bool toWindows); } diff --git a/core/relinker/cli/src/CliArgs.cpp b/core/relinker/cli/src/CliArgs.cpp index 212d084d8..8fcda4b31 100644 --- a/core/relinker/cli/src/CliArgs.cpp +++ b/core/relinker/cli/src/CliArgs.cpp @@ -1,20 +1,34 @@ #include +#include #include #include #include namespace Cli { +const char* Usage() { + return "Usage: relinker [--help] [--windows | --macos] [--windows-diagnostics] [--windows-gui] [--skip-syscall-check] [--skip-sce-module | --sce-module-path ] [--exclude-sce-module ]... [--to-intel] [unused-filter=0|1|2] [--registry] [--rpath ] [--lazy-binding] [--autorun] \n" + "Example: relinker input.elf output.elf"; +} + Args ParseArgs(int argc, char* argv[]) { Args args; bool unusedFilterSpecified = false; + bool sceModulePathSpecified = false; for (int i = 1; i < argc; ++i) { const std::string arg = argv[i]; - if (arg == "--skip-syscall-check") { + if (arg == "--help") { + args.showHelp = true; + } else if (arg == "--skip-syscall-check") { args.skipSyscallCheck = true; } else if (arg == "--skip-sce-module") { args.skipSceModule = true; + } else if (arg == "--sce-module-path") { + if (sceModulePathSpecified) throw std::runtime_error("--sce-module-path must be specified once"); + if (i + 1 >= argc || std::string(argv[i + 1]).empty() || std::string(argv[i + 1]).starts_with("--")) throw std::runtime_error("--sce-module-path requires a nonempty path"); + args.sceModulePath = argv[++i]; + sceModulePathSpecified = true; } else if (arg == "--exclude-sce-module") { if (i + 1 >= argc) throw std::runtime_error("--exclude-sce-module requires a file name"); @@ -35,6 +49,8 @@ Args ParseArgs(int argc, char* argv[]) { args.runPath = argv[++i]; } else if (arg == "--windows") { args.toWindows = true; + } else if (arg == "--macos") { + args.toMacos = true; } else if (arg == "--lazy-binding") { args.lazyBinding = true; } else if (arg == "--autorun") { @@ -54,20 +70,24 @@ Args ParseArgs(int argc, char* argv[]) { } } + if (args.skipSceModule && sceModulePathSpecified) throw std::runtime_error("--sce-module-path conflicts with --skip-sce-module"); + if (args.skipSceModule && !args.excludedSceModules.empty()) throw std::runtime_error("--exclude-sce-module conflicts with --skip-sce-module"); + if (args.toWindows && args.toMacos) + throw std::runtime_error("--windows conflicts with --macos"); if (args.windowsDiagnostics && !args.toWindows) throw std::runtime_error("--windows-diagnostics requires --windows"); if (args.windowsGui && !args.toWindows) throw std::runtime_error("--windows-gui requires --windows"); - if (args.inputPath.empty() || args.outputPath.empty()) - throw std::runtime_error( - "Usage: relinker [--windows] [--windows-diagnostics] [--windows-gui] [--skip-syscall-check] [--skip-sce-module] [--exclude-sce-module ]... [--to-intel] [unused-filter=0|1|2] [--registry] [--rpath ] [--lazy-binding] [--autorun] \n" - "Example: relinker input.elf output.elf" - ); + if (!args.showHelp && (args.inputPath.empty() || args.outputPath.empty())) + throw std::runtime_error(Usage()); + + + if (!sceModulePathSpecified && !args.skipSceModule && !args.inputPath.empty()) args.sceModulePath = std::filesystem::absolute(args.inputPath).parent_path().string(); return args; } diff --git a/core/relinker/cli/tests/test_help.py b/core/relinker/cli/tests/test_help.py new file mode 100644 index 000000000..8c9659aff --- /dev/null +++ b/core/relinker/cli/tests/test_help.py @@ -0,0 +1,44 @@ +from pathlib import Path +import subprocess +import sys +import tempfile + + +def main(): + relinker = Path(sys.argv[1]).resolve() + with tempfile.TemporaryDirectory(prefix="anyps5-help-") as directory: + work = Path(directory) + source = work / "missing.elf" + output = work / "existing.exe" + output.write_bytes(b"preserve this output") + + def run(*args): + return subprocess.run([str(relinker), *args], cwd=work, + capture_output=True, text=True, timeout=10) + + for args in [("--help",), ("--help", "--registry", "--autorun", str(source), str(output)), + (str(source), str(output), "--help")]: + result = run(*args) + assert result.returncode == 0, result.stderr + assert result.stdout.startswith("Usage: relinker "), result.stdout + assert "--help" in result.stdout and "Example:" in result.stdout + assert result.stderr == "", result.stderr + assert output.read_bytes() == b"preserve this output" + assert sorted(path.name for path in work.iterdir()) == [output.name] + + result = run() + assert result.returncode == 1 and "Usage: relinker" in result.stderr + for args in [("--help", "--unknown"), ("--unknown", "--help")]: + result = run(*args) + assert result.returncode == 1 and "unknown option" in result.stderr + result = run("--help", "--windows-gui") + assert result.returncode == 1 and "requires --windows" in result.stderr + for flag in ["--rpath", "--exclude-sce-module"]: + result = run(flag, "--help", str(source), str(output)) + assert result.returncode == 2 and "Cannot open file" in result.stderr + assert result.stdout == "", result.stdout + assert output.read_bytes() == b"preserve this output" + + +if __name__ == "__main__": + main() diff --git a/core/relinker/codegen/include/codegen/x86/X64OpcodeConstants.hpp b/core/relinker/codegen/include/codegen/x86/X64OpcodeConstants.hpp index 969a1fd30..889d0c447 100644 --- a/core/relinker/codegen/include/codegen/x86/X64OpcodeConstants.hpp +++ b/core/relinker/codegen/include/codegen/x86/X64OpcodeConstants.hpp @@ -220,6 +220,10 @@ inline constexpr std::size_t Vex2PrefixLength = 2; inline constexpr std::size_t Vex3PrefixLength = 3; inline constexpr std::uint8_t Vex3MapMask = 0x1F; inline constexpr std::uint8_t Vex3Map0F3A = 0x03; +inline constexpr std::uint8_t XopPrefix = 0x8F; +inline constexpr std::uint8_t XopMapMin = 0x08; +inline constexpr std::uint8_t XopMap8 = 0x08; +inline constexpr std::uint8_t XopMapA = 0x0A; } diff --git a/core/relinker/codegen/src/Amd64OnlyConverter.cpp b/core/relinker/codegen/src/Amd64OnlyConverter.cpp index 0b3239d47..409b62e87 100644 --- a/core/relinker/codegen/src/Amd64OnlyConverter.cpp +++ b/core/relinker/codegen/src/Amd64OnlyConverter.cpp @@ -3,7 +3,6 @@ #include #include #include -#include #include #include #include @@ -39,7 +38,6 @@ private: Amd64OnlyMatch Substitution; }; - X64InstructionRewriter _rewriter; std::unique_ptr _matcher = MakeAmd64OnlyInstructionMatcher(); std::unique_ptr _scanner = MakeInstructionScanner(); @@ -120,7 +118,7 @@ void Amd64OnlyConverter::_convertSegment( case Amd64OnlyLowering::InPlace: { if (substitution.ReplacementBytes.size() != match.Length) throw CodegenException("Intel substitution changes the instruction length", fileOffset); - seg = _atFileOffset(ph.Offset, [&] { return _rewriter.Rewrite(seg, {match.Offset, substitution.ReplacementBytes}).Bytes; }); + std::copy(substitution.ReplacementBytes.begin(), substitution.ReplacementBytes.end(), seg.begin() + static_cast(match.Offset)); replacementLength = substitution.ReplacementBytes.size(); ++result.ReplacedCount; break; diff --git a/core/relinker/codegen/src/x86/X64InstructionDecoder.cpp b/core/relinker/codegen/src/x86/X64InstructionDecoder.cpp index 7a0b2a001..af8eefc1e 100644 --- a/core/relinker/codegen/src/x86/X64InstructionDecoder.cpp +++ b/core/relinker/codegen/src/x86/X64InstructionDecoder.cpp @@ -55,6 +55,7 @@ std::size_t X64InstructionDecoder::Decode(const std::uint8_t* data, std::size_t bool vexPresent = false; bool evexPresent = false; std::uint8_t vexMap = 0; + std::uint8_t xopMap = 0; std::uint8_t threeByteMap = 0; bool threeByteEscape = false; @@ -76,6 +77,12 @@ std::size_t X64InstructionDecoder::Decode(const std::uint8_t* data, std::size_t throw CodegenException("Instruction truncated after VEX3 prefix"); } pos += 1; + } else if (opcode == XopPrefix && pos < available && (data[pos] & Vex3MapMask) >= XopMapMin) { + if (pos + 1 >= available) { + throw CodegenException("Instruction truncated after XOP prefix"); + } + xopMap = static_cast(data[pos] & Vex3MapMask); + pos += 2; } else if (opcode == EvexPrefix) { if (pos + 2 >= available) { throw CodegenException("Instruction truncated after EVEX prefix"); @@ -101,7 +108,7 @@ std::size_t X64InstructionDecoder::Decode(const std::uint8_t* data, std::size_t } } - if (vexPresent || evexPresent) { + if (vexPresent || evexPresent || xopMap != 0) { if (pos >= available) { throw CodegenException("Instruction truncated after VEX prefix"); } @@ -114,7 +121,14 @@ std::size_t X64InstructionDecoder::Decode(const std::uint8_t* data, std::size_t const bool vectorImmediate = vexMap == 1 && ((opcode >= 0x70 && opcode <= 0x73) || opcode == 0xC2 || opcode == 0xC4 || opcode == 0xC5 || opcode == 0xC6); - if (vexPresent) { + if (xopMap != 0) { + hasModRm = true; + if (xopMap == XopMap8) { + immediateSize = ImmSize8; + } else if (xopMap == XopMapA) { + immediateSize = ImmSize32; + } + } else if (vexPresent) { if (vexMap == 1 && opcode >= VexNoModRmMin && opcode <= VexNoModRmMax) { hasModRm = false; } else { @@ -379,8 +393,9 @@ DecodedInstructionInfo X64InstructionDecoder::DecodeInstruction( std::uint8_t op = data[pos++]; bool twoByteOpcode = false; - if (op == OneByteVex2 || op == OneByteVex3 || op == EvexPrefix) { - pos += (op == OneByteVex2 ? 1 : (op == OneByteVex3 ? 2 : EvexPrefixLength - 1)) + 1; + const bool xop = op == XopPrefix && pos < info.Length && (data[pos] & Vex3MapMask) >= XopMapMin; + if (op == OneByteVex2 || op == OneByteVex3 || op == EvexPrefix || xop) { + pos += (op == OneByteVex2 ? 1 : (op == OneByteVex3 || xop ? 2 : EvexPrefixLength - 1)) + 1; if (pos < info.Length) readModRm(pos, false); info.FlowKind = ControlFlowKind::Sequential; diff --git a/core/relinker/codegen/tests/Amd64OnlyConverterTests.cpp b/core/relinker/codegen/tests/Amd64OnlyConverterTests.cpp index c3e4a1da2..236b8e9cd 100644 --- a/core/relinker/codegen/tests/Amd64OnlyConverterTests.cpp +++ b/core/relinker/codegen/tests/Amd64OnlyConverterTests.cpp @@ -135,7 +135,10 @@ void decoderLengths() { {0x66, 0x48, 0x69, 0xC0, 1, 2, 3, 4}, {0x66, 0x48, 0x68, 1, 2, 3, 4}, {0x66, 0x48, 0xA9, 1, 2, 3, 4}, {0x66, 0x48, 0xF7, 0xC0, 1, 2, 3, 4}, {0x66, 0x81, 0xC0, 0x34, 0x12}, {0x48, 0x66, 0x81, 0xC0, 0x34, 0x12}, {0x66, 0x41, 0x81, 0xC0, 0x34, 0x12}, {0x66, 0xF7, 0xC0, 0x34, 0x12}, - {0x48, 0x64, 0x8B, 0x00}, {0x48, 0xF3, 0x0F, 0x2B, 0x00}, {0x48, 0x67, 0x0F, 0x01, 0xFC}, {0x48, 0x48, 0x0F, 0x01, 0xFC}}; + {0x48, 0x64, 0x8B, 0x00}, {0x48, 0xF3, 0x0F, 0x2B, 0x00}, {0x48, 0x67, 0x0F, 0x01, 0xFC}, {0x48, 0x48, 0x0F, 0x01, 0xFC}, + {0x8F, 0xE8, 0x40, 0xA3, 0xC1, 0x70}, {0x8F, 0xE8, 0xF8, 0xA3, 0x4A, 0x10, 0x60}, {0x8F, 0xE8, 0xC0, 0xA3, 0xBA, 0x70, 0xFF, 0xFF, 0xFF, 0x30}, + {0x8F, 0xE9, 0x78, 0x90, 0xC1}, {0x8F, 0xEA, 0x78, 0x10, 0xC0, 1, 2, 3, 4}, + {0x8F, 0xC0}, {0x41, 0x8F, 0x07}, {0x8F, 0x05, 1, 2, 3, 4}}; Bytes padded; for (const auto& instruction : instructions) { padded = instruction; @@ -156,7 +159,8 @@ const std::vector kRipRelativeVectorLoads = { {0xC4, 0xE2, 0x79, 0x00, 0x05, 0x10, 0x00, 0x00, 0x00}, {0x62, 0xF1, 0xFD, 0x08, 0x6F, 0x05, 0x10, 0x00, 0x00, 0x00}, {0x66, 0x0F, 0x38, 0x00, 0x05, 0x10, 0x00, 0x00, 0x00}, - {0x66, 0x0F, 0x3A, 0x0F, 0x05, 0x10, 0x00, 0x00, 0x00, 0x08}}; + {0x66, 0x0F, 0x3A, 0x0F, 0x05, 0x10, 0x00, 0x00, 0x00, 0x08}, + {0x8F, 0xE8, 0x78, 0xA3, 0x05, 0x10, 0x00, 0x00, 0x00, 0x30}}; const std::vector kRegisterVectorOperations = { {0xC5, 0xF9, 0x6F, 0xC1}, {0x66, 0x0F, 0x38, 0x00, 0xC1}, {0xC5, 0xF8, 0x77}, {0xC4, 0xE2, 0x79, 0x00, 0x00}}; @@ -164,7 +168,7 @@ void decoderRipRelative() { const Codegen::X64InstructionDecoder decoder; for (const auto& instruction : kRipRelativeVectorLoads) { const auto info = decoder.DecodeInstruction(instruction.data(), instruction.size()); - require(info.Length == instruction.size() && info.HasRipRelativeDisp && read(instruction, info.RipRelativeDispOffset) == 0x10, "VEX, EVEX or three-byte RIP-relative operand was not reported"); + require(info.Length == instruction.size() && info.HasRipRelativeDisp && read(instruction, info.RipRelativeDispOffset) == 0x10, "VEX, EVEX, XOP or three-byte RIP-relative operand was not reported"); } for (const auto& instruction : kRegisterVectorOperations) require(!decoder.DecodeInstruction(instruction.data(), instruction.size()).HasRipRelativeDisp, "Vector instruction without a RIP-relative operand was reported as RIP-relative"); @@ -772,7 +776,7 @@ Bytes elfFixture(const Bytes& text) { write(bytes, 24, 0x1000); write(bytes, 32, 64); write(bytes, 54, 56); - write(bytes, 56, 6); + write(bytes, 56, 7); write(bytes, 64, 1); write(bytes, 68, 5); write(bytes, 72, 0x200); diff --git a/core/relinker/codegen/tests/DecoderLengthCases.hpp b/core/relinker/codegen/tests/DecoderLengthCases.hpp new file mode 100644 index 000000000..561f3a64e --- /dev/null +++ b/core/relinker/codegen/tests/DecoderLengthCases.hpp @@ -0,0 +1,960 @@ +#ifndef CODEGEN_TESTS_DECODERLENGTHCASES_HPP +#define CODEGEN_TESTS_DECODERLENGTHCASES_HPP + +#include + +struct DecoderLengthCase { + const char* Bytes; + std::size_t Length; +}; + +inline constexpr DecoderLengthCase kDecoderLengthCases[] = { + {"00846511223344", 7}, + {"01846511223344", 7}, + {"02846511223344", 7}, + {"03846511223344", 7}, + {"0484", 2}, + {"0584651122", 5}, + {"08846511223344", 7}, + {"09846511223344", 7}, + {"0a846511223344", 7}, + {"0b846511223344", 7}, + {"0c84", 2}, + {"0d84651122", 5}, + {"0f00846511223344", 8}, + {"0f01846511223344", 8}, + {"0f02846511223344", 8}, + {"0f03846511223344", 8}, + {"0f05", 2}, + {"0f06", 2}, + {"0f07", 2}, + {"0f08", 2}, + {"0f09", 2}, + {"0f0b", 2}, + {"0f0d846511223344", 8}, + {"0f0e", 2}, + {"0f10846511223344", 8}, + {"0f11846511223344", 8}, + {"0f12846511223344", 8}, + {"0f13846511223344", 8}, + {"0f14846511223344", 8}, + {"0f15846511223344", 8}, + {"0f16846511223344", 8}, + {"0f17846511223344", 8}, + {"0f18846511223344", 8}, + {"0f1c846511223344", 8}, + {"0f1f846511223344", 8}, + {"0f20c1", 3}, + {"0f21c1", 3}, + {"0f22c1", 3}, + {"0f23c1", 3}, + {"0f28846511223344", 8}, + {"0f29846511223344", 8}, + {"0f2a846511223344", 8}, + {"0f2b846511223344", 8}, + {"0f2c846511223344", 8}, + {"0f2d846511223344", 8}, + {"0f2e846511223344", 8}, + {"0f2f846511223344", 8}, + {"0f30", 2}, + {"0f31", 2}, + {"0f32", 2}, + {"0f33", 2}, + {"0f34", 2}, + {"0f35", 2}, + {"0f37", 2}, + {"0f3800846511223344", 9}, + {"0f3801846511223344", 9}, + {"0f3802846511223344", 9}, + {"0f3803846511223344", 9}, + {"0f3804846511223344", 9}, + {"0f3805846511223344", 9}, + {"0f3806846511223344", 9}, + {"0f3807846511223344", 9}, + {"0f3808846511223344", 9}, + {"0f3809846511223344", 9}, + {"0f380a846511223344", 9}, + {"0f380b846511223344", 9}, + {"0f381c846511223344", 9}, + {"0f381d846511223344", 9}, + {"0f381e846511223344", 9}, + {"0f388a846511223344", 9}, + {"0f388b846511223344", 9}, + {"0f38c8846511223344", 9}, + {"0f38c9846511223344", 9}, + {"0f38ca846511223344", 9}, + {"0f38cb846511223344", 9}, + {"0f38cc846511223344", 9}, + {"0f38cd846511223344", 9}, + {"0f38f0846511223344", 9}, + {"0f38f1846511223344", 9}, + {"0f38f6846511223344", 9}, + {"0f38f9846511223344", 9}, + {"0f38fc846511223344", 9}, + {"0f3a0f84651122334455", 10}, + {"0f3acc84651122334455", 10}, + {"0f40846511223344", 8}, + {"0f41846511223344", 8}, + {"0f42846511223344", 8}, + {"0f43846511223344", 8}, + {"0f44846511223344", 8}, + {"0f45846511223344", 8}, + {"0f46846511223344", 8}, + {"0f47846511223344", 8}, + {"0f48846511223344", 8}, + {"0f49846511223344", 8}, + {"0f4a846511223344", 8}, + {"0f4b846511223344", 8}, + {"0f4c846511223344", 8}, + {"0f4d846511223344", 8}, + {"0f4e846511223344", 8}, + {"0f4f846511223344", 8}, + {"0f50c1", 3}, + {"0f51846511223344", 8}, + {"0f52846511223344", 8}, + {"0f53846511223344", 8}, + {"0f54846511223344", 8}, + {"0f55846511223344", 8}, + {"0f56846511223344", 8}, + {"0f57846511223344", 8}, + {"0f58846511223344", 8}, + {"0f59846511223344", 8}, + {"0f5a846511223344", 8}, + {"0f5b846511223344", 8}, + {"0f5c846511223344", 8}, + {"0f5d846511223344", 8}, + {"0f5e846511223344", 8}, + {"0f5f846511223344", 8}, + {"0f60846511223344", 8}, + {"0f61846511223344", 8}, + {"0f62846511223344", 8}, + {"0f63846511223344", 8}, + {"0f64846511223344", 8}, + {"0f65846511223344", 8}, + {"0f66846511223344", 8}, + {"0f67846511223344", 8}, + {"0f68846511223344", 8}, + {"0f69846511223344", 8}, + {"0f6a846511223344", 8}, + {"0f6b846511223344", 8}, + {"0f6e846511223344", 8}, + {"0f6f846511223344", 8}, + {"0f7084651122334455", 9}, + {"0f74846511223344", 8}, + {"0f75846511223344", 8}, + {"0f76846511223344", 8}, + {"0f77", 2}, + {"0f7e846511223344", 8}, + {"0f7f846511223344", 8}, + {"0f8084651122", 6}, + {"0f8184651122", 6}, + {"0f8284651122", 6}, + {"0f8384651122", 6}, + {"0f8484651122", 6}, + {"0f8584651122", 6}, + {"0f8684651122", 6}, + {"0f8784651122", 6}, + {"0f8884651122", 6}, + {"0f8984651122", 6}, + {"0f8a84651122", 6}, + {"0f8b84651122", 6}, + {"0f8c84651122", 6}, + {"0f8d84651122", 6}, + {"0f8e84651122", 6}, + {"0f8f84651122", 6}, + {"0f90846511223344", 8}, + {"0f91846511223344", 8}, + {"0f92846511223344", 8}, + {"0f93846511223344", 8}, + {"0f94846511223344", 8}, + {"0f95846511223344", 8}, + {"0f96846511223344", 8}, + {"0f97846511223344", 8}, + {"0f98846511223344", 8}, + {"0f99846511223344", 8}, + {"0f9a846511223344", 8}, + {"0f9b846511223344", 8}, + {"0f9c846511223344", 8}, + {"0f9d846511223344", 8}, + {"0f9e846511223344", 8}, + {"0f9f846511223344", 8}, + {"0fa0", 2}, + {"0fa1", 2}, + {"0fa2", 2}, + {"0fa3846511223344", 8}, + {"0fa484651122334455", 9}, + {"0fa5846511223344", 8}, + {"0fa8", 2}, + {"0fa9", 2}, + {"0faa", 2}, + {"0fab846511223344", 8}, + {"0fac84651122334455", 9}, + {"0fad846511223344", 8}, + {"0fae846511223344", 8}, + {"0faf846511223344", 8}, + {"0fb0846511223344", 8}, + {"0fb1846511223344", 8}, + {"0fb2846511223344", 8}, + {"0fb3846511223344", 8}, + {"0fb4846511223344", 8}, + {"0fb5846511223344", 8}, + {"0fb6846511223344", 8}, + {"0fb7846511223344", 8}, + {"0fb9846511223344", 8}, + {"0fbb846511223344", 8}, + {"0fbc846511223344", 8}, + {"0fbd846511223344", 8}, + {"0fbe846511223344", 8}, + {"0fbf846511223344", 8}, + {"0fc0846511223344", 8}, + {"0fc1846511223344", 8}, + {"0fc284651122334455", 9}, + {"0fc3846511223344", 8}, + {"0fc484651122334455", 9}, + {"0fc5c111", 4}, + {"0fc684651122334455", 9}, + {"0fc8", 2}, + {"0fc9", 2}, + {"0fca", 2}, + {"0fcb", 2}, + {"0fcc", 2}, + {"0fcd", 2}, + {"0fce", 2}, + {"0fcf", 2}, + {"0fd1846511223344", 8}, + {"0fd2846511223344", 8}, + {"0fd3846511223344", 8}, + {"0fd4846511223344", 8}, + {"0fd5846511223344", 8}, + {"0fd7c1", 3}, + {"0fd8846511223344", 8}, + {"0fd9846511223344", 8}, + {"0fda846511223344", 8}, + {"0fdb846511223344", 8}, + {"0fdc846511223344", 8}, + {"0fdd846511223344", 8}, + {"0fde846511223344", 8}, + {"0fdf846511223344", 8}, + {"0fe0846511223344", 8}, + {"0fe1846511223344", 8}, + {"0fe2846511223344", 8}, + {"0fe3846511223344", 8}, + {"0fe4846511223344", 8}, + {"0fe5846511223344", 8}, + {"0fe7846511223344", 8}, + {"0fe8846511223344", 8}, + {"0fe9846511223344", 8}, + {"0fea846511223344", 8}, + {"0feb846511223344", 8}, + {"0fec846511223344", 8}, + {"0fed846511223344", 8}, + {"0fee846511223344", 8}, + {"0fef846511223344", 8}, + {"0ff1846511223344", 8}, + {"0ff2846511223344", 8}, + {"0ff3846511223344", 8}, + {"0ff4846511223344", 8}, + {"0ff5846511223344", 8}, + {"0ff6846511223344", 8}, + {"0ff7c1", 3}, + {"0ff8846511223344", 8}, + {"0ff9846511223344", 8}, + {"0ffa846511223344", 8}, + {"0ffb846511223344", 8}, + {"0ffc846511223344", 8}, + {"0ffd846511223344", 8}, + {"0ffe846511223344", 8}, + {"10846511223344", 7}, + {"11846511223344", 7}, + {"12846511223344", 7}, + {"13846511223344", 7}, + {"1484", 2}, + {"1584651122", 5}, + {"18846511223344", 7}, + {"19846511223344", 7}, + {"1a846511223344", 7}, + {"1b846511223344", 7}, + {"1c84", 2}, + {"1d84651122", 5}, + {"20846511223344", 7}, + {"21846511223344", 7}, + {"22846511223344", 7}, + {"23846511223344", 7}, + {"2484", 2}, + {"2584651122", 5}, + {"28846511223344", 7}, + {"29846511223344", 7}, + {"2a846511223344", 7}, + {"2b846511223344", 7}, + {"2c84", 2}, + {"2d84651122", 5}, + {"30846511223344", 7}, + {"31846511223344", 7}, + {"32846511223344", 7}, + {"33846511223344", 7}, + {"3484", 2}, + {"3584651122", 5}, + {"38846511223344", 7}, + {"39846511223344", 7}, + {"3a846511223344", 7}, + {"3b846511223344", 7}, + {"3c84", 2}, + {"3d84651122", 5}, + {"480584651122", 6}, + {"480d84651122", 6}, + {"481584651122", 6}, + {"481d84651122", 6}, + {"482584651122", 6}, + {"482d84651122", 6}, + {"483584651122", 6}, + {"483d84651122", 6}, + {"486884651122", 6}, + {"486984651122334455667708", 12}, + {"488184651122334455667708", 12}, + {"48a984651122", 6}, + {"48b88465112233445566", 10}, + {"48b98465112233445566", 10}, + {"48ba8465112233445566", 10}, + {"48bb8465112233445566", 10}, + {"48bc8465112233445566", 10}, + {"48bd8465112233445566", 10}, + {"48be8465112233445566", 10}, + {"48bf8465112233445566", 10}, + {"48c784651122334455667708", 12}, + {"48f784651122334455667708", 12}, + {"50", 1}, + {"51", 1}, + {"52", 1}, + {"53", 1}, + {"54", 1}, + {"55", 1}, + {"56", 1}, + {"57", 1}, + {"58", 1}, + {"59", 1}, + {"5a", 1}, + {"5b", 1}, + {"5c", 1}, + {"5d", 1}, + {"5e", 1}, + {"5f", 1}, + {"63846511223344", 7}, + {"66058465", 4}, + {"660d8465", 4}, + {"660f3810846511223344", 10}, + {"660f3814846511223344", 10}, + {"660f3815846511223344", 10}, + {"660f3817846511223344", 10}, + {"660f3820846511223344", 10}, + {"660f3821846511223344", 10}, + {"660f3822846511223344", 10}, + {"660f3823846511223344", 10}, + {"660f3824846511223344", 10}, + {"660f3825846511223344", 10}, + {"660f3828846511223344", 10}, + {"660f3829846511223344", 10}, + {"660f382a846511223344", 10}, + {"660f382b846511223344", 10}, + {"660f3830846511223344", 10}, + {"660f3831846511223344", 10}, + {"660f3832846511223344", 10}, + {"660f3833846511223344", 10}, + {"660f3834846511223344", 10}, + {"660f3835846511223344", 10}, + {"660f3837846511223344", 10}, + {"660f3838846511223344", 10}, + {"660f3839846511223344", 10}, + {"660f383a846511223344", 10}, + {"660f383b846511223344", 10}, + {"660f383c846511223344", 10}, + {"660f383d846511223344", 10}, + {"660f383e846511223344", 10}, + {"660f383f846511223344", 10}, + {"660f3840846511223344", 10}, + {"660f3841846511223344", 10}, + {"660f3880846511223344", 10}, + {"660f3881846511223344", 10}, + {"660f3882846511223344", 10}, + {"660f38cf846511223344", 10}, + {"660f38db846511223344", 10}, + {"660f38dc846511223344", 10}, + {"660f38dd846511223344", 10}, + {"660f38de846511223344", 10}, + {"660f38df846511223344", 10}, + {"660f38f5846511223344", 10}, + {"660f38f8846511223344", 10}, + {"660f3a0884651122334455", 11}, + {"660f3a0984651122334455", 11}, + {"660f3a0a84651122334455", 11}, + {"660f3a0b84651122334455", 11}, + {"660f3a0c84651122334455", 11}, + {"660f3a0d84651122334455", 11}, + {"660f3a0e84651122334455", 11}, + {"660f3a1484651122334455", 11}, + {"660f3a1584651122334455", 11}, + {"660f3a1684651122334455", 11}, + {"660f3a1784651122334455", 11}, + {"660f3a2084651122334455", 11}, + {"660f3a2184651122334455", 11}, + {"660f3a2284651122334455", 11}, + {"660f3a4084651122334455", 11}, + {"660f3a4184651122334455", 11}, + {"660f3a4284651122334455", 11}, + {"660f3a4484651122334455", 11}, + {"660f3a6084651122334455", 11}, + {"660f3a6184651122334455", 11}, + {"660f3a6284651122334455", 11}, + {"660f3a6384651122334455", 11}, + {"660f3ace84651122334455", 11}, + {"660f3acf84651122334455", 11}, + {"660f3adf84651122334455", 11}, + {"660f6c846511223344", 9}, + {"660f6d846511223344", 9}, + {"660f78c11122", 6}, + {"660f79c1", 4}, + {"660f7c846511223344", 9}, + {"660f7d846511223344", 9}, + {"660fd0846511223344", 9}, + {"660fd6846511223344", 9}, + {"660fe6846511223344", 9}, + {"66158465", 4}, + {"661d8465", 4}, + {"66258465", 4}, + {"662d8465", 4}, + {"66358465", 4}, + {"663d8465", 4}, + {"66480584651122", 7}, + {"66480d84651122", 7}, + {"66481584651122", 7}, + {"66481d84651122", 7}, + {"66482584651122", 7}, + {"66482d84651122", 7}, + {"66483584651122", 7}, + {"66483d84651122", 7}, + {"66486884651122", 7}, + {"66486984651122334455667708", 13}, + {"66488184651122334455667708", 13}, + {"6648a984651122", 7}, + {"6648b88465112233445566", 11}, + {"6648b98465112233445566", 11}, + {"6648ba8465112233445566", 11}, + {"6648bb8465112233445566", 11}, + {"6648bc8465112233445566", 11}, + {"6648bd8465112233445566", 11}, + {"6648be8465112233445566", 11}, + {"6648bf8465112233445566", 11}, + {"6648c784651122334455667708", 13}, + {"6648f784651122334455667708", 13}, + {"66688465", 4}, + {"66698465112233445566", 10}, + {"66818465112233445566", 10}, + {"66a98465", 4}, + {"66b88465", 4}, + {"66b98465", 4}, + {"66ba8465", 4}, + {"66bb8465", 4}, + {"66bc8465", 4}, + {"66bd8465", 4}, + {"66be8465", 4}, + {"66bf8465", 4}, + {"66c78465112233445566", 10}, + {"66f78465112233445566", 10}, + {"676984651122334455667708", 12}, + {"678184651122334455667708", 12}, + {"67a084651122", 6}, + {"67a184651122", 6}, + {"67a284651122", 6}, + {"67a384651122", 6}, + {"67f784651122334455667708", 12}, + {"6884651122", 5}, + {"6984651122334455667708", 11}, + {"6a84", 2}, + {"6b84651122334455", 8}, + {"6c", 1}, + {"6d", 1}, + {"6e", 1}, + {"6f", 1}, + {"7084", 2}, + {"7184", 2}, + {"7284", 2}, + {"7384", 2}, + {"7484", 2}, + {"7584", 2}, + {"7684", 2}, + {"7784", 2}, + {"7884", 2}, + {"7984", 2}, + {"7a84", 2}, + {"7b84", 2}, + {"7c84", 2}, + {"7d84", 2}, + {"7e84", 2}, + {"7f84", 2}, + {"8084651122334455", 8}, + {"8184651122334455667708", 11}, + {"8384651122334455", 8}, + {"84846511223344", 7}, + {"85846511223344", 7}, + {"86846511223344", 7}, + {"87846511223344", 7}, + {"88846511223344", 7}, + {"89846511223344", 7}, + {"8a846511223344", 7}, + {"8b846511223344", 7}, + {"8c846511223344", 7}, + {"8d846511223344", 7}, + {"8e846511223344", 7}, + {"90", 1}, + {"91", 1}, + {"92", 1}, + {"93", 1}, + {"94", 1}, + {"95", 1}, + {"96", 1}, + {"97", 1}, + {"98", 1}, + {"99", 1}, + {"9b", 1}, + {"9c", 1}, + {"9d", 1}, + {"9e", 1}, + {"9f", 1}, + {"a08465112233445566", 9}, + {"a18465112233445566", 9}, + {"a28465112233445566", 9}, + {"a38465112233445566", 9}, + {"a4", 1}, + {"a5", 1}, + {"a6", 1}, + {"a7", 1}, + {"a884", 2}, + {"a984651122", 5}, + {"aa", 1}, + {"ab", 1}, + {"ac", 1}, + {"ad", 1}, + {"ae", 1}, + {"af", 1}, + {"b084", 2}, + {"b184", 2}, + {"b284", 2}, + {"b384", 2}, + {"b484", 2}, + {"b584", 2}, + {"b684", 2}, + {"b784", 2}, + {"b884651122", 5}, + {"b984651122", 5}, + {"ba84651122", 5}, + {"bb84651122", 5}, + {"bc84651122", 5}, + {"bd84651122", 5}, + {"be84651122", 5}, + {"bf84651122", 5}, + {"c084651122334455", 8}, + {"c184651122334455", 8}, + {"c28465", 3}, + {"c3", 1}, + {"c4e17d4bc1", 5}, + {"c4e1f913846511223344", 10}, + {"c4e1f917846511223344", 10}, + {"c4e1f944c1", 5}, + {"c4e1f96e846511223344", 10}, + {"c4e1f990846511223344", 10}, + {"c4e1f991846511223344", 10}, + {"c4e1f998c1", 5}, + {"c4e1f999c1", 5}, + {"c4e1f9c484651122334455", 11}, + {"c4e1f9c5c111", 6}, + {"c4e1f9d6846511223344", 10}, + {"c4e1f9f7c1", 5}, + {"c4e1fa7e846511223344", 10}, + {"c4e1fb92c1", 5}, + {"c4e1fb93c1", 5}, + {"c4e1fc77", 4}, + {"c4e1fd14846511223344", 10}, + {"c4e1fd15846511223344", 10}, + {"c4e1fd28846511223344", 10}, + {"c4e1fd29846511223344", 10}, + {"c4e1fd2b846511223344", 10}, + {"c4e1fd2e846511223344", 10}, + {"c4e1fd2f846511223344", 10}, + {"c4e1fd41c1", 5}, + {"c4e1fd42c1", 5}, + {"c4e1fd45c1", 5}, + {"c4e1fd46c1", 5}, + {"c4e1fd47c1", 5}, + {"c4e1fd4ac1", 5}, + {"c4e1fd50c1", 5}, + {"c4e1fd54846511223344", 10}, + {"c4e1fd55846511223344", 10}, + {"c4e1fd56846511223344", 10}, + {"c4e1fd57846511223344", 10}, + {"c4e1fd60846511223344", 10}, + {"c4e1fd61846511223344", 10}, + {"c4e1fd62846511223344", 10}, + {"c4e1fd63846511223344", 10}, + {"c4e1fd64846511223344", 10}, + {"c4e1fd65846511223344", 10}, + {"c4e1fd66846511223344", 10}, + {"c4e1fd67846511223344", 10}, + {"c4e1fd68846511223344", 10}, + {"c4e1fd69846511223344", 10}, + {"c4e1fd6a846511223344", 10}, + {"c4e1fd6b846511223344", 10}, + {"c4e1fd6c846511223344", 10}, + {"c4e1fd6d846511223344", 10}, + {"c4e1fd74846511223344", 10}, + {"c4e1fd75846511223344", 10}, + {"c4e1fd76846511223344", 10}, + {"c4e1fdc684651122334455", 11}, + {"c4e1fdd1846511223344", 10}, + {"c4e1fdd2846511223344", 10}, + {"c4e1fdd3846511223344", 10}, + {"c4e1fdd4846511223344", 10}, + {"c4e1fdd5846511223344", 10}, + {"c4e1fdd7c1", 5}, + {"c4e1fdd8846511223344", 10}, + {"c4e1fdd9846511223344", 10}, + {"c4e1fdda846511223344", 10}, + {"c4e1fddb846511223344", 10}, + {"c4e1fddc846511223344", 10}, + {"c4e1fddd846511223344", 10}, + {"c4e1fdde846511223344", 10}, + {"c4e1fddf846511223344", 10}, + {"c4e1fde0846511223344", 10}, + {"c4e1fde1846511223344", 10}, + {"c4e1fde2846511223344", 10}, + {"c4e1fde3846511223344", 10}, + {"c4e1fde4846511223344", 10}, + {"c4e1fde5846511223344", 10}, + {"c4e1fde7846511223344", 10}, + {"c4e1fde8846511223344", 10}, + {"c4e1fde9846511223344", 10}, + {"c4e1fdea846511223344", 10}, + {"c4e1fdeb846511223344", 10}, + {"c4e1fdec846511223344", 10}, + {"c4e1fded846511223344", 10}, + {"c4e1fdee846511223344", 10}, + {"c4e1fdef846511223344", 10}, + {"c4e1fdf1846511223344", 10}, + {"c4e1fdf2846511223344", 10}, + {"c4e1fdf3846511223344", 10}, + {"c4e1fdf4846511223344", 10}, + {"c4e1fdf5846511223344", 10}, + {"c4e1fdf6846511223344", 10}, + {"c4e1fdf8846511223344", 10}, + {"c4e1fdf9846511223344", 10}, + {"c4e1fdfa846511223344", 10}, + {"c4e1fdfb846511223344", 10}, + {"c4e1fdfc846511223344", 10}, + {"c4e1fdfd846511223344", 10}, + {"c4e1fdfe846511223344", 10}, + {"c4e1fe16846511223344", 10}, + {"c4e1fe52846511223344", 10}, + {"c4e1fe53846511223344", 10}, + {"c4e1fe5b846511223344", 10}, + {"c4e1fe6f846511223344", 10}, + {"c4e1fe7f846511223344", 10}, + {"c4e1ff10846511223344", 10}, + {"c4e1ff11846511223344", 10}, + {"c4e1ff12846511223344", 10}, + {"c4e1ff2a846511223344", 10}, + {"c4e1ff2c846511223344", 10}, + {"c4e1ff2d846511223344", 10}, + {"c4e1ff51846511223344", 10}, + {"c4e1ff58846511223344", 10}, + {"c4e1ff59846511223344", 10}, + {"c4e1ff5a846511223344", 10}, + {"c4e1ff5c846511223344", 10}, + {"c4e1ff5d846511223344", 10}, + {"c4e1ff5e846511223344", 10}, + {"c4e1ff5f846511223344", 10}, + {"c4e1ff7084651122334455", 11}, + {"c4e1ff7c846511223344", 10}, + {"c4e1ff7d846511223344", 10}, + {"c4e1ffc284651122334455", 11}, + {"c4e1ffd0846511223344", 10}, + {"c4e1ffe6846511223344", 10}, + {"c4e1fff0846511223344", 10}, + {"c4e27948c1", 5}, + {"c4e27949846511223344", 10}, + {"c4e2796e846511223344", 10}, + {"c4e2796f846511223344", 10}, + {"c4e27a5fc1", 5}, + {"c4e27b4a846511223344", 10}, + {"c4e27b4b846511223344", 10}, + {"c4e27b5cc1", 5}, + {"c4e27b5ec1", 5}, + {"c4e27b6bc1", 5}, + {"c4e27b6cc1", 5}, + {"c4e27d0c846511223344", 10}, + {"c4e27d0d846511223344", 10}, + {"c4e27d0e846511223344", 10}, + {"c4e27d0f846511223344", 10}, + {"c4e27d13846511223344", 10}, + {"c4e27d16846511223344", 10}, + {"c4e27d18846511223344", 10}, + {"c4e27d19846511223344", 10}, + {"c4e27d1a846511223344", 10}, + {"c4e27d2c846511223344", 10}, + {"c4e27d2d846511223344", 10}, + {"c4e27d2e846511223344", 10}, + {"c4e27d2f846511223344", 10}, + {"c4e27d36846511223344", 10}, + {"c4e27d46846511223344", 10}, + {"c4e27d52846511223344", 10}, + {"c4e27d53846511223344", 10}, + {"c4e27d58846511223344", 10}, + {"c4e27d59846511223344", 10}, + {"c4e27d5a846511223344", 10}, + {"c4e27d78846511223344", 10}, + {"c4e27d79846511223344", 10}, + {"c4e27dcf846511223344", 10}, + {"c4e27e72846511223344", 10}, + {"c4e27eb1846511223344", 10}, + {"c4e27ed2846511223344", 10}, + {"c4e27ed3846511223344", 10}, + {"c4e27f50846511223344", 10}, + {"c4e27f51846511223344", 10}, + {"c4e27fb0846511223344", 10}, + {"c4e27fcbc1", 5}, + {"c4e27fccc1", 5}, + {"c4e27fcdc1", 5}, + {"c4e27fda846511223344", 10}, + {"c4e2f8f2846511223344", 10}, + {"c4e2f941846511223344", 10}, + {"c4e2f9db846511223344", 10}, + {"c4e2f9e0846511223344", 10}, + {"c4e2f9e1846511223344", 10}, + {"c4e2f9e2846511223344", 10}, + {"c4e2f9e3846511223344", 10}, + {"c4e2f9e4846511223344", 10}, + {"c4e2f9e5846511223344", 10}, + {"c4e2f9e6846511223344", 10}, + {"c4e2f9e7846511223344", 10}, + {"c4e2f9e8846511223344", 10}, + {"c4e2f9e9846511223344", 10}, + {"c4e2f9ea846511223344", 10}, + {"c4e2f9eb846511223344", 10}, + {"c4e2f9ec846511223344", 10}, + {"c4e2f9ed846511223344", 10}, + {"c4e2f9ee846511223344", 10}, + {"c4e2f9ef846511223344", 10}, + {"c4e2fbf5846511223344", 10}, + {"c4e2fbf6846511223344", 10}, + {"c4e2fbf7846511223344", 10}, + {"c4e2fd00846511223344", 10}, + {"c4e2fd01846511223344", 10}, + {"c4e2fd02846511223344", 10}, + {"c4e2fd03846511223344", 10}, + {"c4e2fd04846511223344", 10}, + {"c4e2fd05846511223344", 10}, + {"c4e2fd06846511223344", 10}, + {"c4e2fd07846511223344", 10}, + {"c4e2fd08846511223344", 10}, + {"c4e2fd09846511223344", 10}, + {"c4e2fd0a846511223344", 10}, + {"c4e2fd0b846511223344", 10}, + {"c4e2fd17846511223344", 10}, + {"c4e2fd1c846511223344", 10}, + {"c4e2fd1d846511223344", 10}, + {"c4e2fd1e846511223344", 10}, + {"c4e2fd20846511223344", 10}, + {"c4e2fd21846511223344", 10}, + {"c4e2fd22846511223344", 10}, + {"c4e2fd23846511223344", 10}, + {"c4e2fd24846511223344", 10}, + {"c4e2fd25846511223344", 10}, + {"c4e2fd28846511223344", 10}, + {"c4e2fd29846511223344", 10}, + {"c4e2fd2a846511223344", 10}, + {"c4e2fd2b846511223344", 10}, + {"c4e2fd30846511223344", 10}, + {"c4e2fd31846511223344", 10}, + {"c4e2fd32846511223344", 10}, + {"c4e2fd33846511223344", 10}, + {"c4e2fd34846511223344", 10}, + {"c4e2fd35846511223344", 10}, + {"c4e2fd37846511223344", 10}, + {"c4e2fd38846511223344", 10}, + {"c4e2fd39846511223344", 10}, + {"c4e2fd3a846511223344", 10}, + {"c4e2fd3b846511223344", 10}, + {"c4e2fd3c846511223344", 10}, + {"c4e2fd3d846511223344", 10}, + {"c4e2fd3e846511223344", 10}, + {"c4e2fd3f846511223344", 10}, + {"c4e2fd40846511223344", 10}, + {"c4e2fd45846511223344", 10}, + {"c4e2fd47846511223344", 10}, + {"c4e2fd8c846511223344", 10}, + {"c4e2fd8e846511223344", 10}, + {"c4e2fd90846511223344", 10}, + {"c4e2fd91846511223344", 10}, + {"c4e2fd92846511223344", 10}, + {"c4e2fd93846511223344", 10}, + {"c4e2fd96846511223344", 10}, + {"c4e2fd97846511223344", 10}, + {"c4e2fd98846511223344", 10}, + {"c4e2fd99846511223344", 10}, + {"c4e2fd9a846511223344", 10}, + {"c4e2fd9b846511223344", 10}, + {"c4e2fd9c846511223344", 10}, + {"c4e2fd9d846511223344", 10}, + {"c4e2fd9e846511223344", 10}, + {"c4e2fd9f846511223344", 10}, + {"c4e2fda6846511223344", 10}, + {"c4e2fda7846511223344", 10}, + {"c4e2fda8846511223344", 10}, + {"c4e2fda9846511223344", 10}, + {"c4e2fdaa846511223344", 10}, + {"c4e2fdab846511223344", 10}, + {"c4e2fdac846511223344", 10}, + {"c4e2fdad846511223344", 10}, + {"c4e2fdae846511223344", 10}, + {"c4e2fdaf846511223344", 10}, + {"c4e2fdb4846511223344", 10}, + {"c4e2fdb5846511223344", 10}, + {"c4e2fdb6846511223344", 10}, + {"c4e2fdb7846511223344", 10}, + {"c4e2fdb8846511223344", 10}, + {"c4e2fdb9846511223344", 10}, + {"c4e2fdba846511223344", 10}, + {"c4e2fdbb846511223344", 10}, + {"c4e2fdbc846511223344", 10}, + {"c4e2fdbd846511223344", 10}, + {"c4e2fdbe846511223344", 10}, + {"c4e2fdbf846511223344", 10}, + {"c4e2fddc846511223344", 10}, + {"c4e2fddd846511223344", 10}, + {"c4e2fdde846511223344", 10}, + {"c4e2fddf846511223344", 10}, + {"c4e379de84651122334455", 11}, + {"c4e37d0284651122334455", 11}, + {"c4e37d0484651122334455", 11}, + {"c4e37d0584651122334455", 11}, + {"c4e37d0684651122334455", 11}, + {"c4e37d1884651122334455", 11}, + {"c4e37d1984651122334455", 11}, + {"c4e37d1d84651122334455", 11}, + {"c4e37d3884651122334455", 11}, + {"c4e37d3984651122334455", 11}, + {"c4e37d4684651122334455", 11}, + {"c4e37d4a84651122334455", 11}, + {"c4e37d4b84651122334455", 11}, + {"c4e37d4c84651122334455", 11}, + {"c4e3f91484651122334455", 11}, + {"c4e3f91584651122334455", 11}, + {"c4e3f91684651122334455", 11}, + {"c4e3f91784651122334455", 11}, + {"c4e3f92084651122334455", 11}, + {"c4e3f92184651122334455", 11}, + {"c4e3f92284651122334455", 11}, + {"c4e3f930c111", 6}, + {"c4e3f931c111", 6}, + {"c4e3f932c111", 6}, + {"c4e3f933c111", 6}, + {"c4e3f94184651122334455", 11}, + {"c4e3f96084651122334455", 11}, + {"c4e3f96184651122334455", 11}, + {"c4e3f96284651122334455", 11}, + {"c4e3f96384651122334455", 11}, + {"c4e3f9df84651122334455", 11}, + {"c4e3fbf084651122334455", 11}, + {"c4e3fd0084651122334455", 11}, + {"c4e3fd0184651122334455", 11}, + {"c4e3fd0884651122334455", 11}, + {"c4e3fd0984651122334455", 11}, + {"c4e3fd0a84651122334455", 11}, + {"c4e3fd0b84651122334455", 11}, + {"c4e3fd0c84651122334455", 11}, + {"c4e3fd0d84651122334455", 11}, + {"c4e3fd0e84651122334455", 11}, + {"c4e3fd0f84651122334455", 11}, + {"c4e3fd4084651122334455", 11}, + {"c4e3fd4284651122334455", 11}, + {"c4e3fd4484651122334455", 11}, + {"c4e3fd4884651122334455", 11}, + {"c4e3fd4984651122334455", 11}, + {"c4e3fd5c84651122334455", 11}, + {"c4e3fd5d84651122334455", 11}, + {"c4e3fd5e84651122334455", 11}, + {"c4e3fd5f84651122334455", 11}, + {"c4e3fd6884651122334455", 11}, + {"c4e3fd6984651122334455", 11}, + {"c4e3fd6a84651122334455", 11}, + {"c4e3fd6b84651122334455", 11}, + {"c4e3fd6c84651122334455", 11}, + {"c4e3fd6d84651122334455", 11}, + {"c4e3fd6e84651122334455", 11}, + {"c4e3fd6f84651122334455", 11}, + {"c4e3fd7884651122334455", 11}, + {"c4e3fd7984651122334455", 11}, + {"c4e3fd7a84651122334455", 11}, + {"c4e3fd7b84651122334455", 11}, + {"c4e3fd7c84651122334455", 11}, + {"c4e3fd7d84651122334455", 11}, + {"c4e3fd7e84651122334455", 11}, + {"c4e3fd7f84651122334455", 11}, + {"c4e3fdce84651122334455", 11}, + {"c4e3fdcf84651122334455", 11}, + {"c684651122334455", 8}, + {"c784651122334455667708", 11}, + {"c8846511", 4}, + {"c9", 1}, + {"ca8465", 3}, + {"cb", 1}, + {"cc", 1}, + {"cd84", 2}, + {"cf", 1}, + {"d0846511223344", 7}, + {"d1846511223344", 7}, + {"d2846511223344", 7}, + {"d3846511223344", 7}, + {"d7", 1}, + {"d8846511223344", 7}, + {"d9846511223344", 7}, + {"da846511223344", 7}, + {"db846511223344", 7}, + {"dc846511223344", 7}, + {"dd846511223344", 7}, + {"de846511223344", 7}, + {"df846511223344", 7}, + {"e084", 2}, + {"e184", 2}, + {"e284", 2}, + {"e384", 2}, + {"e484", 2}, + {"e584", 2}, + {"e684", 2}, + {"e784", 2}, + {"e884651122", 5}, + {"e984651122", 5}, + {"eb84", 2}, + {"ec", 1}, + {"ed", 1}, + {"ee", 1}, + {"ef", 1}, + {"f20fd6c1", 4}, + {"f20ff0846511223344", 9}, + {"f30f38d8846511223344", 10}, + {"f30f38fac1", 5}, + {"f30f38fbc1", 5}, + {"f30fb8846511223344", 9}, + {"f3c784651122334455667708", 12}, + {"f4", 1}, + {"f5", 1}, + {"f684651122334455", 8}, + {"f784651122334455667708", 11}, + {"f8", 1}, + {"f9", 1}, + {"fa", 1}, + {"fb", 1}, + {"fc", 1}, + {"fd", 1}, + {"fe846511223344", 7}, + {"ff846511223344", 7}, +}; + +#endif diff --git a/core/relinker/codegen/tests/DecoderLengthTests.cpp b/core/relinker/codegen/tests/DecoderLengthTests.cpp new file mode 100644 index 000000000..0c24a309d --- /dev/null +++ b/core/relinker/codegen/tests/DecoderLengthTests.cpp @@ -0,0 +1,64 @@ +#include "DecoderLengthCases.hpp" +#include +#include +#include +#include +#include +#include +#include +#include + +namespace { + +using Bytes = std::vector; + +void require(bool condition, const std::string& message) { + if (!condition) throw std::runtime_error(message); +} + +Bytes parse(std::string_view hex) { + Bytes bytes; + for (std::size_t index = 0; index + 1 < hex.size(); index += 2) { + bytes.push_back(static_cast(std::stoul(std::string(hex.substr(index, 2)), nullptr, 16))); + } + return bytes; +} + +std::size_t decodedLength(const Codegen::X64InstructionDecoder& decoder, const Bytes& bytes, const std::string& name) { + try { + return decoder.Decode(bytes.data(), bytes.size()); + } catch (const Codegen::CodegenException& error) { + throw std::runtime_error("Cannot decode " + name + ": " + error.what()); + } +} + +bool truncationRejected(const Codegen::X64InstructionDecoder& decoder, const Bytes& bytes, std::size_t available) { + try { + (void)decoder.Decode(bytes.data(), available); + } catch (const Codegen::CodegenException&) { + return true; + } + return false; +} + +} + +int main() { + try { + const Codegen::X64InstructionDecoder decoder; + for (const auto& testCase : kDecoderLengthCases) { + const std::string name = testCase.Bytes; + auto bytes = parse(testCase.Bytes); + require(bytes.size() == testCase.Length, "Decoder length case " + name + " does not hold its own length"); + require(decodedLength(decoder, bytes, name) == testCase.Length, "Wrong length for " + name); + require(truncationRejected(decoder, bytes, bytes.size() - 1), "A truncated " + name + " was accepted"); + bytes.insert(bytes.end(), 15, 0x90); + require(decodedLength(decoder, bytes, name + " followed by NOPs") == testCase.Length, "Wrong length for " + name + " followed by NOPs"); + } + std::cout << "Decoder length tests passed\n"; + } catch (const std::exception& error) { + std::cerr << error.what() << '\n'; + return 1; + } + return 0; +} diff --git a/core/relinker/domain/include/domain/ImportModule.hpp b/core/relinker/domain/include/domain/ImportModule.hpp index 81b0c3b2b..a2bd98e33 100644 --- a/core/relinker/domain/include/domain/ImportModule.hpp +++ b/core/relinker/domain/include/domain/ImportModule.hpp @@ -10,6 +10,8 @@ namespace Domain { inline std::string ImportModuleName(std::string name) { if (name.ends_with(".prx")) name.resize(name.size() - 4); + else if (name.ends_with(".sprx")) name.resize(name.size() - 5); + else if (name.ends_with(".suprx")) name.resize(name.size() - 6); if (name.ends_with("-module")) name.resize(name.size() - 7); std::replace(name.begin(), name.end(), '.', '_'); return name; @@ -36,7 +38,7 @@ inline std::string ImportModule(const std::string& symbol, const std::mapsecond; if (name.empty() || name.find_first_of("/\\:$\r\n") != std::string::npos) throw RelinkerException("Invalid import module name: " + name); - if (!name.ends_with(".prx")) name += ".prx"; + if (!name.ends_with(".prx") && !name.ends_with(".sprx") && !name.ends_with(".suprx")) name += ".prx"; if (std::find(dependencies.begin(), dependencies.end(), name) != dependencies.end()) return name; std::string matched; for (const auto& dependency : dependencies) { diff --git a/core/relinker/elfpatcher/include/elfpatcher/general/ElfConstants.hpp b/core/relinker/elfpatcher/include/elfpatcher/general/ElfConstants.hpp index 51857f69b..60da5e79f 100644 --- a/core/relinker/elfpatcher/include/elfpatcher/general/ElfConstants.hpp +++ b/core/relinker/elfpatcher/include/elfpatcher/general/ElfConstants.hpp @@ -55,7 +55,7 @@ inline constexpr std::uint64_t kDefaultLoadAlignment = 0x1000; inline constexpr std::uint64_t kPhdrHeaderAlignment = 8; inline constexpr std::uint64_t kDynamicHeaderAlignment = 8; inline constexpr std::uint64_t kInterpHeaderAlignment = 1; -inline constexpr std::uint16_t kSyntheticProgramHeaderCount = 4; +inline constexpr std::uint16_t kSyntheticProgramHeaderCount = 5; inline constexpr std::uint32_t PT_LOAD = 1; inline constexpr std::uint32_t PT_DYNAMIC = 2; @@ -63,6 +63,7 @@ inline constexpr std::uint32_t PT_INTERP = 3; inline constexpr std::uint32_t PT_NOTE = 4; inline constexpr std::uint32_t PT_PHDR = 6; inline constexpr std::uint32_t PT_GNU_EH_FRAME = 0x6474e550; +inline constexpr std::uint32_t PT_GNU_STACK = 0x6474e551; inline constexpr std::uint32_t PT_GNU_RELRO = 0x6474e552; inline constexpr std::uint32_t PF_X = 0x1; inline constexpr std::uint32_t PF_W = 0x2; @@ -96,6 +97,8 @@ inline constexpr std::int64_t DT_INIT_ARRAYSZ = 27; inline constexpr std::int64_t DT_FINI_ARRAYSZ = 28; inline constexpr std::int64_t DT_RUNPATH = 29; inline constexpr std::int64_t DT_FLAGS = 30; +inline constexpr std::int64_t DT_PREINIT_ARRAY = 32; +inline constexpr std::int64_t DT_PREINIT_ARRAYSZ = 33; inline constexpr std::uint64_t DF_BIND_NOW = 0x8; inline constexpr std::int64_t DT_OS_INIT = 0x6000000c; @@ -104,6 +107,8 @@ inline constexpr std::int64_t DT_OS_INIT_ARRAY = 0x60000019; inline constexpr std::int64_t DT_OS_FINI_ARRAY = 0x6000001a; inline constexpr std::int64_t DT_OS_INIT_ARRAYSZ = 0x6000001b; inline constexpr std::int64_t DT_OS_FINI_ARRAYSZ = 0x6000001c; +inline constexpr std::int64_t DT_OS_PREINIT_ARRAY = 0x60000020; +inline constexpr std::int64_t DT_OS_PREINIT_ARRAYSZ = 0x60000021; inline constexpr std::int64_t DT_OS_PLTGOT = 0x61000027; inline constexpr std::int64_t DT_OS_JMPREL = 0x61000029; inline constexpr std::int64_t DT_OS_PLTREL = 0x6100002b; @@ -166,6 +171,8 @@ inline constexpr std::uint32_t kDynamicLinkToStrTab = 2; inline constexpr std::uint8_t kStubOpAndRsp0xf0[] = {0x48, 0x83, 0xe4, 0xf0}; inline constexpr std::uint8_t kStubOpMovRdiRsp[] = {0x48, 0x89, 0xe7}; inline constexpr std::uint8_t kStubOpXorRsiRsi[] = {0x48, 0x31, 0xf6}; +inline constexpr std::uint8_t kStubOpPushZero[] = {0x6a, 0x00}; +inline constexpr std::uint8_t kStubOpMovRbpRsp[] = {0x48, 0x89, 0xe5}; inline constexpr std::uint8_t kStubOpCallRel32 = 0xe8; inline constexpr std::size_t kStubCallInstructionSize = 5; inline constexpr std::uint8_t kStubOpUd2[] = {0x0f, 0x0b}; diff --git a/core/relinker/elfpatcher/include/elfpatcher/general/GuestModuleWriter.hpp b/core/relinker/elfpatcher/include/elfpatcher/general/GuestModuleWriter.hpp index c820c3879..a59418bee 100644 --- a/core/relinker/elfpatcher/include/elfpatcher/general/GuestModuleWriter.hpp +++ b/core/relinker/elfpatcher/include/elfpatcher/general/GuestModuleWriter.hpp @@ -9,6 +9,7 @@ class GuestModuleWriter { public: std::vector WriteLinux(const Relinker::GuestImage& image, const std::vector& dependencies, const std::string& runPath) const; std::vector WriteWindows(const Relinker::GuestImage& image, Domain::GuestRuntime& runtime) const; + std::vector WriteMacOs(const Relinker::GuestImage& image, const std::vector& dependencies, const std::string& runPath) const; }; } diff --git a/core/relinker/elfpatcher/include/elfpatcher/general/ProgramHeaderLayoutBuilder.hpp b/core/relinker/elfpatcher/include/elfpatcher/general/ProgramHeaderLayoutBuilder.hpp index fcff63757..6c86cf4cf 100644 --- a/core/relinker/elfpatcher/include/elfpatcher/general/ProgramHeaderLayoutBuilder.hpp +++ b/core/relinker/elfpatcher/include/elfpatcher/general/ProgramHeaderLayoutBuilder.hpp @@ -32,6 +32,7 @@ namespace Elfpatcher { [[nodiscard]] Domain::ProgramHeader _makeDynamicHeader(std::uint64_t offset, std::uint64_t vaddr, std::uint64_t size) const; [[nodiscard]] Domain::ProgramHeader _makeInterpHeader(std::uint64_t offset, std::uint64_t vaddr, std::uint64_t size) const; [[nodiscard]] std::uint32_t _fixLoadFlags(std::uint32_t originalFlags) const; + void _sortLoadHeaders(std::vector& headers) const; void _writeProgramHeader(std::vector& buf, std::size_t offset, const Domain::ProgramHeader& ph) const; }; diff --git a/core/relinker/elfpatcher/include/elfpatcher/macos/MacOsImage.hpp b/core/relinker/elfpatcher/include/elfpatcher/macos/MacOsImage.hpp new file mode 100644 index 000000000..ed2ad7e75 --- /dev/null +++ b/core/relinker/elfpatcher/include/elfpatcher/macos/MacOsImage.hpp @@ -0,0 +1,70 @@ +#ifndef ELFPATCHER_MACOS_IMAGE_HPP +#define ELFPATCHER_MACOS_IMAGE_HPP + +#include +#include +#include +#include +#include +#include + +namespace Elfpatcher::MacOs { + +struct MacOsBind { + std::string Symbol; + std::uint32_t Rva; + std::int64_t Addend = 0; +}; + +struct MacOsExport { + std::string Symbol; + std::uint32_t Rva; +}; + +struct MacOsTlsImport { + std::string Symbol; + std::uint32_t Rva; + bool Module; + std::int64_t Addend = 0; +}; + +struct MacOsTlsExport { + std::string Symbol; + std::uint64_t Offset; +}; + +struct MacOsImageInput { + bool Executable = true; + std::string InstallName; + const std::vector* Source = nullptr; + const std::vector* Headers = nullptr; + Windows::WindowsLoadImage* Image = nullptr; + std::vector Rebases; + std::vector Binds; + std::vector Exports; + std::vector Dylibs; + std::vector RunPaths; + std::uint32_t EntryRva = 0; + std::uint32_t InitRva = 0; + std::uint32_t FiniRva = 0; + std::vector InitArrayRvas; + std::vector FiniArrayRvas; + std::vector TlsModuleSlots; + std::vector TlsImports; + std::vector TlsExports; + const std::vector* Trampolines = nullptr; +}; + +std::uint64_t GuestTlsBlockSize(const Domain::ProgramHeader& tls); + +std::vector WriteMacOsImage(MacOsImageInput& input); + +std::string MachOLoadPath(const std::string& path, const std::string& origin); + +std::string MachODependency(const std::string& name, const std::string& origin); + +std::vector ReadNeededLibraries(const Domain::SysVDynamicSection& dynamicSection); + +} + +#endif diff --git a/core/relinker/elfpatcher/include/elfpatcher/macos/MacOsMachOPatcher.hpp b/core/relinker/elfpatcher/include/elfpatcher/macos/MacOsMachOPatcher.hpp new file mode 100644 index 000000000..2fe447571 --- /dev/null +++ b/core/relinker/elfpatcher/include/elfpatcher/macos/MacOsMachOPatcher.hpp @@ -0,0 +1,24 @@ +#ifndef ELFPATCHER_MACOS_MACHOPATCHER_HPP +#define ELFPATCHER_MACOS_MACHOPATCHER_HPP + +#include + +namespace Elfpatcher::MacOs { + +class MacOsMachOPatcher final : public IElfPatcher { +public: + std::vector Patch( + const std::vector& sourceElf, + const std::vector& originalHeaders, + const Domain::SysVDynamicSection& dynamicSection, + std::uint64_t originalPltGotVaddr, + const std::string& runPath, + bool lazyBinding, + bool dependencyDiagnostics, + const std::vector& trampolines + ) override; +}; + +} + +#endif diff --git a/core/relinker/elfpatcher/src/general/EntryStubBuilder.cpp b/core/relinker/elfpatcher/src/general/EntryStubBuilder.cpp index 21bc763d8..72168e18b 100644 --- a/core/relinker/elfpatcher/src/general/EntryStubBuilder.cpp +++ b/core/relinker/elfpatcher/src/general/EntryStubBuilder.cpp @@ -1,5 +1,7 @@ #include #include +#include +#include namespace Elfpatcher { @@ -17,10 +19,16 @@ std::vector EntryStubBuilder::BuildEntryStub( std::vector s; _appendBytes(s, kStubOpMovRdiRsp, sizeof(kStubOpMovRdiRsp)); _appendBytes(s, kStubOpAndRsp0xf0, sizeof(kStubOpAndRsp0xf0)); + _appendBytes(s, kStubOpPushZero, sizeof(kStubOpPushZero)); + _appendBytes(s, kStubOpPushZero, sizeof(kStubOpPushZero)); + _appendBytes(s, kStubOpMovRbpRsp, sizeof(kStubOpMovRbpRsp)); _appendBytes(s, kStubOpXorRsiRsi, sizeof(kStubOpXorRsiRsi)); const std::uint64_t callInsnVaddr = stubVaddr + s.size(); const std::uint64_t callNextVaddr = callInsnVaddr + kStubCallInstructionSize; - const auto rel32 = static_cast(realEntryVaddr - callNextVaddr); + const auto displacement = static_cast(realEntryVaddr - callNextVaddr); + if (displacement < std::numeric_limits::min() || displacement > std::numeric_limits::max()) + throw Domain::RelinkerException("Entry stub call exceeds rel32 range"); + const auto rel32 = static_cast(displacement); s.push_back(kStubOpCallRel32); s.push_back(static_cast(rel32 & 0xff)); s.push_back(static_cast((rel32 >> 8) & 0xff)); diff --git a/core/relinker/elfpatcher/src/general/ProgramHeaderLayoutBuilder.cpp b/core/relinker/elfpatcher/src/general/ProgramHeaderLayoutBuilder.cpp index d844e8176..d70c3385d 100644 --- a/core/relinker/elfpatcher/src/general/ProgramHeaderLayoutBuilder.cpp +++ b/core/relinker/elfpatcher/src/general/ProgramHeaderLayoutBuilder.cpp @@ -1,9 +1,11 @@ #include #include #include +#include #include #include #include +#include namespace Elfpatcher { @@ -131,23 +133,24 @@ std::uint16_t ProgramHeaderLayoutBuilder::WriteLayout( std::vector& buf, const ProgramHeaderLayoutRequest& request ) const { - std::uint16_t keptCount = 0; - std::uint16_t frameCount = 0; + std::vector kept; + std::size_t frameCount = 0; for (const auto& ph : request.OriginalHeaders) { if (_segmentFilter->ShouldSkip(ph)) continue; - keptCount++; + kept.push_back(ph); if (ph.Type == PT_GNU_EH_FRAME) frameCount++; } - const bool keepFrames = keptCount + kSyntheticProgramHeaderCount < request.PhNum; + const bool keepFrames = kept.size() + kSyntheticProgramHeaderCount <= request.PhNum; if (!keepFrames && frameCount > 0) { - keptCount -= frameCount; + std::erase_if(kept, [](const Domain::ProgramHeader& ph) { return ph.Type == PT_GNU_EH_FRAME; }); std::cerr << "WARNING: No free program header slot for PT_GNU_EH_FRAME; C++ exceptions thrown in the executable cannot be caught.\n"; } + _sortLoadHeaders(kept); - const std::uint16_t neededPh = keptCount + kSyntheticProgramHeaderCount; + const std::size_t neededPh = kept.size() + kSyntheticProgramHeaderCount; if (neededPh > request.PhNum) throw Domain::RelinkerException( "Not enough program header slots: need " + std::to_string(neededPh) + @@ -174,49 +177,38 @@ std::uint16_t ProgramHeaderLayoutBuilder::WriteLayout( const std::uint64_t dynamicSegmentVaddr = request.ExtraBlockVaddr + (request.DynamicSegmentOffset - request.ExtraBlockOffset); const std::uint64_t interpVaddr = request.ExtraBlockVaddr + (request.InterpOffset - request.ExtraBlockOffset); - std::uint16_t writtenPh = 0; - - const std::size_t phdrEntOff = static_cast(request.PhOff) + writtenPh * request.PhEntSize; - _writeProgramHeader(buf, phdrEntOff, _makePhdrHeader(request.PhOff, headerBlockVaddr + request.PhOff, static_cast(neededPh) * request.PhEntSize)); - writtenPh++; - - const std::size_t headerLoadEntOff = static_cast(request.PhOff) + writtenPh * request.PhEntSize; - _writeProgramHeader(buf, headerLoadEntOff, _makeHeaderBlockLoad(headerBlockVaddr, headerBlockSize, headerBlockAlign)); - writtenPh++; - - for (const auto& ph : request.OriginalHeaders) { - if (_segmentFilter->ShouldSkip(ph) || (!keepFrames && ph.Type == PT_GNU_EH_FRAME)) - continue; - const std::size_t phEntOff = static_cast(request.PhOff) + writtenPh * request.PhEntSize; - if (ph.Type == PT_LOAD) { - Domain::ProgramHeader fixed = ph; - fixed.Flags = _fixLoadFlags(ph.Flags); - _writeProgramHeader(buf, phEntOff, fixed); - } else { - _writeProgramHeader(buf, phEntOff, ph); - } - writtenPh++; + std::vector layout; + layout.push_back(_makePhdrHeader(request.PhOff, headerBlockVaddr + request.PhOff, static_cast(neededPh) * request.PhEntSize)); + layout.push_back(_makeInterpHeader(request.InterpOffset, interpVaddr, request.InterpSize)); + for (auto ph : kept) { + if (ph.Type == PT_LOAD) + ph.Flags = _fixLoadFlags(ph.Flags); + layout.push_back(ph); } + layout.push_back(_makeLoadHeader(request.ExtraBlockOffset, request.ExtraBlockVaddr, request.ExtraBlockSize)); + layout.push_back(_makeHeaderBlockLoad(headerBlockVaddr, headerBlockSize, headerBlockAlign)); + layout.push_back(_makeDynamicHeader(request.DynamicSegmentOffset, dynamicSegmentVaddr, request.DynamicSegmentSize)); - { - const std::size_t phEntOff = static_cast(request.PhOff) + writtenPh * request.PhEntSize; - _writeProgramHeader(buf, phEntOff, _makeLoadHeader(request.ExtraBlockOffset, request.ExtraBlockVaddr, request.ExtraBlockSize)); - writtenPh++; + for (std::size_t index = 0; index < layout.size(); ++index) + _writeProgramHeader(buf, static_cast(request.PhOff) + index * request.PhEntSize, layout[index]); + + return static_cast(layout.size()); +} + +void ProgramHeaderLayoutBuilder::_sortLoadHeaders(std::vector& headers) const { + std::vector loads; + for (const auto& ph : headers) { + if (ph.Type == PT_LOAD) + loads.push_back(ph); } - - { - const std::size_t phEntOff = static_cast(request.PhOff) + writtenPh * request.PhEntSize; - _writeProgramHeader(buf, phEntOff, _makeDynamicHeader(request.DynamicSegmentOffset, dynamicSegmentVaddr, request.DynamicSegmentSize)); - writtenPh++; + std::stable_sort(loads.begin(), loads.end(), [](const Domain::ProgramHeader& left, const Domain::ProgramHeader& right) { + return left.MappedAddress < right.MappedAddress; + }); + std::size_t next = 0; + for (auto& ph : headers) { + if (ph.Type == PT_LOAD) + ph = loads[next++]; } - - { - const std::size_t phEntOff = static_cast(request.PhOff) + writtenPh * request.PhEntSize; - _writeProgramHeader(buf, phEntOff, _makeInterpHeader(request.InterpOffset, interpVaddr, request.InterpSize)); - writtenPh++; - } - - return writtenPh; } } diff --git a/core/relinker/elfpatcher/src/linux/LinuxGuestModuleWriter.cpp b/core/relinker/elfpatcher/src/linux/LinuxGuestModuleWriter.cpp index f7744ddfa..76598e35a 100644 --- a/core/relinker/elfpatcher/src/linux/LinuxGuestModuleWriter.cpp +++ b/core/relinker/elfpatcher/src/linux/LinuxGuestModuleWriter.cpp @@ -1,4 +1,5 @@ #include +#include #include #include #include @@ -13,7 +14,7 @@ std::vector GuestModuleWriter::WriteLinux(const Relinker::GuestIma std::vector headers; std::uint64_t end = 0; for (auto header : image.Headers) { - if (header.Type != 1 && header.Type != 7 && header.Type != 0x6474e550 && header.Type != 0x6474e551) continue; + if (header.Type != 1 && header.Type != 7 && header.Type != 0x6474e550 && header.Type != PT_GNU_STACK) continue; if (header.Type == 1) { end = std::max(end, header.MappedAddress + header.MemorySize); if (header.Flags == 0) continue; @@ -21,13 +22,26 @@ std::vector GuestModuleWriter::WriteLinux(const Relinker::GuestIma } headers.push_back(header); } + if (std::none_of(headers.begin(), headers.end(), [&](const auto& header) { return header.Type == PT_GNU_STACK; })) + headers.push_back({PT_GNU_STACK, PF_R | PF_W, 0, 0, 0, 0, 0, 16}); if (end > std::numeric_limits::max() - 0x4000) throw Domain::RelinkerException("Guest virtual address overflow"); Io::AlignBuffer(bytes, 0x4000); const auto extraOffset = bytes.size(); const auto extraAddress = Io::AlignUp64(end, 0x4000); const auto address = [&] { return extraAddress + bytes.size() - extraOffset; }; + const bool passesModuleArgs = image.Init != 0; + std::uint64_t argsSlot = 0; + std::uint64_t resultSlot = 0; + if (passesModuleArgs) { + Io::AlignBuffer(bytes, 8); + argsSlot = address(); + bytes.insert(bytes.end(), 8, 0); + resultSlot = address(); + bytes.insert(bytes.end(), 8, 0); + } auto strings = image.Dynamic.DynStrData; auto symbols = image.Dynamic.DynSymData; + constexpr std::uint64_t kGlobDat = 6; const auto addString = [&](const std::string& value) { if (strings.size() > std::numeric_limits::max()) throw Domain::RelinkerException("Guest string table too large"); const auto offset = static_cast(strings.size()); @@ -46,6 +60,30 @@ std::vector GuestModuleWriter::WriteLinux(const Relinker::GuestIma std::vector needed; for (const auto& dependency : dependencies) needed.push_back(addString(dependency)); if (needsTlsResolver) needed.push_back(addString("ld-linux-x86-64.so.2")); + const auto addUndefinedImport = [&](const std::string& name) { + const auto nameOffset = addString(name); + const auto entry = symbols.size(); + symbols.resize(entry + 24, 0); + Io::WriteU32(symbols, entry, nameOffset); + symbols[entry + 4] = 0x12; + Io::WriteU16(symbols, entry + 6, 0); + return entry / 24; + }; + auto relaData = image.Dynamic.RelaData; + std::uint64_t argsImport = 0; + std::uint64_t resultImport = 0; + const auto addGlobDat = [&](std::uint64_t slot, std::uint64_t symbol) { + const auto entry = relaData.size(); + relaData.resize(entry + 24, 0); + Io::WriteU64(relaData, entry, slot); + Io::WriteU64(relaData, entry + 8, (symbol << 32) | kGlobDat); + }; + if (passesModuleArgs) { + argsImport = addUndefinedImport("__aps5_get_pending_module_args_nid_no_patch"); + resultImport = addUndefinedImport("__aps5_set_module_init_result_nid_no_patch"); + addGlobDat(argsSlot, argsImport); + addGlobDat(resultSlot, resultImport); + } const auto soname = addString(image.OutputName); const auto search = addString(runPath); const auto strAddress = address(); @@ -54,27 +92,67 @@ std::vector GuestModuleWriter::WriteLinux(const Relinker::GuestIma const auto symAddress = address(); bytes.insert(bytes.end(), symbols.begin(), symbols.end()); const auto hashAddress = address(); - const auto count = static_cast(image.Symbols.size()); - Io::AppendU32(bytes, 1); + const auto count = static_cast(symbols.size() / 24); + const auto bucketCount = std::max(count, 1); + std::vector buckets(bucketCount), chains(count); + for (std::uint32_t index = count; index-- > 1;) { + std::uint32_t hash = 0; + for (auto offset = Io::ReadU32(symbols, index * 24); offset < strings.size() && strings[offset] != 0; ++offset) { + hash = (hash << 4u) + strings[offset]; + hash = (hash ^ ((hash & 0xf0000000u) >> 24u)) & 0x0fffffffu; + } + chains[index] = buckets[hash % bucketCount]; + buckets[hash % bucketCount] = index; + } + Io::AppendU32(bytes, bucketCount); Io::AppendU32(bytes, count); - Io::AppendU32(bytes, count > 1 ? 1 : 0); - for (std::uint32_t index = 0; index < count; ++index) Io::AppendU32(bytes, index != 0 && index + 1 < count ? index + 1 : 0); + for (const auto bucket : buckets) Io::AppendU32(bytes, bucket); + for (const auto chain : chains) Io::AppendU32(bytes, chain); Io::AlignBuffer(bytes, 8); const auto relaAddress = address(); - bytes.insert(bytes.end(), image.Dynamic.RelaData.begin(), image.Dynamic.RelaData.end()); + bytes.insert(bytes.end(), relaData.begin(), relaData.end()); const auto pltAddress = address(); bytes.insert(bytes.end(), image.Dynamic.RelaPltData.begin(), image.Dynamic.RelaPltData.end()); + constexpr std::size_t kArgsDisplacement = 3, kTargetDisplacement = 0x11, kResultDisplacement = 0x19; + constexpr std::size_t kStubSize = 0x1f; + const auto stubDisplacement = [](std::uint64_t target, std::uint64_t next) { + const auto distance = static_cast(target) - static_cast(next); + if (distance < std::numeric_limits::min() || distance > std::numeric_limits::max()) + throw Domain::RelinkerException("Guest initializer exceeds relative branch range"); + return static_cast(static_cast(distance)); + }; const auto lifecycle = [&](std::uint64_t target) { if (target == 0) return std::uint64_t{}; const auto start = address(); - bytes.insert(bytes.end(), {0x31, 0xff, 0x31, 0xf6, 0x31, 0xd2, 0xe9}); - const auto displacement = static_cast(target) - static_cast(address() + 4); - if (displacement < std::numeric_limits::min() || displacement > std::numeric_limits::max()) throw Domain::RelinkerException("Guest initializer exceeds relative branch range"); - Io::AppendU32(bytes, static_cast(displacement)); + std::vector stub(kStubSize, 0xc3); + stub[0x00] = 0x50; + stub[0x01] = 0xff; stub[0x02] = 0x15; + Io::WriteU32(stub, kArgsDisplacement, stubDisplacement(argsSlot, start + 7)); + stub[0x07] = 0x48; stub[0x08] = 0x8b; stub[0x09] = 0x38; + stub[0x0a] = 0x48; stub[0x0b] = 0x8b; stub[0x0c] = 0x70; stub[0x0d] = 0x08; + stub[0x0e] = 0x31; stub[0x0f] = 0xd2; + stub[0x10] = 0xe8; + Io::WriteU32(stub, kTargetDisplacement, stubDisplacement(target, start + 0x15)); + stub[0x15] = 0x89; stub[0x16] = 0xc7; + stub[0x17] = 0xff; stub[0x18] = 0x15; + Io::WriteU32(stub, kResultDisplacement, stubDisplacement(resultSlot, start + 0x1d)); + stub[0x1d] = 0x58; + stub[0x1e] = 0xc3; + bytes.insert(bytes.end(), stub.begin(), stub.end()); + return start; + }; + const auto callThrough = [&](std::uint64_t target) { + if (target == 0) return std::uint64_t{}; + const auto start = address(); + const auto distance = static_cast(target) - static_cast(start + 5); + if (distance < std::numeric_limits::min() || distance > std::numeric_limits::max()) + throw Domain::RelinkerException("Guest finaliser exceeds relative branch range"); + bytes.push_back(0xe9); + Io::AppendU32(bytes, static_cast(distance)); return start; }; const auto init = lifecycle(image.Init); - const auto fini = lifecycle(image.Fini); + const auto fini = callThrough(image.Fini); Io::AlignBuffer(bytes, 8); const auto dynamicOffset = bytes.size(); const auto dynamicAddress = address(); @@ -87,9 +165,9 @@ std::vector GuestModuleWriter::WriteLinux(const Relinker::GuestIma tag(10, strings.size()); tag(6, symAddress); tag(11, 24); - if (!image.Dynamic.RelaData.empty()) { + if (!relaData.empty()) { tag(7, relaAddress); - tag(8, image.Dynamic.RelaData.size()); + tag(8, relaData.size()); tag(9, 24); } if (!image.Dynamic.RelaPltData.empty()) { diff --git a/core/relinker/elfpatcher/src/macos/MacOsGuestModuleWriter.cpp b/core/relinker/elfpatcher/src/macos/MacOsGuestModuleWriter.cpp new file mode 100644 index 000000000..6e1912dd9 --- /dev/null +++ b/core/relinker/elfpatcher/src/macos/MacOsGuestModuleWriter.cpp @@ -0,0 +1,121 @@ +#include +#include +#include +#include +#include +#include +#include + +namespace Elfpatcher { + +namespace { + +std::string DynamicName(const Relinker::GuestImage& guest, const std::size_t index) { + const auto& symbols = guest.Dynamic.DynSymData; + if ((index + 1) * 24 > symbols.size()) throw Domain::RelinkerException("Guest symbol index outside the dynamic symbol table"); + const auto offset = Io::ReadU32(symbols, index * 24); + if (offset >= guest.Dynamic.DynStrData.size()) throw Domain::RelinkerException("Guest symbol name offset is out of bounds", offset); + return Windows::ReadString(guest.Dynamic.DynStrData, offset); +} + +} + +std::vector GuestModuleWriter::WriteMacOs(const Relinker::GuestImage& guest, const std::vector& dependencies, const std::string& runPath) const { + using namespace MacOs; + Windows::WindowsLoadImage image(guest.Bytes, guest.Headers, false); + MacOsImageInput input; + input.Executable = false; + input.InstallName = "@rpath/" + guest.OutputName; + input.Source = &guest.Bytes; + input.Headers = &guest.Headers; + input.Image = ℑ + + const auto tls = std::find_if(guest.Headers.begin(), guest.Headers.end(), [](const auto& header) { return header.Type == 7; }); + std::map targets; + const auto apply = [&](const std::vector& table) { + for (std::size_t offset = 0; offset < table.size(); offset += 24) { + const auto target = Io::ReadU64(table, offset); + const auto info = Io::ReadU64(table, offset + 8); + const auto addend = Io::ReadU64(table, offset + 16); + const auto type = static_cast(info); + const auto symbolIndex = info >> 32; + const auto& symbol = guest.Symbols.at(symbolIndex); + const bool imported = symbolIndex != 0 && symbol.Section == 0; + const bool tlsSymbol = (symbol.Info & 15) == 6; + const auto rva = image.GetRva(target, 8); + for (const auto& header : guest.Headers) { + if (header.Type != 7 || header.FileSize == 0) continue; + const auto templateRva = image.GetRva(header.MappedAddress, header.FileSize); + const auto templateEnd = static_cast(templateRva) + header.FileSize; + if (rva >= templateEnd || static_cast(rva) + 8 <= templateRva) continue; + if (rva < templateRva || templateEnd - rva < 8) throw Domain::RelinkerException("Guest relocation crosses the TLS template boundary", target); + if (type == 16 || imported) throw Domain::RelinkerException("Runtime-bound guest TLS template relocation is not supported", target); + } + const auto next = targets.lower_bound(target); + if ((next != targets.end() && next->first < target + 8) || (next != targets.begin() && std::prev(next)->second > target)) throw Domain::RelinkerException("Overlapping guest relocations", target); + targets.emplace(target, target + 8); + image.RequireWritable(target, 8); + if (type == 8) { + if (symbolIndex != 0) throw Domain::RelinkerException("RELATIVE guest relocation has a symbol", target); + image.WritePointer(target, image.GetRelocatedAddress(addend)); + input.Rebases.push_back(rva); + } else if (type == 16 || type == 17) { + if ((type == 16 && addend != 0) || (symbolIndex != 0 && !tlsSymbol)) throw Domain::RelinkerException("Invalid guest dynamic TLS relocation", target); + if (imported) { + image.WritePointer(target, 0); + input.TlsImports.push_back({"_" + DynamicName(guest, symbolIndex), rva, type == 16, static_cast(addend)}); + } else if (type == 16) { + image.WritePointer(target, 0); + input.TlsModuleSlots.push_back(rva); + } else image.WritePointer(target, symbol.Value + addend); + } else if (type == 18) { + if (tls == guest.Headers.end() || (symbolIndex != 0 && (imported || !tlsSymbol))) throw Domain::RelinkerException("Unsupported external static TLS relocation", target); + image.WritePointer(target, symbol.Value + addend - GuestTlsBlockSize(*tls)); + } else if (type == 1 || type == 6 || type == 7) { + if (symbolIndex == 0 || (type != 1 && addend != 0) || tlsSymbol) throw Domain::RelinkerException("Invalid guest symbol relocation", target); + if (!imported) { + if (symbol.Section >= 0xff00) throw Domain::RelinkerException("Unsupported special guest symbol section", target); + image.WritePointer(target, image.GetRelocatedAddress(symbol.Value) + addend); + input.Rebases.push_back(rva); + } else { + if (symbol.Name.empty()) throw Domain::RelinkerException("Empty guest import", target); + image.WritePointer(target, 0); + const auto name = guest.UsePlatformTlsResolver && symbol.Name == "vNe1w4diLCs" ? std::string("Aps5GuestTlsGetAddr_nid_no_patch") : DynamicName(guest, symbolIndex); + input.Binds.push_back({"_" + name, rva, static_cast(addend)}); + } + } else throw Domain::RelinkerException("Unsupported macOS guest relocation " + std::to_string(type), target); + } + }; + apply(guest.Dynamic.RelaData); + apply(guest.Dynamic.RelaPltData); + + for (std::size_t index = 1; index < guest.Symbols.size(); ++index) { + const auto& symbol = guest.Symbols[index]; + if (symbol.Section == 0 || (symbol.Info >> 4) == 0 || symbol.Visibility == 1 || symbol.Visibility == 2) continue; + if (symbol.Section >= 0xff00) throw Domain::RelinkerException("Unsupported guest export section: " + symbol.Name); + const auto name = "_" + DynamicName(guest, index); + if ((symbol.Info & 15) == 6) { + input.TlsExports.push_back({name, symbol.Value}); + continue; + } + input.Exports.push_back({name, image.GetRva(symbol.Value, std::max(symbol.Size, 1))}); + } + + for (const auto& dependency : dependencies) input.Dylibs.push_back(MachODependency(dependency, "@loader_path")); + if (std::find(input.Dylibs.begin(), input.Dylibs.end(), "@rpath/libkernel.prx") == input.Dylibs.end()) + input.Dylibs.push_back("@rpath/libkernel.prx"); + input.RunPaths.push_back(MachOLoadPath(runPath, "@loader_path")); + for (const auto slot : guest.InitArray) input.InitArrayRvas.push_back(image.GetRva(slot, 8)); + for (const auto slot : guest.FiniArray) input.FiniArrayRvas.push_back(image.GetRva(slot, 8)); + input.InitRva = guest.Init == 0 ? 0 : image.GetRva(guest.Init); + input.FiniRva = guest.Fini == 0 ? 0 : image.GetRva(guest.Fini); + input.Trampolines = &guest.Trampolines; + try { + return WriteMacOsImage(input); + } catch (Domain::RelinkerException& error) { + error.InputPath = guest.SourcePath.string(); + throw; + } +} + +} diff --git a/core/relinker/elfpatcher/src/macos/MacOsImage.cpp b/core/relinker/elfpatcher/src/macos/MacOsImage.cpp new file mode 100644 index 000000000..098505bf8 --- /dev/null +++ b/core/relinker/elfpatcher/src/macos/MacOsImage.cpp @@ -0,0 +1,799 @@ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace Elfpatcher::MacOs { + +namespace { + +using Windows::ImageBase; +using Windows::LoadRva; + +constexpr std::uint64_t PageSize = 0x1000; + +constexpr std::uint32_t MhMagic64 = 0xfeedfacf; +constexpr std::uint32_t CpuTypeX8664 = 0x01000007; +constexpr std::uint32_t CpuSubtypeX8664All = 3; +constexpr std::uint32_t MhExecute = 2; +constexpr std::uint32_t MhDylib = 6; +constexpr std::uint32_t MhNoUndefs = 0x1; +constexpr std::uint32_t MhDyldLink = 0x4; +constexpr std::uint32_t MhTwoLevel = 0x80; + +constexpr std::uint32_t LcSegment64 = 0x19; +constexpr std::uint32_t LcSymtab = 0x2; +constexpr std::uint32_t LcDysymtab = 0xb; +constexpr std::uint32_t LcLoadDylib = 0xc; +constexpr std::uint32_t LcIdDylib = 0xd; +constexpr std::uint32_t LcLoadDylinker = 0xe; +constexpr std::uint32_t LcUuid = 0x1b; +constexpr std::uint32_t LcRpath = 0x8000001c; +constexpr std::uint32_t LcDyldInfoOnly = 0x80000022; +constexpr std::uint32_t LcMain = 0x80000028; +constexpr std::uint32_t LcBuildVersion = 0x32; + +constexpr std::uint32_t VmProtRead = 1; +constexpr std::uint32_t VmProtWrite = 2; +constexpr std::uint32_t VmProtExecute = 4; + +constexpr std::uint32_t SectionModInitFuncPointers = 0x9; + +constexpr std::uint8_t RebaseTypePointer = 1; +constexpr std::uint8_t RebaseOpcodeDone = 0x00; +constexpr std::uint8_t RebaseOpcodeSetTypeImm = 0x10; +constexpr std::uint8_t RebaseOpcodeSetSegmentAndOffsetUleb = 0x20; +constexpr std::uint8_t RebaseOpcodeDoRebaseImmTimes = 0x50; + +constexpr std::uint8_t BindTypePointer = 1; +constexpr std::uint8_t BindSpecialDylibFlatLookup = 0x0e; +constexpr std::uint8_t BindOpcodeDone = 0x00; +constexpr std::uint8_t BindOpcodeSetDylibSpecialImm = 0x30; +constexpr std::uint8_t BindOpcodeSetSymbolTrailingFlagsImm = 0x40; +constexpr std::uint8_t BindOpcodeSetTypeImm = 0x50; +constexpr std::uint8_t BindOpcodeSetAddendSleb = 0x60; +constexpr std::uint8_t BindOpcodeSetSegmentAndOffsetUleb = 0x70; +constexpr std::uint8_t BindOpcodeDoBind = 0x90; + +constexpr std::uint64_t MetaVersion = 2; +constexpr std::size_t MetaLoadBias = 0x08; +constexpr std::size_t MetaAllocateSlot = 0x10; +constexpr std::size_t MetaRegisterSlot = 0x18; +constexpr std::size_t MetaTlsDescriptor = 0x20; +constexpr std::size_t MetaTlsKey = MetaTlsDescriptor + 0x28; +constexpr std::size_t MetaCxaAtexitSlot = 0x50; +constexpr std::size_t MetaHeaderCount = 0x58; +constexpr std::size_t MetaHeaders = 0x60; + +constexpr std::uint8_t EntryStub[] = { + 0x31, 0xc9, 0x48, 0x83, 0x3c, 0xca, 0x00, 0x74, 0x05, 0x48, 0xff, 0xc1, 0xeb, 0xf4, 0x48, 0x8d, + 0x44, 0x0f, 0x03, 0x48, 0xc1, 0xe0, 0x03, 0x48, 0x29, 0xc4, 0x48, 0x83, 0xe4, 0xf0, 0x48, 0x89, + 0x3c, 0x24, 0x45, 0x31, 0xc0, 0x49, 0x39, 0xf8, 0x74, 0x0e, 0x4e, 0x8b, 0x0c, 0xc6, 0x4e, 0x89, + 0x4c, 0xc4, 0x08, 0x49, 0xff, 0xc0, 0xeb, 0xed, 0x48, 0xc7, 0x44, 0xfc, 0x08, 0x00, 0x00, 0x00, + 0x00, 0x4c, 0x8d, 0x54, 0xfc, 0x10, 0x45, 0x31, 0xc0, 0x49, 0x39, 0xc8, 0x74, 0x0d, 0x4e, 0x8b, + 0x0c, 0xc2, 0x4f, 0x89, 0x0c, 0xc2, 0x49, 0xff, 0xc0, 0xeb, 0xee, 0x49, 0xc7, 0x04, 0xca, 0x00, + 0x00, 0x00, 0x00, 0x48, 0x89, 0xe6, 0x48, 0x8d, 0x3d, 0x00, 0x00, 0x00, 0x00, 0xff, 0x15, 0x00, + 0x00, 0x00, 0x00, 0x0f, 0x0b, +}; +constexpr std::size_t EntryStubEntryDisplacement = 0x69; +constexpr std::size_t EntryStubStartDisplacement = 0x6f; + +struct Section { + std::string Name; + std::uint64_t Offset; + std::uint64_t Size; + std::uint32_t Flags; + std::uint32_t AlignShift; +}; + +struct Segment { + std::string Name; + std::uint64_t Rva; + std::uint64_t MemorySize; + std::uint32_t Protection; + std::vector Data; + std::vector
Sections; + std::uint64_t FileOffset = 0; +}; + +std::uint64_t AlignUp(const std::uint64_t value, const std::uint64_t alignment) { + return (value + alignment - 1) & ~(alignment - 1); +} + +void AppendUleb(std::vector& out, std::uint64_t value) { + do { + std::uint8_t byte = value & 0x7f; + value >>= 7; + if (value != 0) byte |= 0x80; + out.push_back(byte); + } while (value != 0); +} + +std::size_t UlebSize(std::uint64_t value) { + std::size_t size = 1; + while (value >>= 7) ++size; + return size; +} + +void AppendSleb(std::vector& out, std::int64_t value) { + for (bool more = true; more;) { + std::uint8_t byte = value & 0x7f; + value >>= 7; + more = !((value == 0 && (byte & 0x40) == 0) || (value == -1 && (byte & 0x40) != 0)); + if (more) byte |= 0x80; + out.push_back(byte); + } +} + +void AppendCString(std::vector& out, const std::string& text) { + out.insert(out.end(), text.begin(), text.end()); + out.push_back(0); +} + +void AppendName16(std::vector& out, const std::string& name) { + if (name.size() > 16) throw Domain::RelinkerException("Mach-O segment or section name is too long: " + name); + out.insert(out.end(), name.begin(), name.end()); + out.insert(out.end(), 16 - name.size(), 0); +} + +void PadTo(std::vector& out, const std::size_t alignment) { + while (out.size() % alignment != 0) out.push_back(0); +} + +std::pair Locate(const std::vector& segments, const std::uint64_t rva, const std::uint64_t size) { + for (std::size_t index = 0; index < segments.size(); ++index) { + const auto& segment = segments[index]; + if (segment.MemorySize == 0 || rva < segment.Rva || rva - segment.Rva >= segment.MemorySize || size > segment.MemorySize - (rva - segment.Rva)) continue; + if (index > 15) throw Domain::RelinkerException("Too many Mach-O segments for dyld opcodes", rva); + return {static_cast(index), rva - segment.Rva}; + } + throw Domain::RelinkerException("Mach-O fixup target is outside the image", rva); +} + +Segment& SegmentAt(std::vector& segments, const std::uint64_t rva, const std::uint64_t size) { + for (auto& segment : segments) { + if (rva >= segment.Rva && rva - segment.Rva + size <= segment.Data.size()) return segment; + } + throw Domain::RelinkerException("Mach-O image patch is outside the image data", rva); +} + +void AppendDylibCommand(std::vector& out, const std::uint32_t cmd, const std::string& path) { + const auto size = static_cast(AlignUp(24 + path.size() + 1, 8)); + const auto start = out.size(); + Io::AppendU32(out, cmd); + Io::AppendU32(out, size); + Io::AppendU32(out, 24); + Io::AppendU32(out, 2); + Io::AppendU32(out, 0x10000); + Io::AppendU32(out, 0x10000); + AppendCString(out, path); + out.resize(start + size, 0); +} + +void AppendPathCommand(std::vector& out, const std::uint32_t cmd, const std::string& path) { + const auto size = static_cast(AlignUp(12 + path.size() + 1, 8)); + const auto start = out.size(); + Io::AppendU32(out, cmd); + Io::AppendU32(out, size); + Io::AppendU32(out, 12); + AppendCString(out, path); + out.resize(start + size, 0); +} + +struct TlsAccess { + std::uint32_t Rva; + std::size_t Length; + bool StoreImmediate; + std::uint32_t Immediate; + std::uint8_t Register; + std::uint32_t Displacement; +}; + +struct TlsScan { + const Domain::ProgramHeader* Tls = nullptr; + std::vector Accesses; + std::set BranchTargets; +}; + +TlsScan ScanTls(const std::vector& source, const std::vector& headers, const Windows::WindowsLoadImage& image) { + TlsScan scan; + const Codegen::X64InstructionDecoder decoder; + const auto instructions = Relinker::CodeInstructionCollector().Collect(source, headers); + for (const auto& header : headers) { + if (header.Type == 7) { + if (scan.Tls != nullptr) throw Domain::RelinkerException("Multiple ELF TLS segments"); + scan.Tls = &header; + } + if (header.Type != 1 || (header.Flags & 1) == 0) continue; + for (auto instruction = instructions.lower_bound(header.MappedAddress); instruction != instructions.end() && *instruction - header.MappedAddress < header.FileSize; ++instruction) { + const auto offset = *instruction - header.MappedAddress; + const auto* bytes = source.data() + header.Offset + offset; + const auto info = decoder.DecodeInstruction(bytes, header.FileSize - offset); + const auto rva = image.GetRva(header.MappedAddress + offset, info.Length); + if (info.HasBranchTarget && !info.HasRipRelativeDisp) { + const auto target = static_cast(rva) + static_cast(info.Length) + info.BranchDisp; + if (target >= 0 && target <= std::numeric_limits::max()) scan.BranchTargets.insert(static_cast(target)); + } + if (info.SegmentPrefix == 0) continue; + const auto position = info.OpcodeOffset; + bool hasOperandSizePrefix = false; + bool supportedPrefixes = info.SegmentPrefix == 0x64; + for (std::size_t prefix = 0; prefix < position; ++prefix) { + const auto value = bytes[prefix]; + if (value == 0x66) hasOperandSizePrefix = true; + else if (value != 0x64 && !(prefix + 1 == position && value >= 0x40 && value <= 0x4f)) supportedPrefixes = false; + } + const auto loadRegister = info.Length - position == 7 ? static_cast(((bytes[position + 1] >> 3) & 7) | ((info.RexPrefix & 4) << 1)) : std::uint8_t{4}; + const bool loadValue = supportedPrefixes && (info.RexPrefix == 0x48 || info.RexPrefix == 0x4c) && loadRegister != 4 && bytes[position] == 0x8b && (bytes[position + 1] & 0xc7) == 0x04 && bytes[position + 2] == 0x25; + const bool storeImmediate = supportedPrefixes && !hasOperandSizePrefix && (info.RexPrefix == 0 || info.RexPrefix == 0x40) && info.Length - position == 11 && bytes[position] == 0xc7 && bytes[position + 1] == 0x04 && bytes[position + 2] == 0x25 && Io::ReadU32(source, header.Offset + offset + position + 3) == 0x28; + if (!loadValue && !storeImmediate) { + std::ostringstream message; + message << "Unsupported macOS guest TLS instruction (bytes:" << std::hex << std::setfill('0'); + for (std::size_t index = 0; index < info.Length; ++index) message << ' ' << std::setw(2) << static_cast(bytes[index]); + message << ')'; + throw Domain::RelinkerException(message.str(), header.Offset + offset); + } + scan.Accesses.push_back({rva, info.Length, storeImmediate, storeImmediate ? Io::ReadU32(source, header.Offset + offset + position + 7) : 0, storeImmediate ? std::uint8_t{0} : loadRegister, storeImmediate ? 0 : Io::ReadU32(source, header.Offset + offset + position + 3)}); + } + } + return scan; +} + +class TrieBuilder { +public: + explicit TrieBuilder(const std::vector& exports) { + for (const auto& entry : exports) Insert(entry); + } + + std::vector Build() { + std::vector order; + Collect(&root, order); + for (bool changed = true; changed;) { + changed = false; + std::uint64_t offset = 0; + for (auto* node : order) { + if (node->Offset != offset) { + node->Offset = offset; + changed = true; + } + offset += NodeSize(*node); + } + } + std::vector out; + for (auto* node : order) { + const auto terminal = TerminalBytes(*node); + AppendUleb(out, terminal.size()); + out.insert(out.end(), terminal.begin(), terminal.end()); + if (node->Children.size() > 255) throw Domain::RelinkerException("Mach-O export trie node has too many children"); + out.push_back(static_cast(node->Children.size())); + for (const auto& [label, child] : node->Children) { + AppendCString(out, label); + AppendUleb(out, child->Offset); + } + } + PadTo(out, 8); + return out; + } + +private: + struct Node { + std::vector>> Children; + std::optional Rva; + std::uint64_t Offset = 0; + }; + + Node root; + + void Insert(const MacOsExport& entry) { + Node* node = &root; + std::string_view rest = entry.Symbol; + while (!rest.empty()) { + bool descended = false; + for (auto& [label, child] : node->Children) { + std::size_t common = 0; + while (common < label.size() && common < rest.size() && label[common] == rest[common]) ++common; + if (common == 0) continue; + if (common < label.size()) { + auto middle = std::make_unique(); + middle->Children.emplace_back(label.substr(common), std::move(child)); + label.resize(common); + child = std::move(middle); + } + node = child.get(); + rest.remove_prefix(common); + descended = true; + break; + } + if (!descended) { + node->Children.emplace_back(std::string(rest), std::make_unique()); + node = node->Children.back().second.get(); + rest = {}; + } + } + if (node->Rva) throw Domain::RelinkerException("Duplicate Mach-O export: " + entry.Symbol); + node->Rva = entry.Rva; + } + + static void Collect(Node* node, std::vector& order) { + order.push_back(node); + for (auto& [label, child] : node->Children) Collect(child.get(), order); + } + + static std::vector TerminalBytes(const Node& node) { + std::vector bytes; + if (!node.Rva) return bytes; + AppendUleb(bytes, 0); + AppendUleb(bytes, *node.Rva); + return bytes; + } + + static std::uint64_t NodeSize(const Node& node) { + const auto terminal = TerminalBytes(node).size(); + std::uint64_t size = UlebSize(terminal) + terminal + 1; + for (const auto& [label, child] : node.Children) size += label.size() + 1 + UlebSize(child->Offset); + return size; + } +}; + +} + +std::uint64_t GuestTlsBlockSize(const Domain::ProgramHeader& tls) { + const auto alignment = std::max(tls.Alignment, 1); + if (tls.FileSize > tls.MemorySize || !std::has_single_bit(alignment) || alignment > 8192 || tls.MemorySize > 0x7fff0000u) + throw Domain::RelinkerException("Invalid or unsupported ELF TLS layout", tls.Offset); + return AlignUp(tls.MemorySize, alignment); +} + +std::vector WriteMacOsImage(MacOsImageInput& input) { + auto& image = *input.Image; + const auto& headers = *input.Headers; + const auto tls = ScanTls(*input.Source, headers, image); + const bool hasTls = tls.Tls != nullptr && tls.Tls->MemorySize != 0; + if (!hasTls && !tls.Accesses.empty()) throw Domain::RelinkerException("Guest TLS access without a non-empty PT_TLS"); + if (!hasTls && !input.TlsModuleSlots.empty()) throw Domain::RelinkerException("Guest TLS module relocation without a non-empty PT_TLS"); + + std::vector segments; + if (input.Executable) segments.push_back({"__PAGEZERO", 0, 0, 0, {}, {}}); + segments.push_back({"__TEXT", 0, LoadRva, VmProtRead | VmProtExecute, {}, {}}); + const auto guestSegments = segments.size(); + auto sections = image.BuildSections(); + if (input.Trampolines != nullptr && !input.Trampolines->empty()) { + for (const auto& access : tls.Accesses) { + for (const auto& site : *input.Trampolines) { + const auto siteRva = image.GetRva(site.Address, site.Length); + if (siteRva < access.Rva + access.Length && access.Rva < siteRva + site.Length) + throw Domain::RelinkerException("AMD-only instruction is also a guest TLS access", siteRva); + } + } + auto stubRva = Windows::AlignRva(image.GetEndRva()); + Windows::WindowsTrampolineBuilder().Build(*input.Trampolines, image, sections, stubRva); + } + std::uint64_t guestEnd = image.GetEndRva(); + for (auto& section : sections) { + std::uint32_t protection = 0; + if (section.Characteristics & Windows::SectionRead) protection |= VmProtRead; + if (section.Characteristics & Windows::SectionWrite) protection |= VmProtWrite; + if (section.Characteristics & Windows::SectionExecute) protection |= VmProtExecute; + auto name = section.Name == ".amdstub" ? std::string("__AMDSTUB") : "__ELF" + std::to_string(segments.size() - guestSegments); + guestEnd = std::max(guestEnd, section.Rva + section.Data.size()); + segments.push_back({std::move(name), section.Rva, section.Data.size(), protection, std::move(section.Data), {}}); + } + + const auto metaRva = static_cast(AlignUp(guestEnd, PageSize)); + const auto metaSize = MetaHeaders + headers.size() * 56; + std::vector meta(metaSize, 0); + Io::WriteU64(meta, 0, MetaVersion); + const auto firstLoad = std::find_if(headers.begin(), headers.end(), [](const auto& header) { return header.Type == 1; }); + if (firstLoad == headers.end()) throw Domain::RelinkerException("No PT_LOAD segments found"); + Io::WriteU64(meta, MetaLoadBias, ImageBase + image.GetRva(firstLoad->MappedAddress) - firstLoad->MappedAddress); + input.Rebases.push_back(metaRva + MetaLoadBias); + Io::WriteU64(meta, MetaHeaderCount, headers.size()); + for (std::size_t index = 0; index < headers.size(); ++index) { + const auto& header = headers[index]; + const auto at = MetaHeaders + index * 56; + Io::WriteU32(meta, at, header.Type); + Io::WriteU32(meta, at + 4, header.Flags); + Io::WriteU64(meta, at + 8, header.Offset); + Io::WriteU64(meta, at + 16, header.MappedAddress); + Io::WriteU64(meta, at + 24, header.PhysicalAddress); + Io::WriteU64(meta, at + 32, header.FileSize); + Io::WriteU64(meta, at + 40, header.MemorySize); + Io::WriteU64(meta, at + 48, header.Alignment); + } + const auto descriptor = metaRva + static_cast(MetaTlsDescriptor); + const auto initPointer = AlignUp(metaSize, 8); + const auto startSlot = initPointer + 8; + const auto tlsExports = startSlot + 8; + std::map tlsImportSlots; + for (const auto& import : input.TlsImports) tlsImportSlots.emplace(import.Symbol, 0); + auto metaEnd = tlsExports + input.TlsExports.size() * 16; + for (auto& [symbol, slot] : tlsImportSlots) { + slot = metaEnd; + metaEnd += 8; + input.Binds.push_back({symbol, metaRva + static_cast(slot)}); + } + meta.resize(metaEnd, 0); + for (std::size_t index = 0; index < input.TlsExports.size(); ++index) { + const auto& variable = input.TlsExports[index]; + if (!hasTls) throw Domain::RelinkerException("Guest TLS export has no TLS block: " + variable.Symbol); + if (variable.Offset >= tls.Tls->MemorySize) throw Domain::RelinkerException("Guest TLS export is outside the TLS block: " + variable.Symbol); + const auto at = tlsExports + index * 16; + Io::WriteU64(meta, at, ImageBase + descriptor); + Io::WriteU64(meta, at + 8, variable.Offset); + input.Rebases.push_back(metaRva + static_cast(at)); + input.Exports.push_back({variable.Symbol, metaRva + static_cast(at)}); + } + const auto codeRva = static_cast(AlignUp(metaRva + metaEnd, PageSize)); + Windows::WindowsStubEmitter code(codeRva); + + if (hasTls) { + const auto& header = *tls.Tls; + const auto blockSize = GuestTlsBlockSize(header); + constexpr std::int64_t StackGuardDisplacement = 0x28; + for (const auto& access : tls.Accesses) { + if (access.StoreImmediate || access.Displacement == 0) continue; + const auto displacement = static_cast(std::bit_cast(access.Displacement)); + if (displacement != StackGuardDisplacement && (displacement > 0 || displacement < -static_cast(blockSize))) + throw Domain::RelinkerException("macOS guest TLS load displacement " + std::to_string(displacement) + " is not in the thread TLS block, 0 or the stack guard at 0x28"); + } + Io::WriteU64(meta, MetaTlsDescriptor, 1); + if (header.FileSize != 0) { + Io::WriteU64(meta, MetaTlsDescriptor + 8, ImageBase + image.GetRva(header.MappedAddress, header.FileSize)); + input.Rebases.push_back(descriptor + 8); + } + Io::WriteU64(meta, MetaTlsDescriptor + 0x10, header.FileSize); + Io::WriteU64(meta, MetaTlsDescriptor + 0x18, blockSize); + Io::WriteU64(meta, MetaTlsDescriptor + 0x20, std::max(header.Alignment, 1)); + input.Binds.push_back({"_Aps5GuestTlsAllocate_nid_no_patch", metaRva + static_cast(MetaAllocateSlot)}); + input.Binds.push_back({"_Aps5GuestTlsRegister_nid_no_patch", metaRva + static_cast(MetaRegisterSlot)}); + for (const auto slot : input.TlsModuleSlots) { + auto& segment = SegmentAt(segments, slot, 8); + Io::WriteU64(segment.Data, slot - segment.Rva, ImageBase + descriptor); + input.Rebases.push_back(slot); + } + + for (const auto& access : tls.Accesses) { + const auto target = tls.BranchTargets.upper_bound(access.Rva); + if (target != tls.BranchTargets.end() && *target < access.Rva + access.Length) + throw Domain::RelinkerException("Branch enters a guest TLS instruction", *target); + const auto stub = code.GetRva(); + const bool preserveAccumulator = access.StoreImmediate || access.Register != 0; + code.Emit({0x48, 0x8d, 0x64, 0x24, 0x80}); + code.Emit({0x9c}); + if (preserveAccumulator) code.Emit({0x50}); + code.Rip({0x48, 0x8b, 0x05}, metaRva + static_cast(MetaTlsKey)); + code.Emit({0x65, 0x48, 0x8b, 0x04, 0xc5, 0, 0, 0, 0}); + code.Emit({0x48, 0x85, 0xc0}); + const auto allocated = code.Branch({0x0f, 0x85}); + code.Rip({0x48, 0x8d, 0x05}, descriptor); + code.Rip({0xff, 0x15}, metaRva + static_cast(MetaAllocateSlot)); + code.PatchBranch(allocated, code.GetRva()); + if (access.StoreImmediate) { + code.Emit({0xc7, 0x40, 0x28}); + code.U32(access.Immediate); + } else if (access.Displacement != 0) { + code.Emit({0x48, 0x8b, 0x80}); + code.U32(access.Displacement); + } + if (!access.StoreImmediate && access.Register != 0) + code.Emit({static_cast(0x48 | (access.Register >> 3)), 0x89, static_cast(0xc0 | (access.Register & 7))}); + if (preserveAccumulator) code.Emit({0x58}); + code.Emit({0x9d}); + code.Emit({0x48, 0x8d, 0xa4, 0x24, 0x80, 0, 0, 0}); + code.Rip({0xe9}, static_cast(access.Rva + access.Length)); + + auto& site = SegmentAt(segments, access.Rva, access.Length); + const auto at = access.Rva - site.Rva; + const auto displacement = static_cast(stub) - static_cast(access.Rva + 5); + std::fill_n(site.Data.begin() + static_cast(at), access.Length, 0x90); + site.Data[at] = 0xe9; + Io::WriteU32(site.Data, at + 1, static_cast(static_cast(displacement))); + } + } + + const bool hasFini = input.FiniRva != 0 || !input.FiniArrayRvas.empty(); + std::uint32_t finiStub = 0; + const auto callZeroArgs = [&] { code.Emit({0x31, 0xff, 0x31, 0xf6, 0x31, 0xd2}); }; + const auto callSlot = [&](const std::uint32_t slot) { + code.Rip({0x48, 0x8b, 0x05}, slot); + code.Emit({0x48, 0x85, 0xc0, 0x74, 0x08}); + callZeroArgs(); + code.Emit({0xff, 0xd0}); + }; + if (hasFini) { + finiStub = code.GetRva(); + code.Emit({0x55, 0x48, 0x89, 0xe5}); + for (auto slot = input.FiniArrayRvas.rbegin(); slot != input.FiniArrayRvas.rend(); ++slot) callSlot(*slot); + if (input.FiniRva != 0) { + callZeroArgs(); + code.Rip({0xe8}, input.FiniRva); + } + code.Emit({0x5d, 0xc3}); + input.Binds.push_back({"___cxa_atexit", metaRva + static_cast(MetaCxaAtexitSlot)}); + } + + const bool hasInit = hasTls || !input.TlsImports.empty() || input.InitRva != 0 || !input.InitArrayRvas.empty() || hasFini; + std::vector
dataSections{{"__meta", 0, metaSize, 0, 3}}; + if (hasInit) { + const auto initStub = code.GetRva(); + code.Emit({0x55, 0x48, 0x89, 0xe5}); + if (hasTls) { + code.Rip({0x48, 0x8d, 0x3d}, descriptor); + code.Rip({0xff, 0x15}, metaRva + static_cast(MetaRegisterSlot)); + } + for (const auto& import : input.TlsImports) { + code.Rip({0x48, 0x8b, 0x05}, metaRva + static_cast(tlsImportSlots.at(import.Symbol))); + if (import.Module) code.Emit({0x48, 0x8b, 0x00}); + else code.Emit({0x48, 0x8b, 0x40, 0x08}); + if (import.Addend != 0) { + code.Emit({0x48, 0xba}); + code.U64(static_cast(import.Addend)); + code.Emit({0x48, 0x01, 0xd0}); + } + code.Rip({0x48, 0x89, 0x05}, import.Rva); + } + if (input.InitRva != 0) { + callZeroArgs(); + code.Rip({0xe8}, input.InitRva); + } + for (const auto slot : input.InitArrayRvas) callSlot(slot); + if (hasFini) { + code.Rip({0x48, 0x8d, 0x3d}, finiStub); + code.Emit({0x31, 0xf6}); + code.Rip({0x48, 0x8d, 0x15}, 0); + code.Rip({0xff, 0x15}, metaRva + static_cast(MetaCxaAtexitSlot)); + } + code.Emit({0x5d, 0xc3}); + Io::WriteU64(meta, initPointer, ImageBase + initStub); + input.Rebases.push_back(metaRva + static_cast(initPointer)); + dataSections.push_back({"__mod_init_func", initPointer, 8, SectionModInitFuncPointers, 3}); + } + if (input.Executable) { + input.Binds.push_back({"_Aps5StartGuest_nid_no_patch", metaRva + static_cast(startSlot)}); + } + const auto metaBytes = meta.size(); + segments.push_back({"__APS5DATA", metaRva, metaBytes, VmProtRead | VmProtWrite, std::move(meta), std::move(dataSections)}); + auto stubs = code.TakeBytes(); + if (!stubs.empty()) segments.push_back({"__APS5TEXT", codeRva, stubs.size(), VmProtRead | VmProtExecute, std::move(stubs), {}}); + + std::sort(segments.begin() + static_cast(guestSegments), segments.end(), [](const Segment& left, const Segment& right) { return left.Rva < right.Rva; }); + for (std::size_t index = guestSegments; index < segments.size(); ++index) { + auto& data = segments[index].Data; + if (segments[index].Sections.empty()) { + auto used = data.size(); + while (used != 0 && data[used - 1] == 0) --used; + data.resize(used); + } + data.resize(AlignUp(data.size(), PageSize), 0); + } + + std::vector rebase; + rebase.push_back(RebaseOpcodeSetTypeImm | RebaseTypePointer); + auto rebaseTargets = input.Rebases; + std::sort(rebaseTargets.begin(), rebaseTargets.end()); + for (const auto rva : rebaseTargets) { + const auto [segment, offset] = Locate(segments, rva, 8); + rebase.push_back(RebaseOpcodeSetSegmentAndOffsetUleb | segment); + AppendUleb(rebase, offset); + rebase.push_back(RebaseOpcodeDoRebaseImmTimes | 1); + } + rebase.push_back(RebaseOpcodeDone); + PadTo(rebase, 8); + + std::vector bind; + bind.push_back(BindOpcodeSetDylibSpecialImm | BindSpecialDylibFlatLookup); + bind.push_back(BindOpcodeSetTypeImm | BindTypePointer); + for (const auto& item : input.Binds) { + const auto [segment, offset] = Locate(segments, item.Rva, 8); + bind.push_back(BindOpcodeSetSymbolTrailingFlagsImm); + AppendCString(bind, item.Symbol); + bind.push_back(BindOpcodeSetAddendSleb); + AppendSleb(bind, item.Addend); + bind.push_back(BindOpcodeSetSegmentAndOffsetUleb | segment); + AppendUleb(bind, offset); + bind.push_back(BindOpcodeDoBind); + } + bind.push_back(BindOpcodeDone); + PadTo(bind, 8); + + const auto exports = input.Exports.empty() ? std::vector{} : TrieBuilder(input.Exports).Build(); + + std::vector commands; + std::uint32_t commandCount = 0; + std::vector segmentCommandOffsets; + for (const auto& segment : segments) { + segmentCommandOffsets.push_back(commands.size()); + Io::AppendU32(commands, LcSegment64); + Io::AppendU32(commands, static_cast(72 + 80 * segment.Sections.size())); + AppendName16(commands, segment.Name); + for (int field = 0; field < 4; ++field) Io::AppendU64(commands, 0); + Io::AppendU32(commands, segment.Protection); + Io::AppendU32(commands, segment.Protection); + Io::AppendU32(commands, static_cast(segment.Sections.size())); + Io::AppendU32(commands, 0); + for (const auto& section : segment.Sections) { + AppendName16(commands, section.Name); + AppendName16(commands, segment.Name); + Io::AppendU64(commands, 0); + Io::AppendU64(commands, section.Size); + Io::AppendU32(commands, 0); + Io::AppendU32(commands, section.AlignShift); + Io::AppendU32(commands, 0); + Io::AppendU32(commands, 0); + Io::AppendU32(commands, section.Flags); + for (int field = 0; field < 3; ++field) Io::AppendU32(commands, 0); + } + ++commandCount; + } + const auto linkeditCommand = commands.size(); + Io::AppendU32(commands, LcSegment64); + Io::AppendU32(commands, 72); + AppendName16(commands, "__LINKEDIT"); + for (int field = 0; field < 4; ++field) Io::AppendU64(commands, 0); + Io::AppendU32(commands, VmProtRead); + Io::AppendU32(commands, VmProtRead); + Io::AppendU32(commands, 0); + Io::AppendU32(commands, 0); + ++commandCount; + + if (!input.Executable) { + AppendDylibCommand(commands, LcIdDylib, input.InstallName); + ++commandCount; + } + + const auto dyldInfoCommand = commands.size(); + Io::AppendU32(commands, LcDyldInfoOnly); + Io::AppendU32(commands, 48); + for (int field = 0; field < 10; ++field) Io::AppendU32(commands, 0); + ++commandCount; + + const auto symtabCommand = commands.size(); + Io::AppendU32(commands, LcSymtab); + Io::AppendU32(commands, 24); + for (int field = 0; field < 4; ++field) Io::AppendU32(commands, 0); + ++commandCount; + + Io::AppendU32(commands, LcDysymtab); + Io::AppendU32(commands, 80); + for (int field = 0; field < 18; ++field) Io::AppendU32(commands, 0); + ++commandCount; + + if (input.Executable) { + AppendPathCommand(commands, LcLoadDylinker, "/usr/lib/dyld"); + ++commandCount; + } + + const auto uuidCommand = commands.size(); + Io::AppendU32(commands, LcUuid); + Io::AppendU32(commands, 24); + for (int field = 0; field < 4; ++field) Io::AppendU32(commands, 0); + ++commandCount; + + Io::AppendU32(commands, LcBuildVersion); + Io::AppendU32(commands, 24); + Io::AppendU32(commands, 1); + Io::AppendU32(commands, 0x000b0000); + Io::AppendU32(commands, 0x000b0000); + Io::AppendU32(commands, 0); + ++commandCount; + + std::size_t mainCommand = 0; + if (input.Executable) { + mainCommand = commands.size(); + Io::AppendU32(commands, LcMain); + Io::AppendU32(commands, 24); + Io::AppendU64(commands, 0); + Io::AppendU64(commands, 0); + ++commandCount; + } + + AppendDylibCommand(commands, LcLoadDylib, "/usr/lib/libSystem.B.dylib"); + ++commandCount; + for (const auto& library : input.Dylibs) { + AppendDylibCommand(commands, LcLoadDylib, library); + ++commandCount; + } + for (const auto& path : input.RunPaths) { + AppendPathCommand(commands, LcRpath, path); + ++commandCount; + } + + std::vector file; + constexpr std::size_t HeaderSize = 32; + const auto stubOffset = AlignUp(HeaderSize + commands.size(), 16); + if (stubOffset + (input.Executable ? sizeof(EntryStub) : 0) > LoadRva) + throw Domain::RelinkerException("Mach-O load commands do not fit before the guest image"); + file.resize(LoadRva, 0); + if (input.Executable) { + std::copy(std::begin(EntryStub), std::end(EntryStub), file.begin() + static_cast(stubOffset)); + const auto writeDisplacement = [&](const std::size_t field, const std::uint64_t targetRva) { + const auto displacement = static_cast(targetRva) - static_cast(stubOffset + field + 4); + if (displacement < std::numeric_limits::min() || displacement > std::numeric_limits::max()) + throw Domain::RelinkerException("macOS entry stub target is out of rel32 range"); + Io::WriteU32(file, stubOffset + field, static_cast(static_cast(displacement))); + }; + writeDisplacement(EntryStubEntryDisplacement, input.EntryRva); + writeDisplacement(EntryStubStartDisplacement, metaRva + startSlot); + } + + for (std::size_t index = guestSegments; index < segments.size(); ++index) { + auto& segment = segments[index]; + segment.FileOffset = segment.Data.empty() ? 0 : file.size(); + file.insert(file.end(), segment.Data.begin(), segment.Data.end()); + } + + const auto linkeditOffset = file.size(); + const auto rebaseOffset = file.size(); + file.insert(file.end(), rebase.begin(), rebase.end()); + const auto bindOffset = file.size(); + file.insert(file.end(), bind.begin(), bind.end()); + const auto exportOffset = file.size(); + file.insert(file.end(), exports.begin(), exports.end()); + const auto stringOffset = file.size(); + file.insert(file.end(), 8, 0); + const auto linkeditSize = file.size() - linkeditOffset; + const auto imageEnd = segments.back().Rva + AlignUp(segments.back().MemorySize, PageSize); + if (file.size() > std::numeric_limits::max()) + throw Domain::RelinkerException("Mach-O output exceeds 4 GiB"); + + const auto writeSegment = [&](const std::size_t at, const std::uint64_t vmaddr, const std::uint64_t vmsize, const std::uint64_t fileoff, const std::uint64_t filesize) { + Io::WriteU64(commands, at + 24, vmaddr); + Io::WriteU64(commands, at + 32, vmsize); + Io::WriteU64(commands, at + 40, fileoff); + Io::WriteU64(commands, at + 48, filesize); + }; + std::size_t index = 0; + if (input.Executable) writeSegment(segmentCommandOffsets[index++], 0, ImageBase, 0, 0); + writeSegment(segmentCommandOffsets[index++], ImageBase, LoadRva, 0, LoadRva); + for (; index < segments.size(); ++index) { + const auto& segment = segments[index]; + writeSegment(segmentCommandOffsets[index], ImageBase + segment.Rva, AlignUp(segment.MemorySize, PageSize), segment.FileOffset, segment.Data.size()); + for (std::size_t sectionIndex = 0; sectionIndex < segment.Sections.size(); ++sectionIndex) { + const auto& section = segment.Sections[sectionIndex]; + const auto at = segmentCommandOffsets[index] + 72 + sectionIndex * 80; + Io::WriteU64(commands, at + 32, ImageBase + segment.Rva + section.Offset); + Io::WriteU32(commands, at + 48, static_cast(segment.FileOffset + section.Offset)); + } + } + writeSegment(linkeditCommand, ImageBase + imageEnd, AlignUp(linkeditSize, PageSize), linkeditOffset, linkeditSize); + Io::WriteU32(commands, dyldInfoCommand + 8, static_cast(rebaseOffset)); + Io::WriteU32(commands, dyldInfoCommand + 12, static_cast(rebase.size())); + Io::WriteU32(commands, dyldInfoCommand + 16, static_cast(bindOffset)); + Io::WriteU32(commands, dyldInfoCommand + 20, static_cast(bind.size())); + Io::WriteU32(commands, dyldInfoCommand + 40, static_cast(exportOffset)); + Io::WriteU32(commands, dyldInfoCommand + 44, static_cast(exports.size())); + Io::WriteU32(commands, symtabCommand + 8, static_cast(stringOffset)); + Io::WriteU32(commands, symtabCommand + 16, static_cast(stringOffset)); + Io::WriteU32(commands, symtabCommand + 20, 8); + if (input.Executable) Io::WriteU64(commands, mainCommand + 8, stubOffset); + + std::uint64_t hash = 0xcbf29ce484222325ull; + for (const auto byte : file) hash = (hash ^ byte) * 0x100000001b3ull; + Io::WriteU64(commands, uuidCommand + 8, hash); + Io::WriteU64(commands, uuidCommand + 16, hash * 0x9e3779b97f4a7c15ull); + commands[uuidCommand + 14] = static_cast((commands[uuidCommand + 14] & 0x0f) | 0x40); + + Io::WriteU32(file, 0, MhMagic64); + Io::WriteU32(file, 4, CpuTypeX8664); + Io::WriteU32(file, 8, CpuSubtypeX8664All); + Io::WriteU32(file, 12, input.Executable ? MhExecute : MhDylib); + Io::WriteU32(file, 16, commandCount); + Io::WriteU32(file, 20, static_cast(commands.size())); + Io::WriteU32(file, 24, MhNoUndefs | MhDyldLink | MhTwoLevel); + std::copy(commands.begin(), commands.end(), file.begin() + HeaderSize); + return file; +} + +} diff --git a/core/relinker/elfpatcher/src/macos/MacOsMachOPatcher.cpp b/core/relinker/elfpatcher/src/macos/MacOsMachOPatcher.cpp new file mode 100644 index 000000000..8e7603773 --- /dev/null +++ b/core/relinker/elfpatcher/src/macos/MacOsMachOPatcher.cpp @@ -0,0 +1,69 @@ +#include +#include +#include +#include +#include +#include +#include +#include + +namespace Elfpatcher::MacOs { + +std::string MachOLoadPath(const std::string& path, const std::string& origin) { + if (path == "$ORIGIN") return origin; + if (path.starts_with("$ORIGIN/")) return origin + path.substr(7); + return path; +} + +std::vector ReadNeededLibraries(const Domain::SysVDynamicSection& dynamicSection) { + const auto& bytes = dynamicSection.DynamicSegmentData; + if (bytes.size() % 16 != 0) throw Domain::RelinkerException("Invalid dynamic segment size"); + std::vector result; + std::set unique; + for (std::size_t offset = 0; offset < bytes.size(); offset += 16) { + if (Io::ReadU64(bytes, offset) != 1) throw Domain::RelinkerException("Unexpected tag in rebuilt ELF dependency table", offset); + const auto nameOffset = Io::ReadU64(bytes, offset + 8); + if (nameOffset >= dynamicSection.DynStrData.size()) throw Domain::RelinkerException("DT_NEEDED string offset is out of bounds", nameOffset); + auto name = Windows::ReadString(dynamicSection.DynStrData, static_cast(nameOffset)); + if (name.empty()) throw Domain::RelinkerException("Invalid DT_NEEDED library: " + name); + if (unique.insert(name).second) result.push_back(std::move(name)); + } + if (unique.contains("libSceLibcInternal.prx") && !unique.contains("libc.prx")) result.push_back("libc.prx"); + return result; +} + +std::string MachODependency(const std::string& name, const std::string& origin) { + if (name.starts_with("$ORIGIN")) return MachOLoadPath(name, origin); + if (name.find('/') != std::string::npos) throw Domain::RelinkerException("Unsupported dependency path: " + name); + return "@rpath/" + name; +} + +std::vector MacOsMachOPatcher::Patch(const std::vector& sourceElf, const std::vector& originalHeaders, const Domain::SysVDynamicSection& dynamicSection, const std::uint64_t originalPltGotVaddr, const std::string& runPath, const bool lazyBinding, const bool dependencyDiagnostics, const std::vector& trampolines) { + if (dependencyDiagnostics) + throw Domain::RelinkerException("macOS target does not support --windows-diagnostics"); + static_cast(lazyBinding); + + Windows::WindowsLoadImage image(sourceElf, originalHeaders); + if (originalPltGotVaddr != 0) + image.GetRva(originalPltGotVaddr, 8); + const auto relocations = Windows::WindowsRelocationBuilder().Apply(image, dynamicSection); + + MacOsImageInput input; + input.Executable = true; + input.Source = &sourceElf; + input.Headers = &originalHeaders; + input.Image = ℑ + input.Rebases = relocations.BaseRelocations; + for (const auto& import : relocations.Imports) + input.Binds.push_back({"_" + import.Name, import.TargetRva, static_cast(import.Addend)}); + for (const auto& library : ReadNeededLibraries(dynamicSection)) + input.Dylibs.push_back(MachODependency(library, "@executable_path")); + if (std::find(input.Dylibs.begin(), input.Dylibs.end(), "@rpath/libkernel.prx") == input.Dylibs.end()) + input.Dylibs.push_back("@rpath/libkernel.prx"); + input.RunPaths.push_back(MachOLoadPath(runPath, "@executable_path")); + input.EntryRva = image.GetEntryRva(); + input.Trampolines = &trampolines; + return WriteMacOsImage(input); +} + +} diff --git a/core/relinker/elfpatcher/src/windows/WindowsEntryStubBuilder.cpp b/core/relinker/elfpatcher/src/windows/WindowsEntryStubBuilder.cpp index 068954f28..b696e76a4 100644 --- a/core/relinker/elfpatcher/src/windows/WindowsEntryStubBuilder.cpp +++ b/core/relinker/elfpatcher/src/windows/WindowsEntryStubBuilder.cpp @@ -146,6 +146,8 @@ WindowsEntryStub WindowsEntryStubBuilder::Build(const std::uint32_t dataRva, con (import.Library.empty() ? std::string{} : " from " + import.Library) + "\n"); const auto argumentError = errors.size(); errors.push_back("FAIL: cannot prepare command-line arguments\n"); + const auto directoryError = errors.size(); + errors.push_back("FAIL: cannot make the executable directory the current directory\n"); std::vector errorRvas; for (const auto& error : errors) errorRvas.push_back(addString(error)); @@ -271,6 +273,10 @@ WindowsEntryStub WindowsEntryStubBuilder::Build(const std::uint32_t dataRva, con code.Emit({0x49, 0xff, 0xcc, 0x41, 0x80, 0x3c, 0x24, 0x5c}); code.Rip({0x0f, 0x85}, findSeparator); code.Emit({0x49, 0xff, 0xc4}); + code.Emit({0x41, 0xc6, 0x04, 0x24, 0x00}); + code.Rip({0x48, 0x8d, 0x0d}, modulePath); + call("SetCurrentDirectoryA"); + requireNonzero(directoryError, 0xc000000du); for (std::size_t index = 0; index < libraries.size(); ++index) { if (absolutePath && index >= guestModules.size()) { @@ -426,6 +432,7 @@ WindowsEntryStub WindowsEntryStubBuilder::Build(const std::uint32_t dataRva, con call("FreeLibrary"); code.Rip({0x48, 0x8b, 0x3d}, argumentBlock); const auto exitCallback = code.Branch({0x48, 0x8d, 0x35}); + code.Emit({0x48, 0xc7, 0x44, 0x24, 0x40, 0, 0, 0, 0, 0x48, 0xc7, 0x44, 0x24, 0x48, 0, 0, 0, 0, 0x48, 0x8d, 0x6c, 0x24, 0x40}); code.Rip({0xe8}, entryRva); code.Emit({0x89, 0x44, 0x24, 0x58}); guestStartup.Finalize(code, guestModules, handles, guestFinished); diff --git a/core/relinker/elfpatcher/src/windows/WindowsImportBuilder.cpp b/core/relinker/elfpatcher/src/windows/WindowsImportBuilder.cpp index 26a775b87..070970f7d 100644 --- a/core/relinker/elfpatcher/src/windows/WindowsImportBuilder.cpp +++ b/core/relinker/elfpatcher/src/windows/WindowsImportBuilder.cpp @@ -5,7 +5,7 @@ namespace Elfpatcher::Windows { WindowsImports WindowsImportBuilder::Build(const std::uint32_t sectionRva) const { - const std::vector names = {"ExitProcess", "FormatMessageA", "FreeLibrary", "GetCommandLineW", "GetFileAttributesA", "GetLastError", "GetModuleFileNameA", "GetModuleHandleA", "GetProcAddress", "GetStdHandle", "GetSystemDirectoryA", "LoadLibraryExA", "LocalFree", "RaiseException", "VirtualAlloc", "WideCharToMultiByte", "WriteFile", "lstrcatA", "lstrcmpA", "lstrcmpiA", "lstrcpyA", "lstrlenA"}; + const std::vector names = {"ExitProcess", "FormatMessageA", "FreeLibrary", "GetCommandLineW", "GetFileAttributesA", "GetLastError", "GetModuleFileNameA", "GetModuleHandleA", "GetProcAddress", "GetStdHandle", "GetSystemDirectoryA", "LoadLibraryExA", "LocalFree", "RaiseException", "SetCurrentDirectoryA", "VirtualAlloc", "WideCharToMultiByte", "WriteFile", "lstrcatA", "lstrcmpA", "lstrcmpiA", "lstrcpyA", "lstrlenA"}; WindowsImports result{{".idata", sectionRva, SectionRead | SectionWrite | 0x40u, std::vector(40)}, {sectionRva, 40}, {}, {}}; auto& bytes = result.Section.Data; const auto lookupOffset = bytes.size(); @@ -44,9 +44,10 @@ std::vector WindowsImportBuilder::ReadLibraries(const Domain::SysVD if (nameOffset >= dynamicSection.DynStrData.size()) throw Domain::RelinkerException("DT_NEEDED string offset is out of bounds", nameOffset); auto name = ReadString(dynamicSection.DynStrData, static_cast(nameOffset)); - if (name.empty() || name.find_first_of("/\\:") != std::string::npos || !unique.insert(name).second) - throw Domain::RelinkerException("Invalid or duplicate DT_NEEDED library: " + name); - result.push_back(std::move(name)); + if (name.empty() || name.find_first_of("/\\:") != std::string::npos) + throw Domain::RelinkerException("Invalid DT_NEEDED library: " + name); + if (unique.insert(name).second) + result.push_back(std::move(name)); } // AnyPS5 implements the C runtime in libc.prx. Windows GetProcAddress // does not search a module's dependencies as ELF symbol lookup does. diff --git a/core/relinker/elfpatcher/src/windows/WindowsPeWriter.cpp b/core/relinker/elfpatcher/src/windows/WindowsPeWriter.cpp index 3c4e9ac34..e91f5e59f 100644 --- a/core/relinker/elfpatcher/src/windows/WindowsPeWriter.cpp +++ b/core/relinker/elfpatcher/src/windows/WindowsPeWriter.cpp @@ -18,7 +18,7 @@ std::vector WindowsPeWriter::Write(const std::vector& s Io::WriteU16(result, peOffset + 4, 0x8664); Io::WriteU16(result, peOffset + 6, static_cast(sections.size())); Io::WriteU16(result, peOffset + 20, 240); - Io::WriteU16(result, peOffset + 22, directories[5].Size == 0 ? 0x23 : 0x22); + Io::WriteU16(result, peOffset + 22, 0x22); Io::WriteU16(result, optionalOffset, 0x20b); Io::WriteU32(result, optionalOffset + 16, entryRva); Io::WriteU64(result, optionalOffset + 24, ImageBase); @@ -44,8 +44,9 @@ std::vector WindowsPeWriter::Write(const std::vector& s const auto& section = sections[index]; if (section.Name.empty() || section.Name.size() > 8 || section.Data.empty() || section.Rva != endRva) throw Domain::RelinkerException("Invalid PE section layout: " + section.Name, section.Rva); - const auto rawSize = Io::AlignUp(CheckedRva(section.Data.size()), FileAlignment); - const auto rawOffset = CheckedRva(result.size()); + const auto initialized = static_cast(std::find_if(section.Data.rbegin(), section.Data.rend(), [](const std::uint8_t value) { return value != 0; }).base() - section.Data.begin()); + const auto rawSize = Io::AlignUp(CheckedRva(initialized), FileAlignment); + const auto rawOffset = rawSize == 0 ? 0u : CheckedRva(result.size()); const auto header = sectionTable + index * 40; std::copy(section.Name.begin(), section.Name.end(), result.begin() + static_cast(header)); Io::WriteU32(result, header + 8, CheckedRva(section.Data.size())); @@ -61,8 +62,10 @@ std::vector WindowsPeWriter::Write(const std::vector& s } else { dataSize = CheckedRva(static_cast(dataSize) + rawSize); } - result.insert(result.end(), section.Data.begin(), section.Data.end()); - result.resize(static_cast(rawOffset) + rawSize); + if (rawSize != 0) { + result.insert(result.end(), section.Data.begin(), section.Data.begin() + static_cast(initialized)); + result.resize(static_cast(rawOffset) + rawSize); + } endRva = AlignRva(section.Rva + static_cast(section.Data.size())); } if (!foundEntry) diff --git a/core/relinker/elfpatcher/src/windows/WindowsTlsBuilder.cpp b/core/relinker/elfpatcher/src/windows/WindowsTlsBuilder.cpp index 3fb5379dc..54414f33e 100644 --- a/core/relinker/elfpatcher/src/windows/WindowsTlsBuilder.cpp +++ b/core/relinker/elfpatcher/src/windows/WindowsTlsBuilder.cpp @@ -135,7 +135,8 @@ PeDirectory WindowsTlsBuilder::Build(const std::vector& source, co const bool storeImmediate = supportedPrefixes && !hasOperandSizePrefix && (info.RexPrefix == 0 || info.RexPrefix == 0x40) && info.Length - position == 11 && bytes[position] == 0xc7 && bytes[position + 1] == 0x04 && bytes[position + 2] == 0x25 && Io::ReadU32(source, header.Offset + offset + position + 3) == 0x28; const bool aluRead = supportedPrefixes && (info.RexPrefix == 0x48 || info.RexPrefix == 0x4c) && loadRegister != 4 && info.Length - position == 7 && isAluReadOpcode(bytes[position]) && (bytes[position + 1] & 0xc7) == 0x04 && bytes[position + 2] == 0x25; const bool wide = (info.RexPrefix & 8) != 0; - const auto address = supportedPrefixes && !loadValue && !aluRead && bytes[position] == 0x8b && (wide || !hasOperandSizePrefix) ? registerAddress(bytes, position, info.Length, info.RexPrefix) : std::nullopt; + const bool isAlu = isAluReadOpcode(bytes[position]); + const auto address = supportedPrefixes && !loadValue && !aluRead && (bytes[position] == 0x8b || isAlu) && (wide || !hasOperandSizePrefix) ? registerAddress(bytes, position, info.Length, info.RexPrefix) : std::nullopt; const auto addressRegister = static_cast(((bytes[position + 1] >> 3) & 7) | ((info.RexPrefix & 4) << 1)); if (address && addressRegister != 4) { std::vector moved; @@ -149,7 +150,7 @@ PeDirectory WindowsTlsBuilder::Build(const std::vector& source, co moved.push_back({CheckedRva(rva + length), std::vector(bytes + length, bytes + length + next.Length), next.HasRipRelativeDisp ? std::optional(next.RipRelativeDispOffset) : std::nullopt}); length += next.Length; } - accesses.push_back({rva, header.Offset + offset, length, false, 0, addressRegister, 0, 0, *address, wide, std::move(moved)}); + accesses.push_back({rva, header.Offset + offset, length, false, 0, addressRegister, 0, isAlu ? bytes[position] : std::uint8_t{0}, *address, wide, std::move(moved)}); continue; } if (!loadValue && !storeImmediate && !aluRead) { @@ -175,11 +176,8 @@ PeDirectory WindowsTlsBuilder::Build(const std::vector& source, co throw Domain::RelinkerException("Invalid or unsupported ELF TLS layout", tls->Offset); // Native homebrew linkers can emit an empty PT_TLS placeholder. // It needs no Windows TLS directory unless guest code accesses TLS. - if (tls->MemorySize == 0) { - if (!accesses.empty()) - throw Domain::RelinkerException("Guest TLS access with empty PT_TLS", tls->Offset); + if (tls->MemorySize == 0 && accesses.empty()) return {}; - } const auto alignment = std::max(tls->Alignment, 16); const auto blockSize = CheckedRva((tls->MemorySize + alignment - 1) & ~(alignment - 1)); const auto templateOffset = CheckedRva((64 + alignment - 1) & ~(alignment - 1)); @@ -224,26 +222,84 @@ PeDirectory WindowsTlsBuilder::Build(const std::vector& source, co patchAccess(sections, access, code.GetRva()); code.Emit({0x48, 0x8d, 0x64, 0x24, 0x80}); if (!access.Address.empty()) { - code.Emit({0x51}); - code.Emit({0x50}); - for (const auto byte : access.Address) - code.Emit({byte}); - code.Emit({0x50}); - loadPointer(); - code.Emit({0x59}); - const auto rex = static_cast((access.Wide ? 0x48 : 0x40) | ((access.Register >> 3) << 2)); - if (rex != 0x40) - code.Emit({rex}); - code.Emit({0x8b, static_cast(0x04 | ((access.Register & 7) << 3)), 0x08}); - if (access.Register == 0) { - code.Emit({0x48, 0x8d, 0x64, 0x24, 0x08}); + if (access.AluOpcode != 0 && access.Register == 0) { + code.Emit({0x51}); + code.Emit({0x52}); + code.Emit({0x50}); + for (const auto byte : access.Address) + code.Emit({byte}); + code.Emit({0x50}); + loadPointer(); code.Emit({0x59}); - } else if (access.Register == 1) { + const auto rex = static_cast((access.Wide ? 0x48 : 0x40) | 0x00); + if (rex != 0x40) + code.Emit({rex}); + code.Emit({0x8b, 0x14, 0x08}); + code.Emit({0x58}); + if (access.Wide) + code.Emit({0x48, access.AluOpcode, 0xc2}); + else + code.Emit({access.AluOpcode, 0xc2}); + code.Emit({0x5a}); + code.Emit({0x59}); + } else if (access.AluOpcode != 0 && access.Register == 1) { + code.Emit({0x51}); + code.Emit({0x50}); + for (const auto byte : access.Address) + code.Emit({byte}); + code.Emit({0x50}); + loadPointer(); + code.Emit({0x59}); + const auto rex = static_cast((access.Wide ? 0x48 : 0x40) | 0x00); + if (rex != 0x40) + code.Emit({rex}); + code.Emit({0x8b, 0x04, 0x08}); + if (access.Wide) + code.Emit({0x48, 0x8b, 0x4c, 0x24, 0x08, 0x48, access.AluOpcode, 0xc8}); + else + code.Emit({0x48, 0x8b, 0x4c, 0x24, 0x08, access.AluOpcode, 0xc8}); code.Emit({0x58}); code.Emit({0x48, 0x8d, 0x64, 0x24, 0x08}); + } else if (access.AluOpcode != 0) { + code.Emit({0x51}); + code.Emit({0x50}); + for (const auto byte : access.Address) + code.Emit({byte}); + code.Emit({0x50}); + loadPointer(); + code.Emit({0x59}); + const auto rexLoad = static_cast((access.Wide ? 0x48 : 0x40) | 0x00); + if (rexLoad != 0x40) + code.Emit({rexLoad}); + code.Emit({0x8b, 0x04, 0x08}); + const auto rexAlu = static_cast((access.Wide ? 0x48 : 0x40) | ((access.Register >> 3) << 2)); + if (rexAlu != 0x40) + code.Emit({rexAlu}); + code.Emit({access.AluOpcode, static_cast(0xc0 | ((access.Register & 7) << 3))}); + code.Emit({0x58}); + code.Emit({0x59}); } else { - code.Emit({0x58}); + code.Emit({0x51}); + code.Emit({0x50}); + for (const auto byte : access.Address) + code.Emit({byte}); + code.Emit({0x50}); + loadPointer(); code.Emit({0x59}); + const auto rex = static_cast((access.Wide ? 0x48 : 0x40) | ((access.Register >> 3) << 2)); + if (rex != 0x40) + code.Emit({rex}); + code.Emit({0x8b, static_cast(0x04 | ((access.Register & 7) << 3)), 0x08}); + if (access.Register == 0) { + code.Emit({0x48, 0x8d, 0x64, 0x24, 0x08}); + code.Emit({0x59}); + } else if (access.Register == 1) { + code.Emit({0x58}); + code.Emit({0x48, 0x8d, 0x64, 0x24, 0x08}); + } else { + code.Emit({0x58}); + code.Emit({0x59}); + } } } else if (access.AluOpcode != 0 && access.Register == 0) { code.Emit({0x51}); diff --git a/core/relinker/elfpatcher/tests/Amd64OnlyWindowsTests.cpp b/core/relinker/elfpatcher/tests/Amd64OnlyWindowsTests.cpp index 749aa1923..f88d3d93f 100644 --- a/core/relinker/elfpatcher/tests/Amd64OnlyWindowsTests.cpp +++ b/core/relinker/elfpatcher/tests/Amd64OnlyWindowsTests.cpp @@ -557,7 +557,7 @@ void cpuExecution() { for (std::uint8_t src = 0; src < 16; ++src) executeSha256(harness, *matcher, {{opcode, dst, src}}, random, 2); for (int sequence = 0; sequence < 256; ++sequence) { - std::vector steps(2 + random() % 2); + std::vector steps(2 + random() % 2, Sha256Step{}); for (auto& step : steps) step = {static_cast(0xCB + random() % 3), static_cast(random() % 16), static_cast(random() % 16)}; executeSha256(harness, *matcher, steps, random, 2); } @@ -616,6 +616,7 @@ void peBuilder() { std::array directories{}; const auto file = WindowsPeWriter().Write(sections, LoadRva, directories); require(file.size() > 0x400, "PE writer rejected the stub section"); + require(read(file, 0x96) == 0x22, "PE writer marked a fixed image as relocation-stripped"); auto altered = converted.Bytes; altered[0x205] = 0xDC; requireFailure([&] { diff --git a/core/relinker/elfpatcher/tests/EntryStubTests.cpp b/core/relinker/elfpatcher/tests/EntryStubTests.cpp new file mode 100644 index 000000000..cd3737ed0 --- /dev/null +++ b/core/relinker/elfpatcher/tests/EntryStubTests.cpp @@ -0,0 +1,49 @@ +#include +#include +#include +#include +#include +#include +#include + +namespace { + +void require(const bool condition, const std::string& message) { + if (!condition) throw std::runtime_error(message); +} + +bool bytesAt(const std::vector& stub, std::size_t& offset, const std::vector& expected) { + if (offset + expected.size() > stub.size() || std::memcmp(stub.data() + offset, expected.data(), expected.size()) != 0) return false; + offset += expected.size(); + return true; +} + +void stubTerminatesTheFrameChain() { + constexpr std::uint64_t stubVaddr = 0xb32db98; + constexpr std::uint64_t entryVaddr = 0x80; + const auto stub = Elfpatcher::EntryStubBuilder().BuildEntryStub(stubVaddr, entryVaddr); + std::size_t offset = 0; + require(bytesAt(stub, offset, {0x48, 0x89, 0xe7}), "the stub does not pass the initial stack pointer in rdi"); + require(bytesAt(stub, offset, {0x48, 0x83, 0xe4, 0xf0}), "the stub does not align the stack to 16 bytes"); + require(bytesAt(stub, offset, {0x6a, 0x00, 0x6a, 0x00}), "the stub does not push a zero frame record"); + require(bytesAt(stub, offset, {0x48, 0x89, 0xe5}), "the stub does not point rbp at the zero frame record"); + require(bytesAt(stub, offset, {0x48, 0x31, 0xf6}), "the stub does not clear rsi"); + require(offset < stub.size() && stub[offset] == 0xe8, "the stub does not call the entry point"); + std::int32_t rel32 = 0; + std::memcpy(&rel32, stub.data() + offset + 1, sizeof(rel32)); + offset += 5; + require(stubVaddr + offset + static_cast(rel32) == entryVaddr, "the stub calls the wrong address"); + require(bytesAt(stub, offset, {0x0f, 0x0b}) && offset == stub.size(), "the stub does not end with ud2 after the call"); +} + +} + +int main() { + try { + stubTerminatesTheFrameChain(); + } catch (const std::exception& error) { + std::cerr << error.what() << '\n'; + return 1; + } + return 0; +} diff --git a/core/relinker/elfpatcher/tests/GuestModuleArgsHost.cpp b/core/relinker/elfpatcher/tests/GuestModuleArgsHost.cpp new file mode 100644 index 000000000..708d39dc7 --- /dev/null +++ b/core/relinker/elfpatcher/tests/GuestModuleArgsHost.cpp @@ -0,0 +1,56 @@ +#include "prx/libc/include/general/VabiMacros.hpp" +#include +#include +#include + +namespace { + +struct Pending { + std::size_t args = 0; + const void* argp = nullptr; +} pending; + +[[noreturn]] void Fail(const char* message) noexcept { + std::fprintf(stderr, "Guest module argument fixture: %s\n", message); + std::fflush(stderr); + std::_Exit(EXIT_FAILURE); +} + +FILE* Events() { + const auto* path = std::getenv("ANYPS5_GUEST_MODULE_ARGS"); + if (!path) Fail("Missing ANYPS5_GUEST_MODULE_ARGS"); + auto* file = std::fopen(path, "ab"); + if (!file) Fail("Cannot open argument event file"); + return file; +} + +void Write(const char* text) { + auto* file = Events(); + const auto written = std::fputs(text, file); + const auto closed = std::fclose(file); + if (written == EOF || closed != 0) Fail("Cannot write argument event"); +} + +} + +extern "C" void APS5_VABI RecordArgs(std::size_t args, const void* argp) { + char line[80]; + std::snprintf(line, sizeof(line), "args=%016zx argp=%016zx\n", args, + static_cast(reinterpret_cast(argp))); + Write(line); +} + +extern "C" void SetPendingArgs(std::size_t args, const void* argp) { + pending.args = args; + pending.argp = argp; +} + +extern "C" const void* __aps5_get_pending_module_args_nid_no_patch() { + return &pending; +} + +extern "C" void __aps5_set_module_init_result_nid_no_patch(int result) { + char line[40]; + std::snprintf(line, sizeof(line), "result=%d\n", result); + Write(line); +} diff --git a/core/relinker/elfpatcher/tests/WindowsDependencyDiagnosticsTests.cpp b/core/relinker/elfpatcher/tests/WindowsDependencyDiagnosticsTests.cpp index 9866f1f07..a5951091c 100644 --- a/core/relinker/elfpatcher/tests/WindowsDependencyDiagnosticsTests.cpp +++ b/core/relinker/elfpatcher/tests/WindowsDependencyDiagnosticsTests.cpp @@ -5,9 +5,12 @@ #include #include #include +#include +#include #include #include #include +#include #include namespace { @@ -15,6 +18,17 @@ namespace { namespace Fs = std::filesystem; using namespace Elfpatcher::Windows; +Fs::path createFixtureDirectory(const Fs::path& parent) { + std::random_device random; + for (int attempt = 0; attempt < 100; ++attempt) { + const auto directory = parent / ("windows-diagnostic-fixtures-" + + std::to_string(GetCurrentProcessId()) + "-" + std::to_string(random())); + if (Fs::create_directory(directory)) + return directory; + } + throw std::runtime_error("Cannot create a unique diagnostic fixture directory"); +} + void checkRuntimeDependencies() { const auto check = [](const std::vector& input, const std::vector& expected) { Domain::SysVDynamicSection dynamic; @@ -36,12 +50,30 @@ void checkRuntimeDependencies() { {"libSceLibcInternal.prx", "libkernel.prx", "libc.prx"}); check({"libc.prx", "libSceLibcInternal.prx"}, {"libc.prx", "libSceLibcInternal.prx"}); + check({"libSceVideoOut.prx", "libkernel.prx", "libSceVideoOut.prx"}, + {"libSceVideoOut.prx", "libkernel.prx"}); + check({"libSceVideoOut.prx", "libSceVideoOut.prx"}, {"libSceVideoOut.prx"}); } void writeFile(const Fs::path& path, const std::vector& bytes) { + const auto fail = [&](const char* phase) { + const auto error = errno; + const auto nativeError = _doserrno; + throw std::runtime_error("Cannot write diagnostic fixture: phase=" + std::string(phase) + + " path=" + path.string() + " errno=" + std::to_string(error) + + " native_error=" + std::to_string(nativeError) + + " bytes=" + std::to_string(bytes.size())); + }; + errno = 0; + _set_doserrno(0); std::ofstream stream(path, std::ios::binary); + if (!stream.is_open()) + fail("open"); if (!stream.write(reinterpret_cast(bytes.data()), static_cast(bytes.size()))) - throw std::runtime_error("Cannot write diagnostic fixture"); + fail("write"); + stream.close(); + if (!stream) + fail("close"); } void createImage(const Fs::path& path, const std::vector>& imports, const std::string& exported, const std::string& forwarded = {}) { @@ -183,8 +215,7 @@ int main() { const auto size = GetModuleFileNameA(nullptr, filename.data(), static_cast(filename.size())); if (size == 0 || size >= filename.size()) throw std::runtime_error("Cannot locate diagnostic tests"); - const auto directory = Fs::path(filename.data()).parent_path() / "windows-diagnostic-fixtures"; - Fs::create_directories(directory); + const auto directory = createFixtureDirectory(Fs::path(filename.data()).parent_path()); const auto root = directory / "diagnostic-root.prx"; const auto middle = directory / "diagnostic-middle.dll"; const auto leaf = directory / "diagnostic-leaf.dll"; @@ -225,6 +256,7 @@ int main() { expectDiagnostic(runner, {"invalid or unsupported PE dependency metadata"}); writeFile(root, {0, 1, 2}); expectDiagnostic(runner, {"Windows API failed"}); + Fs::remove_all(directory); std::cout << "Windows dependency machine-code tests passed\n"; } catch (const std::exception& error) { std::cerr << error.what() << '\n'; diff --git a/core/relinker/elfpatcher/tests/test_windows_diagnostic_fixture_isolation.py b/core/relinker/elfpatcher/tests/test_windows_diagnostic_fixture_isolation.py new file mode 100644 index 000000000..3d7f58285 --- /dev/null +++ b/core/relinker/elfpatcher/tests/test_windows_diagnostic_fixture_isolation.py @@ -0,0 +1,43 @@ +import concurrent.futures +import ctypes +import mmap +from pathlib import Path +import shutil +import subprocess +import sys +import tempfile + + +def run(executable): + result = subprocess.run([str(executable)], capture_output=True, text=True, timeout=25) + assert result.returncode == 0, (result.returncode, result.stdout, result.stderr) + assert "Windows dependency machine-code tests passed" in result.stdout, result.stdout + + +def main(): + ctypes.windll.kernel32.SetErrorMode(0x0001 | 0x0002 | 0x8000) + original = Path(sys.argv[1]).resolve() + with tempfile.TemporaryDirectory(prefix="anyps5-diagnostic-isolation-") as temporary: + directory = Path(temporary) + executable = directory / original.name + shutil.copy2(original, executable) + legacy = directory / "windows-diagnostic-fixtures" + legacy.mkdir() + fixture = legacy / "diagnostic-runner.exe" + expected = b"previous diagnostic fixture" * 256 + fixture.write_bytes(expected) + with fixture.open("rb") as stream, mmap.mmap(stream.fileno(), 0, access=mmap.ACCESS_READ) as mapping: + run(executable) + with concurrent.futures.ThreadPoolExecutor(max_workers=2) as pool: + futures = [pool.submit(run, executable) for _ in range(2)] + for future in futures: + future.result() + assert mapping[:] == expected, "Mapped previous fixture changed" + assert fixture.read_bytes() == expected, "Previous fixture was overwritten" + assert {path.name for path in legacy.iterdir()} == {fixture.name}, "Previous fixture directory was reused" + assert {path.name for path in directory.iterdir()} == {executable.name, legacy.name}, "Successful diagnostic runs left fixture directories" + print("Windows diagnostic fixture isolation passed") + + +if __name__ == "__main__": + main() diff --git a/core/relinker/io/src/FileReader.cpp b/core/relinker/io/src/FileReader.cpp index 65bff45a7..935676b6d 100644 --- a/core/relinker/io/src/FileReader.cpp +++ b/core/relinker/io/src/FileReader.cpp @@ -1,14 +1,20 @@ #include #include +#include #include namespace Io { std::vector FileReader::Read(const std::string& path) { + std::error_code error; + if (!std::filesystem::is_regular_file(path, error)) + throw Domain::RelinkerException("Cannot open file: " + path); std::ifstream f(path, std::ios::binary | std::ios::ate); if (!f) throw Domain::RelinkerException("Cannot open file: " + path); const std::streamsize size = f.tellg(); + if (size < 0) + throw Domain::RelinkerException("Cannot read file: " + path); f.seekg(0); std::vector buf(static_cast(size)); if (!f.read(reinterpret_cast(buf.data()), size)) diff --git a/core/relinker/main.cpp b/core/relinker/main.cpp index 1b4db2e0d..89564d51b 100644 --- a/core/relinker/main.cpp +++ b/core/relinker/main.cpp @@ -8,6 +8,7 @@ #include #include #include +#include #include #include #include @@ -36,14 +37,21 @@ int main(const int argc, char* argv[]) { return 1; } + if (args.showHelp) { + std::cout << Cli::Usage() << '\n'; + return 0; + } + try { auto extension = std::filesystem::path(args.outputPath).extension().string(); for (auto& character : extension) if (character >= 'A' && character <= 'Z') character = static_cast(character + ('a' - 'A')); - if (!args.toWindows && extension == ".exe") std::cerr << "WARNING: Output filename ends with .exe, but --windows was not specified. The output will be a Linux ELF executable.\n"; + if (!args.toWindows && !args.toMacos && extension == ".exe") std::cerr << "WARNING: Output filename ends with .exe, but --windows was not specified. The output will be a Linux ELF executable.\n"; Io::FileReader fileReader; Io::FileWriter fileWriter; auto sourceBytes = fileReader.Read(args.inputPath); + if (std::filesystem::exists(args.outputPath) && std::filesystem::equivalent(args.inputPath, args.outputPath)) + throw Domain::RelinkerException("Executable output would overwrite the input executable"); const std::string absPath = std::filesystem::absolute(args.outputPath).string(); std::vector trampolines; @@ -79,8 +87,8 @@ int main(const int argc, char* argv[]) { args.unusedFilterLevel ); - std::cout << "System: " << (args.toWindows ? "Windows" : "Linux") << "; unused-filter=" << args.unusedFilterLevel << "\n"; - std::cout << "sce_module/sce_modules/prx processing: " << (args.skipSceModule ? "disabled (--skip-sce-module)" : "enabled") << '\n'; + std::cout << "System: " << (args.toWindows ? "Windows" : args.toMacos ? "macOS" : "Linux") << "; unused-filter=" << args.unusedFilterLevel << "\n"; + std::cout << Relinker::GuestModulePattern << " processing: " << (args.skipSceModule ? "disabled (--skip-sce-module)" : "enabled") << '\n'; for (const auto& name : args.excludedSceModules) std::cout << "Guest module excluded: " << name << '\n'; auto result = pipeline->Relink(sourceBytes); for (const auto& patch : result.Patches) { @@ -91,19 +99,26 @@ int main(const int argc, char* argv[]) { std::vector guestArtifacts; if (!args.skipSceModule) { - guestArtifacts = Relinker::GuestModuleBuilder().Build(args.inputPath, absPath, result.DynamicSection, args.toWindows, args.toIntel, *syscallScanner, args.lazyBinding, args.runPath, args.excludedSceModules); + guestArtifacts = Relinker::GuestModuleBuilder().Build(args.inputPath, absPath, result.DynamicSection, args.toWindows, args.toMacos, args.toIntel, *syscallScanner, args.lazyBinding, args.runPath, args.excludedSceModules, args.sceModulePath); } if (args.writeRegistry) { const std::filesystem::path outFsPath(absPath); const std::string registryPath = (outFsPath.parent_path() / (outFsPath.stem().string() + ".registry.json")).string(); - fileWriter.Write(registryPath, std::make_shared()->WriteCallRegistry(result.RegistryEntries)); + const auto registryWriter = std::make_shared(); + fileWriter.Write(registryPath, registryWriter->WriteCallRegistry(result.RegistryEntries)); + for (const auto& artifact : guestArtifacts) { + const auto modulePath = outFsPath.parent_path() / (outFsPath.stem().string() + "." + artifact.Path.filename().string() + ".registry.json"); + fileWriter.Write(modulePath.string(), registryWriter->WriteModuleImports(artifact.Imports)); + } } auto byteWriter = std::make_shared(); std::shared_ptr patcher; - if (args.toWindows) { + if (args.toMacos) { + patcher = std::make_shared(); + } else if (args.toWindows) { patcher = std::make_shared(args.windowsGui, std::filesystem::path(args.inputPath).parent_path() / "sce_sys" / "icon0.png"); } else { patcher = std::make_shared( diff --git a/core/relinker/relinker/include/relinker/analysis/UnusedNidFilter/IControlFlowGraph.hpp b/core/relinker/relinker/include/relinker/analysis/UnusedNidFilter/IControlFlowGraph.hpp index 421029d69..a4290599b 100644 --- a/core/relinker/relinker/include/relinker/analysis/UnusedNidFilter/IControlFlowGraph.hpp +++ b/core/relinker/relinker/include/relinker/analysis/UnusedNidFilter/IControlFlowGraph.hpp @@ -5,6 +5,7 @@ #include #include #include +#include #include namespace Relinker::UnusedNidFilter { @@ -16,12 +17,32 @@ public: virtual bool IsReachable(VirtualAddress vaddr) const = 0; }; +std::unique_ptr BuildControlFlowGraph( + std::span text, + VirtualAddress textVaddr, + VirtualAddress entryVaddr, + const std::vector& extraEntries, + const IRelativeRelocationIndex& relativeRelocations, + bool followCodeAddresses = false +); + +std::unique_ptr BuildControlFlowGraph( + std::span text, + VirtualAddress textVaddr, + VirtualAddress entryVaddr, + const std::vector& extraEntries, + const IRelativeRelocationIndex& relativeRelocations, + bool followCodeAddresses, + std::unordered_set& reachable +); + std::unique_ptr BuildControlFlowGraph( const std::vector& text, VirtualAddress textVaddr, VirtualAddress entryVaddr, const std::vector& extraEntries, - const IRelativeRelocationIndex& relativeRelocations + const IRelativeRelocationIndex& relativeRelocations, + bool followCodeAddresses = false ); } diff --git a/core/relinker/relinker/include/relinker/analysis/UnusedNidFilter/IEntryPointCollector.hpp b/core/relinker/relinker/include/relinker/analysis/UnusedNidFilter/IEntryPointCollector.hpp index 0241329ee..fc9127d18 100644 --- a/core/relinker/relinker/include/relinker/analysis/UnusedNidFilter/IEntryPointCollector.hpp +++ b/core/relinker/relinker/include/relinker/analysis/UnusedNidFilter/IEntryPointCollector.hpp @@ -2,6 +2,7 @@ #define RELINKER_ANALYSIS_UNUSED_NID_FILTER_IENTRYPOINTCOLLECTOR_HPP #include +#include #include #include @@ -13,7 +14,8 @@ public: virtual std::vector Collect( const std::vector& elfBytes, VirtualAddress textVaddr, - std::size_t textSize + std::size_t textSize, + const IRelativeRelocationIndex& relativeRelocations ) const = 0; }; diff --git a/core/relinker/relinker/include/relinker/domain/ICallRegistryWriter.hpp b/core/relinker/relinker/include/relinker/domain/ICallRegistryWriter.hpp index cc88e831c..d528667e9 100644 --- a/core/relinker/relinker/include/relinker/domain/ICallRegistryWriter.hpp +++ b/core/relinker/relinker/include/relinker/domain/ICallRegistryWriter.hpp @@ -12,6 +12,7 @@ public: virtual ~ICallRegistryWriter() = default; virtual std::string WriteCallRegistry(const std::vector& entries) = 0; + virtual std::string WriteModuleImports(const std::vector& entries) = 0; }; } diff --git a/core/relinker/relinker/include/relinker/guest/GuestImage.hpp b/core/relinker/relinker/include/relinker/guest/GuestImage.hpp index fe34092b5..e80610e72 100644 --- a/core/relinker/relinker/include/relinker/guest/GuestImage.hpp +++ b/core/relinker/relinker/include/relinker/guest/GuestImage.hpp @@ -12,6 +12,7 @@ namespace Relinker { inline constexpr char GuestSymbolSuffix[] = "#guest"; inline constexpr char GuestModuleSuffix[] = ".guest.prx"; +inline constexpr char GuestModulePattern[] = "sce_module/*.prx"; inline constexpr std::uint16_t AbsoluteSection = 0xfff1; struct GuestSymbol { @@ -28,6 +29,7 @@ struct GuestImage { std::filesystem::path SourcePath; std::string OutputName; std::string Soname; + std::vector ModuleNames; std::vector Bytes; std::vector Trampolines; std::vector Headers; @@ -44,17 +46,19 @@ struct GuestImage { class GuestImageReader { public: + std::vector ReadModuleNames(const std::vector& bytes) const; GuestImage Read(const std::filesystem::path& path, std::vector bytes) const; }; struct GuestArtifact { std::filesystem::path Path; std::vector Bytes; + std::vector Imports; }; class GuestModuleBuilder { public: - std::vector Build(const std::filesystem::path& inputPath, const std::filesystem::path& outputPath, Domain::SysVDynamicSection& dynamic, bool windows, bool toIntel, ISyscallScanner& syscallScanner, bool lazyBinding, const std::string& runPath, const std::set& excludedModules) const; + std::vector Build(const std::filesystem::path& inputPath, const std::filesystem::path& outputPath, Domain::SysVDynamicSection& dynamic, bool windows, bool macos, bool toIntel, ISyscallScanner& syscallScanner, bool lazyBinding, const std::string& runPath, const std::set& excludedModules, const std::filesystem::path& sceModulePath) const; }; } diff --git a/core/relinker/relinker/include/relinker/output/CallRegistryWriter.hpp b/core/relinker/relinker/include/relinker/output/CallRegistryWriter.hpp index f6b1c64e7..79b196f86 100644 --- a/core/relinker/relinker/include/relinker/output/CallRegistryWriter.hpp +++ b/core/relinker/relinker/include/relinker/output/CallRegistryWriter.hpp @@ -8,6 +8,7 @@ namespace Relinker { class CallRegistryWriter : public ICallRegistryWriter { public: std::string WriteCallRegistry(const std::vector& entries) override; + std::string WriteModuleImports(const std::vector& entries) override; }; } diff --git a/core/relinker/relinker/src/analysis/CallSiteResolver.cpp b/core/relinker/relinker/src/analysis/CallSiteResolver.cpp index 8628e151c..6f08ab26f 100644 --- a/core/relinker/relinker/src/analysis/CallSiteResolver.cpp +++ b/core/relinker/relinker/src/analysis/CallSiteResolver.cpp @@ -2,6 +2,7 @@ #include #include #include +#include #include #include @@ -74,6 +75,8 @@ private: }; std::vector CallSiteResolver::ResolveCallSites(const std::vector& textSection, const FileByteOffset textSectionVAddr, const VirtualAddress targetGotOrPltAddress, const ByteCount targetGotOrPltSize) { + if (targetGotOrPltSize > std::numeric_limits::max() - targetGotOrPltAddress) + throw RelinkerException("Relocation target range exceeds the address space"); if (textSection.empty()) return {}; if (_cachedTextPtr != &textSection || _cachedTextVAddr != textSectionVAddr) { auto scanner = Codegen::MakeInstructionScanner(); diff --git a/core/relinker/relinker/src/analysis/CodeInstructionCollector.cpp b/core/relinker/relinker/src/analysis/CodeInstructionCollector.cpp index b346b598c..e3a6f2201 100644 --- a/core/relinker/relinker/src/analysis/CodeInstructionCollector.cpp +++ b/core/relinker/relinker/src/analysis/CodeInstructionCollector.cpp @@ -5,10 +5,13 @@ #include #include #include +#include #include #include #include +#include #include +#include namespace Relinker { @@ -80,6 +83,11 @@ std::set CodeInstructionCollector::Collect(const std::ve std::set roots; std::map functions; const auto addRoot = [&](std::uint64_t address) { if (isCode(address)) roots.insert(address); }; + const auto addAddressTakenRoot = [&](std::uint64_t address, const std::uint8_t* code, const Codegen::DecodedInstructionInfo& info) { + if (!info.HasRipRelativeDisp || code[info.OpcodeOffset] != 0x8d || !(info.RexPrefix & 8)) return; + const auto displacement = std::bit_cast(Io::ReadU32(bytes, fileOffset(address, info.Length) + info.RipRelativeDispOffset)); + addRoot(address + info.Length + static_cast(displacement)); + }; const auto addFunction = [&](std::uint64_t begin, std::uint64_t size, bool symbolAlias) { if (size == 0) return; if (!isCode(begin) || size > std::numeric_limits::max() - begin) @@ -201,6 +209,7 @@ std::set CodeInstructionCollector::Collect(const std::ve if (info.Length == 0 || info.Length > end - address) throw Domain::RelinkerException("Code analysis: instruction crosses function boundary", address); instructions.insert(address); + addAddressTakenRoot(address, bytes.data() + offset + address - begin, info); if (info.HasBranchTarget && !info.HasRipRelativeDisp) { const auto target = address + info.Length + static_cast(info.BranchDisp); staticTargets.insert(target); @@ -266,20 +275,25 @@ std::set CodeInstructionCollector::Collect(const std::ve } } std::size_t previousRoots = 0; + std::unordered_set decodedInstructions; + std::vector> sectionReachable(headers.size()); do { previousRoots = roots.size(); - for (const auto& header : headers) { + for (std::size_t i = 0; i < headers.size(); ++i) { + const auto& header = headers[i]; if (header.Type != 1 || (header.Flags & 1) == 0 || header.FileSize == 0) continue; range(header.Offset, header.FileSize); std::vector entries; for (const auto root : roots) if (root >= header.MappedAddress && root - header.MappedAddress < header.FileSize) entries.push_back(root); if (entries.empty()) continue; - const std::vector text(bytes.begin() + header.Offset, bytes.begin() + header.Offset + header.FileSize); - const auto graph = UnusedNidFilter::BuildControlFlowGraph(text, header.MappedAddress, entries.front(), entries, pointers); + const std::span text(bytes.data() + header.Offset, header.FileSize); + const auto graph = UnusedNidFilter::BuildControlFlowGraph(text, header.MappedAddress, entries.front(), entries, pointers, true, sectionReachable[i]); for (const auto address : graph->ReachableVaddrs()) { + if (!decodedInstructions.insert(address).second) continue; instructions.insert(address); const auto offset = address - header.MappedAddress; const auto info = decoder.DecodeInstruction(text.data() + offset, text.size() - offset); + addAddressTakenRoot(address, text.data() + offset, info); if (info.HasBranchTarget && !info.HasRipRelativeDisp) addRoot(static_cast(static_cast(address + info.Length) + info.BranchDisp)); } } diff --git a/core/relinker/relinker/src/analysis/SyscallScanner.cpp b/core/relinker/relinker/src/analysis/SyscallScanner.cpp index f57cede70..8476e026f 100644 --- a/core/relinker/relinker/src/analysis/SyscallScanner.cpp +++ b/core/relinker/relinker/src/analysis/SyscallScanner.cpp @@ -1,5 +1,6 @@ #include #include +#include #include #include @@ -26,17 +27,16 @@ void SyscallScanner::ScanCodeSectionForSyscalls( const std::vector& codeSection, const FileByteOffset codeSectionOffset, const FileByteOffset codeSectionSize) { - const std::size_t limit = std::min(codeSection.size(), static_cast(codeSectionSize)); - const auto scanner = Codegen::MakeInstructionScanner(); const auto matches = scanner->ScanCodeSection(codeSection, codeSectionOffset, codeSectionSize); for (const auto& [Offset, Length] : matches) { const std::size_t i = Offset - codeSectionOffset; + const auto opcodeOffset = Codegen::DecodedInstruction{codeSection.data() + i, Length}.OpcodeOffset(); - if (i + 1 < limit) { - const std::uint8_t b0 = codeSection[i]; - const std::uint8_t b1 = codeSection[i + 1]; + if (opcodeOffset + 1 < Length) { + const std::uint8_t b0 = codeSection[i + opcodeOffset]; + const std::uint8_t b1 = codeSection[i + opcodeOffset + 1]; const bool isSyscall = (b0 == SYSCALL_BYTE0 && b1 == SYSCALL_BYTE1); const bool isInt80 = (b0 == INT80_BYTE0 && b1 == INT80_BYTE1); diff --git a/core/relinker/relinker/src/analysis/UnusedNidFilter.cpp b/core/relinker/relinker/src/analysis/UnusedNidFilter.cpp index e6b796348..e3396f536 100644 --- a/core/relinker/relinker/src/analysis/UnusedNidFilter.cpp +++ b/core/relinker/relinker/src/analysis/UnusedNidFilter.cpp @@ -16,15 +16,15 @@ public: ) override { if (textSection.empty()) throw RelinkerException("Cannot filter NIDs: text section is empty"); + auto relativeRelocations = UnusedNidFilter::BuildRelativeRelocationIndex(elfBytes); auto collector = UnusedNidFilter::MakeEntryPointCollector(); - auto entries = collector->Collect(elfBytes, textVAddr, textSection.size()); + auto entries = collector->Collect(elfBytes, textVAddr, textSection.size(), *relativeRelocations); if (entries.empty()) throw RelinkerException("Cannot filter NIDs: no entry points found"); VirtualAddress primary = entries[0]; std::vector extra(entries.begin() + 1, entries.end()); - auto relativeRelocations = UnusedNidFilter::BuildRelativeRelocationIndex(elfBytes); auto cfg = UnusedNidFilter::BuildControlFlowGraph(textSection, textVAddr, primary, extra, *relativeRelocations); auto index = UnusedNidFilter::BuildGotAccessIndex(*cfg, textSection, textVAddr); diff --git a/core/relinker/relinker/src/analysis/UnusedNidFilter/ControlFlowGraph.cpp b/core/relinker/relinker/src/analysis/UnusedNidFilter/ControlFlowGraph.cpp index bb98c8277..203ae2b08 100644 --- a/core/relinker/relinker/src/analysis/UnusedNidFilter/ControlFlowGraph.cpp +++ b/core/relinker/relinker/src/analysis/UnusedNidFilter/ControlFlowGraph.cpp @@ -9,24 +9,33 @@ namespace Relinker::UnusedNidFilter { class ControlFlowGraph : public IControlFlowGraph { public: explicit ControlFlowGraph(std::unordered_set reachable) - : _reachable(std::move(reachable)) {} + : _owned(std::move(reachable)), _borrowed(nullptr) {} - const std::unordered_set& ReachableVaddrs() const override { return _reachable; } - bool IsReachable(VirtualAddress vaddr) const override { return _reachable.count(vaddr) > 0; } + explicit ControlFlowGraph(const std::unordered_set* reachable) + : _borrowed(reachable) {} + + const std::unordered_set& ReachableVaddrs() const override { + return _borrowed ? *_borrowed : _owned; + } + bool IsReachable(VirtualAddress vaddr) const override { + return (_borrowed ? *_borrowed : _owned).count(vaddr) > 0; + } private: - std::unordered_set _reachable; + std::unordered_set _owned; + const std::unordered_set* _borrowed = nullptr; }; std::unique_ptr BuildControlFlowGraph( - const std::vector& text, + std::span text, VirtualAddress textVaddr, VirtualAddress entryVaddr, const std::vector& extraEntries, - const IRelativeRelocationIndex& relativeRelocations + const IRelativeRelocationIndex& relativeRelocations, + bool followCodeAddresses, + std::unordered_set& reachable ) { const Codegen::X64InstructionDecoder decoder; - std::unordered_set reachable; std::queue worklist; auto enqueue = [&](VirtualAddress va) { @@ -54,6 +63,12 @@ std::unique_ptr BuildControlFlowGraph( VirtualAddress nextVaddr = va + static_cast(info.Length); + if (followCodeAddresses && info.HasRipRelativeDisp && text[bufOff + info.OpcodeOffset] == 0x8d && (info.RexPrefix & 8)) { + std::int32_t displacement = 0; + std::memcpy(&displacement, text.data() + bufOff + info.RipRelativeDispOffset, 4); + enqueue(static_cast(static_cast(nextVaddr) + displacement)); + } + switch (info.FlowKind) { using enum Codegen::ControlFlowKind; case Sequential: @@ -103,7 +118,31 @@ std::unique_ptr BuildControlFlowGraph( } } + return std::make_unique(&reachable); +} + +std::unique_ptr BuildControlFlowGraph( + std::span text, + VirtualAddress textVaddr, + VirtualAddress entryVaddr, + const std::vector& extraEntries, + const IRelativeRelocationIndex& relativeRelocations, + bool followCodeAddresses +) { + std::unordered_set reachable; + BuildControlFlowGraph(text, textVaddr, entryVaddr, extraEntries, relativeRelocations, followCodeAddresses, reachable); return std::make_unique(std::move(reachable)); } +std::unique_ptr BuildControlFlowGraph( + const std::vector& text, + VirtualAddress textVaddr, + VirtualAddress entryVaddr, + const std::vector& extraEntries, + const IRelativeRelocationIndex& relativeRelocations, + bool followCodeAddresses +) { + return BuildControlFlowGraph(std::span(text.data(), text.size()), textVaddr, entryVaddr, extraEntries, relativeRelocations, followCodeAddresses); +} + } diff --git a/core/relinker/relinker/src/analysis/UnusedNidFilter/EhFrameReader.cpp b/core/relinker/relinker/src/analysis/UnusedNidFilter/EhFrameReader.cpp index 50cbf3ee2..84d8b64d2 100644 --- a/core/relinker/relinker/src/analysis/UnusedNidFilter/EhFrameReader.cpp +++ b/core/relinker/relinker/src/analysis/UnusedNidFilter/EhFrameReader.cpp @@ -21,6 +21,7 @@ public: std::vector result; for (const auto& segment : headers) { if (segment.Type != 0x6474E550 || segment.FileSize == 0) continue; + if (segment.FileSize < 4) throw RelinkerException("Strict filter: EH frame header exceeds segment", segment.MappedAddress); auto position = segment.MappedAddress; const auto version = read(position); const auto frameEncoding = read(position); @@ -30,6 +31,8 @@ public: throw RelinkerException("Strict filter: unsupported EH frame header", segment.MappedAddress); encoded(position, frameEncoding, segment.MappedAddress); const auto count = encoded(position, countEncoding, segment.MappedAddress); + if (position < segment.MappedAddress || position - segment.MappedAddress > segment.FileSize) + throw RelinkerException("Strict filter: EH frame header exceeds segment", position); if (count > segment.FileSize / 2) throw RelinkerException("Strict filter: invalid EH frame count", position); for (std::uint64_t index = 0; index < count; ++index) { const auto begin = encoded(position, tableEncoding, segment.MappedAddress); @@ -67,7 +70,7 @@ private: unsigned shift = 0; for (;;) { const auto byte = read(position); - if (shift == 63 && ((byte & 0x7F) != 0 && (byte & 0x7F) != 1 && (!signedValue || (byte & 0x7F) != 0x7F))) + if (shift == 63 && (byte & 0x7F) != 0 && (byte & 0x7F) != (signedValue ? 0x7F : 1)) throw RelinkerException("Strict filter: overflowing LEB128 value", position); value |= static_cast(byte & 0x7F) << shift; shift += 7; @@ -180,7 +183,7 @@ private: if (augmentationEnd > end) throw RelinkerException("Strict filter: FDE augmentation exceeds record", address); if (cie.LsdaEncoding != 0xFF) { const auto lsda = encoded(position, cie.LsdaEncoding, 0, begin); - if (lsda != 0) readLsda(lsda, region); + if (lsda != 0 && cie.Personality != 0) readLsda(lsda, region); } if (position > augmentationEnd) throw RelinkerException("Strict filter: FDE augmentation size mismatch", address); } diff --git a/core/relinker/relinker/src/analysis/UnusedNidFilter/EntryPointCollector.cpp b/core/relinker/relinker/src/analysis/UnusedNidFilter/EntryPointCollector.cpp index d24bcaf3d..8e87a10d8 100644 --- a/core/relinker/relinker/src/analysis/UnusedNidFilter/EntryPointCollector.cpp +++ b/core/relinker/relinker/src/analysis/UnusedNidFilter/EntryPointCollector.cpp @@ -1,6 +1,7 @@ #include #include #include +#include namespace Relinker::UnusedNidFilter { @@ -26,6 +27,16 @@ std::uint64_t read64(const std::vector& b, std::size_t off) { return v; } +bool fitsAt(std::uint64_t base, std::uint64_t offset, std::uint64_t length, std::size_t size) { + return base <= size && offset <= size - base && length <= size - base - offset; +} + +struct LoadSegment { + VirtualAddress vaddr; + std::uint64_t fileOffset; + std::uint64_t fileSize; +}; + bool inText(VirtualAddress va, VirtualAddress textVaddr, std::size_t textSize) { return va >= textVaddr && va < textVaddr + static_cast(textSize); } @@ -37,7 +48,8 @@ public: std::vector Collect( const std::vector& elfBytes, VirtualAddress textVaddr, - std::size_t textSize + std::size_t textSize, + const IRelativeRelocationIndex& relativeRelocations ) const override { if (elfBytes.size() < 64) throw RelinkerException("ELF too small for header"); if (elfBytes[0] != 0x7f || elfBytes[1] != 'E' || @@ -62,9 +74,18 @@ public: if (phEntSize < 56) throw RelinkerException("ELF program header entry too small"); + std::vector loads; for (std::uint16_t i = 0; i < phCount; ++i) { std::size_t phPos = static_cast(phOff) + i * phEntSize; if (phPos + 56 > elfBytes.size()) throw RelinkerException("Program header out of bounds"); + if (read32(elfBytes, phPos) != PT_LOAD) continue; + loads.push_back({read64(elfBytes, phPos + 16), read64(elfBytes, phPos + 8), read64(elfBytes, phPos + 32)}); + } + + for (std::uint16_t i = 0; i < phCount; ++i) { + const std::uint64_t phIndexOff = static_cast(i) * phEntSize; + if (!fitsAt(phOff, phIndexOff, 56, elfBytes.size())) throw RelinkerException("Program header out of bounds"); + std::size_t phPos = static_cast(phOff + phIndexOff); std::uint32_t type = read32(elfBytes, phPos); @@ -73,12 +94,14 @@ public: std::uint64_t segOff = read64(elfBytes, phPos + 8); std::uint64_t segSz = read64(elfBytes, phPos + 32); - if (segOff + segSz > elfBytes.size()) throw RelinkerException("PT_DYNAMIC segment out of bounds"); + if (!fitsAt(segOff, 0, segSz, elfBytes.size())) throw RelinkerException("PT_DYNAMIC segment out of bounds"); VirtualAddress initArrayVa = 0; std::uint64_t initArraySz = 0; VirtualAddress finiArrayVa = 0; std::uint64_t finiArraySz = 0; + VirtualAddress preinitArrayVa = 0; + std::uint64_t preinitArraySz = 0; for (std::uint64_t off = 0; off + 16 <= segSz; off += 16) { std::size_t pos = static_cast(segOff + off); @@ -92,24 +115,37 @@ public: if (tag == DT_INIT_ARRAYSZ || tag == DT_OS_INIT_ARRAYSZ) initArraySz = val; if (tag == DT_FINI_ARRAY || tag == DT_OS_FINI_ARRAY) finiArrayVa = val; if (tag == DT_FINI_ARRAYSZ || tag == DT_OS_FINI_ARRAYSZ) finiArraySz = val; + if (tag == DT_PREINIT_ARRAY || tag == DT_OS_PREINIT_ARRAY) preinitArrayVa = val; + if (tag == DT_PREINIT_ARRAYSZ || tag == DT_OS_PREINIT_ARRAYSZ) preinitArraySz = val; } auto collectArray = [&](VirtualAddress arrayVa, std::uint64_t arraySz) { if (arrayVa == 0 || arraySz == 0) return; - std::uint64_t phFileOff = read64(elfBytes, phPos + 8); - std::uint64_t phVaddr = read64(elfBytes, phPos + 16); - if (arrayVa < phVaddr) return; - std::uint64_t arrayFileOff = phFileOff + (arrayVa - phVaddr); - std::uint64_t count = arraySz / 8; - for (std::uint64_t k = 0; k < count; ++k) { - std::size_t entPos = static_cast(arrayFileOff + k * 8); - if (entPos + 8 > elfBytes.size()) break; - addIfInText(read64(elfBytes, entPos)); + if (arraySz % 8 != 0) throw RelinkerException("Init/fini array size is not a multiple of 8", arrayVa); + const LoadSegment* container = nullptr; + for (const auto& load : loads) { + if (arrayVa >= load.vaddr && arrayVa - load.vaddr <= load.fileSize && + arraySz <= load.fileSize - (arrayVa - load.vaddr)) { + container = &load; + break; + } + } + if (!container) throw RelinkerException("Init/fini array is not file-backed", arrayVa); + std::uint64_t arrayFileOff = container->fileOffset + (arrayVa - container->vaddr); + if (arrayFileOff > elfBytes.size() || arraySz > elfBytes.size() - arrayFileOff) + throw RelinkerException("Init/fini array is out of bounds", arrayVa); + for (std::uint64_t k = 0; k < arraySz / 8; ++k) { + VirtualAddress slot = arrayVa + k * 8; + auto relocated = relativeRelocations.TargetOfSlot(slot); + std::uint64_t target = relocated ? *relocated : read64(elfBytes, static_cast(arrayFileOff + k * 8)); + if (target == 0 || target == std::numeric_limits::max()) continue; + addIfInText(target); } }; collectArray(initArrayVa, initArraySz); collectArray(finiArrayVa, finiArraySz); + collectArray(preinitArrayVa, preinitArraySz); break; } @@ -120,14 +156,16 @@ public: std::uint16_t shStrIdx = read16(elfBytes, 62); if (shEntSize >= 64 && shOff != 0 && shCount > 0 && shStrIdx < shCount) { - std::size_t shStrPos = static_cast(shOff) + shStrIdx * shEntSize; - if (shStrPos + 64 <= elfBytes.size()) { + const std::uint64_t shStrIndexOff = static_cast(shStrIdx) * shEntSize; + if (fitsAt(shOff, shStrIndexOff, 64, elfBytes.size())) { + std::size_t shStrPos = static_cast(shOff + shStrIndexOff); std::uint64_t strTabOff = read64(elfBytes, shStrPos + 24); std::uint64_t strTabSz = read64(elfBytes, shStrPos + 32); for (std::uint16_t si = 0; si < shCount; ++si) { - std::size_t shPos = static_cast(shOff) + si * shEntSize; - if (shPos + 64 > elfBytes.size()) break; + const std::uint64_t shIndexOff = static_cast(si) * shEntSize; + if (!fitsAt(shOff, shIndexOff, 64, elfBytes.size())) break; + std::size_t shPos = static_cast(shOff + shIndexOff); std::uint32_t shType = read32(elfBytes, shPos + 4); if (shType != SHT_DYNSYM && shType != SHT_SYMTAB) continue; @@ -140,9 +178,9 @@ public: std::uint64_t strOff = 0; if (symLink < shCount) { - std::size_t strShPos = static_cast(shOff) + symLink * shEntSize; - if (strShPos + 64 <= elfBytes.size()) - strOff = read64(elfBytes, strShPos + 24); + const std::uint64_t strShIndexOff = static_cast(symLink) * shEntSize; + if (fitsAt(shOff, strShIndexOff, 64, elfBytes.size())) + strOff = read64(elfBytes, static_cast(shOff + strShIndexOff) + 24); } else { strOff = strTabOff; (void)strTabSz; @@ -151,8 +189,8 @@ public: std::uint64_t symCount = symSz / entSz; for (std::uint64_t k = 0; k < symCount; ++k) { + if (!fitsAt(symOff, k * entSz, 24, elfBytes.size())) break; std::size_t sPos = static_cast(symOff + k * entSz); - if (sPos + 24 > elfBytes.size()) break; std::uint8_t info = elfBytes[sPos + 4]; std::uint8_t stBind = info >> 4; std::uint8_t stType = info & 0xF; diff --git a/core/relinker/relinker/src/analysis/UnusedNidFilter/PltCompactor.cpp b/core/relinker/relinker/src/analysis/UnusedNidFilter/PltCompactor.cpp index 832de8d4f..c21acd03b 100644 --- a/core/relinker/relinker/src/analysis/UnusedNidFilter/PltCompactor.cpp +++ b/core/relinker/relinker/src/analysis/UnusedNidFilter/PltCompactor.cpp @@ -1,15 +1,15 @@ #include #include #include -#include -#include +#include +#include namespace Relinker::UnusedNidFilter { CompactedPlt CompactPlt(const std::vector& originalReferences, const std::vector& keptReferences, const std::vector& text, VirtualAddress textVaddr, FileByteOffset textOffset, FileByteOffset tableOffset) { CompactedPlt result; result.References = keptReferences; - std::map originalSlots; + std::unordered_map originalSlots; for (const auto& reference : originalReferences) { if (reference.RelocationTypeValue != 7) continue; if (reference.RelocationTableOffset < tableOffset || (reference.RelocationTableOffset - tableOffset) % 24 != 0 || (reference.RelocationTableOffset - tableOffset) / 24 > std::numeric_limits::max()) @@ -17,22 +17,21 @@ CompactedPlt CompactPlt(const std::vector& originalReferences, con if (!originalSlots.emplace(reference.RelocationAddress, static_cast((reference.RelocationTableOffset - tableOffset) / 24)).second) throw RelinkerException("Strict filter: duplicate original PLT slot", reference.RelocationAddress); } - std::map newSlots; + std::unordered_map newSlots; for (auto& reference : result.References) { if (reference.RelocationTypeValue != 7) continue; if (!originalSlots.contains(reference.RelocationAddress) || !newSlots.emplace(reference.RelocationAddress, result.SlotCount).second) throw RelinkerException("Strict filter: invalid retained PLT slot", reference.RelocationAddress); reference.RelocationTableOffset = tableOffset + static_cast(result.SlotCount++) * 24; } - std::set found; + std::unordered_set found; for (std::size_t offset = 0; offset + 16 <= text.size(); ++offset) { - if (text[offset] != 0xFF || text[offset + 1] != 0x25) continue; + if (text[offset] != 0xFF || text[offset + 1] != 0x25 || text[offset + 6] != 0x68 || text[offset + 11] != 0xE9) continue; std::int32_t displacement; std::memcpy(&displacement, text.data() + offset + 2, sizeof(displacement)); const auto slot = textVaddr + offset + 6 + static_cast(static_cast(displacement)); const auto original = originalSlots.find(slot); if (original == originalSlots.end()) continue; - if (text[offset + 6] != 0x68 || text[offset + 11] != 0xE9) continue; std::uint32_t index; std::memcpy(&index, text.data() + offset + 7, sizeof(index)); if (index != original->second) throw RelinkerException("Strict filter: PLT thunk index disagrees with its relocation", textVaddr + offset); diff --git a/core/relinker/relinker/src/analysis/UnusedNidFilter/RelativeRelocationIndex.cpp b/core/relinker/relinker/src/analysis/UnusedNidFilter/RelativeRelocationIndex.cpp index fd508b987..4e050df8c 100644 --- a/core/relinker/relinker/src/analysis/UnusedNidFilter/RelativeRelocationIndex.cpp +++ b/core/relinker/relinker/src/analysis/UnusedNidFilter/RelativeRelocationIndex.cpp @@ -2,6 +2,7 @@ #include #include #include +#include namespace Relinker::UnusedNidFilter { @@ -33,10 +34,22 @@ struct LoadSegment { std::uint64_t fileSize; }; -std::uint64_t vaddrToFileOffset(const std::vector& loads, VirtualAddress va) { +std::uint64_t vaddrToFileOffset( + const std::vector& loads, + VirtualAddress va, + std::uint64_t size +) { for (const auto& seg : loads) { - if (va >= seg.vaddr && va < seg.vaddr + seg.fileSize) - return seg.fileOffset + (va - seg.vaddr); + if (va >= seg.vaddr) { + std::uint64_t relative = va - seg.vaddr; + if (relative < seg.fileSize) { + if (size > seg.fileSize - relative) + throw RelinkerException("Relocation table out of bounds", va); + if (relative > std::numeric_limits::max() - seg.fileOffset) + throw RelinkerException("Relocation table file offset overflow", va); + return seg.fileOffset + relative; + } + } } throw RelinkerException("Cannot translate virtual address to file offset", va); } @@ -50,12 +63,19 @@ void collectRelativeEntries( std::uint64_t tableSize, std::unordered_map& out ) { - if (tableVaddr == 0 || tableSize == 0) return; - std::uint64_t tableFileOff = vaddrToFileOffset(loads, tableVaddr); + if (tableSize == 0) return; + if (tableSize % RelaEntSize != 0) + throw RelinkerException("Invalid relocation table size", tableSize); - for (std::uint64_t off = 0; off + RelaEntSize <= tableSize; off += RelaEntSize) { + std::uint64_t tableFileOff = vaddrToFileOffset(loads, tableVaddr, tableSize); + if (tableFileOff > elfBytes.size()) + throw RelinkerException("Relocation table out of bounds", tableFileOff); + if (tableSize > elfBytes.size() - tableFileOff) + throw RelinkerException("Relocation table out of bounds", tableFileOff); + + for (std::uint64_t off = 0; off < tableSize; off += RelaEntSize) { std::size_t pos = static_cast(tableFileOff + off); - if (pos + RelaEntSize > elfBytes.size()) + if (pos > elfBytes.size() || RelaEntSize > elfBytes.size() - pos) throw RelinkerException("Relocation entry out of bounds", pos); std::uint64_t rOffset = read64(elfBytes, pos); @@ -126,14 +146,14 @@ std::unique_ptr BuildRelativeRelocationIndex( std::uint64_t segOff = read64(elfBytes, phPos + 8); std::uint64_t segSz = read64(elfBytes, phPos + 32); - if (segOff + segSz > elfBytes.size()) throw RelinkerException("PT_DYNAMIC segment out of bounds"); + if (segOff > elfBytes.size() || segSz > elfBytes.size() - segOff) throw RelinkerException("PT_DYNAMIC segment out of bounds"); VirtualAddress relaVa = 0; std::uint64_t relaSz = 0; VirtualAddress jmprelVa = 0; std::uint64_t pltrelsz = 0; - for (std::uint64_t off = 0; off + 16 <= segSz; off += 16) { + for (std::uint64_t off = 0; off <= segSz && segSz - off >= 16; off += 16) { std::size_t pos = static_cast(segOff + off); std::int64_t tag = static_cast(read64(elfBytes, pos)); std::uint64_t val = read64(elfBytes, pos + 8); diff --git a/core/relinker/relinker/src/analysis/UnusedNidFilter/StrictReachability.cpp b/core/relinker/relinker/src/analysis/UnusedNidFilter/StrictReachability.cpp index a7d7cd658..60dcf67b8 100644 --- a/core/relinker/relinker/src/analysis/UnusedNidFilter/StrictReachability.cpp +++ b/core/relinker/relinker/src/analysis/UnusedNidFilter/StrictReachability.cpp @@ -102,16 +102,17 @@ private: return {address, info}; } - bool isZeroPadding(VirtualAddress begin, VirtualAddress end) const { - const auto first = input.Text.begin() + static_cast(begin - input.TextVaddr); - const auto last = input.Text.begin() + static_cast(end - input.TextVaddr); - return std::all_of(first, last, [](std::uint8_t byte) { return byte == 0; }); + VirtualAddress trailingZeroStart(VirtualAddress begin, VirtualAddress end) const { + auto pos = input.Text.data() + static_cast(end - input.TextVaddr); + const auto first = input.Text.data() + static_cast(begin - input.TextVaddr); + while (pos != first && *(pos - 1) == 0) --pos; + return input.TextVaddr + static_cast(pos - input.Text.data()); } void addGap(VirtualAddress begin, VirtualAddress end) { + const auto effectiveEnd = trailingZeroStart(begin, end); VirtualAddress regionBegin = begin; - for (auto address = begin; address < end;) { - if (isZeroPadding(address, end)) break; + for (auto address = begin; address < effectiveEnd;) { const auto instruction = decode(address, end); address += instruction.Info.Length; if (endsFlow(instruction.Info.FlowKind)) { @@ -145,12 +146,10 @@ private: void buildEdges() { for (auto& [begin, region] : regions) { + const auto effectiveEnd = trailingZeroStart(begin, region.End); Codegen::ControlFlowKind lastFlow = Codegen::ControlFlowKind::Sequential; - for (auto address = begin; address < region.End;) { - if (isZeroPadding(address, region.End)) { - lastFlow = Codegen::ControlFlowKind::Sequential; - break; - } + auto address = begin; + while (address < effectiveEnd) { const auto instruction = decode(address, region.End); const auto& info = instruction.Info; const auto next = address + info.Length; @@ -185,6 +184,7 @@ private: lastFlow = info.FlowKind; address = next; } + if (address < region.End) lastFlow = Codegen::ControlFlowKind::Sequential; if (!endsFlow(lastFlow) && isCode(region.End)) region.Edges.insert(region.End); } } diff --git a/core/relinker/relinker/src/guest/GuestImageReader.cpp b/core/relinker/relinker/src/guest/GuestImageReader.cpp index 56551a48d..22dfe0274 100644 --- a/core/relinker/relinker/src/guest/GuestImageReader.cpp +++ b/core/relinker/relinker/src/guest/GuestImageReader.cpp @@ -11,6 +11,82 @@ namespace Relinker { +std::vector GuestImageReader::ReadModuleNames(const std::vector& bytes) const { + if (bytes.size() < 64 || Io::ReadU32(bytes, 0) != 0x464c457f || bytes[4] != 2 || bytes[5] != 1 || bytes[6] != 1 || Io::ReadU16(bytes, 18) != 62) + throw Domain::RelinkerException("Expected little-endian ELF64 x86-64 module metadata"); + const auto type = Io::ReadU16(bytes, 16); + if (type != 3 && type != 0xfe18) return {}; + if (Io::ReadU16(bytes, 52) != 64 || Io::ReadU16(bytes, 54) != 56 || Io::ReadU16(bytes, 56) == 0) + throw Domain::RelinkerException("Invalid module metadata program headers"); + const auto range = [&](std::uint64_t offset, std::uint64_t size) { + if (offset > bytes.size() || size > bytes.size() - offset) throw Domain::RelinkerException("Module metadata exceeds file"); + }; + range(Io::ReadU64(bytes, 32), static_cast(Io::ReadU16(bytes, 56)) * 56); + ElfReader reader(bytes); + const auto headers = reader.ReadProgramHeaders(); + const Domain::ProgramHeader* dynamic = nullptr; + const Domain::ProgramHeader* dynlib = nullptr; + for (const auto& header : headers) { + if (header.Type == 2) { + if (dynamic) throw Domain::RelinkerException("Duplicate module metadata dynamic segment"); + dynamic = &header; + } + if (header.Type == 0x61000000) { + if (dynlib) throw Domain::RelinkerException("Duplicate module metadata dynamic data"); + dynlib = &header; + } + } + if (!dynamic) throw Domain::RelinkerException("Missing module metadata dynamic segment"); + std::map strings; + std::vector modules; + for (const auto& tag : reader.ReadDynamicTags(*dynamic)) { + if (tag.Tag == 0x6100000d || tag.Tag == 0x61000043) modules.push_back(tag.Value); + else if (tag.Tag == 5 || tag.Tag == 10 || tag.Tag == 0x61000035 || tag.Tag == 0x61000037) { + if (!strings.emplace(tag.Tag, tag.Value).second) throw Domain::RelinkerException("Duplicate module metadata string tag"); + } + } + if (modules.empty()) return {}; + const auto get = [&](std::uint64_t standard, std::uint64_t sce) { + if (strings.contains(standard) == strings.contains(sce)) throw Domain::RelinkerException("Missing or ambiguous module metadata string tag"); + return strings.at(strings.contains(standard) ? standard : sce); + }; + const auto size = get(10, 0x61000037); + const auto address = get(5, 0x61000035); + std::uint64_t offset = 0; + if (strings.contains(5)) { + bool found = false; + for (const auto& header : headers) { + if (header.Type != 1 || address < header.MappedAddress || address - header.MappedAddress > header.FileSize || size > header.FileSize - (address - header.MappedAddress)) continue; + range(header.Offset, header.FileSize); + if (found) throw Domain::RelinkerException("Ambiguous module metadata string mapping"); + offset = header.Offset + address - header.MappedAddress; + found = true; + } + if (!found) throw Domain::RelinkerException("Unmapped module metadata strings"); + } else { + if (!dynlib || address > dynlib->FileSize || size > dynlib->FileSize - address) throw Domain::RelinkerException("Module metadata strings exceed dynamic data"); + range(dynlib->Offset, dynlib->FileSize); + offset = dynlib->Offset + address; + } + range(offset, size); + if (size == 0 || bytes[offset] != 0) throw Domain::RelinkerException("Invalid module metadata strings"); + std::vector names; + std::set ids; + for (const auto value : modules) { + const auto nameOffset = value & 0xffffffffu; + if (nameOffset >= size) throw Domain::RelinkerException("Module metadata string offset out of bounds"); + const auto start = bytes.begin() + static_cast(offset + nameOffset); + const auto end = bytes.begin() + static_cast(offset + size); + const auto zero = std::find(start, end, 0); + if (zero == end) throw Domain::RelinkerException("Unterminated module metadata string"); + const std::string name(start, zero); + if (name.empty() || name.find_first_of("/\\:$\r\n") != std::string::npos) throw Domain::RelinkerException("Invalid module metadata name"); + if (!ids.insert(value >> 48).second) throw Domain::RelinkerException("Duplicate module metadata export ID"); + names.push_back(name); + } + return names; +} + GuestImage GuestImageReader::Read(const std::filesystem::path& path, std::vector bytes) const { const auto fail = [&](const std::string& message) { throw Domain::RelinkerException(path.string() + ": " + message); }; const auto range = [&](std::uint64_t offset, std::uint64_t size) { @@ -63,6 +139,7 @@ GuestImage GuestImageReader::Read(const std::filesystem::path& path, std::vector std::map tags; std::vector needed; std::vector moduleImports; + std::vector moduleExports; bool terminated = false; for (std::uint64_t offset = dynamic->Offset; offset < dynamic->Offset + dynamic->FileSize; offset += 16) { const auto tag = Io::ReadU64(bytes, offset); @@ -70,6 +147,7 @@ GuestImage GuestImageReader::Read(const std::filesystem::path& path, std::vector if (tag == 0) { terminated = true; break; } if (tag == 1) needed.push_back(value); else if (tag == 0x61000045) moduleImports.push_back(value); + else if (tag == 0x6100000d || tag == 0x61000043) moduleExports.push_back(value); else if (tag < 0x60000000 || tag == 0x6100003f || (tag >= 0x61000027 && tag <= 0x6100003b)) { if (!tags.emplace(tag, value).second) fail("Duplicate dynamic tag " + std::to_string(tag)); } @@ -159,11 +237,18 @@ GuestImage GuestImageReader::Read(const std::filesystem::path& path, std::vector image.Soname = string(tags.at(14)); if (image.Soname.empty() || image.Soname.find_first_of("\\:$\r\n") != std::string::npos) fail("Invalid guest SONAME: " + image.Soname); } + std::set moduleIds; + for (const auto value : moduleExports) { + const auto name = string(value & 0xffffffffu); + if (name.empty() || name.find_first_of("/\\:$\r\n") != std::string::npos) fail("Invalid guest module name: " + name); + if (!moduleIds.insert(value >> 48).second) fail("Duplicate export module ID"); + image.ModuleNames.push_back(name); + } std::set dependencies; for (const auto offset : needed) { const auto name = string(offset); - if (name.empty() || name.find_first_of("\\:$\r\n") != std::string::npos || !dependencies.insert(name).second) fail("Invalid or duplicate dependency: " + name); - image.Dependencies.push_back(name); + if (name.empty() || name.find_first_of("\\:$\r\n") != std::string::npos) fail("Invalid dependency: " + name); + if (dependencies.insert(name).second) image.Dependencies.push_back(name); } std::map relocationTargets; const auto copyRelocations = [&](std::uint64_t addressTag, std::uint64_t sceAddressTag, std::uint64_t sizeTag, std::uint64_t sceSizeTag, std::vector& output, bool plt) { diff --git a/core/relinker/relinker/src/guest/GuestModuleBuilder.cpp b/core/relinker/relinker/src/guest/GuestModuleBuilder.cpp index b2cd06d9a..42dbc7403 100644 --- a/core/relinker/relinker/src/guest/GuestModuleBuilder.cpp +++ b/core/relinker/relinker/src/guest/GuestModuleBuilder.cpp @@ -8,7 +8,12 @@ #include #include #include +#include #include +#include +#include +#include +#include namespace Relinker { @@ -29,18 +34,34 @@ std::vector ReadNeededNames(const Domain::SysVDynamicSection& dynam return names; } +std::string FoldFilename(std::string name) { + for (auto& character : name) if (character >= 'A' && character <= 'Z') character = static_cast(character + ('a' - 'A')); + return name; } -std::vector GuestModuleBuilder::Build(const std::filesystem::path& inputPath, const std::filesystem::path& outputPath, Domain::SysVDynamicSection& dynamic, const bool windows, const bool toIntel, ISyscallScanner& syscallScanner, const bool lazyBinding, const std::string& runPath, const std::set& excludedModules) const { - const auto root = std::filesystem::absolute(inputPath).parent_path(); +std::string ModuleStem(std::string name, const bool windows) { + if (windows) name = FoldFilename(std::move(name)); + for (const std::string_view suffix : {".debug_prx", ".sprx", ".prx"}) { + if (name.size() > suffix.size() && name.ends_with(suffix)) return name.substr(0, name.size() - suffix.size()); + } + return {}; +} + +} + +std::vector GuestModuleBuilder::Build(const std::filesystem::path& inputPath, const std::filesystem::path& outputPath, Domain::SysVDynamicSection& dynamic, const bool windows, const bool macos, const bool toIntel, ISyscallScanner& syscallScanner, const bool lazyBinding, const std::string& runPath, const std::set& excludedModules, const std::filesystem::path& sceModulePath) const { + const auto root = std::filesystem::absolute(sceModulePath).lexically_normal(); + if (!std::filesystem::exists(root)) throw Domain::RelinkerException("Guest module parent directory does not exist: " + root.string()); + if (!std::filesystem::is_directory(root)) throw Domain::RelinkerException("Guest module parent path is not a directory: " + root.string()); + const auto missingModuleMessage = std::string(GuestModulePattern) + " was not found in: " + root.string() + ". Use --skip-sce-module to disable guest module processing."; const auto singular = root / "sce_module"; const auto plural = root / "sce_modules"; const auto prx = root / "prx"; const bool hasSingular = std::filesystem::exists(singular); const bool hasPlural = std::filesystem::exists(plural); const bool hasPrx = std::filesystem::exists(prx); - if (hasSingular && hasPlural) throw Domain::RelinkerException("Both sce_module and sce_modules exist beside the input executable"); - if (!hasSingular && !hasPlural && !hasPrx) throw Domain::RelinkerException("sce_module/sce_modules/prx was not found beside the input executable: " + root.string() + ". Use --skip-sce-module only if this game can run without these modules."); + if (hasSingular && hasPlural) throw Domain::RelinkerException("Both sce_module and sce_modules exist in the guest module parent directory"); + if (!hasSingular && !hasPlural && !hasPrx) throw Domain::RelinkerException(missingModuleMessage); std::vector directories; if (hasSingular || hasPlural) directories.push_back(hasSingular ? singular : plural); if (hasPrx) directories.push_back(prx); @@ -52,7 +73,11 @@ std::vector GuestModuleBuilder::Build(const std::filesystem::path char magic[4]{}; stream.read(magic, 4); if (stream.bad()) throw Domain::RelinkerException("Cannot read guest candidate magic: " + path.string()); - return stream.gcount() == 4 && static_cast(magic[0]) == 0x7f && magic[1] == 'E' && magic[2] == 'L' && magic[3] == 'F'; + if (stream.gcount() != 4) return false; + const auto byte = [&](const std::size_t index) { return static_cast(magic[index]); }; + const bool self = (byte(0) == 0x4f && byte(1) == 0x15 && byte(2) == 0x3d && byte(3) == 0x1d) || (byte(0) == 0x54 && byte(1) == 0x14 && byte(2) == 0xf5 && byte(3) == 0xee); + if (self) throw Domain::RelinkerException("Guest module is a SELF container, not an ELF: " + path.string()); + return byte(0) == 0x7f && magic[1] == 'E' && magic[2] == 'L' && magic[3] == 'F'; }; for (const auto& directory : directories) { if (!std::filesystem::is_directory(directory)) throw Domain::RelinkerException("Guest module path is not a directory: " + directory.string()); @@ -68,34 +93,39 @@ std::vector GuestModuleBuilder::Build(const std::filesystem::path } const auto neededNames = ReadNeededNames(dynamic); std::set missingNeeded; + std::map neededAliases; for (const auto& name : neededNames) { if (excludedModules.contains(name)) continue; - if (std::none_of(paths.begin(), paths.end(), [&](const auto& path) { return path.filename().string() == name; })) missingNeeded.insert(name); - } - if (!missingNeeded.empty()) { - std::map found; - for (auto it = std::filesystem::recursive_directory_iterator(root); it != std::filesystem::recursive_directory_iterator(); ++it) { - if (it->is_directory() && std::find(directories.begin(), directories.end(), it->path()) != directories.end()) { - it.disable_recursion_pending(); - continue; - } - const auto name = it->path().filename().string(); - if (!it->is_regular_file() || !missingNeeded.contains(name) || !isElf(it->path())) continue; - if (!found.emplace(name, it->path()).second) throw Domain::RelinkerException("Ambiguous needed module: " + found.at(name).string() + " and " + it->path().string()); - } - for (const auto& [name, path] : found) paths.push_back(path); + if (std::any_of(paths.begin(), paths.end(), [&](const auto& path) { return path.filename().string() == name || (windows && FoldFilename(path.filename().string()) == FoldFilename(name)); })) continue; + const auto stem = ModuleStem(name, windows); + const auto alias = stem.empty() ? paths.end() : std::find_if(paths.begin(), paths.end(), [&](const auto& path) { return ModuleStem(path.filename().string(), windows) == stem; }); + if (alias != paths.end()) neededAliases.emplace(name, *alias); + else missingNeeded.insert(name); } + Io::FileReader reader; + std::map discovered; + const auto matchesIdentity = [](const std::string& name, const std::vector& identities) { + return std::any_of(identities.begin(), identities.end(), [&](const auto& identity) { + return name == identity || name == identity + ".prx" || name == identity + ".sprx" || name == identity + ".suprx"; + }); + }; + for (const auto& path : paths) discovered.emplace(path, GuestImageReader().Read(path, reader.Read(path.string()))); if (!unmatchedExclusions.empty()) throw Domain::RelinkerException("Excluded guest module file not found: " + *unmatchedExclusions.begin()); std::sort(paths.begin(), paths.end()); + paths.erase(std::unique(paths.begin(), paths.end()), paths.end()); if (paths.empty()) return {}; if (lazyBinding) throw Domain::RelinkerException("Guest modules require eager binding; --lazy-binding is incompatible"); std::vector images; + if (std::filesystem::exists(outputPath)) { + for (const auto& path : paths) + if (std::filesystem::equivalent(path, outputPath)) + throw Domain::RelinkerException("Executable output would overwrite an input module: " + outputPath.string()); + } std::map> exports; std::map> sharedExports; std::set outputNames; - Io::FileReader reader; for (const auto& path : paths) { - auto image = GuestImageReader().Read(path, reader.Read(path.string())); + auto image = std::move(discovered.at(path)); if (image.OutputName.find_first_of("$\r\n") != std::string::npos) throw Domain::RelinkerException("Unsupported guest filename: " + image.OutputName); std::string folded = image.OutputName; if (windows) { @@ -128,24 +158,41 @@ std::vector GuestModuleBuilder::Build(const std::filesystem::path } std::map guestNames; std::map windowsGuestFiles; - const auto foldFilename = [](std::string name) { - for (auto& character : name) if (character >= 'A' && character <= 'Z') character = static_cast(character + ('a' - 'A')); - return name; - }; for (std::size_t index = 0; index < images.size(); ++index) { for (const auto& name : {images[index].SourcePath.filename().string(), images[index].Soname}) { if (name.empty()) continue; const auto [found, inserted] = guestNames.emplace(name, index); if (!inserted && found->second != index) throw Domain::RelinkerException("Ambiguous guest dependency name: " + name); } - if (windows) windowsGuestFiles.emplace(foldFilename(images[index].SourcePath.filename().string()), index); + if (windows) windowsGuestFiles.emplace(FoldFilename(images[index].SourcePath.filename().string()), index); + } + for (const auto& [name, path] : neededAliases) { + if (guestNames.contains(name)) continue; + const auto image = std::find_if(images.begin(), images.end(), [&](const auto& candidate) { return candidate.SourcePath == path; }); + if (image == images.end()) throw Domain::RelinkerException("Needed module alias has no guest image: " + name); + const auto index = static_cast(image - images.begin()); + const auto [found, inserted] = guestNames.emplace(name, index); + if (!inserted && found->second != index) throw Domain::RelinkerException("Ambiguous guest dependency name: " + name); } const auto findGuest = [&](const std::string& name) { const auto exact = guestNames.find(name); if (exact != guestNames.end() || !windows) return exact; - const auto file = windowsGuestFiles.find(foldFilename(name)); + const auto file = windowsGuestFiles.find(FoldFilename(name)); return file == windowsGuestFiles.end() ? guestNames.end() : guestNames.emplace(name, file->second).first; }; + const auto resolveIdentity = [&](const std::string& name) { + if (findGuest(name) != guestNames.end()) return; + std::size_t match = images.size(); + for (std::size_t index = 0; index < images.size(); ++index) { + const auto& identities = images[index].ModuleNames; + if (!matchesIdentity(name, identities)) continue; + if (match != images.size()) throw Domain::RelinkerException("Ambiguous guest module identity: " + name); + match = index; + } + if (match != images.size()) guestNames.emplace(name, match); + }; + for (const auto& name : missingNeeded) resolveIdentity(name); + for (const auto& image : images) for (const auto& name : image.Dependencies) resolveIdentity(name); const auto rejectSharedImport = [&](const std::string& name, const std::string& importer) { const auto shared = sharedExports.find(name); if (shared == sharedExports.end()) return; @@ -171,6 +218,7 @@ std::vector GuestModuleBuilder::Build(const std::filesystem::path if (!windows && exports.contains(name)) rename(dynamic.DynSymData, dynamic.DynStrData, offset / 24, name); } std::vector> dependencies(images.size()); + std::vector> systemImports(images.size()); for (auto& image : images) { image.UsePlatformTlsResolver = !windows ? !exports.contains("vNe1w4diLCs") : std::none_of(image.Symbols.begin(), image.Symbols.end(), [](const auto& symbol) { return symbol.Name == "vNe1w4diLCs" && symbol.Section != 0 && symbol.Section != AbsoluteSection && (symbol.Info >> 4) != 0 && symbol.Visibility != 1 && symbol.Visibility != 2; @@ -181,7 +229,8 @@ std::vector GuestModuleBuilder::Build(const std::filesystem::path const auto found = findGuest(name); if (found != guestNames.end() && found->second != index) dependencies[index].insert(found->second); } - for (const auto& symbol : images[index].Symbols) { + for (std::size_t symbolIndex = 0; symbolIndex < images[index].Symbols.size(); ++symbolIndex) { + const auto& symbol = images[index].Symbols[symbolIndex]; if (symbol.Section != 0 || symbol.Name.empty()) continue; rejectSharedImport(symbol.Name, images[index].SourcePath.string()); const auto found = exports.find(symbol.Name); @@ -199,6 +248,7 @@ std::vector GuestModuleBuilder::Build(const std::filesystem::path if (exported == provider.Symbols.end() || ((symbol.Info & 15) != 0 && (symbol.Info & 15) != (exported->Info & 15))) throw Domain::RelinkerException("Guest import/export type mismatch: " + symbol.Name); if (providers.front() != index) dependencies[index].insert(providers.front()); } else if (windows && (symbol.Info & 15) == 6) throw Domain::RelinkerException("Windows guest TLS import requires a guest TLS export: " + symbol.Name); + if (providers.empty()) systemImports[index].insert(symbolIndex); } } if (!windows) { @@ -239,17 +289,13 @@ std::vector GuestModuleBuilder::Build(const std::filesystem::path }; for (const auto index : order) if (!windows) addNeeded("$ORIGIN/app0/" + images[index].SourcePath.parent_path().lexically_relative(root).generic_string() + "/" + images[index].OutputName); for (const auto& name : hostLibraries) addNeeded(name); - std::string guestRunPath = runPath; - if (!windows) { - if (guestRunPath == "$ORIGIN") guestRunPath = "$ORIGIN/../.."; - else if (guestRunPath.starts_with("$ORIGIN/")) guestRunPath.insert(8, "../../"); - else if (!std::filesystem::path(guestRunPath).is_absolute()) throw Domain::RelinkerException("Guest Linux run path must be absolute or begin with $ORIGIN"); - } + if (!windows && runPath != "$ORIGIN" && !runPath.starts_with("$ORIGIN/") && !std::filesystem::path(runPath).is_absolute()) throw Domain::RelinkerException("Guest Linux run path must be absolute or begin with $ORIGIN"); + const auto outputDirectory = std::filesystem::absolute(outputPath).parent_path().lexically_normal(); std::vector artifacts; for (const auto index : order) { const auto& image = images[index]; const auto relativeDirectory = "app0/" + image.SourcePath.parent_path().lexically_relative(root).generic_string(); - const auto destination = std::filesystem::absolute(outputPath).parent_path() / relativeDirectory; + const auto destination = outputDirectory / relativeDirectory; const auto target = destination / image.OutputName; if (target.lexically_normal() == std::filesystem::absolute(outputPath).lexically_normal()) throw Domain::RelinkerException("Guest output collides with the executable output"); for (const auto& source : paths) if (std::filesystem::exists(target) && std::filesystem::equivalent(source, target)) throw Domain::RelinkerException("Guest output would overwrite an input module: " + target.string()); @@ -261,21 +307,35 @@ std::vector GuestModuleBuilder::Build(const std::filesystem::path std::vector output; if (windows) output = Elfpatcher::GuestModuleWriter().WriteWindows(image, runtime); else { + std::string guestRunPath = runPath; + if (runPath == "$ORIGIN" || runPath.starts_with("$ORIGIN/")) { + const auto relativeRoot = outputDirectory.lexically_relative(destination.lexically_normal()).generic_string(); + guestRunPath = "$ORIGIN/" + relativeRoot + runPath.substr(7); + } std::vector needed; for (const auto dependency : dependencies[index]) { const auto dependencyPath = images[dependency].SourcePath.parent_path() / images[dependency].OutputName; needed.push_back("$ORIGIN/" + dependencyPath.lexically_relative(image.SourcePath.parent_path()).generic_string()); } needed.insert(needed.end(), hostLibraries.begin(), hostLibraries.end()); - output = Elfpatcher::GuestModuleWriter().WriteLinux(image, needed, guestRunPath); + output = macos ? Elfpatcher::GuestModuleWriter().WriteMacOs(image, needed, guestRunPath) : Elfpatcher::GuestModuleWriter().WriteLinux(image, needed, guestRunPath); } if (windows) { for (const auto& [name, provider] : guestNames) { if (provider == index && std::find(runtime.Names.begin(), runtime.Names.end(), name) == runtime.Names.end()) runtime.Names.push_back(name); } } + std::vector imports; + for (const auto* table : {&image.Dynamic.RelaData, &image.Dynamic.RelaPltData}) { + for (std::size_t position = 0; position < table->size(); position += 24) { + const auto symbolIndex = static_cast(Io::ReadU64(*table, position + 8) >> 32); + if (!systemImports[index].contains(symbolIndex)) continue; + const auto& symbol = image.Symbols[symbolIndex]; + imports.push_back({symbol.Name, symbol.Library, {}, 0, {}, Io::ReadU64(*table, position), {}, false}); + } + } dynamic.GuestModules.push_back(std::move(runtime)); - artifacts.push_back({target, std::move(output)}); + artifacts.push_back({target, std::move(output), std::move(imports)}); } return artifacts; } diff --git a/core/relinker/relinker/src/output/CallRegistryWriter.cpp b/core/relinker/relinker/src/output/CallRegistryWriter.cpp index 39ca268aa..e598056d6 100644 --- a/core/relinker/relinker/src/output/CallRegistryWriter.cpp +++ b/core/relinker/relinker/src/output/CallRegistryWriter.cpp @@ -62,4 +62,23 @@ std::string CallRegistryWriter::WriteCallRegistry(const std::vector& entries) { + std::ostringstream out; + out << "[\n"; + + for (std::size_t i = 0; i < entries.size(); ++i) { + const auto& e = entries[i]; + out << " {\n"; + out << " \"nid\": " << _jsonString(e.Nid) << ",\n"; + out << " \"library\": " << _jsonString(e.Library) << ",\n"; + out << " \"targetOffset\": " << _hexOffset(e.TargetOffset) << "\n"; + out << " }"; + if (i + 1 < entries.size()) out << ","; + out << "\n"; + } + + out << "]\n"; + return out.str(); +} + } diff --git a/core/relinker/relinker/src/parsing/ElfReader.cpp b/core/relinker/relinker/src/parsing/ElfReader.cpp index 84b845c19..a92ef6238 100644 --- a/core/relinker/relinker/src/parsing/ElfReader.cpp +++ b/core/relinker/relinker/src/parsing/ElfReader.cpp @@ -83,6 +83,8 @@ ElfHeader ElfReader::ReadHeader() const { _fileBuffer[2] != 'L' || _fileBuffer[3] != 'F') { throw RelinkerException("Invalid ELF magic number: " + formatMagic()); } + if (_fileBuffer[4] != 2 || _fileBuffer[5] != 1 || _fileBuffer[6] != 1) + throw RelinkerException("Expected little-endian ELF64 version 1"); ElfHeader header{}; header.Machine = _readU16At(0x12); diff --git a/core/relinker/relinker/src/pipeline/RelinkerPipeline.cpp b/core/relinker/relinker/src/pipeline/RelinkerPipeline.cpp index 63d311a13..d631eea28 100644 --- a/core/relinker/relinker/src/pipeline/RelinkerPipeline.cpp +++ b/core/relinker/relinker/src/pipeline/RelinkerPipeline.cpp @@ -165,6 +165,9 @@ RelinkResult RelinkerPipeline::Relink(const std::vector& sourceElf if (hasPltRelocations && (dynJmpRelOffset > raw.size() || dynJmpRelSize > raw.size() - dynJmpRelOffset)) throw RelinkerException("Jump relocation table is out of bounds", dynJmpRelOffset); + if (dynRelaOffset > raw.size() || dynRelaSize > raw.size() - dynRelaOffset) + throw RelinkerException("Relocation table is out of bounds", dynRelaOffset); + auto readCStr = [&](FileByteOffset strOff) -> std::string { if (strOff >= dynStrTabSize) throw RelinkerException("Dynamic string offset is outside DT_STRSZ", strOff); @@ -216,7 +219,7 @@ RelinkResult RelinkerPipeline::Relink(const std::vector& sourceElf } const FileByteOffset symOff = dynSymTabOffset + static_cast(symIdx) * symEntSize; - if (symOff + 4 > raw.size()) + if (symOff < dynSymTabOffset || symOff > raw.size() || raw.size() - symOff < 4) throw RelinkerException("Symbol table entry out of bounds", symOff); std::uint32_t nameOff = 0; diff --git a/core/relinker/relinker/tests/DynamicTableBoundsTests.cpp b/core/relinker/relinker/tests/DynamicTableBoundsTests.cpp new file mode 100644 index 000000000..c91f3640e --- /dev/null +++ b/core/relinker/relinker/tests/DynamicTableBoundsTests.cpp @@ -0,0 +1,63 @@ +#include "ElfFixture.hpp" +#include "RelinkerProcess.hpp" +#include +#include +#include +#include + +namespace { + +using namespace RelinkerTests; + +void require(const bool condition, const std::string& message) { + if (!condition) throw std::runtime_error(message); +} + +const Bytes kCode = {0xC3}; + +constexpr std::size_t kTagStride = 16; +constexpr std::size_t kTagBase = kCodeOffset + kDynamicVaddr; +constexpr std::size_t kSymTabTag = 2; +constexpr std::size_t kRelaSzTag = 5; +constexpr std::int64_t kDtOsSymtab = 0x61000039; +constexpr std::int64_t kDtRelaSz = 8; +constexpr std::uint64_t kBogusOffset = 0xFFFFFFFFFFFFFFFEull; +constexpr std::size_t kRelaEntry = kCodeOffset + 0x700; + +void WriteTag(Bytes& image, const std::size_t index, const std::int64_t tag, const std::uint64_t value) { + Write(image, kTagBase + index * kTagStride, tag); + Write(image, kTagBase + index * kTagStride + 8, value); +} + +void WriteRelaEntry(Bytes& image, const std::uint64_t info) { + Write(image, kRelaEntry, 0); + Write(image, kRelaEntry + 8, info); + Write(image, kRelaEntry + 16, 0); +} + +} + +int main(const int argc, char** argv) { + try { + require(argc == 2, "usage: dynamic_table_bounds_tests "); + const std::string binary = argv[1]; + const TempDirectory directory; + const auto input = directory.Path() / "input.elf"; + const auto output = directory.Path() / "output.elf"; + + Bytes image = MakeExecutable(kCode); + WriteRelaEntry(image, (1ull << 32) | 6ull); + WriteTag(image, kRelaSzTag, kDtRelaSz, 24); + WriteTag(image, kSymTabTag, kDtOsSymtab, kBogusOffset); + WriteFile(input, image); + + const auto run = RunRelinker(binary, {"--skip-sce-module", "--to-intel", input.string(), output.string()}, directory.Path() / "relinker.log"); + require(run.ExitCode != 0, "Relinker read a symbol table outside the file and exited 0:\n" + run.Output); + require(run.Output.find("Symbol table entry out of bounds") != std::string::npos, + "Relinker did not report the symbol table entry as out of bounds:\n" + run.Output); + return 0; + } catch (const std::exception& error) { + std::cerr << "dynamic_table_bounds_tests: " << error.what() << '\n'; + return 1; + } +} diff --git a/core/relinker/relinker/tests/ElfFixture.hpp b/core/relinker/relinker/tests/ElfFixture.hpp index 8c44f1179..daa1f322c 100644 --- a/core/relinker/relinker/tests/ElfFixture.hpp +++ b/core/relinker/relinker/tests/ElfFixture.hpp @@ -17,7 +17,7 @@ using Bytes = std::vector; inline constexpr std::uint64_t kCodeOffset = 0x4000; inline constexpr std::uint64_t kCodeSize = 0x1000; inline constexpr std::uint64_t kDynamicVaddr = 0x600; -inline constexpr std::uint16_t kSpareProgramHeaders = 3; +inline constexpr std::uint16_t kSpareProgramHeaders = 4; inline constexpr std::uint64_t kModuleDataOffset = 0x5000; inline constexpr std::uint64_t kModuleDataVaddr = 0x2000; inline constexpr std::uint64_t kModuleDataSize = 0x400; diff --git a/core/relinker/relinker/tests/RelaBoundsTests.cpp b/core/relinker/relinker/tests/RelaBoundsTests.cpp index a445c4f69..9fb373944 100644 --- a/core/relinker/relinker/tests/RelaBoundsTests.cpp +++ b/core/relinker/relinker/tests/RelaBoundsTests.cpp @@ -35,8 +35,8 @@ void RunAndRequireRejected(const std::string& binary, const Bytes& image, const WriteFile(input, image); const auto run = RunRelinker(binary, {"--skip-sce-module", "--to-intel", input.string(), output.string()}, directory.Path() / "relinker.log"); require(run.ExitCode != 0, "Relinker accepted a relocation table " + what + " and exited 0:\n" + run.Output); - require(run.Output.find("Relocation entry out of bounds") != std::string::npos, - "Relinker did not report the entry as out of bounds for a table " + what + ":\n" + run.Output); + require(run.Output.find("Relocation table is out of bounds") != std::string::npos, + "Relinker did not report the table as out of bounds for a table " + what + ":\n" + run.Output); } } diff --git a/core/relinker/relinker/tests/RelocTableBoundsTests.cpp b/core/relinker/relinker/tests/RelocTableBoundsTests.cpp new file mode 100644 index 000000000..fc0abc51f --- /dev/null +++ b/core/relinker/relinker/tests/RelocTableBoundsTests.cpp @@ -0,0 +1,126 @@ +#include "ElfFixture.hpp" +#include "RelinkerProcess.hpp" +#include +#include +#include +#include +#include +#include + +namespace { + +using namespace RelinkerTests; + +void require(const bool condition, const std::string& message) { + if (!condition) throw std::runtime_error(message); +} + +const Bytes kCode = {0xC3}; + +constexpr std::size_t kTagStride = 16; +constexpr std::size_t kTagBase = kCodeOffset + kDynamicVaddr; +constexpr std::size_t kEntryPointOffset = 24; +constexpr std::uint64_t kEntryVaddr = 0x100; +constexpr std::size_t kRelaTag = 4; +constexpr std::size_t kRelaSzTag = 5; +constexpr std::int64_t kDtRela = 7; +constexpr std::int64_t kDtRelaSz = 8; +constexpr std::int64_t kDtOsRela = 0x6100002f; +constexpr std::int64_t kDtOsRelaSz = 0x61000031; +constexpr std::uint64_t kImageSize = 0x8000; +constexpr std::uint64_t kUnmappedVaddr = 0x2000; +constexpr std::uint64_t kOffsetPastEnd = kImageSize + 0x1000; +constexpr std::uint64_t kOffsetBeforeEnd = kImageSize - 16; +constexpr std::size_t kRelaEntry = kCodeOffset + 0x700; +constexpr std::uint64_t kRelativeEntryInfo = 8; + +struct OutputMode { + std::string Flag; + std::string Name; +}; + +const std::vector& OutputModes() { + static const std::vector modes{{"--windows", "windows"}, {"--to-intel", "intel"}}; + return modes; +} + +void WriteTag(Bytes& image, const std::size_t index, const std::int64_t tag, const std::uint64_t value) { + Write(image, kTagBase + index * kTagStride, tag); + Write(image, kTagBase + index * kTagStride + 8, value); +} + +void WriteRelativeEntry(Bytes& image) { + Write(image, kRelaEntry, 0x300); + Write(image, kRelaEntry + 8, kRelativeEntryInfo); + Write(image, kRelaEntry + 16, 0); +} + +Bytes ImageWithRelocationTable(const std::int64_t offsetTag, const std::int64_t sizeTag, + const std::uint64_t offset, const std::uint64_t size) { + Bytes image = MakeExecutable(kCode); + Write(image, kEntryPointOffset, kEntryVaddr); + Write(image, kCodeOffset + kEntryVaddr, 0xC3); + WriteTag(image, kRelaTag, offsetTag, offset); + WriteTag(image, kRelaSzTag, sizeTag, size); + return image; +} + +void RunAndRequireAccepted(const std::string& binary, const Bytes& image, const OutputMode& mode, const std::string& what) { + const TempDirectory directory; + const auto input = directory.Path() / "input.elf"; + const auto output = directory.Path() / ("output." + mode.Name); + WriteFile(input, image); + const auto run = RunRelinker(binary, {"--skip-sce-module", mode.Flag, input.string(), output.string()}, directory.Path() / "relinker.log"); + require(run.ExitCode == 0 && std::filesystem::exists(output), + "Relinker rejected " + what + " for " + mode.Flag + ":\n" + run.Output); +} + +void RunAndRequireRejected(const std::string& binary, const Bytes& image, const OutputMode& mode, const std::string& what, const std::string& expected) { + const TempDirectory directory; + const auto input = directory.Path() / "input.elf"; + const auto output = directory.Path() / ("output." + mode.Name); + WriteFile(input, image); + const auto run = RunRelinker(binary, {"--skip-sce-module", mode.Flag, input.string(), output.string()}, directory.Path() / "relinker.log"); + require(run.ExitCode != 0 && !std::filesystem::exists(output), + "Relinker accepted " + what + " for " + mode.Flag + ":\n" + run.Output); + require(run.Output.find(expected) != std::string::npos, + "Relinker did not report " + expected + " for " + what + ":\n" + run.Output); +} + +} + +int main(const int argc, char** argv) { + try { + require(argc == 2, "usage: reloc_table_bounds_tests "); + const std::string binary = argv[1]; + const std::string tableError = "Relocation table is out of bounds"; + const std::string unmappedError = "Virtual address not mapped by any PT_LOAD segment"; + + for (const auto& mode : OutputModes()) { + RunAndRequireAccepted(binary, ImageWithRelocationTable(kDtRela, kDtRelaSz, 0x700, 0), mode, "an empty relocation table in the file"); + RunAndRequireAccepted(binary, ImageWithRelocationTable(kDtOsRela, kDtOsRelaSz, kRelaEntry, 0), mode, "an empty DT_OS_RELA table in the file"); + + { + Bytes image = ImageWithRelocationTable(kDtRela, kDtRelaSz, 0x700, 24); + WriteRelativeEntry(image); + RunAndRequireAccepted(binary, image, mode, "a relocation table that fits in the file"); + } + + RunAndRequireRejected(binary, ImageWithRelocationTable(kDtOsRela, kDtOsRelaSz, kOffsetPastEnd, 0), mode, + "an empty relocation table whose offset is past the end of the file", tableError); + RunAndRequireRejected(binary, ImageWithRelocationTable(kDtOsRela, kDtOsRelaSz, 0xFFFFFFFFFFFFFFFFull, 0), mode, + "an empty relocation table whose offset is near UINT64_MAX", tableError); + RunAndRequireRejected(binary, ImageWithRelocationTable(kDtOsRela, kDtOsRelaSz, kOffsetBeforeEnd, 0x108), mode, + "a relocation table that does not fit in the file", tableError); + RunAndRequireRejected(binary, ImageWithRelocationTable(kDtRela, kDtRelaSz, 0x700, 0xFFFFFFFFFFFFFFF0ull), mode, + "a relocation table whose size does not fit in the file", tableError); + RunAndRequireRejected(binary, ImageWithRelocationTable(kDtRela, kDtRelaSz, kUnmappedVaddr, 0), mode, + "a relocation table whose address no segment maps", unmappedError); + } + + return 0; + } catch (const std::exception& error) { + std::cerr << "reloc_table_bounds_tests: " << error.what() << '\n'; + return 1; + } +} \ No newline at end of file diff --git a/core/relinker/relinker/tests/StrictNidFilterTests.cpp b/core/relinker/relinker/tests/StrictNidFilterTests.cpp index 77d77d3f7..284bc616d 100644 --- a/core/relinker/relinker/tests/StrictNidFilterTests.cpp +++ b/core/relinker/relinker/tests/StrictNidFilterTests.cpp @@ -3,6 +3,7 @@ #include #include #include +#include #include #include #include @@ -86,6 +87,15 @@ void conditionalTailCall() { require(AnalyzeStrictReachability(input).ImportSlots.contains(0x2000), "Conditional tail call was removed"); } +void fallthroughAfterZeroPadding() { + auto input = fixture(); + ripOperand(input, 0, {0xE9}, 0x1000); + std::fill(input.Text.begin() + 5, input.Text.begin() + 16, 0); + importThunk(input, 16, 0x2000); + input.Functions = {{0x1000, 0x1010, {}}, {0x1010, 0x1018, {}}}; + require(AnalyzeStrictReachability(input).ImportSlots.contains(0x2000), "Zero padding after a region's last jump dropped its fall-through edge"); +} + void callbackAndRelocationRoots() { auto input = fixture(); ripOperand(input, 0, {0x48, 0x8D, 0x3D}, 0x1020); @@ -158,6 +168,45 @@ void relativeTableAndWholeFunction() { require(AnalyzeStrictReachability(input).ImportSlots.contains(0x2000), "Unknown switch target inside a live function was removed"); } +void exceptionLeb128Bounds() { + const auto check = [](const std::vector& codeAlignment, const std::vector& dataAlignment, bool overflowing) { + std::vector bytes(0x400); + std::vector cie{1, 0}; + cie.insert(cie.end(), codeAlignment.begin(), codeAlignment.end()); + cie.insert(cie.end(), dataAlignment.begin(), dataAlignment.end()); + cie.push_back(16); + write(bytes, 0x200, static_cast(4 + cie.size())); + std::copy(cie.begin(), cie.end(), bytes.begin() + 0x208); + write(bytes, 0x240, 20); + write(bytes, 0x244, 0x44); + write(bytes, 0x248, 0x1000); + write(bytes, 0x250, 0x20); + write(bytes, 0x300, 1); + write(bytes, 0x302, 3); + write(bytes, 0x304, 0x200); + write(bytes, 0x30C, 1); + write(bytes, 0x310, 0x1000); + write(bytes, 0x318, 0x240); + const std::vector headers = {{1, 4, 0, 0, 0, bytes.size(), bytes.size(), 8}, {0x6474E550, 4, 0x300, 0x300, 0, 32, 32, 4}}; + try { + const auto functions = Relinker::UnusedNidFilter::ReadExceptionFunctions(bytes, headers, {}, {}); + require(!overflowing, "Overflowing CIE LEB128 value was accepted"); + require(functions.size() == 1 && functions[0].Begin == 0x1000 && functions[0].End == 0x1020 && functions[0].ExtraTargets.empty(), "Valid CIE LEB128 changed the function range"); + } catch (const Relinker::RelinkerException& error) { + require(overflowing && std::string(error.what()).find("overflowing LEB128 value") != std::string::npos, "CIE LEB128 was rejected for an unexpected reason"); + } + }; + check({1}, {0x78}, false); + for (const std::uint8_t prefix : {0x80, 0xFF}) { + for (unsigned terminal = 0; terminal < 0x80; ++terminal) { + std::vector encoded(10, prefix); + encoded.back() = static_cast(terminal); + check({1}, encoded, terminal != 0 && terminal != 0x7F); + check(encoded, {0x78}, terminal != 0 && terminal != 1); + } + } +} + void vectorInstructionLengths() { const Codegen::X64InstructionDecoder decoder; const std::vector> instructions = { @@ -174,10 +223,13 @@ void vectorInstructionLengths() { void exceptionLandingPads() { std::vector bytes(0x400); - write(bytes, 0x200, 15); + write(bytes, 0x200, 25); write(bytes, 0x208, 1); - const std::vector augmentation = {'z', 'L', 'R', 0, 1, 0x78, 16, 2, 0, 0}; + const std::vector augmentation = {'z', 'P', 'L', 'R', 0, 1, 0x78, 16, 11, 0}; std::copy(augmentation.begin(), augmentation.end(), bytes.begin() + 0x209); + write(bytes, 0x213, 0x1000); + write(bytes, 0x21B, 0); + write(bytes, 0x21C, 0); write(bytes, 0x220, 29); write(bytes, 0x224, 0x24); write(bytes, 0x228, 0x1000); @@ -195,7 +247,7 @@ void exceptionLandingPads() { const std::vector headers = {{1, 4, 0, 0, 0, bytes.size(), bytes.size(), 8}, {0x6474E550, 4, 0x300, 0x300, 0, 32, 32, 4}}; auto input = fixture(); input.Functions = Relinker::UnusedNidFilter::ReadExceptionFunctions(bytes, headers, {}, {}); - require(input.Functions.size() == 1 && input.Functions[0].ExtraTargets == std::vector{0x1040}, "LSDA landing pad outside the function was not recovered"); + require(input.Functions.size() == 1 && input.Functions[0].ExtraTargets == std::vector{0x1000, 0x1040}, "LSDA landing pad outside the function was not recovered"); emit(input, 0, {0x0F, 0x0B}); emit(input, 31, {0xC3}); importThunk(input, 64, 0x2000); @@ -204,6 +256,31 @@ void exceptionLandingPads() { requireFailure([&] { Relinker::UnusedNidFilter::ReadExceptionFunctions(bytes, headers, {}, {}); }, "Malformed LSDA was accepted"); } +void exceptionDataWithoutPersonality() { + std::vector bytes(0x400); + write(bytes, 0x200, 15); + write(bytes, 0x208, 1); + const std::vector augmentation = {'z', 'L', 'R', 0, 1, 0x78, 16, 2, 0, 0}; + std::copy(augmentation.begin(), augmentation.end(), bytes.begin() + 0x209); + write(bytes, 0x220, 29); + write(bytes, 0x224, 0x24); + write(bytes, 0x228, 0x1000); + write(bytes, 0x230, 0x20); + write(bytes, 0x238, 8); + write(bytes, 0x239, 0x280); + const std::vector lsda = {0xFF, 0xFF, 1, 3, 0, 1, 0x40, 0}; + std::copy(lsda.begin(), lsda.end(), bytes.begin() + 0x280); + write(bytes, 0x300, 1); + write(bytes, 0x302, 3); + write(bytes, 0x304, 0x200); + write(bytes, 0x30C, 1); + write(bytes, 0x310, 0x1000); + write(bytes, 0x318, 0x220); + const std::vector headers = {{1, 4, 0, 0, 0, bytes.size(), bytes.size(), 8}, {0x6474E550, 4, 0x300, 0x300, 0, 32, 32, 4}}; + const auto functions = Relinker::UnusedNidFilter::ReadExceptionFunctions(bytes, headers, {}, {}); + require(functions.size() == 1 && functions[0].Begin == 0x1000 && functions[0].End == 0x1020 && functions[0].ExtraTargets.empty(), "LSDA without a personality routine was read as a C++ call-site table"); +} + std::vector elfFixture(const StrictReachabilityInput& input) { std::vector bytes(0x400); bytes[0] = 0x7F; @@ -289,6 +366,105 @@ void filterAndPltCompaction() { requireFailure([&] { Relinker::MakeStrictUnusedNidFilter()->Filter(references, exceptional, input.Text, input.TextVaddr); }, "Exception metadata was ignored"); } +void relativeRelocationTableBounds() { + std::vector bytes(0x400, 0); + bytes[0] = 0x7F; + bytes[1] = 'E'; + bytes[2] = 'L'; + bytes[3] = 'F'; + bytes[4] = 2; + bytes[5] = 1; + bytes[6] = 1; + write(bytes, 16, 3); + write(bytes, 18, 62); + write(bytes, 32, 64); + write(bytes, 54, 56); + write(bytes, 56, 2); + + write(bytes, 64, 1); + write(bytes, 68, 7); + write(bytes, 72, 0x200); + write(bytes, 80, 0x1000); + write(bytes, 96, 0x100); + write(bytes, 104, 0x100); + + write(bytes, 120, 2); + write(bytes, 124, 6); + write(bytes, 128, 0x100); + write(bytes, 136, 0x2000); + write(bytes, 152, 0x80); + write(bytes, 160, 0x80); + + write(bytes, 0x100, 7); + write(bytes, 0x108, 0x10E8); + write(bytes, 0x110, 8); + write(bytes, 0x118, 24); + write(bytes, 0x120, 0); + write(bytes, 0x128, 0); + + write(bytes, 0x2E8, 0x5000); + write(bytes, 0x2F0, 8); + write(bytes, 0x2F8, 0x6000); + + write(bytes, 0x300, 0x7000); + write(bytes, 0x308, 8); + write(bytes, 0x310, 0x8000); + + auto valid = Relinker::UnusedNidFilter::BuildRelativeRelocationIndex(bytes); + require(valid->TargetOfSlot(0x5000) == 0x6000, "Valid relative relocation was not indexed"); + require(!valid->TargetOfSlot(0x7000).has_value(), "Out of segment relocation was indexed in valid table"); + + auto emptyRelocationTable = bytes; + write(emptyRelocationTable, 0x108, ~std::uint64_t{0}); + write(emptyRelocationTable, 0x118, 0); + auto emptyIndex = Relinker::UnusedNidFilter::BuildRelativeRelocationIndex(emptyRelocationTable); + require(!emptyIndex->TargetOfSlot(0x5000).has_value(), + "Empty relocation table was unexpectedly parsed"); + + auto overextended = bytes; + write(overextended, 0x118, 48); + requireFailure([&] { Relinker::UnusedNidFilter::BuildRelativeRelocationIndex(overextended); }, "Relocation table extending beyond PT_LOAD was accepted"); + + auto badSize = bytes; + write(badSize, 0x118, 25); + requireFailure([&] { Relinker::UnusedNidFilter::BuildRelativeRelocationIndex(badSize); }, "Unaligned relocation table size was accepted"); + + auto pltOverextended = bytes; + write(pltOverextended, 0x100, 23); + write(pltOverextended, 0x108, 0x10E8); + write(pltOverextended, 0x110, 2); + write(pltOverextended, 0x118, 48); + requireFailure([&] { Relinker::UnusedNidFilter::BuildRelativeRelocationIndex(pltOverextended); }, "PLT relocation table extending beyond PT_LOAD was accepted"); + + auto pltBadSize = bytes; + write(pltBadSize, 0x100, 23); + write(pltBadSize, 0x108, 0x10E8); + write(pltBadSize, 0x110, 2); + write(pltBadSize, 0x118, 1); + requireFailure([&] { Relinker::UnusedNidFilter::BuildRelativeRelocationIndex(pltBadSize); }, "Unaligned PLT relocation table size was accepted"); + + auto overflowSize = bytes; + write(overflowSize, 0x118, (~std::uint64_t{0} / 24) * 24); + requireFailure([&] { Relinker::UnusedNidFilter::BuildRelativeRelocationIndex(overflowSize); }, "Relocation table size near UINT64_MAX was accepted"); + + auto overflowRelaVa = bytes; + write(overflowRelaVa, 0x108, ~std::uint64_t{0} - 15); + requireFailure([&] { Relinker::UnusedNidFilter::BuildRelativeRelocationIndex(overflowRelaVa); }, "Relocation table vaddr near UINT64_MAX was accepted"); + + auto overflowLoadOffset = bytes; + write(overflowLoadOffset, 72, ~std::uint64_t{0} - 16); + requireFailure([&] { Relinker::UnusedNidFilter::BuildRelativeRelocationIndex(overflowLoadOffset); }, "PT_LOAD file offset near UINT64_MAX was accepted"); + + auto overflowLoadSize = bytes; + write(overflowLoadSize, 96, ~std::uint64_t{0} - 16); + write(overflowLoadSize, 0x118, 0x300); + requireFailure([&] { Relinker::UnusedNidFilter::BuildRelativeRelocationIndex(overflowLoadSize); }, "Relocation table beyond file with fileSize near UINT64_MAX was accepted"); + + auto overflowDynamicOffset = bytes; + write(overflowDynamicOffset, 128, ~std::uint64_t{0} - 16); + requireFailure([&] { Relinker::UnusedNidFilter::BuildRelativeRelocationIndex(overflowDynamicOffset); }, "PT_DYNAMIC offset near UINT64_MAX was accepted"); +} + } int main() { @@ -296,16 +472,20 @@ int main() { deadTailAndExplicitEntry(); ud1DeadTail(); conditionalTailCall(); + fallthroughAfterZeroPadding(); callbackAndRelocationRoots(); registerImportCall(); mergedJumpTable(); conservativeIndirectTargets(); isolatedFunctionCycle(); relativeTableAndWholeFunction(); + exceptionLeb128Bounds(); vectorInstructionLengths(); exceptionLandingPads(); + exceptionDataWithoutPersonality(); filterCallbackDataImports(); filterAndPltCompaction(); + relativeRelocationTableBounds(); std::cout << "Strict NID filter tests passed\n"; } catch (const std::exception& error) { std::cerr << error.what() << '\n'; diff --git a/core/relinker/relinker/tests/macos/import_fixture.py b/core/relinker/relinker/tests/macos/import_fixture.py new file mode 100644 index 000000000..5e3c0a491 --- /dev/null +++ b/core/relinker/relinker/tests/macos/import_fixture.py @@ -0,0 +1,35 @@ +"""A minimal PS5-style ELF that imports puts and exit from libc.prx by NID.""" +import struct, sys +sys.path.insert(0, __import__("os").path.dirname(__import__("os").path.abspath(__file__))) +from nid import nid + +def fixture(message=b"hello from a PS5 ELF on macOS\0"): + image = bytearray(0x6000) + image[:16] = b"\x7fELF\x02\x01\x01" + bytes(9) + struct.pack_into("> 18) & 63] + B64[(t >> 12) & 63] + B64[(t >> 6) & 63] + B64[t & 63] + t = (r[6] << 16) | (r[7] << 8) + out += B64[(t >> 18) & 63] + B64[(t >> 12) & 63] + B64[(t >> 6) & 63] + return out +if __name__ == "__main__": + import sys + for n in sys.argv[1:]: print(n, nid(n)) diff --git a/core/relinker/relinker/tests/macos/tls_fixture.py b/core/relinker/relinker/tests/macos/tls_fixture.py new file mode 100644 index 000000000..2acca4919 --- /dev/null +++ b/core/relinker/relinker/tests/macos/tls_fixture.py @@ -0,0 +1,40 @@ +"""A PS5-style ELF that uses static TLS through %fs and exits with a value computed from it.""" +import struct, sys +sys.path.insert(0, __import__("os").path.dirname(__import__("os").path.abspath(__file__))) +from nid import nid + +def fixture(): + image = bytearray(0x6000) + image[:16] = b"\x7fELF\x02\x01\x01" + bytes(9) + struct.pack_into("= 0, (tags, strings) + actual = strings[offset:end].decode() + configured = run_path if run_path is not None else "$ORIGIN/libs" + expected = Path(configured.replace("$ORIGIN", str(output.parent))).resolve() + resolved = Path(actual.replace("$ORIGIN", str(artifact.parent))).resolve() + if resolved != expected or (label == "absolute" and actual != run_path): + failures.append((str(relative), label, actual, str(resolved), str(expected))) + checked += 1 + assert not failures, failures + print(f"Guest run path tests passed: {checked} module/path combinations") + + +if __name__ == "__main__": + main() diff --git a/core/relinker/relinker/tests/test_guest_symbol_names.py b/core/relinker/relinker/tests/test_guest_symbol_names.py index 35558254c..a99180830 100644 --- a/core/relinker/relinker/tests/test_guest_symbol_names.py +++ b/core/relinker/relinker/tests/test_guest_symbol_names.py @@ -85,6 +85,27 @@ def module_symbols(image): return [symbol_name(image, tags, index) for index in range(1, count)] +def elf_hash(name): + value = 0 + for byte in name.encode(): + value = (value << 4) + byte + value = (value ^ ((value & 0xF0000000) >> 24)) & 0x0FFFFFFF + return value + + +def check_hash_table(image): + tags = dynamic_tags(image) + table = file_offset(image, tags[4]) + buckets, count = struct.unpack_from("> 3), 0xb8 | (target & 7)]) + struct.pack("= 8 else b"") + bytes([0x50 | (target & 7)])) + emit(bytes.fromhex("48 8d a4 24 00 ff ff ff")) + emit(fs_load(0, 0)) + for displacement, value in REGISTER_VALUES.items(): + immediate(1, value) + emit(bytes([0x48, 0x89, 0x48, displacement & 255])) + markers = [0x1020304050607000 + target * 0x101 for target in range(16)] + expected = list(markers) + for target in range(16): + if target != 4: + immediate(target, markers[target]) + for target, value in address.items(): + immediate(target, value) + expected[target] = value & mask + value = REGISTER_VALUES[offset] + base_val = expected[register] if wide else expected[register] & 0xffffffff + operand_val = value if wide else value & 0xffffffff + if opcode == 0x33: + res = base_val ^ operand_val + elif opcode == 0x23: + res = base_val & operand_val + elif opcode == 0x0b: + res = base_val | operand_val + elif opcode == 0x03: + res = (base_val + operand_val) & (mask if wide else 0xffffffff) + elif opcode == 0x2b: + res = (base_val - operand_val) & (mask if wide else 0xffffffff) + elif opcode == 0x3b: + res = base_val + elif opcode == 0x13: + res = (base_val + operand_val + (flags & 1)) & (mask if wide else 0xffffffff) + elif opcode == 0x1b: + res = (base_val - operand_val - (flags & 1)) & (mask if wide else 0xffffffff) + else: + raise ValueError(f"Unknown opcode: {opcode}") + if opcode != 0x3b: + expected[register] = res if wide else res & 0xffffffff + emit(b"\x68" + struct.pack("> 3) << 2, 0x89, 0x84 | (target & 7) << 3, 0x24]) + struct.pack("= 8 else b"") + bytes([0x58 | (target & 7)])) + emit(b"\xc3") + for position in failures: + struct.pack_into("> 3) << 2]) if wide or register >= 8 else b"") + alu = rex_alu + bytes([opcode, 0xc0 | (register & 7) << 3]) + return bytes.fromhex("48 8d 64 24 80 51 50") + address + b"\x50", b"\x59" + load + alu + bytes.fromhex("58 59 48 8d a4 24 80 00 00 00") + + + def make_image(transfer, metadata, extent=None, body=None): image = fixture() image.extend(b"\x90" * 0x1000) @@ -389,8 +510,16 @@ def add_alias(image, size, begin=0x1200): def main(): - relinker = Path(sys.argv[1]).resolve() - with tempfile.TemporaryDirectory(prefix="anyps5-tls-coverage-") as directory: + groups = ("metadata", "padding", "register_loads", "displacement_bounds", "alu_execution", + "register_address_loads", "rejected") + groups += tuple(f"displacement_r{register}" for register in range(16) if register != 4) + groups += tuple(f"alu_{name}" for name in ALU_READ) + parser = argparse.ArgumentParser(description="Run Windows guest TLS function coverage fixtures") + parser.add_argument("relinker", type=Path) + parser.add_argument("group", nargs="?", default="all", choices=("all",) + groups) + args = parser.parse_args() + relinker = args.relinker.resolve() + with tempfile.TemporaryDirectory(prefix=f"anyps5-tls-{args.group}-") as directory: work = Path(directory) def convert(name, image, error=None, tls_address=0x1240, displacement=0, error_offset=None): @@ -422,144 +551,167 @@ def main(): executed = subprocess.run([sys.executable, str(RUNNER), str(output), hex(entry)], capture_output=True, timeout=30) assert executed.returncode == 42, (name, executed.returncode, executed.stderr) - for metadata in ("unwind", "symbol"): - for transfer in ("table", "register", "memory"): - convert(metadata + "-" + transfer, make_image(transfer, metadata)) - convert(metadata + "-truncated", make_image("register", metadata, 0x46), "Code analysis:") - convert(metadata + "-outside", make_image("register", metadata, 0x1000), "Code analysis: function exceeds executable segment") - for first, second in ((0x47, 0x51), (0x51, 0x47), (0, 0x51), (0x51, 0)): - convert(f"symbol-alias-{first:x}-{second:x}", - add_alias(make_image("register", "symbol", first), second)) - for first, second in ((0x47, 0x1000), (0x1000, 0x47)): - convert(f"symbol-alias-outside-{first:x}-{second:x}", - add_alias(make_image("register", "symbol", first), second), - "Code analysis: function exceeds executable segment") - alias_tail = add_alias(make_image("register", "symbol", 0x51), 0x60) - alias_tail[0x1258:0x1260] = bytes.fromhex("64 8b 04 25 28 00 00 00") - convert("symbol-alias-unreachable-tls-tail", alias_tail, - "Unsupported Windows guest TLS instruction", error_offset=0x1258) - for metadata in ("unwind", "symbol"): - convert(f"branch-into-cut-tail-{metadata}", - make_image("register", metadata, 0x50, TLS_LOAD + bytes.fromhex("eb 00 8b 40 f0 c3"))) - convert("branch-into-undecodable-tail", - make_image("register", "unwind", 0x50, TLS_LOAD + bytes.fromhex("eb 01 66 0f 78 c0 01 02 c3")), - "Code analysis: branch into skipped range tail", error_offset=0x124f) - split_body = TLS_LOAD + bytes.fromhex("eb 00 8b 40 f0 48 b9 00 00 00 65 2e 62 69 6e") + b"\xc3" * 8 - convert("function-begins-inside-instruction", - add_alias(make_image("register", "symbol", 0x55, split_body), 0x0c, 0x1256)) - overlapping = make_image("register", "unwind") - overlapping[0x1200:0x1205] = b"\xe9" + struct.pack("> 3) << 2), opcode, 0xc0 | ((register & 7) << 3)]) - assert expected in stub, (case, expected.hex(), stub.hex()) - for name, image in alu_execution_cases(): - convert(name, image) - for name, image, address, instruction, register, wide, moved in register_load_cases(): - convert(name, image, tls_address=address) - pe = (work / (name + ".exe")).read_bytes() - patched = pe_bytes_at(pe, 0x10000 + address, 5) - stub = pe_bytes_at(pe, 0x10000 + address + 5 + struct.unpack_from("> 3) << 2), opcode, 0xc0 | ((register & 7) << 3)]) + assert expected in stub, (case, expected.hex(), stub.hex()) + if args.group in ("all", "alu_execution"): + for name, image in alu_execution_cases(): + convert(name, image) + if args.group in ("all", "register_address_loads"): + for name, image, address, instruction, register, wide, moved in register_load_cases(): + convert(name, image, tls_address=address) + pe = (work / (name + ".exe")).read_bytes() + patched = pe_bytes_at(pe, 0x10000 + address, 5) + stub = pe_bytes_at(pe, 0x10000 + address + 5 + struct.unpack_from("= memory_size and rva + virtual_size <= image_size, first + assert 0 < raw_size < SECTION_ALIGNMENT * 2, first + for name, virtual_size, rva, raw_size, raw in table: + assert raw_size % FILE_ALIGNMENT == 0 and raw % FILE_ALIGNMENT == 0, name + assert raw_size <= -(-virtual_size // FILE_ALIGNMENT) * FILE_ALIGNMENT, name + assert (raw_size == 0) == (raw == 0) and raw + raw_size <= len(pe), name + mapped = bytearray(image_size) + for name, virtual_size, rva, raw_size, raw in table: + size = min(raw_size, virtual_size) + mapped[rva:rva + size] = pe[raw:raw + size] + loaded = mapped[LOAD_RVA:LOAD_RVA + memory_size] + assert loaded[0x210:0x21a] == image[0x210:0x21a] + assert not any(loaded[0x1000:]) + if os.name == "nt": + executed = subprocess.run([str(output)], capture_output=True, text=True, timeout=20) + assert executed.returncode == 42, f"exit code {executed.returncode:#x}" + print("Windows zero-filled memory tests passed") + + +if __name__ == "__main__": + main() diff --git a/core/relinker/relinker/tests/test_windows_import_modules.py b/core/relinker/relinker/tests/test_windows_import_modules.py index 2ad1d9399..83e83ff56 100644 --- a/core/relinker/relinker/tests/test_windows_import_modules.py +++ b/core/relinker/relinker/tests/test_windows_import_modules.py @@ -107,34 +107,51 @@ def executable(owner, symbol='shared#A#B', module_name=None, extra_dependencies= return image -def shell_execute_exit_code(executable_path): - see_mask_nocloseprocess = 0x00000040 +def exit_code_without_standard_handles(executable_path): + sem_failcriticalerrors = 0x0001 + sem_nogpfaulterrorbox = 0x0002 + startf_usestdhandles = 0x00000100 wait_object_0 = 0 wait_timeout = 0x00000102 - class ShellExecuteInfo(ctypes.Structure): + class StartupInfo(ctypes.Structure): _fields_ = [ - ('cbSize', wintypes.DWORD), - ('fMask', wintypes.DWORD), - ('hwnd', wintypes.HWND), - ('lpVerb', wintypes.LPCWSTR), - ('lpFile', wintypes.LPCWSTR), - ('lpParameters', wintypes.LPCWSTR), - ('lpDirectory', wintypes.LPCWSTR), - ('nShow', ctypes.c_int), - ('hInstApp', wintypes.HANDLE), - ('lpIDList', wintypes.LPVOID), - ('lpClass', wintypes.LPCWSTR), - ('hkeyClass', wintypes.HANDLE), - ('dwHotKey', wintypes.DWORD), - ('hIconOrMonitor', wintypes.HANDLE), - ('hProcess', wintypes.HANDLE), + ('cb', wintypes.DWORD), + ('lpReserved', wintypes.LPWSTR), + ('lpDesktop', wintypes.LPWSTR), + ('lpTitle', wintypes.LPWSTR), + ('dwX', wintypes.DWORD), + ('dwY', wintypes.DWORD), + ('dwXSize', wintypes.DWORD), + ('dwYSize', wintypes.DWORD), + ('dwXCountChars', wintypes.DWORD), + ('dwYCountChars', wintypes.DWORD), + ('dwFillAttribute', wintypes.DWORD), + ('dwFlags', wintypes.DWORD), + ('wShowWindow', wintypes.WORD), + ('cbReserved2', wintypes.WORD), + ('lpReserved2', wintypes.LPVOID), + ('hStdInput', wintypes.HANDLE), + ('hStdOutput', wintypes.HANDLE), + ('hStdError', wintypes.HANDLE), + ] + + class ProcessInformation(ctypes.Structure): + _fields_ = [ + ('hProcess', wintypes.HANDLE), + ('hThread', wintypes.HANDLE), + ('dwProcessId', wintypes.DWORD), + ('dwThreadId', wintypes.DWORD), ] - shell32 = ctypes.WinDLL('shell32', use_last_error=True) kernel32 = ctypes.WinDLL('kernel32', use_last_error=True) - shell32.ShellExecuteExW.argtypes = [ctypes.POINTER(ShellExecuteInfo)] - shell32.ShellExecuteExW.restype = wintypes.BOOL + kernel32.GetErrorMode.restype = wintypes.UINT + kernel32.SetErrorMode.argtypes = [wintypes.UINT] + kernel32.SetErrorMode.restype = wintypes.UINT + kernel32.CreateProcessW.argtypes = [wintypes.LPCWSTR, wintypes.LPWSTR, wintypes.LPVOID, wintypes.LPVOID, wintypes.BOOL, + wintypes.DWORD, wintypes.LPVOID, wintypes.LPCWSTR, ctypes.POINTER(StartupInfo), + ctypes.POINTER(ProcessInformation)] + kernel32.CreateProcessW.restype = wintypes.BOOL kernel32.WaitForSingleObject.argtypes = [wintypes.HANDLE, wintypes.DWORD] kernel32.WaitForSingleObject.restype = wintypes.DWORD kernel32.GetExitCodeProcess.argtypes = [wintypes.HANDLE, ctypes.POINTER(wintypes.DWORD)] @@ -143,28 +160,30 @@ def shell_execute_exit_code(executable_path): kernel32.TerminateProcess.restype = wintypes.BOOL kernel32.CloseHandle.argtypes = [wintypes.HANDLE] kernel32.CloseHandle.restype = wintypes.BOOL - info = ShellExecuteInfo() - info.cbSize = ctypes.sizeof(info) - info.fMask = see_mask_nocloseprocess - info.lpVerb = 'open' - info.lpFile = str(executable_path) - info.nShow = 1 - if not shell32.ShellExecuteExW(ctypes.byref(info)): - raise OSError(ctypes.get_last_error(), 'ShellExecuteExW failed') + kernel32.SetErrorMode(kernel32.GetErrorMode() | sem_failcriticalerrors | sem_nogpfaulterrorbox) + info = StartupInfo() + info.cb = ctypes.sizeof(info) + info.dwFlags = startf_usestdhandles + process = ProcessInformation() + command = ctypes.create_unicode_buffer(f'"{executable_path}"') + if not kernel32.CreateProcessW(str(executable_path), command, None, None, False, 0, None, None, + ctypes.byref(info), ctypes.byref(process)): + raise OSError(ctypes.get_last_error(), 'CreateProcessW failed') try: - result = kernel32.WaitForSingleObject(info.hProcess, 30_000) + result = kernel32.WaitForSingleObject(process.hProcess, 30_000) if result == wait_timeout: - kernel32.TerminateProcess(info.hProcess, 0xffffffff) - kernel32.WaitForSingleObject(info.hProcess, 0xffffffff) - raise TimeoutError('ShellExecuteExW child did not exit') + kernel32.TerminateProcess(process.hProcess, 0xffffffff) + kernel32.WaitForSingleObject(process.hProcess, 0xffffffff) + raise TimeoutError('GUI child did not exit') if result != wait_object_0: raise OSError(f'WaitForSingleObject returned {result}') exit_code = wintypes.DWORD() - if not kernel32.GetExitCodeProcess(info.hProcess, ctypes.byref(exit_code)): + if not kernel32.GetExitCodeProcess(process.hProcess, ctypes.byref(exit_code)): raise OSError(ctypes.get_last_error(), 'GetExitCodeProcess failed') return exit_code.value finally: - kernel32.CloseHandle(info.hProcess) + kernel32.CloseHandle(process.hThread) + kernel32.CloseHandle(process.hProcess) def main(): @@ -288,7 +307,7 @@ def main(): ('missing-dependency-gui', 'missing.prx', 'shared#A#B', 0xc0000135)]: result, output = convert(name, owner, symbol, windows_gui=True) assert result.returncode == 0, result.stderr - run_status = shell_execute_exit_code(output) + run_status = exit_code_without_standard_handles(output) assert run_status == expected, (name, hex(run_status), hex(expected)) result, output = convert('missing-symbol', 'a.prx', 'absent#A#B') diff --git a/core/relinker/relinker/tests/test_windows_working_directory.py b/core/relinker/relinker/tests/test_windows_working_directory.py new file mode 100644 index 000000000..426acb896 --- /dev/null +++ b/core/relinker/relinker/tests/test_windows_working_directory.py @@ -0,0 +1,110 @@ +"""A converted Windows executable makes its own directory the current directory before it loads libraries.""" + +import ctypes +import os +from pathlib import Path +import struct +import subprocess +import sys +import tempfile +import time + +from test_windows_import_modules import executable, provider + + +class ProcessBasicInformation(ctypes.Structure): + _fields_ = [ + ('ExitStatus', ctypes.c_void_p), + ('PebBaseAddress', ctypes.c_void_p), + ('AffinityMask', ctypes.c_void_p), + ('BasePriority', ctypes.c_void_p), + ('UniqueProcessId', ctypes.c_void_p), + ('InheritedFromUniqueProcessId', ctypes.c_void_p), + ] + + +def current_directory(pid): + kernel32 = ctypes.WinDLL('kernel32', use_last_error=True) + ntdll = ctypes.WinDLL('ntdll') + kernel32.OpenProcess.restype = ctypes.c_void_p + kernel32.OpenProcess.argtypes = [ctypes.c_uint32, ctypes.c_int, ctypes.c_uint32] + kernel32.ReadProcessMemory.argtypes = [ctypes.c_void_p, ctypes.c_void_p, ctypes.c_void_p, ctypes.c_size_t, + ctypes.POINTER(ctypes.c_size_t)] + kernel32.CloseHandle.argtypes = [ctypes.c_void_p] + ntdll.NtQueryInformationProcess.argtypes = [ctypes.c_void_p, ctypes.c_int, ctypes.c_void_p, ctypes.c_uint32, + ctypes.c_void_p] + handle = kernel32.OpenProcess(0x1000 | 0x0010, 0, pid) + if not handle: + raise OSError(ctypes.get_last_error(), 'OpenProcess failed') + try: + def read(address, size): + buffer = ctypes.create_string_buffer(size) + done = ctypes.c_size_t() + if not kernel32.ReadProcessMemory(handle, address, buffer, size, ctypes.byref(done)) or done.value != size: + raise OSError(ctypes.get_last_error(), 'ReadProcessMemory failed') + return buffer.raw + + information = ProcessBasicInformation() + status = ntdll.NtQueryInformationProcess(handle, 0, ctypes.byref(information), ctypes.sizeof(information), None) + assert status == 0, f'NtQueryInformationProcess returned {status:#x}' + parameters = struct.unpack(' +#include +#include + +namespace ShaderRecompiler { + +std::string toHexString(std::uint32_t value); +bool contains(const std::vector& values, std::uint32_t value); +std::vector intersectSorted(const std::vector& first, const std::vector& second); +void addUnique(std::vector& values, std::uint32_t value); +void sortUnique(std::vector& values); +std::uint32_t remapId(std::uint32_t id, const std::vector& idMap); +void remapIds(std::vector& values, const std::vector& idMap); +void rebuildPredecessors(ControlFlowGraph& graph); + +} + +#endif diff --git a/core/shader/recompiler/ControlFlow/include/ControlFlow/GraphBuilder.hpp b/core/shader/recompiler/ControlFlow/include/ControlFlow/GraphBuilder.hpp index bca0da8c9..1b450325e 100644 --- a/core/shader/recompiler/ControlFlow/include/ControlFlow/GraphBuilder.hpp +++ b/core/shader/recompiler/ControlFlow/include/ControlFlow/GraphBuilder.hpp @@ -4,6 +4,7 @@ #include "ControlFlow/ControlFlowGraph.hpp" #include "RdnaDecoder/RdnaProgram.hpp" #include +#include #include namespace ShaderRecompiler { @@ -24,6 +25,8 @@ struct SwappcCall { std::uint32_t returnTargetProgramCounter = 0; }; +[[nodiscard]] std::optional UnresolvableSwappcTarget(const RdnaProgram& program, const SwappcInfo* swappc); + class GraphBuilder { public: [[nodiscard]] ControlFlowGraph Build(const RdnaProgram& program, const SwappcInfo* swappc = nullptr) const; diff --git a/core/shader/recompiler/ControlFlow/include/ControlFlow/Structurizer.hpp b/core/shader/recompiler/ControlFlow/include/ControlFlow/Structurizer.hpp index d643041ac..15318dc57 100644 --- a/core/shader/recompiler/ControlFlow/include/ControlFlow/Structurizer.hpp +++ b/core/shader/recompiler/ControlFlow/include/ControlFlow/Structurizer.hpp @@ -10,6 +10,8 @@ public: void Structurize(ControlFlowGraph& graph) const; private: + void structurize(ControlFlowGraph& graph, bool privatizeReturns) const; + void privatizeSharedReturns(ControlFlowGraph& graph) const; void computeDominatorTree(ControlFlowGraph& graph) const; void detectNaturalLoops(ControlFlowGraph& graph) const; void computePostDominators(ControlFlowGraph& graph) const; diff --git a/core/shader/recompiler/ControlFlow/src/ControlFlowGraph.cpp b/core/shader/recompiler/ControlFlow/src/ControlFlowGraph.cpp index 12ebb915d..3f331f1ec 100644 --- a/core/shader/recompiler/ControlFlow/src/ControlFlowGraph.cpp +++ b/core/shader/recompiler/ControlFlow/src/ControlFlowGraph.cpp @@ -1,4 +1,5 @@ #include "ControlFlow/ControlFlowGraph.hpp" +#include "ControlFlow/ControlFlowHelpers.hpp" #include #include #include @@ -7,8 +8,6 @@ namespace ShaderRecompiler { -namespace { - std::string toHexString(std::uint32_t value) { char buffer[11]; std::snprintf(buffer, sizeof(buffer), "0x%08x", value); @@ -25,6 +24,45 @@ std::vector intersectSorted(const std::vector& fir return result; } +void addUnique(std::vector& values, std::uint32_t value) { + if (std::find(values.begin(), values.end(), value) == values.end()) { + values.push_back(value); + } +} + +void sortUnique(std::vector& values) { + std::sort(values.begin(), values.end()); + values.erase(std::unique(values.begin(), values.end()), values.end()); +} + +std::uint32_t remapId(std::uint32_t id, const std::vector& idMap) { + return id != InvalidControlFlowId && id < idMap.size() ? idMap[id] : id; +} + +void remapIds(std::vector& values, const std::vector& idMap) { + for (auto& value : values) { + value = remapId(value, idMap); + } + sortUnique(values); +} + +void rebuildPredecessors(ControlFlowGraph& graph) { + for (auto& block : graph.blocks) { + block.predecessors.clear(); + sortUnique(block.successors); + } + for (const auto& block : graph.blocks) { + for (const auto successor : block.successors) { + addUnique(graph.blocks[successor].predecessors, block.id); + } + } + for (auto& block : graph.blocks) { + sortUnique(block.predecessors); + } +} + +namespace { + std::string joinIds(const std::vector& values) { std::string text; for (std::size_t i = 0; i < values.size(); ++i) { @@ -68,11 +106,13 @@ BasicBlock& ControlFlowGraph::FindBlockByProgramCounter(std::uint32_t programCou } bool ControlFlowGraph::Dominates(std::uint32_t dominator, std::uint32_t blockId) const { - return contains(FindBlock(blockId).dominators, dominator); + const auto& dominators = FindBlock(blockId).dominators; + return std::binary_search(dominators.begin(), dominators.end(), dominator); } bool ControlFlowGraph::PostDominates(std::uint32_t postDominator, std::uint32_t blockId) const { - return contains(FindBlock(blockId).postDominators, postDominator); + const auto& postDominators = FindBlock(blockId).postDominators; + return std::binary_search(postDominators.begin(), postDominators.end(), postDominator); } std::uint32_t ControlFlowGraph::FindNearestCommonPostDominator(std::uint32_t firstBlock, std::uint32_t secondBlock) const { diff --git a/core/shader/recompiler/ControlFlow/src/GraphBuilder.cpp b/core/shader/recompiler/ControlFlow/src/GraphBuilder.cpp index 1bd3c7776..0128c6474 100644 --- a/core/shader/recompiler/ControlFlow/src/GraphBuilder.cpp +++ b/core/shader/recompiler/ControlFlow/src/GraphBuilder.cpp @@ -1,4 +1,5 @@ #include "ControlFlow/GraphBuilder.hpp" +#include "ControlFlow/ControlFlowHelpers.hpp" #include #include #include @@ -12,38 +13,10 @@ namespace ShaderRecompiler { namespace { -std::string toHexString(std::uint32_t value) { - char buffer[11]; - std::snprintf(buffer, sizeof(buffer), "0x%08x", value); - return std::string(buffer); -} - std::uint32_t instructionEndProgramCounter(const RdnaInstruction& instruction) { return instruction.programCounter + instruction.wordCount * 4u; } -void addUnique(std::vector& values, std::uint32_t value) { - if (std::find(values.begin(), values.end(), value) == values.end()) { - values.push_back(value); - } -} - -void sortUnique(std::vector& values) { - std::sort(values.begin(), values.end()); - values.erase(std::unique(values.begin(), values.end()), values.end()); -} - -std::uint32_t remapId(std::uint32_t id, const std::vector& idMap) { - return id != InvalidControlFlowId && id < idMap.size() ? idMap[id] : id; -} - -void remapIds(std::vector& values, const std::vector& idMap) { - for (auto& value : values) { - value = remapId(value, idMap); - } - sortUnique(values); -} - std::uint32_t estimatedSpirvWords(const RdnaInstruction& instruction) { switch (instruction.op) { case RdnaOpcode::VReadfirstlaneB32: @@ -463,6 +436,18 @@ const SwappcCall* findReturnAt(const std::vector& calls, std::uint32 return nullptr; } +bool isFetchCall(const RdnaProgram& program, std::uint32_t index, const SwappcInfo* swappc, bool outsideProgram) { + const auto& instruction = program.instructions[index]; + const bool positional = std::all_of(program.instructions.begin(), program.instructions.begin() + index, isFetchCallPrefixOpcode); + const std::uint32_t targetRegister = scalarIndex(instruction.source0); + const bool userDataPair = swappc != nullptr && targetRegister != NoScalarRegister && + targetRegister >= swappc->userDataBaseRegister && targetRegister + 1u < swappc->userDataBaseRegister + swappc->userDataCount && + std::none_of(program.instructions.begin(), program.instructions.end(), [&](const RdnaInstruction& other) { + return writesScalar(other, targetRegister) || writesScalar(other, targetRegister + 1u); + }); + return swappc != nullptr && swappc->fetchCallAllowed && positional && (userDataPair || outsideProgram); +} + std::vector analyzeSwappcCalls(const RdnaProgram& program, const SwappcInfo* swappc) { std::vector calls; for (std::uint32_t index = 0; index < program.instructions.size(); ++index) { @@ -489,15 +474,7 @@ std::vector analyzeSwappcCalls(const RdnaProgram& program, const Swa if (instruction.op == RdnaOpcode::SCallB64 && call.targetIndex == InvalidControlFlowId) { throw std::invalid_argument("s_call_b64 at program counter " + toHexString(instruction.programCounter) + " targets invalid instruction boundary " + toHexString(target)); } - const bool positional = std::all_of(program.instructions.begin(), program.instructions.begin() + index, isFetchCallPrefixOpcode); - const std::uint32_t targetRegister = scalarIndex(instruction.source0); - const bool userDataPair = swappc != nullptr && targetRegister != NoScalarRegister && - targetRegister >= swappc->userDataBaseRegister && targetRegister + 1u < swappc->userDataBaseRegister + swappc->userDataCount && - std::none_of(program.instructions.begin(), program.instructions.end(), [&](const RdnaInstruction& other) { - return writesScalar(other, targetRegister) || writesScalar(other, targetRegister + 1u); - }); - const bool outsideProgram = staticTarget && call.targetIndex == InvalidControlFlowId; - if (instruction.op == RdnaOpcode::SSwappcB64 && swappc != nullptr && swappc->fetchCallAllowed && positional && (userDataPair || outsideProgram)) { + if (instruction.op == RdnaOpcode::SSwappcB64 && isFetchCall(program, index, swappc, staticTarget && call.targetIndex == InvalidControlFlowId)) { call.fetch = true; call.returnTargetProgramCounter = instructionEndProgramCounter(instruction); calls.push_back(call); @@ -641,21 +618,6 @@ BranchCondition conditionForOpcode(RdnaOpcode opcode) { throw std::logic_error("unreachable branch condition for opcode " + std::to_string(static_cast(opcode))); } -void rebuildPredecessors(ControlFlowGraph& graph) { - for (auto& block : graph.blocks) { - block.predecessors.clear(); - sortUnique(block.successors); - } - for (const auto& block : graph.blocks) { - for (const auto successor : block.successors) { - addUnique(graph.blocks[successor].predecessors, block.id); - } - } - for (auto& block : graph.blocks) { - sortUnique(block.predecessors); - } -} - void pruneUnreachableBlocks(ControlFlowGraph& graph) { if (graph.entryBlock >= graph.blocks.size()) { throw std::invalid_argument("control flow graph entry block is out of range"); @@ -890,6 +852,22 @@ void GraphBuilder::linkBlocks(std::vector& blocks, const RdnaProgram } } +std::optional UnresolvableSwappcTarget(const RdnaProgram& program, const SwappcInfo* swappc) { + for (std::uint32_t index = 0; index < program.instructions.size(); ++index) { + const auto& instruction = program.instructions[index]; + if (instruction.op != RdnaOpcode::SSwappcB64) { + continue; + } + std::uint32_t target = instruction.branchTarget; + const bool staticTarget = resolveSetpcTarget(program, index, target); + const bool inProgram = staticTarget && instructionIndexOfProgramCounter(program, target) != InvalidControlFlowId; + if (!inProgram && !isFetchCall(program, index, swappc, staticTarget)) { + return instruction.programCounter; + } + } + return std::nullopt; +} + ControlFlowGraph GraphBuilder::Build(const RdnaProgram& program, const SwappcInfo* swappc) const { const std::vector calls = analyzeSwappcCalls(program, swappc); ControlFlowGraph graph; diff --git a/core/shader/recompiler/ControlFlow/src/RequestSerializer.cpp b/core/shader/recompiler/ControlFlow/src/RequestSerializer.cpp index 48386cd20..e5b8a2f7c 100644 --- a/core/shader/recompiler/ControlFlow/src/RequestSerializer.cpp +++ b/core/shader/recompiler/ControlFlow/src/RequestSerializer.cpp @@ -235,6 +235,14 @@ ConservativeZExport readConservativeZExport(Reader& reader) { return static_cast(value); } +ColorExportPacking readColorExportPacking(Reader& reader) { + const auto value = reader.ReadU8(); + if (value > static_cast(ColorExportPacking::Unorm10_11_11)) { + throw std::runtime_error("RequestSerializer: invalid ColorExportPacking value"); + } + return static_cast(value); +} + void writeShaderBinary(Writer& writer, const ShaderBinary& binary) { writer.WriteU8(static_cast(binary.stage)); writer.WriteU64(binary.codeAddress); @@ -312,6 +320,10 @@ void writePixelInfo(Writer& writer, const ShaderPixelStageInfo& info) { for (const std::uint8_t value : info.targetExportMapping) { writer.WriteU8(value); } + for (const ColorExportPacking value : info.targetExportPacking) { + writer.WriteU8(static_cast(value)); + } + writer.WriteBool(info.dualSourceBlend); } ShaderPixelStageInfo readPixelInfo(Reader& reader, std::uint32_t version) { @@ -353,6 +365,12 @@ ShaderPixelStageInfo readPixelInfo(Reader& reader, std::uint32_t version) { } else { info.targetExportMapping.fill(0u); } + if (version >= 14u) { + for (ColorExportPacking& value : info.targetExportPacking) { + value = readColorExportPacking(reader); + } + } + if (version >= 15u) info.dualSourceBlend = reader.ReadBool(); if (version < 5u) { const auto input = [](PixelInput value, bool present) { return present ? PixelInputBit(value) : 0u; }; info.inputAddr = input(PixelInput::PerspectiveSample, info.hasPerspectiveCenterVgpr && inputAddrOrCenterVgpr == 2u) | input(PixelInput::PerspectiveCenter, info.hasPerspectiveCenterVgpr) | @@ -714,7 +732,7 @@ std::string RequestSerializer::Serialize(const RecompileRequest& request) const std::string buffer; Writer writer(buffer); writer.WriteU32(0x41505335u); - writer.WriteU32(13u); + writer.WriteU32(15u); writeShaderBinary(writer, request.shader); writeGuestContext(writer, request.context); writeSpirvTarget(writer, request.target); @@ -741,7 +759,7 @@ DeserializedRequest RequestSerializer::Deserialize(std::string_view text) const Reader reader(decoded); if (reader.ReadU32() != 0x41505335u) throw std::runtime_error("invalid recompile request signature"); const auto version = reader.ReadU32(); - if (version < 1u || version > 13u) throw std::runtime_error("unsupported recompile request serialization version"); + if (version < 1u || version > 15u) throw std::runtime_error("unsupported recompile request serialization version"); DeserializedRequest result{}; result.request.shader = readShaderBinary(reader, result.shaderCode, result.shaderHeader); result.request.context = readGuestContext(reader, result, version); diff --git a/core/shader/recompiler/ControlFlow/src/Structurizer.cpp b/core/shader/recompiler/ControlFlow/src/Structurizer.cpp index f23bd9258..d86026724 100644 --- a/core/shader/recompiler/ControlFlow/src/Structurizer.cpp +++ b/core/shader/recompiler/ControlFlow/src/Structurizer.cpp @@ -1,4 +1,5 @@ #include "ControlFlow/Structurizer.hpp" +#include "ControlFlow/ControlFlowHelpers.hpp" #include #include #include @@ -19,27 +20,13 @@ std::vector allBlockIds(std::uint32_t count) { return ids; } -std::vector intersectSorted(const std::vector& first, const std::vector& second) { - std::vector result; - std::set_intersection(first.begin(), first.end(), second.begin(), second.end(), std::back_inserter(result)); - return result; -} - -void sortUnique(std::vector& values) { - std::sort(values.begin(), values.end()); - values.erase(std::unique(values.begin(), values.end()), values.end()); -} - -void addUnique(std::vector& values, std::uint32_t value) { - if (std::find(values.begin(), values.end(), value) == values.end()) { - values.push_back(value); +void addSortedUnique(std::vector& values, std::uint32_t value) { + const auto position = std::lower_bound(values.begin(), values.end(), value); + if (position == values.end() || *position != value) { + values.insert(position, value); } } -bool contains(const std::vector& values, std::uint32_t value) { - return std::find(values.begin(), values.end(), value) != values.end(); -} - bool replaceValue(std::vector& values, std::uint32_t oldValue, std::uint32_t newValue) { bool changed = false; for (auto& value : values) { @@ -73,32 +60,6 @@ bool replaceTerminatorTarget(Terminator& terminator, std::uint32_t oldValue, std return changed; } -std::uint32_t remapId(std::uint32_t id, const std::vector& idMap) { - return id != InvalidControlFlowId && id < idMap.size() ? idMap[id] : id; -} - -void remapIds(std::vector& values, const std::vector& idMap) { - for (auto& value : values) { - value = remapId(value, idMap); - } - sortUnique(values); -} - -void rebuildPredecessors(ControlFlowGraph& graph) { - for (auto& block : graph.blocks) { - block.predecessors.clear(); - sortUnique(block.successors); - } - for (const auto& block : graph.blocks) { - for (const auto successor : block.successors) { - addUnique(graph.blocks[successor].predecessors, block.id); - } - } - for (auto& block : graph.blocks) { - sortUnique(block.predecessors); - } -} - std::vector applyBlockOrder(ControlFlowGraph& graph, std::vector blocks) { std::vector idMap(blocks.size(), InvalidControlFlowId); for (std::uint32_t i = 0; i < blocks.size(); ++i) { @@ -155,18 +116,23 @@ std::uint32_t moveBlockBefore(ControlFlowGraph& graph, std::uint32_t blockId, st std::vector dominatedBlocks(const ControlFlowGraph& graph, std::uint32_t headerBlock, std::uint32_t stopBlock = InvalidControlFlowId) { std::vector blocks; std::vector stack = {headerBlock}; + std::vector visited(graph.blocks.size()); blocks.reserve(graph.blocks.size()); stack.reserve(graph.blocks.size()); while (!stack.empty()) { const auto blockId = stack.back(); stack.pop_back(); - if (blockId == stopBlock || contains(blocks, blockId) || !graph.Dominates(headerBlock, blockId)) { + if (blockId >= visited.size()) { + visited.resize(blockId + 1); + } + if (blockId == stopBlock || visited[blockId] || !graph.Dominates(headerBlock, blockId)) { continue; } const auto& block = graph.FindBlock(blockId); - addUnique(blocks, blockId); + visited[blockId] = true; + blocks.push_back(blockId); for (const auto successor : block.successors) { if (successor != stopBlock && graph.Dominates(headerBlock, successor)) { stack.push_back(successor); @@ -837,6 +803,24 @@ void cloneBlocks(ControlFlowGraph& graph, const std::vector& bloc applyBlockOrder(graph, std::move(ordered)); } +bool privatizeOneSharedReturn(ControlFlowGraph& graph) { + for (const auto& block : graph.blocks) { + if (!block.successors.empty() || block.terminator.kind != TerminatorKind::Return || block.predecessors.size() < 2u || block.estimatedSpirvWords > CloneWordFloor) { + continue; + } + for (const auto predecessor : block.predecessors) { + const auto& from = graph.FindBlock(predecessor); + if (from.terminator.kind != TerminatorKind::ConditionalBranch || findInnermostContainingLoop(graph, predecessor) != nullptr) { + continue; + } + const auto shared = block.id; + cloneBlocks(graph, {shared}, {{predecessor, shared}}); + return true; + } + } + return false; +} + std::optional privatizeMergeTail(const ControlFlowGraph& graph, std::uint32_t header, std::uint32_t merge, std::uint32_t& cost, const std::function& recompute) { const auto& headerBlock = graph.FindBlock(header); if ((headerBlock.terminator.trueBlock != merge && headerBlock.terminator.falseBlock != merge) || graph.FindBlock(merge).predecessors.size() < 2u || !hasLinearPathToTerminal(graph, merge)) { @@ -1251,8 +1235,32 @@ void tarjanVisit(TarjanState& state, std::uint32_t blockId) { } void Structurizer::Structurize(ControlFlowGraph& graph) const { + const auto original = graph; + try { + structurize(graph, false); + } catch (const std::runtime_error&) { + graph = original; + structurize(graph, true); + } +} + +void Structurizer::privatizeSharedReturns(ControlFlowGraph& graph) const { + const auto budget = std::max(16u, graph.blocks.size() * 4u); + for (std::size_t clones = 0; privatizeOneSharedReturn(graph); ++clones) { + if (clones == budget) { + throw std::runtime_error("CFG shared return privatization exceeded budget"); + } + rebuildPredecessors(graph); + recomputeAnalyses(graph); + } +} + +void Structurizer::structurize(ControlFlowGraph& graph, bool privatizeReturns) const { recomputeAnalyses(graph); verifyReducibility(graph); + if (privatizeReturns) { + privatizeSharedReturns(graph); + } canonicalizeNaturalLoops(graph); splitSharedMergeBlocks(graph); isolateSemanticLoopHeaders(graph); @@ -1334,8 +1342,7 @@ void Structurizer::computeDominatorTree(ControlFlowGraph& graph) const { for (std::size_t i = 1; i < block.predecessors.size(); ++i) { next = intersectSorted(next, graph.blocks[block.predecessors[i]].dominators); } - addUnique(next, block.id); - sortUnique(next); + addSortedUnique(next, block.id); } if (next != block.dominators) { @@ -1389,10 +1396,25 @@ void Structurizer::computePostDominators(ControlFlowGraph& graph) const { block.postDominators = block.successors.empty() ? std::vector{block.id} : all; } + std::vector order; + order.reserve(count); + std::vector ordered(count, false); + if (graph.entryBlock < count) { + const auto forward = reversePostOrder(graph); + for (auto it = forward.rbegin(); it != forward.rend(); ++it) { + order.push_back(*it); + ordered[*it] = true; + } + } + for (std::uint32_t id = 0; id < count; ++id) { + if (!ordered[id]) order.push_back(id); + } + bool changed = true; while (changed) { changed = false; - for (auto& block : graph.blocks) { + for (const auto id : order) { + auto& block = graph.blocks[id]; std::vector next; if (block.successors.empty()) { next = {block.id}; @@ -1401,8 +1423,7 @@ void Structurizer::computePostDominators(ControlFlowGraph& graph) const { for (std::size_t i = 1; i < block.successors.size(); ++i) { next = intersectSorted(next, graph.blocks[block.successors[i]].postDominators); } - addUnique(next, block.id); - sortUnique(next); + addSortedUnique(next, block.id); } if (next != block.postDominators) { diff --git a/core/shader/recompiler/Deserialize.cpp b/core/shader/recompiler/Deserialize.cpp index c7379f1a7..b4e12b4b5 100644 --- a/core/shader/recompiler/Deserialize.cpp +++ b/core/shader/recompiler/Deserialize.cpp @@ -167,6 +167,7 @@ int main(int argc, char** argv) { translateOptions.userDataBaseRegister = request.context.userDataBaseRegister; translateOptions.userDataCount = static_cast(request.context.userData.size()); translateOptions.scratchDwords = request.context.compute.has_value() ? request.context.compute->scratchDwords : 0u; + translateOptions.sharedMemoryBytes = request.context.compute.has_value() && static_cast(request.context.compute->ldsSizeDwords) * 4u + 4u > request.target.maxWorkgroupSharedMemoryBytes ? request.context.compute->ldsSizeDwords * 4u : 0u; translateOptions.inputInfo = inputInfo; InstructionTranslator translator; diff --git a/core/shader/recompiler/FragmentOutputSpecialization.hpp b/core/shader/recompiler/FragmentOutputSpecialization.hpp new file mode 100644 index 000000000..8bd331e9a --- /dev/null +++ b/core/shader/recompiler/FragmentOutputSpecialization.hpp @@ -0,0 +1,145 @@ +#ifndef CORE_SHADER_RECOMPILER_FRAGMENTOUTPUTSPECIALIZATION_HPP +#define CORE_SHADER_RECOMPILER_FRAGMENTOUTPUTSPECIALIZATION_HPP + +#include "PipelineSpecialization.hpp" +#include "Recompiler.hpp" +#include +#include +#include +#include +#include +#include +#include +#include + +namespace ShaderRecompiler { + +inline std::vector SpecializeFragmentOutputs(std::vector words, std::span constants, const std::set& specializedIds) { + std::set packed; + bool dualSource = false; + for (const auto& constant : constants) { + if (constant.id == PipelineSpecialization::DualSourceBlend) { + if (constant.value > 1u) throw std::runtime_error("invalid prepared dual-source blend flag"); + dualSource = constant.value != 0u; + } + if (constant.id < PipelineSpecialization::ExportPackingBase || constant.id >= PipelineSpecialization::ExportPackingBase + 8u) continue; + if (constant.value > static_cast(ColorExportPacking::Unorm10_11_11)) throw std::runtime_error("invalid prepared fragment export packing"); + if (constant.value != static_cast(ColorExportPacking::None)) packed.insert(constant.id - PipelineSpecialization::ExportPackingBase); + } + if (packed.empty() && !dualSource) return words; + if (words.size() < 5u || words[0] != spv::MagicNumber) throw std::runtime_error("invalid fragment output specialization module"); + std::map locations; + std::set indexed; + std::map> typeKeys; + std::map, std::uint32_t> types; + std::map outputs; + for (std::size_t cursor = 5; cursor < words.size();) { + const auto count = words[cursor] >> 16u; + const auto op = static_cast(words[cursor] & 0xffffu); + if (count == 0u || count > words.size() - cursor) throw std::runtime_error("truncated fragment output specialization instruction"); + if (op == spv::OpDecorate && count == 4u && words[cursor + 2u] == spv::DecorationLocation) locations.emplace(words[cursor + 1u], words[cursor + 3u]); + if (op == spv::OpDecorate && count == 4u && words[cursor + 2u] == spv::DecorationIndex) indexed.insert(words[cursor + 1u]); + if ((op == spv::OpTypeInt || op == spv::OpTypeFloat || op == spv::OpTypeVector || op == spv::OpTypePointer) && count >= 3u) { + std::vector key{static_cast(op)}; + key.insert(key.end(), words.begin() + cursor + 2u, words.begin() + cursor + count); + typeKeys.emplace(words[cursor + 1u], key); + types.emplace(std::move(key), words[cursor + 1u]); + } + if (op == spv::OpVariable && count >= 4u && words[cursor + 3u] == spv::StorageClassOutput) outputs.emplace(words[cursor + 2u], words[cursor + 1u]); + cursor += count; + } + const auto key = [&](std::uint32_t id) { + const auto found = typeKeys.find(id); + return found != typeKeys.end() ? found->second : std::vector{}; + }; + const auto floatVectorPointer = [&](std::uint32_t pointer) { + const auto outer = key(pointer); + if (outer.size() != 3u || outer[0] != spv::OpTypePointer || outer[1] != spv::StorageClassOutput) return false; + const auto vector = key(outer[2]); + return vector.size() == 3u && vector[0] == spv::OpTypeVector && vector[2] == 4u && key(vector[1]) == std::vector{spv::OpTypeFloat, 32u}; + }; + const auto output = [&](std::uint32_t location) { + std::uint32_t variable = 0; + for (const auto& [id, pointer] : outputs) { + const auto found = locations.find(id); + if (found == locations.end() || found->second != location || indexed.contains(id)) continue; + if (variable != 0u) throw std::runtime_error("fragment outputs share location " + std::to_string(location)); + variable = id; + } + return variable; + }; + std::set retyped; + for (const auto location : packed) { + const auto variable = output(location); + if (variable == 0u) continue; + if (!specializedIds.contains(PipelineSpecialization::ExportPackingBase + location)) throw std::runtime_error("fragment output " + std::to_string(location) + " has no 10_11_11 unorm packing path"); + if (!floatVectorPointer(outputs.at(variable))) throw std::runtime_error("packed fragment output " + std::to_string(location) + " is not a float4"); + retyped.insert(variable); + } + const auto secondSource = dualSource ? output(1u) : 0u; + if (retyped.contains(secondSource)) throw std::runtime_error("the second dual-source blend color cannot be packed"); + if (retyped.empty() && secondSource == 0u) return words; + auto bound = words[3]; + std::vector> declarations; + const auto declare = [&](std::vector declaration) { + const auto found = types.find(declaration); + if (found != types.end()) return found->second; + if (bound == std::numeric_limits::max()) throw std::runtime_error("fragment output specialization module ID overflow"); + const auto id = bound++; + std::vector instruction{(static_cast(declaration.size() + 1u) << 16u) | declaration.front(), id}; + instruction.insert(instruction.end(), declaration.begin() + 1, declaration.end()); + types.emplace(std::move(declaration), id); + declarations.push_back(std::move(instruction)); + return id; + }; + const auto wordType = retyped.empty() ? 0u : declare({spv::OpTypeInt, 32u, 0u}); + const auto vectorType = retyped.empty() ? 0u : declare({spv::OpTypeVector, wordType, 4u}); + const auto pointerType = retyped.empty() ? 0u : declare({spv::OpTypePointer, spv::StorageClassOutput, vectorType}); + std::vector result(words.begin(), words.begin() + 5); + std::vector> variables; + bool inserted = false; + for (std::size_t cursor = 5; cursor < words.size();) { + const auto count = words[cursor] >> 16u; + const auto op = static_cast(words[cursor] & 0xffffu); + const auto operand = [&](std::size_t index) { return index < count ? words[cursor + index] : 0u; }; + if (op == spv::OpVariable && retyped.contains(operand(2u))) { + if (count != 4u) throw std::runtime_error("a packed fragment output has an initializer"); + std::vector variable(words.begin() + cursor, words.begin() + cursor + count); + variable[1] = pointerType; + variables.push_back(std::move(variable)); + cursor += count; + continue; + } + if (op == spv::OpDecorate && count == 4u && secondSource != 0u && operand(1u) == secondSource && operand(2u) == spv::DecorationLocation) { + result.insert(result.end(), {(4u << 16u) | spv::OpDecorate, secondSource, spv::DecorationLocation, 0u, (4u << 16u) | spv::OpDecorate, secondSource, spv::DecorationIndex, 1u}); + cursor += count; + continue; + } + if (op == spv::OpFunction && !inserted) { + for (const auto& declaration : declarations) result.insert(result.end(), declaration.begin(), declaration.end()); + for (const auto& variable : variables) result.insert(result.end(), variable.begin(), variable.end()); + inserted = true; + } + if (op == spv::OpStore && count >= 3u && retyped.contains(operand(1u))) { + if (bound == std::numeric_limits::max()) throw std::runtime_error("fragment output specialization module ID overflow"); + const auto bits = bound++; + result.insert(result.end(), {(4u << 16u) | spv::OpBitcast, vectorType, bits, operand(2u)}); + result.insert(result.end(), words.begin() + cursor, words.begin() + cursor + count); + result[result.size() - count + 2u] = bits; + cursor += count; + continue; + } + const bool pointerUse = ((op == spv::OpLoad || op == spv::OpAccessChain || op == spv::OpInBoundsAccessChain || op == spv::OpPtrAccessChain || op == spv::OpInBoundsPtrAccessChain || op == spv::OpCopyObject) && retyped.contains(operand(3u))) || + ((op == spv::OpCopyMemory || op == spv::OpCopyMemorySized) && (retyped.contains(operand(1u)) || retyped.contains(operand(2u)))); + if (pointerUse) throw std::runtime_error("a packed fragment output is accessed other than by whole stores"); + result.insert(result.end(), words.begin() + cursor, words.begin() + cursor + count); + cursor += count; + } + if (!inserted) throw std::runtime_error("fragment output specialization module has no function"); + result[3] = bound; + return result; +} + +} + +#endif diff --git a/core/shader/recompiler/IntermediateRepresentation/include/IntermediateRepresentation/IrMetadata/ControlFlowInfo.hpp b/core/shader/recompiler/IntermediateRepresentation/include/IntermediateRepresentation/IrMetadata/ControlFlowInfo.hpp index 5447bef4e..eed5b9551 100644 --- a/core/shader/recompiler/IntermediateRepresentation/include/IntermediateRepresentation/IrMetadata/ControlFlowInfo.hpp +++ b/core/shader/recompiler/IntermediateRepresentation/include/IntermediateRepresentation/IrMetadata/ControlFlowInfo.hpp @@ -50,6 +50,14 @@ struct SrtRead { bool operator==(const SrtRead& other) const = default; }; +struct SrtReadPoison { + std::uint32_t slot = 0; + std::uint32_t pc = 0; + std::uint64_t address = 0; + + bool operator==(const SrtReadPoison& other) const = default; +}; + struct ResourceBlock { IrValue* condition = nullptr; std::vector successors; diff --git a/core/shader/recompiler/IntermediateRepresentation/include/IntermediateRepresentation/IrMetadata/DescriptorBinding.hpp b/core/shader/recompiler/IntermediateRepresentation/include/IntermediateRepresentation/IrMetadata/DescriptorBinding.hpp index 91952766e..c1a9500b7 100644 --- a/core/shader/recompiler/IntermediateRepresentation/include/IntermediateRepresentation/IrMetadata/DescriptorBinding.hpp +++ b/core/shader/recompiler/IntermediateRepresentation/include/IntermediateRepresentation/IrMetadata/DescriptorBinding.hpp @@ -58,6 +58,9 @@ struct IrBindingLayout { [[nodiscard]] bool UsesPushData() const { return pushDataStartDword != PushData::NoStart; } + [[nodiscard]] std::uint32_t PushSlotDword() const { + return UsesPushData() ? pushDataStartDword / PushData::DwordCount * PushData::DwordCount : 0u; + } void AdvancePushData(std::uint32_t& cursor) const { if (UsesPushData()) { cursor = pushDataStartDword + ShaderDataDwords(); diff --git a/core/shader/recompiler/IntermediateRepresentation/include/IntermediateRepresentation/IrMetadata/ProgramMetadata.hpp b/core/shader/recompiler/IntermediateRepresentation/include/IntermediateRepresentation/IrMetadata/ProgramMetadata.hpp index 99cf118a7..adfeafae6 100644 --- a/core/shader/recompiler/IntermediateRepresentation/include/IntermediateRepresentation/IrMetadata/ProgramMetadata.hpp +++ b/core/shader/recompiler/IntermediateRepresentation/include/IntermediateRepresentation/IrMetadata/ProgramMetadata.hpp @@ -19,6 +19,7 @@ struct IrProgramMetadata { std::vector exportInfo; std::uint32_t pixelLinearInputs = 0; std::uint32_t pixelPerspectiveInputs = 0; + bool barycentricEmulation = false; std::vector dynamicReads; bool shaderInfoComplete = false; IrBindingLayout bindings; diff --git a/core/shader/recompiler/IntermediateRepresentation/include/IntermediateRepresentation/IrMetadata/ResourceInfo.hpp b/core/shader/recompiler/IntermediateRepresentation/include/IntermediateRepresentation/IrMetadata/ResourceInfo.hpp index a8216cc16..7c8d5d26d 100644 --- a/core/shader/recompiler/IntermediateRepresentation/include/IntermediateRepresentation/IrMetadata/ResourceInfo.hpp +++ b/core/shader/recompiler/IntermediateRepresentation/include/IntermediateRepresentation/IrMetadata/ResourceInfo.hpp @@ -50,6 +50,7 @@ struct MemoryInfo { bool planningOnly = false; bool coherent = false; bool gpuDescriptor = false; + bool flushDenormals = false; bool operator==(const MemoryInfo& other) const = default; }; diff --git a/core/shader/recompiler/IntermediateRepresentation/include/IntermediateRepresentation/IrMetadata/ResourcePlan.hpp b/core/shader/recompiler/IntermediateRepresentation/include/IntermediateRepresentation/IrMetadata/ResourcePlan.hpp index a59dbab67..3f1b27f6d 100644 --- a/core/shader/recompiler/IntermediateRepresentation/include/IntermediateRepresentation/IrMetadata/ResourcePlan.hpp +++ b/core/shader/recompiler/IntermediateRepresentation/include/IntermediateRepresentation/IrMetadata/ResourcePlan.hpp @@ -41,6 +41,7 @@ struct ResourceSnapshot { std::vector flattenedSrt; std::vector userData; UniformFill uniformFill; + std::vector srtPoison; }; struct UniformFillPlan { @@ -48,7 +49,8 @@ struct UniformFillPlan { std::array values{}; }; -inline constexpr std::uint32_t NativePushConstantSize = sizeof(PushData); +inline constexpr std::uint32_t NativePushSlotSize = sizeof(PushData); +inline constexpr std::uint32_t NativePushConstantSize = 2u * NativePushSlotSize; struct IrResourcePlan { IrShaderStage stage = IrShaderStage::Unknown; @@ -63,6 +65,8 @@ struct IrResourcePlan { std::vector controlFlow; std::vector materializationSources; std::vector srtReads; + std::vector guardedSrtSlots; + std::uint32_t srtGuardOffset = 0; std::vector cleanFlatSlots; // One byte per srtReads slot, 1 when the CPU walk never consumes the slot's value (see // Detail::ComputePureFlatSlots): a driver may reuse a capture whose words differ only there. diff --git a/core/shader/recompiler/IntermediateRepresentation/include/IntermediateRepresentation/IrMetadata/Resources.hpp b/core/shader/recompiler/IntermediateRepresentation/include/IntermediateRepresentation/IrMetadata/Resources.hpp index ec2d0c79f..882457f94 100644 --- a/core/shader/recompiler/IntermediateRepresentation/include/IntermediateRepresentation/IrMetadata/Resources.hpp +++ b/core/shader/recompiler/IntermediateRepresentation/include/IntermediateRepresentation/IrMetadata/Resources.hpp @@ -83,6 +83,10 @@ struct ImageResource { bool packed = false; bool fmaskCompatible = true; bool depthBitsCompatible = true; + bool constantSwizzle = false; + bool constantSwizzleCompatible = true; + bool flatVolumeCompatible = true; + bool flatLineCompatible = true; std::uint32_t byElements = 0; std::uint32_t byComponents = 0; IrBufferFormat packedFormat = IrBufferFormat::Invalid; diff --git a/core/shader/recompiler/IntermediateRepresentation/include/IntermediateRepresentation/IrMetadata/ShaderInfo.hpp b/core/shader/recompiler/IntermediateRepresentation/include/IntermediateRepresentation/IrMetadata/ShaderInfo.hpp index 1a3f4f790..aa1f183ee 100644 --- a/core/shader/recompiler/IntermediateRepresentation/include/IntermediateRepresentation/IrMetadata/ShaderInfo.hpp +++ b/core/shader/recompiler/IntermediateRepresentation/include/IntermediateRepresentation/IrMetadata/ShaderInfo.hpp @@ -33,11 +33,16 @@ struct ShaderInfo { bool hasBitwiseXor = false; bool usesDma = false; bool bdaWrites = false; + bool usesFaultBuffer = false; bool dispatchThreadLimit = false; bool operator==(const ShaderInfo& other) const = default; }; +[[nodiscard]] inline std::uint32_t WorkgroupMemoryStrideDwords(const ShaderInfo& info) { + return info.sharedMemoryBytes == 0u ? 0u : info.sharedMemoryBytes / 4u + 1u; +} + } #endif diff --git a/core/shader/recompiler/IntermediateRepresentation/include/IntermediateRepresentation/IrOpcode.hpp b/core/shader/recompiler/IntermediateRepresentation/include/IntermediateRepresentation/IrOpcode.hpp index d223ca696..f08b2da82 100644 --- a/core/shader/recompiler/IntermediateRepresentation/include/IntermediateRepresentation/IrOpcode.hpp +++ b/core/shader/recompiler/IntermediateRepresentation/include/IntermediateRepresentation/IrOpcode.hpp @@ -293,6 +293,9 @@ enum class IrOpcode : std::uint16_t { StoreAddressU8, StoreAddressU16, StoreAddressU32, + StoreAddressU32x2, + StoreAddressU32x3, + StoreAddressU32x4, AddressAtomicSwap32, AddressAtomicCmpSwap32, AddressAtomicIAdd32, @@ -460,6 +463,8 @@ enum class IrOpcode : std::uint16_t { GetAttribute, GetInterpolationParameter, GetInterpolationParameterF16, + InterpolateHostP1, + InterpolateHostP2, SetAttribute, ControlNop, Waitcnt, diff --git a/core/shader/recompiler/IntermediateRepresentation/src/IrOpcode.cpp b/core/shader/recompiler/IntermediateRepresentation/src/IrOpcode.cpp index a06c8e9f4..ec6ddd79e 100644 --- a/core/shader/recompiler/IntermediateRepresentation/src/IrOpcode.cpp +++ b/core/shader/recompiler/IntermediateRepresentation/src/IrOpcode.cpp @@ -334,6 +334,9 @@ constexpr std::array(IrOpcode::Count)> Meta makeMeta("StoreAddressU8", Void, AddressResource, U32, U32, U8, U1), makeMeta("StoreAddressU16", Void, AddressResource, U32, U32, U16, U1), makeMeta("StoreAddressU32", Void, AddressResource, U32, U32, U32, U1), + makeMeta("StoreAddressU32x2", Void, AddressResource, U32, U32, U32x2, U1), + makeMeta("StoreAddressU32x3", Void, AddressResource, U32, U32, U32x3, U1), + makeMeta("StoreAddressU32x4", Void, AddressResource, U32, U32, U32x4, U1), makeMeta("AddressAtomicSwap32", U32, AddressResource, U32, U32, U32, U1), makeMeta("AddressAtomicCmpSwap32", U32, AddressResource, U32, U32, U32, U32, U1), makeMeta("AddressAtomicIAdd32", U32, AddressResource, U32, U32, U32, U1), @@ -501,6 +504,8 @@ constexpr std::array(IrOpcode::Count)> Meta makeMeta("GetAttribute", U32, U32, U32), makeMeta("GetInterpolationParameter", U32, U32, U32, U32), makeMeta("GetInterpolationParameterF16", F32, U32, U32, U32, U32), + makeMeta("InterpolateHostP1", F32, U32, U32, F32), + makeMeta("InterpolateHostP2", Void, U32, U32, F32, F32, U1), makeMeta("SetAttribute", Void, U32x4, U1), makeMeta("ControlNop", Void), makeMeta("Waitcnt", Void), @@ -747,6 +752,12 @@ AddressOpcodeInfo AddressOpcodeInfoOf(IrOpcode opcode) { return {AddressAccess::Write, 16u}; case IrOpcode::StoreAddressU32: return {AddressAccess::Write, 32u}; + case IrOpcode::StoreAddressU32x2: + return {AddressAccess::Write, 32u, 2u}; + case IrOpcode::StoreAddressU32x3: + return {AddressAccess::Write, 32u, 3u}; + case IrOpcode::StoreAddressU32x4: + return {AddressAccess::Write, 32u, 4u}; case IrOpcode::AddressAtomicSwap32: case IrOpcode::AddressAtomicCmpSwap32: case IrOpcode::AddressAtomicIAdd32: @@ -873,6 +884,7 @@ bool IrOpcodeHasSideEffects(IrOpcode opcode) { case IrOpcode::Reference: case IrOpcode::ReferenceU32: case IrOpcode::SetTessellationAttribute: + case IrOpcode::InterpolateHostP2: case IrOpcode::SetThreadBitScalarRegister: case IrOpcode::SetScalarMaskTag: case IrOpcode::SetScalarRegister: diff --git a/core/shader/recompiler/Optimization/include/Optimization/DenormalFlushEliminator.hpp b/core/shader/recompiler/Optimization/include/Optimization/DenormalFlushEliminator.hpp new file mode 100644 index 000000000..a75f0be76 --- /dev/null +++ b/core/shader/recompiler/Optimization/include/Optimization/DenormalFlushEliminator.hpp @@ -0,0 +1,20 @@ +#ifndef CORE_SHADER_RECOMPILIER_OPTIMIZATION_INCLUDE_OPTIMIZATION_DENORMALFLUSHELIMINATOR_HPP +#define CORE_SHADER_RECOMPILIER_OPTIMIZATION_INCLUDE_OPTIMIZATION_DENORMALFLUSHELIMINATOR_HPP + +#include "IntermediateRepresentation/IrProgram.hpp" +#include + +namespace ShaderRecompiler { + +struct DenormalFlushEliminationStats { + std::uint32_t removedFlushes = 0; +}; + +class DenormalFlushEliminator { +public: + [[nodiscard]] DenormalFlushEliminationStats Eliminate(IrProgram& program) const; +}; + +} + +#endif diff --git a/core/shader/recompiler/Optimization/include/Optimization/HostInterpolationChecker.hpp b/core/shader/recompiler/Optimization/include/Optimization/HostInterpolationChecker.hpp new file mode 100644 index 000000000..a909f6440 --- /dev/null +++ b/core/shader/recompiler/Optimization/include/Optimization/HostInterpolationChecker.hpp @@ -0,0 +1,16 @@ +#ifndef CORE_SHADER_RECOMPILIER_OPTIMIZATION_INCLUDE_OPTIMIZATION_HOSTINTERPOLATIONCHECKER_HPP +#define CORE_SHADER_RECOMPILIER_OPTIMIZATION_INCLUDE_OPTIMIZATION_HOSTINTERPOLATIONCHECKER_HPP + +#include "IntermediateRepresentation/IrProgram.hpp" +#include "Optimization/ShaderStageInputInfo.hpp" + +namespace ShaderRecompiler { + +class HostInterpolationChecker { +public: + [[nodiscard]] bool Lower(IrProgram& program, const ShaderPixelInputInfo& pixel) const; +}; + +} + +#endif diff --git a/core/shader/recompiler/Optimization/include/Optimization/RequestMemoryView.hpp b/core/shader/recompiler/Optimization/include/Optimization/RequestMemoryView.hpp index 5abfd7503..321474d50 100644 --- a/core/shader/recompiler/Optimization/include/Optimization/RequestMemoryView.hpp +++ b/core/shader/recompiler/Optimization/include/Optimization/RequestMemoryView.hpp @@ -16,6 +16,7 @@ public: private: static bool ReadGuestMemory(void* userContext, std::uint64_t address, std::uint32_t* value); + static bool Covered(void* userContext, std::uint64_t address, std::uint64_t bytes); std::vector regions; }; diff --git a/core/shader/recompiler/Optimization/include/Optimization/ResourceProgram.hpp b/core/shader/recompiler/Optimization/include/Optimization/ResourceProgram.hpp index a0c470a7b..f23d1b8f6 100644 --- a/core/shader/recompiler/Optimization/include/Optimization/ResourceProgram.hpp +++ b/core/shader/recompiler/Optimization/include/Optimization/ResourceProgram.hpp @@ -9,7 +9,17 @@ namespace ShaderRecompiler { +struct PreparedControlFlow; +class ShaderPreparationContext { +public: + [[nodiscard]] std::shared_ptr AcquireFrontend(const RecompileRequest& request); + +private: + std::shared_ptr frontend; +}; + [[nodiscard]] IrProgram PrepareResourceProgram(const RecompileRequest& request); +[[nodiscard]] IrProgram PrepareResourceProgram(const RecompileRequest& request, ShaderPreparationContext* preparation); [[nodiscard]] std::shared_ptr GetResourcePlan(const RecompileRequest& request); struct SourceEntry; @@ -55,6 +65,7 @@ private: }; [[nodiscard]] std::span GetPreparedCode(const SourceHandle& handle); [[nodiscard]] std::shared_ptr PrepareShader(const RecompileRequest& request); +[[nodiscard]] std::shared_ptr PrepareShader(const RecompileRequest& request, ShaderPreparationContext* preparation); [[nodiscard]] bool MatchesPreparedShader(const RecompileRequest& request, const SourceHandle& handle); void BuildPreparedShaderKey(const RecompileRequest& request, std::vector& key); [[nodiscard]] bool MatchesPreparedShader(const RecompileRequest& request, const SourceHandle& handle, std::span key); diff --git a/core/shader/recompiler/Optimization/include/Optimization/ShaderStageInputInfo.hpp b/core/shader/recompiler/Optimization/include/Optimization/ShaderStageInputInfo.hpp index 996730b67..8f72d9821 100644 --- a/core/shader/recompiler/Optimization/include/Optimization/ShaderStageInputInfo.hpp +++ b/core/shader/recompiler/Optimization/include/Optimization/ShaderStageInputInfo.hpp @@ -191,6 +191,10 @@ struct ShaderPixelInputInfo { return input < 32u && ((customInterpolationMask & (1u << input)) != 0u || InputIsPassthrough(input)); } + [[nodiscard]] bool InputIsFlat(std::uint32_t input) const { + return input < inputNum && input < 32u && (interpolatorSettings[input] & 0x400u) != 0u && !InputIsCustom(input); + } + [[nodiscard]] std::uint32_t InputSlot(std::uint32_t input) const { return input < 32u ? interpolatorSettings[input] & 0x1fu : input; } diff --git a/core/shader/recompiler/Optimization/include/Optimization/SrtWalker.hpp b/core/shader/recompiler/Optimization/include/Optimization/SrtWalker.hpp index f5f5a4070..16cdedac5 100644 --- a/core/shader/recompiler/Optimization/include/Optimization/SrtWalker.hpp +++ b/core/shader/recompiler/Optimization/include/Optimization/SrtWalker.hpp @@ -11,6 +11,7 @@ namespace ShaderRecompiler { using SrtMemoryReader = bool (*)(void* userData, std::uint64_t address, std::uint32_t* value); +using SrtMemoryProbe = bool (*)(void* userData, std::uint64_t address, std::uint64_t bytes); // The guest addresses a walk dereferenced (Detail::Evaluator::EvaluateRawRead): the leaf read of // each pure flat slot (IrResourcePlan::pureFlatSlots) as (flat offset, address), set by the @@ -30,6 +31,7 @@ struct SrtRuntime { void* userContext = nullptr; SrtMemoryReader readSpecializationMemory = nullptr; SrtReadTrace* readTrace = nullptr; + SrtMemoryProbe accessible = nullptr; }; enum class RuntimeValueType { @@ -44,7 +46,7 @@ public: void EvaluateUniformValues(const IrResourcePlan& program, std::span values, const SrtRuntime& runtime, std::span results) const; void EvaluateDescriptorSource(const IrResourcePlan& program, std::uint32_t source, const SrtRuntime& runtime, DescriptorValue& result) const; void EvaluateDescriptorSources(const IrResourcePlan& program, std::span sources, const SrtRuntime& runtime, std::vector& results) const; - void EvaluateRuntimeSources(const IrResourcePlan& program, std::span sources, const SrtRuntime& runtime, std::vector& results, std::vector& flat, std::span cleanFlatSlots, std::vector& activeSources) const; + void EvaluateRuntimeSources(const IrResourcePlan& program, std::span sources, const SrtRuntime& runtime, std::vector& results, std::vector& flat, std::span cleanFlatSlots, std::vector& activeSources, std::vector* poison = nullptr) const; void Walk(const IrResourcePlan& program, const SrtRuntime& runtime, std::vector& flat) const; }; diff --git a/core/shader/recompiler/Optimization/include/Optimization/SrtWalker/SrtDescriptorEvaluation.hpp b/core/shader/recompiler/Optimization/include/Optimization/SrtWalker/SrtDescriptorEvaluation.hpp index c1b008e9b..eb9c568ad 100644 --- a/core/shader/recompiler/Optimization/include/Optimization/SrtWalker/SrtDescriptorEvaluation.hpp +++ b/core/shader/recompiler/Optimization/include/Optimization/SrtWalker/SrtDescriptorEvaluation.hpp @@ -15,7 +15,7 @@ namespace ShaderRecompiler::Detail { // Why the last EvaluateRuntimeSourcesImpl call on this thread returned false. const std::string& RuntimeSourceFailureReason(); -bool EvaluateRuntimeSourcesImpl(const IrResourcePlan& program, std::span sources, const SrtRuntime& runtime, std::vector& results, std::vector& flat, bool evaluateFlat, std::span cleanFlatSlots, std::vector& activeSources); +bool EvaluateRuntimeSourcesImpl(const IrResourcePlan& program, std::span sources, const SrtRuntime& runtime, std::vector& results, std::vector& flat, bool evaluateFlat, std::span cleanFlatSlots, std::vector& activeSources, std::vector* poison = nullptr); } diff --git a/core/shader/recompiler/Optimization/include/Optimization/SrtWalker/SrtEvaluator.hpp b/core/shader/recompiler/Optimization/include/Optimization/SrtWalker/SrtEvaluator.hpp index 7877c5928..269bfceb4 100644 --- a/core/shader/recompiler/Optimization/include/Optimization/SrtWalker/SrtEvaluator.hpp +++ b/core/shader/recompiler/Optimization/include/Optimization/SrtWalker/SrtEvaluator.hpp @@ -66,12 +66,18 @@ private: std::size_t _count = 0; }; +struct InaccessibleRead { + const IrValue* read = nullptr; + std::uint64_t address = 0; +}; + class Evaluator { public: Evaluator(const IrResourcePlan& program, const SrtRuntime& runtime, std::span cleanFlatSlots = {}, Evaluator* cleanEvaluator = nullptr, IrValue* activeMask = nullptr) : _program(program), _runtime(runtime), _cleanFlatSlots(cleanFlatSlots), _cleanEvaluator(cleanEvaluator), _activeMask(activeMask != nullptr ? activeMask->Resolve() : nullptr) {} bool Evaluate(IrValue* value, std::uint32_t& result); bool EvaluateWide(IrValue* raw, std::uint64_t& result); + void ReportInaccessibleReads(InaccessibleRead* sink) { _inaccessible = sink; } private: static float Float32(std::uint64_t bits); @@ -88,6 +94,7 @@ private: std::span _cleanFlatSlots; Evaluator* _cleanEvaluator = nullptr; IrValue* _activeMask = nullptr; + InaccessibleRead* _inaccessible = nullptr; EvaluatedValues _cache; std::vector _visiting; }; diff --git a/core/shader/recompiler/Optimization/include/Optimization/SrtWalker/SrtFlatSlotClasses.hpp b/core/shader/recompiler/Optimization/include/Optimization/SrtWalker/SrtFlatSlotClasses.hpp index f1f806f08..85bd0e7ab 100644 --- a/core/shader/recompiler/Optimization/include/Optimization/SrtWalker/SrtFlatSlotClasses.hpp +++ b/core/shader/recompiler/Optimization/include/Optimization/SrtWalker/SrtFlatSlotClasses.hpp @@ -16,6 +16,7 @@ namespace ShaderRecompiler::Detail { // fill) has no pure slot. Reachability over-approximates what the evaluators consume, so an // unknown opcode only makes more slots impure. std::vector ComputePureFlatSlots(const IrResourcePlan& plan); +std::vector ComputeGuardedFlatSlots(const IrResourcePlan& plan); } diff --git a/core/shader/recompiler/Optimization/include/Optimization/SrtWalker/SrtInstructionPredicates.hpp b/core/shader/recompiler/Optimization/include/Optimization/SrtWalker/SrtInstructionPredicates.hpp index e4ab1ce06..00c598e3b 100644 --- a/core/shader/recompiler/Optimization/include/Optimization/SrtWalker/SrtInstructionPredicates.hpp +++ b/core/shader/recompiler/Optimization/include/Optimization/SrtWalker/SrtInstructionPredicates.hpp @@ -6,6 +6,7 @@ namespace ShaderRecompiler::Detail { bool IsRawRead(const IrResourcePlan& program, const IrValue& inst); +bool LoadedFromMemory(const IrValue& handle); bool IsDescriptorHandle(IrOpcode opcode); bool IsRuntimeSelect(IrOpcode opcode); bool IsRuntimeUniformOp(IrOpcode opcode); diff --git a/core/shader/recompiler/Optimization/src/BindingAllocator.cpp b/core/shader/recompiler/Optimization/src/BindingAllocator.cpp index 7d18b8a79..f99616a96 100644 --- a/core/shader/recompiler/Optimization/src/BindingAllocator.cpp +++ b/core/shader/recompiler/Optimization/src/BindingAllocator.cpp @@ -89,6 +89,9 @@ BindingAllocationResult BindingAllocator::Allocate(IrProgram& program, const Bin if (layout.pushConstantOffsetBytes + layout.pushConstantSizeBytes > NativePushConstantSize) { fail("shader binding layout failed: push constant range exceeds the native push constant size"); } + if (layout.pushConstantSizeBytes != 0u && layout.pushConstantOffsetBytes / NativePushSlotSize != (layout.pushConstantOffsetBytes + layout.pushConstantSizeBytes - 1u) / NativePushSlotSize) { + fail("shader binding layout failed: push constant range crosses a stage slot"); + } const ShaderInfo& info = program.Resources().info; @@ -114,7 +117,7 @@ BindingAllocationResult BindingAllocator::Allocate(IrProgram& program, const Bin std::array, ImageBindingCount> imageGroups; const auto place = [&](std::uint32_t i) { const bool dynamic = info.images[i].mipMode == ImageMipMode::DynamicStorage; - const std::uint32_t count = dynamic ? RuntimeAbi::StorageHeapCapacity : 1u; + const std::uint32_t count = dynamic ? RuntimeAbi::StorageMipSlots : 1u; std::array placed{}; for (const auto& mode : ResourceMaterializer::RuntimeImageModes(info.images[i])) { const auto group = ImageBindingIndex(DescriptorBindingForImage(mode)); @@ -161,6 +164,8 @@ BindingAllocationResult BindingAllocator::Allocate(IrProgram& program, const Bin } if (info.usesDma) { addBinding(next, DescriptorBindingKind::BdaPagetable); + } + if (info.usesDma || info.usesFaultBuffer) { addBinding(next, DescriptorBindingKind::FaultBuffer); } diff --git a/core/shader/recompiler/Optimization/src/ConstantFolder.cpp b/core/shader/recompiler/Optimization/src/ConstantFolder.cpp index a05a6c7a8..a37056fc7 100644 --- a/core/shader/recompiler/Optimization/src/ConstantFolder.cpp +++ b/core/shader/recompiler/Optimization/src/ConstantFolder.cpp @@ -5,6 +5,7 @@ #include #include #include +#include #include #include @@ -53,6 +54,28 @@ template bool foldU64Shift(IrBuilder& builder, IrValue& ins return true; } +bool foldConstantBitTest(IrBuilder& builder, IrValue& inst) { + auto& left = resolveArg(inst, 0); + auto& right = resolveArg(inst, 1); + if (!isImmediate(right, IrType::U32) || right.ImmediateU32() != 0u || left.Opcode() != IrOpcode::BitwiseAnd32) return false; + auto& first = resolveArg(left, 0); + auto& second = resolveArg(left, 1); + IrValue* shifted = nullptr; + if (isImmediate(second, IrType::U32) && second.ImmediateU32() == 1u) shifted = &first; + else if (isImmediate(first, IrType::U32) && first.ImmediateU32() == 1u) shifted = &second; + if (shifted == nullptr || shifted->Opcode() != IrOpcode::ShiftRightLogical32) return false; + auto& word = resolveArg(*shifted, 0); + auto& amount = resolveArg(*shifted, 1); + if (!isImmediate(word, IrType::U32) || (word.ImmediateU32() != 0u && word.ImmediateU32() != 0xffffffffu)) return false; + if (amount.Opcode() != IrOpcode::BitwiseAnd32) return false; + auto& amountLeft = resolveArg(amount, 0); + auto& amountRight = resolveArg(amount, 1); + const bool masked = (isImmediate(amountLeft, IrType::U32) && amountLeft.ImmediateU32() == 31u) || (isImmediate(amountRight, IrType::U32) && amountRight.ImmediateU32() == 31u); + if (!masked) return false; + replaceWith(inst, builder.ConstantBool(word.ImmediateU32() != 0u)); + return true; +} + template bool foldU32Compare(IrBuilder& builder, IrValue& inst, TFunction function) { auto& lhs = resolveArg(inst, 0); auto& rhs = resolveArg(inst, 1); @@ -184,23 +207,87 @@ bool foldCompositeExtract(IrBuilder& builder, IrValue& inst, IrOpcode construct, return false; } -void collectUses(IrValue& value, std::vector& uses) { - for (const IrUse& use : value.OperandUses()) { - if (use.user->Opcode() == IrOpcode::Identity) { - collectUses(*use.user, uses); +bool forwardsWord(const IrUse& use) { + return use.user->Opcode() == IrOpcode::Identity || use.user->Opcode() == IrOpcode::Phi || (use.user->Opcode() == IrOpcode::SelectU32 && use.operand != 0u); +} + +void collectReaders(IrValue& value, std::vector& readers, std::vector& visited) { + if (std::ranges::find(visited, &value) != visited.end()) { + return; + } + visited.push_back(&value); + const std::vector uses = value.OperandUses(); + for (const IrUse& use : uses) { + if (forwardsWord(use)) { + collectReaders(*use.user, readers, visited); } else { - uses.push_back(use); + readers.push_back(use); } } } -bool lowerPackedAncillary(IrProgram& program, IrBuilder& builder, IrValue& ancillary) { +void collectReaders(IrValue& value, std::vector& readers) { + std::vector visited; + collectReaders(value, readers, visited); +} + +constexpr std::array, 2> packedFields {{{8u, 4u}, {16u, 13u}}}; + +std::optional packedField(std::uint32_t offset, std::uint32_t count) { + const auto field = std::ranges::find_if(packedFields, [&](const auto& candidate) { + return offset >= candidate.first && count <= candidate.second && offset - candidate.first <= candidate.second - count; + }); + if (field == packedFields.end()) { + return std::nullopt; + } + return static_cast(field - packedFields.begin()); +} + +void lowerPackedAncillary(IrProgram& program, IrBuilder& builder, IrValue& ancillary) { auto& kind = resolveArg(ancillary, 0); if (!isImmediate(kind, IrType::U32) || static_cast(kind.ImmediateU32()) != StageInputKind::PackedAncillary) { - return false; + return; + } + const std::vector uses = ancillary.OperandUses(); + std::vector direct; + std::vector forwarded; + for (const IrUse& use : uses) { + if (forwardsWord(use)) { + forwarded.push_back(use); + } else { + direct.push_back(use); + } + } + std::vector readers = direct; + for (const IrUse& use : forwarded) { + collectReaders(*use.user, readers); + } + const auto extractsField = [](const IrUse& use) { + const IrValue& user = *use.user; + if ((user.Opcode() != IrOpcode::BitFieldUExtract && user.Opcode() != IrOpcode::BitFieldSExtract) || use.operand != 0u) { + return false; + } + auto& offset = resolveArg(user, 1); + auto& count = resolveArg(user, 2); + return isImmediate(offset, IrType::U32) && isImmediate(count, IrType::U32) && count.ImmediateU32() != 0u && + packedField(offset.ImmediateU32(), count.ImmediateU32()).has_value(); + }; + const bool whole = std::ranges::all_of(readers, extractsField); + if (!whole) { + std::vector extracts; + const auto collectExtracts = [&](const auto& self, IrValue& value) -> void { + for (const IrUse& use : value.OperandUses()) { + if (use.user->Opcode() == IrOpcode::Identity) { + self(self, *use.user); + } else if (extractsField(use)) { + extracts.push_back(use); + } + } + }; + collectExtracts(collectExtracts, ancillary); + direct = std::move(extracts); + forwarded.clear(); } - std::vector uses; - collectUses(ancillary, uses); std::array fields {}; const auto field = [&](std::size_t index) -> IrValue& { if (fields[index] == nullptr) { @@ -213,30 +300,29 @@ bool lowerPackedAncillary(IrProgram& program, IrBuilder& builder, IrValue& ancil } return *fields[index]; }; - constexpr std::array, 2> ranges {{{8u, 4u}, {16u, 13u}}}; - bool lowered = false; - for (const IrUse& use : uses) { - IrValue& user = *use.user; - if ((user.Opcode() != IrOpcode::BitFieldUExtract && user.Opcode() != IrOpcode::BitFieldSExtract) || use.operand != 0u) { - continue; + const auto emit = [&](IrOpcode opcode, IrValue& lhs, IrValue& rhs) -> IrValue& { + IrValue& created = program.CreateValue(opcode, IrType::U32); + created.AddArgument(&lhs); + created.AddArgument(&rhs); + ancillary.Parent()->InsertInstructionBefore(&ancillary, &created); + return created; + }; + if (!forwarded.empty()) { + IrValue& sample = emit(IrOpcode::ShiftLeftLogical32, field(0u), builder.Constant(8u)); + IrValue& layer = emit(IrOpcode::ShiftLeftLogical32, field(1u), builder.Constant(16u)); + IrValue& packed = emit(IrOpcode::BitwiseOr32, sample, layer); + for (const IrUse& use : forwarded) { + use.user->ReplaceArgument(use.operand, &packed); } + } + for (const IrUse& use : direct) { + IrValue& user = *use.user; auto& offset = resolveArg(user, 1); auto& count = resolveArg(user, 2); - if (!isImmediate(offset, IrType::U32) || !isImmediate(count, IrType::U32) || count.ImmediateU32() == 0u) { - continue; - } - const auto range = std::ranges::find_if(ranges, [&](const auto& candidate) { - return offset.ImmediateU32() >= candidate.first && count.ImmediateU32() <= candidate.second && - offset.ImmediateU32() - candidate.first <= candidate.second - count.ImmediateU32(); - }); - if (range == ranges.end()) { - continue; - } - user.ReplaceArgument(0, &field(static_cast(range - ranges.begin()))); - user.ReplaceArgument(1, &builder.Constant(offset.ImmediateU32() - range->first)); - lowered = true; + const auto index = *packedField(offset.ImmediateU32(), count.ImmediateU32()); + user.ReplaceArgument(0, &field(index)); + user.ReplaceArgument(1, &builder.Constant(offset.ImmediateU32() - packedFields[index].first)); } - return lowered; } } // namespace @@ -245,7 +331,6 @@ bool ConstantFolder::tryFoldValue(IrProgram& program, IrValue& value) const { IrBuilder builder(program); switch (value.Opcode()) { case IrOpcode::Phi: return foldPhi(value); - case IrOpcode::GetBuiltin: return lowerPackedAncillary(program, builder, value); case IrOpcode::SelectU1: case IrOpcode::SelectF32: case IrOpcode::SelectU32: return foldSelect(value); @@ -554,7 +639,7 @@ bool ConstantFolder::tryFoldValue(IrProgram& program, IrValue& value) const { return foldU32(builder, value, [](std::uint32_t a, std::uint32_t b) { return std::max(a, b); }); } case IrOpcode::IEqual32: return foldU32Compare(builder, value, [](std::uint32_t a, std::uint32_t b) { return a == b; }); - case IrOpcode::INotEqual32: return foldU32Compare(builder, value, [](std::uint32_t a, std::uint32_t b) { return a != b; }); + case IrOpcode::INotEqual32: return foldConstantBitTest(builder, value) || foldU32Compare(builder, value, [](std::uint32_t a, std::uint32_t b) { return a != b; }); case IrOpcode::ULessThan32: return foldU32Compare(builder, value, [](std::uint32_t a, std::uint32_t b) { return a < b; }); case IrOpcode::ULessThanEqual32: return foldU32Compare(builder, value, [](std::uint32_t a, std::uint32_t b) { return a <= b; }); case IrOpcode::UGreaterThan32: return foldU32Compare(builder, value, [](std::uint32_t a, std::uint32_t b) { return a > b; }); @@ -652,6 +737,18 @@ void ConstantFolder::Fold(IrProgram& program, std::span blocks) const auto success = tryFoldValue(program, *inst); } } + std::vector words; + for (IrBlock* block : blocks) { + for (IrValue* inst : block->Instructions()) { + if (inst->Opcode() == IrOpcode::GetBuiltin) { + words.push_back(inst); + } + } + } + IrBuilder builder(program); + for (IrValue* word : words) { + lowerPackedAncillary(program, builder, *word); + } } void ConstantFolder::Fold(IrProgram& program) const { diff --git a/core/shader/recompiler/Optimization/src/DenormalFlushEliminator.cpp b/core/shader/recompiler/Optimization/src/DenormalFlushEliminator.cpp new file mode 100644 index 000000000..7de56772e --- /dev/null +++ b/core/shader/recompiler/Optimization/src/DenormalFlushEliminator.cpp @@ -0,0 +1,100 @@ +#include "Optimization/DenormalFlushEliminator.hpp" +#include +#include + +namespace ShaderRecompiler { + +namespace { + +bool isImmediate(const IrValue* value, std::uint32_t bits) { + return value->HasImmediate() && value->ImmediateU32() == bits; +} + +const IrValue* maskedOperand(const IrValue* value, IrOpcode opcode, std::uint32_t bits) { + if (value->Opcode() != opcode || value->ArgumentCount() != 2u) return nullptr; + const IrValue* lhs = value->Argument(0)->Resolve(); + const IrValue* rhs = value->Argument(1)->Resolve(); + if (isImmediate(rhs, bits)) return lhs; + if (isImmediate(lhs, bits)) return rhs; + return nullptr; +} + +IrValue* flushedSource(const IrValue& value) { + if (value.Opcode() != IrOpcode::SelectU32 || value.ArgumentCount() != 3u) return nullptr; + IrValue* source = value.Argument(2)->Resolve(); + const IrValue* exponent = maskedOperand(value.Argument(0)->Resolve(), IrOpcode::IEqual32, 0u); + if (exponent == nullptr || maskedOperand(exponent, IrOpcode::BitwiseAnd32, 0x7f800000u) != source) return nullptr; + return maskedOperand(value.Argument(1)->Resolve(), IrOpcode::BitwiseAnd32, 0x80000000u) == source ? source : nullptr; +} + +class DenormalFreeValues { +public: + bool Contains(const IrValue* value) { + value = value->Resolve(); + if (const auto found = known.find(value); found != known.end()) return found->second; + known.emplace(value, false); + const bool result = compute(*value); + known[value] = result; + return result; + } + +private: + bool compute(const IrValue& value) { + if (value.HasImmediate()) { + const std::uint32_t bits = value.ImmediateU32(); + return (bits & 0x7f800000u) != 0u || (bits & 0x7fffffffu) == 0u; + } + if (flushedSource(value) != nullptr) return true; + switch (value.Opcode()) { + case IrOpcode::ConvertIToF32: + case IrOpcode::ConvertUToF32: + case IrOpcode::ConvertF32S32: + case IrOpcode::ConvertF32U32: + case IrOpcode::ConvertF32F16: + return true; + case IrOpcode::BitCastU32F32: + case IrOpcode::BitCastF32U32: + case IrOpcode::FPAbs32: + case IrOpcode::FPNeg32: + case IrOpcode::FPSaturate32: + return Contains(value.Argument(0)); + case IrOpcode::BitwiseXor32: + if (const IrValue* operand = maskedOperand(&value, IrOpcode::BitwiseXor32, 0x80000000u)) return Contains(operand); + return false; + case IrOpcode::BitwiseAnd32: + if (const IrValue* operand = maskedOperand(&value, IrOpcode::BitwiseAnd32, 0x7fffffffu)) return Contains(operand); + return false; + case IrOpcode::SelectU32: + case IrOpcode::SelectF32: + return Contains(value.Argument(1)) && Contains(value.Argument(2)); + case IrOpcode::Phi: + for (std::size_t index = 0; index < value.ArgumentCount(); ++index) { + if (!Contains(value.Argument(index))) return false; + } + return true; + default: + return false; + } + } + + std::unordered_map known; +}; + +} + +DenormalFlushEliminationStats DenormalFlushEliminator::Eliminate(IrProgram& program) const { + DenormalFlushEliminationStats stats; + DenormalFreeValues denormalFree; + for (auto* block : program.BlockOrder()) { + for (auto* inst : block->Instructions()) { + if (!inst->HasUses()) continue; + IrValue* source = flushedSource(*inst); + if (source == nullptr || !denormalFree.Contains(source)) continue; + inst->ReplaceAllUsesWith(source); + ++stats.removedFlushes; + } + } + return stats; +} + +} diff --git a/core/shader/recompiler/Optimization/src/DescriptorBindingBuilder.cpp b/core/shader/recompiler/Optimization/src/DescriptorBindingBuilder.cpp index 0005d4c46..7455a1384 100644 --- a/core/shader/recompiler/Optimization/src/DescriptorBindingBuilder.cpp +++ b/core/shader/recompiler/Optimization/src/DescriptorBindingBuilder.cpp @@ -187,6 +187,9 @@ UnnormalizedProof ProveUnnormalized(const ShaderInfo& info, const ResourceSnapsh if (image.indirectRoot != ImageResource::NoIndirectImage) { failUnnormalized("samples an image selected at run time"); } + if (image.constantSwizzle) { + continue; + } if ((image.dimension != RdnaImageDimension::Dim1D && image.dimension != RdnaImageDimension::Dim2D) || image.cube) { failUnnormalized("samples a 1D-array, 2D-array, 3D, cube or multisampled image"); } @@ -295,7 +298,7 @@ DescriptorBindingPlan DescriptorBindingBuilder::Prepare(const IrBindingLayout& l for (std::uint32_t component = 0; component < 4u; ++component) plan.specialization.push_back({PipelineSpecialization::ExportBase + target * 4u + component, (exportMappings[target] >> (component * 2u)) & 3u}); } std::vector samplerModes(info.samplers.size(), 0u); - const auto samplerMode = [](const ImageResource& image) { return image.numericClass == IrTextureNumericClass::Sint || image.conversionFormat != IrBufferFormat::Invalid || image.depthBits ? 2u : 1u; }; + const auto samplerMode = [](const ImageResource& image) { return !image.constantSwizzle && (image.numericClass == IrTextureNumericClass::Sint || image.conversionFormat != IrBufferFormat::Invalid || image.depthBits) ? 2u : 1u; }; for (const auto& pair : info.sampledPairs) { const auto& image = info.images.at(pair.image); if (image.indirectRoot == ImageResource::NoIndirectImage) samplerModes.at(pair.sampler) |= samplerMode(info.runtimeImageModes.at(pair.image).at(imageModes[pair.image])); @@ -309,7 +312,7 @@ DescriptorBindingPlan DescriptorBindingBuilder::Prepare(const IrBindingLayout& l const auto word = snapshot.images.at(index).dwords[3]; const auto first = (word >> 12u) & 0xfu; const auto last = (word >> 16u) & 0xfu; - if (last < first || last - first >= RuntimeAbi::StorageHeapCapacity) fail("invalid dynamic storage mip range"); + if (last < first || last - first >= RuntimeAbi::StorageMipSlots) fail("invalid dynamic storage mip range"); plan.specialization.push_back({PipelineSpecialization::MipCountBase + index, last - first + 1u}); } for (std::uint32_t resource = 0; resource < info.images.size(); ++resource) { @@ -343,7 +346,7 @@ DescriptorBindingPlan DescriptorBindingBuilder::Prepare(const IrBindingLayout& l previous = resource; { const auto word = snapshot.images.at(resource).dwords[3]; - active = DescriptorBindingForImage(mode) == logical.kind && mode.packedFormat != IrBufferFormat::Fmask8_S2_F1 && (image.mipMode != ImageMipMode::DynamicStorage || mip <= ((word >> 16u) & 0xfu) - ((word >> 12u) & 0xfu)); + active = DescriptorBindingForImage(mode) == logical.kind && mode.packedFormat != IrBufferFormat::Fmask8_S2_F1 && !mode.constantSwizzle && (image.mipMode != ImageMipMode::DynamicStorage || mip <= ((word >> 16u) & 0xfu) - ((word >> 12u) & 0xfu)); } } else if (samplerHeap) { active = (samplerModes.at(resource) & (1u << (element & 1u))) != 0u; @@ -368,6 +371,7 @@ DescriptorBindingPlan DescriptorBindingBuilder::Prepare(const IrBindingLayout& l const auto& buffer = info.buffers.at(resource); physical.bufferAtomic.push_back(buffer.atomic); physical.bufferWritten.push_back(buffer.written || buffer.atomic); + physical.bufferRead.push_back(buffer.read); if (buffer.written || buffer.atomic) ++entry.writtenBuffers; else ++entry.readOnlyBuffers; } diff --git a/core/shader/recompiler/Optimization/src/HostInterpolationChecker.cpp b/core/shader/recompiler/Optimization/src/HostInterpolationChecker.cpp new file mode 100644 index 000000000..2100af1bd --- /dev/null +++ b/core/shader/recompiler/Optimization/src/HostInterpolationChecker.cpp @@ -0,0 +1,142 @@ +#include "Optimization/HostInterpolationChecker.hpp" +#include "IntermediateRepresentation/IrBlock.hpp" +#include "IntermediateRepresentation/IrMetadata/StageIO.hpp" +#include "Optimization/DeadCodeEliminator.hpp" +#include "Optimization/MaskedSelectEliminator.hpp" +#include +#include + +namespace ShaderRecompiler { +namespace { + +bool isCopy(const IrValue& value) { + return value.IsIdentity() || value.Opcode() == IrOpcode::BitCastF32U32 || value.Opcode() == IrOpcode::BitCastU32F32; +} + +const IrValue* lookThrough(const IrValue* value) { + value = value->Resolve(); + while (value->Opcode() == IrOpcode::BitCastF32U32 || value->Opcode() == IrOpcode::BitCastU32F32) value = value->Argument(0)->Resolve(); + return value; +} + +const IrValue* partialOf(const IrValue& interpolation) { + const auto* exec = interpolation.Argument(4)->Resolve(); + const auto* value = lookThrough(interpolation.Argument(3)); + while (value->Opcode() == IrOpcode::SelectU32) { + if (value->Argument(0)->Resolve() != exec) return value; + value = lookThrough(value->Argument(1)); + } + return value; +} + +bool isBarycentric(const IrValue& value) { + if (value.Opcode() != IrOpcode::GetBuiltin) return false; + const auto kind = static_cast(value.Argument(0)->Resolve()->ImmediateU32()); + return kind == StageInputKind::BaryCoordSmooth || kind == StageInputKind::BaryCoordNoPerspective; +} + +bool isBarycentric(const IrValue& value, StageInputKind kind, std::uint32_t component) { + return isBarycentric(value) && static_cast(value.Argument(0)->Resolve()->ImmediateU32()) == kind && value.Argument(1)->Resolve()->ImmediateU32() == component; +} + +template +bool usesAll(const IrValue& value, const TAccept& accept) { + for (const auto& use : value.OperandUses()) { + if (isCopy(*use.user)) { + if (!usesAll(*use.user, accept)) return false; + } else if (!accept(*use.user, use.operand)) { + return false; + } + } + return true; +} + +bool sameCondition(const IrValue& left, const IrValue& right) { + return left.Argument(0)->Resolve() == right.Argument(0)->Resolve(); +} + +bool partialSelect(const IrValue& select) { + return usesAll(select, [&](const IrValue& user, std::size_t operand) { + return (user.Opcode() == IrOpcode::InterpolateHostP2 && operand == 3u) || (user.Opcode() == IrOpcode::SelectU32 && operand == 2u && sameCondition(user, select)); + }); +} + +bool sameParameter(const IrValue& left, const IrValue& right) { + return left.Argument(0)->Resolve()->ImmediateU32() == right.Argument(0)->Resolve()->ImmediateU32() && left.Argument(1)->Resolve()->ImmediateU32() == right.Argument(1)->Resolve()->ImmediateU32(); +} + +bool exact(const IrValue& inst, const IrProgram& program, const ShaderPixelInputInfo& pixel, std::vector& selects) { + if (inst.Opcode() == IrOpcode::InterpolateHostP1) { + return usesAll(inst, [&](const IrValue& user, std::size_t operand) { + if (user.Opcode() == IrOpcode::InterpolateHostP2 && operand == 3u) return true; + if (user.Opcode() != IrOpcode::SelectU32 || operand != 1u || !partialSelect(user)) return false; + selects.push_back(const_cast(&user)); + return true; + }); + } + if (inst.Opcode() == IrOpcode::GetInterpolationParameter) { + const auto input = inst.Argument(0)->Resolve()->ImmediateU32(); + return pixel.InputIsDefault(input) || (inst.Argument(2)->Resolve()->ImmediateU32() == 2u && pixel.InputIsFlat(input)); + } + if (inst.Opcode() != IrOpcode::InterpolateHostP2) return true; + const auto* partial = partialOf(inst); + if (partial->Opcode() != IrOpcode::InterpolateHostP1 || !sameParameter(inst, *partial)) return false; + const auto& metadata = program.Metadata(); + const auto kind = pixel.InputIsLinear(inst.Argument(0)->Resolve()->ImmediateU32(), metadata.pixelLinearInputs, metadata.pixelPerspectiveInputs) ? StageInputKind::BaryCoordNoPerspective : StageInputKind::BaryCoordSmooth; + return isBarycentric(*lookThrough(partial->Argument(2)), kind, 0u) && isBarycentric(*lookThrough(inst.Argument(2)), kind, 1u); +} + +bool sharesSlots(const IrProgram& program, const ShaderPixelInputInfo& pixel) { + constexpr std::uint32_t unassigned = 3u; + std::array interpolation {}; + interpolation.fill(unassigned); + const auto& metadata = program.Metadata(); + for (const auto& block : program.Blocks()) { + for (const IrValue* inst : block->Instructions()) { + if (inst->Opcode() != IrOpcode::GetAttribute && inst->Opcode() != IrOpcode::GetInterpolationParameter) continue; + const auto input = inst->Argument(0)->Resolve()->ImmediateU32(); + if (input >= pixel.inputNum || pixel.InputIsDefault(input)) continue; + const auto mode = pixel.InputIsFlat(input) ? 2u : pixel.InputIsLinear(input, metadata.pixelLinearInputs, metadata.pixelPerspectiveInputs) ? 1u : 0u; + auto& slot = interpolation[pixel.InputSlot(input)]; + if (slot != unassigned && slot != mode) return true; + slot = mode; + } + } + return false; +} + +} + +bool HostInterpolationChecker::Lower(IrProgram& program, const ShaderPixelInputInfo& pixel) const { + std::vector selects; + for (const auto& block : program.Blocks()) { + for (const IrValue* inst : block->Instructions()) { + if (!exact(*inst, program, pixel, selects)) return false; + } + } + if (sharesSlots(program, pixel)) return false; + for (auto* select : selects) select->ReplaceAllUsesWith(select->Argument(2)); + for (const auto& block : program.Blocks()) { + auto& instructions = block->Instructions(); + for (auto it = instructions.begin(); it != instructions.end();) { + IrValue* inst = *it; + if (inst->Opcode() != IrOpcode::InterpolateHostP2) { + ++it; + continue; + } + inst->Invalidate(); + inst->SetParent(nullptr); + it = instructions.erase(it); + } + } + DeadCodeEliminator{}.Eliminate(program); + if (MaskedSelectEliminator{}.Eliminate(program).removedSelects != 0u) DeadCodeEliminator{}.Eliminate(program); + for (const auto& block : program.Blocks()) { + for (const IrValue* inst : block->Instructions()) { + if (isBarycentric(*inst) || inst->Opcode() == IrOpcode::InterpolateHostP1) return false; + } + } + return true; +} + +} diff --git a/core/shader/recompiler/Optimization/src/RequestMemoryView.cpp b/core/shader/recompiler/Optimization/src/RequestMemoryView.cpp index 249778f06..58981be3a 100644 --- a/core/shader/recompiler/Optimization/src/RequestMemoryView.cpp +++ b/core/shader/recompiler/Optimization/src/RequestMemoryView.cpp @@ -64,6 +64,29 @@ bool RequestMemoryView::ReadGuestMemory(void* userContext, std::uint64_t address return true; } +bool RequestMemoryView::Covered(void* userContext, std::uint64_t address, std::uint64_t bytes) { + const auto* self = static_cast(userContext); + if (bytes > std::numeric_limits::max() - address) { + return false; + } + const std::uint64_t end = address + bytes; + while (address < end) { + const auto it = std::upper_bound(self->regions.begin(), self->regions.end(), address, [](std::uint64_t addressValue, const MemoryRegion& region) { + return addressValue < region.guestAddress; + }); + if (it == self->regions.begin()) { + return false; + } + const MemoryRegion& region = *std::prev(it); + const std::uint64_t regionEnd = region.guestAddress + region.bytes.size(); + if (address >= regionEnd) { + return false; + } + address = regionEnd; + } + return true; +} + SrtRuntime RequestMemoryView::MakeRuntime(std::span userData, std::uint64_t shaderBase) { SrtRuntime runtime; runtime.userData = userData; @@ -71,6 +94,7 @@ SrtRuntime RequestMemoryView::MakeRuntime(std::span userDat runtime.userContext = this; runtime.readMemory = &ReadGuestMemory; runtime.readSpecializationMemory = &ReadGuestMemory; + runtime.accessible = &Covered; return runtime; } diff --git a/core/shader/recompiler/Optimization/src/ResourceMaterializer.cpp b/core/shader/recompiler/Optimization/src/ResourceMaterializer.cpp index 1623afd77..f1e3a5545 100644 --- a/core/shader/recompiler/Optimization/src/ResourceMaterializer.cpp +++ b/core/shader/recompiler/Optimization/src/ResourceMaterializer.cpp @@ -444,7 +444,7 @@ void materializeSnapshot(const IrResourcePlan& plan, const SrtRuntime& runtime, std::vector values; std::vector activeSources; - walker.EvaluateRuntimeSources(plan, plan.materializationSources, runtime, values, snapshot.flattenedSrt, plan.cleanFlatSlots, activeSources); + walker.EvaluateRuntimeSources(plan, plan.materializationSources, runtime, values, snapshot.flattenedSrt, plan.cleanFlatSlots, activeSources, &snapshot.srtPoison); std::size_t cursor = 0; if (values.size() < plan.info.buffers.size()) { @@ -527,7 +527,7 @@ std::uint32_t emulatedCompareState(const ShaderInfo& info, const ResourceSnapsho if ((image.emulatedCompare & EmulatedCompare::RequiresSingleLevel) != 0u && ((descriptor.dwords[3] >> 12u) & 0xfu) != ((descriptor.dwords[3] >> 16u) & 0xfu)) throw std::runtime_error("color comparison requires a single mip level"); const auto reference = colorCompareReference(format); const auto type = rawImageType(descriptor); - if (type != ImageType::Color2D && type != ImageType::Color2DArray) throw std::runtime_error("comparison sampling of a color texture is implemented only for 2D and 2D array views"); + if (type != ImageType::Color2D && type != ImageType::Color2DArray && type != ImageType::Cube) throw std::runtime_error("comparison sampling of a color texture is implemented only for 2D, 2D array and cube views"); if ((descriptorImageSwizzle(descriptor) & 0x7u) != 4u) throw std::runtime_error("comparison sampling of a color texture is implemented only when the view's X channel is red"); std::optional samplerState; for (const auto& pair : info.sampledPairs) { @@ -541,6 +541,9 @@ std::uint32_t emulatedCompareState(const ShaderInfo& info, const ResourceSnapsho if (((words[0] >> 29u) & 0x3u) != 0u) throw std::runtime_error("comparison sampling of a color texture through a min or max reduction sampler is not implemented"); const auto magFilter = (words[2] >> 20u) & 0x3u; const auto minFilter = (words[2] >> 22u) & 0x3u; + if (type == ImageType::Cube && magFilter == 1u && ((words[0] >> 28u) & 1u) == 0u) { + throw std::runtime_error("bilinear cube comparison requires DISABLE_CUBE_WRAP to avoid seamless face filtering"); + } const auto addressMode = [](std::uint32_t clamp) { if (clamp == 0u) return EmulatedCompare::AddressWrap; if (clamp == 2u) return EmulatedCompare::AddressEdge; @@ -593,6 +596,19 @@ void materializeTables(const IrResourcePlan& plan, ResourceSnapshot& snapshot, c } } +void materializeSrtGuards(const IrResourcePlan& plan, ResourceSnapshot& snapshot) { + const auto& guarded = plan.guardedSrtSlots; + const auto flags = snapshot.flattenedSrt.size(); + snapshot.flattenedSrt.resize(flags + guarded.size(), 0u); + for (std::uint32_t record = 0u; record < snapshot.srtPoison.size(); ++record) { + const auto& poison = snapshot.srtPoison[record]; + const auto found = std::lower_bound(guarded.begin(), guarded.end(), poison.slot); + if (found == guarded.end() || *found != poison.slot) throw std::runtime_error("SRT read at flat offset " + std::to_string(poison.slot) + " reads inaccessible memory but has no guard"); + snapshot.flattenedSrt[flags + static_cast(found - guarded.begin())] = record + 1u; + snapshot.flattenedSrt.insert(snapshot.flattenedSrt.end(), {poison.pc, static_cast(poison.address), static_cast(poison.address >> 32u)}); + } +} + } std::vector ResourceMaterializer::RuntimeImageModes(const ImageResource& image) { @@ -606,7 +622,7 @@ std::vector ResourceMaterializer::RuntimeImageModes(const ImageRe mode.depthBits = depth; mode.depthUnorm16 = unorm16; mode.cube = false; - mode.mipCount = mode.mipMode == ImageMipMode::DynamicStorage ? RuntimeAbi::StorageHeapCapacity : 1u; + mode.mipCount = mode.mipMode == ImageMipMode::DynamicStorage ? RuntimeAbi::StorageMipSlots : 1u; mode.shaderSwizzle = ShaderImageIdentitySwizzle; if (conversion == IrBufferFormat::Format11_11_10UNorm || conversion == IrBufferFormat::Format10_11_11Float) mode.shaderSwizzle = 0x2acu; modes.push_back(mode); @@ -615,6 +631,21 @@ std::vector ResourceMaterializer::RuntimeImageModes(const ImageRe volume.dimension = RdnaImageDimension::Dim3D; modes.push_back(volume); } + if (image.dimension == RdnaImageDimension::Dim3D && image.flatVolumeCompatible && !storage && !depth && conversion == IrBufferFormat::Invalid && packed == IrBufferFormat::Invalid) { + auto plane = mode; + plane.dimension = RdnaImageDimension::Dim2D; + modes.push_back(plane); + } + if (image.dimension == RdnaImageDimension::Dim2D && storage && image.written && !image.read && !image.atomic && image.mipMode != ImageMipMode::DynamicStorage && !depth && packed == IrBufferFormat::Invalid && image.byElements == 0u) { + auto line = mode; + line.dimension = RdnaImageDimension::Dim1D; + modes.push_back(line); + } + if (image.dimension == RdnaImageDimension::Dim2D && image.flatLineCompatible && !storage && !depth && conversion == IrBufferFormat::Invalid && packed == IrBufferFormat::Invalid) { + auto line = mode; + line.dimension = RdnaImageDimension::Dim1D; + modes.push_back(line); + } if (image.dimension == RdnaImageDimension::Dim1DArray || image.dimension == RdnaImageDimension::Dim2DArray || image.dimension == RdnaImageDimension::Dim2DMsaaArray) { auto plain = mode; plain.dimension = image.dimension == RdnaImageDimension::Dim1DArray ? RdnaImageDimension::Dim1D : image.dimension == RdnaImageDimension::Dim2DArray ? RdnaImageDimension::Dim2D : RdnaImageDimension::Dim2DMsaa; @@ -679,6 +710,10 @@ std::vector ResourceMaterializer::RuntimeImageModes(const ImageRe mode.packedFormat = IrBufferFormat::Invalid; mode.shaderSwizzle = ShaderImageIdentitySwizzle; modes.push_back(mode); + if (image.dimension == RdnaImageDimension::Dim2DArray) { + mode.cube = true; + modes.push_back(mode); + } } if (image.srgbDecodeFormats != 0u && image.srgbDecodeCompatible && !storage && !image.depthCompare && !image.packed) { const auto count = modes.size(); @@ -690,6 +725,21 @@ std::vector ResourceMaterializer::RuntimeImageModes(const ImageRe modes.push_back(mode); } } + if (image.constantSwizzleCompatible && !storage && !image.depthCompare && !image.packed && image.indirectRoot == ImageResource::NoIndirectImage) { + for (const auto numeric : {IrTextureNumericClass::Float, IrTextureNumericClass::Uint, IrTextureNumericClass::Sint}) { + auto mode = image; + mode.numericClass = numeric; + mode.conversionFormat = IrBufferFormat::Invalid; + mode.packedFormat = IrBufferFormat::Invalid; + mode.depthBits = false; + mode.depthUnorm16 = false; + mode.cube = false; + mode.mipCount = 1u; + mode.shaderSwizzle = ShaderImageIdentitySwizzle; + mode.constantSwizzle = true; + modes.push_back(mode); + } + } if (modes.empty()) throw std::runtime_error("image instruction has no supported runtime modes"); return modes; } @@ -713,7 +763,12 @@ std::uint32_t ResourceMaterializer::RuntimeImageMode(const ImageResource& image, if (!exact) throw std::runtime_error(storage ? "runtime packed image bits are not reproducible through the view" : "runtime packed image bits are not recoverable from the view"); } } - if (decoded.mipCount > (image.mipMode == ImageMipMode::DynamicStorage ? RuntimeAbi::StorageHeapCapacity : 1u)) throw std::runtime_error("runtime storage image mip capacity exceeded"); + if (decoded.mipCount > (image.mipMode == ImageMipMode::DynamicStorage ? RuntimeAbi::StorageMipSlots : 1u)) throw std::runtime_error("runtime storage image mip capacity exceeded"); + if ((descriptorImageSwizzle(descriptor) & 06666u) == 0u && !decoded.fmask && !decoded.depthBits && decoded.conversionFormat == IrBufferFormat::Invalid && !decoded.srgbDecode) { + for (std::uint32_t index = 0u; index < modes.size(); ++index) { + if (modes[index].constantSwizzle && modes[index].numericClass == decoded.numericClass) return index; + } + } for (std::uint32_t index = 0u; index < modes.size(); ++index) { const auto& mode = modes[index]; if (((mode.emulatedCompare & EmulatedCompare::Enabled) != 0u) != emulated) continue; @@ -757,7 +812,7 @@ void ResourceMaterializer::ApplyStaticInterface(IrProgram& program, bool nativeS image.srgbDecodeFormats = resources.srgbDecodeFormats; if (image.indirectRoot != ImageResource::NoIndirectImage) throw std::runtime_error("static image interface was already expanded"); image.numericClass = image.atomic ? IrTextureNumericClass::Uint : IrTextureNumericClass::Float; - image.mipCount = image.mipMode == ImageMipMode::DynamicStorage ? RuntimeAbi::StorageHeapCapacity : 1u; + image.mipCount = image.mipMode == ImageMipMode::DynamicStorage ? RuntimeAbi::StorageMipSlots : 1u; if (resources.descriptorSources.at(image.source).indirectImage.has_value()) { if (images.size() + slots - 1u > ShaderInfo::MaxImages) throw std::runtime_error("static bindless image capacity exceeded"); image.indirectRoot = index; @@ -780,6 +835,9 @@ void ResourceMaterializer::ApplyStaticInterface(IrProgram& program, bool nativeS sampler.depthCompare = sampler.depthCompare || images[pair.image].depthCompare; } resources.info.images = std::move(images); + resources.guardedSrtSlots = Detail::ComputeGuardedFlatSlots(resources); + resources.srtGuardOffset = mappingOffset; + if (!resources.guardedSrtSlots.empty()) resources.info.usesFaultBuffer = true; PrepareImageModes(resources.info); } @@ -893,6 +951,7 @@ IrResourcePlan ResourceMaterializer::ExtractPlan(const IrProgram& program) const } for (const auto& sampler : plan.info.samplers) addSource(sampler.source); plan.pureFlatSlots = Detail::ComputePureFlatSlots(plan); + plan.guardedSrtSlots = Detail::ComputeGuardedFlatSlots(plan); return plan; } @@ -915,6 +974,7 @@ void ResourceMaterializer::Materialize(const IrResourcePlan& program, const SrtR } const auto started = MaterializeProfiled() ? std::chrono::steady_clock::now() : std::chrono::steady_clock::time_point{}; materializeTables(plan, nextSnapshot, tables); + materializeSrtGuards(plan, nextSnapshot); if (MaterializeProfiled()) specializationNanoseconds.fetch_add(static_cast(std::chrono::duration_cast(std::chrono::steady_clock::now() - started).count()), std::memory_order_relaxed); snapshot = std::move(nextSnapshot); reportBindless(); @@ -925,7 +985,7 @@ std::uint64_t ResourceMaterializer::SpecializationNanoseconds() { } std::uint32_t ResourceMaterializer::BindlessSlots() { - return RuntimeAbi::SampledHeapCapacity; + return RuntimeAbi::BindlessTableSlots; } void ResourceMaterializer::CountBindlessRejection(BindlessRejection reason) { diff --git a/core/shader/recompiler/Optimization/src/ResourceTracker.cpp b/core/shader/recompiler/Optimization/src/ResourceTracker.cpp index ddeeb2cb0..3f20951dd 100644 --- a/core/shader/recompiler/Optimization/src/ResourceTracker.cpp +++ b/core/shader/recompiler/Optimization/src/ResourceTracker.cpp @@ -11,8 +11,10 @@ #include #include #include +#include #include #include +#include #include #include @@ -599,7 +601,7 @@ private: void SplitDescriptorPhi(IrValue& inst) { const auto info = ImageOpcodeInfoOf(inst.Opcode()); const std::uint32_t handleCount = info.needsSampler ? 2u : 1u; - if (inst.ArgumentCount() < handleCount) { + if (inst.ArgumentCount() < handleCount || inst.Parent() == nullptr) { return; } std::array handles {}; @@ -626,21 +628,27 @@ private: if (phis.empty()) { return; } + if (!SplitDescriptorEdges(inst, handles, split, handleCount, phis)) { + SplitDescriptorWeb(inst, handles, split, handleCount); + } + } + + bool SplitDescriptorEdges(IrValue& inst, const std::array& handles, const std::array& split, std::uint32_t handleCount, const std::vector& phis) { IrBlock* block = phis.front()->Parent(); - if (block == nullptr || inst.Parent() == nullptr) { - return; + if (block == nullptr) { + return false; } const auto& predecessors = block->Predecessors(); if (predecessors.size() < 2u) { - return; + return false; } for (const IrValue* phi : phis) { if (phi->Parent() != block || phi->PhiBlockCount() != predecessors.size()) { - return; + return false; } for (const IrBlock* predecessor : predecessors) { if (PhiIncoming(*phi, predecessor) == nullptr) { - return; + return false; } } } @@ -666,24 +674,40 @@ private: for (std::uint32_t slot = 0; slot < handleCount; slot++) { for (std::size_t dword = 0; split[slot] && dword < handles[slot]->ArgumentCount(); dword++) { if (SubstituteEdge(handles[slot]->Argument(dword), arm, inst, false, phiSearchDepth) == nullptr) { - return; + return false; } } } } + std::vector conditions; + for (std::uint32_t arm = 0; arm + 1u < arms.size(); arm++) { + std::vector edges(edgeArm.size()); + for (std::size_t edge = 0; edge < edgeArm.size(); edge++) { + edges[edge] = edgeArm[edge] == arm; + } + conditions.push_back(&EdgeSelector(*block, edges)); + } + ReplaceWithArmCopies(inst, handles, split, handleCount, conditions, [&](std::uint32_t arm, std::uint32_t slot, std::size_t dword) { + return SubstituteEdge(handles[slot]->Argument(dword), arms[arm], inst, true, phiSearchDepth); + }); + return true; + } + + template + void ReplaceWithArmCopies(IrValue& inst, const std::array& handles, const std::array& split, std::uint32_t handleCount, std::span conditions, Dword&& dword) { std::vector copies; const auto flags = inst.Flags(); const auto memory = m_program.Resources().memoryInfo.at(flags.index); - for (std::uint32_t arm = 0; arm < arms.size(); arm++) { + for (std::uint32_t arm = 0; arm <= conditions.size(); arm++) { std::vector arguments(inst.Arguments().begin(), inst.Arguments().end()); for (std::uint32_t slot = 0; slot < handleCount; slot++) { if (!split[slot]) { continue; } std::vector dwords; - for (std::size_t dword = 0; dword < handles[slot]->ArgumentCount(); dword++) { - dwords.push_back(SubstituteEdge(handles[slot]->Argument(dword), arms[arm], inst, true, phiSearchDepth)); + for (std::size_t index = 0; index < handles[slot]->ArgumentCount(); index++) { + dwords.push_back(dword(arm, slot, index)); } arguments[slot] = EmitCopy(*handles[slot], dwords, inst); } @@ -707,11 +731,7 @@ private: }; IrValue* selected = extract(copies.back()); for (std::uint32_t arm = static_cast(copies.size()) - 1u; arm-- > 0u;) { - std::vector edges(edgeArm.size()); - for (std::size_t edge = 0; edge < edgeArm.size(); edge++) { - edges[edge] = edgeArm[edge] == arm; - } - const std::array arguments {&EdgeSelector(*block, edges), extract(copies[arm]), selected}; + const std::array arguments {conditions[arm], extract(copies[arm]), selected}; selected = EmitBefore(inst, IrOpcode::SelectU32, IrType::U32, arguments); } components[component] = selected; @@ -723,6 +743,140 @@ private: inst.Parent()->RemoveInstruction(&inst); } + static bool HoldsProgramCounter(IrValue* value, std::uint32_t depth) { + value = value->Resolve(); + if (value->Opcode() == IrOpcode::GetShaderBase) { + return true; + } + if (value->HasImmediate() || depth == 0u || !Detail::IsRuntimeUniformOp(value->Opcode())) { + return false; + } + for (std::size_t index = 0; index < value->ArgumentCount(); index++) { + if (HoldsProgramCounter(value->Argument(index), depth - 1u)) { + return true; + } + } + return false; + } + + struct DescriptorWeb { + static constexpr std::size_t maxStates = 512u; + static constexpr std::size_t maxArms = 64u; + std::vector> arms; + std::map, IrValue*> selectors; + std::vector> phis; + std::unordered_map resolved; + }; + + IrValue* WebSelector(std::vector tuple, DescriptorWeb& web, IrValue& position) { + for (IrValue*& value : tuple) { + value = value->Resolve(); + if (!value->IsPhi()) { + continue; + } + const auto [entry, inserted] = web.resolved.try_emplace(value, value); + if (inserted) { + IrValue* invariant = ResolveInvariantPhi(m_program.Resources(), value); + entry->second = invariant == nullptr ? value : invariant->Resolve(); + } + value = entry->second; + } + if (const auto known = web.selectors.find(tuple); known != web.selectors.end()) { + return known->second; + } + const auto phi = std::ranges::find_if(tuple, [](const IrValue* value) { return value->IsPhi(); }); + if (phi == tuple.end()) { + for (IrValue* value : tuple) { + if (HoldsProgramCounter(value, phiSearchDepth) || Rematerialize(value, position, false, phiSearchDepth) == nullptr) { + return nullptr; + } + } + const auto same = std::ranges::find_if(web.arms, [&](const std::vector& arm) { + return std::ranges::equal(tuple, arm, [&](IrValue* left, IrValue* right) { return EquivalentValue(m_program.Resources(), left, right); }); + }); + if (same != web.arms.end()) { + return &m_builder.Constant(static_cast(same - web.arms.begin())); + } + if (web.arms.size() == DescriptorWeb::maxArms) { + return nullptr; + } + web.arms.push_back(std::move(tuple)); + return &m_builder.Constant(static_cast(web.arms.size() - 1u)); + } + IrBlock* block = (*phi)->Parent(); + if (block == nullptr || web.selectors.size() == DescriptorWeb::maxStates) { + return nullptr; + } + const auto& predecessors = block->Predecessors(); + if (predecessors.size() < 2u) { + return nullptr; + } + for (const IrValue* value : tuple) { + if (value->IsPhi() && (value->Parent() != block || value->PhiBlockCount() != predecessors.size())) { + return nullptr; + } + } + IrValue& selector = m_program.CreateValue(IrOpcode::Phi, IrType::U32); + web.selectors.emplace(tuple, &selector); + web.phis.emplace_back(block, &selector); + for (IrBlock* predecessor : predecessors) { + std::vector incoming; + for (IrValue* value : tuple) { + IrValue* edge = value->IsPhi() ? PhiIncoming(*value, predecessor) : value; + if (edge == nullptr) { + return nullptr; + } + incoming.push_back(edge); + } + IrValue* edgeSelector = WebSelector(std::move(incoming), web, position); + if (edgeSelector == nullptr) { + return nullptr; + } + selector.AddPhiOperand(predecessor, edgeSelector); + } + return &selector; + } + + void SplitDescriptorWeb(IrValue& inst, const std::array& handles, const std::array& split, std::uint32_t handleCount) { + std::vector tuple; + std::array offsets {}; + for (std::uint32_t slot = 0; slot < handleCount; slot++) { + offsets[slot] = tuple.size(); + for (std::size_t dword = 0; split[slot] && dword < handles[slot]->ArgumentCount(); dword++) { + tuple.push_back(handles[slot]->Argument(dword)); + } + } + DescriptorWeb web; + IrValue* selector = WebSelector(tuple, web, inst); + const bool select = selector != nullptr && web.arms.size() > 1u; + if (select) { + for (const auto& [block, phi] : web.phis) { + block->InsertInstructionBefore(nullptr, phi); + } + } else { + for (const auto& [block, phi] : web.phis) { + for (std::size_t index = 0; index < phi->ArgumentCount(); index++) { + phi->ReplaceArgument(index, nullptr); + } + } + for (const auto& [block, phi] : web.phis) { + phi->Invalidate(); + } + } + if (selector == nullptr || web.arms.empty()) { + return; + } + + std::vector conditions; + for (std::uint32_t arm = 0; arm + 1u < web.arms.size(); arm++) { + const std::array compare {selector, &m_builder.Constant(arm)}; + conditions.push_back(EmitBefore(inst, IrOpcode::IEqual32, IrType::Bool, compare)); + } + ReplaceWithArmCopies(inst, handles, split, handleCount, conditions, [&](std::uint32_t arm, std::uint32_t slot, std::size_t dword) { + return Rematerialize(web.arms[arm][offsets[slot] + dword], inst, true, phiSearchDepth); + }); + } + void SplitDescriptorPhis() { std::vector candidates; for (auto& block : m_program.Blocks()) { @@ -907,6 +1061,13 @@ private: image.srgbDecodeCompatible = image.srgbDecodeCompatible && !ImageOpcodeInfoOf(op).needsSampler; image.fmaskCompatible = image.fmaskCompatible && op == IrOpcode::ImageRead && memory.dataBits == 32u; image.depthBitsCompatible = image.depthBitsCompatible && memory.dataBits == 32u; + image.constantSwizzleCompatible = image.constantSwizzleCompatible && (op == IrOpcode::ImageSampleRaw || op == IrOpcode::ImageGatherRaw); + const bool flatVolumeLoad = op == IrOpcode::ImageRead && !memory.imageHasMip && memory.imageSampleFlags == 0u; + const bool flatVolumeSample = op == IrOpcode::ImageSampleRaw && memory.imageSampleFlags == RdnaImageSampleFlagLevelZero; + image.flatVolumeCompatible = image.flatVolumeCompatible && (flatVolumeLoad || flatVolumeSample) && !memory.imagePacked && memory.imageByElements == 0u; + const bool flatLineLoad = op == IrOpcode::ImageRead && !memory.imageHasMip && memory.imageSampleFlags == 0u; + const bool flatLineSample = op == IrOpcode::ImageSampleRaw && memory.imageSampleFlags == RdnaImageSampleFlagLevelZero; + image.flatLineCompatible = image.flatLineCompatible && (flatLineLoad || flatLineSample) && !memory.imagePacked && memory.imageByElements == 0u; if ((memory.imageSampleFlags & RdnaImageSampleFlagCompare) != 0u) { constexpr auto unsupported = RdnaImageSampleFlagLod | RdnaImageSampleFlagDerivative; if (op == IrOpcode::ImageGatherRaw || (memory.imageSampleFlags & unsupported) != 0u) image.emulatedCompare |= EmulatedCompare::Unsupported; diff --git a/core/shader/recompiler/Optimization/src/ShaderInfoCollector.cpp b/core/shader/recompiler/Optimization/src/ShaderInfoCollector.cpp index 93f66f1c3..672f01ef4 100644 --- a/core/shader/recompiler/Optimization/src/ShaderInfoCollector.cpp +++ b/core/shader/recompiler/Optimization/src/ShaderInfoCollector.cpp @@ -129,8 +129,18 @@ void ValidateValueReferences(const IrProgram& program, ShaderStageInputInfo inpu const auto kind = static_cast(kindValue->ImmediateU32()); const auto component = componentValue->ImmediateU32(); switch (kind) { - case StageInputKind::PackedAncillary: - return Fail("packed pixel ancillary input has an unsupported live use"); + case StageInputKind::PackedAncillary: { + std::string uses; + for (const IrValue* user : inst->Uses()) { + uses += uses.empty() ? " (used by " : ", "; + uses += IrOpcodeName(user->Opcode()); + for (std::size_t i = 1; i < user->ArgumentCount(); ++i) { + const IrValue* argument = user->Argument(i)->Resolve(); + uses += argument->HasImmediate() && argument->Type() == IrType::U32 ? " " + std::to_string(argument->ImmediateU32()) : " ?"; + } + } + return Fail("packed pixel ancillary input has an unsupported live use" + uses + (uses.empty() ? "" : ")")); + } case StageInputKind::Layer: case StageInputKind::SampleId: if (program.Resources().stage != IrShaderStage::Pixel || component != 0u) { diff --git a/core/shader/recompiler/Optimization/src/SrtWalker/SrtDescriptorEvaluation.cpp b/core/shader/recompiler/Optimization/src/SrtWalker/SrtDescriptorEvaluation.cpp index 7ea4c4a78..752c49c96 100644 --- a/core/shader/recompiler/Optimization/src/SrtWalker/SrtDescriptorEvaluation.cpp +++ b/core/shader/recompiler/Optimization/src/SrtWalker/SrtDescriptorEvaluation.cpp @@ -6,6 +6,7 @@ #include #include #include +#include namespace ShaderRecompiler::Detail { @@ -44,7 +45,7 @@ const DescriptorSource* Source(const IrResourcePlan& program, std::uint32_t sour } -bool EvaluateRuntimeSourcesImpl(const IrResourcePlan& program, std::span sources, const SrtRuntime& runtime, std::vector& results, std::vector& flat, bool evaluateFlat, std::span cleanFlatSlots, std::vector& activeSources) { +bool EvaluateRuntimeSourcesImpl(const IrResourcePlan& program, std::span sources, const SrtRuntime& runtime, std::vector& results, std::vector& flat, bool evaluateFlat, std::span cleanFlatSlots, std::vector& activeSources, std::vector* poison) { failureReason().clear(); static const bool debug = std::getenv("APS5_SRT_DEBUG") != nullptr; if (debug) { @@ -92,6 +93,12 @@ bool EvaluateRuntimeSourcesImpl(const IrResourcePlan& program, std::span> zeroedSources; std::vector evaluated; evaluated.reserve(sources.size()); for (const auto sourceIndex : sources) { @@ -103,7 +110,14 @@ bool EvaluateRuntimeSourcesImpl(const IrResourcePlan& program, std::spandwordCount; if (!evaluateFlat || active[sourceIndex]) { for (std::uint32_t index = 0; index < source->dwordCount; index++) { + inaccessible = {}; if (!evaluator.Evaluate(source->dwords[index], value.dwords[index])) { + if (inaccessible.read != nullptr) { + if (debug) std::fprintf(stderr, "[srt] descriptor source %u reads inaccessible 0x%llx at pc 0x%x: zero words\n", sourceIndex, static_cast(inaccessible.address), inaccessible.read->Flags().pc); + value.dwords.fill(0u); + zeroedSources.emplace_back(sourceIndex, inaccessible); + break; + } std::string detail = DescribeValue(source->dwords[index], 4); const IrValue* dword = source->dwords[index]->Resolve(); if (dword->Opcode() == IrOpcode::ReadConst && dword->ArgumentCount() == 2 && dword->Argument(1)->Resolve()->HasImmediate()) { @@ -117,6 +131,7 @@ bool EvaluateRuntimeSourcesImpl(const IrResourcePlan& program, std::span flattened; + std::vector poisoned; if (evaluateFlat) { flattened.resize(program.srtReads.size()); for (const auto& read : program.srtReads) { @@ -131,18 +146,36 @@ bool EvaluateRuntimeSourcesImpl(const IrResourcePlan& program, std::spanleaf = read.value->Resolve(); trace->leafSlot = read.flatOffset; } + inaccessible = {}; const bool evaluated = read.flatOffset < flattened.size() && selected.Evaluate(read.value, flattened[read.flatOffset]); if (pure) trace->leaf = nullptr; + if (!evaluated && inaccessible.read != nullptr) { + if (debug) std::fprintf(stderr, "[srt] flat offset %u reads inaccessible 0x%llx at pc 0x%x: poisoned\n", read.flatOffset, static_cast(inaccessible.address), inaccessible.read->Flags().pc); + flattened[read.flatOffset] = 0u; + poisoned.push_back({read.flatOffset, inaccessible.read->Flags().pc, inaccessible.address}); + continue; + } if (!evaluated) { return Fail(std::string(clean ? "clean " : "") + "SRT read at flat offset " + std::to_string(read.flatOffset) + ": " + DescribeValue(read.value, 4)); } } + for (const auto& [sourceIndex, read] : zeroedSources) { + const auto pc = read.read->Flags().pc; + if (std::none_of(poisoned.begin(), poisoned.end(), [&](const SrtReadPoison& entry) { return entry.pc == pc && entry.address == read.address; })) { + char address[64]; + std::snprintf(address, sizeof(address), "0x%llx at pc 0x%x", static_cast(read.address), pc); + return Fail("descriptor source " + std::to_string(sourceIndex) + " reads inaccessible " + address + ", which no flat SRT slot records"); + } + } } results = std::move(evaluated); activeSources = std::move(active); if (evaluateFlat) { flat = std::move(flattened); } + if (poison != nullptr) { + *poison = std::move(poisoned); + } return true; } diff --git a/core/shader/recompiler/Optimization/src/SrtWalker/SrtEvaluator.cpp b/core/shader/recompiler/Optimization/src/SrtWalker/SrtEvaluator.cpp index ec3347d8f..76c6ceb53 100644 --- a/core/shader/recompiler/Optimization/src/SrtWalker/SrtEvaluator.cpp +++ b/core/shader/recompiler/Optimization/src/SrtWalker/SrtEvaluator.cpp @@ -152,6 +152,10 @@ bool Evaluator::EvaluateRawRead(IrValue& inst, std::uint64_t& result) { return false; } } + if (_inaccessible != nullptr && _runtime.accessible != nullptr && !_runtime.accessible(_runtime.userContext, address, sizeof(std::uint32_t)) && LoadedFromMemory(*handle)) { + *_inaccessible = {&inst, address}; + return false; + } if (auto* trace = _runtime.readTrace; trace != nullptr) { if (&inst == trace->leaf) trace->leaves.emplace_back(trace->leafSlot, address); else trace->otherReads.push_back(address); @@ -187,6 +191,7 @@ bool Evaluator::EvaluateInst(IrValue& inst, std::uint64_t& result) { case IrOpcode::Phi: return EvaluatePhi(inst, result); case IrOpcode::ReadFirstLane: { Evaluator active(_program, _runtime, _cleanFlatSlots, _cleanEvaluator, inst.Argument(1)); + active.ReportInaccessibleReads(_inaccessible); return active.EvaluateWide(inst.Argument(0), result); } case IrOpcode::BitCastU32F32: @@ -439,6 +444,12 @@ bool Evaluator::EvaluateInst(IrValue& inst, std::uint64_t& result) { return true; } return false; + case IrOpcode::SLessThan32: + if (binary()) { + result = std::bit_cast(static_cast(a)) < std::bit_cast(static_cast(b)) ? 1u : 0u; + return true; + } + return false; case IrOpcode::ULessThan32: if (binary()) { result = static_cast(a) < static_cast(b) ? 1u : 0u; diff --git a/core/shader/recompiler/Optimization/src/SrtWalker/SrtFlatSlotClasses.cpp b/core/shader/recompiler/Optimization/src/SrtWalker/SrtFlatSlotClasses.cpp index d0b787b7f..cca8ad626 100644 --- a/core/shader/recompiler/Optimization/src/SrtWalker/SrtFlatSlotClasses.cpp +++ b/core/shader/recompiler/Optimization/src/SrtWalker/SrtFlatSlotClasses.cpp @@ -1,4 +1,5 @@ #include "Optimization/SrtWalker/SrtFlatSlotClasses.hpp" +#include "Optimization/SrtWalker/SrtInstructionPredicates.hpp" #include #include @@ -61,4 +62,47 @@ std::vector ComputePureFlatSlots(const IrResourcePlan& plan) { return pure; } +std::vector ComputeGuardedFlatSlots(const IrResourcePlan& plan) { + std::unordered_map> users; + std::unordered_set seen; + std::vector pending; + std::vector poisonable; + for (const auto& read : plan.srtReads) { + if (read.value != nullptr) pending.push_back(read.value->Resolve()); + } + while (!pending.empty()) { + const IrValue* value = pending.back(); + pending.pop_back(); + if (!seen.insert(value).second) continue; + const IrValue* handle = value->ArgumentCount() != 0u && value->Argument(0) != nullptr ? value->Argument(0)->Resolve() : nullptr; + if (IsRawRead(plan, *value) && handle != nullptr && LoadedFromMemory(*handle)) poisonable.push_back(value); + const auto follow = [&](const IrValue* argument) { + if (argument == nullptr) return; + argument = argument->Resolve(); + users[argument].push_back(value); + pending.push_back(argument); + }; + for (std::size_t i = 0; i < value->ArgumentCount(); ++i) follow(value->Argument(i)); + if (value->Opcode() == IrOpcode::ReadConst && value->ArgumentCount() >= 2 && value->Argument(1) != nullptr) { + const IrValue* slot = value->Argument(1)->Resolve(); + if (slot->HasImmediate() && slot->Type() == IrType::U32 && slot->ImmediateU32() < plan.srtReads.size()) follow(plan.srtReads[slot->ImmediateU32()].value); + } + } + std::unordered_set reaching(poisonable.begin(), poisonable.end()); + while (!poisonable.empty()) { + const IrValue* value = poisonable.back(); + poisonable.pop_back(); + const auto found = users.find(value); + if (found == users.end()) continue; + for (const IrValue* user : found->second) { + if (reaching.insert(user).second) poisonable.push_back(user); + } + } + std::vector guarded; + for (std::uint32_t slot = 0; slot < plan.srtReads.size(); ++slot) { + if (plan.srtReads[slot].value != nullptr && reaching.contains(plan.srtReads[slot].value->Resolve())) guarded.push_back(slot); + } + return guarded; +} + } diff --git a/core/shader/recompiler/Optimization/src/SrtWalker/SrtInstructionPredicates.cpp b/core/shader/recompiler/Optimization/src/SrtWalker/SrtInstructionPredicates.cpp index 46735d2cc..274524be7 100644 --- a/core/shader/recompiler/Optimization/src/SrtWalker/SrtInstructionPredicates.cpp +++ b/core/shader/recompiler/Optimization/src/SrtWalker/SrtInstructionPredicates.cpp @@ -1,5 +1,8 @@ #include "Optimization/SrtWalker/SrtInstructionPredicates.hpp" +#include +#include + namespace ShaderRecompiler::Detail { bool IsRawRead(const IrResourcePlan& program, const IrValue& inst) { @@ -15,6 +18,24 @@ bool IsRawRead(const IrResourcePlan& program, const IrValue& inst) { return (op == IrOpcode::LoadAddressU32 && kind == ResourceKind::ScalarAddress) || (op == IrOpcode::ReadConstBuffer && kind == ResourceKind::ScalarBuffer); } +bool LoadedFromMemory(const IrValue& handle) { + std::vector pending; + std::vector visited; + for (std::size_t index = 0; index < handle.ArgumentCount(); ++index) pending.push_back(handle.Argument(index)); + while (!pending.empty()) { + const IrValue* value = pending.back(); + pending.pop_back(); + if (value == nullptr) continue; + value = value->Resolve(); + if (value->HasImmediate() || std::find(visited.begin(), visited.end(), value) != visited.end()) continue; + visited.push_back(value); + const auto opcode = value->Opcode(); + if (opcode == IrOpcode::ReadConst || opcode == IrOpcode::LoadAddressU32 || opcode == IrOpcode::ReadConstBuffer) return true; + for (std::size_t index = 0; index < value->ArgumentCount(); ++index) pending.push_back(value->Argument(index)); + } + return false; +} + bool IsDescriptorHandle(IrOpcode opcode) { switch (opcode) { case IrOpcode::GetBufferResource: diff --git a/core/shader/recompiler/Optimization/src/SrtWalker/SrtWalker.cpp b/core/shader/recompiler/Optimization/src/SrtWalker/SrtWalker.cpp index 2000b1c90..4e8a85d89 100644 --- a/core/shader/recompiler/Optimization/src/SrtWalker/SrtWalker.cpp +++ b/core/shader/recompiler/Optimization/src/SrtWalker/SrtWalker.cpp @@ -49,8 +49,8 @@ void SrtWalker::EvaluateDescriptorSources(const IrResourcePlan& program, std::sp } } -void SrtWalker::EvaluateRuntimeSources(const IrResourcePlan& program, std::span sources, const SrtRuntime& runtime, std::vector& results, std::vector& flat, std::span cleanFlatSlots, std::vector& activeSources) const { - if (!Detail::EvaluateRuntimeSourcesImpl(program, sources, runtime, results, flat, true, cleanFlatSlots, activeSources)) { +void SrtWalker::EvaluateRuntimeSources(const IrResourcePlan& program, std::span sources, const SrtRuntime& runtime, std::vector& results, std::vector& flat, std::span cleanFlatSlots, std::vector& activeSources, std::vector* poison) const { + if (!Detail::EvaluateRuntimeSourcesImpl(program, sources, runtime, results, flat, true, cleanFlatSlots, activeSources, poison)) { throw std::runtime_error("SrtWalker::EvaluateRuntimeSources failed to evaluate runtime sources: " + Detail::RuntimeSourceFailureReason()); } } diff --git a/core/shader/recompiler/PipelineSpecialization.hpp b/core/shader/recompiler/PipelineSpecialization.hpp index 4b7fea929..f6ae06676 100644 --- a/core/shader/recompiler/PipelineSpecialization.hpp +++ b/core/shader/recompiler/PipelineSpecialization.hpp @@ -24,6 +24,8 @@ inline constexpr std::uint32_t VertexBase = 4096u; inline constexpr std::uint32_t VertexWords = 6u; inline constexpr std::uint32_t PushDataOffset = 8188u; inline constexpr std::uint32_t ExportBase = 8192u; +inline constexpr std::uint32_t ExportPackingBase = 8224u; +inline constexpr std::uint32_t DualSourceBlend = 8232u; inline constexpr std::uint32_t HeapCountBase = 8256u; inline constexpr std::uint32_t MipCountBase = 8320u; inline constexpr std::uint32_t DescriptorIndexBase = 16384u; diff --git a/core/shader/recompiler/RdnaDecoder/include/RdnaDecoder/RdnaInstruction.hpp b/core/shader/recompiler/RdnaDecoder/include/RdnaDecoder/RdnaInstruction.hpp index 1c7598815..f038fdc0f 100644 --- a/core/shader/recompiler/RdnaDecoder/include/RdnaDecoder/RdnaInstruction.hpp +++ b/core/shader/recompiler/RdnaDecoder/include/RdnaDecoder/RdnaInstruction.hpp @@ -4,10 +4,16 @@ #include "RdnaDecoder/RdnaOpcode.hpp" #include #include +#include #include namespace ShaderRecompiler { +class UnsupportedInstructionError : public std::runtime_error { +public: + using std::runtime_error::runtime_error; +}; + enum class RdnaInstructionFamily { Unknown, SOP1, diff --git a/core/shader/recompiler/RdnaDecoder/include/RdnaDecoder/RdnaOpcode.hpp b/core/shader/recompiler/RdnaDecoder/include/RdnaDecoder/RdnaOpcode.hpp index bdffac7f0..0b83c3ae6 100644 --- a/core/shader/recompiler/RdnaDecoder/include/RdnaDecoder/RdnaOpcode.hpp +++ b/core/shader/recompiler/RdnaDecoder/include/RdnaDecoder/RdnaOpcode.hpp @@ -70,9 +70,6 @@ enum class RdnaOpcode : std::uint16_t { VAndB32, VOrB32, VXorB32, - VLshlB32, - VLshrB32, - VAshrI32, VCmpEqF32, VCmpLtF32, VCmpGtF32, diff --git a/core/shader/recompiler/RdnaDecoder/src/RdnaDescriptorFormat.cpp b/core/shader/recompiler/RdnaDecoder/src/RdnaDescriptorFormat.cpp index 7cefea341..04f2b9e07 100644 --- a/core/shader/recompiler/RdnaDecoder/src/RdnaDescriptorFormat.cpp +++ b/core/shader/recompiler/RdnaDecoder/src/RdnaDescriptorFormat.cpp @@ -15,7 +15,7 @@ struct FormatInfo { constexpr std::array kFormatInfoTable {{ {IrBufferFormat::Format8UNorm, true, false, false}, - {IrBufferFormat::Format8SNorm, false, false, false}, + {IrBufferFormat::Format8SNorm, true, false, false}, {IrBufferFormat::Format8UInt, true, true, false}, {IrBufferFormat::Format8SInt, true, false, true}, {IrBufferFormat::Format16UNorm, true, false, false}, @@ -144,6 +144,7 @@ IrTextureNumericClass VertexInputNumericClass(IrBufferFormat format) { case IrBufferFormat::Format8_8UInt: case IrBufferFormat::Format32UInt: case IrBufferFormat::Format16_16UInt: + case IrBufferFormat::Format10_10_10_2UInt: case IrBufferFormat::Format8_8_8_8UInt: case IrBufferFormat::Format32_32UInt: case IrBufferFormat::Format16_16_16_16UInt: @@ -154,6 +155,7 @@ IrTextureNumericClass VertexInputNumericClass(IrBufferFormat format) { case IrBufferFormat::Format8_8SInt: case IrBufferFormat::Format32SInt: case IrBufferFormat::Format16_16SInt: + case IrBufferFormat::Format10_10_10_2SInt: case IrBufferFormat::Format8_8_8_8SInt: case IrBufferFormat::Format32_32SInt: case IrBufferFormat::Format16_16_16_16SInt: diff --git a/core/shader/recompiler/RdnaDecoder/src/RdnaExportOpDecoder.cpp b/core/shader/recompiler/RdnaDecoder/src/RdnaExportOpDecoder.cpp index 89e5645f7..24e45182b 100644 --- a/core/shader/recompiler/RdnaDecoder/src/RdnaExportOpDecoder.cpp +++ b/core/shader/recompiler/RdnaDecoder/src/RdnaExportOpDecoder.cpp @@ -14,7 +14,7 @@ RdnaInstruction DecodeRdnaExportOp(std::span code, std::uin RdnaInstruction DecodeRdnaExportOp(std::uint32_t programCounter, std::span code, std::uint32_t wordIndex) { const std::size_t index = wordIndex; if (index >= code.size() || code.size() - index < 2u) { - throw std::runtime_error("truncated export instruction"); + throw std::out_of_range("truncated export instruction"); } if (programCounter % 4u != 0u || programCounter > std::numeric_limits::max() - 7u) { throw std::runtime_error("invalid export program counter"); diff --git a/core/shader/recompiler/RdnaDecoder/src/RdnaImageOpDecoder.cpp b/core/shader/recompiler/RdnaDecoder/src/RdnaImageOpDecoder.cpp index 640be3b59..0ab4c4645 100644 --- a/core/shader/recompiler/RdnaDecoder/src/RdnaImageOpDecoder.cpp +++ b/core/shader/recompiler/RdnaDecoder/src/RdnaImageOpDecoder.cpp @@ -162,7 +162,9 @@ const ImageOpcodeInfo& lookupOpcode(std::uint32_t opcode) { return entry; } } - throw std::runtime_error("unsupported MIMG opcode"); + char message[48]; + std::snprintf(message, sizeof(message), "unsupported MIMG opcode 0x%02x", opcode); + throw UnsupportedInstructionError(message); } void validateFlags(std::uint32_t flags) { @@ -323,7 +325,7 @@ RdnaInstruction DecodeRdnaImageOp(std::span code, std::uint RdnaInstruction DecodeRdnaMimg(std::uint32_t programCounter, std::span code, std::uint32_t wordIndex) { const std::size_t index = wordIndex; if (index >= code.size() || code.size() - index < 2u) { - throw std::runtime_error("truncated MIMG instruction"); + throw std::out_of_range("truncated MIMG instruction"); } const auto word0 = code[index]; const auto word1 = code[index + 1u]; @@ -343,7 +345,7 @@ RdnaInstruction DecodeRdnaMimg(std::uint32_t programCounter, std::span> 1u) & 3u; const auto wordCount = 2u + nsa; if (code.size() - index < wordCount) { - throw std::runtime_error("truncated MIMG NSA payload"); + throw std::out_of_range("truncated MIMG NSA payload"); } if (programCounter % 4u != 0u || programCounter > std::numeric_limits::max() - (wordCount * 4u - 1u)) { throw std::runtime_error("invalid MIMG program counter"); diff --git a/core/shader/recompiler/RdnaDecoder/src/RdnaMemoryOpDecoder.cpp b/core/shader/recompiler/RdnaDecoder/src/RdnaMemoryOpDecoder.cpp index 8f8c5c142..f794274c6 100644 --- a/core/shader/recompiler/RdnaDecoder/src/RdnaMemoryOpDecoder.cpp +++ b/core/shader/recompiler/RdnaDecoder/src/RdnaMemoryOpDecoder.cpp @@ -696,7 +696,7 @@ void setRawWords(RdnaInstruction& instruction, std::span co void requireTwoWords(std::span code, std::uint32_t wordIndex, std::uint32_t programCounter, const char* reason) { const std::size_t index = wordIndex; if (index >= code.size() || code.size() - index < 2u) { - throw std::runtime_error(reason); + throw std::out_of_range(reason); } if (programCounter % 4u != 0u || programCounter > std::numeric_limits::max() - 7u) { throw std::runtime_error("invalid memory instruction program counter"); @@ -813,6 +813,9 @@ RdnaInstruction DecodeRdnaMubuf(std::uint32_t programCounter, std::span> 16u) & 1u) != 0u) { + throw std::runtime_error("unsupported MUBUF lds modifier"); + } RdnaInstruction instruction{}; instruction.programCounter = programCounter; @@ -1036,7 +1039,7 @@ RdnaInstruction DecodeRdnaDs(std::uint32_t programCounter, std::span code, std::uint32_t wordIndex) { const auto programCounter = toProgramCounter(wordIndex); if (static_cast(wordIndex) >= code.size()) { - throw std::runtime_error("truncated memory instruction"); + throw std::out_of_range("truncated memory instruction"); } switch (code[wordIndex] >> 26u) { case 0x36u: return DecodeRdnaDs(programCounter, code, wordIndex); diff --git a/core/shader/recompiler/RdnaDecoder/src/RdnaOpcode.cpp b/core/shader/recompiler/RdnaDecoder/src/RdnaOpcode.cpp index 79affa1cd..374315cb1 100644 --- a/core/shader/recompiler/RdnaDecoder/src/RdnaOpcode.cpp +++ b/core/shader/recompiler/RdnaDecoder/src/RdnaOpcode.cpp @@ -183,9 +183,6 @@ bool IsVectorAluOpcode(RdnaOpcode opcode) { case RdnaOpcode::VAndB32: case RdnaOpcode::VOrB32: case RdnaOpcode::VXorB32: - case RdnaOpcode::VLshlB32: - case RdnaOpcode::VLshrB32: - case RdnaOpcode::VAshrI32: case RdnaOpcode::VCmpEqF32: case RdnaOpcode::VCmpLtF32: case RdnaOpcode::VCmpGtF32: diff --git a/core/shader/recompiler/RdnaDecoder/src/RdnaScalarOpDecoder.cpp b/core/shader/recompiler/RdnaDecoder/src/RdnaScalarOpDecoder.cpp index 02abfe5a4..c85b8d563 100644 --- a/core/shader/recompiler/RdnaDecoder/src/RdnaScalarOpDecoder.cpp +++ b/core/shader/recompiler/RdnaDecoder/src/RdnaScalarOpDecoder.cpp @@ -74,7 +74,7 @@ RdnaOpcode decodeSop1Opcode(std::uint32_t opcode) { case 0x46u: return RdnaOpcode::SAndn1WrexecB32; case 0x47u: return RdnaOpcode::SAndn2WrexecB32; case 0x49u: return RdnaOpcode::SMovrelsd2B32; - default: throw std::invalid_argument("unsupported SOP1 opcode " + std::to_string(opcode)); + default: throw UnsupportedInstructionError("unsupported SOP1 opcode " + std::to_string(opcode)); } } @@ -131,7 +131,7 @@ RdnaOpcode decodeSop2Opcode(std::uint32_t opcode) { case 0x34u: return RdnaOpcode::SPackHhB32B16; case 0x35u: return RdnaOpcode::SMulHiU32; case 0x36u: return RdnaOpcode::SMulHiI32; - default: throw std::invalid_argument("unsupported SOP2 opcode " + std::to_string(opcode)); + default: throw UnsupportedInstructionError("unsupported SOP2 opcode " + std::to_string(opcode)); } } @@ -155,7 +155,7 @@ RdnaOpcode decodeSopcOpcode(std::uint32_t opcode) { case 0x0fu: return RdnaOpcode::SBitcmp1B64; case 0x12u: return RdnaOpcode::SCmpEqU64; case 0x13u: return RdnaOpcode::SCmpLgU64; - default: throw std::invalid_argument("unsupported SOPC opcode " + std::to_string(opcode)); + default: throw UnsupportedInstructionError("unsupported SOPC opcode " + std::to_string(opcode)); } } @@ -188,7 +188,7 @@ RdnaOpcode decodeSopkOpcode(std::uint32_t opcode) { case 0x1au: return RdnaOpcode::SWaitcnt; case 0x1bu: return RdnaOpcode::SSubvectorLoopBegin; case 0x1cu: return RdnaOpcode::SSubvectorLoopEnd; - default: throw std::invalid_argument("unsupported SOPK opcode " + std::to_string(opcode)); + default: throw UnsupportedInstructionError("unsupported SOPK opcode " + std::to_string(opcode)); } } @@ -231,7 +231,7 @@ RdnaOpcode decodeSoppOpcode(std::uint32_t opcode) { case 0x24u: return RdnaOpcode::SRoundMode; case 0x25u: return RdnaOpcode::SDenormMode; case 0x28u: return RdnaOpcode::STtracedata; - default: throw std::invalid_argument("unsupported SOPP opcode " + std::to_string(opcode)); + default: throw UnsupportedInstructionError("unsupported SOPP opcode " + std::to_string(opcode)); } } diff --git a/core/shader/recompiler/RdnaDecoder/src/RdnaVectorOpDecoder.cpp b/core/shader/recompiler/RdnaDecoder/src/RdnaVectorOpDecoder.cpp index 326343f03..521e629c2 100644 --- a/core/shader/recompiler/RdnaDecoder/src/RdnaVectorOpDecoder.cpp +++ b/core/shader/recompiler/RdnaDecoder/src/RdnaVectorOpDecoder.cpp @@ -58,11 +58,8 @@ constexpr Vop2OpcodeInfo vop2Opcodes[] = { {0x12u, RdnaOpcode::VMaxI32, Vop2SdwaProfile::Integer}, {0x13u, RdnaOpcode::VMinU32, Vop2SdwaProfile::Integer}, {0x14u, RdnaOpcode::VMaxU32, Vop2SdwaProfile::Integer}, - {0x15u, RdnaOpcode::VLshrB32}, {0x16u, RdnaOpcode::VLshrrevB32, Vop2SdwaProfile::Integer}, - {0x17u, RdnaOpcode::VAshrI32}, {0x18u, RdnaOpcode::VAshrrevI32, Vop2SdwaProfile::Integer}, - {0x19u, RdnaOpcode::VLshlB32}, {0x1au, RdnaOpcode::VLshlrevB32, Vop2SdwaProfile::Integer}, {0x1bu, RdnaOpcode::VAndB32, Vop2SdwaProfile::Integer}, {0x1cu, RdnaOpcode::VOrB32, Vop2SdwaProfile::Integer}, @@ -71,9 +68,6 @@ constexpr Vop2OpcodeInfo vop2Opcodes[] = { {0x1fu, RdnaOpcode::VMacF32}, {0x20u, RdnaOpcode::VMadmkF32}, {0x21u, RdnaOpcode::VMadakF32}, - {0x22u, RdnaOpcode::VBcntU32B32}, - {0x23u, RdnaOpcode::VMbcntLoU32B32}, - {0x24u, RdnaOpcode::VMbcntHiU32B32}, {0x25u, RdnaOpcode::VAddNcU32, Vop2SdwaProfile::Integer}, {0x28u, RdnaOpcode::VAddcU32, Vop2SdwaProfile::Integer}, {0x29u, RdnaOpcode::VSubCoCiU32, Vop2SdwaProfile::Integer}, @@ -630,11 +624,11 @@ RdnaOpcode lookupVectorOpcode(const TEntry (&table)[Size], std::uint32_t encodin } bool isVop2LiteralMadOpcode(std::uint32_t opcode) { - return opcode == 0x20u || opcode == 0x21u || opcode == 0x2cu || opcode == 0x2du; + return opcode == 0x20u || opcode == 0x21u || opcode == 0x2cu || opcode == 0x2du || opcode == 0x37u || opcode == 0x38u; } bool isUnsupportedVop3EncodedVop2Alias(std::uint32_t opcode) { - return isVop2LiteralMadOpcode(opcode) || opcode == 0x02u || opcode == 0x0du; + return isVop2LiteralMadOpcode(opcode) || opcode == 0x02u || opcode == 0x0du || opcode == 0x3cu; } bool isVop3EncodedVopc(std::uint32_t opcode) { @@ -757,6 +751,9 @@ bool isNativeVop3AlignOpcode(RdnaOpcode opcode) { bool usesIntegerSaturateClamp(RdnaOpcode opcode) { switch (opcode) { + case RdnaOpcode::VAddNcU32: + case RdnaOpcode::VSubNcU32: + case RdnaOpcode::VSubrevNcU32: case RdnaOpcode::VAddNcI32: case RdnaOpcode::VSubNcI32: case RdnaOpcode::VMadU32U24: @@ -782,6 +779,10 @@ bool isCubeOpcode(RdnaOpcode opcode) { opcode == RdnaOpcode::VCubemaF32; } +bool isVop3Integer16AddSubOpcode(RdnaOpcode opcode) { + return opcode == RdnaOpcode::VAddNcU16 || opcode == RdnaOpcode::VSubNcU16 || opcode == RdnaOpcode::VAddNcI16 || opcode == RdnaOpcode::VSubNcI16; +} + bool isNativeVop3B16BinaryOpcode(RdnaOpcode opcode) { switch (opcode) { case RdnaOpcode::VAddNcU16: @@ -1119,7 +1120,7 @@ constexpr Vop1SdwaRule vop1SdwaRules[] = { {RdnaOpcode::VCvtF32U32, sdwaSelAll(), 0, 0, false}, {RdnaOpcode::VCvtF32I32, sdwaSelAll(), 0, 0, false}, {RdnaOpcode::VCvtF32Ubyte0, sdwaSelAll(), 0, 0, false}, - {RdnaOpcode::VNotB32, sdwaSelAll(), 0, 0, false}, + {RdnaOpcode::VNotB32, sdwaSelAll(), sdwaSelBytes() | sdwaSelWords(), sdwaSelAll(), false}, {RdnaOpcode::VFfblB32, sdwaSelAll(), 0, 0, false}, {RdnaOpcode::VFfbhU32, sdwaSelAll(), 0, 0, false}, {RdnaOpcode::VBfrevB32, sdwaSelAll(), 0, 0, false}, @@ -1981,7 +1982,7 @@ bool supportsNativeVop3Clamp(RdnaOpcode opcode) { return true; } return supportsNativeVop3ResultModifiers(opcode) || usesInexactClampControl(opcode) || usesIntegerSaturateClamp(opcode) || - isF32DivisionStepOpcode(opcode) || isCubeOpcode(opcode); + isVop3Integer16AddSubOpcode(opcode) || isF32DivisionStepOpcode(opcode) || isCubeOpcode(opcode); } void checkNativeVop3Modifiers(RdnaOpcode opcode, bool permlane, bool carryInOut, bool scalarDst, std::uint32_t abs, std::uint32_t opSel, std::uint32_t clamp, std::uint32_t omod, std::uint32_t neg) { @@ -2007,7 +2008,7 @@ void checkNativeVop3Modifiers(RdnaOpcode opcode, bool permlane, bool carryInOut, return; } if (isNativeVop3B16BinaryOpcode(opcode)) { - if (abs != 0u || clamp != 0u || omod != 0u || neg != 0u) { + if (abs != 0u || (clamp != 0u && !isVop3Integer16AddSubOpcode(opcode)) || omod != 0u || neg != 0u) { throw std::invalid_argument("VOP3 source modifiers are not implemented"); } return; diff --git a/core/shader/recompiler/Recompiler.cpp b/core/shader/recompiler/Recompiler.cpp index 358b31a24..430b18975 100644 --- a/core/shader/recompiler/Recompiler.cpp +++ b/core/shader/recompiler/Recompiler.cpp @@ -7,6 +7,7 @@ #include "CacheKey.hpp" #include "CompiledVariant.hpp" #include "VertexInputSpecialization.hpp" +#include "FragmentOutputSpecialization.hpp" #include "SpirvBackend/SpirvSpecialization.hpp" #include "RdnaDecoder/RdnaDescriptorFormat.hpp" #include "ShaderDiskCache.hpp" @@ -24,7 +25,9 @@ #include "Optimization/include/Optimization/BindingAllocator.hpp" #include "Optimization/include/Optimization/ConstantFolder.hpp" #include "Optimization/include/Optimization/DeadCodeEliminator.hpp" +#include "Optimization/include/Optimization/DenormalFlushEliminator.hpp" #include "Optimization/include/Optimization/DescriptorBindingBuilder.hpp" +#include "Optimization/include/Optimization/HostInterpolationChecker.hpp" #include "Optimization/include/Optimization/MaskedSelectEliminator.hpp" #include "Optimization/include/Optimization/ReadLaneEliminator.hpp" #include "Optimization/include/Optimization/RequestMemoryView.hpp" @@ -95,24 +98,62 @@ ShaderStageInputInfo RequestInputInfo(const RecompileRequest& request) { return BuildShaderStageInputInfo(toShaderStageKind(request.shader.stage), request.context, HostSubgroupSize(request), mesh, tessellation); } +std::uint32_t DeviceMemoryLdsBytes(const RecompileRequest& request) { + if (!request.context.compute.has_value()) return 0u; + const auto bytes = static_cast(request.context.compute->ldsSizeDwords) * 4u; + return bytes + 4u > request.target.maxWorkgroupSharedMemoryBytes ? static_cast(bytes) : 0u; +} + +} + +struct PreparedControlFlow { + PreparedControlFlow(const RecompileRequest& request, const SwappcInfo& swappcInfo) + : stage(request.shader.stage), swappc(swappcInfo), code(request.shader.code.begin(), request.shader.code.end()), + decoded(RdnaInstructionDecoder{}.Decode(code)), cfg(GraphBuilder{}.Build(decoded, &swappc)) { + Structurizer{}.Structurize(cfg); + } + + PreparedControlFlow(const PreparedControlFlow&) = delete; + PreparedControlFlow& operator=(const PreparedControlFlow&) = delete; + + bool Matches(const RecompileRequest& request, const SwappcInfo& swappcInfo) const { + return stage == request.shader.stage && swappc.fetchCallAllowed == swappcInfo.fetchCallAllowed && + swappc.userDataBaseRegister == swappcInfo.userDataBaseRegister && swappc.userDataCount == swappcInfo.userDataCount && + std::ranges::equal(code, request.shader.code); + } + + ShaderStage stage; + SwappcInfo swappc; + std::vector code; + RdnaProgram decoded; + ControlFlowGraph cfg; +}; + +std::shared_ptr ShaderPreparationContext::AcquireFrontend(const RecompileRequest& request) { + const auto inputInfo = RequestInputInfo(request); + const SwappcInfo swappc{inputInfo.vertex != nullptr, request.context.userDataBaseRegister, static_cast(request.context.userData.size())}; + static const bool reuse = std::getenv("APS5_NO_PERF_FRONTEND_PAIR") == nullptr; + if (reuse && frontend != nullptr && frontend->Matches(request, swappc)) { + return frontend; + } + auto prepared = std::make_shared(request, swappc); + if (reuse) { + frontend = prepared; + } + return prepared; } IrProgram PrepareResourceProgram(const RecompileRequest& request) { + return PrepareResourceProgram(request, nullptr); +} + +IrProgram PrepareResourceProgram(const RecompileRequest& request, ShaderPreparationContext* preparation) { + ShaderPreparationContext local; + const auto frontend = (preparation != nullptr ? *preparation : local).AcquireFrontend(request); const auto stageKind = toShaderStageKind(request.shader.stage); const auto inputInfo = RequestInputInfo(request); - - constexpr RdnaInstructionDecoder decoder; - const auto decoded = decoder.Decode(request.shader.code); - - constexpr GraphBuilder graphBuilder; - SwappcInfo swappcInfo; - swappcInfo.fetchCallAllowed = inputInfo.vertex != nullptr; - swappcInfo.userDataBaseRegister = request.context.userDataBaseRegister; - swappcInfo.userDataCount = static_cast(request.context.userData.size()); - auto cfg = graphBuilder.Build(decoded, &swappcInfo); - - constexpr Structurizer structurizer; - structurizer.Structurize(cfg); + const auto& decoded = frontend->decoded; + const auto& cfg = frontend->cfg; TranslateOptions translateOptions {}; translateOptions.stage = stageKind; @@ -121,6 +162,7 @@ IrProgram PrepareResourceProgram(const RecompileRequest& request) { translateOptions.userDataBaseRegister = request.context.userDataBaseRegister; translateOptions.userDataCount = static_cast(request.context.userData.size()); translateOptions.scratchDwords = request.context.compute.has_value() ? request.context.compute->scratchDwords : 0u; + translateOptions.sharedMemoryBytes = DeviceMemoryLdsBytes(request); translateOptions.fragmentShaderBarycentricEnabled = request.target.fragmentShaderBarycentricEnabled; translateOptions.floatMode = request.context.floatMode; translateOptions.inputInfo = inputInfo; @@ -142,32 +184,45 @@ IrProgram PrepareResourceProgram(const RecompileRequest& request) { if (list != "all" && list.find(address) == std::string::npos) return; std::fprintf(stderr, "==== IR 0x%s after %s\n%s\n", address, pass, ProgramToString(program).c_str()); }; - dumpIr("translate"); constexpr SsaBuilder ssaBuilder; - ssaBuilder.Rewrite(program); - dumpIr("ssa"); - constexpr ConstantFolder constantFolder; constexpr DeadCodeEliminator deadCodeEliminator; - - constantFolder.Fold(program); - ResolveControlFlowIdentities(program); - deadCodeEliminator.RemoveIdentities(program); - deadCodeEliminator.Eliminate(program); - dumpIr("fold"); - constexpr ReadLaneEliminator readLaneEliminator; - const auto readLaneStats = readLaneEliminator.Eliminate(program, translateOptions.waveSize); - if (readLaneStats.rewrittenReads != 0u) { + constexpr MaskedSelectEliminator maskedSelectEliminator; + const auto simplify = [&] { + dumpIr("translate"); + ssaBuilder.Rewrite(program); + dumpIr("ssa"); + constantFolder.Fold(program); ResolveControlFlowIdentities(program); deadCodeEliminator.RemoveIdentities(program); deadCodeEliminator.Eliminate(program); + dumpIr("fold"); + + const auto readLaneStats = readLaneEliminator.Eliminate(program, translateOptions.waveSize); + if (readLaneStats.rewrittenReads != 0u) { + constantFolder.Fold(program); + ResolveControlFlowIdentities(program); + deadCodeEliminator.RemoveIdentities(program); + deadCodeEliminator.Eliminate(program); + } + + if (maskedSelectEliminator.Eliminate(program).removedSelects != 0u) { + deadCodeEliminator.Eliminate(program); + } + }; + simplify(); + if (stageKind == ShaderStageKind::Pixel && !translateOptions.fragmentShaderBarycentricEnabled && !HostInterpolationChecker{}.Lower(program, *inputInfo.pixel)) { + translateOptions.fragmentShaderBarycentricEnabled = true; + program = translator.Translate(decoded, cfg, translateOptions); + program.Metadata().barycentricEmulation = true; + simplify(); } - constexpr MaskedSelectEliminator maskedSelectEliminator; - if (maskedSelectEliminator.Eliminate(program).removedSelects != 0u) { + constexpr DenormalFlushEliminator denormalFlushEliminator; + if (denormalFlushEliminator.Eliminate(program).removedFlushes != 0u) { deadCodeEliminator.Eliminate(program); } @@ -222,7 +277,7 @@ struct SourceEntry { namespace { struct ResourceProgram { - explicit ResourceProgram(const RecompileRequest& request) : program(std::make_unique(PrepareResourceProgram(request))), plan(std::make_shared(ResourceMaterializer{}.ExtractPlan(*program))) {} + explicit ResourceProgram(const RecompileRequest& request, ShaderPreparationContext* preparation = nullptr) : program(std::make_unique(PrepareResourceProgram(request, preparation))), plan(std::make_shared(ResourceMaterializer{}.ExtractPlan(*program))) {} std::unique_ptr program; std::shared_ptr plan; @@ -248,7 +303,7 @@ bool FailureMemo() { return memo; } -std::shared_ptr getSource(const RecompileRequest& request) { +std::shared_ptr getSource(const RecompileRequest& request, ShaderPreparationContext* preparation = nullptr) { static std::shared_mutex mutex; // Entries whose code hashes alike share a bucket; the code comparison picks the right one. static std::unordered_map, std::vector>, SourceKeyHash> sources; @@ -292,7 +347,7 @@ std::shared_ptr getSource(const RecompileRequest& request) { if (source->plan == nullptr) { if (FailureMemo() && source->planFailure) std::rethrow_exception(source->planFailure); try { - ResourceProgram resource(request); + ResourceProgram resource(request, preparation); source->plan = std::move(resource.plan); source->program = std::move(resource.program); } catch (...) { @@ -344,7 +399,7 @@ CompiledVariant compileVariant(const RecompileRequest& request, IrProgram progra result.variantId = nextVariantId(); result.spirv = spirvEmitter.Emit(program, inputInfo, bindings, targetOptions); - result.bdaAbiVersion = program.Info().usesDma ? request.target.bdaAbiVersion : 0u; + result.bdaAbiVersion = program.Info().usesDma || program.Info().usesFaultBuffer ? request.target.bdaAbiVersion : 0u; result.memoryOffsetDword = bindings.layout.memoryOffsetDword; result.shaderDataDwords = bindings.layout.ShaderDataDwords(); result.imageMetadataDword = bindings.layout.ImageMetadataDword(); @@ -362,7 +417,12 @@ CompiledVariant compileVariant(const RecompileRequest& request, IrProgram progra for (const auto& output : program.Info().outputs) { if (output.kind == StageOutputKind::Parameter) result.parameterExports.push_back(output.location); } - if (request.shader.stage == ShaderStage::Fragment) result.fragmentParameters = DescribeFragmentParameters(program, inputInfo); + if (request.shader.stage == ShaderStage::Fragment) { + result.fragmentParameters = DescribeFragmentParameters(program, inputInfo); + const auto& inputs = program.Info().inputs; + const auto reads = [&](StageInputKind kind) { return std::any_of(inputs.begin(), inputs.end(), [&](const auto& input) { return input.kind == kind; }); }; + if (program.Metadata().barycentricEmulation) result.barycentricEmulation = {true, reads(StageInputKind::BaryCoordSmooth), reads(StageInputKind::BaryCoordNoPerspective)}; + } if (request.shader.stage == ShaderStage::Vertex || request.shader.stage == ShaderStage::Local) { if (inputInfo.vertex == nullptr) throw std::runtime_error("vertex input metadata is missing"); for (const auto& input : program.Info().inputs) { @@ -415,6 +475,7 @@ std::shared_ptr buildSpecializedModule(const CompiledSh if (!supplied.emplace(constant.id, constant.value).second) throw std::runtime_error("duplicate prepared specialization ID"); } std::map values; + std::set specializedIds; const auto& words = source.Words(); if (words.size() < 5u || words[0] != spv::MagicNumber) throw std::runtime_error("invalid prepared specialization module"); for (std::size_t cursor = 5; cursor < words.size();) { @@ -424,6 +485,7 @@ std::shared_ptr buildSpecializedModule(const CompiledSh if (op == spv::OpDecorate && count == 4u && words[cursor + 2u] == spv::DecorationSpecId) { const auto found = supplied.find(words[cursor + 3u]); if (found == supplied.end() || !values.emplace(words[cursor + 1u], found->second).second) throw std::runtime_error("missing or duplicate prepared specialization value"); + specializedIds.insert(found->first); } cursor += count; } @@ -439,7 +501,7 @@ std::shared_ptr buildSpecializedModule(const CompiledSh } cursor += count; } - materialized = SpecializeSpirv(materialized); + materialized = SpecializeFragmentOutputs(SpecializeSpirv(materialized), constants, specializedIds); #if ANYPS5_ENABLE_SPIRV_TOOLS materialized = ValidateAndOptimizeSpirv(materialized, target.vulkanVersion, target.spirvVersion, target.nonConstantImageOffsets, true, true); #endif @@ -495,11 +557,13 @@ std::shared_ptr buildSpecializedModule(const CompiledSh struct SpecializedModuleEntry { std::once_flag ready; + std::exception_ptr failure; std::shared_ptr module; }; struct PreparedModuleEntry { std::once_flag ready; + std::exception_ptr failure; std::shared_ptr module; DescriptorBindingPlan bindings; }; @@ -580,7 +644,14 @@ std::shared_ptr specializeModule(const CompiledShaderAr const auto found = modules.find(key); entry = found != modules.end() ? found->second : modules.emplace(key, std::make_shared()).first->second; } - std::call_once(entry->ready, [&] { entry->module = buildSpecializedModule(artifact, classes, constants, target); }); + std::call_once(entry->ready, [&] { + try { + entry->module = buildSpecializedModule(artifact, classes, constants, target); + } catch (...) { + entry->failure = std::current_exception(); + } + }); + if (entry->failure) std::rethrow_exception(entry->failure); return entry->module; } @@ -605,6 +676,7 @@ RecompileResult materializeResult(const CompiledVariant& variant, const Recompil .hostSubgroupSize = artifact.hostSubgroupSize, .parameterExports = artifact.parameterExports, .fragmentParameters = artifact.fragmentParameters, + .barycentricEmulation = artifact.barycentricEmulation, .variantId = artifact.variantId }; const auto plan = preparedBindingPlan(variant, snapshot, request.context.pixel ? std::span(request.context.pixel->targetExportMapping) : std::span{}); @@ -614,6 +686,10 @@ RecompileResult materializeResult(const CompiledVariant& variant, const Recompil auto& moduleKey = HostThreadLocal, LocalModuleKeyStorage>(); moduleKey.clear(); if (variant.bindings.layout.UsesPushData()) moduleKey.push_back(request.layout.pushConstantOffsetBytes / 4u); + if (request.context.pixel) { + for (const auto packing : request.context.pixel->targetExportPacking) moduleKey.push_back(static_cast(packing)); + moduleKey.push_back(request.context.pixel->dualSourceBlend ? 1u : 0u); + } result.vertexAttributes.reserve(result.vertexInputs.size()); std::array vertexClasses{}; for (const auto& input : result.vertexInputs) { @@ -639,7 +715,7 @@ RecompileResult materializeResult(const CompiledVariant& variant, const Recompil moduleKey.push_back(selectors); } } - if (!plan->bindings.specialization.empty() || variant.bindings.layout.UsesPushData() || !artifact.vertexInputPatches.empty()) { + if (!plan->bindings.specialization.empty() || variant.bindings.layout.UsesPushData() || !artifact.vertexInputPatches.empty() || request.context.pixel) { { std::shared_lock lock(plan->mutex); if (const auto found = plan->modules.find(moduleKey); found != plan->modules.end()) entry = found->second; @@ -649,10 +725,14 @@ RecompileResult materializeResult(const CompiledVariant& variant, const Recompil const auto found = plan->modules.find(moduleKey); entry = found != plan->modules.end() ? found->second : plan->modules.emplace(moduleKey, std::make_shared()).first->second; } - std::call_once(entry->ready, [&] { + const auto prepare = [&] { auto constants = plan->bindings.specialization; std::size_t index = 0; if (variant.bindings.layout.UsesPushData()) constants.push_back({PipelineSpecialization::PushDataOffset, moduleKey[index++]}); + if (request.context.pixel) { + for (std::uint32_t target = 0; target < request.context.pixel->targetExportPacking.size(); ++target) constants.push_back({PipelineSpecialization::ExportPackingBase + target, moduleKey[index++]}); + constants.push_back({PipelineSpecialization::DualSourceBlend, moduleKey[index++]}); + } if (!artifact.vertexInputPatches.empty()) { for (const auto& input : result.vertexInputs) { const auto selectors = moduleKey[index++]; @@ -668,7 +748,15 @@ RecompileResult materializeResult(const CompiledVariant& variant, const Recompil auto selected = DescriptorBindingBuilder{}.Select(plan->bindings, module->bindings); entry->bindings = std::move(selected); entry->module = module; + }; + std::call_once(entry->ready, [&] { + try { + prepare(); + } catch (...) { + entry->failure = std::current_exception(); + } }); + if (entry->failure) std::rethrow_exception(entry->failure); const auto& module = entry->module; result.specializationId = module->specializationId; result.spirv = module->spirv; @@ -678,8 +766,10 @@ RecompileResult materializeResult(const CompiledVariant& variant, const Recompil } BindingAllocationResult bindings; DescriptorBindingBuilder{}.Populate(bindings, variant.bindings, *bindingPlan, variant.info.userDataBase, snapshot, partialThreads(request)); + result.workgroupMemoryDwords = WorkgroupMemoryStrideDwords(variant.info.info); result.bindings = std::move(bindings.bindings); result.pushConstants = std::move(bindings.pushConstants); + result.poisonedSrtReads = static_cast(snapshot.srtPoison.size()); return result; } @@ -687,7 +777,7 @@ bool sameLayout(const BindingLayout& left, const BindingLayout& right) { return left.descriptorSet == right.descriptorSet && left.firstBinding == right.firstBinding && left.pushConstantOffsetBytes == right.pushConstantOffsetBytes && left.pushConstantSizeBytes == right.pushConstantSizeBytes; } -std::shared_ptr findOrCompileVariant(SourceEntry& source, const RecompileRequest& request, bool& cacheHit) { +std::shared_ptr findOrCompileVariant(SourceEntry& source, const RecompileRequest& request, bool& cacheHit, ShaderPreparationContext* preparation = nullptr) { for (const auto& candidate : source.variants) { if (sameLayout(candidate->layout, request.layout)) { cacheHit = true; @@ -711,7 +801,7 @@ std::shared_ptr findOrCompileVariant(SourceEntry& source, } } if (variant == nullptr) { - auto program = source.program != nullptr ? std::move(*source.program) : PrepareResourceProgram(request); + auto program = source.program != nullptr ? std::move(*source.program) : PrepareResourceProgram(request, preparation); source.program.reset(); std::exception_ptr emissionFailure; try { @@ -827,6 +917,8 @@ std::uint64_t snapshotHash(const RecompileRequest& request, const ResourceSnapsh mix(request.layout.pushConstantOffsetBytes); if (request.context.pixel) { for (const auto mapping : request.context.pixel->targetExportMapping) mix(mapping); + for (const auto packing : request.context.pixel->targetExportPacking) mix(static_cast(packing)); + mix(request.context.pixel->dualSourceBlend); } if (request.context.vertex) { const auto& vertex = *request.context.vertex; @@ -982,15 +1074,19 @@ std::shared_ptr ResolveSource(const RecompileRequest& reques } std::shared_ptr PrepareShader(const RecompileRequest& request) { + return PrepareShader(request, nullptr); +} + +std::shared_ptr PrepareShader(const RecompileRequest& request, ShaderPreparationContext* preparation) { return recompileReporting(request, [&]() -> std::shared_ptr { static_cast(RequestInputInfo(request)); auto handle = std::make_shared(); - handle->source = getSource(request); + handle->source = getSource(request, preparation); RecompileCacheKey::BuildInterface(request, handle->staticKey); handle->staticKey.push_back(HostSubgroupSize(request)); bool cacheHit = false; std::lock_guard lock(handle->source->mutex); - handle->artifact = findOrCompileVariant(*handle->source, request, cacheHit); + handle->artifact = findOrCompileVariant(*handle->source, request, cacheHit, preparation); return handle; }); } diff --git a/core/shader/recompiler/Recompiler.hpp b/core/shader/recompiler/Recompiler.hpp index 3ca2409ea..3d177b9de 100644 --- a/core/shader/recompiler/Recompiler.hpp +++ b/core/shader/recompiler/Recompiler.hpp @@ -10,6 +10,7 @@ #include #include #include +#include #include #include @@ -107,6 +108,11 @@ enum class ConservativeZExport : std::uint8_t { GreaterThanZ }; +enum class ColorExportPacking : std::uint8_t { + None, + Unorm10_11_11 +}; + struct ShaderPixelStageInfo { std::uint32_t interpolatorCount; std::array interpolatorSettings; @@ -132,6 +138,8 @@ struct ShaderPixelStageInfo { bool orderedPixelShader; std::array targetOutputMode; std::array targetExportMapping; + std::array targetExportPacking; + bool dualSourceBlend; }; struct ShaderVertexBufferResource { @@ -217,6 +225,7 @@ struct SpirvTarget { bool nonConstantImageOffsets = false; std::uint32_t srgbDecodeFormats = 0; bool narrowSubgroupClock = false; + bool fixedPushSlots = false; }; struct BindingLayout { @@ -270,6 +279,7 @@ inline constexpr std::uint32_t MeshArgumentIndexCountDword = 3; inline constexpr std::uint32_t MeshArgumentFirstIndexDword = 4; inline constexpr std::uint32_t MeshArgumentBytes = 20; inline constexpr std::uint32_t MeshIndexBufferUserWord = 4; +inline constexpr std::uint32_t WorkgroupMemoryDescriptorSet = 1; struct GraphicsDrawParameters { std::uint64_t indexAddress; @@ -353,6 +363,7 @@ struct DescriptorBinding { std::vector samplerUnnormalized; std::vector imageUnnormalized; std::vector imageSamplers; + std::vector bufferRead; }; struct VertexAttribute { @@ -371,6 +382,21 @@ struct FragmentParameter { bool custom = false; }; +struct BarycentricEmulation { + bool active = false; + bool smooth = false; + bool linear = false; +}; + +struct BarycentricEmulationLayout { + static constexpr std::uint32_t NoLocation = 0xffffffffu; + std::uint32_t smoothLocation = NoLocation; + std::uint32_t linearLocation = NoLocation; + std::vector> perVertexLocations; +}; + +[[nodiscard]] BarycentricEmulationLayout LayoutBarycentricEmulation(std::span parameters, const BarycentricEmulation& emulation); + // Compiled SPIR-V shared between a cached variant and every result materialized from it: results // are copied per dispatch and draw, so the words are reference counted and only duplicated when a // holder writes to them (tests and tools patch modules in place). Reads look like a vector. @@ -453,6 +479,7 @@ struct CompiledShaderArtifact { std::uint32_t hostSubgroupSize = 0; std::vector parameterExports; std::vector fragmentParameters; + BarycentricEmulation barycentricEmulation; std::uint64_t variantId = 0; }; @@ -462,10 +489,12 @@ struct ShaderInvocation { std::vector bindings; std::vector pushConstants; std::vector vertexAttributes; + std::uint32_t poisonedSrtReads = 0; }; struct RecompileResult : CompiledShaderArtifact, ShaderInvocation { bool cacheHit = false; + std::uint32_t workgroupMemoryDwords = 0; [[nodiscard]] std::uint64_t PipelineVariantId() const { return specializationId != 0 ? specializationId : variantId; } }; @@ -489,6 +518,16 @@ struct RectListShaders { [[nodiscard]] RectListShaders BuildRectListShaders(const RecompileResult& vertex, const RecompileResult& fragment, const SpirvTarget& target); +struct GeometryStageLimits { + std::uint32_t maxGeometryInputComponents; + std::uint32_t maxGeometryOutputComponents; + std::uint32_t maxGeometryOutputVertices; + std::uint32_t maxGeometryTotalOutputComponents; + std::uint32_t maxFragmentInputComponents; +}; + +[[nodiscard]] RecompileResult BuildBarycentricGeometryShader(const RecompileResult& vertex, const RecompileResult& fragment, const SpirvTarget& target, const std::optional& limits); + } #endif diff --git a/core/shader/recompiler/RuntimeAbi.hpp b/core/shader/recompiler/RuntimeAbi.hpp index 66f7f8a2f..131372e68 100644 --- a/core/shader/recompiler/RuntimeAbi.hpp +++ b/core/shader/recompiler/RuntimeAbi.hpp @@ -9,7 +9,7 @@ namespace ShaderRecompiler::RuntimeAbi { -inline constexpr std::uint32_t Version = 10u; +inline constexpr std::uint32_t Version = 11u; inline constexpr std::uint32_t DescriptorSet = 0u; inline constexpr std::uint32_t StageCount = 4u; inline constexpr std::uint32_t PushConstantDwords = 32u; @@ -34,9 +34,11 @@ enum class Stage : std::uint32_t { Main, Fragment, TessellationControl, Tessella inline constexpr std::uint32_t UserDataCapacity = 128u; inline constexpr std::uint32_t BufferCapacity = 128u; inline constexpr std::uint32_t ImageCapacity = 256u; -inline constexpr std::uint32_t SampledHeapCapacity = 16u; -inline constexpr std::uint32_t StorageHeapCapacity = 4u; -inline constexpr std::uint32_t SamplerHeapCapacity = 16u; +inline constexpr std::uint32_t SampledHeapCapacity = 64u; +inline constexpr std::uint32_t BindlessTableSlots = 16u; +inline constexpr std::uint32_t StorageMipSlots = 4u; +inline constexpr std::uint32_t StorageHeapCapacity = 16u; +inline constexpr std::uint32_t SamplerHeapCapacity = 32u; struct ResourceMetadata { std::uint32_t binding; @@ -74,9 +76,9 @@ inline std::uint32_t HeapCapacity(Binding binding) { } static_assert(std::is_standard_layout_v && std::is_trivially_copyable_v && sizeof(ResourceMetadata) == 48u); -static_assert(std::is_standard_layout_v && std::is_trivially_copyable_v && sizeof(ShaderData) == 13760u); +static_assert(std::is_standard_layout_v && std::is_trivially_copyable_v && sizeof(ShaderData) == 14528u); static_assert(UserDataDword == 4u && BufferOffsetsDword == 132u && DispatchThreadLimitDword == 164u); -static_assert(ExportMappingsDword == 3432u); +static_assert(ExportMappingsDword == 3624u); static_assert(offsetof(ResourceMetadata, descriptor) == 16u && offsetof(ShaderData, images) == 672u && offsetof(ShaderData, samplers) == 12960u); inline void RequireVersion(std::uint32_t version) { diff --git a/core/shader/recompiler/ShaderDiskCache.cpp b/core/shader/recompiler/ShaderDiskCache.cpp index 10a60c556..8a5d0d497 100644 --- a/core/shader/recompiler/ShaderDiskCache.cpp +++ b/core/shader/recompiler/ShaderDiskCache.cpp @@ -64,17 +64,17 @@ std::filesystem::path ShaderCacheDirectory() { namespace ShaderRecompiler::ShaderDiskCache { #if defined(__linux__) && defined(__x86_64__) && defined(__GLIBCXX__) -static_assert(sizeof(CompiledShaderArtifact) == 184, "CompiledShaderArtifact changed: update the artifact encoder"); -static_assert(sizeof(ShaderInvocation) == 104, "ShaderInvocation changed: update the invocation encoder"); -static_assert(sizeof(RecompileResult) == 296, "RecompileResult changed: update EncodeResult and DecodeResult"); -static_assert(sizeof(DescriptorBinding) == 448, "DescriptorBinding changed: update the binding encoder"); +static_assert(sizeof(CompiledShaderArtifact) == 192, "CompiledShaderArtifact changed: update the artifact encoder"); +static_assert(sizeof(ShaderInvocation) == 112, "ShaderInvocation changed: update the invocation encoder"); +static_assert(sizeof(RecompileResult) == 312, "RecompileResult changed: update EncodeResult and DecodeResult"); +static_assert(sizeof(DescriptorBinding) == 488, "DescriptorBinding changed: update the binding encoder"); static_assert(sizeof(VertexAttribute) == 32, "VertexAttribute changed: update the attribute encoder"); static_assert(sizeof(VertexInput) == 16, "VertexInput changed: update the vertex input encoder"); static_assert(sizeof(FragmentParameter) == 12, "FragmentParameter changed: update the parameter encoder"); -static_assert(sizeof(CompiledShaderInfo) == 336, "CompiledShaderInfo changed: update the info encoder"); -static_assert(sizeof(ShaderInfo) == 224, "ShaderInfo changed: update the info encoder"); +static_assert(sizeof(CompiledShaderInfo) == 344, "CompiledShaderInfo changed: update the info encoder"); +static_assert(sizeof(ShaderInfo) == 232, "ShaderInfo changed: update the info encoder"); static_assert(sizeof(BufferResource) == 32, "BufferResource changed: update the info encoder"); -static_assert(sizeof(ImageResource) == 112, "ImageResource changed: update the info encoder"); +static_assert(sizeof(ImageResource) == 120, "ImageResource changed: update the info encoder"); static_assert(sizeof(SamplerResource) == 16, "SamplerResource changed: update the info encoder"); static_assert(sizeof(SampledResourcePair) == 12, "SampledResourcePair changed: update the info encoder"); static_assert(sizeof(StageInput) == 56, "StageInput changed: update the info encoder"); @@ -257,6 +257,7 @@ void encodeBinding(Writer& writer, const DescriptorBinding& binding) { writer.Flags(binding.samplerUnnormalized); writer.Flags(binding.imageUnnormalized); writer.Values(std::span(binding.imageSamplers)); + writer.Flags(binding.bufferRead); } void decodeBinding(Reader& reader, DescriptorBinding& binding) { @@ -280,6 +281,7 @@ void decodeBinding(Reader& reader, DescriptorBinding& binding) { reader.Flags(binding.samplerUnnormalized); reader.Flags(binding.imageUnnormalized); reader.Values(binding.imageSamplers); + reader.Flags(binding.bufferRead); } void encodeArtifact(Writer& writer, const CompiledShaderArtifact& result) { @@ -317,6 +319,9 @@ void encodeArtifact(Writer& writer, const CompiledShaderArtifact& result) { out.Value(parameter.perVertex); out.Value(parameter.custom); }); + writer.Value(result.barycentricEmulation.active); + writer.Value(result.barycentricEmulation.smooth); + writer.Value(result.barycentricEmulation.linear); } void decodeArtifact(Reader& reader, CompiledShaderArtifact& result) { @@ -356,6 +361,9 @@ void decodeArtifact(Reader& reader, CompiledShaderArtifact& result) { in.Value(parameter.perVertex); in.Value(parameter.custom); }); + reader.Value(result.barycentricEmulation.active); + reader.Value(result.barycentricEmulation.smooth); + reader.Value(result.barycentricEmulation.linear); result.variantId = 0; } @@ -374,6 +382,7 @@ void encodeInvocation(Writer& writer, const ShaderInvocation& invocation) { out.Value(attribute.fetchIndex); out.Value(attribute.formatComponents); }); + writer.Value(invocation.poisonedSrtReads); } void decodeInvocation(Reader& reader, ShaderInvocation& invocation) { @@ -394,6 +403,7 @@ void decodeInvocation(Reader& reader, ShaderInvocation& invocation) { in.Value(attribute.fetchIndex); in.Value(attribute.formatComponents); }); + reader.Value(invocation.poisonedSrtReads); } void encodeLayout(Writer& writer, const IrBindingLayout& layout) { @@ -472,6 +482,10 @@ void encodeInfo(Writer& writer, const CompiledShaderInfo& compiled) { out.Value(image.packed); out.Value(image.fmaskCompatible); out.Value(image.depthBitsCompatible); + out.Value(image.constantSwizzle); + out.Value(image.constantSwizzleCompatible); + out.Value(image.flatVolumeCompatible); + out.Value(image.flatLineCompatible); out.Value(image.byElements); out.Value(image.byComponents); out.Value(image.packedFormat); @@ -518,6 +532,7 @@ void encodeInfo(Writer& writer, const CompiledShaderInfo& compiled) { writer.Value(info.hasBitwiseXor); writer.Value(info.usesDma); writer.Value(info.bdaWrites); + writer.Value(info.usesFaultBuffer); writer.Value(info.dispatchThreadLimit); encodeLayout(writer, compiled.bindings); } @@ -547,7 +562,7 @@ void decodeInfo(Reader& reader, CompiledShaderInfo& compiled) { in.Value(buffer.scalar); in.Value(buffer.typedAlignment); }); - reader.List(info.images, 72, [](Reader& in, ImageResource& image) { + reader.List(info.images, 74, [](Reader& in, ImageResource& image) { in.Value(image.source); in.Value(image.firstUsePc); in.Value(image.resourceClass); @@ -572,6 +587,10 @@ void decodeInfo(Reader& reader, CompiledShaderInfo& compiled) { in.Value(image.packed); in.Value(image.fmaskCompatible); in.Value(image.depthBitsCompatible); + in.Value(image.constantSwizzle); + in.Value(image.constantSwizzleCompatible); + in.Value(image.flatVolumeCompatible); + in.Value(image.flatLineCompatible); in.Value(image.byElements); in.Value(image.byComponents); in.Value(image.packedFormat); @@ -620,6 +639,7 @@ void decodeInfo(Reader& reader, CompiledShaderInfo& compiled) { reader.Value(info.hasBitwiseXor); reader.Value(info.usesDma); reader.Value(info.bdaWrites); + reader.Value(info.usesFaultBuffer); reader.Value(info.dispatchThreadLimit); decodeLayout(reader, compiled.bindings); } @@ -641,6 +661,7 @@ constexpr std::string_view NeutralSwitches[] = { "APS5_DUMP_IR", "APS5_NO_CODE_HASH_KEY", "APS5_NO_FAILURE_MEMO", + "APS5_NO_PERF_FRONTEND_PAIR", "APS5_NO_RESULT_MEMO", }; @@ -880,12 +901,14 @@ void EncodeResult(const RecompileResult& result, std::vector& out) { Writer writer(out); encodeArtifact(writer, result); encodeInvocation(writer, result); + writer.Value(result.workgroupMemoryDwords); } bool DecodeResult(std::span bytes, RecompileResult& result) { Reader reader(bytes); decodeArtifact(reader, result); decodeInvocation(reader, result); + reader.Value(result.workgroupMemoryDwords); result.cacheHit = false; return reader.Done() && result.runtimeAbiVersion == RuntimeAbi::Version; } diff --git a/core/shader/recompiler/ShaderDiskCache.hpp b/core/shader/recompiler/ShaderDiskCache.hpp index d8aa5338d..6111e47c1 100644 --- a/core/shader/recompiler/ShaderDiskCache.hpp +++ b/core/shader/recompiler/ShaderDiskCache.hpp @@ -12,7 +12,7 @@ namespace ShaderRecompiler::ShaderDiskCache { -inline constexpr std::uint32_t FormatVersion = 20; +inline constexpr std::uint32_t FormatVersion = 24; enum class LoadStatus { Loaded, diff --git a/core/shader/recompiler/SpirvBackend/include/SpirvBackend/SpirvBda.hpp b/core/shader/recompiler/SpirvBackend/include/SpirvBackend/SpirvBda.hpp index 4b68725c7..0b308f556 100644 --- a/core/shader/recompiler/SpirvBackend/include/SpirvBackend/SpirvBda.hpp +++ b/core/shader/recompiler/SpirvBackend/include/SpirvBackend/SpirvBda.hpp @@ -15,16 +15,21 @@ std::uint32_t BdaLoadWord(SpirvEmitterState& state, std::uint32_t index); std::uint32_t BdaLoadAddress(SpirvEmitterState& state, std::uint32_t index); void DefineBdaFaultFunction(SpirvEmitterState& state); void DefineBdaDwordReadFunctions(SpirvEmitterState& state); +void DefineBdaSpanReadFunctions(SpirvEmitterState& state); void RecordBdaFault(SpirvEmitterState& state, std::uint32_t address, std::uint32_t bytes, std::uint32_t instruction, BdaAbi::FaultReason reason); +void RecordBdaFaultWords(SpirvEmitterState& state, std::uint32_t addressLow, std::uint32_t addressHigh, std::uint32_t bytes, std::uint32_t instruction, BdaAbi::FaultReason reason); void ReturnBdaFailureIf(SpirvEmitterState& state, std::uint32_t condition, std::uint32_t address, std::uint32_t bytes, std::uint32_t instruction, BdaAbi::FaultReason reason); void ValidateBdaTarget(const IrProgram& program, const SpirvTargetOptions& target); // Whether a faulting BDA access may end its invocation. Programs with workgroup barriers must keep // every invocation running, so their faulting reads return zero instead. bool BdaInvocationsMayStop(const IrProgram& program); void StopBdaInvocationIf(SpirvEmitterState& state, std::uint32_t condition); +std::uint32_t BdaInstructionPc(SpirvEmitterState& state, const IrValue& inst); std::uint32_t EmitBdaRead(SpirvValueEmitContext& ctx, const IrValue& inst, std::uint32_t address, std::uint32_t bits); void EmitBdaWrite(SpirvValueEmitContext& ctx, const IrValue& inst, std::uint32_t address, std::uint32_t value, std::uint32_t bits = 32u); void EmitBdaStore(SpirvValueEmitContext& ctx, const IrValue& inst, std::uint32_t address, std::uint32_t value, std::uint32_t bits); +void DefineBdaByteWriteFunctions(SpirvEmitterState& state); +void EmitBdaDwordWrites(SpirvValueEmitContext& ctx, const IrValue& inst, std::uint32_t address, std::uint32_t offset, std::uint32_t dwords, std::uint32_t value); std::uint32_t EmitBdaAtomic(SpirvValueEmitContext& ctx, const IrValue& inst, std::uint32_t address, std::uint32_t bytes, const std::function& operation); // Reads the dwords of a 1-4 dword load at address + offset that the program extracts, with one // table lookup for the whole span; the per-byte lookups of EmitBdaRead remain the fallback (and diff --git a/core/shader/recompiler/SpirvBackend/include/SpirvBackend/SpirvEmitterHelpers.hpp b/core/shader/recompiler/SpirvBackend/include/SpirvBackend/SpirvEmitterHelpers.hpp index 07af37ad0..6eb3597c6 100644 --- a/core/shader/recompiler/SpirvBackend/include/SpirvBackend/SpirvEmitterHelpers.hpp +++ b/core/shader/recompiler/SpirvBackend/include/SpirvBackend/SpirvEmitterHelpers.hpp @@ -101,6 +101,9 @@ std::uint32_t EmitBinaryU32(SpirvEmitterState& state, std::uint32_t opcode, std: std::uint32_t EmitShaderDataDwordLoad(SpirvEmitterState& state, std::uint32_t dwordIndex); void EmitMemoryOffsets(SpirvEmitterState& state); std::uint32_t LdsDwordCount(const SpirvEmitterState& state); +bool LdsInDeviceMemory(const SpirvEmitterState& state); +std::uint32_t LdsMemorySemantics(const SpirvEmitterState& state); +void EmitLdsBufferBase(SpirvEmitterState& state); std::uint32_t EmitLdsLockPointer(SpirvEmitterState& state); MemoryResourceAccess PrepareMemoryResourceAccess(SpirvEmitterState& state, const MemoryInfo& mem); std::uint32_t EmitMemoryElementIndex(SpirvEmitterState& state, const MemoryResourceAccess& access, std::uint32_t rawIndex); @@ -223,10 +226,10 @@ std::uint32_t EmitValueIfElse(SpirvEmitterState& state, std::uint32_t condition, template std::uint32_t AtomicUpdateTyped(SpirvEmitterState& state, std::uint32_t pointer, ResourceKind kind, std::uint32_t type, TFunction&& function) { const auto scope = kind == ResourceKind::Lds ? spv::ScopeWorkgroup : spv::ScopeDevice; - const auto memory = [&]() { + const auto memory = [&]() -> std::uint32_t { switch (kind) { case ResourceKind::Lds: - return spv::MemorySemanticsWorkgroupMemoryMask; + return LdsMemorySemantics(state); case ResourceKind::Image: return spv::MemorySemanticsImageMemoryMask; default: diff --git a/core/shader/recompiler/SpirvBackend/include/SpirvBackend/SpirvEmitterInstructions.hpp b/core/shader/recompiler/SpirvBackend/include/SpirvBackend/SpirvEmitterInstructions.hpp index a2f5aa8d5..8e275b607 100644 --- a/core/shader/recompiler/SpirvBackend/include/SpirvBackend/SpirvEmitterInstructions.hpp +++ b/core/shader/recompiler/SpirvBackend/include/SpirvBackend/SpirvEmitterInstructions.hpp @@ -243,6 +243,9 @@ void EmitLoadAddressU32x4(SpirvValueEmitContext& ctx, const IrValue& inst); void EmitStoreAddressU8(SpirvValueEmitContext& ctx, const IrValue& inst); void EmitStoreAddressU16(SpirvValueEmitContext& ctx, const IrValue& inst); void EmitStoreAddressU32(SpirvValueEmitContext& ctx, const IrValue& inst); +void EmitStoreAddressU32x2(SpirvValueEmitContext& ctx, const IrValue& inst); +void EmitStoreAddressU32x3(SpirvValueEmitContext& ctx, const IrValue& inst); +void EmitStoreAddressU32x4(SpirvValueEmitContext& ctx, const IrValue& inst); std::uint32_t EmitAddressAtomic(SpirvValueEmitContext& ctx, const IrValue& inst); void EmitLoadBufferU8(SpirvValueEmitContext& ctx, const IrValue& inst); void EmitLoadBufferU16(SpirvValueEmitContext& ctx, const IrValue& inst); diff --git a/core/shader/recompiler/SpirvBackend/include/SpirvBackend/SpirvEmitterState.hpp b/core/shader/recompiler/SpirvBackend/include/SpirvBackend/SpirvEmitterState.hpp index a2498d029..a89724a0a 100644 --- a/core/shader/recompiler/SpirvBackend/include/SpirvBackend/SpirvEmitterState.hpp +++ b/core/shader/recompiler/SpirvBackend/include/SpirvBackend/SpirvEmitterState.hpp @@ -7,6 +7,7 @@ #include #include #include +#include #include #include #include @@ -102,6 +103,8 @@ struct SpirvEmitterState { std::uint32_t storageBufferVariable = 0; std::uint32_t storageBufferU64Variable = 0; std::vector memoryByteOffsets; + std::map, std::uint32_t> formattedBufferFunctions; + std::map, std::uint32_t> formattedGpuBufferFunctions; std::uint32_t bdaPagetableVariable = 0; std::uint32_t faultBufferVariable = 0; std::uint32_t bdaPointerFunction = 0; @@ -109,13 +112,19 @@ struct SpirvEmitterState { // every read takes the byte path. std::uint32_t bdaProbeFunction = 0; std::uint32_t bdaWritePointerFunction = 0; + std::uint32_t bdaWriteProbeFunction = 0; std::uint32_t bdaAtomicPointerFunction = 0; std::uint32_t bdaNoteWriteFunction = 0; + std::array bdaByteWriteFunctions {}; std::uint32_t bdaFaultFunction = 0; + bool nativeF16ModesEmitted = false; std::array, 2> bdaDwordReadFunctions {}; + std::array, 2> bdaSpanReadFunctions {}; std::uint32_t bdaStopValue = 0; + std::uint32_t bdaPcOverride = 0; // False for programs with workgroup barriers: faulting BDA accesses then continue (see BdaInvocationsMayStop). bool bdaStopsInvocations = true; + bool continueTarget = false; // Execution scope of the barriers that keep one guest wave's LDS accesses in program order across // host invocations (see WaveLdsScope); 0 when none are emitted. std::uint32_t waveLdsScope = 0; @@ -131,6 +140,9 @@ struct SpirvEmitterState { std::uint32_t shaderDataStorageVariable = 0; std::uint32_t flattenedSrtVariable = 0; std::uint32_t ldsVariable = 0; + std::uint32_t ldsBufferVariable = 0; + std::uint32_t ldsBufferBase = 0; + std::uint32_t numWorkgroupsVariable = 0; std::array scratchVariable {}; std::array imageVariables {}; std::uint32_t samplerVariable = 0; diff --git a/core/shader/recompiler/SpirvBackend/include/SpirvBackend/SpirvMemory/SpirvBufferAccess.hpp b/core/shader/recompiler/SpirvBackend/include/SpirvBackend/SpirvMemory/SpirvBufferAccess.hpp index ee30b6506..c89d6a211 100644 --- a/core/shader/recompiler/SpirvBackend/include/SpirvBackend/SpirvMemory/SpirvBufferAccess.hpp +++ b/core/shader/recompiler/SpirvBackend/include/SpirvBackend/SpirvMemory/SpirvBufferAccess.hpp @@ -11,6 +11,9 @@ std::uint32_t EmitBinaryU32(SpirvEmitterState& state, std::uint32_t opcode, std: std::uint32_t EmitShaderDataDwordLoad(SpirvEmitterState& state, std::uint32_t dwordIndex); void EmitMemoryOffsets(SpirvEmitterState& state); std::uint32_t LdsDwordCount(const SpirvEmitterState& state); +bool LdsInDeviceMemory(const SpirvEmitterState& state); +std::uint32_t LdsMemorySemantics(const SpirvEmitterState& state); +void EmitLdsBufferBase(SpirvEmitterState& state); std::uint32_t EmitLdsLockPointer(SpirvEmitterState& state); MemoryResourceAccess PrepareMemoryResourceAccess(SpirvEmitterState& state, const MemoryInfo& mem); std::uint32_t EmitMemoryElementIndex(SpirvEmitterState& state, const MemoryResourceAccess& access, std::uint32_t rawIndex); diff --git a/core/shader/recompiler/SpirvBackend/include/SpirvBackend/SpirvModule.hpp b/core/shader/recompiler/SpirvBackend/include/SpirvBackend/SpirvModule.hpp index 15e5f762c..21043b76c 100644 --- a/core/shader/recompiler/SpirvBackend/include/SpirvBackend/SpirvModule.hpp +++ b/core/shader/recompiler/SpirvBackend/include/SpirvBackend/SpirvModule.hpp @@ -55,6 +55,8 @@ public: void AddMemoryModel(std::uint32_t addressingModel, std::uint32_t memoryModel); void AddName(std::uint32_t target, const std::string& name); void AddFunction(std::span words); + void BeginHelperFunction(); + void EndHelperFunction(); [[nodiscard]] SpirvDeferredPhi AddDeferredPhi(std::uint32_t type, std::uint32_t result, std::size_t incomingCount); void PatchDeferredPhi(SpirvDeferredPhi phi, std::size_t incoming, std::uint32_t value, std::uint32_t parent); @@ -253,6 +255,8 @@ private: std::vector typeDeclarations; std::vector globalVariables; std::vector functionInstructions; + std::vector helperFunctionInstructions; + bool inHelperFunction = false; }; } diff --git a/core/shader/recompiler/SpirvBackend/src/BarycentricGeometryShader.cpp b/core/shader/recompiler/SpirvBackend/src/BarycentricGeometryShader.cpp new file mode 100644 index 000000000..7c86bc826 --- /dev/null +++ b/core/shader/recompiler/SpirvBackend/src/BarycentricGeometryShader.cpp @@ -0,0 +1,204 @@ +#include "Recompiler.hpp" +#include "SpirvBackend/SpirvModule.hpp" +#if ANYPS5_ENABLE_SPIRV_TOOLS +#include "SpirvBackend/SpirvOptimizer.hpp" +#endif +#include +#include +#include +#include +#include +#include + +namespace ShaderRecompiler { +namespace { + +constexpr std::uint32_t ParameterLocations = 32u; + +void require(bool condition, const std::string& message) { + if (!condition) throw std::runtime_error("Barycentric geometry SPIR-V: " + message); +} + +struct PerVertexParameter { + std::uint32_t sourceLocation; + std::uint32_t location; +}; + +class BarycentricGeometryEmitter { +public: + explicit BarycentricGeometryEmitter(std::uint32_t version) : builder(version) { + builder.AddMemoryModel(spv::AddressingModelLogical, spv::MemoryModelGLSL450); + voidType = builder.Type(spv::OpTypeVoid); + uintType = builder.Type(spv::OpTypeInt, 32u, 0u); + intType = builder.Type(spv::OpTypeInt, 32u, 1u); + floatType = builder.Type(spv::OpTypeFloat, 32u); + vec3FloatType = builder.Type(spv::OpTypeVector, floatType, 3u); + vec4FloatType = builder.Type(spv::OpTypeVector, floatType, 4u); + vertexArrayType = builder.Type(spv::OpTypeArray, vec4FloatType, constant(uintType, 3u)); + functionType = builder.Type(spv::OpTypeFunction, voidType); + perVertexType = builder.DecoratedType(spv::OpTypeStruct, {{spv::OpMemberDecorate, {0u, spv::DecorationBuiltIn, spv::BuiltInPosition}}, {spv::OpDecorate, {spv::DecorationBlock}}}, vec4FloatType); + } + + std::vector Emit(const std::vector& passed, const std::vector& perVertex, const BarycentricEmulationLayout& layout) { + builder.EmitCapability(spv::CapabilityShader); + builder.EmitCapability(spv::CapabilityGeometry); + main = result(spv::OpFunction, voidType, spv::FunctionControlMaskNone, functionType); + builder.AddExecutionMode(main, spv::ExecutionModeTriangles); + builder.AddExecutionMode(main, spv::ExecutionModeInvocations, 1u); + builder.AddExecutionMode(main, spv::ExecutionModeOutputTriangleStrip); + builder.AddExecutionMode(main, spv::ExecutionModeOutputVertices, 3u); + + const auto glIn = addInterface(spv::StorageClassInput, builder.Type(spv::OpTypeArray, perVertexType, constant(uintType, 3u))); + std::set sources(passed.begin(), passed.end()); + for (const auto& parameter : perVertex) sources.insert(parameter.sourceLocation); + std::array inputs {}; + for (const auto location : sources) { + inputs[location] = addInterface(spv::StorageClassInput, vertexArrayType); + decorate(inputs[location], spv::DecorationLocation, location); + } + const auto position = addInterface(spv::StorageClassOutput, vec4FloatType); + decorate(position, spv::DecorationBuiltIn, spv::BuiltInPosition); + std::vector passedOutputs; + for (const auto location : passed) { + passedOutputs.push_back(addInterface(spv::StorageClassOutput, vec4FloatType)); + decorate(passedOutputs.back(), spv::DecorationLocation, location); + } + const auto barycentricOutput = [&](std::uint32_t location) { + if (location == BarycentricEmulationLayout::NoLocation) return 0u; + const auto variable = addInterface(spv::StorageClassOutput, vec3FloatType); + decorate(variable, spv::DecorationLocation, location); + return variable; + }; + const auto smooth = barycentricOutput(layout.smoothLocation); + const auto linear = barycentricOutput(layout.linearLocation); + std::vector perVertexOutputs; + for (const auto& parameter : perVertex) { + perVertexOutputs.push_back(addInterface(spv::StorageClassOutput, vertexArrayType)); + decorate(perVertexOutputs.back(), spv::DecorationLocation, parameter.location); + } + builder.EmitEntryPoint(spv::ExecutionModelGeometry, main, "main", interfaces); + builder.AddFunction(spv::OpLabel, builder.AllocateId()); + + const auto ptrInputVec4 = builder.Type(spv::OpTypePointer, spv::StorageClassInput, vec4FloatType); + const auto ptrOutputVec4 = builder.Type(spv::OpTypePointer, spv::StorageClassOutput, vec4FloatType); + const auto zero = constant(floatType, 0u); + const auto one = constant(floatType, std::bit_cast(1.0f)); + std::array, ParameterLocations> values {}; + for (const auto location : sources) { + for (std::uint32_t vertex = 0; vertex < 3u; ++vertex) values[location][vertex] = load(vec4FloatType, result(spv::OpAccessChain, ptrInputVec4, inputs[location], constant(intType, vertex))); + } + for (std::uint32_t vertex = 0; vertex < 3u; ++vertex) { + store(position, load(vec4FloatType, result(spv::OpAccessChain, ptrInputVec4, glIn, constant(intType, vertex), constant(intType, 0u)))); + for (std::size_t i = 0; i < passed.size(); ++i) store(passedOutputs[i], values[passed[i]][vertex]); + const auto weights = result(spv::OpCompositeConstruct, vec3FloatType, vertex == 0u ? one : zero, vertex == 1u ? one : zero, vertex == 2u ? one : zero); + if (smooth != 0u) store(smooth, weights); + if (linear != 0u) store(linear, weights); + for (std::size_t i = 0; i < perVertex.size(); ++i) { + for (std::uint32_t source = 0; source < 3u; ++source) store(result(spv::OpAccessChain, ptrOutputVec4, perVertexOutputs[i], constant(intType, source)), values[perVertex[i].sourceLocation][source]); + } + builder.AddFunction(spv::OpEmitVertex); + } + builder.AddFunction(spv::OpEndPrimitive); + builder.AddFunction(spv::OpReturn); + builder.AddFunction(spv::OpFunctionEnd); + return builder.Finalize(); + } + +private: + SpirvModule builder; + std::vector interfaces; + std::uint32_t main = 0; + std::uint32_t voidType = 0; + std::uint32_t uintType = 0; + std::uint32_t intType = 0; + std::uint32_t floatType = 0; + std::uint32_t vec3FloatType = 0; + std::uint32_t vec4FloatType = 0; + std::uint32_t vertexArrayType = 0; + std::uint32_t functionType = 0; + std::uint32_t perVertexType = 0; + + std::uint32_t constant(std::uint32_t type, std::uint32_t value) { return builder.Constant(spv::OpConstant, type, value); } + + template + std::uint32_t result(spv::Op opcode, std::uint32_t type, TArgs... operands) { + const auto id = builder.AllocateId(); + builder.AddFunction(opcode, type, id, operands...); + return id; + } + + std::uint32_t load(std::uint32_t type, std::uint32_t pointer) { return result(spv::OpLoad, type, pointer); } + + void store(std::uint32_t pointer, std::uint32_t value) { builder.AddFunction(spv::OpStore, pointer, value); } + + void decorate(std::uint32_t target, spv::Decoration decoration, std::uint32_t value) { builder.AddAnnotation(spv::OpDecorate, target, decoration, value); } + + std::uint32_t addInterface(spv::StorageClass storage, std::uint32_t type) { + const auto variable = builder.DefineGlobalVariable(builder.Type(spv::OpTypePointer, storage, type), storage); + interfaces.push_back(variable); + return variable; + } +}; + +} + +BarycentricEmulationLayout LayoutBarycentricEmulation(std::span parameters, const BarycentricEmulation& emulation) { + BarycentricEmulationLayout layout; + if (!emulation.active) return layout; + std::array used {}; + for (const auto& parameter : parameters) { + require(parameter.location < ParameterLocations, "fragment parameter location " + std::to_string(parameter.location) + " is out of range"); + if (!parameter.perVertex) used[parameter.location] = true; + } + const auto take = [&](std::uint32_t count) { + for (std::uint32_t first = 0; first + count <= ParameterLocations; ++first) { + if (std::any_of(used.begin() + first, used.begin() + first + count, [](bool taken) { return taken; })) continue; + std::fill(used.begin() + first, used.begin() + first + count, true); + return first; + } + throw std::runtime_error("Barycentric geometry SPIR-V: the emulated barycentric inputs exceed " + std::to_string(ParameterLocations) + " locations"); + }; + if (emulation.smooth) layout.smoothLocation = take(1u); + if (emulation.linear) layout.linearLocation = take(1u); + for (const auto& parameter : parameters) { + if (parameter.perVertex) layout.perVertexLocations.emplace_back(parameter.location, take(3u)); + } + return layout; +} + +RecompileResult BuildBarycentricGeometryShader(const RecompileResult& vertex, const RecompileResult& fragment, const SpirvTarget& target, const std::optional& limits) { + require(fragment.barycentricEmulation.active, "the fragment shader does not emulate barycentrics"); + require(limits.has_value(), "geometry shaders are unavailable"); + require(target.spirvVersion >= 0x00010300u && target.spirvVersion <= 0x00010500u, "unsupported SPIR-V target version"); + require(std::find(target.supportedCapabilities.begin(), target.supportedCapabilities.end(), spv::CapabilityGeometry) != target.supportedCapabilities.end(), "geometry capability is unavailable"); + const auto exported = [&](std::uint32_t location) { return std::find(vertex.parameterExports.begin(), vertex.parameterExports.end(), location) != vertex.parameterExports.end(); }; + const auto layout = LayoutBarycentricEmulation(fragment.fragmentParameters, fragment.barycentricEmulation); + std::vector passed; + std::vector perVertex; + for (const auto& parameter : fragment.fragmentParameters) { + require(exported(parameter.sourceLocation), "fragment parameter " + std::to_string(parameter.location) + " is not exported by the vertex shader"); + if (!parameter.perVertex) { + passed.push_back(parameter.sourceLocation); + continue; + } + const auto relocated = std::find_if(layout.perVertexLocations.begin(), layout.perVertexLocations.end(), [&](const auto& entry) { return entry.first == parameter.location; }); + require(relocated != layout.perVertexLocations.end(), "per-vertex parameter " + std::to_string(parameter.location) + " has no emulated location"); + perVertex.push_back({parameter.sourceLocation, relocated->second}); + } + const auto barycentrics = (layout.smoothLocation != BarycentricEmulationLayout::NoLocation ? 1u : 0u) + (layout.linearLocation != BarycentricEmulationLayout::NoLocation ? 1u : 0u); + const auto outputComponents = 4u + 4u * static_cast(passed.size()) + 3u * barycentrics + 12u * static_cast(perVertex.size()); + std::set sources(passed.begin(), passed.end()); + for (const auto& parameter : perVertex) sources.insert(parameter.sourceLocation); + const auto inputComponents = 4u + 4u * static_cast(sources.size()); + const auto fragmentComponents = 4u * (static_cast(passed.size()) + barycentrics + 3u * static_cast(perVertex.size())); + require(limits->maxGeometryOutputVertices >= 3u && outputComponents <= limits->maxGeometryOutputComponents && 3u * outputComponents <= limits->maxGeometryTotalOutputComponents && inputComponents <= limits->maxGeometryInputComponents && fragmentComponents <= limits->maxFragmentInputComponents, "the emulated barycentric interface exceeds device limits"); + BarycentricGeometryEmitter emitter(target.spirvVersion); + RecompileResult result; + result.spirv = emitter.Emit(passed, perVertex, layout); +#if ANYPS5_ENABLE_SPIRV_TOOLS + result.spirv = ValidateAndOptimizeSpirv(result.spirv, target.vulkanVersion, target.spirvVersion); +#endif + return result; +} + +} diff --git a/core/shader/recompiler/SpirvBackend/src/RectListShaders.cpp b/core/shader/recompiler/SpirvBackend/src/RectListShaders.cpp index ed7a17055..f8af830fa 100644 --- a/core/shader/recompiler/SpirvBackend/src/RectListShaders.cpp +++ b/core/shader/recompiler/SpirvBackend/src/RectListShaders.cpp @@ -1,5 +1,6 @@ #include "Recompiler.hpp" #include "BdaAbi.hpp" +#include "RuntimeAbi.hpp" #include "SpirvBackend/SpirvModule.hpp" #if ANYPS5_ENABLE_SPIRV_TOOLS #include "SpirvBackend/SpirvOptimizer.hpp" @@ -393,13 +394,7 @@ RectListShaders BuildRectListShaders(const RecompileResult& vertex, const Recomp } const auto components = static_cast((parameters.size() + 1) * 4); require(limits.maxPatchSize >= 4 && components <= limits.maxControlPerVertexInputComponents && components <= limits.maxControlPerVertexOutputComponents && components <= limits.maxEvaluationInputComponents && components <= limits.maxEvaluationOutputComponents && limits.maxControlPerPatchOutputComponents >= 6 && components * 4 + 6 <= limits.maxControlTotalOutputComponents, "tessellation interface exceeds device limits"); - std::uint32_t faultBinding = 0; - for (const auto* shader : {&vertex, &fragment}) { - for (const auto& binding : shader->bindings) { - require(binding.binding < std::numeric_limits::max(), "descriptor binding overflow"); - faultBinding = std::max(faultBinding, binding.binding + 1); - } - } + const auto faultBinding = RuntimeAbi::BindingNumber(RuntimeAbi::Stage::TessellationControl, RuntimeAbi::Binding::FaultBuffer); RectListEmitter control(parameters, spv::ExecutionModelTessellationControl, target.spirvVersion, faultBinding); RectListEmitter evaluation(parameters, spv::ExecutionModelTessellationEvaluation, target.spirvVersion, faultBinding); RectListShaders shaders; diff --git a/core/shader/recompiler/SpirvBackend/src/SpirvAlu/SpirvAluEmitterFloat64.cpp b/core/shader/recompiler/SpirvBackend/src/SpirvAlu/SpirvAluEmitterFloat64.cpp index 95c2d7450..1fb812638 100644 --- a/core/shader/recompiler/SpirvBackend/src/SpirvAlu/SpirvAluEmitterFloat64.cpp +++ b/core/shader/recompiler/SpirvBackend/src/SpirvAlu/SpirvAluEmitterFloat64.cpp @@ -306,32 +306,89 @@ std::uint32_t FusedMultiplyAdd(SpirvEmitterState& state, std::uint32_t arg0, std }; const auto signedInt = [&](std::uint32_t value) { return Unary(state, spv::OpBitcast, TypeI32(state), value); }; - const auto product = Binary(state, spv::OpISub, TypeI32(state), signedInt(Binary(state, spv::OpIAdd, TypeU32(state), exponentA, exponentB)), ConstantI32(state, 2046)); - const auto distance = Binary(state, spv::OpISub, TypeI32(state), Binary(state, spv::OpISub, TypeI32(state), signedInt(exponentC), ConstantI32(state, 1023)), product); - const auto far = Binary(state, spv::OpLogicalAnd, TypeBool(state), Binary(state, spv::OpSGreaterThanEqual, TypeBool(state), distance, ConstantI32(state, 60)), Unary(state, spv::OpLogicalNot, TypeBool(state), c.zero)); + const auto both = [&](std::uint32_t lhs, std::uint32_t rhs) { return Binary(state, spv::OpLogicalAnd, TypeBool(state), lhs, rhs); }; + const auto either = [&](std::uint32_t lhs, std::uint32_t rhs) { return Binary(state, spv::OpLogicalOr, TypeBool(state), lhs, rhs); }; + const auto negate = [&](std::uint32_t value) { return Unary(state, spv::OpLogicalNot, TypeBool(state), value); }; + struct Normalized { + F64Bits bits; + std::uint32_t exponent = 0; + }; + const auto normalize = [&](const F64Class& value, std::uint32_t bits, std::uint32_t field) { + const auto subnormal = isField(field, 0u); + const auto widened = Split(state, FromF64(state, Exact(state, spv::OpFMul, ToF64(state, bits), ConstantF64(state, 0x43f0000000000000ull)))); + const auto normal = SelectBits(state, subnormal, widened, value.bits); + const auto bias = Select(state, TypeI32(state), subnormal, ConstantI32(state, 64), ConstantI32(state, 0)); + return Normalized{normal, Binary(state, spv::OpISub, TypeI32(state), signedInt(ExponentField(state, normal.high)), bias)}; + }; + const auto normalA = normalize(a, arg0, exponentA); + const auto normalB = normalize(b, arg1, exponentB); + const auto normalC = normalize(c, arg2, exponentC); + const auto product = Binary(state, spv::OpISub, TypeI32(state), Binary(state, spv::OpIAdd, TypeI32(state), normalA.exponent, normalB.exponent), ConstantI32(state, 2046)); + const auto distance = Binary(state, spv::OpISub, TypeI32(state), Binary(state, spv::OpISub, TypeI32(state), normalC.exponent, ConstantI32(state, 1023)), product); + const auto far = both(Binary(state, spv::OpSGreaterThanEqual, TypeBool(state), distance, ConstantI32(state, 60)), negate(c.zero)); const auto tiny = Binary(state, spv::OpSLessThan, TypeBool(state), distance, ConstantI32(state, -150)); const auto clamped = Glsl(state, TypeI32(state), GLSLstd450SClamp, distance, ConstantI32(state, -150), ConstantI32(state, 60)); const auto sign = Binary(state, spv::OpBitwiseAnd, TypeU32(state), c.bits.high, ConstantU32(state, 0x80000000u)); const F64Bits tinyBits{ConstantU32(state, 0u), Binary(state, spv::OpBitwiseOr, TypeU32(state), sign, ConstantU32(state, (1023u - 150u) << 20u))}; const auto rebiased = Unary(state, spv::OpBitcast, TypeU32(state), Binary(state, spv::OpIAdd, TypeI32(state), clamped, ConstantI32(state, 1023))); - const auto scaledC = SelectBits(state, c.zero, c.bits, SelectBits(state, tiny, tinyBits, WithExponent(state, c.bits, rebiased))); + const auto scaledC = SelectBits(state, c.zero, c.bits, SelectBits(state, tiny, tinyBits, WithExponent(state, normalC.bits, rebiased))); - const auto exact = TwoProduct(state, ToF64(state, Join(state, WithExponent(state, a.bits, ConstantU32(state, 1023u)))), ToF64(state, Join(state, WithExponent(state, b.bits, ConstantU32(state, 1023u))))); + const auto exact = TwoProduct(state, ToF64(state, Join(state, WithExponent(state, normalA.bits, ConstantU32(state, 1023u)))), ToF64(state, Join(state, WithExponent(state, normalB.bits, ConstantU32(state, 1023u))))); const auto sum = TwoSum(state, ToF64(state, Join(state, scaledC)), exact.high); - const auto scaled = Exact(state, spv::OpFAdd, sum.high, AddRoundToOdd(state, sum.low, exact.low)); - const auto power = PowerOfTwo(state, scale); - const auto total = Glsl(state, TypeI32(state), GLSLstd450SClamp, Binary(state, spv::OpIAdd, TypeI32(state), product, scale), ConstantI32(state, -2044), ConstantI32(state, 2046)); - const auto half = Binary(state, spv::OpSDiv, TypeI32(state), total, ConstantI32(state, 2)); - const auto rescaled = Exact(state, spv::OpFMul, Exact(state, spv::OpFMul, scaled, PowerOfTwo(state, half)), PowerOfTwo(state, Binary(state, spv::OpISub, TypeI32(state), total, half))); - const auto farValue = Split(state, FromF64(state, Exact(state, spv::OpFMul, ToF64(state, arg2), power))); - const auto finite = SelectBits(state, far, farValue, Split(state, FromF64(state, rescaled))); + const auto remainder = TwoSum(state, sum.low, exact.low); + const auto odd = AddRoundToOdd(state, sum.low, exact.low); + const auto rounded = TwoSum(state, sum.high, odd); + const auto residual = Exact(state, spv::OpFAdd, rounded.low, Exact(state, spv::OpFAdd, Exact(state, spv::OpFSub, remainder.high, odd), remainder.low)); + const auto productSign = Binary(state, spv::OpBitwiseAnd, TypeU32(state), Binary(state, spv::OpBitwiseXor, TypeU32(state), a.bits.high, b.bits.high), ConstantU32(state, 0x80000000u)); + const auto productNegative = Binary(state, spv::OpINotEqual, TypeBool(state), productSign, ConstantU32(state, 0u)); + const auto above = either(both(far, negate(productNegative)), both(negate(far), Binary(state, spv::OpFOrdGreaterThan, TypeBool(state), residual, ConstantF64(state, 0u)))); + const auto below = either(both(far, productNegative), both(negate(far), Binary(state, spv::OpFOrdLessThan, TypeBool(state), residual, ConstantF64(state, 0u)))); + auto value = SelectBits(state, far, normalC.bits, Split(state, FromF64(state, rounded.high))); + const auto farTotal = Binary(state, spv::OpISub, TypeI32(state), Binary(state, spv::OpIAdd, TypeI32(state), normalC.exponent, scale), signedInt(ExponentField(state, normalC.bits.high))); + const auto total = Select(state, TypeI32(state), far, farTotal, Binary(state, spv::OpIAdd, TypeI32(state), product, scale)); + const auto resultExponent = [&](F64Bits bits) { return Binary(state, spv::OpIAdd, TypeI32(state), signedInt(ExponentField(state, bits.high)), total); }; + const auto magnitude = Binary(state, spv::OpBitwiseOr, TypeU32(state), Binary(state, spv::OpBitwiseAnd, TypeU32(state), value.high, ConstantU32(state, 0x7fffffffu)), value.low); + const auto zero = Binary(state, spv::OpIEqual, TypeBool(state), magnitude, ConstantU32(state, 0u)); + const auto unrounded = resultExponent(value); + const auto shift = Binary(state, spv::OpISub, TypeI32(state), ConstantI32(state, 1), unrounded); + const auto dropped = Unary(state, spv::OpBitcast, TypeU32(state), Glsl(state, TypeI32(state), GLSLstd450SClamp, shift, ConstantI32(state, 1), ConstantI32(state, 54))); + const auto isHalf = [&](std::uint32_t word, std::uint32_t bits) { + const auto mask = Binary(state, spv::OpShiftRightLogical, TypeU32(state), ConstantU32(state, 0xffffffffu), Binary(state, spv::OpISub, TypeU32(state), ConstantU32(state, 32u), bits)); + const auto half = Binary(state, spv::OpShiftLeftLogical, TypeU32(state), ConstantU32(state, 1u), Binary(state, spv::OpISub, TypeU32(state), bits, ConstantU32(state, 1u))); + return Binary(state, spv::OpIEqual, TypeBool(state), Binary(state, spv::OpBitwiseAnd, TypeU32(state), word, mask), half); + }; + const auto lowHalf = isHalf(value.low, Glsl(state, TypeU32(state), GLSLstd450UMin, dropped, ConstantU32(state, 32u))); + const auto significandHigh = Binary(state, spv::OpBitwiseOr, TypeU32(state), Binary(state, spv::OpBitwiseAnd, TypeU32(state), value.high, ConstantU32(state, 0x000fffffu)), ConstantU32(state, 0x00100000u)); + const auto highBits = Binary(state, spv::OpISub, TypeU32(state), Glsl(state, TypeU32(state), GLSLstd450UMax, dropped, ConstantU32(state, 33u)), ConstantU32(state, 32u)); + const auto highHalf = both(Binary(state, spv::OpIEqual, TypeBool(state), value.low, ConstantU32(state, 0u)), isHalf(significandHigh, highBits)); + const auto wide = Binary(state, spv::OpUGreaterThan, TypeBool(state), dropped, ConstantU32(state, 32u)); + const auto onHalf = either(both(wide, highHalf), both(negate(wide), lowHalf)); + const auto inRange = both(Binary(state, spv::OpSLessThanEqual, TypeBool(state), unrounded, ConstantI32(state, 0)), Binary(state, spv::OpSLessThanEqual, TypeBool(state), shift, ConstantI32(state, 54))); + const auto midpoint = both(both(negate(zero), inRange), both(onHalf, either(above, below))); + const auto negative = Binary(state, spv::OpINotEqual, TypeBool(state), Binary(state, spv::OpBitwiseAnd, TypeU32(state), value.high, ConstantU32(state, 0x80000000u)), ConstantU32(state, 0u)); + const auto away = either(both(above, negate(negative)), both(below, negative)); + const auto carry = Select(state, TypeU32(state), Binary(state, spv::OpIEqual, TypeBool(state), value.low, ConstantU32(state, 0xffffffffu)), ConstantU32(state, 1u), ConstantU32(state, 0u)); + const auto borrow = Select(state, TypeU32(state), Binary(state, spv::OpIEqual, TypeBool(state), value.low, ConstantU32(state, 0u)), ConstantU32(state, 1u), ConstantU32(state, 0u)); + const F64Bits increased{Binary(state, spv::OpIAdd, TypeU32(state), value.low, ConstantU32(state, 1u)), Binary(state, spv::OpIAdd, TypeU32(state), value.high, carry)}; + const F64Bits decreased{Binary(state, spv::OpISub, TypeU32(state), value.low, ConstantU32(state, 1u)), Binary(state, spv::OpISub, TypeU32(state), value.high, borrow)}; + value = SelectBits(state, midpoint, SelectBits(state, away, increased, decreased), value); + + const auto exponent = resultExponent(value); + const auto normalResult = WithExponent(state, value, Unary(state, spv::OpBitcast, TypeU32(state), Glsl(state, TypeI32(state), GLSLstd450SClamp, exponent, ConstantI32(state, 1), ConstantI32(state, 2046)))); + const auto lowest = PowerOfTwo(state, Glsl(state, TypeI32(state), GLSLstd450SMax, Binary(state, spv::OpISub, TypeI32(state), exponent, ConstantI32(state, 1)), ConstantI32(state, -60))); + const auto subnormalResult = Split(state, FromF64(state, Exact(state, spv::OpFMul, ToF64(state, Join(state, WithExponent(state, value, ConstantU32(state, 1u)))), lowest))); + const F64Bits infinity{ConstantU32(state, 0u), Binary(state, spv::OpBitwiseOr, TypeU32(state), Binary(state, spv::OpBitwiseAnd, TypeU32(state), value.high, ConstantU32(state, 0x80000000u)), ConstantU32(state, 0x7ff00000u))}; + auto finite = SelectBits(state, Binary(state, spv::OpSGreaterThanEqual, TypeBool(state), exponent, ConstantI32(state, 1)), normalResult, subnormalResult); + finite = SelectBits(state, Binary(state, spv::OpSGreaterThanEqual, TypeBool(state), exponent, ConstantI32(state, 2047)), infinity, finite); + finite = SelectBits(state, zero, value, finite); + + const auto power = PowerOfTwo(state, scale); auto fallback = Split(state, FromF64(state, Exact(state, spv::OpFMul, Exact(state, spv::OpFAdd, Exact(state, spv::OpFMul, ToF64(state, arg0), ToF64(state, arg1)), ToF64(state, arg2)), power))); const auto finiteProduct = Binary(state, spv::OpLogicalAnd, TypeBool(state), Unary(state, spv::OpLogicalNot, TypeBool(state), isField(exponentA, 0x7ffu)), Unary(state, spv::OpLogicalNot, TypeBool(state), isField(exponentB, 0x7ffu))); fallback = SelectBits(state, Binary(state, spv::OpLogicalAnd, TypeBool(state), c.infinite, finiteProduct), c.bits, fallback); fallback = SelectBits(state, Classify(state, Join(state, fallback)).nan, {ConstantU32(state, 0u), ConstantU32(state, 0xfff80000u)}, fallback); - const auto subnormalC = Binary(state, spv::OpLogicalAnd, TypeBool(state), isField(exponentC, 0u), Unary(state, spv::OpLogicalNot, TypeBool(state), c.zero)); - const auto special = any({isField(exponentA, 0u), isField(exponentB, 0u), isField(exponentA, 0x7ffu), isField(exponentB, 0x7ffu), isField(exponentC, 0x7ffu), subnormalC}); + const auto special = any({a.zero, b.zero, isField(exponentA, 0x7ffu), isField(exponentB, 0x7ffu), isField(exponentC, 0x7ffu)}); auto result = SelectBits(state, special, fallback, finite); result = SelectBits(state, c.nan, Quiet(state, c.bits), result); diff --git a/core/shader/recompiler/SpirvBackend/src/SpirvAlu/SpirvAluEmitterMath.cpp b/core/shader/recompiler/SpirvBackend/src/SpirvAlu/SpirvAluEmitterMath.cpp index ab3a31dfe..6a78209dc 100644 --- a/core/shader/recompiler/SpirvBackend/src/SpirvAlu/SpirvAluEmitterMath.cpp +++ b/core/shader/recompiler/SpirvBackend/src/SpirvAlu/SpirvAluEmitterMath.cpp @@ -53,19 +53,17 @@ std::uint32_t CompareOrdered64(SpirvEmitterState& state, std::uint32_t lhsValue, } std::uint32_t EmitMulHigh(SpirvEmitterState& state, std::uint32_t lhs, std::uint32_t rhs, bool signedValue) { - const auto operandType = signedValue ? TypeI32(state) : TypeU32(state); - const auto pairType = signedValue ? TypeI32Pair(state) : TypeU32Pair(state); - auto lhsOperand = lhs; - auto rhsOperand = rhs; - if (signedValue) { - lhsOperand = Unary(state, spv::OpBitcast, TypeI32(state), lhs); - rhsOperand = Unary(state, spv::OpBitcast, TypeI32(state), rhs); - } + const auto pairType = TypeU32Pair(state); const auto extended = state.module.AllocateId(); - state.module.AddFunction(signedValue ? spv::OpSMulExtended : spv::OpUMulExtended, pairType, extended, lhsOperand, rhsOperand); + state.module.AddFunction(spv::OpUMulExtended, pairType, extended, lhs, rhs); const auto high = state.module.AllocateId(); - state.module.AddFunction(spv::OpCompositeExtract, operandType, high, extended, 1u); - return signedValue ? Unary(state, spv::OpBitcast, TypeU32(state), high) : high; + state.module.AddFunction(spv::OpCompositeExtract, TypeU32(state), high, extended, 1u); + if (!signedValue) return high; + const auto aNeg = Binary(state, spv::OpSLessThan, TypeBool(state), lhs, ConstantU32(state, 0u)); + const auto bNeg = Binary(state, spv::OpSLessThan, TypeBool(state), rhs, ConstantU32(state, 0u)); + const auto corrA = Select(state, TypeU32(state), aNeg, rhs, ConstantU32(state, 0u)); + const auto corrB = Select(state, TypeU32(state), bNeg, lhs, ConstantU32(state, 0u)); + return Binary(state, spv::OpISub, TypeU32(state), high, Binary(state, spv::OpIAdd, TypeU32(state), corrA, corrB)); } std::uint32_t EmitShift64(SpirvEmitterState& state, spv::Op opcode, std::uint32_t value, std::uint32_t shift) { @@ -234,9 +232,10 @@ struct WaveReduction { IrOpcode opcode; std::uint32_t identity; spv::Op reduce; + bool floating = false; }; -constexpr std::array WaveReductions{{ +constexpr std::array WaveReductions{{ {IrOpcode::UMin32, 0xffffffffu, spv::OpGroupNonUniformUMin}, {IrOpcode::UMax32, 0u, spv::OpGroupNonUniformUMax}, {IrOpcode::SMin32, 0x7fffffffu, spv::OpGroupNonUniformSMin}, @@ -244,6 +243,8 @@ constexpr std::array WaveReductions{{ {IrOpcode::IAdd32, 0u, spv::OpGroupNonUniformIAdd}, {IrOpcode::BitwiseAnd32, 0xffffffffu, spv::OpGroupNonUniformBitwiseAnd}, {IrOpcode::BitwiseOr32, 0u, spv::OpGroupNonUniformBitwiseOr}, + {IrOpcode::FPMin32, 0x7f800000u, spv::OpNop, true}, + {IrOpcode::FPMax32, 0xff800000u, spv::OpNop, true}, }}; struct HalfWaveScan { @@ -269,6 +270,11 @@ bool EitherOrder(const IrValue* value, TMatch&& match) { return match(Resolved(value->Argument(0)), Resolved(value->Argument(1))) || match(Resolved(value->Argument(1)), Resolved(value->Argument(0))); } +template +bool EitherOrder(const std::array& operands, TMatch&& match) { + return match(operands[0], operands[1]) || match(operands[1], operands[0]); +} + std::optional> LaneBitWords(const IrValue* bit) { const IrValue* masked = nullptr; const IrValue* shifted = nullptr; @@ -297,10 +303,15 @@ bool AllLanesBit(const IrValue* bit) { class EntryLaneWalk { public: + explicit EntryLaneWalk(IrShaderStage stage) : everyInvocationEnters(stage == IrShaderStage::Vertex || stage == IrShaderStage::Local || stage == IrShaderStage::TessellationControl || stage == IrShaderStage::TessellationEvaluation) {} + bool ZeroBit(const IrValue* bit) { bit = Resolved(bit); if (bit == nullptr || !Spend()) return false; + if (bit->HasImmediate()) return bit->Type() == IrType::Bool && !bit->ImmediateBool(); if (Is(bit, IrOpcode::LogicalAnd)) return ZeroBit(bit->Argument(0)) || ZeroBit(bit->Argument(1)); + if (Is(bit, IrOpcode::SelectU1)) return ZeroBit(bit->Argument(1)) && ZeroBit(bit->Argument(2)); + if (Is(bit, IrOpcode::Phi)) return Incoming(bit, [this](const IrValue* incoming) { return ZeroBit(incoming); }); const auto words = LaneBitWords(bit); return words && Word((*words)[0], 0u) && Word((*words)[1], 1u); } @@ -314,7 +325,8 @@ private: bool NotHelper(const IrValue* predicate) { predicate = Resolved(predicate); - if (predicate == nullptr || predicate->HasImmediate() || !Spend()) return false; + if (predicate == nullptr || !Spend()) return false; + if (predicate->HasImmediate()) return everyInvocationEnters && predicate->Type() == IrType::Bool && predicate->ImmediateBool(); switch (predicate->Opcode()) { case IrOpcode::LogicalAnd: return NotHelper(predicate->Argument(0)) || NotHelper(predicate->Argument(1)); case IrOpcode::IEqual32: return EitherOrder(predicate, [](const IrValue* builtin, const IrValue* zero) { return IsU32(zero, 0u) && Is(builtin, IrOpcode::GetBuiltin) && IsU32(builtin->Argument(0), static_cast(StageInputKind::HelperInvocation)) && IsU32(builtin->Argument(1), 0u); }); @@ -359,6 +371,7 @@ private: return result; } + bool everyInvocationEnters; std::unordered_set phis; std::uint32_t budget = 512u; }; @@ -368,11 +381,38 @@ bool RowShiftFlags(const IrValue* value, std::uint32_t control) { return flags.control == control && flags.rowMask == 0xfu && flags.bankMask == 0xfu && !flags.boundControl && !flags.fetchInactive; } +const IrValue* Unflushed(const IrValue* value) { + if (!Is(value, IrOpcode::SelectU32)) return value; + const auto* kept = Resolved(value->Argument(2)); + const auto masked = [kept](const IrValue* candidate, std::uint32_t mask) { + return Is(candidate, IrOpcode::BitwiseAnd32) && EitherOrder(candidate, [&](const IrValue* source, const IrValue* constant) { return source == kept && IsU32(constant, mask); }); + }; + const auto* condition = Resolved(value->Argument(0)); + const bool exponentZero = Is(condition, IrOpcode::IEqual32) && EitherOrder(condition, [&](const IrValue* exponent, const IrValue* zero) { return IsU32(zero, 0u) && masked(exponent, 0x7f800000u); }); + return exponentZero && masked(Resolved(value->Argument(1)), 0x80000000u) ? kept : value; +} + +std::optional> ReductionOperands(const IrValue* combined, const WaveReduction& reduction) { + if (reduction.floating) { + combined = Unflushed(combined); + if (!Is(combined, IrOpcode::BitCastU32F32)) return std::nullopt; + combined = Resolved(combined->Argument(0)); + } + if (!Is(combined, reduction.opcode)) return std::nullopt; + std::array operands{Resolved(combined->Argument(0)), Resolved(combined->Argument(1))}; + if (!reduction.floating) return operands; + for (auto& operand : operands) { + if (!Is(operand, IrOpcode::BitCastF32U32)) return std::nullopt; + operand = Unflushed(Resolved(operand->Argument(0))); + } + return operands; +} + const IrValue* RowScanStepInput(const IrValue* step, const WaveReduction& reduction, std::uint32_t control) { if (!Is(step, IrOpcode::DppUpdateU32) || !RowShiftFlags(step, control) || !AllLanesBit(step->Argument(2))) return nullptr; const auto* previous = Resolved(step->Argument(1)); - const auto* combined = Resolved(step->Argument(0)); - const bool matched = Is(combined, reduction.opcode) && EitherOrder(combined, [&](const IrValue* moved, const IrValue* own) { + const auto operands = ReductionOperands(Resolved(step->Argument(0)), reduction); + const bool matched = operands && EitherOrder(*operands, [&](const IrValue* moved, const IrValue* own) { return own == previous && Is(moved, IrOpcode::DppMoveU32) && RowShiftFlags(moved, control) && Resolved(moved->Argument(0)) == previous && AllLanesBit(moved->Argument(1)); }); return matched ? previous : nullptr; @@ -390,25 +430,30 @@ bool CrossesRowsOf(const IrValue* value, const IrValue* scan) { return flags.x16 && !flags.fetchInactive && Resolved(permlane->Argument(0)) == scan && IsU32(permlane->Argument(1), 0xffffffffu) && IsU32(permlane->Argument(2), 0xffffffffu) && AllLanesBit(permlane->Argument(3)); } -std::optional MatchHalfWaveScan(const IrValue* value) { +std::optional MatchHalfWaveScan(const IrValue* value, IrShaderStage stage) { const auto* combined = Unmasked(Resolved(value)); - const auto reduction = std::find_if(WaveReductions.begin(), WaveReductions.end(), [&](const WaveReduction& candidate) { return Is(combined, candidate.opcode); }); + std::optional> operands; + const auto reduction = std::find_if(WaveReductions.begin(), WaveReductions.end(), [&](const WaveReduction& candidate) { return (operands = ReductionOperands(combined, candidate)).has_value(); }); const IrValue* scan = nullptr; - if (reduction == WaveReductions.end() || !EitherOrder(combined, [&](const IrValue* own, const IrValue* crossed) { scan = own; return CrossesRowsOf(crossed, own); })) return std::nullopt; + if (reduction == WaveReductions.end() || !EitherOrder(*operands, [&](const IrValue* own, const IrValue* crossed) { scan = own; return CrossesRowsOf(crossed, own); })) return std::nullopt; for (const auto control : {0x118u, 0x114u, 0x112u, 0x111u}) { scan = RowScanStepInput(scan, *reduction, control); if (scan == nullptr) return std::nullopt; } const auto* source = Unmasked(scan); - if (!Is(source, IrOpcode::SelectU32) || !IsU32(source->Argument(2), reduction->identity) || !EntryLaneWalk{}.ZeroBit(source->Argument(0))) return std::nullopt; + if (!Is(source, IrOpcode::SelectU32) || !IsU32(source->Argument(2), reduction->identity) || !EntryLaneWalk{stage}.ZeroBit(source->Argument(0))) return std::nullopt; return HalfWaveScan{&*reduction, source}; } std::optional EmitHalfWaveReduction(SpirvValueEmitContext& ctx, const IrValue& inst, std::uint32_t lane) { auto& state = ctx.state; if (state.program.WaveSize() != 64u || (lane & 31u) != 31u) return std::nullopt; - const auto scan = MatchHalfWaveScan(inst.Argument(0)); + const auto scan = MatchHalfWaveScan(inst.Argument(0), state.program.Resources().stage); if (!scan) return std::nullopt; + if (scan->reduction->floating) { + if (lane < 32u || state.hostSubgroupSize > 32u) return std::nullopt; + return ConstantU32(state, scan->reduction->identity); + } const auto read = "v_readlane_b32 of lane " + std::to_string(lane) + " of a wave64 half-wave reduction scan"; const auto& capabilities = state.supportedCapabilities; if (std::find(capabilities.begin(), capabilities.end(), static_cast(spv::CapabilityGroupNonUniformArithmetic)) == capabilities.end()) ctx.Fail(inst, (read + " needs subgroup arithmetic, which the device lacks").c_str()); @@ -643,14 +688,29 @@ std::uint32_t EmitFPMaxTri32(SpirvEmitterState& state, std::uint32_t arg0, std:: } std::uint32_t EmitFPMedTri32(SpirvEmitterState& state, std::uint32_t arg0, std::uint32_t arg1, std::uint32_t arg2) { - const auto minAb = EmitMinMaxF32Value(state, arg0, arg1, false); - const auto min3 = EmitMinMaxF32Value(state, minAb, arg2, false); - const auto maxAb = EmitMinMaxF32Value(state, arg0, arg1, true); - const auto highMin = EmitMinMaxF32Value(state, maxAb, arg2, false); - const auto median = EmitMinMaxF32Value(state, minAb, highMin, true); - const auto nanAb = Binary(state, spv::OpLogicalOr, TypeBool(state), EmitClassifyF32(state, arg0).nan, EmitClassifyF32(state, arg1).nan); - const auto anyNan = Binary(state, spv::OpLogicalOr, TypeBool(state), nanAb, EmitClassifyF32(state, arg2).nan); - return Select(state, TypeF32(state), anyNan, min3, median); + const auto u32 = TypeU32(state); + const auto boolean = TypeBool(state); + const auto bits = [&](std::uint32_t value) { return Unary(state, spv::OpBitcast, u32, value); }; + const auto value = [&](std::uint32_t valueBits) { return Unary(state, spv::OpBitcast, TypeF32(state), valueBits); }; + const auto isNan = [&](std::uint32_t valueBits) { return EmitCompareU32Constant(state, spv::OpUGreaterThan, EmitAndConstant(state, valueBits, 0x7fffffffu), 0x7f800000u); }; + const auto numeric = [&](std::uint32_t lhs, std::uint32_t rhs, bool maxValue) { + const auto either = Binary(state, spv::OpBitwiseOr, u32, lhs, rhs); + const auto bothZero = EmitCompareU32Constant(state, spv::OpIEqual, EmitAndConstant(state, either, 0x7fffffffu), 0u); + const auto zeroBits = maxValue ? Binary(state, spv::OpBitwiseAnd, u32, lhs, rhs) : either; + const auto ordered = Select(state, u32, Binary(state, maxValue ? spv::OpFOrdGreaterThanEqual : spv::OpFOrdLessThan, boolean, value(lhs), value(rhs)), lhs, rhs); + return Select(state, u32, bothZero, zeroBits, ordered); + }; + const auto a = bits(arg0); + const auto b = bits(arg1); + const auto c = bits(arg2); + const auto nanA = isNan(a); + const auto nanB = isNan(b); + const auto nanC = isNan(c); + const auto minAb = Select(state, u32, nanB, a, numeric(a, b, false)); + const auto min3 = Select(state, u32, nanC, minAb, numeric(minAb, c, false)); + const auto median = numeric(minAb, numeric(numeric(a, b, true), c, false), true); + const auto anyNan = Binary(state, spv::OpLogicalOr, boolean, Binary(state, spv::OpLogicalOr, boolean, nanA, nanB), nanC); + return value(Select(state, u32, anyNan, min3, median)); } std::uint32_t EmitFPRecip32(SpirvEmitterState& state, std::uint32_t arg0) { @@ -678,16 +738,31 @@ std::uint32_t EmitFPLog2(SpirvEmitterState& state, std::uint32_t arg0) { return EmitExt(state, TypeF32(state), GLSLstd450Log2, {EmitFlushF32DenormToSignedZero(state, arg0)}); } +std::uint32_t TrigLinearNearZero(SpirvEmitterState& state, std::uint32_t value, std::uint32_t distance, std::uint32_t linearConstant) { + const auto linear = Binary(state, spv::OpFMul, TypeF32(state), distance, linearConstant); + const auto magnitude = Binary(state, spv::OpBitwiseAnd, TypeU32(state), Unary(state, spv::OpBitcast, TypeU32(state), distance), ConstantU32(state, 0x7fffffffu)); + const auto near = Binary(state, spv::OpULessThan, TypeBool(state), magnitude, ConstantU32(state, 0x39800000u)); + return Select(state, TypeF32(state), near, linear, value); +} + std::uint32_t EmitFPSin(SpirvEmitterState& state, std::uint32_t arg0) { const auto cycle = EmitTrigCycleF32(state, arg0, true); - const auto source = Binary(state, spv::OpFMul, TypeF32(state), cycle, ConstantF32(state, 0x40c90fdbu)); - return EmitExt(state, TypeF32(state), GLSLstd450Sin, {source}); + const auto absolute = EmitExt(state, TypeF32(state), GLSLstd450FAbs, {cycle}); + const auto signBits = Binary(state, spv::OpBitwiseAnd, TypeU32(state), Unary(state, spv::OpBitcast, TypeU32(state), cycle), ConstantU32(state, 0x80000000u)); + const auto value = EmitExt(state, TypeF32(state), GLSLstd450Sin, {Binary(state, spv::OpFMul, TypeF32(state), absolute, ConstantF32(state, 0x40c90fdbu))}); + const auto nearHalf = Binary(state, spv::OpFOrdGreaterThan, TypeBool(state), absolute, ConstantF32(state, 0x3e800000u)); + const auto distance = Select(state, TypeF32(state), nearHalf, Binary(state, spv::OpFSub, TypeF32(state), ConstantF32(state, 0x3f000000u), absolute), absolute); + const auto linearConstant = Select(state, TypeF32(state), nearHalf, ConstantF32(state, 0x40c90fdbu), ConstantF32(state, 0x40c90fd5u)); + const auto result = TrigLinearNearZero(state, value, distance, linearConstant); + return Unary(state, spv::OpBitcast, TypeF32(state), Binary(state, spv::OpBitwiseOr, TypeU32(state), Unary(state, spv::OpBitcast, TypeU32(state), result), signBits)); } std::uint32_t EmitFPCos(SpirvEmitterState& state, std::uint32_t arg0) { const auto cycle = EmitTrigCycleF32(state, arg0, false); - const auto source = Binary(state, spv::OpFMul, TypeF32(state), cycle, ConstantF32(state, 0x40c90fdbu)); - return EmitExt(state, TypeF32(state), GLSLstd450Cos, {source}); + const auto absolute = EmitExt(state, TypeF32(state), GLSLstd450FAbs, {cycle}); + const auto value = EmitExt(state, TypeF32(state), GLSLstd450Cos, {Binary(state, spv::OpFMul, TypeF32(state), absolute, ConstantF32(state, 0x40c90fdbu))}); + const auto distance = Binary(state, spv::OpFSub, TypeF32(state), ConstantF32(state, 0x3e800000u), absolute); + return TrigLinearNearZero(state, value, distance, ConstantF32(state, 0x40c90fdbu)); } std::uint32_t EmitIdentity(SpirvValueEmitContext&, std::uint32_t value) { @@ -1028,7 +1103,12 @@ std::uint32_t EmitFPUnordEqual32(SpirvEmitterState& state, std::uint32_t arg0, s } std::uint32_t EmitFPOrdNotEqual32(SpirvEmitterState& state, std::uint32_t arg0, std::uint32_t arg1) { - return EmitNative(state, arg0, arg1); + const auto ordered = [&](std::uint32_t value) { + const auto magnitude = EmitNative(state, EmitNative(state, value), ConstantU32(state, 0x7fffffffu)); + return EmitNative(state, magnitude, ConstantU32(state, 0x7f800000u)); + }; + const auto bothOrdered = EmitNative(state, ordered(arg0), ordered(arg1)); + return EmitNative(state, EmitNative(state, arg0, arg1), bothOrdered); } std::uint32_t EmitFPUnordNotEqual32(SpirvEmitterState& state, std::uint32_t arg0, std::uint32_t arg1) { @@ -1131,7 +1211,40 @@ std::uint32_t EmitFAbsValue(SpirvEmitterState& state, std::uint32_t arg0) { return EmitGlsl(state, arg0); } +static bool NativeF16Rte(const SpirvEmitterState& state) { + const auto supported = [&](spv::Capability capability) { + return std::find(state.supportedCapabilities.begin(), state.supportedCapabilities.end(), static_cast(capability)) != state.supportedCapabilities.end(); + }; + return supported(spv::CapabilityFloat16) && supported(spv::CapabilityRoundingModeRTE) && supported(spv::CapabilityDenormPreserve); +} + +static std::uint32_t EmitNativeF32ToF16BitsRte(SpirvEmitterState& state, std::uint32_t value) { + state.module.EmitCapability(spv::CapabilityFloat16); + state.module.EmitCapability(spv::CapabilityRoundingModeRTE); + state.module.EmitCapability(spv::CapabilityDenormPreserve); + if (!state.nativeF16ModesEmitted) { + state.module.AddExecutionMode(state.mainFunc, spv::ExecutionModeRoundingModeRTE, 16u); + state.module.AddExecutionMode(state.mainFunc, spv::ExecutionModeDenormPreserve, 16u); + state.module.AddExecutionMode(state.mainFunc, spv::ExecutionModeSignedZeroInfNanPreserve, 16u); + state.nativeF16ModesEmitted = true; + } + const auto u32 = TypeU32(state); + const auto f16 = state.module.Type(spv::OpTypeFloat, 16u); + const auto half = Unary(state, spv::OpFConvert, f16, value); + const auto pair = state.module.AllocateId(); + state.module.AddFunction(spv::OpCompositeConstruct, state.module.Type(spv::OpTypeVector, f16, 2u), pair, half, state.module.Constant(spv::OpConstant, f16, 0u)); + const auto bits = Unary(state, spv::OpBitcast, u32, pair); + const auto source = Unary(state, spv::OpBitcast, u32, value); + const auto magnitude = Binary(state, spv::OpBitwiseAnd, u32, source, ConstantU32(state, 0x7fffffffu)); + const auto sign = Binary(state, spv::OpBitwiseAnd, u32, Binary(state, spv::OpShiftRightLogical, u32, source, ConstantU32(state, 16u)), ConstantU32(state, 0x8000u)); + const auto payload = Binary(state, spv::OpShiftRightLogical, u32, Binary(state, spv::OpBitwiseAnd, u32, magnitude, ConstantU32(state, 0x7fffffu)), ConstantU32(state, 13u)); + const auto nan = Binary(state, spv::OpBitwiseOr, u32, ConstantU32(state, 0x7e00u), payload); + const auto result = Select(state, u32, Binary(state, spv::OpUGreaterThan, TypeBool(state), magnitude, ConstantU32(state, 0x7f800000u)), nan, Binary(state, spv::OpBitwiseAnd, u32, bits, ConstantU32(state, 0x7fffu))); + return Binary(state, spv::OpBitwiseOr, u32, sign, result); +} + std::uint32_t EmitF32ToF16BitsRte(SpirvEmitterState& state, std::uint32_t value) { + if (NativeF16Rte(state)) return EmitNativeF32ToF16BitsRte(state, value); const auto u32 = TypeU32(state); const auto boolean = TypeBool(state); const auto constant = [&](std::uint32_t bits) { return ConstantU32(state, bits); }; diff --git a/core/shader/recompiler/SpirvBackend/src/SpirvBdaFault.cpp b/core/shader/recompiler/SpirvBackend/src/SpirvBdaFault.cpp index 411f7fc0a..ca8c1a267 100644 --- a/core/shader/recompiler/SpirvBackend/src/SpirvBdaFault.cpp +++ b/core/shader/recompiler/SpirvBackend/src/SpirvBdaFault.cpp @@ -29,16 +29,20 @@ std::uint32_t BdaLoadAddress(SpirvEmitterState& state, std::uint32_t index) { void DefineBdaFaultFunction(SpirvEmitterState& state) { const auto u32 = TypeU32(state); - const auto u64 = TypeScalarU64(state); + const bool words = !state.program.Info().usesDma; + const auto addressType = words ? u32 : TypeScalarU64(state); state.bdaFaultFunction = state.module.AllocateId(); state.module.AddName(state.bdaFaultFunction, "record_bda_fault"); - state.module.AddFunction(spv::OpFunction, TypeVoid(state), state.bdaFaultFunction, spv::FunctionControlDontInlineMask, state.module.Type(spv::OpTypeFunction, TypeVoid(state), u32, u64, u32, u32)); + const auto type = words ? state.module.Type(spv::OpTypeFunction, TypeVoid(state), u32, addressType, u32, u32, u32) : state.module.Type(spv::OpTypeFunction, TypeVoid(state), u32, addressType, u32, u32); + state.module.AddFunction(spv::OpFunction, TypeVoid(state), state.bdaFaultFunction, spv::FunctionControlDontInlineMask, type); const auto reason = state.module.AllocateId(); const auto address = state.module.AllocateId(); + const auto addressHigh = words ? state.module.AllocateId() : 0u; const auto bytes = state.module.AllocateId(); const auto instruction = state.module.AllocateId(); state.module.AddFunction(spv::OpFunctionParameter, u32, reason); - state.module.AddFunction(spv::OpFunctionParameter, u64, address); + state.module.AddFunction(spv::OpFunctionParameter, addressType, address); + if (words) state.module.AddFunction(spv::OpFunctionParameter, u32, addressHigh); state.module.AddFunction(spv::OpFunctionParameter, u32, bytes); state.module.AddFunction(spv::OpFunctionParameter, u32, instruction); EmitLabel(state, state.module.AllocateId()); @@ -53,8 +57,8 @@ void DefineBdaFaultFunction(SpirvEmitterState& state) { state.module.AddFunction(spv::OpStore, BdaWord(state, state.faultBufferVariable, ConstantU32(state, index)), value); }; store(1, reason); - store(2, Unary(state, spv::OpUConvert, TypeU32(state), address)); - store(3, Unary(state, spv::OpUConvert, TypeU32(state), Binary(state, spv::OpShiftRightLogical, TypeScalarU64(state), address, BdaConstant(state, 32u)))); + store(2, words ? address : Unary(state, spv::OpUConvert, TypeU32(state), address)); + store(3, words ? addressHigh : Unary(state, spv::OpUConvert, TypeU32(state), Binary(state, spv::OpShiftRightLogical, TypeScalarU64(state), address, BdaConstant(state, 32u)))); store(4, bytes); store(5, ConstantU32(state, static_cast(state.program.Resources().stage))); store(6, instruction); @@ -67,10 +71,21 @@ void DefineBdaFaultFunction(SpirvEmitterState& state) { } void RecordBdaFault(SpirvEmitterState& state, std::uint32_t address, std::uint32_t bytes, std::uint32_t instruction, BdaAbi::FaultReason reason) { - if (state.bdaFaultFunction == 0) throw std::runtime_error("BDA fault function is missing"); + if (state.bdaFaultFunction == 0 || !state.program.Info().usesDma) throw std::runtime_error("BDA fault function is missing"); state.module.AddFunction(spv::OpFunctionCall, TypeVoid(state), state.module.AllocateId(), state.bdaFaultFunction, ConstantU32(state, static_cast(reason)), address, bytes, instruction); } +void RecordBdaFaultWords(SpirvEmitterState& state, std::uint32_t addressLow, std::uint32_t addressHigh, std::uint32_t bytes, std::uint32_t instruction, BdaAbi::FaultReason reason) { + if (state.program.Info().usesDma) { + const auto u64 = TypeScalarU64(state); + const auto high = Binary(state, spv::OpShiftLeftLogical, u64, Unary(state, spv::OpUConvert, u64, addressHigh), BdaConstant(state, 32u)); + RecordBdaFault(state, Binary(state, spv::OpBitwiseOr, u64, Unary(state, spv::OpUConvert, u64, addressLow), high), bytes, instruction, reason); + return; + } + if (state.bdaFaultFunction == 0) throw std::runtime_error("BDA fault function is missing"); + state.module.AddFunction(spv::OpFunctionCall, TypeVoid(state), state.module.AllocateId(), state.bdaFaultFunction, ConstantU32(state, static_cast(reason)), addressLow, addressHigh, bytes, instruction); +} + void ReturnBdaFailureIf(SpirvEmitterState& state, std::uint32_t condition, std::uint32_t address, std::uint32_t bytes, std::uint32_t instruction, BdaAbi::FaultReason reason) { const auto failed = state.module.AllocateId(); const auto next = state.module.AllocateId(); diff --git a/core/shader/recompiler/SpirvBackend/src/SpirvBdaLookup.cpp b/core/shader/recompiler/SpirvBackend/src/SpirvBdaLookup.cpp index e6d905db3..8cc0fa6d2 100644 --- a/core/shader/recompiler/SpirvBackend/src/SpirvBdaLookup.cpp +++ b/core/shader/recompiler/SpirvBackend/src/SpirvBdaLookup.cpp @@ -130,22 +130,44 @@ static std::uint32_t DefineBdaNoteWrite(SpirvEmitterState& state) { EmitLabel(state, state.module.AllocateId()); const auto done = state.module.AllocateId(); state.module.AddFunction(spv::OpVariable, TypePointer(state, spv::StorageClassFunction, boolean), done, spv::StorageClassFunction); + const auto counter = state.module.AllocateId(); + state.module.AddFunction(spv::OpVariable, TypePointer(state, spv::StorageClassFunction, u32), counter, spv::StorageClassFunction); state.module.AddFunction(spv::OpStore, done, ConstantBool(state, false)); + state.module.AddFunction(spv::OpStore, counter, constant(0u)); const auto page = binary(spv::OpIAdd, u32, Unary(state, spv::OpUConvert, u32, binary(spv::OpShiftRightLogical, TypeScalarU64(state), address, BdaConstant(state, BdaAbi::WrittenPageShift))), constant(1)); const auto hash = binary(spv::OpShiftRightLogical, u32, binary(spv::OpIMul, u32, page, constant(0x9e3779b1u)), constant(32u - static_cast(std::countr_zero(BdaAbi::WrittenPageSlots)))); const auto scope = constant(spv::ScopeDevice); const auto relaxed = constant(spv::MemorySemanticsMaskNone); - for (std::uint32_t probe = 0; probe < BdaAbi::WrittenPageProbes; probe++) { - const auto pending = state.module.AllocateId(); - state.module.AddFunction(spv::OpLoad, boolean, pending, done); - EmitIfCondition(state, Unary(state, spv::OpLogicalNot, boolean, pending), [&] { - const auto slot = binary(spv::OpIAdd, u32, constant(BdaAbi::WrittenSlotsWord), binary(spv::OpBitwiseAnd, u32, binary(spv::OpIAdd, u32, hash, constant(probe)), constant(BdaAbi::WrittenPageSlots - 1u))); - const auto previous = state.module.AllocateId(); - state.module.AddFunction(spv::OpAtomicCompareExchange, u32, previous, BdaWord(state, state.faultBufferVariable, slot), scope, relaxed, relaxed, page, constant(0u)); - const auto taken = binary(spv::OpLogicalOr, boolean, binary(spv::OpIEqual, boolean, previous, constant(0u)), binary(spv::OpIEqual, boolean, previous, page)); - state.module.AddFunction(spv::OpStore, done, taken); - }); - } + const auto header = state.module.AllocateId(); + const auto body = state.module.AllocateId(); + const auto continuation = state.module.AllocateId(); + const auto merge = state.module.AllocateId(); + state.module.AddFunction(spv::OpBranch, header); + EmitLabel(state, header); + const auto probe = state.module.AllocateId(); + state.module.AddFunction(spv::OpLoad, u32, probe, counter); + const auto found = state.module.AllocateId(); + state.module.AddFunction(spv::OpLoad, boolean, found, done); + const auto searching = binary(spv::OpLogicalAnd, boolean, Unary(state, spv::OpLogicalNot, boolean, found), binary(spv::OpULessThan, boolean, probe, constant(BdaAbi::WrittenPageProbes))); + state.module.AddFunction(spv::OpLoopMerge, merge, continuation, spv::LoopControlMaskNone); + state.module.AddFunction(spv::OpBranchConditional, searching, body, merge); + EmitLabel(state, body); + const auto slot = binary(spv::OpIAdd, u32, constant(BdaAbi::WrittenSlotsWord), binary(spv::OpBitwiseAnd, u32, binary(spv::OpIAdd, u32, hash, probe), constant(BdaAbi::WrittenPageSlots - 1u))); + const auto current = state.module.AllocateId(); + state.module.AddFunction(spv::OpAtomicLoad, u32, current, BdaWord(state, state.faultBufferVariable, slot), scope, relaxed); + const auto held = binary(spv::OpIEqual, boolean, current, page); + state.module.AddFunction(spv::OpStore, done, held); + EmitIfCondition(state, Unary(state, spv::OpLogicalNot, boolean, held), [&] { + const auto previous = state.module.AllocateId(); + state.module.AddFunction(spv::OpAtomicCompareExchange, u32, previous, BdaWord(state, state.faultBufferVariable, slot), scope, relaxed, relaxed, page, constant(0u)); + const auto taken = binary(spv::OpLogicalOr, boolean, binary(spv::OpIEqual, boolean, previous, constant(0u)), binary(spv::OpIEqual, boolean, previous, page)); + state.module.AddFunction(spv::OpStore, done, taken); + }); + state.module.AddFunction(spv::OpBranch, continuation); + EmitLabel(state, continuation); + state.module.AddFunction(spv::OpStore, counter, binary(spv::OpIAdd, u32, probe, constant(1u))); + state.module.AddFunction(spv::OpBranch, header); + EmitLabel(state, merge); const auto noted = state.module.AllocateId(); state.module.AddFunction(spv::OpLoad, boolean, noted, done); EmitIfCondition(state, Unary(state, spv::OpLogicalNot, boolean, noted), [&] { @@ -157,15 +179,18 @@ static std::uint32_t DefineBdaNoteWrite(SpirvEmitterState& state) { } void DefineGetBdaPointer(SpirvEmitterState& state) { + if (state.faultBufferVariable != 0) DefineBdaFaultFunction(state); if (!state.program.Info().usesDma) return; - DefineBdaFaultFunction(state); state.bdaPointerFunction = DefineBdaLookup(state, "get_bda_pointer", true); if (!BdaByteReadsForced()) state.bdaProbeFunction = DefineBdaLookup(state, "probe_bda_pointer", false); DefineBdaDwordReadFunctions(state); + DefineBdaSpanReadFunctions(state); if (state.program.Info().bdaWrites) { state.bdaWritePointerFunction = DefineBdaLookup(state, "get_bda_write_pointer", true, BdaAbi::Write); + if (!BdaByteReadsForced()) state.bdaWriteProbeFunction = DefineBdaLookup(state, "probe_bda_write_pointer", false, BdaAbi::Write); state.bdaAtomicPointerFunction = DefineBdaLookup(state, "get_bda_atomic_pointer", true, BdaAbi::Read | BdaAbi::Write); state.bdaNoteWriteFunction = DefineBdaNoteWrite(state); + DefineBdaByteWriteFunctions(state); } } diff --git a/core/shader/recompiler/SpirvBackend/src/SpirvBdaRead.cpp b/core/shader/recompiler/SpirvBackend/src/SpirvBdaRead.cpp index 60df7b542..371d7fc06 100644 --- a/core/shader/recompiler/SpirvBackend/src/SpirvBdaRead.cpp +++ b/core/shader/recompiler/SpirvBackend/src/SpirvBdaRead.cpp @@ -1,6 +1,7 @@ #include "SpirvBackend/SpirvBda.hpp" #include "SpirvBackend/SpirvMemory/SpirvTypes.hpp" #include "SpirvBackend/SpirvMemory/SpirvConstants.hpp" +#include #include #include #include @@ -107,6 +108,108 @@ std::uint32_t BdaAddressUnaligned(SpirvEmitterState& state, std::uint32_t addres return Binary(state, spv::OpINotEqual, TypeBool(state), Binary(state, spv::OpBitwiseAnd, TypeScalarU64(state), address, BdaConstant(state, 3u)), BdaConstant(state, 0u)); } +std::uint32_t AddBdaSignedOffset(SpirvEmitterState& state, std::uint32_t address, std::uint32_t offset, std::uint32_t instruction) { + const auto u32 = TypeU32(state); + const auto u64 = TypeScalarU64(state); + const auto boolean = TypeBool(state); + const auto negative = Binary(state, spv::OpSLessThan, boolean, offset, ConstantU32(state, 0u)); + const auto magnitude = Unary(state, spv::OpUConvert, u64, Select(state, u32, negative, Binary(state, spv::OpISub, u32, ConstantU32(state, 0u), offset), offset)); + const auto sum = Binary(state, spv::OpIAdd, u64, address, magnitude); + const auto result = Select(state, u64, negative, Binary(state, spv::OpISub, u64, address, magnitude), sum); + const auto overflow = Select(state, boolean, negative, Binary(state, spv::OpUGreaterThan, boolean, magnitude, address), Binary(state, spv::OpULessThan, boolean, sum, address)); + EmitIfCondition(state, overflow, [&] { RecordBdaFault(state, address, ConstantU32(state, 0u), instruction, BdaAbi::FaultReason::Overflow); }); + StopBdaInvocationIf(state, overflow); + return result; +} + +std::uint32_t BdaSpanReadResultType(SpirvEmitterState& state) { + const auto values = TypeU32Vector(state, 4); + return state.bdaStopsInvocations ? state.module.Type(spv::OpTypeStruct, values, TypeU32(state)) : values; +} + +std::uint32_t DefineBdaSpanReadFunction(SpirvEmitterState& state, bool coherent) { + const auto u32 = TypeU32(state); + const auto u64 = TypeScalarU64(state); + const auto boolean = TypeBool(state); + const auto values = TypeU32Vector(state, 4); + const auto zero = ConstantU32(state, 0u); + const bool stops = state.bdaStopsInvocations; + const auto result = BdaSpanReadResultType(state); + const auto reader = state.bdaDwordReadFunctions[stops][coherent]; + const auto function = state.module.AllocateId(); + state.module.AddName(function, std::string("read_bda_span") + (stops ? "_stop" : "") + (coherent ? "_coherent" : "")); + state.module.AddFunction(spv::OpFunction, result, function, spv::FunctionControlDontInlineMask, state.module.Type(spv::OpTypeFunction, result, u64, u32, u32, u32)); + const auto address = state.module.AllocateId(); + const auto offset = state.module.AllocateId(); + const auto used = state.module.AllocateId(); + const auto instruction = state.module.AllocateId(); + state.module.AddFunction(spv::OpFunctionParameter, u64, address); + state.module.AddFunction(spv::OpFunctionParameter, u32, offset); + state.module.AddFunction(spv::OpFunctionParameter, u32, used); + state.module.AddFunction(spv::OpFunctionParameter, u32, instruction); + EmitLabel(state, state.module.AllocateId()); + const auto first = EmitGlsl(state, used); + const auto last = EmitGlsl(state, used); + const auto firstBytes = Binary(state, spv::OpShiftLeftLogical, u32, first, ConstantU32(state, 2u)); + const auto firstAddress = AddBdaSignedOffset(state, address, Binary(state, spv::OpIAdd, u32, offset, firstBytes), instruction); + const auto span = Binary(state, spv::OpISub, u32, Binary(state, spv::OpShiftLeftLogical, u32, Binary(state, spv::OpIAdd, u32, last, ConstantU32(state, 1u)), ConstantU32(state, 2u)), firstBytes); + const auto physical = state.module.AllocateId(); + state.module.AddFunction(spv::OpFunctionCall, u64, physical, state.bdaProbeFunction, firstAddress, span, instruction); + const auto mapped = Binary(state, spv::OpINotEqual, boolean, physical, BdaConstant(state, 0u)); + const auto wide = Binary(state, spv::OpLogicalAnd, boolean, mapped, Unary(state, spv::OpLogicalNot, boolean, BdaAddressUnaligned(state, physical))); + const auto isUsed = [&](std::uint32_t dword) { + return Binary(state, spv::OpINotEqual, boolean, Binary(state, spv::OpBitwiseAnd, u32, used, ConstantU32(state, 1u << dword)), zero); + }; + const auto composite = [&](const std::array& parts) { + const auto value = state.module.AllocateId(); + state.module.AddFunction(spv::OpCompositeConstruct, values, value, parts[0], parts[1], parts[2], parts[3]); + return value; + }; + const auto read = EmitValueIfElse(state, wide, values, [&] { + std::array loaded{}; + for (std::uint32_t dword = 0; dword < 4u; ++dword) { + loaded[dword] = EmitValueOrZeroIfCondition(state, isUsed(dword), [&] { + const auto delta = Unary(state, spv::OpUConvert, u64, Binary(state, spv::OpISub, u32, ConstantU32(state, dword * 4u), firstBytes)); + const auto pointer = state.module.AllocateId(); + state.module.AddFunction(spv::OpConvertUToPtr, TypePhysicalU32Pointer(state), pointer, Binary(state, spv::OpIAdd, u64, physical, delta)); + const auto value = state.module.AllocateId(); + state.module.AddFunction(spv::OpLoad, u32, value, pointer, BdaAccessMask(coherent), 4u); + return value; + }); + } + return composite(loaded); + }, [&] { + std::array bytes{}; + for (std::uint32_t dword = 0; dword < 4u; ++dword) { + bytes[dword] = EmitValueOrZeroIfCondition(state, isUsed(dword), [&] { + const auto isFirst = Binary(state, spv::OpIEqual, boolean, first, ConstantU32(state, dword)); + const auto dwordAddress = EmitValueIfElse(state, isFirst, u64, [&] { return firstAddress; }, [&] { + return AddBdaSignedOffset(state, address, Binary(state, spv::OpIAdd, u32, offset, ConstantU32(state, dword * 4u)), instruction); + }); + const auto pair = state.module.AllocateId(); + state.module.AddFunction(spv::OpFunctionCall, TypeU32Vector(state, 2), pair, reader, dwordAddress, instruction); + if (stops) { + const auto stopped = state.module.AllocateId(); + state.module.AddFunction(spv::OpCompositeExtract, u32, stopped, pair, 1u); + StopBdaInvocationIf(state, Binary(state, spv::OpINotEqual, boolean, stopped, zero)); + } + const auto value = state.module.AllocateId(); + state.module.AddFunction(spv::OpCompositeExtract, u32, value, pair, 0u); + return value; + }); + } + return composite(bytes); + }); + auto returned = read; + if (stops) { + returned = state.module.AllocateId(); + state.module.AddFunction(spv::OpCompositeConstruct, result, returned, read, zero); + } + state.module.AddFunction(spv::OpReturnValue, returned); + state.module.AddFunction(spv::OpFunctionEnd); + return function; +} + } void DefineBdaDwordReadFunctions(SpirvEmitterState& state) { @@ -143,17 +246,40 @@ void DefineBdaDwordReadFunctions(SpirvEmitterState& state) { state.bdaStopValue = 0; } +void DefineBdaSpanReadFunctions(SpirvEmitterState& state) { + if (state.bdaProbeFunction == 0) return; + const auto& memory = state.program.Resources().memoryInfo; + const bool coherentAccesses = std::any_of(memory.begin(), memory.end(), [](const MemoryInfo& info) { return info.coherent; }); + const bool stops = state.bdaStopsInvocations; + const auto zero = ConstantU32(state, 0u); + for (const bool stop : {false, true}) { + if (stop && !stops) continue; + state.bdaStopsInvocations = stop; + state.bdaStopValue = stop ? state.module.Constant(spv::OpConstantComposite, BdaSpanReadResultType(state), state.module.Constant(spv::OpConstantComposite, TypeU32Vector(state, 4), zero, zero, zero, zero), ConstantU32(state, 1u)) : 0u; + for (const bool coherent : {false, true}) { + if (coherent && !coherentAccesses) continue; + state.bdaSpanReadFunctions[stop][coherent] = DefineBdaSpanReadFunction(state, coherent); + } + } + state.bdaStopsInvocations = stops; + state.bdaStopValue = 0; +} + // APS5_BDA_BYTE_READS=1 restores one lookup and one byte load per byte for every read. bool BdaByteReadsForced() { static const bool forced = std::getenv("APS5_BDA_BYTE_READS") != nullptr; return forced; } +std::uint32_t BdaInstructionPc(SpirvEmitterState& state, const IrValue& inst) { + return state.bdaPcOverride != 0u ? state.bdaPcOverride : ConstantU32(state, inst.Flags().pc); +} + std::uint32_t AddBdaAddress(SpirvValueEmitContext& ctx, const IrValue& inst, std::uint32_t address, std::uint32_t offset, bool subtract) { auto& state = ctx.state; const auto result = Binary(state, subtract ? spv::OpISub : spv::OpIAdd, TypeScalarU64(state), address, offset); const auto overflow = subtract ? Binary(state, spv::OpUGreaterThan, TypeBool(state), offset, address) : Binary(state, spv::OpULessThan, TypeBool(state), result, address); - EmitIfCondition(state, overflow, [&] { RecordBdaFault(state, address, ConstantU32(state, 0u), ConstantU32(state, inst.Flags().pc), BdaAbi::FaultReason::Overflow); }); + EmitIfCondition(state, overflow, [&] { RecordBdaFault(state, address, ConstantU32(state, 0u), BdaInstructionPc(state, inst), BdaAbi::FaultReason::Overflow); }); StopBdaInvocationIf(state, overflow); return result; } @@ -167,6 +293,7 @@ std::uint32_t AddBdaImmediate(SpirvValueEmitContext& ctx, const IrValue& inst, s } void ValidateBdaTarget(const IrProgram& program, const SpirvTargetOptions& target) { + if (program.Info().usesFaultBuffer && target.bdaAbiVersion != BdaAbi::Version) throw std::runtime_error("the fault buffer needs the BDA fault ABI, which the target lacks"); if (!program.Info().usesDma) return; if (target.bdaAbiVersion != BdaAbi::Version) throw std::runtime_error("unsupported BDA ABI version"); for (const auto capability : {spv::CapabilityInt64, spv::CapabilityPhysicalStorageBufferAddresses, spv::CapabilityStorageBuffer8BitAccess}) { @@ -192,7 +319,7 @@ std::uint32_t EmitBdaRead(SpirvValueEmitContext& ctx, const IrValue& inst, std:: if (bits != 8u && bits != 16u && bits != 32u) ctx.Fail(inst, "unsupported BDA read width"); if (state.bdaPointerFunction == 0) ctx.Fail(inst, "BDA lookup function is missing"); if (bits == 32u) return EmitBdaDwordReads(ctx, inst, address, 0u, 1u)[0]; - const auto instruction = ConstantU32(state, inst.Flags().pc); + const auto instruction = BdaInstructionPc(state, inst); EmitBdaOverflowCheck(state, address, bits / 8u, instruction); return EmitBdaBytes(state, address, bits / 8u, instruction, BdaAccessMask(state, inst)); } @@ -200,7 +327,7 @@ std::uint32_t EmitBdaRead(SpirvValueEmitContext& ctx, const IrValue& inst, std:: void EmitBdaWrite(SpirvValueEmitContext& ctx, const IrValue& inst, std::uint32_t address, std::uint32_t value, std::uint32_t bits) { auto& state = ctx.state; if (state.bdaWritePointerFunction == 0 || state.bdaNoteWriteFunction == 0) ctx.Fail(inst, "BDA write functions are missing"); - const auto instruction = ConstantU32(state, inst.Flags().pc); + const auto instruction = BdaInstructionPc(state, inst); if (bits != 8u && bits != 16u && bits != 32u) ctx.Fail(inst, "unsupported BDA write width"); if (bits != 32u) { EmitBdaOverflowCheck(state, address, bits / 8u, instruction); @@ -226,20 +353,67 @@ void EmitBdaWrite(SpirvValueEmitContext& ctx, const IrValue& inst, std::uint32_t }); } +void DefineBdaByteWriteFunctions(SpirvEmitterState& state) { + const auto u32 = TypeU32(state); + const auto u64 = TypeScalarU64(state); + const auto voidType = state.module.Type(spv::OpTypeVoid); + const auto byteType = state.module.Type(spv::OpTypeInt, 8u, 0u); + const auto bytePointer = TypePointer(state, spv::StorageClassPhysicalStorageBuffer, byteType); + const auto type = state.module.Type(spv::OpTypeFunction, voidType, u64, u32, u32, u32); + const auto& memory = state.program.Resources().memoryInfo; + const bool coherentAccesses = std::any_of(memory.begin(), memory.end(), [](const MemoryInfo& info) { return info.coherent; }); + for (const bool coherent : {false, true}) { + if (coherent && !coherentAccesses) continue; + const auto function = state.module.AllocateId(); + state.module.AddName(function, std::string("write_bda_bytes") + (coherent ? "_coherent" : "")); + state.module.AddFunction(spv::OpFunction, voidType, function, spv::FunctionControlDontInlineMask, type); + const auto address = state.module.AllocateId(); + const auto value = state.module.AllocateId(); + const auto bytes = state.module.AllocateId(); + const auto instruction = state.module.AllocateId(); + state.module.AddFunction(spv::OpFunctionParameter, u64, address); + state.module.AddFunction(spv::OpFunctionParameter, u32, value); + state.module.AddFunction(spv::OpFunctionParameter, u32, bytes); + state.module.AddFunction(spv::OpFunctionParameter, u32, instruction); + EmitLabel(state, state.module.AllocateId()); + const auto counter = state.module.AllocateId(); + state.module.AddFunction(spv::OpVariable, TypePointer(state, spv::StorageClassFunction, u32), counter, spv::StorageClassFunction); + state.module.AddFunction(spv::OpStore, counter, ConstantU32(state, 0u)); + const auto header = state.module.AllocateId(); + const auto body = state.module.AllocateId(); + const auto continuation = state.module.AllocateId(); + const auto merge = state.module.AllocateId(); + state.module.AddFunction(spv::OpBranch, header); + EmitLabel(state, header); + const auto byte = state.module.AllocateId(); + state.module.AddFunction(spv::OpLoad, u32, byte, counter); + const auto pending = Binary(state, spv::OpULessThan, TypeBool(state), byte, bytes); + state.module.AddFunction(spv::OpLoopMerge, merge, continuation, spv::LoopControlMaskNone); + state.module.AddFunction(spv::OpBranchConditional, pending, body, merge); + EmitLabel(state, body); + const auto guest = Binary(state, spv::OpIAdd, u64, address, Unary(state, spv::OpUConvert, u64, byte)); + const auto shifted = Binary(state, spv::OpShiftRightLogical, u32, value, Binary(state, spv::OpShiftLeftLogical, u32, byte, ConstantU32(state, 3u))); + EmitBdaStoreAt(state, guest, 1u, bytePointer, Unary(state, spv::OpUConvert, byteType, shifted), instruction, BdaAccessMask(coherent)); + state.module.AddFunction(spv::OpBranch, continuation); + EmitLabel(state, continuation); + state.module.AddFunction(spv::OpStore, counter, Binary(state, spv::OpIAdd, u32, byte, ConstantU32(state, 1u))); + state.module.AddFunction(spv::OpBranch, header); + EmitLabel(state, merge); + state.module.AddFunction(spv::OpReturn); + state.module.AddFunction(spv::OpFunctionEnd); + state.bdaByteWriteFunctions[coherent] = function; + } +} + void EmitBdaStore(SpirvValueEmitContext& ctx, const IrValue& inst, std::uint32_t address, std::uint32_t value, std::uint32_t bits) { auto& state = ctx.state; if (bits != 8u && bits != 16u && bits != 32u) ctx.Fail(inst, "unsupported BDA store width"); if (state.bdaWritePointerFunction == 0 || state.bdaNoteWriteFunction == 0) ctx.Fail(inst, "BDA write functions are missing"); - const auto instruction = ConstantU32(state, inst.Flags().pc); - const auto accessMask = BdaAccessMask(state, inst); + const auto instruction = BdaInstructionPc(state, inst); const auto storeBytes = [&] { - const auto byteType = state.module.Type(spv::OpTypeInt, 8u, 0u); - const auto bytePointer = TypePointer(state, spv::StorageClassPhysicalStorageBuffer, byteType); - for (std::uint32_t byte = 0; byte < bits / 8u; ++byte) { - const auto guest = byte == 0u ? address : Binary(state, spv::OpIAdd, TypeScalarU64(state), address, BdaConstant(state, byte)); - const auto shifted = byte == 0u ? value : Binary(state, spv::OpShiftRightLogical, TypeU32(state), value, ConstantU32(state, byte * 8u)); - EmitBdaStoreAt(state, guest, 1u, bytePointer, Unary(state, spv::OpUConvert, byteType, shifted), instruction, accessMask); - } + const auto function = state.bdaByteWriteFunctions[BdaCoherent(state, inst)]; + if (function == 0) ctx.Fail(inst, "BDA byte write function is missing"); + state.module.AddFunction(spv::OpFunctionCall, state.module.Type(spv::OpTypeVoid), state.module.AllocateId(), function, address, value, ConstantU32(state, bits / 8u), instruction); }; if (bits != 32u) { storeBytes(); @@ -248,15 +422,87 @@ void EmitBdaStore(SpirvValueEmitContext& ctx, const IrValue& inst, std::uint32_t const auto unaligned = BdaAddressUnaligned(state, address); EmitIfCondition(state, unaligned, storeBytes); EmitIfCondition(state, Unary(state, spv::OpLogicalNot, TypeBool(state), unaligned), [&] { - EmitBdaStoreAt(state, address, 4u, TypePhysicalU32Pointer(state), value, instruction, accessMask); + EmitBdaStoreAt(state, address, 4u, TypePhysicalU32Pointer(state), value, instruction, BdaAccessMask(state, inst)); }); } +void EmitBdaDwordWrites(SpirvValueEmitContext& ctx, const IrValue& inst, std::uint32_t address, std::uint32_t offset, std::uint32_t dwords, std::uint32_t value) { + auto& state = ctx.state; + if (dwords < 2u || dwords > 4u) ctx.Fail(inst, "unsupported BDA dword store count"); + if (state.bdaWritePointerFunction == 0 || state.bdaNoteWriteFunction == 0) ctx.Fail(inst, "BDA write functions are missing"); + const auto u32 = TypeU32(state); + const auto u64 = TypeScalarU64(state); + const auto boolean = TypeBool(state); + std::array values{}; + std::array addresses{}; + for (std::uint32_t dword = 0; dword < dwords; ++dword) { + values[dword] = state.module.AllocateId(); + state.module.AddFunction(spv::OpCompositeExtract, u32, values[dword], value, dword); + addresses[dword] = AddBdaImmediate(ctx, inst, address, static_cast(offset + dword * 4u)); + } + const auto storeDwords = [&] { + for (std::uint32_t dword = 0; dword < dwords; ++dword) EmitBdaStore(ctx, inst, addresses[dword], values[dword], 32u); + }; + if (state.bdaWriteProbeFunction == 0) { + storeDwords(); + return; + } + const auto instruction = BdaInstructionPc(state, inst); + const auto accessMask = BdaAccessMask(state, inst); + const auto physical = state.module.AllocateId(); + state.module.AddFunction(spv::OpFunctionCall, u64, physical, state.bdaWriteProbeFunction, addresses[0], ConstantU32(state, dwords * 4u), instruction); + const auto mapped = Binary(state, spv::OpINotEqual, boolean, physical, BdaConstant(state, 0u)); + const auto aligned = Binary(state, spv::OpLogicalAnd, boolean, Unary(state, spv::OpLogicalNot, boolean, BdaAddressUnaligned(state, addresses[0])), Unary(state, spv::OpLogicalNot, boolean, BdaAddressUnaligned(state, physical))); + const auto wide = Binary(state, spv::OpLogicalAnd, boolean, mapped, aligned); + const auto wideLabel = state.module.AllocateId(); + const auto dwordLabel = state.module.AllocateId(); + const auto merge = state.module.AllocateId(); + state.module.AddFunction(spv::OpSelectionMerge, merge, spv::SelectionControlMaskNone); + state.module.AddFunction(spv::OpBranchConditional, wide, wideLabel, dwordLabel); + EmitLabel(state, wideLabel); + const auto vectorBytes = dwords == 3u ? 4u : dwords * 4u; + const auto separate = [&] { + for (std::uint32_t dword = 0; dword < dwords; ++dword) { + const auto element = dword == 0u ? physical : Binary(state, spv::OpIAdd, u64, physical, BdaConstant(state, dword * 4u)); + const auto pointer = state.module.AllocateId(); + state.module.AddFunction(spv::OpConvertUToPtr, TypePhysicalU32Pointer(state), pointer, element); + state.module.AddFunction(spv::OpStore, pointer, values[dword], accessMask, 4u); + } + }; + if (dwords == 3u) { + separate(); + } else { + const auto vectorAligned = Binary(state, spv::OpIEqual, boolean, Binary(state, spv::OpBitwiseAnd, u64, physical, BdaConstant(state, vectorBytes - 1u)), BdaConstant(state, 0u)); + const auto vectorType = TypeU32Vector(state, dwords); + const auto vector = state.module.AllocateId(); + if (dwords == 2u) state.module.AddFunction(spv::OpCompositeConstruct, vectorType, vector, values[0], values[1]); + else state.module.AddFunction(spv::OpCompositeConstruct, vectorType, vector, values[0], values[1], values[2], values[3]); + EmitIfCondition(state, vectorAligned, [&] { + const auto pointer = state.module.AllocateId(); + state.module.AddFunction(spv::OpConvertUToPtr, TypePointer(state, spv::StorageClassPhysicalStorageBuffer, vectorType), pointer, physical); + state.module.AddFunction(spv::OpStore, pointer, vector, accessMask, vectorBytes); + }); + EmitIfCondition(state, Unary(state, spv::OpLogicalNot, boolean, vectorAligned), separate); + } + const auto last = addresses[dwords - 1u]; + state.module.AddFunction(spv::OpFunctionCall, state.module.Type(spv::OpTypeVoid), state.module.AllocateId(), state.bdaNoteWriteFunction, addresses[0]); + const auto pageShift = BdaConstant(state, BdaAbi::WrittenPageShift); + const auto crosses = Binary(state, spv::OpINotEqual, boolean, Binary(state, spv::OpShiftRightLogical, u64, addresses[0], pageShift), Binary(state, spv::OpShiftRightLogical, u64, last, pageShift)); + EmitIfCondition(state, crosses, [&] { + state.module.AddFunction(spv::OpFunctionCall, state.module.Type(spv::OpTypeVoid), state.module.AllocateId(), state.bdaNoteWriteFunction, last); + }); + state.module.AddFunction(spv::OpBranch, merge); + EmitLabel(state, dwordLabel); + storeDwords(); + state.module.AddFunction(spv::OpBranch, merge); + EmitLabel(state, merge); +} + std::uint32_t EmitBdaAtomic(SpirvValueEmitContext& ctx, const IrValue& inst, std::uint32_t address, std::uint32_t bytes, const std::function& operation) { auto& state = ctx.state; if (bytes != 4u && bytes != 8u) ctx.Fail(inst, "unsupported BDA atomic width"); if (state.bdaWritePointerFunction == 0 || state.bdaNoteWriteFunction == 0) ctx.Fail(inst, "BDA write functions are missing"); - const auto instruction = ConstantU32(state, inst.Flags().pc); + const auto instruction = BdaInstructionPc(state, inst); const auto type = bytes == 8u ? TypeScalarU64(state) : TypeU32(state); const auto zero = bytes == 8u ? BdaConstant(state, 0u) : ConstantU32(state, 0u); const auto unaligned = Binary(state, spv::OpINotEqual, TypeBool(state), Binary(state, spv::OpBitwiseAnd, TypeScalarU64(state), address, BdaConstant(state, bytes - 1u)), BdaConstant(state, 0u)); @@ -285,8 +531,7 @@ std::array EmitBdaDwordReads(SpirvValueEmitContext& ctx, const auto& state = ctx.state; if (dwords == 0u || dwords > 4u) ctx.Fail(inst, "unsupported BDA read width"); if (state.bdaPointerFunction == 0) ctx.Fail(inst, "BDA lookup function is missing"); - const auto u64 = TypeScalarU64(state); - const auto instruction = ConstantU32(state, inst.Flags().pc); + const auto instruction = BdaInstructionPc(state, inst); const auto used = everyDword ? (1u << dwords) - 1u : UsedBdaDwords(inst, dwords); const auto isUsed = [&](std::uint32_t dword) { return (used & (1u << dword)) != 0u; }; std::array values{}; @@ -319,44 +564,25 @@ std::array EmitBdaDwordReads(SpirvValueEmitContext& ctx, const StopBdaInvocationIf(state, Binary(state, spv::OpINotEqual, TypeBool(state), stopped, ConstantU32(state, 0u))); } }; - addresses[first] = dwordAddress(first); if (state.bdaProbeFunction == 0) { + addresses[first] = dwordAddress(first); readBytes(values); return values; } - const auto physical = state.module.AllocateId(); - state.module.AddFunction(spv::OpFunctionCall, u64, physical, state.bdaProbeFunction, addresses[first], ConstantU32(state, (last - first + 1u) * 4u), instruction); - const auto mapped = Binary(state, spv::OpINotEqual, TypeBool(state), physical, BdaConstant(state, 0u)); - const auto aligned = Binary(state, spv::OpIEqual, TypeBool(state), Binary(state, spv::OpBitwiseAnd, u64, physical, BdaConstant(state, 3u)), BdaConstant(state, 0u)); - const auto wide = Binary(state, spv::OpLogicalAnd, TypeBool(state), mapped, aligned); - const auto wideLabel = state.module.AllocateId(); - const auto byteLabel = state.module.AllocateId(); - const auto byteExit = state.module.AllocateId(); - const auto merge = state.module.AllocateId(); - state.module.AddFunction(spv::OpSelectionMerge, merge, spv::SelectionControlMaskNone); - state.module.AddFunction(spv::OpBranchConditional, wide, wideLabel, byteLabel); - EmitLabel(state, wideLabel); - std::array wideValues{}; - for (std::uint32_t dword = first; dword <= last; ++dword) { - if (!isUsed(dword)) continue; - const auto element = dword == first ? physical : Binary(state, spv::OpIAdd, u64, physical, BdaConstant(state, (dword - first) * 4u)); - const auto pointer = state.module.AllocateId(); - state.module.AddFunction(spv::OpConvertUToPtr, TypePhysicalU32Pointer(state), pointer, element); - wideValues[dword] = state.module.AllocateId(); - state.module.AddFunction(spv::OpLoad, TypeU32(state), wideValues[dword], pointer, BdaAccessMask(state, inst), 4u); + const auto function = state.bdaSpanReadFunctions[state.bdaStopsInvocations][BdaCoherent(state, inst)]; + if (function == 0) ctx.Fail(inst, "BDA span read function is missing"); + const auto read = state.module.AllocateId(); + state.module.AddFunction(spv::OpFunctionCall, BdaSpanReadResultType(state), read, function, address, ConstantU32(state, offset), ConstantU32(state, used), instruction); + if (state.bdaStopsInvocations) { + const auto stopped = state.module.AllocateId(); + state.module.AddFunction(spv::OpCompositeExtract, TypeU32(state), stopped, read, 1u); + StopBdaInvocationIf(state, Binary(state, spv::OpINotEqual, TypeBool(state), stopped, ConstantU32(state, 0u))); } - state.module.AddFunction(spv::OpBranch, merge); - EmitLabel(state, byteLabel); - std::array byteValues{}; - readBytes(byteValues); - state.module.AddFunction(spv::OpBranch, byteExit); - EmitLabel(state, byteExit); - state.module.AddFunction(spv::OpBranch, merge); - EmitLabel(state, merge); for (std::uint32_t dword = first; dword <= last; ++dword) { if (!isUsed(dword)) continue; values[dword] = state.module.AllocateId(); - state.module.AddFunction(spv::OpPhi, TypeU32(state), values[dword], wideValues[dword], wideLabel, byteValues[dword], byteExit); + if (state.bdaStopsInvocations) state.module.AddFunction(spv::OpCompositeExtract, TypeU32(state), values[dword], read, 0u, dword); + else state.module.AddFunction(spv::OpCompositeExtract, TypeU32(state), values[dword], read, dword); } return values; } diff --git a/core/shader/recompiler/SpirvBackend/src/SpirvFlowEmitter.cpp b/core/shader/recompiler/SpirvBackend/src/SpirvFlowEmitter.cpp index c08132cb3..d4e8eaa77 100644 --- a/core/shader/recompiler/SpirvBackend/src/SpirvFlowEmitter.cpp +++ b/core/shader/recompiler/SpirvBackend/src/SpirvFlowEmitter.cpp @@ -85,7 +85,8 @@ void EmitReturnTerminator(SpirvValueEmitContext& ctx) { const auto pc = state.module.AllocateId(); state.module.AddFunction(spv::OpLoad, TypeU32(state), pc, state.loopGuardPc); EmitIfCondition(state, Binary(state, spv::OpINotEqual, TypeBool(state), pc, ConstantU32(state, 0u)), [&] { - RecordBdaFault(state, BdaConstant(state, state.program.Resources().shaderHash), ConstantU32(state, state.loopGuardLimit), EmitBinaryU32(state, spv::OpISub, pc, ConstantU32(state, 1u)), BdaAbi::FaultReason::LoopLimit); + const auto hash = state.program.Resources().shaderHash; + RecordBdaFaultWords(state, ConstantU32(state, static_cast(hash)), ConstantU32(state, static_cast(hash >> 32u)), ConstantU32(state, state.loopGuardLimit), EmitBinaryU32(state, spv::OpISub, pc, ConstantU32(state, 1u)), BdaAbi::FaultReason::LoopLimit); }); } if (OrderedPixelShader(state)) { @@ -507,6 +508,9 @@ void EmitDirectInstruction(SpirvValueEmitContext& ctx, const IrValue& inst) { case IrOpcode::StoreAddressU8: return Invoke(EmitStoreAddressU8, ctx, inst); case IrOpcode::StoreAddressU16: return Invoke(EmitStoreAddressU16, ctx, inst); case IrOpcode::StoreAddressU32: return Invoke(EmitStoreAddressU32, ctx, inst); + case IrOpcode::StoreAddressU32x2: return Invoke(EmitStoreAddressU32x2, ctx, inst); + case IrOpcode::StoreAddressU32x3: return Invoke(EmitStoreAddressU32x3, ctx, inst); + case IrOpcode::StoreAddressU32x4: return Invoke(EmitStoreAddressU32x4, ctx, inst); case IrOpcode::AddressAtomicSwap32: return Invoke(EmitAddressAtomic, ctx, inst); case IrOpcode::AddressAtomicCmpSwap32: return Invoke(EmitAddressAtomic, ctx, inst); case IrOpcode::AddressAtomicIAdd32: return Invoke(EmitAddressAtomic, ctx, inst); @@ -729,7 +733,7 @@ void EmitStructuredBlock(SpirvValueEmitContext& ctx, StructuredFunctionState& fu } else if (access != SharedAccess::None) { const bool writes = access != SharedAccess::Read; if (ldsWritten || (writes && ldsRead)) { - state.module.AddFunction(spv::OpControlBarrier, ConstantU32(state, state.waveLdsScope), ConstantU32(state, spv::ScopeWorkgroup), ConstantU32(state, spv::MemorySemanticsAcquireReleaseMask | spv::MemorySemanticsWorkgroupMemoryMask)); + state.module.AddFunction(spv::OpControlBarrier, ConstantU32(state, state.waveLdsScope), ConstantU32(state, spv::ScopeWorkgroup), ConstantU32(state, spv::MemorySemanticsAcquireReleaseMask | LdsMemorySemantics(state))); ldsWritten = false; ldsRead = false; } @@ -811,11 +815,13 @@ void EmitControlFlow(SpirvValueEmitContext& context, StructuredFunctionState& fu context.Fail("structured control flow block has no terminator metadata"); } const bool stops = state.bdaStopsInvocations; - state.bdaStopsInvocations = stops && !IsContinueTarget(program, info->id); + state.continueTarget = IsContinueTarget(program, info->id); + state.bdaStopsInvocations = stops && !state.continueTarget; EmitStructuredBlock(context, functionState, block); functionState.blockExitLabels.emplace(block, state.currentLabel); EmitStructuredTerminator(context, program, *info); state.bdaStopsInvocations = stops; + state.continueTarget = false; } PatchStructuredPhis(context, functionState); } @@ -826,7 +832,7 @@ void EmitVoid(SpirvValueEmitContext&) { void EmitBarrier(SpirvEmitterState& state) { const bool tessellation = state.program.Resources().stage == IrShaderStage::TessellationControl; const auto memoryScope = tessellation ? spv::ScopeInvocation : spv::ScopeWorkgroup; - const auto semantics = tessellation ? spv::MemorySemanticsMaskNone : spv::MemorySemanticsAcquireReleaseMask | spv::MemorySemanticsWorkgroupMemoryMask; + const std::uint32_t semantics = tessellation ? static_cast(spv::MemorySemanticsMaskNone) : spv::MemorySemanticsAcquireReleaseMask | LdsMemorySemantics(state); state.module.AddFunction(spv::OpControlBarrier, ConstantU32(state, spv::ScopeWorkgroup), ConstantU32(state, memoryScope), ConstantU32(state, semantics)); } diff --git a/core/shader/recompiler/SpirvBackend/src/SpirvImageEmitter.cpp b/core/shader/recompiler/SpirvBackend/src/SpirvImageEmitter.cpp index e9fde5d68..9ca9a0dd1 100644 --- a/core/shader/recompiler/SpirvBackend/src/SpirvImageEmitter.cpp +++ b/core/shader/recompiler/SpirvBackend/src/SpirvImageEmitter.cpp @@ -950,7 +950,10 @@ void EmitReadOp(SpirvValueEmitContext& ctx, const ImageEmitAccess& access) { auto& state = ctx.state; const auto& dimensionInfo = RdnaImageDimensionInfoFor(access.image.dimension); const auto numericClass = access.image.numericClass; - const auto condition = ctx.Arg(access.inst, 2); + auto condition = ctx.Arg(access.inst, 2); + if (access.mem.imageDimension == RdnaImageDimension::Dim3D && access.image.dimension == RdnaImageDimension::Dim2D) { + condition = Binary(state, spv::OpLogicalAnd, TypeBool(state), condition, Binary(state, spv::OpIEqual, TypeBool(state), AddressU32(ctx, access, 2u), ConstantU32(state, 0u))); + } ctx.Define(access.inst, EmitValueOrDefaultIfCondition(state, condition, TypeU32Vector(state, 4), ConstantU32CompositeZero(state, 4), [&]() { const auto descriptor = LoadSampledImageDescriptor(state, access.mem.resource, access.slot); const auto color = state.module.AllocateId(); @@ -1294,7 +1297,12 @@ void EmitEmulatedCompareSample(SpirvValueEmitContext& ctx, const ImageEmitAccess reference = Select(state, f32, equal(parameter(4u), EmulatedCompare::ReferenceUnorm), ext(f32, GLSLstd450FClamp, {reference, f32Constant(0.0f), f32Constant(1.0f)}), reference); reference = Select(state, f32, equal(parameter(4u), EmulatedCompare::ReferenceSnorm), ext(f32, GLSLstd450FClamp, {reference, f32Constant(-1.0f), f32Constant(1.0f)}), reference); const auto address = [&](std::uint32_t index, std::uint32_t extent, std::uint32_t mode) { - return Select(state, i32, equal(mode, EmulatedCompare::AddressWrap), Binary(state, spv::OpSMod, i32, index, extent), ext(i32, GLSLstd450SClamp, {index, ConstantI32(state, 0), Binary(state, spv::OpISub, i32, extent, ConstantI32(state, 1))})); + const auto remainder = Binary(state, spv::OpSRem, i32, index, extent); + const auto negative = Binary(state, spv::OpSLessThan, TypeBool(state), remainder, ConstantI32(state, 0)); + const auto correction = Select(state, i32, negative, extent, ConstantI32(state, 0)); + const auto wrapped = Binary(state, spv::OpIAdd, i32, remainder, correction); + const auto clamped = ext(i32, GLSLstd450SClamp, {index, ConstantI32(state, 0), Binary(state, spv::OpISub, i32, extent, ConstantI32(state, 1))}); + return Select(state, i32, equal(mode, EmulatedCompare::AddressWrap), wrapped, clamped); }; const auto inside = [&](std::uint32_t index, std::uint32_t extent) { const auto notBelow = Binary(state, spv::OpSGreaterThanEqual, TypeBool(state), index, ConstantI32(state, 0)); @@ -1470,7 +1478,28 @@ TableSelection TableSlot(SpirvValueEmitContext& ctx, const IrValue& inst, const return EmitIndirectImageSelector(ctx, image, key); } +void EmitConstantSwizzleSample(SpirvValueEmitContext& ctx, const ImageEmitAccess& access) { + auto& state = ctx.state; + const auto numericClass = access.image.numericClass; + const bool gather = access.inst.Opcode() == IrOpcode::ImageGatherRaw; + const auto scalarType = ImageScalarType(state, numericClass); + const auto one = numericClass == IrTextureNumericClass::Float ? ConstantF32(state, 0x3f800000u) : numericClass == IrTextureNumericClass::Sint ? ConstantI32(state, 1) : ConstantU32(state, 1u); + const auto zero = SampledComponentZero(state, numericClass); + std::uint32_t channels[4] = {}; + for (std::uint32_t channel = 0; channel < 4u; channel++) { + const auto selector = RuntimeImageSwizzle(state, access.mem.resource, gather ? ImageGatherComponent(EffectiveDmask(access.mem)) : channel); + channels[channel] = Select(state, scalarType, Binary(state, spv::OpIEqual, TypeBool(state), selector, ConstantU32(state, 1u)), one, zero); + } + const auto value = state.module.AllocateId(); + state.module.AddFunction(spv::OpCompositeConstruct, ImageVectorType(state, numericClass, 4), value, channels[0], channels[1], channels[2], channels[3]); + ctx.Define(access.inst, ResultVector(ctx, access, value, numericClass, false, gather)); +} + void EmitSamplingOp(SpirvValueEmitContext& ctx, const ImageEmitAccess& access) { + if (access.image.constantSwizzle) { + EmitConstantSwizzleSample(ctx, access); + return; + } if (access.image.srgbDecode) { ctx.Fail(access.inst, "samples or gathers an sRGB image the device cannot sample, which is not implemented"); } @@ -1548,7 +1577,10 @@ void EmitImage(SpirvValueEmitContext& ctx, const IrValue& inst) { if (inst.Opcode() == IrOpcode::ImageRead && memory.dataBits == 32u) { const auto value = state.module.AllocateId(); state.module.AddFunction(spv::OpCompositeConstruct, TypeU32Vector(state, 4u), value, ConstantU32(state, 0x76543210u), ConstantU32(state, 0xfedcba98u), ConstantU32(state, 0u), ConstantU32(state, 0u)); - ctx.Define(inst, Select(state, TypeU32Vector(state, 4u), ctx.Arg(inst, 2u), value, ConstantU32CompositeZero(state, 4u))); + const auto active = ctx.Arg(inst, 2u); + const auto mapped = state.module.AllocateId(); + state.module.AddFunction(spv::OpCompositeConstruct, TypeBoolVector(state, 4u), mapped, active, active, active, active); + ctx.Define(inst, Select(state, TypeU32Vector(state, 4u), mapped, value, ConstantU32CompositeZero(state, 4u))); } else { ctx.Fail(inst, "FMASK requires a 32-bit image read"); } @@ -1568,17 +1600,23 @@ void EmitImage(SpirvValueEmitContext& ctx, const IrValue& inst) { const bool returnsValue = inst.Opcode() != IrOpcode::ImageWrite; const auto resultId = returnsValue ? ctx.Result(inst) : 0u; ctx.definitions.erase(&inst); + const bool exits = !state.continueTarget; const auto merge = state.module.AllocateId(); - const auto invalid = state.module.AllocateId(); + const auto invalid = exits ? state.module.AllocateId() : merge; + const auto undefined = returnsValue && !exits ? state.module.AllocateId() : 0u; std::vector labels(modes.size()); std::vector words{spv::OpSwitch, selector, invalid}; for (std::uint32_t index = 0u; index < modes.size(); ++index) { labels[index] = state.module.AllocateId(); words.insert(words.end(), {index, labels[index]}); } + std::vector incoming; + if (undefined != 0u) { + state.module.AddFunction(spv::OpUndef, TypeId(state, inst.Type()), undefined); + incoming.insert(incoming.end(), {undefined, state.currentLabel}); + } state.module.AddFunction(spv::OpSelectionMerge, merge, spv::SelectionControlMaskNone); state.module.AddFunction(words); - std::vector incoming; for (std::uint32_t index = 0u; index < modes.size(); ++index) { EmitLabel(state, labels[index]); std::uint32_t modeMerge = 0u; @@ -1586,18 +1624,26 @@ void EmitImage(SpirvValueEmitContext& ctx, const IrValue& inst) { const auto enabled = state.module.SpecializationConstant(TypeU32(state), PipelineSpecialization::ImageModeBase + memory.resource * PipelineSpecialization::ImageModeStride + index, 1u); const auto active = Binary(state, spv::OpINotEqual, TypeBool(state), enabled, ConstantU32(state, 0u)); const auto activeLabel = state.module.AllocateId(); - const auto inactiveLabel = state.module.AllocateId(); modeMerge = state.module.AllocateId(); + const auto inactiveLabel = exits ? state.module.AllocateId() : modeMerge; state.module.AddFunction(spv::OpSelectionMerge, modeMerge, spv::SelectionControlMaskNone); state.module.AddFunction(spv::OpBranchConditional, active, activeLabel, inactiveLabel); - EmitLabel(state, inactiveLabel); - state.module.AddFunction(spv::OpUnreachable); + if (exits) { + EmitLabel(state, inactiveLabel); + state.module.AddFunction(spv::OpUnreachable); + } EmitLabel(state, activeLabel); } emitMode(modes[index]); if (modeMerge != 0u) { + const auto activeEnd = state.currentLabel; state.module.AddFunction(spv::OpBranch, modeMerge); EmitLabel(state, modeMerge); + if (undefined != 0u) { + const auto joined = state.module.AllocateId(); + state.module.AddFunction(spv::OpPhi, TypeId(state, inst.Type()), joined, ctx.Def(&inst), activeEnd, undefined, labels[index]); + ctx.definitions.insert_or_assign(&inst, joined); + } } if (returnsValue) incoming.insert(incoming.end(), {ctx.Def(&inst), state.currentLabel}); ctx.definitions.erase(&inst); @@ -1605,8 +1651,10 @@ void EmitImage(SpirvValueEmitContext& ctx, const IrValue& inst) { } state.runtimeImage = nullptr; state.runtimeImageMetadata = 0u; - EmitLabel(state, invalid); - state.module.AddFunction(spv::OpUnreachable); + if (exits) { + EmitLabel(state, invalid); + state.module.AddFunction(spv::OpUnreachable); + } EmitLabel(state, merge); if (returnsValue) { std::vector phi{spv::OpPhi, TypeId(state, inst.Type()), resultId}; diff --git a/core/shader/recompiler/SpirvBackend/src/SpirvMemory/SpirvBufferAccess.cpp b/core/shader/recompiler/SpirvBackend/src/SpirvMemory/SpirvBufferAccess.cpp index 7416ff383..b0c4e7b32 100644 --- a/core/shader/recompiler/SpirvBackend/src/SpirvMemory/SpirvBufferAccess.cpp +++ b/core/shader/recompiler/SpirvBackend/src/SpirvMemory/SpirvBufferAccess.cpp @@ -38,7 +38,7 @@ std::uint32_t EmitShaderDataDwordLoad(SpirvEmitterState& state, std::uint32_t dw const auto value = state.module.AllocateId(); if (layout.UsesPushData()) { const auto base = state.module.SpecializationConstant(TypeU32(state), PipelineSpecialization::PushDataOffset, layout.pushDataStartDword); - const auto index = Binary(state, spv::OpIAdd, TypeU32(state), base, ConstantU32(state, dwordIndex)); + const auto index = Binary(state, spv::OpIAdd, TypeU32(state), Binary(state, spv::OpISub, TypeU32(state), base, ConstantU32(state, layout.PushSlotDword())), ConstantU32(state, dwordIndex)); state.module.AddFunction(spv::OpAccessChain, TypePushConstantElementPointer(state), pointer, state.pushConstantVariable, ConstantU32(state, 0), index); } else if (state.shaderDataStorageVariable != 0) { state.module.AddFunction(spv::OpAccessChain, TypeStorageBufferElementPointer(state), pointer, state.shaderDataStorageVariable, ConstantU32(state, 0), ConstantU32(state, dwordIndex)); @@ -65,10 +65,46 @@ std::uint32_t LdsDwordCount(const SpirvEmitterState& state) { return state.requirements.functionLdsDwords != 0u ? state.requirements.functionLdsDwords : FunctionLdsDwordLimit; } +bool LdsInDeviceMemory(const SpirvEmitterState& state) { + return state.program.Info().sharedMemoryBytes != 0u; +} + +std::uint32_t LdsMemorySemantics(const SpirvEmitterState& state) { + return spv::MemorySemanticsWorkgroupMemoryMask | (LdsInDeviceMemory(state) ? spv::MemorySemanticsUniformMemoryMask : 0u); +} + +void EmitLdsBufferBase(SpirvEmitterState& state) { + if (!LdsInDeviceMemory(state)) { + return; + } + if (state.ldsBufferVariable == 0u || state.numWorkgroupsVariable == 0u) { + FailEmit("device memory LDS was not declared before function emission"); + } + if (ShaderWorkgroupInput(state) == nullptr || state.program.Resources().stage != IrShaderStage::Compute || state.program.Info().sharedMemoryBytes != LdsDwordCount(state) * 4u) { + FailEmit("device memory LDS does not match the compute program's LDS size"); + } + const auto groups = [&](std::uint32_t component) { + const auto pointer = state.module.AllocateId(); + const auto value = state.module.AllocateId(); + state.module.AddFunction(spv::OpAccessChain, TypePointer(state, spv::StorageClassInput, TypeU32(state)), pointer, state.numWorkgroupsVariable, ConstantU32(state, component)); + state.module.AddFunction(spv::OpLoad, TypeU32(state), value, pointer); + return value; + }; + const auto group = [&](std::uint32_t component) { return EmitInputComponentU32(state, StageInputKind::WorkgroupId, component); }; + const auto plane = EmitAddU32(state, EmitBinaryU32(state, spv::OpIMul, group(2u), groups(1u)), group(1u)); + const auto linear = EmitAddU32(state, EmitBinaryU32(state, spv::OpIMul, plane, groups(0u)), group(0u)); + state.ldsBufferBase = EmitBinaryU32(state, spv::OpIMul, linear, ConstantU32(state, WorkgroupMemoryStrideDwords(state.program.Info()))); +} + std::uint32_t EmitLdsLockPointer(SpirvEmitterState& state) { if (!state.requirements.ldsLock) { FailEmit("LDS lock was not requested by the program analysis"); } + if (LdsInDeviceMemory(state)) { + const auto pointer = state.module.AllocateId(); + state.module.AddFunction(spv::OpAccessChain, TypeStorageBufferElementPointer(state), pointer, state.ldsBufferVariable, ConstantU32(state, 0u), EmitAddU32(state, state.ldsBufferBase, ConstantU32(state, LdsDwordCount(state)))); + return pointer; + } EnsureLdsStorage(state); const auto pointer = state.module.AllocateId(); state.module.AddFunction(spv::OpAccessChain, TypeU32ElementPointer(state, spv::StorageClassWorkgroup), pointer, state.ldsVariable, ConstantU32(state, LdsDwordCount(state))); @@ -80,6 +116,14 @@ MemoryResourceAccess PrepareMemoryResourceAccess(SpirvEmitterState& state, const access.kind = mem.kind; switch (mem.kind) { case ResourceKind::Lds: + if (LdsInDeviceMemory(state)) { + if (state.ldsBufferVariable == 0u || state.ldsBufferBase == 0u) { + FailEmit("device memory LDS was not prepared at function entry"); + } + access.objectPointer = state.ldsBufferVariable; + access.length = ConstantU32(state, LdsDwordCount(state)); + return access; + } EnsureLdsStorage(state); access.objectPointer = state.ldsVariable; access.length = ConstantU32(state, LdsDwordCount(state)); @@ -125,6 +169,9 @@ std::uint32_t EmitMemoryElementInBounds(SpirvEmitterState& state, const MemoryRe } std::uint32_t EmitMemoryElementPointer(SpirvEmitterState& state, const MemoryResourceAccess& access, std::uint32_t index) { + if (access.kind == ResourceKind::Lds && LdsInDeviceMemory(state)) { + return EmitStorageBufferElementPointer(state, access, EmitAddU32(state, state.ldsBufferBase, index), TypeStorageBufferElementPointer(state)); + } if (access.kind == ResourceKind::Lds || access.kind == ResourceKind::Scratch) { const auto pointer = state.module.AllocateId(); const std::uint32_t storageClass = access.kind == ResourceKind::Scratch ? spv::StorageClassFunction : ShaderWorkgroupInput(state) != nullptr ? spv::StorageClassWorkgroup : spv::StorageClassFunction; diff --git a/core/shader/recompiler/SpirvBackend/src/SpirvMemory/SpirvEmitterHelpers.cpp b/core/shader/recompiler/SpirvBackend/src/SpirvMemory/SpirvEmitterHelpers.cpp index efb0d8e3f..79c762646 100644 --- a/core/shader/recompiler/SpirvBackend/src/SpirvMemory/SpirvEmitterHelpers.cpp +++ b/core/shader/recompiler/SpirvBackend/src/SpirvMemory/SpirvEmitterHelpers.cpp @@ -82,6 +82,7 @@ std::uint32_t BuiltInForInput(StageInputKind kind) { std::uint32_t PerVertexType(SpirvEmitterState& state) { return state.module.DecoratedType(spv::OpTypeStruct, {{spv::OpMemberDecorate, {0u, spv::DecorationBuiltIn, spv::BuiltInPosition}}, + {spv::OpMemberDecorate, {0u, spv::DecorationInvariant}}, {spv::OpDecorate, {spv::DecorationBlock}}}, TypeF32Vector(state, 4u)); } @@ -119,7 +120,7 @@ std::uint32_t PushConstantArrayType(SpirvEmitterState& state) { std::uint32_t PushConstantBlockType(SpirvEmitterState& state) { return state.module.DecoratedType(spv::OpTypeStruct, - {{spv::OpMemberDecorate, {0u, spv::DecorationOffset, 0u}}, + {{spv::OpMemberDecorate, {0u, spv::DecorationOffset, state.program.Metadata().bindings.PushSlotDword() * 4u}}, {spv::OpDecorate, {spv::DecorationBlock}}}, PushConstantArrayType(state)); } @@ -171,7 +172,25 @@ void DefineInputs(SpirvEmitterState& state) { addBuiltin(StageInputKind::WorkgroupId, 3u, "gl_WorkGroupID"); } } + if (LdsInDeviceMemory(state)) { + if (std::none_of(state.inputs.begin(), state.inputs.end(), [](const SpirvInputBinding& input) { + return input.kind == StageInputKind::WorkgroupId; + })) { + state.inputs.push_back(SpirvInputBinding {{StageInputKind::WorkgroupId, 0u, 3u, "gl_WorkGroupID", false}}); + } + state.numWorkgroupsVariable = DefineInterfaceVariable(state, TypeU32Vector(state, 3u), spv::StorageClassInput, "gl_NumWorkGroups"); + state.module.AddAnnotation(spv::OpDecorate, state.numWorkgroupsVariable, spv::DecorationBuiltIn, spv::BuiltInNumWorkgroups); + } const bool pixelStage = state.program.Resources().stage == IrShaderStage::Pixel; + const bool emulated = pixelStage && state.program.Metadata().barycentricEmulation; + BarycentricEmulationLayout emulation; + if (emulated) { + const auto parameters = DescribeFragmentParameters(state.program, state.inputInfo); + const auto reads = [&](StageInputKind kind) { + return std::any_of(state.inputs.begin(), state.inputs.end(), [&](const SpirvInputBinding& input) { return input.kind == kind; }); + }; + emulation = LayoutBarycentricEmulation(parameters, {true, reads(StageInputKind::BaryCoordSmooth), reads(StageInputKind::BaryCoordNoPerspective)}); + } for (auto& input : state.inputs) { if (pixelStage && input.kind == StageInputKind::Parameter) { const auto location = PixelParameterLocation(state, input.location); @@ -235,7 +254,15 @@ void DefineInputs(SpirvEmitterState& state) { } if (input.kind == StageInputKind::Parameter) { const auto flat = PixelParameterIsFlat(state, input.location); - if (input.perVertex) { + auto location = PixelParameterLocation(state, input.location); + if (input.perVertex && emulated) { + const auto relocated = std::find_if(emulation.perVertexLocations.begin(), emulation.perVertexLocations.end(), [&](const auto& entry) { return entry.first == location; }); + if (relocated == emulation.perVertexLocations.end()) { + throw std::runtime_error("SPIR-V module emission failed: per-vertex parameter " + std::to_string(location) + " has no emulated location"); + } + location = relocated->second; + state.module.AddAnnotation(spv::OpDecorate, input.variableId, spv::DecorationFlat); + } else if (input.perVertex) { state.module.AddAnnotation(spv::OpDecorate, input.variableId, spv::DecorationPerVertexKHR); } else if (flat) { state.module.AddAnnotation(spv::OpDecorate, input.variableId, spv::DecorationFlat); @@ -243,7 +270,13 @@ void DefineInputs(SpirvEmitterState& state) { if (!flat && !input.perVertex && PixelParameterIsLinear(state, input.location)) { state.module.AddAnnotation(spv::OpDecorate, input.variableId, spv::DecorationNoPerspective); } - state.module.AddAnnotation(spv::OpDecorate, input.variableId, spv::DecorationLocation, PixelParameterLocation(state, input.location)); + state.module.AddAnnotation(spv::OpDecorate, input.variableId, spv::DecorationLocation, location); + } else if (emulated && (input.kind == StageInputKind::BaryCoordSmooth || input.kind == StageInputKind::BaryCoordNoPerspective)) { + const bool linear = input.kind == StageInputKind::BaryCoordNoPerspective; + if (linear) { + state.module.AddAnnotation(spv::OpDecorate, input.variableId, spv::DecorationNoPerspective); + } + state.module.AddAnnotation(spv::OpDecorate, input.variableId, spv::DecorationLocation, linear ? emulation.linearLocation : emulation.smoothLocation); } else if (const auto builtin = BuiltInForInput(input.kind); builtin != NoBuiltIn) { state.module.AddAnnotation(spv::OpDecorate, input.variableId, spv::DecorationBuiltIn, builtin); } @@ -395,6 +428,13 @@ void DefineDescriptors(SpirvEmitterState& state) { } } } + if (LdsInDeviceMemory(state)) { + state.ldsBufferVariable = state.module.DefineGlobalVariable(TypePointer(state, spv::StorageClassStorageBuffer, StorageBufferBlockType(state)), spv::StorageClassStorageBuffer); + state.module.AddName(state.ldsBufferVariable, "workgroup_memory"); + state.module.AddAnnotation(spv::OpDecorate, state.ldsBufferVariable, spv::DecorationDescriptorSet, WorkgroupMemoryDescriptorSet); + state.module.AddAnnotation(spv::OpDecorate, state.ldsBufferVariable, spv::DecorationBinding, 0u); + state.module.AddAnnotation(spv::OpDecorate, state.ldsBufferVariable, spv::DecorationCoherent); + } } } diff --git a/core/shader/recompiler/SpirvBackend/src/SpirvMemory/SpirvMemoryInstructions.cpp b/core/shader/recompiler/SpirvBackend/src/SpirvMemory/SpirvMemoryInstructions.cpp index 4ef487bfc..7faaadbe1 100644 --- a/core/shader/recompiler/SpirvBackend/src/SpirvMemory/SpirvMemoryInstructions.cpp +++ b/core/shader/recompiler/SpirvBackend/src/SpirvMemory/SpirvMemoryInstructions.cpp @@ -562,7 +562,7 @@ std::uint32_t Atomic32(SpirvValueEmitContext& ctx, const IrValue& inst, const Me const auto operation = [&]() { const auto old = EmitAtomicOperation(ctx, inst, pointer, scope); if (lds) { - const std::uint32_t semantics = spv::MemorySemanticsAcquireReleaseMask | spv::MemorySemanticsWorkgroupMemoryMask; + const std::uint32_t semantics = spv::MemorySemanticsAcquireReleaseMask | LdsMemorySemantics(state); state.module.AddFunction(spv::OpMemoryBarrier, ConstantU32(state, scope), ConstantU32(state, semantics)); } else { EmitDeviceAtomicMemoryBarrier(state); @@ -611,11 +611,15 @@ struct AtomicFloatBits { std::uint64_t sign() const { return wide ? 0x8000000000000000ull : 0x80000000ull; } std::uint64_t quiet() const { return wide ? 0x0008000000000000ull : 0x00400000ull; } std::uint64_t infinity() const { return wide ? 0x7ff0000000000000ull : 0x7f800000ull; } + std::uint64_t minNormal() const { return wide ? 0x0010000000000000ull : 0x00800000ull; } std::uint32_t op(spv::Op opcode, std::uint32_t lhs, std::uint32_t rhs) const { return Binary(state, opcode, type(), lhs, rhs); } std::uint32_t test(spv::Op opcode, std::uint32_t lhs, std::uint32_t rhs) const { return Binary(state, opcode, TypeBool(state), lhs, rhs); } std::uint32_t select(std::uint32_t condition, std::uint32_t lhs, std::uint32_t rhs) const { return Select(state, type(), condition, lhs, rhs); } std::uint32_t magnitude(std::uint32_t value) const { return op(spv::OpBitwiseAnd, value, constant(sign() - 1u)); } std::uint32_t nan(std::uint32_t value) const { return test(spv::OpUGreaterThan, magnitude(value), constant(infinity())); } + std::uint32_t flush(std::uint32_t value, bool enabled) const { + return enabled ? select(test(spv::OpULessThan, magnitude(value), constant(minNormal())), op(spv::OpBitwiseAnd, value, constant(sign())), value) : value; + } std::uint32_t signalingNan(std::uint32_t value) const { return Binary(state, spv::OpLogicalAnd, TypeBool(state), nan(value), test(spv::OpIEqual, op(spv::OpBitwiseAnd, value, constant(quiet())), constant(0u))); } @@ -675,7 +679,7 @@ std::uint32_t SharedAtomicUpdate(SpirvValueEmitContext& ctx, const IrValue& inst const auto& mem = SharedMemory(ctx, inst); return EmitAtomicAccess(ctx, inst, mem, [&](std::uint32_t pointer) { const auto old = AtomicUpdate(state, pointer, mem.kind, [&](std::uint32_t current) { return replacement(state, current); }); - const std::uint32_t semantics = spv::MemorySemanticsAcquireReleaseMask | spv::MemorySemanticsWorkgroupMemoryMask; + const std::uint32_t semantics = spv::MemorySemanticsAcquireReleaseMask | LdsMemorySemantics(state); state.module.AddFunction(spv::OpMemoryBarrier, ConstantU32(state, spv::ScopeWorkgroup), ConstantU32(state, semantics)); return old; }); @@ -683,19 +687,10 @@ std::uint32_t SharedAtomicUpdate(SpirvValueEmitContext& ctx, const IrValue& inst std::uint32_t SharedFloatMinMax(SpirvValueEmitContext& ctx, const IrValue& inst, bool maxValue) { const auto data = ctx.Arg(inst, 1); + const bool flush = SharedMemory(ctx, inst).flushDenormals; return SharedAtomicUpdate(ctx, inst, [&](SpirvEmitterState& state, std::uint32_t old) { - const auto oldFloat = Unary(state, spv::OpBitcast, TypeF32(state), old); - const auto dataFloat = Unary(state, spv::OpBitcast, TypeF32(state), data); - const auto oldNan = Unary(state, spv::OpIsNan, TypeBool(state), oldFloat); - const auto dataNan = Unary(state, spv::OpIsNan, TypeBool(state), dataFloat); - const auto better = Binary(state, maxValue ? spv::OpFOrdGreaterThan : spv::OpFOrdLessThan, TypeBool(state), dataFloat, oldFloat); - const auto magnitudes = Binary(state, spv::OpBitwiseAnd, TypeU32(state), Binary(state, spv::OpBitwiseOr, TypeU32(state), old, data), ConstantU32(state, 0x7fffffffu)); - const auto zeros = Binary(state, spv::OpIEqual, TypeBool(state), magnitudes, ConstantU32(state, 0u)); - const auto zero = Binary(state, maxValue ? spv::OpBitwiseAnd : spv::OpBitwiseOr, TypeU32(state), old, data); - auto result = Select(state, TypeU32(state), better, data, old); - result = Select(state, TypeU32(state), zeros, zero, result); - result = Select(state, TypeU32(state), dataNan, old, result); - return Select(state, TypeU32(state), oldNan, data, result); + const AtomicFloatBits bits{state, false}; + return bits.minMax(bits.flush(old, flush), bits.flush(data, flush), maxValue); }); } @@ -884,14 +879,14 @@ std::uint32_t LockedLdsUpdate(SpirvEmitterState& state, TUpdate&& update) { state.module.AddFunction(spv::OpBranch, spinBody); EmitLabel(state, spinBody); const auto previous = state.module.AllocateId(); - state.module.AddFunction(spv::OpAtomicCompareExchange, TypeU32(state), previous, lock, ConstantU32(state, spv::ScopeWorkgroup), ConstantU32(state, spv::MemorySemanticsAcquireMask | spv::MemorySemanticsWorkgroupMemoryMask), ConstantU32(state, spv::MemorySemanticsMaskNone), ConstantU32(state, 1u), ConstantU32(state, 0u)); + state.module.AddFunction(spv::OpAtomicCompareExchange, TypeU32(state), previous, lock, ConstantU32(state, spv::ScopeWorkgroup), ConstantU32(state, spv::MemorySemanticsAcquireMask | LdsMemorySemantics(state)), ConstantU32(state, spv::MemorySemanticsMaskNone), ConstantU32(state, 1u), ConstantU32(state, 0u)); const auto acquired = Binary(state, spv::OpIEqual, TypeBool(state), previous, ConstantU32(state, 0u)); state.module.AddFunction(spv::OpBranchConditional, acquired, critical, spinCont); EmitLabel(state, spinCont); state.module.AddFunction(spv::OpBranch, spinHeader); EmitLabel(state, critical); const auto old = update(); - state.module.AddFunction(spv::OpAtomicStore, lock, ConstantU32(state, spv::ScopeWorkgroup), ConstantU32(state, spv::MemorySemanticsReleaseMask | spv::MemorySemanticsWorkgroupMemoryMask), ConstantU32(state, 0u)); + state.module.AddFunction(spv::OpAtomicStore, lock, ConstantU32(state, spv::ScopeWorkgroup), ConstantU32(state, spv::MemorySemanticsReleaseMask | LdsMemorySemantics(state)), ConstantU32(state, 0u)); const auto criticalExit = state.currentLabel; state.module.AddFunction(spv::OpBranch, servedMerge); EmitLabel(state, servedMerge); @@ -972,11 +967,10 @@ std::uint32_t SharedAtomic64Select(SpirvValueEmitContext& ctx, const IrValue& in } std::uint32_t SharedFloatMinMax64(SpirvValueEmitContext& ctx, const IrValue& inst, bool maxValue) { + const bool flush = SharedMemory(ctx, inst).flushDenormals; return SharedAtomic64Binary(ctx, inst, [&](SpirvEmitterState& state, std::uint32_t old, std::uint32_t data) { const AtomicFloatBits bits{state, true}; - auto result = bits.select(bits.test(maxValue ? spv::OpUGreaterThan : spv::OpULessThan, bits.orderKey(data), bits.orderKey(old)), data, old); - result = bits.select(bits.nan(data), old, result); - return bits.select(bits.nan(old), data, result); + return bits.minMax(bits.flush(old, flush), bits.flush(data, flush), maxValue); }); } @@ -1020,6 +1014,41 @@ std::uint32_t EmitReadConst(SpirvValueEmitContext& ctx, const IrValue& inst) { if (state.flattenedSrtVariable == 0) { ctx.Fail(inst, "requires the flattened SRT descriptor"); } + const auto& resources = state.program.Resources(); + const auto& guarded = resources.guardedSrtSlots; + if (!guarded.empty()) { + const IrValue* slot = inst.Argument(1)->Resolve(); + if (!slot->HasImmediate()) { + ctx.Fail(inst, "reads the flattened SRT at a runtime slot while some slots are guarded"); + } + const auto found = std::lower_bound(guarded.begin(), guarded.end(), slot->ImmediateU32()); + if (found != guarded.end() && *found == slot->ImmediateU32()) { + if (state.faultBufferVariable == 0) { + ctx.Fail(inst, "reads a guarded SRT slot without a fault buffer"); + } + const auto u32 = TypeU32(state); + const auto load = [&](std::uint32_t index) { + const auto pointer = state.module.AllocateId(); + state.module.AddFunction(spv::OpAccessChain, TypeStorageBufferElementPointer(state), pointer, state.flattenedSrtVariable, ConstantU32(state, 0u), index); + const auto value = state.module.AllocateId(); + state.module.AddFunction(spv::OpLoad, u32, value, pointer); + return value; + }; + const auto guard = static_cast(found - guarded.begin()); + const auto flag = load(ConstantU32(state, resources.srtGuardOffset + guard)); + const auto poisoned = Binary(state, spv::OpINotEqual, TypeBool(state), flag, ConstantU32(state, 0u)); + EmitIfCondition(state, poisoned, [&] { + const auto records = resources.srtGuardOffset + static_cast(guarded.size()); + const auto record = Binary(state, spv::OpIAdd, u32, ConstantU32(state, records), Binary(state, spv::OpIMul, u32, Binary(state, spv::OpISub, u32, flag, ConstantU32(state, 1u)), ConstantU32(state, 3u))); + const auto word = [&](std::uint32_t offset) { return load(Binary(state, spv::OpIAdd, u32, record, ConstantU32(state, offset))); }; + const auto pc = word(0u); + const auto low = word(1u); + const auto high = word(2u); + RecordBdaFaultWords(state, low, high, ConstantU32(state, 4u), pc, BdaAbi::FaultReason::Unmapped); + }); + StopBdaInvocationIf(state, poisoned); + } + } const auto pointer = state.module.AllocateId(); state.module.AddFunction(spv::OpAccessChain, TypeStorageBufferElementPointer(state), pointer, state.flattenedSrtVariable, ConstantU32(state, 0u), ctx.Arg(inst, 1)); const auto value = state.module.AllocateId(); @@ -1086,6 +1115,26 @@ void EmitStoreAddressU32(SpirvValueEmitContext& ctx, const IrValue& inst) { StoreAddress(ctx, inst, 32u); } +void StoreAddressWide(SpirvValueEmitContext& ctx, const IrValue& inst, std::uint32_t components) { + const auto& mem = ctx.Memory(inst); + if (mem.kind != ResourceKind::Flat && mem.kind != ResourceKind::Global) ctx.Fail(inst, "must write a physical address resource"); + EmitIfCondition(ctx.state, ActiveArgument(ctx, inst), [&]() { + EmitBdaDwordWrites(ctx, inst, GuestAddressBase(ctx, inst, mem), mem.offset, components, ctx.Arg(inst, inst.ArgumentCount() - 2u)); + }); +} + +void EmitStoreAddressU32x2(SpirvValueEmitContext& ctx, const IrValue& inst) { + StoreAddressWide(ctx, inst, 2u); +} + +void EmitStoreAddressU32x3(SpirvValueEmitContext& ctx, const IrValue& inst) { + StoreAddressWide(ctx, inst, 3u); +} + +void EmitStoreAddressU32x4(SpirvValueEmitContext& ctx, const IrValue& inst) { + StoreAddressWide(ctx, inst, 4u); +} + std::uint32_t EmitAddressAtomic(SpirvValueEmitContext& ctx, const IrValue& inst) { return AddressAtomic(ctx, inst); } @@ -1338,9 +1387,10 @@ std::uint32_t EmitSharedAtomicCmpst32(SpirvValueEmitContext& ctx, const IrValue& std::uint32_t EmitSharedAtomicCmpstF32(SpirvValueEmitContext& ctx, const IrValue& inst) { const auto compare = ctx.Arg(inst, 1); const auto source = ctx.Arg(inst, 2); + const bool flush = SharedMemory(ctx, inst).flushDenormals; return SharedAtomicUpdate(ctx, inst, [&](SpirvEmitterState& state, std::uint32_t old) { - const auto equal = Binary(state, spv::OpFOrdEqual, TypeBool(state), Unary(state, spv::OpBitcast, TypeF32(state), old), Unary(state, spv::OpBitcast, TypeF32(state), compare)); - return Select(state, TypeU32(state), equal, source, old); + const AtomicFloatBits bits{state, false}; + return bits.select(bits.equal(bits.flush(old, flush), bits.flush(compare, flush)), source, old); }); } @@ -1440,9 +1490,10 @@ std::uint32_t EmitSharedAtomicCmpst64(SpirvValueEmitContext& ctx, const IrValue& std::uint32_t EmitSharedAtomicCmpstF64(SpirvValueEmitContext& ctx, const IrValue& inst) { const auto compare = ScalarU64Argument(ctx, inst, 3u); const auto source = ScalarU64Argument(ctx, inst, 2u); + const bool flush = SharedMemory(ctx, inst).flushDenormals; return SharedAtomic64(ctx, inst, [&](SpirvEmitterState& state, std::uint32_t old) { const AtomicFloatBits bits{state, true}; - return bits.select(bits.equal(old, compare), source, old); + return bits.select(bits.equal(bits.flush(old, flush), bits.flush(compare, flush)), source, old); }); } diff --git a/core/shader/recompiler/SpirvBackend/src/SpirvMemory/SpirvModuleSetup.cpp b/core/shader/recompiler/SpirvBackend/src/SpirvMemory/SpirvModuleSetup.cpp index 2689e24de..b8d68e795 100644 --- a/core/shader/recompiler/SpirvBackend/src/SpirvMemory/SpirvModuleSetup.cpp +++ b/core/shader/recompiler/SpirvBackend/src/SpirvMemory/SpirvModuleSetup.cpp @@ -134,7 +134,7 @@ void DefineModule(SpirvEmitterState& state) { const bool fragmentBarycentric = StageOf(state) == IrShaderStage::Pixel && std::any_of(state.inputs.begin(), state.inputs.end(), [](const SpirvInputBinding& input) { return input.perVertex || input.kind == StageInputKind::BaryCoordSmooth || input.kind == StageInputKind::BaryCoordNoPerspective; }); - if (fragmentBarycentric) { + if (fragmentBarycentric && !state.program.Metadata().barycentricEmulation) { state.module.EmitCapability(spv::CapabilityFragmentBarycentricKHR); state.module.EmitExtension("SPV_KHR_fragment_shader_barycentric"); } diff --git a/core/shader/recompiler/SpirvBackend/src/SpirvModule.cpp b/core/shader/recompiler/SpirvBackend/src/SpirvModule.cpp index 9b6d701cc..3dbdbc300 100644 --- a/core/shader/recompiler/SpirvBackend/src/SpirvModule.cpp +++ b/core/shader/recompiler/SpirvBackend/src/SpirvModule.cpp @@ -87,8 +87,9 @@ std::vector SpirvModule::Finalize() const { if (unpatchedPhiIncomings != 0) { throw std::runtime_error("SpirvModule::Finalize called with unpatched OpPhi incoming pairs"); } + if (inHelperFunction) throw std::runtime_error("SpirvModule::Finalize called inside a helper function"); std::vector module; - module.reserve(5u + capabilities.size() + extensions.size() + extInstImports.size() + memoryModel.size() + entryPoints.size() + executionModes.size() + debug.size() + annotations.size() + typeDeclarations.size() + declarations.size() + globalVariables.size() + functionInstructions.size()); + module.reserve(5u + capabilities.size() + extensions.size() + extInstImports.size() + memoryModel.size() + entryPoints.size() + executionModes.size() + debug.size() + annotations.size() + typeDeclarations.size() + declarations.size() + globalVariables.size() + functionInstructions.size() + helperFunctionInstructions.size()); module.push_back(spv::MagicNumber); module.push_back(version); module.push_back(0u); @@ -106,9 +107,22 @@ std::vector SpirvModule::Finalize() const { module.insert(module.end(), declarations.begin(), declarations.end()); module.insert(module.end(), globalVariables.begin(), globalVariables.end()); module.insert(module.end(), functionInstructions.begin(), functionInstructions.end()); + module.insert(module.end(), helperFunctionInstructions.begin(), helperFunctionInstructions.end()); return module; } +void SpirvModule::BeginHelperFunction() { + if (inHelperFunction) throw std::runtime_error("SpirvModule helper functions cannot nest"); + std::swap(functionInstructions, helperFunctionInstructions); + inHelperFunction = true; +} + +void SpirvModule::EndHelperFunction() { + if (!inHelperFunction) throw std::runtime_error("SpirvModule::EndHelperFunction without a helper function"); + std::swap(functionInstructions, helperFunctionInstructions); + inHelperFunction = false; +} + void SpirvModule::RequireVersion(std::uint32_t version) { this->version = std::max(this->version, version); } diff --git a/core/shader/recompiler/SpirvBackend/src/SpirvModuleEmitter.cpp b/core/shader/recompiler/SpirvBackend/src/SpirvModuleEmitter.cpp index 4fb0e39aa..baf755e18 100644 --- a/core/shader/recompiler/SpirvBackend/src/SpirvModuleEmitter.cpp +++ b/core/shader/recompiler/SpirvBackend/src/SpirvModuleEmitter.cpp @@ -1,4 +1,5 @@ #include "SpirvBackend/SpirvModuleEmitter.hpp" +#include "SpirvBackend/SpirvBufferFormat.hpp" #include "SpirvBackend/SpirvEmitterHelpers.hpp" #include "SpirvBackend/SpirvEmitterInstructions.hpp" #include "SpirvBackend/SpirvFlowEmitter.hpp" @@ -242,6 +243,35 @@ std::uint32_t MrtOutputMode(const SpirvEmitterState& state, const ExportInfo& ex return state.inputInfo.pixel->targetOutputMode[exp.index]; } +std::uint32_t PackUnorm10_11_11(SpirvEmitterState& state, std::uint32_t color) { + const auto info = GetFormatInfo(IrBufferFormat::Format11_11_10UNorm); + auto packed = ConstantU32(state, 0u); + for (std::uint32_t component = 0; component < info.componentCount; component++) { + const auto value = state.module.AllocateId(); + state.module.AddFunction(spv::OpCompositeExtract, TypeF32(state), value, color, component); + const auto bits = state.module.AllocateId(); + state.module.AddFunction(spv::OpBitcast, TypeU32(state), bits, value); + const auto field = EmitFormatStoreComponent(state, info, component, bits); + packed = EmitOrU32(state, packed, info.componentBitOffset[component] == 0u ? field : EmitBinaryU32(state, spv::OpShiftLeftLogical, field, ConstantU32(state, info.componentBitOffset[component]))); + } + return packed; +} + +std::uint32_t SelectExportPacking(SpirvEmitterState& state, std::uint32_t target, std::uint32_t color) { + const auto packing = state.module.SpecializationConstant(TypeU32(state), PipelineSpecialization::ExportPackingBase + target, static_cast(ColorExportPacking::None)); + const auto packed = EmitCompareU32Constant(state, spv::OpIEqual, packing, static_cast(ColorExportPacking::Unorm10_11_11)); + const auto word = PackUnorm10_11_11(state, color); + std::array components{}; + for (std::uint32_t component = 0; component < components.size(); ++component) { + const auto unpacked = state.module.AllocateId(); + state.module.AddFunction(spv::OpCompositeExtract, TypeF32(state), unpacked, color, component); + components[component] = Select(state, TypeF32(state), packed, component == 0u ? Unary(state, spv::OpBitcast, TypeF32(state), word) : ConstantF32(state, 0u), unpacked); + } + const auto result = state.module.AllocateId(); + state.module.AddFunction(spv::OpCompositeConstruct, TypeF32Vector(state, 4u), result, components[0], components[1], components[2], components[3]); + return result; +} + std::uint32_t ExportRawComponent(SpirvValueEmitContext& ctx, std::uint32_t vector, std::uint32_t component) { const auto value = ctx.state.module.AllocateId(); ctx.state.module.AddFunction(spv::OpCompositeExtract, TypeU32(ctx.state), value, vector, component); @@ -572,6 +602,9 @@ void DefineMeshOutputs(SpirvEmitterState& state) { if (output.kind == StageOutputKind::Layer) { state.module.AddAnnotation(spv::OpDecorate, output.variableId, spv::DecorationPerPrimitiveEXT); } + if (output.kind == StageOutputKind::Position) { + state.module.AddAnnotation(spv::OpDecorate, output.variableId, spv::DecorationInvariant); + } } state.meshAllocation = MeshArray(state, spv::StorageClassWorkgroup, TypeU32(state), 2u); state.meshPrimitiveData = MeshArray(state, spv::StorageClassPrivate, TypeU32(state), state.laneCount); @@ -775,11 +808,7 @@ F32Class EmitClassifyF32Bits(SpirvEmitterState& state, std::uint32_t bits) { F32Class cls; cls.bits = bits; const auto absBits = EmitAndConstant(state, cls.bits, 0x7fffffffu); - const auto exponentBits = EmitAndConstant(state, absBits, 0x7f800000u); - const auto mantissaBits = EmitAndConstant(state, absBits, 0x007fffffu); - const auto exponentMax = EmitCompareU32Constant(state, spv::OpIEqual, exponentBits, 0x7f800000u); - const auto mantissaNonzero = EmitCompareU32Constant(state, spv::OpINotEqual, mantissaBits, 0u); - cls.nan = Binary(state, spv::OpLogicalAnd, TypeBool(state), exponentMax, mantissaNonzero); + cls.nan = EmitCompareU32Constant(state, spv::OpUGreaterThan, absBits, 0x7f800000u); cls.zero = EmitCompareU32Constant(state, spv::OpIEqual, absBits, 0u); return cls; } @@ -842,8 +871,7 @@ std::uint32_t EmitMinMaxF32Value(SpirvEmitterState& state, std::uint32_t lhs, st const auto zeroBits = Binary(state, maxValue ? spv::OpBitwiseAnd : spv::OpBitwiseOr, TypeU32(state), lhsClass.bits, rhsClass.bits); const auto numericBits = Select(state, TypeU32(state), bothZero, zeroBits, orderedBits); - const auto lhsNanBits = Select(state, TypeU32(state), lhsClass.nan, rhsClass.bits, numericBits); - const auto resultBits = Select(state, TypeU32(state), rhsClass.nan, lhsClass.bits, lhsNanBits); + const auto resultBits = Select(state, TypeU32(state), rhsClass.nan, lhsClass.bits, numericBits); return Unary(state, spv::OpBitcast, TypeF32(state), resultBits); } @@ -868,25 +896,19 @@ std::uint32_t EmitFlushF32DenormToSignedZero(SpirvEmitterState& state, std::uint } std::uint32_t EmitTrigCycleF32(SpirvEmitterState& state, std::uint32_t src, bool preserveSignedZero) { - const auto fract = state.module.AllocateId(); - const auto bits = state.module.AllocateId(); - const auto absBits = state.module.AllocateId(); - const auto large = state.module.AllocateId(); - const auto finite = state.module.AllocateId(); - const auto largeFinite = state.module.AllocateId(); + const auto rounded = state.module.AllocateId(); const auto reduced = state.module.AllocateId(); - state.module.AddFunction(spv::OpExtInst, TypeF32(state), fract, GlslStd450(state), GLSLstd450Fract, src); - state.module.AddFunction(spv::OpBitcast, TypeU32(state), bits, src); - state.module.AddFunction(spv::OpBitwiseAnd, TypeU32(state), absBits, bits, ConstantU32(state, 0x7fffffffu)); - state.module.AddFunction(spv::OpUGreaterThanEqual, TypeBool(state), large, absBits, ConstantU32(state, 0x4b000000u)); - state.module.AddFunction(spv::OpULessThan, TypeBool(state), finite, absBits, ConstantU32(state, 0x7f800000u)); - state.module.AddFunction(spv::OpLogicalAnd, TypeBool(state), largeFinite, large, finite); - state.module.AddFunction(spv::OpSelect, TypeF32(state), reduced, largeFinite, ConstantF32Value(state, 0.0f), fract); + state.module.AddFunction(spv::OpExtInst, TypeF32(state), rounded, GlslStd450(state), GLSLstd450RoundEven, src); + state.module.AddFunction(spv::OpFSub, TypeF32(state), reduced, src, rounded); if (!preserveSignedZero) { return reduced; } + const auto bits = state.module.AllocateId(); + const auto absBits = state.module.AllocateId(); const auto zero = state.module.AllocateId(); const auto result = state.module.AllocateId(); + state.module.AddFunction(spv::OpBitcast, TypeU32(state), bits, src); + state.module.AddFunction(spv::OpBitwiseAnd, TypeU32(state), absBits, bits, ConstantU32(state, 0x7fffffffu)); state.module.AddFunction(spv::OpIEqual, TypeBool(state), zero, absBits, ConstantU32(state, 0u)); state.module.AddFunction(spv::OpSelect, TypeF32(state), result, zero, src, reduced); return result; @@ -949,9 +971,10 @@ void EmitProgram(SpirvEmitterState& state) { state.module.AddFunction(spv::OpVariable, TypePointer(state, spv::StorageClassFunction, TypeU32(state)), lane.scratchU32Variable, spv::StorageClassFunction); } } + EmitLdsBufferBase(state); if (state.requirements.ldsLock) { state.module.AddFunction(spv::OpStore, EmitLdsLockPointer(state), ConstantU32(state, 0u)); - state.module.AddFunction(spv::OpControlBarrier, ConstantU32(state, spv::ScopeWorkgroup), ConstantU32(state, spv::ScopeWorkgroup), ConstantU32(state, spv::MemorySemanticsAcquireReleaseMask | spv::MemorySemanticsWorkgroupMemoryMask)); + state.module.AddFunction(spv::OpControlBarrier, ConstantU32(state, spv::ScopeWorkgroup), ConstantU32(state, spv::ScopeWorkgroup), ConstantU32(state, spv::MemorySemanticsAcquireReleaseMask | LdsMemorySemantics(state))); } if (state.gdsVariable != 0u) { state.gdsLength = state.module.AllocateId(); @@ -1180,6 +1203,8 @@ void EmitSetAttribute(SpirvValueEmitContext& ctx, const IrValue& inst) { const auto mapped = state.module.AllocateId(); state.module.AddFunction(spv::OpCompositeConstruct, vectorType, mapped, components[0], components[1], components[2], components[3]); value = mapped; + const auto mode = MrtOutputMode(state, exp); + if (!uintOutput && mode != 5u && mode != 6u) value = SelectExportPacking(state, exp.index, value); } if (exp.kind == ExportTargetKind::Position) { if (state.inputInfo.vertex == nullptr) { diff --git a/core/shader/recompiler/SpirvBackend/src/SpirvRuntimeBuffer.cpp b/core/shader/recompiler/SpirvBackend/src/SpirvRuntimeBuffer.cpp index 59279223c..f841ffd0b 100644 --- a/core/shader/recompiler/SpirvBackend/src/SpirvRuntimeBuffer.cpp +++ b/core/shader/recompiler/SpirvBackend/src/SpirvRuntimeBuffer.cpp @@ -7,6 +7,7 @@ #include "PipelineSpecialization.hpp" #include #include +#include #include namespace ShaderRecompiler { @@ -20,6 +21,8 @@ struct RuntimeDescriptor { std::uint32_t swizzle; std::uint32_t index; std::uint32_t valid; + std::uint32_t offset = 0; + std::uint32_t soffset = 0; }; struct RuntimeAddress { @@ -45,7 +48,7 @@ std::uint32_t nonzero(SpirvEmitterState& state, std::uint32_t value) { void faultIf(SpirvValueEmitContext& context, const IrValue& instruction, std::uint32_t condition, std::uint32_t address, std::uint32_t bytes, BdaAbi::FaultReason reason) { auto& state = context.state; - EmitIfCondition(state, condition, [&] { RecordBdaFault(state, address, ConstantU32(state, bytes), ConstantU32(state, instruction.Flags().pc), reason); }); + EmitIfCondition(state, condition, [&] { RecordBdaFault(state, address, ConstantU32(state, bytes), BdaInstructionPc(state, instruction), reason); }); StopBdaInvocationIf(state, condition); } @@ -94,8 +97,8 @@ RuntimeAddress address(SpirvValueEmitContext& context, const IrValue& instructio const auto fits = [&](std::uint32_t offset, std::uint32_t size) { return both(state, Binary(state, spv::OpUGreaterThanEqual, boolean, size, constant(bytes)), Binary(state, spv::OpULessThanEqual, boolean, offset, Binary(state, spv::OpISub, u32, size, constant(bytes)))); }; - const auto offset = add(context.Arg(instruction, 2u), constant(context.Memory(instruction).offset + componentOffset)); - const auto soffset = context.Arg(instruction, 3u); + const auto offset = resource.offset != 0u ? add(resource.offset, constant(componentOffset)) : add(context.Arg(instruction, 2u), constant(context.Memory(instruction).offset + componentOffset)); + const auto soffset = resource.soffset != 0u ? resource.soffset : context.Arg(instruction, 3u); const auto indexShift = add(field(state, resource.word3, 21u, 2u), constant(3u)); const auto indices = Binary(state, spv::OpShiftLeftLogical, u32, constant(1u), indexShift); const auto indexMsb = Binary(state, spv::OpShiftRightLogical, u32, resource.index, indexShift); @@ -231,10 +234,12 @@ std::uint32_t composite(SpirvEmitterState& state, std::uint32_t components, cons return result; } -std::array storeValues(SpirvValueEmitContext& context, const IrValue& instruction, std::uint32_t components) { - auto& state = context.state; +std::uint32_t storeData(SpirvValueEmitContext& context, const IrValue& instruction) { + return context.Arg(instruction, instruction.ArgumentCount() - 2u); +} + +std::array storeValues(SpirvEmitterState& state, std::uint32_t data, std::uint32_t components) { std::array result{}; - const auto data = context.Arg(instruction, instruction.ArgumentCount() - 2u); for (std::uint32_t component = 0; component < components; ++component) { result[component] = data; if (components != 1u) { @@ -245,7 +250,7 @@ std::array storeValues(SpirvValueEmitContext& context, const I return result; } -std::uint32_t formatted(SpirvValueEmitContext& context, const IrValue& instruction, const RuntimeDescriptor& resource, const SpirvBufferFormatInfo& info, std::uint32_t components, bool store) { +std::uint32_t formatted(SpirvValueEmitContext& context, const IrValue& instruction, const RuntimeDescriptor& resource, const SpirvBufferFormatInfo& info, std::uint32_t components, bool store, std::uint32_t data) { auto& state = context.state; const auto& memory = context.Memory(instruction); const auto u32 = TypeU32(state); @@ -271,11 +276,11 @@ std::uint32_t formatted(SpirvValueEmitContext& context, const IrValue& instructi inBounds = both(state, inBounds, Binary(state, spv::OpLogicalOr, TypeBool(state), Unary(state, spv::OpLogicalNot, TypeBool(state), required[component]), addresses[component].inBounds)); } if (store) { - const auto data = storeValues(context, instruction, components); + const auto values = storeValues(state, data, components); EmitIfCondition(state, inBounds, [&] { auto packed = ConstantU32(state, 0u); for (std::uint32_t component = 0; component < info.componentCount; ++component) { - const auto encoded = component >= components ? ConstantU32(state, 0u) : memory.d16 ? EmitD16StoreComponent(state, info, component, data[component]) : EmitFormatStoreComponent(state, info, component, data[component]); + const auto encoded = component >= components ? ConstantU32(state, 0u) : memory.d16 ? EmitD16StoreComponent(state, info, component, values[component]) : EmitFormatStoreComponent(state, info, component, values[component]); if (info.packedBitfield) { packed = EmitOrU32(state, packed, Binary(state, spv::OpShiftLeftLogical, u32, EmitAndConstant(state, encoded, (1u << info.componentBits[component]) - 1u), ConstantU32(state, info.componentBitOffset[component]))); } else { @@ -313,13 +318,17 @@ std::uint32_t formatted(SpirvValueEmitContext& context, const IrValue& instructi return composite(state, components, outputs); } -std::uint32_t formattedDispatch(SpirvValueEmitContext& context, const IrValue& instruction, const RuntimeDescriptor& resource, std::uint32_t components, bool store) { +std::uint32_t formatSwitch(SpirvValueEmitContext& context, const IrValue& instruction, const RuntimeDescriptor& resource, std::uint32_t components, bool store, std::uint32_t data); +std::uint32_t formattedCall(SpirvValueEmitContext& context, const IrValue& instruction, const RuntimeDescriptor& resource, std::uint32_t components, bool store, std::uint32_t data); +std::uint32_t formattedGpuCall(SpirvValueEmitContext& context, const IrValue& instruction, const RuntimeDescriptor& resource, std::uint32_t components, bool store, std::uint32_t data); + +std::uint32_t formattedDispatch(SpirvValueEmitContext& context, const IrValue& instruction, const RuntimeDescriptor& resource, std::uint32_t components, bool store, std::uint32_t data) { auto& state = context.state; const auto& memory = context.Memory(instruction); if (memory.typed) { const auto info = GetFormatInfo(DecodeTBufferFormat(memory.dataFormat, memory.numberFormat)); if (info.type == SpirvFormatComponentType::Unknown) context.Fail(instruction, "typed buffer instruction has an invalid format"); - return formatted(context, instruction, resource, info, components, store); + return formatted(context, instruction, resource, info, components, store, data); } if (!store && memory.gpuDescriptor) { auto invalid = ConstantBool(state, false); @@ -333,8 +342,15 @@ std::uint32_t formattedDispatch(SpirvValueEmitContext& context, const IrValue& i if (memory.gpuDescriptor && word->HasImmediate()) { const auto format = static_cast((word->ImmediateU32() >> 12u) & 0x7fu); if (format == IrBufferFormat::Invalid) context.Fail(instruction, "formatted buffer instruction has an invalid constant format"); - return formatted(context, instruction, resource, GetFormatInfo(format), components, store); + return formatted(context, instruction, resource, GetFormatInfo(format), components, store, data); } + if (!memory.gpuDescriptor) return formattedCall(context, instruction, resource, components, store, data); + return formattedGpuCall(context, instruction, resource, components, store, data); +} + +std::uint32_t formatSwitch(SpirvValueEmitContext& context, const IrValue& instruction, const RuntimeDescriptor& resource, std::uint32_t components, bool store, std::uint32_t data) { + auto& state = context.state; + const auto& memory = context.Memory(instruction); const auto format = field(state, resource.word3, 12u, 7u); const auto merge = state.module.AllocateId(); const auto invalid = state.module.AllocateId(); @@ -352,7 +368,7 @@ std::uint32_t formattedDispatch(SpirvValueEmitContext& context, const IrValue& i state.module.AddFunction(branch); for (std::uint32_t index = 0; index < count; ++index) { EmitLabel(state, labels[index]); - const auto value = formatted(context, instruction, resource, GetFormatInfo(static_cast(index + 1u)), components, store); + const auto value = formatted(context, instruction, resource, GetFormatInfo(static_cast(index + 1u)), components, store, data); const auto exit = state.module.AllocateId(); state.module.AddFunction(spv::OpBranch, exit); EmitLabel(state, exit); @@ -360,7 +376,7 @@ std::uint32_t formattedDispatch(SpirvValueEmitContext& context, const IrValue& i phi.insert(phi.end(), {value, exit}); } EmitLabel(state, invalid); - if (memory.gpuDescriptor) RecordBdaFault(state, resource.base, ConstantU32(state, 0u), ConstantU32(state, instruction.Flags().pc), BdaAbi::FaultReason::InvalidDescriptor); + if (memory.gpuDescriptor) RecordBdaFault(state, resource.base, ConstantU32(state, 0u), BdaInstructionPc(state, instruction), BdaAbi::FaultReason::InvalidDescriptor); const auto zero = store || components == 1u ? ConstantU32(state, 0u) : ConstantU32CompositeZero(state, components); const auto invalidExit = state.module.AllocateId(); state.module.AddFunction(spv::OpBranch, invalidExit); @@ -374,6 +390,134 @@ std::uint32_t formattedDispatch(SpirvValueEmitContext& context, const IrValue& i return result; } +std::uint32_t formattedCall(SpirvValueEmitContext& context, const IrValue& instruction, const RuntimeDescriptor& resource, std::uint32_t components, bool store, std::uint32_t data) { + auto& state = context.state; + const auto& memory = context.Memory(instruction); + const auto u32 = TypeU32(state); + const auto valueType = components == 1u ? u32 : TypeU32Composite(state, components); + const auto resultType = store ? state.module.Type(spv::OpTypeVoid) : valueType; + const auto element = ResourceForDescriptor(state, DescriptorBindingKind::Buffers, memory.resource); + const std::array key{memory.resource, components, store ? 1u : 0u, memory.d16 ? 1u : 0u, memory.coherent ? 1u : 0u, static_cast(memory.kind)}; + auto function = state.formattedBufferFunctions.find(key); + if (function == state.formattedBufferFunctions.end()) { + const auto id = state.module.AllocateId(); + const auto functionType = store ? state.module.Type(spv::OpTypeFunction, resultType, u32, u32, u32, u32, u32, valueType) : state.module.Type(spv::OpTypeFunction, resultType, u32, u32, u32, u32, u32); + const auto label = state.currentLabel; + const auto byteOffset = state.memoryByteOffsets.at(element); + struct Restore { + SpirvEmitterState& state; + std::uint32_t element; + std::uint32_t label; + std::uint32_t byteOffset; + ~Restore() { + state.module.EndHelperFunction(); + state.memoryByteOffsets.at(element) = byteOffset; + state.currentLabel = label; + } + }; + state.module.BeginHelperFunction(); + const Restore restore{state, element, label, byteOffset}; + state.module.AddFunction(spv::OpFunction, resultType, id, spv::FunctionControlDontInlineMask, functionType); + std::array parameters{}; + for (std::uint32_t index = 0; index < (store ? 6u : 5u); ++index) { + parameters[index] = state.module.AllocateId(); + state.module.AddFunction(spv::OpFunctionParameter, index == 5u ? valueType : u32, parameters[index]); + } + EmitLabel(state, state.module.AllocateId()); + state.memoryByteOffsets.at(element) = parameters[4]; + const auto first = PipelineSpecialization::BufferBase + memory.resource * PipelineSpecialization::BufferWords; + const auto word1 = state.module.SpecializationConstant(u32, first, 0u); + const auto word3 = state.module.SpecializationConstant(u32, first + 1u, 0u); + const auto stride = field(state, word1, 16u, 14u); + const auto swizzle = both(state, nonzero(state, stride), nonzero(state, field(state, word1, 31u, 1u))); + const auto valid = nonzero(state, state.module.SpecializationConstant(u32, first + 2u, 0u)); + const RuntimeDescriptor shared{ConstantU32(state, 0u), parameters[3], stride, word3, swizzle, parameters[0], valid, parameters[1], parameters[2]}; + const auto value = formatSwitch(context, instruction, shared, components, store, store ? parameters[5] : 0u); + if (store) state.module.AddFunction(spv::OpReturn); + else state.module.AddFunction(spv::OpReturnValue, value); + state.module.AddFunction(spv::OpFunctionEnd); + function = state.formattedBufferFunctions.emplace(key, id).first; + } + const auto offset = Binary(state, spv::OpIAdd, u32, context.Arg(instruction, 2u), ConstantU32(state, memory.offset)); + const auto result = state.module.AllocateId(); + if (store) state.module.AddFunction(spv::OpFunctionCall, resultType, result, function->second, resource.index, offset, context.Arg(instruction, 3u), resource.records, state.memoryByteOffsets.at(element), data); + else state.module.AddFunction(spv::OpFunctionCall, resultType, result, function->second, resource.index, offset, context.Arg(instruction, 3u), resource.records, state.memoryByteOffsets.at(element)); + return store ? ConstantU32(state, 0u) : result; +} + +std::uint32_t formattedGpuCall(SpirvValueEmitContext& context, const IrValue& instruction, const RuntimeDescriptor& resource, std::uint32_t components, bool store, std::uint32_t data) { + auto& state = context.state; + const auto& memory = context.Memory(instruction); + const auto u32 = TypeU32(state); + const auto u64 = TypeScalarU64(state); + const auto boolean = TypeBool(state); + const bool stops = state.bdaStopsInvocations; + const auto valueType = components == 1u ? u32 : TypeU32Composite(state, components); + const auto resultType = stops ? (store ? u32 : state.module.Type(spv::OpTypeStruct, valueType, u32)) : (store ? state.module.Type(spv::OpTypeVoid) : valueType); + const std::array key{components, store ? 1u : 0u, memory.d16 ? 1u : 0u, memory.coherent ? 1u : 0u, static_cast(memory.kind), stops ? 1u : 0u}; + auto function = state.formattedGpuBufferFunctions.find(key); + if (function == state.formattedGpuBufferFunctions.end()) { + const auto id = state.module.AllocateId(); + const auto functionType = store ? state.module.Type(spv::OpTypeFunction, resultType, u64, u32, u32, u32, boolean, u32, boolean, u32, u32, u32, valueType) + : state.module.Type(spv::OpTypeFunction, resultType, u64, u32, u32, u32, boolean, u32, boolean, u32, u32, u32); + struct Restore { + SpirvEmitterState& state; + std::uint32_t label; + std::uint32_t stopValue; + ~Restore() { + state.module.EndHelperFunction(); + state.currentLabel = label; + state.bdaStopValue = stopValue; + state.bdaPcOverride = 0u; + } + }; + state.module.BeginHelperFunction(); + const Restore restore{state, state.currentLabel, state.bdaStopValue}; + state.module.AddName(id, std::string(store ? "store" : "load") + "_formatted_gpu_buffer_x" + std::to_string(components)); + state.module.AddFunction(spv::OpFunction, resultType, id, spv::FunctionControlDontInlineMask, functionType); + const std::array types{u64, u32, u32, u32, boolean, u32, boolean, u32, u32, u32, valueType}; + std::array parameters{}; + for (std::uint32_t index = 0; index < (store ? 11u : 10u); ++index) { + parameters[index] = state.module.AllocateId(); + state.module.AddFunction(spv::OpFunctionParameter, types[index], parameters[index]); + } + EmitLabel(state, state.module.AllocateId()); + const auto zero = components == 1u ? ConstantU32(state, 0u) : ConstantU32CompositeZero(state, components); + if (stops) state.bdaStopValue = store ? ConstantU32(state, 1u) : state.module.Constant(spv::OpConstantComposite, resultType, zero, ConstantU32(state, 1u)); + state.bdaPcOverride = parameters[9]; + const RuntimeDescriptor shared{parameters[0], parameters[1], parameters[2], parameters[3], parameters[4], parameters[5], parameters[6], parameters[7], parameters[8]}; + const auto value = formatSwitch(context, instruction, shared, components, store, store ? parameters[10] : 0u); + if (!stops && store) { + state.module.AddFunction(spv::OpReturn); + } else { + auto returned = stops && store ? ConstantU32(state, 0u) : value; + if (stops && !store) { + returned = state.module.AllocateId(); + state.module.AddFunction(spv::OpCompositeConstruct, resultType, returned, value, ConstantU32(state, 0u)); + } + state.module.AddFunction(spv::OpReturnValue, returned); + } + state.module.AddFunction(spv::OpFunctionEnd); + function = state.formattedGpuBufferFunctions.emplace(key, id).first; + } + const auto offset = Binary(state, spv::OpIAdd, u32, context.Arg(instruction, 2u), ConstantU32(state, memory.offset)); + std::vector call{spv::OpFunctionCall, resultType, state.module.AllocateId(), function->second, resource.base, resource.records, resource.stride, resource.word3, resource.swizzle, resource.index, resource.valid, offset, context.Arg(instruction, 3u), BdaInstructionPc(state, instruction)}; + if (store) call.push_back(data); + state.module.AddFunction(call); + const auto result = call[2]; + if (!stops) return store ? ConstantU32(state, 0u) : result; + if (store) { + StopBdaInvocationIf(state, Binary(state, spv::OpINotEqual, boolean, result, ConstantU32(state, 0u))); + return ConstantU32(state, 0u); + } + const auto stopped = state.module.AllocateId(); + state.module.AddFunction(spv::OpCompositeExtract, u32, stopped, result, 1u); + StopBdaInvocationIf(state, Binary(state, spv::OpINotEqual, boolean, stopped, ConstantU32(state, 0u))); + const auto value = state.module.AllocateId(); + state.module.AddFunction(spv::OpCompositeExtract, valueType, value, result, 0u); + return value; +} + } std::uint32_t EmitRuntimeBufferLoad(SpirvValueEmitContext& context, const IrValue& instruction, std::uint32_t components) { @@ -383,7 +527,7 @@ std::uint32_t EmitRuntimeBufferLoad(SpirvValueEmitContext& context, const IrValu const auto zero = components == 1u ? ConstantU32(state, 0u) : ConstantU32CompositeZero(state, components); return EmitValueOrDefaultIfCondition(state, context.Arg(instruction, instruction.ArgumentCount() - 1u), type, zero, [&] { const auto resource = descriptor(context, instruction); - if (memory.formatted) return EmitValueOrDefaultIfCondition(state, resource.valid, type, zero, [&] { return formattedDispatch(context, instruction, resource, components, false); }); + if (memory.formatted) return EmitValueOrDefaultIfCondition(state, resource.valid, type, zero, [&] { return formattedDispatch(context, instruction, resource, components, false, 0u); }); const auto readComponents = [&] { std::array values{}; for (std::uint32_t component = 0; component < components; ++component) { @@ -415,10 +559,10 @@ void EmitRuntimeBufferStore(SpirvValueEmitContext& context, const IrValue& instr EmitIfCondition(state, context.Arg(instruction, instruction.ArgumentCount() - 1u), [&] { const auto resource = descriptor(context, instruction); if (memory.formatted) { - EmitIfCondition(state, resource.valid, [&] { static_cast(formattedDispatch(context, instruction, resource, components, true)); }); + EmitIfCondition(state, resource.valid, [&] { static_cast(formattedDispatch(context, instruction, resource, components, true, storeData(context, instruction))); }); return; } - const auto values = storeValues(context, instruction, components); + const auto values = storeValues(state, storeData(context, instruction), components); for (std::uint32_t component = 0; component < components; ++component) { const auto access = address(context, instruction, resource, component * 4u, memory.dataBits / 8u); EmitIfCondition(state, access.inBounds, [&] { writeBuffer(context, instruction, access.guest, values[component], memory.dataBits); }); @@ -469,7 +613,7 @@ std::uint32_t EmitRuntimeBufferAtomic(SpirvValueEmitContext& context, const IrVa faultIf(context, instruction, unaligned, access.guest, bits / 8u, BdaAbi::FaultReason::Unaligned); return EmitValueOrDefaultIfCondition(state, Unary(state, spv::OpLogicalNot, TypeBool(state), unaligned), type, zero, [&] { const auto physical = state.module.AllocateId(); - state.module.AddFunction(spv::OpFunctionCall, TypeScalarU64(state), physical, state.bdaAtomicPointerFunction, access.guest, ConstantU32(state, bits / 8u), ConstantU32(state, instruction.Flags().pc)); + state.module.AddFunction(spv::OpFunctionCall, TypeScalarU64(state), physical, state.bdaAtomicPointerFunction, access.guest, ConstantU32(state, bits / 8u), BdaInstructionPc(state, instruction)); const auto mapped = Binary(state, spv::OpINotEqual, TypeBool(state), physical, BdaConstant(state, 0u)); return EmitValueOrDefaultIfCondition(state, mapped, type, zero, [&] { const auto pointer = state.module.AllocateId(); diff --git a/core/shader/recompiler/Translation/include/Translation/EmbeddedVertexFetch.hpp b/core/shader/recompiler/Translation/include/Translation/EmbeddedVertexFetch.hpp index 238056cd6..9729baeeb 100644 --- a/core/shader/recompiler/Translation/include/Translation/EmbeddedVertexFetch.hpp +++ b/core/shader/recompiler/Translation/include/Translation/EmbeddedVertexFetch.hpp @@ -24,6 +24,8 @@ struct EmbeddedFetchPlan { bool instanceOffsetShared = false; bool vertexOffsetConflict = false; bool instanceOffsetConflict = false; + bool vertexIndexObserved = false; + bool instanceIndexObserved = false; }; class EmbeddedVertexFetchAnalyzer { diff --git a/core/shader/recompiler/Translation/include/Translation/InstructionTranslator.hpp b/core/shader/recompiler/Translation/include/Translation/InstructionTranslator.hpp index 3085542e0..4f5674034 100644 --- a/core/shader/recompiler/Translation/include/Translation/InstructionTranslator.hpp +++ b/core/shader/recompiler/Translation/include/Translation/InstructionTranslator.hpp @@ -30,6 +30,7 @@ struct TranslateOptions { std::uint32_t userDataBaseRegister = 0; std::uint32_t userDataCount = 64; std::uint32_t scratchDwords = 0; + std::uint32_t sharedMemoryBytes = 0; std::uint64_t shaderHash = 0; bool fragmentShaderBarycentricEnabled = false; std::optional floatMode; diff --git a/core/shader/recompiler/Translation/include/Translation/TranslationContext.hpp b/core/shader/recompiler/Translation/include/Translation/TranslationContext.hpp index 34e79fefe..1d01b32f0 100644 --- a/core/shader/recompiler/Translation/include/Translation/TranslationContext.hpp +++ b/core/shader/recompiler/Translation/include/Translation/TranslationContext.hpp @@ -51,12 +51,14 @@ private: IrF32 applyF16ResultModifiers(const RdnaOperand& operand, IrF32 value); bool outputModifierApplies(std::uint32_t denormalShift) const; bool dx10Clamp() const { return !floatMode.has_value() || floatMode->dx10Clamp; } + bool fp16Overflow() const { return floatMode.has_value() && floatMode->fp16Overflow; } + IrF32 clampF16Overflow(IrF32 value, std::initializer_list sources); void rejectHalfOrDoubleOutputModifier(const RdnaOperand& operand) const; IrU32 clampF16Bits(const RdnaOperand& operand, IrU32 bits); void writeOperand(const RdnaOperand& operand, IrValue* value); IrU32 packHalf2x16(IrF32 low, IrF32 high); void write16Bits(const RdnaOperand& operand, IrU32 value); - void writeF16(const RdnaOperand& operand, IrF32 value); + void writeF16(const RdnaOperand& operand, IrF32 value, std::initializer_list sources); IrU32 readU32(const RdnaOperand& operand); IrU32 flushF32Denormal(IrU32 bits); IrF32 flushTinyProduct(IrValue* lhs, IrValue* rhs, IrValue* product, IrValue* addend = nullptr); @@ -64,6 +66,7 @@ private: IrU32 quietNan16(IrU32 bits); IrValue* nanResultF32(std::initializer_list sources, IrValue* result, IrValue* invalidProduct = nullptr); IrValue& invalidProductF32(IrValue* lhs, IrValue* rhs); + std::array quietNan64(const std::array& bits); std::array readU32Pair(const RdnaOperand& operand); IrU64 readU64(const RdnaOperand& operand); std::array readF64Bits(const RdnaOperand& operand); @@ -129,6 +132,7 @@ private: bool dsWrite2(const RdnaInstruction& inst); bool dsSrc2(const RdnaInstruction& inst, IrOpcode opcode); bool dsAtomic2(const RdnaInstruction& inst, IrOpcode opcode, bool returnsValue); + bool sharedFloatFlush(IrOpcode opcode) const; bool dsAtomic64(const RdnaInstruction& inst, IrOpcode opcode, bool returnsValue); bool dsCondxchg32(const RdnaInstruction& inst); bool dsAppendConsume(const RdnaInstruction& inst, IrOpcode opcode); @@ -178,15 +182,17 @@ private: bool vDivFixupF16(const RdnaInstruction& inst); bool float16Binary(const RdnaInstruction& inst, IrOpcode opcode, bool reverse); bool float16Ternary(const RdnaInstruction& inst, IrOpcode opcode, bool accumulator, bool mix); + IrF32 fmaF16RoundedToOdd(IrF32 lhs, IrF32 rhs, IrF32 addend); bool floatUnary(const RdnaInstruction& inst, IrOpcode opcode); bool floatBinary(const RdnaInstruction& inst, IrOpcode opcode, bool reverse); bool floatTernary(const RdnaInstruction& inst, IrOpcode opcode, bool accumulator, bool mix); bool ieeeMinMaxF32(const RdnaInstruction& inst, IrOpcode opcode); + bool minMaxF16(const RdnaInstruction& inst, IrOpcode opcode); bool vDivScaleF32(const RdnaInstruction& inst); bool vDivFmasF32(const RdnaInstruction& inst); bool vDivFixupF32(const RdnaInstruction& inst); bool vFrexpMantF32(const RdnaInstruction& inst); - bool vFmaLegacyF32(const RdnaInstruction& inst); + bool vFmaLegacyF32(const RdnaInstruction& inst, bool accumulate); bool vLdexpF32(const RdnaInstruction& inst); IrU32 readF16Bits(const RdnaOperand& operand); IrU32 normF16(IrU32 bits, bool signedValue); @@ -196,10 +202,13 @@ private: bool vCvtNormF16(const RdnaInstruction& inst, bool signedValue); bool vCvtPknormF16(const RdnaInstruction& inst, bool signedValue); bool vSatPkU8I16(const RdnaInstruction& inst); - bool vMulLegacyF32(const RdnaInstruction& inst, bool accumulate); + bool vMulLegacyF32(const RdnaInstruction& inst); bool vMullitF32(const RdnaInstruction& inst); void emitFloat16ClassCompare(const RdnaInstruction& inst, bool cmpx); bool float64Operation(const RdnaInstruction& inst, IrOpcode opcode); + bool nonIeeeMinMaxF64(const RdnaInstruction& inst, IrOpcode opcode); + bool float64Unary(const RdnaInstruction& inst, IrOpcode opcode); + bool vCvtF64F32(const RdnaInstruction& inst); void writeF64Result(const RdnaOperand& operand, IrValue& value); bool vDivScaleF64(const RdnaInstruction& inst); bool vDivFmasF64(const RdnaInstruction& inst); @@ -294,6 +303,7 @@ private: void sSwappcB64(const RdnaInstruction& inst); void sCselectB32(const RdnaInstruction& inst); void scalarSelect64(const RdnaInstruction& inst, const RdnaOperand& falseSource); + void scalarSelectMask64(const RdnaInstruction& inst); void movB32(const RdnaInstruction& inst, bool applyFloatModifiers); void sMovB64(const RdnaInstruction& inst); void sWqm(const RdnaInstruction& inst, bool wide); diff --git a/core/shader/recompiler/Translation/src/AttributeInstructions.cpp b/core/shader/recompiler/Translation/src/AttributeInstructions.cpp index aebe411e0..7d45f1d7f 100644 --- a/core/shader/recompiler/Translation/src/AttributeInstructions.cpp +++ b/core/shader/recompiler/Translation/src/AttributeInstructions.cpp @@ -53,7 +53,10 @@ ExportFlags TranslationContext::addExportInfo(const RdnaInstruction& inst) { } void TranslationContext::vInterpP1F32(const RdnaInstruction& inst) { - if (!fragmentShaderBarycentricEnabled) return; + if (!fragmentShaderBarycentricEnabled) { + writeOperand(inst.destination, &ir.Emit(IrOpcode::InterpolateHostP1, IrType::F32, {&ir.Constant(inst.source1.value), &ir.Constant(inst.source2.value), readOperand(inst.source0, IrType::F32)})); + return; + } auto& delta = ir.Emit(IrOpcode::GetInterpolationParameter, IrType::U32, {&ir.Constant(inst.source1.value), &ir.Constant(inst.source2.value), &ir.Constant(0u)}); auto& origin = ir.Emit(IrOpcode::GetInterpolationParameter, IrType::U32, {&ir.Constant(inst.source1.value), &ir.Constant(inst.source2.value), &ir.Constant(2u)}); auto& product = ir.Emit(IrOpcode::FPMul32, IrType::F32, {&ir.BitCastF32(delta), readOperand(inst.source0, IrType::F32)}); @@ -81,6 +84,7 @@ void TranslationContext::vInterpP2F32(const RdnaInstruction& inst) { program.Metadata().pixelPerspectiveInputs |= bit; } } + ir.Emit(IrOpcode::InterpolateHostP2, IrType::Void, {&ir.Constant(inst.source1.value), &ir.Constant(inst.source2.value), readOperand(inst.source0, IrType::F32), readOperand(inst.destination, IrType::F32), &ir.GetExec()}); IrValue& value = ir.Emit(IrOpcode::GetAttribute, IrType::U32, {&ir.Constant(inst.source1.value), &ir.Constant(inst.source2.value)}); writeOperand(inst.destination, &value); } @@ -124,7 +128,9 @@ void TranslationContext::vInterpP1F16(const RdnaInstruction& inst) { void TranslationContext::vInterpP2F16(const RdnaInstruction& inst) { const auto mode = interpolationModeF16(inst); const IrF32 delta = interpolationParameterF16(inst, 1u); - writeF16(inst.destination, IrF32(ir.Emit(IrOpcode::FPInterpolateF16, IrType::F32, {&delta.Value(), readOperand(inst.source0, IrType::F32), readOperand(inst.source3, IrType::F32), &ir.Constant(mode)}))); + IrValue* coordinate = readOperand(inst.source0, IrType::F32); + IrValue* partial = readOperand(inst.source3, IrType::F32); + writeF16(inst.destination, IrF32(ir.Emit(IrOpcode::FPInterpolateF16, IrType::F32, {&delta.Value(), coordinate, partial, &ir.Constant(mode)})), {&delta.Value(), coordinate, partial}); } void TranslationContext::eXP(const RdnaInstruction& inst) { diff --git a/core/shader/recompiler/Translation/src/ControlFlowInstructions.cpp b/core/shader/recompiler/Translation/src/ControlFlowInstructions.cpp index 798eaf34c..838346dc4 100644 --- a/core/shader/recompiler/Translation/src/ControlFlowInstructions.cpp +++ b/core/shader/recompiler/Translation/src/ControlFlowInstructions.cpp @@ -244,6 +244,21 @@ void TranslationContext::scalarSelect64(const RdnaInstruction& inst, const RdnaO writeU32Pair(inst.destination, {low, high}); } +void TranslationContext::scalarSelectMask64(const RdnaInstruction& inst) { + if (inst.destination.kind != RdnaOperandKind::ScalarRegister) { + scalarSelect64(inst, sourceAt(inst, 1u)); + return; + } + const RdnaOperand& trueSource = sourceAt(inst, 0u); + const RdnaOperand& falseSource = sourceAt(inst, 1u); + IrValue& valid = ir.LogicalAnd(readMaskValid(trueSource).Value(), readMaskValid(falseSource).Value()); + IrValue& bit = ir.Emit(IrOpcode::SelectU1, IrType::U1, {&ir.GetScc(), &readMask(trueSource).Value(), &readMask(falseSource).Value()}); + scalarSelect64(inst, falseSource); + const auto dst = static_cast(inst.destination.reg); + ir.SetThreadBitScalarReg(dst, bit); + ir.SetScalarMaskTag(dst, valid); +} + void TranslationContext::movB32(const RdnaInstruction& inst, bool applyFloatModifiers) { if (applyFloatModifiers) { IrF32 value(*readOperand(sourceAt(inst, 0u), IrType::F32)); diff --git a/core/shader/recompiler/Translation/src/ConvertInstructions.cpp b/core/shader/recompiler/Translation/src/ConvertInstructions.cpp index b0a367c0d..91241ac31 100644 --- a/core/shader/recompiler/Translation/src/ConvertInstructions.cpp +++ b/core/shader/recompiler/Translation/src/ConvertInstructions.cpp @@ -79,7 +79,7 @@ void TranslationContext::vCvtI32F32(const RdnaInstruction& inst) { void TranslationContext::vCvtF16F32(const RdnaInstruction& inst) { const IrF32 value(*readOperand(sourceAt(inst, 0u), IrType::F32)); - writeF16(inst.destination, value); + writeF16(inst.destination, value, {&value.Value()}); } void TranslationContext::vCvtF32F16(const RdnaInstruction& inst) { @@ -91,7 +91,7 @@ void TranslationContext::vCvtF1616(const RdnaInstruction& inst, bool signedValue const IrU32 source = readU16AsU32(sourceAt(inst, 0u), signedValue); const IrOpcode opcode = signedValue ? IrOpcode::ConvertF32S32 : IrOpcode::ConvertF32U32; const IrF32 value(ir.Emit(opcode, IrType::F32, {&source.Value()})); - writeF16(inst.destination, value); + writeF16(inst.destination, value, {}); } void TranslationContext::vCvt16F16(const RdnaInstruction& inst, bool signedValue) { @@ -188,8 +188,13 @@ void TranslationContext::vCvtPkU8F32(const RdnaInstruction& inst) { } void TranslationContext::vPackB32F16(const RdnaInstruction& inst) { - const IrU32 low = readF16Bits(sourceAt(inst, 0u)); - const IrU32 high(ir.ShiftLeftLogical(readF16Bits(sourceAt(inst, 1u)).Value(), ir.Constant(16u))); + const auto half = [&](std::uint32_t index) { + const IrU32 bits = readF16Bits(sourceAt(inst, index)); + const IrU1 nan(ir.UGreaterThan(ir.BitwiseAnd(bits.Value(), ir.Constant(0x7fffu)), ir.Constant(0x7c00u))); + return IrU32(ir.Select(nan.Value(), quietNan16(bits).Value(), bits.Value())); + }; + const IrU32 low = half(0u); + const IrU32 high(ir.ShiftLeftLogical(half(1u).Value(), ir.Constant(16u))); const IrU32 result(ir.BitwiseOr(low.Value(), high.Value())); writeOperand(inst.destination, &result.Value()); } diff --git a/core/shader/recompiler/Translation/src/EmbeddedVertexFetch.cpp b/core/shader/recompiler/Translation/src/EmbeddedVertexFetch.cpp index 4fda5df13..a5ba98424 100644 --- a/core/shader/recompiler/Translation/src/EmbeddedVertexFetch.cpp +++ b/core/shader/recompiler/Translation/src/EmbeddedVertexFetch.cpp @@ -149,6 +149,15 @@ bool isAttributePropagationAlu(RdnaOpcode opcode) { } } +std::uint32_t scalarBfeU32(std::uint32_t source, std::uint32_t field) { + const std::uint32_t count = std::min((field >> 16u) & 0x7fu, 32u); + if (count == 0u) { + return 0u; + } + const std::uint32_t shifted = source >> (field & 31u); + return count == 32u ? shifted : shifted & ((1u << count) - 1u); +} + std::int32_t bufferAttributeFromOffset(std::uint32_t rawOffset, std::uint32_t dword) { return static_cast((rawOffset + dword * 4u) / 16u); } @@ -208,6 +217,159 @@ bool touchesSgpr(const RdnaInstruction& inst, std::uint32_t sgpr) { return false; } +bool isVectorAlu(const RdnaInstruction& inst) { + switch (inst.family) { + case RdnaInstructionFamily::VOP1: + case RdnaInstructionFamily::VOP2: + case RdnaInstructionFamily::VOP3: + case RdnaInstructionFamily::VOP3P: + case RdnaInstructionFamily::VOPC: + return true; + default: + return false; + } +} + +bool isSequentialProgramControl(RdnaOpcode opcode) { + switch (opcode) { + case RdnaOpcode::SNop: + case RdnaOpcode::SWaitcnt: + case RdnaOpcode::SWaitcntDepctr: + case RdnaOpcode::SSetprio: + case RdnaOpcode::SSendmsg: + case RdnaOpcode::SBarrier: + case RdnaOpcode::SSleep: + case RdnaOpcode::SClause: + case RdnaOpcode::SInstPrefetch: + case RdnaOpcode::SRoundMode: + case RdnaOpcode::SDenormMode: + case RdnaOpcode::SIcacheInv: + case RdnaOpcode::STtracedata: + case RdnaOpcode::SIncperflevel: + case RdnaOpcode::SDecperflevel: + return true; + default: + return false; + } +} + +bool leavesStraightLine(const RdnaInstruction& inst) { + if (inst.op == RdnaOpcode::Unknown || inst.op == RdnaOpcode::SSetpcB64 || inst.op == RdnaOpcode::SRfeB64) return true; + return inst.family == RdnaInstructionFamily::SOPP && !isSequentialProgramControl(inst.op); +} + +bool readsSgprRange(const RdnaInstruction& inst, std::uint32_t first, std::uint32_t count) { + const std::array sources{&inst.source0, &inst.source1, &inst.source2, &inst.source3}; + for (std::uint32_t i = 0u; i < sources.size(); i++) { + if (!isScalarOperand(*sources[i])) continue; + const auto slot = scalarSlot(*sources[i]); + if (slot < first + count && first < slot + scalarOperandWidth(inst, i)) return true; + } + return false; +} + +bool readsVccImplicitly(const RdnaInstruction& inst) { + switch (inst.op) { + case RdnaOpcode::VCndmaskB32: + case RdnaOpcode::VAddcU32: + case RdnaOpcode::VSubCoCiU32: + case RdnaOpcode::VSubrevCoCiU32: + return !isScalarOperand(inst.source2); + case RdnaOpcode::VDivFmasF32: + case RdnaOpcode::VDivFmasF64: + return true; + default: + return false; + } +} + +bool mayReachVgpr(const RdnaInstruction& inst, std::uint32_t reg) { + switch (inst.family) { + case RdnaInstructionFamily::SOP1: + case RdnaInstructionFamily::SOP2: + case RdnaInstructionFamily::SOPK: + case RdnaInstructionFamily::SOPC: + case RdnaInstructionFamily::SMEM: + case RdnaInstructionFamily::SOPP: + return leavesStraightLine(inst); + default: + break; + } + if (!isVectorAlu(inst)) return true; + switch (inst.op) { + case RdnaOpcode::VMovrelsB32: + case RdnaOpcode::VMovreldB32: + case RdnaOpcode::VMovrelsdB32: + case RdnaOpcode::VMovrelsd2B32: + case RdnaOpcode::VSwaprelB32: + return true; + default: + break; + } + const std::array sources{&inst.source0, &inst.source1, &inst.source2, &inst.source3}; + for (std::uint32_t i = 0u; i < sources.size(); i++) { + const auto width = inst.op == RdnaOpcode::VMqsadU32U8 && i == 2u ? 4u : 2u; + if (isVectorOperand(*sources[i]) && reg >= sources[i]->reg && reg < sources[i]->reg + width) return true; + } + return isVectorOperand(inst.destination) && reg >= inst.destination.reg && reg < inst.destination.reg + std::max(embeddedFetchDstSize(inst), 2u); +} + +bool writesExec(const RdnaInstruction& inst) { + const auto exec = [](const RdnaOperand& operand) { return operand.kind == RdnaOperandKind::ExecLo || operand.kind == RdnaOperandKind::ExecHi; }; + if (exec(inst.destination) || exec(inst.destination2)) return true; + switch (inst.op) { + case RdnaOpcode::SAndSaveexecB32: + case RdnaOpcode::SOrSaveexecB32: + case RdnaOpcode::SXorSaveexecB32: + case RdnaOpcode::SAndn2SaveexecB32: + case RdnaOpcode::SOrn2SaveexecB32: + case RdnaOpcode::SNandSaveexecB32: + case RdnaOpcode::SNorSaveexecB32: + case RdnaOpcode::SXnorSaveexecB32: + case RdnaOpcode::SAndn1SaveexecB32: + case RdnaOpcode::SOrn1SaveexecB32: + case RdnaOpcode::SAndn1WrexecB32: + case RdnaOpcode::SAndn2WrexecB32: + case RdnaOpcode::SAndSaveexecB64: + case RdnaOpcode::SOrSaveexecB64: + case RdnaOpcode::SXorSaveexecB64: + case RdnaOpcode::SAndn2SaveexecB64: + case RdnaOpcode::SOrn2SaveexecB64: + case RdnaOpcode::SNandSaveexecB64: + case RdnaOpcode::SNorSaveexecB64: + case RdnaOpcode::SXnorSaveexecB64: + case RdnaOpcode::SAndn1SaveexecB64: + case RdnaOpcode::SOrn1SaveexecB64: + case RdnaOpcode::SAndn1WrexecB64: + case RdnaOpcode::SAndn2WrexecB64: + return true; + default: + return false; + } +} + +bool writesSgprMask(const RdnaInstruction& inst, std::uint32_t sgpr, std::uint32_t waveSize) { + const auto covers = [&](const RdnaOperand& operand, std::uint32_t width) { return isScalarOperand(operand) && sgpr >= scalarSlot(operand) && sgpr < scalarSlot(operand) + width; }; + return covers(inst.destination2, 2u) || (isVectorAlu(inst) && covers(inst.destination, waveSize == 64u ? 2u : 1u)); +} + +bool carryObserved(const RdnaProgram& program, std::size_t addIndex, std::uint32_t waveSize) { + const auto& add = program.instructions[addIndex]; + if (add.destination2.kind == RdnaOperandKind::None || add.destination2.kind == RdnaOperandKind::Null) return false; + if (!isScalarOperand(add.destination2)) return true; + const auto first = scalarSlot(add.destination2); + const std::uint32_t count = waveSize == 64u ? 2u : 1u; + for (std::size_t index = addIndex + 1u; index < program.instructions.size(); index++) { + const auto& inst = program.instructions[index]; + if (inst.op == RdnaOpcode::SEndpgm) return false; + if (leavesStraightLine(inst) || readsSgprRange(inst, first, count) || (first == kVccLoSlot && readsVccImplicitly(inst))) return true; + const bool laneMask = isVectorAlu(inst) && inst.op != RdnaOpcode::VReadlaneB32 && inst.op != RdnaOpcode::VReadfirstlaneB32; + const auto rewrites = [&](const RdnaOperand& operand) { return isScalarOperand(operand) && scalarSlot(operand) == first && (count == 1u || laneMask); }; + if (rewrites(inst.destination) || rewrites(inst.destination2)) return false; + } + return false; +} + } EmbeddedFetchPlan EmbeddedVertexFetchAnalyzer::Analyze(const RdnaProgram& program, std::uint32_t attributeTableRegister, std::uint32_t bufferTableRegister, std::uint32_t userDataBaseRegister, std::uint32_t userDataCount, std::uint32_t waveSize) const { @@ -393,7 +555,7 @@ EmbeddedFetchPlan EmbeddedVertexFetchAnalyzer::Analyze(const RdnaProgram& progra dst.constant = src0 << (src1 & 31u); break; case RdnaOpcode::SBfeU32: - dst.constant = src0 >> (src1 & 31u); + dst.constant = scalarBfeU32(src0, src1); break; default: dst.constant = src0 + src1; @@ -447,6 +609,21 @@ EmbeddedFetchPlan EmbeddedVertexFetchAnalyzer::Analyze(const RdnaProgram& progra }; plan.vertexOffsetShared = shared(plan.vertexOffsetSgpr, vertexAddPcs); plan.instanceOffsetShared = shared(plan.instanceOffsetSgpr, instanceAddPcs); + const bool relativeScalarAccess = std::any_of(program.instructions.begin(), program.instructions.end(), [](const RdnaInstruction& inst) { + return inst.op == RdnaOpcode::SMovrelsB32 || inst.op == RdnaOpcode::SMovrelsB64 || inst.op == RdnaOpcode::SMovrelsd2B32 || inst.op == RdnaOpcode::SMovreldB32 || inst.op == RdnaOpcode::SMovreldB64; + }); + const auto observed = [&](std::int32_t sgpr, const std::vector& addPcs, std::uint32_t indexVgpr) { + if (sgpr < 0) return false; + if (addPcs.size() != 1u || relativeScalarAccess) return true; + const auto add = std::find_if(program.instructions.begin(), program.instructions.end(), [&](const RdnaInstruction& inst) { return inst.programCounter == addPcs.front(); }); + if (add == program.instructions.end() || add->destination.reg != indexVgpr || add->destination.clamp || add->clampResult) return true; + if (std::any_of(program.instructions.begin(), program.instructions.end(), [&](const RdnaInstruction& inst) { return inst.programCounter != add->programCounter && writesSgprMask(inst, static_cast(sgpr), waveSize); })) return true; + const auto reg = add->destination.reg; + if (std::any_of(program.instructions.begin(), add, [&](const RdnaInstruction& inst) { return writesExec(inst) || mayReachVgpr(inst, reg); })) return true; + return carryObserved(program, static_cast(add - program.instructions.begin()), waveSize); + }; + plan.vertexIndexObserved = observed(plan.vertexOffsetSgpr, vertexAddPcs, kVertexIndexVgpr); + plan.instanceIndexObserved = observed(plan.instanceOffsetSgpr, instanceAddPcs, kInstanceIndexVgpr); return plan; } diff --git a/core/shader/recompiler/Translation/src/Float64Instructions.cpp b/core/shader/recompiler/Translation/src/Float64Instructions.cpp index 47e19cf03..00a0bac5b 100644 --- a/core/shader/recompiler/Translation/src/Float64Instructions.cpp +++ b/core/shader/recompiler/Translation/src/Float64Instructions.cpp @@ -33,6 +33,48 @@ bool TranslationContext::float64Operation(const RdnaInstruction& inst, IrOpcode return true; } +bool TranslationContext::nonIeeeMinMaxF64(const RdnaInstruction& inst, IrOpcode opcode) { + const std::array, 2> bits{readF64Bits(sourceAt(inst, 0u)), readF64Bits(sourceAt(inst, 1u))}; + const auto isNan = [&](const std::array& value) { + IrValue& magnitude = ir.BitwiseAnd(value[1].Value(), ir.Constant(0x7fffffffu)); + return &ir.LogicalOr(ir.UGreaterThan(magnitude, ir.Constant(0x7ff00000u)), ir.LogicalAnd(ir.IEqual(magnitude, ir.Constant(0x7ff00000u)), ir.INotEqual(value[0].Value(), ir.Constant(0u)))); + }; + IrValue& plain = ir.Emit(opcode, IrType::U64, {&ir.ConstructU64(bits[0][0].Value(), bits[0][1].Value()), &ir.ConstructU64(bits[1][0].Value(), bits[1][1].Value())}); + IrValue* lhsNan = isNan(bits[0]); + IrValue* rhsNan = isNan(bits[1]); + std::array words{}; + for (std::uint32_t index = 0u; index < 2u; ++index) { + IrValue& lhsChecked = ir.Select(*lhsNan, bits[1][index].Value(), ir.CompositeExtract(plain, index)); + words[index] = &ir.Select(*rhsNan, bits[0][index].Value(), lhsChecked); + } + writeF64Result(inst.destination, ir.ConstructU64(*words[0], *words[1])); + return true; +} + +bool TranslationContext::float64Unary(const RdnaInstruction& inst, IrOpcode opcode) { + const auto bits = readF64Bits(sourceAt(inst, 0u)); + IrValue& argument = ir.ConstructU64(bits[0].Value(), bits[1].Value()); + IrValue& result = opcode == IrOpcode::FPLdexp64 ? ir.Emit(opcode, IrType::U64, {&argument, &readU32(sourceAt(inst, 1u)).Value()}) : ir.Emit(opcode, IrType::U64, {&argument}); + const IrU32 magnitude(ir.BitwiseAnd(bits[1].Value(), ir.Constant(0x7fffffffu))); + const IrU1 nan(ir.LogicalOr(ir.UGreaterThan(magnitude.Value(), ir.Constant(0x7ff00000u)), + ir.LogicalAnd(ir.IEqual(magnitude.Value(), ir.Constant(0x7ff00000u)), ir.INotEqual(bits[0].Value(), ir.Constant(0u))))); + const auto quiet = quietNan64(bits); + writeF64Result(inst.destination, ir.ConstructU64(ir.Select(nan.Value(), quiet[0].Value(), ir.CompositeExtract(result, 0u)), ir.Select(nan.Value(), quiet[1].Value(), ir.CompositeExtract(result, 1u)))); + return true; +} + +bool TranslationContext::vCvtF64F32(const RdnaInstruction& inst) { + IrValue* value = readOperand(sourceAt(inst, 0u), IrType::F32); + IrValue& result = ir.Emit(IrOpcode::ConvertF64F32, IrType::U64, {value}); + const IrU32 bits(ir.BitCastU32(*value)); + const IrU1 nan(ir.UGreaterThan(ir.BitwiseAnd(bits.Value(), ir.Constant(0x7fffffffu)), ir.Constant(0x7f800000u))); + const IrU32 mantissa(ir.BitwiseAnd(bits.Value(), ir.Constant(0x007fffffu))); + const IrU32 high(ir.BitwiseOr(ir.BitwiseOr(ir.BitwiseAnd(bits.Value(), ir.Constant(0x80000000u)), ir.Constant(0x7ff00000u)), ir.ShiftRightLogical(mantissa.Value(), ir.Constant(3u)))); + const auto quiet = quietNan64({IrU32(ir.ShiftLeftLogical(mantissa.Value(), ir.Constant(29u))), high}); + writeF64Result(inst.destination, ir.ConstructU64(ir.Select(nan.Value(), quiet[0].Value(), ir.CompositeExtract(result, 0u)), ir.Select(nan.Value(), quiet[1].Value(), ir.CompositeExtract(result, 1u)))); + return true; +} + void TranslationContext::writeF64Result(const RdnaOperand& operand, IrValue& value) { rejectHalfOrDoubleOutputModifier(operand); RdnaOperand destination = operand; @@ -129,9 +171,6 @@ bool TranslationContext::vDivFixupF64(const RdnaInstruction& inst) { const std::array zero{IrU32(ir.Constant(0u)), sign}; const std::array infinity{IrU32(ir.Constant(0u)), IrU32(ir.BitwiseOr(sign.Value(), ir.Constant(0x7ff00000u)))}; const std::array defaultNan{IrU32(ir.Constant(0u)), IrU32(ir.Constant(0xfff80000u))}; - const auto quiet = [&](const std::array& word) { - return std::array{word[0], IrU32(ir.BitwiseOr(word[1].Value(), ir.Constant(0x00080000u)))}; - }; const IrU32 difference(ir.ISub(exponent(numerator[1]).Value(), exponent(denominator[1]).Value())); const IrU1 underflow(ir.Emit(IrOpcode::SLessThan32, IrType::U1, {&difference.Value(), &ir.Constant(static_cast(-1075))})); std::array result{quotient[0], IrU32(ir.BitwiseOr(sign.Value(), magnitude(quotient[1]).Value()))}; @@ -143,8 +182,8 @@ bool TranslationContext::vDivFixupF64(const RdnaInstruction& inst) { choose(IrU1(ir.LogicalOr(isInf(denominator).Value(), isZero(numerator).Value())), zero); choose(IrU1(ir.LogicalOr(isZero(denominator).Value(), isInf(numerator).Value())), infinity); choose(IrU1(ir.LogicalOr(ir.LogicalAnd(isZero(denominator).Value(), isZero(numerator).Value()), ir.LogicalAnd(isInf(denominator).Value(), isInf(numerator).Value()))), defaultNan); - choose(isNan(denominator), quiet(denominator)); - choose(isNan(numerator), quiet(numerator)); + choose(isNan(denominator), quietNan64(denominator)); + choose(isNan(numerator), quietNan64(numerator)); writeF64Result(inst.destination, ir.ConstructU64(result[0].Value(), result[1].Value())); return true; } diff --git a/core/shader/recompiler/Translation/src/FloatInstructions.cpp b/core/shader/recompiler/Translation/src/FloatInstructions.cpp index 502e601a3..b85979973 100644 --- a/core/shader/recompiler/Translation/src/FloatInstructions.cpp +++ b/core/shader/recompiler/Translation/src/FloatInstructions.cpp @@ -17,18 +17,22 @@ bool TranslationContext::packedFloat16(const RdnaInstruction& inst, IrOpcode opc const auto translateLane = [&](bool high) -> IrF32 { const IrF32 lhs = readF16LaneAsF32(sourceAt(inst, 0u), high, true); const IrF32 rhs = readF16LaneAsF32(sourceAt(inst, 1u), high, true); + const auto ternary = [&](const IrF32& third) -> IrF32 { + if (opcode == IrOpcode::FPFma32) return fmaF16RoundedToOdd(lhs, rhs, third); + return IrF32(ir.Emit(opcode, IrType::F32, {&lhs.Value(), &rhs.Value(), &third.Value()})); + }; if (accumulator) { const IrF32 acc = readF16LaneAsF32(accumulatorOperand(inst), high, true); - return applyF32ResultModifiers(inst.destination, IrF32(ir.Emit(opcode, IrType::F32, {&lhs.Value(), &rhs.Value(), &acc.Value()}))); + return applyF32ResultModifiers(inst.destination, clampF16Overflow(ternary(acc), {&lhs.Value(), &rhs.Value(), &acc.Value()})); } if (inst.sourceCount == 3u) { const IrF32 third = readF16LaneAsF32(sourceAt(inst, 2u), high, true); - return applyF32ResultModifiers(inst.destination, IrF32(ir.Emit(opcode, IrType::F32, {&lhs.Value(), &rhs.Value(), &third.Value()}))); + return applyF32ResultModifiers(inst.destination, clampF16Overflow(ternary(third), {&lhs.Value(), &rhs.Value(), &third.Value()})); } - return applyF32ResultModifiers(inst.destination, IrF32(ir.Emit(opcode, IrType::F32, {&lhs.Value(), &rhs.Value()}))); + return applyF32ResultModifiers(inst.destination, clampF16Overflow(IrF32(ir.Emit(opcode, IrType::F32, {&lhs.Value(), &rhs.Value()})), {&lhs.Value(), &rhs.Value()})); }; IrU32 result = packHalf2x16(translateLane(false), translateLane(true)); - if (!inst.destination.clamp || !dx10Clamp()) { + if (!inst.destination.clamp || !dx10Clamp() || quietSnan) { const auto laneBits = [&](const RdnaOperand& operand, bool high) { const IrU32 source = readF16SourceBits(operand); const bool selectHigh = high ? operand.opSelHi : operand.opSel; @@ -86,7 +90,7 @@ bool TranslationContext::float16Unary(const RdnaInstruction& inst, IrOpcode opco const RdnaOperand& operand = sourceAt(inst, 0u); const IrF32 argument = readF16AsF32(operand); const auto [result, invalid] = unaryFloatSpecials(opcode, argument, IrF32(ir.Emit(opcode, IrType::F32, {&argument.Value()}))); - IrU32 bits = packHalf2x16(applyF16ResultModifiers(inst.destination, result), IrF32(ir.ConstantF32(0.0f))); + IrU32 bits = packHalf2x16(applyF16ResultModifiers(inst.destination, clampF16Overflow(result, {&argument.Value()})), IrF32(ir.ConstantF32(0.0f))); if (opcode == IrOpcode::FPFract32) { bits = IrU32(ir.Select(ir.IEqual(bits.Value(), ir.Constant(0x3c00u)), ir.Constant(0x3bffu), bits.Value())); } @@ -136,6 +140,7 @@ std::pair TranslationContext::unaryFloatSpecials(IrOpcode opcode, I const IrU1 overflow(ir.LogicalAnd(ir.IEqual(sign.Value(), ir.Constant(0u)), ir.UGreaterThan(magnitude.Value(), ir.Constant(0x42ffffffu)))); const IrU1 underflow(ir.LogicalAnd(negative.Value(), ir.UGreaterThan(magnitude.Value(), ir.Constant(0x42fc0000u)))); value = pick(overflow, ir.Constant(0x7f800000u), value); + value = IrU32(ir.Emit(IrOpcode::UMax32, IrType::U32, {&value.Value(), &ir.Constant(0x00800000u)})); value = pick(underflow, ir.Constant(0u), value); value = pick(zero, ir.Constant(0x3f800000u), value); break; @@ -196,6 +201,11 @@ bool TranslationContext::vDivFixupF16(const RdnaInstruction& inst) { result = IrU32(ir.Select(ir.LogicalOr(bothZero.Value(), bothInf.Value()), ir.Constant(0xfe00u), result.Value())); result = IrU32(ir.Select(isNan(1u).Value(), quietNan16(bits[1]).Value(), result.Value())); result = IrU32(ir.Select(isNan(2u).Value(), quietNan16(bits[2]).Value(), result.Value())); + if (fp16Overflow()) { + const IrU1 keep(ir.LogicalOr(isZero(1u).Value(), isInf(2u).Value())); + const IrU1 overflow(ir.LogicalAnd(ir.IEqual(ir.BitwiseAnd(result.Value(), ir.Constant(0x7fffu)), ir.Constant(0x7c00u)), ir.LogicalNot(keep.Value()))); + result = IrU32(ir.Select(overflow.Value(), ir.BitwiseOr(ir.BitwiseAnd(result.Value(), ir.Constant(0x8000u)), ir.Constant(0x7bffu)), result.Value())); + } write16Bits(inst.destination, clampF16Bits(inst.destination, result)); return true; } @@ -203,20 +213,46 @@ bool TranslationContext::vDivFixupF16(const RdnaInstruction& inst) { bool TranslationContext::float16Binary(const RdnaInstruction& inst, IrOpcode opcode, bool reverse) { const IrF32 lhs = readF16AsF32(sourceAt(inst, reverse ? 1u : 0u)); const IrF32 rhs = readF16AsF32(sourceAt(inst, reverse ? 0u : 1u)); - writeF16(inst.destination, IrF32(ir.Emit(opcode, IrType::F32, {&lhs.Value(), &rhs.Value()}))); + writeF16(inst.destination, IrF32(ir.Emit(opcode, IrType::F32, {&lhs.Value(), &rhs.Value()})), {&lhs.Value(), &rhs.Value()}); return true; } bool TranslationContext::float16Ternary(const RdnaInstruction& inst, IrOpcode opcode, bool accumulator, bool mix) { + if (mix && floatMode.has_value() && (floatMode->floatMode & 0xccu) != 0xc0u) { + throw std::runtime_error("v_fma_mixlo/mixhi_f16 at pc " + std::to_string(inst.programCounter) + " in FLOAT_MODE " + std::to_string(floatMode->floatMode) + " (f16 denormals not kept or rounding not to nearest even) is not implemented"); + } std::array args{}; for (std::uint32_t index = 0u; index < args.size(); ++index) { const RdnaOperand& operand = accumulator && index == 2u ? accumulatorOperand(inst) : sourceAt(inst, index); args[index] = mix ? &readMixF32(operand).Value() : &readF16AsF32(operand).Value(); } - writeF16(inst.destination, IrF32(ir.Emit(opcode, IrType::F32, {args[0], args[1], args[2]}))); + if (opcode == IrOpcode::FPFma32 && !mix) { + writeF16(inst.destination, fmaF16RoundedToOdd(IrF32(*args[0]), IrF32(*args[1]), IrF32(*args[2])), {args[0], args[1], args[2]}); + return true; + } + if (mix) { + writeF16(inst.destination, IrF32(ir.Emit(IrOpcode::FPInterpolateF16, IrType::F32, {args[0], args[1], args[2], &ir.Constant(0u)})), {args[0], args[1], args[2]}); + return true; + } + writeF16(inst.destination, IrF32(ir.Emit(opcode, IrType::F32, {args[0], args[1], args[2]})), {args[0], args[1], args[2]}); return true; } +IrF32 TranslationContext::fmaF16RoundedToOdd(IrF32 lhs, IrF32 rhs, IrF32 addend) { + IrValue& product = ir.Emit(IrOpcode::FPMul32, IrType::F32, {&lhs.Value(), &rhs.Value()}); + IrValue& sum = ir.Emit(IrOpcode::FPAdd32, IrType::F32, {&product, &addend.Value()}); + IrValue& addendPart = ir.Emit(IrOpcode::FPSub32, IrType::F32, {&sum, &product}); + IrValue& productPart = ir.Emit(IrOpcode::FPSub32, IrType::F32, {&sum, &addendPart}); + IrValue& productError = ir.Emit(IrOpcode::FPSub32, IrType::F32, {&product, &productPart}); + IrValue& addendError = ir.Emit(IrOpcode::FPSub32, IrType::F32, {&addend.Value(), &addendPart}); + IrValue& error = ir.Emit(IrOpcode::FPAdd32, IrType::F32, {&productError, &addendError}); + IrValue& inexact = ir.Emit(IrOpcode::FPOrdNotEqual32, IrType::U1, {&error, &ir.ConstantF32(0.0f)}); + IrValue& sumBits = ir.BitCastU32(sum); + IrValue& sameSign = ir.IEqual(ir.BitwiseAnd(ir.BitwiseXor(sumBits, ir.BitCastU32(error)), ir.Constant(0x80000000u)), ir.Constant(0u)); + IrValue& towardZero = ir.Select(sameSign, sumBits, ir.ISub(sumBits, ir.Constant(1u))); + return IrF32(ir.BitCastF32(ir.Select(inexact, ir.BitwiseOr(towardZero, ir.Constant(1u)), sumBits))); +} + IrU32 TranslationContext::readF16Bits(const RdnaOperand& operand) { const IrU32 source = readF16SourceBits(operand); IrU32 bits(ir.BitwiseAnd((operand.opSel ? ir.ShiftRightLogical(source.Value(), ir.Constant(16u)) : source.Value()), ir.Constant(0xffffu))); @@ -279,10 +315,12 @@ bool TranslationContext::vLdexpF16(const RdnaInstruction& inst) { const IrU32 bits = readF16Bits(sourceAt(inst, 0u)); const IrF16 half(ir.Emit(IrOpcode::BitCastF16U16, IrType::F16, {&ir.Emit(IrOpcode::ConvertU16U32, IrType::U16, {&bits.Value()})})); const IrF32 value(ir.Emit(IrOpcode::ConvertF32F16, IrType::F32, {&half.Value()})); - const IrU32 exponent(ir.Emit(IrOpcode::BitFieldSExtract, IrType::U32, {&readU32(sourceAt(inst, 1u)).Value(), &ir.Constant(0u), &ir.Constant(16u)})); + const RdnaOperand& exponentSource = sourceAt(inst, 1u); + const IrU32 exponentBits = inst.family == RdnaInstructionFamily::VOP3 ? readF16SourceBits(exponentSource) : readU32(exponentSource); + const IrU32 exponent(ir.Emit(IrOpcode::BitFieldSExtract, IrType::U32, {&exponentBits.Value(), &ir.Constant(0u), &ir.Constant(16u)})); const IrU32 clamped(ir.Emit(IrOpcode::SMax32, IrType::U32, {&ir.Emit(IrOpcode::SMin32, IrType::U32, {&exponent.Value(), &ir.Constant(64u)}), &ir.Constant(static_cast(-64))})); IrValue& power = ir.BitCastF32(ir.ShiftLeftLogical(ir.IAdd(clamped.Value(), ir.Constant(127u)), ir.Constant(23u))); - const IrF16 scaled(ir.Emit(IrOpcode::ConvertF16F32, IrType::F16, {&ir.Emit(IrOpcode::FPMul32, IrType::F32, {&value.Value(), &power})})); + const IrF16 scaled(ir.Emit(IrOpcode::ConvertF16F32, IrType::F16, {&clampF16Overflow(IrF32(ir.Emit(IrOpcode::FPMul32, IrType::F32, {&value.Value(), &power})), {&value.Value()}).Value()})); const IrU32 result(ir.Emit(IrOpcode::ConvertU32U16, IrType::U32, {&ir.Emit(IrOpcode::BitCastU16F16, IrType::U16, {&scaled.Value()})})); const IrU1 nan(ir.UGreaterThan(ir.BitwiseAnd(bits.Value(), ir.Constant(0x7fffu)), ir.Constant(0x7c00u))); write16Bits(inst.destination, clampF16Bits(inst.destination, IrU32(ir.Select(nan.Value(), quietNan16(bits).Value(), result.Value())))); @@ -333,24 +371,19 @@ bool TranslationContext::vSatPkU8I16(const RdnaInstruction& inst) { return true; } -bool TranslationContext::vMulLegacyF32(const RdnaInstruction& inst, bool accumulate) { +bool TranslationContext::vMulLegacyF32(const RdnaInstruction& inst) { IrValue* lhs = readOperand(sourceAt(inst, 0u), IrType::F32); IrValue* rhs = readOperand(sourceAt(inst, 1u), IrType::F32); const auto isZero = [&](IrValue* value) { return IrU1(ir.IEqual(ir.BitwiseAnd(ir.BitCastU32(*value), ir.Constant(0x7fffffffu)), ir.Constant(0u))); }; const IrU1 zero(ir.LogicalOr(isZero(lhs).Value(), isZero(rhs).Value())); - IrValue* result = &ir.Emit(IrOpcode::SelectF32, IrType::F32, {&zero.Value(), &ir.ConstantF32(0.0f), nanResultF32({lhs, rhs}, &flushTinyProduct(lhs, rhs, &ir.Emit(IrOpcode::FPMul32, IrType::F32, {lhs, rhs})).Value())}); - if (accumulate) { - IrValue* addend = readOperand(accumulatorOperand(inst), IrType::F32); - result = &ir.Emit(IrOpcode::FPAdd32, IrType::F32, {result, addend}); - } - writeOperand(inst.destination, result); + writeOperand(inst.destination, &ir.Emit(IrOpcode::SelectF32, IrType::F32, {&zero.Value(), &ir.ConstantF32(0.0f), nanResultF32({lhs, rhs}, &flushTinyProduct(lhs, rhs, &ir.Emit(IrOpcode::FPMul32, IrType::F32, {lhs, rhs})).Value())})); return true; } bool TranslationContext::vMullitF32(const RdnaInstruction& inst) { - const IrU32 lhs = readU32(sourceAt(inst, 0u)); - const IrU32 rhs = readU32(sourceAt(inst, 1u)); - const IrU32 limit = readU32(sourceAt(inst, 2u)); + const IrU32 lhs = flushF32Denormal(readU32(sourceAt(inst, 0u))); + const IrU32 rhs = flushF32Denormal(readU32(sourceAt(inst, 1u))); + const IrU32 limit = flushF32Denormal(readU32(sourceAt(inst, 2u))); const auto magnitude = [&](const IrU32& bits) { return IrU32(ir.BitwiseAnd(bits.Value(), ir.Constant(0x7fffffffu))); }; const auto isNan = [&](const IrU32& bits) { return IrU1(ir.UGreaterThan(magnitude(bits).Value(), ir.Constant(0x7f800000u))); }; const auto isZero = [&](const IrU32& bits) { return IrU1(ir.IEqual(magnitude(bits).Value(), ir.Constant(0u))); }; @@ -358,7 +391,9 @@ bool TranslationContext::vMullitF32(const RdnaInstruction& inst) { const IrU1 rhsNegativeMax(ir.UGreaterThan(rhs.Value(), ir.Constant(0xff7ffffeu))); IrU1 lowest(ir.LogicalOr(rhsNegativeMax.Value(), ir.LogicalOr(isNan(limit).Value(), limitNotPositive.Value()))); lowest = IrU1(ir.LogicalOr(lowest.Value(), isNan(rhs).Value())); - const IrU32 product(ir.BitCastU32(ir.Emit(IrOpcode::FPMul32, IrType::F32, {&ir.BitCastF32(lhs.Value()), &ir.BitCastF32(rhs.Value())}))); + IrValue& lhsF32 = ir.BitCastF32(lhs.Value()); + IrValue& rhsF32 = ir.BitCastF32(rhs.Value()); + const IrU32 product(ir.BitCastU32(flushTinyProduct(&lhsF32, &rhsF32, &ir.Emit(IrOpcode::FPMul32, IrType::F32, {&lhsF32, &rhsF32})).Value())); IrU32 result(ir.Select(isNan(lhs).Value(), quietNan32(lhs).Value(), product.Value())); result = IrU32(ir.Select(ir.LogicalOr(isZero(lhs).Value(), isZero(rhs).Value()), ir.Constant(0u), result.Value())); result = IrU32(ir.Select(lowest.Value(), ir.Constant(0xff7fffffu), result.Value())); @@ -439,15 +474,13 @@ bool TranslationContext::vDivFmasF32(const RdnaInstruction& inst) { const IrU1 up(ir.Emit(IrOpcode::UGreaterThanEqual32, IrType::U1, {&exponent(*addend).Value(), &ir.Constant(128u)})); IrValue& power = ir.Emit(IrOpcode::SelectF32, IrType::F32, {&up.Value(), &ir.ConstantF32(18446744073709551616.0f), &ir.ConstantF32(5.42101086242752217e-20f)}); IrValue& plain = ir.Emit(IrOpcode::FPFma32, IrType::F32, {lhs, rhs, addend}); - const IrU32 plainExponent = exponent(plain); const IrU1 plainInfinite(ir.IEqual(ir.BitwiseAnd(ir.BitCastU32(plain), ir.Constant(0x7fffffffu)), ir.Constant(0x7f800000u))); - const IrU1 rescale(ir.LogicalOr(ir.LogicalAnd(up.Value(), ir.IEqual(plainExponent.Value(), ir.Constant(0u))), ir.LogicalAnd(ir.LogicalNot(up.Value()), plainInfinite.Value()))); + const IrU1 rescale(ir.LogicalAnd(ir.LogicalNot(up.Value()), plainInfinite.Value())); IrValue& lhsMagnitude = ir.BitwiseAnd(ir.BitCastU32(*lhs), ir.Constant(0x7fffffffu)); IrValue& rhsMagnitude = ir.BitwiseAnd(ir.BitCastU32(*rhs), ir.Constant(0x7fffffffu)); const IrU1 lhsLarger(ir.UGreaterThan(lhsMagnitude, rhsMagnitude)); - const IrU1 scaleLhs(ir.LogicalOr(ir.LogicalAnd(up.Value(), ir.LogicalNot(lhsLarger.Value())), ir.LogicalAnd(ir.LogicalNot(up.Value()), lhsLarger.Value()))); - IrValue& scaled = ir.Emit(IrOpcode::SelectF32, IrType::F32, {&scaleLhs.Value(), lhs, rhs}); - IrValue& other = ir.Emit(IrOpcode::SelectF32, IrType::F32, {&scaleLhs.Value(), rhs, lhs}); + IrValue& scaled = ir.Emit(IrOpcode::SelectF32, IrType::F32, {&lhsLarger.Value(), lhs, rhs}); + IrValue& other = ir.Emit(IrOpcode::SelectF32, IrType::F32, {&lhsLarger.Value(), rhs, lhs}); IrValue& rescaled = ir.Emit(IrOpcode::FPFma32, IrType::F32, {&ir.Emit(IrOpcode::FPMul32, IrType::F32, {&scaled, &power}), &other, &ir.Emit(IrOpcode::FPMul32, IrType::F32, {addend, &power})}); IrValue& afterwards = ir.Emit(IrOpcode::FPMul32, IrType::F32, {&plain, &power}); @@ -464,9 +497,9 @@ bool TranslationContext::vDivFmasF32(const RdnaInstruction& inst) { } bool TranslationContext::vDivFixupF32(const RdnaInstruction& inst) { - const IrU32 quotient(ir.BitCastU32(*readOperand(sourceAt(inst, 0u), IrType::F32))); - const IrU32 denominator(ir.BitCastU32(*readOperand(sourceAt(inst, 1u), IrType::F32))); - const IrU32 numerator(ir.BitCastU32(*readOperand(sourceAt(inst, 2u), IrType::F32))); + const IrU32 quotient = readU32(sourceAt(inst, 0u)); + const IrU32 denominator = flushF32Denormal(readU32(sourceAt(inst, 1u))); + const IrU32 numerator = flushF32Denormal(readU32(sourceAt(inst, 2u))); const auto magnitude = [&](IrU32 word) { return IrU32(ir.BitwiseAnd(word.Value(), ir.Constant(0x7fffffffu))); }; const auto isNan = [&](IrU32 word) { return IrU1(ir.UGreaterThan(magnitude(word).Value(), ir.Constant(0x7f800000u))); }; const auto isInf = [&](IrU32 word) { return IrU1(ir.IEqual(magnitude(word).Value(), ir.Constant(0x7f800000u))); }; @@ -565,16 +598,64 @@ bool TranslationContext::ieeeMinMaxF32(const RdnaInstruction& inst, IrOpcode opc return true; } -bool TranslationContext::vFmaLegacyF32(const RdnaInstruction& inst) { +bool TranslationContext::minMaxF16(const RdnaInstruction& inst, IrOpcode opcode) { + struct Half { + IrValue* value; + IrU32 bits; + }; + const bool ternary = opcode == IrOpcode::FPMinTri32 || opcode == IrOpcode::FPMaxTri32 || opcode == IrOpcode::FPMedTri32; + const auto isNan = [&](const IrU32& bits) { + return IrU1(ir.UGreaterThan(ir.BitwiseAnd(bits.Value(), ir.Constant(0x7fffu)), ir.Constant(0x7c00u))); + }; + const auto isSnan = [&](const IrU32& bits) { + return IrU1(ir.LogicalAnd(isNan(bits).Value(), ir.IEqual(ir.BitwiseAnd(bits.Value(), ir.Constant(0x0200u)), ir.Constant(0u)))); + }; + const auto toBits = [&](IrValue& value) { return packHalf2x16(IrF32(value), IrF32(ir.ConstantF32(0.0f))); }; + std::array args{}; + for (std::uint32_t index = 0u; index < (ternary ? 3u : 2u); ++index) { + const RdnaOperand& operand = sourceAt(inst, index); + args[index] = Half{&readF16AsF32(operand).Value(), readF16Bits(operand)}; + } + const auto pair = [&](IrOpcode pairOpcode, const Half& lhs, const Half& rhs) { + IrValue& value = ir.Emit(pairOpcode, IrType::F32, {lhs.value, rhs.value}); + IrU32 bits(ir.Select(isNan(lhs.bits).Value(), rhs.bits.Value(), toBits(value).Value())); + bits = IrU32(ir.Select(isNan(rhs.bits).Value(), lhs.bits.Value(), bits.Value())); + if (ieeeMode) { + bits = IrU32(ir.Select(isSnan(rhs.bits).Value(), quietNan16(rhs.bits).Value(), bits.Value())); + bits = IrU32(ir.Select(isSnan(lhs.bits).Value(), quietNan16(lhs.bits).Value(), bits.Value())); + } + return Half{&value, bits}; + }; + IrU32 result; + if (opcode == IrOpcode::FPMin32 || opcode == IrOpcode::FPMax32) { + result = pair(opcode, args[0], args[1]).bits; + } else if (opcode == IrOpcode::FPMinTri32 || opcode == IrOpcode::FPMaxTri32) { + const auto pairOpcode = opcode == IrOpcode::FPMinTri32 ? IrOpcode::FPMin32 : IrOpcode::FPMax32; + result = pair(pairOpcode, pair(pairOpcode, args[0], args[1]), args[2]).bits; + } else { + const IrU1 anyNan(ir.LogicalOr(ir.LogicalOr(isNan(args[0].bits).Value(), isNan(args[1].bits).Value()), isNan(args[2].bits).Value())); + const Half minimum = pair(IrOpcode::FPMin32, pair(IrOpcode::FPMin32, args[0], args[1]), args[2]); + IrValue& median = ir.Emit(IrOpcode::FPMedTri32, IrType::F32, {args[0].value, args[1].value, args[2].value}); + result = IrU32(ir.Select(anyNan.Value(), minimum.bits.Value(), toBits(median).Value())); + } + write16Bits(inst.destination, clampF16Bits(inst.destination, result)); + return true; +} + +bool TranslationContext::vFmaLegacyF32(const RdnaInstruction& inst, bool accumulate) { IrValue* lhs = readOperand(sourceAt(inst, 0u), IrType::F32); IrValue* rhs = readOperand(sourceAt(inst, 1u), IrType::F32); - IrValue* addend = readOperand(sourceAt(inst, 2u), IrType::F32); + IrValue* addend = readOperand(accumulate ? accumulatorOperand(inst) : sourceAt(inst, 2u), IrType::F32); const auto isZero = [&](IrValue* value) { return IrU1(ir.IEqual(ir.BitwiseAnd(ir.BitCastU32(*value), ir.Constant(0x7fffffffu)), ir.Constant(0u))); }; const IrU1 zero(ir.LogicalOr(isZero(lhs).Value(), isZero(rhs).Value())); const auto factor = [&](IrValue* value) { return &ir.Emit(IrOpcode::SelectF32, IrType::F32, {&zero.Value(), &ir.ConstantF32(0.0f), value}); }; IrValue* lhsFactor = factor(lhs); IrValue* rhsFactor = factor(rhs); - writeOperand(inst.destination, nanResultF32({lhsFactor, rhsFactor, addend}, &ir.Emit(IrOpcode::FPFma32, IrType::F32, {lhsFactor, rhsFactor, addend}), &invalidProductF32(lhsFactor, rhsFactor))); + IrValue* product = &flushTinyProduct(lhsFactor, rhsFactor, &ir.Emit(IrOpcode::FPMul32, IrType::F32, {lhsFactor, rhsFactor})).Value(); + const IrU1 zeroSum(ir.LogicalAnd(isZero(product).Value(), isZero(addend).Value())); + IrValue& zeroSumBits = ir.BitwiseAnd(ir.BitwiseAnd(ir.BitCastU32(*product), ir.BitCastU32(*addend)), ir.Constant(0x80000000u)); + IrValue* sum = &ir.Emit(IrOpcode::SelectF32, IrType::F32, {&zeroSum.Value(), &ir.BitCastF32(zeroSumBits), &ir.Emit(IrOpcode::FPAdd32, IrType::F32, {product, addend})}); + writeOperand(inst.destination, nanResultF32({lhsFactor, rhsFactor, addend}, sum, &invalidProductF32(lhsFactor, rhsFactor))); return true; } @@ -730,7 +811,9 @@ bool TranslationContext::floatCube(const RdnaInstruction& inst, std::uint32_t re default: throw std::runtime_error("invalid cube result kind"); } - writeOperand(inst.destination, &result.Value()); + const IrU32 bits(ir.BitCastU32(result.Value())); + const IrU1 nan(ir.UGreaterThan(ir.BitwiseAnd(bits.Value(), ir.Constant(0x7fffffffu)), ir.Constant(0x7f800000u))); + writeOperand(inst.destination, &ir.BitCastF32(ir.Select(nan.Value(), quietNan32(bits).Value(), bits.Value()))); return true; } diff --git a/core/shader/recompiler/Translation/src/InstructionTranslator.cpp b/core/shader/recompiler/Translation/src/InstructionTranslator.cpp index 2f5a7d6c5..b764b46af 100644 --- a/core/shader/recompiler/Translation/src/InstructionTranslator.cpp +++ b/core/shader/recompiler/Translation/src/InstructionTranslator.cpp @@ -164,6 +164,12 @@ void emitEntryPrologue(IrProgram& program, IrBlock& entryBlock, const TranslateO const auto builtin = [&entryIr](StageInputKind kind, std::uint32_t component = 0u) -> IrValue& { return entryIr.Emit(IrOpcode::GetBuiltin, IrOpcodeType(IrOpcode::GetBuiltin), {&entryIr.Constant(static_cast(kind)), &entryIr.Constant(component)}); }; + const auto drawIndex = [&entryIr, &options, &builtin](StageInputKind kind) -> IrValue& { + const auto* fetch = options.embeddedFetch; + const std::int32_t folded = fetch == nullptr ? -1 : kind == StageInputKind::VertexIndex ? fetch->vertexOffsetSgpr : fetch->instanceOffsetSgpr; + IrValue& index = builtin(kind); + return folded < 0 ? index : entryIr.ISub(index, entryIr.GetUserData(static_cast(folded))); + }; for (std::uint32_t index = 0; index < options.userDataCount; index++) { const auto reg = static_cast(options.userDataBaseRegister + index); @@ -219,7 +225,7 @@ void emitEntryPrologue(IrProgram& program, IrBlock& entryBlock, const TranslateO const auto& mesh = options.inputInfo.vertex->mesh; const std::uint32_t size = mesh.InputPrimitiveSize(); const std::uint32_t stepCount = mesh.InputPrimitiveStep(); - if (options.waveSize != 64u || mesh.primitivesPerGroup == 0u || mesh.verticesPerGroup != mesh.InputVertexCount(mesh.primitivesPerGroup) || mesh.verticesPerGroup > totalThreads || mesh.primitivesPerGroup > totalThreads || totalThreads % 64u != 0u || totalThreads > 15u * 64u || mesh.esgsItemSize == 0u || mesh.esgsItemSize * mesh.verticesPerGroup > 0xffffu) { + if (mesh.primitivesPerGroup == 0u || mesh.verticesPerGroup != mesh.InputVertexCount(mesh.primitivesPerGroup) || mesh.verticesPerGroup > totalThreads || mesh.primitivesPerGroup > totalThreads || totalThreads % options.waveSize != 0u || totalThreads / options.waveSize > 15u || mesh.esgsItemSize == 0u || mesh.esgsItemSize * mesh.verticesPerGroup > 0xffffu) { throw std::runtime_error("mesh shader translation configuration is not supported (wave " + std::to_string(options.waveSize) + ", primitives per group " + std::to_string(mesh.primitivesPerGroup) + ", vertices per group " + std::to_string(mesh.verticesPerGroup) + ", threads " + std::to_string(totalThreads) + ", ESGS item size " + std::to_string(mesh.esgsItemSize) + ")"); } constexpr std::uint32_t kTriFanPrimitiveType = 5u; @@ -250,11 +256,11 @@ void emitEntryPrologue(IrProgram& program, IrBlock& entryBlock, const TranslateO IrValue& vertices = minimum(subtractSaturate(indexCount, firstVertex), u32(mesh.verticesPerGroup)); IrValue& primitives = entryIr.Select(entryIr.ULessThan(vertices, u32(size)), u32(0u), entryIr.IAdd(entryIr.Emit(IrOpcode::UDiv32, IrOpcodeType(IrOpcode::UDiv32), {&subtractSaturate(vertices, u32(size)), &step}), u32(1u))); entryIr.SetScalarReg(static_cast(2), entryIr.BitwiseOr(entryIr.ShiftLeftLogical(vertices, u32(12u)), entryIr.ShiftLeftLogical(primitives, u32(22u)))); - IrValue& wave = entryIr.ShiftRightLogical(local, u32(6u)); - IrValue& waveBase = entryIr.BitwiseAnd(local, u32(~63u)); - IrValue& vertexCount = minimum(subtractSaturate(vertices, waveBase), u32(64u)); - IrValue& primitiveCount = minimum(subtractSaturate(primitives, waveBase), u32(64u)); - IrValue& waveInfo = entryIr.BitwiseOr(entryIr.ShiftLeftLogical(wave, u32(24u)), u32((totalThreads / 64u) << 28u)); + IrValue& wave = entryIr.ShiftRightLogical(local, u32(options.waveSize == 32u ? 5u : 6u)); + IrValue& waveBase = entryIr.BitwiseAnd(local, u32(~(options.waveSize - 1u))); + IrValue& vertexCount = minimum(subtractSaturate(vertices, waveBase), u32(options.waveSize)); + IrValue& primitiveCount = minimum(subtractSaturate(primitives, waveBase), u32(options.waveSize)); + IrValue& waveInfo = entryIr.BitwiseOr(entryIr.ShiftLeftLogical(wave, u32(24u)), u32((totalThreads / options.waveSize) << 28u)); entryIr.SetScalarReg(static_cast(3), entryIr.BitwiseOr(waveInfo, entryIr.BitwiseOr(entryIr.ShiftLeftLogical(primitiveCount, u32(8u)), vertexCount))); IrValue& parity = mesh.inputPrimitive == kTriStripPrimitiveType ? entryIr.BitwiseAnd(entryIr.IAdd(firstPrimitive, local), u32(1u)) : u32(0u); IrValue& vertex = entryIr.IMul(local, step); @@ -288,9 +294,9 @@ void emitEntryPrologue(IrProgram& program, IrBlock& entryBlock, const TranslateO entryIr.SetVectorReg(static_cast(8), entryIr.IAdd(draw(2u), builtin(StageInputKind::WorkgroupId, 1u))); } else if (options.stage == ShaderStageKind::Local) { entryIr.SetScalarReg(static_cast(3), entryIr.Constant(64u)); - entryIr.SetVectorReg(static_cast(2), builtin(StageInputKind::VertexIndex)); + entryIr.SetVectorReg(static_cast(2), drawIndex(StageInputKind::VertexIndex)); entryIr.SetVectorReg(static_cast(3), entryIr.Constant(0u)); - entryIr.SetVectorReg(static_cast(5), builtin(StageInputKind::InstanceIndex)); + entryIr.SetVectorReg(static_cast(5), drawIndex(StageInputKind::InstanceIndex)); } else if (options.stage == ShaderStageKind::TessellationControl) { const auto& tess = options.inputInfo.vertex->tess; entryIr.SetScalarReg(static_cast(2), entryIr.Emit(IrOpcode::TessellationBase, IrOpcodeType(IrOpcode::TessellationBase), {&entryIr.Constant(0u)})); @@ -309,13 +315,11 @@ void emitEntryPrologue(IrProgram& program, IrBlock& entryBlock, const TranslateO const auto* ps = options.inputInfo.pixel; const auto vgpr = [&](PixelInput input) { return ps->psInputVgpr[static_cast(input)]; }; const auto loaded = [&](PixelInput input) { return vgpr(input) != ShaderPixelInputInfo::NoPixelInputVgpr; }; - if (options.fragmentShaderBarycentricEnabled) { - for (const auto [input, kind] : {std::pair{PixelInput::PerspectiveCenter, StageInputKind::BaryCoordSmooth}, std::pair{PixelInput::PerspectiveCentroid, StageInputKind::BaryCoordSmooth}, - std::pair{PixelInput::LinearCenter, StageInputKind::BaryCoordNoPerspective}, std::pair{PixelInput::LinearCentroid, StageInputKind::BaryCoordNoPerspective}}) { - if (!loaded(input)) continue; - entryIr.SetVectorReg(static_cast(vgpr(input)), builtin(kind, 0u)); - entryIr.SetVectorReg(static_cast(vgpr(input) + 1u), builtin(kind, 1u)); - } + for (const auto [input, kind] : {std::pair{PixelInput::PerspectiveCenter, StageInputKind::BaryCoordSmooth}, std::pair{PixelInput::PerspectiveCentroid, StageInputKind::BaryCoordSmooth}, + std::pair{PixelInput::LinearCenter, StageInputKind::BaryCoordNoPerspective}, std::pair{PixelInput::LinearCentroid, StageInputKind::BaryCoordNoPerspective}}) { + if (!loaded(input)) continue; + entryIr.SetVectorReg(static_cast(vgpr(input)), builtin(kind, 0u)); + entryIr.SetVectorReg(static_cast(vgpr(input) + 1u), builtin(kind, 1u)); } if (loaded(PixelInput::PositionX)) { entryIr.SetVectorReg(static_cast(vgpr(PixelInput::PositionX)), builtin(StageInputKind::FragCoord, 0u)); @@ -337,12 +341,20 @@ void emitEntryPrologue(IrProgram& program, IrBlock& entryBlock, const TranslateO if (loaded(PixelInput::Ancillary)) { entryIr.SetVectorReg(static_cast(vgpr(PixelInput::Ancillary)), builtin(StageInputKind::PackedAncillary)); } + if (loaded(PixelInput::LineStipple)) { + entryIr.SetVectorReg(static_cast(vgpr(PixelInput::LineStipple)), entryIr.Constant(0u)); + } + if (loaded(PixelInput::PositionFixedPoint)) { + IrValue& x = entryIr.Emit(IrOpcode::ConvertU32F32, IrType::U32, {&entryIr.BitCastF32(builtin(StageInputKind::FragCoord, 0u))}); + IrValue& y = entryIr.Emit(IrOpcode::ConvertU32F32, IrType::U32, {&entryIr.BitCastF32(builtin(StageInputKind::FragCoord, 1u))}); + entryIr.SetVectorReg(static_cast(vgpr(PixelInput::PositionFixedPoint)), entryIr.BitwiseOr(entryIr.BitwiseAnd(x, entryIr.Constant(0xffffu)), entryIr.ShiftLeftLogical(y, entryIr.Constant(16u)))); + } } else if (options.stage == ShaderStageKind::Vertex) { if (options.userDataBaseRegister >= 8u) { entryIr.SetScalarReg(static_cast(3), entryIr.Constant(options.waveSize | (options.waveSize << 8u))); } - entryIr.SetVectorReg(static_cast(5), builtin(StageInputKind::VertexIndex)); - entryIr.SetVectorReg(static_cast(8), builtin(StageInputKind::InstanceIndex)); + entryIr.SetVectorReg(static_cast(5), drawIndex(StageInputKind::VertexIndex)); + entryIr.SetVectorReg(static_cast(8), drawIndex(StageInputKind::InstanceIndex)); } } @@ -375,11 +387,12 @@ IrProgram InstructionTranslator::Translate(const RdnaProgram& decoded, const Con program.Resources().userDataBase = options.userDataBaseRegister; program.Resources().userDataCount = options.userDataCount; program.Info().scratchDwords = options.scratchDwords; + program.Info().sharedMemoryBytes = options.sharedMemoryBytes; if (options.embeddedFetch != nullptr) { program.Info().vertexOffsetSgpr = options.embeddedFetch->vertexOffsetSgpr; program.Info().instanceOffsetSgpr = options.embeddedFetch->instanceOffsetSgpr; - program.Info().vertexOffsetShared = options.embeddedFetch->vertexOffsetShared; - program.Info().instanceOffsetShared = options.embeddedFetch->instanceOffsetShared; + program.Info().vertexOffsetShared = options.embeddedFetch->vertexOffsetShared || options.embeddedFetch->vertexIndexObserved; + program.Info().instanceOffsetShared = options.embeddedFetch->instanceOffsetShared || options.embeddedFetch->instanceIndexObserved; program.Info().vertexOffsetConflict = options.embeddedFetch->vertexOffsetConflict; program.Info().instanceOffsetConflict = options.embeddedFetch->instanceOffsetConflict; } diff --git a/core/shader/recompiler/Translation/src/IntegerInstructions.cpp b/core/shader/recompiler/Translation/src/IntegerInstructions.cpp index 4a9400e1a..d1af902f6 100644 --- a/core/shader/recompiler/Translation/src/IntegerInstructions.cpp +++ b/core/shader/recompiler/Translation/src/IntegerInstructions.cpp @@ -20,7 +20,7 @@ bool TranslationContext::integer16Shift(const RdnaInstruction& inst, IrOpcode op bool TranslationContext::integer16Binary(const RdnaInstruction& inst, IrOpcode opcode, bool sign) { const IrU32 lhs = readU16AsU32(sourceAt(inst, 0u), sign); const IrU32 rhs = readU16AsU32(sourceAt(inst, 1u), sign); - const IrU32 result(ir.Emit(opcode, IrType::U32, {&lhs.Value(), &rhs.Value()})); + const IrU32 result = saturateInteger16(inst.destination, IrU32(ir.Emit(opcode, IrType::U32, {&lhs.Value(), &rhs.Value()})), sign); write16Bits(inst.destination, IrU32(ir.BitwiseAnd(result.Value(), ir.Constant(0xffffu)))); return true; } @@ -625,7 +625,8 @@ bool TranslationContext::sQuadmask(const RdnaInstruction& inst, bool wide) { bool TranslationContext::bfmB32(const RdnaInstruction& inst) { const IrU32 count(ir.BitwiseAnd(readU32(sourceAt(inst, 0u)).Value(), ir.Constant(31u))); const IrU32 offset(ir.BitwiseAnd(readU32(sourceAt(inst, 1u)).Value(), ir.Constant(31u))); - const IrU32 result(ir.Emit(IrOpcode::BitFieldInsert, IrType::U32, {&ir.Constant(0u), &ir.Constant(0xffffffffu), &offset.Value(), &count.Value()})); + const IrU32 mask = rightMask32(count); + const IrU32 result(ir.ShiftLeftLogical(mask.Value(), offset.Value())); writeOperand(inst.destination, &result.Value()); return true; } diff --git a/core/shader/recompiler/Translation/src/MemoryInstructions/FlatInstructions.cpp b/core/shader/recompiler/Translation/src/MemoryInstructions/FlatInstructions.cpp index 9fbd72b3e..862c65f29 100644 --- a/core/shader/recompiler/Translation/src/MemoryInstructions/FlatInstructions.cpp +++ b/core/shader/recompiler/Translation/src/MemoryInstructions/FlatInstructions.cpp @@ -1,6 +1,7 @@ #include "Translation/MemoryInstructions.hpp" #include "Translation/TranslationContext.hpp" #include +#include #include #include @@ -15,6 +16,22 @@ bool wideAddressLoadsEnabled() { return !byteReads; } +bool wideAddressStoresEnabled() { + static const bool byteWrites = std::getenv("APS5_BDA_DWORD_STORES") != nullptr; + return !byteWrites; +} + +IrOpcode wideAddressStoreOpcode(std::uint32_t dwords) { + switch (dwords) { + case 2u: + return IrOpcode::StoreAddressU32x2; + case 3u: + return IrOpcode::StoreAddressU32x3; + default: + return IrOpcode::StoreAddressU32x4; + } +} + IrOpcode wideAddressLoadOpcode(std::uint32_t dwords) { switch (dwords) { case 2u: @@ -136,6 +153,19 @@ bool TranslationContext::flatStore(const RdnaInstruction& inst) { const AddressOperands address = readAddressOperands(inst, 0u); IrValue& active = ir.GetExec(); const std::uint32_t count = memory.dataBits == 32u ? memory.dataDwords : 1u; + if (count > 1u && count <= 4u && memory.kind != ResourceKind::Scratch && wideAddressStoresEnabled()) { + MemoryInfo group = memory; + group.dataDwords = count; + group.componentCount = count; + group.componentIndex = 0u; + std::array parts{}; + for (std::uint32_t index = 0u; index < count; ++index) parts[index] = &readU32(offsetOperand(inst.destination, index)).Value(); + IrValue& data = count == 2u ? ir.Emit(IrOpcode::CompositeConstructU32x2, IrOpcodeType(IrOpcode::CompositeConstructU32x2), {parts[0], parts[1]}) + : count == 3u ? ir.Emit(IrOpcode::CompositeConstructU32x3, IrOpcodeType(IrOpcode::CompositeConstructU32x3), {parts[0], parts[1], parts[2]}) + : ir.Emit(IrOpcode::CompositeConstructU32x4, IrOpcodeType(IrOpcode::CompositeConstructU32x4), {parts[0], parts[1], parts[2], parts[3]}); + (void)ir.Emit(wideAddressStoreOpcode(count), IrType::Void, {address.resource, address.low, address.high, &data, &active}, addMemoryInfo(group, inst.programCounter)); + return true; + } for (std::uint32_t index = 0u; index < count; ++index) { MemoryInfo component = memory; component.offset += index * 4u; diff --git a/core/shader/recompiler/Translation/src/MemoryInstructions/SharedMemoryInstructions.cpp b/core/shader/recompiler/Translation/src/MemoryInstructions/SharedMemoryInstructions.cpp index be58eaf47..04115cbb3 100644 --- a/core/shader/recompiler/Translation/src/MemoryInstructions/SharedMemoryInstructions.cpp +++ b/core/shader/recompiler/Translation/src/MemoryInstructions/SharedMemoryInstructions.cpp @@ -31,7 +31,8 @@ MemoryInfo sharedMemoryInfoFromInstruction(const RdnaInstruction& inst) { } bool TranslationContext::dsAtomic(const RdnaInstruction& inst, IrOpcode opcode, bool returnsValue) { - const MemoryInfo memory = sharedMemoryInfoFromInstruction(inst); + MemoryInfo memory = sharedMemoryInfoFromInstruction(inst); + memory.flushDenormals = sharedFloatFlush(opcode); const IrU32 address = readU32(inst.source0); const IrU32 value = readU32(inst.source1); IrValue& active = ir.GetExec(); @@ -211,8 +212,24 @@ bool TranslationContext::dsSrc2(const RdnaInstruction& inst, IrOpcode opcode) { return true; } +bool TranslationContext::sharedFloatFlush(IrOpcode opcode) const { + switch (opcode) { + case IrOpcode::SharedAtomicFMin32: + case IrOpcode::SharedAtomicFMax32: + case IrOpcode::SharedAtomicCmpstF32: + return !floatMode.has_value() || ((floatMode->floatMode >> 4u) & 1u) == 0u; + case IrOpcode::SharedAtomicFMin64: + case IrOpcode::SharedAtomicFMax64: + case IrOpcode::SharedAtomicCmpstF64: + return floatMode.has_value() && ((floatMode->floatMode >> 6u) & 1u) == 0u; + default: + return false; + } +} + bool TranslationContext::dsAtomic2(const RdnaInstruction& inst, IrOpcode opcode, bool returnsValue) { - const MemoryInfo memory = sharedMemoryInfoFromInstruction(inst); + MemoryInfo memory = sharedMemoryInfoFromInstruction(inst); + memory.flushDenormals = sharedFloatFlush(opcode); const IrU32 address = readU32(inst.source0); const IrU32 data0 = readU32(inst.source1); const IrU32 data1 = readU32(inst.source2); @@ -226,7 +243,8 @@ bool TranslationContext::dsAtomic64(const RdnaInstruction& inst, IrOpcode opcode if (inst.gds) { throw std::runtime_error("64-bit GDS atomics are not supported"); } - const MemoryInfo memory = sharedMemoryInfoFromInstruction(inst); + MemoryInfo memory = sharedMemoryInfoFromInstruction(inst); + memory.flushDenormals = sharedFloatFlush(opcode); const IrU32 address = readU32(inst.source0); const IrU64 data0 = readU64(inst.source1); IrValue& active = ir.GetExec(); diff --git a/core/shader/recompiler/Translation/src/ScalarInstructions.cpp b/core/shader/recompiler/Translation/src/ScalarInstructions.cpp index 7136560e7..8484da202 100644 --- a/core/shader/recompiler/Translation/src/ScalarInstructions.cpp +++ b/core/shader/recompiler/Translation/src/ScalarInstructions.cpp @@ -43,7 +43,7 @@ bool TranslationContext::emitScalar(const RdnaInstruction& inst) { sCselectB32(inst); return true; case RdnaOpcode::SCselectB64: - scalarSelect64(inst, sourceAt(inst, 1u)); + scalarSelectMask64(inst); return true; case RdnaOpcode::SCmovB32: { const IrU32 source = readU32(sourceAt(inst, 0u)); @@ -56,6 +56,11 @@ bool TranslationContext::emitScalar(const RdnaInstruction& inst) { scalarSelect64(inst, inst.destination); return true; case RdnaOpcode::SSetregB32: + if ((inst.source1.value & 0x3fu) == 1u) { + throw std::runtime_error("s_setreg_b32 at pc " + std::to_string(inst.programCounter) + " writes MODE from an SGPR: runtime MODE changes are not implemented"); + } + emitControlNop(); + return true; case RdnaOpcode::SVersion: emitControlNop(); return true; @@ -64,8 +69,12 @@ bool TranslationContext::emitScalar(const RdnaInstruction& inst) { const std::uint32_t offset = (field >> 6u) & 0x1fu; const std::uint32_t size = ((field >> 11u) & 0x1fu) + 1u; const std::uint64_t written = static_cast(inst.source0.value) & ((std::uint64_t{1} << size) - 1u); - if ((field & 0x3fu) != 1u || offset + size > 4u || written != 0u) { - throw std::runtime_error("s_setreg_imm32_b32 at pc " + std::to_string(inst.programCounter) + " writes a hardware register field other than round to nearest even in MODE"); + if ((field & 0x3fu) != 1u || offset + size > 4u) { + throw std::runtime_error("s_setreg_imm32_b32 at pc " + std::to_string(inst.programCounter) + " writes a hardware register field other than the MODE rounding fields"); + } + const std::uint32_t initial = floatMode.has_value() ? floatMode->floatMode : 0u; + if (written != ((initial >> offset) & ((1u << size) - 1u))) { + throw std::runtime_error("s_setreg_imm32_b32 at pc " + std::to_string(inst.programCounter) + " changes the initial MODE rounding fields: runtime MODE changes are not implemented"); } emitControlNop(); return true; @@ -77,7 +86,8 @@ bool TranslationContext::emitScalar(const RdnaInstruction& inst) { if ((field & 0x3fu) != 1u || offset + size > 4u) { throw std::runtime_error("s_getreg_b32 at pc " + std::to_string(inst.programCounter) + " reads hardware register " + std::to_string(field & 0x3fu) + " bits " + std::to_string(offset) + ".." + std::to_string(offset + size - 1u) + ": only the MODE round mode fields are modeled"); } - writeRawU32(inst.destination, IrU32(ir.Constant(0u))); + const std::uint32_t initial = floatMode.has_value() ? floatMode->floatMode : 0u; + writeRawU32(inst.destination, IrU32(ir.Constant((initial >> offset) & ((1u << size) - 1u)))); return true; } case RdnaOpcode::SCmovkI32: { @@ -413,12 +423,14 @@ bool TranslationContext::emitScalar(const RdnaInstruction& inst) { case RdnaOpcode::SDecperflevel: emitControlNop(); return true; - case RdnaOpcode::SRoundMode: - if (inst.source0.value != 0u) { - throw std::runtime_error("s_round_mode " + std::to_string(inst.source0.value) + " at pc " + std::to_string(inst.programCounter) + " selects a rounding mode other than round to nearest even"); + case RdnaOpcode::SRoundMode: { + const std::uint32_t initial = floatMode.has_value() ? floatMode->floatMode : 0u; + if ((inst.source0.value & 0xfu) != (initial & 0xfu)) { + throw std::runtime_error("s_round_mode " + std::to_string(inst.source0.value) + " at pc " + std::to_string(inst.programCounter) + " changes the initial MODE rounding fields: runtime MODE changes are not implemented"); } emitControlNop(); return true; + } case RdnaOpcode::SDenormMode: throw std::runtime_error("s_denorm_mode " + std::to_string(inst.source0.value) + " at pc " + std::to_string(inst.programCounter) + ": the recompiler does not model denormal modes"); case RdnaOpcode::SSethalt: diff --git a/core/shader/recompiler/Translation/src/ShaderInputInfoBuilder.cpp b/core/shader/recompiler/Translation/src/ShaderInputInfoBuilder.cpp index c75a92eec..c14e244be 100644 --- a/core/shader/recompiler/Translation/src/ShaderInputInfoBuilder.cpp +++ b/core/shader/recompiler/Translation/src/ShaderInputInfoBuilder.cpp @@ -121,6 +121,8 @@ ShaderStageInputInfo BuildShaderStageInputInfo(ShaderStageKind stage, const Gues place(PixelInput::PositionW, pixel.posW); place(PixelInput::FrontFace, pixel.frontFace); place(PixelInput::Ancillary, pixel.ancillary); + place(PixelInput::LineStipple, (pixel.inputAddr & PixelInputBit(PixelInput::LineStipple)) != 0u); + place(PixelInput::PositionFixedPoint, (pixel.inputAddr & PixelInputBit(PixelInput::PositionFixedPoint)) != 0u); for (std::uint32_t i = 0; i < 8; ++i) { pixelStorage.targetOutputMode[i] = pixel.targetOutputMode[i]; } diff --git a/core/shader/recompiler/Translation/src/TranslationContext/ExecMask.cpp b/core/shader/recompiler/Translation/src/TranslationContext/ExecMask.cpp index 66b55249d..6e94905dc 100644 --- a/core/shader/recompiler/Translation/src/TranslationContext/ExecMask.cpp +++ b/core/shader/recompiler/Translation/src/TranslationContext/ExecMask.cpp @@ -19,6 +19,13 @@ IrU32 TranslationContext::hostExecWord(std::uint32_t half) { } IrU1 TranslationContext::threadBit(const std::array& mask) { + const bool wide = program.WaveSize() == 64u; + if (mask[0].Value().HasImmediate() && (!wide || mask[1].Value().HasImmediate())) { + const std::uint32_t low = mask[0].Value().ImmediateU32(); + const std::uint32_t high = wide ? mask[1].Value().ImmediateU32() : low; + if (low == 0u && high == 0u) return IrU1(ir.ConstantBool(false)); + if (low == 0xffffffffu && high == 0xffffffffu) return IrU1(ir.ConstantBool(true)); + } const IrU32 lane(ir.Emit(IrOpcode::LaneId, IrType::U32, {})); const IrU32 word = program.WaveSize() == 64u ? IrU32(ir.Select(ir.ULessThan(lane.Value(), ir.Constant(32u)), mask[0].Value(), mask[1].Value())) : mask[0]; const IrU32 bit(ir.BitwiseAnd(lane.Value(), ir.Constant(31u))); diff --git a/core/shader/recompiler/Translation/src/TranslationContext/OperandAccess.cpp b/core/shader/recompiler/Translation/src/TranslationContext/OperandAccess.cpp index 3136b2f11..63d6d727b 100644 --- a/core/shader/recompiler/Translation/src/TranslationContext/OperandAccess.cpp +++ b/core/shader/recompiler/Translation/src/TranslationContext/OperandAccess.cpp @@ -12,6 +12,24 @@ DppMoveFlags dppFlags(const RdnaOperand& operand) { return {control, static_cast(operand.dppRowMask), static_cast(operand.dppBankMask), operand.dppFetchInactive, operand.dppBoundCtrl}; } +const IrValue* constantF32Bits(const IrValue& value) { + if (value.Opcode() != IrOpcode::BitCastF32U32) return nullptr; + const IrValue* bits = value.Argument(0); + return bits->HasImmediate() ? bits : nullptr; +} + +bool keepsProductOutOfTinyRange(const IrValue& factor) { + const IrValue* bits = constantF32Bits(factor); + if (bits == nullptr) return false; + const std::uint32_t exponent = (bits->ImmediateU32() >> 23u) & 0xffu; + return exponent == 0u || exponent >= 127u; +} + +bool hasUnitSignificand(const IrValue& factor) { + const IrValue* bits = constantF32Bits(factor); + return bits != nullptr && (bits->ImmediateU32() & 0x007fffffu) == 0u; +} + } const RdnaOperand& TranslationContext::sourceAt(const RdnaInstruction& inst, std::uint32_t index) { @@ -127,12 +145,16 @@ RdnaOperand TranslationContext::plainOperand(const RdnaOperand& operand) { IrU32 TranslationContext::flushF32Denormal(IrU32 bits) { if ((f32DenormalFlush & 1u) == 0u) return bits; + if (bits.Value().HasImmediate()) { + const std::uint32_t value = bits.Value().ImmediateU32(); + return (value & 0x7f800000u) == 0u ? IrU32(ir.Constant(value & 0x80000000u)) : bits; + } const IrU1 denormal(ir.IEqual(ir.BitwiseAnd(bits.Value(), ir.Constant(0x7f800000u)), ir.Constant(0u))); return IrU32(ir.Select(denormal.Value(), ir.BitwiseAnd(bits.Value(), ir.Constant(0x80000000u)), bits.Value())); } IrF32 TranslationContext::flushTinyProduct(IrValue* lhs, IrValue* rhs, IrValue* product, IrValue* addend) { - if ((f32DenormalFlush & 2u) == 0u) return IrF32(*product); + if ((f32DenormalFlush & 2u) == 0u || keepsProductOutOfTinyRange(*lhs) || keepsProductOutOfTinyRange(*rhs)) return IrF32(*product); IrValue& lhsBits = ir.BitCastU32(*lhs); IrValue& rhsBits = ir.BitCastU32(*rhs); const auto exponent = [&](IrValue& bits) -> IrValue& { return ir.Emit(IrOpcode::BitFieldUExtract, IrType::U32, {&bits, &ir.Constant(23u), &ir.Constant(8u)}); }; @@ -140,7 +162,7 @@ IrF32 TranslationContext::flushTinyProduct(IrValue* lhs, IrValue* rhs, IrValue* IrValue& lhsExponent = exponent(lhsBits); IrValue& rhsExponent = exponent(rhsBits); const auto finiteNonZero = [&](IrValue& value) -> IrValue& { return ir.LogicalAnd(ir.INotEqual(value, ir.Constant(0u)), ir.INotEqual(value, ir.Constant(0xffu))); }; - IrValue& carry = ir.Select(ir.UGreaterThan(ir.Emit(IrOpcode::UMulHi, IrType::U32, {&significand(lhsBits), &significand(rhsBits)}), ir.Constant(0x7fffu)), ir.Constant(1u), ir.Constant(0u)); + IrValue& carry = hasUnitSignificand(*lhs) || hasUnitSignificand(*rhs) ? ir.Constant(0u) : ir.Select(ir.UGreaterThan(ir.Emit(IrOpcode::UMulHi, IrType::U32, {&significand(lhsBits), &significand(rhsBits)}), ir.Constant(0x7fffu)), ir.Constant(1u), ir.Constant(0u)); IrValue* tiny = &ir.LogicalAnd(ir.LogicalAnd(finiteNonZero(lhsExponent), finiteNonZero(rhsExponent)), ir.ULessThan(ir.IAdd(ir.IAdd(lhsExponent, rhsExponent), carry), ir.Constant(128u))); if (addend != nullptr) tiny = &ir.LogicalAnd(*tiny, ir.IEqual(ir.BitwiseAnd(ir.BitCastU32(*addend), ir.Constant(0x7fffffffu)), ir.Constant(0u))); IrValue& sign = ir.BitwiseAnd(ir.BitwiseXor(lhsBits, rhsBits), ir.Constant(0x80000000u)); @@ -173,11 +195,17 @@ IrValue* TranslationContext::readOperand(const RdnaOperand& operand, IrType type return &ir.ConstructU64(pair[0].Value(), pair[1].Value()); } IrU32 bits = applyBitSourceModifiers(operand, readRawU32(operand)); - if (operand.absolute) { - bits = IrU32(ir.BitwiseAnd(bits.Value(), ir.Constant(0x7fffffffu))); - } - if (operand.negate) { - bits = IrU32(ir.BitwiseXor(bits.Value(), ir.Constant(0x80000000u))); + if (bits.Value().HasImmediate()) { + const std::uint32_t value = bits.Value().ImmediateU32(); + const std::uint32_t magnitude = operand.absolute ? value & 0x7fffffffu : value; + bits = IrU32(ir.Constant(operand.negate ? magnitude ^ 0x80000000u : magnitude)); + } else { + if (operand.absolute) { + bits = IrU32(ir.BitwiseAnd(bits.Value(), ir.Constant(0x7fffffffu))); + } + if (operand.negate) { + bits = IrU32(ir.BitwiseXor(bits.Value(), ir.Constant(0x80000000u))); + } } if (TypesOverlap(type, IrType::F32) && !TypesOverlap(type, IrType::U32)) { return &ir.BitCastF32(flushF32Denormal(bits).Value()); @@ -311,6 +339,10 @@ IrU32 TranslationContext::quietNan16(IrU32 bits) { return ieeeMode ? IrU32(ir.BitwiseOr(bits.Value(), ir.Constant(0x0200u))) : bits; } +std::array TranslationContext::quietNan64(const std::array& bits) { + return ieeeMode ? std::array{bits[0], IrU32(ir.BitwiseOr(bits[1].Value(), ir.Constant(0x00080000u)))} : bits; +} + IrF32 TranslationContext::applyF32ResultModifiers(const RdnaOperand& operand, IrF32 value) { if (operand.omod != 0u && outputModifierApplies(4u)) { IrValue& bits = ir.BitCastU32(value.Value()); @@ -346,6 +378,21 @@ IrF32 TranslationContext::applyF16ResultModifiers(const RdnaOperand& operand, Ir return dx10Clamp() ? clamped : selectF32(IrU1(ir.Emit(IrOpcode::FPIsNan32, IrType::U1, {&value.Value()})), value, clamped); } +IrF32 TranslationContext::clampF16Overflow(IrF32 value, std::initializer_list sources) { + if (!fp16Overflow()) { + return value; + } + const IrU32 bits(ir.BitCastU32(value.Value())); + const IrU32 magnitude(ir.BitwiseAnd(bits.Value(), ir.Constant(0x7fffffffu))); + IrU1 overflow(ir.LogicalAnd(ir.LogicalNot(ir.ULessThan(magnitude.Value(), ir.Constant(0x477ff000u))), ir.ULessThan(magnitude.Value(), ir.Constant(0x7f800001u)))); + for (IrValue* source : sources) { + const IrU1 infinite(ir.IEqual(ir.BitwiseAnd(ir.BitCastU32(*source), ir.Constant(0x7fffffffu)), ir.Constant(0x7f800000u))); + overflow = IrU1(ir.LogicalAnd(overflow.Value(), ir.LogicalNot(infinite.Value()))); + } + const IrU32 largest(ir.BitwiseOr(ir.BitwiseAnd(bits.Value(), ir.Constant(0x80000000u)), ir.Constant(0x477fe000u))); + return IrF32(ir.BitCastF32(ir.Select(overflow.Value(), largest.Value(), bits.Value()))); +} + IrU32 TranslationContext::clampF16Bits(const RdnaOperand& operand, IrU32 bits) { rejectHalfOrDoubleOutputModifier(operand); if (!operand.clamp) { diff --git a/core/shader/recompiler/Translation/src/TranslationContext/SubwordPacking.cpp b/core/shader/recompiler/Translation/src/TranslationContext/SubwordPacking.cpp index fc3a74a66..69f5f4549 100644 --- a/core/shader/recompiler/Translation/src/TranslationContext/SubwordPacking.cpp +++ b/core/shader/recompiler/Translation/src/TranslationContext/SubwordPacking.cpp @@ -156,8 +156,8 @@ void TranslationContext::write16Bits(const RdnaOperand& operand, IrU32 value) { writeRawU32(operand, IrU32(ir.BitwiseOr(cleared.Value(), masked.Value()))); } -void TranslationContext::writeF16(const RdnaOperand& operand, IrF32 value) { - const IrF16 half(ir.Emit(IrOpcode::ConvertF16F32, IrType::F16, {&applyF16ResultModifiers(operand, value).Value()})); +void TranslationContext::writeF16(const RdnaOperand& operand, IrF32 value, std::initializer_list sources) { + const IrF16 half(ir.Emit(IrOpcode::ConvertF16F32, IrType::F16, {&applyF16ResultModifiers(operand, clampF16Overflow(value, sources)).Value()})); const IrU16 bits(ir.Emit(IrOpcode::BitCastU16F16, IrType::U16, {&half.Value()})); write16Bits(operand, IrU32(ir.Emit(IrOpcode::ConvertU32U16, IrType::U32, {&bits.Value()}))); } diff --git a/core/shader/recompiler/Translation/src/VectorInstructions.cpp b/core/shader/recompiler/Translation/src/VectorInstructions.cpp index 9b5fd4580..56c9a588d 100644 --- a/core/shader/recompiler/Translation/src/VectorInstructions.cpp +++ b/core/shader/recompiler/Translation/src/VectorInstructions.cpp @@ -31,6 +31,7 @@ std::uint32_t TranslationContext::f32DenormalFlushFor(const RdnaInstruction& ins case RdnaOpcode::VSubrevF32: case RdnaOpcode::VMulF32: case RdnaOpcode::VMulLegacyF32: + case RdnaOpcode::VMullitF32: case RdnaOpcode::VMinF32: case RdnaOpcode::VMaxF32: case RdnaOpcode::VMin3F32: @@ -85,8 +86,14 @@ std::uint32_t TranslationContext::f32DenormalFlushFor(const RdnaInstruction& ins case RdnaOpcode::VCvtPkU8F32: case RdnaOpcode::VCvtPkrtzF16F32: case RdnaOpcode::VCvtF16F32: + case RdnaOpcode::VDivFixupF32: + case RdnaOpcode::VMadMixloF16: + case RdnaOpcode::VMadMixhiF16: flush = 1u; break; + case RdnaOpcode::VMadLegacyF32: + case RdnaOpcode::VMacLegacyF32: + return 3u; default: return 0u; } @@ -616,9 +623,9 @@ bool TranslationContext::emitVector(const RdnaInstruction& inst) { case RdnaOpcode::VSatPkU8I16: return vSatPkU8I16(inst); case RdnaOpcode::VMulLegacyF32: - return vMulLegacyF32(inst, false); + return vMulLegacyF32(inst); case RdnaOpcode::VMacLegacyF32: - return vMulLegacyF32(inst, true); + return vFmaLegacyF32(inst, true); case RdnaOpcode::VMullitF32: return vMullitF32(inst); case RdnaOpcode::VCmpClassF32: @@ -809,11 +816,13 @@ bool TranslationContext::emitVector(const RdnaInstruction& inst) { case RdnaOpcode::VAshrrevI16: return integer16Shift(inst, IrOpcode::ShiftRightArithmetic32, true); case RdnaOpcode::VAddNcU16: - case RdnaOpcode::VAddNcI16: return integer16Binary(inst, IrOpcode::IAdd32, false); + case RdnaOpcode::VAddNcI16: + return integer16Binary(inst, IrOpcode::IAdd32, true); case RdnaOpcode::VSubNcU16: - case RdnaOpcode::VSubNcI16: return integer16Binary(inst, IrOpcode::ISub32, false); + case RdnaOpcode::VSubNcI16: + return integer16Binary(inst, IrOpcode::ISub32, true); case RdnaOpcode::VMed3I16: return vMed3I16(inst); case RdnaOpcode::VMin3I16: @@ -883,9 +892,9 @@ bool TranslationContext::emitVector(const RdnaInstruction& inst) { case RdnaOpcode::VMulF16: return float16Binary(inst, IrOpcode::FPMul32, false); case RdnaOpcode::VMinF16: - return float16Binary(inst, IrOpcode::FPMin32, false); + return minMaxF16(inst, IrOpcode::FPMin32); case RdnaOpcode::VMaxF16: - return float16Binary(inst, IrOpcode::FPMax32, false); + return minMaxF16(inst, IrOpcode::FPMax32); case RdnaOpcode::VFmacF16: return float16Ternary(inst, IrOpcode::FPFma32, true, false); case RdnaOpcode::VFmamkF16: @@ -920,11 +929,11 @@ bool TranslationContext::emitVector(const RdnaInstruction& inst) { case RdnaOpcode::VCosF16: return float16Unary(inst, IrOpcode::FPCos); case RdnaOpcode::VMin3F16: - return float16Ternary(inst, IrOpcode::FPMinTri32, false, false); + return minMaxF16(inst, IrOpcode::FPMinTri32); case RdnaOpcode::VMax3F16: - return float16Ternary(inst, IrOpcode::FPMaxTri32, false, false); + return minMaxF16(inst, IrOpcode::FPMaxTri32); case RdnaOpcode::VMed3F16: - return float16Ternary(inst, IrOpcode::FPMedTri32, false, false); + return minMaxF16(inst, IrOpcode::FPMedTri32); case RdnaOpcode::VDivFixupF16: return vDivFixupF16(inst); case RdnaOpcode::VFrexpMantF32: @@ -936,37 +945,37 @@ bool TranslationContext::emitVector(const RdnaInstruction& inst) { case RdnaOpcode::VFmaF64: return float64Operation(inst, IrOpcode::FPFma64); case RdnaOpcode::VMinF64: - return float64Operation(inst, IrOpcode::FPMin64); + return ieeeMode ? float64Operation(inst, IrOpcode::FPMin64) : nonIeeeMinMaxF64(inst, IrOpcode::FPMin64); case RdnaOpcode::VMaxF64: - return float64Operation(inst, IrOpcode::FPMax64); + return ieeeMode ? float64Operation(inst, IrOpcode::FPMax64) : nonIeeeMinMaxF64(inst, IrOpcode::FPMax64); case RdnaOpcode::VLdexpF64: - return float64Operation(inst, IrOpcode::FPLdexp64); + return float64Unary(inst, IrOpcode::FPLdexp64); case RdnaOpcode::VTruncF64: - return float64Operation(inst, IrOpcode::FPTrunc64); + return float64Unary(inst, IrOpcode::FPTrunc64); case RdnaOpcode::VCeilF64: - return float64Operation(inst, IrOpcode::FPCeil64); + return float64Unary(inst, IrOpcode::FPCeil64); case RdnaOpcode::VRndneF64: - return float64Operation(inst, IrOpcode::FPRoundEven64); + return float64Unary(inst, IrOpcode::FPRoundEven64); case RdnaOpcode::VFloorF64: - return float64Operation(inst, IrOpcode::FPFloor64); + return float64Unary(inst, IrOpcode::FPFloor64); case RdnaOpcode::VFractF64: - return float64Operation(inst, IrOpcode::FPFract64); + return float64Unary(inst, IrOpcode::FPFract64); case RdnaOpcode::VRcpF64: - return float64Operation(inst, IrOpcode::FPRcp64); + return float64Unary(inst, IrOpcode::FPRcp64); case RdnaOpcode::VRsqF64: - return float64Operation(inst, IrOpcode::FPRsq64); + return float64Unary(inst, IrOpcode::FPRsq64); case RdnaOpcode::VSqrtF64: - return float64Operation(inst, IrOpcode::FPSqrt64); + return float64Unary(inst, IrOpcode::FPSqrt64); case RdnaOpcode::VTrigPreopF64: return float64Operation(inst, IrOpcode::FPTrigPreop64); case RdnaOpcode::VFrexpMantF64: - return float64Operation(inst, IrOpcode::FPFrexpMant64); + return float64Unary(inst, IrOpcode::FPFrexpMant64); case RdnaOpcode::VFrexpExpI32F64: return float64Operation(inst, IrOpcode::FPFrexpExp64); case RdnaOpcode::VCvtF32F64: return float64Operation(inst, IrOpcode::ConvertF32F64); case RdnaOpcode::VCvtF64F32: - return float64Operation(inst, IrOpcode::ConvertF64F32); + return vCvtF64F32(inst); case RdnaOpcode::VCvtF64I32: return float64Operation(inst, IrOpcode::ConvertF64S32); case RdnaOpcode::VCvtF64U32: @@ -1035,7 +1044,7 @@ bool TranslationContext::emitVector(const RdnaInstruction& inst) { case RdnaOpcode::VFmaF32: return floatTernary(inst, roundsProductSeparately(inst) ? IrOpcode::FPMad32 : IrOpcode::FPFma32, false, true); case RdnaOpcode::VMadLegacyF32: - return vFmaLegacyF32(inst); + return vFmaLegacyF32(inst, false); case RdnaOpcode::VMin3F32: return ieeeMode ? ieeeMinMaxF32(inst, IrOpcode::FPMinTri32) : floatTernary(inst, IrOpcode::FPMinTri32, false, false); case RdnaOpcode::VMax3F32: @@ -1113,16 +1122,10 @@ bool TranslationContext::emitVector(const RdnaInstruction& inst) { return simpleInteger(inst, IrOpcode::BitReverse32, IrType::U32, false, false, false); case RdnaOpcode::VFfblB32: return simpleInteger(inst, IrOpcode::FindILsb32, IrType::U32, false, false, false); - case RdnaOpcode::VLshlB32: - return simpleInteger(inst, IrOpcode::ShiftLeftLogical32, IrType::U32, false, true, false); case RdnaOpcode::VLshlrevB32: return simpleInteger(inst, IrOpcode::ShiftLeftLogical32, IrType::U32, true, true, false); - case RdnaOpcode::VLshrB32: - return simpleInteger(inst, IrOpcode::ShiftRightLogical32, IrType::U32, false, true, false); case RdnaOpcode::VLshrrevB32: return simpleInteger(inst, IrOpcode::ShiftRightLogical32, IrType::U32, true, true, false); - case RdnaOpcode::VAshrI32: - return simpleInteger(inst, IrOpcode::ShiftRightArithmetic32, IrType::U32, false, true, false); case RdnaOpcode::VAshrrevI32: return simpleInteger(inst, IrOpcode::ShiftRightArithmetic32, IrType::U32, true, true, false); case RdnaOpcode::VLshlrevB64: diff --git a/docs/dev/ARCHITECTURE.md b/docs/dev/ARCHITECTURE.md index 1a617546a..3713b767c 100644 --- a/docs/dev/ARCHITECTURE.md +++ b/docs/dev/ARCHITECTURE.md @@ -67,3 +67,17 @@ flowchart LR - [`Recompiler.cpp`](../../core/shader/recompiler/Recompiler.cpp) runs the stages in this order. With `ANYPS5_ENABLE_SPIRV_TOOLS`, the SPIR-V is also validated and optimized with SPIRV-Tools. - `ShaderRecompiler::Recompile` keeps compiled variants in memory, and `ShaderDiskCache` stores them on disk so later runs reuse them. +- A compute shader whose LDS plus its lock dword exceeds the host's `maxComputeSharedMemorySize` is not given a `Workgroup` array. `Recompile` sets `ShaderInfo::sharedMemoryBytes` for it and the SPIR-V backend addresses the LDS in a coherent storage buffer at descriptor set 1 (`WorkgroupMemoryDescriptorSet`), with a slice of `RecompileResult::workgroupMemoryDwords` per workgroup, indexed by `WorkgroupID` and `NumWorkGroups`. `VulkanDevice` keeps one such buffer, sized from the dispatch dimensions, and binds it after the resource set; programs that fit keep using shared memory. +- `Driver::RegisterShader` splits shader preparation in two. `PlanRegistered` runs on the registering guest thread: it validates the registers and entry point, decodes the code and builds the recompile request, so invalid registrations still fail there. The RDNA to IR translation and resource plan (`PrepareShader`) run on a background pool of `hardware_concurrency() / 2` detached threads (at least 2, 64 MiB stacks). +- The snapshot is published as pending. `SourceHandleFor`, `InvocationFor` and `ShaderPreparationTransaction::Edit` / `Read` wait for the pending job before reading `PreparedShaders`, and rethrow its failure. `APS5_SYNC_SHADER_PREPARE=1` prepares on the registering thread instead. +- A draw's `Graphics::Pipeline` comes from `CachedPipeline`. When the device has `VK_EXT_graphics_pipeline_library` with fast linking, `VK_EXT_extended_dynamic_state` and `VK_KHR_dynamic_rendering`, draws render with `vkCmdBeginRenderingKHR` instead of render passes and the pipeline is linked from four libraries (vertex input, pre-rasterization shaders, fragment shader, fragment output) kept per device in [`PipelineLibrary.cpp`](../../core/libs/prx/libSceAgcDriver/Graphics/src/PipelineLibrary.cpp). A library built for one pipeline is reused by the next: another blend, vertex layout or target format builds only that part (the shader libraries name no attachment format), and cull mode, front face, depth and stencil state are dynamic, so changing them only links. The fast link is what the draw waits for; a background thread then links the same libraries with link-time optimization and the pipeline binds that one from the next draw on. Push constants then use a 256-byte block with a 128-byte slot per side: the pre-rasterization stages stack from byte 0, the pixel shader always starts at byte 128 (`StagePushOffset`), so its library does not depend on how much push data its vertex shader uses. Pipelines with per-face depth bias, rect lists, geometry shaders or a stage without a variant id are created whole, still with dynamic rendering; devices without the extensions (or with `APS5_NO_GPL=1`) keep render passes and whole pipelines. + +### Shader MODE register access + +`GuestContext::floatMode` carries the initial shader mode through `Recompile` and `TranslateOptions` into each block's `TranslationContext`. `s_getreg_b32` reads confined to MODE bits 0–3 return the selected initial rounding bits in the destination's low bits. Bits 0–1 select f32 rounding; bits 2–3 select f16/f64 rounding. Missing mode metadata retains the legacy round-to-nearest-even assumption. Reading other MODE fields or hardware registers throws. + +An immediate MODE write confined to the rounding fields is accepted only when its selected replacement bits equal the initial state. Source bits above the selected width are ignored. `s_round_mode` similarly compares its low four immediate bits with the initial rounding fields. A state-changing write or an SGPR-sourced MODE write throws with its instruction and program counter. Runtime transitions require control-flow-aware mode tracking and matching arithmetic lowering. Writes to other hardware registers retain their existing behavior and remain technical debt. + +The register layout and instruction contract follow [AMD RDNA2 ISA sections 5.8 and 6.4, Table 24](https://www.amd.com/content/dam/amd/en/documents/radeon-tech-docs/instruction-set-architectures/rdna2-shader-instruction-set-architecture.pdf), cross-checked against LLVM 20.1.8's [hardware-register encoding](https://github.com/llvm/llvm-project/blob/87f0227cb60147a26a1eeb4fb06e3b505e9c7261/llvm/lib/Target/AMDGPU/Utils/AMDGPUBaseInfo.h) and [MODE state analysis](https://github.com/llvm/llvm-project/blob/87f0227cb60147a26a1eeb4fb06e3b505e9c7261/llvm/lib/Target/AMDGPU/SIModeRegister.cpp). This contract does not establish arithmetic support for every initial mode or console-specific behavior. + +`agc_shader_mode_register` decodes instruction words, builds the control-flow graph and checks production translation without a Vulkan device. It covers rounding subfield reads, masked constant writes, branches, unsupported runtime changes and diagnostics, plus mode propagation through full recompilation. The requirements remain active in Release; CTest bounds execution to 20 seconds. diff --git a/docs/dev/BUILD.md b/docs/dev/BUILD.md index 2b5eea0f9..15edb681c 100644 --- a/docs/dev/BUILD.md +++ b/docs/dev/BUILD.md @@ -10,6 +10,8 @@ cmake --build build-relinker --parallel ctest --test-dir build-relinker --output-on-failure ``` +Python must be found during CMake configuration for Python tests to be registered. List registered tests with `ctest --test-dir build-relinker -N` and look for `input_magic`, a Python-backed relinker test. If automatic discovery fails, add `-DPython3_EXECUTABLE="C:/path/to/python.exe"` to the configure command, using the path to your Python interpreter, then list the tests again. + This mode builds the conversion tool and its tests on Linux, Windows and macOS, including Apple Silicon. macOS uses AppleClang from the Xcode command-line tools; Windows uses the MinGW-w64 toolchain below. The executable is `build-relinker/core/relinker/relinker` (`relinker.exe` on Windows with Ninja). The output remains x86-64 Linux ELF or Windows PE. Converted games need system libraries built for the target OS and a compatible x86-64 host. This mode does not build those libraries or provide macOS game execution. Tests inspect both output formats; execution checks run only on their compatible hosts. @@ -26,7 +28,9 @@ git submodule update --init --recursive - x86-64, Git, CMake 3.22.1 or newer, Ninja, C++20. - Linux: GCC, G++, binutils. SDL's X11 backend requires X11 and Xext development headers (`libx11-dev` and `libxext-dev` on Debian/Ubuntu). +- Fedora: if CMake reports `Could NOT find Threads`, install `glibc-static` and `libstdc++-static` with `sudo dnf install glibc-static libstdc++-static`, then rerun configuration. The project enables static executable linking, and FFmpeg's thread check needs these libraries. - Windows: only MinGW-w64 GCC 15.2.0 (WinLibs `x86_64-ucrt-posix-seh`, release `15.2.0posix-14.0.0-ucrt-r7`) is currently supported. Add its `mingw64/bin` directory to `PATH` before configuring. +- Windows antivirus software can quarantine the executables that the relinker tests create in the temp directory, which fails `optional_plt`, `empty_tls`, `windows_address_space` and `windows_icon`. Set `TEMP` and `TMP` to a directory the scanner excludes before running `ctest`. - FFmpeg binaries are downloaded during configuration unless `FFMPEG_PREBUILT_DIR` is set. With the WinLibs CMake, the download fails with status 60 (`SSL peer certificate or SSH remote key was not OK`) unless `SSL_CERT_FILE` names a CA bundle, for example `C:\Program Files\Git\mingw64\etc\ssl\certs\ca-bundle.crt` from Git for Windows, as in CI. ## Commands @@ -70,6 +74,18 @@ Build configuration parameters: SDL and FreeType settings forced by the root `CMakeLists.txt` cannot be overridden with `-D`. +## Write watch tests + +The macOS write watch can be built and tested separately, without submodules or the prx libraries: + +```sh +cmake -S core/libs/tests/write_watch -B build/write-watch -DCMAKE_BUILD_TYPE=Release -DCMAKE_OSX_ARCHITECTURES=x86_64 +cmake --build build/write-watch +ctest --test-dir build/write-watch --output-on-failure +``` + +With `CMAKE_OSX_ARCHITECTURES=x86_64` Apple silicon runs the tests through Rosetta, which is how relinked titles run; without it they run natively. The full `BUILD_TESTING` build on macOS also registers them. + ## Pipeline statistics Set `APS5_PIPELINE_STATS=1` to capture and print driver statistics for each newly created graphics or compute pipeline. This requires `VK_KHR_pipeline_executable_properties` and `pipelineExecutableInfo`; an unsupported device fails with an error. Statistic names and units are driver-specific. Capturing statistics can increase pipeline compilation cost. The setting is disabled by default. @@ -77,3 +93,15 @@ Set `APS5_PIPELINE_STATS=1` to capture and print driver statistics for each newl ## Shader recompiler The shader recompilation logic in [core/shader/recompiler](../../core/shader/recompiler) is isolated from the rest of the project and is a pure function of its input data, designed for integration into any other project. The current CMake target also includes cache support and links a supplied runtime target, glslang, and optionally SPIRV-Tools. + +## Register context tests + +The x86-64 register capture and restore tests can be built separately on Linux and macOS, without submodules or the prx libraries. On macOS: + +```sh +cmake -S core/libs/tests/register_context -B build/register-context -DCMAKE_BUILD_TYPE=Release -DCMAKE_OSX_ARCHITECTURES=x86_64 +cmake --build build/register-context +ctest --test-dir build/register-context --output-on-failure +``` + +On Linux x86-64, omit `CMAKE_OSX_ARCHITECTURES`; the full `BUILD_TESTING` build also registers these tests. Apple silicon runs the x86-64 tests through Rosetta. They cover the unwinder's register transfer, not the complete macOS runtime or guest exception handling. diff --git a/docs/dev/HW_ORACLE.md b/docs/dev/HW_ORACLE.md index 464033fa2..e48f86ed1 100644 --- a/docs/dev/HW_ORACLE.md +++ b/docs/dev/HW_ORACLE.md @@ -26,9 +26,12 @@ from the GPU (`HW_ORACLE_TARGET` overrides it). - `v4`-`v7` hold the row's 4 input dwords. `v10`-`v25` start at 0 and are stored as the result (16 dwords). - Leave `v0` (lane id), `v1` (input offset), `v2` (output offset) and `s[4:7]` (input and output addresses) unchanged. -- 4 KiB of LDS. Rows run in workgroups of up to 1024 lanes, padded to whole waves with zero rows. +- 4 KiB of LDS, or the group segment size `--lds` (Python `lds`) gives in bytes, up to 64 KiB. Rows run in workgroups of up to + 1024 lanes, padded to whole waves with zero rows. - Bytes passed as `extra` follow the rows of each dispatch, at `s[4:5] + 16 * rows`: buffer contents, texels, etc. - Build buffer and image descriptors in SGPRs from that address. + Build buffer and image descriptors in SGPRs from that address. A kernel that needs more than 4 input dwords per lane + takes the rest from there too, for example `v_lshlrev_b32 v40, 3, v0` / `v_add_nc_u32 v40, 16 * rows, v40` / + `global_load_dwordx2 v[8:9], v40, s[4:5]` for two more dwords per lane, with `rows` the padded count. Every run must explicitly set all floating-point controls, including runs of integer instructions. The CLI flags use hyphens; Python keywords use underscores. Record these settings alongside the GPU and measured results. @@ -39,7 +42,7 @@ use hyphens; Python keywords use underscores. Record these settings alongside th | `dx10_clamp` | 0 or 1: DX10 clamp disabled or enabled | | `denorm32`, `denorm16` | 0: flush input/output; 1: preserve input, flush output; 2: flush input, preserve output; 3: preserve both | | `round32`, `round16` | 0: nearest even; 1: toward +infinity; 2: toward -infinity; 3: toward zero | -| `fp16_overflow` | 0: overflow to infinity; 1: clamp computed overflow to the largest finite value (infinite inputs and division by zero still produce infinity) | +| `fp16_overflow` | 0: overflow to infinity; 1: clamp computed overflow to the largest finite value (an infinite source operand still produces infinity, as does `v_div_fixup_f16` for a zero denominator or an infinite numerator, but not for an infinite quotient; `v_rcp_f16`, `v_rsq_f16` and `v_log_f16` of zero are clamped) | `denorm16` and `round16` control both f16 and f64. These values follow the [LLVM AMDGPU kernel descriptor documentation](https://llvm.org/docs/AMDGPUUsage.html#amdhsa-kernel-descriptor). diff --git a/docs/dev/PROGRESS.md b/docs/dev/PROGRESS.md index 36d9a1e08..03517e712 100644 --- a/docs/dev/PROGRESS.md +++ b/docs/dev/PROGRESS.md @@ -6,10 +6,12 @@ The scanner reads `APS5_VABI` function definitions under `core/libs/prx//`, excluding `tests` subdirectories and their descendants. Test callbacks do not contribute to library counts. A definition is pending when its body calls `NotImplemented_nid_no_patch`, directly or through a recognized local stub wrapper. Other definitions are counted as implemented. These counts describe declared functions, not every firmware export, verified ABI compatibility or playable games. Silent behavior and unverified assumptions remain in [Technical debt](TechnicalDebt.md). -A library with no scanned definitions can include a sibling library's source builder. The scanner recognizes literal `include(${CMAKE_CURRENT_SOURCE_DIR}/..//.cmake)` references, including quoted paths. It does not evaluate arbitrary CMake expressions. +A library with no scanned definitions can include a sibling library's source builder. The scanner recognizes literal `include(${CMAKE_CURRENT_SOURCE_DIR}/..//.cmake)` references, including quoted paths. It does not evaluate arbitrary CMake expressions. It also recognizes a top-level `.cpp` that includes another library's source with `#include "prx//.cpp"`. For these wrappers, the JSON group adds `shared_sources` with the source library's directory name. Its own counts and name lists remain empty. The HTML table links to the source library instead of showing `0/0`; the source library's map tooltip lists its wrappers. Global totals count the definitions at the source library once. +A library with no scanned definitions and no shared sources, such as one whose `Export.cpp` only has `APS5_DUMMY_FUN` because its functions are implemented in another library, is listed as having no exports instead of `0/0`. It is left out of the map. + ## Comparisons Comparisons identify functions by library and name. When a head group becomes a shared-source wrapper, its previous definitions are compared under the source library's identity. Removing a duplicate implementation is reported as sharing sources, without reporting the retained functions as removed. diff --git a/docs/dev/TechnicalDebt.md b/docs/dev/TechnicalDebt.md index 8f2178189..4656d0fbb 100644 --- a/docs/dev/TechnicalDebt.md +++ b/docs/dev/TechnicalDebt.md @@ -7,6 +7,7 @@ - libgcc_s_seh-1.dll - libstdc++-6.dll - libwinpthread-1.dll +- winpthreads initializes a statically initialized rwlock on its first lock and fails a concurrent first lock with `EINVAL`, which `std::shared_mutex` and `std::shared_timed_mutex` ignore, so that lock does not exclude. Guest rwlocks ([Pthread.hpp](../../core/libs/prx/libkernel/Pthread/include/Pthread.hpp)) are locked once when created. The host `std::shared_mutex` objects in [Recompiler.cpp](../../core/shader/recompiler/Recompiler.cpp) and [GuestWriteWatch.cpp](../../core/libs/prx/libc/src/GuestWriteWatch.cpp) are not, and can still race on their first contended use. ### Silent stubs @@ -23,24 +24,31 @@ Throughout the project, every function at every stage either **does exactly what - [libSceNpCommerce](../../core/libs/prx/libSceNpCommerce/Export.cpp) - the PS Store icon show/hide calls and `sceNpCommerceSetPsStoreIconLayout` do nothing - [libSceUsbd](../../core/libs/prx/libSceUsbd/Export.cpp) implements only the no-device path: `sceUsbdInit` and `sceUsbdExit` succeed without a host USB context, the device list is always empty and event handling waits for its timeout; calls that need a device throw - [libSceSystemService](../../core/libs/prx/libSceSystemService/Export.cpp) - `sceSystemServicePowerTick`, `sceSystemServiceReportAbnormalTermination`, `sceSystemServiceDisableMusicPlayer`, `sceSystemServiceReenableMusicPlayer`, `sceSystemServiceDisableMediaPlay` and `sceSystemServiceReenableMediaPlay` return success and do nothing: the host has no auto power-off timer to reset, no system crash reporter to notify, no system music player to pause or resume and no system media playback to disable. PPSA01521 calls `sceSystemServiceDisableMediaPlay` during start-up, and its wrappers for both pass no arguments +- [sceSystemServiceLaunchWebBrowser](../../core/libs/prx/libSceSystemService/Export.cpp) (libSceSystemService) - AnyPS5 launches no web browser, so the call fails and nothing is shown. RommPS PPSA76677 then tells the player to scan its QR code instead - [libSceBluetoothHid](../../core/libs/prx/libSceBluetoothHid/Export.cpp) implements only the no-device path: initialization and callback and device registration succeed, the callback is never called because no device connects, and calls that need a connected device throw - [sceVideoOutOpen](../../core/libs/prx/libSceVideoOut/src/Output.cpp) validates the priority and CPU affinity that the open param requests for the VideoOut service thread but does not apply them: the port's present and vblank threads are host threads, and guest priorities and affinities do not reach host scheduling -- Without `VK_KHR_fragment_shader_barycentric` the shader recompiler [does not load the barycentric VGPRs](../../core/shader/recompiler/Translation/src/InstructionTranslator.cpp): [`v_interp_p1_f32` is dropped and `v_interp_p2_f32` reads the attribute Vulkan interpolated](../../core/shader/recompiler/Translation/src/AttributeInstructions.cpp), so pixel shaders that use the barycentrics for anything other than `v_interp` get wrong values. -- [libSceAudio3d](../../core/libs/prx/libSceAudio3d/Export.cpp) - the port produces no sound: objects can be reserved but the library has no bed exports or `sceAudio3dObjectSetAttributes` to give them audio, so `sceAudio3dPortPush` only paces the queue at `granularity` samples per frame at 48 kHz. `sceAudio3dPortSetAttribute` accepts the late reverb level and the downmix spread attributes and ignores them +- [sceVideoOutAddVrrStatusFlagsPrivilege](../../core/libs/prx/libSceVideoOut/src/Output.cpp) returns 0 and does nothing: the host output has no VRR state to report. The name comes from an assert message in a title that requires `SCE_VIDEO_OUT_OK`; the NID hashes to it, the arguments are unknown (it is called without any) +- [libSceAudio3d](../../core/libs/prx/libSceAudio3d/Export.cpp) - the port produces no sound: objects can be reserved and `sceAudio3dObjectSetAttributes` accepts their attributes, but the library has no bed exports or mixing to give them audio, the attributes have no effect and `sceAudio3dPortPush` only paces the queue at `granularity` samples per frame at 48 kHz. `sceAudio3dPortSetAttribute` accepts the late reverb level and the downmix spread attributes and ignores them. `sceAudio3dObjectSetAttributes` validates the port, the reserved object and the attribute array (an object id that was never reserved returns the invalid-object error, while shadPS4 returns success for it and the error only for a released object; neither is measured on a console), treating the reset-state attribute as valid with a null value and requiring a value for the others; the attribute identifiers and the value of each are not interpreted. `sceAudio3dPortFlush` drops the frames queued by `sceAudio3dPortAdvance` instead of mixing and submitting them to an output, as there is no output - [libSceTextToSpeech2](../../core/libs/prx/libSceTextToSpeech2/Export.cpp) has no speech engine: `sceTextToSpeech2Initialize` fails and the other exports throw. PPSA01317 skips its narration set-up when initialization fails - [libSceAvPlayer](../../core/libs/prx/libSceAvPlayer/Export.cpp): `sceAvPlayerSetLogCallback` accepts a callback that is never called, as the player produces no log messages, and `sceAvPlayerSetAvailableBandwidth` has no effect, as it governs HLS sources, which `sceAvPlayerAddSource` does not implement. - [libSceHttp2](../../core/libs/prx/libSceHttp2/Export.cpp) - no request reaches the network, so `sceHttp2SetMinSslVersion` has no TLS connection to restrict and returns success for any `id`, like `sceHttp2SslEnableOption` (prosper returns the invalid-id error for an unknown one) - [libSceNpTrophy2](../../core/libs/prx/libSceNpTrophy2/src/GameInfo.cpp): `sceNpTrophy2RegisterUnlockCallback` accepts a callback that is never called, as no trophy is ever unlocked, and `sceNpTrophy2UnregisterUnlockCallback` only returns success. - [sceAvPlayerSetTrickSpeed](../../core/libs/prx/libSceAvPlayer/src/Source.cpp) (libSceAvPlayer) with a negative speed runs the clock backwards but delivers no frames; when a forward speed is set again, playback resumes from the rewound time. +- [sceRudpActivate](../../core/libs/prx/libSceRudp/Export.cpp) (libSceRudp) - there is no reliable-UDP stack on the host to activate, so it returns success without doing anything, as [shadPS4](https://github.com/shadps4-emu/shadPS4/blob/main/src/core/libraries/rudp/rudp.cpp) does for its own `sceRudpActivate`. PPSA01325 calls it with no arguments during start-up; what the console activates, and whether it reports an error before `sceRudpInit`, is unverified - [ulobjmgr](../../core/libs/prx/ulobjmgr/Export.cpp) registers no object: `_sceUlobjmgrRegisterObject` always hands out id 0 and `_sceUlobjmgrUnregisterObject` releases nothing, as shadPS4 does - [libSceHttp](../../core/libs/prx/libSceHttp/Export.cpp) - no request reaches the network, so `sceHttpSetResponseHeaderMaxSize` has no response header to limit and `sceHttpRedirectCacheFlush` no redirect to forget; `sceHttpsUnloadCert` returns success like `sceHttpsLoadCert`, which keeps no certificate; `sceHttpSetRedirectCallback` and `sceHttpSetCookieRecvCallback` accept callbacks that are never called, `sceHttpsSetSslVersion` has no TLS connection to configure and `sceHttpsGetSslError` reports error 0 with detail 0; `sceHttpSetAuthInfoCallback` accepts a callback that is never called, as no server asks for authentication - [sceFontGlyphDefineAttribute](../../core/libs/prx/libSceFont/src/Render.cpp) (libSceFont) checks the glyph and ignores the attribute and its value, whose meaning is unknown. PPSA01325 sets attribute 0x11 to 0 on a glyph from `sceFontGenerateCharGlyph` that it deletes unused, and draws the character with `sceFontRenderCharGlyphImageHorizontal` ### Unknown function info +- [sceVideoOutGetOutputStatusWithVrr](../../core/libs/prx/libSceVideoOut/Unimplemented.cpp) (libSceVideoOut, NID `gWT7X8H0bYs`) - the name hashes to the imported NID. PPSA26344 version 02.002.000 passes a VideoOut handle and an output pointer at three call sites. It compares a 32-bit value at byte 4 with 2, tests bit 0x10 at byte 16, and, when set, compares low 16-bit values at bytes 48, 50 and 52 with 48, 60 and 120. The full output layout, size, field meanings and console errors are unknown. Every call throws without reading or writing caller memory. +- [getuid, geteuid, getgid, getegid and issetugid](../../core/libs/prx/libkernel/System/src/Process.cpp) (libkernel) report the root identity (0) and a process that never changed credentials (`issetugid` 0), as a jailbroken process has; the host user and group are not exposed to the guest. The values a title sees on a console are unverified +- [Packed pixel ancillary word](../../core/shader/recompiler/Optimization/src/ConstantFolder.cpp) (shader recompiler) - the sample index at bits 8-11 and the render target index at bits 16-28 follow #295, and their equality with the SampleId and Layer builtins is inferred, not measured. A value passed through a select, move or phi is rebuilt from those two fields with bits 0-7, 12-15 and 29-31 zero. That is exact only because every read of the word must be an extract inside one modelled field; any other read throws. The other bits are unmodelled and their RDNA 2 layout is unknown: the RDNA 3.5 ISA is reported to give prim type at bits 0-1, VRS rates at bits 2-5 and eye id at bit 7, which was not checked against RDNA 2 +- [sceAudioPropagationSourceSetAudioPath](../../core/libs/prx/libSceAudioPropagation/Export.cpp) (libSceAudioPropagation, NID `tKSmk2JsMAA`) - `(uint64_t path, const void* data, float gain)` follows [SharpProspero](https://github.com/SvenGDK/SharpProspero/blob/5dfdb4b0b166e83dc9ae275001afb695f12c7174/src/SharpProspero/Interop/Audio/AudioPropagation.cs) and SAROS's call, which passes a handle from `sceAudioPropagationSourceGetAudioPath`, path data and gain 1.0. Path updates are unsupported and every call throws without reading or writing caller memory. The no-propagation model yields no path handles; path allocation, ownership, rendering and console error codes remain unknown. +- [Color comparison descriptor forms](../../core/shader/recompiler/Optimization/src/ResourceMaterializer.cpp): emulated color comparison supports UNORM, SNORM and float formats for 2D, 2D-array and cube views. sRGB formats, anisotropic filters and bilinear cube comparisons without `DISABLE_CUBE_WRAP` are refused; cube-edge sampling behavior and console coverage on PS5's gfx1013 remain unverified. - [sceNpEntitlementAccessGetEntitlementKey](../../core/libs/prx/libSceNpEntitlementAccess/Export.cpp) - the three-argument signature, 16-byte key and errors `0x817D0002`/`0x817D0007` follow [KytyPS5](https://github.com/KytyPS5/KytyPS5/blob/main/src/libs/libNet.cpp); PPSA01521 passes a service label, entitlement label and output pointer. Only the no-key path is implemented: local entitlement labels carry no keys, and the output is untouched on failure. Console error precedence and key provisioning are unverified - [sceNpEntitlementAccess{Request,Poll}{Unified,Service}EntitlementInfoList](../../core/libs/prx/libSceNpEntitlementAccess/Export.cpp) - signatures unknown; no reference implements them, so they are declared without parameters. Like the consume requests in the same library, they report the signed-out NP state with `SCE_NP_ERROR_SIGNED_OUT` (`0x80550006`) and write nothing. PPSA15657 calls the unified request during start-up and continues past the error. Whether the console fails at the request or at the poll when signed out is unverified -- [sceRudpGetStatus](../../core/libs/prx/libSceRudp/Export.cpp) - PPSA01521 passes an output pointer and byte count `0xF8`; the status layout, size rules and error behavior are unknown. The export throws without writing output, including after initialization. [shadPS4](https://github.com/shadps4-emu/shadPS4/blob/main/src/core/libraries/rudp/rudp.cpp) declares it without arguments and returns success without writing status +- [sceRudpGetStatus](../../core/libs/prx/libSceRudp/Export.cpp) - PPSA01521 passes an output pointer and byte count `0xF8`; the status layout and size rules are unknown, so the buffer is zeroed like `sceRudpGetContextStatus` (state 0 = idle, no connections) and the size is not validated. It returns `RUDP_ERROR_NOT_INITIALIZED` before initialization and 0 otherwise, as `sceRudpGetContextStatus` does. [shadPS4](https://github.com/shadps4-emu/shadPS4/blob/main/src/core/libraries/rudp/rudp.cpp) declares it without arguments and returns success without writing status - [sceHttpSetCookieEnabled](../../core/libs/prx/libSceHttp/Export.cpp) - the two-argument signature and 0/1 enable values follow [VitaSDK](https://github.com/vitasdk/vita-headers/blob/master/include/psp2/net/http.h) and [shadPS4](https://github.com/shadps4-emu/shadPS4/blob/main/src/core/libraries/network/http.cpp); PPSA01521 passes 0 to disable cookies. Disabling succeeds because the offline HTTP backend has no cookie engine; enabling throws. Invalid enable values use the same error as `sceHttpSetInflateGZIPEnabled`. Like the existing offline HTTP setters, the handle is not validated; requests still fail with the network error. Console behavior is unverified - [sceHmd2Initialize](../../core/libs/prx/libSceHmd2/Export.cpp) (libSceHmd2) - the error the console returns when the PS VR2 runtime is unavailable is unknown; it returns libSceHmd's `UNSUPPORTED_FEATURE` (0x81110016). PPSA01317 passes a pointer to 16 zeroed bytes and only tests the result for non-zero @@ -48,29 +56,32 @@ Throughout the project, every function at every stage either **does exactly what - [libSceMbus](../../core/libs/prx/libSceMbus/Export.cpp) (`libSceDeviceService`) - signatures taken from PPSA01317, which calls `sceDeviceServiceInitialize(3, 0)`, polls `sceDeviceServiceGetEventState(1)` and queries again when bit 0 or 1 is set, and calls `sceDeviceServiceQueryDeviceInfo_(0x7001, 0, 0, infos, 1, &count, &unknown, 0x70)`, treating a negative result as an error and `count` as the number of entries. The host has no devices: the event state is 0 and a query returns 0 with `count` 0 - [NGS2 ATRAC9 waveform loops](../../core/libs/prx/libSceNgs2.native/src/Atrac9.cpp) accepts one integer forward `smpl` loop. Inclusive RIFF endpoints and encoder-delay adjustment follow [vgmstream riff.c](https://github.com/vgmstream/vgmstream/blob/7dc938fa2f210943b37c7b6511852b516ef432ab/src/meta/riff.c). PPSA03747 supplies a 60-byte sampler chunk with one loop, `samplerData` 24, start 256, inclusive end 299224 and delay 256 for 298969 audible samples. The chunk size and loop count bound the records; `samplerData` is not interpreted. Output positions are assumed to use audible samples and an exclusive end. Blocks split into an optional intro, a repeated range and an optional tail; zero play count means infinite repeats, positive play count includes the first pass. ATRAC9 ranges include a preceding codec frame for decoder history. The output block layout, repeat-count mapping and error codes are unverified on hardware. Multiple, reverse, alternating and fractional loops throw. - [NGS2 submixer UserFx](../../core/libs/prx/libSceNgs2.native/src/Custom.cpp) (control `0x20000004`) invokes the title's callback after input mixing on every playing grain, including silence. The 40-byte control and 48-byte process context follow [shadPS4](https://github.com/shadps4-emu/shadPS4/blob/945dbc3cc3eee80ac3e053b438502ed936fa6bb2/src/core/libraries/ngs2/ngs2_submixer.h). PPSA03747 installs a peak-meter callback; its code confirms the channel pointers, user data 0, channel count, grain size and sample-rate offsets and returns 0. Other user data words are preserved. Flags are assumed to be 1 after registration/restart and 0 thereafter, following UserFx2's first-process convention; this callback does not inspect them. Null handlers disable the effect, setup clears it, and nonzero callback results throw. These lifecycle and error semantics are unverified on console hardware. -- PPSA01341 imports declared without parameters, signatures unknown: [sceAgcSetSemaphoreMemory](../../core/libs/prx/libSceAgc/Unimplemented.cpp), [sceAgcDriverRegisterMultipleResources](../../core/libs/prx/libSceAgcDriver/Unimplemented.cpp). Names from the shadPS4 aerolib NID list +- [NGS2 ATRAC9 streaming](../../core/libs/prx/libSceNgs2.native/src/Atrac9.cpp) keeps waveform duration and skip counts across compressed input blocks. The shared `APPEND` flag (2), called `ONLY_DATA` in the compressed-data reference, ignores an ATRAC9 continuation block's sample, skip and repeat counts, following [KytyPS5](https://github.com/KytyPS5/KytyPS5/blob/bb98f26433fa9e711c219aac1711fe8ecba5bf3a/src/libs/ngs2.cpp); PPSA03747 confirms a 262044-byte initial buffer declaring 11310532 samples, followed by 16384-byte buffers with flags 3. Incomplete superframes are retained until more input arrives. Input-block completion callbacks allow refills; the final callback of a logical waveform waits for its buffered PCM to drain, preserving the existing complete-block repeat behaviour. This buffering, callback timing, decoded-byte accounting and starvation silence are unverified on hardware. A closed ATRAC9 stream ending before its declared samples throws; fragmented ATRAC9 waveforms with repeats are unsupported. PCM streaming retains its existing append validation and early-close behaviour. +- PPSA01341 imports declared without parameters, signatures unknown: [sceAgcSetSemaphoreMemory](../../core/libs/prx/libSceAgc/Unimplemented.cpp). Name from the shadPS4 aerolib NID list - [sceAppContentDownloadDataGetAvailableSpaceKb](../../core/libs/prx/libSceAppContent/Export.cpp) (libSceAppContent) - a title whose `param.json` declares `downloadDataSize` 0 gets 0 KB and OK; the console's behaviour for a title without a download data area is unverified (it may return an error instead). KytyPS5 does not special-case it, and shadPS4 stubs 1048576 KB -- [7CxI50-xlCk, pMxXhNozUX8](../../core/libs/prx/libSceNpPartner001/Export.cpp) (libSceNpPartner001) - unknown names and signatures, imported by PPSA23566; declared without parameters +- [sceNpEAAccessInitialize, sceNpEAAccessTerminate](../../core/libs/prx/libSceNpPartner001/Export.cpp) (libSceNpPartner001) - names and the no-argument signatures follow [shadPS4](https://github.com/shadps4-emu/shadPS4/blob/main/src/core/libraries/np/np_partner.cpp); imported by PPSA23566, not confirmed on a PS5 title - [libSceBluetoothHid](../../core/libs/prx/libSceBluetoothHid/Export.cpp) (libSceBluetoothHid) - signatures taken from PPSA01317: `sceBluetoothHidInit()` and `sceBluetoothHidUnregisterCallback()` without arguments, `sceBluetoothHidRegisterCallback(callback, 0, param)` with a 16-byte param, and `sceBluetoothHidRegisterDevice` / `sceBluetoothHidUnregisterDevice(vendorId, productId)`; the report and device name exports are declared without parameters - [sceAgcWaitRegMemPatchMask](../../core/libs/prx/libSceAgc/Patch/src/WaitRegMem.cpp) (libSceAgc) - follows `sceAgcWaitRegMemPatchReference`: the mask is taken as a 32-bit value and written to the low mask word of a 32- or 64-bit wait, as [OpenAGC](https://github.com/OpenAGC/OpenAGC/blob/main/src/game_compat.c) does; [sharpemu](https://github.com/sharpemu/sharpemu/blob/main/src/SharpEmu.Libs/Agc/AgcExports.PacketPatching.cs) writes both words of a 64-bit wait's mask. A mask above 32 bits throws - [scePsmlMfsr2CreateContext, scePsmlMfsr2CreateSharedResources, scePsmlMfsr2GetContextInitRequirement, scePsmlMfsr2GetDispatchPackets, scePsmlMfsr2GetDispatchPacketsSizeInDwords, scePsmlMfsr2GetSharedResourcesInitRequirement, scePsmlMfsr2Init, scePsmlMfsr2ReleaseContext, scePsmlMfsr2ReleaseSharedResources](../../core/libs/prx/libScePsml_debug/Export.cpp) (libScePsml_mfsr2) - unknown signatures, imported by PPSA23566; declared without parameters - [sceAgcDcbSetCfRegisterDirect, sceAgcDcbSetCfRegisterRangeDirect](../../core/libs/prx/libSceAgc/DcbState/src/Registers.cpp) (libSceAgc) - signatures follow [OpenAGC](https://github.com/OpenAGC/OpenAGC/blob/main/src/game_compat.c) and [SharpProspero](https://github.com/SvenGDK/SharpProspero/blob/main/src/SharpProspero/Interop/Agc/SceAgc.cs), which agree; the SET_CONFIG_REG (0x68) packet is OpenAGC's. The driver does not execute SET_CONFIG_REG, so a submitted packet throws - [scePadOpenExt](../../core/libs/prx/libScePad/Export.cpp) (libScePad) - only special ports (type 2) are handled, and they return `DEVICE_NOT_CONNECTED` (0x80920007) because the host has no special controller; whether the console opens a handle and reports the device as disconnected later is unknown. PPSA01317 passes a 16-byte param that names the device by vendor and product id (`b7 0e 04 0e ...` for 0x0eb7:0x0e04) and skips the user on any negative result. Other port types throw - PPSA23566 imports declared without parameters, signatures unknown: [sceAudioOut2UserGetSupportedAttributes](../../core/libs/prx/libSceAudioOut/src/AudioOut2System.cpp), [sceConvertKeycodeGetCharacterFromKeyboardData](../../core/libs/prx/libSceConvertKeycode/Export.cpp), [sce::Json::MemAllocator::notifyError](../../core/libs/prx/libSceJson2/Export.cpp), [sceNetResolverAbort](../../core/libs/prx/libSceNet/Export.cpp), [sceNpCommerceDialogOpen2](../../core/libs/prx/libSceNpCommerce/Export.cpp), [sceNpEntitlementAccessGetPftFlag](../../core/libs/prx/libSceNpEntitlementAccess/Export.cpp), [sceProprietaryVoiceChatHelper{Initialize,Terminate,SetVoiceChatState,GetVoiceChatUsageState}](../../core/libs/prx/libSceProprietaryVoiceChatHelperPrx/Export.cpp), [sceVoice{GetResourceInfo,ResetPort,EnableChat,DisableChat}](../../core/libs/prx/libSceVoice/Export.cpp). Names from the shadPS4 aerolib NID list -- PPSA01325 imports declared without parameters, signature unknown: [sceRudpActivate](../../core/libs/prx/libSceRudp/Export.cpp). Name from the shadPS4 aerolib NID list - [sceVoiceChatRequestCreateVoiceChatGroup, sceVoiceChatRequestJoinVoiceChatGroup, sceVoiceChatRequestLeaveVoiceChatGroup, sceVoiceChatRequestDeleteVoiceChatGroup](../../core/libs/prx/libSceVoiceChat/Export.cpp) (libSceVoiceChat) - assumed to take a request id and two pointers like the other `sceVoiceChatRequest*` calls; offline, every argument is ignored and `SIGNED_OUT` is returned, as for those - [sceAvPlayerAddSource](../../core/libs/prx/libSceAvPlayer/src/Source.cpp) (libSceAvPlayer) - requests up to 4 additional decode-ahead video framebuffers from the memory replacement beyond `num_output_video_framebuffers` when memory is available; the console's internal buffering behaviour is unknown - [sceTextToSpeech2Initialize](../../core/libs/prx/libSceTextToSpeech2/Export.cpp) (libSceTextToSpeech2) - the console's error for an unavailable speech engine is unknown; it returns the kernel's `EOPNOTSUPP` (0x8002002D). PPSA01317 passes a param that starts with 0x2000000 and only tests the result for non-zero; the other exports it imports are declared without parameters +- [sceSystemServiceLaunchWebBrowser](../../core/libs/prx/libSceSystemService/Export.cpp) (libSceSystemService) - the console's error for an unavailable web browser is unknown; it returns the kernel's `EOPNOTSUPP` (0x8002002D) for any arguments. The (uri, param) signature follows [RommPS](https://github.com/s0liton/RommPS/blob/754d4d60f25938a5136b9e2efcd074313c4386c5/platform/ps5/app/rommps/examples/rommps/kit/credits.cpp#L86-L95), which passes a URL and a null param and only tests the result for zero; the param layout and the console's argument checks are unknown - [sceAgcDcbClearState](../../core/libs/prx/libSceAgc/DcbFlow/src/Control.cpp) (libSceAgc) - the (buffer, command) signature follows [SharpProspero](https://github.com/SvenGDK/SharpProspero/blob/main/src/SharpProspero/Interop/Agc/SceAgc.cs) and [OpenAGC](https://github.com/OpenAGC/OpenAGC/blob/main/src/game_compat.c), and the 2-dword CLEAR_STATE (0x12) packet is OpenAGC's; OpenAGC masks the command to 4 bits, here a command above 0xf throws, as the driver rejects that payload - [Command buffer cursors](../../core/libs/prx/libSceAgc/Command/src/Packet.cpp) (libSceAgc) - a null `cursor_down` is taken as a buffer without downward allocations, whose limit is `top`. PPSA16064 builds one-packet buffers of 8 dwords that way; the console's handling is unconfirmed - [sceVideoOutRegisterBuffers2](../../core/libs/prx/libSceVideoOut/src/Buffer.cpp) (libSceVideoOut) - buffer option 8 (STRICT_COLORIMETRY) is presented as option 0, with the stored values unchanged; what the console changes for it is unknown. Only options 0 and 8 are accepted - [sceAgcGetRegisterDefaultsInternal](../../core/libs/prx/libSceAgc/Misc/src/RegisterDefaults.cpp) (libSceAgc) - returns the version 0 internal table, as `sceAgcGetRegisterDefaults` returns the version 0 public one; [OpenAGC](https://github.com/OpenAGC/OpenAGC/blob/main/src/game_compat.c) returns the table of the version passed to `sceAgcInit` instead, and null before it - [sceVideoOutRegisterBuffers2](../../core/libs/prx/libSceVideoOut/src/Buffer.cpp) (libSceVideoOut) - DCC buffers (category 1): `dcc_control` is taken as the CB_DCC_CONTROL block layout, only bits 0x10026c are accepted; `dcc_cb_register_clear_color` as the register-clear texel, only 32-bit values are presented -- [sceVideoOutIsOutputSupported](../../core/libs/prx/libSceVideoOut/src/Output.cpp) (libSceVideoOut) - output mode encoding unknown: every mode except the default reports not supported (0), and `sceVideoOutConfigureOutput` returns `VIDEO_OUT_ERROR_UNAVAILABLE_OUTPUT_MODE` for it, instead of returning `VIDEO_OUT_ERROR_UNSUPPORTED_OUTPUT_MODE` for values the console does not define. PPSA12544 probes `0xd000000a` and selects 7680 pixels when it is supported, 3840 otherwise +- [sceVideoOutIsOutputSupported](../../core/libs/prx/libSceVideoOut/src/Output.cpp) (libSceVideoOut) - only the default mode (1) is reported as supported. The console answers per display and per title: with a 120 Hz TV it also answers 1 for mode 15 (119.88 Hz), and [PS5_Vulkan](https://github.com/mihawk-99/PS5_Vulkan/blob/5b5e4fc2d80fdb67a4f61ba9e6a8a424026bb8a5/docs/HARDWARE_FINDINGS.md) measured that `sceVideoOutConfigureOutput` accepts 15 only from a title whose param.json declares high-frame-rate output (attribute3 0x80040). Modes 4, 7, 8, 12-17 and 19 return `VIDEO_OUT_ERROR_UNSUPPORTED_OUTPUT_MODE` here, as they did on the measured display. Mode 18 throws, because one console answered 0x80290016 and another 0x8029001E. The name `VIDEO_OUT_ERROR_UNKNOWN_OUTPUT_MODE` for 0x8029001E is ours. Word 3 of the output options is accepted with any value, as on the console, and its meaning is unknown. PPSA12544 probes `0xd000000a`, which the console answers with 0x8029001E, so the title selects 3840 pixels +- [sceVideoOutGetResolutionStatus](../../core/libs/prx/libSceVideoOut/src/Output.cpp) (libSceVideoOut) - the 48-byte layout follows the declarations in ps5-opengl and EVO-PLAYER-PS5 (#1578), which agree; not confirmed on a console. `screenSizeInInch` is 0, the value EDID uses for an undefined size, since the host display's size is not read; `flags` is 0, its bits are unknown. The pane equals the full size because `sceVideoOutSetWindowModeMargins` applies no margins - [sceVideoOutOpen](../../core/libs/prx/libSceVideoOut/src/Output.cpp) (libSceVideoOut) - the open param's first word is unknown (PPSA21564 passes 16; it is not the byte size, since the affinity mask is at offset 16); only 16 is accepted. Whether the param continues past offset 24 is also unknown - [s_setkill](../../core/shader/recompiler/RdnaDecoder/src/RdnaScalarOpDecoder.cpp) (core/shader/recompiler) - decoded as `s_endpgm` for any immediate: measured on RDNA2 hardware (gfx1035), a store before it is visible and nothing after it runs ([#771](https://github.com/boykopovar/AnyPS5/pull/771)); whether the KILL status bit it sets is observable differently from `s_endpgm` is unverified - [sceRtcParseDateTime](../../core/libs/prx/libSceRtc/Export.cpp) (libSceRtc) - the console's zone name table is only known from the names measured on a PS5 (firmware 13.60); any other name is assumed to take the military letter offset of its first character, as every measured name outside the table does (`HST`, `MSK`, `XYZ`), so a name the console's table holds but nobody measured gets the wrong offset - [sceAudio3dPortOpen](../../core/libs/prx/libSceAudio3d/Export.cpp) (libSceAudio3d) - buffer modes 0 and 1 (also selected by the 0x10 and 0x18 parameter sizes) and 3 beds depend on an SDK version check in the module ([prosper](https://github.com/mattias800/prosper)) and throw. A repeated `sceAudio3dInitialize` (`NOT_READY` in shadPS4, 0 in prosper) and `sceAudio3dPortAdvance` on a full queue (`NOT_READY` in shadPS4, unhandled in KytyPS5) throw too +- [sceAudiodecDecode](../../core/libs/prx/libSceAudiodec/src/Codecs.cpp) (libSceAudiodec) - a raw M4AAC stream carries no channel configuration, so the decoder tries the mono/stereo configuration and then configurations 3 to 7 within `ui_max_channels` until one decodes the first access unit, and keeps that one. No reference documents how the library picks the layout (shadPS4 maps the channel count straight to the configuration; SharpProspero only describes the maximum). With the stereo configuration a mono raw stream decodes as 2 channels, so 2 is reported, not 1 - [sceAudio3dObjectReserve](../../core/libs/prx/libSceAudio3d/Export.cpp) (libSceAudio3d) - object ids, the `max_objects` limit and the checks of `sceAudio3dObjectReserve` and `sceAudio3dObjectUnreserve` follow [shadPS4](https://github.com/shadps4-emu/shadPS4) (PS4), the only reference with a body: ids count up from 1 per port, skipping 0, 0xFFFFFFFF and ids in use, and a failed reserve writes 0xFFFFFFFF - [s_sethalt](../../core/shader/recompiler/Translation/src/ScalarInstructions.cpp) (core/shader/recompiler) - with SIMM16[0] clear it is a no-op: the RDNA 2 ISA guide describes that value as resuming a halted wave, and a wave that executes it is not halted; measured on a gfx1035 (RDNA 2) wave: `s_sethalt 0` followed by a VALU op completes and gives the normal result. Only SIMM16[0] is read; with it set the instruction throws - [AMPR WriteAddressFromCounterPairOnCompletion](../../core/libs/prx/libSceAmpr/Export.cpp) (libSceAmpr) - the pair of an even counter n is assumed to give counter n in the low 32 bits and n + 1 in the high 32 bits. Wait compare values, wait flush and counter index ranges follow the [ampr_emu](https://github.com/drakmor/ampr_emu) reimplementation; the flush is accepted and has no effect, as commands run in order @@ -92,17 +103,21 @@ Throughout the project, every function at every stage either **does exactly what - [sceFontWritingLineGetOrderingSpace](../../core/libs/prx/libSceFont/src/WritingLine.cpp) (libSceFont) - signature taken from the IDA type database of [Orbital](https://github.com/AlexAltea/orbital/blob/master/tools/ida/db_types.json) and the [SharpProspero](https://github.com/SvenGDK/SharpProspero/blob/main/src/SharpProspero/Interop/Font/SceFont.Engine.cs) bindings, not confirmed on a title. The modelled line applies no spacing, so the head, inline, tail and advance spaces are 0; what spacing the console applies is unknown. A null output throws - [sceSaveDataTransferringMountPs4, sceSaveDataDirNameSearchPs4](../../core/libs/prx/libSceSaveData.native/Export.cpp) (libSceSaveData.native) - arguments assumed to be those of `sceSaveDataTransferringMount` and `sceSaveDataDirNameSearch`: KytyPS5 binds both NIDs to the PS5 functions, and [prosper](https://github.com/mattias800/prosper) captured the same result layout on a title for the search. There is no PS4 save area, so the mount returns `NOT_FOUND` and the search reports zero hits - [sceCoredumpWriteUserString](../../core/libs/prx/libkernel/System/src/Coredump.cpp) and [sceSaveDataConvert, sceSaveDataGetConvertProgress](../../core/libs/prx/libSceSaveData.native/Export.cpp) - imported by PPSA16064, not called during its start-up and added without an implementation; unknown signatures +- [sceSaveDataCancel](../../core/libs/prx/libSceSaveData.native/Export.cpp) (libSceSaveData.native) - imported by PPSA23126, which fails to load without it; added without an implementation because its signature is unknown. Every save data call here finishes before it returns, so nothing is ever in flight to cancel, but what the console returns then is unknown - [sce::Json::Parser::parse](../../core/libs/prx/libSceJson2/Export.cpp) (libSceJson2) - a number that overflows a double (e.g. `1e309`) throws: whether the console rejects the document or stores an infinite real is unknown - [sceUserServiceGetNpAccountId](../../core/libs/prx/libSceUserService/Export.cpp) (libSceUserService) - the account id for a user with no linked PSN account is assumed to be 0 with success; libSceNpManager's `sceNpGetAccountIdA` answers the same question with `SCE_NP_ERROR_SIGNED_OUT` - [sceUserServiceGetUserColor](../../core/libs/prx/libSceUserService/Export.cpp) (libSceUserService) - the default profile colour is assumed to be blue (0) -- [kP2L8t3j-aM](../../core/libs/prx/libSceVideoOut/src/Output.cpp) (libSceVideoOut) - unknown name, signature -- [LibwuIonIBw](../../core/libs/prx/libSceVideoOut/src/Output.cpp) (libSceVideoOut) - unknown name, signature +- [libSceVdecsw](../../core/libs/prx/libSceVdecsw/Export.cpp) (libSceVdecsw) - structure layouts follow shadPS4's `vdecsw.h` and were checked against PPSA05684's movie decoder (`movie-decoder-vdecsw.cpp`), which asserts `pictureCount == 1` at offset 0xb of `OutputInfo` and reads `InputResult` offset 0x10 as the number of frames an access unit produced. The compute, decoder memory and decode input structures and the first 0x28 bytes of the 0x50-byte decoder config are those of libSceVideodec2, and decoding goes through it. The 0x78-byte AVC picture info is filled with Videodec2's leading fields and the cropping fields only. OUTPUT_PENDING (0x81510115), INPUT_QUEUE_EMPTY (0x81510116) and DECODE_PENDING (0x81510117) follow from the codes each call's assert accepts. At most four decoded pictures are queued and the last one is held until `sceVdecswFinalizeDecodeSequence`; the console's queue depth is unknown +- [sceGameLiveStreamingGetCurrentStatus2](../../core/libs/prx/libSceGameLiveStreaming/Export.cpp) (libSceGameLiveStreaming) - the status layout (user id, on-air flag, spectator, text and command message counts, broadcast resolution, 48 reserved bytes) is taken from shadPS4's `OrbisGameLiveStreamingStatus2`; the call reports that no stream is on air, with an invalid user id +- [Depth clear draws](../../core/libs/prx/libSceAgcDriver/Graphics/src/State.cpp) (AGC driver) - a draw with `DB_RENDER_CONTROL.DEPTH_CLEAR_ENABLE` is taken to write `DB_DEPTH_CLEAR` to every pixel it covers whatever the depth test, depth bounds and depth bias say: it is drawn with an always-passing depth write and the viewport depth range pinned to the clear value. The draw is rasterized as submitted, so its coverage is the draw's own. Whether the DB still applies the depth test or HiZ to such a draw is not measured +- [GPU flips](../../core/libs/prx/libSceVideoOut/src/VideoOutDriver.cpp) (libSceVideoOut) - a command-buffer flip of a buffer index with no registered buffer when the GPU reaches it presents black and completes like any flip (flip event, count, `flipPendingNum`). The console's behaviour is unknown; PPSA05684 requires that it completes: its renderer submits a flip of buffer 0 while another thread waits for `sceVideoOutIsFlipPending` to reach 0 before registering the buffers, and `flipPendingNum` counts that flip from its submission as in shadPS4 - [sceAgcDriverGetEqContextId](../../core/libs/prx/libSceAgcDriver/Eq/src/Query.cpp) (libSceAgcDriver) - returns `ident`: an end-of-pipe interrupt is delivered only to the registration whose id equals the raising queue, so `ident` is that context's id (titles register 0 or 1 for graphics and `0x20 + 8 * pipe + queue` for compute). The console's value is not known; KytyPS5 returns the event's `data`, which it fills with the interrupt's context id, whereas `data` here is the EV_CLEAR trigger count. Other event filters throw - [sceAgcSetSubmitMode](../../core/libs/prx/libSceAgc/Misc/src/Suspend.cpp) (libSceAgc) - mode values unknown; only 0 is accepted - [User data registers](../../core/libs/prx/libSceAgcDriver/Execution/src/Driver/Diagnostics.cpp) (AGC driver) - a `SPI_SHADER_USER_DATA_*` or `COMPUTE_USER_DATA_*` register the title never wrote reads as 0 when a stage's RSRC2 count covers it, as Kyty (zeroed `UserSgprInfo`) and SharpEmu (missing registers read 0) do, and as AMD PAL and Mesa initialize them; the console's value is not confirmed - [SET_PREDICATION](../../core/libs/prx/libSceAgcDriver/Execution/src/Pm4.cpp) (AGC driver) - boolean predication follows the AMD PM4 encoding [Mesa](https://gitlab.freedesktop.org/mesa/mesa/-/blob/main/src/amd/common/sid.h) uses: operation 3 reads 64 bits and 4 reads 32 bits, and bit 8 runs predicated packets when the value is non-zero (clear: when it is zero); the bit 12 hint is ignored. The address must have its low 4 bits clear, as `sceAgcDcbSetPredication` writes it. Query predication (Z-pass, primitive count), a predicated COND_EXEC, predicated command buffer chains and flips in predicated command buffers throw - [sceAgcGetGsOversubscription](../../core/libs/prx/libSceAgc/Shader/src/GsOversubscription.cpp) (libSceAgc) - signature and occupancy formula (base and expanded vertex/export capacities, budget shift, `GE_PC_ALLOC` / `SPI_SHADER_PGM_RSRC4_GS` encoding) follow KytyPS5 only, not confirmed on hardware or a title; a negative or NaN factor that gives a negative target, a missing GS context register or a zero `GE_MAX_OUTPUT_PER_SUBGROUP` throws - [Vertex entry merged wave info](../../core/shader/recompiler/Translation/src/InstructionTranslator.cpp) (recompiler) - `s3` of every vertex-path shader is set to `waveSize | waveSize << 8` (vertex and primitive count of a full wave, wave index 0). Layout taken from the prologue Mesa's ACO emits for NGG vertex shaders (`s_bfe_u64 exec` from `s3`), not measured on hardware. The driver gives every graphics program user data from `s8`, so the condition does not tell NGG from legacy vertex shaders: they just never read `s3`. A draw whose last wave is partial still reports a full wave. +- [Zero-stride vertex fetch range](../../core/libs/prx/libSceAgcDriver/Graphics/include/VertexInput.hpp) (libSceAgcDriver) - for a stride-0 V# with OOB_SELECT 2 the whole fetch reads memory unless NUM_RECORDS is 0; with OOB_SELECT 3 it reads zero when NUM_RECORDS is 0 and throws when the fetch only partly fits. Rules taken from the GFX10 OOB_SELECT table in Mesa [ac_descriptors.c](https://gitlab.freedesktop.org/mesa/mesa/-/blob/main/src/amd/common/ac_descriptors.c), not measured on hardware. Whether OOB_SELECT 3 checks each dword of a multi-dword fetch separately is unknown. OOB_SELECT 0 and 1 keep the structured record checks. - [sceAgcCbCondWrite](../../core/libs/prx/libSceAgc/Cb/src/Branch.cpp) (libSceAgc) - argument order and the 9-dword COND_WRITE layout follow the clean-room [OpenAGC](https://github.com/OpenAGC/OpenAGC/blob/main/src/game_compat.c) and [sharpemu](https://github.com/sharpemu/sharpemu/blob/main/src/SharpEmu.Libs/Agc/AgcExports.CommandPackets.cs), which agree on them; SharpProspero lists another order. The write space is accepted only as 1 (memory), the one value both encode the same way, and other values throw. - [sceAgcGetIsTrinityMode](../../core/libs/prx/libSceAgc/Misc/src/Platform.cpp) (libSceAgc) - signature from the only PPSA26344 call (a pointer to a one-byte flag, result ignored); a null pointer throws because its error code is unknown - [zARR5aCmkoY](../../core/libs/prx/libSceAgc/DcbFlow/src/Control.cpp) (libSceAgc) - unknown name, signature @@ -110,21 +125,25 @@ Throughout the project, every function at every stage either **does exactly what - [sceAgcBranchPatchSetThenTarget_0300, sceAgcBranchPatchSetElseTarget_0300](../../core/libs/prx/libSceAgc/Unimplemented.cpp) (libSceAgc) - names from their NIDs (`7Wa3aeJgeVU`, `rP5xLdOf26k`), signatures unknown - [sceAgcUpdateInterpolantMapping_0100](../../core/libs/prx/libSceAgc/Misc/src/ShaderFusion.cpp) (libSceAgc) - name from its NID (`k0E7vkgqAuE`); the `(regs, vs, ps)` arguments are the declared ones, behaviour unknown - [sceAgcWriteDataPatchSetDst, sceAgcWriteDataPatchSetCachePolicy and their Async forms](../../core/libs/prx/libSceAgc/Patch/src/WaitRegMem.cpp) (libSceAgc) - the signatures `(packet, uint8 value)` are inferred from the other patch functions, as no public declaration was found; the value is taken in the encoding of the `dst` and `cachePolicy` arguments of `sceAgcDcbWriteData` (graphics) and `sceAgcAcbWriteData` (Async, compute), and the patched packet is the one those calls would write. `sceAgcAsyncWriteDataPatchSetAddressOrOffset` is assumed to patch the address like `sceAgcWriteDataPatchSetAddressOrOffset`, as the compute WRITE_DATA packet keeps it in the same dwords -- [dbOlWdppb4o](../../core/libs/prx/libSceAgc/Shader/src/InterpolantMapping.cpp) (libSceAgc) - unknown name; same arguments as `sceAgcCreateInterpolantMapping`; the `SPI_PS_INPUT_CNTL` bits for `is_f16` 2 inputs follow KytyPS5, not confirmed on hardware +- [dbOlWdppb4o](../../core/libs/prx/libSceAgc/Shader/src/InterpolantMapping.cpp) (libSceAgc) - unknown name; same arguments as `sceAgcCreateInterpolantMapping_0100`; the `SPI_PS_INPUT_CNTL` bits for `is_f16` 2 inputs follow KytyPS5, not confirmed on hardware - [sceAgcGetDataPacketPayloadRange](../../core/libs/prx/libSceAgc/Misc/src/PacketInfo.cpp) (libSceAgc) - signature, the `{base, size in bytes}` output and the type 0 range (header + 1, one dword longer than the type 1 payload) from KytyPS5 only, matching `sceAgcGetDataPacketPayloadAddress_0090`; prosper returns header + 2 for both types from that address function - [sceAgcAsyncCondExecPatchSetCommandAddress, sceAgcAsyncCondExecPatchSetEnd](../../core/libs/prx/libSceAgc/Patch/src/CondExec.cpp) (libSceAgc) - no reference has a body for them; they are assumed to patch the COND_EXEC packet like `sceAgcCondExecPatchSetCommandAddress` and `sceAgcCondExecPatchSetEnd`, since `sceAgcAcbCondExec` writes the same packet as `sceAgcDcbCondExec`, as `sceAgcAsyncRewindPatchSetRewindState` does for REWIND - [sceAgcAcbAtomicGds_0900](../../core/libs/prx/libSceAgc/Acb/src/Control.cpp) (libSceAgc) - name from its NID (`gQkqkLttcpw`), signature unknown - [sceAgcDcbPrimeUtcl2](../../core/libs/prx/libSceAgc/DcbState/src/Display.cpp) (libSceAgc) - assumed to write the packet `sceAgcAcbPrimeUtcl2` writes: PRIME_UTCL2 has the same 5 dwords on the graphics and compute rings. Not confirmed on a title. Both write a NOP of that size, as there is no TLB to prime - [sceAgcDcbQueueEndOfShaderActionGetSize](../../core/libs/prx/libSceAgc/DcbState/src/ContextState.cpp) (libSceAgc) - the builder is inline in the SDK and not exported; the size is assumed to be that of the RELEASE_MEM packet `sceAgcCbReleaseMem` writes (32 bytes), the same as `sceAgcAcbQueueEndOfShaderActionGetSize`, as in [OpenAGC](https://github.com/OpenAGC/OpenAGC/blob/main/src/game_compat.c) and [REmu](https://github.com/xnetcat/REmu/blob/main/crates/remu-runtime/src/agc_dcb.rs) - [sceKernelInternalMemoryGetModuleSegmentInfo](../../core/libs/prx/libkernel/Module/src/Module.cpp) (libkernel) - unknown signature -- [sceKernelGetModuleInfoForUnwind](../../core/libs/prx/libkernel/Module/src/Module.cpp) (libkernel) - `eh_frame_size` is assumed to cover the records up to, not including, the zero terminator. LLVM libunwind's `CFI_Parser::findFDE` stops at the terminator or the length, whichever comes first, so it finds the same FDEs either way; what the console reports is not known. -- [sceKernelGetModuleInfoFromAddr](../../core/libs/prx/libkernel/Module/src/ModuleInfo.cpp) (libkernel) - the 0x1A8-byte info layout follows the [shadPS4](https://github.com/shadps4-emu/shadPS4) reimplementation; a title's `libc.prx` reads only `id` (PPSA14632). The meaning of `flags` is unknown; only 2 is accepted, as is only the full `st_size`. The id is the `sceKernelLoadStartModule` handle of an image opened through the loader and otherwise a stable id from the same range. The name is the host file name without `.guest.prx`. An image with more than 4 load segments (the converted executable has 7) reports its first 4. `tls_offset` is 0 and `ref_count` is 1, as the host loader exposes neither. Not implemented on Windows +- [sceKernelGetModuleInfoForUnwind](../../core/libs/prx/libkernel/Module/src/Module.cpp) (libkernel) - on Linux the tables come from `sceKernelGetModuleInfoFromAddr`, whose `eh_frame_size` includes the zero terminator, as measured on firmware 13.60 (reported in [#2419](https://github.com/boykopovar/AnyPS5/issues/2419), not reproduced here); on Windows it is assumed to stop before the terminator. LLVM libunwind's `CFI_Parser::findFDE` stops at the terminator or the length, whichever comes first, so it finds the same FDEs either way. Only the main executable and `.guest.prx` images report tables; other host images report none, as on Windows +- [sceKernelGetModuleInfoFromAddr](../../core/libs/prx/libkernel/Module/src/ModuleInfo.cpp) (libkernel) - the 0x1A8-byte info layout follows the [shadPS4](https://github.com/shadps4-emu/shadPS4) reimplementation; a title's `libc.prx` reads only `id` (PPSA14632). The meaning of `flags` is unknown; only 2 is accepted, as is only the full `st_size`. The id is the `sceKernelLoadStartModule` handle of an image opened through the loader and otherwise a stable id from the same range. The name is the host file name without `.guest.prx`. An image with more than 4 load segments (the converted executable has 7) reports its first 4. `tls_offset` is 0 and `ref_count` is 1, as the host loader exposes neither. On Windows the module is the PE image holding the address: a relinked image reports its `.elf` sections as segments and the eh_frame_hdr recorded in `.ehmeta`, any other image its first 4 sections with memory access and the `.ehfram` section without an eh_frame_hdr; the TLS fields and `init_proc_addr`/`fini_proc_addr` are 0 +- [sceKernelGetModuleList](../../core/libs/prx/libkernel/Module/src/ModuleInfo.cpp) (libkernel) - lists the main program and every loaded `.prx` image in load order; host runtime libraries are left out. The handles are the ids `sceKernelGetModuleInfoFromAddr` reports, not the console's. The `(int32_t* handles, size_t count, size_t* actual)` signature and returning `ENOMEM` without writing `actual` after filling the first `count` handles when more images are loaded follow [shadPS4](https://github.com/shadps4-emu/shadPS4/blob/0fe263a4760dfbfa973366890061749b4af0de97/src/core/libraries/kernel/process.cpp#L244-L264); whether the console writes any handles in that case is unknown +- [sceKernelGetModuleInfo](../../core/libs/prx/libkernel/Module/src/ModuleInfo.cpp) (libkernel) - the 0x160-byte layout (`st_size`, `name[256]`, 4 segments, `segment_count`, `fingerprint[20]`) follows [fpPS4](https://github.com/red-prig/fpPS4/blob/04cefd43e6fddd1ab033e7980cd356d14c964905/sys/sys_types.pas#L319-L337) and [shadPS4](https://github.com/shadps4-emu/shadPS4/blob/0fe263a4760dfbfa973366890061749b4af0de97/src/core/module.h#L15-L31). Only that `st_size` is accepted, as in shadPS4; other sizes return `EINVAL`, a null pointer `EFAULT` and a handle `sceKernelGetModuleList` does not list `ESRCH`. Name and segments are the ones `sceKernelGetModuleInfoFromAddr` reports, on Windows from the same PE sections, so `prot` uses the read/write/execute bits 1/2/4, while fpPS4 and shadPS4 store the ELF `p_flags`. The fingerprint is left zero, as the host image does not carry it - [sceKernelReleaseFlexibleMemory](../../core/libs/prx/libkernel/DirectMemory/Export.cpp) (libkernel) - unmaps the range like sceKernelMunmap, as fpPS4 and prosper do; whether the console keeps the virtual range reserved after releasing its pages is unverified - [Memory protection bits 0x100 and 0x200](../../core/libs/prx/libkernel/DirectMemory/DirectMemory.cpp) (libkernel) - taken as audio coprocessor (ACP) read and write, as in the [KytyPS5](https://github.com/KytyPS5/KytyPS5) and [etaHEN](https://github.com/etaHEN/etaHEN) headers; the coprocessor work runs on the host CPU, so they grant host read and read/write access like the GPU bits - [sceKernelSyncOnAddressWait](../../core/libs/prx/libkernel/SyncOnAddress/src/SyncOnAddress.cpp) (libkernel) - the only known caller passes a null timeout and a name string as the fourth argument; the timeout is assumed to point to microseconds like the other kernel waits, and the name is ignored - [sceKernelSyncOnAddressWait8 / sceKernelSyncOnAddressWait16 / sceKernelSyncOnAddressWait32 / sceKernelSyncOnAddressWait64](../../core/libs/prx/libkernel/SyncOnAddress/src/SyncOnAddress.cpp) (libkernel) - libc's `__std_atomic_wait_direct_8/16/32/64` pass a microsecond timeout or null and ignore the result; the by-value comparand, return values and the 2- and 8-byte alignment of Wait16 and Wait64 are assumed -- [sceKernelAioSubmitReadCommandsMultiple / sceKernelAioSubmitWriteCommandsMultiple / sceKernelAioWaitRequests / sceKernelAioCancelRequest / sceKernelAioCancelRequests / sceKernelAioDeleteRequests](../../core/libs/prx/libkernel/Aio/src/Aio.cpp) (libkernel) - behaviour from shadPS4 (one id per request for the Multiple submits, cancel turns any request aborted and id 0 reports processing, wait mode 2 returns once one request completed); the 128 requests or ids per batch limit is from SharpProspero, with an unknown error, so larger batches throw; wait modes other than 1 (and) and 2 (or) throw; null pointers, negative counts and invalid ids return the codes the single-request siblings use, checked before any state is written +- [scePthreadCondWait / scePthreadCondTimedwait](../../core/libs/prx/libkernel/Pthread/src/Cond.cpp) (libkernel) - a recursive mutex locked several times is released completely for the wait and relocked to the same depth afterwards, as FreeBSD libthr does (`_mutex_cv_unlock`, `_mutex_cv_lock`); unverified on hardware +- [sceKernelAioSubmitReadCommandsMultiple / sceKernelAioSubmitWriteCommandsMultiple / sceKernelAioWaitRequests / sceKernelAioPollRequests / sceKernelAioCancelRequest / sceKernelAioCancelRequests / sceKernelAioDeleteRequests](../../core/libs/prx/libkernel/Aio/src/Aio.cpp) (libkernel) - behaviour from shadPS4 (one id per request for the Multiple submits, cancel turns any request aborted and id 0 reports processing, wait mode 2 returns once one request completed, poll copies every state without waiting); the 128 requests or ids per batch limit is from SharpProspero, with an unknown error, so larger batches throw; wait modes other than 1 (and) and 2 (or) throw; null pointers, negative counts and invalid ids return the codes the single-request siblings use, checked before any state is written - [sceKernelGetAvailableCpumask](../../core/libs/prx/libkernel/Pthread/src/Thread.cpp) (libkernel) - returns the default thread affinity, 0x1FFF (CPUs 0 to 12), as KytyPS5 does. Titles pin threads within it (PPSA02664 passes 0x1FFB, Hades II PPSA36082 0x3 and 0x3F); whether CPU 13 is available to a title is not confirmed +- [sceKernelGetCurrentCpu](../../core/libs/prx/libkernel/System/src/Process.cpp) (libkernel) - guest affinities are not applied to host threads, so the host CPU index is mapped into the calling thread's affinity clipped to `sceKernelGetAvailableCpumask` (the available mask when they do not overlap): host CPU n reports the (n modulo the CPU count)-th CPU of that mask, so a thread pinned to one CPU reports that CPU, as a FreeBSD `cpuset` guarantees on the console. Which CPU the console reports for a thread allowed on several is not known; shadPS4 returns 0 - [sceKernelMapNamedFlexibleMemoryInternal](../../core/libs/prx/libkernel/DirectMemory/Export.cpp) (libkernel) - flag 0x8000 is accepted and ignored; its meaning is not established. KytyPS5 (`GUEST_MAP_UNKNOWN_8000`) and sharpemu accept it without reading it. Every known caller is the SDK 9 `libc.prx` heap (Astro Bot PPSA21564, PPSA22520), which passes it with read/write protection and uses the mapping as an older `libc.prx` uses `sceKernelMapNamedFlexibleMemory` with flags 0 - [sceKernelMtypeprotect](../../core/libs/prx/libkernel/DirectMemory/DirectMemory.cpp) (libkernel) - sets the memory type of the direct mappings in the page-rounded range and of the direct memory behind them, as shadPS4 and prosper do (KytyPS5 ignores the type); the type is not validated, flexible pages in the range only get the protection, and a later map of the retyped direct memory reports the type it was allocated with. `sceKernelBatchMap2` operation 4 takes the same path with the entry type - [Guest arena](../../core/libs/prx/libc/src/GuestArena.cpp) (libc) - the application map area is assumed to end at 0xFC_0000_0000, as `SCE_KERNEL_APP_MAP_AREA_END_ADDR` does on the PS4. On Windows, host allocations made in the arena before libc loads stay, and a fixed guest mapping over one throws @@ -132,10 +151,14 @@ Throughout the project, every function at every stage either **does exactly what - [sceLibcInternalHeapErrorReportForGame](../../core/libs/prx/libc/src/HeapDiagnostics.cpp) (libSceLibcInternal, implemented in libc) - unknown signature - [_sceLibcInternalThreadAtexit, _sceLibcInternalThreadDtors](../../core/libs/prx/libSceLibcInternal/Export.cpp) (libSceLibcInternal) - FreeBSD `__cxa_thread_atexit` semantics assumed; a null or non-image destructor and an allocation failure throw - [_sceKernelRtldThreadAtexitIncrement, _sceKernelRtldThreadAtexitDecrement](../../core/libs/prx/libkernel/System/src/Rtld.cpp) (libkernel) - the argument is the `dso` address a title's `libc.prx` passes to `__cxa_thread_atexit` (PPSA14632), which fails when Increment returns nonzero. As in the [fpPS4](https://github.com/red-prig/fpPS4) reimplementation, Increment takes a reference on the loaded image containing the address and Decrement releases one; the variants without the leading underscore are assumed to match +- [sceImeGetPanelPositionAndForm](../../core/libs/prx/libSceIme/Export.cpp) (libSceIme) - the output is taken to be the `PositionAndForm` that `sceImeDialogGetPanelPositionAndForm` fills; it is never written, since no panel can be open. That a closed panel is reported before a null output is assumed by analogy with `sceImeSetCaret`, not confirmed on a PS5 title - [_sceLibcInternalForceTlsDestructor](../../core/libs/prx/libSceLibcInternal/Export.cpp) (libSceLibcInternal) - behaviour follows `sceLibcForceTlsDestructor(int)` in a title's `libc.prx` (disassembly): the calling thread's destructors whose dso belongs to the module run newest first, only if the dso's module is loaded and the dso points to itself, in up to 4 passes. The return type is unknown (that function returns nothing), 0 is returned. The module of each dso is looked up when the destructors are forced, not when they are registered as `libc.prx` does, and the lookup is Linux only, so on Windows the call throws when the thread has destructors - [std_execute_once](../../core/libs/prx/libc/Export.cpp) (libc) - assumed to be the plain-named twin of `std::_Execute_once` (`_ZSt13_Execute_onceRSt9once_flagPFiPvS1_PS1_ES1_`, the callee of Dinkumware `call_once`): it delegates there and gets the [thread.once.callonce] semantics already verified for that symbol (one invocation per flag, concurrent callers block until it completes, and a failing or throwing invocation leaves the flag unready so the next caller runs it). The name equivalence is not confirmed on a title, and the callback convention (called with (null, arg, null), non-zero return meaning success) is the twin's, unverified for this symbol +- [tcgetattr, tcsetattr](../../core/libs/prx/libkernel/File/src/Terminal.cpp) (libkernel) - every file and pipe descriptor is taken as not a terminal (`ENOTTY`), and a socket gets `EADDRNOTAVAIL` for `AF_INET` and `EOPNOTSUPP` otherwise, as FreeBSD's `in_control` and `pru_control_notsupp` answer a termios ioctl; what descriptors 0 to 2 of a title are on the console, and whether any of them is a terminal there, is unknown - [__progname](../../core/libs/prx/libkernel/System/src/Process.cpp) (libkernel) - unknown data export +- [seteuid, setegid](../../core/libs/prx/libkernel/System/src/Process.cpp) (libkernel) - the guest process is assumed to run as root, with real, effective and saved user and group ids 0. As in FreeBSD's `sys_seteuid` and `sys_setegid`, id 0 is then accepted without a privilege check and changes no credential, so 0 is returned; any other id would change the effective id, which is not modelled, and throws. The ids and privileges a title has on a console are unverified - [pthread_barrierattr_setpshared](../../core/libs/prx/libkernel/Pthread/Posix/Barrier.cpp) (libkernel) - PTHREAD_PROCESS_SHARED is accepted on the assumption that the console follows FreeBSD 11.0 libthr, which accepts it (9.0 rejects it with EINVAL); unverified on hardware. Guest code runs in one process, so a shared barrier behaves as a private one +- [pthread_getname_np](../../core/libs/prx/libkernel/Pthread/Posix/Thread.cpp) (libkernel) - a thread that has already been joined returns `ESRCH` (3) on a PS5 (firmware 13.60), but its object is freed after the join here, so the pointer cannot be told apart from a live thread and its name is read as for a live one - [sceSslClose](../../core/libs/prx/libSceSsl/Export.cpp) (libSceSsl) - unknown signature - [sceSslGetSerialNumber](../../core/libs/prx/libSceSsl/Export.cpp) (libSceSsl) - unknown signature - [X+4jdIS75P0](../../core/libs/prx/libSceAudioIn/Export.cpp) (libSceAudioIn) - unknown name, signature @@ -147,7 +170,9 @@ Throughout the project, every function at every stage either **does exactly what - [sceAudioOut2PortCreate](../../core/libs/prx/libSceAudioOut/src/AudioOut2Port.cpp) (libSceAudioOut) - MAIN (type 0) format validation follows PS5 firmware 13.60 measurements: bits 8-11 accept 1, 2, 8 or 12, bits 0-6 accept 0 or 1, bit 7 requires 8 channels, and bits 12-31 do not affect validation. The sample types are still read as float and int16; bit 7's layout, the PCM effect of bits 12-31 and the port `flags` are unverified. Other port types retain unverified format validation, including 6 channels and rejection of bits 12-31 - [sceAudioOut2PortCreate](../../core/libs/prx/libSceAudioOut/src/AudioOut2Port.cpp) (libSceAudioOut) - 12 channels are assumed to be L R C LFE Ls Rs Lb Rb Ltf Rtf Ltb Rtb as in KytyPS5's 12-to-8 fold, and 6 and 8 channels to be its first 6 and 8; no title or SDK header confirms the order - [sceAudioOut2PortCreate](../../core/libs/prx/libSceAudioOut/src/AudioOut2Port.cpp) (libSceAudioOut) - the stereo fold is assumed, not the console's downmix: C at -3 dB into both sides, Ls/Rs, Lb/Rb and Ltf/Rtf at -3 dB and Ltb/Rtb at -6 dB into their own side, LFE dropped +- [sceAudioOut2PortSetAttributes](../../core/libs/prx/libSceAudioOut/src/AudioOut2Port.cpp) (libSceAudioOut) - each PCM buffer set (attribute 0) is taken as one grain: buffers set between two pushes are queued and played one per grain, at most 16 queued. A grain with no new buffer repeats the last one only when the port has always been given that same buffer (rewritten in place); a port whose buffers move on plays silence instead. Inferred from PPSA12544, whose object port (type 6) is given one grain of a 64-grain ring per set, often two or three between pushes and then none for a few; the console behaviour for several sets per push and for a push without a new buffer is not documented - [scePsmlMfsrGetContextBufferRequirement1100, scePsmlMfsrCreateContext1100, scePsmlMfsrGetDispatchMfsrPacket1100](../../core/libs/prx/libScePsml_debug/Export.cpp) (libScePsml_debug) - argument layouts unknown; only 0x8A810001 (not initialized) is returned +- [scePsmlMfsrGetDispatchMfsrPacket1000](../../core/libs/prx/libScePsml_debug/Export.cpp) (libScePsml) - unknown signature, imported by PPSA25344; declared without parameters. libScePsml and libScePsml_debug are built from the same source - [scePngEncEncode](../../core/libs/prx/libScePngEnc/Export.cpp) (libScePngEnc) - whether rows may stay unfiltered when `filter_type` names a subset of the filters is unknown; as in shadPS4, they may only for 0 and for the all-filters mask - [sceJpegEncEncode](../../core/libs/prx/libSceJpegEnc/Export.cpp) (libSceJpegEnc) - `restart_interval` follows SharpProspero's field doc (0 none, -1 every row of blocks, positive a block count); a positive value is taken as the JPEG DRI interval in MCUs and -1 as one restart per MCU row, which no reference confirms. Other negative values throw - [sceVoiceQoSInit](../../core/libs/prx/libSceVoiceQoS/Export.cpp) (libSceVoiceQoS) - `0x804E0905` for a second initialization comes from [prosper](https://github.com/mattias800/prosper)'s reading of the module and was not measured: on a PS5 a valid call blocks inside a homebrew payload, so it never returns there. The app type and the block are not used, as no endpoint is implemented @@ -156,7 +181,7 @@ Throughout the project, every function at every stage either **does exactly what - [libSceCdlgG2P](../../core/libs/prx/libSceCdlgG2P/Export.cpp) - `sceG2PDialogInitialize`, `Terminate`, `Open`, `GetStatus`, `UpdateStatus` and `GetResult`, imported by PPSA01289, are added without an implementation and declared without parameters - [vieBRwlh1Lw](../../core/libs/prx/libSceAgc/Unimplemented.cpp) (libSceAgc) - unknown name, signature - [scePadIsRemoteController](../../core/libs/prx/libScePad/Export.cpp) (libScePad) - the name hashes to NID `fCWdlnmB1Ks`; the signature `(int handle, bool* remote)` comes from PPSA16064, which zeroes a byte, passes its address after `scePadGetControllerInformation` and ignores the result. The error codes follow the other pad calls. Controllers are local devices, so the answer is always false -- [scePadSetAngularVelocityBiasCorrectionState](../../core/libs/prx/libScePad/Export.cpp) (libScePad) - imported by PPSA16064 and not called during its start-up; unknown signature +- [scePadSetAngularVelocityBiasCorrectionState](../../core/libs/prx/libScePad/Export.cpp) (libScePad) - inferred signature `(int handle, bool enabled)` follows `scePadSetTiltCorrectionState` and `scePadSetAngularVelocityDeadbandState`; an invalid handle returns `PAD_ERROR_INVALID_HANDLE`, disabling is a no-op, and enabling throws because bias correction is not modelled - [sceKernelGetOperationMode](../../core/libs/prx/libkernel/System/src/Process.cpp) (libkernel) - the signature `(int* mode, int* submode)` comes from PPSA12544, which logs both values after the call; the values are unknown and 0 is reported for both, as in the [prosper](https://github.com/mattias800/prosper/pull/4153) reimplementation. Null outputs throw - Font, Http2, Net, Ssl, SystemService and libkernel exports imported by PPSA12544 and added without an implementation have unknown signatures - [sceHttp2GetMemoryPoolStats, sceSslGetMemoryPoolStats, sceNetGetMemoryPoolStats](../../core/libs/prx/libSceNet/Export.cpp) (libSceHttp2, libSceSsl, libSceNet) - signatures `(int id, stats*)` and the statistics layouts (pool size, peak and current use, and for Http2 and Ssl a 32-bit reserved field) follow the PS4 SDK's `SceHttp2MemoryPoolStats`, `SceSslMemoryPoolStats` and `SceNetMemoryPoolStats`, not confirmed for the PS5. The pool size is the one given at init or pool creation; nothing is allocated from these pools, so the peak and current use are 0. Net returns `EBADF` for an unknown pool and `EINVAL` for a null output, as its other pool calls do; for Http2 and Ssl the error codes are unknown, so both cases throw @@ -169,14 +194,17 @@ Throughout the project, every function at every stage either **does exactly what - [NGS2 mastering voice gain](../../core/libs/prx/libSceNgs2.native/src/Voice.cpp) (libSceNgs2.native) - the param id 0x30000004 is taken from PPSA22520 and the layout from [shadPS4](https://github.com/shadps4-emu/shadPS4)'s `OrbisNgs2MasteringVoiceGainParam`; no reference gives the units or range. The levels are assumed to be linear factors applied when the mastering voice is mixed into the render buffer, so the voice's own samples stay unscaled; the LFE level applies to channel 3 of 6- and 8-channel voices, assuming the L R C LFE order of the AudioOut2 entries above, and the full-band level to the rest. Negative levels are accepted, and a new setup resets both levels to 1 - [NGS2 stereo mastering voice into a surround buffer](../../core/libs/prx/libSceNgs2.native/src/Render.cpp) (libSceNgs2.native) - PPSA21402 renders a 2-channel mastering voice into a 6- or 8-channel render buffer, which the mixing stage rejected. The console's conversion is unknown: the left and right channels are assumed to go to the first two buffer channels (front left and right) and the remaining channels stay silent, with no centre, LFE or surround content. Any other channel count mismatch still throws - [NGS2 streaming PCM waveforms](../../core/libs/prx/libSceNgs2.native/src/Voice.cpp) (libSceNgs2.native) - PPSA21402 streams its music by adding one block whose `num_samples` is the whole track and whose `data_size` is a 16 KiB first piece, with `CONTINUE`, and then adds each following piece with flag `0x2` (called `APPEND` here) and the same `num_samples`. No reference describes flag `0x2`. A 16-bit PCM block without skipped samples or repeats whose `num_samples` exceed its data is assumed to start such a stream; flag `0x2` is assumed to append the block's data to the last streaming block, ignoring its `num_samples`; the voice is assumed to output silence without advancing while it has played all appended data and `CONTINUE` is set; and the waveform is assumed to end at `num_samples` or, once `CONTINUE` is cleared, at the end of the appended data. `sceNgs2VoiceGetState` reports the bytes played as `decoded_data_size`, which the title uses to release its queued data, and the read position as `waveform_data`. Appending to a block that is not a stream, or other formats, skipped samples, repeats and partial frames, throw +- [NGS2 matrix levels](../../core/libs/prx/libSceNgs2.native/src/Voice.cpp) (libSceNgs2.native) - levels are clamped to -4..4 and NaN is kept, as the PS4 firmware 11.00 `libSceNgs2.sprx` stores them; not yet confirmed on the PS5 library. That library rejects a level count of 0 and keeps the matrix; the PS5 error code is unknown, so a count of 0 throws. `sceNgs2VoiceRunCommands` is not in that library, so its matrix levels command is assumed to clamp the same way - [sceAvPlayerStartEx](../../core/libs/prx/libSceAvPlayer/Export.cpp) (libSceAvPlayer) - start info layout unknown; it is ignored and playback starts as with `sceAvPlayerStart` - [sceAvPlayerInit / sceAvPlayerInitEx](../../core/libs/prx/libSceAvPlayer/src/Player.cpp) (libSceAvPlayer) - behaviour without a memory replacement unknown; frame and sample buffers then come from the guest heap +- [sceAvPlayerInitEx](../../core/libs/prx/libSceAvPlayer/Export.cpp) (libSceAvPlayer) - the PS5 init data layout (`this_size` 0x230: auto start at 0x74, nine 0x30-byte thread parameter blocks from 0x78, output video frame buffer count at 0x228) follows [KytyPS5](https://github.com/KytyPS5/KytyPS5/blob/main/src/libs/avPlayer.cpp), [prosper](https://github.com/mattias800/prosper/blob/main/prosper/src/hle/video/avplayer.cpp), [SharpProspero](https://github.com/SvenGDK/SharpProspero/blob/main/src/SharpProspero/Interop/Media/AvPlayer.cs) and [sharpemu](https://github.com/sharpemu/sharpemu/blob/main/src/SharpEmu.Libs/AvPlayer/AvPlayerExports.cs), which agree on it. PPSA25063 fills the same fields, writes auto start 0 and starts playback with `sceAvPlayerStart`; a title that sets auto start has not been seen. A smaller `this_size` is read with the PS4 offsets. The thread parameters are ignored - [SceAvPlayerVideoEx](../../core/libs/SceTypes.hpp) (libSceAvPlayer) - frame rate field and encoding unknown; it is left zero in frame and stream info - [sceRazorCpuFlushOccurred](../../core/libs/prx/libSceRazorCpu/Export.cpp) (libSceRazorCpu) - signature and return values (1 = flush since the last call, 0 = none) from [SharpProspero](https://github.com/SvenGDK/SharpProspero); without a capture nothing is flushed, so it returns 0 and writes 0 cycles to the optional out pointer (whether the real library writes it when nothing was flushed is unknown) - [sceRazorCpuPushMarkerStatic, sceRazorCpuPopMarker](../../core/libs/prx/libSceRazorCpu/Export.cpp) (libSceRazorCpu) - signatures from PPSA26344's calls; (name, color, flags) assumed for the push; both do nothing without a capture - [sceRazorCpuPlotValue, sceRazorCpuWriteBookmark](../../core/libs/prx/libSceRazorCpu/Export.cpp) (libSceRazorCpu) - signatures (series, float value) and (label, optional description) from [SharpProspero](https://github.com/SvenGDK/SharpProspero), matching PPSA06320's calls; both do nothing without a capture and return 0 - [sceAjmBatchJobGetInfo](../../core/libs/prx/libSceAjm.native/src/Ajm.cpp) (libSceAjm.native) - writes the result and format sideband of a run job without buffers with the `SIDEBAND_FORMAT` flag (0x400000000000), as [KytyPS5](https://github.com/KytyPS5/KytyPS5) does, not confirmed on a PS5 title; the format reports bitrate 0 for ATRAC9 as in KytyPS5, while [shadPS4](https://github.com/shadps4-emu/shadPS4) derives it from the superframe size; before the instance is initialized it returns the not-initialized result (1) and no format, like the other run jobs here, while KytyPS5 returns result 0 with the format of the uninitialized decoder - [sceAjmDecMp3ParseFrame](../../core/libs/prx/libSceAjm.native/src/Ajm.cpp) (libSceAjm.native) - signature, result layout and rate tables taken from the [shadPS4](https://github.com/shadps4-emu/shadPS4) reimplementation of the PS4 library, not confirmed on a PS5 title: the layer field is not checked and MPEG-2.5 stops at 64 kbps; the original file length (`parse_ofl`) layout (Xing/Info + LAME tag, VBRI, Fraunhofer `0xB4` block with its CRC, encoder delay = samples per frame + LAME delay + 529) also comes from shadPS4 only, and the Fraunhofer block has no other public description +- [NGS2 port matrix](../../core/libs/prx/libSceNgs2.native/src/Render.cpp) (libSceNgs2.native) - a port without a matrix, or with a selected matrix whose levels were never set, mixes through a default channel map: mono reaches both stereo channels at -3 dB and the 5.1 and 7.1 centre, stereo folds to mono at -3 dB per side, and wider sources fold down with 0.5 and -3 dB gains. The map and the equivalence of an unset matrix with no matrix were measured from the PS4 firmware 11.00 `libSceNgs2.sprx`, not yet from the PS5 library. That library only accepts submixer and mastering voices with 1, 2, 6 or 8 channels, so the map into 3, 4, 5 or 7 channels throws. A matrix set with fewer levels than the destination needs still mixes only the rows it has - [sceAjmBatchJobGetCodecInfo, sceAjmBatchJobGetGaplessDecode](../../core/libs/prx/libSceAjm.native/src/Ajm.cpp) (libSceAjm.native) - the `RUN_GET_CODEC_INFO` flag (0x800), the ATRAC9 codec info layout and its next frame size are taken from the [shadPS4](https://github.com/shadps4-emu/shadPS4) and [RPCSX](https://github.com/RPCSX/rpcsx) reimplementations, not confirmed on a PS5 title; a run job without buffers returns result 0, as in shadPS4. The gapless sideband reports the total and skip counts the title set and does not count them down: RPCSX does the same, shadPS4 counts them down and BryKytyPS5 returns its current counts - [AJM Opus decoder](../../core/libs/prx/libSceAjm.native/src/Ajm.cpp) (libSceAjm.native, codec 24) - the initialize parameter layout (`u32 channels, u32 sample rate, u32`, seen as `2, 48000, 0`) and the little-endian `u16` byte count before each packet come from one title (The Smurfs Dreams); only 48000 Hz and a zero third word are accepted - [sceAjmBatchJobControl](../../core/libs/prx/libSceAjm.native/src/Ajm.cpp) (libSceAjm.native) - RESET (bit 13), INITIALIZE (bit 14) and the sideband input order (gapless decode, then initialize parameters) come from [shadPS4](https://github.com/shadps4-emu/shadPS4) (PS4); other flags throw @@ -184,6 +212,7 @@ Throughout the project, every function at every stage either **does exactly what - [sceAjmBatchJobControl](../../core/libs/prx/libSceAjm.native/src/Ajm.cpp) (libSceAjm.native) - initialize parameters are 8 bytes for ATRAC9, 12 for Opus and none for MP3, as one title passes them; other codecs and sizes, and an output other than the 8-byte result, throw - [sceAjmBatchJobControl](../../core/libs/prx/libSceAjm.native/src/Ajm.cpp) (libSceAjm.native) - SIDEBAND_GAPLESS_DECODE with RESET overwrites both counts as in [RPCSX](https://github.com/RPCSX/rpcsx); shadPS4 ignores a zero count when the other is nonzero. Without RESET they differ, so it throws - [NGS2 custom submixer rack](../../core/libs/prx/libSceNgs2.native/src/Custom.cpp) (libSceNgs2.native, rack 0x4002) - structure layouts from [shadPS4](https://github.com/shadps4-emu/shadPS4)'s `ngs2_custom.h`; module id 0x1f for UserFx2, the voice parameter id `0x40001f00 | module index`, the process flags (1 on the first process call after setup, 2 after a parameter change, 0 otherwise) and the call order (setup per voice at rack creation, process after the inputs are mixed, cleanup at destruction) come only from [KytyPS5](https://github.com/KytyPS5/KytyPS5). Only UserFx2 modules on a single buffer are accepted: other module ids, more buffers, a module control handler, different input and output channel counts, setup flags and default rack options throw. Each module's state is a separate buffer, not the `state_offset` slice of the voice state, and `sceNgs2VoiceGetState` still throws for this rack. As in KytyPS5, process runs only on grains where an input produced samples, so an effect tail stops with the input and a pending flag 2 waits for the next audible grain +- [NGS2 reverb rack](../../core/libs/prx/libSceNgs2.native/src/Reverb.cpp) (libSceNgs2.native, rack 0x2001) - follows the PS4 firmware 11.00 `libSceNgs2.sprx`, read and checked against its output at 11.025 to 48 kHz, including parameter changes during playback (differences at float rounding level); not yet compared with the PS5 library. Structure layouts come from [shadPS4](https://github.com/shadps4-emu/shadPS4)'s `ngs2_reverb.h` and the param ids 0x20010000 (setup) and 0x20010001 (I3DL2) from PPSA22520. Only the default `reverb_size` of 1 is handled, with its 14400-sample reflection buffer; other sizes and the library's half-rate mode throw. System sample rates below about 7.5 kHz, where an allpass delay would have no samples, throw. Setup only accepts equal input and output channel counts - [sceHttpParseResponseHeader](../../core/libs/prx/libSceHttp/Export.cpp) (libSceHttp) - signature follows [SharpProspero](https://github.com/SvenGDK/SharpProspero/blob/main/src/SharpProspero/Interop/Net/Http.cs); parsing and error codes follow [shadPS4's reverse-engineered PS4 library](https://github.com/shadps4-emu/shadPS4/blob/main/src/core/libraries/network/http.cpp), not confirmed on a PS5 title or hardware. Folded values retain their line breaks and whitespace; an empty first line starts the value scan after its line feed. A consumed byte count above `INT_MAX` throws instead of wrapping. - [sceHttpParseStatusLine](../../core/libs/prx/libSceHttp/src/Parse.cpp) (libSceHttp) - behaviour and error codes follow the [shadPS4](https://github.com/shadps4-emu/shadPS4) reimplementation of the PS4 library, not confirmed on a PS5 title; unlike it, no byte past `lineLen` is read - [sceHttpUriMerge](../../core/libs/prx/libSceHttp/src/Uri.cpp) (libSceHttp) - signature follows [SharpProspero](https://github.com/SvenGDK/SharpProspero/blob/main/src/SharpProspero/Interop/Net/Http.cs); validation, size formula, absolute/relative split and path join follow the disassembly-like body in [shadPS4](https://github.com/shadps4-emu/shadPS4/blob/main/src/core/libraries/network/http.cpp), not confirmed on a PS5 title or hardware. The base URL goes through our `sceHttpUriParse`, which removes dot segments from the base path, while the relative part is appended unswept; base query and fragment are dropped. The relative reference is only classified by its scheme and a leading `//`, and only references with `//` are validated; as in shadPS4, `//host/x` is returned unchanged instead of taking the base scheme, a reference with a scheme but no `//` (`mailto:x`) is joined onto the base directory, and `?q` and `#f` are appended to the base directory instead of the base path. @@ -192,9 +221,7 @@ Throughout the project, every function at every stage either **does exactly what - [sceHttpSetResponseHeaderMaxSize, sceHttpRedirectCacheFlush, sceHttpsUnloadCert](../../core/libs/prx/libSceHttp/Export.cpp) (libSceHttp) - signatures follow [shadPS4](https://github.com/shadps4-emu/shadPS4/blob/main/src/core/libraries/network/http.cpp) (PS4); no PS5 reference has them - [sceHttpSetRedirectCallback, sceHttpSetCookieRecvCallback, sceHttpsSetSslVersion, sceHttpsGetSslError](../../core/libs/prx/libSceHttp/Export.cpp) (libSceHttp) - signatures follow [shadPS4](https://github.com/shadps4-emu/shadPS4/blob/main/src/core/libraries/network/http.cpp) (PS4), which only logs and returns 0; a null `errNum` or `detail` in `sceHttpsGetSslError` is assumed to return `SCE_HTTP_ERROR_INVALID_VALUE` like `sceHttpGetLastErrno` - [sceHttpSetAuthInfoCallback](../../core/libs/prx/libSceHttp/Export.cpp) (libSceHttp) - signature and callback type follow [shadPS4](https://github.com/shadps4-emu/shadPS4/blob/main/src/core/libraries/network/http.h) (PS4); no PS5 reference has them. The `id` is not checked, as in the other libSceHttp setters -- [sceHttp2GetMemoryPoolStats](../../core/libs/prx/libSceHttp2/Export.cpp) (libSceHttp2) - unknown signature - [libSceHttp2 cookie boxes](../../core/libs/prx/libSceHttp2/Export.cpp) (libSceHttp2) - `sceHttp2CreateCookieBox` is taken to take the library context id first, as in the [prosper](https://github.com/mattias800/prosper) reimplementation; further arguments are unknown and ignored. `sceHttp2SetRequestNoContentLength` is assumed to take only the request id. Cookies are never stored, so `sceHttp2CookieFlush` has nothing to discard -- [sceNetGetMemoryPoolStats](../../core/libs/prx/libSceNet/Export.cpp) (libSceNet) - unknown signature - [sceHttpSetRequestStatusCallback](../../core/libs/prx/libSceHttp/Export.cpp) (libSceHttp) - no reference implements it (shadPS4 only logs and returns 0); when the callback runs and what status it gets is unknown - [sceNpSessionSignalingGetMemoryInfo](../../core/libs/prx/libSceNpSessionSignaling/Export.cpp) (libSceNpSessionSignaling) - unknown signature - [sceNpSessionSignalingGetConnectionStatistics](../../core/libs/prx/libSceNpSessionSignaling/Export.cpp) (libSceNpSessionSignaling) - unknown signature @@ -209,66 +236,107 @@ Throughout the project, every function at every stage either **does exactly what - [sceSaveDataInitialize3](../../core/libs/prx/libSceSaveData.native/Export.cpp) (libSceSaveData.native) - a repeated initialize succeeds: PPSA12544's executable and its Unity SaveData plugin both initialize, and the plugin fails on any error. Initializations are counted and `sceSaveDataTerminate` ends the session at the last one; how the console pairs them is unknown - [sceVoiceSetMuteFlag](../../core/libs/prx/libSceVoice/Export.cpp) (libSceVoice) - unknown signature - [sceTextToSpeech2GetSystemStatus](../../core/libs/prx/libSceTextToSpeech2/Export.cpp) (libSceTextToSpeech2) - unknown signature -- [sceVdecswQueryComputeMemoryInfo](../../core/libs/prx/libSceVdecsw/Export.cpp) (libSceVdecsw) - unknown signature -- [sceVdecswFinalizeDecodeSequence](../../core/libs/prx/libSceVdecsw/Export.cpp) (libSceVdecsw) - unknown signature -- [sceVdecswQueryDecoderMemoryInfo](../../core/libs/prx/libSceVdecsw/Export.cpp) (libSceVdecsw) - unknown signature -- [sceVdecswSetDecodeInput](../../core/libs/prx/libSceVdecsw/Export.cpp) (libSceVdecsw) - unknown signature -- [sceVdecswReleaseComputeQueue](../../core/libs/prx/libSceVdecsw/Export.cpp) (libSceVdecsw) - unknown signature -- [sceVdecswAllocateComputeQueue](../../core/libs/prx/libSceVdecsw/Export.cpp) (libSceVdecsw) - unknown signature -- [sceVdecswGetAvcPictureInfo](../../core/libs/prx/libSceVdecsw/Export.cpp) (libSceVdecsw) - unknown signature -- [sceVdecswTrySyncDecodeOutput](../../core/libs/prx/libSceVdecsw/Export.cpp) (libSceVdecsw) - unknown signature -- [sceVdecswTrySyncDecodeInput](../../core/libs/prx/libSceVdecsw/Export.cpp) (libSceVdecsw) - unknown signature -- [sceVdecswSetDecodeOutput](../../core/libs/prx/libSceVdecsw/Export.cpp) (libSceVdecsw) - unknown signature -- [sceVdecswResetDecoder](../../core/libs/prx/libSceVdecsw/Export.cpp) (libSceVdecsw) - unknown signature - [sceCoredumpAttachUserMemoryFile](../../core/libs/prx/libkernel/System/src/Coredump.cpp) (libkernel) - unknown signature - [sceCoredumpAttachMemoryRegion](../../core/libs/prx/libkernel/System/src/Coredump.cpp) (libkernel) - unknown signature - [sceKernelAddTimerEvent](../../core/libs/prx/libkernel/Equeue/Equeue.cpp) (libkernel) - `usec` is taken as the period in microseconds; re-adding an existing timer keeps its pending expiration count and restarts the period from the time of the call, where FreeBSD 12+ `filt_timertouch` clears pending expirations on `EV_ADD`; `usec == 0` fires on every wait. None of this is confirmed on a PS5 title - [sceRtcFormatRFC2822, sceRtcFormatRFC2822LocalTime, sceRtcFormatRFC3339LocalTime](../../core/libs/prx/libSceRtc/Export.cpp) (libSceRtc) - only shadPS4 (PS4) has bodies. A null tick returns `INVALID_POINTER` like the existing `sceRtcFormatRFC3339`, where shadPS4 formats the current time instead; offsets outside ±1439 minutes return `INVALID_VALUE` like `sceRtcFormatRFC3339`; the local-time variants use the host's offset at the given instant (shadPS4 uses `sceKernelGettimezone`, the current offset) - [snwprintf_s](../../core/libs/prx/libc/src/FormattingWide.cpp) (libc) - follows C11 K.3.9.1.3 with `RSIZE_MAX` assumed to be `SIZE_MAX >> 1`. A runtime-constraint violation only returns a negative value: `set_constraint_handler_s` is not exported and the console's default handler is unknown. An invalid multibyte `%s` argument is copied byte by byte instead of being reported as an encoding error, as in `vswprintf` +- [libSceUlt ulthreads](../../core/libs/prx/libSceUlt/Export.cpp) - `_sceUltUlthreadRuntimeOptParamInitialize`, `_sceUltUlthreadRuntimeCreate` and `_sceUltUlthreadCreate` are assumed to take the same arguments as the existing exports without the underscore, including the trailing build version. `sceUltUlthreadTryJoin` is assumed to return `ULT_ERROR_BUSY` while the ulthread runs, and `sceUltUlthreadRuntimeDestroy` to return `ULT_ERROR_BUSY` while an ulthread created on the runtime is not joined. Neither error has been confirmed on PS5 hardware - [libSceUlt queues](../../core/libs/prx/libSceUlt/Export.cpp) - the added `sceUltQueuePop`, `sceUltQueueTryPush`, `sceUltQueueDestroy` and `sceUltQueueDataResourcePoolDestroy` signatures are inferred from the existing push/pop and object APIs. Queue destruction is assumed to reject blocked callers with `ULT_ERROR_BUSY` and otherwise discard queued data; pool exhaustion returns `ULT_ERROR_AGAIN`. These rules, alignment checks and shared slot limits have not been confirmed on PS5 hardware or a title. Non-null queue and data-pool option parameters throw because their layouts are unknown. +- [AC1FtjqMCL0](../../core/libs/prx/libkernel/Unimplemented.cpp) (libkernel) - unknown name, signature +- [0iEKNAvT600](../../core/libs/prx/libkernel/Unimplemented.cpp) (libkernel) - unknown name, signature +- [2cELEBPdYQ0](../../core/libs/prx/libkernel/Unimplemented.cpp) (libkernel) - unknown name, signature +- [5xYxiOtOccA](../../core/libs/prx/libkernel/Unimplemented.cpp) (libkernel) - unknown name, signature +- [6W+shIH315Q](../../core/libs/prx/libkernel/Unimplemented.cpp) (libkernel) - unknown name, signature +- [7ilObq815O4](../../core/libs/prx/libkernel/Unimplemented.cpp) (libkernel) - unknown name, signature +- [7iq7Hfs8HBk](../../core/libs/prx/libkernel/Unimplemented.cpp) (libkernel) - unknown name, signature +- [AMJZhpC+siY](../../core/libs/prx/libkernel/Unimplemented.cpp) (libkernel) - unknown name, signature +- [E9XkHowt+dY](../../core/libs/prx/libkernel/Unimplemented.cpp) (libkernel) - unknown name, signature +- [GADjreszOgM](../../core/libs/prx/libkernel/Unimplemented.cpp) (libkernel) - unknown name, signature +- [Giyg2xGrXDU](../../core/libs/prx/libkernel/Unimplemented.cpp) (libkernel) - unknown name, signature +- [KC3y21wrqzc](../../core/libs/prx/libkernel/Unimplemented.cpp) (libkernel) - unknown name, signature +- [Qb8wS4GZUsw](../../core/libs/prx/libkernel/Unimplemented.cpp) (libkernel) - unknown name, signature +- [aAQZSjb2fho](../../core/libs/prx/libkernel/Unimplemented.cpp) (libkernel) - unknown name, signature +- [pnz21I6VjDs](../../core/libs/prx/libkernel/Unimplemented.cpp) (libkernel) - unknown name, signature +- [qk043QPsrZU](../../core/libs/prx/libkernel/Unimplemented.cpp) (libkernel) - unknown name, signature +- [uDGXwUKqi8s](../../core/libs/prx/libkernel/Unimplemented.cpp) (libkernel) - unknown name, signature - [LinuxProtFromSce](../../core/libs/prx/libkernel/DirectMemory/DirectMemory.cpp) (libkernel) - protection bits 0x100 and 0x200 are accepted and assumed not to affect host CPU access; host protection is derived only from the read, write and execute bits - [_sceUlobjmgrRegisterObject](../../core/libs/prx/ulobjmgr/Export.cpp) (ulobjmgr) - parameters taken from shadPS4 (PS4): a nonzero 64-bit object, a nonzero 32-bit kind and a 32-bit id output; the meaning of the first two and the PS5 signature are unverified. The id bound of `_sceUlobjmgrUnregisterObject` (below 0x4000) and the raw `EINVAL` (22) return also come from shadPS4 - [sceShareGetCurrentStatus](../../core/libs/prx/libSceShare/Export.cpp) (libSceShare) - validation (feature flag non-zero, status non-null) and the all-zero 16-byte status taken from the [KytyPS5](https://github.com/KytyPS5/KytyPS5) reimplementation, not confirmed on a PS5 title; the meaning of the recording status values is unknown - [sceShareGetRunningStatus](../../core/libs/prx/libSceShare/Export.cpp) (libSceShare) - only the first 32-bit word of the status is known: PPSA01317 treats bits 0x3c of it as a running share feature and gives the call a 40-byte buffer. The host never records or broadcasts, so it stores 0 in that word and leaves the rest untouched; a null status returns `INVALID_PARAM` by analogy with `sceShareGetCurrentStatus` +- [sceShareSetContentParamForApplicationTitle](../../core/libs/prx/libSceShare/Export.cpp) (libSceShare) - imported by PPSA12544's Unity plugins; the signature follows [KytyPS5](https://github.com/KytyPS5/KytyPS5/blob/main/src/libs/libShare.cpp), not confirmed on a PS5 title. The application title is only checked for null, as KytyPS5 does, since nothing is captured - [sceAgcDcbSetZPassPredicationEnableGetSize, sceAgcDcbSetPredicationDisableGetSize, sceAgcDcbSetBoolPredicationEnableGetSize](../../core/libs/prx/libSceAgc/DcbState/src/Predication.cpp) (libSceAgc) - the builders are inline in the SDK and not exported; the size is assumed to be that of the SET_PREDICATION packet `sceAgcDcbSetPredication` writes (16 bytes) - [Resource registration](../../core/libs/prx/libSceAgcDriver/Resource/src/Registration.cpp) (libSceAgcDriver) - registration always fails with `RESOURCE_REGISTRATION_UNAVAILABLE`, so the owner and resource queries, `sceAgcDriverSetResourceUserData`, `sceAgcDriverFindResourcesPublic` and `sceAgcDriverUnregisterAllResourcesForOwner` return the same error without reading their arguments. Their parameter lists follow the libSceGnmDriver counterparts and are unverified. `sceAgcDriverGetResourceRegistrationMaxNameLength` writes 0xfc and returns 0, as [prosper](https://github.com/mattias800/prosper) does from a title's registration configuration; the `uint32_t*` parameter matches the SDK wrapper `sce::Agc::ResourceRegistration::getMaxNameLength(unsigned int*)`. sharpemu writes 256 and KytyPS5 returns `RESOURCE_REGISTRATION_UNAVAILABLE`, so the value is unverified; a null output throws +- [sceAgcDriverRegisterMultipleResources](../../core/libs/prx/libSceAgcDriver/Resource/src/Registration.cpp) uses the existing unavailable-registration state and leaves caller memory untouched. Its four arguments (uint32 handle output, uint32 owner, descriptor pointer, uint32 count) follow the original PPSA34502 / 01.010.000 codzm call. The caller builds 40-byte descriptors and explicitly accepts 0x8A6C9018 with an invalid output handle. Enabled registration and console argument-validation precedence are unverified. - [Shader registration](../../core/libs/prx/libSceAgcDriver/Execution/src/Driver/Shaders/ShaderRegistry.cpp) (libSceAgcDriver) - a shader header is accepted at any address: PPSA26344 passes headers 4 and 1 bytes past an 8-byte boundary; whether the console refuses an alignment is unknown +- [Shader header copies](../../core/libs/prx/libSceAgcDriver/Execution/src/Driver/Shaders/ShaderRegistry.cpp) (libSceAgcDriver) - a header bound for a draw or dispatch at another address than the registered one is accepted when its fixed fields match the registered header except `user_data`; the registered header's metadata is used. PPSA21567 binds such a copy of a vertex shader header with `user_data` set to null. Whether the console reads the copy's `user_data` is unknown - [sceNgs2VoiceControl sampler filter](../../core/libs/prx/libSceNgs2.native/src/Voice.cpp) (libSceNgs2.native) - param `0x1000000a`, its layout and meaning follow KytyPS5 only: location 1 type 1 is an RBJ low-pass biquad at the system sample rate after resampling, scaled by `level`, a set `channel_mask` bit bypasses that channel, type 0 turns the filter off. Other locations and types throw. The voice keeps playing a filter tail after its last block until the history is zero, with outputs below 1e-20 flushed to zero (a KytyPS5 choice); the console's tail length is unknown +- [sceNgs2PanInit / sceNgs2PanGetVolumeMatrix](../../core/libs/prx/libSceNgs2.native/src/Pan.cpp) (libSceNgs2.native) - follow the PS4 firmware 11.00 `libSceNgs2.sprx`, read and checked against its output for 2, 4, 5 and 7 speakers into 1, 2, 6 and 8 channels; not yet confirmed on the PS5 library. Speakers are taken in the order L R, L R Ls Rs, L R C Ls Rs or L R C Ls Rs Lb Rb and kept in a fixed circular order, not sorted; without angles the defaults are 90/270, 40/110/250/320, 0/40/110/250/320 and 0/40/110/150/210/250/320 degrees. As in that library, defaults converted to radians keep the wrap-around entry in degrees, so angles past the last speaker pan almost fully to it. Error codes are not known for the PS5, so invalid arguments throw, including a speaker angle equal to the unit angle, on which that library loops forever. Angles so large that adding or subtracting a turn leaves them unchanged throw for the same reason - [sceRtcGetDosTime, sceRtcSetDosTime](../../core/libs/prx/libSceRtc/Export.cpp) (libSceRtc) - bit layout from shadPS4 (PS4, the standard FAT date/time). A year outside 1980-2107 makes `GetDosTime` return `INVALID_YEAR` after writing 0 for an earlier one and `0xff9fbf7d` (2107-12-31 23:59:58, the largest encodable time) for a later one, and `SetDosTime` decodes the year as the unsigned 7-bit FAT field (1980-2107) without validating the fields, as the disassembly-derived [fpPS4](https://github.com/red-prig/fpPS4) reimplementation of the PS4 library does; not confirmed on a PS5 title, and shadPS4 instead masks out-of-range years into the packed value and sign-extends the 16-bit date - [sce::Json::InitParameter2](../../core/libs/prx/libSceJson2/Export.cpp) (libSceJson2) - layout inferred, not known: assumed to start with the allocator, its user data and the file buffer size, in that order, and the whole struct is assumed to fit in 40 bytes; `Initializer::initialize(InitParameter2)` ignores them as `initialize(InitParameter)` does - [sceAjmBatchJobSetResampleParameters, sceAjmBatchJobGetResampleInfo](../../core/libs/prx/libSceAjm.native/src/Ajm.cpp) (libSceAjm) - the ratio is taken as decoded samples per output sample and num_samples as the decoded samples the resampler holds, both inferred from FMOD (Hades II); the console's interpolation is unknown, a 4-point Catmull-Rom is used. The flags argument is ignored, as in shadPS4 and Kyty - [scePlayerReviewDialogOpen, scePlayerReviewDialogGetResult](../../core/libs/prx/libSceCdlgPlayerReview/Export.cpp) (libSceCdlgPlayerReview) - the PS5-only dialog has no public reference: the parameter is not read and the result is assumed to start with the 32-bit common dialog result, as `sceWebBrowserDialogGetResult` assumes - [sceHttpWaitRequest](../../core/libs/prx/libSceHttp/Export.cpp) (libSceHttp) - the timeout is assumed to be in microseconds and a negative timeout to wait without limit; the invalid-value error for a null handle or event array or a non-positive event count is assumed, not measured. No request is ever pending, so the call waits for the timeout and returns 0 events, and a wait without limit throws +- [sceVideoOutAddPreVblankStartEvent](../../core/libs/prx/libSceVideoOut/src/Event.cpp) (libSceVideoOut) - how long before the vblank the console signals pre-vblank start is unknown. The event is triggered on each emulated vblank tick immediately before the vblank event, with its own count as the event data +- [sched_get_priority_max, sched_get_priority_min](../../core/libs/prx/libkernel/System/src/Process.cpp) (libkernel) - `SCHED_OTHER` (2) reports the same 256 to 767 range as `SCHED_FIFO` and `SCHED_RR`; the console's answer is unverified. FreeBSD accepts `SCHED_OTHER` with its own range, and shadPS4 rejects it with `EINVAL`. `pthread_setschedparam` ignores the policy, so a priority taken from this range applies + +- [gethostbyname](../../core/libs/prx/libScePosixForWebKit/Resolver.cpp) (libScePosixForWebKit) - numeric IPv4 forms, trailing whitespace and 64-bit accumulation follow FreeBSD 11.4 [inet_aton](https://github.com/freebsd/freebsd-src/blob/releng/11.4/lib/libc/inet/inet_addr.c); clearing the resolver error on numeric success follows [fakeaddr](https://github.com/freebsd/freebsd-src/blob/releng/11.4/lib/libc/net/gethostnamadr.c). Exact PS5 behavior is unverified; hostname lookup still uses the host resolver ### Functional +- [libSceOpusDec](../../core/libs/prx/libSceOpusDec/Export.cpp) uses FFmpeg for 48000 Hz mono/stereo signed-16 decoding. The six signatures, 640-byte opaque state allocation and byte-count return follow [public PS5 decoder research](https://github.com/blackbearreloaded/ps5-hardware-audio-decoding/blob/26be5a12f92b2cdc67f8a9e082fd69ae0e9a35a2/docs/DECODING.md#direct-opus-abi) and [ProsperoRadio](https://github.com/blackbearreloaded/ProsperoRadio/blob/2c4029c3a8f43e9e1dfef98374d94d99b14d745d/src/opus_decoder.cpp). Console PCM equivalence and error codes are unverified. Other sample rates, packet loss concealment, invalid inputs and termination with live decoders throw. Context identifiers and host decoder state are private registries; the firmware's opaque state layout and error recovery are unknown. +- [mkstemp and isatty](../../core/libs/prx/libScePosixForWebKit/Filesystem.cpp) (libScePosixForWebKit) follow FreeBSD's [temporary-file API](https://github.com/freebsd/freebsd-src/blob/stable/9/lib/libc/stdio/mktemp.c) and [terminal check](https://github.com/freebsd/freebsd-src/blob/stable/9/lib/libc/gen/isatty.c), without console verification. `mkstemp` replaces all trailing `X` characters, accepts a fixed name, and creates files through the guest path resolver with exclusive read/write access and mode 0600. Windows file permissions follow the existing kernel file implementation. `isatty` tests Windows console handles rather than all character devices; files, pipes and guest sockets return `ENOTTY`, closed descriptors return `EBADF`. +- [libSceZlib](../../core/libs/prx/libSceZlib/Export.cpp) completes RFC1950 decompression during submission and supports one outstanding request, null-timeout waits and one result retrieval. Timed waits, multiple outstanding requests, repeated retrieval and finalization with an unread result throw. The 64-bit initialization length and 64 KiB output limit with 2 KiB destination alignment follow [SharpProspero](https://github.com/SvenGDK/SharpProspero/blob/5dfdb4b0b166e83dc9ae275001afb695f12c7174/src/SharpProspero/Interop/Compression/Zlib.cs); the PS4 initialization length is 32 bits. shadPS4 also requires the output length to be a multiple of 2 KiB and returns `0x81120016` for invalid arguments; this implementation accepts any length up to 64 KiB (SharpProspero documents only the pointer alignment) and throws on invalid arguments. Scratch memory is not used, and its firmware size/alignment requirements are unknown. The initialization errors follow SharpProspero; malformed/checksum/truncated streams report `0x811200FF` and insufficient output `0x8112001C` as in [shadPS4](https://github.com/shadps4-emu/shadPS4/blob/0fe263a4760dfbfa973366890061749b4af0de97/src/core/libraries/zlib/zlib.cpp). PS5 error precedence, scheduling, completion selection, result consumption and cancellation are unverified. The payload SDK's FreeBSD kernel `z_stream` and host errno are not this library's ABI. +- [Windows file timestamps](../../core/libs/prx/libkernel/File/include/WindowsFileTime.hpp) use the host filesystem's timestamp range and resolution. `utimes` and `futimes` reject timestamps outside nonnegative signed 64-bit Windows file-time ticks with `EINVAL`, including exactly 1601-01-01 00:00:00 UTC, whose zero value means "leave unchanged" to `SetFileTime`. Guest directory descriptors still resolve through their recorded path. Filesystems with coarser timestamp resolution cannot retain every guest microsecond +- [Windows file identity](../../core/libs/prx/libkernel/File/src/NativeStat.cpp) (libkernel) - `stat` and `fstat` report the volume serial number as `st_dev` and the 64-bit file index folded into the guest's 32-bit `st_ino` (never 0), so two different files can in principle share an `st_ino`; the fold is not taken from a reference. `st_nlink` is clamped to 65535, the width of the guest field. The call keeps the `_wstat64` values when `GetFileInformationByHandle` fails +- On Linux, [direct memory](../../core/libs/prx/libkernel/DirectMemory/DirectMemory.cpp) lives in one sparse memfd at file offset = physical address, so the [GPU imports](../../core/libs/prx/libSceAgcDriver/Graphics/src/GuestBufferMemory.cpp) can share one udmabuf buffer object per 64 MiB of it. Released direct memory is not punched out of the file, because a live import of its chunk pins the pages and the GPU would keep reading the old ones: its pages stay committed (the process keeps the high-water mark of the direct memory it used) and are zeroed when the range is allocated again. +- [Per-face depth bias](../../core/libs/prx/libSceAgcDriver/Graphics/src/Draw.cpp) (libSceAgcDriver): Vulkan has one depth bias per draw, so a draw that rasterizes both faces with different front and back bias (PA_SU_SC_MODE_CNTL, PA_SU_POLY_OFFSET_FRONT/BACK_*) is recorded twice, front faces with the back faces culled and then back faces with the front faces culled. The vertex stages run twice, so a vertex shader that writes memory writes twice, and all front faces land before all back faces instead of in primitive order, which differs only for blending or stencil updates that depend on the order. +- [Compute LDS beyond the device's shared-memory limit](../../core/libs/prx/libSceAgcDriver/Execution/src/VulkanDevice.cpp) (libSceAgcDriver) lives in a coherent storage buffer with one slice of `LDS bytes / 4 + 1` dwords per workgroup (the last dword is the atomic lock), shared by all such dispatches and grown in 1 MiB steps up to `maxStorageBufferRange`; a dispatch whose slices do not fit throws. It is slower than shared memory and its cost was not measured. An indirect dispatch of such a program always reads its group counts on the CPU. Its results are pinned against the in-limit semantics in `agc_driver_lds_beyond_device_limit`, not measured on hardware. +- [Graphics pipeline libraries](../../core/libs/prx/libSceAgcDriver/Graphics/src/PipelineLibrary.cpp) (libSceAgcDriver): a pipeline is fast-linked at the draw and relinked with `VK_PIPELINE_CREATE_LINK_TIME_OPTIMIZATION_BIT_EXT` on a background thread, and binds the optimized one from the next draw on. Draws recorded before that use the fast link, whose vertex stage keeps the outputs the fragment stage does not read: on RADV (Radeon 660M, Mesa 26.2.4) 346 of 483 Astro's Playroom vertex shaders paired with a fragment stage that reads none of them used more VGPRs (median 8 more, up to 2.5 times), and on an RTX 4090 13 of 15 PPSA36082 vertex shaders up to 68 instead of 38 (#2242). The optimized link compiles to the same code as a whole pipeline +- [sceAvPlayerGetVideoData](../../core/libs/prx/libSceAvPlayer/src/Source.cpp) (libSceAvPlayer) in sync mode `SCE_AVPLAYER_AV_SYNC_MODE_NONE` skips queued frames that the audio clock has already passed while the title pulls audio, so a title that requests video slower than the film's frame rate keeps the picture with the sound. Whether the console hands out every frame in this mode is unknown. PPSA12544 requests 7-19 frames per second for a 30 fps cinematic - [libSceHttp2 asynchronous requests](../../core/libs/prx/libSceHttp2/Export.cpp) (libSceHttp2) never reach the network: `sceHttp2SendRequestAsync` accepts the request, queues a completion whose result is the library's network error and triggers the user event named by the kqueue option (equeue at offset 0, event id at 8, user data at 16, layout inferred from PPSA17221); `sceHttp2WaitAsync` returns that completion. The completion's `event_type` value is unknown and left 0. PPSA17221 completes its request object only after `sceHttp2SendRequestAsync` returns, and its handler for a synchronous failure calls `shared_from_this` before that, so a send must not fail synchronously. A kqueue option or user event that is missing, an extra option, and a wait with no completed operation throw +- [system](../../core/libs/prx/libkernel/System/src/Process.cpp) (libkernel) runs no command. It follows FreeBSD 11's `lib/libc/stdlib/system.c` for a sandbox without `/bin/sh`: a null command returns 1 unconditionally, as that version does, and any command returns the wait status of a child that could not exec the shell and called `_exit(127)` (`0x7f00`). Whether the console's application sandbox has no shell, and whether `vfork` succeeds there (FreeBSD returns -1 when it fails), is unverified - [sceKernelWriteThrottlingStatus](../../core/libs/prx/libkernel/File/src/Stdio.cpp) (libkernel) reports no write throttling: the host SSD has no write budget, so the first qword of the 32-byte status is set to `UINT32_MAX` and the rest is zeroed. PPSA17221 reads only that qword: it multiplies it by 64 KiB and compares the product with byte thresholds up to 0x4AF00000 to pick one of five tiers. When the call fails the title assumes 0x4B000001 bytes, which selects the same top tier as `UINT32_MAX`. What the value measures on hardware (a remaining budget or a rate) and the meaning of the other three qwords are unknown -- [Wide-character conversions](../../core/libs/prx/libc/src/Strings.cpp) (libc) - `wcsrtombs` and [`swprintf`](../../core/libs/prx/libc/src/Formatting.cpp) still pass guest strings to the host's `wchar_t` functions. The guest's `wchar_t` is 16 bits, so on Linux, where the host's is 32 bits, they misread and miswrite guest strings. The wide `wcsto*` functions parse the ASCII prefix of the guest string, as the C locale accepts only ASCII white space and digits +- [/dev/random and /dev/urandom](../../core/libs/prx/libkernel/File/src/RandomDevice.cpp) (libkernel) opened for reading return a descriptor backed by the host null device whose reads are filled from `std::random_device`; both paths behave the same, never block, and positioned reads ignore the offset (FreeBSD `random(4)`). Only the exact paths are recognised, a `dup`ed descriptor reads as an empty file, `fstat`, `lseek`, `mmap` and `select` see the null device, and opening them for writing throws +- [Wide-character conversions](../../core/libs/prx/libc/src/Multibyte.cpp) (libc) - The wide `wcsto*` functions parse the ASCII prefix of the guest string, as the C locale accepts only ASCII white space and digits. [`wcsrtombs`](../../core/libs/prx/libc/src/Multibyte.cpp) uses the guest's single-byte C locale through `wcrtomb`; other encodings and shift states are not supported - [SPIRV-Tools optimization](../../core/shader/recompiler/SpirvBackend/src/SpirvOptimizer.cpp) skips CCP and Simplification for modules with storage-only 8- or 16-bit types because the pinned folder produces illegal narrow constants. This also disables unrelated constant folding; the restricted full pipeline mirrors the pinned performance recipe. +- [SRT signed comparison](../../core/shader/recompiler/Optimization/src/SrtWalker/SrtEvaluator.cpp) evaluates `SLessThan32` as signed 32-bit less-than, matching the existing SPIR-V backend and [SPIR-V `OpSLessThan`](https://registry.khronos.org/SPIR-V/specs/unified1/SPIRV.html#OpSLessThan). Signed boundary descriptor expressions are tested on the host; native RDNA2/PS5 execution and title behavior remain unverified. +- On macOS the [write watch](../../core/libs/prx/libc/src/GuestWriteWatch.cpp) write-protects watched pages and records the first write to each from a SIGBUS/SIGSEGV handler, because macOS has no counterpart to `userfaultfd` write-protect or `MEM_WRITE_WATCH`. The protection is real, so: + - a system call that writes into a protected page fails with `EFAULT` instead of faulting. `GuestArena::HostWrite` lifts the protection for the calls it brackets (the file reads); a host call that writes guest memory without it (sockets, `clock_gettime` and the like into a watched page) fails; + - code that changes the protection of watched pages, or maps over them, has to call `GuestWriteWatchProtectionChanged`, and code that asks the kernel for a page's protection has to ask `GuestWriteWatchProtection` first, since a watched page is read-only between a collection and the next write. Nothing calls either yet: the macOS memory paths are not in the tree; + - the handler is installed when libc loads and passes other faults to the handler installed before it. A handler installed later has to do the same. +- [Pending storage-image results](../../core/libs/prx/libSceAgcDriver/Graphics/src/Texture.cpp) (libSceAgcDriver) are stored when something reads their memory, not when the GPU work that wrote them completes. A title that waits for that work, frees the image and reuses its memory can have the new data overwritten by the late store (#1970). `APS5_STORE_AT_FLIP=1` stores every pending result at each flip; storing them at the title's GPU waits (labels, fences) is unimplemented. - Windows host-pointer imports of the guest mapping itself are compared instead of write-watched: a driver can mark imported pages dirty after the synchronous import window. Shared memory imported through a separate writable alias retains protection-based tracking on the guest views; driver access to the import alias does not disarm those views. [GuestMemory](../../core/libs/prx/libSceAgcDriver/Execution/src/GuestMemory.cpp) excludes every overlapping 64 KiB tracker block while its backing is retained. A full block replaced by fresh private backing regains tracking and gets a new generation; protection changes, partial replacements and shared remappings do not restore it. This preserves CPU edits through byte comparisons but disables snapshot reuse and unit shadows for excluded blocks; restoring tracking on import retirement without replacing the backing and measuring the comparison cost remain unimplemented. +- [shm_open](../../core/libs/prx/libkernel/File/src/SharedMemory.cpp) (libkernel) creates only anonymous objects (`SHM_ANON`), each backed by a host anonymous file: a `memfd` on Linux, a delete-on-close temporary file on Windows. The flag checks follow FreeBSD 11's `kern_shm_open`; the mode is ignored. Named objects throw. The descriptor cannot be mapped yet: `mmap` accepts only private anonymous mappings and fails a descriptor mapping with `EOPNOTSUPP`, so the known caller (the bundled Mesa in PPSA59322, which truncates the object and maps it `MAP_SHARED`) gets that failure - [sceNetSendmsg and sceNetRecvmsg](../../core/libs/prx/libSceNet/Export.cpp) (libSceNet) copy the scatter/gather list through one host buffer and accept only the flags `sceNetSend`/`sceNetRecv` accept. Sending control data throws; a receive never returns control data (`msg_controllen` 0), since no socket option that queues it is supported - [Additional content](../../core/libs/prx/libSceAppContent/Export.cpp) (libSceAppContent) is not installed or mounted: `sceAppContentAddcontMount` answers `NOT_FOUND` for every entitlement label and `sceAppContentAddcontUnmount` for every mount point, as no add-content mount is ever handed out; `sceAppContentGetAddcontInfo` answers `DRM_NO_ENTITLEMENT` for every label and `sceAppContentGetAddcontInfoList` reports 0 entries - [sceKernelPread, sceKernelPwrite, sceKernelPreadv, sceKernelPwritev](../../core/libs/prx/libkernel/File/src/Stdio.cpp) (libkernel) on Windows transfer at the given offset with an overlapped `ReadFile`/`WriteFile` and restore the descriptor's file pointer afterwards, so a concurrent `read`/`write` on the same descriptor can see the moved pointer in between, unlike on the console. Handles that are not disk files return `ESPIPE`. `readv` on a pipe or console handle stops after the first iovec read, a short read the console is assumed to allow; each iovec is transferred separately, so a failure after the first transfer returns the bytes already done -- [sceKernelSync](../../core/libs/prx/libc/src/General.cpp) (libkernel) on Windows flushes only the host files and directories recorded through `RecordWrittenPath_nid_no_patch` (the libc, libkernel, save data and app content file functions) and their changed parent directories. A host write in a library that does not record its path is not flushed by `sceKernelSync` -- [sceKernelSync](../../core/libs/prx/libc/src/General.cpp) (libkernel) on Windows skips a recorded file it cannot open for writing (read-only, or held open by a handle that denies write sharing), since `FlushFileBuffers` needs a writable handle. Such a file's data may not reach the disk before a host crash; the guest reads the same data either way +- [sceKernelAioSubmitReadCommands, sceKernelAioSubmitWriteCommands and their Multiple variants](../../core/libs/prx/libkernel/Aio/src/Aio.cpp) (libkernel) on Windows transfer through the same positioned path as `sceKernelPread`/`sceKernelPwrite` and restore the descriptor's file pointer afterwards, so a concurrent `read`/`write` on the same descriptor can see the moved pointer in between, unlike on the console +- [sceKernelGetFsSandboxRandomWord](../../core/libs/prx/libkernel/System/src/Sandbox.cpp) (libkernel) returns a fixed directory name, `gkbHtXc3Wq`. On the console the name is the process's randomized path (syscall 602, `randomized_path`), and the sandbox exposes system files under it, such as `common/lib` and `common/cert`. Nothing is mounted under the name here, so `/gkbHtXc3Wq/common/...` resolves under the guest root like any other path and does not exist +- [sceKernelSync](../../core/libs/prx/libc/src/General.cpp) (libkernel) flushes only the host files and directories recorded through `RecordWrittenPath_nid_no_patch` (the libc, libkernel, save data and app content file functions) and their changed parent directories. A host write in a library that does not record its path is not flushed by `sceKernelSync` +- [sceKernelSync](../../core/libs/prx/libc/src/General.cpp) (libkernel) on Windows skips a recorded file it cannot open for writing (read-only, or held open by a handle that denies write sharing), since `FlushFileBuffers` needs a writable handle. On Linux, it skips a recorded file it cannot open for reading. Such a file's data may not reach the disk before a host crash; the guest reads the same data either way - [sceKernelInstallExceptionHandler](../../core/libs/prx/libkernel/System/src/Exception.cpp) records the handler, but a host fault still ends the process: only signals raised with `sceKernelRaiseException` are delivered. Unity's PS5Util module installs a SIGUSR1 handler that reads `uc_mcontext.mc_rsp` (offset 0xf8 of the context) and raises SIGUSR1 on other threads to stop them for garbage collection - [sceKernelRaiseException](../../core/libs/prx/libkernel/System/src/Exception.cpp) delivers on Windows only and throws on Linux - [sceKernelRaiseException](../../core/libs/prx/libkernel/System/src/Exception.cpp) builds the handler's `ucontext` with FreeBSD's amd64 layout (16-byte sigmask, `mcontext` at 0x40, `mc_rsp` at 0xf8); the console's layout is not verified beyond the fields PS5Util reads - [sceKernelRaiseException](../../core/libs/prx/libkernel/System/src/Exception.cpp) does not consult the target thread's signal mask -- [sceKernelRaiseException](../../core/libs/prx/libkernel/System/src/Exception.cpp) saves and restores only `CONTEXT_FULL | CONTEXT_FLOATING_POINT` on Windows, so the upper halves of the YMM registers are not preserved if the handler or the entry uses AVX - [sceKernelRaiseException](../../core/libs/prx/libkernel/System/src/Exception.cpp) reaches a thread that waits in an alertable `TimedWait` wait through an APC. A thread blocked anywhere else, including all waits under `APS5_COARSE_TIMED_WAITS` and host waits inside the emulator, is redirected and runs the handler only when that wait returns +- [sceKernelRaiseException](../../core/libs/prx/libkernel/System/src/Exception.cpp) on Windows runs the handler wherever the target thread was interrupted. It waits only while the thread is inside winpthreads or restoring a context through `NtContinue`, and throws if that lasts 1 s, so a handler that calls into other host code the thread was interrupted in, such as the C runtime's heap or a lock of the emulator, can deadlock on it or reenter it +- [sigaltstack](../../core/libs/prx/libkernel/System/src/Signals.cpp) (libkernel) records each thread's alternate signal stack and reports it back, following FreeBSD's `kern_sigaltstack` (`EINVAL` for flags other than `SS_DISABLE`, `ENOMEM` below `MINSIGSTKSZ` 2048), but handlers never run on it, so `SS_ONSTACK` and `EPERM` are never reported - [libSceUlt queues](../../core/libs/prx/libSceUlt/Export.cpp) use preallocated host storage instead of the supplied guest work area; its existing size formula and the 512-byte object layouts remain unverified. Waiting-queue pool thread limits and waiter priority ordering are not implemented. Finalization wakes blocked queue callers with `ULT_ERROR_STATE`. -- [Thread cancellation](../../core/libs/prx/libkernel/Pthread/src/Thread.cpp) (libkernel) is not modelled: `scePthreadCancel` throws, so no cancel request is ever pending and `scePthreadTestcancel` and `pthread_testcancel` return without acting; the cancel state and type set by `scePthreadSetcancelstate` and `scePthreadSetcanceltype` are stored and not used +- [Thread cancellation](../../core/libs/prx/libkernel/Pthread/src/Thread.cpp) (libkernel) follows FreeBSD libthr (states and types 0/1 and 0/2, `PTHREAD_CANCELED` 1, the `_pthread_cleanup_info` node layout), not measured on the console. A request is acted on only in `scePthreadTestcancel`, the condition variable, `sem` and `scePthreadSem` waits, `scePthreadJoin` and the sleeps, and their POSIX names; file and socket calls are not cancellation points. A sleep is not interrupted: the thread exits when it ends. The asynchronous type acts like the deferred one, except at the two set calls. - [MIMG `tfe` and `lwe`](../../core/shader/recompiler/RdnaDecoder/src/RdnaImageOpDecoder.cpp) never write the status VGPR after the data. On hardware it is written only when a texel fetch hits an unmapped page of a partially resident texture, or, for `lwe` on sample/gather, when the LOD is below the T# `MIN_LOD_WARN`; texture memory is always resident here, the LOD warning (texture-streaming feedback, like the T# `MIN_LOD_WARN` itself) is never reported, and `lwe` on `image_get_lod` throws. - [Packed horizontal gathers](../../core/shader/recompiler/SpirvBackend/src/SpirvImageEmitter.cpp) (`image_gather4h_pck`, MIMG opcode 98): the four window texels are read raw and concatenated into a little-endian bitstream, with DMASK bit i writing stream word i to the next destination register; measured on RDNA2 (gfx1031) at every edge: the anchor is floor(x-0.5) with elements at -1, 0, +1 and +2 clamped horizontally into the width, the row is floor(y), negative rows clamp to row 0, rows at or past the height return zeros, sampler addressing modes do not affect the window, and formats wider than 32 bits per texel write zeros in every register. The emitter fetches each window texel at LOD 0 and packs its bits like `image_load_pck`, so the sampler's filter, addressing and LOD state are not used; formats whose bits are not recoverable from the view and D16 data throw. The anchor and row multiply the normalized coordinate by the image size without contraction. The stream rule was measured for R8_UNORM, RG8_UNORM and RGBA8; only RGBA8 has an execution test. `image_gather8h_pck` (opcode 99) never returns texel data on this hardware - across dmasks 0x1 to 0xFF including the upper nibble, RGBA8, RGBA16 and RGBA32 formats, 1D, 2D and 3D descriptors and normalized and unnormalized sampling it only zero-fills popcount(dmask) registers (sentinel-prefilled destinations) - so it decodes but throws at translation; its eight-texel semantics remain unmeasured. +- [waitpid](../../core/libs/prx/libkernel/System/src/Process.cpp) (libkernel) returns -1 with `ECHILD` (`EINVAL` for unknown options): nothing here creates a child process; child processes would need a real `waitpid`. - [MIMG BY2/BY4 loads](../../core/shader/recompiler/SpirvBackend/src/SpirvImageEmitter.cpp) (`image_load[_mip]_by2/by4`, opcodes 0x42, 0x43, 0x4a and 0x4b) were measured on RDNA2 (gfx1035) with the [hardware oracle](HW_ORACLE.md) for 2D images with identity swizzle. They read N texels (BY2: 2, BY4: 4) from `x & ~(N - 1)` on row y, texel by texel and component by component within each texel, with each texel converted like `image_load`. Every register is zero when the mip level is past the last level, `x + N - 1` is outside the level's width (unaligned x, unsigned), or y is outside its height. Data only comes back when the N texels fit in one dword: R8 and R16 (DMASK 0x3) or RG8 (0xf) for BY2, and R8 (0xf) for BY4. With 32-bit texels (R32, RG32, RG16, RGBA8) the hardware returns zeros. Other formats, masks, dimensions, swizzles, D16/A16 and indirect descriptors throw. The stores (0x52, 0x53, 0x5a and 0x5b) decode but throw at translation. Measured on hardware, they write from the unaligned x up to the end of its dword, which is not implemented. +- [2D stores to 1D storage images](../../core/shader/recompiler/SpirvBackend/src/SpirvImageEmitter.cpp): an `image_store` with DIM 2D through a 1D T# writes texel x and ignores y, as measured on RDNA2 (gfx1035) for y from 0 to 4 and 0xffffffff, single- and four-component formats and a 1x1 8_8_8_8_UNORM T#: the T# type decides the addressing, DIM only how many address VGPRs are read. Storage loads, atomics and `image_store_mip` through such a pair still throw - [MIMG PCK2/PCK4 loads](../../core/shader/recompiler/SpirvBackend/src/SpirvImageEmitter.cpp) (`image_load[_mip]_pck2/pck4`, opcodes 0x70, 0x71, 0x73 and 0x74) were measured on RDNA2 (gfx1035) with the [hardware oracle](HW_ORACLE.md) for 2D images with identity swizzle. With data mask 0x1, they write one dword: the raw bits of the N texels (PCK2: 2, PCK4: 4) from `x & ~(N - 1)` on row y, concatenated (element k at bit `k * texel bits`) and zero-extended. The dword is zero when the mip level is past the last level, `x + N - 1` is outside the level's width (unaligned x, unsigned), or y is outside its height. Data only comes back when N texels fit in 32 bits: R8, R16 or RG8 for PCK2, R8 for PCK4. Wider texels (R32, RG16, RGBA8) and masks other than 0x1 write zeros on hardware. Those masks throw at decode, and formats outside UNORM/UINT/SINT R8, R16 and RG8 throw. #1314's earlier measurement saw only zeros because it used masks 0x3 and 0xf. The stores (0x76, 0x77, 0x79 and 0x7a) decode but throw at translation. They are not measured, and opcode 0x76 on 32_32_32_32_FLOAT hung the GPU (#1314). +- [3D image accesses bound to a 2D T#](../../core/shader/recompiler/Optimization/src/ResourceMaterializer.cpp) follow CouchRiv's [hardware oracle](HW_ORACLE.md) measurement on RDNA2 (gfx1036, #2023), not on the PS5's gfx1013: `image_sample_lz` ignores r and reads the (u,v) texel, and `image_load` without a mip reads the (x,y) texel when z is 0 and returns zero otherwise. Every other access of such an image, including samples with an LOD, bias, derivatives, offset or comparison, gathers, queries and stores, keeps throwing because it is not measured. - [Unnormalized S# coordinates](../../core/libs/prx/libSceAgcDriver/Graphics/src/GuestSamplerResource.cpp) (libSceAgcDriver) - `FORCE_UNNORMALIZED` uses a Vulkan `unnormalizedCoordinates` sampler, so the S# LOD range, LOD bias and mip filter are not applied, which is exact only on a one-level view. - [Unnormalized S#s](../../core/shader/recompiler/Optimization/src/DescriptorBindingBuilder.cpp) are accepted only when direct and used only by explicit-LOD `image_sample` without offset, comparison, derivatives or `_a`, on a single-level, single-layer 1D or 2D view of mip 0, even when the image is also sampled through a normalized S#. - An [unnormalized S#](../../core/libs/prx/libSceAgcDriver/Graphics/src/GuestSamplerResource.cpp) (libSceAgcDriver) throws with unequal minification and magnification filters, anisotropy, `TRUNC_COORD`, `MC_COORD_TRUNC` or X/Y clamping other than last-texel or border. - [Image atomics](../../core/shader/recompiler/Optimization/src/ResourceMaterializer.cpp) on 32_SINT and 32_FLOAT surfaces act on the raw dword through an R32_UINT storage view; their float results are assumed to match those measured on 32_UINT surfaces. Other formats throw. -- [`s_memtime`](../../core/shader/recompiler/SpirvBackend/src/SpirvModuleEmitter.cpp) on RADV with a GFX10.3 to GFX11.7 GPU (RDNA 2, RDNA 3, RDNA 3.5) reads the Vulkan device clock instead of the subgroup clock, which [ACO](https://gitlab.freedesktop.org/mesa/mesa/-/blob/main/src/amd/compiler/instruction_selection/aco_select_nir_intrinsics.cpp) reads from the 20-bit `SHADER_CYCLES` register there and which wraps every 2^20 cycles. The device clock is a monotonic 64-bit counter, but it counts the 100 MHz reference clock (the same one as `s_memrealtime`), not shader cycles, so `s_memtime` deltas are about 20 times smaller than on hardware. The families are matched by the name RADV reports ([`NarrowSubgroupClock`](../../core/libs/prx/libSceAgcDriver/Execution/src/SubgroupClock.cpp)); a family added to Mesa later reads the subgroup clock until it is listed +- [`s_memtime`](../../core/shader/recompiler/SpirvBackend/src/SpirvModuleEmitter.cpp) on RADV with a GFX10.3 to GFX11.7 GPU (RDNA 2, RDNA 3, RDNA 3.5) reads the Vulkan device clock instead of the subgroup clock, which [ACO](https://gitlab.freedesktop.org/mesa/mesa/-/blob/main/src/amd/compiler/instruction_selection/aco_select_nir_intrinsics.cpp) reads from the 20-bit `SHADER_CYCLES` register there and which wraps every 2^20 cycles. The device clock is a monotonic 64-bit counter, but it counts the 100 MHz reference clock (the same one as `s_memrealtime`), not shader cycles, so `s_memtime` deltas are about 20 times smaller than on hardware. The families are matched by the name RADV reports ([`NarrowSubgroupClock`](../../core/libs/prx/libSceAgcDriver/Execution/src/SubgroupClock.cpp)); a family added to Mesa later reads the subgroup clock until it is listed. AMD's proprietary Windows driver also exposes a 20-bit subgroup clock on device 0x1114 (Radeon 860M, measured in #1615), device 0x744c (Radeon RX 7900 XT, measured in #1632), and device 0x747e (Radeon RX 7800 XT, measured in #1632); other AMD proprietary devices remain unverified. +- [`s_memtime` and `s_memrealtime`](../../core/shader/recompiler/SpirvBackend/src/SpirvModuleEmitter.cpp) on llvmpipe read only the low 32 bits of the host's nanosecond clock at either scope: Mesa's [`emit_clock`](https://gitlab.freedesktop.org/mesa/mesa/-/blob/main/src/gallium/auxiliary/gallivm/lp_bld_nir_soa.c) builds the high dword with a left shift by 32, so it is always 0 and both counters wrap every 2^32 ns (about 4.3 s). The [shader clock test](../../core/libs/prx/libSceAgcDriver/tests/execution/ShaderClock.cpp) compares them modulo 2^32 there - [`ds_wrxchg2_rtn_b32`, `ds_wrxchg2st64_rtn_b32`, `ds_wrxchg2_rtn_b64` and `ds_wrxchg2st64_rtn_b64`](../../core/shader/recompiler/Translation/src/MemoryInstructions/SharedMemoryInstructions.cpp) are translated as two separate 32-bit or 64-bit atomic swaps per invocation. RDNA2 hardware (measured on gfx1035) performs every lane's first exchange before any lane's second one, so when lanes of the same wave or of different waves overlap addresses, the result can differ from hardware. `offset0 == offset1` throws, although on hardware it is two exchanges in order. - [`ds_condxchg32_rtn_b64`](../../core/shader/recompiler/Translation/src/MemoryInstructions/SharedMemoryInstructions.cpp) uses two 32-bit atomic conditional exchanges. RDNA2 (gfx1035) measurements confirmed aligned return values, conditional bit clearing and 16-bit offsets; out-of-range accesses return zero without modifying LDS. Misaligned accesses fault the wave on gfx1035; the recompiler does not model this fault. Radeon 780M (gfx1103, RDNA3) measurements confirmed aligned return values, conditional bit clearing, EXEC masking and operand overlap in wave32 and wave64; contended cases passed per-dword conservation checks. Contended lane ordering and gfx1013 remain unmeasured; pair-wide indivisibility is not assumed, and concurrent 64-bit LDS atomics use a separate lock. GDS throws. VGPR pairs extending beyond v255 throw; allocated-VGPR EXEC nullification is not emulated. - [64-bit image atomics](../../core/shader/recompiler/Translation/src/MemoryInstructions/ImageInstructions.cpp) (`image_atomic_*` with a two-dword data mask) need `VK_EXT_shader_image_atomic_int64` and act on 32_32 UINT, SINT and FLOAT surfaces through an R64_UINT storage view. 64-bit `image_atomic_inc`, `image_atomic_dec` and the float atomics throw, as does any other access in the same shader to an image it updates with 64-bit atomics. @@ -276,51 +344,69 @@ Throughout the project, every function at every stage either **does exactly what - [8_SRGB and 8_8_SRGB textures](../../core/libs/prx/libSceAgcDriver/Graphics/src/TextureFormat.cpp) (libSceAgcDriver) that the device samples as R8_SRGB or R8G8_SRGB use its own decode, which need not match gfx1035: NVIDIA's R8_SRGB is one bf16 step off in 29 of 256 codes. - [Samples and gathers](../../core/shader/recompiler/SpirvBackend/src/SpirvImageEmitter.cpp) of 8_SRGB and 8_8_SRGB textures throw on a device that cannot sample R8_SRGB or R8G8_SRGB. - [Bindless image table](../../core/shader/recompiler/Optimization/src/ResourceMaterializer.cpp) entries of 8_SRGB and 8_8_SRGB textures read zeros, like converted formats, on a device that cannot sample R8_SRGB or R8G8_SRGB. -- The shader recompiler [ignores `s_setreg_b32`](../../core/shader/recompiler/Translation/src/ScalarInstructions.cpp): writes to MODE (float rounding and denormal controls) and the other hardware registers have no effect, and `s_getreg_b32` reads of the MODE round fields report round to nearest even even after `s_setreg_b32` wrote another mode. The initial round mode from the shader's program registers is not read either. +- [Shader MODE register access](../../core/shader/recompiler/Translation/src/ScalarInstructions.cpp) reads the initial rounding fields from `GuestContext::floatMode`; missing metadata retains the legacy round-to-nearest-even assumption. Immediate rounding writes are accepted only when they preserve that state. State-changing or SGPR-sourced MODE writes throw until runtime mode tracking and affected arithmetic are implemented. Non-rounding register reads still throw; `s_setreg_b32` writes to other hardware registers are still ignored. Tests follow the public RDNA2/LLVM contract, not console measurements. +- A [flat SRT read](../../core/shader/recompiler/Optimization/src/SrtWalker/SrtDescriptorEvaluation.cpp) through a null or unmapped pointer loaded from memory raises a [BDA fault](../../core/libs/prx/libSceAgcDriver/Graphics/src/BdaResources.cpp) only if the program executes it, then stops or reads zero like a BDA read. +- Repeated `s_load`s of one [flat SRT](../../core/shader/recompiler/Optimization/src/SrtWalker/SrtDescriptorEvaluation.cpp) dword through a null or unmapped pointer report one of their pcs, and an `s_load` whose inaccessible dwords are all unused, or only address repeated loads, reports no fault of its own. +- A value read through a null or unmapped pointer still throws at record time if a [branch condition, uniform fill or table base](../../core/shader/recompiler/Optimization/src/SrtWalker/SrtDescriptorEvaluation.cpp) needs it, or if the pointer comes from user data, even if the console skips the read. +- A [V#, T# or S#](../../core/shader/recompiler/Optimization/src/SrtWalker/SrtDescriptorEvaluation.cpp) loaded through a null or unmapped pointer read from memory is zero when the work is recorded, and the capture throws if no flat SRT slot records that read. +- [2D accesses through a 1D T#](../../core/shader/recompiler/Optimization/src/ResourceMaterializer.cpp) read the texel the 1D addressing selects and ignore the second coordinate, as simonbbby measured on RDNA2 (gfx1036) in #2237 for `image_sample_lz` and mip-less `image_load` with point filtering and wrap, clamp and border addressing; the PS5's gfx1013 is not measured. Other 2D accesses through a 1D T# (storage, depth, packed, BY loads, mips, derivatives, comparison) still throw. - On a host subgroup narrower than 32 lanes, [`threadBit`](../../core/shader/recompiler/Translation/src/TranslationContext/ExecMask.cpp) reads an EXEC written as a value (`s_mov_b32`/`s_mov_b64`) at bit (subgroup invocation id) of EXEC_LO, so lanes at or above the subgroup size read a lower lane's bit. - [EXEC read as a value](../../core/shader/recompiler/Translation/src/TranslationContext/ExecMask.cpp) (`s_mov_b64 sN, exec`, the `s_*_saveexec` copy, ALU sources) holds only lanes with a host invocation; the console's also holds the wave's other lanes, e.g. 32-63 of a wave64 on 32 host lanes. - [`v_readlane_b32`](../../core/shader/recompiler/SpirvBackend/src/SpirvAlu/SpirvAluEmitterMath.cpp) and fetch-inactive `v_permlane16_b32`/`v_permlanex16_b32` read undefined data from a lane that the host subgroup does not hold or whose host invocation is inactive; the console reads the lane's VGPR. - Without fetch-inactive, [`v_permlane16_b32` and `v_permlanex16_b32`](../../core/shader/recompiler/SpirvBackend/src/SpirvAlu/SpirvAluEmitterMath.cpp) take a source lane that the host subgroup does not hold as inactive and keep the old value (0 with bound_ctrl); the console reads that lane's VGPR. - On a host subgroup narrower than the wave, [`v_readlane_b32`](../../core/shader/recompiler/SpirvBackend/src/SpirvAlu/SpirvAluEmitterMath.cpp) of a constant lane outside it builds only as lane 63 of a half-wave scan (DPP row_shr 1/2/4/8, `v_permlanex16_b32`, EXEC -1); other such reads throw. +- [NGG mesh entry](../../core/shader/recompiler/Translation/src/InstructionTranslator.cpp) uses the guest wave size (32 or 64) for wave indices, per-wave vertex and primitive counts, and the workgroup wave count in `s3`. The four-bit wave fields follow Mesa's [get_wave_id_in_tg, get_tgsize and get_thread_id_in_tg](https://chromium.googlesource.com/chromiumos/third_party/mesa/+/refs/heads/stabilize-13982.70.B-chromeos-amd/src/gallium/drivers/radeonsi/gfx10_shader_ngg.c); partial waves and groups exceeding 15 waves throw. Indexed and non-indexed wave32 mesh rendering with 32, 64 and 128 threads was checked by pixel readback on Windows with an NVIDIA RTX 4070 Laptop GPU. The guest register values have not been compared with console hardware. - [Shader recompilation](../../core/shader/recompiler/Recompiler.cpp) currently occurs right before it was transferred to Vulkan with caching, but should be moved to the [relinker](../../core/relinker/main.cpp) stage. For this purpose, [shader/recompiler](../../core/shader/recompiler) was written completely independently from [libs/prx](../../core/libs/prx). - [v_interp_p1ll_f16, v_interp_p1lv_f16 and v_interp_p2_f16](../../core/shader/recompiler/Translation/src/AttributeInstructions.cpp) read the 16-bit halves of each vertex's parameter through `VK_KHR_fragment_shader_barycentric` and throw without it, as fixed-function interpolation cannot split a 32-bit attribute. P10 and P20 are the vertex differences rounded to f16 to nearest even, which is not confirmed on hardware. v_interp_p1ll_f16 and v_interp_p1lv_f16 round the exact `P10 * I + P0` once to f32 and v_interp_p2_f16 rounds the exact `P20 * J + S2` once to f16, as measured on RDNA2 (#697); the exact sums are computed in f64, so these instructions need the Float64 capability. They follow the shader's float mode, measured on RDNA2 (gfx1035) in all eight combinations of `IEEE_MODE` and kept/flushed f32 and f16 denormals over 768 special and 1,024 random rows. With IEEE=1 a NaN operand is returned quieted; with IEEE=0 it is returned unquieted, except that v_interp_p2_f16's f16 result is always quiet. Flushed f32 denormals flush I/J, v_interp_p2_f16's f32 S2 and the f32 result; flushed f16 denormals flush P0, P10/P20, v_interp_p1lv_f16's f16 S2 and the f16 result. Denormal modes 1 and 2 (flush only inputs or only outputs) and rounding other than to nearest even are not measured and throw. An unknown mode follows IEEE=0 with denormals kept. Lavapipe has no barycentrics, so the draw test is skipped in CI; `agc_driver_interpolation_modes` checks the arithmetic in every mode against the hardware rows on any device with Float64. -- [v_fma_f64](../../core/shader/recompiler/SpirvBackend/src/SpirvAlu/SpirvAluEmitterFloat64.cpp) is fused exactly only for normal inputs: with a subnormal input it is a separate multiply and add, which rounds twice. +- Without `VK_KHR_fragment_shader_barycentric`, a pixel shader whose I/J VGPRs [only feed `v_interp_p1_f32`/`v_interp_p2_f32` of their own pair](../../core/shader/recompiler/Optimization/src/HostInterpolationChecker.cpp) reads the attribute Vulkan interpolated. Any other read of them or of a raw vertex parameter takes the weights and the three vertex values from a [generated geometry stage](../../core/shader/recompiler/SpirvBackend/src/BarycentricGeometryShader.cpp), which needs `geometryShader` and a vertex shader before it: tessellation, mesh and rect-list pipelines, primitives other than triangles, more than 32 fragment input locations and interfaces beyond the device's geometry limits throw. +- [v_fma_f64 and v_div_fmas_f64](../../core/shader/recompiler/SpirvBackend/src/SpirvAlu/SpirvAluEmitterFloat64.cpp) with a subnormal operand or result round once, as IEEE 754 fusedMultiplyAdd does with f64 denormals kept, measured on RDNA2 (gfx1035) at IEEE=0 and IEEE=1, `v_div_fmas_f64` with VCC clear and set. The f16/f64 denormal field of `FLOAT_MODE` (bits 7:6) is not read: when it flushes, the hardware flushes the operands and the result of `v_fma_f64` (tininess before rounding) and only the result of `v_div_fmas_f64`, while these keep them. - [SINT storage images](../../core/shader/recompiler/SpirvBackend/src/SpirvImageEmitter.cpp) of 8- and 16-bit components saturate each stored 32-bit value to the component range, as adriadam10 measured for image stores on RDNA2 (gfx1035) in #944 (448 rows match); the PS5's gfx1013 is not measured. - [`StoreTexel`](../../core/shader/recompiler/SpirvBackend/src/SpirvImageEmitter.cpp) writes channel 1 of a two-channel format from the data slot whose DST_SEL is Y, for SINT and UINT alike. With DST_SEL X,Y,Z,W, gfx1035 writes it from the W slot, as adriadam10 measured in #944. - [COND_EXEC](../../core/libs/prx/libSceAgcDriver/Execution/src/Driver/Packets/PacketExecution.cpp) reads its condition after the earlier packets of its queue complete. Without PFP_SYNC_ME the console's graphics PFP reads it ahead of the ME, so a title that races the read sees a later value here. - A [flip inside a COND_EXEC range](../../core/libs/prx/libSceAgcDriver/Execution/src/Driver/Packets/CommandValidation.cpp) throws: the flip is reserved with the video output when the buffer is submitted, and how the console withdraws a skipped one is unknown. - A [COND_EXEC range](../../core/libs/prx/libSceAgcDriver/Execution/src/Driver/Queues/Submission.cpp) must hold whole packets and end inside its command buffer. A chained INDIRECT_BUFFER or a REWIND inside a range throws, as what the CP does with the skipped dwords then is unknown. - A [COND_INDIRECT_BUFFER](../../core/libs/prx/libSceAgcDriver/Execution/src/Driver/Queues/Submission.cpp) (`sceAgcCbBranch`) runs only with the always-pass compare function; a comparison or a predicated packet throws. Modes 1 and 2 run the first buffer, assumed to be the if-then and if-then-else modes of AMD's PM4 packet. [sceAgcCbBranch](../../core/libs/prx/libSceAgc/Cb/src/Branch.cpp) does not check the compare address of an always-pass branch or the address of an empty buffer, as PPSA14632 passes an uninitialised compare address and a null else buffer of 0 dwords. +- A [label or store](../../core/libs/prx/libSceAgcDriver/Execution/src/VulkanDevice.cpp) (`RELEASE_MEM`, `WRITE_DATA`, `DMA_DATA`) into guest memory the device has no host import of waits for all recorded work and is then written by the CPU, so the queue worker stalls once per such label. The console's CP stores it in stream order without that stall. - [SET_*_REG_INDIRECT](../../core/libs/prx/libSceAgcDriver/Execution/src/Driver/Queues/Submission.cpp) (libSceAgcDriver) reads its register list when the command buffer is submitted, with the packets. The console's graphics PFP reads it when it reaches the packet, so a list written after the submission (by the CPU, or by GPU work ahead of the packet behind PFP_SYNC_ME) is read here before that write. -- The [shader float mode](../../core/libs/prx/libSceAgcDriver/Execution/src/Driver/Shaders/ShaderRegistry.cpp) (`FLOAT_MODE`, `DX10_CLAMP`, `IEEE_MODE` and `FP16_OVFL` of `COMPUTE_PGM_RSRC1` and `SPI_SHADER_PGM_RSRC1_PS/GS/HS`, bit positions from Mesa's `gfx10.json` and `gfx103.json`) is decoded into `GuestContext::floatMode` and is part of the shader cache keys and the serialized request (#981). `IEEE_MODE` is read by `v_min_f32`, `v_max_f32`, `v_min3_f32`, `v_max3_f32` and `v_med3_f32`, `IEEE_MODE` and the `FLOAT_MODE` output denormal bits by the output modifier, `DX10_CLAMP` by the clamp modifier, and the whole mode by the 16-bit interpolation instructions (see their entries). With IEEE=1 a signaling NaN operand returns that NaN quieted (the first operand checked first, min3/max3 as two nested pairs, med3 as min3 when any operand is NaN), measured on RDNA2 (gfx1035) over every combination of ±0, ±1, ±inf, denormals, quiet and signaling NaNs with f32 denormals kept and flushed. IEEE=0 and unknown modes keep the previous lowering, which matches IEEE=0. The f16 and f64 min/max/med3, NaN quieting in f16 arithmetic (the unary instructions and the f32 arithmetic follow it, see below), `FP16_OVFL` and `FLOAT_MODE` (f32 denormals flushed before the min/max compare in Astro's Playroom's 0xc0) still follow a single mode, as the entries below record. The mode comes from the registered shader header's `sh_registers`, the same source at registration and at draw/dispatch so the prepared artifact keys match. A title that rewrites RSRC1 at run time, or a raw compute program with no header, compiles with that mode or none. -- [v_mullit_f32](../../core/shader/recompiler/Translation/src/FloatInstructions.cpp) follows the hardware measured with the MODE register at IEEE=1 and denormals kept (a signaling NaN src0 is quieted only with IEEE=1, as measured at IEEE=0 too; a positive denormal src2 counts as positive): other denormal settings are not modelled. -- [v_fma_legacy_f32](../../core/shader/recompiler/Translation/src/FloatInstructions.cpp) (VOP3 0x140) is assumed to be fused, which has not been measured on hardware. LLVM decodes this encoding as `v_mad_legacy_f32` for gfx1013, and `v_mac_legacy_f32` (VOP2 0x06) was measured to round its product before the add, so 0x140 may do the same. -- [v_fma_f16, v_pk_fma_f16 and v_pk_fmac_f16](../../core/shader/recompiler/Translation/src/FloatInstructions.cpp) are an f32 FMA rounded to f16, which rounds twice: 0x62e9 * 0xd159 + 0x23ec gives 0xf89e, the hardware 0xf89f. -- [Signaling NaN inputs](../../core/shader/recompiler/Translation/src/FloatInstructions.cpp) of the f32 and f16 unary instructions (`v_sqrt/rcp/rsq/log/exp/sin/cos/fract/floor/ceil/trunc/rndne/rcp_iflag/frexp_mant`), `v_ldexp_f32/f16`, `v_frexp_mant_f16`, `v_div_fixup_f32/f16` and `v_mullit_f32` are returned quieted only with IEEE=1 and unchanged with IEEE=0, measured on RDNA2 (gfx1035) in both modes; shaders without a known float mode follow IEEE=0. `v_add/sub/subrev/mul/fma/fmac/fmamk/fmaak/mul_legacy/fma_legacy_f32` and `v_fma_mix_f32` with f32 sources choose their NaN result on the bits, measured over 1000 operand triples in both modes: the first NaN source in operand order after its modifiers (the subtrahend negated, as the hardware adds its negation), quieted only with IEEE=1, or the default NaN 0xffc00000 for an invalid operation; in the fma forms an invalid product (inf * 0) gives the default NaN even when src2 is a NaN, and in the legacy forms a zero factor makes the product 0 even when the other factor is a NaN (a NaN src2 of `v_fma_legacy_f32` still wins). The f16 arithmetic, `v_fma_mix_f32` with f16 sources, `v_cubema`, `v_mac_legacy_f32` and the separately rounded mad/mac forms (not measurable on RDNA2) follow the same rule on hardware but their NaN bits are left to the driver. `v_cvt_f16_f32` and `v_dot2_f32_f16` always quiet and `v_div_scale_f32` never does. +- The [shader float mode](../../core/libs/prx/libSceAgcDriver/Execution/src/Driver/Shaders/ShaderRegistry.cpp) (`FLOAT_MODE`, `DX10_CLAMP`, `IEEE_MODE` and `FP16_OVFL` of `COMPUTE_PGM_RSRC1` and `SPI_SHADER_PGM_RSRC1_PS/GS/HS`, bit positions from Mesa's `gfx10.json` and `gfx103.json`) is decoded into `GuestContext::floatMode` and is part of the shader cache keys and the serialized request (#981). `IEEE_MODE` is read by `v_min_f32`, `v_max_f32`, `v_min3_f32`, `v_max3_f32` and `v_med3_f32`, their f16 forms, `v_min_f64` and `v_max_f64`, `IEEE_MODE` and the `FLOAT_MODE` output denormal bits by the output modifier, `DX10_CLAMP` by the clamp modifier, `FP16_OVFL` by every instruction with an f16 result (see its entry), and the whole mode by the 16-bit interpolation instructions (see their entries). With IEEE=1 a signaling NaN operand returns that NaN quieted (the first operand checked first, min3/max3 as two nested pairs, med3 as min3 when any operand is NaN), measured on RDNA2 (gfx1035) over every combination of ±0, ±1, ±inf, denormals, quiet and signaling NaNs with f32 denormals kept and flushed. IEEE=0 and unknown modes keep the previous f32 lowering, which matches IEEE=0. The f16 forms choose their NaN results on the bits: a NaN operand returns the other operand unchanged (src0 when both are NaN), and with IEEE=1 a signaling NaN operand returns that NaN quieted, in the same order as f32. `v_min_f64` and `v_max_f64` quiet a signaling NaN operand the same way with IEEE=1; with IEEE=0 they return src0 unchanged when src1 is NaN, otherwise src1 when src0 is NaN. Both were measured on RDNA2 (gfx1036) at IEEE=0 and IEEE=1 with f32 denormals kept and flushed and f16/f64 denormals kept; RDNA2 has no f64 min3, max3 or med3. Unknown modes follow IEEE=0. With f16/f64 denormals flushed the hardware also flushes their denormal inputs (`v_min_f16(0x0001, 0x8001)` gives 0x8000), which these lowerings do not. NaN quieting in f16 arithmetic (the unary instructions and the f32 arithmetic follow it, see below) and `FLOAT_MODE` (f32 denormals flushed before the min/max compare in Astro's Playroom's 0xc0) still follow a single mode, as the entries below record. The mode comes from the registered shader header's `sh_registers`, the same source at registration and at draw/dispatch so the prepared artifact keys match. A title that rewrites RSRC1 at run time, or a raw compute program with no header, compiles with that mode or none. +- A [WAIT_REG_MEM](../../core/libs/prx/libSceAgcDriver/Execution/src/Driver/Synchronization/WaitMemory.cpp) (libSceAgcDriver) that a compute queue passes on another queue's recorded label store, before the GPU has executed it, is landed before the queue's next dispatch, draw, COND_EXEC or predicated packet reads guest memory: the worker polls until memory satisfies the wait or no unfinished batch writes the label. The graphics queue (queue 0) is exempt, as landing its waits costs PPSA01341 about a quarter of its frame rate; a title that fills memory a graphics command buffer reads after it submits the buffer behind such a wait reads it early there. `APS5_LAND_WAITS_ALL_QUEUES=1` lands the graphics queue's waits too. +- The [dispatches that follow a compute queue's wait](../../core/libs/prx/libSceAgcDriver/Execution/src/Driver/Dispatch/DispatchAhead.cpp) (libSceAgcDriver), up to the next wait that memory does not satisfy, a packet that writes guest memory other than a label, or a conditional packet, are resolved (memory captured, shader recompiled) when the wait passes, as the CP reads them right after it; the graphics queue is exempt (`APS5_GROUP_CAPTURE_ALL_QUEUES=1` includes it). The device still reads guest memory when the dispatch is issued; when that memory no longer matches the capture, the dispatch is resolved again from current memory. +- [v_mullit_f32](../../core/shader/recompiler/Translation/src/FloatInstructions.cpp) follows the hardware measured with the MODE register at IEEE=1 and denormals kept (a signaling NaN src0 is quieted only with IEEE=1, as measured at IEEE=0 too; a positive denormal src2 counts as positive); with f32 denormals flushed it follows the f32 denormals entry below, and denormal modes 1 and 2 throw. +- [v_mad_legacy_f32 and v_mac_legacy_f32](../../core/shader/recompiler/Translation/src/FloatInstructions.cpp) (VOP3 0x140, VOP2 0x06) round the product to f32 before the add, give a +0 product for a zero factor, and flush their f32 denormal sources, their product and their result to a zero of the same sign in every f32 denormal mode, an exception to the `FLOAT_MODE` model in [f32DenormalFlushFor](../../core/shader/recompiler/Translation/src/VectorInstructions.cpp). Measured on gfx1010 (RX 5700 XT) with `tools/hw-oracle/template.s`, assembled by the AMD Windows driver's `amd_comgr_2.dll` and launched through its `amdhip64_6.dll` instead of `oracle.c` and ROCr, over the 32 rows of the five `FmaLegacy.cpp` and the four `MacLegacy.cpp` encodings: identical in wave32 and wave64 at IEEE 0 and 1 with f32 denormal modes 0 to 3, except that the two signaling NaN rows are quieted only with IEEE=1 (#1281, #1290). A sum of two zeros takes its sign on the bits (+0 unless both are -0), as lavapipe returned -0 for +0 + -0. The PS5's gfx1013 is not measured; LLVM decodes the encoding as `v_mad_legacy_f32` there, as on gfx1010. RDNA2 differs: on gfx1031 (RX 6700 XT, measured by pandather in #1281) the encoding is `v_fma_legacy_f32` and fused, with the same zero rule and denormal-source flush, so an RDNA2 measurement is not evidence for the console. +- [v_fma_mixlo_f16 and v_fma_mixhi_f16](../../core/shader/recompiler/Translation/src/FloatInstructions.cpp) round the exact product-sum once to f16 with the f64 arithmetic of v_interp_p2_f16, so they need the Float64 capability. On RDNA2 (gfx1036) they follow the `FLOAT_MODE` f16 round and denormal fields, not the f32 round field: -1.5 * 2^-24 + 2^-126 from f32 sources (0xbfc00000, 0x33800000, 0x00800000) gives 0x8001 with f16 denormals kept and round to nearest even, 0x8000 with f16 denormals flushed and 0x8002 with f16 round toward -infinity. Only f16 denormals kept with round to nearest even is translated; another f16 mode throws, as for packed f16. `FP16_OVFL` set is not measured. +- [Signaling NaN inputs](../../core/shader/recompiler/Translation/src/FloatInstructions.cpp) of the f32 and f16 unary instructions (`v_sqrt/rcp/rsq/log/exp/sin/cos/fract/floor/ceil/trunc/rndne/rcp_iflag/frexp_mant`), `v_ldexp_f32/f16`, `v_frexp_mant_f16`, `v_div_fixup_f32/f16` and `v_mullit_f32` are returned quieted only with IEEE=1 and unchanged with IEEE=0, measured on RDNA2 (gfx1035) in both modes, and so are those of the f64 forms (`v_sqrt/rcp/rsq/fract/floor/ceil/trunc/rndne/frexp_mant/ldexp/div_fixup_f64`) and `v_cvt_f64_f32`, measured on RDNA2 (gfx1036) in both modes; shaders without a known float mode follow IEEE=0. `v_pack_b32_f16` quiets each signaling NaN half by the same rule, measured on RDNA2 (gfx1036) in both modes. A signaling NaN result of `v_cubesc/cubetc/cubema_f32` is quieted by the same rule and keeps the sign the instruction gives it, measured on RDNA2 (gfx1036) in both modes. `v_add/sub/subrev/mul/fma/fmac/fmamk/fmaak/mul_legacy_f32` and `v_fma_mix_f32` with f32 sources choose their NaN result on the bits, measured over 1000 operand triples in both modes: the first NaN source in operand order after its modifiers (the subtrahend negated, as the hardware adds its negation), quieted only with IEEE=1, or the default NaN 0xffc00000 for an invalid operation; in the fma forms an invalid product (inf * 0) gives the default NaN even when src2 is a NaN, and in the legacy forms a zero factor makes the product 0 even when the other factor is a NaN (a NaN src2 still wins). `v_mad_legacy_f32` and `v_mac_legacy_f32` follow the same rule with their separately rounded product, as measured on gfx1010 (#1281, #1290): a NaN source quieted only with IEEE=1, inf - inf giving 0xffc00000, the zero rule hiding a NaN factor and a NaN src2 winning. The f16 and f64 arithmetic, `v_fma_mix_f32` with f16 sources and the separately rounded mad/mac forms (not measurable on RDNA2) follow the same rule on hardware but their NaN bits are left to the driver. `v_cvt_f16_f32`, `v_cvt_f32_f64`, `v_dot2_f32_f16`, `v_fma_mixlo_f16` and `v_fma_mixhi_f16` always quiet and `v_div_scale_f32` never does: the f32 signaling NaN 0x7f812345 as src0 or src2 gives 0x7e09 from mixlo and mixhi at IEEE=0 and IEEE=1 on RDNA2 (gfx1036), where `v_fma_mix_f32` gives 0x7f812345 at IEEE=0 and 0x7fc12345 at IEEE=1. The payload of an f16 NaN source of mixlo and mixhi is left to the driver's f16 to f32 conversion, which NVIDIA replaces. - [Packed f16 NaN results](../../core/shader/recompiler/Translation/src/FloatInstructions.cpp) (`v_pk_add/mul/fma/min/max_f16`) follow the hardware measured on RDNA2 (gfx1035) at IEEE=0 and IEEE=1 with f16 round-to-nearest-even and f16 denormals kept: with IEEE=1 a signaling NaN input is quieted and min/max return the other operand only for a quiet NaN; with IEEE=0 NaN inputs pass through unquieted and min/max ignore signaling. In `v_pk_fma_f16` an invalid product (inf * 0) gives the default NaN even when src2 is a NaN. A shader without a known float mode is translated as IEEE=0, the mode titles use; a mode with another f16 rounding or that flushes f16 denormals throws. -- [f32 denormals](../../core/shader/recompiler/Translation/src/TranslationContext/OperandAccess.cpp) follow the shader's `FLOAT_MODE` f32 denormal field (bits 5:4) when it flushes them (0, and shaders without a known float mode, as in the 0xc0 most titles use), measured on RDNA2 (gfx1035) over 1444 rows per instruction at IEEE=0 and IEEE=1. `v_add/sub/subrev/mul/mul_legacy/fma/fmac/fmamk/fmaak/min/max/min3/max3/med3/fract/floor/ceil/frexp_mant/sin/cos_f32` flush their f32 inputs and their result to a zero of the same sign; the f32 compares, `v_cvt_i32/u32/rpi_i32/flr_i32_f32`, `v_frexp_exp_i32_f32`, `v_cvt_f64_f32`, `v_cvt_f16_f32`, `v_cvt_pkrtz_f16_f32`, `v_cvt_pknorm_i16/u16_f32` and `v_cvt_pk_u8_f32` flush their f32 inputs. Tininess is detected before rounding: a product whose exact value is below 2^-126 gives zero even when it rounds to 2^-126, computed exactly from the significands for `v_mul_f32`, `v_mul_legacy_f32`, and for `v_fma_f32`, `v_fmac_f32`, `v_fmamk_f32`, `v_fmaak_f32` and `v_fma_mix_f32` when the addend is zero; with a nonzero addend the fused result is kept, so an exact result just below 2^-126 that rounds to it is not flushed. `v_fma_mix_f32` was measured with f32 sources only; its f16 sources are not flushed. Denormal modes 1 and 2 throw. Not covered, and not measurable on RDNA2 (no such opcodes) or not measured: `v_mac_f32`, `v_mad_f32`, `v_madak_f32` and `v_madmk_f32` (rounded separately), `v_mac_legacy_f32`, `v_dot2_f32_f16`, the division steps (on hardware `v_div_fixup_f32` flushes only src1 and src2, `v_div_fmas_f32` only its result), f32 interpolation and LDS float atomics keep the previous lowering. With f32 denormals kept (3) the results depend on the driver's default denormal behaviour, as no `DenormPreserve` execution mode is requested, and `v_ldexp_f32` flushes in every mode. +- [f32 denormals](../../core/shader/recompiler/Translation/src/TranslationContext/OperandAccess.cpp) follow the shader's `FLOAT_MODE` f32 denormal field (bits 5:4) when it flushes them (0, and shaders without a known float mode, as in the 0xc0 most titles use), measured on RDNA2 (gfx1035) over 1444 rows per instruction at IEEE=0 and IEEE=1. `v_add/sub/subrev/mul/mul_legacy/mullit/fma/fmac/fmamk/fmaak/min/max/min3/max3/med3/fract/floor/ceil/frexp_mant/sin/cos_f32` flush their f32 inputs and their result to a zero of the same sign (`v_mullit_f32` measured on RDNA2 (gfx1036) over 6907 rows per mode at IEEE=0 and IEEE=1, so a positive denormal src2 counts as zero); the f32 compares, `v_cvt_i32/u32/rpi_i32/flr_i32_f32`, `v_frexp_exp_i32_f32`, `v_cvt_f64_f32`, `v_cvt_f16_f32`, `v_cvt_pkrtz_f16_f32`, `v_cvt_pknorm_i16/u16_f32` and `v_cvt_pk_u8_f32` flush their f32 inputs. `v_div_fixup_f32` flushes its src1 and src2 and keeps a denormal src0 and result, measured on RDNA2 (gfx1036) over 6859 rows per mode at IEEE=0 and IEEE=1. A product is checked for tininess on its exact value: one below 2^-126 gives zero even when it rounds to 2^-126, computed exactly from the significands for `v_mul_f32`, `v_mul_legacy_f32`, `v_mullit_f32`, and for `v_fma_f32`, `v_fmac_f32`, `v_fmamk_f32`, `v_fmaak_f32` and `v_fma_mix_f32` when the addend is zero; with a nonzero addend the fused result is kept, so an exact result just below 2^-126 that rounds to it is not flushed. The hardware detects tininess after rounding to 24 bits with an unbounded exponent, measured on RDNA2 (gfx1036) for `v_mul_f32` and `v_mullit_f32` at IEEE=0 and IEEE=1: `0x3f7ffffe * 0x00800001` (2^-126 * (1 - 2^-46)) gives `0x00800000` there and zero here, while `0x3f7fffff * 0x00800000` (2^-126 * (1 - 2^-24)) gives zero on both. `v_fma_mix_f32` was measured with f32 sources only; its f16 sources are not flushed. `v_fma_mixlo_f16` and `v_fma_mixhi_f16` flush their f32 sources too and keep a product below 2^-126, measured on RDNA2 (gfx1036). `v_mad_legacy_f32` and `v_mac_legacy_f32` flush their inputs, their product and their result in every mode, see its entry. Denormal modes 1 and 2 throw for the other instructions. Not covered, and not measurable on RDNA2 (no such opcodes) or not measured: `v_mac_f32`, `v_mad_f32`, `v_madak_f32` and `v_madmk_f32` (rounded separately), `v_dot2_f32_f16`, `v_div_scale_f32` and `v_div_fmas_f32` (on hardware `v_div_fmas_f32` flushes only its result), f32 interpolation and `ds_add_f32` keep the previous lowering; `ds_min/max/cmpst_f32` flush both operands by the field, see the LDS float atomics entry. With f32 denormals kept (3) the results depend on the driver's default denormal behaviour, as no `DenormPreserve` execution mode is requested, and `v_ldexp_f32` flushes in every mode. - The executable file that [relinker](../../core/relinker/elfpatcher/src/windows/WindowsPeWriter.cpp) generates opens the console when launched, which is inconvenient for playability. +- [Overlay output bus](../../core/libs/prx/libSceVideoOut/src/VideoOutDriver.cpp) (libSceVideoOut) - flips on a port opened with `SCE_VIDEO_OUT_BUS_TYPE_OVERLAY` complete and signal their events like main-bus flips, but their buffers are not shown: the overlay is not composited over the main output. Presenting them as whole frames made the window alternate between the two buses - [PA_CL_CLIP_CNTL](../../core/libs/prx/libSceAgcDriver/Graphics/src/State.cpp) (libSceAgcDriver) - `DX_LINEAR_ATTR_CLIP_ENA` (bit 24) clear is drawn like bit 24 set, with Vulkan's clipping of `NoPerspective` outputs; what a clear bit changes on the console is undocumented. -- [Per-stage descriptor count](../../core/libs/prx/libSceAgcDriver/Graphics/src/ShaderResources.cpp) (libSceAgcDriver) - a pixel shader's colour attachments are not counted against `maxPerStageResources`, so its buffers and images can fit the limit while the pipeline exceeds it -- [v_rcp, v_rsq, v_sqrt, v_log, v_exp, v_sin and v_cos](../../core/shader/recompiler/Translation/src/FloatInstructions.cpp) (f32 and f16) use the driver's approximation for finite results, not the console's: the low bits can differ. Only NaN, invalid, zero and infinite results and the exact sin/cos zeros are chosen on the bits. v_sin_f32/v_cos_f32 with inputs beyond ±256 cycles were not compared with the console. +- [10_11_11 unorm color targets](../../core/libs/prx/libSceAgcDriver/Graphics/src/State.cpp) (libSceAgcDriver) render into R32_UINT attachments: the pixel shader packs each export, rounded to nearest even like format-30 image stores, which is assumed to match the CB's conversion. +- [PA_SU_SC_MODE_CNTL.PROVOKING_VTX_LAST](../../core/libs/prx/libSceAgcDriver/Graphics/src/Pipeline.cpp) (libSceAgcDriver) uses Vulkan's last-vertex mode for native primitives. Bit 19 follows [opengnm's `flatshade_first` mapping](https://github.com/blackbearreloaded/ps5-opengl/blob/db752da4c89380af228497b5b6852163a4e61069/src/gallium/ps5/ps5_screen.c#L2244); vertex selection follows [Vulkan](https://docs.vulkan.org/spec/latest/chapters/drawing.html#drawing-primitive-topologies), without console measurements. Last mode throws without `VK_EXT_provoking_vertex`, for generated rectangle, mesh or tessellation primitives, explicit per-vertex flat inputs, and triangle strips with per-vertex inputs, whose [ordering depends on the device](https://docs.vulkan.org/refpages/latest/refpages/source/VkPhysicalDeviceFragmentShaderBarycentricPropertiesKHR.html). +- [v_rcp, v_rsq, v_sqrt, v_log, v_exp, v_sin and v_cos](../../core/shader/recompiler/Translation/src/FloatInstructions.cpp) (f32 and f16) use the driver's approximation for finite results, not the console's: the low bits can differ. Only NaN, invalid, zero and infinite results and the exact sin/cos zeros are chosen on the bits. Measured on RDNA2 (gfx1036) over every f16 input and 32,832 f32 inputs: RADV returns the hardware's bits for every input of rcp, rsq, sqrt and log and all but one of exp (f16 0x11c5, 1 ULP); NVIDIA (615.71.09) stays within 2 ULP except `v_log_f32` near 1.0, where its log2 is up to 24 million ULP off. [v_sin and v_cos](../../core/shader/recompiler/SpirvBackend/src/SpirvAlu/SpirvAluEmitterMath.cpp) take the driver's sine or cosine of the radians of the exact cycle, except within 2^-12 of a turn from a zero of the function, where the result is the linear term of the exact distance to the zero: the hardware's constant 6.2831826 (0x40c90fd5) at the integers, which the hardware applies exactly to inputs below 2^-12 of a turn, and the f32 2Ï€ at the half and quarter turns, where the hardware is within 7 ULP of it. RADV returns the hardware's bits for every f16 input and for 32,422 sine and 32,486 cosine results of the 32,832 f32 inputs; the rest are 1 ULP off, except up to 14 ULP on either side of non-zero integers and up to 15 ULP beyond ±256 cycles, where the hardware's own reduction differs from the exact fraction, and up to 6 ULP inside the half- and quarter-turn bands. NVIDIA's sine is within 1 ULP on 93% of the f32 inputs and up to 1,166 ULP off just outside the bands, its cosine within 163 ULP. A denormal f32 input with f32 denormals kept gives the hardware the input times 6.2831826; both hosts flush it. +- [Dual-source blending](../../core/libs/prx/libSceAgcDriver/Graphics/src/State.cpp) (libSceAgcDriver) takes the MRT1 export as the second source of the MRT0 blend, as RADV and radeonsi program it; what the CB does with other export or target layouts is unknown, so they throw. - `--to-intel` does not lower RDPRU/MCOMMIT; the [matcher](../../core/relinker/codegen/src/x86/Amd64OnlyInstructionMatcher.cpp) fails the relink instead. +- [Ordered not-equal compares](../../core/shader/recompiler/SpirvBackend/src/SpirvAlu/SpirvAluEmitterMath.cpp) (shader recompiler) - `FPOrdNotEqual32` is emitted as `OpFUnordNotEqual` with both operands checked for NaN on their bits instead of `OpFOrdNotEqual`; the two agree for every input. The AMD Windows driver 32.0.13058.8001 (Radeon 860M, RDNA 3.5) returns 1 for a NaN operand when any float form of the compare feeds a ballot on a 32-wide subgroup (#825) - The length-changing path of the [instruction rewriter](../../core/relinker/codegen/src/x86/X64InstructionRewriter.cpp) is not used by the [converter](../../core/relinker/codegen/src/Amd64OnlyConverter.cpp): it does not adjust data-to-code references (relocations, FDEs, jump tables) or segment sizes, so every substitution keeps the instruction length. - [DB_SHADER_CONTROL.CONSERVATIVE_Z_EXPORT](../../core/libs/prx/libSceAgcDriver/Graphics/src/ShaderInputState.cpp) (libSceAgcDriver) becomes `DepthLess`/`DepthGreater`. For a pixel shader that breaks the promise, the host driver decides whether its interpolated Z is depth-tested before shading; what the console does then was not measured. - [DB_SHADER_CONTROL.PRIMITIVE_ORDERED_PIXEL_SHADER](../../core/shader/recompiler/SpirvBackend/src/SpirvModuleEmitter.cpp) (shader recompiler) - the host pixel interlock spans the whole shader, not only up to `s_sendmsg` ORDERED_PS_DONE. - [POPS collision SGPRs](../../core/shader/recompiler/SpirvBackend/src/SpirvModuleEmitter.cpp) (shader recompiler) - the collision wave ID and intrawave collision mask read 0, so a title skips its POPS wait and runs its per-collision loop once. - [Image loads](../../core/shader/recompiler/Optimization/src/ResourceTracker.cpp) (shader recompiler) - an image_load reads a sampled binding, also of an image the shader stores to, so in a primitive-ordered pixel shader it misses an earlier overlapping primitive's image_store in the same draw. -- [PA_SC_CONSERVATIVE_RASTERIZATION_CNTL](../../core/libs/prx/libSceAgcDriver/Graphics/src/State.cpp) (libSceAgcDriver) - 0x6001 uses Vulkan overestimation on devices that add at most 1/256 pixel and keep degenerate triangles; the console's uncertainty region and degenerate handling are unmeasured. +- [PA_SC_CONSERVATIVE_RASTERIZATION_CNTL](../../core/libs/prx/libSceAgcDriver/Graphics/src/State.cpp) (libSceAgcDriver) - 0x6001 uses Vulkan overestimation on devices that add at most 1/256 pixel and keep degenerate triangles; the console's uncertainty region and degenerate handling are unmeasured. A value with `OVER_RAST_ENABLE` and `UNDER_RAST_ENABLE` clear is decoded as disabled even when `UNCERTAINTY_REGION_MODE` or the `OUTER_`/`INNER_UNCERTAINTY_EDGERULE_OVERRIDE` bits (19:16) are set, as PPSA01474 writes 0xe6000; that these fields have no effect without over- or underestimation is assumed from their names, not measured. - The Linux placement of the `--to-intel` stubs in the [ELF patcher](../../core/relinker/elfpatcher/src/linux/LinuxElfPatcher.cpp) is covered only by a synthetic test. - The libc [SSE4a trap emulation](../../core/libs/prx/libc/src/specifics/windows/Sse4aEmulation.hpp) on Windows is superseded by `--to-intel` and remains only until the relinked title has been verified without it. - [Descriptor pool chain](../../core/libs/prx/libSceAgcDriver/Graphics/src/ShaderResources.cpp) (libSceAgcDriver) - each pool holds two storage buffer descriptors more than a set can take, as a workaround: RADV (Mesa 26.0 and later) lets a `FREE_DESCRIPTOR_SET` pool place a set up to 32 bytes past the pool's buffer ([`util_vma_heap_init`](https://gitlab.freedesktop.org/mesa/mesa/-/blob/main/src/amd/vulkan/radv_descriptor_pool.c) is given `bo_size + RADV_POOL_HEAP_OFFSET` as a size), and with a page-multiple pool size those bytes are an unmapped page -- [CMASK fast clears](../../core/libs/prx/libSceAgcDriver/Graphics/src/Draw.cpp) (libSceAgcDriver) - only a CMASK whose nibbles are all fast-cleared (0) or all expanded (0xF) is modeled, checked over the target's whole CMASK, 4 KiB per 1024x512-pixel metablock (the size PS5 titles clear, for every color format seen); the addressing within a metablock is not decoded. The first draw or fast-clear eliminate over a fast-cleared CMASK fills the whole target with `CB_COLOR_CLEAR_WORD` and stores those bytes as expanded, so a draw that covers only part of the target reads the clear value in the rest as the eliminate would leave it. On a DCC target the clear is in the DCC keys, so its CMASK must be all expanded. A mixed CMASK throws, as does a DCC target's CMASK that is not all expanded and a fast clear of a mipmapped, 3D or array color target +- [CMASK fast clears](../../core/libs/prx/libSceAgcDriver/Graphics/src/Draw.cpp) (libSceAgcDriver) - each fast-cleared CMASK tile (code 0) is stored as `CB_COLOR_CLEAR_WORD` into its 8x8 texels at the first draw into the target or at its fast-clear eliminate or DCC decompress pass, not as the color block writes the tile, and the CMASK is then stored as expanded (0xF): a texture or copy read of the target after that draw and before the title's eliminate sees the clear value in tiles no draw wrote, where the console keeps their previous bytes. Tile addresses follow the pipe-aligned SW_64KB_Z_X CMASK equation for 16 pipes; other CMASK swizzle modes, codes other than 0 and 0xF, a cleared tile of a DCC target and a fast clear of a mipmapped, 3D or array color target throw. When the clear runs as a compute pass (a resident target whose CMASK is host-imported), an invalid code throws at the next fast-clear materialization, not at the one that read it +- [HTILE depth fast clears](../../core/libs/prx/libSceAgcDriver/Graphics/src/DepthSurface.cpp) (libSceAgcDriver) - HTILE is not modeled. Only RADV's `clear_htile_mask` compute kernel (17 GFX10 words, 64x1x1 groups) is recognized, and only when it leaves ZMASK at 0 (`user[4]` and `user[5]` low nibbles both 0); the dispatch still runs, and the host depth image is cleared to `DB_DEPTH_CLEAR` the next time the surface with that `DB_HTILE_DATA_BASE` address is bound. The address match is exact, so a clear of a different HTILE range does not hit. HTILE initialization, the stencil-only update, and fast clears made by the SDK's own paths, CP DMA fills or other kernels are not covered and leave the depth image as it was. The kernel words are the ones the PR author saw RADV submit; they are not compared with Mesa's source. - [Host imports](../../core/libs/prx/libSceAgcDriver/Graphics/src/GuestBufferMemory.cpp) (libSceAgcDriver) on Linux skip ranges that are not mapped read-write, which are copied instead. The NVIDIA driver (595.91.07) refuses `VK_EXT_external_memory_host` imports of read-only or executable mappings (and shared mappings of regular files) with `VK_ERROR_OUT_OF_DEVICE_MEMORY`, after which every `vkAllocateMemory` on the device fails the same way. Read-only ranges are never imported, even on drivers that accept them. Shared mappings of regular files are not checked for. -- [Cube texture descriptors](../../core/libs/prx/libSceAgcDriver/Graphics/src/GuestTextureResource.cpp) with equal `BASE_ARRAY` and `DEPTH` expose six faces starting at `BASE_ARRAY`. Measured with `image_load`, `image_sample` and `image_get_resinfo` on RX 6600 (gfx1032), using 64x64 R32_UINT faces and base slices 0, 5 and 6. Other ranges containing fewer than six faces or incomplete cube arrays still throw: hardware load bounds and filtered-sample addressing differ. Seamless cube filtering and out-of-range cube-array sampling remain unimplemented. PS5 equivalence has not been measured on console hardware. +- [Cube texture descriptors](../../core/libs/prx/libSceAgcDriver/Graphics/src/GuestTextureResource.cpp) with equal `BASE_ARRAY` and `DEPTH` expose six faces starting at `BASE_ARRAY`. Measured with `image_load`, `image_sample` and `image_get_resinfo` on RX 6600 (gfx1032), using 64x64 R32_UINT faces and base slices 0, 5 and 6. Other ranges containing fewer than six faces or incomplete cube arrays still throw: hardware load bounds and filtered-sample addressing differ. Out-of-range cube-array sampling remains unimplemented. Samplers with `DISABLE_CUBE_WRAP` are created with `VK_SAMPLER_CREATE_NON_SEAMLESS_CUBE_MAP_BIT_EXT`, the flag RADV maps to that bit (Mesa `src/amd/vulkan/radv_sampler.c`, `src/amd/common/ac_descriptors.c`); without `VK_EXT_non_seamless_cube_map` a cube texture sampled through such a sampler throws. PS5 equivalence has not been measured on console hardware. - [DCC display buffers](../../core/libs/prx/libSceAgcDriver/Execution/src/VulkanDevice.cpp) are presented only when all keys of the console's DCC extent agree on uncompressed or a fast-clear code, padding keys included; other keys, and register-clear keys over a pending image, throw. +- [Storage image write-back](../../core/libs/prx/libSceAgcDriver/Graphics/src/Texture.cpp) (libSceAgcDriver) stores the GPU results over a CPU-stamped range whose guest bytes equal the copy taken when the image's layers got their generation (images up to 1 MiB; larger ones keep the previous rule: the CPU's bytes win). A CPU store that rewrites the baseline value after the GPU changed it is not told apart from no store, and the GPU result wins there. The 1 MiB limit is a choice, not a console value. - [DCC key stores](../../core/libs/prx/libSceAgcDriver/Graphics/src/DccMetadata.cpp) cover the console's DCC extent only for pipe-aligned DCC of single-level, single-layer 4-byte SW_64KB_R_X surfaces; other surfaces, and every texture and color target key read, cover one byte per 256 bytes. +- [DCC key fills over other images](../../core/libs/prx/libSceAgcDriver/Graphics/src/Texture.cpp) (libSceAgcDriver) - a uniform fill at a live image's `dccAddress`, at least one key per 256 bytes long, is a key fill of that image even when other live images overlap it, but only while it is no longer than the image's key extent (`DccKeyCount`, the pipe-aligned count); a longer one is classified by its overlaps as before. The bound is inferred, not measured on a title: a longer fill also writes the bytes past the keys, which the overlapping image may own. Whether the title's bloom-target key fills stay within the extent was not checked (`APS5_TRACE_DCC_KEYS` prints the length of the fills classified as keys, not of the refused ones). - [8_SRGB color targets](../../core/libs/prx/libSceAgcDriver/Graphics/src/Draw.cpp) (libSceAgcDriver) on a device that cannot attach and blend R8_SRGB render into an RGBA8_SRGB copy of the target, so the sRGB encode, decode and blend rounding are the device's; the console's are not measured. - [8_SRGB color targets](../../core/libs/prx/libSceAgcDriver/Graphics/src/Texture.cpp) (libSceAgcDriver) on such a device throw unless they are single-mip 2D surfaces with a resident image; linear targets have none. - [COPY_DATA from the GPU clock](../../core/libs/prx/libSceAgcDriver/Execution/src/Pm4.cpp) (libSceAgcDriver, `SRC_SEL` 9) writes the host steady clock in 10 ns ticks when the packet runs, the same counter as the `RELEASE_MEM` timestamp, not the console's GPU clock. The `COPY_DATA` cache policy fields are accepted and have no effect. +- [UINT16_ABGR color exports](../../core/libs/prx/libSceAgcDriver/Graphics/src/State.cpp) (libSceAgcDriver) - export format 7 is accepted only into an unsigned integer target, and each 16-bit value reaches the target unchanged; a value beyond a narrower target's range (`R8_UINT`) is undefined in Vulkan, and whether the console clamps or truncates it is not confirmed. SINT16_ABGR (8) throws - `--to-intel` guest module trampolines are covered only by a synthetic relinker test; no game title has been verified with them on Linux or Windows. - `--to-intel` replaces the approximate VEX.128 register forms of VRSQRTPS and VRCPPS with [correctly rounded](../../core/relinker/codegen/src/x86/ReciprocalLowering.cpp) 1/sqrt(x) and 1/x, not with the AMD approximation: its tables are not public. Intel's approximation of 1/sqrt(1.0) is 0x3F7FF000, and two Newton-Raphson steps on it settle on 0.99999994 instead of 1.0; a renormalised identity quaternion then turns into a NaN axis in PPSA21564, which the console does not show. Refined results now reach the exact value; unrefined ones can still differ from the console in the low bits. Each site costs an out-of-line stub (two jumps, a spill below the red zone, SQRTPS/DIVPS). The 256-bit, scalar, legacy and memory forms keep the native instruction, as does a 4-byte site with no movable instruction after it for the jump (5 sites in PPSA21564, counted before a following RIP-relative instruction could move into the stub; not recounted). - [sceKeyboardGetKey2Char](../../core/libs/prx/libSceKeyboard/src/keyboard_impl.cpp) (libSceKeyboard) translates the 101-key (US) arrangement and the alphanumeric layer of the 106-key (Japanese) one, which follows the JIS layout of the HID usages: the Yen (0x89) and Ro (0x87) keys return `\` (0x5C) unshifted, as JIS X 0201 places the yen sign there; whether the console returns U+00A5 instead is unverified. With the Kana LED (bit 4, the HID position) set the 106-key arrangement throws, as the kana the console returns are unknown. Ctrl and Alt do not change the character. @@ -329,35 +415,57 @@ Throughout the project, every function at every stage either **does exactly what - [AudioOut2 contexts](../../core/libs/prx/libSceAudioOut/src/AudioOut2Context.cpp) (libSceAudioOut) with an SDL device read a grain's port buffers at the next push, at least 100 us after the grain was pushed, and play the mix behind a 40 ms cushion that the queue level leaves out. When the hardware reads the buffers is not known. The timing follows a title whose mixer fills a buffer after pushing it, reuses it two pushes later, and may still be writing it when the next push follows within microseconds. The last grain pushed before a title stops pushing is never read or played. - [libSceAudiodec](../../core/libs/prx/libSceAudiodec/Export.cpp) throws for the 24-bit PCM word size (`iBwPcm` 0): its sample layout is unknown. ATRAC9 decoding is covered only by configuration and error tests, as no ATRAC9 encoder is available for a fixture. - [sceAjmDecMp3ParseFrame](../../core/libs/prx/libSceAjm.native/src/Ajm.cpp) (libSceAjm.native) throws when the original file length is requested (`parse_ofl` non-zero) for a layer other than III, for a CRC-protected frame, or for a VBRI header outside MPEG-1 stereo: shadPS4 looks for the tag after the CRC and after the side information, while LAME and FFmpeg place it without the CRC and VBRI always at byte 36. +- [ATRAC9 gapless segments](../../core/libs/prx/libSceAjm.native/src/Ajm.cpp) (libSceAjm.native) - a run job stops after the gapless total of output samples counted from the start of the segment, leaving the rest of its input unconsumed, and the next run starts the next segment with the decoder as it is and without a second skip. FMOD in PPSA29141 sets a total shorter than a sound's frames and reads the frames after it on later runs; with the output held at the total its stream reader waited for them forever and the last buffer repeated. The console's handling of the frames after the total is not documented; a reset (`SetGaplessDecode` with the reset flag, `ClearContext`, initialization) starts a new segment. An ATRAC9 run job without `RUN_MULTIPLE_FRAMES` decodes its whole input as one with it: FMOD in PPSA29141 puts two run jobs in one batch with the second's input right after the whole of the first's. Whether the console stops after one frame there is not verified; MP3 and Opus run jobs still stop after one frame - The [output modifier](../../core/shader/recompiler/Translation/src/TranslationContext/OperandAccess.cpp) (`mul:2`, `mul:4`, `div:2`) follows the shader's float mode as measured on RDNA2 (gfx1035): the hardware applies it only with IEEE mode off and output denormals of the result's precision flushed (bit 1 of the `FLOAT_MODE` denormal field clear), and ignores it otherwise. On f32 results it is applied as the hardware does with f32 denormals flushed; a shader without a known float mode is translated with IEEE mode off and f32 denormals flushed, the mode titles use. On f16 and f64 results it is ignored, and it throws in the modes where the hardware applies it. -- The [clamp modifier](../../core/shader/recompiler/Translation/src/TranslationContext/OperandAccess.cpp) follows `DX10_CLAMP`, measured on RDNA2 (gfx1035) for f32, f16, packed f16 and f64 results at IEEE=0 and IEEE=1: with `DX10_CLAMP`=1 (and in shaders without a known float mode) a NaN result clamps to +0; with `DX10_CLAMP`=0 a NaN result is written unchanged and every other value clamps to [0, 1] as before. f64 results follow the same rule on hardware, but a clamped f64 result with `DX10_CLAMP`=0 throws until it has an execution test. With `DX10_CLAMP`=0 the NaN's bits are those the translation already produces for the instruction without clamp, so where those are left to the driver (f32 and scalar f16 arithmetic) the payload is the driver's, not the hardware's input propagation. `v_dot2_f32_f16` and the integer clamps are unchanged. +- The [clamp modifier](../../core/shader/recompiler/Translation/src/TranslationContext/OperandAccess.cpp) follows `DX10_CLAMP`, measured on RDNA2 (gfx1035) for f32, f16, packed f16 and f64 results at IEEE=0 and IEEE=1: with `DX10_CLAMP`=1 (and in shaders without a known float mode) a NaN result clamps to +0, including the quieted NaN that `v_pk_min_f16` and `v_pk_max_f16` return for a signaling NaN operand with IEEE=1 (measured on RDNA2 gfx1036 at both `IEEE_MODE` and both `DX10_CLAMP` settings); with `DX10_CLAMP`=0 a NaN result is written unchanged and every other value clamps to [0, 1] as before. f64 results follow the same rule on hardware, but a clamped f64 result with `DX10_CLAMP`=0 throws until it has an execution test. With `DX10_CLAMP`=0 the NaN's bits are those the translation already produces for the instruction without clamp, so where those are left to the driver (f32 and scalar f16 arithmetic) the payload is the driver's, not the hardware's input propagation. `v_dot2_f32_f16` and the integer clamps are unchanged. +- [FP16_OVFL](../../core/shader/recompiler/Translation/src/TranslationContext/OperandAccess.cpp) follows the shader's float mode as measured on RDNA2 (gfx1036) at IEEE=0 and IEEE=1 with f16 denormals kept and flushed: with `FP16_OVFL`=1 an f16 result that rounds to ±infinity is written as ±0x7bff unless a source operand is infinite, so `v_rcp_f16`, `v_rsq_f16` and `v_log_f16` of ±0 and `v_exp_f16` of 16 or more are clamped while an infinite input passes through, and `v_div_fixup_f16` keeps the infinity it writes for a zero denominator or an infinite numerator and clamps any other infinite quotient, a NaN one included. `v_cvt_pkrtz_f16_f32` rounds toward zero and never overflows. The sources are checked as the f32 values the translation computes with; the rule is applied to `v_fma_mix*_f16` with f32 sources and to `v_interp_p2_f16` without a measurement of infinite f32 sources or parameters. Round modes other than nearest even are not modelled, so the overflow threshold is the nearest-even one (65520). - [Comparison sampling of a color texture](../../core/shader/recompiler/SpirvBackend/src/SpirvImageEmitter.cpp) (`image_sample_c`/`image_sample_c_lz` on a texture that is not R32 float or R16 unorm) is emulated in the shader, since Vulkan compares only depth formats: the red channel is fetched, compared and, for bilinear filtering, blended (compare, then filter). Only 2D and 2D array views, float/unorm/snorm formats, wrap, clamp-to-edge and clamp-to-border addressing (border red 0 for the black borders, 1 for opaque white) and point or bilinear filtering at one level are implemented, with texel offsets added to the texel indices; LOD biases and clamps are accepted, as they cannot select another level of a single-level sample. Half-border addressing, the border color table, gradients, gathers, other dimensions, anisotropy and mip filtering throw. Under bilinear filtering, taps past a clamp-to-border edge compare the border value and blend with the texel taps, so results near the edge depend on the filter weights. AMD documentation does not say how a color format's reference is handled: it is clamped to [0, 1] for unorm and [-1, 1] for snorm, as Vulkan does for unorm depth, and not clamped for float. Bilinear weights are full float, not the hardware's fixed-point subtexel weights +- [Negative color-texture comparison wrap indices](../../core/shader/recompiler/SpirvBackend/src/SpirvImageEmitter.cpp) (libSceAgcDriver) use explicit `OpSRem` normalization to implement modulo; the negative-offset point-wrap regression fails with direct `OpSMod` and passes with normalization on Mesa 22.3.6 llvmpipe (LLVM 15.0.6), but negative-index wrap has not been measured on PS5 hardware. - [libScePlayerInvitationDialog](../../core/libs/prx/libScePlayerInvitationDialog/libScePlayerInvitationDialog.cpp) simulates dialog completion without displaying UI or sending invitations, so `scePlayerInvitationDialogGetResult` reports `USER_CANCELED`; its parameter ABI remains unverified. The 40-byte result layout (error code, result, 32 reserved bytes) is taken from [SharpProspero](https://github.com/SvenGDK/SharpProspero/blob/main/src/SharpProspero/Interop/Dialog/PlayerInvitationDialog.cs) and the error codes from the other common dialogs, not confirmed on a title. - [Converted unorm images](../../core/shader/recompiler/SpirvBackend/src/SpirvImageEmitter.cpp) (10_11_11_UNORM, read and written through an R32_UINT view): samples, gathers, LOD queries, 16-bit data and a DST_SEL of the missing W channel throw. - [Converted unorm image loads](../../core/shader/recompiler/SpirvBackend/src/SpirvImageEmitter.cpp) divide by 2^n - 1 with the buffer format path's `OpFDiv`, which Vulkan allows to be up to 2.5 ULP off, so some drivers are not bit-exact. -- [64-bit LDS atomics](../../core/shader/recompiler/SpirvBackend/src/SpirvMemory/SpirvMemoryInstructions.cpp) (`ds_*_u64`, `ds_*_b64`, `ds_*_f64`) read and write the two dwords under a workgroup lock stored after the guest LDS, as LDS is a 32-bit array without 64-bit atomics. They are atomic with respect to each other only: 32-bit accesses to the same dwords issued concurrently are not ordered by the lock. 64-bit GDS atomics throw. `ds_min/max(_rtn)_f64` and `ds_cmpst(_rtn)_f64` follow the NaN and signed-zero rules measured for the f32 forms; the f64 forms were not measured on hardware. +- [64-bit LDS atomics](../../core/shader/recompiler/SpirvBackend/src/SpirvMemory/SpirvMemoryInstructions.cpp) (`ds_*_u64`, `ds_*_b64`, `ds_*_f64`) read and write the two dwords under a workgroup lock stored after the guest LDS, as LDS is a 32-bit array without 64-bit atomics. They are atomic with respect to each other only: 32-bit accesses to the same dwords issued concurrently are not ordered by the lock. 64-bit GDS atomics throw. On the hardware the LDS serialises 32-bit and 64-bit atomics on the same address (256 lanes mixing `ds_add_u32` and `ds_add_u64` on one slot sum exactly), and under the lock the 32-bit adds can be lost: in 20 runs per host and wave size, RADV lost 32 to 96 of the 128 in every run and the RTX 3080 16 to 80 in 18 of 20 wave32 runs and 16 in 3 of 20 wave64 runs, always a multiple of 16. A 64-bit atomic at an address that is 4-byte but not 8-byte aligned faults the wave on gfx1036, as on gfx1035. +- [LDS float atomics](../../core/shader/recompiler/SpirvBackend/src/SpirvMemory/SpirvMemoryInstructions.cpp) (`ds_min/max/cmpst_f32`, `ds_min/max/cmpst_f64` and their `_rtn` forms) follow the hardware measured on RDNA2 (gfx1036) over 14 x 14 operand sets (zeros, ones, infinities, denormals, quiet and signalling NaNs of both signs and with payloads) in IEEE 0 and 1 with every value 0-3 of the type's denormal field: both operands are flushed to a signed zero when bit 0 of the field is clear (0 and 2; 1 and 3 keep them; the f32 field for the f32 forms, the f16/f64 field for the f64 forms; a shader without a known float mode flushes f32 and keeps f64), then a signalling-NaN data operand is stored quieted, else a signalling-NaN old value is stored quieted, else a quiet-NaN old value is replaced by the data, else a quiet-NaN data operand loses, else the ordered compare with -0 below +0 decides, and the flushed value is what is written, also when the old value wins. `cmpst` compares the flushed bits (both zeros equal, NaN never equal) and leaves memory unchanged when they differ. The returned value is always the old value as it was in memory. `IEEE_MODE` has no effect. The same bit rule serves the buffer float atomics (`AtomicFloatBits`). Not measured: a denormal or signalling-NaN `data1` of `cmpst` (stored as given), and `ds_add_f32`, which keeps the host's `OpFAdd`. - [V# bases off a DWORD boundary](../../core/shader/recompiler/SpirvBackend/src/SpirvMemory/SpirvMemoryInstructions.cpp) (shader recompiler) - raw accesses use the exact bytes and `s_buffer_load` reads `(base & ~3) + (offset & ~3)`, as measured on gfx1035 (Linux, alignment mode UNALIGNED; the console's mode is unknown). Formatted accesses off their element's alignment and atomics throw. +- [32-bit bitfield masks](../../core/shader/recompiler/Translation/src/IntegerInstructions.cpp) (`S_BFM_B32`, `V_BFM_B32`) use the truncating 32-bit shift specified by the [RDNA 1.0 ISA](https://www.amd.com/content/dam/amd/en/documents/radeon-tech-docs/instruction-set-architectures/rdna-shader-instruction-set-architecture.pdf), sections 12.1 (opcode 36) and 12.12 (opcode 867). Width and offset use their low five bits. This matches gfx1035 (Linux) for every width/offset pair, with and without upper operand bits set, and both leave SCC unchanged; not measured on the console. - [Vector buffer offsets that are not a multiple of 4](../../core/shader/recompiler/SpirvBackend/src/SpirvMemory/SpirvMemoryInstructions.cpp) (shader recompiler) - dword accesses count whole dwords from the V# base, where the hardware reads the exact bytes at base + offset. - [Min/max S# reduction](../../core/libs/prx/libSceAgcDriver/Graphics/src/GuestSamplerResource.cpp) (libSceAgcDriver) - `FILTER_MODE` 1 and 2 use Vulkan's min and max reduction (the same `SQ_IMG_FILTER_MODE` values RADV programs). Measured on RDNA2 (gfx1035), not on the console: only texels of non-zero weight are reduced, and the weight is taken after rounding to the 8-bit subtexel grid (a sample within 1/512 texel of a centre reduces that texel alone); a driver with other `subTexelPrecisionBits` reduces a different set there. Gathers ignore the mode. - [Min/max S#](../../core/libs/prx/libSceAgcDriver/Graphics/src/GuestSamplerResource.cpp) (libSceAgcDriver, shader recompiler) - throws with anisotropy, a linear mip filter, depth or color-texture compare, or bilinear filtering of an image the recompiler point-samples (sint, converted, depth-bits). +- [FORCE_DEGAMMA](../../core/libs/prx/libSceAgcDriver/Graphics/src/Sampler.cpp) (libSceAgcDriver) - an S# that forces sRGB decoding adds nothing to an sRGB texture, as in shadPS4 (PS4). Measured on RDNA2 (gfx1035), not on the console: `8`, `8_8` and `8_8_8_8` sRGB textures sample bit for bit the same with and without the bit (point and bilinear; BC formats not measured). Other textures throw; on the same hardware their UNORM `8`, `8_8` and `8_8_8_8` forms sample bit for bit as the sRGB format (decoded before filtering). +- [Tessellation factor ring](../../core/libs/prx/libSceAgcDriver/State/src/Configuration.cpp) (libSceAgcDriver) - `sceAgcDriverSetTFRing` records the ring and `sceAgcDriverGetTFRing` reports it. Before a title sets one, it reports the console's default, 0xFF0000000 and 128 KiB, as measured in [PS5_Vulkan's hardware findings](https://github.com/mihawk-99/PS5_Vulkan/blob/fde9e3798c2ec1d8fccc90d5067cd6060490607f/docs/HARDWARE_FINDINGS.md). Nothing is written to the recorded ring: tessellated draws run as Vulkan tessellation stages, which keep their factors on the host, so a title that reads tessellation factors from the ring finds none. A null base or a zero size throws; [ps5-opengl's AGC runtime](https://github.com/blackbearreloaded/ps5-opengl/blob/67c873f985eaefe0a72f713529e5632eaa01139c/src/platform/ps5_agc_native_runtime.c) calls `sceAgcDriverSetTFRing(0, 0)` in its cleanup, and what that does on the console is unknown. - [Raw `buffer_load_dwordx2`](../../core/shader/recompiler/SpirvBackend/src/SpirvMemory/SpirvMemoryInstructions.cpp) is one 64-bit access on the hardware (with or without glc), so it never sees half of a `buffer_atomic_swap_x2`. Only a glc load that is 8-byte aligned and in bounds, in a program that has the 64-bit storage view (it uses 64-bit buffer atomics), is read with one 64-bit atomic load; every other dwordx2 load is two dword loads and can tear against a concurrent 64-bit write: loads without glc, and loads in a program without 64-bit buffer atomics while another shader writes the pair with `swap_x2`. - [DS src2 operations](../../core/shader/recompiler/Translation/src/MemoryInstructions/SharedMemoryInstructions.cpp) (`ds_*_src2_*`, GFX10.1 only) read `DS[B]`, with `B = A + 4 * sext(offset1[6:0]:offset0)`, and then update `DS[A]` with the matching 32-bit atomic, so the read of `B` is not atomic with the update of `A`. The form that takes the offset from the address (`offset1[7]` set), the 64-bit forms and GDS throw: whether the LDS address keeps `A[31:17]` in that form, and whether the 64-bit forms scale the offset by 4 or 8, is not documented. `ds_min_src2_f32`, `ds_max_src2_f32` and `ds_add_src2_f32` throw: on RDNA2 (gfx1035, the oracle's hardware, which has no src2 opcodes) `ds_min_f32`, `ds_max_f32` and `ds_add_f32` ignore `IEEE_MODE` but depend on whether f32 denormals are flushed (28, 28 and 12 of 196 special rows differ), and that is not modelled for LDS float atomics yet (#981). - [wcsrtombs_s](../../core/libs/prx/libc/src/Multibyte.cpp) (libc) follows the console's `libc.prx`, but a runtime-constraint violation only returns its error: the default handler there writes the message and a newline to `stderr` before returning, which is not done here. Conversion uses the C locale of `wcrtomb` (`MB_CUR_MAX` 1), so the path where a multibyte character does not fit the remaining space is not reached -- [The narrow printf family](../../core/libs/prx/libc/src/Formatting.cpp) (libc) passes `%ls` and `%S` to glibc on Linux, which reads them as strings of 32-bit `wchar_t`; the guest's `wchar_t` is 16 bits (UTF-16), so these conversions fail with -1. The Windows path converts them to UTF-8, and `guest_formatting` checks wide conversions on Windows only +- [The narrow printf family](../../core/libs/prx/libc/src/Formatting.cpp) (libc) converts 16-bit guest wide arguments to UTF-8 on both platforms, independently of the host locale. It does not use the guest's single-byte `wcrtomb`; the console's treatment of invalid UTF-16 and locale-specific encoding remains unverified. Linux retains host formatting for formats without wide arguments; positional arguments, the `'` flag and `%m` throw when a format requires the guest formatter - [collate::_Getcat](../../core/libs/prx/libc/src/LocaleSupport.cpp) (libc) builds the facet for the C locale only and throws for any other locale name. The console's `libc.prx` stores pointers to its collation state tables at offsets 0x10 and 0x18 of the facet; ours leaves them null, as only the facet's own functions read them. `do_transform` throws for input with embedded null characters, which the console drops depending on the output buffer size, and for results of 4095 characters or more, whose string storage uses the guest's over-aligned large allocation. +- [The scanf family](../../core/libs/prx/libc/src/Formatting.cpp) (libc) hands `%i` to glibc on Linux, which since 2.38 reads the C23 `0b`/`0B` binary prefix when the library is built as GNU C++: `sscanf("0b101", "%i", &value)` stores 5 where C17 stores 0 and stops at the `b`. The strtol family stops at the prefix as C17 does - [socketpair](../../core/libs/prx/libkernel/Socket/src/Udp.cpp) (libkernel) - the pair is the host's `AF_UNIX` pair on Linux and, on Windows, a loopback TCP connection for `SOCK_STREAM` or two connected UDP sockets for `SOCK_DGRAM`; `SOCK_SEQPACKET` throws on Windows. A datagram sent to a full receiver gets `ENOBUFS` on the console, waits or gets `EAGAIN` on Linux and is dropped on Windows, and on Windows a datagram `recv` after the peer closed reports `ECONNRESET` instead of the console's `ENOTCONN`, and a stream `send` after the peer closed reports `ECONNABORTED` instead of the console's `EPIPE`. `read` and `write` on a pair end go to the host like on every guest socket. `getpeername` on a pair end succeeds after the peer closed, where the console returns `ENOTCONN`, and `connect`, `bind` and `sendto` with an address on a pair end report an address-family error instead of the console's `EISCONN`. On Windows `poll` on a stream end reports `POLLHUP` without `POLLIN` after the peer's `shutdown(SHUT_WR)` or `close`, where the console reports `POLLIN`, with `POLLHUP` only after `close` +- [read and write](../../core/libs/prx/libkernel/File/src/Stdio.cpp) (libkernel) on a guest socket are `recv` and `send` without flags, and `sceKernelRead` and `sceKernelWrite` return the kernel error for them; `EPIPE` raises the host's `SIGPIPE` on Linux, which the guest can neither ignore (`signal` maps only six signals) nor suppress (`SO_NOSIGPIPE` is not an option here), and raises nothing on Windows; a `write` longer than `INT_MAX` bytes is `EMSGSIZE` where the console sends it; `readv`, `writev`, `pread`, `pwrite`, their `sceKernel` forms and AIO on a guest socket still go to the host, which has no such descriptor, where the console reads and writes through `readv` and `writev` and returns `ESPIPE` for the positioned forms - [poll](../../core/libs/prx/libkernel/Socket/src/Udp.cpp) (libkernel) waits only on guest sockets, through `WSAPoll` on Windows and `poll` on Linux; the `pollfd` layout and flag values are FreeBSD's. A descriptor below the socket range (files, pipes, stdio) throws, as do event bits other than `POLLIN`, `POLLPRI`, `POLLOUT`, `POLLRDNORM`, `POLLRDBAND` and `POLLWRBAND` (the `POLLERR`, `POLLHUP` and `POLLNVAL` bits are ignored in `events`), `POLLPRI` on Windows, which `WSAPoll` rejects, and an infinite timeout with no socket to wait on. A closed socket descriptor reports `POLLNVAL`; the console's limit on the descriptor count is unknown and not checked +- [msync](../../core/libs/prx/libkernel/DirectMemory/PosixMemory.cpp) (libkernel) follows FreeBSD 11's `kern_msync` and `vm_map_sync`, not measured on the console: the address is rounded down to the page, a range that wraps is `EINVAL`, as is `MS_ASYNC` with `MS_INVALIDATE`, and a range whose start or any page is not mapped (as `sceKernelVirtualQuery` reports it, reservations included) is `ENOMEM`; a page unmapped from inside a larger mapping still counts as mapped there, since `sceKernelVirtualQuery` keeps reporting the whole mapping. The guest has no file mappings, so a successful call writes nothing back. `MS_INVALIDATE` on a mapped range throws: FreeBSD returns `EBUSY` when the range holds `mlock`ed pages, and locked pages are not tracked - [Thread exit destructors](../../core/libs/prx/libSceLibcInternal/Export.cpp) (libSceLibcInternal) run when a libkernel thread finishes; on the main or a host thread only `_sceLibcInternalThreadDtors` runs them, as `exit` does, else they are dropped - [sceFontOpenFontSet](../../core/libs/prx/libSceFont/src/SystemFont.cpp) (libSceFont) opens only the primary file of a system font set. Characters the console takes from the set's other files (Hangul, Thai and Arabic in Japanese and Chinese sets, Arabic in European sets) are reported as unsupported glyphs. The console file names are those of the PS4 firmware; the names of the PS5-only JG2 sets (`0x1A......`) are unknown, so they use the SSTJpPro files. When a console file is missing, an openly licensed substitute (Noto Sans, Noto Sans Mono, Noto Sans Thai, Noto Sans CJK) is loaded instead. It is not the system font: its metrics differ, so text width, line height and wrapping differ from the console. System font faces are loaded by FreeType in host memory, not in the font memory the title passes to the library: FreeType needs about 1 MiB per CJK face, which the title's font memory is not sized for. +- [sceFontGetCharGlyphMetrics, sceFontGetRenderCharGlyphMetrics](../../core/libs/prx/libSceFont/src/Layout.cpp) (libSceFont) report the vertical metrics the way the [FreeType driver](../../core/libs/prx/libSceFontFt/src/Driver.cpp) reports them for a rendered glyph, not confirmed on a title: FreeType's vertical advance, and for a face without vertical metrics a bearing of half the horizontal advance left of the horizontal bearing and of half the advance left over by the glyph height. - [Font writing lines](../../core/libs/prx/libSceFont/src/WritingLine.cpp) (libSceFont) - `sceFontCreateWritingLine`, `sceFontWritingLineWritesOrder`, `sceFontWritingLineRefersRenderStep`, `sceFontWritingLineGetRenderMetrics` and `sceFontWritingLineClear` follow [KytyPS5](https://github.com/KytyPS5/KytyPS5/blob/main/src/libs/libFont.cpp): runs are placed left to right, each at the line's advance so far, with no spacing or adjustment. Only horizontal and left-to-right lines, writing attribute 0 and non-null run metrics are modelled; other values, the metrics of an empty line and unknown or null handles throw, since no reference gives their result or error code. `sceFontWritingLineGetOrderingSpace` still throws (KytyPS5's values are placeholders). Lines are allocated on the host, not in the font memory the title passes to the library: FreeType needs about 1 MiB per CJK face, which the title's font memory is not sized for. -- [s_swappc_b64](../../core/shader/recompiler/Translation/src/ControlFlowInstructions.cpp) writes the link pair with the same model as `s_getpc_b64`: the guest return address `pc + 4` as a byte offset over `GetShaderBase`. Any other use of the link pair (arithmetic, compare, hash, store, or a second `s_swappc_b64` sharing it) throws; constant-offset folding over the link is not modeled. The special encoding `s_swappc_b64` with SDST 125 still decodes as `s_setpc_b64`, keeping the existing decode quirk. +- [s_swappc_b64](../../core/shader/recompiler/Translation/src/ControlFlowInstructions.cpp) writes the link pair with the same model as `s_getpc_b64`: the guest return address `pc + 4` as a byte offset over `GetShaderBase`. Any other use of the link pair (arithmetic, compare, hash, store, or a second `s_swappc_b64` sharing it) throws; constant-offset folding over the link is not modeled. The special encoding `s_swappc_b64` with SDST 125 still decodes as `s_setpc_b64`, which matches the hardware: measured on RDNA2 (gfx1036) in wave32 and wave64, it branches to the source pair and leaves a marker SGPR pair untouched, and with an SGPR destination the link is `PC + 4`. - The [s_swappc_b64 fetch-shader call at vertex/local/tessellation/mesh entry](../../core/shader/recompiler/ControlFlow/src/GraphBuilder.cpp) assumes the called fetch-shader blob ends in `s_setpc_b64` to the link pair (the GNM fetch-shader shape shadPS4's `EmitFetch` copies); the blob is not part of the shader program, so its return closes the pairing by assumption and any in-program reader of the link pair throws. Attribute fetches are inlined from the request's parsed fetch-shader plan through the same `GetAttribute` path as the embedded fetch analyzer, honoring the buffer resource `dst_sel`; an attribute writing more than four registers, or an entry call without a parsed plan, throws. The vertex/instance offset accumulator inside the fetch blob is not replayed for external blobs. +- [VertexBufferReadSize](../../core/libs/prx/libSceAgcDriver/Graphics/include/VertexInput.hpp) and the [vertex staging copy](../../core/libs/prx/libSceAgcDriver/Graphics/src/Draw.cpp) (libSceAgcDriver) - a vertex fetch whose index is at or past the descriptor's `num_records` no longer throws for `OOB_SELECT` 0 and 1. The copy still spans the whole fetch, but the bytes from `num_records * stride` onward read as zeros, so the out-of-range vertex is `(0, 0, 0, 0)`. Measured with `buffer_load_format_xyzw` (`idxen`) through the [hardware oracle](HW_ORACLE.md) on RDNA2 gfx1036 for PPSA01325's word 3 `0x0004dfac` (`OOB_SELECT` 0) in #2241: indices 3, 4 and 5 over 3 records each return `00000000 00000000 00000000 00000000`, W included, unlike Vulkan `robustBufferAccess`, which returns `(0, 0, 0, 1)`. The zeros come from the record check, not from memory: stored values placed past the records are ignored, and a `num_records` 6 control returns them. `RESOURCE_LEVEL` 1, `OOB_SELECT` 1 and FORMAT 74 `32_32_32_FLOAT` give the same zeros, in wave32 and wave64; gfx1013 (the console) is not measured. `OOB_SELECT` 2 returns the stored values past the records and `OOB_SELECT` 3 compares `num_records` against the byte offset, both measured on the same setup in #2241; the two keep throwing past `num_records` here, since each needs its own rule. Zero-stride descriptors (`stride` 0) keep their separate byte-count check and are not covered here. Each out-of-range fetch is copied alone and outside the draw-snapshot cache, which is neither read nor written for it: a reused snapshot would carry another draw's bytes into the zero region, and a stored zero-padded copy would hand zeros to a later in-range draw at the same address. The valid prefix is still flushed for GPU writes before it is read. - [sceImeKeyboardGetResourceId](../../core/libs/prx/libSceIme/Export.cpp) (libSceIme) reports that no USB keyboard is connected (`0x80bc0004`, as [shadPS4](https://github.com/shadps4-emu/shadPS4) does) because IME keyboard events are not delivered to the handler, although [libSceKeyboard](../../core/libs/prx/libSceKeyboard/src/keyboard_impl.cpp) exposes the host keyboard - [Player review dialog](../../core/libs/prx/libSceCdlgPlayerReview/Export.cpp) (libSceCdlgPlayerReview) shows nothing: `scePlayerReviewDialogOpen` finishes at once and the result reports that the user canceled, as `sceWebBrowserDialogOpen` does - [NP reachability callbacks](../../core/libs/prx/libSceNpManager/Export.cpp) retain one callback and its user data; duplicate registration and unregistering an absent callback return the errors used by [shadPS4](https://github.com/shadps4-emu/shadPS4/blob/945dbc3cc3eee80ac3e053b438502ed936fa6bb2/src/core/libraries/np/np_manager.cpp). No callback is delivered because the existing NP model remains signed out with reachability unavailable. - [Running application ID](../../core/libs/prx/libSceSystemService/Export.cpp) (libSceSystemService) - `sceSystemServiceGetAppIdOfRunningBigApp` reports 1 as the running title's app ID. The value is AnyPS5's own: the console's shell assigns app IDs and their values are not known here, so other calls that take an app ID must use the same one. `sceSystemServiceKillApp` with that ID and the arguments -1, 0 and 0, which [SwanStationPS5](https://github.com/darkxex/SwanStationPS5/blob/4cdc981d1a64ffb7f2af73f7a27a9e91a3cb5eb4/src/main.c#L1050-L1058) passes to close itself and [ps5-payload-dev's hbldr](https://github.com/ps5-payload-dev/websrv/blob/46374edee6627885abac717de24da7a1d73af6f1/src/ps5/hbldr.c#L425-L435) passes to close the running game, ends the title as its own `exit(0)` would: the guest's shutdown and `atexit` handlers run and the process exits with status 0. On the console the call returns and the shell then ends the title, and whether those handlers run there is unknown; here there is no shell, so the call does not return. Another app ID or other arguments throw. +- [sceNpCheckNpAvailability](../../core/libs/prx/libSceNpManager/Export.cpp) (libSceNpManager) - the `(req_id, online_id)` signature and the errors follow [shadPS4](https://github.com/shadps4-emu/shadPS4/blob/0fe263a4760dfbfa973366890061749b4af0de97/src/core/libraries/np/np_manager.cpp) with no user signed in to PSN: a null online id is `SCE_NP_ERROR_INVALID_ARGUMENT`, and any other online id matches no user, `SCE_NP_ERROR_USER_NOT_FOUND`; the request id is not checked, as in the other NpManager requests - [Temporary app-content storage](../../core/libs/prx/libSceAppContent/Export.cpp) tracks one `/temp0` mount, preserves its files on unmount, and rejects new path resolutions until remount. Mount, format, space-query and unmount errors follow [fpPS4](https://github.com/red-prig/fpPS4/blob/04cefd43e6fddd1ab033e7980cd356d14c964905/sys/sys_path.pas); the pointer ABI is also declared by [SharpProspero](https://github.com/SvenGDK/SharpProspero/blob/main/src/SharpProspero/Interop/AppContent/AppContent.cs). Existing open host descriptors remain usable after unmount; console behaviour with open files is unverified. Host filesystem failures still throw. - [Guest path resolution](../../core/libs/prx/libc/src/General.cpp) falls back to ASCII case-insensitive component lookup, following [shadPS4's host filesystem](https://github.com/shadps4-emu/shadPS4/blob/945dbc3cc3eee80ac3e053b438502ed936fa6bb2/src/core/file_sys/backends/host_fs.cpp). PPSA03747 requests `Skyrim.INI`, `Data/Video` and lowercase archive/plugin names that differ from the dump's filenames. Exact host names take precedence; multiple fallback matches throw instead of selecting an arbitrary file. Existing parent names are resolved when creating a new file, and guest mount aliases use the same ASCII comparison. Results are not cached, so renames and newly created files remain visible. Unicode case folding and PS5 behaviour for case-colliding host entries are unverified; host roots are used as supplied. +- [localtime, mktime](../../core/libs/prx/libc/src/Time.cpp) (libc) on Windows convert through the CRT's `localtime_s` and `mktime`, which accept about 1970 to January 3001 UTC and apply the CRT's daylight-saving rule; outside that range `localtime` returns null and `mktime` returns -1, where the console converts. `gmtime` converts every time whose year fits an `int` on both hosts. +- [LINE_STIPPLE_TEX](../../core/shader/recompiler/Translation/src/InstructionTranslator.cpp) (recompiler) reads as 0: line stipple is not emulated. RADV enables the input only as the required pixel input of a shader that reads no interpolants, because it initializes the fewest VGPRs ([Mesa](https://gitlab.freedesktop.org/mesa/mesa/-/blob/main/src/amd/vulkan/radv_shader.c) `radv_compute_spi_ps_input`), and never reads it +- [Draw snapshot buffer descriptors](../../core/libs/prx/libSceAgcDriver/Graphics/src/ShaderResources.cpp) rewrite each affected array from element zero, preserving unchanged descriptors. MoltenVK 1.4.2 writes auxiliary buffer lengths at index `i` instead of `start + i` for partial array updates ([MVKDescriptorSet.mm, BufferAuxSize](https://github.com/KhronosGroup/MoltenVK/blob/v1.4.2/MoltenVK/MoltenVK/GPUObjects/MVKDescriptorSet.mm#L1051)). Per-element writes therefore let one buffer length replace another even when Vulkan buffer handles and ranges are correct. - [Static scalar calls](../../core/shader/recompiler/ControlFlow/src/GraphBuilder.cpp) (`s_call_b64` and `s_swappc_b64`) require one call per ordinary even-aligned SGPR link pair (s0:s1 through s104:s105) and one later `s_setpc_b64` return reading that exact pair. Regions must nest or be disjoint; recursion, shared links, escaping link values (including overlapping tuples and resource descriptors), invalid targets, interleaved regions, cross-region branches or fall-through, relative SGPR reads and dynamic `s_setpc_b64` in a program with calls throw. The analysis checks all decoded instructions, including unreachable code. Arbitrary indirect calls and general call frames are not implemented. -- [s_call_b64](../../core/shader/recompiler/Translation/src/ScalarInstructions.cpp) saves `PC + 4` over `GetShaderBase` and branches to `PC + 4 + signext(SIMM16) * 4`, following [AMD RDNA2 ISA](https://gpuopen.com/download/rdna2-shader-instruction-set-architecture.pdf) sections 3.6.3, 12.2 and 12.3. Forward callees after `s_endpgm` are decoded. Native RDNA2/PS5 execution and title compatibility for these bounded scalar calls remain unverified; `s_rfe_b64` still throws in translation. +- [DB_RENDER_CONTROL resummarize draws](../../core/libs/prx/libSceAgcDriver/Graphics/src/State.cpp) (libSceAgcDriver) - HTILE is not modeled, so a RESUMMARIZE_ENABLE draw without depth or stencil tests and without color writes changes no modeled state and draws nothing; any other resummarize, copy or decompress draw throws +- [s_call_b64](../../core/shader/recompiler/Translation/src/ScalarInstructions.cpp) saves `PC + 4` over `GetShaderBase` and branches to `PC + 4 + signext(SIMM16) * 4`, following [AMD RDNA2 ISA](https://gpuopen.com/download/rdna2-shader-instruction-set-architecture.pdf) sections 3.6.3, 12.2 and 12.3. Forward callees after `s_endpgm` are decoded. Measured on RDNA2 (gfx1036) in wave32 and wave64 with SIMM16 = 3: the link pair receives `PC + 4`, the branch goes to `PC + 4 + signext(SIMM16) * 4`, and `s_setpc_b64` of the link returns to the instruction after the call. Execution on the PS5 (gfx1013) and title compatibility for these bounded scalar calls remain unverified; `s_rfe_b64` still throws in translation. - [sceVideodec2CreateDecoder](../../core/libs/prx/libSceVideodec2/Export.cpp) (libSceVideodec2) decodes HEVC (codec type 974921) and VP9 (2382845) through FFmpeg, besides H.264. The tags come from SharpProspero's `Videodec2CodecType` and EVO-PLAYER-PS5's `sce_videodec2.h`, which records them from hardware runs. HEVC input is converted from length-prefixed NAL units to Annex B as for H.264, and VP9 frames are passed through. Output is the same 8-bit NV12 for every codec; a decoder for HEVC Main10 or VP9 Profile 2 is created, but a frame deeper than 8 bits throws, since the console's 10-bit output layout is unknown, and `QueryDecoderMemoryInfo` reports the 8-bit frame size for it. `sceVideodec2GetPictureInfo` and `sceVideodec2GetAvcPictureInfo` throw for HEVC and VP9 pictures, which use other layouts (`SceVideodec2HevcPictureInfo`, `SceVideodec2Vp9PictureInfo`), and `sceVideodec2GetHevcPictureInfo` and `sceVideodec2GetVp9PictureInfo` are not exported +- [syslog](../../core/libs/prx/libc/src/Syslog.cpp) (libc) writes each message to `stderr` as `[syslog:] `, with a newline added when missing, instead of sending it to the console's system log. Priority and facility handling follow FreeBSD's `vsyslog`: `%m` expands to the guest errno's message, unknown bits are reported (as priority 0x23, since FreeBSD passes `LOG_ERR|LOG_CONS|LOG_PERROR|LOG_PID` as the priority) and masked, a missing facility becomes `LOG_USER`, and errno is preserved. `openlog`, `setlogmask` and `closelog` are not implemented, so the identity, options and mask they set are not applied +- [sigaction](../../core/libs/prx/libkernel/System/src/Signals.cpp) (libkernel) records the handler, flags and mask of every signal from 1 to 128 and reports them back, following FreeBSD's `sigaction` (`EINVAL` for other signals, and for a handler on `SIGKILL` or `SIGSTOP`); `signal` records its handler with `SA_RESTART`, as FreeBSD's libc does. Only handlers without `SA_SIGINFO` on the signals `signal` already delivers (`SIGINT`, `SIGILL`, `SIGABRT`, `SIGFPE`, `SIGSEGV`, `SIGTERM`) run, the same way `signal` handlers do; their mask and flags are not applied. `SA_SIGINFO` handlers and other signals (`SIGBUS`, `SIGHUP`, ...) are recorded but never called, since that needs FreeBSD's `siginfo_t` and `ucontext_t`; `raise` of an `SA_SIGINFO` handler on a signal `signal` delivers throws instead of terminating, and `raise` of the other signals still returns `EINVAL` +- [kill](../../core/libs/prx/libkernel/System/src/Signals.cpp) (libkernel) follows FreeBSD's `kern_kill` and `killpg1` for the only guest process: `EINVAL` for signals outside 0 to 128, signal 0 only checks that the target exists, and its own pid, 0 and minus its own pid (the guest is taken to lead its own process group) signal it through `raise`, so they share `raise`'s limits above. Any other pid, and -1, which skips the caller, report `ESRCH`, so `EPERM` is never reported +- [AJM](../../core/libs/prx/libSceAjm.native/src/Ajm.cpp), [APR](../../core/libs/prx/libkernel/Apr/src/Apr.cpp) and [NGS2](../../core/libs/prx/libSceNgs2.native/src/System.cpp) state is allocated with `new` and never freed on purpose: title threads still call into these libraries while `std::exit` runs static destructors, so ordinary statics would be destroyed under them. Keep them leaked; `guest_ajm_teardown` covers the AJM case on Unix. +- [Null vertex buffer V#](../../core/libs/prx/libSceAgcDriver/Graphics/include/VertexInput.hpp) (libSceAgcDriver) - a V# whose four dwords are all zero is bound to a zeroed vertex buffer of the attribute's size with stride 0, so every component of its attribute reads 0. This follows the V# fields in the [AMD RDNA2 ISA](https://gpuopen.com/download/rdna2-shader-instruction-set-architecture.pdf) (`NUM_RECORDS` 0, `DST_SEL` 0 selects 0), not measured on hardware. diff --git a/docs/user/USAGE.md b/docs/user/USAGE.md index 8405c9cd6..713119e86 100644 --- a/docs/user/USAGE.md +++ b/docs/user/USAGE.md @@ -2,7 +2,7 @@ ## Input and conversion -Use a clean ELF executable. Place its bundled ELF modules in `sce_module/`, `sce_modules/`, or `prx/` beside the input executable. `prx/` can coexist with either `sce_module/` or `sce_modules/`. Both `sce_module/` and `sce_modules/` present, or all three absent, is an error. +Use a clean ELF executable. Place its bundled ELF modules in `sce_module/`, `sce_modules/`, or `prx/` beside the input executable, or under the parent directory selected with `--sce-module-path `. The option accepts absolute paths and paths relative to the current working directory; do not include the `sce_module` directory itself. Only files directly inside the module directories are processed. A bundled module in a SELF container is an error, as the executable is. `prx/` can coexist with either `sce_module/` or `sce_modules/`. Both `sce_module/` and `sce_modules/` present, or all three absent, is an error. ```text source/ @@ -27,28 +27,38 @@ Windows output: relinker --windows source/input.elf app.exe ``` +macOS output (x86-64 Mach-O, which runs under Rosetta on Apple silicon): + +```sh +relinker --macos source/input.elf eboot +``` + Add `--to-intel` for Intel hosts. The output format defaults to Linux ELF regardless of the filename; `.exe` alone does not select Windows. +The executable output must not refer to the input executable or a bundled module being converted, including through a hard link or symbolic link. An existing output file can be replaced if it is separate from those inputs. + ## Options All switches are disabled by default. `unused-filter` defaults to `0`; `--rpath` defaults to `$ORIGIN/libs`. -| Option | Effect | -|-------------------------------|-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| -| `--windows` | Produce a Windows PE executable. | -| `--windows-diagnostics` | Include startup dependency diagnostics. Requires `--windows`. | -| `--windows-gui` | Select the Windows GUI subsystem instead of the console subsystem. Requires `--windows`. | -| `--to-intel` | Convert supported AMD-only instructions in the executable and bundled modules. Unsupported instructions or unreachable conversion stubs cause an error. | -| `unused-filter=0` | Keep all imported NID references. | -| `unused-filter=1` | Filter unused non-PLT imports using control-flow and GOT access analysis; preserve PLT imports. | -| `unused-filter=2` | Apply strict unused-import analysis and compact the PLT. Unsupported analysis cases cause an error. | -| `--registry` | Write `.registry.json` beside the output executable. | -| `--rpath ` | Set the system library search path. Quote `$ORIGIN` to prevent shell expansion, for example `--rpath '$ORIGIN/libs'` in Bash or PowerShell. Linux guest modules require an absolute path or a path beginning with `$ORIGIN`. Windows requires a nonempty ASCII path and supports `$ORIGIN` as the executable directory. | -| `--autorun` | Run the output after conversion, print its exit code, and wait for Enter. Adds executable permissions for Linux output. Requires the target OS and prepared runtime layout. | -| `--skip-sce-module` | Deprecated. Skip all bundled module processing. | -| `--exclude-sce-module ` | Deprecated. Exclude a bundled module by exact filename, not path. Repeat for multiple files; a missing filename is an error. Conflicts with `--skip-sce-module`. | -| `--skip-syscall-check` | Deprecated. Disable syscall scanning in the executable and bundled modules. | -| `--lazy-binding` | Deprecated. Enable lazy symbol binding instead of eager binding. Incompatible with bundled ELF modules. | +| Option | Effect | +|-------------------------------|-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| `--windows` | Produce a Windows PE executable. | +| `--macos` | Produce an x86-64 Mach-O executable, with bundled modules as Mach-O libraries in `app0/sce_module`. Conflicts with `--windows`. | +| `--windows-diagnostics` | Include startup dependency diagnostics. Requires `--windows`. | +| `--windows-gui` | Select the Windows GUI subsystem instead of the console subsystem. Requires `--windows`. | +| `--to-intel` | Convert supported AMD-only instructions in the executable and bundled modules. Unsupported instructions or unreachable conversion stubs cause an error. | +| `unused-filter=0` | Keep all imported NID references. | +| `unused-filter=1` | Filter unused non-PLT imports using control-flow and GOT access analysis; preserve PLT imports. | +| `unused-filter=2` | Apply strict unused-import analysis and compact the PLT. Unsupported analysis cases cause an error. | +| `--registry` | Write `.registry.json` beside the output executable, and `..registry.json` for each converted bundled module. | +| `--rpath ` | Set the system library search path. Quote `$ORIGIN` to prevent shell expansion, for example `--rpath '$ORIGIN/libs'` in Bash or PowerShell. Linux guest modules require an absolute path or a path beginning with `$ORIGIN`. Windows requires a nonempty ASCII path and supports `$ORIGIN` as the executable directory. | +| `--autorun` | Run the output after conversion, print its exit code, and wait for Enter. Adds executable permissions for Linux output. Requires the target OS and prepared runtime layout. | +| `--sce-module-path ` | Parent of the module directories, not the module directory itself. Default: input ELF parent; relative input paths resolve from the process working directory. Explicit absolute path: used directly. Explicit relative path (including `.` and `..`): relative to the process working directory. Output path has no effect. Requires one nonempty value; repeats, nonexistent or nondirectory paths, access failures, and combination with `--skip-sce-module` are errors. | +| `--skip-sce-module` | Deprecated. Disable bundled module processing and module directory checks. Conflicts with explicit `--sce-module-path`. | +| `--exclude-sce-module ` | Deprecated. Exclude a bundled module by exact filename, not path. Repeat for multiple files; a missing filename is an error. Conflicts with `--skip-sce-module`. | +| `--skip-syscall-check` | Deprecated. Disable syscall scanning in the executable and bundled modules. | +| `--lazy-binding` | Deprecated. Enable lazy symbol binding instead of eager binding. Incompatible with bundled ELF modules. | Specify `unused-filter=0|1|2` without `--`, at most once. Unknown options and extra positional arguments are errors. There is no `--help` flag; invoking `relinker` without arguments prints the usage syntax and exits with an error. @@ -72,8 +82,12 @@ Use `sce_modules/` or `prx/` instead of `sce_module/` if that is the input direc Use the generated files printed as `Guest module:` for bundled title modules. `libs/` is for AnyPS5 system libraries, not the original PS5 `.prx` files. Placing an original PS5 module in `libs/` on Windows makes Windows try to load it as a DLL and can fail with error 193 (not a valid Win32 application). +On Windows, a self-built `libs/` also needs `libgcc_s_seh-1.dll`, `libstdc++-6.dll` and `libwinpthread-1.dll` from the `mingw64/bin` directory of the toolchain that built the libraries; the release archives already contain them. The libraries are loaded without searching `PATH`, so a copy elsewhere on the system is not used, and a library that needs one of them fails with error 126 (the specified module could not be found) although its `.prx` file is present. + On Windows, direct memory (`sceKernelAllocateDirectMemory`, up to 13824 MiB per title) is committed in full when the title allocates it, not when its pages are first used. The system commit limit (installed memory plus page file size, the second value of Committed in Task Manager) must cover it together with all other committed memory. Otherwise the allocation throws `create direct memory backing of 0x bytes ( MiB)` with the Windows error; enlarge the page file or close other applications. +On Linux, when the Vulkan driver imports dma-buf memory (not the NVIDIA proprietary driver), shared direct memory is imported through `/dev/udmabuf`, and the user who runs the game needs read-write access to it. Many distributions create it as `root:kvm` with mode `0660`: add the user to the `kvm` group and log in again (an ACL such as `setfacl -m u:$USER:rw /dev/udmabuf` lasts until the next reboot). Without access, startup prints `[gpu] open /dev/udmabuf: Permission denied`, these ranges are copied instead of imported, and GPU stores to them through FLAT/GLOBAL addresses fail with `BDA access failed`. Ranges above udmabuf's `size_limit_mb` (64 MiB by default) are copied as well. + Linux: ```sh @@ -95,6 +109,10 @@ Games that open the console's system font sets (`sceFontOpenFontSet`) need font The game runs on the first Vulkan 1.1 device with graphics and compute queues and swapchain presentation, preferring a discrete GPU over an integrated one. Set `ANYPS5_GPU` to a part of a device name, compared without regard to case, to run on another device; the names are printed at start-up in the `Physical device candidate` lines. When no usable device contains the text, the start fails and the error lists the device names. +### Storing GPU results at each flip + +GPU results in storage images and render targets stay on the GPU until something reads their memory. A title that frees such an image after waiting for its GPU work and reuses the memory can have its new data overwritten when the results are stored later ([TechnicalDebt](../dev/TechnicalDebt.md)). `APS5_STORE_AT_FLIP=1` stores every pending result when a display buffer is flipped, at the cost of one write-back per pending image and frame. Any other value stops the title at its first flip. + ## Exit codes `0`: conversion succeeded. `1`: invalid arguments. `2`: conversion failed; the error is printed to stderr. With `--autorun`, successful conversion returns the launched application's exit code. @@ -105,10 +123,10 @@ The game runs on the first Vulkan 1.1 device with graphics and compute queues an ```sh relinker --registry source/input.elf app.elf -python3 tools/import_audit.py app.registry.json --libs build/core/libs/libs --modules source/sce_module +python3 tools/import_audit.py app.registry.json app.libc.prx.guest.prx.registry.json --libs build/core/libs/libs --modules source/sce_module ``` -`--registry` writes `app.registry.json` beside the output. `--libs` is the directory the `libs` target fills ([build instructions](../dev/BUILD.md)); the exports are read from the built `.prx` files, so the result matches what the loader finds. Imports are counted once per NID and library, and every reference lands in exactly one class: +`--registry` writes `app.registry.json` for the executable and one `app..registry.json` for each converted bundled module; pass all of them. `--libs` is the directory the `libs` target fills ([build instructions](../dev/BUILD.md)); the exports are read from the built `.prx` files, so the result matches what the loader finds. Imports are counted once per NID and library, and every reference lands in exactly one class: | Class | Meaning | |-------|---------| @@ -129,4 +147,4 @@ The report also lists needed libraries that have no file in `--libs` or `--modul Exit codes: `0` nothing blocks loading, `1` there are `absent` imports or needed libraries without a file, `2` an input could not be read; the message names the file and the value. -The registry lists the imports of the executable, not of its bundled modules. +A module registry lists `nid`, `library` and `targetOffset` for each relocation against a function that no other bundled module exports, without the `#` suffix of the executable's NIDs. diff --git a/tools/check_conventions.py b/tools/check_conventions.py index 78a136d13..eb53a431d 100644 --- a/tools/check_conventions.py +++ b/tools/check_conventions.py @@ -216,7 +216,7 @@ def allowed_notes(path): def function_body(lines, start): text = "\n".join(lines[start:start + 400]) - text = re.sub(r"//.*|/\*.*?\*/", "", STRINGS.sub('""', text), flags=re.S) + text = re.sub(r"//[^\r\n]*|/\*.*?\*/", "", STRINGS.sub('""', text), flags=re.S) brace, semicolon = text.find("{"), text.find(";") if brace < 0 or 0 <= semicolon < brace: return None diff --git a/tools/check_skips.py b/tools/check_skips.py new file mode 100644 index 000000000..f2f8fe1aa --- /dev/null +++ b/tools/check_skips.py @@ -0,0 +1,47 @@ +import argparse +import re +import xml.etree.ElementTree as ET +from pathlib import Path + +DEVICE_LIMITATION = re.compile(r"skipped,.*(subgroup|the device has no|the device reports|no display)", re.IGNORECASE) + +NO_DISPLAY = re.compile(r"skipped,.*no display", re.IGNORECASE) + +WINDOWS_PREFIX = "agc" + + +def skips(path): + for case in ET.parse(path).getroot().iter("testcase"): + if case.find("skipped") is None: + continue + name = case.get("name") + if not name: + continue + output = case.find("system-out") + yield name, (output.text or "") if output is not None else "" + + +if __name__ == "__main__": + parser = argparse.ArgumentParser(description="fail when a test is skipped that the job's device is not expected to skip") + parser.add_argument("junit", type=Path, help="the file written by ctest --output-junit") + parser.add_argument("--job", choices=("linux", "windows"), default="linux") + parser.add_argument("--list", action="store_true", help="print the skipped tests and why the file says they skipped") + args = parser.parse_args() + found = sorted(skips(args.junit)) + if args.list: + for name, output in found: + reason = next((line.strip() for line in output.splitlines() if line.startswith("skipped")), "") + print(f"{name}: {reason}") + raise SystemExit + if args.job == "windows": + unexpected = [name for name, output in found if not name.startswith(WINDOWS_PREFIX) and not NO_DISPLAY.search(output)] + expected = f"any {WINDOWS_PREFIX}* test, which the runner has no Vulkan driver for, or a test that reports no display or Vulkan device" + else: + unexpected = [name for name, output in found if not DEVICE_LIMITATION.search(output)] + expected = "any test that reports the device limitation it needs, such as narrow subgroups, a missing extension or no display for a Vulkan window" + for name in unexpected: + print(f"error: {name} skipped") + if unexpected: + print(f"{len(unexpected)} test(s) skipped for a reason that is not expected: the job lost that coverage and ctest still reports every test as passed") + raise SystemExit(1) + print(f"{len(found)} test(s) skipped, all expected ({expected})") diff --git a/tools/check_test_registration.py b/tools/check_test_registration.py index a0115b2cd..991cdb59c 100644 --- a/tools/check_test_registration.py +++ b/tools/check_test_registration.py @@ -5,10 +5,7 @@ from pathlib import Path TEST_OUTPUT = r"tests[\\/]" -KNOWN_UNRUN = { - "agc_driver_recorder_tests", - "video_out_flip_tests", -} +KNOWN_UNRUN = set() def built(build): @@ -37,7 +34,8 @@ def registered(build): found = re.match(r"\s*add_test\s*\((.*)\)\s*$", line) if not found: continue - for token in re.findall(r'"([^"]*)"|(\S+)', found.group(1)): + tokens = re.findall(r'"([^"]*)"|(\S+)', found.group(1)) + for token in tokens[1:]: argument = token[0] or token[1] base = re.split(r"[\\/]", argument)[-1] if base.endswith(".exe"): diff --git a/tools/hw-oracle/hw_oracle.py b/tools/hw-oracle/hw_oracle.py index b3d6e4ed7..6cb1c4738 100644 --- a/tools/hw-oracle/hw_oracle.py +++ b/tools/hw-oracle/hw_oracle.py @@ -11,6 +11,8 @@ from pathlib import Path HERE = Path(__file__).resolve().parent CACHE = Path(os.environ.get("HW_ORACLE_CACHE") or Path(os.environ.get("XDG_CACHE_HOME") or Path.home() / ".cache") / "anyps5-hw-oracle") ROWS_PER_DISPATCH = 1024 +DEFAULT_LDS = 4096 +MAX_LDS = 65536 def rocm_root(): @@ -53,18 +55,20 @@ def target(): return os.environ.get("HW_ORACLE_TARGET") or subprocess.run([oracle(), "--target"], capture_output=True, text=True, check=True).stdout.strip() -def assemble(body, work, wave64, *, ieee, denorm32, denorm16, dx10_clamp, round32, round16, fp16_overflow): +def assemble(body, work, wave64, *, ieee, denorm32, denorm16, dx10_clamp, round32, round16, fp16_overflow, lds=DEFAULT_LDS): modes = (("ieee", ieee, 2), ("denorm32", denorm32, 4), ("denorm16", denorm16, 4), ("dx10_clamp", dx10_clamp, 2), ("round32", round32, 4), ("round16", round16, 4), ("fp16_overflow", fp16_overflow, 2)) for name, value, limit in modes: if not isinstance(value, int) or value not in range(limit): raise ValueError(f"{name} must be an integer from 0 to {limit - 1}") + if not isinstance(lds, int) or lds not in range(MAX_LDS + 1): + raise ValueError(f"lds must be an integer from 0 to {MAX_LDS}") text = (HERE / "template.s").read_text() for key, value in (("@TARGET@", target()), ("@WAVE32@", "0" if wave64 else "1"), ("@WAVESIZE@", "64" if wave64 else "32"), ("@DENORM@", str(int(denorm32))), ("@DENORM16@", str(int(denorm16))), ("@IEEE@", str(int(ieee))), ("@DX10_CLAMP@", str(int(dx10_clamp))), ("@ROUND32@", str(int(round32))), ("@ROUND16@", str(int(round16))), - ("@FP16_OVERFLOW@", str(int(fp16_overflow))), ("@BODY@", body)): + ("@FP16_OVERFLOW@", str(int(fp16_overflow))), ("@LDS@", str(lds)), ("@BODY@", body)): text = text.replace(key, value) (work / "k.s").write_text(text) subprocess.run([tool("clang"), "-x", "assembler", "-target", "amdgcn-amd-amdhsa", f"-mcpu={target()}", *(["-mwavefrontsize64"] if wave64 else []), "-c", str(work / "k.s"), "-o", str(work / "k.o")], check=True) @@ -72,7 +76,7 @@ def assemble(body, work, wave64, *, ieee, denorm32, denorm16, dx10_clamp, round3 return work / "k.co" -def run(body, rows, extra=b"", wave64=False, coarse=False, *, ieee, denorm32, denorm16, dx10_clamp, round32, round16, fp16_overflow): +def run(body, rows, extra=b"", wave64=False, coarse=False, *, ieee, denorm32, denorm16, dx10_clamp, round32, round16, fp16_overflow, lds=DEFAULT_LDS): rows = [tuple(r) for r in rows] wave = 64 if wave64 else 32 padded = rows + [(0, 0, 0, 0)] * (-len(rows) % wave) @@ -84,7 +88,7 @@ def run(body, rows, extra=b"", wave64=False, coarse=False, *, ieee, denorm32, de with tempfile.TemporaryDirectory() as tmp: work = Path(tmp) code = assemble(body, work, wave64, ieee=ieee, denorm32=denorm32, denorm16=denorm16, - dx10_clamp=dx10_clamp, round32=round32, round16=round16, fp16_overflow=fp16_overflow) + dx10_clamp=dx10_clamp, round32=round32, round16=round16, fp16_overflow=fp16_overflow, lds=lds) for start in range(0, len(padded), ROWS_PER_DISPATCH): chunk = padded[start:start + ROWS_PER_DISPATCH] (work / "in.bin").write_bytes(b"".join(struct.pack("<4I", *r) for r in chunk) + extra) @@ -107,6 +111,7 @@ def main(): parser.add_argument("--round16", type=int, choices=range(4), required=True) parser.add_argument("--fp16-overflow", type=int, choices=(0, 1), required=True) parser.add_argument("--coarse", action="store_true", help="input and output in coarse-grained GPU memory") + parser.add_argument("--lds", type=int, default=DEFAULT_LDS, metavar="BYTES", help=f"group segment size, 0 to {MAX_LDS} (default {DEFAULT_LDS})") parser.add_argument("--extra", type=Path, help="bytes appended after the rows") parser.add_argument("--outs", type=int, choices=range(1, 17), default=16, metavar="N", help="print v10..v(10+N-1)") args = parser.parse_args() @@ -117,7 +122,7 @@ def main(): for result in run(args.body.read_text(), rows, extra, args.wave64, args.coarse, ieee=args.ieee, denorm32=args.denorm32, denorm16=args.denorm16, dx10_clamp=args.dx10_clamp, round32=args.round32, round16=args.round16, - fp16_overflow=args.fp16_overflow): + fp16_overflow=args.fp16_overflow, lds=args.lds): print(" ".join(f"{v:08x}" for v in result[:args.outs])) diff --git a/tools/hw-oracle/template.s b/tools/hw-oracle/template.s index 915409b61..f14a96c03 100644 --- a/tools/hw-oracle/template.s +++ b/tools/hw-oracle/template.s @@ -40,7 +40,7 @@ probe: .amdhsa_kernel probe .amdhsa_user_sgpr_kernarg_segment_ptr 1 .amdhsa_kernarg_size 16 - .amdhsa_group_segment_fixed_size 4096 + .amdhsa_group_segment_fixed_size @LDS@ .amdhsa_next_free_vgpr 128 .amdhsa_next_free_sgpr 48 .amdhsa_wavefront_size32 @WAVE32@ @@ -60,7 +60,7 @@ amdhsa.kernels: - .name: probe .symbol: probe.kd .kernarg_segment_size: 16 - .group_segment_fixed_size: 4096 + .group_segment_fixed_size: @LDS@ .private_segment_fixed_size: 0 .kernarg_segment_align: 8 .wavefront_size: @WAVESIZE@ diff --git a/tools/hw-oracle/test_hw_oracle.py b/tools/hw-oracle/test_hw_oracle.py index 15a33fd05..0b8ad0ea9 100644 --- a/tools/hw-oracle/test_hw_oracle.py +++ b/tools/hw-oracle/test_hw_oracle.py @@ -112,14 +112,49 @@ class FloatModeTests(unittest.TestCase): argv += [item for name, value in modes.items() for item in ["--" + name.replace("_", "-"), str(value)]] with patch("sys.argv", argv), patch.object(hw_oracle, "run", return_value=[]) as run: hw_oracle.main() - run.assert_called_once_with("s_nop 0", [(1, 2, 3, 4)], b"", True, True, **modes) + run.assert_called_once_with("s_nop 0", [(1, 2, 3, 4)], b"", True, True, **modes, lds=4096) def test_run_forwards_modes_to_assembler(self): with patch.object(hw_oracle, "assemble") as assemble: hw_oracle.run("s_nop 0", [], wave64=True, **MODES) self.assertEqual(assemble.call_args.args[0], "s_nop 0") self.assertTrue(assemble.call_args.args[2]) - self.assertEqual(assemble.call_args.kwargs, MODES) + self.assertEqual(assemble.call_args.kwargs, MODES | {"lds": 4096}) + + +class GroupSegmentTests(unittest.TestCase): + def test_cli_forwards_lds(self): + with tempfile.TemporaryDirectory() as tmp: + body = Path(tmp) / "body.s" + rows = Path(tmp) / "rows.txt" + body.write_text("s_nop 0") + rows.write_text("1 2 3 4\n") + argv = ["hw_oracle.py", str(body), str(rows), "--lds", "8192"] + argv += [item for name, value in MODES.items() for item in ["--" + name.replace("_", "-"), str(value)]] + with patch("sys.argv", argv), patch.object(hw_oracle, "run", return_value=[]) as run: + hw_oracle.main() + self.assertEqual(run.call_args.kwargs["lds"], 8192) + + def test_run_forwards_lds_to_assembler(self): + with patch.object(hw_oracle, "assemble") as assemble: + hw_oracle.run("s_nop 0", [], lds=65536, **MODES) + self.assertEqual(assemble.call_args.kwargs["lds"], 65536) + + def test_invalid_lds_fails_before_tool_lookup(self): + with tempfile.TemporaryDirectory() as tmp, patch.object(hw_oracle, "target") as target: + for value in (-1, 65537, 0.5, "4096"): + with self.subTest(value=value), self.assertRaisesRegex(ValueError, "lds"): + hw_oracle.assemble("s_nop 0", Path(tmp), False, lds=value, **MODES) + target.assert_not_called() + + def test_template_takes_lds(self): + with tempfile.TemporaryDirectory() as tmp, patch.object(hw_oracle, "target", return_value="gfx1036"), \ + patch.object(hw_oracle.subprocess, "run"), patch.object(hw_oracle, "tool", return_value="clang"): + hw_oracle.assemble("s_nop 0", Path(tmp), False, lds=8192, **MODES) + text = (Path(tmp) / "k.s").read_text() + self.assertIn(".amdhsa_group_segment_fixed_size 8192", text) + self.assertIn(".group_segment_fixed_size: 8192", text) + self.assertNotIn("@LDS@", text) class AssemblyTests(unittest.TestCase): diff --git a/tools/import_audit.py b/tools/import_audit.py index ea046eab5..25d6f611b 100644 --- a/tools/import_audit.py +++ b/tools/import_audit.py @@ -312,6 +312,9 @@ def main(argv=None): except AuditError as error: print(f"FAIL: {error}", file=sys.stderr) return 2 + except OSError as error: + print(f"FAIL: {error.strerror}: {error.filename}" if error.filename else f"FAIL: {error}", file=sys.stderr) + return 2 print(render(summary, records, missing)) return 1 if summary["unique_by_class"]["absent"] or missing else 0 diff --git a/tools/nid_names.py b/tools/nid_names.py index 2eb43a804..e4c0c98f0 100644 --- a/tools/nid_names.py +++ b/tools/nid_names.py @@ -41,7 +41,13 @@ def load_db(path): if not path.exists(): path.parent.mkdir(parents=True, exist_ok=True) print("downloading NID database to %s ..." % path, file=sys.stderr) - urllib.request.urlretrieve(DB_URL, str(path)) + with tempfile.NamedTemporaryFile(dir=path.parent, prefix=path.name + ".", suffix=".tmp", delete=False) as download: + temporary = Path(download.name) + try: + urllib.request.urlretrieve(DB_URL, str(temporary)) + temporary.replace(path) + finally: + temporary.unlink(missing_ok=True) db = {} with open(path, errors="replace") as handle: for line in handle: diff --git a/tools/pr_overlap.py b/tools/pr_overlap.py index db2bbaa90..a2b60a362 100644 --- a/tools/pr_overlap.py +++ b/tools/pr_overlap.py @@ -3,8 +3,9 @@ import json import os import re import subprocess -from itertools import combinations +from itertools import chain, combinations from pathlib import Path +from urllib.parse import urlencode PARENT = 'if .parent then "\\(.parent.owner.login)/\\(.parent.name)" else .nameWithOwner end' EXPORT = re.compile(r"^\+.*\bAPS5_VABI\s+(\w+)\s*\(") @@ -26,14 +27,15 @@ def upstream(): def open_prs(base): - out = gh("pr", "list", "-R", REPO, "--base", base, "-L", "200", "--json", "number,author,headRefOid,body") + query = urlencode({"state": "open", "base": base, "per_page": 100}) + out = gh("api", "--paginate", "--slurp", f"repos/{REPO}/pulls?{query}") prs = {} - for pr in json.loads(out): + for pr in chain.from_iterable(json.loads(out)): depends = DEPENDS.search(pr["body"] or "") prs[pr["number"]] = { "ref": f"refs/pr/{pr['number']}", - "sha": pr["headRefOid"], - "author": pr["author"]["login"], + "sha": pr["head"]["sha"], + "author": pr["user"]["login"], "depends": {int(n) for n in re.findall(r"#(\d+)", depends.group(1))} if depends else set(), } return prs @@ -58,6 +60,15 @@ def commit(tree): "-p", "refs/pr/base", "-m", "pr_overlap").strip() +def exports(diff): + found = set() + for line in diff.splitlines(): + m = EXPORT.match(line) + if m and not m.group(1).endswith("_nid_no_patch") and not line.rstrip().endswith(";"): + found.add(m.group(1)) + return found + + def scan(pr): tree, conflicted = merge("refs/pr/base", pr["ref"]) pr["tree"] = None if conflicted else commit(tree) @@ -69,8 +80,7 @@ def scan(pr): if status == "A": pr["added"].add(path) diff = git("diff", "-U0", "--no-color", old, new, "--", "core/libs/prx/*.cpp") - pr["exports"] = {m.group(1) for m in map(EXPORT.match, diff.splitlines()) - if m and not m.group(1).endswith("_nid_no_patch")} + pr["exports"] = exports(diff) def hunks(pr, path): @@ -131,6 +141,12 @@ def report(rows, blob): return "\n".join(lines) + "\n" +def write_reports(out, rows, blob): + out.mkdir(parents=True, exist_ok=True) + for number, found in rows.items(): + (out / f"{number}.md").write_text(report(found, blob), encoding="utf-8") + + if __name__ == "__main__": parser = argparse.ArgumentParser(description="report open pull requests that overlap each other") parser.add_argument("--base", default="main") @@ -158,6 +174,4 @@ if __name__ == "__main__": if args.branch: print(report(rows[0], blob) or "No overlap with open pull requests.", end="") raise SystemExit - args.out.mkdir(parents=True, exist_ok=True) - for number, found in rows.items(): - (args.out / f"{number}.md").write_text(report(found, blob)) + write_reports(args.out, rows, blob) diff --git a/tools/progress.py b/tools/progress.py index dc566a69a..505c684d7 100644 --- a/tools/progress.py +++ b/tools/progress.py @@ -2,6 +2,7 @@ import argparse import json import os import re +import sys from html import escape from pathlib import Path @@ -12,6 +13,8 @@ ISA = Path(__file__).resolve().parent / "rdna_isa.txt" SOURCE = f'https://github.com/{os.environ.get("GITHUB_REPOSITORY", "boykopovar/AnyPS5")}/blob/main' DEFINITION = re.compile(r"\bAPS5_VABI\s+(\w+)\s*\([^;{]*\)\s*(?:noexcept\s*)?(?:try\s*)?\{") STUB = "NotImplemented_nid_no_patch" +CMAKE_SHARED = re.compile(r'^\s*include\(\s*"?\$\{CMAKE_CURRENT_SOURCE_DIR\}/\.\./([\w.-]+)/[\w.-]+\.cmake"?\s*\)', re.M) +SOURCE_SHARED = re.compile(r'^\s*#\s*include\s*"prx/([\w.-]+)/[\w./-]+\.cpp"', re.M) STUB_WRAPPER = re.compile(r"\bstatic\s+(?:\[\[noreturn\]\]\s+)?void\s+(\w+)\s*\([^;{]*\)\s*\{") FLAT_SEGMENTS = ("GLOBAL_", "SCRATCH_") OPCODE_SENTINELS = {"Invalid", "Count", "Unknown", "Unsupported"} @@ -95,11 +98,14 @@ def scan_library(path): todo -= done group = {"name": path.name, "label": path.name.removeprefix("libSce"), "done": len(done), "todo": len(todo), "done_names": sorted(done), "todo_names": sorted(todo)} - cmake = path / "CMakeLists.txt" - if not (done or todo) and cmake.is_file(): - shared = re.search(r'^\s*include\(\s*"?\$\{CMAKE_CURRENT_SOURCE_DIR\}/\.\./([\w.-]+)/[\w.-]+\.cmake"?\s*\)', cmake.read_text(), re.M) - if shared and (path.parent / shared[1]).is_dir() and shared[1] != path.name: - group["shared_sources"] = shared[1] + if not (done or todo): + cmake = path / "CMakeLists.txt" + texts = [(CMAKE_SHARED, cmake.read_text())] if cmake.is_file() else [] + texts += [(SOURCE_SHARED, source.read_text(errors="ignore")) for source in sorted(path.glob("*.cpp"))] + owners = [m[1] for pattern, content in texts for m in pattern.finditer(content)] + owners = [o for o in owners if o != path.name and (path.parent / o).is_dir()] + if owners: + group["shared_sources"] = owners[0] return group @@ -254,7 +260,10 @@ def table(heading, column, data): f'{owner}') continue total = group["done"] + group["todo"] - percent = f'{100 * group["done"] / total:.0f}%' if total else "-" + if not total: + rows.append(f'{escape(group["name"])}No exports') + continue + percent = f'{100 * group["done"] / total:.0f}%' rows.append(f'{escape(group["name"])}{group["done"]}{total}{percent}') rows.append(f'Total{data["done"]}{data["total"]}{data["percent"]}%') rows.append("") @@ -278,6 +287,7 @@ def summary(libraries, shaders): "

A function is implemented when it no longer calls NotImplemented_nid_no_patch. " f'The total only includes functions already declared in core/libs/prx, ' "not every function exported by the PS5 firmware. Shared-source libraries are counted only at their source library. " + "Libraries with no exports only provide an empty module for titles that load them. " f'Counting rules.

', table("GPU shader instructions", "Encoding", shaders), f'

The total is the AMD RDNA 1 + RDNA 2 instruction list (tools/rdna_isa.txt). ' @@ -333,6 +343,8 @@ def report(base, head): if __name__ == "__main__": + if hasattr(sys.stdout, "reconfigure"): + sys.stdout.reconfigure(encoding="utf-8") parser = argparse.ArgumentParser() parser.add_argument("output", type=Path, nargs="?") parser.add_argument("--root", type=Path, help="source tree to measure instead of the one containing this script") @@ -340,7 +352,7 @@ if __name__ == "__main__": help="print a markdown report of the changes between two progress.json files") args = parser.parse_args() if args.compare: - base, head = (json.loads(path.read_text()) for path in args.compare) + base, head = (json.loads(path.read_text(encoding="utf-8")) for path in args.compare) print(report(base, head), end="") raise SystemExit if not args.output: diff --git a/tools/stubgen.py b/tools/stubgen.py new file mode 100644 index 000000000..2dca532f4 --- /dev/null +++ b/tools/stubgen.py @@ -0,0 +1,212 @@ +#!/usr/bin/env python3 +import argparse +import re +import sys +from pathlib import Path + +try: + from tools.nid_names import CHARSET, SUFFIX, compute_nid +except ImportError: + from nid_names import CHARSET, SUFFIX, compute_nid + + +def strip_nid_token(raw): + s = raw.lstrip("\ufeff").strip() + if not s: + return "" + if s.startswith("#"): + return "" + if "#" in s: + s = s.split("#", 1)[0].strip() + if not s: + return "" + return s.split()[0] + + +def parse_nids(path): + nids = [] + seen = set() + with open(path, "r", encoding="utf-8-sig", errors="replace") as f: + for line in f: + nid = strip_nid_token(line) + if not nid: + continue + if nid in seen: + continue + seen.add(nid) + nids.append(nid) + return nids + + +def parse_db(path): + mapping = {} + with open(path, "r", encoding="utf-8-sig", errors="replace") as f: + for line in f: + s = line.lstrip("\ufeff").strip() + if not s or s.startswith("#"): + continue + parts = s.split() + if len(parts) < 2: + continue + nid = parts[0].split("#", 1)[0].strip().lstrip("\ufeff") + name = parts[1].split("#", 1)[0].strip().lstrip("\ufeff") + if not nid or not name: + continue + if nid not in mapping: + mapping[nid] = name + return mapping + + +def sanitize_identifier(name): + s = re.sub(r"[^0-9A-Za-z_]", "_", name) + if not s: + return "_" + if s[0].isdigit(): + s = "_" + s + return s + + +def module_prefix(module): + base = module + if base.startswith("lib") and len(base) > 3 and base[3].isupper(): + base = base[3:] + base = base[0].lower() + base[1:] if base else module + return sanitize_identifier(base) + + +def is_runtime_name(name): + if name.startswith("_"): + return True + if name.startswith("sce") or name.startswith("Sce"): + return False + return True + + +def is_valid_ident(name): + return re.match(r"^[A-Za-z_][A-Za-z0-9_]*$", name) is not None + + +def _default_td_path(module): + candidates = [ + "core/libs/prx/" + module + "/Export.cpp", + "core/libs/prx/" + module + "/Unimplemented.cpp", + "core/libs/prx/" + module + "/src/Unimplemented.cpp", + ] + root = Path(__file__).resolve().parent.parent + for cand in candidates: + try: + if (root / cand).is_file(): + return cand + except OSError: + continue + return candidates[0] + + +def td_target(module, td_path, td_lib): + if td_path is None: + td_path = _default_td_path(module) + if td_lib is None: + td_lib = module + return (td_path, td_lib) + + +def generate(module, nids, db, td_path=None, td_lib=None): + prefix = module_prefix(module) + path, lib = td_target(module, td_path, td_lib) + used = set() + unknown_counter = 0 + chunks = [] + td = [] + for nid in nids: + real = db.get(nid) + func = None + display = None + if real is not None: + if is_valid_ident(real): + if compute_nid(real) == nid: + if is_runtime_name(real): + candidate = real + "_nid_postfix" + else: + candidate = real + if candidate not in used: + func = candidate + display = real + if func is None: + while True: + func = "%sUnknown%02d" % (prefix, unknown_counter) + unknown_counter += 1 + if func not in used: + break + display = None + used.add(func) + if display is not None: + chunks.append("") + chunks.append("int APS5_VABI %s(void) {" % func) + chunks.append(' NotImplemented_nid_no_patch("%s");' % nid) + chunks.append(" return 0;") + chunks.append("}") + td.append("- [%s](%s) (%s) - unknown signature" % (display, path, lib)) + else: + chunks.append("") + chunks.append('APS5_EXPORT("%s", %s);' % (nid, func)) + chunks.append("int APS5_VABI %s(void) {" % func) + chunks.append(' NotImplemented_nid_no_patch("%s");' % nid) + chunks.append(" return 0;") + chunks.append("}") + td.append("- [%s](%s) (%s) - unknown name, signature" % (nid, path, lib)) + header = [] + header.append("#include ") + header.append("#include ") + header.append('#include "SceTypes.hpp"') + header.append('#include "prx/libc/include/General.hpp"') + header.append("") + header.append('extern "C" {') + footer = ["", "}"] + code = "\n".join(header + chunks + footer) + "\n" + td_text = "\n".join(td) + ("\n" if td else "") + return (code, td_text) + + +def main(argv=None): + ap = argparse.ArgumentParser(description="Generate AnyPS5 Export.cpp skeletons and TechnicalDebt lines") + ap.add_argument("--module", required=True, help="target module name, e.g. libSceAmpr, used for UnknownNN prefix and default TechnicalDebt path core/libs/prx/MODULE/Export.cpp") + ap.add_argument("--nids", required=True, help="path to missing-NIDs file, one 11-char NID per line") + ap.add_argument("--db", required=False, default=None, help="path to NID name CSV, NID name per line like aerolib.csv") + ap.add_argument("--out", required=False, default=None, help="output file for code, default stdout") + ap.add_argument("--td-path", required=False, default=None, help="override file path used inside TechnicalDebt lines, default derived from module") + ap.add_argument("--td-lib", required=False, default=None, help="override lib label used inside TechnicalDebt lines, default derived from module") + ap.add_argument("--td-out", required=False, default=None, help="output file for TechnicalDebt lines, default stdout") + ap.add_argument("--td-only", action="store_true", help="print only TechnicalDebt lines, no code") + args = ap.parse_args(argv) + nids = parse_nids(args.nids) + db = {} + if args.db: + db = parse_db(args.db) + code, td = generate(args.module, nids, db, args.td_path, args.td_lib) + if args.td_only: + if args.td_out: + with open(args.td_out, "w", encoding="utf-8", newline="\n") as f: + f.write(td) + else: + sys.stdout.write(td) + return 0 + if args.out: + with open(args.out, "w", encoding="utf-8", newline="\n") as f: + f.write(code) + else: + sys.stdout.write(code) + if args.td_out: + with open(args.td_out, "w", encoding="utf-8", newline="\n") as f: + f.write(td) + else: + if args.out: + sys.stdout.write(td) + else: + if td: + sys.stdout.write("\n") + sys.stdout.write(td) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tools/tests/test_check_conventions_comments.py b/tools/tests/test_check_conventions_comments.py new file mode 100644 index 000000000..8f577a370 --- /dev/null +++ b/tools/tests/test_check_conventions_comments.py @@ -0,0 +1,86 @@ +import os +import subprocess +import sys +import tempfile +import unittest +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) +from check_conventions import STUB_BODY, function_body + + +class FunctionBodyTests(unittest.TestCase): + def test_line_comments_preserve_following_statements(self): + sources = ( + "int APS5_VABI sceExample() {\n// pending implementation\nreturn 0;\n}", + "int APS5_VABI sceExample() // pending implementation\n{\nreturn 0;\n}", + "int APS5_VABI sceExample() {\n(void)arg; // unused argument\nreturn 0;\n}", + "int APS5_VABI sceExample() {\n// ignored } ; { tokens\nreturn 0;\n}", + ) + for source in sources: + with self.subTest(source=source): + body = function_body(source.splitlines(), 0) + self.assertIsNotNone(body) + self.assertIsNotNone(STUB_BODY.fullmatch(body)) + + def test_line_comment_does_not_turn_an_implementation_into_a_stub(self): + source = "int APS5_VABI sceExample() {\n// validate the request\nvalidate();\nreturn 0;\n}" + body = function_body(source.splitlines(), 0) + self.assertEqual(body, "validate();return0;") + self.assertIsNone(STUB_BODY.fullmatch(body)) + + def test_block_comments_preserve_following_statements(self): + source = "int APS5_VABI sceExample() {\n/* pending\nimplementation */\nreturn 0;\n}" + self.assertEqual(function_body(source.splitlines(), 0), "return0;") + + def test_comment_markers_and_braces_in_strings_are_ignored(self): + source = 'int APS5_VABI sceExample() {\nuse("// }", "/* { */");\nreturn 0;\n}' + self.assertEqual(function_body(source.splitlines(), 0), 'use("","");return0;') + + def test_declaration_has_no_body(self): + source = "int APS5_VABI sceExample();\nint APS5_VABI sceOther() { return 0; }" + self.assertIsNone(function_body(source.splitlines(), 0)) + + def test_unterminated_function_has_no_body(self): + source = "int APS5_VABI sceExample() {\n// pending implementation\nreturn 0;" + self.assertIsNone(function_body(source.splitlines(), 0)) + + +class SilentStubTests(unittest.TestCase): + def test_commented_stub_is_reported_when_debt_change_allows_comments(self): + checker = Path(__file__).resolve().parents[1] / "check_conventions.py" + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + + def git(*args): + return subprocess.run( + ["git", "-c", "user.name=Conventions Test", "-c", "user.email=test@example.com", + "-c", "commit.gpgsign=false", *args], + cwd=root, capture_output=True, text=True, check=True, + ) + + git("init", "-q") + debt = root / "docs/dev/TechnicalDebt.md" + debt.parent.mkdir(parents=True) + debt.write_text("# Project technical debt\n", encoding="utf-8") + git("add", ".") + git("commit", "-qm", "docs: record technical debt") + source = root / "core/libs/prx/libSceExample/Export.cpp" + source.parent.mkdir(parents=True) + source.write_text("int APS5_VABI sceExample() {\n// pending implementation\nreturn 0;\n}\n", encoding="utf-8") + debt.write_text("# Project technical debt\n\nAn unrelated build limitation.\n", encoding="utf-8") + git("add", ".") + git("commit", "-qm", "feat: add example export") + result = subprocess.run( + [sys.executable, str(checker), "--base", "HEAD~1"], + cwd=root, capture_output=True, text=True, + env={key: value for key, value in os.environ.items() + if key not in ("GITHUB_EVENT_PATH", "GITHUB_STEP_SUMMARY", "GITHUB_ACTIONS")}, + ) + self.assertEqual(result.returncode, 1, result.stdout + result.stderr) + self.assertIn("[silent-stub]", result.stdout) + self.assertNotIn("[comment]", result.stdout) + + +if __name__ == "__main__": + unittest.main() diff --git a/tools/tests/test_import_audit.py b/tools/tests/test_import_audit.py index 860a5b9e5..2242cae3c 100644 --- a/tools/tests/test_import_audit.py +++ b/tools/tests/test_import_audit.py @@ -273,6 +273,22 @@ class ImportAuditTests(unittest.TestCase): self.assertEqual((code, out), (2, "")) self.assertIn("game.json: entry 0 has an invalid NID 'short'", err) + def test_unreadable_library_and_unwritable_json_exit_two(self): + work = self.work + library = work.libs / "libSceA.prx" + library.write_bytes(exporting("AAAAAAAAAAA")) + registry = work.root / "game.json" + write_registry(registry, [("AAAAAAAAAAA", "libSceA.prx")]) + code, out, err = work.run(registry, "--json", str(work.modules)) + self.assertEqual((code, out), (2, "")) + self.assertIn("FAIL: ", err) + self.assertIn(str(work.modules), err) + denied = PermissionError(13, "Permission denied", str(library)) + with patch.object(Path, "read_bytes", side_effect=denied): + code, out, err = work.run(registry) + self.assertEqual((code, out), (2, "")) + self.assertIn(f"FAIL: Permission denied: {library}", err) + if __name__ == "__main__": unittest.main() diff --git a/tools/tests/test_nid_names.py b/tools/tests/test_nid_names.py new file mode 100644 index 000000000..ca268c741 --- /dev/null +++ b/tools/tests/test_nid_names.py @@ -0,0 +1,72 @@ +import tempfile +import sys +import unittest +from pathlib import Path +from unittest.mock import patch +from urllib.error import URLError + +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) +import nid_names + + +class DatabaseCacheTests(unittest.TestCase): + def test_interrupted_download_does_not_publish_a_partial_cache(self): + with tempfile.TemporaryDirectory() as directory: + cache = Path(directory) / "cache" / "aerolib.csv" + + def interrupted(url, filename): + Path(filename).write_text("nid1 scePartial\n", encoding="utf-8") + raise URLError("connection interrupted") + + with patch.object(nid_names.urllib.request, "urlretrieve", side_effect=interrupted): + with self.assertRaises(URLError): + nid_names.load_db(cache) + self.assertFalse(cache.exists()) + self.assertEqual(list(cache.parent.iterdir()), []) + + def test_retry_after_interruption_downloads_the_complete_database(self): + with tempfile.TemporaryDirectory() as directory: + cache = Path(directory) / "aerolib.csv" + + def interrupted(url, filename): + Path(filename).write_text("nid1 scePartial\n", encoding="utf-8") + raise URLError("connection interrupted") + + def complete(url, filename): + Path(filename).write_text("nid1 sceComplete\nnid2 sceOther\n", encoding="utf-8") + + with patch.object(nid_names.urllib.request, "urlretrieve", side_effect=interrupted): + with self.assertRaises(URLError): + nid_names.load_db(cache) + with patch.object(nid_names.urllib.request, "urlretrieve", side_effect=complete) as download: + self.assertEqual(nid_names.load_db(cache), {"nid1": "sceComplete", "nid2": "sceOther"}) + download.assert_called_once() + self.assertEqual(list(cache.parent.iterdir()), [cache]) + + def test_cache_is_published_only_after_download_completes(self): + with tempfile.TemporaryDirectory() as directory: + cache = Path(directory) / "aerolib.csv" + + def complete(url, filename): + self.assertEqual(url, nid_names.DB_URL) + self.assertNotEqual(Path(filename), cache) + self.assertEqual(Path(filename).parent, cache.parent) + self.assertFalse(cache.exists()) + Path(filename).write_text("nid1 sceComplete\n", encoding="utf-8") + + with patch.object(nid_names.urllib.request, "urlretrieve", side_effect=complete): + self.assertEqual(nid_names.load_db(cache), {"nid1": "sceComplete"}) + self.assertEqual(cache.read_text(encoding="utf-8"), "nid1 sceComplete\n") + self.assertEqual(list(cache.parent.iterdir()), [cache]) + + def test_existing_cache_is_used_without_downloading(self): + with tempfile.TemporaryDirectory() as directory: + cache = Path(directory) / "aerolib.csv" + cache.write_text("nid1 sceExisting\n", encoding="utf-8") + with patch.object(nid_names.urllib.request, "urlretrieve") as download: + self.assertEqual(nid_names.load_db(cache), {"nid1": "sceExisting"}) + download.assert_not_called() + + +if __name__ == "__main__": + unittest.main() diff --git a/tools/tests/test_pr_overlap.py b/tools/tests/test_pr_overlap.py new file mode 100644 index 000000000..1670876dc --- /dev/null +++ b/tools/tests/test_pr_overlap.py @@ -0,0 +1,60 @@ +import json +import sys +import unittest +from pathlib import Path +from unittest.mock import patch + +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) +import pr_overlap + + +class OpenPullRequestsTests(unittest.TestCase): + def test_collects_pull_requests_beyond_the_previous_limit(self): + pull_requests = [ + {"number": number, "user": {"login": f"author{number}"}, + "head": {"sha": f"sha{number}"}, "body": None} + for number in range(1, 202) + ] + pull_requests[-1]["body"] = "Depends on: #12\n- #34\n" + + def response(*args): + if args[0] == "api": + return json.dumps([pull_requests[start:start + 100] + for start in range(0, len(pull_requests), 100)]) + limit = int(args[args.index("-L") + 1]) + return json.dumps([ + {"number": pr["number"], "author": pr["user"], "headRefOid": pr["head"]["sha"], + "body": pr["body"]} for pr in pull_requests[:limit] + ]) + + with patch.object(pr_overlap, "REPO", "owner/project", create=True), \ + patch.object(pr_overlap, "gh", side_effect=response): + found = pr_overlap.open_prs("main") + self.assertEqual(set(found), set(range(1, 202))) + self.assertEqual(found[201], { + "ref": "refs/pr/201", "sha": "sha201", "author": "author201", "depends": {12, 34}, + }) + self.assertEqual(found[1]["depends"], set()) + + def test_base_branch_is_encoded_in_the_paginated_request(self): + with patch.object(pr_overlap, "REPO", "owner/project", create=True), \ + patch.object(pr_overlap, "gh", return_value="[[]]") as gh: + self.assertEqual(pr_overlap.open_prs("topic/one&two"), {}) + gh.assert_called_once_with( + "api", "--paginate", "--slurp", + "repos/owner/project/pulls?state=open&base=topic%2Fone%26two&per_page=100", + ) + + def test_single_page_preserves_author_head_and_dependencies(self): + pages = [[{"number": 7, "user": {"login": "contributor"}, "head": {"sha": "head-sha"}, + "body": "Depends on: none\nUnrelated issue #42"}]] + with patch.object(pr_overlap, "REPO", "owner/project", create=True), \ + patch.object(pr_overlap, "gh", return_value=json.dumps(pages)): + found = pr_overlap.open_prs("main") + self.assertEqual(found, { + 7: {"ref": "refs/pr/7", "sha": "head-sha", "author": "contributor", "depends": set()}, + }) + + +if __name__ == "__main__": + unittest.main() diff --git a/tools/tests/test_pr_overlap_encoding.py b/tools/tests/test_pr_overlap_encoding.py index 2272bcc13..9a4c3f449 100644 --- a/tools/tests/test_pr_overlap_encoding.py +++ b/tools/tests/test_pr_overlap_encoding.py @@ -3,6 +3,7 @@ import json from pathlib import Path import subprocess import sys +from tempfile import TemporaryDirectory import unittest from unittest.mock import patch @@ -73,6 +74,20 @@ class EncodingTests(unittest.TestCase): with self.assertRaisesRegex(RuntimeError, UNICODE): pr_overlap.merge("main", "topic") + def test_report_is_written_as_utf8(self): + with TemporaryDirectory() as directory: + path = Path(directory) / "1.md" + write_text = Path.write_text + + def write_with_utf8(path, text, *args, **kwargs): + self.assertEqual(kwargs.get("encoding"), "utf-8") + return write_text(path, text, *args, **kwargs) + + with patch.object(Path, "write_text", new=write_with_utf8), \ + patch.object(pr_overlap, "report", return_value=UNICODE): + pr_overlap.write_reports(Path(directory), {1: {}}, "https://example.test") + self.assertEqual(path.read_bytes(), UNICODE.encode("utf-8")) + if __name__ == "__main__": unittest.main() diff --git a/tools/tests/test_pr_overlap_exports.py b/tools/tests/test_pr_overlap_exports.py new file mode 100644 index 000000000..eb2e18db2 --- /dev/null +++ b/tools/tests/test_pr_overlap_exports.py @@ -0,0 +1,44 @@ +import sys +import unittest +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) +import pr_overlap + +TRAILING = "+extern \"C\" std::int64_t APS5_VABI recv_nid_postfix(int, void*, std::uint64_t, int) noexcept; \n" +DIFF = """diff --git a/core/libs/prx/libkernel/File/src/Open.cpp b/core/libs/prx/libkernel/File/src/Open.cpp +@@ -96,0 +97,2 @@ ++extern "C" int* APS5_VABI __error_nid_postfix(); +""" + TRAILING + """@@ -135,0 +136,5 @@ ++int APS5_VABI mlock_nid_postfix(const void* address, std::size_t length) { ++std::int64_t APS5_VABI sceKernelRead(int d, void* buf, std::size_t nbytes) noexcept { ++int APS5_VABI getpid_nid_postfix() { return 1; } ++int APS5_VABI sceNpUniversalDataSystemCreateEvent( ++void APS5_VABI Helper_nid_no_patch(int value) { +@@ -200 +205,0 @@ +-int APS5_VABI removed_nid_postfix(int) { +diff --git a/core/libs/prx/libkernel/File/src/Stdio.cpp b/core/libs/prx/libkernel/File/src/Stdio.cpp +@@ -10,2 +10,2 @@ + int APS5_VABI context_nid_postfix(int) { +""" + + +class ExportTests(unittest.TestCase): + def test_added_definitions_are_exports(self): + self.assertEqual(pr_overlap.exports(DIFF), {"mlock_nid_postfix", "sceKernelRead", "getpid_nid_postfix", "sceNpUniversalDataSystemCreateEvent"}) + + def test_declarations_are_not_exports(self): + self.assertTrue(TRAILING.endswith("; \n")) + found = pr_overlap.exports(DIFF) + self.assertNotIn("__error_nid_postfix", found) + self.assertNotIn("recv_nid_postfix", found) + + def test_removed_context_and_helper_lines_are_not_exports(self): + found = pr_overlap.exports(DIFF) + self.assertNotIn("removed_nid_postfix", found) + self.assertNotIn("context_nid_postfix", found) + self.assertNotIn("Helper_nid_no_patch", found) + + +if __name__ == "__main__": + unittest.main() diff --git a/tools/tests/test_progress.py b/tools/tests/test_progress.py index 144a8d998..ddefe767e 100644 --- a/tools/tests/test_progress.py +++ b/tools/tests/test_progress.py @@ -81,6 +81,28 @@ class ProgressTests(unittest.TestCase): self.assertIn("-1 removed", report) self.assertIn("Pending", report) + def test_empty_libraries(self): + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + owner = root / "libSceExample" + alias = root / "libSceExampleBackend" + dummy = root / "libSceDummy" + for path in (owner, alias, dummy): + path.mkdir() + (owner / "Export.cpp").write_text("int APS5_VABI Ready() { return 0; }\n") + (alias / "Export.cpp").write_text('#include "prx/libSceExample/Export.cpp"\n') + (dummy / "Export.cpp").write_text('#include "prx/libc/include/General.hpp"\nextern "C" {\nAPS5_DUMMY_FUN\n}\n') + with patch.object(progress, "PRX", root): + data = progress.collect_libraries() + groups = {group["name"]: group for group in data["groups"]} + self.assertEqual(groups[alias.name]["shared_sources"], owner.name) + self.assertNotIn("shared_sources", groups[dummy.name]) + self.assertEqual((data["done"], data["total"]), (1, 1)) + html = progress.table("Libraries", "Library", data) + self.assertIn(f'{alias.name}Shared sources:', html) + self.assertIn(f'{dummy.name}No exports', html) + self.assertNotIn("0", html) + if __name__ == "__main__": unittest.main() diff --git a/tools/tests/test_progress_encoding.py b/tools/tests/test_progress_encoding.py new file mode 100644 index 000000000..289389fc5 --- /dev/null +++ b/tools/tests/test_progress_encoding.py @@ -0,0 +1,62 @@ +import json +import os +import subprocess +import sys +import tempfile +import unittest +from pathlib import Path + +PROGRESS = Path(__file__).resolve().parents[1] / "progress.py" + + +def snapshot(implemented, library="libSceExample"): + return { + "libraries": { + "done": int(implemented), "total": 1, "percent": 100 if implemented else 0, + "groups": [{"name": library, "done": int(implemented), "todo": int(not implemented), + "done_names": ["Ready"] if implemented else [], + "todo_names": [] if implemented else ["Ready"]}], + }, + "shaders": {"done": 0, "total": 0, "percent": 0, "groups": []}, + } + + +class ProgressEncodingTests(unittest.TestCase): + def compare(self, base, head, encoding="cp1252"): + with tempfile.TemporaryDirectory(prefix="progress encoding ") as directory: + paths = [Path(directory) / name for name in ("base.json", "head.json")] + for path, data in zip(paths, (base, head)): + path.write_text(json.dumps(data, ensure_ascii=False), encoding="utf-8") + env = {**os.environ, "PYTHONIOENCODING": encoding, "PYTHONUTF8": "0"} + result = subprocess.run([sys.executable, "-X", "utf8=0", str(PROGRESS), "--compare", *map(str, paths)], + env=env, capture_output=True, timeout=10) + self.assertEqual(result.returncode, 0, result.stderr.decode("utf-8", errors="replace")) + return result.stdout.decode("utf-8").replace("\r\n", "\n") + + def test_implemented_report_with_cp1252_stdout(self): + report = self.compare(snapshot(False), snapshot(True)) + self.assertIn("\U0001f4c8 **System libraries**: 100%", report) + self.assertIn("\u2705 1 implemented", report) + self.assertIn("| libSceExample | `Ready` |", report) + + def test_regression_report_with_cp1252_stdout(self): + report = self.compare(snapshot(True), snapshot(False)) + self.assertIn("\U0001f4c9 **System libraries**: 0%", report) + self.assertIn("\u26a0\ufe0f 1 went back to stubs", report) + self.assertIn("-1 reverted", report) + + def test_utf8_snapshot_names_are_preserved(self): + library = "libSce\u6e2c\u8a66\U0001f3ae" + report = self.compare(snapshot(False, library), snapshot(True, library)) + self.assertIn(f"| {library} | `Ready` |", report) + + def test_unchanged_snapshots_produce_no_output(self): + self.assertEqual(self.compare(snapshot(False), snapshot(False)), "") + + def test_utf8_stdout_report_is_unchanged(self): + base, head = snapshot(False), snapshot(True) + self.assertEqual(self.compare(base, head, "utf-8"), self.compare(base, head)) + + +if __name__ == "__main__": + unittest.main() diff --git a/tools/tests/test_test_registration.py b/tools/tests/test_test_registration.py new file mode 100644 index 000000000..4010ca6bd --- /dev/null +++ b/tools/tests/test_test_registration.py @@ -0,0 +1,37 @@ +from pathlib import Path +import sys +import tempfile +import unittest + +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) +import check_test_registration + + +class RegistrationTests(unittest.TestCase): + def registered(self, content): + with tempfile.TemporaryDirectory() as directory: + build = Path(directory) + (build / "CTestTestfile.cmake").write_text(content, encoding="utf-8") + return check_test_registration.registered(build) + + def test_test_name_does_not_register_executable(self): + names = self.registered('add_test("missing_tests" "/usr/bin/python3" "unrelated.py")\n') + self.assertNotIn("missing_tests", names) + + def test_direct_executable(self): + names = self.registered('add_test("bounds" "/build/tests/buffer_bounds_tests")\n') + self.assertIn("buffer_bounds_tests", names) + self.assertNotIn("bounds", names) + + def test_wrapped_executable(self): + names = self.registered('add_test("bounds" "/usr/bin/python3" "runner.py" "/build/tests/buffer_bounds_tests")\n') + self.assertIn("buffer_bounds_tests", names) + + def test_windows_executable_and_test_name(self): + names = self.registered('add_test("missing_tests.exe" "C:/Program Files/Python/python.exe" "runner.py" "C:/build/tests/buffer_bounds_tests.exe")\n') + self.assertIn("buffer_bounds_tests", names) + self.assertNotIn("missing_tests", names) + + +if __name__ == "__main__": + unittest.main()