diff --git a/.agents/skills/reverse-engineer-anything/references/connection-and-recovery.md b/.agents/skills/reverse-engineer-anything/references/connection-and-recovery.md index 35d0042c7..4d6d08bfc 100644 --- a/.agents/skills/reverse-engineer-anything/references/connection-and-recovery.md +++ b/.agents/skills/reverse-engineer-anything/references/connection-and-recovery.md @@ -73,7 +73,7 @@ rea setup --client codex --dry-run --json Use the current supported client's ID: `claude_code`, `claude_desktop`, `codex`, `cursor`, `gemini_cli`, `windsurf`, `devin`, `opencode`, `antigravity`, `copilot_cli`, `commandcode`, `qwen_code`, `vscode`, `trae`, `grok_build`, `omp`, `pi`, -`hermes`, `qoder`, `zcode`, `dsh`, or `grok_bot`. If unknown, inspect `doctor --json` before choosing a +`hermes`, `qoder`, `droid`, `zcode`, `dsh`, or `grok_bot`. If unknown, inspect `doctor --json` before choosing a scope. `grok_bot` uses a chat registration step rather than local `mcp.json`. For unsupported clients, use version-pinned manual stdio registration or CLI. diff --git a/.agents/skills/reverse-engineer-anything/references/native-and-artifacts.md b/.agents/skills/reverse-engineer-anything/references/native-and-artifacts.md index 6a942c73c..a4fc6d18f 100644 --- a/.agents/skills/reverse-engineer-anything/references/native-and-artifacts.md +++ b/.agents/skills/reverse-engineer-anything/references/native-and-artifacts.md @@ -224,10 +224,13 @@ context. PSP/stack/device state remains unmodeled. See [DOS guide](https://githu For other headerless images (embedded firmware without ELF/Mach-O/PE/MZ headers), open with `format: "raw-binary"`, `architecture` (`x86`, `x86_64`, -`arm`, or `arm64`), and `entry_point` (`0x` hex). File offset 0 is mapped at +`arm`, `arm64`, or `6502`), and `entry_point` (`0x` hex). File offset 0 is mapped at that address. Hopper and IDA refuse this interpretation; Ghidra uses `BinaryLoader` with a little-endian generic language. Big-endian, Thumb-only, -and Cortex-M variants are unsupported. See [raw binary guide](https://github.com/morluto/rea/blob/main/docs/raw-binary.md). +and Cortex-M variants are unsupported. `6502` selects NMOS 6502; the entry and +complete file mapping must fit within `0x0000..0xffff`. 65C02, 65816, cartridge +containers, bank switching, and mapper semantics are unsupported. Windows +Ghidra P0 admits raw images only for `6502`. See [raw binary guide](https://github.com/morluto/rea/blob/main/docs/raw-binary.md). ### Function annotations in Ghidra diff --git a/.github/actions/portable-build-cache/action.yml b/.github/actions/portable-build-cache/action.yml index cee775ec5..799b9476b 100644 --- a/.github/actions/portable-build-cache/action.yml +++ b/.github/actions/portable-build-cache/action.yml @@ -12,7 +12,7 @@ runs: steps: - name: Download portable build outputs from this workflow run if: inputs.build_cache_artifact != '' - uses: actions/download-artifact@v4 + uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0 with: name: ${{ inputs.build_cache_artifact }} path: .turbo/cache/ @@ -39,14 +39,14 @@ runs: ' >> "$GITHUB_OUTPUT" - name: Restore portable build outputs if: inputs.build_cache_artifact == '' && inputs.save != 'true' - uses: actions/cache/restore@v4 + uses: actions/cache/restore@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.3.0 with: path: .turbo/cache/ key: ${{ steps.plan.outputs.key }} restore-keys: ${{ steps.plan.outputs.prefix }} - name: Restore and save portable build outputs if: inputs.build_cache_artifact == '' && inputs.save == 'true' - uses: actions/cache@v4 + uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4.3.0 with: path: .turbo/cache/ key: ${{ steps.plan.outputs.key }} diff --git a/.github/workflows/ci-timings.yml b/.github/workflows/ci-timings.yml index 30fa19464..73a9ba4b9 100644 --- a/.github/workflows/ci-timings.yml +++ b/.github/workflows/ci-timings.yml @@ -20,11 +20,11 @@ jobs: timeout-minutes: 5 steps: # Consume only API data and execute batch code from the trusted default branch. - - uses: actions/checkout@v4 + - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0 with: ref: ${{ github.event.repository.default_branch }} persist-credentials: false - - uses: actions/setup-node@v4 + - uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4.4.0 with: node-version-file: .nvmrc - name: Record queue, execution, and matched step timing @@ -36,7 +36,7 @@ jobs: - name: Upload CI timing artifacts if: always() && hashFiles('.cache/ci-job-timings/*') != '' continue-on-error: true - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 with: name: ci-job-timings-batch path: .cache/ci-job-timings/ diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index e0516da85..142e179bb 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -63,11 +63,11 @@ jobs: matrix: ${{ steps.plan.outputs.matrix }} test_selection: ${{ steps.plan.outputs.test_selection }} steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0 with: ref: ${{ inputs.source_sha || github.sha }} fetch-depth: 0 - - uses: actions/setup-node@v4 + - uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4.4.0 with: node-version-file: .nvmrc - name: Bind qualification to the caller revision @@ -118,8 +118,8 @@ jobs: env: CHANGES_RESULT: ${{ needs.changes.result }} run: test "${CHANGES_RESULT}" = "success" - - uses: actions/checkout@v4 - - uses: actions/setup-node@v4 + - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0 + - uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4.4.0 with: node-version-file: .nvmrc cache: npm @@ -138,7 +138,7 @@ jobs: run: npm run format:check - name: Upload complete static diagnostics if: always() && needs.changes.outputs.code == 'true' - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 with: name: static-check-reports path: .cache/static-checks/ @@ -157,13 +157,13 @@ jobs: matrix: shard: [1/6, 2/6, 3/6, 4/6, 5/6, 6/6] steps: - - uses: actions/checkout@v4 - - uses: actions/setup-node@v4 + - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0 + - uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4.4.0 with: node-version-file: .nvmrc cache: npm - run: npm ci - - uses: actions/download-artifact@v4 + - uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0 with: name: rea-test-runtime path: . @@ -177,7 +177,7 @@ jobs: run: npm run test:run -- --reporter=default --reporter=blob --shard=${{ matrix.shard }} env: REA_CI_TEST_HISTORY: .cache/ci-test-history/test-results.json - - uses: actions/upload-artifact@v4 + - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 if: ${{ !cancelled() }} with: name: vitest-report-${{ strategy.job-index }} @@ -208,8 +208,8 @@ jobs: contents: read actions: read steps: - - uses: actions/checkout@v4 - - uses: actions/setup-node@v4 + - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0 + - uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4.4.0 with: node-version-file: .nvmrc cache: npm @@ -231,13 +231,13 @@ jobs: - name: Download previous timing report if: steps.history.outputs.run-id != '' continue-on-error: true - uses: actions/download-artifact@v4 + uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0 with: name: test-timings run-id: ${{ steps.history.outputs.run-id }} github-token: ${{ github.token }} path: .cache/ci-test-history - - uses: actions/upload-artifact@v4 + - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 with: name: rea-test-runtime path: | @@ -254,7 +254,7 @@ jobs: # test:prepare caches only portable JS, skill, and catalog outputs. # Native package jobs still run their build and prepack checks; Turbo # restores matching outputs or rebuilds when their inputs differ. - - uses: actions/upload-artifact@v4 + - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 with: name: rea-build-cache path: .turbo/cache/ @@ -280,16 +280,16 @@ jobs: env: CHANGES_RESULT: ${{ needs.changes.result }} run: test "${CHANGES_RESULT}" = "success" - - uses: actions/checkout@v4 + - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0 if: needs.changes.outputs.code == 'true' && success() - - uses: actions/setup-node@v4 + - uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4.4.0 if: needs.changes.outputs.code == 'true' && success() with: node-version-file: .nvmrc cache: npm - if: needs.changes.outputs.code == 'true' && success() run: npm ci - - uses: actions/download-artifact@v4 + - uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0 if: needs.changes.outputs.code == 'true' && success() with: path: .vitest/blob @@ -330,7 +330,7 @@ jobs: CI_TEST_SELECTION: ${{ needs.changes.outputs.test_selection }} CI_PR_HEAD_SHA: ${{ github.event.pull_request.head.sha }} run: node scripts/ci/test-report.mjs - - uses: actions/upload-artifact@v4 + - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 if: needs.changes.outputs.code == 'true' && !cancelled() && hashFiles('test-results.json') != '' with: name: test-results @@ -345,7 +345,7 @@ jobs: retention-days: 30 - name: Publish successful timing history if: needs.changes.outputs.code == 'true' && needs.changes.outputs.tier == 'full' && success() && needs.test-shard.result == 'success' - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 with: name: test-timings path: .cache/ci-test-history/test-results.json @@ -366,7 +366,7 @@ jobs: runs-on: ubuntu-latest timeout-minutes: 10 steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0 - name: Verify fixtures from a clean checkout run: npm run verify:fixtures @@ -377,8 +377,8 @@ jobs: runs-on: ubuntu-latest timeout-minutes: 10 steps: - - uses: actions/checkout@v4 - - uses: actions/setup-node@v4 + - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0 + - uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4.4.0 with: node-version-file: .nvmrc - name: Prepare cross compiler @@ -395,12 +395,12 @@ jobs: - run: npm run build:windows-native - run: npm run verify:windows-native:artifact - run: node scripts/build-windows-native.mjs --inspection-fixture - - uses: actions/upload-artifact@v4 + - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 with: name: windows-native-inspection-fixture path: .cache/windows-native-inspection-fixture/ if-no-files-found: error - - uses: actions/upload-artifact@v4 + - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 with: name: windows-native-controls path: native/windows/build/ @@ -423,8 +423,8 @@ jobs: NODE_OPTIONS: "--max-old-space-size=1024" UV_THREADPOOL_SIZE: "2" steps: - - uses: actions/checkout@v4 - - uses: actions/setup-node@v4 + - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0 + - uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4.4.0 with: node-version-file: .nvmrc cache: npm @@ -438,11 +438,11 @@ jobs: assert.equal(process.arch, process.env.EXPECTED_CI_ARCHITECTURE); console.log(JSON.stringify({ platform: process.platform, architecture: process.arch, node: process.version }));" - run: npm ci - - uses: actions/download-artifact@v4 + - uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0 with: name: rea-build-cache path: .turbo/cache/ - - uses: actions/download-artifact@v4 + - uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0 with: name: windows-native-controls path: native/windows/build/ @@ -463,7 +463,7 @@ jobs: run: npm run verify:inspector - name: Retain the exact qualified release package if: needs.changes.outputs.tier == 'full' && matrix.platform == 'linux' && matrix.architecture == 'x64' - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 with: name: verified-release-package path: .cache/verified-package/ @@ -479,13 +479,13 @@ jobs: runs-on: macos-15 timeout-minutes: 15 steps: - - uses: actions/checkout@v4 - - uses: actions/setup-node@v4 + - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0 + - uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4.4.0 with: node-version-file: .nvmrc cache: npm - run: npm ci - - uses: actions/download-artifact@v4 + - uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0 with: name: rea-build-cache path: .turbo/cache/ @@ -527,8 +527,8 @@ jobs: runs-on: ubuntu-latest timeout-minutes: 15 steps: - - uses: actions/checkout@v4 - - uses: actions/setup-node@v4 + - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0 + - uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4.4.0 with: node-version-file: .nvmrc cache: npm @@ -539,7 +539,7 @@ jobs: run: | set -o pipefail npm run verify:readiness | tee readiness-summary.json - - uses: actions/upload-artifact@v4 + - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 if: ${{ !cancelled() }} with: name: reconstruction-readiness @@ -556,18 +556,18 @@ jobs: runs-on: windows-latest timeout-minutes: 25 steps: - - uses: actions/checkout@v4 - - uses: actions/setup-node@v4 + - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0 + - uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4.4.0 with: node-version-file: .nvmrc cache: npm - run: npm ci - - uses: actions/download-artifact@v4 + - uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0 with: name: rea-build-cache path: .turbo/cache/ - run: npm run build:cached - - uses: actions/download-artifact@v4 + - uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0 with: name: windows-native-controls path: native/windows/build/ @@ -597,6 +597,7 @@ jobs: tests/boundary/filesystem/setupUnavailableProvider.test.ts tests/boundary/filesystem/windowsNpxRegistration.test.ts tests/acceptance/setup/homeDirectoryFallback.test.ts + tests/boundary/filesystem/javascriptApplicationWorkflows.test.ts tests/boundary/filesystem/webScriptManifestPaths.test.ts tests/boundary/process/ProcessEvidence.host.test.ts tests/acceptance/webModuleCli.test.ts @@ -622,7 +623,7 @@ jobs: tests/process-global/updateMaintenance.test.ts tests/conformance/providers/ghidra/realGhidraWindowsWorkflow.test.ts - run: npm run verify:windows-native - - uses: actions/download-artifact@v4 + - uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0 with: name: windows-native-inspection-fixture path: .cache/windows-native-inspection-fixture/ @@ -641,7 +642,7 @@ jobs: tests/boundary/filesystem/directoryArtifactReaderOpen.test.ts tests/boundary/filesystem/javascriptPackagePrecedence.test.ts tests/boundary/filesystem/javascriptArtifactContentIdentity.test.ts - - uses: actions/setup-node@v4 + - uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4.4.0 with: node-version: 22.19.0 - name: Verify native path boundaries on Node 22 @@ -651,7 +652,7 @@ jobs: tests/boundary/filesystem/javascriptPackagePrecedence.test.ts tests/boundary/filesystem/javascriptArtifactContentIdentity.test.ts tests/boundary/process/sourceMapUncHosts.test.ts - - uses: actions/upload-artifact@v4 + - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 if: always() with: name: windows-capabilities @@ -665,8 +666,8 @@ jobs: runs-on: ubuntu-latest timeout-minutes: 10 steps: - - uses: actions/checkout@v4 - - uses: actions/setup-node@v4 + - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0 + - uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4.4.0 with: node-version-file: .nvmrc cache: npm @@ -674,7 +675,7 @@ jobs: - run: npm run docs:check - name: Generation leaves authored files unchanged run: git diff --exit-code - - uses: actions/upload-artifact@v4 + - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 with: name: generated-docs path: | @@ -781,8 +782,8 @@ jobs: runs-on: ubuntu-latest timeout-minutes: 5 steps: - - uses: actions/checkout@v4 - - uses: actions/setup-node@v4 + - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0 + - uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4.4.0 with: node-version-file: .nvmrc - name: Require every selected lane to pass @@ -793,7 +794,7 @@ jobs: run: node scripts/ci/required.mjs - name: Upload full qualification receipt artifact if: needs.changes.outputs.tier == 'full' && success() - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 with: name: ${{ steps.gate.outputs.receipt_name }} path: ${{ steps.gate.outputs.receipt_path }} diff --git a/.github/workflows/nativeaot-fixtures.yml b/.github/workflows/nativeaot-fixtures.yml index ce7dc1ea1..b2cc5801f 100644 --- a/.github/workflows/nativeaot-fixtures.yml +++ b/.github/workflows/nativeaot-fixtures.yml @@ -19,16 +19,16 @@ jobs: DOTNET_PROCESSOR_COUNT: "1" COMPlus_GCHeapHardLimit: "30000000" steps: - - uses: actions/checkout@v4 - - uses: actions/setup-node@v4 + - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0 + - uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4.4.0 with: node-version-file: .nvmrc - - uses: actions/setup-dotnet@v4 + - uses: actions/setup-dotnet@67a3573c9a986a3f9c594539f4ab511d57bb3ce9 # v4.3.1 with: dotnet-version: "8.0.416" - name: Build benign fixture and independent byte oracle run: node scripts/generate-nativeaot-fixtures.mjs - - uses: actions/upload-artifact@v4 + - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 with: name: nativeaot-win-x64-${{ github.event.pull_request.head.sha || github.sha }} path: | diff --git a/.github/workflows/pages.yml b/.github/workflows/pages.yml index e276a9f0d..6b8a05b46 100644 --- a/.github/workflows/pages.yml +++ b/.github/workflows/pages.yml @@ -19,8 +19,8 @@ jobs: runs-on: ubuntu-latest timeout-minutes: 10 steps: - - uses: actions/checkout@v4 - - uses: actions/setup-node@v4 + - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0 + - uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4.4.0 with: node-version-file: .nvmrc cache: npm diff --git a/.github/workflows/real-binary-layout.yml b/.github/workflows/real-binary-layout.yml index 5cec952ce..0218543f0 100644 --- a/.github/workflows/real-binary-layout.yml +++ b/.github/workflows/real-binary-layout.yml @@ -28,8 +28,8 @@ jobs: runs-on: ubuntu-24.04 timeout-minutes: 15 steps: - - uses: actions/checkout@v4 - - uses: actions/setup-node@v4 + - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0 + - uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4.4.0 with: node-version-file: .nvmrc cache: npm diff --git a/.github/workflows/real-browser.yml b/.github/workflows/real-browser.yml index 912fdafb1..3c3fe955a 100644 --- a/.github/workflows/real-browser.yml +++ b/.github/workflows/real-browser.yml @@ -21,8 +21,8 @@ jobs: runs-on: ubuntu-latest timeout-minutes: 10 steps: - - uses: actions/checkout@v4 - - uses: actions/setup-node@v4 + - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0 + - uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4.4.0 with: node-version-file: .nvmrc cache: npm diff --git a/.github/workflows/real-evm-interface.yml b/.github/workflows/real-evm-interface.yml index dfdd3970f..6e510c973 100644 --- a/.github/workflows/real-evm-interface.yml +++ b/.github/workflows/real-evm-interface.yml @@ -22,8 +22,8 @@ jobs: runs-on: ubuntu-24.04 timeout-minutes: 15 steps: - - uses: actions/checkout@v4 - - uses: actions/setup-node@v4 + - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0 + - uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4.4.0 with: node-version-file: .nvmrc cache: npm diff --git a/.github/workflows/real-ghidra-windows.yml b/.github/workflows/real-ghidra-windows.yml index d6de7469f..db92cd0f8 100644 --- a/.github/workflows/real-ghidra-windows.yml +++ b/.github/workflows/real-ghidra-windows.yml @@ -16,8 +16,8 @@ jobs: if: github.ref == 'refs/heads/main' runs-on: ubuntu-latest steps: - - uses: actions/checkout@v4 - - uses: actions/setup-node@v4 + - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0 + - uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4.4.0 with: node-version-file: .nvmrc - name: Prepare cross compiler @@ -31,7 +31,7 @@ jobs: sudo apt-get install --no-install-recommends -y g++-mingw-w64-x86-64 - run: npm run build:windows-native - run: npm run verify:windows-native:artifact - - uses: actions/upload-artifact@v4 + - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 with: name: real-ghidra-windows-native path: native/windows/build/ @@ -45,12 +45,12 @@ jobs: runs-on: [self-hosted, Windows, x64, ghidra-12-1-2] timeout-minutes: 20 steps: - - uses: actions/checkout@v4 - - uses: actions/setup-node@v4 + - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0 + - uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4.4.0 with: node-version-file: .nvmrc - run: npm ci - - uses: actions/download-artifact@v4 + - uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0 with: name: real-ghidra-windows-native path: native/windows/build/ @@ -88,7 +88,7 @@ jobs: env: REA_ANALYSIS_PROVIDER: ghidra run: npm run verify:ghidra:windows:package -- --x86 --dll | Tee-Object -FilePath windows-ghidra-x86-dll-package-proof.log - - uses: actions/upload-artifact@v4 + - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 if: always() with: name: windows-ghidra-proof diff --git a/.github/workflows/real-go-binary.yml b/.github/workflows/real-go-binary.yml index b886bd70a..c7d74b001 100644 --- a/.github/workflows/real-go-binary.yml +++ b/.github/workflows/real-go-binary.yml @@ -44,12 +44,12 @@ jobs: runs-on: ubuntu-24.04 timeout-minutes: 20 steps: - - uses: actions/checkout@v4 - - uses: actions/setup-node@v4 + - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0 + - uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4.4.0 with: node-version-file: .nvmrc cache: npm - - uses: actions/setup-go@v5 + - uses: actions/setup-go@40f1582b2485089dde7abd97c1529aa768e1baff # v5.6.0 with: go-version: "1.26.1" cache: false diff --git a/.github/workflows/real-hopper-linux.yml b/.github/workflows/real-hopper-linux.yml index 2355229de..d5e5f8c33 100644 --- a/.github/workflows/real-hopper-linux.yml +++ b/.github/workflows/real-hopper-linux.yml @@ -15,8 +15,8 @@ jobs: runs-on: ubuntu-24.04 timeout-minutes: 25 steps: - - uses: actions/checkout@v4 - - uses: actions/setup-node@v4 + - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0 + - uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4.4.0 with: node-version-file: .nvmrc cache: npm @@ -36,8 +36,8 @@ jobs: runs-on: [self-hosted, Linux, hopper] timeout-minutes: 20 steps: - - uses: actions/checkout@v4 - - uses: actions/setup-node@v4 + - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0 + - uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4.4.0 with: node-version-file: .nvmrc cache: npm diff --git a/.github/workflows/real-hopper.yml b/.github/workflows/real-hopper.yml index 51135f4b6..b3ceeb7d9 100644 --- a/.github/workflows/real-hopper.yml +++ b/.github/workflows/real-hopper.yml @@ -15,8 +15,8 @@ jobs: runs-on: [self-hosted, macOS, hopper] timeout-minutes: 20 steps: - - uses: actions/checkout@v4 - - uses: actions/setup-node@v4 + - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0 + - uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4.4.0 with: node-version-file: .nvmrc cache: npm diff --git a/.github/workflows/real-javascript-recovery.yml b/.github/workflows/real-javascript-recovery.yml index 817dc4dc0..a5043b62e 100644 --- a/.github/workflows/real-javascript-recovery.yml +++ b/.github/workflows/real-javascript-recovery.yml @@ -23,8 +23,8 @@ jobs: runs-on: ubuntu-latest timeout-minutes: 10 steps: - - uses: actions/checkout@v4 - - uses: actions/setup-node@v4 + - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0 + - uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4.4.0 with: node-version-file: .nvmrc cache: npm diff --git a/.github/workflows/real-packet-capture.yml b/.github/workflows/real-packet-capture.yml index b5e9f9201..8daebfd8a 100644 --- a/.github/workflows/real-packet-capture.yml +++ b/.github/workflows/real-packet-capture.yml @@ -43,8 +43,8 @@ jobs: runs-on: ubuntu-24.04 timeout-minutes: 15 steps: - - uses: actions/checkout@v4 - - uses: actions/setup-node@v4 + - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0 + - uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4.4.0 with: node-version-file: .nvmrc cache: npm diff --git a/.github/workflows/real-recorded-crash.yml b/.github/workflows/real-recorded-crash.yml index 04467a608..0463df27d 100644 --- a/.github/workflows/real-recorded-crash.yml +++ b/.github/workflows/real-recorded-crash.yml @@ -25,8 +25,8 @@ jobs: runs-on: ubuntu-24.04 timeout-minutes: 15 steps: - - uses: actions/checkout@v4 - - uses: actions/setup-node@v4 + - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0 + - uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4.4.0 with: node-version-file: .nvmrc cache: npm diff --git a/.github/workflows/real-signal-recording.yml b/.github/workflows/real-signal-recording.yml index cc902037e..5c1380e2d 100644 --- a/.github/workflows/real-signal-recording.yml +++ b/.github/workflows/real-signal-recording.yml @@ -36,8 +36,8 @@ jobs: runs-on: ubuntu-24.04 timeout-minutes: 15 steps: - - uses: actions/checkout@v4 - - uses: actions/setup-node@v4 + - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0 + - uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4.4.0 with: node-version-file: .nvmrc cache: npm diff --git a/.github/workflows/real-termux.yml b/.github/workflows/real-termux.yml index 204acc1c6..e6f3890e1 100644 --- a/.github/workflows/real-termux.yml +++ b/.github/workflows/real-termux.yml @@ -18,7 +18,7 @@ jobs: runs-on: ubuntu-latest timeout-minutes: 45 steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0 - name: Download the pinned Termux app and verify its release checksum shell: bash run: | @@ -38,7 +38,7 @@ jobs: sudo udevadm control --reload-rules sudo udevadm trigger --name-match=kvm - name: Build and capture a page inside Termux - uses: reactivecircus/android-emulator-runner@4c44018e59b437e86cdfc41da381398f93ed8808 # v2 + uses: reactivecircus/android-emulator-runner@a421e43855164a8197daf9d8d40fe71c6996bb0d # v2.38.0 with: api-level: 30 arch: x86_64 @@ -50,7 +50,7 @@ jobs: script: bash scripts/verify/termux/emulator.sh - name: Upload Termux and Android diagnostics if: always() - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 with: name: termux-browser-diagnostics path: ${{ runner.temp }}/rea-termux/diagnostics diff --git a/.github/workflows/real-web-network-captures.yml b/.github/workflows/real-web-network-captures.yml index d41bb37a5..3ee59d2c8 100644 --- a/.github/workflows/real-web-network-captures.yml +++ b/.github/workflows/real-web-network-captures.yml @@ -22,8 +22,8 @@ jobs: runs-on: ubuntu-24.04 timeout-minutes: 15 steps: - - uses: actions/checkout@v4 - - uses: actions/setup-node@v4 + - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0 + - uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4.4.0 with: node-version-file: .nvmrc cache: npm diff --git a/.github/workflows/real-web-runtime.yml b/.github/workflows/real-web-runtime.yml index 2575c55f9..f5b9f201a 100644 --- a/.github/workflows/real-web-runtime.yml +++ b/.github/workflows/real-web-runtime.yml @@ -17,8 +17,8 @@ jobs: runs-on: ubuntu-latest timeout-minutes: 10 steps: - - uses: actions/checkout@v4 - - uses: actions/setup-node@v4 + - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0 + - uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4.4.0 with: node-version-file: .nvmrc cache: npm diff --git a/.github/workflows/real-web-source-map.yml b/.github/workflows/real-web-source-map.yml index 8e0869eb7..2a1e65dc0 100644 --- a/.github/workflows/real-web-source-map.yml +++ b/.github/workflows/real-web-source-map.yml @@ -17,8 +17,8 @@ jobs: runs-on: ubuntu-latest timeout-minutes: 10 steps: - - uses: actions/checkout@v4 - - uses: actions/setup-node@v4 + - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0 + - uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4.4.0 with: node-version-file: .nvmrc cache: npm diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 7cc798cfa..61e39d029 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -48,7 +48,7 @@ jobs: echo "frozen=true" >> "$GITHUB_OUTPUT" fi - - uses: googleapis/release-please-action@v4 + - uses: googleapis/release-please-action@5c625bfb5d1ff62eadeeb3772007f7f66fdcf071 # v4.4.1 if: steps.editorial.outputs.frozen != 'true' with: token: ${{ secrets.RELEASE_PLEASE_TOKEN || secrets.GITHUB_TOKEN }} @@ -70,13 +70,13 @@ jobs: sha: ${{ steps.release.outputs.sha }} steps: - name: Check out reviewed release merge - uses: actions/checkout@v6 + uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6.1.0 with: ref: ${{ github.event.pull_request.merge_commit_sha }} fetch-depth: 0 persist-credentials: false - - uses: actions/setup-node@v6 + - uses: actions/setup-node@249970729cb0ef3589644e2896645e5dc5ba9c38 # v6.5.0 with: node-version-file: .nvmrc package-manager-cache: false @@ -104,7 +104,7 @@ jobs: node scripts/release-notes.mjs publication --repository "$GH_REPO" \ --source "$SOURCE_SHA" --pr "$RELEASE_PR" - - uses: googleapis/release-please-action@v4 + - uses: googleapis/release-please-action@5c625bfb5d1ff62eadeeb3772007f7f66fdcf071 # v4.4.1 id: release with: token: ${{ secrets.RELEASE_PLEASE_TOKEN || secrets.GITHUB_TOKEN }} @@ -191,14 +191,14 @@ jobs: fi - name: Check out release controller and Git history - uses: actions/checkout@v6 + uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6.1.0 with: ref: ${{ github.sha }} fetch-depth: 0 persist-credentials: false - name: Set up Node.js for checkpoint validation - uses: actions/setup-node@v6 + uses: actions/setup-node@249970729cb0ef3589644e2896645e5dc5ba9c38 # v6.5.0 with: node-version-file: .nvmrc package-manager-cache: false @@ -226,7 +226,7 @@ jobs: node scripts/release-notes.mjs publication --repository "$GH_REPO" \ --source "$SOURCE_SHA" - - uses: googleapis/release-please-action@v4 + - uses: googleapis/release-please-action@5c625bfb5d1ff62eadeeb3772007f7f66fdcf071 # v4.4.1 id: release with: # A dedicated credential starts PR CI automatically; the built-in @@ -255,7 +255,7 @@ jobs: - name: Check out release pull request if: inputs.phase == 'prepare' && steps.release.outputs.prs_created == 'true' - uses: actions/checkout@v6 + uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6.1.0 with: ref: ${{ fromJSON(steps.release.outputs.pr).headBranchName }} fetch-depth: 0 @@ -263,7 +263,7 @@ jobs: - name: Set up Node.js for generated documentation if: inputs.phase == 'prepare' && steps.release.outputs.prs_created == 'true' - uses: actions/setup-node@v6 + uses: actions/setup-node@249970729cb0ef3589644e2896645e5dc5ba9c38 # v6.5.0 with: node-version-file: .nvmrc package-manager-cache: false @@ -314,12 +314,12 @@ jobs: contents: read id-token: write steps: - - uses: actions/checkout@v6 + - uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6.1.0 with: ref: ${{ needs.merged-release.outputs.sha || needs.release-please.outputs.sha }} persist-credentials: false - - uses: actions/setup-node@v6 + - uses: actions/setup-node@249970729cb0ef3589644e2896645e5dc5ba9c38 # v6.5.0 with: node-version-file: .nvmrc package-manager-cache: false @@ -328,7 +328,7 @@ jobs: - run: npm ci - name: Download the exact qualified package - uses: actions/download-artifact@v4 + uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0 with: name: verified-release-package path: .cache/verified-package/ @@ -392,7 +392,7 @@ jobs: contents: read id-token: write steps: - - uses: actions/checkout@v6 + - uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6.1.0 with: ref: ${{ needs.merged-release.outputs.sha || needs.release-please.outputs.sha }} persist-credentials: false diff --git a/.github/workflows/sqlite-runtime.yml b/.github/workflows/sqlite-runtime.yml index 6e5bbcd10..fc379d1dc 100644 --- a/.github/workflows/sqlite-runtime.yml +++ b/.github/workflows/sqlite-runtime.yml @@ -22,14 +22,14 @@ jobs: # The repository-pinned Node 24 version also runs the complete test suite. node: ["22.x", "24.15.0", "24.x", "26.x"] steps: - - uses: actions/checkout@v4 - - uses: actions/setup-node@v4 + - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0 + - uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4.4.0 with: node-version: ${{ matrix.node }} check-latest: true cache: npm - run: npm ci - - uses: actions/download-artifact@v4 + - uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0 with: name: rea-test-runtime path: . diff --git a/.github/workflows/website-check.yml b/.github/workflows/website-check.yml index 2a2c5406f..dc17b4b96 100644 --- a/.github/workflows/website-check.yml +++ b/.github/workflows/website-check.yml @@ -14,9 +14,9 @@ jobs: timeout-minutes: 10 steps: - name: Check out the repository - uses: actions/checkout@v4 + uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0 - name: Set up Python for website assets - uses: actions/setup-python@v5 + uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0 with: python-version: "3.12" cache: pip diff --git a/.github/workflows/website-pages.yml b/.github/workflows/website-pages.yml index 3c2251658..491b028a6 100644 --- a/.github/workflows/website-pages.yml +++ b/.github/workflows/website-pages.yml @@ -47,9 +47,9 @@ jobs: exit 1 fi - name: Check out the repository - uses: actions/checkout@v4 + uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0 - name: Set up Python for website assets - uses: actions/setup-python@v5 + uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0 with: python-version: "3.12" cache: pip @@ -74,9 +74,9 @@ jobs: ) PY - name: Configure Pages - uses: actions/configure-pages@v5 + uses: actions/configure-pages@983d7736d9b0ae728b81ab479565c72886d7745b # v5.0.0 - name: Upload the website - uses: actions/upload-pages-artifact@v4 + uses: actions/upload-pages-artifact@7b1f4a764d45c48632c6b24a0339c27f5614fb0b # v4.0.0 with: path: website/_site @@ -95,15 +95,15 @@ jobs: steps: - name: Check out publishing configuration if: inputs.target == 'both' - uses: actions/checkout@v4 + uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0 - name: Set up Node.js for Wrangler if: inputs.target == 'both' - uses: actions/setup-node@v4 + uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4.4.0 with: node-version-file: .nvmrc - name: Download the verified website if: inputs.target == 'both' - uses: actions/download-artifact@v4 + uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0 with: name: github-pages path: ${{ runner.temp }}/website-archive @@ -114,7 +114,7 @@ jobs: tar -xf "$RUNNER_TEMP/website-archive/artifact.tar" -C "$RUNNER_TEMP/rea-website-public" - name: Publish the website to Cloudflare if: inputs.target == 'both' - uses: cloudflare/wrangler-action@v4 + uses: cloudflare/wrangler-action@953926a2e2182532811c01a25e53647d93bf07c0 # v4.1.3 with: apiToken: ${{ secrets.CLOUDFLARE_API_TOKEN }} accountId: ${{ secrets.CLOUDFLARE_ACCOUNT_ID }} @@ -123,7 +123,7 @@ jobs: command: deploy --cwd website --env production --assets "${{ runner.temp }}/rea-website-public" - name: Publish to GitHub Pages id: deployment - uses: actions/deploy-pages@v4 + uses: actions/deploy-pages@d6db90164ac5ed86f2b6aed7e0febac5b3c0c03e # v4.0.5 - name: Verify the selected published revisions env: PAGES_URL: ${{ steps.deployment.outputs.page_url }} diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index a8d5f3635..50674afa7 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -142,6 +142,21 @@ caches deterministic builds and static checks across Git worktrees. After a pack Repository invariants live in [AGENTS.md](AGENTS.md). Keep provider-specific protocols in adapters and shared CLI/MCP workflows in the application layer. +### Windows source checkouts + +The published npm package includes the Windows x64 native addon and its +manifest. A source checkout does not: `native/windows/build/` is ignored by +Git, and neither `npm ci` nor `npm run build:cached` (or `npm run build`) +produces `rea-windows-x64.node` or `manifest.json` in that directory. +Windows workflows and tests that require native controls need these artifacts; +without them, those controls are unavailable even after a successful runtime build. + +Follow the [Windows native build guide](native/windows/README.md#build) before +running those workflows from source. The artifact build requires Linux, +MinGW-w64, and `tar`; the build script does not run on Windows. See that guide +for the build and artifact-verification commands and the separate real Windows +verification prerequisites. Keep the generated addon and manifest uncommitted. + ## Documentation website The VitePress site uses the Markdown files in `docs/`. Run `npm run docs:dev` diff --git a/bridge/ghidra/ReaGhidraBridge.java b/bridge/ghidra/ReaGhidraBridge.java index 02fc0f148..c180c5866 100644 --- a/bridge/ghidra/ReaGhidraBridge.java +++ b/bridge/ghidra/ReaGhidraBridge.java @@ -13,6 +13,8 @@ import java.net.UnixDomainSocketAddress; import java.nio.channels.Channels; import java.nio.channels.ServerSocketChannel; import java.nio.channels.SocketChannel; +import java.nio.ByteBuffer; +import java.nio.charset.CharacterCodingException; import java.nio.charset.StandardCharsets; import java.nio.file.AtomicMoveNotSupportedException; import java.nio.file.Files; @@ -22,7 +24,9 @@ import java.nio.file.StandardCopyOption; import java.nio.file.StandardOpenOption; import java.nio.file.attribute.PosixFilePermissions; import java.security.MessageDigest; +import java.util.ArrayDeque; import java.util.ArrayList; +import java.util.Arrays; import java.util.Comparator; import java.util.HashSet; import java.util.IdentityHashMap; @@ -82,6 +86,7 @@ import ghidra.program.model.address.SegmentedAddress; import ghidra.program.model.reloc.Relocation; import java.io.ByteArrayOutputStream; import ghidra.program.model.symbol.Namespace; +import ghidra.program.model.symbol.FlowType; import ghidra.program.model.symbol.RefType; import ghidra.program.model.symbol.Reference; import ghidra.program.model.symbol.ReferenceIterator; @@ -100,6 +105,9 @@ import ghidra.program.model.pcode.PcodeBlock; import ghidra.program.model.pcode.PcodeBlockBasic; import ghidra.program.model.pcode.Varnode; import ghidra.program.model.pcode.VarnodeAST; +import ghidra.program.util.ContextEvaluatorAdapter; +import ghidra.program.util.SymbolicPropogator; +import ghidra.program.util.VarnodeContext; public final class ReaGhidraBridge extends HeadlessScript { private static final int MAX_HIGH_PCODE_OPS = 3000; @@ -804,7 +812,7 @@ public final class ReaGhidraBridge extends HeadlessScript { return functionObservation(result, boundaryLimitations); } - private JsonArray procedureCalls(JsonObject params, boolean callers) throws Exception { + private JsonObject procedureCalls(JsonObject params, boolean callers) throws Exception { requireKeys(params, Set.of("document", "procedure")); requireDocument(params); Function function = resolveProcedure(requireString(params, "procedure")); @@ -817,7 +825,12 @@ public final class ReaGhidraBridge extends HeadlessScript { for (Function item : ordered) { result.add(canonicalAddress(item.getEntryPoint())); } - return result; + return functionObservation( + result, + noReturnCallLimitations( + currentProgram.getListing().getInstructions(function.getBody(), true) + ) + ); } private JsonObject procedureInfo(JsonObject params) throws Exception { @@ -1319,6 +1332,13 @@ public final class ReaGhidraBridge extends HeadlessScript { result.add("outgoing_references", outgoing); result.add("referenced_strings", referencedStrings); result.add("referenced_names", referencedNames); + boolean swift = swiftProgram(); + boolean aarch64 = currentProgram.getLanguageID().getIdAsString().startsWith("AARCH64:"); + SwiftStringLiteralDecode swiftLiterals = swift && aarch64 ? swiftStringLiterals(function) : null; + if (swiftLiterals != null) { + result.add("swift_string_literals", swiftLiterals.literals()); + result.add("swift_string_literal_omissions", swiftLiterals.omissions()); + } result.add("basic_blocks", blocks); result.add( "native_api", @@ -1338,6 +1358,20 @@ public final class ReaGhidraBridge extends HeadlessScript { limitations.add( "Synthetic Ghidra entry-point references without actionable memory sources are omitted." ); + if (swift && aarch64) { + limitations.add( + "Swift String literals are decoded from constant register pairs at calls, returns and paired stores; literals stored one word at a time, chosen by conditional selects, passed through unresolved jumps, or built at runtime are not reported." + ); + if (swiftLiterals.omissions().size() > 0) { + limitations.add( + "Swift String literal records listed in swift_string_literal_omissions could not be retained or decoded within this dossier's remaining heap budget; later symbolic flows remain unexamined." + ); + } + } else if (swift) { + limitations.add( + "Swift String literal decoding is implemented for AArch64 register pairs only; no Swift literals were decoded for this architecture." + ); + } List boundaryLimitations = noReturnCallLimitations(scan.instructions); for (String limitation : boundaryLimitations) { limitations.add(limitation); @@ -3391,6 +3425,349 @@ public final class ReaGhidraBridge extends HeadlessScript { return result; } + private boolean swiftProgram() { + for (MemoryBlock block : currentProgram.getMemory().getBlocks()) { + String name = block.getName(); + if (name.startsWith("__swift5_") || name.startsWith("swift5_")) return true; + } + return false; + } + + /** + * Swift passes a String as two words: small literals carry their bytes in the words and + * immortal large literals carry their count and a biased pointer to the bytes. Ghidra keeps + * neither as a data reference, so decode constant pairs where Swift consumes them: argument + * and return registers at calls, tail calls and returns, and both registers of one STP. + * The canonical empty string is a small literal with a zero count. + */ + private SwiftStringLiteralDecode swiftStringLiterals(Function function) throws Exception { + Register[] registers = new Register[31]; + for (int index = 0; index < registers.length; index++) { + registers[index] = currentProgram.getRegister("x" + index); + } + TreeMap observed = new TreeMap<>(); + TreeMap omitted = new TreeMap<>(); + SwiftLiteralBudget budget = new SwiftLiteralBudget(); + Map selectedValues = new HashMap<>(); + SymbolicPropogator propagator = new SymbolicPropogator(currentProgram, false); + propagator.flowConstants(function.getEntryPoint(), function.getBody(), new ContextEvaluatorAdapter() { + @Override + public boolean evaluateContextBefore(VarnodeContext context, Instruction instruction) { + // Returning true ends only the current flow, so every queued flow checks the shared stop. + if (budget.stopped()) return true; + for (Register[] pair : swiftStringPairs(function, instruction, registers)) { + Long word = registerConstant(context, pair[0]); + Long object = registerConstant(context, pair[1]); + if (word == null || object == null) continue; + SwiftLiteralLayout layout = swiftLiteralLayout(word, object); + if (layout == null) continue; + // A selected word can combine with a path-dependent ordinary move. Cache the + // provenance question per site/register, charging retained cache state to the + // same dossier budget before allocating it. + boolean selected = false; + boolean provenanceOverCapacity = false; + for (Register register : pair) { + String provenanceKey = canonicalAddress(instruction.getAddress()) + "\u0000" + + register.getBaseRegister().getName(); + Boolean value = selectedValues.get(provenanceKey); + if (value == null) { + if (!budget.takeState()) { + provenanceOverCapacity = true; + break; + } + value = selectedValue(function, instruction, register); + selectedValues.put(provenanceKey, value); + } + if (value) { + selected = true; + break; + } + } + if (selected) continue; + // Forks can reach one site with different constants, so the words are part of the key. + String key = canonicalAddress(instruction.getAddress()) + "\u0000" + pair[0].getName() + + "\u0000" + pair[1].getName() + "\u0000" + Long.toUnsignedString(word, 16) + + "\u0000" + Long.toUnsignedString(object, 16); + if (observed.containsKey(key) || omitted.containsKey(key)) continue; + // Every record draws on the budget; the first one that does not fit is kept as an + // omission and decoding stops, so later sites are never examined. + String value = layout.smallValue(); + boolean overCapacity = provenanceOverCapacity || !budget.take(layout.byteLength()); + if (!overCapacity && value == null) { + try { + value = swiftLargeLiteralText(layout); + } catch (OutOfMemoryError exhausted) { + overCapacity = true; + } + } + if (value == null && !overCapacity) continue; + JsonObject item = overCapacity + ? swiftStringOmission(layout, word, object) + : swiftStringItem(value, layout, word, object); + item.addProperty("source_address", canonicalAddress(instruction.getAddress())); + JsonArray pairRegisters = new JsonArray(); + pairRegisters.add(pair[0].getName()); + pairRegisters.add(pair[1].getName()); + item.add("registers", pairRegisters); + (overCapacity ? omitted : observed).put(key, item); + if (overCapacity) { + budget.stop(); + return true; + } + } + return false; + } + }, false, monitor); + JsonArray literals = new JsonArray(); + observed.values().forEach(literals::add); + JsonArray omissions = new JsonArray(); + omitted.values().forEach(omissions::add); + return new SwiftStringLiteralDecode(literals, omissions); + } + + private static List swiftStringPairs( + Function function, + Instruction instruction, + Register[] registers + ) { + List pairs = new ArrayList<>(); + if (instruction.getMnemonicString().equals("stp")) { + Register first = instruction.getRegister(0); + Register second = instruction.getRegister(1); + if (first != null && second != null && first.getBitLength() == 64 && second.getBitLength() == 64) { + pairs.add(new Register[] {first, second}); + } + return pairs; + } + FlowType flow = instruction.getFlowType(); + boolean returns = false; + for (PcodeOp operation : instruction.getPcode()) { + if (operation.getOpcode() == PcodeOp.RETURN) returns = true; + } + // A jump is a proven tail call only when it has resolved targets and all leave the + // body. A targetless computed jump can still dispatch inside the function. + Address[] targets = instruction.getFlows(); + boolean tailCall = flow.isJump() && !flow.isConditional() && targets.length > 0 && + Arrays.stream(targets).allMatch(target -> !function.getBody().contains(target)); + if (!flow.isCall() && !returns && !tailCall) return pairs; + // A String occupies two consecutive registers among x0-x7, or x0 and x1 when returned. + int lastFirstRegister = returns ? 0 : 6; + for (int index = 0; index <= lastFirstRegister; index++) { + pairs.add(new Register[] {registers[index], registers[index + 1]}); + } + return pairs; + } + + private static Long registerConstant(VarnodeContext context, Register register) { + Varnode value = context.getRegisterVarnodeValue(register); + return value != null && context.isConstant(value) ? value.getOffset() : null; + } + + // Search every predecessor path for the reaching definitions, following register copies, + // since a select copied into an argument register or defined before a branch forks just as + // independently as one written there directly. + private boolean selectedValue(Function function, Instruction use, Register register) { + ArrayDeque instructions = new ArrayDeque<>(); + ArrayDeque targets = new ArrayDeque<>(); + Set visited = new HashSet<>(); + for (Instruction predecessor : predecessors(function, use)) { + instructions.push(predecessor); + targets.push(register.getBaseRegister()); + } + while (!instructions.isEmpty()) { + Instruction instruction = instructions.pop(); + Register target = targets.pop(); + if (!visited.add(canonicalAddress(instruction.getAddress()) + "\u0000" + target.getName())) continue; + boolean defines = false; + for (Object result : instruction.getResultObjects()) { + if (result instanceof Register written && written.getBaseRegister().equals(target)) defines = true; + } + Register next = target; + if (defines) { + PcodeOp[] operations = instruction.getPcode(); + for (PcodeOp operation : operations) { + if (operation.getOpcode() == PcodeOp.CBRANCH && operation.getInput(0).isConstant()) return true; + } + Register source = copiedRegister(operations); + if (source == null) continue; + next = source.getBaseRegister(); + } + for (Instruction predecessor : predecessors(function, instruction)) { + instructions.push(predecessor); + targets.push(next); + } + } + return false; + } + + private List predecessors(Function function, Instruction instruction) { + List result = new ArrayList<>(); + Address from = instruction.getFallFrom(); + if (from != null) { + Instruction previous = currentProgram.getListing().getInstructionAt(from); + if (previous != null) result.add(previous); + } + for (Reference reference : currentProgram.getReferenceManager().getReferencesTo(instruction.getAddress())) { + if (!reference.getReferenceType().isFlow() || reference.getReferenceType().isCall()) continue; + if (!function.getBody().contains(reference.getFromAddress())) continue; + Instruction source = currentProgram.getListing().getInstructionAt(reference.getFromAddress()); + if (source != null) result.add(source); + } + return result; + } + + private Register copiedRegister(PcodeOp[] operations) { + if (operations.length != 1) return null; + int opcode = operations[0].getOpcode(); + Varnode input = operations[0].getInput(0); + if ((opcode != PcodeOp.COPY && opcode != PcodeOp.INT_ZEXT) || input == null || !input.isRegister()) return null; + return currentProgram.getRegister(input); + } + + /** Validate the ABI layout of a pair; a small literal is decoded here, a large one only located. */ + private SwiftLiteralLayout swiftLiteralLayout(long word, long object) { + int discriminator = (int) (object >>> 60); + if (discriminator == 0xE || discriminator == 0xA) { + int count = (int) ((object >>> 56) & 0xF); + byte[] bytes = new byte[15]; + for (int index = 0; index < 8; index++) bytes[index] = (byte) (word >>> (8 * index)); + for (int index = 0; index < 7; index++) bytes[8 + index] = (byte) (object >>> (8 * index)); + for (int index = count; index < bytes.length; index++) { + if (bytes[index] != 0) return null; + } + // Count zero is the canonical empty string (object 0xE000000000000000, word 0). + String value = swiftStringText(bytes, count, discriminator == 0xE); + return value == null ? null : new SwiftLiteralLayout("small", null, count, true, value); + } + int flags = (int) (word >>> 60); + long count = word & 0xFFFFFFFFFFFFL; + // Immortal literals: ASCII and NFC or neither, tail allocated, with a pointer biased by 32. + if ((object >>> 56) != 0x80 || (flags != 0xD && flags != 0x1) || ((word >>> 48) & 0xFFF) != 0) { + return null; + } + try { + Address address = currentProgram.getAddressFactory().getDefaultAddressSpace() + .getAddress((object & 0x00FFFFFFFFFFFFFFL) + 0x20); + MemoryBlock block = currentProgram.getMemory().getBlock(address); + // The bytes and their terminator must lie in one initialized, non-executable block, + // and Java arrays bound one read. + if (block == null || !block.isInitialized() || block.isExecute() || + count >= block.getEnd().subtract(address) + 1 || count >= Integer.MAX_VALUE) { + return null; + } + return new SwiftLiteralLayout("large", address, count, flags == 0xD, null); + } catch (Exception unaddressable) { + return null; + } + } + + private String swiftLargeLiteralText(SwiftLiteralLayout layout) { + int count = (int) layout.byteLength(); + byte[] bytes = new byte[count + 1]; + try { + if (currentProgram.getMemory().getBytes(layout.address(), bytes) != bytes.length) return null; + } catch (Exception unreadable) { + return null; + } + return bytes[count] == 0 ? swiftStringText(bytes, count, layout.ascii()) : null; + } + + /** + * Every literal record of one dossier draws on half of the heap that was free when decoding + * began. Text is charged at the worst case per UTF-8 byte: two bytes of UTF-16 text, then up + * to six escaped JSON characters ("\\u0000") held as UTF-16 in Gson's growing writer buffer + * (up to twice its length), again in the returned String, and once more as UTF-8 on the + * socket. Each record also pays a fixed cost for its map key, JSON members, addresses, words + * and their serialized form, so many empty or small literals are bounded too. Retained + * provenance-cache entries draw on that same fixed-state allowance. + */ + private static final class SwiftLiteralBudget { + private static final long BYTES_PER_LITERAL_BYTE = 2 + 6 * 2 * 2 + 6 * 2 + 6; + private static final long BYTES_PER_RECORD = 4096; + private long remaining; + private boolean stopped; + + SwiftLiteralBudget() { + Runtime runtime = Runtime.getRuntime(); + remaining = Math.max(0, (runtime.maxMemory() - runtime.totalMemory() + runtime.freeMemory()) / 2); + } + + boolean takeState() { + if (remaining < BYTES_PER_RECORD) return false; + remaining -= BYTES_PER_RECORD; + return true; + } + + boolean take(long bytes) { + if (remaining < BYTES_PER_RECORD || + bytes > (remaining - BYTES_PER_RECORD) / BYTES_PER_LITERAL_BYTE) { + return false; + } + remaining -= BYTES_PER_RECORD + bytes * BYTES_PER_LITERAL_BYTE; + return true; + } + + void stop() { + stopped = true; + } + + boolean stopped() { + return stopped; + } + } + + // Swift String values may contain any scalar, including controls; JSON escapes them. + private static String swiftStringText(byte[] bytes, int length, boolean ascii) { + if (length < 0 || length > bytes.length) return null; + for (int index = 0; index < length; index++) { + if (ascii && bytes[index] < 0) return null; + } + try { + return StandardCharsets.UTF_8.newDecoder().decode(ByteBuffer.wrap(bytes, 0, length)).toString(); + } catch (CharacterCodingException invalid) { + return null; + } + } + + private static JsonObject swiftStringOmission(SwiftLiteralLayout layout, long word, long object) { + JsonObject item = new JsonObject(); + item.addProperty("representation", layout.representation()); + if (layout.address() == null) { + item.add("address", JsonNull.INSTANCE); + } else { + item.addProperty("address", canonicalAddress(layout.address())); + } + item.addProperty("byte_length", layout.byteLength()); + item.addProperty("reason", "capacity"); + item.add("words", swiftStringWords(word, object)); + return item; + } + + private static JsonObject swiftStringItem( + String value, + SwiftLiteralLayout layout, + long word, + long object + ) { + JsonObject item = new JsonObject(); + item.addProperty("value", value); + item.addProperty("representation", layout.representation()); + if (layout.address() == null) { + item.add("address", JsonNull.INSTANCE); + } else { + item.addProperty("address", canonicalAddress(layout.address())); + } + item.add("words", swiftStringWords(word, object)); + return item; + } + + private static JsonArray swiftStringWords(long word, long object) { + JsonArray words = new JsonArray(); + words.add("0x" + Long.toUnsignedString(word, 16)); + words.add("0x" + Long.toUnsignedString(object, 16)); + return words; + } + private static String exclusiveAddress(Address inclusive) { Address next = inclusive.next(); if (next != null && next.getAddressSpace().equals(inclusive.getAddressSpace())) { @@ -3930,6 +4307,14 @@ public final class ReaGhidraBridge extends HeadlessScript { private record AddressNameRequest(String inputAddress, Address address, String name) {} private record FunctionEntry(Function function, InventoryItem item) {} private record FunctionAnalysisObservation(JsonObject dossier, List limitations) {} + private record SwiftStringLiteralDecode(JsonArray literals, JsonArray omissions) {} + private record SwiftLiteralLayout( + String representation, + Address address, + long byteLength, + boolean ascii, + String smallValue + ) {} private record InstructionScan(List instructions, boolean truncated) {} private record SessionDescriptor( String transport, diff --git a/bridge/ghidra/ReaGhidraPrepareRaw.java b/bridge/ghidra/ReaGhidraPrepareRaw.java index 1a5db9627..143331390 100644 --- a/bridge/ghidra/ReaGhidraPrepareRaw.java +++ b/bridge/ghidra/ReaGhidraPrepareRaw.java @@ -1,6 +1,8 @@ import ghidra.app.script.GhidraScript; import ghidra.program.model.address.Address; import ghidra.program.model.listing.Program; +import ghidra.program.model.mem.MemoryBlock; +import ghidra.program.model.mem.MemoryBlockSourceInfo; /** Seed the caller-supplied raw-binary entry before default auto-analysis. */ public final class ReaGhidraPrepareRaw extends GhidraScript { @@ -21,8 +23,19 @@ public final class ReaGhidraPrepareRaw extends GhidraScript { ); } Address entry = currentProgram.getAddressFactory().getDefaultAddressSpace().getAddress(entryHex); - Address start = currentProgram.getMinAddress(); - if (entry == null || !entry.equals(start)) { + MemoryBlock block = entry == null ? null : currentProgram.getMemory().getBlock(entry); + boolean fileStart = false; + if (block != null) { + // Processor-defined zero page/stack blocks can precede the imported file. + for (MemoryBlockSourceInfo source : block.getSourceInfos()) { + if (source.getFileBytes().isPresent() && source.getFileBytesOffset() == 0 && + entry.equals(source.getMinAddress())) { + fileStart = true; + break; + } + } + } + if (!fileStart) { throw new IllegalArgumentException( "Raw binary import must map file offset 0 at the caller-supplied entry" ); diff --git a/docs/binary-diagnostics.md b/docs/binary-diagnostics.md index 29e10a33c..0e6a3f196 100644 --- a/docs/binary-diagnostics.md +++ b/docs/binary-diagnostics.md @@ -98,8 +98,10 @@ section, symbol, mitigation or linkage views. The default `complete` detail and CLI output are unchanged. Complete results have a 32 MiB input, 64 MiB reply, 1 MiB combined diagnostics -and 30-second owned command deadline. The reply budget applies to the decoder -record; the CLI/MCP Evidence envelope adds copies and encoding overhead. The +and 30-second owned command deadline. `REA_PWNTOOLS_MAX_INPUT_BYTES` raises the +input bound alone, up to 1 GiB; the reply and deadline bounds are unchanged. The +reply budget applies to the decoder record; the CLI/MCP Evidence envelope adds +copies and encoding overhead. The pinned MCP SDK defaults to a 10 MiB receive buffer. Large complete results need a caller-configured `StdioClientTransport({ maxBufferSize: ... })` and a matching server `REA_MCP_MAX_RESPONSE_BYTES`. See [MCP delivery budgets](mcp-contracts.md#tool-results). diff --git a/docs/browser-observation.md b/docs/browser-observation.md index 2f73f4261..dca1f994b 100644 --- a/docs/browser-observation.md +++ b/docs/browser-observation.md @@ -80,6 +80,27 @@ google-chrome \ REA does not launch, own, or terminate this browser. Use a dedicated profile and stop it yourself when the investigation is complete. Do not expose the debugging port on a non-loopback interface. +Other Chromium-family browsers, such as Chromium or Brave, work the same way +with the same two flags. On macOS, run the executable inside the application +bundle, for example +`"/Applications/Google Chrome.app/Contents/MacOS/Google Chrome"` or +`"/Applications/Brave Browser.app/Contents/MacOS/Brave Browser"`. Fedora installs +Chromium as `chromium-browser`. + +REA needs the browser's HTTP discovery endpoint on a loopback port. A browser +driven over `--remote-debugging-pipe` or a private channel, for example by an +automation SDK, exposes no such endpoint, so `list_browser_targets` has nothing +to attach to. Connection failures report `endpoint_unreachable`; a reachable +HTTP service that returns a non-200 or non-JSON discovery response reports +`invalid_endpoint_response`. +Reproduce that flow in a Chromium-family browser started with +`--remote-debugging-port`. + +In a minimal Linux container, headed Chromium under Xvfb can keep running while +its DevTools port never opens. Install the GLES, EGL, and Mesa DRI libraries +(on Fedora: `libglvnd-gles`, `mesa-libEGL`, and `mesa-dri-drivers`), then +restart the browser. + On Linux, a reachable but locked desktop keyring can stall Chrome before its first navigation. For a dedicated investigation profile, add `--password-store=basic` to bypass that keyring. Empty or placeholder frame URLs @@ -207,7 +228,7 @@ commands fail closed. A correlated reply's session ID must match the command's selected session. An optional method is unavailable only when the browser rejects it as unknown; other command rejections retain their reported failure. -Network and console coverage starts only after REA attaches and enables the relevant CDP domains. `prior_activity_available` is always `false`; absence from these arrays is not evidence that an event never occurred. If CDP reuses a request ID without supplying redirect metadata, REA retains the predecessor observation and marks network coverage incomplete. +Network and console coverage starts only after REA attaches and enables the relevant CDP domains. `prior_activity_available` is always `false`; absence from these arrays is not evidence that an event never occurred. Scripts are different: scripts the page parsed before REA attached are still reported under `scripts`. A page that loaded before attach can therefore list its scripts while `network.requests` lacks the requests that fetched them. If CDP reuses a request ID without supplying redirect metadata, REA retains the predecessor observation and marks network coverage incomplete. Source maps are not fetched by inspection. Bundle analysis fetches them only when `fetch_source_maps` is selected, with exact-origin redirect checks and no credentials, cookies, or referrer. Each fetch operation has a 30-second deadline and a 64 MiB aggregate streamed-response budget across redirects. A deadline or byte-limit failure remains explicitly unavailable and makes bundle completeness partial. Source-map decoding also bounds raw resolved identities to 64 MiB and aggregate inline evidence to 32 MiB. Failed maps preserve their failure context when it fits; if another context cannot fit, collection stops with the full requested count, the returned processed count, and an explicit unknown-results limitation. Retained mappings and original sources are returned inline. See [source-location limits](web-source-location.md) for the structural decoder bounds. Original source artifacts with `.ts`, `.tsx`, `.mts`, or `.cts` suffixes carry `text/typescript`; other source names retain `text/javascript`. diff --git a/docs/cli.md b/docs/cli.md index dcd949e11..6d81ec7c1 100644 --- a/docs/cli.md +++ b/docs/cli.md @@ -174,7 +174,9 @@ rea analyze /absolute/path/to/program --provider ghidra --snapshot /absolute/pat rea analyze /absolute/path/to/program --provider ghidra --snapshot /absolute/path/to/analysis/program.json ``` -An exact CLI cache hit is read before starting a provider process. +An exact CLI cache hit is normally read before starting a provider process. +On Windows, Ghidra first admits the selected source through its native snapshot +owner, so replay can start that provider. In MCP, `open_binary` accepts `snapshot_path` to import a snapshot atomically for its matching target; an MCP provider may still start before returning a cached result. `close_binary` accepts `snapshot_path` and optional diff --git a/docs/client-configuration.md b/docs/client-configuration.md index 00e83e179..e27866f7d 100644 --- a/docs/client-configuration.md +++ b/docs/client-configuration.md @@ -165,6 +165,14 @@ For [Qoder](https://docs.qoder.com/cli/mcp-reference), setup writes from `~/.qoder/skills` (following `QODER_CONFIG_DIR` when set), so setup installs its skill copy there. +## Factory Droid + +For [Factory Droid](https://docs.factory.com/harness/mcp), setup writes +`mcpServers.rea` into the user-level `~/.factory/mcp.json`. Entries carry +`command`, `args`, and `env`. Droid discovers personal skills from +`~/.factory/skills` ([skills](https://docs.factory.com/harness/skills)), so +setup installs its skill copy there. + ## Trae For [Trae](https://docs.trae.ai/ide/add-mcp-servers), setup writes @@ -209,8 +217,15 @@ replaces) the REA entry in the home patch `$DSH_HOME/cordis.patch.yml` patches and comments. `rea doctor --client dsh` checks that same entry id. `rea uninstall` removes it by scanning supported clients; uninstall takes no `--client` flag. The home patch applies to every Harness profile, so the -registration is written once and no per-profile edit is needed. The written -entry is: +registration is written once and no per-profile edit is needed. + +Setup, doctor and uninstall resolve `DSH_HOME` and `DSH_AGENTS_HOME` as Harness +does: unset, empty or whitespace-only values use `~/.dsh` and `~/.agents`, +respectively. Bare `~`, `~/` and `~\` expand to the user home; other relative +values resolve against the command's working directory. Nonblank values retain +their original spaces, and the agents root is independent of the Harness home. +Run REA and Harness from the same working directory when using relative overrides. +The written entry is: ```yaml - insert: diff --git a/docs/client-verification.md b/docs/client-verification.md index 42e059382..d9ece9aad 100644 --- a/docs/client-verification.md +++ b/docs/client-verification.md @@ -18,14 +18,26 @@ pass `-- chat` to check plain chat with REA enabled. Qwen also reads the full result that its client offloads, and Pi exercises default codemode execution. Use `REA_VERIFY_RUNTIME_ROOT` to select a production-only installed REA package. Receipts retain client versions, result digests, host coverage and owned-process -cleanup. These POSIX lanes use a deterministic loopback model, so they do not -prove live model-provider or native Windows compatibility. Qwen, Pi and Hermes +cleanup. These lanes use a deterministic loopback model and do not prove live +model-provider compatibility. Pi and Hermes remain POSIX-only. Qwen, Pi and Hermes bind both `HOME` and `USERPROFILE` to the disposable account. Qwen and Pi discover the shared personal skill through their default search paths without an explicit skill setting; Hermes discovers it through the selected `HERMES_HOME`. Set `REA_VERIFY_HERMES_STICKY_PROFILE=1` to exercise native Hermes selection of a named sticky profile with `HERMES_HOME` still pointing to its root. +The Qwen lane also supports native Windows (verified with Qwen Code 0.25.1 and +Node 24.19.0 for both `chat` and `call`). With Node 24.18.0, the same `call` +journey reached full Evidence readback but the client aborted during exit with +`UV_HANDLE_CLOSING`; that run remains a failure, not a compatibility claim. +Select the npm install's +`node_modules/@qwen-code/qwen-code/cli-entry.js` with `REA_VERIFY_QWEN_COMMAND`, +because `execFile` cannot launch the `qwen.cmd` shim directly. A selected `.js`, +`.mjs` or `.cjs` entry point runs under the verifier's Node. Windows path checks +use native separators; persisted output is read through Qwen's real shell tool +with a shell-independent encoded Node payload. Windows receipts report process +lineage as unavailable when the verifier has no retained native Job Object lease. + `verify:gemini-client` requires an installed Gemini CLI (verified with `@google/gemini-cli@0.63.0` on POSIX and native Windows); select it with `REA_VERIFY_GEMINI_COMMAND`. A selected `.js`, `.mjs` or `.cjs` entry point runs diff --git a/docs/ghidra-mips.md b/docs/ghidra-mips.md index 59d04a8b0..d5b58f00b 100644 --- a/docs/ghidra-mips.md +++ b/docs/ghidra-mips.md @@ -111,7 +111,7 @@ MIPS header and ABI record facts are committed in the analysis profile so interpretation changes invalidate profile-bound snapshots. The support-lane identity is versioned independently of file identity. Evidence, lifecycle outputs and saved snapshots preserve the `mips` family. Hopper and IDA adapters -do not implicitly gain MIPS support. Windows Ghidra P0 remains PE x86/x86-64 only. +do not implicitly gain MIPS support. Windows Ghidra P0 does not admit MIPS targets. ### Soft-float prototype diff --git a/docs/ghidra-provider.md b/docs/ghidra-provider.md index 32edbd320..5b83ee152 100644 --- a/docs/ghidra-provider.md +++ b/docs/ghidra-provider.md @@ -131,9 +131,10 @@ overview merely because its source artifact is unchanged. Function extents and instruction completeness refer to Ghidra's analyzed body. An inferred no-return call can terminate that body before subsequent bytes. -Pseudocode, assembly, procedure info, instruction windows, and dossiers report -observed terminal call sites in Evidence limitations, including resolved callee -names, addresses, and FunctionManager flags when available. Instruction windows +Pseudocode, assembly, procedure info, instruction windows, callee and caller +reads, and dossiers report observed terminal call sites in Evidence limitations, +including resolved callee names, addresses, and FunctionManager flags when +available. Instruction windows and dossiers also retain these warnings in their result limitations. An unresolved callee or unestablished terminal-flow basis remains explicit. These observations qualify the analysis; they do not prove that a no-return flag is diff --git a/docs/ghidra-riscv.md b/docs/ghidra-riscv.md index 5b3407361..afae01948 100644 --- a/docs/ghidra-riscv.md +++ b/docs/ghidra-riscv.md @@ -11,8 +11,8 @@ ELF `EM_RISCV` (243) is retained as the provider-neutral `riscv` family. The target also retains the raw ELF declarations that decide a lane: class, byte order, `e_type`, and `e_flags`. Recognition does not mean every provider or RISC-V variant is supported. Existing PE, managed and Mach-O architecture sets -are unchanged, and caller-supplied headerless raw-binary entry still accepts only -its existing four families. +are unchanged. Headerless inputs use the separate [raw-binary admission +boundary](raw-binary.md); raw RISC-V images are not supported. The Ghidra lane admits little-endian ELF32 and ELF64 `ET_EXEC` images that declare `EF_RISCV_FLOAT_ABI_DOUBLE` in `e_flags`, with no `EF_RISCV_RVE`, diff --git a/docs/ida-provider.md b/docs/ida-provider.md index b22a35cce..fbf67c8a2 100644 --- a/docs/ida-provider.md +++ b/docs/ida-provider.md @@ -88,6 +88,8 @@ REA copies the admitted binary into a private temporary directory, checks its di One REA session owns one database. Stdio registrations default `IDA_MCP_MAX_WORKERS` to `1`; the explicit upstream `--max-workers 1` also bounds a supervisor. Database size and engine memory depend on the target and upstream; REA does not impose an engine memory limit. The original binary is never executed or overwritten. Optional `workspaceRoot` selects an existing absolute temporary-directory parent on the REA host. Ensure it has space for the input copy and IDA working files. +When a completed `idb_open` response reports success or is otherwise ambiguous, but no active owned private worker can be verified, REA retains the private workspace even if `idb_list` appears empty. A later close can retry verification if the original owned worker becomes discoverable. A completed explicit `idb_open` rejection retains the existing no-worker cleanup path. This is a local admission/ownership safeguard and **does not** provide a process-termination receipt or change the legacy reaped-worker protocol. + Headless cleanup uses the upstream database lifecycle. Abruptly terminating REA, its proxy, or the supervisor does not establish worker release. Prefer `close_binary` for MCP sessions; after an interrupted process, inspect upstream session inventory and retained workspace diagnostics before removing working files. This adapter does not claim Windows Job Object containment for the upstream detached workers. ## Connect an agent once @@ -126,6 +128,8 @@ For guided registration, set `REA_IDA_MCP_CONFIG` before `rea setup --client =22.19), 24.x (>=24.11), or 26+; - an operator-installed Ghidra 12.1.x distribution (verified with 12.1.4); - a 64-bit full JDK inside that installation's declared Java range (JDK 21 or newer, with no maximum, for current 12.1 releases); -- an explicit native, non-managed x86 or x86-64 PE application or DLL; and +- an explicit native, non-managed x86 or x86-64 PE application/DLL, or a + [raw NMOS 6502 image](raw-binary.md#nmos-6502) with a caller-supplied entry; and - the 25 read-only Ghidra inventory, memory, and function-analysis operations. Loaded memory reads and file offsets preserve Ghidra's observed source mappings. PE load-image inspection returns an explicit unsupported attestation status. Independent verification covers admitted DOS MZ, explicit COM, and explicit -[raw-binary imports](raw-binary.md) on their supported hosts. This does not -extend Windows P0 admission beyond native PE. For x64 PE +[raw-binary imports](raw-binary.md) on their supported hosts, including the +admitted raw NMOS 6502 mapping on Windows. Windows P0 admission remains limited +to native PE and raw NMOS 6502 images. For x64 PE NativeAOT RTR 9.1, the read-only snapshot parser can report metadata through `inspect_native_load_image` and `inspect_native_data_type` without a JAR or database mutation. A real Windows-host NativeAOT Ghidra run is still required @@ -35,6 +37,12 @@ state, annotations, other target architectures and formats, and Hopper-only operations remain unavailable through this provider. Managed PE/CLI inspection uses its separate execution-free provider. +Raw 6502 imports retain the same native filesystem and process ownership +requirements. The complete image must fit in the 16-bit address space; no +console-specific cartridge loader, CPU variant, or bank-switching model is +implied. Run `npm run verify:ghidra:6502` with caller-installed `ca65`/`ld65` +to check the source-built fixture through real CLI and MCP workflows. + REA does not install or upgrade Ghidra, Java, Python, npm, Node.js, Hopper, or a compiler. The adapter uses the packaged Java `HeadlessScript`; Python and PyGhidra are not prerequisites. Users do not build the native addon. diff --git a/experiments/analysis-efficiency/agent-trial.py b/experiments/analysis-efficiency/agent-trial.py index 6dcc7d7f8..dc13e8781 100644 --- a/experiments/analysis-efficiency/agent-trial.py +++ b/experiments/analysis-efficiency/agent-trial.py @@ -94,6 +94,9 @@ def main(): if undeclared: raise ValueError('Declare inherited provider settings explicitly in --providers: ' + ', '.join(undeclared) + '. Values are not logged.') + source_auth_home = (Path(os.environ['CODEX_HOME']) if 'CODEX_HOME' in os.environ + else Path.home() / '.codex') + source_auth = source_auth_home / 'auth.json' args.output.mkdir(parents=True, exist_ok=False, mode=0o700) runtime_files, runtime_digest = runtime_fingerprint(repository) (args.output / 'runtime-files.json').write_text(json.dumps(runtime_files)) @@ -103,7 +106,6 @@ def main(): workspace = profile / 'workspace' for directory in [trial_codex_home, account, workspace]: directory.mkdir(mode=0o700) - source_auth = Path(os.environ.get('CODEX_HOME', str(Path.home() / '.codex'))) / 'auth.json' auth = trial_codex_home / 'auth.json' started = time.monotonic() timeout = False diff --git a/experiments/analysis-efficiency/test_codex_context.py b/experiments/analysis-efficiency/test_codex_context.py index 37df83a7c..0b8e0247c 100644 --- a/experiments/analysis-efficiency/test_codex_context.py +++ b/experiments/analysis-efficiency/test_codex_context.py @@ -71,7 +71,9 @@ class ContextCaptureTests(unittest.TestCase): self.assertNotIn(marker, text) self.assertIn('caller-selected evidence and paths', text) self.assertIn('original-response', text) - self.assertEqual((output / 'codex-tool-context.jsonl').stat().st_mode & 0o777, 0o600) + # Windows mode bits do not describe access control; this is a POSIX assertion. + if os.name == 'posix': + self.assertEqual((output / 'codex-tool-context.jsonl').stat().st_mode & 0o777, 0o600) def test_missing_rollouts_are_unknown_and_malformed_lines_remain_partial(self): with tempfile.TemporaryDirectory() as directory: @@ -124,6 +126,7 @@ class ContextCaptureTests(unittest.TestCase): patch.object(agent.subprocess, 'check_output', return_value='v24.18.0\n'), \ patch.object(agent.subprocess, 'Popen', return_value=child), \ patch.object(agent, 'capture_tool_context', side_effect=OSError('fixture capture failure')), \ + patch.object(agent.Path, 'home', side_effect=AssertionError('Explicit CODEX_HOME must not resolve the default home')), \ patch('builtins.print'): with self.assertRaisesRegex(OSError, 'fixture capture failure'): agent.main() diff --git a/experiments/analysis-efficiency/windows-posix-reconciliation.md b/experiments/analysis-efficiency/windows-posix-reconciliation.md new file mode 100644 index 000000000..0df06ac3e --- /dev/null +++ b/experiments/analysis-efficiency/windows-posix-reconciliation.md @@ -0,0 +1,128 @@ +# Windows reconciliation of recorded POSIX paths + +Follow-up to [#2422](https://github.com/morluto/rea/pull/2422), including +@Aurantium-47's diagnosis of the host-dependent relative-path operation. + +## Source pins and real data + +The initial reproduction and passive producer used main +fa4b8288655188426222773ead2ccbdaf76334fd. Before branching, main was updated +to 8b181716aca2cb467be499cba226abfb5ae19141. The owning matcher and its +affected reconciliation consumers were unchanged between those revisions. +The newer baseline again reproduced both errors on the real Windows host. + +Input: npm 11.16.0's published lib/utils/validate-lockfile.js, 1,023 bytes, +SHA-256 2295e2dac1c5816e1eae2d417804960ebbef17b64a0a44db4f3483ce6e6d9526. +The installed copy was independently checked against the exact module in the +[official registry package](https://registry.npmjs.org/npm/11.16.0). +The 2,945,850-byte tarball matched its published SHA-512 SRI before reading +that member; no archive extraction or installation occurred, and the tarball +was not retained. + +Two actual Linux filesystem directories, Runtime/App and Runtime/app, +held byte-identical module copies and had distinct directory identities. +A controlled Node 24.18.0 process loaded the public module and ran its real +function against 587 locked dependency records: compatible trees returned zero +errors, and one controlled version mismatch returned one error. A passive REA +Inspector capture recorded 114 loaded scripts, including the selected module. +The provider did not evaluate, pause, resume or read source. The owned process +exited normally. Function execution is established by the harness receipt; +REA's passive script metadata establishes loaded status. + +Production static analysis inspected both directories plus an independent +Static/public-module copy. The latter prevents implicit root matching from +hiding the explicit mapping result. Complete static/runtime Evidence was then +consumed by the actual **Windows** CLI and SDK 2.3.1 stdio MCP server, using +Windows 11 10.0.22631.0 and official Windows x64 Node 24.18.0. +No Electron GUI, IDA or Ghidra engine was used. + +## Correctness and provenance + +| Windows reconciliation | Baseline | Candidate | +| -------------------------------------- | ------------------------------ | ----------------------------------- | +| Exact POSIX input root | Matched, artifact path | Same complete normalized result | +| Case-mismatched POSIX input root | Incorrectly matched and loaded | Unknown; static load states unknown | +| Exact explicit POSIX mapping | Matched, operator file mapping | Same complete normalized result | +| Case-mismatched explicit POSIX mapping | Incorrectly matched and loaded | Unknown; static load states unknown | + +All four CLI and same-connection MCP normalized results agree. The input +Evidence, recorded runtime facts and caller-selected root spellings are +preserved. Rejected location membership does not establish that a module was +absent or unexecuted. The operation remains offline and performs no target or +filesystem inspection. + +The shared matcher now chooses posix.relative after identifying POSIX input; +Windows coordinates continue to use win32.relative. Both location matching +and active-layer scope use this owner. The obsolete host-default import is +removed; no downstream repair or new path abstraction is introduced. + +The related suite passed 102 tests: 15 workflow tests, 85 CI scope tests and +two curated-path checks. Four simulated Electron Evidence controls cover +POSIX mismatch/exact matching, Windows case-insensitive matching, the resulting +asset load state and retained mapping spelling. They are service regressions, +not real Electron-provider verification. The existing workflow test file is +registered in both the Windows execution lane and its package-change scope. + +## Delivery and resource observations + +Probes ran serially at BelowNormal priority with one CPU and 512 MiB Node heap +settings for CLI/server/client work. Heap settings do not bound aggregate memory. +Four corrected CLI exit-time observations reported 7.343–15.827 seconds CPU and +322,308–324,280 KiB peak working set, excluding the probe parent. Wall times +ranged from 7.785 to 17.601 seconds. There is no paired resource baseline. + +One complete MCP trial connected in 18,296 ms and discovered 189 tools in +875 ms (2,588,336 serialized bytes). The four subsequent same-connection calls +took 879, 320, 353 and 422 ms. Their serialized replies were 918,133 / 913,765 / +918,571 / 914,171 bytes; inputs totalled 681,312 bytes. Discovery plus these +inputs and replies totalled 6,934,288 bytes, excluding initialization, framing +and notifications. This is delivery volume, not model token usage. Actual +model context and total model task cost were not measured. + +A process-generation sampler recorded 233 observations over 25,671 ms: +15,797 ms aggregate observed CPU as a lower bound and a peak sampled working-set +sum of 491,597,824 bytes (about 469 MiB). Exited-process tails can be missed; +shared pages can be counted twice. This excludes the observer and is not an +exact physical-memory peak. The server wrote four structured completion records +to stderr; tool replies and normalized results passed the recorded checks. + +Cold/warm caches and observer differences were not normalized. No latency, +CPU, memory or model-token improvement is claimed. The fix adds no lookup I/O, +engine invocation, retry or cache. Complete default replies remain available; +only compact selected findings and hashes were sent to the investigating agent. + +## Main integration replay + +Runtime cb0d8a9fef2b907cff2ba27f1e017493e1173eb2, rebased onto main +73dd8d6ed, again passed the 101 related tests and all four real Windows CLI/MCP +cases using the unchanged older producer Evidence. Positive results and original +runtime/layer facts remain identical. Typecheck, lint, dependency and metadata +checks passed; all five changed files passed formatting independently. +The full format gate at that main revision failed on the unchanged upstream +GhidraDoctor.test.ts and GhidraSnapshotCache.test.ts files. The PR does not +repair those unrelated baseline files. Resource figures above refer to the +earlier candidate; the integration replay does not establish a speedup. + +Review found that owning only the Windows test path would still skip that lane +on a later implementation-only edit. The matcher implementation is now also +registered in package scope. A source-only classifier case verifies that it +selects both Windows package and JavaScript recovery lanes; 85 scope checks and +two curated-path checks pass. Runtime code and the real-data checks are unchanged. + +The original production fix merged as +[1a1b67c88](https://github.com/morluto/rea/commit/1a1b67c88) while this follow-up +was open. The follow-up was then rebased onto main fb28ec153 and contains only +the stronger workflow assertions, Windows lane ownership, scope classification +and this verification record. The updated 131 related tests, two representative +Ghidra boundary tests, seven IDA input-thinning tests, 24 semantic-structure +tests, 16 required-lane checks, 39 release-checkpoint tests and 11 dyld CLI +tests passed after the rebase (230 total). +Upstream a0317cf73 corrected the invalid FAT fixture and consolidated unrelated +duplicated helpers; e05d3399f aligned the older parameter-provenance assertion +with the upstream semantics. Those changes, the threshold change and the new +CI qualification tiers are part of main, outside this follow-up's diff. +Upstream 747e72991 restored release preflight coverage for reusable workflow +jobs. The independently reviewed, fully green #2755 fixture correction merged +as fb28ec153 and restored real x64 dyld CLI admission without altering the +original same-named arm64e assertions. +No production source is duplicated by the follow-up. diff --git a/package.json b/package.json index 0bc15c982..4af1b2ae1 100644 --- a/package.json +++ b/package.json @@ -178,6 +178,7 @@ "verify:ghidra:seeds": "npm run build:cached && node scripts/verify/ghidra/seeds.mjs", "verify:ghidra:mips": "npm run build:cached && node scripts/verify-real-ghidra-mips.mjs", "verify:ghidra:riscv": "npm run build:cached && node scripts/verify-real-ghidra-riscv.mjs", + "verify:ghidra:6502": "npm run build:cached && node scripts/verify-real-ghidra-6502.mjs", "verify:ghidra:mips-shared": "npm run build:cached && node scripts/verify-real-ghidra-mips-shared.mjs", "verify:ghidra:mips-relocatable": "npm run build:cached && node scripts/verify-real-ghidra-mips-relocatable.mjs", "verify:ghidra:swift": "npm run build:cached && node scripts/verify-real-ghidra-swift.mjs", diff --git a/scripts/ci/scopes.mjs b/scripts/ci/scopes.mjs index a8679726c..ffe15cb3d 100644 --- a/scripts/ci/scopes.mjs +++ b/scripts/ci/scopes.mjs @@ -56,6 +56,8 @@ export const SCOPES = { "tests/boundary/filesystem/setupUnavailableProvider.test.ts", "tests/boundary/filesystem/windowsNpxRegistration.test.ts", "tests/acceptance/setup/homeDirectoryFallback.test.ts", + "tests/boundary/filesystem/javascriptApplicationWorkflows.test.ts", + "src/domain/javascript/javascriptRuntimeStaticCandidates.ts", "tests/boundary/filesystem/webScriptManifestPaths.test.ts", "tests/boundary/process/ProcessEvidence.host.test.ts", "tests/acceptance/webModuleCli.test.ts", diff --git a/scripts/lib/ida-feature-trace.d.mts b/scripts/lib/ida-feature-trace.d.mts new file mode 100644 index 000000000..13c84df27 --- /dev/null +++ b/scripts/lib/ida-feature-trace.d.mts @@ -0,0 +1,9 @@ +export function assertFeatureTrace( + trace: unknown, + procedure: { readonly address: string }, +): void; + +export function findMappedNonFunction( + strings: readonly { readonly address: string }[], + resolveAt: (address: string) => Promise, +): Promise; diff --git a/scripts/lib/ida-feature-trace.mjs b/scripts/lib/ida-feature-trace.mjs new file mode 100644 index 000000000..3c4778d25 --- /dev/null +++ b/scripts/lib/ida-feature-trace.mjs @@ -0,0 +1,52 @@ +import assert from "node:assert/strict"; + +const NEGATIVE_REASONS = new Set(["not_in_procedure", "outside_segments"]); + +/** + * Assert the `trace_feature` result of the real IDA lane for one selected + * procedure. The query is a substring search, so the trace may also match other + * procedures and strings; each reference is checked against its own matched + * target, and a source outside every function (a data reference) is valid. + */ +export function assertFeatureTrace(trace, procedure) { + assert.ok( + trace.matches.some( + ({ type, address }) => + type === "procedure" && address === procedure.address, + ), + "The trace omitted the selected procedure's match.", + ); + assert.equal(typeof trace.truncated, "boolean"); + const matched = new Set(trace.matches.map(({ address }) => address)); + for (const reference of trace.references) { + assert.ok( + matched.has(reference.target_address), + `A reference targets ${reference.target_address}, which is not a match.`, + ); + const { containing_procedure: containing } = reference; + if (containing.found) + assert.equal(typeof containing.procedure.address, "string"); + else assert.ok(NEGATIVE_REASONS.has(containing.reason)); + } +} + +/** + * Find a string that IDA maps outside every function. A string inside a + * function chunk (inline data in code) is valid and skipped, so the strings are + * tried in order. Any other answer must be "not in a procedure": IDA lists only + * mapped strings, so an unmapped answer is a containment defect and fails the + * lane. Returns the address, or undefined when every string is inside a function. + */ +export async function findMappedNonFunction(strings, resolveAt) { + for (const { address } of strings) { + const resolved = await resolveAt(address); + if (resolved.found) continue; + assert.equal( + resolved.reason, + "not_in_procedure", + `The listed string at ${address} resolved as ${resolved.reason}.`, + ); + return address; + } + return undefined; +} diff --git a/scripts/verify-qwen-client.mjs b/scripts/verify-qwen-client.mjs index ed60ccd4c..fff4338b7 100644 --- a/scripts/verify-qwen-client.mjs +++ b/scripts/verify-qwen-client.mjs @@ -8,7 +8,7 @@ import { realpath, } from "node:fs/promises"; import { tmpdir } from "node:os"; -import { join } from "node:path"; +import { join, resolve, sep } from "node:path"; import { fileURLToPath } from "node:url"; import { createHash } from "node:crypto"; import { exec } from "./lib/verify-package-core.mjs"; @@ -18,10 +18,6 @@ import { toolContract, } from "../dist/contracts/toolContracts.js"; -assert( - process.platform !== "win32", - "The Qwen real-client verifier currently requires a POSIX host; native Windows is unverified.", -); const repo = fileURLToPath(new URL("..", import.meta.url)); const runtimeRoot = process.env.REA_VERIFY_RUNTIME_ROOT ?? repo; const lab = await mkdtemp(join(tmpdir(), "rea-qwen-client-")); @@ -32,6 +28,13 @@ assert( "Usage: node scripts/verify-qwen-client.mjs [chat|call]", ); const command = process.env.REA_VERIFY_QWEN_COMMAND ?? "qwen"; +// npm exposes only qwen.cmd on Windows. Resolve a selected JavaScript entry +// before the verifier switches to its disposable workspace. +const entryPoint = /\.(c|m)?js$/iu.test(command) ? resolve(command) : undefined; +const launch = (args) => + entryPoint === undefined + ? [command, args] + : [process.execPath, [entryPoint, ...args]]; const verifier = createVerifierRun(); const profile = join(lab, "selected profile"); const account = join(lab, "account"); @@ -54,14 +57,18 @@ const baseEnvironment = { let clientVersion; try { clientVersion = ( - await exec(command, ["--version"], { + await exec(...launch(["--version"]), { env: baseEnvironment, timeout: 30_000, }) ).stdout.trim(); } catch (cause) { + const windowsHint = + process.platform === "win32" + ? " On Windows, select node_modules/@qwen-code/qwen-code/cli-entry.js." + : ""; throw new Error( - `Qwen client preflight failed for ${command}; install Qwen Code or set REA_VERIFY_QWEN_COMMAND. ${String(cause)}`, + `Qwen client preflight failed for ${command}; install Qwen Code or set REA_VERIFY_QWEN_COMMAND.${windowsHint} ${String(cause)}`, ); } const original = JSON.stringify( @@ -93,8 +100,8 @@ for (const step of [ assert( result.plannedActions.every( (a) => - a.target.startsWith(profile + "/") || - a.target.startsWith(sharedSkills + "/"), + a.target.startsWith(profile + sep) || + a.target.startsWith(sharedSkills + sep), ), ); await writeFile( @@ -163,7 +170,14 @@ const fixture = await createOpenAiModelFixture({ skillSeen ||= JSON.stringify(body.messages ?? []).includes( "reverse-engineer-anything", ); - selectedSkillLoaded ||= JSON.stringify(body.messages ?? []).includes( + const messageText = (body.messages ?? []) + .map((message) => + typeof message.content === "string" + ? message.content + : (message.content ?? []).map((part) => part.text ?? "").join("\n"), + ) + .join("\n"); + selectedSkillLoaded ||= messageText.includes( join(sharedSkills, "reverse-engineer-anything"), ); const toolText = JSON.stringify( @@ -243,9 +257,7 @@ const fixture = await createOpenAiModelFixture({ const invoke = tools.find((t) => t.name === "tool_call"); const skill = tools.find((t) => t.name === "skill"); const shell = tools.find((t) => t.name === "run_shell_command"); - const persisted = toolText.match( - /Full output saved to: ([^\\"\n]+)(?:\\n|\n)/u, - ); + const persisted = messageText.match(/Full output saved to: ([^\r\n]+)/u); const call = (name, args) => ({ role: "assistant", content: null, @@ -268,13 +280,14 @@ const fixture = await createOpenAiModelFixture({ artifactPath = persisted[1]; assert( (await realpath(artifactPath)).startsWith( - (await realpath(runtime)) + "/", + (await realpath(runtime)) + sep, ), ); const script = `const fs=require('node:fs');const e=JSON.parse(fs.readFileSync(${JSON.stringify(artifactPath)},'utf8'));console.log(JSON.stringify({REA_CLIENT_FIXTURE:JSON.stringify(e.normalized_result).includes('REA_CLIENT_FIXTURE'),evidence_id:e.evidence_id,graph_nodes:e.normalized_result.graph.nodes.length}));`; - const quote = (value) => "'" + value.replaceAll("'", "'\\''") + "'"; + // Keep paths and JSON quotes out of shell syntax on Windows and POSIX. + const encodedScript = Buffer.from(script).toString("base64"); message = call(shell.name, { - command: `node -e ${quote(script)}`, + command: `node -e "eval(Buffer.from('${encodedScript}','base64').toString('utf8'))"`, description: "Read the complete saved REA Evidence and verify the fixture facts.", }); @@ -329,7 +342,7 @@ const args = [ let response; let failure; try { - response = await exec(command, args, { + response = await exec(...launch(args), { env, cwd: workspace, timeout: 120_000, diff --git a/scripts/verify-real-ghidra-6502.mjs b/scripts/verify-real-ghidra-6502.mjs new file mode 100644 index 000000000..87fb01d41 --- /dev/null +++ b/scripts/verify-real-ghidra-6502.mjs @@ -0,0 +1,479 @@ +import assert from "node:assert/strict"; +import { execFile } from "node:child_process"; +import { createHash } from "node:crypto"; +import { + mkdir, + mkdtemp, + readFile, + readdir, + rm, + symlink, + unlink, + writeFile, +} from "node:fs/promises"; +import { join } from "node:path"; +import { fileURLToPath } from "node:url"; +import { promisify } from "node:util"; + +import { Client } from "@modelcontextprotocol/client"; +import { StdioClientTransport } from "@modelcontextprotocol/client/stdio"; +import { parseEvidence } from "../dist/domain/evidence.js"; +import { requireMcpOperationResult } from "./lib/mcp-verifier-results.mjs"; +import { createVerifierRun, completeVerifierRun } from "./lib/verifier-run.mjs"; + +if (process.argv.length !== 2) + throw new Error("Usage: node scripts/verify-real-ghidra-6502.mjs"); +if (process.env.GHIDRA_INSTALL_DIR === undefined) + throw new Error( + "Set GHIDRA_INSTALL_DIR to a bring-your-own Ghidra installation.", + ); +const exec = promisify(execFile); +const run = createVerifierRun(); +const root = fileURLToPath(new URL("..", import.meta.url)); +const ca65 = process.env.REA_CA65 ?? "ca65"; +const ld65 = process.env.REA_LD65 ?? "ld65"; +const toolchain = {}; +for (const [name, command] of Object.entries({ ca65, ld65 })) { + const version = await exec(command, ["--version"], { timeout: 10000 }); + toolchain[name] = (version.stdout + version.stderr).trim(); +} +await mkdir(join(root, "build"), { recursive: true }); +const workspace = await mkdtemp(join(root, "build", "rea-6502-proof-")); +const runtime = join(workspace, "runtime"); +await mkdir(runtime); +const path = join(workspace, "image.bin"); +const object = join(workspace, "image.o"); +const entrypoint = join(root, "scripts", "rea.mjs"); +const env = { + ...process.env, + TMPDIR: runtime, + TMP: runtime, + TEMP: runtime, + REA_LOG_LEVEL: "silent", + REA_ANALYSIS_PROVIDER: "ghidra", + REA_PROCESS_RUN_ID: run.run_id, +}; +const transport = new StdioClientTransport({ + command: process.execPath, + args: [entrypoint, "mcp"], + env, + stderr: "pipe", +}); +const client = new Client({ name: "rea-real-6502-proof", version: "1" }); +let stderr = ""; +transport.stderr?.on("data", (chunk) => { + stderr = (stderr + chunk.toString()).slice(-65536); +}); +let report; +try { + await exec( + ca65, + [join(root, "tests/conformance/ghidra/6502.s"), "-o", object], + { timeout: 10000 }, + ); + await exec( + ld65, + ["-C", join(root, "tests/conformance/ghidra/6502.cfg"), "-o", path, object], + { timeout: 10000 }, + ); + const bytes = await readFile(path); + // Independently known opcodes and link addresses from the source fixture. + assert.equal(bytes.toString("hex"), "a907201080c90ad003a92a60a9006018690360"); + const sha256 = createHash("sha256").update(bytes).digest("hex"); + await client.connect(transport); + const opened = await call("open_binary", { + path, + format: "raw-binary", + architecture: "6502", + entry_point: "0x008001", + provider_id: "ghidra", + }); + assert.equal(opened.architecture, "6502"); + assert.equal(opened.entry_point, "0x8001"); + const status = await call("binary_session"); + const image = evidence(await call("inspect_native_load_image"), sha256); + assert.equal( + image.normalized_result.status, + "verified", + JSON.stringify(image.normalized_result), + ); + assert.equal(image.normalized_result.architecture, "6502"); + assert.equal(image.normalized_result.base_address, "0x8001"); + assert.equal(image.normalized_result.language_id, "6502:LE:16:default"); + assert.equal(image.normalized_result.compiler_spec_id, "default"); + assert.equal(image.normalized_result.file_bytes, bytes.length); + const memory = evidence( + await call("read_bytes", { address: "0x8001", length: bytes.length }), + sha256, + ); + assert.equal(memory.normalized_result.bytes_hex, bytes.toString("hex")); + const procedures = evidence(await call("list_procedures"), sha256); + for (const address of ["0x8001", "0x8010"]) + assert.ok( + procedures.normalized_result.some( + (procedure) => procedure.address === address, + ), + ); + const instruction = evidence( + await call("inspect_native_instruction", { address: "0x8003" }), + sha256, + ); + assert.equal(instruction.normalized_result.mnemonic.toUpperCase(), "JSR"); + assert.ok( + instruction.normalized_result.references.some( + (reference) => reference.call && reference.target_address === "0x8010", + ), + ); + const pseudo = evidence( + await call("procedure_pseudo_code", { procedure: "0x8010" }), + sha256, + ); + assert.match(pseudo.normalized_result, /\+\s*(?:3|0x3|'\\x03')/u); + const filePseudo = evidence( + await call("procedure_pseudo_code", { procedure: "file-offset:0xf" }), + sha256, + ); + assert.equal(filePseudo.normalized_result, pseudo.normalized_result); + assert.ok( + filePseudo.locations.some( + (location) => location.kind === "file-offset" && location.offset === 15, + ), + ); + const rejected = await client.callTool({ + name: "open_binary", + arguments: { + path, + format: "raw-binary", + architecture: "6502", + entry_point: "0xffff", + }, + }); + assert.equal(rejected.isError, true); + assert.match(JSON.stringify(rejected.content), /6502 address width/u); + const preserved = await call("binary_session"); + assert.equal(preserved.analysis_run.run_id, status.analysis_run.run_id); + assert.equal(preserved.entry_point, "0x8001"); + await call("close_binary"); + await client.ping(); + const { stdout } = await exec( + process.execPath, + [ + entrypoint, + "inspect-native-load-image", + path, + "--target-format", + "raw-binary", + "--architecture", + "6502", + "--entry-point", + "0x8001", + "--provider", + "ghidra", + "--json", + ], + { env, timeout: 240000, maxBuffer: 16 * 1024 * 1024 }, + ); + const cli = parseEvidence(JSON.parse(stdout)); + assert.deepEqual(cli.normalized_result, image.normalized_result); + assert.equal(cli.analysis_profile.digest, image.analysis_profile.digest); + assert.equal(cli.subject.digest.sha256, sha256); + const fileCli = await exec( + process.execPath, + [ + entrypoint, + "decompile", + path, + "--file-offset=0xf", + "--target-format", + "raw-binary", + "--architecture", + "6502", + "--entry-point", + "0x8001", + "--provider", + "ghidra", + "--json", + ], + { env, timeout: 240000, maxBuffer: 16 * 1024 * 1024 }, + ); + const fileCliEvidence = parseEvidence(JSON.parse(fileCli.stdout)); + assert.equal(fileCliEvidence.normalized_result, filePseudo.normalized_result); + assert.deepEqual(fileCliEvidence.locations, filePseudo.locations); + assert.equal(fileCliEvidence.subject.digest.sha256, sha256); + assert.deepEqual(await readFile(path), bytes); + assert.deepEqual(await readdir(runtime), []); + const lowMemory = []; + // BinaryLoader maps file bytes before trying processor-defined blocks. Prove + // that overlapping zero-page/stack definitions never overlay the selected image. + for (const [entry, fullAddressSpace] of [ + [0xff, false], + [0, true], + ]) { + const lowPath = join(workspace, `low-${entry}.bin`); + await exec( + ld65, + [ + "-C", + join(root, "tests/conformance/ghidra/6502.cfg"), + "--start-addr", + String(entry), + "-o", + lowPath, + object, + ], + { timeout: 10000 }, + ); + const linked = await readFile(lowPath); + const expectedCode = Buffer.from( + "a907200000c90ad003a92a60a9006018690360", + "hex", + ); + expectedCode.writeUInt16LE(entry + 15, 3); + assert.deepEqual(linked, expectedCode); + const lowBytes = fullAddressSpace + ? Buffer.concat([linked, Buffer.alloc(65536 - linked.length)]) + : linked; + await writeFile(lowPath, lowBytes); + const lowSha256 = createHash("sha256").update(lowBytes).digest("hex"); + const lowEntry = `0x${entry.toString(16)}`; + const lowOpened = await call("open_binary", { + path: lowPath, + format: "raw-binary", + architecture: "6502", + entry_point: lowEntry, + provider_id: "ghidra", + }); + assert.equal(lowOpened.entry_point, lowEntry); + const lowImage = evidence( + await call("inspect_native_load_image"), + lowSha256, + ); + assert.equal( + lowImage.normalized_result.status, + "verified", + JSON.stringify(lowImage.normalized_result), + ); + assert.equal(lowImage.normalized_result.base_address, lowEntry); + assert.equal(lowImage.normalized_result.file_bytes, lowBytes.length); + const fileMappings = + lowImage.normalized_result.observations.mappings.filter( + (mapping) => mapping.source_file_index !== null, + ); + assert.equal(fileMappings.length, 1); + assert.equal( + fileMappings[0].address_space, + lowImage.normalized_result.observations.default_address_space, + ); + assert.equal(fileMappings[0].overlay, false); + assert.equal(fileMappings[0].start, lowEntry); + assert.equal(fileMappings[0].length, lowBytes.length); + const lowRead = evidence( + await call("read_bytes", { address: lowEntry, length: linked.length }), + lowSha256, + ); + assert.equal(lowRead.normalized_result.bytes_hex, linked.toString("hex")); + const lowCall = evidence( + await call("inspect_native_instruction", { + address: `0x${(entry + 2).toString(16)}`, + }), + lowSha256, + ); + assert.equal(lowCall.normalized_result.mnemonic.toUpperCase(), "JSR"); + assert.ok( + lowCall.normalized_result.references.some( + (reference) => + reference.call && + reference.target_address === `0x${(entry + 15).toString(16)}`, + ), + ); + await call("close_binary"); + const lowCli = await exec( + process.execPath, + [ + entrypoint, + "inspect-native-load-image", + lowPath, + "--target-format", + "raw-binary", + "--architecture", + "6502", + "--entry-point", + lowEntry, + "--provider", + "ghidra", + "--json", + ], + { env, timeout: 240000, maxBuffer: 16 * 1024 * 1024 }, + ); + const lowCliEvidence = evidence(JSON.parse(lowCli.stdout), lowSha256); + assert.deepEqual( + lowCliEvidence.normalized_result, + lowImage.normalized_result, + ); + assert.equal( + lowCliEvidence.analysis_profile.digest, + lowImage.analysis_profile.digest, + ); + assert.deepEqual(await readFile(lowPath), lowBytes); + assert.deepEqual(await readdir(runtime), []); + lowMemory.push({ + entry: lowEntry, + bytes: lowBytes.length, + sha256: lowSha256, + }); + } + let windowsReparseAdmission = "not-applicable"; + if (process.platform === "win32") { + const junction = join(workspace, "linked"); + await symlink(workspace, junction, "junction"); + try { + const linkedPath = join(junction, "image.bin"); + // Reopening the same bytes through a junction must replace the direct session. + await call("open_binary", { + path, + format: "raw-binary", + architecture: "6502", + entry_point: "0x8001", + provider_id: "ghidra", + }); + const directImage = evidence( + await call("inspect_native_load_image"), + sha256, + ); + assert.equal(directImage.normalized_result.status, "verified"); + // Canonical file identity must not bypass admission of the new path. + await call("open_binary", { + path: linkedPath, + format: "raw-binary", + architecture: "6502", + entry_point: "0x8001", + provider_id: "ghidra", + }); + // open_binary selects a lazy session; the first analysis admits the snapshot. + const linkedImage = await client.callTool( + { name: "inspect_native_load_image", arguments: {} }, + { timeout: 240000 }, + ); + assert.equal(linkedImage.isError, true); + assert.match( + JSON.stringify(linkedImage), + /Reparse path component is unsupported/u, + ); + await call("close_binary"); + const snapshotPath = join(workspace, "direct-snapshot.json"); + const cliArgs = (selectedPath) => [ + entrypoint, + "analyze", + selectedPath, + "--target-format", + "raw-binary", + "--architecture", + "6502", + "--entry-point", + "0x8001", + "--provider", + "ghidra", + "--snapshot", + snapshotPath, + "--json", + ]; + const saved = await exec(process.execPath, cliArgs(path), { + env, + timeout: 240000, + maxBuffer: 16 * 1024 * 1024, + }); + assert.equal( + evidence(JSON.parse(saved.stdout), sha256).operation, + "binary_overview", + ); + const savedSnapshot = JSON.parse(await readFile(snapshotPath, "utf8")); + assert.ok( + savedSnapshot.workflow_entries.some( + ({ operation }) => operation === "binary_overview", + ), + ); + assert.deepEqual(await readdir(runtime), []); + // Replaying a matching snapshot must still reach native path admission. + await assert.rejects( + exec(process.execPath, cliArgs(linkedPath), { + env, + timeout: 240000, + maxBuffer: 16 * 1024 * 1024, + }), + (cause) => { + assert.match( + cause.stdout + cause.stderr, + /Reparse path component is unsupported/u, + ); + return true; + }, + ); + assert.deepEqual(await readFile(path), bytes); + assert.deepEqual(await readdir(runtime), []); + windowsReparseAdmission = "cli-snapshot-mcp-rejected"; + } finally { + await unlink(junction); + } + } + report = { + ok: true, + mocked: false, + lane: "ghidra-6502", + host: process.platform, + toolchain, + provider: image.provider, + target: { + sha256, + bytes: bytes.length, + architecture: "6502", + entry: "0x8001", + }, + cli_mcp_load_image_parity: true, + instructions_and_call_reference: true, + decompilation: true, + file_offset_decompilation_cli_mcp: true, + invalid_mapping_preserves_session: true, + low_memory_cli_mcp_parity: lowMemory, + windows_reparse_admission: windowsReparseAdmission, + source_unchanged: true, + runtime_cleanup: "complete", + }; +} catch (cause) { + if (stderr) process.stderr.write(stderr); + throw cause; +} finally { + try { + await client.close(); + } finally { + await transport.close(); + } +} +const verifierRun = await completeVerifierRun(run); +if (process.platform !== "win32") { + assert.equal(verifierRun.process_lineage.status, "verified"); + assert.deepEqual(verifierRun.process_lineage.descendants, []); +} +// Keep the workspace for diagnosis unless verification and cleanup both pass. +await rm(workspace, { recursive: true, force: true }); +process.stdout.write( + JSON.stringify({ verifier_run: verifierRun, ...report }) + "\n", +); + +async function call(name, args = {}) { + const result = await client.callTool( + { name, arguments: args }, + { timeout: 240000 }, + ); + assert.notEqual(result.isError, true, JSON.stringify(result)); + if (["open_binary", "binary_session", "close_binary"].includes(name)) + return requireMcpOperationResult(result, name); + return result.structuredContent; +} + +function evidence(value, sha256) { + const result = parseEvidence(value); + assert.equal(result.subject.digest.sha256, sha256); + assert.equal(result.subject.architecture, "6502"); + assert.equal(result.subject.format, "raw-binary"); + return result; +} diff --git a/scripts/verify-real-ghidra-swift.mjs b/scripts/verify-real-ghidra-swift.mjs index 0593408b0..0aa58e2f0 100644 --- a/scripts/verify-real-ghidra-swift.mjs +++ b/scripts/verify-real-ghidra-swift.mjs @@ -45,8 +45,13 @@ const transport = new StdioClientTransport({ const source = fileURLToPath( new URL("../tests/conformance/swift/type-inventory.swift", import.meta.url), ); +const literalSource = fileURLToPath( + new URL("../tests/conformance/swift/string-literals.swift", import.meta.url), +); const binary = join(root, "swift-inventory"); +const literalBinary = join(root, "swift-literals"); const reports = []; +let literalReport; try { await execute("swiftc", ["-O", "-g", source, "-o", binary], { timeout: 60_000, @@ -54,6 +59,19 @@ try { await mkdir(join(root, "copied")); const copied = join(root, "copied", "swift-inventory"); await copyFile(binary, copied); + // Decoding covers AArch64 register pairs, so build that slice on any Mac host. + await execute( + "swiftc", + [ + "-O", + "-target", + "arm64-apple-macosx13.0", + literalSource, + "-o", + literalBinary, + ], + { timeout: 60_000 }, + ); await client.connect(transport); for (const [variant, path] of [ ["debug", binary], @@ -132,6 +150,7 @@ try { await call("close_binary", {}); } } + literalReport = await verifyStringLiterals(); } finally { await client.close(); await transport.close(); @@ -143,6 +162,7 @@ console.log( ok: true, provider: "ghidra", fixtures: reports, + swift_string_literals: literalReport, verifier_run: await completeVerifierRun(run), }, null, @@ -159,3 +179,51 @@ async function call(name, arguments_) { toolContract(name).outputSchema.parse(response.structuredContent); return requireMcpOperationResult(response, name); } + +// Literal text lives only in register constants, so it must come from the decoded pairs. +async function verifyStringLiterals() { + await call("open_binary", { path: literalBinary, provider_id: "ghidra" }); + try { + const procedures = await call("list_procedures", {}); + const expected = { + smallLiteral: [["SleepDisabled", "small"]], + largeLiteral: [["The power setting could not be changed.", "large"]], + unicodeLiterals: [ + ["café", "small"], + ["Résumé of the naïve décor, written twice", "large"], + ], + emptyAndControlLiterals: [ + ["", "small"], + ["tab\there", "small"], + ], + returnedLiteral: [["Holding awake", "small"]], + }; + const decoded = {}; + for (const [name, literals] of Object.entries(expected)) { + const procedure = procedures.find(({ value }) => value.includes(name)); + assert(procedure, `${name} is missing from the procedure inventory`); + const dossier = await call("analyze_function", { + procedure: procedure.address, + }); + const observed = dossier.swift_string_literals; + assert(Array.isArray(observed), `${name} recorded no Swift literals`); + assert.deepEqual(dossier.swift_string_literal_omissions, []); + assert.deepEqual( + observed.map(({ value, representation }) => [value, representation]), + literals, + ); + for (const literal of observed) { + assert.deepEqual(literal.registers, ["x0", "x1"]); + assert.equal( + literal.address === null, + literal.representation === "small", + `${literal.value} storage does not match its representation`, + ); + } + decoded[name] = observed.length; + } + return decoded; + } finally { + await call("close_binary", {}); + } +} diff --git a/scripts/verify-real-ida.mjs b/scripts/verify-real-ida.mjs index cedbf5076..bed0519a9 100644 --- a/scripts/verify-real-ida.mjs +++ b/scripts/verify-real-ida.mjs @@ -1,4 +1,8 @@ #!/usr/bin/env node +import { + assertFeatureTrace, + findMappedNonFunction, +} from "./lib/ida-feature-trace.mjs"; import { requireMcpToolError } from "./lib/mcp-verifier-results.mjs"; import assert from "node:assert/strict"; @@ -6,7 +10,7 @@ import { execFile } from "node:child_process"; import { createHash } from "node:crypto"; import { createReadStream } from "node:fs"; import { access, writeFile } from "node:fs/promises"; -import { dirname, join, resolve } from "node:path"; +import { basename, dirname, join, resolve } from "node:path"; import { fileURLToPath, pathToFileURL } from "node:url"; import { parseArgs, promisify } from "node:util"; import { Client } from "@modelcontextprotocol/client"; @@ -261,6 +265,61 @@ const mcp = async (cliDossier) => { break; } } + if (config.mode === "headless") { + const documents = observe(await call("list_documents", {})); + assert.equal(documents.length, 1); + assert.equal(basename(documents[0]), basename(target)); + assert.ok( + !workspaces.has(dirname(documents[0])), + "list_documents reported the private workspace copy.", + ); + const resolveAt = async (address) => + observe(await call("resolve_containing_procedure", { address })); + const instructionAddresses = dossier.assembly + .map((line) => /^(0x[0-9a-f]+)/iu.exec(line)?.[1]) + .filter((address) => address !== undefined); + const queries = [ + dossier.procedure.address, + ...instructionAddresses.slice(1, 2), + ]; + for (const queried of queries) { + const containing = await resolveAt(queried); + assert.equal(containing.found, true); + assert.equal(containing.procedure.address, dossier.procedure.address); + assert.equal(containing.procedure.name, dossier.procedure.name); + } + const unmapped = await resolveAt("0xffffffffffffff00"); + assert.equal(unmapped.found, false); + assert.equal(unmapped.reason, "outside_segments"); + const strings = observe(await call("list_strings", {})); + const mappedString = await findMappedNonFunction(strings, resolveAt); + if (mappedString === undefined) + report.unverified.push( + "Mapped non-function containment: no string of the target is mapped outside every function.", + ); + const context = await client.callTool({ + name: "inspect_address_context", + arguments: { address: dossier.procedure.address }, + }); + assert.notEqual(context.isError, true); + const contextResult = context.structuredContent.result; + assert.equal(contextResult.document, documents[0]); + assert.equal(contextResult.procedure.state, "available"); + assert.equal( + contextResult.procedure.value.procedure.address, + dossier.procedure.address, + ); + const traced = await client.callTool({ + name: "trace_feature", + arguments: { query: dossier.procedure.name }, + }); + assert.notEqual(traced.isError, true); + const trace = traced.structuredContent.normalized_result; + assertFeatureTrace(trace, dossier.procedure); + report.checks.push( + "headless containing procedures, the admitted program as the document, address context, and the procedure match and references of feature tracing", + ); + } if (values["expected-string"] === undefined) report.unverified.push( "String inventory/search correctness: no independent expected literal supplied.", diff --git a/scripts/verify-windows-native-artifact.mjs b/scripts/verify-windows-native-artifact.mjs index 1a6777322..4db66e0ee 100644 --- a/scripts/verify-windows-native-artifact.mjs +++ b/scripts/verify-windows-native-artifact.mjs @@ -27,9 +27,9 @@ if ( ); const offset = bytes.length >= 64 ? bytes.readUInt32LE(0x3c) : bytes.length; if ( - bytes.toString("ascii", 0, 2) !== "MZ" || + bytes.toString("latin1", 0, 2) !== "MZ" || offset > bytes.length - 26 || - bytes.toString("ascii", offset, offset + 4) !== "PE\0\0" || + bytes.toString("latin1", offset, offset + 4) !== "PE\0\0" || bytes.readUInt16LE(offset + 4) !== 0x8664 || bytes.readUInt16LE(offset + 24) !== 0x20b ) diff --git a/scripts/verify/firmware/analysis.mjs b/scripts/verify/firmware/analysis.mjs index 1f84681c9..804e8fffa 100644 --- a/scripts/verify/firmware/analysis.mjs +++ b/scripts/verify/firmware/analysis.mjs @@ -1,15 +1,8 @@ import assert from "node:assert/strict"; import { execFile } from "node:child_process"; -import { - access, - mkdir, - mkdtemp, - readFile, - rm, - symlink, -} from "node:fs/promises"; +import * as fs from "node:fs/promises"; import { tmpdir } from "node:os"; -import { dirname, join, resolve } from "node:path"; +import * as nodePath from "node:path"; import { fileURLToPath } from "node:url"; import { promisify } from "node:util"; import { Client } from "@modelcontextprotocol/client"; @@ -18,247 +11,348 @@ import { firmwareResultSchemas } from "../../../dist/domain/firmware/firmwareAna import { parseEvidence } from "../../../dist/domain/evidence.js"; import { hashFirmwareFile } from "../../../dist/firmware/FirmwareFiles.js"; -const exec = promisify(execFile); -const repository = resolve(dirname(fileURLToPath(import.meta.url)), "../../.."); -const entrypoint = resolve( +const execFileAsync = promisify(execFile); +const { join, resolve, dirname } = nodePath; + +const CALL_TIMEOUT_MS = 150_000; +const CONNECT_TIMEOUT_MS = 30_000; +const SEVENZIP_TIMEOUT_MS = 10_000; +const MAX_STDOUT_BYTES = 16 * 1024 * 1024; + +const REPO_ROOT = resolve(dirname(fileURLToPath(import.meta.url)), "../../.."); +const ENTRYPOINT = resolve( process.env.REA_FIRMWARE_TEST_ENTRYPOINT ?? - join(repository, "scripts/rea.mjs"), + join(REPO_ROOT, "scripts/rea.mjs"), ); -if (process.platform !== "linux") - throw new Error( - "verify:firmware requires Linux; other hosts are not verified", - ); -for (const variable of ["REA_BINWALK_COMMAND", "REA_UNBLOB_COMMAND"]) { - const path = process.env[variable]; - if (path === undefined) - throw new Error( - `verify:firmware requires ${variable}; see docs/firmware-analysis.md`, - ); - await access(path); -} -try { - await exec("7z", ["i"], { timeout: 10_000 }); -} catch (cause) { - throw new Error( - "verify:firmware prerequisite unavailable: Unblob's gzip/USTAR fixture extraction requires 7z on PATH", - { cause }, - ); -} -const fixtureRoot = resolve( +const FIXTURE_ROOT = resolve( process.env.REA_FIRMWARE_FIXTURE_ROOT ?? - join(repository, "_reference/firmware-integration/generated"), + join(REPO_ROOT, "_reference/firmware-integration/generated"), ); -await access(join(fixtureRoot, "oracle.json")).catch((cause) => { - throw new Error( - "verify:firmware requires generated fixtures; run npm run fixtures:firmware", - { cause }, - ); -}); + +// --------------------------------------------------------------------------- +// Prerequisite checks +// --------------------------------------------------------------------------- + +async function checkPrerequisites() { + if (process.platform !== "linux") { + throw new Error( + "verify:firmware requires Linux; other hosts are not verified", + ); + } + + for (const name of ["REA_BINWALK_COMMAND", "REA_UNBLOB_COMMAND"]) { + const value = process.env[name]; + if (value === undefined) { + throw new Error( + `verify:firmware requires ${name}; see docs/firmware-analysis.md`, + ); + } + await fs.access(value); + } + + try { + await execFileAsync("7z", ["i"], { timeout: SEVENZIP_TIMEOUT_MS }); + } catch (cause) { + throw new Error( + "verify:firmware prerequisite unavailable: Unblob's gzip/USTAR fixture extraction requires 7z on PATH", + { cause }, + ); + } + + try { + await fs.access(join(FIXTURE_ROOT, "oracle.json")); + } catch (cause) { + throw new Error( + "verify:firmware requires generated fixtures; run npm run fixtures:firmware", + { cause }, + ); + } +} + +await checkPrerequisites(); + +// --------------------------------------------------------------------------- +// Shared state +// --------------------------------------------------------------------------- + const oracle = JSON.parse( - await readFile(join(fixtureRoot, "oracle.json"), "utf8"), + await fs.readFile(join(FIXTURE_ROOT, "oracle.json"), "utf8"), ); -const path = join(fixtureRoot, "firmware.bin"); -assert.equal(await hashFirmwareFile(path), oracle.sha256); -const root = await mkdtemp(join(tmpdir(), "rea-real-firmware-")); -const environment = Object.fromEntries( - Object.entries(process.env).filter(([, value]) => value !== undefined), +const firmwarePath = join(FIXTURE_ROOT, "firmware.bin"); +assert.equal(await hashFirmwareFile(firmwarePath), oracle.sha256); + +const workDir = await fs.mkdtemp(join(tmpdir(), "rea-real-firmware-")); +const baseEnv = Object.fromEntries( + Object.entries(process.env).filter(([, v]) => v !== undefined), ); -const client = new Client({ name: "rea-real-firmware-verifier", version: "1" }); -const cli = async (args, env = environment) => { - const response = await exec( +const mcpClient = new Client({ + name: "rea-real-firmware-verifier", + version: "1", +}); + +// --------------------------------------------------------------------------- +// Invocation helpers +// --------------------------------------------------------------------------- + +async function runCli(args, env = baseEnv) { + const { stdout } = await execFileAsync( process.execPath, - [entrypoint, ...args, "--format", "json"], + [ENTRYPOINT, ...args, "--format", "json"], { env, - cwd: repository, - timeout: 150_000, - maxBuffer: 16 * 1024 * 1024, + cwd: REPO_ROOT, + timeout: CALL_TIMEOUT_MS, + maxBuffer: MAX_STDOUT_BYTES, }, ); - return parseEvidence(JSON.parse(response.stdout)); -}; -const call = async (name, args) => { - const response = await client.callTool( + return parseEvidence(JSON.parse(stdout)); +} + +async function callTool(name, args) { + const result = await mcpClient.callTool( { name, arguments: args }, - { timeout: 150_000 }, + { timeout: CALL_TIMEOUT_MS }, ); - assert.notEqual(response.isError, true, JSON.stringify(response)); - assert.ok(response.structuredContent !== undefined); - return response.structuredContent; -}; -const verifyRegions = (evidence) => { + assert.notEqual(result.isError, true, JSON.stringify(result)); + assert.ok(result.structuredContent !== undefined); + return result.structuredContent; +} + +// --------------------------------------------------------------------------- +// Result verifiers +// --------------------------------------------------------------------------- + +function verifyRegions(evidence) { assert.equal(evidence.subject.digest.sha256, oracle.sha256); - const result = firmwareResultSchemas.inspect_firmware_regions.parse( + + const parsed = firmwareResultSchemas.inspect_firmware_regions.parse( evidence.normalized_result, ); - const gzip = result.regions.find( - (region) => region.signature === "gzip" && region.offset === oracle.offset, + + const gzipRegion = parsed.regions.find( + (r) => r.signature === "gzip" && r.offset === oracle.offset, ); assert.ok( - gzip !== undefined, + gzipRegion !== undefined, "Binwalk must find the independently generated gzip offset", ); - assert.equal(gzip.reported_size, oracle.length); - assert.equal(gzip.size_basis, "provider_reported_validation_unknown"); - return result.regions.map( - ({ provider_id: _provider_id, ...region }) => region, - ); -}; -const verifyFiles = async (evidence) => { - const result = firmwareResultSchemas.extract_firmware.parse( + assert.equal(gzipRegion.reported_size, oracle.length); + assert.equal(gzipRegion.size_basis, "provider_reported_validation_unknown"); + + // Strip provider-specific ids so CLI and MCP results compare equal. + return parsed.regions.map((region) => { + const { provider_id: _ignored, ...rest } = region; + return rest; + }); +} + +async function verifyFiles(evidence) { + const parsed = firmwareResultSchemas.extract_firmware.parse( evidence.normalized_result, ); - assert.ok(result.chunks.some((chunk) => chunk.handler === "gzip")); + + assert.ok(parsed.chunks.some((c) => c.handler === "gzip")); + for (const [suffix, expected] of Object.entries(oracle.files)) { - const file = result.files.find((candidate) => - candidate.relative_path.endsWith(`/${suffix}`), + const entry = parsed.files.find((f) => + f.relative_path.endsWith(`/${suffix}`), ); - assert.ok(file !== undefined, `Missing extracted ${suffix}`); - assert.equal(file.sha256, expected.sha256); - assert.equal(file.size, expected.size); - assert.equal(await hashFirmwareFile(file.path), expected.sha256); - assert.equal(file.original_file_range, null); - assert.equal(file.runtime_address, null); + assert.ok(entry !== undefined, `Missing extracted ${suffix}`); + assert.equal(entry.sha256, expected.sha256); + assert.equal(entry.size, expected.size); + assert.equal(await hashFirmwareFile(entry.path), expected.sha256); + assert.equal(entry.original_file_range, null); + assert.equal(entry.runtime_address, null); } - return result; -}; -try { - const inspected = await cli(["inspect-firmware-regions", path]); - const cliRegions = verifyRegions(inspected); - const extracted = await cli([ - "extract-firmware", - path, - join(root, "cli-output"), + + return parsed; +} + +const toContentIdentity = (parsed) => + parsed.files.map(({ relative_path, sha256, size }) => ({ + relative_path, + sha256, + size, + })); + +// --------------------------------------------------------------------------- +// Verification stages +// --------------------------------------------------------------------------- + +async function stageCli() { + const regionsEvidence = await runCli([ + "inspect-firmware-regions", + firmwarePath, ]); - const cliFiles = await verifyFiles(extracted); - assert.equal(cliFiles.coverage, "partial"); + const regions = verifyRegions(regionsEvidence); + + const extractEvidence = await runCli([ + "extract-firmware", + firmwarePath, + join(workDir, "cli-output"), + ]); + const files = await verifyFiles(extractEvidence); + + assert.equal(files.coverage, "partial"); assert.ok( - cliFiles.chunks.some( - (chunk) => - chunk.handler === null && - chunk.range.offset === 0 && - chunk.range.length === 64, + files.chunks.some( + (c) => + c.handler === null && c.range.offset === 0 && c.range.length === 64, ), ); - await client.connect( - new StdioClientTransport({ - command: process.execPath, - args: [entrypoint, "mcp"], - cwd: repository, - env: environment, - stderr: "pipe", - }), - { timeout: 30_000 }, - ); - const inspectedMcp = await call("inspect_firmware_regions", { path }); - const mcpEvidence = parseEvidence(inspectedMcp); - assert.deepEqual(verifyRegions(mcpEvidence), cliRegions); - const extractedMcp = await call("extract_firmware", { - path, - output_directory: join(root, "mcp-output"), + + return { regions, files }; +} + +async function connectMcp() { + const transport = new StdioClientTransport({ + command: process.execPath, + args: [ENTRYPOINT, "mcp"], + cwd: REPO_ROOT, + env: baseEnv, + stderr: "pipe", }); - const mcpFiles = await verifyFiles(parseEvidence(extractedMcp)); - const contentIdentity = (result) => - result.files.map(({ relative_path, sha256, size }) => ({ - relative_path, - sha256, - size, - })); - assert.deepEqual(contentIdentity(mcpFiles), contentIdentity(cliFiles)); - const selected = await call("extract_firmware", { - path, - output_directory: join(root, "selected-output"), + await mcpClient.connect(transport, { timeout: CONNECT_TIMEOUT_MS }); +} + +async function stageMcp(cli) { + const regionsRaw = await callTool("inspect_firmware_regions", { + path: firmwarePath, + }); + assert.deepEqual(verifyRegions(parseEvidence(regionsRaw)), cli.regions); + + const extractRaw = await callTool("extract_firmware", { + path: firmwarePath, + output_directory: join(workDir, "mcp-output"), + }); + const mcpFiles = await verifyFiles(parseEvidence(extractRaw)); + assert.deepEqual(toContentIdentity(mcpFiles), toContentIdentity(cli.files)); + + const rangedRaw = await callTool("extract_firmware", { + path: firmwarePath, + output_directory: join(workDir, "selected-output"), range: { offset: oracle.offset, length: oracle.length }, }); - const selectedFiles = await verifyFiles(parseEvidence(selected)); - assert.equal(selectedFiles.selection.offset, oracle.offset); - assert.equal(selectedFiles.selection.length, oracle.length); - const depth = await call("extract_firmware", { - path, - output_directory: join(root, "depth-output"), + const ranged = await verifyFiles(parseEvidence(rangedRaw)); + assert.equal(ranged.selection.offset, oracle.offset); + assert.equal(ranged.selection.length, oracle.length); + + const depthRaw = await callTool("extract_firmware", { + path: firmwarePath, + output_directory: join(workDir, "depth-output"), max_depth: 1, }); - const depthResult = firmwareResultSchemas.extract_firmware.parse( - depth.normalized_result, + const depthParsed = firmwareResultSchemas.extract_firmware.parse( + depthRaw.normalized_result, ); - assert.equal(depthResult.coverage, "partial"); - assert.ok(depthResult.depth_limited_paths.length > 0); + assert.equal(depthParsed.coverage, "partial"); + assert.ok(depthParsed.depth_limited_paths.length > 0); + console.log( "PASS real Binwalk/Unblob: CLI, MCP, independent offsets/digests, range extraction, unknown chunks and depth limits", ); - if (process.env.REA_FIRMWARE_VERIFY_EXT4 === "1") { - const ext4 = join(fixtureRoot, "ext4.bin"); - await access(ext4); - const response = await call("extract_firmware", { - path: ext4, - output_directory: join(root, "ext4-output"), - }); - const result = firmwareResultSchemas.extract_firmware.parse( - response.normalized_result, + + return mcpFiles; +} + +async function stageExt4() { + const ext4Path = join(FIXTURE_ROOT, "ext4.bin"); + await fs.access(ext4Path); + + const raw = await callTool("extract_firmware", { + path: ext4Path, + output_directory: join(workDir, "ext4-output"), + }); + const parsed = firmwareResultSchemas.extract_firmware.parse( + raw.normalized_result, + ); + assert.ok(parsed.chunks.some((c) => c.handler === "extfs")); + assert.equal(parsed.coverage, "complete"); + + for (const [suffix, expected] of Object.entries(oracle.files)) { + const match = parsed.files.find((f) => + f.relative_path.endsWith(`/${suffix}`), ); - assert.ok(result.chunks.some((chunk) => chunk.handler === "extfs")); - assert.equal(result.coverage, "complete"); - for (const [suffix, expected] of Object.entries(oracle.files)) - assert.equal( - result.files.find((file) => file.relative_path.endsWith(`/${suffix}`)) - ?.sha256, - expected.sha256, - ); - console.log("PASS real ext4 extraction with debugfs; no mounts"); - // Preserve the process-ownership utility while withholding all extractors. - const withoutExtractors = join(root, "without-extractors"); - await access("/bin/ps"); - await mkdir(withoutExtractors); - await symlink("/bin/ps", join(withoutExtractors, "ps")); - const missing = await cli( - ["extract-firmware", ext4, join(root, "missing-extractor-output")], - { ...environment, PATH: withoutExtractors }, - ); - const partial = firmwareResultSchemas.extract_firmware.parse( - missing.normalized_result, - ); - assert.equal(partial.coverage, "partial"); - assert.ok(JSON.stringify(partial.diagnostics).includes("debugfs")); - console.log( - "PASS real missing-extractor diagnostics retained despite Unblob exit 1", - ); - } - if (process.env.REA_FIRMWARE_VERIFY_GHIDRA === "1") { - if (process.env.GHIDRA_INSTALL_DIR === undefined) - throw new Error( - "Firmware Ghidra handoff requires GHIDRA_INSTALL_DIR and Java; see docs/ghidra.md", - ); - const child = mcpFiles.files.find((file) => - file.relative_path.endsWith("/usr/bin/probe"), - ); - assert.ok(child !== undefined); - await call("open_binary", { path: child.path, provider_id: "ghidra" }); - const dossier = await call("analyze_function", { - procedure: "firmware_probe", - }); - const evidence = parseEvidence(dossier); - assert.equal( - evidence.subject.digest.sha256, - oracle.files["usr/bin/probe"].sha256, - ); - const pseudocode = await call("procedure_pseudo_code", { - procedure: "firmware_probe", - }); - assert.equal(parseEvidence(pseudocode).provider.id, "ghidra"); - assert.ok( - JSON.stringify(evidence.normalized_result).includes("firmware_probe"), - ); - await call("close_binary", {}); - console.log( - "PASS real Ghidra handoff of the selected extracted host ELF, bound to its digest", + assert.equal(match?.sha256, expected.sha256); + } + console.log("PASS real ext4 extraction with debugfs; no mounts"); + + // Keep the process-ownership utility available but hide every extractor. + const emptyBin = join(workDir, "without-extractors"); + await fs.access("/bin/ps"); + await fs.mkdir(emptyBin); + await fs.symlink("/bin/ps", join(emptyBin, "ps")); + + const degraded = await runCli( + ["extract-firmware", ext4Path, join(workDir, "missing-extractor-output")], + { ...baseEnv, PATH: emptyBin }, + ); + const degradedParsed = firmwareResultSchemas.extract_firmware.parse( + degraded.normalized_result, + ); + assert.equal(degradedParsed.coverage, "partial"); + assert.ok(JSON.stringify(degradedParsed.diagnostics).includes("debugfs")); + console.log( + "PASS real missing-extractor diagnostics retained despite Unblob exit 1", + ); +} + +async function stageGhidra(mcpFiles) { + if (process.env.GHIDRA_INSTALL_DIR === undefined) { + throw new Error( + "Firmware Ghidra handoff requires GHIDRA_INSTALL_DIR and Java; see docs/ghidra.md", ); } + + const probe = mcpFiles.files.find((f) => + f.relative_path.endsWith("/usr/bin/probe"), + ); + assert.ok(probe !== undefined); + + await callTool("open_binary", { path: probe.path, provider_id: "ghidra" }); + + const dossierRaw = await callTool("analyze_function", { + procedure: "firmware_probe", + }); + const dossier = parseEvidence(dossierRaw); assert.equal( - await hashFirmwareFile(path), + dossier.subject.digest.sha256, + oracle.files["usr/bin/probe"].sha256, + ); + + const pseudoRaw = await callTool("procedure_pseudo_code", { + procedure: "firmware_probe", + }); + assert.equal(parseEvidence(pseudoRaw).provider.id, "ghidra"); + assert.ok( + JSON.stringify(dossier.normalized_result).includes("firmware_probe"), + ); + + await callTool("close_binary", {}); + console.log( + "PASS real Ghidra handoff of the selected extracted host ELF, bound to its digest", + ); +} + +// --------------------------------------------------------------------------- +// Entry point +// --------------------------------------------------------------------------- + +try { + const cli = await stageCli(); + await connectMcp(); + const mcpFiles = await stageMcp(cli); + + if (process.env.REA_FIRMWARE_VERIFY_EXT4 === "1") await stageExt4(); + if (process.env.REA_FIRMWARE_VERIFY_GHIDRA === "1") + await stageGhidra(mcpFiles); + + assert.equal( + await hashFirmwareFile(firmwarePath), oracle.sha256, "Original firmware bytes must remain unchanged", ); } finally { - await client.close(); - await rm(root, { recursive: true, force: true }); + await mcpClient.close(); + await fs.rm(workDir, { recursive: true, force: true }); } diff --git a/src/adb/AdbProvider.test.ts b/src/adb/AdbProvider.test.ts index ae81353c7..23645e456 100644 --- a/src/adb/AdbProvider.test.ts +++ b/src/adb/AdbProvider.test.ts @@ -810,6 +810,36 @@ exit 1`, expect(result.error.message).toContain("overwrite"); }); +it.each([ + ["offline", false, "error: device offline"], + ["unauthorized", false, "error: device unauthorized"], + ["offline", true, "error: device offline"], +] as const)( + "reports the %s device state when the push existence probe cannot run (overwrite %s)", + async (state, overwrite, stderr) => { + const { binary, root } = await writeAdbStub( + (stubRoot) => `if [ "$1" = "version" ]; then ${VERSION_BLOCK}; exit 0; fi +if [ "$1" = "-s" ]; then printf '%s\\n' "$3" >> "${stubRoot}/calls"; printf '%s\\n' "${stderr}" >&2; exit 1; fi`, + ); + const localFile = join(root, "payload.bin"); + await writeFile(localFile, "x"); + const projected = await refusedBy(providerOver(binary), { + operation: "push_adb_file", + input: { + serial: "emulator-5554", + local_path: localFile, + device_path: "/data/local/tmp/payload.bin", + overwrite, + }, + }); + // Existence is unknown, so nothing is claimed about the path and no push runs. + expect(projected).toMatchObject({ code: "unsupported_target" }); + expect(projected.message).toContain(state); + expect(projected.message).not.toContain("already exists"); + expect(await readFile(join(root, "calls"), "utf8")).toBe("shell\n"); + }, +); + it("reports a local push source that does not exist", async () => { const { binary } = await writeAdbStub( () => `if [ "$1" = "version" ]; then ${VERSION_BLOCK}; exit 0; fi diff --git a/src/adb/AdbProvider.ts b/src/adb/AdbProvider.ts index b9af6d14f..d26345eba 100644 --- a/src/adb/AdbProvider.ts +++ b/src/adb/AdbProvider.ts @@ -1623,17 +1623,15 @@ export class AdbProvider { { timeoutMs: SETTINGS_TIMEOUT_MS, maxOutputBytes: 1024 }, signal, ); - let deviceExists: boolean; - if (existsProbe.ok) deviceExists = true; - else if (existsProbe.error instanceof AnalysisProtocolError) - deviceExists = false; - else if (existsProbe.error instanceof AnalysisTimeoutError) + // `test -e` exits nonzero with no adb diagnosis when the path is absent. + // Any other probe failure leaves existence unknown, so it is reported as + // is rather than read as an existing path or followed by a push. + if ( + !existsProbe.ok && + !(existsProbe.error instanceof AnalysisProtocolError) + ) return existsProbe; - else if (existsProbe.error instanceof AnalysisCancelledError) - return existsProbe; - else if (existsProbe.error instanceof AnalysisResourceConstraintError) - return existsProbe; - else deviceExists = true; // authorization/refusal states surface on push instead + const deviceExists = existsProbe.ok; if (deviceExists && !input.overwrite) return err( new AnalysisOutputError( diff --git a/src/application/AnalysisProvider.ts b/src/application/AnalysisProvider.ts index f3b51257a..376036fa1 100644 --- a/src/application/AnalysisProvider.ts +++ b/src/application/AnalysisProvider.ts @@ -50,6 +50,13 @@ export interface AnalysisExecution { readonly limitations: readonly string[]; /** Provider-owned completeness assessment for caller/callee observations. */ readonly relationshipCoverage?: RelationshipCoverage; + /** + * Whether the provider capped an `xrefs` read: true when it reported a + * truncation, false when it explicitly reported none. Absent when it reports + * nothing, which is not a completeness claim. Never part of Evidence or tool + * output. + */ + readonly referencesTruncated?: boolean; readonly locations: readonly EvidenceLocation[]; readonly subject: EvidenceSubjectTarget | null; } @@ -59,6 +66,7 @@ interface CreateAnalysisExecutionOptions { readonly analysisProfile?: AnalysisProfileCommitment; readonly limitations?: readonly string[]; readonly relationshipCoverage?: RelationshipCoverage; + readonly referencesTruncated?: boolean; readonly locations?: readonly EvidenceLocation[]; readonly subject?: EvidenceSubjectTarget; } @@ -103,6 +111,9 @@ export function createAnalysisExecution( options.relationshipCoverage, ), }), + ...(options.referencesTruncated === undefined + ? {} + : { referencesTruncated: options.referencesTruncated }), locations: [...(options.locations ?? [])], subject: options.subject ?? null, }; diff --git a/src/application/BinaryTargetResolver.ts b/src/application/BinaryTargetResolver.ts index ac691b55e..6f1bc4366 100644 --- a/src/application/BinaryTargetResolver.ts +++ b/src/application/BinaryTargetResolver.ts @@ -76,6 +76,8 @@ export const parseBinaryTarget = async ( await access(candidate, constants.R_OK); throwIfTargetResolutionCancelled(signal); const canonical = await realpath(candidate); + // Windows native admission must inspect the caller-selected reparse components. + const sourcePath = process.platform === "win32" ? candidate : canonical; throwIfTargetResolutionCancelled(signal); const resolved = await resolveAppBundle(canonical); throwIfTargetResolutionCancelled(signal); @@ -113,7 +115,7 @@ export const parseBinaryTarget = async ( ); return ok({ path, - sourcePath: canonical, + sourcePath, sha256: await sha256Handle(handle, path, metadata, signal), kind: "executable", format: "raw-binary", @@ -152,7 +154,7 @@ export const parseBinaryTarget = async ( ); return ok({ path, - sourcePath: canonical, + sourcePath, sha256: await sha256Handle(handle, path, metadata, signal), kind: "executable", format: "dos-com", @@ -166,7 +168,7 @@ export const parseBinaryTarget = async ( ) return ok({ path, - sourcePath: process.platform === "win32" ? candidate : canonical, + sourcePath, sha256: await sha256Handle(handle, path, metadata, signal), kind: "database", format: "analysis-database", @@ -175,7 +177,7 @@ export const parseBinaryTarget = async ( if (artifactFormat !== undefined) { const identity = { path, - sourcePath: process.platform === "win32" ? candidate : canonical, + sourcePath, sha256: await sha256Handle(handle, path, metadata, signal), }; return isArchiveFormat(artifactFormat) @@ -195,7 +197,7 @@ export const parseBinaryTarget = async ( ); return ok({ path, - sourcePath: process.platform === "win32" ? candidate : canonical, + sourcePath, ...bundle, sha256: await sha256Handle(handle, path, metadata, signal), kind: "executable", diff --git a/src/application/CapabilityInventory.test.ts b/src/application/CapabilityInventory.test.ts index 1b8ec6411..4286ff1d0 100644 --- a/src/application/CapabilityInventory.test.ts +++ b/src/application/CapabilityInventory.test.ts @@ -497,3 +497,71 @@ it("reports host denial before target mismatch but keeps composed workflow prere ).reason, ).toBe("target_required"); }); + +describe("trace_feature requirements", () => { + const capabilities = (operations: readonly string[]) => + operations.map((operation) => ({ + operation, + available: true, + reason: null, + })); + + it("needs the search operations the workflow actually calls", () => { + expect( + entry( + "trace_feature", + status({ + open: true, + kind: "executable", + capabilities: capabilities([ + "search_strings", + "search_procedures", + "xrefs", + "resolve_containing_procedure", + ]), + }), + ), + ).toMatchObject({ available: true, reason: "available" }); + }); + + it.each(["search_strings", "search_procedures"])( + "is unavailable without %s", + (missing) => { + const operations = [ + "search_strings", + "search_procedures", + "xrefs", + "resolve_containing_procedure", + ].filter((operation) => operation !== missing); + + expect( + entry( + "trace_feature", + status({ + open: true, + kind: "executable", + capabilities: capabilities(operations), + }), + ), + ).toMatchObject({ available: false, reason: "provider_missing" }); + }, + ); + + it("is unavailable when only the list operations exist", () => { + expect( + entry( + "trace_feature", + status({ + open: true, + kind: "executable", + capabilities: capabilities([ + "list_strings", + "list_procedures", + "xrefs", + "resolve_containing_procedure", + ]), + }), + ), + ).toMatchObject({ available: false, reason: "provider_missing" }); + }); +}); diff --git a/src/application/CapabilityInventory.ts b/src/application/CapabilityInventory.ts index 6fefb8e20..f4584eb29 100644 --- a/src/application/CapabilityInventory.ts +++ b/src/application/CapabilityInventory.ts @@ -121,8 +121,8 @@ const ENHANCED_REQUIREMENTS: Readonly> = { analyze_function: ["analyze_function"], inspect_native_api: ["analyze_function"], trace_feature: [ - "list_strings", - "list_procedures", + "search_strings", + "search_procedures", "xrefs", "resolve_containing_procedure", ], @@ -141,6 +141,7 @@ const FRIDA_OPERATION_NAMES: ReadonlySet = new Set( /** Native contracts defined only for an active Mach-O executable. */ const MACHO_TARGET_OPERATIONS: ReadonlySet = new Set([ + "inspect_signature", "inspect_macho", "list_architectures", "observe_native_calls", diff --git a/src/application/ClientConfigurationDocument.ts b/src/application/ClientConfigurationDocument.ts index b32df3fd0..1b241a98b 100644 --- a/src/application/ClientConfigurationDocument.ts +++ b/src/application/ClientConfigurationDocument.ts @@ -1,9 +1,5 @@ import { parse as parseToml, stringify as stringifyToml } from "smol-toml"; -import { - parse as parseYaml, - parseDocument as parseYamlDocument, - stringify as stringifyYaml, -} from "yaml"; +import { stringify as stringifyYaml } from "yaml"; import { isDeepStrictEqual } from "node:util"; import { @@ -12,6 +8,10 @@ import { editDshDocument, dshClientRegistrationEntry, } from "./DshCordisPatch.js"; +import { + editHermesConfigurationYaml, + parseHermesConfigurationYaml, +} from "./HermesConfigurationYaml.js"; import { applyEdits, findNodeAtLocation, @@ -118,7 +118,10 @@ const parseDocument = ( if (isEmptyClientConfigurationText(text)) return {}; // Hermes stores MCP servers in `mcp_servers` inside a YAML document. if (format === "hermes") - return requireObject(parseYaml(text), "The configuration root"); + return requireObject( + parseHermesConfigurationYaml(text), + "The configuration root", + ); // ZCode loads nonempty native files with strict JSON.parse. if (format === "zcode") return requireObject(JSON.parse(text), "The configuration root"); @@ -1419,12 +1422,11 @@ const serializeHermesConfiguration = ( typeof servers === "object" && servers !== null ? (servers as Record)[PRODUCT_IDENTITY.mcpServerKey] : undefined; - const yamlDocument = parseYamlDocument(originalText); - if (entry === undefined) - yamlDocument.deleteIn([serversKey, PRODUCT_IDENTITY.mcpServerKey]); - else yamlDocument.setIn([serversKey, PRODUCT_IDENTITY.mcpServerKey], entry); - // Editing REA must not reflow unrelated user prompts or other YAML scalars. - return yamlDocument.toString({ lineWidth: 0 }); + return editHermesConfigurationYaml( + originalText, + [serversKey, PRODUCT_IDENTITY.mcpServerKey], + entry, + ); }; /** Build the stdio entry shape expected by one client's configuration dialect. */ diff --git a/src/application/DirectAnalysis.ts b/src/application/DirectAnalysis.ts index c4d130f63..0bdb69864 100644 --- a/src/application/DirectAnalysis.ts +++ b/src/application/DirectAnalysis.ts @@ -23,6 +23,7 @@ import { projectAnalysisError } from "../domain/analysisErrorProjection.js"; import type { AnalysisError } from "../domain/analysisErrorBase.js"; import { access } from "node:fs/promises"; import { readAnalysisSnapshot } from "./binary/AnalysisSnapshotFiles.js"; +import { requiresNativeSourceAdmissionBeforeReplay } from "./binary/AnalysisSnapshotCache.js"; import { parseBinaryTarget } from "./BinaryTargetResolver.js"; import type { BinaryTarget } from "../domain/binaryTargetTypes.js"; import { @@ -254,9 +255,14 @@ const runAnalysis = async ( const route = preview.value.route; const bindingProfile = route.profile ?? undefined; const evidenceProfile = analysisProfileForRoute(route, tool); + // A parsed Windows path does not prove Ghidra's native snapshot owner + // admitted the selected source; CLI replay must wait for that client. if ( bindingProfile !== undefined && evidenceProfile !== undefined && + !requiresNativeSourceAdmissionBeforeReplay( + route.binding?.identity.id, + ) && allowsSnapshotReplay(route, tool) ) { const cached = snapshotEvidenceForQuery(snapshot, { diff --git a/src/application/EnhancedLiteralTracing.ts b/src/application/EnhancedLiteralTracing.ts index 9c86781bc..79684ce0f 100644 --- a/src/application/EnhancedLiteralTracing.ts +++ b/src/application/EnhancedLiteralTracing.ts @@ -1,5 +1,8 @@ import { z } from "zod"; -import type { AnalysisOperation } from "./AnalysisProvider.js"; +import type { + AnalysisExecution, + AnalysisOperation, +} from "./AnalysisProvider.js"; import { AnalysisOutputError } from "../domain/analysisErrorCore.js"; import type { AnalysisError } from "../domain/analysisErrorBase.js"; import type { JsonValue } from "../domain/jsonValue.js"; @@ -16,6 +19,12 @@ type AnalysisCall = ( signal?: AbortSignal, ) => Promise>; +type ObserveCall = ( + name: AnalysisOperation, + arguments_: Readonly>, + signal?: AbortSignal, +) => Promise>; + type LiteralSearchSource = readonly [ "search_strings" | "search_procedures", "string" | "procedure", @@ -39,6 +48,7 @@ export interface LiteralTraceInput { /** Trace matching strings and procedures through provider xrefs. */ export const traceLiteralFeature = async ( call: AnalysisCall, + observe: ObserveCall, input: LiteralTraceInput, signal?: AbortSignal, ): EnhancedResult => { @@ -49,7 +59,7 @@ export const traceLiteralFeature = async ( signal, ); if (!searched.ok) return searched; - const traced = await traceReferences(call, { + const traced = await traceReferences(call, observe, { matches: searched.value.matches, ...(signal === undefined ? {} : { signal }), }); @@ -59,8 +69,8 @@ export const traceLiteralFeature = async ( search_mode: "literal", matches: searched.value.matches, references: traced.value.references, - truncated: false, - residual_unknowns: [], + truncated: traced.value.residual.length > 0, + residual_unknowns: traced.value.residual, }); }; @@ -126,24 +136,39 @@ const addPageMatches = (input: { const traceReferences = async ( call: AnalysisCall, + observe: ObserveCall, request: ReferenceTraceRequest, -): Promise> => { +): Promise< + Result<{ references: TraceReference[]; residual: string[] }, AnalysisError> +> => { const references: TraceReference[] = []; + const residual: string[] = []; + // A source address is resolved once however many matches reference it. + const containing = new Map(); for (const match of request.matches) { - const xrefs = await call( + const observed = await observe( "xrefs", { address: match.address }, request.signal, ); - if (!xrefs.ok) return xrefs; - if (!Array.isArray(xrefs.value)) + if (!observed.ok) return observed; + // Only an explicit producer report of truncation is a coverage gap; a + // provider that reports nothing keeps its previous, unqualified result. + const unknown = `References to ${match.address} were truncated by the provider; additional references may exist.`; + if ( + observed.value.referencesTruncated === true && + !residual.includes(unknown) + ) + residual.push(unknown); + const sources = observed.value.result; + if (!Array.isArray(sources)) return err( new AnalysisOutputError( "xrefs", "provider returned a non-array result", ), ); - for (const source of xrefs.value) { + for (const source of sources) { if (typeof source !== "string") return err( new AnalysisOutputError( @@ -151,18 +176,23 @@ const traceReferences = async ( "provider returned a non-address value", ), ); - const resolved = await call( - "resolve_containing_procedure", - { address: source }, - request.signal, - ); - if (!resolved.ok) return resolved; + let containingProcedure = containing.get(source); + if (containingProcedure === undefined) { + const resolved = await call( + "resolve_containing_procedure", + { address: source }, + request.signal, + ); + if (!resolved.ok) return resolved; + containingProcedure = resolved.value; + containing.set(source, containingProcedure); + } references.push({ target_address: match.address, source_address: source, - containing_procedure: resolved.value, + containing_procedure: containingProcedure, }); } } - return ok({ references }); + return ok({ references, residual }); }; diff --git a/src/application/EnhancedTools.ts b/src/application/EnhancedTools.ts index 87d27fc5f..207186ccc 100644 --- a/src/application/EnhancedTools.ts +++ b/src/application/EnhancedTools.ts @@ -163,6 +163,8 @@ export class EnhancedTools { return traceLiteralFeature( (name, arguments_, operationSignal) => this.#call(name, arguments_, operationSignal), + (name, arguments_, operationSignal) => + this.#observe(name, arguments_, operationSignal), call.input, signal, ); diff --git a/src/application/HermesConfigurationYaml.test.ts b/src/application/HermesConfigurationYaml.test.ts new file mode 100644 index 000000000..41b74fcc9 --- /dev/null +++ b/src/application/HermesConfigurationYaml.test.ts @@ -0,0 +1,21 @@ +import { describe, expect, it } from "vitest"; + +import { + editHermesConfigurationYaml, + parseHermesConfigurationYaml, +} from "./HermesConfigurationYaml.js"; + +describe("Hermes configuration YAML", () => { + it("writes an edited number from its new value", () => { + expect( + editHermesConfigurationYaml("mode: 0600\nmask: 0x1F\n", ["mode"], 384), + ).toBe("mode: 384\nmask: 0x1F\n"); + }); + + it("accepts keys Hermes reads as distinct and rejects a repeated key", () => { + expect(() => + parseHermesConfigurationYaml("0600: octal\n600: decimal\n"), + ).not.toThrow(); + expect(() => parseHermesConfigurationYaml('a: 1\n"a": 2\n')).toThrow(); + }); +}); diff --git a/src/application/HermesConfigurationYaml.ts b/src/application/HermesConfigurationYaml.ts new file mode 100644 index 000000000..8d245da37 --- /dev/null +++ b/src/application/HermesConfigurationYaml.ts @@ -0,0 +1,74 @@ +import { + isScalar, + parse, + parseDocument, + type ParsedNode, + type ScalarTag, + type Tags, +} from "yaml"; + +const NUMBER_TAGS = new Set([ + "tag:yaml.org,2002:int", + "tag:yaml.org,2002:float", +]); + +/** + * Write a parsed number with its source spelling while that spelling still + * reads as the node's value. Hermes reads config.yaml as YAML 1.1 with exact + * integers, so re-printing a JavaScript number changes what it reads: a + * Discord channel ID loses its low digits, `0600` (octal) becomes `600`, and + * `0.0000001` becomes `1e-7`, which YAML 1.1 does not define as a float. + */ +const keepNumberSource = (tag: Tags[number]): Tags[number] => { + if (typeof tag === "string" || tag.collection !== undefined) return tag; + const { stringify } = tag; + if (!NUMBER_TAGS.has(tag.tag) || stringify === undefined) return tag; + return { + ...tag, + stringify: (item, context, ...rest) => { + const { source } = item; + if (source === undefined) return stringify(item, context, ...rest); + const reader = context.doc.schema.tags.find( + (candidate): candidate is ScalarTag => + candidate.default === true && candidate.test?.test(source) === true, + ); + const read = reader?.resolve( + source, + () => undefined, + context.doc.options, + ); + return Object.is(isScalar(read) ? read.value : read, item.value) + ? source + : stringify(item, context, ...rest); + }, + }; +}; + +const HERMES_YAML_OPTIONS = { + customTags: (tags: Tags) => tags.map(keepNumberSource), + // Hermes keeps integers exact, so IDs that round to one double, or `0600` + // and `600`, are distinct keys. + uniqueKeys: (a: ParsedNode, b: ParsedNode) => + a === b || + (isScalar(a) && + isScalar(b) && + a.value === b.value && + a.source === b.source), +}; + +/** Parse a Hermes `config.yaml`. */ +export const parseHermesConfigurationYaml = (text: string): unknown => + parse(text, HERMES_YAML_OPTIONS); + +/** Set or delete one value in a Hermes `config.yaml`, keeping the rest as written. */ +export const editHermesConfigurationYaml = ( + text: string, + path: readonly string[], + value: unknown, +): string => { + const document = parseDocument(text, HERMES_YAML_OPTIONS); + if (value === undefined) document.deleteIn(path); + else document.setIn(path, value); + // Editing REA must not reflow unrelated user prompts or other YAML scalars. + return document.toString({ lineWidth: 0 }); +}; diff --git a/src/application/InvestigationProviders.ts b/src/application/InvestigationProviders.ts index 5dcb18150..af23463f4 100644 --- a/src/application/InvestigationProviders.ts +++ b/src/application/InvestigationProviders.ts @@ -20,7 +20,9 @@ export const workflowAnalysisProfile = ( upstream_analysis_profile: upstream, ...(operation === "binary_overview" ? { workflow_revision: 3 } - : operation === "batch_decompile" || operation === "get_call_graph" + : operation === "batch_decompile" || + operation === "get_call_graph" || + operation === "trace_feature" ? { workflow_revision: 2 } : {}), }); diff --git a/src/application/ReferenceSourceImportEntries.ts b/src/application/ReferenceSourceImportEntries.ts index 36dae9949..49c6f82f5 100644 --- a/src/application/ReferenceSourceImportEntries.ts +++ b/src/application/ReferenceSourceImportEntries.ts @@ -97,18 +97,22 @@ const resolveInternalSpecifier = ( ); if (logicalPathEscapesRoot(normalized)) return { to: normalized, resolution: "unresolved" }; + // Preserve directory intent before normalization removes terminal dot segments. + const directoryReference = + specifier.endsWith("/") || [".", ".."].includes(posix.basename(specifier)); const suffixes = ["", ".ts", ".js", ".mjs", ".cjs", ".tsx", ".jsx"]; const candidates = suffixes.flatMap((suffix) => [ - `${normalized}${suffix}`, - `${normalized}/index${suffix}`, + ...(directoryReference ? [] : [`${normalized}${suffix}`]), + posix.join(normalized, `index${suffix}`), ]); // Retain present runtime files first; substitute only a missing counterpart. // Counterparts cover explicit filenames, not directory/index resolution. - candidates.push( - ...typescriptEmittedSourceCandidates(fromPath, normalized).map( - ({ path }) => path, - ), - ); + if (!directoryReference) + candidates.push( + ...typescriptEmittedSourceCandidates(fromPath, normalized).map( + ({ path }) => path, + ), + ); const match = lookupPath.resolveCandidates(candidates); return match === undefined ? { to: normalized, resolution: "unresolved" } diff --git a/src/application/SupportedClients.ts b/src/application/SupportedClients.ts index 488dda49d..0a7352627 100644 --- a/src/application/SupportedClients.ts +++ b/src/application/SupportedClients.ts @@ -135,6 +135,9 @@ const codexDirectory = ({ home, env }: ClientPathContext): string => const qoderDirectory = ({ home, env }: ClientPathContext): string => env.QODER_CONFIG_DIR || join(home, ".qoder"); +const factoryDirectory = ({ home }: ClientPathContext): string => + join(home, ".factory"); + /** Match Qwen Code's home override, including tilde and cwd-relative paths. */ const qwenCodeDirectory = ({ home, env }: ClientPathContext): string => { const override = env.QWEN_HOME; @@ -157,8 +160,27 @@ const qwenCodeDirectory = ({ home, env }: ClientPathContext): string => { const grokDirectory = ({ home, env }: ClientPathContext): string => env.GROK_HOME || join(home, ".grok"); +/** Match Harness home-paths for both independent roots; trim only to detect blanks. */ +const dshDirectory = ( + home: string, + override: string | undefined, + fallback: string, +): string => { + const selected = + override !== undefined && override.trim().length > 0 + ? override + : join(home, fallback); + const expanded = + selected === "~" + ? home + : selected.startsWith("~/") || selected.startsWith("~\\") + ? join(home, selected.slice(2)) + : selected; + return resolve(expanded); +}; + const dshHomeDirectory = ({ home, env }: ClientPathContext): string => - env.DSH_HOME ?? join(home, ".dsh"); + dshDirectory(home, env.DSH_HOME, ".dsh"); const dshPatchDirectory = (context: ClientPathContext): string => join(dshHomeDirectory(context), "cordis.patch.yml"); @@ -539,6 +561,16 @@ export const SUPPORTED_CLIENT_DEFINITIONS = [ markerPath: qoderDirectory, format: "json", }, + { + name: "droid", + displayName: "Factory Droid", + skillPath: (context: ClientPathContext) => + join(factoryDirectory(context), "skills"), + configPath: (context: ClientPathContext) => + join(factoryDirectory(context), "mcp.json"), + markerPath: factoryDirectory, + format: "json", + }, { name: "zcode", zcodeFallbackPath: ({ home }: ClientPathContext) => @@ -555,7 +587,7 @@ export const SUPPORTED_CLIENT_DEFINITIONS = [ // The home Cordis patch applies to every profile; the skill directory is // shared agent storage Harness discovers. skillPath: ({ home, env }: ClientPathContext) => - join(env.DSH_AGENTS_HOME ?? join(home, ".agents"), "skills"), + join(dshDirectory(home, env.DSH_AGENTS_HOME, ".agents"), "skills"), configPath: dshPatchDirectory, markerPath: dshHomeDirectory, format: "dsh", diff --git a/src/application/binary/AnalysisSnapshotCache.test.ts b/src/application/binary/AnalysisSnapshotCache.test.ts index 48c2a54bd..91b9f8807 100644 --- a/src/application/binary/AnalysisSnapshotCache.test.ts +++ b/src/application/binary/AnalysisSnapshotCache.test.ts @@ -661,3 +661,52 @@ it("does not replay pre-envelope call graphs under the revised workflow profile" }).success, ).toBe(false); }); + +describe("executions that report truncated references", () => { + const record = (referencesTruncated: boolean | undefined) => { + const cache = new AnalysisSnapshotCache(); + const parameters = { address: "0x1000", document: "fixture" }; + cache.record({ + target: ANALYSIS_SNAPSHOT_TARGET, + profile: ANALYSIS_SNAPSHOT_PROFILE, + operation: "xrefs", + parameters, + execution: createAnalysisExecution( + ["0x2000"], + ANALYSIS_SNAPSHOT_PROVIDER, + { + subject: ANALYSIS_SNAPSHOT_TARGET, + analysisProfile: ANALYSIS_SNAPSHOT_PROFILE, + ...(referencesTruncated === undefined ? {} : { referencesTruncated }), + }, + ), + }); + return cache.lookup( + ANALYSIS_SNAPSHOT_TARGET, + ANALYSIS_SNAPSHOT_PROFILE, + "xrefs", + parameters, + ); + }; + + it("are not replayed, because a replay would lose the truncation", () => { + expect(record(true)).toBeUndefined(); + }); + + it("do not stop an execution that is complete or silent from being replayed", () => { + expect(record(false)?.result).toEqual(["0x2000"]); + expect(record(undefined)?.result).toEqual(["0x2000"]); + }); +}); + +it("revises the trace_feature workflow profile so earlier results are not replayed", () => { + const profile = ANALYSIS_SNAPSHOT_PROFILE; + + expect(workflowAnalysisProfile(profile, "trace_feature")).not.toEqual( + workflowAnalysisProfile(profile), + ); + expect( + workflowAnalysisProfile(profile, "trace_call_path"), + "other workflows keep their profile", + ).toEqual(workflowAnalysisProfile(profile)); +}); diff --git a/src/application/binary/AnalysisSnapshotCache.ts b/src/application/binary/AnalysisSnapshotCache.ts index bdaa9f68d..79e212cac 100644 --- a/src/application/binary/AnalysisSnapshotCache.ts +++ b/src/application/binary/AnalysisSnapshotCache.ts @@ -84,6 +84,11 @@ export const isSnapshotCacheable = ( descriptor.effects.mayWriteFilesystem === false) && descriptor.effects.changesPermissions === false; +/** Windows Ghidra must admit the selected source through its native snapshot owner. */ +export const requiresNativeSourceAdmissionBeforeReplay = ( + providerId: string | undefined, +): boolean => process.platform === "win32" && providerId === "ghidra"; + /** Bounded in-memory cache for one immutable binary identity. */ export class AnalysisSnapshotCache { readonly #entries = new Map(); @@ -337,6 +342,9 @@ export class AnalysisSnapshotCache { }): void { const { target, profile, operation, parameters, execution } = input; this.select(target, profile); + // The snapshot format does not carry reference truncation; a replay would + // present a capped read as complete, so it is not recorded. + if (execution.referencesTruncated === true) return; const queryId = analysisQueryId( snapshotTarget(target), snapshotBinding(profile), diff --git a/src/application/binary/BinarySession.ts b/src/application/binary/BinarySession.ts index f7b2d3529..3da571832 100644 --- a/src/application/binary/BinarySession.ts +++ b/src/application/binary/BinarySession.ts @@ -58,6 +58,7 @@ interface SessionBinding { readonly profile: AnalysisProfileCommitment | null; readonly route: SessionProviderRoute; readonly runId: string; + nativeSourceAdmitted: boolean; } /** @@ -323,6 +324,7 @@ export class BinarySession profile, route, runId, + nativeSourceAdmitted: false, }; this.#clearRuntimeAvailability(); this.resetSnapshotInvalidation(); @@ -609,6 +611,7 @@ export class BinarySession active: this.#active, operation: name, parameters: arguments_, + nativeSourceAdmitted: this.#active?.nativeSourceAdmitted === true, unboundOperationError: (operation, route) => this.#providerRouter.unboundOperationError(operation, route), lookupSnapshot: (target, profile, operation, parameters) => @@ -645,6 +648,14 @@ export class BinarySession }, }) : profiled; + // A successful Ghidra operation proves this client's native source snapshot + // was admitted; a composite health response and parsed path do not. + if ( + execution.ok && + capability.provider.id === "ghidra" && + this.#active === active + ) + this.#active.nativeSourceAdmitted = true; this.#observeRuntimeAvailability(name, execution); if ( execution.ok && @@ -838,6 +849,7 @@ export class BinarySession profile: previous.profile, route: previous.route, runId, + nativeSourceAdmitted: false, }; return ok(null); } diff --git a/src/application/binary/BinarySessionExecution.ts b/src/application/binary/BinarySessionExecution.ts index 996749aae..73dfe4eea 100644 --- a/src/application/binary/BinarySessionExecution.ts +++ b/src/application/binary/BinarySessionExecution.ts @@ -13,7 +13,10 @@ import type { AnalysisOperation, CapabilityDescriptor, } from "../AnalysisProvider.js"; -import { isSnapshotCacheable } from "./AnalysisSnapshotCache.js"; +import { + isSnapshotCacheable, + requiresNativeSourceAdmissionBeforeReplay, +} from "./AnalysisSnapshotCache.js"; import type { SessionProviderRoute } from "./SessionProviderRouter.js"; interface ActiveExecutionBinding { @@ -35,6 +38,7 @@ interface PrepareSessionExecutionInput { readonly active: ActiveExecutionBinding | undefined; readonly operation: AnalysisOperation; readonly parameters: Readonly>; + readonly nativeSourceAdmitted: boolean; readonly unboundOperationError: ( operation: AnalysisOperation, route: SessionProviderRoute, @@ -55,6 +59,7 @@ export const prepareSessionExecution = ( active, operation, parameters, + nativeSourceAdmitted, unboundOperationError, lookupSnapshot, } = input; @@ -79,9 +84,12 @@ export const prepareSessionExecution = ( const cacheable = profile !== undefined && isSnapshotCacheable(operation, capability, parameters); - const cached = cacheable - ? lookupSnapshot(active.target, profile, operation, parameters) - : undefined; + const cached = + cacheable && + (!requiresNativeSourceAdmissionBeforeReplay(capability.provider.id) || + nativeSourceAdmitted) + ? lookupSnapshot(active.target, profile, operation, parameters) + : undefined; return ok({ active, capability, profile, cacheable, cached }); }; diff --git a/src/application/binary/BinarySessionOpen.ts b/src/application/binary/BinarySessionOpen.ts index 8d810e671..cd5f6be4a 100644 --- a/src/application/binary/BinarySessionOpen.ts +++ b/src/application/binary/BinarySessionOpen.ts @@ -92,9 +92,7 @@ export const resolveSessionTarget = async ( input: ResolveSessionTargetInput, ): Promise> => { const { router, current, target, options } = input; - const sameTarget = - current?.target.path === target.path && - snapshotMatchesTarget(snapshotTarget(current.target), target); + const sameTarget = sameSessionTarget(current?.target, target); const resolvedRoute = sameTarget && options.providerId === undefined && current !== undefined ? ({ ok: true, value: current.route } as const) @@ -113,9 +111,7 @@ export const validateResolvedSessionTarget = ( }, ): Result => { const { current, target, route, options, stagedSnapshotMatches } = input; - const sameTarget = - current?.target.path === target.path && - snapshotMatchesTarget(snapshotTarget(current.target), target); + const sameTarget = sameSessionTarget(current?.target, target); const snapshotError = validateSnapshot({ snapshot: options.snapshot, target, @@ -127,6 +123,16 @@ export const validateResolvedSessionTarget = ( return { ok: true, value: { target, route, sameTarget } }; }; +const sameSessionTarget = ( + current: BinaryTarget | undefined, + target: BinaryTarget, +): boolean => + current !== undefined && + current.path === target.path && + // Native admission is bound to the requested path, not just its realpath. + (current.sourcePath ?? current.path) === (target.sourcePath ?? target.path) && + snapshotMatchesTarget(snapshotTarget(current), target); + interface ValidateSnapshotInput { readonly snapshot: AnalysisSnapshot | undefined; readonly target: BinaryTarget; diff --git a/src/artifacts/ArtifactProvider.ts b/src/artifacts/ArtifactProvider.ts index c95cec6d5..47d5906ea 100644 --- a/src/artifacts/ArtifactProvider.ts +++ b/src/artifacts/ArtifactProvider.ts @@ -40,6 +40,7 @@ import { ArtifactResourceScope } from "./ArtifactResourceScope.js"; import { artifactCapabilities } from "./ArtifactProviderMetadata.js"; import { ARTIFACT_GRAPH_PROVIDER } from "../application/InvestigationProviders.js"; import { createEvidence } from "../domain/evidence.js"; +import { freezeJsonSnapshot } from "../domain/immutableJson.js"; import { createArtifactInspection } from "../domain/artifactInspection.js"; import { ProviderCleanupError } from "../domain/providerCleanupError.js"; @@ -341,7 +342,11 @@ class ArtifactClient implements AnalysisClient { total: 1, message: "inventory substep started", }); - const inventory = await this.inventory(inventoryParameters, options); + // The freshly parsed inventory is exclusively owned here. Sealing it lets + // the nested Evidence share it instead of cloning and re-digesting it. + const inventory = freezeJsonSnapshot( + await this.inventory(inventoryParameters, options), + ); const subject = subjectFor( this.target.sourcePath ?? this.target.path, inventory.manifest, @@ -355,7 +360,10 @@ class ArtifactClient implements AnalysisClient { // The nested inventory already names every occurrence's logical path. locations: [], }); - const result = createArtifactInspection(inventoryEvidence); + // The returned execution and its Evidence then reuse this owned report. + const result = freezeJsonSnapshot( + createArtifactInspection(inventoryEvidence), + ); await options?.progress?.report({ phase: "inspect_artifact.inventory", completed: 1, diff --git a/src/artifacts/MachOSliceArtifactReader.ts b/src/artifacts/MachOSliceArtifactReader.ts index f567458c7..b57b0247e 100644 --- a/src/artifacts/MachOSliceArtifactReader.ts +++ b/src/artifacts/MachOSliceArtifactReader.ts @@ -8,7 +8,7 @@ import { type NativeCommandRunner, } from "../native/CommandRunner.js"; import { - parseLipoArchitectures, + parseLipoOutput, type LipoArchitecture, } from "../native/parsers/lipo.js"; import type { MachoSlice } from "../domain/apple/dylibResolution.js"; @@ -114,7 +114,7 @@ export class MachOSliceArtifactReader implements ArtifactReader { }; await this.#verifySource(source); const fileSize = source.initial.size; - const architectures = parseLipoArchitectures(captured.value.stdout); + const { architectures } = parseLipoOutput(captured.value.stdout, this.path); const structural = await readStructuralSlices(source.handle, fileSize); await this.#verifySource(source); if (structural.status === "malformed") diff --git a/src/artifacts/ZipArtifactReader.ts b/src/artifacts/ZipArtifactReader.ts index dfbee8b27..066b005b8 100644 --- a/src/artifacts/ZipArtifactReader.ts +++ b/src/artifacts/ZipArtifactReader.ts @@ -2,7 +2,29 @@ import type { FileHandle } from "node:fs/promises"; import { PassThrough, type Readable } from "node:stream"; import { once } from "node:events"; -import { Reader, ZipReader, type Entry, type FileEntry } from "@zip.js/zip.js"; +import { + ERR_AMBIGUOUS_ARCHIVE, + ERR_BAD_FORMAT, + ERR_CENTRAL_DIRECTORY_NOT_FOUND, + ERR_ENCRYPTED_CENTRAL_DIRECTORY, + ERR_ENTRY_DATA_OUT_OF_BOUNDS, + ERR_EOCDR_LOCATOR_ZIP64_NOT_FOUND, + ERR_EOCDR_NOT_FOUND, + ERR_EXTRAFIELD_ZIP64_NOT_FOUND, + ERR_INVALID_COMPRESSED_DATA, + ERR_INVALID_CRC32, + ERR_INVALID_UNCOMPRESSED_SIZE, + ERR_LOCAL_FILE_HEADER_NOT_FOUND, + ERR_OVERLAPPING_ENTRY, + ERR_SPLIT_ZIP_FILE, + ERR_UNSUPPORTED_COMPRESSION, + ERR_UNSUPPORTED_ENCRYPTION, + ERR_UNSUPPORTED_UINT64, + Reader, + ZipReader, + type Entry, + type FileEntry, +} from "@zip.js/zip.js"; import { ArtifactReaderFailure, @@ -159,9 +181,9 @@ export class ZipArtifactReader implements ArtifactReader { let index = 0; // A traversal owns its accounting, including overlapping or repeated passes. let declaredFileBytes = 0; - for await (const entry of this.#reader.getEntriesGenerator({ - decodeText: decodeZipEntryText, - })) { + for await (const entry of typedZipFailures( + this.#reader.getEntriesGenerator({ decodeText: decodeZipEntryText }), + )) { abortIfNeeded(signal); // ZIP member identity is its directory occurrence, not its filename. // Logical path uniqueness belongs to the artifact inventory boundary. @@ -327,7 +349,7 @@ const extractStream = (entry: FileEntry, signal?: AbortSignal): Readable => { output.end(); }, abort: (reason) => { - output.destroy(toError(reason)); + output.destroy(toError(typedZipFailure(reason))); }, }); void entry @@ -337,10 +359,61 @@ const extractStream = (entry: FileEntry, signal?: AbortSignal): Readable => { checkOverlappingEntry: true, onprogress: () => abortIfNeeded(signal), }) - .catch((cause: unknown) => output.destroy(toError(cause))); + .catch((cause: unknown) => output.destroy(toError(typedZipFailure(cause)))); return output; }; +/** zip.js reports archive faults as plain errors identified by message. */ +const ZIP_FAILURE_REASONS: ReadonlyMap = + new Map([ + ...[ + ERR_AMBIGUOUS_ARCHIVE, + ERR_BAD_FORMAT, + ERR_CENTRAL_DIRECTORY_NOT_FOUND, + ERR_ENTRY_DATA_OUT_OF_BOUNDS, + ERR_EOCDR_LOCATOR_ZIP64_NOT_FOUND, + ERR_EOCDR_NOT_FOUND, + ERR_EXTRAFIELD_ZIP64_NOT_FOUND, + ERR_INVALID_COMPRESSED_DATA, + ERR_INVALID_CRC32, + ERR_INVALID_UNCOMPRESSED_SIZE, + ERR_LOCAL_FILE_HEADER_NOT_FOUND, + ERR_OVERLAPPING_ENTRY, + ERR_UNSUPPORTED_UINT64, + ].map((message) => [message, "format"] as const), + ...[ + ERR_ENCRYPTED_CENTRAL_DIRECTORY, + ERR_SPLIT_ZIP_FILE, + ERR_UNSUPPORTED_COMPRESSION, + ERR_UNSUPPORTED_ENCRYPTION, + ].map((message) => [message, "unavailable"] as const), + ]); + +/** Type a zip.js archive fault so consumers do not report it as file I/O. */ +const typedZipFailure = (cause: unknown): unknown => { + if (!(cause instanceof Error) || cause instanceof ArtifactReaderFailure) + return cause; + const reason = ZIP_FAILURE_REASONS.get(cause.message); + if (reason === undefined) return cause; + return new ArtifactReaderFailure( + reason, + reason === "format" + ? `ZIP archive is malformed: ${cause.message}` + : `ZIP archive uses an unsupported feature: ${cause.message}`, + { cause }, + ); +}; + +async function* typedZipFailures( + source: AsyncIterable, +): AsyncIterable { + try { + yield* source; + } catch (cause: unknown) { + throw typedZipFailure(cause); + } +} + const isSymlink = (entry: Entry): boolean => entry.unixMode !== undefined && (entry.unixMode & 0o170000) === 0o120000; diff --git a/src/artifacts/apple/InterfaceBuilderAnalysis.ts b/src/artifacts/apple/InterfaceBuilderAnalysis.ts index fb90594b3..02ecd9821 100644 --- a/src/artifacts/apple/InterfaceBuilderAnalysis.ts +++ b/src/artifacts/apple/InterfaceBuilderAnalysis.ts @@ -56,6 +56,7 @@ export const analyzeInterfaceBuilderBundle = async (input: { const omittedArchives: Array<{ path: string; reason: string }> = []; let unreportedOmittedArchives = 0; const incompleteHierarchies = new Map(); + const unresolvedConnections = new Map(); const prototypeKeyOmissions = new Map(); let omitted = 0; let attempted = 0; @@ -136,6 +137,8 @@ export const analyzeInterfaceBuilderBundle = async (input: { : { value: nib.raw, omittedPrototypeKeys: 0 }; if (nib !== null && nib.omitted > 0) incompleteHierarchies.set(entry.path, nib.omitted); + if (nib !== null && nib.unresolvedConnections > 0) + unresolvedConnections.set(entry.path, nib.unresolvedConnections); if (omittedPrototypeKeys !== 0) prototypeKeyOmissions.set(entry.path, omittedPrototypeKeys); const documentHash = createHash("sha256").update(bytes).digest("hex"); @@ -170,6 +173,7 @@ export const analyzeInterfaceBuilderBundle = async (input: { }); return finalizeHierarchyCoverage(result, { incompleteHierarchies, + unresolvedConnections, prototypeKeyOmissions, invalid, omittedArchives, @@ -237,6 +241,7 @@ const finalizeHierarchyCoverage = ( result: ReturnType, counts: { incompleteHierarchies: ReadonlyMap; + unresolvedConnections: ReadonlyMap; prototypeKeyOmissions: ReadonlyMap; invalid: readonly string[]; omittedArchives: readonly { path: string; reason: string }[]; @@ -247,6 +252,7 @@ const finalizeHierarchyCoverage = ( ) => { const { incompleteHierarchies, + unresolvedConnections, prototypeKeyOmissions, invalid, omittedArchives, @@ -260,6 +266,13 @@ const finalizeHierarchyCoverage = ( ...result, documents: result.documents.map((document) => ({ ...document, + // The dictionary parser only sees connectors that were projected, so the + // omission recorded where an ownerless connector was dropped has to be + // added here for this producer to report the same document count as the + // keyed-archive producer for the same recognized connector. + connection_count: + document.connection_count + + (unresolvedConnections.get(document.relative_path) ?? 0), hierarchy_complete: document.hierarchy_complete && !incompleteHierarchies.has(document.relative_path), @@ -268,15 +281,32 @@ const finalizeHierarchyCoverage = ( ...result.graph, coverage: [ ...result.graph.coverage.map((facet) => { - const omitted = incompleteHierarchies.get( + const omittedHierarchy = incompleteHierarchies.get( facet.facet.replace(/^hierarchy:/u, ""), ); - return facet.facet.startsWith("hierarchy:") && omitted !== undefined + if ( + facet.facet.startsWith("hierarchy:") && + omittedHierarchy !== undefined + ) + return { + ...facet, + status: "partial" as const, + reason: "serialized_view_hierarchy_incomplete", + omitted: facet.omitted + omittedHierarchy, + }; + const omittedConnection = unresolvedConnections.get( + facet.facet.replace(/^connections:/u, ""), + ); + return facet.facet.startsWith("connections:") && + omittedConnection !== undefined ? { ...facet, status: "partial" as const, - reason: "serialized_view_hierarchy_incomplete", - omitted: facet.omitted + omitted, + // A connection dropped by the limit is recoverable by raising + // it, so that cause keeps the reason the graph already named. + // Only a facet with no other cause reports the omission. + reason: facet.reason ?? "connection_unresolved", + omitted: facet.omitted + omittedConnection, } : facet; }), @@ -306,7 +336,8 @@ const finalizeHierarchyCoverage = ( result.graph.truncated || counts.omitted > 0 || archiveDecodePartial || - incompleteHierarchies.size > 0, + incompleteHierarchies.size > 0 || + unresolvedConnections.size > 0, }, limitations: [ ...result.limitations, @@ -316,6 +347,11 @@ const finalizeHierarchyCoverage = ( : [ "Some serialized hierarchy links could not be projected within the bounded hierarchy; their parentage remains unknown.", ]), + ...(unresolvedConnections.size === 0 + ? [] + : [ + "Some recognized NIBArchive connectors declare no resolvable owner; they carry no connection evidence and are reported as omissions.", + ]), ...(invalid.length === 0 ? [] : [ @@ -496,6 +532,10 @@ const projectNibArchive = ( ); const connections: Record = {}; const hierarchy: NibHierarchyNode[] = []; + // A recognized connector whose owner reference cannot be resolved produces no + // connection entry, so the omission has to be counted here: the projected + // dictionary carries no trace of it for a downstream consumer to find. + let unresolvedConnections = 0; const reference = (value: JsonValue | undefined): number | null => { if (typeof value !== "object" || value === null || Array.isArray(value)) return null; @@ -512,7 +552,10 @@ const projectNibArchive = ( const values = object.values; const connectionFields = interfaceBuilderConnectionFieldNames(className); const source = reference(values[connectionFields.source]); - if (source === null) continue; + if (source === null) { + unresolvedConnections += 1; + continue; + } const destination = reference(values[connectionFields.destination]); const label = referencedString(values[connectionFields.label]); const type = className.includes("Outlet") @@ -608,6 +651,7 @@ const projectNibArchive = ( ); return { omitted: viewHierarchy.omitted + hierarchyOmitted + merged.omitted, + unresolvedConnections, raw: jsonValueSchema.parse({ "com.apple.ibtool.document.objects": objects, "com.apple.ibtool.document.connections": connections, diff --git a/src/artifacts/inventory/ArtifactGraphConstruction.test.ts b/src/artifacts/inventory/ArtifactGraphConstruction.test.ts new file mode 100644 index 000000000..925d3543a --- /dev/null +++ b/src/artifacts/inventory/ArtifactGraphConstruction.test.ts @@ -0,0 +1,103 @@ +import { describe, expect, it } from "vitest"; + +import { CPU, fatImage, machoImage } from "../apple/MachoImage.fixture.js"; +import { classifyArtifactContent } from "./ArtifactGraphConstruction.js"; + +const fat = (wide = false): Buffer => + Buffer.from(fatImage([{ cpu: CPU.arm64, bytes: machoImage({}) }], wide)); + +describe("bounded FAT inventory classification", () => { + it("requires the complete table but not slice bytes in the retained prefix", () => { + const bytes = fat(); + for (const length of [4, 7, 8, 27]) { + expect( + classifyArtifactContent( + "native", + bytes.subarray(0, length), + bytes.length, + ), + ).toEqual({ kind: "unknown", format: "unknown" }); + } + expect( + classifyArtifactContent( + "native.class", + bytes.subarray(0, 28), + bytes.length, + ), + ).toEqual({ kind: "executable", format: "mach-o-universal" }); + }); + + it.each([ + ["empty table", (bytes: Buffer) => bytes.writeUInt32BE(0, 4)], + ["slice overlapping table", (bytes: Buffer) => bytes.writeUInt32BE(0, 16)], + ["empty slice", (bytes: Buffer) => bytes.writeUInt32BE(0, 20)], + [ + "slice beyond file", + (bytes: Buffer) => bytes.writeUInt32BE(bytes.length, 20), + ], + ["unaligned slice", (bytes: Buffer) => bytes.writeUInt32BE(4097, 16)], + ["unsupported alignment", (bytes: Buffer) => bytes.writeUInt32BE(53, 24)], + ] as const)("keeps %s unknown", (_label, mutate) => { + const bytes = fat(); + mutate(bytes); + expect(classifyArtifactContent("native", bytes)).toEqual({ + kind: "unknown", + format: "unknown", + }); + }); + + it("does not infer Mach-O from an out-of-bounds architecture count", () => { + const bytes = fat(); + bytes.writeUInt32BE(0xffffffff, 4); + expect(classifyArtifactContent("native", bytes)).toEqual({ + kind: "resource", + format: "file", + }); + }); + + it("does not lose unsafe FAT64 range values to number conversion", () => { + const bytes = fat(true); + bytes.writeBigUInt64BE(BigInt(Number.MAX_SAFE_INTEGER) + 1n, 16); + expect(classifyArtifactContent("native", bytes)).toEqual({ + kind: "unknown", + format: "unknown", + }); + }); + + it("requires valid ranges for every declared slice", () => { + const bytes = Buffer.from( + fatImage([ + { cpu: CPU.arm64, bytes: machoImage({}) }, + { cpu: CPU.x86_64, bytes: machoImage({ cpu: CPU.x86_64 }) }, + ]), + ); + bytes.writeUInt32BE(bytes.length, 40); + expect(classifyArtifactContent("native", bytes)).toEqual({ + kind: "unknown", + format: "unknown", + }); + }); + + it("requires observed slice magic for an unknown CPU type", () => { + const cpu = { type: 0x12345678, subtype: 0 }; + const bytes = Buffer.from(fatImage([{ cpu, bytes: machoImage({ cpu }) }])); + expect( + classifyArtifactContent("native", bytes.subarray(0, 28), bytes.length), + ).toEqual({ kind: "unknown", format: "unknown" }); + expect(classifyArtifactContent("native", bytes)).toEqual({ + kind: "executable", + format: "mach-o-universal", + }); + }); + + it.each([NaN, Infinity, -1, 8.5, Number.MAX_SAFE_INTEGER + 1, 27])( + "keeps invalid or contradictory file size %s unknown", + (size) => { + const bytes = fat(); + expect(classifyArtifactContent("native", bytes, size)).toEqual({ + kind: "unknown", + format: "unknown", + }); + }, + ); +}); diff --git a/src/artifacts/javascript/JavaScriptArtifactFiles.ts b/src/artifacts/javascript/JavaScriptArtifactFiles.ts index e3849f814..78d8cf750 100644 --- a/src/artifacts/javascript/JavaScriptArtifactFiles.ts +++ b/src/artifacts/javascript/JavaScriptArtifactFiles.ts @@ -37,6 +37,7 @@ interface ExpectedFile { interface ExpectedContainer { readonly hash_status: ArtifactOccurrence["hash_status"]; + readonly format: ArtifactOccurrence["artifact_format"]; readonly inventory: JavaScriptArtifactContainer | null; } @@ -219,6 +220,9 @@ const visitReader = async ( "integrity", `Nested ASAR was not integrity-verified: ${path}`, ); + // Inventory expands an ASAR-named file only when its bytes are an + // ASAR. One it recorded as ordinary bytes has no members to read. + if (container.format !== "asar") continue; const inventory = container.inventory; if (inventory === null) throw new ArtifactReaderFailure( @@ -395,6 +399,7 @@ const expectedInventory = ( if (isNestedAsar) containers.set(occurrence.logical_path, { hash_status: occurrence.hash_status, + format: occurrence.artifact_format, inventory: null, }); continue; @@ -412,6 +417,7 @@ const expectedInventory = ( if (isNestedAsar) containers.set(occurrence.logical_path, { hash_status: occurrence.hash_status, + format: occurrence.artifact_format, inventory: { path: occurrence.logical_path, sha256: node.sha256, diff --git a/src/browser/CdpCaptureDocuments.baseUrl.test.ts b/src/browser/CdpCaptureDocuments.baseUrl.test.ts index a5481c161..e3201eade 100644 --- a/src/browser/CdpCaptureDocuments.baseUrl.test.ts +++ b/src/browser/CdpCaptureDocuments.baseUrl.test.ts @@ -2,6 +2,8 @@ import { describe, expect, it } from "vitest"; import { inspectWebPageInputSchema } from "../domain/browserObservation.js"; import { captureDom } from "./CdpCaptureDocuments.js"; +import { capturedSvgNodeKind } from "./CdpCaptureDomSvg.js"; +import { CdpCaptureCompleteness } from "./CdpCaptureCompleteness.js"; const origin = "https://app.example.test"; const documentUrl = `${origin}/screens/app`; @@ -50,8 +52,11 @@ const snapshot = ( ], }); -const capture = (value: ReturnType) => - captureDom(value, new Set([origin]), input); +const capture = (value: ReturnType) => { + const step = captureDom(value, new Set([origin]), input).next(); + if (!step.done) throw new Error("Unexpected SVG identity query"); + return step.value; +}; describe("DOM metadata document base URLs", () => { it("resolves links, actions and agent hints against the captured document base", () => { @@ -282,6 +287,7 @@ interface SvgSnapshotNode { readonly name: string; readonly parent: number; readonly attributes?: readonly (readonly [string, string])[]; + readonly svg?: boolean; } const svgDocument = ( @@ -300,7 +306,13 @@ const svgDocument = ( const nodeValue = [intern("")]; const parentIndex = [-1]; const attributes: number[][] = [[]]; + const backendNodeId = [1]; + const svgNodeKinds = new Map(); for (const element of elements) { + const id = backendNodeId.length + 1; + backendNodeId.push(id); + if (element.svg !== undefined) + svgNodeKinds.set(`${String(id)}:${element.name}`, element.svg); nodeType.push(1); nodeName.push(intern(element.name)); nodeValue.push(intern("")); @@ -316,14 +328,35 @@ const svgDocument = ( { documentURL: 0, baseURL: 1, - nodes: { nodeType, nodeName, nodeValue, parentIndex, attributes }, + nodes: { + nodeType, + nodeName, + nodeValue, + parentIndex, + attributes, + backendNodeId, + }, + svgNodeKinds, }, ], }; }; -const captureSvg = (value: ReturnType) => - captureDom(value, new Set([origin]), input); +const captureSvg = (value: ReturnType) => { + const kinds = new Map( + value.documents.flatMap((document) => [...document.svgNodeKinds]), + ); + const capture = captureDom(value, new Set([origin]), input); + let step = capture.next(); + while (!step.done) { + step = capture.next( + step.value.map((query) => + kinds.get(`${String(query.backendNodeId)}:${query.nodeName}`), + ), + ); + } + return step.value; +}; const urlFor = ( result: ReturnType, @@ -331,6 +364,136 @@ const urlFor = ( ): (typeof result.urls)[number] | undefined => result.urls.find((url) => url.node_index === nodeIndex); +describe("browser SVG identity", () => { + const query = { backendNodeId: 42, nodeName: "a" }; + const node = { ...query, nodeType: 1 }; + + it("uses only a matching described element's SVG flag", () => { + expect(capturedSvgNodeKind(query, { node: { ...node, isSVG: true } })).toBe( + true, + ); + expect( + capturedSvgNodeKind(query, { node: { ...node, isSVG: false } }), + ).toBe(false); + expect(capturedSvgNodeKind(query, { node })).toBe(false); + for (const invalid of [ + {}, + { node: { ...node, backendNodeId: 43, isSVG: true } }, + { node: { ...node, nodeName: "A", isSVG: true } }, + { node: { ...node, nodeType: 3, isSVG: true } }, + { node: { ...node, isSVG: "true" } }, + ]) + expect(capturedSvgNodeKind(query, invalid)).toBeUndefined(); + }); + + it("distinguishes identically named SVG and MathML elements regardless of parent", () => { + const result = captureSvg( + svgDocument([ + { name: "svg", parent: 0 }, + { name: "desc", parent: 1 }, + { + name: "a", + svg: true, + parent: 2, + attributes: [["xlink:href", "/svg"]], + }, + { + name: "a", + svg: false, + parent: 2, + attributes: [["xlink:href", "/math"]], + }, + { + name: "A", + svg: false, + parent: 1, + attributes: [["xlink:href", "/html"]], + }, + { + name: "a", + svg: true, + parent: 0, + attributes: [["xlink:href", "/orphan"]], + }, + ]), + ); + expect(result.urls.map(({ url }) => url)).toEqual([ + `${origin}/svg`, + `${origin}/orphan`, + ]); + }); + + it("records unresolved identity without inventing a destination or hiding attributes", () => { + const value = svgDocument([ + { name: "a", parent: 0, attributes: [["xlink:href", "/unknown"]] }, + ]); + const completeness = new CdpCaptureCompleteness(); + const capture = captureDom(value, new Set([origin]), input, completeness); + expect(capture.next()).toEqual({ + done: false, + value: [{ backendNodeId: 2, nodeName: "a" }], + }); + const step = capture.next([undefined]); + expect(step.done).toBe(true); + if (!step.done) throw new Error("Unexpected second query"); + expect(step.value.unresolvedSvgNodes).toBe(1); + expect(step.value.urls).toEqual([]); + expect(step.value.nodes[1]?.attribute_names).toEqual(["xlink:href"]); + expect(completeness.snapshot().excluded).toEqual( + expect.arrayContaining([ + expect.objectContaining({ + section: "metadata", + reason: "provider_unavailable", + count: 1, + }), + ]), + ); + }); +}); + +it("keeps batched SVG identities aligned around missing IDs and ordinary links", () => { + const value = svgDocument([ + { name: "a", parent: 0, attributes: [["xlink:href", "/svg"]] }, + { name: "A", parent: 0, attributes: [["href", "/ordinary"]] }, + { name: "a", parent: 0, attributes: [["xlink:href", "/missing-id"]] }, + { name: "a", parent: 0, attributes: [["xlink:href", "/non-svg"]] }, + ]); + const completeness = new CdpCaptureCompleteness(); + const capture = captureDom( + { + ...value, + documents: value.documents.map((document) => ({ + ...document, + nodes: { ...document.nodes, backendNodeId: [1, 2, 3, null, 5] }, + })), + }, + new Set([origin]), + input, + completeness, + ); + expect(capture.next()).toEqual({ + done: false, + value: [ + { backendNodeId: 2, nodeName: "a" }, + { backendNodeId: 5, nodeName: "a" }, + ], + }); + const step = capture.next([true, false]); + if (!step.done) throw new Error("Unexpected second batch"); + expect( + step.value.urls.map(({ node_index, url }) => [node_index, url]), + ).toEqual([ + [1, `${origin}/svg`], + [2, `${origin}/ordinary`], + ]); + expect(step.value.unresolvedSvgNodes).toBe(1); + expect(completeness.snapshot().excluded).toContainEqual({ + section: "metadata", + reason: "provider_unavailable", + count: 1, + }); +}); + describe("SVG xlink:href destinations", () => { it("reports an SVG anchor that only has xlink:href, keeping the first value", () => { const result = captureSvg( @@ -339,6 +502,7 @@ describe("SVG xlink:href destinations", () => { { name: "g", parent: 1 }, { name: "a", + svg: true, parent: 2, attributes: [ ["xlink:href", "/xl-s2h"], @@ -371,6 +535,7 @@ describe("SVG xlink:href destinations", () => { { name: "svg", parent: 0 }, { name: "a", + svg: true, parent: 1, attributes: [ [first, valueFor(first)], @@ -395,18 +560,21 @@ describe("SVG xlink:href destinations", () => { svgDocument([ { name: "A", + svg: false, parent: 0, attributes: [["xlink:href", "/xl-html"]], }, { name: "math", parent: 0 }, { name: "mrow", + svg: false, parent: 2, attributes: [["xlink:href", "/xl-math"]], }, { name: "svg", parent: 0 }, { name: "a", + svg: true, parent: 4, attributes: [["xlink:href", "/xl-s2h"]], }, @@ -425,6 +593,7 @@ describe("SVG xlink:href destinations", () => { { name: "svg", parent: 0 }, { name: "a", + svg: true, parent: 1, attributes: [["xlink:href", "https://cdn.example.test/out"]], }, @@ -444,8 +613,18 @@ describe("SVG xlink:href destinations", () => { const result = captureSvg( svgDocument([ { name: "svg", parent: 0 }, - { name: "image", parent: 1, attributes: [["xlink:href", "guide"]] }, - { name: "use", parent: 1, attributes: [["xlink:href", "icon.svg"]] }, + { + name: "image", + svg: true, + parent: 1, + attributes: [["xlink:href", "guide"]], + }, + { + name: "use", + svg: true, + parent: 1, + attributes: [["xlink:href", "icon.svg"]], + }, ]), ); expect(result.urls).toEqual([ @@ -466,14 +645,24 @@ describe("SVG xlink:href destinations", () => { }); describe("SVG xlink:href context", () => { - it("keeps forward ancestor context local to each document", () => { + it("binds SVG identity to the selected element across documents", () => { const svg = svgDocument([ - { name: "a", parent: 2, attributes: [["xlink:href", "/svg-link"]] }, + { + name: "a", + svg: true, + parent: 2, + attributes: [["xlink:href", "/svg-link"]], + }, { name: "g", parent: 3 }, { name: "svg", parent: 0 }, ]); const html = svgDocument([ - { name: "A", parent: 2, attributes: [["xlink:href", "/html-link"]] }, + { + name: "A", + svg: false, + parent: 2, + attributes: [["xlink:href", "/html-link"]], + }, { name: "DIV", parent: 3 }, { name: "BODY", parent: 0 }, ]); @@ -507,32 +696,39 @@ describe("SVG xlink:href context", () => { ]); }); - it("stops SVG context at foreignObject and resumes it for a nested svg", () => { - const result = captureSvg( - svgDocument([ - { name: "svg", parent: 0 }, - { name: "foreignObject", parent: 1 }, - { - name: "A", - parent: 2, - attributes: [["xlink:href", "/xl-html-fo"]], - }, - { name: "svg", parent: 2 }, - { - name: "a", - parent: 4, - attributes: [["xlink:href", "/xl-nested"]], - }, - ]), - ); - expect(urlFor(result, 3)).toBeUndefined(); - expect(urlFor(result, 5)).toMatchObject({ - attribute: "href", - url: `${origin}/xl-nested`, - destination_scope: "approved", - }); - }); + it.each(["foreignObject", "desc", "title"])( + "preserves browser SVG identity across %s HTML integration points", + (integrationPoint) => { + const result = captureSvg( + svgDocument([ + { name: "svg", parent: 0 }, + { name: integrationPoint, parent: 1 }, + { + name: "A", + svg: false, + parent: 2, + attributes: [["xlink:href", "/xl-html-fo"]], + }, + { name: "svg", parent: 2 }, + { + name: "a", + svg: true, + parent: 4, + attributes: [["xlink:href", "/xl-nested"]], + }, + ]), + ); + expect(urlFor(result, 3)).toBeUndefined(); + expect(urlFor(result, 5)).toMatchObject({ + attribute: "href", + url: `${origin}/xl-nested`, + destination_scope: "approved", + }); + }, + ); +}); +describe("SVG XLink precedence and unknown identity", () => { it("keeps an empty href ahead of xlink:href", () => { const baseUrl = `${origin}/screens/assets/`; const result = captureSvg( @@ -541,6 +737,7 @@ describe("SVG xlink:href context", () => { { name: "svg", parent: 0 }, { name: "a", + svg: true, parent: 1, attributes: [ ["href", ""], @@ -567,6 +764,7 @@ describe("SVG xlink:href context", () => { { name: "svg", parent: 0 }, { name: "a", + svg: true, parent: 1, attributes: [["xlink:href", "javascript:void(0)"]], }, @@ -582,7 +780,7 @@ describe("SVG xlink:href context", () => { ]); }); - it("does not treat a cyclic or dangling parent chain as SVG", () => { + it("keeps an unavailable SVG identity unknown despite ancestor names", () => { const cyclic = captureSvg( svgDocument([ { name: "g", parent: 2 }, @@ -652,7 +850,12 @@ describe("XML SVG roots and namespace scope restoration", () => { ["xmlns:p", XLINK_NAMESPACE], ], }, - { name: "a", parent: 1, attributes: [["p:href", "/linked"]] }, + { + name: "a", + svg: true, + parent: 1, + attributes: [["p:href", "/linked"]], + }, ]), ); expect(result.urls).toEqual([approvedHref(2, `${origin}/linked`)]); @@ -764,6 +967,7 @@ describe("XHTML and XML SVG namespace prefixes", () => { { name: "svg:foreignObject", parent: 2 }, { name: "a", + svg: true, parent: 3, attributes: [["xlink:href", "/xl-html-fo"]], }, @@ -799,6 +1003,7 @@ describe("XHTML and XML SVG namespace prefixes", () => { }, { name: "a", + svg: true, parent: 2, attributes: [ ["xlink:href", "/not-xlink"], @@ -918,11 +1123,13 @@ describe("XHTML SVG exclusions, HTML documents, and prefix scope", () => { }, { name: "a", + svg: true, parent: 2, attributes: [["xlink:href", "/xl-literal"]], }, { name: "a", + svg: true, parent: 2, attributes: [["x:href", "/xl-custom-html"]], }, @@ -937,6 +1144,7 @@ describe("XHTML SVG exclusions, HTML documents, and prefix scope", () => { }, { name: "A", + svg: false, parent: 1, attributes: [["xlink:href", "/xl-html-anchor"]], }, @@ -944,7 +1152,9 @@ describe("XHTML SVG exclusions, HTML documents, and prefix scope", () => { ); expect(result.urls).toEqual([approvedHref(3, `${origin}/xl-literal`)]); }); +}); +describe("XML namespace resolution ordering and capacity", () => { it("applies an element's own xmlns declarations to its name and href", () => { const result = captureSvg( svgDocument([ diff --git a/src/browser/CdpCaptureDocuments.ts b/src/browser/CdpCaptureDocuments.ts index 3c7fb98ed..f7eb4fe31 100644 --- a/src/browser/CdpCaptureDocuments.ts +++ b/src/browser/CdpCaptureDocuments.ts @@ -11,7 +11,11 @@ import { } from "./CdpCaptureValues.js"; import { exclusionReasonForUrl } from "./CdpCaptureEventHelpers.js"; import type { CdpCaptureCompleteness } from "./CdpCaptureCompleteness.js"; -import { createDocumentSvgAncestry, domSvgLink } from "./CdpCaptureDomSvg.js"; +import { + createDocumentSvgAncestry, + domSvgLink, + type DomSvgNodeQuery, +} from "./CdpCaptureDomSvg.js"; import { compareUnicodeCodePoints } from "../domain/unicodeCodePointOrder.js"; export type CapturedResource = Omit< @@ -135,18 +139,39 @@ export const walkFrameTrees = function* ( } }; -export const captureDom = ( +interface DomMetadataCapture { + readonly urls: WebPageInspection["metadata"]["dom_urls"]; + readonly agentHints: WebPageInspection["metadata"]["agent_hints"]; + readonly excludedUrls: number; + readonly unresolvedSvgNodes: number; +} + +export interface DomCapture extends DomMetadataCapture { + readonly total: number; + readonly nodes: WebPageInspection["dom"]["nodes"]; +} + +// Bound retained projections and outstanding CDP reads independently of page size. +// Eight leaves room for the session's other cancellation listeners. +const DOM_METADATA_BATCH_SIZE = 8; + +interface PendingDomMetadata { + readonly options: DomMetadataOptions; + // Undefined means no query is needed; null means the candidate has no valid ID. + readonly svgQuery: DomSvgNodeQuery | null | undefined; +} + +/** Project one snapshot, requesting bounded batches of SVG identity facts. */ +export function* captureDom( result: unknown, allowedOrigins: ReadonlySet, input: InspectWebPageInput, completeness?: CdpCaptureCompleteness, -): { - readonly total: number; - readonly nodes: WebPageInspection["dom"]["nodes"]; - readonly urls: WebPageInspection["metadata"]["dom_urls"]; - readonly agentHints: WebPageInspection["metadata"]["agent_hints"]; - readonly excludedUrls: number; -} => { +): Generator< + readonly DomSvgNodeQuery[], + DomCapture, + readonly (boolean | undefined)[] +> { const root = requiredRecord(result); const strings = Array.isArray(root.strings) ? root.strings.map((value) => cdpStringValue(value) ?? "") @@ -156,6 +181,7 @@ export const captureDom = ( const agentHints: WebPageInspection["metadata"]["agent_hints"] = []; let excludedUrls = 0; let total = 0; + let unresolvedSvgNodes = 0; for (const document of recordsValue(root.documents)) { const documentUrl = indexedString(strings, document.documentURL); if (allowedSanitizedUrl(documentUrl, allowedOrigins) === undefined) { @@ -172,6 +198,9 @@ export const captureDom = ( const attributes = Array.isArray(documentNodes.attributes) ? documentNodes.attributes : []; + const backendNodeIds = Array.isArray(documentNodes.backendNodeId) + ? documentNodes.backendNodeId + : []; const baseIndex = nodes.length; const ancestry = createDocumentSvgAncestry({ nodeTypes, @@ -181,12 +210,29 @@ export const captureDom = ( attributes, }); total += nodeTypes.length; + const pendingMetadata: PendingDomMetadata[] = []; for (let index = 0; index < nodeTypes.length; index += 1) { const attributeIndexes = numberArray(attributes[index]); const parent = Math.trunc(parents[index] ?? -1); const nodeIndex = nodes.length; const nodeName = indexedString(strings, nodeNames[index]); const attributeNames = uniqueAttributeNames(attributeIndexes, strings); + const svgLink = domSvgLink(ancestry, index); + let svgQuery: DomSvgNodeQuery | null | undefined; + if ( + !svgLink.xmlDocument && + nodeTypes[index] === 1 && + attributeNames.includes("xlink:href") && + !attributeNames.includes("href") + ) { + const backendNodeId = numberValue(backendNodeIds[index]); + svgQuery = + backendNodeId !== undefined && + Number.isSafeInteger(backendNodeId) && + backendNodeId > 0 + ? { backendNodeId, nodeName } + : null; + } nodes.push({ index: nodeIndex, parent_index: parent < 0 ? -1 : baseIndex + parent, @@ -195,30 +241,34 @@ export const captureDom = ( node_value_length: indexedString(strings, nodeValues[index]).length, attribute_names: attributeNames, }); - const metadata = domMetadata({ - strings, - attributes: attributeIndexes, - // The BASE element itself resolves against the document fallback URL. - baseUrl: nodeName.toLowerCase() === "base" ? documentUrl : baseUrl, - documentUrl, - nodeIndex, - nodeName, - allowedOrigins, - ...domSvgLink(ancestry, index), + pendingMetadata.push({ + svgQuery, + options: { + strings, + attributes: attributeIndexes, + // The BASE element itself resolves against the document fallback URL. + baseUrl: nodeName.toLowerCase() === "base" ? documentUrl : baseUrl, + documentUrl, + nodeIndex, + nodeName, + allowedOrigins, + ...svgLink, + }, }); - for (const url of metadata.urls) { - urls.push(url); - if (url.destination_scope !== "approved") { - excludedUrls += 1; - completeness?.exclude( - "metadata", - url.destination_scope === "outside_policy" - ? "disallowed_origin" - : "unsupported_url", - ); - } + if ( + pendingMetadata.length === DOM_METADATA_BATCH_SIZE || + index === nodeTypes.length - 1 + ) { + const metadata = yield* captureDomMetadata( + pendingMetadata, + completeness, + ); + urls.push(...metadata.urls); + agentHints.push(...metadata.agentHints); + excludedUrls += metadata.excludedUrls; + unresolvedSvgNodes += metadata.unresolvedSvgNodes; + pendingMetadata.length = 0; } - agentHints.push(...metadata.agentHints); } } return { @@ -227,8 +277,54 @@ export const captureDom = ( urls, agentHints, excludedUrls, + unresolvedSvgNodes, }; -}; +} + +function* captureDomMetadata( + pending: readonly PendingDomMetadata[], + completeness: CdpCaptureCompleteness | undefined, +): Generator< + readonly DomSvgNodeQuery[], + DomMetadataCapture, + readonly (boolean | undefined)[] +> { + const queries = pending.flatMap(({ svgQuery }) => + svgQuery == null ? [] : [svgQuery], + ); + const kinds = queries.length > 0 ? yield queries : []; + const urls: DomMetadataCapture["urls"] = []; + const agentHints: DomMetadataCapture["agentHints"] = []; + let queryIndex = 0; + let unresolvedSvgNodes = 0; + let excludedUrls = 0; + for (const { options, svgQuery } of pending) { + let svgElement = options.svgElement; + if (svgQuery !== undefined) { + const kind = svgQuery === null ? undefined : kinds[queryIndex++]; + svgElement = kind === true; + if (kind === undefined) { + unresolvedSvgNodes += 1; + completeness?.exclude("metadata", "provider_unavailable"); + } + } + const metadata = domMetadata({ ...options, svgElement }); + for (const url of metadata.urls) { + urls.push(url); + if (url.destination_scope !== "approved") { + excludedUrls += 1; + completeness?.exclude( + "metadata", + url.destination_scope === "outside_policy" + ? "disallowed_origin" + : "unsupported_url", + ); + } + } + agentHints.push(...metadata.agentHints); + } + return { urls, agentHints, unresolvedSvgNodes, excludedUrls }; +} export const captureAccessibility = ( results: readonly unknown[], diff --git a/src/browser/CdpCaptureDomSvg.ts b/src/browser/CdpCaptureDomSvg.ts index 3d1a5c977..4d021766c 100644 --- a/src/browser/CdpCaptureDomSvg.ts +++ b/src/browser/CdpCaptureDomSvg.ts @@ -1,4 +1,4 @@ -import { numberValue } from "./CdpCaptureValues.js"; +import { numberValue, recordValue } from "./CdpCaptureValues.js"; const SVG_NAMESPACE = "http://www.w3.org/2000/svg"; const XLINK_NAMESPACE = "http://www.w3.org/1999/xlink"; @@ -18,11 +18,34 @@ interface XmlSvgLink { interface DocumentSvgAncestry { readonly xml: boolean; - readonly nodes: DomSnapshotNodes; - readonly svgContexts: Map; readonly xmlLinks: Map; } +export interface DomSvgNodeQuery { + readonly backendNodeId: number; + readonly nodeName: string; +} + +/** Bind the browser's SVG flag to the exact snapshotted element. */ +export const capturedSvgNodeKind = ( + query: DomSvgNodeQuery, + result: unknown, +): boolean | undefined => { + const node = recordValue(recordValue(result)?.node); + if ( + node?.backendNodeId !== query.backendNodeId || + node.nodeName !== query.nodeName || + node.nodeType !== 1 + ) + return undefined; + // CDP omits isSVG for non-SVG nodes. Other values are malformed, not false. + return node.isSVG === undefined + ? false + : typeof node.isSVG === "boolean" + ? node.isSVG + : undefined; +}; + interface DomSvgLink { readonly svgElement: boolean; readonly xmlDocument: boolean; @@ -79,8 +102,6 @@ export const createDocumentSvgAncestry = ( const xml = xmlDocument(nodes); return { xml, - nodes, - svgContexts: new Map(), xmlLinks: xml ? resolveXmlElements(nodes) : new Map(), }; }; @@ -91,11 +112,7 @@ export const domSvgLink = ( ): DomSvgLink => { if (!ancestry.xml) { return { - svgElement: elementInSvgContext( - index, - ancestry.nodes, - ancestry.svgContexts, - ), + svgElement: false, xmlDocument: false, xlinkHref: undefined, }; @@ -108,43 +125,6 @@ export const domSvgLink = ( }; }; -// Chrome's HTML parser records SVG local names (`svg`, `foreignObject`). -// Descendants of foreignObject are HTML until a nested `svg`. -const elementInSvgContext = ( - index: number, - nodes: DomSnapshotNodes, - contexts: Map, -): boolean => { - let current = index; - const seen = new Set(); - let inSvg = false; - while ( - current >= 0 && - current < nodes.nodeNames.length && - !seen.has(current) - ) { - const cached = contexts.get(current); - if (cached !== undefined) { - inSvg = cached; - break; - } - seen.add(current); - if (stringAt(nodes.strings, nodes.nodeNames[current]) === "svg") { - inSvg = true; - break; - } - const parent = Math.trunc(nodes.parents[current] ?? -1); - if (parent < 0 || parent === current) break; - if (stringAt(nodes.strings, nodes.nodeNames[parent]) === "foreignObject") - break; - current = parent; - } - // Cache the entire examined chain, including forward references and cycles, - // so shared ancestors are examined once per document. - for (const node of seen) contexts.set(node, inSvg); - return inSvg; -}; - interface NamespaceRestore { readonly prefix: string; readonly previous: string | undefined; diff --git a/src/browser/CdpPageCapture.ts b/src/browser/CdpPageCapture.ts index 0d97576b5..f49d4e186 100644 --- a/src/browser/CdpPageCapture.ts +++ b/src/browser/CdpPageCapture.ts @@ -17,8 +17,10 @@ import { captureFrames, captureResources, type CapturedResource, + type DomCapture, mainFrameUrl, } from "./CdpCaptureDocuments.js"; +import { capturedSvgNodeKind } from "./CdpCaptureDomSvg.js"; import { allowedSanitizedUrl, delayWithCancellation, @@ -52,7 +54,7 @@ interface CaptureState { interface CapturedSections { readonly attachedUrl: string; readonly frames: WebPageInspection["frames"]; - readonly dom: ReturnType; + readonly dom: DomCapture; readonly accessibility: ReturnType; readonly scripts: WebPageInspection["scripts"]; readonly resources: WebPageInspection["resources"]; @@ -260,9 +262,7 @@ const authorizeObservationWindow = async ( throw new BrowserObservationError("inspect_web_page", "target_not_allowed"); }; -const capturePageDom = async ( - state: CaptureState, -): Promise> => { +const capturePageDom = async (state: CaptureState): Promise => { const { context, allowedOrigins } = state; const result = await context.connection.send( "DOMSnapshot.captureSnapshot", @@ -274,12 +274,56 @@ const capturePageDom = async ( context.sessionId, context.signal, ); - return captureDom( + const capture = captureDom( result, allowedOrigins, context.input, state.events.completeness, ); + let step = capture.next(); + let descriptionUnavailable = false; + const pendingController = new AbortController(); + const signal = + context.signal === undefined + ? pendingController.signal + : AbortSignal.any([context.signal, pendingController.signal]); + while (!step.done) { + const pending = step.value.map(async (query) => { + if (descriptionUnavailable) return undefined; + try { + return capturedSvgNodeKind( + query, + await context.connection.send( + "DOM.describeNode", + { backendNodeId: query.backendNodeId, depth: 0 }, + context.sessionId, + signal, + ), + ); + } catch (cause: unknown) { + // A detached node can disappear between snapshot and description. + // Preserve partial metadata, but propagate transport/cancellation errors. + if (!(cause instanceof CdpCommandRejection)) throw cause; + if (cause.code === -32_601) descriptionUnavailable = true; + else if (cause.code !== -32_000) throw cause; + return undefined; + } + }); + try { + // Promise.all preserves snapshot order even when CDP replies out of order. + step = capture.next(await Promise.all(pending)); + } catch (cause: unknown) { + // Stop and settle this bounded batch before the session can be detached. + pendingController.abort(); + await Promise.allSettled(pending); + throw cause; + } + } + if (step.value.unresolvedSvgNodes > 0) + state.limitations.push( + `SVG identity was unavailable for ${String(step.value.unresolvedSvgNodes)} snapshotted XLink candidate nodes; their URL destinations remain unknown.`, + ); + return step.value; }; const normalizedInspection = ( diff --git a/src/cli/cliResultOutput.ts b/src/cli/cliResultOutput.ts index 67e745b0b..5efde8b33 100644 --- a/src/cli/cliResultOutput.ts +++ b/src/cli/cliResultOutput.ts @@ -216,8 +216,9 @@ export const createCliResultOutput = ( : filtered === undefined ? { ok: true, meta } : { ok: true, data: filtered, meta } - : filtered === undefined || - filtered === null || + : // A selected null is a value; only a selection with no JSON value + // falls back to the empty document. + filtered === undefined || typeof filtered === "function" || typeof filtered === "symbol" ? {} diff --git a/src/cliOutput.ts b/src/cliOutput.ts index 5119eebc3..ccfaf6b62 100644 --- a/src/cliOutput.ts +++ b/src/cliOutput.ts @@ -9,7 +9,7 @@ export type CliOutputArgumentValidation = | { readonly ok: false; readonly format: StructuredOutputFormat; - readonly code: "UNSUPPORTED_OUTPUT_COMBINATION"; + readonly code: "UNSUPPORTED_OUTPUT_COMBINATION" | "INVALID_OUTPUT_FILTER"; readonly message: string; }; @@ -89,13 +89,39 @@ export const parseCliOutputArguments = ( return parsed(false); }; -/** Reject text-window controls that would corrupt a structured document. */ +/** Reject slices that cannot advance within Incur's comma-delimited paths. */ +const hasUnclosedFilterSlice = (expression: string): boolean => { + let depth = 0; + let unclosed = false; + for (const character of expression) { + if (character === "[") { + depth += 1; + unclosed = true; + } else if (character === "]") { + depth -= 1; + // Incur consumes everything through the next closing bracket, including + // nested opening brackets. Do not impose a different filter grammar. + unclosed = false; + } else if (character === "," && depth === 0 && unclosed) return true; + } + return unclosed; +}; + +/** Reject output controls that cannot safely produce a complete result. */ export const validateCliOutputArguments = ( arguments_: readonly string[], ): CliOutputArgumentValidation => { - const { format, parseError, tokenWindow } = + const { filterOutput, format, parseError, tokenWindow } = parseCliOutputArguments(arguments_); if (parseError) return { ok: true }; + if (filterOutput !== undefined && hasUnclosedFilterSlice(filterOutput)) + return { + ok: false, + format: format === "yaml" || format === "jsonl" ? format : "json", + code: "INVALID_OUTPUT_FILTER", + message: + "The --filter-output expression has a slice without a closing ]. Close the slice or remove --filter-output.", + }; if ( tokenWindow && (format === "json" || format === "jsonl" || format === "yaml") diff --git a/src/contracts/native/nativeToolContracts.ts b/src/contracts/native/nativeToolContracts.ts index 16101368b..d45b62cc6 100644 --- a/src/contracts/native/nativeToolContracts.ts +++ b/src/contracts/native/nativeToolContracts.ts @@ -107,7 +107,7 @@ export const NATIVE_TOOL_CONTRACTS = [ ), native( "inspect_signature", - "Inspect the active artifact's code-signing identity, hashes, authorities, requirements, entitlements, hardened-runtime state, and exact command provenance.", + "Inspect the active Mach-O executable's code-signing identity, hashes, authorities, requirements, entitlements, hardened-runtime state, and exact command provenance.", z.object({}), ), native( diff --git a/src/contracts/officialToolContracts.ts b/src/contracts/officialToolContracts.ts index 82f1dc83d..4750528ad 100644 --- a/src/contracts/officialToolContracts.ts +++ b/src/contracts/officialToolContracts.ts @@ -149,12 +149,12 @@ export const OFFICIAL_TOOL_CONTRACTS = [ ), official( "procedure_callees", - "Return the provider's resolved direct callees for one procedure identified by symbol or address. Unresolved indirect calls may be absent; use analyze_function and typed references to preserve available edge uncertainty.", + "Return the provider's resolved direct callees for one procedure identified by symbol or address. Unresolved indirect calls may be absent; use analyze_function and typed references to preserve available edge uncertainty. Ghidra terminal-call assumptions that may cut the analyzed body are reported in Evidence limitations.", z.object({ procedure, document }), ), official( "procedure_callers", - "Return the provider's resolved direct callers for one procedure identified by symbol or address. Results reflect completed static analysis and may omit unresolved indirect references.", + "Return the provider's resolved direct callers for one procedure identified by symbol or address. Results reflect completed static analysis and may omit unresolved indirect references. Ghidra terminal-call assumptions that may cut the analyzed body are reported in Evidence limitations.", z.object({ procedure, document }), ), official( diff --git a/src/contracts/toolOutputReferences.test.ts b/src/contracts/toolOutputReferences.test.ts index cda49f1d5..1cd890bf3 100644 --- a/src/contracts/toolOutputReferences.test.ts +++ b/src/contracts/toolOutputReferences.test.ts @@ -6,6 +6,34 @@ import { z } from "zod"; import { toolContract } from "./toolContracts.js"; import { toolOutputSchemaWithMetadata } from "./toolSchemaMetadata.js"; +import { createEvidence } from "../domain/evidence.js"; + +it.each(["draft-2020-12", "draft-07"] as const)( + "advertises the raw-only 6502 Evidence subject constraint in %s", + (target) => { + const valid = createEvidence( + { + path: "/fixture/image.bin", + sha256: "a".repeat(64), + format: "raw-binary", + architecture: "6502", + }, + { id: "fixture", name: "Fixture", version: "1" }, + { operation: "address_name", parameters: {}, result: "entry" }, + ); + const advertised = toolOutputSchemaWithMetadata( + toolContract("address_name"), + )["~standard"].jsonSchema.output({ target }); + const validate = new AjvJsonSchemaValidator().getValidator(advertised); + expect(validate(valid).valid).toBe(true); + expect( + validate({ + ...valid, + subject: { ...valid.subject, format: "pe" }, + }).valid, + ).toBe(false); + }, +); it("keeps dialect-specific validators distinct after JSON schema serialization", () => { const contract = { diff --git a/src/contracts/toolOutputSchemaGroups.ts b/src/contracts/toolOutputSchemaGroups.ts index dcf7ad033..76294e2db 100644 --- a/src/contracts/toolOutputSchemaGroups.ts +++ b/src/contracts/toolOutputSchemaGroups.ts @@ -444,11 +444,11 @@ const lifecycleTargetFields = { path: z.string(), kind: targetKindSchema, sha256: z.string().regex(/^[a-f0-9]{64}$/u), - architecture: z.enum(BINARY_ARCHITECTURES).nullable(), }; const detectedLifecycleTargetSchema = z.object({ ...lifecycleTargetFields, format: targetFormatSchema.exclude(["raw-binary"]), + architecture: z.enum(BINARY_ARCHITECTURES).exclude(["6502"]).nullable(), }); const rawLifecycleTargetSchema = z.object({ ...lifecycleTargetFields, diff --git a/src/domain/analysisSnapshot.test.ts b/src/domain/analysisSnapshot.test.ts index ee3920be5..d83ddac30 100644 --- a/src/domain/analysisSnapshot.test.ts +++ b/src/domain/analysisSnapshot.test.ts @@ -52,7 +52,7 @@ const snapshotWithHistoricalEvidence = () => { }; return { current, - snapshot: { + snapshot: analysisSnapshotSchema.parse({ target, binding, entries: [entry], @@ -66,11 +66,32 @@ const snapshotWithHistoricalEvidence = () => { result: { summary: "unprofiled" }, }), ]), - } satisfies AnalysisSnapshot, + }), }; }; describe("analysis snapshot target identity", () => { + it("keeps 6502 confined to executable raw snapshots", () => { + const target = { + sha256: ANALYSIS_SNAPSHOT_TARGET.sha256, + kind: "executable", + architecture: "6502", + }; + expect( + analysisSnapshotSchema.shape.target.safeParse({ + ...target, + format: "pe", + }).success, + ).toBe(false); + expect( + analysisSnapshotSchema.shape.target.safeParse({ + ...target, + format: "raw-binary", + entry_point: "0x8001", + }).success, + ).toBe(true); + }); + it("partitions raw snapshots and query identities by the canonical entry", () => { const target: BinaryTarget = { path: "/fixture/image.bin", @@ -120,6 +141,48 @@ describe("analysis snapshot target identity", () => { }); }); +describe("analysis snapshot nested subject contract", () => { + it("rejects raw-only 6502 in nested detected execution subjects", () => { + const { snapshot } = snapshotWithHistoricalEvidence(); + const entry = snapshot.entries[0]; + if (entry === undefined) throw new Error("Fixture has no snapshot entry"); + const malformedSubject = { + ...entry.execution.subject, + format: "pe", + architecture: "6502", + }; + expect( + analysisSnapshotSchema.safeParse({ + ...snapshot, + entries: [ + { + ...entry, + execution: { + ...entry.execution, + subject: malformedSubject, + }, + }, + ], + }).success, + ).toBe(false); + expect( + analysisSnapshotSchema.safeParse({ + ...snapshot, + workflow_entries: [ + { + ...entry, + execution: { + ...entry.execution, + analysis_profile: snapshot.binding.analysis_profile, + subject: malformedSubject, + }, + }, + ], + }).success, + ).toBe(false); + }); +}); + describe("analysis snapshot contract", () => { it.each([ ["dos-mz", "x86"], diff --git a/src/domain/analysisSnapshot.ts b/src/domain/analysisSnapshot.ts index 57e876653..4e1af5fa8 100644 --- a/src/domain/analysisSnapshot.ts +++ b/src/domain/analysisSnapshot.ts @@ -41,7 +41,10 @@ import { digestSchema } from "./../domain/digests.js"; import { prefixedDigestSchema } from "./../domain/digests.js"; import { canonicalJsonDigestSteps } from "./canonicalJsonDigestSteps.js"; import { freezeJsonSnapshot } from "./immutableJson.js"; -import { rawBinaryEntryPointSchema } from "./rawBinary.js"; +import { + binaryArchitectureSchema, + rawBinaryEntryPointSchema, +} from "./rawBinary.js"; import { relationshipCoverageSchema, type RelationshipCoverage, @@ -73,20 +76,30 @@ const formatSchema = z.enum([ "source-map", ]); const kindSchema = z.enum(["executable", "database", "archive", "artifact"]); -const subjectSchema = z.object({ +const subjectBaseSchema = z.object({ path: localPathStringSchema, sha256: digestSchema, - format: z.enum([ - ...formatSchema.options, - "hopper", - "directory", - "file", - "unknown", - "javascript-bundle", - "entitlements", - ]), - architecture: architectureSchema.nullable(), }); +const subjectFormatSchema = z.enum([ + ...formatSchema.options, + "hopper", + "directory", + "file", + "unknown", + "javascript-bundle", + "entitlements", +]); +// Snapshot entries retain the same raw-only architecture boundary as targets. +const subjectSchema = z.discriminatedUnion("format", [ + subjectBaseSchema.extend({ + format: subjectFormatSchema.exclude(["raw-binary"]), + architecture: architectureSchema.exclude(["6502"]).nullable(), + }), + subjectBaseSchema.extend({ + format: z.literal("raw-binary"), + architecture: architectureSchema.nullable(), + }), +]); const targetIdentitySchema = z.object({ sha256: digestSchema, kind: kindSchema, @@ -97,9 +110,12 @@ const targetSchema = z.discriminatedUnion("format", [ format: z.enum( formatSchema.options.filter((format) => format !== "raw-binary"), ), + architecture: architectureSchema.exclude(["6502"]).nullable(), }), targetIdentitySchema.extend({ format: z.literal("raw-binary"), + kind: z.literal("executable"), + architecture: binaryArchitectureSchema, entry_point: rawBinaryEntryPointSchema, }), ]); @@ -332,18 +348,21 @@ const canonicalJson = (value: unknown): string => { }; /** Project a binary into the path-free identity used for cache invalidation. */ -export const snapshotTarget = ( - target: BinaryTarget, -): AnalysisSnapshotTarget => { - const identity = { - sha256: target.sha256, - kind: target.kind, - architecture: target.architecture ?? null, - }; - return target.format === "raw-binary" - ? { ...identity, format: target.format, entry_point: target.entryPoint } - : { ...identity, format: target.format }; -}; +export const snapshotTarget = (target: BinaryTarget): AnalysisSnapshotTarget => + target.format === "raw-binary" + ? { + sha256: target.sha256, + kind: target.kind, + architecture: target.architecture, + format: target.format, + entry_point: target.entryPoint, + } + : { + sha256: target.sha256, + kind: target.kind, + architecture: target.architecture ?? null, + format: target.format, + }; /** Build the immutable provider/profile binding used by a cached snapshot. */ export const snapshotBinding = ( @@ -475,10 +494,10 @@ export const createAnalysisSnapshotWorkflowEntry = (input: { subject: execution.subject === null ? null - : { + : subjectSchema.parse({ ...execution.subject, architecture: execution.subject.architecture ?? null, - }, + }), }, }; }; @@ -538,10 +557,10 @@ export const createAnalysisSnapshotEntry = (input: { subject: execution.subject === null ? null - : { + : subjectSchema.parse({ ...execution.subject, architecture: execution.subject.architecture ?? null, - }, + }), }, }; }; diff --git a/src/domain/analysisView/analysisView.test.ts b/src/domain/analysisView/analysisView.test.ts index f1aa3e829..07e6909c3 100644 --- a/src/domain/analysisView/analysisView.test.ts +++ b/src/domain/analysisView/analysisView.test.ts @@ -169,7 +169,8 @@ it("pages every retained dossier collection and the native API record", () => { | "comments" | "unresolved_calls" | "referenced_strings" - | "referenced_names", + | "referenced_names" + | "swift_string_literals", result: unknown = original, ) => { const view = projectAnalysisView(nativeParent(result), { @@ -195,6 +196,23 @@ it("pages every retained dossier collection and the native API record", () => { }), ).toMatchObject({ item: [{ text: "entry" }] }); expect(page("referenced_names")).toMatchObject({ coverage: { total: 0 } }); + const literal = { + source_address: "0x401004", + value: "SleepDisabled", + representation: "small", + address: null, + registers: ["x0", "x1"], + words: ["0x7369447065656c53", "0xed000064656c6261"], + }; + expect( + page("swift_string_literals", { + ...original, + swift_string_literals: [literal], + }), + ).toMatchObject({ item: [literal], coverage: { total: 1 } }); + expect(page("swift_string_literals")).toMatchObject({ + item: { available: false, reason: "not recorded" }, + }); const { unresolved_calls: _omitted, ...withoutUnresolved } = original; expect( page("unresolved_calls", { diff --git a/src/domain/analysisView/analysisView.ts b/src/domain/analysisView/analysisView.ts index 7f985f349..b08001e35 100644 --- a/src/domain/analysisView/analysisView.ts +++ b/src/domain/analysisView/analysisView.ts @@ -160,6 +160,8 @@ const nativeViewSchema = z.strictObject({ "unresolved_calls", "referenced_strings", "referenced_names", + "swift_string_literals", + "swift_string_literal_omissions", "native_api", "value_flow_summary", "value_flow_operations", diff --git a/src/domain/analysisView/nativeFunctionView.ts b/src/domain/analysisView/nativeFunctionView.ts index f5a29e0e1..b56de24a1 100644 --- a/src/domain/analysisView/nativeFunctionView.ts +++ b/src/domain/analysisView/nativeFunctionView.ts @@ -79,6 +79,8 @@ const nativeRows = ( unresolved_calls: dossier.unresolved_calls, referenced_strings: dossier.referenced_strings, referenced_names: dossier.referenced_names, + swift_string_literals: dossier.swift_string_literals, + swift_string_literal_omissions: dossier.swift_string_literal_omissions, value_flow_operations: flow?.operations, value_flow_def_use: flow?.def_use, value_flow_effects: flow?.effects, @@ -163,7 +165,8 @@ export const projectNativeFunctionView = ( singleton({ available: false, // Only value-flow facets inherit the value-flow failure reason; an - // older dossier simply has no unresolved_calls record. + // older dossier simply has no unresolved_calls record, and a target + // whose provider does not decode Swift literals has no such record. reason: view.facet.startsWith("value_flow_") && flow?.available === false ? flow.reason diff --git a/src/domain/apple/interfaceBuilderGraph.test.ts b/src/domain/apple/interfaceBuilderGraph.test.ts index bfa2c71de..96c6b4915 100644 --- a/src/domain/apple/interfaceBuilderGraph.test.ts +++ b/src/domain/apple/interfaceBuilderGraph.test.ts @@ -510,6 +510,106 @@ describe("AppKit keyed archive connectors", () => { }); }); +describe("keyed archive unresolved connectors", () => { + // A rooted UIKit archive whose only connector is UID 3, with UID 1 as the + // owner, UID 2 as the destination and UID 7 as the "view" label. + const rootedOutletArchive = (connection: Record) => ({ + $archiver: "NSKeyedArchiver", + $version: 100000, + $objects: [ + "$null", + { $class: { CF$UID: 4 } }, + { $class: { CF$UID: 5 } }, + { $class: { CF$UID: 6 }, ...connection }, + { + $classname: "UIViewController", + $classes: ["UIViewController", "UIResponder", "NSObject"], + }, + { $classname: "UIView", $classes: ["UIView", "UIResponder", "NSObject"] }, + { + $classname: "UIRuntimeOutletConnection", + $classes: [ + "UIRuntimeOutletConnection", + "UIRuntimeConnection", + "NSObject", + ], + }, + "view", + { + $class: { CF$UID: 11 }, + "NS.objects": [{ CF$UID: 1 }, { CF$UID: 2 }, { CF$UID: 3 }], + }, + { $class: { CF$UID: 11 }, "NS.objects": [{ CF$UID: 3 }] }, + { + $class: { CF$UID: 11 }, + "NS.objects": [{ CF$UID: 1 }, { CF$UID: 2 }], + }, + { $classname: "NSArray", $classes: ["NSArray", "NSObject"] }, + ], + $top: { + root: { CF$UID: 8 }, + UINibConnectionsKey: { CF$UID: 9 }, + UINibObjectsKey: { CF$UID: 10 }, + UINibTopLevelObjectsKey: { CF$UID: 10 }, + }, + }); + + const project = (raw: unknown) => + buildInterfaceBuilderAnalysis({ + targetSha256: hash, + toolVersion: "test", + documents: [ + { + relativePath: "Main.nib", + archiveSha256: hash, + documentKind: "nib", + raw, + }, + ], + limits: interfaceBuilderLimitsSchema.parse({}), + }); + + it("covers a resolvable connector as a complete examined connection", () => { + const raw = rootedOutletArchive({ + UISource: { CF$UID: 1 }, + UIDestination: { CF$UID: 2 }, + UILabel: { CF$UID: 7 }, + }); + expect(parseInterfaceBuilderRecords(raw).unresolvedConnections).toBe(0); + expect(project(raw).graph.coverage).toContainEqual( + expect.objectContaining({ + facet: "connections:Main.nib", + status: "complete", + examined: 1, + omitted: 0, + }), + ); + }); + + it("reports a recognized connector with no resolvable owner as unresolved", () => { + const raw = rootedOutletArchive({ + UIDestination: { CF$UID: 2 }, + UILabel: { CF$UID: 7 }, + }); + expect(parseInterfaceBuilderRecords(raw).unresolvedConnections).toBe(1); + + const analysis = project(raw); + expect(analysis.graph.nodes.some(({ kind }) => kind === "outlet")).toBe( + false, + ); + expect(analysis.graph.coverage).toContainEqual( + expect.objectContaining({ + facet: "connections:Main.nib", + status: "partial", + reason: "connection_unresolved", + examined: 0, + omitted: 1, + }), + ); + expect(analysis.graph.truncated).toBe(true); + }); +}); + it("names the archive once per evidence record", () => { const relativePath = "Views/Main.storyboardc/scene.nib/objects.nib"; const result = buildInterfaceBuilderAnalysis({ diff --git a/src/domain/apple/interfaceBuilderGraph.ts b/src/domain/apple/interfaceBuilderGraph.ts index f25c8b9b1..b3e5216dc 100644 --- a/src/domain/apple/interfaceBuilderGraph.ts +++ b/src/domain/apple/interfaceBuilderGraph.ts @@ -410,18 +410,26 @@ export const buildInterfaceBuilderAnalysis = (input: { const connectionsTruncated = parserOmittedConnections > 0 || parsed.connections.length > input.limits.max_connections; + // A recognized connector the parser could not attach to an owner produces + // no connection record, so its coverage is partial with its own reason + // rather than absent or truncated. + const connectionsUnresolved = parsed.unresolvedConnections > 0; omittedObjects += parsed.omittedObjects + Math.max(0, parsed.objects.length - documentObjectCount); omittedConnections += Math.max(0, parsed.connections.length - input.limits.max_connections) + - parserOmittedConnections; + parserOmittedConnections + + parsed.unresolvedConnections; summaries.push({ relative_path: document.relativePath, archive_sha256: document.archiveSha256, document_kind: document.documentKind, object_count: parsed.objectCount, - connection_count: parsed.connections.length + parserOmittedConnections, + connection_count: + parsed.connections.length + + parserOmittedConnections + + parsed.unresolvedConnections, hierarchy_complete: parsed.hierarchy.length > 0 && !truncated && @@ -440,13 +448,17 @@ export const buildInterfaceBuilderAnalysis = (input: { { facet: `connections:${document.relativePath}`, status: - connectionsTruncated || documentOmittedConnections > 0 + connectionsTruncated || + documentOmittedConnections > 0 || + connectionsUnresolved ? "partial" : "complete", reason: connectionsTruncated || documentOmittedConnections > 0 ? "connection_limit_reached" - : null, + : connectionsUnresolved + ? "connection_unresolved" + : null, examined: Math.min( parsed.connections.length, input.limits.max_connections, @@ -457,7 +469,8 @@ export const buildInterfaceBuilderAnalysis = (input: { parsed.connections.length - input.limits.max_connections, ) + documentOmittedConnections + - parserOmittedConnections, + parserOmittedConnections + + parsed.unresolvedConnections, }, { facet: `hierarchy:${document.relativePath}`, diff --git a/src/domain/apple/interfaceBuilderKeyedArchive.ts b/src/domain/apple/interfaceBuilderKeyedArchive.ts index 207950bf8..57ebda803 100644 --- a/src/domain/apple/interfaceBuilderKeyedArchive.ts +++ b/src/domain/apple/interfaceBuilderKeyedArchive.ts @@ -55,6 +55,7 @@ export const parseInterfaceBuilderRecords = ( readonly objectCount: number; readonly omittedObjects: number; readonly omittedConnections: number; + readonly unresolvedConnections: number; readonly omittedHierarchyReferences: number; } => { const root = record(value); @@ -135,6 +136,7 @@ export const parseInterfaceBuilderRecords = ( objectCount: objectEntries.length, omittedObjects: Math.max(0, objectEntries.length - 20_000), omittedConnections, + unresolvedConnections: 0, omittedHierarchyReferences: 0, }; }; @@ -320,6 +322,7 @@ const parseKeyedArchive = ( const connections: InterfaceBuilderConnection[] = []; let objectCount = 0; let omittedConnections = 0; + let unresolvedConnections = 0; for (const [index, raw] of objectTable.entries()) { if (index === 0) continue; const resolved = record(resolve(raw)); @@ -371,7 +374,13 @@ const parseKeyedArchive = ( controlAction ? fields.to : fields.from, controlAction ? fields.target : fields.owner, ); - if (source === null) continue; + if (source === null) { + // A recognized connector that cannot name its owner is an unresolved + // connection, not an absent one, and cannot be represented as a + // connection because connection identity requires a source. + unresolvedConnections += 1; + continue; + } const kind = controlAction ? "action" : classifyConnection(className); const parsedConnection = connection.parse({ id: `archive:${objectId}`, @@ -426,6 +435,7 @@ const parseKeyedArchive = ( objectCount, omittedObjects: Math.max(0, objectCount - 20_000), omittedConnections, + unresolvedConnections, omittedHierarchyReferences: hierarchy.omittedReferences, }; }; diff --git a/src/domain/apple/machoContainer.ts b/src/domain/apple/machoContainer.ts index 7b552be64..c23cf7f64 100644 --- a/src/domain/apple/machoContainer.ts +++ b/src/domain/apple/machoContainer.ts @@ -63,6 +63,11 @@ export const readFatContainerHeader = ( const magic = view.getUint32(0, false); if (![FAT_MAGIC, FAT_MAGIC_64, FAT_CIGAM, FAT_CIGAM_64].includes(magic)) return { state: "not-fat" }; + if (!Number.isSafeInteger(fileSize) || fileSize < bytes.byteLength) + return { + state: "malformed", + reason: "FAT file size is invalid or contradicts retained bytes", + }; if (bytes.byteLength < 8) { return fileSize < 8 ? { state: "malformed", reason: "FAT header is truncated" } diff --git a/src/domain/binaryTarget.ts b/src/domain/binaryTarget.ts index 48da65960..94828ee6e 100644 --- a/src/domain/binaryTarget.ts +++ b/src/domain/binaryTarget.ts @@ -278,8 +278,12 @@ export const parsePeRecord = ( record.toString("binary", 0, 4) !== "PE\u0000\u0000" ) return err("invalid or truncated PE header"); - const machine = decodePeMachine(record.readUInt16LE(4)); - if (machine === undefined) return err("unsupported PE architecture"); + const machineType = record.readUInt16LE(4); + const machine = decodePeMachine(machineType); + if (machine === undefined) + return err( + `unsupported PE architecture (COFF machine 0x${machineType.toString(16).padStart(4, "0")})`, + ); const { architecture } = machine; const optionalHeaderSize = record.readUInt16LE(20); if (optionalHeaderSize > 4096 || record.length < 24 + optionalHeaderSize) @@ -347,7 +351,9 @@ const machArchitecture = (cpu: number): HeaderArchitecture | undefined => { return undefined; }; -const elfArchitecture = (machine: number): BinaryArchitecture | undefined => { +const elfArchitecture = ( + machine: number, +): Exclude | undefined => { switch (machine) { case 3: return "x86"; diff --git a/src/domain/binaryTargetTypes.ts b/src/domain/binaryTargetTypes.ts index ab5046549..99ad8487f 100644 --- a/src/domain/binaryTargetTypes.ts +++ b/src/domain/binaryTargetTypes.ts @@ -1,12 +1,13 @@ import type { ZipPackageFormat } from "./zipPackageFormat.js"; -/** Provider-neutral CPU families detected from supported executable headers. */ +/** Provider-neutral CPU families detected from headers or explicitly selected raw loads. */ export const BINARY_ARCHITECTURES = [ "x86", "x86_64", "arm", "arm64", "mips", + "6502", "riscv", ] as const; export type BinaryArchitecture = (typeof BINARY_ARCHITECTURES)[number]; @@ -40,7 +41,7 @@ export const isElfMetadataArchitecture = ( /** Recognized family with no ELF metadata member; the header alone is the fact. */ export type HeaderArchitecture = Exclude< BinaryArchitecture, - ElfMetadataArchitecture + ElfMetadataArchitecture | "6502" >; /** Raw ELF facts; family recognition does not establish ISA or ABI support. */ @@ -144,8 +145,17 @@ export type BinaryTarget = readonly executableRole?: never; readonly managed?: never; }) - | (ExecutableTarget & { + | (BinaryTargetIdentity & { + readonly kind: "executable"; readonly format: "raw-binary"; + readonly architecture: Exclude< + BinaryArchitecture, + ElfMetadataArchitecture + >; + readonly availableArchitectures: readonly Exclude< + BinaryArchitecture, + ElfMetadataArchitecture + >[]; readonly entryPoint: string; readonly entryPointSource: string; readonly executableRole?: never; diff --git a/src/domain/callPath.test.ts b/src/domain/callPath.test.ts index e0f343fca..e78a5f936 100644 --- a/src/domain/callPath.test.ts +++ b/src/domain/callPath.test.ts @@ -1,6 +1,7 @@ import { describe, expect, it } from "vitest"; import { buildCallPath } from "./callPath.js"; +import { combineFunctionRelationshipCoverage } from "./functionCallCoverage.js"; import { callPathResultSchema, type CallPathInput } from "./callPathSchemas.js"; import { createEvidence, type Evidence } from "./evidence.js"; import type { JsonValue } from "./jsonValue.js"; @@ -396,6 +397,60 @@ describe("explicit call relationship coverage", () => { expect(result.limitations).toEqual([reason]); }); + it("establishes not_found when only caller coverage is unknown", () => { + const reason = + "Callers: Upstream xrefs do not establish direct caller relationships."; + const result = run([ + observe("0x1000", ["0x2000"], "rea-workflow", { + relationshipCoverage: { status: "unknown", reason }, + }), + observe("0x2000", [], "rea-workflow", { + relationshipCoverage: { status: "unknown", reason }, + }), + ]); + expect(result).toMatchObject({ + status: "not_found", + search_scope: { exhaustive: true }, + }); + expect(result.limitations).toContain(reason); + }); + + it("stays unknown when callee coverage is unknown", () => { + const reason = + "Callees: Upstream callees capped results at 500; additional callees remain unknown."; + const result = run([ + observe("0x1000", ["0x2000"], "rea-workflow", { + relationshipCoverage: { status: "unknown", reason }, + }), + observe("0x2000", [], "rea-workflow", { + relationshipCoverage: { status: "unknown", reason }, + }), + ]); + expect(result).toMatchObject({ + status: "unknown", + search_scope: { exhaustive: false }, + }); + expect(result.limitations).toContain(reason); + }); + + it("stays unknown when both caller and callee coverage are unknown", () => { + const reason = + "Callers: Upstream xrefs do not establish direct caller relationships. Callees: Upstream callees are not observed."; + const result = run([ + observe("0x1000", ["0x2000"], "rea-workflow", { + relationshipCoverage: { status: "unknown", reason }, + }), + observe("0x2000", [], "rea-workflow", { + relationshipCoverage: { status: "unknown", reason }, + }), + ]); + expect(result).toMatchObject({ + status: "unknown", + search_scope: { exhaustive: false }, + }); + expect(result.limitations).toContain(reason); + }); + it("ignores incomplete relationship coverage outside the reachable search closure", () => { const reason = "An unrelated function's callees were not observed."; const result = run([ @@ -415,6 +470,34 @@ describe("explicit call relationship coverage", () => { }); }); +describe("IDA-capped callee coverage beside unknown callers", () => { + it("keeps the search non-exhaustive for the combined coverage value", () => { + const combined = combineFunctionRelationshipCoverage( + { + status: "unknown", + reason: "Upstream xrefs do not establish direct caller relationships.", + }, + { + status: "partial", + limitations: [ + "Upstream callees capped results at 500; additional callees remain unknown.", + ], + }, + ); + expect(combined.status).toBe("unknown"); + const result = run([ + observe("0x1000", [], "rea-workflow", { relationshipCoverage: combined }), + observe("0x4000", []), + ]); + expect(result).toMatchObject({ + status: "unknown", + search_scope: { exhaustive: false }, + shortest_hops: null, + paths: [], + }); + }); +}); + describe("call path evidence admission", () => { it("normalizes hex addresses and rejects duplicates and mixed providers", () => { expect( diff --git a/src/domain/callPath.ts b/src/domain/callPath.ts index 2186c8010..0bd5e88e1 100644 --- a/src/domain/callPath.ts +++ b/src/domain/callPath.ts @@ -10,6 +10,7 @@ import { parseFunctionEvidence, type FunctionSnapshot, } from "./functionDossierEvidence.js"; +import { isCallerOnlyUnknownCoverage } from "./functionCallCoverage.js"; import { compareUnicodeCodePoints } from "./unicodeCodePointOrder.js"; /** Minimal directed caller-to-callee adjacency for call-path search. */ @@ -255,10 +256,14 @@ const inspectSearch = ({ // Explicit, Evidence-bound relationship coverage owns completeness; // generic provider/dossier limitations remain contextual. Legacy Evidence // without that metadata retains its conservative limitation-based rule. + // The search only expands callee edges, so coverage that is unknown + // solely because callers are unobservable does not make the searched + // callee closure incomplete; the caller caveat stays in limitations. if ( coverage === null ? snapshot.limitations.length > 0 - : coverage.status !== "complete" + : coverage.status !== "complete" && + !isCallerOnlyUnknownCoverage(coverage) ) incompleteExpandedCoverage = true; } else if (node === goal) { diff --git a/src/domain/evidence.test.ts b/src/domain/evidence.test.ts index 5cdf30a27..be923e3a6 100644 --- a/src/domain/evidence.test.ts +++ b/src/domain/evidence.test.ts @@ -19,6 +19,33 @@ const TARGET: BinaryTarget = { const PROVIDER = { id: "fixture", name: "Fixture provider", version: "1" }; const PROFILE = createAnalysisProfile(PROVIDER, { loader: "default" }); +it("rejects a detected Evidence subject with the raw-only 6502 architecture", () => { + const selected = { + path: TARGET.path, + sha256: TARGET.sha256, + format: "raw-binary" as const, + architecture: "6502" as const, + }; + expect( + createEvidence(selected, PROVIDER, { + operation: "inspect", + parameters: {}, + result: null, + }).subject, + ).toMatchObject({ format: "raw-binary", architecture: "6502" }); + const malformed = { + ...selected, + format: "pe" as const, + }; + expect(() => + createEvidence(malformed, PROVIDER, { + operation: "inspect", + parameters: {}, + result: null, + }), + ).toThrow(); +}); + it("preserves omitted relationship diagnostics and authenticates reported diagnostics", () => { const observation = { operation: "procedure_callees", diff --git a/src/domain/evidence.ts b/src/domain/evidence.ts index 8a450f478..49cb2a229 100644 --- a/src/domain/evidence.ts +++ b/src/domain/evidence.ts @@ -38,42 +38,52 @@ export const providerSchema = z.strictObject({ name: z.string().min(1), version: z.string().nullable(), }); -const subjectSchema = z.strictObject({ +const subjectFormatSchema = z.enum([ + "hopper", + "analysis-database", + "mach-o", + "elf", + "pe", + "dos-mz", + "dos-com", + "raw-binary", + "zip", + "ipa", + "apk", + "msix", + "appx", + "hap", + "hsp", + "app-pack", + "asar", + "dmg", + "pkg", + "plist", + "javascript", + "source-map", + "directory", + "file", + "unknown", + "mach-o-universal", + "javascript-bundle", + "entitlements", +]); +const subjectBaseSchema = z.strictObject({ name: z.string().min(1), digest: z.strictObject({ sha256: digestSchema }), - format: z.enum([ - "hopper", - "analysis-database", - "mach-o", - "elf", - "pe", - "dos-mz", - "dos-com", - "raw-binary", - "zip", - "ipa", - "apk", - "msix", - "appx", - "hap", - "hsp", - "app-pack", - "asar", - "dmg", - "pkg", - "plist", - "javascript", - "source-map", - "directory", - "file", - "unknown", - "mach-o-universal", - "javascript-bundle", - "entitlements", - ]), - architecture: z.enum(BINARY_ARCHITECTURES).nullable(), local_path: nulFreeStringSchema, }); +// The format owns the architecture claim, including nested Evidence subjects. +const subjectSchema = z.discriminatedUnion("format", [ + subjectBaseSchema.extend({ + format: subjectFormatSchema.exclude(["raw-binary"]), + architecture: z.enum(BINARY_ARCHITECTURES).exclude(["6502"]).nullable(), + }), + subjectBaseSchema.extend({ + format: z.literal("raw-binary"), + architecture: z.enum(BINARY_ARCHITECTURES).nullable(), + }), +]); /** Source location attached to an evidence observation. */ /** Source-location schema shared by evidence-bearing persistence formats. */ export const evidenceLocationSchema = z.discriminatedUnion("kind", [ @@ -186,7 +196,7 @@ const immutableResultSchema = z }); export type EvidenceLocation = z.infer; -/** Minimal immutable local artifact identity accepted by Evidence. */ +/** Candidate local artifact identity validated by the Evidence subject schema. */ export interface EvidenceSubjectTarget { readonly path: string; readonly sha256: string; diff --git a/src/domain/evidenceBundle.ts b/src/domain/evidenceBundle.ts index 7e6fd2b3b..a70969b8e 100644 --- a/src/domain/evidenceBundle.ts +++ b/src/domain/evidenceBundle.ts @@ -16,14 +16,17 @@ import { } from "./residualUnknown.js"; import { freezeJsonSnapshot } from "./immutableJson.js"; -const artifactManifestSchema = evidenceSchema.shape.subject - .unwrap() - .pick({ - digest: true, - format: true, - architecture: true, - }) - .strict(); +const [detectedSubjectSchema, rawSubjectSchema] = + evidenceSchema.shape.subject.unwrap().options; +const manifestFields = { + digest: true, + format: true, + architecture: true, +} as const; +const artifactManifestSchema = z.discriminatedUnion("format", [ + detectedSubjectSchema.pick(manifestFields), + rawSubjectSchema.pick(manifestFields), +]); const providerManifestSchema = providerSchema; const environmentManifestSchema = evidenceSchema.shape.environment.unwrap(); const scenarioManifestSchema = z.strictObject({ @@ -85,11 +88,11 @@ export const createEvidenceBundle = ( subject === null ? [] : [ - { + artifactManifestSchema.parse({ digest: subject.digest, format: subject.format, architecture: subject.architecture, - }, + }), ], ), ), diff --git a/src/domain/functionCallCoverage.test.ts b/src/domain/functionCallCoverage.test.ts new file mode 100644 index 000000000..6fd3fb045 --- /dev/null +++ b/src/domain/functionCallCoverage.test.ts @@ -0,0 +1,64 @@ +import { describe, expect, it } from "vitest"; + +import { + combineFunctionRelationshipCoverage, + isCallerOnlyUnknownCoverage, +} from "./functionCallCoverage.js"; +import type { RelationshipCoverage } from "./relationshipCoverage.js"; + +const unknownCallers: RelationshipCoverage = { + status: "unknown", + reason: "Upstream xrefs do not establish direct caller relationships.", +}; +const unknownCallees: RelationshipCoverage = { + status: "unknown", + reason: "Upstream callees are not observed in this profile.", +}; +const cappedCallees: RelationshipCoverage = { + status: "partial", + limitations: [ + "Upstream callees capped results at 500; additional callees remain unknown.", + ], +}; +const complete: RelationshipCoverage = { status: "complete" }; + +describe("isCallerOnlyUnknownCoverage", () => { + it("detects unknown callers beside complete callees", () => { + const combined = combineFunctionRelationshipCoverage( + unknownCallers, + complete, + ); + expect(combined.status).toBe("unknown"); + expect(isCallerOnlyUnknownCoverage(combined)).toBe(true); + }); + + it("rejects unknown callees beside complete callers", () => { + const combined = combineFunctionRelationshipCoverage( + complete, + unknownCallees, + ); + expect(isCallerOnlyUnknownCoverage(combined)).toBe(false); + }); + + it("rejects unknown coverage on both sides", () => { + const combined = combineFunctionRelationshipCoverage( + unknownCallers, + unknownCallees, + ); + expect(isCallerOnlyUnknownCoverage(combined)).toBe(false); + }); + + it("rejects unknown callers beside IDA-capped callees", () => { + const combined = combineFunctionRelationshipCoverage( + unknownCallers, + cappedCallees, + ); + expect(combined.status).toBe("unknown"); + expect(isCallerOnlyUnknownCoverage(combined)).toBe(false); + }); + + it("rejects complete coverage", () => { + const combined = combineFunctionRelationshipCoverage(complete, complete); + expect(isCallerOnlyUnknownCoverage(combined)).toBe(false); + }); +}); diff --git a/src/domain/functionCallCoverage.ts b/src/domain/functionCallCoverage.ts index eb2c05a13..b64e78cd6 100644 --- a/src/domain/functionCallCoverage.ts +++ b/src/domain/functionCallCoverage.ts @@ -2,6 +2,9 @@ import type { RelationshipCoverage } from "./relationshipCoverage.js"; import type { FunctionDossier } from "./hopperValues.js"; import { uniqueSorted } from "./canonicalOrdering.js"; +const callerCoveragePrefix = "Callers: "; +const calleeCoveragePrefix = "Callees: "; + /** Combine caller and callee read outcomes without losing either explanation. */ export const combineFunctionRelationshipCoverage = ( callers: RelationshipCoverage | undefined, @@ -20,7 +23,8 @@ export const combineFunctionRelationshipCoverage = ( ); const details = diagnostics.length > 0 ? { diagnostics } : {}; const reasons = reads.flatMap(({ direction, coverage }) => { - const label = direction === "callers" ? "Callers" : "Callees"; + const label = + direction === "callers" ? callerCoveragePrefix : calleeCoveragePrefix; const limitations = coverage === undefined ? ["Provider did not report relationship coverage."] @@ -29,7 +33,7 @@ export const combineFunctionRelationshipCoverage = ( : coverage.status === "partial" ? coverage.limitations : []; - return limitations.map((reason) => `${label}: ${reason}`); + return limitations.map((reason) => `${label}${reason}`); }); if ( reads.some( @@ -50,6 +54,20 @@ export const combineFunctionRelationshipCoverage = ( return { status: "complete", ...details }; }; +/** + * Whether combined relationship coverage is unknown only on the caller side. + * The combine helper above prefixes each direction's reasons, so a reason + * that carries the caller prefix and no callee prefix records observed + * callees with unobservable callers. Forward (callee-edge) searches stay + * exhaustive under that caveat; the caller reason remains a limitation. + */ +export const isCallerOnlyUnknownCoverage = ( + coverage: RelationshipCoverage, +): boolean => + coverage.status === "unknown" && + coverage.reason.startsWith(callerCoveragePrefix) && + !coverage.reason.includes(calleeCoveragePrefix); + /** Explain why two resolved call sets cannot establish call-set equality. */ export const functionCallCoverageLimitations = ( coverage: RelationshipCoverage | null, diff --git a/src/domain/functionComparison.test.ts b/src/domain/functionComparison.test.ts index e03a27748..677fdc0cc 100644 --- a/src/domain/functionComparison.test.ts +++ b/src/domain/functionComparison.test.ts @@ -716,6 +716,236 @@ describe("function comparison identity and malformed CFG", () => { }); }); +const withLiterals = (base: "0x1000" | "0x2000", values: readonly string[]) => + functionDossierSchema.parse({ + ...dossier("return 0;", base), + swift_string_literals: values.map((value) => ({ + source_address: base, + value, + representation: "small" as const, + address: null, + registers: ["x0", "x1"] as const, + words: ["0x0", "0xe"] as const, + })), + }); +const stringsDimension = ( + leftDigit: string, + left: ReturnType | ReturnType, + rightDigit: string, + right: ReturnType | ReturnType, +) => + compareFunctions( + observe(leftDigit, left), + observe(rightDigit, right), + ).dimensions.find(({ dimension }) => dimension === "strings_names"); + +describe("function comparison Swift string literals", () => { + it("treats differing decoded literals as a strings and names change", () => { + expect( + stringsDimension( + "1", + withLiterals("0x1000", ["SleepDisabled"]), + "2", + withLiterals("0x2000", ["Holding awake"]), + ), + ).toMatchObject({ status: "changed" }); + }); + + it("matches the same decoded literal after the function is relocated", () => { + expect( + stringsDimension( + "3", + withLiterals("0x1000", ["SleepDisabled"]), + "4", + withLiterals("0x2000", ["SleepDisabled"]), + ), + ).toMatchObject({ status: "unchanged" }); + }); + + it("keeps strings and names unknown when only one dossier recorded Swift literals", () => { + expect( + stringsDimension( + "5", + withLiterals("0x1000", []), + "6", + dossier("return 0;", "0x2000"), + ), + ).toMatchObject({ + status: "unknown", + limitations: [ + "Swift String literal coverage differs or is partial; referenced strings and names match, and complete coverage does not establish a differing literal.", + ], + }); + }); + + it("reports a referenced name change even when Swift literal coverage differs", () => { + const named = functionDossierSchema.parse({ + ...dossier("return 0;", "0x2000"), + referenced_names: [ + { address: "0x3000", value: "helper", source_address: "0x2000" }, + ], + }); + expect( + stringsDimension("7", withLiterals("0x1000", []), "8", named), + ).toMatchObject({ status: "changed" }); + }); +}); + +describe("function comparison partial Swift literal coverage", () => { + it("does not equate a partial literal record with a complete one", () => { + const partial = functionDossierSchema.parse({ + ...withLiterals("0x2000", ["SleepDisabled"]), + swift_string_literal_omissions: [ + { + source_address: "0x2000", + representation: "large", + address: "0x3000", + byte_length: 64, + reason: "capacity", + registers: ["x0", "x1"], + words: ["0xd000000000000040", "0x8000000000002fe0"], + }, + ], + }); + expect( + stringsDimension( + "9", + withLiterals("0x1000", ["SleepDisabled"]), + "a", + partial, + ), + ).toMatchObject({ status: "unknown" }); + }); + + it("reports a literal that a complete dossier shows absent at a site the partial one decoded", () => { + const partial = functionDossierSchema.parse({ + ...withLiterals("0x2000", ["SleepDisabled"]), + swift_string_literal_omissions: [ + { + source_address: "0x2004", + representation: "small", + address: null, + byte_length: 4, + reason: "capacity", + registers: ["x0", "x1"], + words: ["0x732f4220", "0xe400000000000000"], + }, + ], + }); + expect( + stringsDimension("d", withLiterals("0x1000", []), "e", partial), + ).toMatchObject({ status: "changed" }); + }); + + it("does not interpret an unexamined path as a missing literal at a retained site", () => { + const partial = functionDossierSchema.parse({ + ...withLiterals("0x2000", ["A"]), + swift_string_literal_omissions: [ + { + source_address: "0x2004", + representation: "small", + address: null, + byte_length: 0, + reason: "capacity", + registers: ["x0", "x1"], + words: ["0x0", "0xe000000000000000"], + }, + ], + }); + const complete = withLiterals("0x1000", ["A", "B"]); + expect(stringsDimension("1", complete, "2", partial)).toMatchObject({ + status: "unknown", + }); + expect(stringsDimension("3", partial, "4", complete)).toMatchObject({ + status: "unknown", + }); + expect( + stringsDimension("5", withLiterals("0x1000", []), "6", partial), + ).toMatchObject({ status: "changed" }); + }); + + it("treats an omission record without decoded literals as partial coverage", () => { + const omissionOnly = functionDossierSchema.parse({ + ...dossier("return 0;", "0x2000"), + swift_string_literal_omissions: [ + { + source_address: "0x2004", + representation: "small", + address: null, + byte_length: 4, + reason: "capacity", + registers: ["x0", "x1"], + words: ["0x732f4220", "0xe400000000000000"], + }, + ], + }); + expect( + stringsDimension("f", dossier("return 0;", "0x1000"), "0", omissionOnly), + ).toMatchObject({ status: "unknown" }); + }); + + it("keeps different retained representations unknown when both dossiers are partial", () => { + const omission = { + source_address: "0x1004", + representation: "small", + address: null, + byte_length: 4, + reason: "capacity", + registers: ["x0", "x1"], + words: ["0x732f4220", "0xe400000000000000"], + } as const; + const small = functionDossierSchema.parse({ + ...withLiterals("0x1000", ["SleepDisabled"]), + swift_string_literal_omissions: [omission], + }); + const large = functionDossierSchema.parse({ + ...dossier("return 0;", "0x2000"), + swift_string_literals: [ + { + source_address: "0x2000", + value: "SleepDisabled", + representation: "large", + address: "0x3000", + registers: ["x0", "x1"], + words: ["0xd00000000000000d", "0x8000000000002fe0"], + }, + ], + swift_string_literal_omissions: [ + { ...omission, source_address: "0x2004" }, + ], + }); + expect(stringsDimension("1", small, "2", large)).toMatchObject({ + status: "unknown", + }); + }); + + it("keeps different retained literals unknown when both dossiers are partial", () => { + const partial = (base: "0x1000" | "0x2000", value: string) => + functionDossierSchema.parse({ + ...withLiterals(base, [value]), + swift_string_literal_omissions: [ + { + source_address: base === "0x1000" ? "0x1004" : "0x2004", + representation: "large", + address: "0x3000", + byte_length: 64, + reason: "capacity", + registers: ["x0", "x1"], + words: ["0xd000000000000040", "0x8000000000002fe0"], + }, + ], + }); + expect( + stringsDimension( + "b", + partial("0x1000", "SleepDisabled"), + "c", + partial("0x2000", "Holding awake"), + ), + ).toMatchObject({ status: "unknown" }); + }); +}); + const expectInlineComparisonEvidence = ( result: ReturnType, ): void => { diff --git a/src/domain/functionComparisonDimensions.ts b/src/domain/functionComparisonDimensions.ts index 27bb7b176..c5a0f373f 100644 --- a/src/domain/functionComparisonDimensions.ts +++ b/src/domain/functionComparisonDimensions.ts @@ -11,6 +11,7 @@ import { referenceProjection, sorted, stringAndNameProjection, + swiftLiteralSitesConflict, } from "./functionComparisonNormalization.js"; import type { DimensionName, @@ -139,8 +140,12 @@ const compareStringsAndNames = ( right: FunctionSnapshot, context: ComparisonContext, ): FunctionDimension => { - const leftValues = stringAndNameProjection(left); - const rightValues = stringAndNameProjection(right); + const complete = + swiftLiteralCoverage(left) === swiftLiteralCoverage(right) && + swiftLiteralCoverage(left) !== "partial"; + // Facts both dossiers record decide a change before Swift literal coverage can. + const leftValues = stringAndNameProjection(left, complete); + const rightValues = stringAndNameProjection(right, complete); if (leftValues === null || rightValues === null) return unresolvedDimension({ dimension: "strings_names", @@ -151,9 +156,40 @@ const compareStringsAndNames = ( "Reference source locations could not be normalized to function offsets.", ], }); - return compareValues("strings_names", leftValues, rightValues, context); + const compared = compareValues( + "strings_names", + leftValues, + rightValues, + context, + ); + if (complete || compared.status === "changed") return compared; + // A retained literal differs only when the opposite dossier establishes its absence. + if (swiftLiteralSitesConflict(left, right)) { + const leftAll = stringAndNameProjection(left); + const rightAll = stringAndNameProjection(right); + if (leftAll !== null && rightAll !== null) + return compareValues("strings_names", leftAll, rightAll, context); + } + return unresolvedDimension({ + dimension: "strings_names", + links: context.links, + leftCount: null, + rightCount: null, + limitations: [ + "Swift String literal coverage differs or is partial; referenced strings and names match, and complete coverage does not establish a differing literal.", + ], + }); }; +const swiftLiteralCoverage = ( + snapshot: FunctionSnapshot, +): "absent" | "complete" | "partial" => + (snapshot.dossier.swift_string_literal_omissions ?? []).length > 0 + ? "partial" + : snapshot.dossier.swift_string_literals === undefined + ? "absent" + : "complete"; + const compareReferences = ( left: FunctionSnapshot, right: FunctionSnapshot, diff --git a/src/domain/functionComparisonNormalization.ts b/src/domain/functionComparisonNormalization.ts index 5282272e1..f32220b0b 100644 --- a/src/domain/functionComparisonNormalization.ts +++ b/src/domain/functionComparisonNormalization.ts @@ -142,6 +142,7 @@ export const commentProjection = ( export const stringAndNameProjection = ( snapshot: FunctionSnapshot, + includeSwiftLiterals = true, ): readonly unknown[] | null => { const values = [ ...snapshot.dossier.referenced_strings.map( @@ -164,12 +165,90 @@ export const stringAndNameProjection = ( value, }), ), + ...(includeSwiftLiterals + ? (snapshot.dossier.swift_string_literals ?? []) + : [] + ).map( + ({ + source_address: sourceAddress, + value, + representation, + registers, + }) => ({ + kind: "swift_string", + source_offset: relativeAddress( + sourceAddress, + snapshot.dossier.procedure.address, + ), + value, + representation, + registers, + }), + ), ]; return values.some(({ source_offset: offset }) => offset === null) ? null : sorted(values); }; +/** + * A retained literal proves presence, not that every path through its site was examined. + * A differing retained fact proves a change only when the opposite dossier is complete + * and therefore establishes its absence. + */ +export const swiftLiteralSitesConflict = ( + left: FunctionSnapshot, + right: FunctionSnapshot, +): boolean => { + const leftSites = swiftLiteralSites(left); + const rightSites = swiftLiteralSites(right); + for (const site of new Set([ + ...leftSites.values.keys(), + ...rightSites.values.keys(), + ])) { + const leftValues = + leftSites.values.get(site) ?? (leftSites.complete ? [] : undefined); + const rightValues = + rightSites.values.get(site) ?? (rightSites.complete ? [] : undefined); + if (leftValues === undefined || rightValues === undefined) continue; + const leftFacts = new Set(leftValues); + const rightFacts = new Set(rightValues); + if ( + (rightSites.complete && + leftValues.some((value) => !rightFacts.has(value))) || + (leftSites.complete && rightValues.some((value) => !leftFacts.has(value))) + ) + return true; + } + return false; +}; + +const swiftLiteralSites = (snapshot: FunctionSnapshot) => { + const base = snapshot.dossier.procedure.address; + const omissions = snapshot.dossier.swift_string_literal_omissions ?? []; + const site = (sourceAddress: string, registers: readonly string[]) => { + const offset = relativeAddress(sourceAddress, base); + return offset === null ? null : canonicalJson([offset, registers]); + }; + // Keep the same literal facts the full strings_names projection compares. + const values = new Map(); + for (const literal of snapshot.dossier.swift_string_literals ?? []) { + const key = site(literal.source_address, literal.registers); + if (key === null) continue; + values.set( + key, + [ + ...(values.get(key) ?? []), + canonicalJson([literal.value, literal.representation]), + ].sort(compareUnicodeCodePoints), + ); + } + const complete = + snapshot.dossier.swift_string_literals !== undefined && + omissions.length === 0; + return { values, complete }; +}; + export const sorted = (values: readonly unknown[]): readonly unknown[] => values .map((value) => ({ diff --git a/src/domain/hopperValues.ts b/src/domain/hopperValues.ts index 769d0f040..d6344ff3e 100644 --- a/src/domain/hopperValues.ts +++ b/src/domain/hopperValues.ts @@ -290,6 +290,70 @@ export const unresolvedCallSchema = z.strictObject({ reason: z.string(), }); +const swiftStringLiteralFields = { + source_address: z + .string() + .describe( + "Instruction that consumes both words: a call, tail call, return, or paired store.", + ), + value: z + .string() + .describe("Decoded UTF-8 text, including the canonical empty string."), + registers: z.tuple([z.string(), z.string()]), + words: z + .tuple([z.string(), z.string()]) + .describe("Observed constant register values, in register order."), +}; + +/** A Swift String literal decoded from the two words of its ABI value, not a data reference. */ +export const swiftStringLiteralSchema = z.discriminatedUnion("representation", [ + z.strictObject({ + ...swiftStringLiteralFields, + representation: z + .literal("small") + .describe("The UTF-8 bytes are inside the two words."), + address: z.null().describe("A small literal has no storage of its own."), + }), + z.strictObject({ + ...swiftStringLiteralFields, + representation: z + .literal("large") + .describe( + "An immortal literal whose words hold its count and a pointer to its bytes.", + ), + address: z.string().describe("Address of a large literal's bytes."), + }), +]); + +const swiftStringLiteralOmissionFields = { + source_address: swiftStringLiteralFields.source_address, + byte_length: z.number().int().min(0).max(Number.MAX_SAFE_INTEGER), + reason: z + .literal("capacity") + .describe( + "The record did not fit the provider's memory budget for this dossier; decoding stopped here, so later use sites were not examined.", + ), + registers: swiftStringLiteralFields.registers, + words: swiftStringLiteralFields.words, +}; + +/** A Swift literal whose layout matched but whose record did not fit; decoding stopped there. */ +export const swiftStringLiteralOmissionSchema = z.discriminatedUnion( + "representation", + [ + z.strictObject({ + ...swiftStringLiteralOmissionFields, + representation: z.literal("small"), + address: z.null(), + }), + z.strictObject({ + ...swiftStringLiteralOmissionFields, + representation: z.literal("large"), + address: z.string().describe("Address of the literal's bytes."), + }), + ], +); + /** Observed endpoints with complete or partial reference classification evidence. */ export const referenceEdgeSchema = z .object({ @@ -341,6 +405,18 @@ export const functionDossierSchema = z }) .strict(), ), + swift_string_literals: z + .array(swiftStringLiteralSchema) + .exactOptional() + .describe( + "Swift String literals decoded from constant register values. Absent when the provider does not decode them for this target.", + ), + swift_string_literal_omissions: z + .array(swiftStringLiteralOmissionSchema) + .exactOptional() + .describe( + "The literal at which decoding stopped for capacity. Present exactly when swift_string_literals is present; empty when decoding examined every use site.", + ), basic_blocks: z.array( z .object({ diff --git a/src/domain/javascript/javascriptSemanticAnalysis.getter_aliases.test.ts b/src/domain/javascript/javascriptSemanticAnalysis.getter_aliases.test.ts new file mode 100644 index 000000000..1eeaec350 --- /dev/null +++ b/src/domain/javascript/javascriptSemanticAnalysis.getter_aliases.test.ts @@ -0,0 +1,351 @@ +import { describe, expect, it } from "vitest"; +import { analyzeJavaScriptSemantics } from "./javascriptSemanticAnalysis.js"; +import { onlyCallable } from "./javascriptSemanticAnalysis.fixture.js"; + +const returnedValue = (getter: string, write = "box.v.x = 2;") => + onlyCallable( + analyzeJavaScriptSemantics(`export function result() { + const shared = { x: 1, y: 1 }; + const box = { s: shared, get v() { ${getter} } }; + ${write} + return shared.x; + }`), + "result", + ).returnSites[0]?.value; + +describe("projected private getter receivers", () => { + it.each([ + [ + "destructured self field", + "shared", + "peer = this;", + "const { peer: self } = this; return self.#s;", + true, + "unknown", + ], + [ + "unrelated private child", + "{ peer: shared }", + "peer = this;", + "const { peer: self } = this; return self.#s;", + true, + "literal", + ], + [ + "read without write", + "shared", + "peer = this;", + "const { peer: self } = this; return self.#s;", + false, + "literal", + ], + [ + "two self fields", + "shared", + "peer = this; next = this.peer;", + "const { next: self } = this; return self.#s;", + true, + "unknown", + ], + [ + "nested receiver projection", + "shared", + "peer = this;", + "const { peer: { peer: self } } = this; return self.#s;", + true, + "unknown", + ], + [ + "member receiver alias", + "shared", + "peer = this;", + "const self = this.peer.peer; return self.#s;", + true, + "unknown", + ], + [ + "projected result suffix", + "{ child: shared }", + "peer = this;", + "const { peer: self } = this; return self.#s.child;", + true, + "unknown", + ], + [ + "external receiver", + "shared", + "peer = external;", + "const { peer: self } = this; return self.#s;", + true, + "literal", + ], + [ + "field receiver cycle", + "shared", + "peer = this.other; other = this.peer;", + "const { peer: self } = this; return self.#s;", + true, + "literal", + ], + ] as const)( + "keeps receiver and result paths separate for %s", + (_name, value, fields, getter, write, status) => { + const ir = analyzeJavaScriptSemantics(`export function result() { + const shared = { x: 1 }; + class Box { + #s = ${value}; ${fields} + get v() { ${getter} } + } + const box = new Box(); + ${write ? "box.v.x = 2;" : "const saved = box.v;"} + return shared.x; + }`); + expect(onlyCallable(ir, "result").returnSites[0]?.value).toMatchObject( + status === "literal" ? { status, value: 1 } : { status }, + ); + }, + ); +}); + +describe("private receiver field ownership", () => { + it("bounds growing receiver projections under cyclic class inheritance", () => { + const ir = analyzeJavaScriptSemantics(`export function result() { + const shared = { x: 1 }; + class A extends B { + #s = shared; + peer = this.other.other; + get v() { const { peer: self } = this; return self.#s; } + } + class B extends A { other = this.peer.peer; } + new A().v.x = 2; + return shared.x; + }`); + expect(onlyCallable(ir, "result").returnSites[0]?.value).toMatchObject({ + status: "literal", + value: 1, + }); + }, 1000); + + it.each([ + [ + "own static receiver", + "class Box { static #s=shared; static peer=this; static get v(){const {peer:self}=this;return self.#s;} } Box.v.x=2;", + "unknown", + ], + [ + "inherited instance receiver", + "class Base {peer=this;} class Box extends Base {#s=shared;get v(){const {peer:self}=this;return self.#s;}} new Box().v.x=2;", + "unknown", + ], + [ + "inherited static field retains its declaring class", + "class Base {static peer=this;} class Box extends Base {static #s=shared;static get v(){const {peer:self}=this;return self.#s;}} Box.v.x=2;", + "literal", + ], + [ + "inherited static getter's own private field", + "class Base {static #s=shared;static peer=this;static get v(){const {peer:self}=this;return self.#s;}} class Box extends Base {} Box.v.x=2;", + "unknown", + ], + [ + "a field that is not initialized yet", + "class Box {#s=shared;peer=this.later;later=this;get v(){const {peer:self}=this;return self.#s;}} new Box().v.x=2;", + "literal", + ], + [ + "remaining getter projection after a captured field", + "class Box {#s=shared;peer=this;next=this.peer;later=this;get v(){const {next:{later:self}}=this;return self.#s;}} new Box().v.x=2;", + "unknown", + ], + [ + "a replaced self field", + "class Box {#s=shared;peer=this;peer=external;get v(){const {peer:self}=this;return self.#s;}} new Box().v.x=2;", + "literal", + ], + ] as const)("respects %s", (_name, body, status) => { + const ir = analyzeJavaScriptSemantics(`export function result() { + const shared = { x: 1 }; + ${body} + return shared.x; + }`); + expect(onlyCallable(ir, "result").returnSites[0]?.value).toMatchObject( + status === "literal" ? { status, value: 1 } : { status }, + ); + }); +}); + +describe("getter-local receiver aliases (#2653)", () => { + it.each([ + ["initialized alias", "const self = this; return self.s;"], + ["assigned alias", "let self; self = this; return self.s;"], + [ + "chained assignment", + "let self; let alias; self = alias = this; return self.s;", + ], + ["destructured receiver", "const { s: value } = this; return value;"], + ["alias chain", "const self = this; const saved = self; return saved.s;"], + [ + "capture before reassignment", + "let self = this; const saved = self; self = { s: { x: 1 } }; return saved.s;", + ], + [ + "conditional receiver alias", + "const self = flag ? this : { s: { x: 1 } }; return self.s;", + ], + ])("invalidates shared values through %s", (_name, getter) => { + expect(returnedValue(getter)?.status).toBe("unknown"); + }); + + it.each([ + ["no write", "const self = this; return self.s;", "const saved = box.v;"], + [ + "an unrelated property write", + "const self = this; return self.s;", + "box.v.y = 2;", + ], + [ + "a cycle that does not grow the selected path", + "let self = this; while (flag) self = self; return self.s;", + "box.v.y = 2;", + ], + [ + "an alias replaced before return", + "let self = this; self = { s: { x: 1 } }; return self.s;", + "box.v.x = 2;", + ], + [ + "a shadowing block binding", + "const self = this; { const self = { s: { x: 1 } }; return self.s; }", + "box.v.x = 2;", + ], + [ + "a nested function's receiver", + "function other() { const self = this; return self.s; } return other();", + "box.v.x = 2;", + ], + [ + "an alias assigned in a different receiver scope", + "let self; function assign() { self = this; } assign(); return self.s;", + "box.v.x = 2;", + ], + [ + "an excluded destructuring default", + "const { value = shared } = { value: { x: 1 } }; return value;", + "box.v.x = 2;", + ], + ])("preserves known values with %s", (_name, getter, write) => { + expect(returnedValue(getter, write)).toMatchObject({ + status: "literal", + value: 1, + }); + }); +}); + +describe("getter receiver alias boundaries", () => { + it("does not reinterpret an outer this binding as the getter receiver", () => { + const ir = analyzeJavaScriptSemantics(`export function result() { + const shared = { x: 1 }; + const outside = this; + const box = { s: shared, get v() { return outside.s; } }; + box.v.x = 2; + return shared.x; + }`); + expect(onlyCallable(ir, "result").returnSites[0]?.value).toMatchObject({ + status: "literal", + value: 1, + }); + }); + + it("retains a private field's lexical owner through a receiver alias", () => { + const ir = analyzeJavaScriptSemantics(`export function result() { + const shared = { x: 1 }; + class Box { + #s = shared; + get v() { const self = this; return self.#s; } + } + new Box().v.x = 2; + return shared.x; + }`); + expect(onlyCallable(ir, "result").returnSites[0]?.value.status).toBe( + "unknown", + ); + }); + + it("does not append a receiver projection to a private field's result path", () => { + const ir = analyzeJavaScriptSemantics(`export function result() { + const shared = { x: 1 }; + class Box { + #s = { peer: shared }; + peer = this; + get v() { const { peer: self } = this; return self.#s; } + } + new Box().v.x = 2; + return shared.x; + }`); + expect(onlyCallable(ir, "result").returnSites[0]?.value).toMatchObject({ + status: "literal", + value: 1, + }); + }); + + it("resolves a long local alias chain without recursive expansion", () => { + const aliases = ["const a0 = this;"]; + for (let index = 1; index <= 2000; index++) + aliases.push(`const a${String(index)} = a${String(index - 1)};`); + expect(returnedValue(`${aliases.join("\n")} return a2000.s;`)?.status).toBe( + "unknown", + ); + }); + + it("bounds a growing alias cycle without losing the original receiver", () => { + expect( + returnedValue( + "let self = this; while (flag) self = self.s; return self.s;", + )?.status, + ).toBe("unknown"); + }); + + it.each([true, false])( + "retains deeper shared references through an alias cycle (write=%s)", + (write) => { + const ir = analyzeJavaScriptSemantics(`export function result() { + const shared = { x: 1 }; + const box = { + start: { child: { child: shared } }, + get v() { + let self = this.start; + while (flag) self = self.child; + return self; + } + }; + ${write ? "box.v.x = 2;" : "const saved = box.v;"} + return shared.x; + }`); + expect(onlyCallable(ir, "result").returnSites[0]?.value).toMatchObject( + write ? { status: "unknown" } : { status: "literal", value: 1 }, + ); + }, + ); + + it("does not widen a growing cycle to unrelated receiver properties", () => { + const ir = analyzeJavaScriptSemantics(`export function result() { + const shared = { x: 1 }; + const unrelated = { x: 1 }; + const box = { + start: { child: { child: shared } }, + unrelated, + get v() { + let self = this.start; + while (flag) self = self.child; + return self; + } + }; + box.v.x = 2; + return unrelated.x; + }`); + expect(onlyCallable(ir, "result").returnSites[0]?.value).toMatchObject({ + status: "literal", + value: 1, + }); + }); +}); diff --git a/src/domain/javascript/javascriptSemanticAnalysis.getter_receivers.test.ts b/src/domain/javascript/javascriptSemanticAnalysis.getter_receivers.test.ts index 3b83bf482..c3b38842a 100644 --- a/src/domain/javascript/javascriptSemanticAnalysis.getter_receivers.test.ts +++ b/src/domain/javascript/javascriptSemanticAnalysis.getter_receivers.test.ts @@ -33,6 +33,38 @@ const getterReceivers = [ "nested owner", "const outer = { box: { x: 1, get self() { return this; } } }; outer.box.self.x = 2; return outer.box.x;", ], + [ + "array element", + "const arr = [{ x: 1, get self() { return this; } }]; arr[0].self.x = 2; return arr[0].x;", + ], + [ + "array element after a hole", + "const arr = [, { x: 1, get self() { return this; } }]; arr[1].self.x = 2; return arr[1].x;", + ], + [ + "mixed object and array owners", + "const outer = { rows: [{ box: { x: 1, get self() { return this; } } }] }; outer.rows[0].box.self.x = 2; return outer.rows[0].box.x;", + ], + [ + "assigned owner", + "let box; box = { x: 1, get self() { return this; } }; box.self.x = 2; return box.x;", + ], + [ + "saved array receiver alias", + "const arr = [{ x: 1, get self() { return this; } }]; const saved = arr[0].self; saved.x = 2; return arr[0].x;", + ], + [ + "old receiver captured inside its replacement initializer", + "const old = { x: 1 }; let box = old; box = { old: box, get self() { return this; } }; box.self.old.x = 2; return old.x;", + ], + [ + "array receiver after an unknown spread", + "const arr = [...items, { x: 1, get self() { return this; } }]; arr[index].self.x = 2; return arr[index].x;", + ], + [ + "saved receiver before its array alias is reassigned", + "const original = [{ x: 1, get self() { return this; } }]; let arr = original; const saved = arr[0].self; arr = [{ x: 1 }]; saved.x = 2; return original[0].x;", + ], [ "inherited receiver field", "const shared = { x: 1 }; class A { get v() { return this.s; } } class B extends A { s = shared; } new B().v.x = 2; return shared.x;", @@ -75,7 +107,7 @@ const getterReceivers = [ ], ] as const; -describe("getter receiver mutation aliases (#1665)", () => { +describe("getter receiver mutation aliases (#1665, #2654)", () => { it.each(getterReceivers)( "does not report a stale literal through %s", (_shape, body) => { @@ -103,6 +135,42 @@ describe("getter receiver mutation aliases (#1665)", () => { "unrelated sibling", "const box = { x: 1, y: 1, get self() { return this; } }; box.self.y = 2; return box.x;", ], + [ + "array receiver read without write", + "const arr = [{ x: 1, get self() { return this; } }]; const saved = arr[0].self; return arr[0].x;", + ], + [ + "assigned receiver read without write", + "let box; box = { x: 1, get self() { return this; } }; const saved = box.self; return box.x;", + ], + [ + "unrelated array receiver property", + "const arr = [{ x: 1, y: 1, get self() { return this; } }]; arr[0].self.y = 2; return arr[0].x;", + ], + [ + "unrelated array element", + "const arr = [{ x: 1, get self() { return this; } }, { x: 1 }]; arr[0].self.x = 2; return arr[1].x;", + ], + [ + "array replacement after a receiver was captured", + "const original = [{ x: 1, get self() { return this; } }]; const replacement = [{ x: 1 }]; let arr = original; const saved = arr[0].self; arr = replacement; saved.x = 2; return replacement[0].x;", + ], + [ + "known array prefix before a spread receiver", + "const arr = [{ x: 1 }, ...items, { x: 1, get self() { return this; } }]; arr[1].self.x = 2; return arr[0].x;", + ], + [ + "outer binding shadowed by an assigned receiver", + "let box; box = { x: 1 }; { let box; box = { x: 1, get self() { return this; } }; box.self.x = 2; } return box.x;", + ], + [ + "old object before an unconditional receiver assignment", + "const old = { x: 1 }; let box = old; box = { x: 1, get self() { return this; } }; box.self.x = 2; return old.x;", + ], + [ + "unmodified old receiver captured inside its replacement initializer", + "const old = { x: 1 }; let box = old; box = { x: 1, old: box, get self() { return this; } }; box.self.x = 2; return old.x;", + ], [ "replace getter", "const shared = { x: 1 }; const box = { s: shared, get v() { return this.s; } }; box.v = {}; return shared.x;", diff --git a/src/domain/javascript/javascriptSemanticMemberMutations.ts b/src/domain/javascript/javascriptSemanticMemberMutations.ts index c2f7efba3..7b988af88 100644 --- a/src/domain/javascript/javascriptSemanticMemberMutations.ts +++ b/src/domain/javascript/javascriptSemanticMemberMutations.ts @@ -10,19 +10,21 @@ import { evaluateSemanticBinding, evaluateSemanticExpression, } from "./javascriptSemanticValues.js"; -import { - semanticContainer, - semanticSlotAtPath, -} from "./javascriptSemanticSlots.js"; +import { semanticContainer } from "./javascriptSemanticSlots.js"; import { semanticClassCandidates, semanticClassPropertyReferences, semanticObjectPropertyReferences, + type JavaScriptSemanticObjectReference, } from "./javascriptSemanticObjectReferences.js"; import { semanticIterationSources } from "./javascriptSemanticIterationSources.js"; import { semanticCallableResultExpressions } from "./javascriptSemanticCallableResults.js"; import { createSemanticReturnedReferences } from "./javascriptSemanticReturnedReferences.js"; import { semanticMutationInitializers } from "./javascriptSemanticMutationInitializers.js"; +import { + semanticReferenceSourcesInactive, + semanticSelectedReferenceSlot, +} from "./javascriptSemanticReferenceSources.js"; import { semanticArrayIndex, SemanticPropertyPathCoverage, @@ -98,8 +100,20 @@ export function* collectSemanticMemberMutationsSteps( effect: ValueEffect = "write", mutation?: t.Node, originAt: t.Node | undefined = mutation, + ownerInitializer?: t.Node, ): void => { - const pending = [{ node, path, bindings, originAt }]; + const pending: (ReferencedValue & { + readonly bindings: ReadonlySet; + readonly originAt: t.Node | undefined; + })[] = [ + { + node, + path, + bindings, + originAt, + ...(ownerInitializer === undefined ? {} : { ownerInitializer }), + }, + ]; // Visit shallower alias traversals first so their exact prefixes suppress // longer loop paths before those paths enumerate every assignment order. for (let cursor = 0; cursor < pending.length; cursor++) { @@ -132,6 +146,8 @@ export function* collectSemanticMemberMutationsSteps( binding.bindingId, current.originAt?.start, current.originAt?.end, + current.ownerInitializer?.start, + current.ownerInitializer?.end, ]; const lifetime = JSON.stringify([...capture, current.path]); if (current.bindings.has(lifetime)) continue; @@ -165,6 +181,7 @@ export function* collectSemanticMemberMutationsSteps( "escape", mutation, current.originAt, + current.ownerInitializer, ); continue; } @@ -195,6 +212,8 @@ export function* collectSemanticMemberMutationsSteps( mutation?.end, current.originAt?.start, current.originAt?.end, + current.ownerInitializer?.start, + current.ownerInitializer?.end, ]); const previous = selections.get(selectionIdentity); const excludedKeys = @@ -220,6 +239,8 @@ export function* collectSemanticMemberMutationsSteps( mutation?.end, current.originAt?.start, current.originAt?.end, + current.ownerInitializer?.start, + current.ownerInitializer?.end, ]); let covering = coveringPaths.get(pathState); if (covering === undefined) { @@ -235,6 +256,8 @@ export function* collectSemanticMemberMutationsSteps( mutation?.end, current.originAt?.start, current.originAt?.end, + current.ownerInitializer?.start, + current.ownerInitializer?.end, ]); if (recordedEffects.has(identity)) continue; // A recorded write invalidates the binding to a non-primitive value, so @@ -248,10 +271,23 @@ export function* collectSemanticMemberMutationsSteps( value?.status === "literal" || value?.status === "union"; recordedEffects.add(identity); const nested = new Set([...current.bindings, lifetime, captureKey]); + // Returning a literal's receiver reaches that allocation, not the + // binding's older values read while evaluating its initializer. const origins = - current.originAt === undefined - ? binding - : semanticMutationInitializers(binding, current.originAt, parents); + current.ownerInitializer !== undefined + ? { + initializers: binding.initializers.filter( + ({ node }) => node === current.ownerInitializer, + ), + referenceInitializers: [], + } + : current.originAt === undefined + ? binding + : semanticMutationInitializers( + binding, + current.originAt, + parents, + ); if (!primitiveWrite) { if (effect === "escape") binding.escapedPaths.push({ path, node: mutation }); @@ -294,7 +330,17 @@ export function* collectSemanticMemberMutationsSteps( for (const value of effect === "escape" ? escapedReferenceLeaves(expression, current.path) : expandedValues(expression, current.path, effect)) { - if (value.iterableFallbackPath !== undefined) + if (value.escapes && effect !== "escape") + markValue( + value.node, + value.path, + new Set(), + "escape", + mutation, + current.originAt, + value.ownerInitializer, + ); + else if (value.iterableFallbackPath !== undefined) deferIterable({ node: value.node, projection: [], @@ -367,6 +413,8 @@ export function* collectSemanticMemberMutationsSteps( binding === undefined ? null : binding.bindingId, binding === undefined ? node.name : null, pathKey, + value.ownerInitializer?.start, + value.ownerInitializer?.end, ]); if (leafIdentities.has(identity)) continue; leafIdentities.add(identity); @@ -467,7 +515,7 @@ export function* collectSemanticMemberMutationsSteps( for (const [index, reference] of pendingReferences.splice(0).entries()) { if (index % 64 === 0) yield; const { initializer } = reference; - if (referenceSourcesInactive(initializer, state)) + if (semanticReferenceSourcesInactive(initializer, state)) pendingReferences.push(reference); else if (t.isForOfStatement(initializer.node)) deferIterable({ @@ -512,7 +560,7 @@ export function* collectSemanticMemberMutationsSteps( } for (const [index, reference] of [...iterableReferences].entries()) { if (index % 64 === 0) yield; - if (referenceSourcesInactive(reference, state)) continue; + if (semanticReferenceSourcesInactive(reference, state)) continue; if (reference.iterationOnly) { const expanded = expandedIterationReferences(reference, state); if (expanded !== null) { @@ -520,7 +568,7 @@ export function* collectSemanticMemberMutationsSteps( continue; } } - const slot = selectedReferenceSlot(reference, state); + const slot = semanticSelectedReferenceSlot(reference, state); // Custom iteration can yield the iterable itself as well as any child. // Treat its root as escaped, including writes through yielded references. const knownArray = @@ -551,9 +599,7 @@ export function* collectSemanticMemberMutationsSteps( clearSemanticValueCaches(state); } -interface ReferencedValue { - readonly node: t.Node; - readonly path: PropertyPath; +interface ReferencedValue extends JavaScriptSemanticObjectReference { readonly iterableFallbackPath?: PropertyPath; } @@ -575,27 +621,6 @@ interface IterableReference extends ReferencedValue { }[]; } -const referenceSourcesInactive = ( - reference: Pick, - state: JavaScriptSemanticAnalysisState, -): boolean => - reference.requiredSources?.some( - (source) => - selectedReferenceSlot(source, state, source.node)?.presence === "absent", - ) === true || - reference.fallbackSources?.some((source) => { - // Selection precedes effects through the chosen reference. A later escape - // cannot make a fallback that was excluded at destructuring reachable. - const slot = selectedReferenceSlot(source, state, source.node); - return ( - slot?.presence === "present" && - (slot.value.status === "literal" || - slot.value.status === "union" || - slot.value.status === "object" || - slot.value.status === "array") - ); - }) === true; - const expandedIterationReferences = ( reference: IterableReference, state: JavaScriptSemanticAnalysisState, @@ -610,7 +635,10 @@ const expandedIterationReferences = ( ...(reference.fallbackSources ?? []), ].find((source) => source.node === iterable && source.projection[0] === null); if (source === undefined) return null; - const slot = selectedReferenceSlot({ node: iterable, projection: [] }, state); + const slot = semanticSelectedReferenceSlot( + { node: iterable, projection: [] }, + state, + ); const container = slot === undefined ? null : semanticContainer(slot.value); if (container?.coverage.status !== "complete") return null; const select = ( @@ -1106,19 +1134,3 @@ const copiedReferencePath = ( } return [...initializer.projection.slice(0, offset), key, ...remaining]; }; - -const selectedReferenceSlot = ( - source: { readonly node: t.Node; readonly projection: PropertyPath }, - state: JavaScriptSemanticAnalysisState, - capturePoint?: t.Node, -) => { - const path: string[] = []; - for (const key of source.projection) { - if (key === null || typeof key === "object") return undefined; - path.push(String(key)); - } - return semanticSlotAtPath( - evaluateSemanticExpression(source.node, state, capturePoint), - path, - ); -}; diff --git a/src/domain/javascript/javascriptSemanticMutationInitializers.ts b/src/domain/javascript/javascriptSemanticMutationInitializers.ts index cfaaf0ef5..6de7217c1 100644 --- a/src/domain/javascript/javascriptSemanticMutationInitializers.ts +++ b/src/domain/javascript/javascriptSemanticMutationInitializers.ts @@ -149,7 +149,50 @@ export const semanticMutationInitializers = ( const mutationPosition = directMutationPosition(mutation, parents) ?? referenceInitializerPosition(mutation, parents); - if (mutationPosition === null) return binding; + return selectInitializers(binding, mutationPosition, parents); +}; + +/** Select a local alias at its read, before any later straight-line reassignment. */ +export const semanticReadInitializers = ( + binding: JavaScriptSemanticBindingState, + read: t.Node, + parents: WeakMap, +): Pick< + JavaScriptSemanticBindingState, + "initializers" | "referenceInitializers" +> => { + let position: StatementPosition | null = null; + let current = read; + for ( + let parent = parents.get(current); + parent; + parent = parents.get(current) + ) { + if (t.isLoop(parent)) return binding; + if (t.isFunction(parent)) break; + if (position === null) { + if (t.isVariableDeclarator(parent)) { + const declaration = parents.get(parent); + if (t.isVariableDeclaration(declaration)) + position = statementPosition(declaration, parents, parent); + } else if (t.isStatement(parent)) + position = statementPosition(parent, parents); + } + current = parent; + } + return selectInitializers(binding, position, parents, true); +}; + +const selectInitializers = ( + binding: JavaScriptSemanticBindingState, + position: StatementPosition | null, + parents: WeakMap, + atRead = false, +): Pick< + JavaScriptSemanticBindingState, + "initializers" | "referenceInitializers" +> => { + if (position === null) return binding; const origins = [ ...binding.initializers.map((initializer) => ({ initializer, @@ -169,12 +212,20 @@ export const semanticMutationInitializers = ( ]; if ( origins.some( - ({ position }) => - position === null || !positionPrecedes(position, mutationPosition), + ({ position: origin }) => + origin === null || + origin.body !== position.body || + (!atRead && !positionPrecedes(origin, position)), ) ) return binding; - const latest = origins.reduce( + const preceding = atRead + ? origins.filter( + ({ position: origin }) => + origin !== null && positionPrecedes(origin, position), + ) + : origins; + const latest = preceding.reduce( (latest, { position }) => position !== null && (latest === null || positionPrecedes(latest, position)) @@ -183,7 +234,7 @@ export const semanticMutationInitializers = ( null, ); const retained = new Set( - origins + preceding .filter( ({ position }) => position?.index === latest?.index && diff --git a/src/domain/javascript/javascriptSemanticObjectReferences.ts b/src/domain/javascript/javascriptSemanticObjectReferences.ts index 387ba09d2..425e5fea5 100644 --- a/src/domain/javascript/javascriptSemanticObjectReferences.ts +++ b/src/domain/javascript/javascriptSemanticObjectReferences.ts @@ -4,6 +4,8 @@ import { type JavaScriptSemanticAnalysisState, } from "./javascriptSemanticState.js"; import { semanticCallableResultExpressions } from "./javascriptSemanticCallableResults.js"; +import { semanticReadInitializers } from "./javascriptSemanticMutationInitializers.js"; +import { semanticReferenceSourcesInactive } from "./javascriptSemanticReferenceSources.js"; import { semanticStaticPropertyKey, unwrapJavaScriptExpression, @@ -14,9 +16,12 @@ import { } from "./javascriptSemanticPropertyPaths.js"; import { compareUnicodeCodePoints } from "../unicodeCodePointOrder.js"; -interface ReferencedValue { +export interface JavaScriptSemanticObjectReference { readonly node: t.Node; readonly path: PropertyPath; + readonly escapes?: true; + /** The allocation owned by this binding, rather than a read of its prior value. */ + readonly ownerInitializer?: t.Node; } export const semanticClassCandidates = ( @@ -71,11 +76,12 @@ const instanceFieldReferences = ( propertyName: string | null, path: PropertyPath, state: JavaScriptSemanticAnalysisState, + initializedBefore?: t.ClassProperty, ): { - readonly references: readonly ReferencedValue[]; + readonly references: readonly JavaScriptSemanticObjectReference[]; readonly definiteNames: ReadonlySet; } => { - const references: ReferencedValue[] = []; + const references: JavaScriptSemanticObjectReference[] = []; const pending = [{ klass: root, excluded: new Set() }]; const selections = new Map>(); let definiteNames: ReadonlySet | undefined; @@ -92,7 +98,15 @@ const instanceFieldReferences = ( if (previous !== undefined && previous.size === excluded.size) continue; selections.set(current.klass, excluded); const shadowed = new Set(excluded); - for (const member of [...current.klass.body.body].reverse()) { + const before = + initializedBefore === undefined + ? -1 + : current.klass.body.body.indexOf(initializedBefore); + const members = + before < 0 + ? current.klass.body.body + : current.klass.body.body.slice(0, before); + for (const member of [...members].reverse()) { if (!t.isClassProperty(member) || member.static) continue; const key = semanticStaticPropertyKey(member.key, member.computed); if (key !== null && shadowed.has(key)) continue; @@ -121,12 +135,15 @@ interface ClassPropertyLookup { readonly owner: t.ClassDeclaration | t.ClassExpression; readonly receiver?: t.ClassDeclaration | t.ClassExpression; readonly kind: "instance" | "static" | "prototype"; + readonly fieldsOnly?: true; + readonly initializedBefore?: t.ClassProperty; } const classPropertyDescriptors = ( klass: t.ClassDeclaration | t.ClassExpression, propertyName: string | null, kind: ClassPropertyLookup["kind"], + initializedBefore?: t.ClassProperty, ) => { const isStatic = kind === "static"; const selectedGetters: t.ClassMethod[] = []; @@ -136,7 +153,12 @@ const classPropertyDescriptors = ( readonly index: number; readonly key: string | null; }[] = []; + const before = + initializedBefore === undefined + ? -1 + : klass.body.body.indexOf(initializedBefore); klass.body.body.forEach((member, index) => { + if (before >= 0 && index >= before && t.isClassProperty(member)) return; if ( (t.isClassMethod(member) || (kind === "static" && t.isClassProperty(member))) && @@ -208,7 +230,7 @@ export const semanticClassPropertyReferences = ( selection: PropertyPath, state: JavaScriptSemanticAnalysisState, activeGetters: ReadonlyMap = new Map(), -): readonly ReferencedValue[] => { +): readonly JavaScriptSemanticObjectReference[] => { const root = lookup.owner; const receiver = lookup.receiver ?? root; const [key, ...path] = selection; @@ -216,12 +238,18 @@ export const semanticClassPropertyReferences = ( key == null || typeof key === "object" ? null : String(key); const pending: (t.ClassDeclaration | t.ClassExpression)[] = [root]; const seen = new Set(); - const possibleResults: ReferencedValue[] = []; + const possibleResults: JavaScriptSemanticObjectReference[] = []; // Instance fields are own properties, including fields initialized by base // constructors. They take precedence over every prototype descriptor. let instanceFields: ReadonlySet = new Set(); if (lookup.kind === "instance") { - const fields = instanceFieldReferences(root, propertyName, path, state); + const fields = instanceFieldReferences( + root, + propertyName, + path, + state, + lookup.initializedBefore, + ); possibleResults.push(...fields.references); instanceFields = fields.definiteNames; if (propertyName !== null && instanceFields.has(propertyName)) @@ -235,11 +263,12 @@ export const semanticClassPropertyReferences = ( klass, propertyName, lookup.kind, + lookup.initializedBefore, ); for (const field of descriptors.fields) if (field.value != null) possibleResults.push({ node: field.value, path }); - for (const getter of descriptors.getters) { + for (const getter of lookup.fieldsOnly ? [] : descriptors.getters) { const key = semanticStaticPropertyKey(getter.key, getter.computed); if (key !== null && instanceFields.has(key)) continue; possibleResults.push( @@ -267,11 +296,11 @@ const objectReceiverReference = ( object: t.ObjectExpression, path: PropertyPath, state: JavaScriptSemanticAnalysisState, -): readonly ReferencedValue[] => { +): readonly JavaScriptSemanticObjectReference[] => { // A literal's enclosing binding owns its slots. Returning the literal // itself would bypass invalidation of that binding's observed properties. let current: t.Node = object; - const projection: (string | number | null)[] = []; + const projection: PropertyPath[number][] = []; while (true) { const parent = state.parentsByNode.get(current); if ( @@ -279,7 +308,43 @@ const objectReceiverReference = ( parent.init === current && t.isIdentifier(parent.id) ) - return [{ node: parent.id, path: [...projection, ...path] }]; + return [ + { + node: parent.id, + path: [...projection, ...path], + ownerInitializer: current, + }, + ]; + if ( + t.isAssignmentExpression(parent, { operator: "=" }) && + parent.right === current && + t.isIdentifier(parent.left) + ) + return [ + { + node: parent.left, + path: [...projection, ...path], + ownerInitializer: current, + }, + ]; + if (t.isArrayExpression(parent)) { + let minimumIndex = 0; + let uncertainIndex = false; + for (const element of parent.elements) { + if (element === current) break; + if (t.isSpreadElement(element)) uncertainIndex = true; + else minimumIndex++; + } + // A spread contributes an unknown number of slots. Retain the known + // prefix without treating the element's syntax position as its index. + projection.unshift( + uncertainIndex + ? { excludedKeys: [], startIndex: minimumIndex } + : minimumIndex, + ); + current = parent; + continue; + } if (t.isObjectProperty(parent) && parent.value === current) { projection.unshift( semanticStaticPropertyKey(parent.key, parent.computed), @@ -307,12 +372,108 @@ interface GetterInvocation { | t.ClassExpression; } +/** Follow only data fields whose initializers retain a class receiver. */ +const privateReceiverProjection = ( + receiver: t.ClassDeclaration | t.ClassExpression, + owner: t.ClassDeclaration | t.ClassExpression, + path: PropertyPath, + isStatic: boolean, + state: JavaScriptSemanticAnalysisState, +): boolean => { + if (path.some((key) => key === null || typeof key === "object")) return false; + type Selection = { readonly key: string; readonly before?: t.ClassProperty }; + const pending: { + receiver: typeof receiver; + path: readonly Selection[]; + visits: ReadonlyMap; + }[] = [ + { + receiver, + path: path.map((key) => ({ key: String(key) })), + visits: new Map(), + }, + ]; + const seen = new Set(); + const declaringClass = (field: t.ClassProperty) => { + const body = state.parentsByNode.get(field); + return body === undefined ? undefined : state.parentsByNode.get(body); + }; + for (let cursor = 0; cursor < pending.length; cursor++) { + const current = pending[cursor]; + if (current === undefined) break; + const [selection, ...remaining] = current.path; + if (selection === undefined) { + if (!isStatic || current.receiver === owner) return true; + continue; + } + const origin = JSON.stringify([ + current.receiver.start, + selection.key, + selection.before?.start, + ]); + const previous = current.visits.get(origin); + // Repeated fields may consume a finite getter path. An initializer cycle + // that preserves or grows it cannot establish another receiver identity. + if (previous !== undefined && previous <= current.path.length) continue; + const visits = new Map(current.visits).set(origin, current.path.length); + const identity = JSON.stringify([ + current.receiver.start, + current.path.map(({ key, before }) => [key, before?.start]), + ]); + if (seen.has(identity)) continue; + seen.add(identity); + const lookupOwner = + selection.before === undefined + ? current.receiver + : declaringClass(selection.before); + if (!t.isClass(lookupOwner)) continue; + for (const reference of semanticClassPropertyReferences( + { + owner: lookupOwner, + receiver: current.receiver, + kind: isStatic ? "static" : "instance", + fieldsOnly: true, + ...(selection.before === undefined + ? {} + : { initializedBefore: selection.before }), + }, + [selection.key], + state, + )) { + const field = state.parentsByNode.get(reference.node); + if (!t.isClassProperty(field)) continue; + const fieldOwner = declaringClass(field); + if (!t.isClass(fieldOwner)) continue; + const projection: Selection[] = []; + let value = unwrapJavaScriptExpression(reference.node).node; + while ( + t.isMemberExpression(value) || + t.isOptionalMemberExpression(value) + ) { + const key = semanticStaticPropertyKey(value.property, value.computed); + if (key === null || t.isPrivateName(value.property)) break; + projection.unshift({ key, before: field }); + value = unwrapJavaScriptExpression(value.object).node; + } + if (!t.isThisExpression(value)) continue; + // Instance fields retain the actual instance. Static field `this` belongs + // to its declaring class, even when a subclass inherits the field. + pending.push({ + receiver: isStatic ? fieldOwner : current.receiver, + path: [...projection, ...remaining], + visits, + }); + } + } + return false; +}; + const getterResultReferences = ( invocation: GetterInvocation, path: PropertyPath, state: JavaScriptSemanticAnalysisState, activeGetters: ReadonlyMap = new Map(), -): readonly ReferencedValue[] => { +): readonly JavaScriptSemanticObjectReference[] => { const { getter, owner, receiver = owner } = invocation; const previousLength = activeGetters.get(getter); if (previousLength !== undefined && previousLength <= path.length) return []; @@ -320,40 +481,206 @@ const getterResultReferences = ( // path; recursive getters that preserve or grow that path cannot progress. const active = new Map(activeGetters).set(getter, path.length); const isStatic = t.isClassMethod(getter) && getter.static; - const resolve = ( - node: t.Node, - remaining: PropertyPath, - ): readonly ReferencedValue[] => { - const expression = unwrapJavaScriptExpression(node).node; - if (t.isThisExpression(expression)) { - if (t.isClass(receiver)) { - return semanticClassPropertyReferences( - { owner: receiver, kind: isStatic ? "static" : "instance" }, - remaining, - state, - active, - ); + interface AliasVisit { + readonly identity: string; + readonly pathLength: number; + readonly parent: AliasVisit | undefined; + } + const pending: { + readonly node: t.Node; + readonly path: PropertyPath; + readonly aliases: AliasVisit | undefined; + readonly escapes?: true; + readonly privateRead?: { + readonly node: t.MemberExpression | t.OptionalMemberExpression; + readonly name: string; + readonly path: PropertyPath; + }; + }[] = semanticCallableResultExpressions(getter).flatMap(({ node }) => + node === null ? [] : [{ node, path, aliases: undefined }], + ); + const references: JavaScriptSemanticObjectReference[] = []; + const expandedAliases = new Map>(); + const getterScope = state.scopeByNode.get(getter)?.scopeId; + for (let cursor = 0; cursor < pending.length; cursor++) { + const current = pending[cursor]; + if (current === undefined) break; + const { path: remaining } = current; + const expression = unwrapJavaScriptExpression(current.node).node; + const retain = (reference: JavaScriptSemanticObjectReference): void => { + references.push( + current.escapes ? { ...reference, escapes: true } : reference, + ); + }; + const retainPrivateRead = (): void => { + if (current.privateRead !== undefined) + retain({ + node: current.privateRead.node, + path: current.escapes ? [] : current.privateRead.path, + }); + }; + if (t.isIdentifier(expression)) { + const binding = resolveSemanticBindingState( + state, + expression, + expression.name, + ); + let scope = + binding === undefined + ? undefined + : state.scopesById.get(binding.scopeId); + while (scope !== undefined && scope.scopeId !== getterScope) { + if (scope.kind === "function" || scope.parentScopeId === null) { + scope = undefined; + break; + } + scope = state.scopesById.get(scope.parentScopeId); } - return objectReceiverReference(receiver, remaining, state); + if ( + binding !== undefined && + scope !== undefined && + binding.referenceInitializers.every( + (initializer) => initializer.copyKind === undefined, + ) + ) { + const identity = JSON.stringify([ + binding.bindingId, + expression.start, + expression.end, + ]); + const known = expandedAliases.get(identity); + const key = JSON.stringify([ + remaining, + current.privateRead?.name, + current.privateRead?.path, + current.escapes, + ]); + let cyclic = false; + let growing = false; + if (known !== undefined) + for ( + let visit = current.aliases; + visit !== undefined; + visit = visit.parent + ) + if (visit.identity === identity) { + cyclic = true; + growing = remaining.length > visit.pathLength; + break; + } + if (cyclic) { + // A repeated alias can select arbitrarily deep children. Revisit its + // roots as escapes rather than enumerating iterations or dropping them. + if (growing && !current.escapes) + pending.push({ + ...current, + path: [], + aliases: undefined, + escapes: true, + }); + continue; + } + if (known?.has(key)) continue; + const selections = known ?? new Set(); + selections.add(key); + expandedAliases.set(identity, selections); + const aliases = { + identity, + pathLength: remaining.length, + parent: current.aliases, + }; + const origins = semanticReadInitializers( + binding, + expression, + state.parentsByNode, + ); + for (const initializer of [ + ...origins.initializers, + ...origins.referenceInitializers.filter( + (initializer) => + !semanticReferenceSourcesInactive(initializer, state), + ), + ]) + pending.push({ + ...current, + node: initializer.node, + path: [...initializer.projection, ...remaining], + aliases, + }); + continue; + } + } + if (t.isThisExpression(expression)) { + // An outer binding or a nested ordinary function has a different receiver. + let ownerNode = state.parentsByNode.get(expression); + while ( + ownerNode !== undefined && + !t.isClass(ownerNode) && + !(t.isFunction(ownerNode) && !t.isArrowFunctionExpression(ownerNode)) + ) + ownerNode = state.parentsByNode.get(ownerNode); + if (ownerNode !== getter) continue; + if (current.privateRead !== undefined && t.isClass(owner)) { + if ( + !t.isClass(receiver) || + !privateReceiverProjection( + receiver, + owner, + remaining, + isStatic, + state, + ) + ) { + retainPrivateRead(); + continue; + } + for (const member of owner.body.body) + if ( + t.isClassPrivateProperty(member) && + member.static === isStatic && + member.key.id.name === current.privateRead.name && + member.value != null + ) + retain({ + node: member.value, + path: current.escapes ? [] : current.privateRead.path, + }); + continue; + } + for (const reference of t.isClass(receiver) + ? semanticClassPropertyReferences( + { owner: receiver, kind: isStatic ? "static" : "instance" }, + remaining, + state, + active, + ) + : objectReceiverReference(receiver, remaining, state)) + retain(reference); + continue; } if ( t.isMemberExpression(expression) || t.isOptionalMemberExpression(expression) ) { if ( - t.isPrivateName(expression.property) && - t.isThisExpression(expression.object) && - t.isClass(owner) + current.privateRead !== undefined && + t.isPrivateName(expression.property) ) { - const privateName = expression.property.id.name; - return owner.body.body.flatMap((member) => - t.isClassPrivateProperty(member) && - member.static === isStatic && - member.key.id.name === privateName && - member.value != null - ? [{ node: member.value, path: remaining }] - : [], - ); + retainPrivateRead(); + continue; + } + if (t.isPrivateName(expression.property) && t.isClass(owner)) { + pending.push({ + ...current, + node: expression.object, + path: [], + privateRead: { + node: expression, + name: expression.property.id.name, + path: remaining, + }, + }); + continue; } const key = semanticStaticPropertyKey( expression.property, @@ -364,50 +691,61 @@ const getterResultReferences = ( t.isClass(owner) && owner.superClass != null ) { - return semanticClassCandidates(owner.superClass, state).flatMap( - (base) => - semanticClassPropertyReferences( - { - owner: base, - receiver: t.isClass(receiver) ? receiver : owner, - kind: isStatic ? "static" : "prototype", - }, - [key, ...remaining], - state, - active, - ), - ); + for (const base of semanticClassCandidates(owner.superClass, state)) + for (const reference of semanticClassPropertyReferences( + { + owner: base, + receiver: t.isClass(receiver) ? receiver : owner, + kind: isStatic ? "static" : "prototype", + }, + [key, ...remaining], + state, + active, + )) + retain(reference); + continue; } - return resolve(expression.object, [key, ...remaining]); - } - if (t.isConditionalExpression(expression)) - return [ - ...resolve(expression.consequent, remaining), - ...resolve(expression.alternate, remaining), - ]; - if (t.isLogicalExpression(expression)) - return [ - ...resolve(expression.left, remaining), - ...resolve(expression.right, remaining), - ]; - if (t.isSequenceExpression(expression)) { + pending.push({ + ...current, + node: expression.object, + path: [key, ...remaining], + }); + } else if (t.isConditionalExpression(expression)) + pending.push( + { ...current, node: expression.consequent }, + { ...current, node: expression.alternate }, + ); + else if (t.isLogicalExpression(expression)) + pending.push( + { ...current, node: expression.left }, + { ...current, node: expression.right }, + ); + else if (t.isSequenceExpression(expression)) { const last = expression.expressions.at(-1); - return last === undefined ? [] : resolve(last, remaining); - } - return [{ node: expression, path: remaining }]; - }; - return semanticCallableResultExpressions(getter).flatMap(({ node }) => - node === null ? [] : resolve(node, path), - ); + if (last !== undefined) pending.push({ ...current, node: last }); + } else if (t.isAssignmentExpression(expression, { operator: "=" })) + pending.push({ ...current, node: expression.right }); + else if ( + t.isAssignmentExpression(expression) && + ["&&=", "||=", "??="].includes(expression.operator) + ) + pending.push( + { ...current, node: expression.left }, + { ...current, node: expression.right }, + ); + else if (current.privateRead !== undefined) retainPrivateRead(); + else retain({ node: expression, path: remaining }); + } + return references; }; export const semanticObjectPropertyReferences = ( node: t.ObjectExpression, path: PropertyPath, state: JavaScriptSemanticAnalysisState, -): readonly ReferencedValue[] => { +): readonly JavaScriptSemanticObjectReference[] => { const [key, ...remaining] = path; - const references: ReferencedValue[] = []; + const references: JavaScriptSemanticObjectReference[] = []; const overwritten = new Set(); const dataOverwritten = new Set(); const accessorNames = new Set(); diff --git a/src/domain/javascript/javascriptSemanticReferenceSources.ts b/src/domain/javascript/javascriptSemanticReferenceSources.ts new file mode 100644 index 000000000..54e1d3ba9 --- /dev/null +++ b/src/domain/javascript/javascriptSemanticReferenceSources.ts @@ -0,0 +1,53 @@ +import type * as t from "@babel/types"; +import type { JavaScriptSemanticAnalysisState } from "./javascriptSemanticState.js"; +import type { JavaScriptSemanticPropertyPath as PropertyPath } from "./javascriptSemanticPropertyPaths.js"; +import { evaluateSemanticExpression } from "./javascriptSemanticValues.js"; +import { semanticSlotAtPath } from "./javascriptSemanticSlots.js"; + +interface ReferenceSource { + readonly node: t.Node; + readonly projection: PropertyPath; +} + +/** Exclude references whose source slot or destructuring fallback is absent. */ +export const semanticReferenceSourcesInactive = ( + reference: { + readonly requiredSources?: readonly ReferenceSource[]; + readonly fallbackSources?: readonly ReferenceSource[]; + }, + state: JavaScriptSemanticAnalysisState, +): boolean => + reference.requiredSources?.some( + (source) => + semanticSelectedReferenceSlot(source, state, source.node)?.presence === + "absent", + ) === true || + reference.fallbackSources?.some((source) => { + // Selection precedes effects through the chosen reference. A later escape + // cannot make a fallback that was excluded at destructuring reachable. + const slot = semanticSelectedReferenceSlot(source, state, source.node); + return ( + slot?.presence === "present" && + (slot.value.status === "literal" || + slot.value.status === "union" || + slot.value.status === "object" || + slot.value.status === "array") + ); + }) === true; + +/** Select a retained source slot at its capture, before later effects. */ +export const semanticSelectedReferenceSlot = ( + source: ReferenceSource, + state: JavaScriptSemanticAnalysisState, + capturePoint?: t.Node, +) => { + const path: string[] = []; + for (const key of source.projection) { + if (key === null || typeof key === "object") return undefined; + path.push(String(key)); + } + return semanticSlotAtPath( + evaluateSemanticExpression(source.node, state, capturePoint), + path, + ); +}; diff --git a/src/domain/javascript/javascriptStaticAnalysisHelpers.ts b/src/domain/javascript/javascriptStaticAnalysisHelpers.ts index 71562e6ca..f2045a1eb 100644 --- a/src/domain/javascript/javascriptStaticAnalysisHelpers.ts +++ b/src/domain/javascript/javascriptStaticAnalysisHelpers.ts @@ -181,18 +181,37 @@ export const staticPath = (node: t.Node): string | undefined => { if (t.isStringLiteral(node)) return node.value; if (t.isTemplateLiteral(node) && node.expressions.length === 0) return stringValue(node); - if (t.isBinaryExpression(node, { operator: "+" })) { - const left = staticPath(node.left); - const right = staticPath(node.right); - return left === undefined || right === undefined - ? undefined - : `${left}${right}`; - } + if (t.isBinaryExpression(node, { operator: "+" })) + return staticConcatenation(node); if (t.isCallExpression(node) || t.isNewExpression(node)) return staticCallPath(node); return undefined; }; +/** + * Concatenate inert text. A leading source directory contributes no text: + * `__dirname + "/preload.js"` names a file inside it, while text without a + * leading separator extends the directory's own name and stays unknown. + */ +const staticConcatenation = (node: t.BinaryExpression): string | undefined => { + const operands: t.Node[] = []; + let current: t.Node = node; + while (t.isBinaryExpression(current, { operator: "+" })) { + operands.push(current.right); + current = current.left; + } + const anchored = isDirectoryPath(current); + if (!anchored) operands.push(current); + let text = ""; + for (const operand of operands.reverse()) { + const value = staticPath(operand); + if (value === undefined) return undefined; + text += value; + } + if (!anchored) return text; + return text.startsWith("/") ? posix.join(".", text) : undefined; +}; + const staticCallPath = ( node: t.CallExpression | t.NewExpression, ): string | undefined => { @@ -204,11 +223,6 @@ const staticCallPath = ( if (name === "fileURLToPath" || name.endsWith(".fileURLToPath")) { return staticFileUrlPath(argumentNode(node.arguments[0])); } - if ( - (name === "dirname" || name.endsWith(".dirname")) && - isFileIdentity(argumentNode(node.arguments[0])) - ) - return ""; if (name.endsWith(".resolve")) return name === "win32.resolve" || name.endsWith(".win32.resolve") ? undefined @@ -216,7 +230,12 @@ const staticCallPath = ( if (!name.endsWith(".join") && name !== "join") return undefined; const parts: string[] = []; for (const argument of node.arguments) { - if (isDirectoryIdentity(argumentNode(argument))) continue; + if (isDirectoryIdentity(argumentNode(argument))) { + // join never resets at an absolute segment, so `/preload.js` after the + // source directory stays inside it. + parts.push("."); + continue; + } const value = t.isNode(argument) ? staticPath(argument) : undefined; if (value === undefined) return undefined; parts.push(value); @@ -331,11 +350,20 @@ const isFilesystemPathExpression = (node: t.Node): boolean => { }; const isDirectoryIdentity = (node: t.Node | undefined): boolean => { - if (t.isIdentifier(node, { name: "__dirname" })) return true; + if (isDirectoryPath(node)) return true; if (!t.isCallExpression(node) && !t.isNewExpression(node)) return false; const name = calleeName(node.callee); - if (name === "URL" || name.endsWith(".URL")) - return isFileIdentity(argumentNode(node.arguments[1])); + return ( + (name === "URL" || name.endsWith(".URL")) && + isFileIdentity(argumentNode(node.arguments[1])) + ); +}; + +/** `__dirname` or the dirname of the source file, as a string. */ +const isDirectoryPath = (node: t.Node | undefined): boolean => { + if (t.isIdentifier(node, { name: "__dirname" })) return true; + if (!t.isCallExpression(node)) return false; + const name = calleeName(node.callee); return ( (name === "dirname" || name.endsWith(".dirname")) && isFileIdentity(argumentNode(node.arguments[0])) diff --git a/src/domain/javascript/javascriptStaticPathResolution.test.ts b/src/domain/javascript/javascriptStaticPathResolution.test.ts index e54122440..30bbc317f 100644 --- a/src/domain/javascript/javascriptStaticPathResolution.test.ts +++ b/src/domain/javascript/javascriptStaticPathResolution.test.ts @@ -38,6 +38,9 @@ const anchoredExpressions = [ ["path.resolve(__dirname)", "."], ['path.posix.resolve(`/app`, `/shared`, "preload.js")', "/shared/preload.js"], ['path.posix.resolve("" + "/app", "preload.js")', "/app/preload.js"], + ['__dirname + "/preload.js"', "preload.js"], + ['__dirname + "/" + "preload.js"', "preload.js"], + ['path.dirname(__filename) + "/lib" + "/preload.js"', "lib/preload.js"], ] as const; const unknownExpressions = [ @@ -58,6 +61,8 @@ const unknownExpressions = [ 'path.win32.resolve(__dirname, "preload.js")', 'win32.resolve("/app", "preload.js")', 'resolve("/app", "preload.js")', + '__dirname + "preload.js"', + 'path.dirname(__filename) + "preload.js"', ]; describe("inert JavaScript path resolution", () => { @@ -100,6 +105,16 @@ describe("inert JavaScript path resolution", () => { "preload.js", "filesystem-expression", ], + [ + 'path.join(__dirname, "/preload.js")', + "preload.js", + "filesystem-expression", + ], + [ + 'path.join(dirname(fileURLToPath(import.meta.url)), "/lib", "/preload.js")', + "lib/preload.js", + "filesystem-expression", + ], [ 'path.join("ignored", __dirname, "preload.js")', "ignored/preload.js", @@ -111,6 +126,7 @@ describe("inert JavaScript path resolution", () => { "filesystem-expression", ], ['join("sub", "preload.js")', "sub/preload.js", "module-specifier"], + ["path.join(__dirname)", ".", "filesystem-expression"], ] as const)("preserves join behavior in %s", (expression, path, context) => { assertPathProjection(expression, path, context); }); @@ -182,7 +198,6 @@ describe("inert JavaScript path resolution", () => { it.each([ "path.join()", - "path.join(__dirname)", 'path.join("/app", runtimeFile)', "path.join(...parts)", ])("preserves unresolved join syntax in %s", (expression) => diff --git a/src/domain/managed/managedArtifact.ts b/src/domain/managed/managedArtifact.ts index 297b60a4e..94eff2398 100644 --- a/src/domain/managed/managedArtifact.ts +++ b/src/domain/managed/managedArtifact.ts @@ -203,10 +203,18 @@ const managedMethodSchema = z.object({ body: managedMethodBodySchema, }); +// Declaring types are namespace-qualified like TypeDef `full_name`, nested +// types keep bare names, and a generic instantiation names its generic type. +// Evidence recorded before declaring types were named reads them as null. +const declaringTypeNameSchema = z.string().nullable().default(null); + const managedMemberReferenceSchema = z.object({ token: tokenSchema, row_offset: offsetSchema, parent_token: tokenSchema.nullable(), + parent_name: declaringTypeNameSchema.describe( + "Name of a TypeDef, TypeRef, or generic-instantiation TypeSpec parent; null for other parents or when unknown", + ), name: z.string(), signature: managedSignatureSchema, }); @@ -281,7 +289,13 @@ const managedCallEdgeSchema = z.object({ opcode: z.string(), target_token: tokenSchema, target_kind: z.enum(["method-def", "member-ref", "method-spec", "unknown"]), - target_name: z.string().nullable(), + target_name: z + .string() + .nullable() + .describe("Method name; a method-spec target reports its generic method"), + target_declaring_type: declaringTypeNameSchema.describe( + "Declaring type of the called method; null when unknown", + ), }); const managedFieldAccessSchema = z.object({ @@ -290,6 +304,9 @@ const managedFieldAccessSchema = z.object({ opcode: z.string(), field_token: tokenSchema, field_name: z.string().nullable(), + field_declaring_type: declaringTypeNameSchema.describe( + "Declaring type of the accessed field; null when unknown", + ), }); const managedParseIssueSchema = z.object({ diff --git a/src/domain/native/nativeAotPe.test.ts b/src/domain/native/nativeAotPe.test.ts index 8ae624d44..ab031cc85 100644 --- a/src/domain/native/nativeAotPe.test.ts +++ b/src/domain/native/nativeAotPe.test.ts @@ -159,3 +159,22 @@ describe("read-only NativeAOT PE parser", () => { }); }); }); + +describe("NativeAOT PE signature admission", () => { + it.each([ + ["MZ", "Not a PE image"], + ["PE", "Invalid PE signature"], + ])( + "rejects a high-bit lookalike of the %s signature", + (signature, reason) => { + const bytes = nativeAotPeFixture(); + const offset = signature === "MZ" ? 0 : bytes.readUInt32LE(0x3c); + for (const position of [offset, offset + 1]) + bytes.writeUInt8(bytes.readUInt8(position) | 0x80, position); + expect( + parseNativeAotPe(bytes, nativeAotPeDigest(bytes), SUMMARY_BUDGET) + .summary, + ).toMatchObject({ status: "not_applicable", reason }); + }, + ); +}); diff --git a/src/domain/native/nativeAotPe.ts b/src/domain/native/nativeAotPe.ts index b67a58398..7f9c5eb55 100644 --- a/src/domain/native/nativeAotPe.ts +++ b/src/domain/native/nativeAotPe.ts @@ -384,11 +384,11 @@ class PeImage { constructor(bytes: Buffer) { this.#bytes = bytes; - if (bytes.length < 0x40 || bytes.toString("ascii", 0, 2) !== "MZ") + if (bytes.length < 0x40 || bytes.toString("latin1", 0, 2) !== "MZ") throw new Error("Not a PE image"); const peOffset = this.#u32(0x3c); this.#range(peOffset, 24); - if (bytes.toString("ascii", peOffset, peOffset + 4) !== "PE\0\0") + if (bytes.toString("latin1", peOffset, peOffset + 4) !== "PE\0\0") throw new Error("Invalid PE signature"); this.#machine = this.#u16(peOffset + 4); const count = this.#u16(peOffset + 6); diff --git a/src/domain/native/nativeInvestigationGraph.test.ts b/src/domain/native/nativeInvestigationGraph.test.ts index 015458319..684b2a02e 100644 --- a/src/domain/native/nativeInvestigationGraph.test.ts +++ b/src/domain/native/nativeInvestigationGraph.test.ts @@ -83,6 +83,61 @@ const graph = nativeInvestigationGraphSchema.parse({ }); describe("native investigation graph traces", () => { + it.each(["forward", "backward"] as const)( + "keeps both endpoints of retained edges within the %s node budget", + (direction) => { + const trace = traceNativeInvestigationGraph(graph, { + start: direction === "forward" ? "a" : "c", + direction, + limits: { max_nodes: 1 }, + }); + expect(trace.nodes).toHaveLength(1); + expect(trace.edges).toEqual([]); + expect(trace).toMatchObject({ + truncated: true, + reason: "max_nodes_reached", + reached_depth: 0, + }); + }, + ); + + it.each([{ max_nodes: 2 }, { max_edges: 1 }])( + "reports retained node depth when a sibling hits the budget %j", + (limits) => { + const branching = { + ...graph, + edges: graph.edges + .filter((edge) => edge.resolution !== "unresolved") + .map((edge) => ({ ...edge, from: "a" })), + }; + const trace = traceNativeInvestigationGraph(branching, { + start: "a", + limits, + }); + expect(trace.nodes.map(({ id }) => id)).toEqual(["a", "b"]); + expect(trace.edges.map(({ id }) => id)).toEqual(["1"]); + expect(trace.reached_depth).toBe(1); + expect(trace.truncated).toBe(true); + }, + ); + + it("retains cycle edges to admitted nodes when the node budget is full", () => { + const cycle = { + ...graph, + nodes: graph.nodes.slice(0, 2), + edges: [ + graph.edges[0], + { ...graph.edges[0], id: "back", from: "b", to: "a" }, + ], + }; + const trace = traceNativeInvestigationGraph( + nativeInvestigationGraphSchema.parse(cycle), + { start: "a", limits: { max_nodes: 2 } }, + ); + expect(trace.edges.map(({ id }) => id)).toEqual(["1", "back"]); + expect(trace).toMatchObject({ reached_depth: 1, truncated: false }); + }); + it("retains observed and inferred route edges plus unresolved dispatch", () => { const trace = traceNativeInvestigationGraph(graph, { start: "a" }); expect(trace.nodes.map(({ id }) => id)).toEqual(["a", "b", "c"]); diff --git a/src/domain/native/nativeInvestigationGraph.ts b/src/domain/native/nativeInvestigationGraph.ts index 0dd0dbe8b..9b9efc412 100644 --- a/src/domain/native/nativeInvestigationGraph.ts +++ b/src/domain/native/nativeInvestigationGraph.ts @@ -207,7 +207,6 @@ export const traceNativeInvestigationGraph = ( while (queue.length > 0) { const current = queue.shift(); if (current === undefined) break; - reachedDepth = Math.max(reachedDepth, current.depth); const adjacent = adjacency.get(current.id) ?? []; for (const edge of adjacent) { @@ -227,19 +226,24 @@ export const traceNativeInvestigationGraph = ( reason ??= "max_depth_reached"; continue; } - selectedEdges.set(edge.id, edge); - if (alreadyVisited) continue; - if (visited.size >= limits.max_nodes) { + // Check destination capacity before retaining its edge. Otherwise the node + // bound leaves a resolved edge pointing outside the returned trace. + if (!alreadyVisited && visited.size >= limits.max_nodes) { truncated = true; reason = "max_nodes_reached"; break; } + selectedEdges.set(edge.id, edge); + if (alreadyVisited) continue; if (!nodeById.has(nextId)) { truncated = true; reason ??= "edge_target_missing_from_graph"; continue; } visited.add(nextId); + // A retained node has already been reached even if a later sibling + // exhausts a budget before this node is dequeued for expansion. + reachedDepth = Math.max(reachedDepth, current.depth + 1); queue.push({ id: nextId, depth: current.depth + 1 }); } if (reason === "max_edges_reached" || reason === "max_nodes_reached") break; diff --git a/src/domain/native/nativeLoadImage.ts b/src/domain/native/nativeLoadImage.ts index 6a3be4421..a4efc2847 100644 --- a/src/domain/native/nativeLoadImage.ts +++ b/src/domain/native/nativeLoadImage.ts @@ -1,6 +1,7 @@ import { z } from "zod"; import { nativeMetadataRecoverySummarySchema } from "./nativeMetadataRecovery.js"; import { jsonValueSchema } from "../jsonValue.js"; +import { binaryArchitectureSchema } from "../rawBinary.js"; const integer = z.number().int().min(0).max(Number.MAX_SAFE_INTEGER); const digest = z.string().regex(/^[a-f0-9]{64}$/u); @@ -112,7 +113,7 @@ export const nativeLoadImageSchema = z.union([ status: z.enum(["verified", "mismatch"]), format: z.literal("raw-binary"), target_sha256: digest, - architecture: z.enum(["x86", "x86_64", "arm", "arm64"]), + architecture: binaryArchitectureSchema, language_id: z.string().min(1), compiler_spec_id: z.string().min(1), base_address: address, diff --git a/src/domain/rawBinary.test.ts b/src/domain/rawBinary.test.ts index f22e5fced..5cb6ed129 100644 --- a/src/domain/rawBinary.test.ts +++ b/src/domain/rawBinary.test.ts @@ -39,6 +39,44 @@ describe("raw-binary entry addresses", () => { }); describe("raw-binary admission", () => { + it("admits byte-aligned 6502 images up to the 16-bit address boundary", () => { + for (const [entryPoint, fileBytes] of [ + ["0x8001", 8], + ["0xffff", 1], + ["0x0", 65536], + ] as const) { + const result = admitRawBinary({ + architecture: "6502", + entryPoint, + fileBytes, + }); + if (!result.ok) throw new Error(result.error.reason); + expect(result.value).toEqual({ + architecture: "6502", + entryPoint, + entryPointSource: entryPoint, + }); + } + }); + it.each([ + ["0x10000", 1], + ["0xffff", 2], + ["0x0", 65537], + ])( + "rejects a 6502 mapping at %s with %i bytes that exceeds 16 bits", + (entryPoint, fileBytes) => { + const result = admitRawBinary({ + architecture: "6502", + entryPoint, + fileBytes, + }); + if (result.ok) throw new Error("expected 16-bit overflow rejection"); + expect(result.error).toEqual({ + reason: `raw-binary mapping from ${entryPoint} must fit in the 6502 address width`, + constraint: "load_selection_rejected", + }); + }, + ); it("admits a little-endian ARM image mapped at the caller entry", () => { expect( admitRawBinary({ diff --git a/src/domain/rawBinary.ts b/src/domain/rawBinary.ts index 94fdf6ce7..22d94f4d6 100644 --- a/src/domain/rawBinary.ts +++ b/src/domain/rawBinary.ts @@ -13,6 +13,7 @@ export const BINARY_ARCHITECTURES = [ "x86_64", "arm", "arm64", + "6502", ] as const satisfies readonly BinaryArchitecture[]; /** Provider-neutral CPU family for a caller-supplied raw-binary load. */ @@ -58,7 +59,7 @@ export const validateRawBinaryLength = ( /** Admitted headerless load identity used by target resolution and profiles. */ export interface RawBinaryAdmission { - readonly architecture: HeaderArchitecture; + readonly architecture: HeaderArchitecture | "6502"; readonly entryPoint: string; readonly entryPointSource: string; } @@ -74,9 +75,11 @@ export interface RawBinaryAdmissionRejection { } const addressWidth = (architecture: BinaryArchitecture): bigint => - architecture === "x86" || architecture === "arm" - ? 0xffffffffn - : 0xffffffffffffffffn; + architecture === "6502" + ? 0xffffn + : architecture === "x86" || architecture === "arm" + ? 0xffffffffn + : 0xffffffffffffffffn; /** Validate architecture, entry, and that the mapped image fits the CPU width. */ export const admitRawBinary = (input: { @@ -88,7 +91,7 @@ export const admitRawBinary = (input: { if (!architecture.success) return err({ reason: - "raw-binary format requires architecture (x86, x86_64, arm, or arm64)", + "raw-binary format requires architecture (x86, x86_64, arm, arm64, or 6502)", constraint: "load_selection_rejected", }); if (typeof input.entryPoint !== "string") diff --git a/src/dotnet/ManagedArtifactInspector.test.ts b/src/dotnet/ManagedArtifactInspector.test.ts index 2d30bf580..50de3a39b 100644 --- a/src/dotnet/ManagedArtifactInspector.test.ts +++ b/src/dotnet/ManagedArtifactInspector.test.ts @@ -292,6 +292,30 @@ describe("managed artifact inventory", () => { ]); expect(result.coverage).toMatchObject({ state: "complete", issues: [] }); }); + + // dotnet/runtime corhdr.h: 32BITREQUIRED and 32BITPREFERRED form one + // two-bit field, so neither bit can be read on its own. + it.each([ + ["neither bit", 0x0000_0001, "anycpu", false], + ["32BITREQUIRED alone", 0x0000_0003, "x86", true], + ["both bits", 0x0002_0003, "anycpu-prefer-32", false], + ["the reserved 32BITPREFERRED alone", 0x0002_0001, "anycpu", false], + ] as const)( + "reads %s as one 32-bit requirement", + (_label, cliFlags, architecture, requires32Bit) => { + const bytes = buildManagedPeFixture({ cliFlags }); + const target = managedPeFixtureTarget(bytes); + + expect( + inspectManagedArtifactBytes(bytes, target).classification + .managed_architecture, + ).toBe(architecture); + expect( + inspectManagedNativeBoundariesBytes(bytes, target).cli_native + ?.requires_32bit, + ).toBe(requires32Bit); + }, + ); }); describe("managed artifact coded indexes", () => { @@ -523,6 +547,63 @@ describe("managed fixture data layout", () => { }); }); +describe("managed implementation classification", () => { + it("classifies crossgen2 ReadyToRun output without ILONLY as modern .NET", () => { + // crossgen2 writes IL_LIBRARY | STRONGNAMESIGNED and clears ILONLY. + const bytes = buildManagedPeFixture({ readyToRun: true, cliFlags: 0xc }); + const result = inspectManagedArtifactBytes( + bytes, + managedPeFixtureTarget(bytes), + ); + + expect(result.classification).toMatchObject({ + runtime_family: "modern-dotnet", + implementation: "cil-and-ready-to-run", + }); + expect(result.classification.evidence).toContainEqual( + expect.objectContaining({ code: "ready-to-run-signature-observed" }), + ); + expect(result.classification.evidence).not.toContainEqual( + expect.objectContaining({ code: "cli-il-only-clear" }), + ); + }); + + it("preserves ReadyToRun runtime identity after metadata capacity is exhausted", () => { + const bytes = buildManagedPeFixture({ + readyToRun: true, + cliFlags: 0xc, + references: ["x".repeat(12 * 1024 * 1024)], + }); + const result = inspectManagedArtifactBytes( + bytes, + managedPeFixtureTarget(bytes), + ); + expect(result.classification).toMatchObject({ + status: "managed", + runtime_family: "modern-dotnet", + }); + expect(result.coverage.issues).toContainEqual( + expect.objectContaining({ code: "resource-limit" }), + ); + }); + + it("keeps a clear ILONLY flag without ReadyToRun as mixed CLR/native", () => { + const bytes = buildManagedPeFixture({ cliFlags: 0 }); + const result = inspectManagedArtifactBytes( + bytes, + managedPeFixtureTarget(bytes), + ); + + expect(result.classification).toMatchObject({ + runtime_family: "mixed-clr-native", + implementation: "cpp-cli-mixed", + }); + expect(result.classification.evidence).toContainEqual( + expect.objectContaining({ code: "cli-il-only-clear" }), + ); + }); +}); + describe("managed artifact metadata identity", () => { it("accepts CLI metadata GUIDs without RFC UUID version or variant bits", () => { const bytes = buildManagedPeFixture({ diff --git a/src/dotnet/ManagedArtifactInspector.ts b/src/dotnet/ManagedArtifactInspector.ts index 7df8aae83..625969aba 100644 --- a/src/dotnet/ManagedArtifactInspector.ts +++ b/src/dotnet/ManagedArtifactInspector.ts @@ -9,6 +9,7 @@ import { type ManagedParseIssue, } from "../domain/managed/managedArtifact.js"; import { + managedCli32BitRequirement, readManagedPeLayout, type ManagedCliHeader, type ManagedPeLayout, @@ -157,6 +158,15 @@ interface RuntimeClassificationInput { readonly inventory: ManagedMetadataInventory; } +const runtimeFamilyFromCliHeader = ( + cli: ManagedCliHeader | null, +): ManagedArtifactInspection["classification"]["runtime_family"] => + cli?.readyToRunSignature === true + ? "modern-dotnet" + : cli !== null && (cli.flags & 1) === 0 + ? "mixed-clr-native" + : "unknown"; + const runtimeFamilyFor = ( cli: ManagedCliHeader, inventory: ManagedMetadataInventory, @@ -179,7 +189,9 @@ const runtimeFamilyFor = ( ); let runtimeFamily: ManagedArtifactInspection["classification"]["runtime_family"] = "unknown"; - if ((cli.flags & 1) === 0) { + // crossgen2 clears ILONLY on ReadyToRun output, so only a clear flag + // without the ReadyToRun signature indicates mixed CLR/native code. + if (runtimeFamilyFromCliHeader(cli) === "mixed-clr-native") { runtimeFamily = "mixed-clr-native"; evidence.push({ code: "cli-il-only-clear", @@ -238,8 +250,8 @@ const implementationFor = ( cli: ManagedCliHeader, metadata: ManagedMetadataLayout, ): ManagedArtifactInspection["classification"]["implementation"] => { - if ((cli.flags & 1) === 0) return "cpp-cli-mixed"; if (cli.readyToRunSignature) return "cil-and-ready-to-run"; + if ((cli.flags & 1) === 0) return "cpp-cli-mixed"; return (metadata.rowCounts[6] ?? 0) === 0 ? "metadata-only" : "cil"; }; @@ -249,9 +261,9 @@ const managedArchitectureFor = ( ): ManagedArtifactInspection["classification"]["managed_architecture"] => { if (layout.architecture !== "x86" || (cli.flags & 1) === 0) return layout.architecture; - if ((cli.flags & 0x0000_0002) !== 0) return "x86"; - if ((cli.flags & 0x0002_0000) !== 0) return "anycpu-prefer-32"; - return "anycpu"; + const requirement = managedCli32BitRequirement(cli.flags); + if (requirement === "required") return "x86"; + return requirement === "preferred" ? "anycpu-prefer-32" : "anycpu"; }; const runtimeClassification = ({ @@ -302,8 +314,7 @@ const partialMetadataResult = ({ classification: { status: issue.code === "resource-limit" ? "managed" : "malformed", container: "pe", - runtime_family: - (layout.cli?.flags ?? 1) & 1 ? "unknown" : "mixed-clr-native", + runtime_family: runtimeFamilyFromCliHeader(layout.cli), implementation: "unknown", managed_architecture: "unknown", evidence: [ diff --git a/src/dotnet/ManagedCilDecoding.test.ts b/src/dotnet/ManagedCilDecoding.test.ts index c1ed21130..4ba33f4c9 100644 --- a/src/dotnet/ManagedCilDecoding.test.ts +++ b/src/dotnet/ManagedCilDecoding.test.ts @@ -345,6 +345,31 @@ describe("managed exception region prefix boundaries", () => { }); }); + it("decodes the no. prefix and keeps it with its opcode", () => { + // ldarg.0; no. nullcheck; ldfld 0x04000001; pop; ret + const il = Buffer.from([2, 0xfe, 0x19, 4, 0x7b, 1, 0, 0, 4, 0x26, 0x2a]); + expect( + managedBodyWithRegion({ il, tryLength: 10, handlerOffset: 10 }), + ).toMatchObject({ + status: "present", + issue: null, + decoded_instruction_count: 5, + opcode_counts: { "ldarg.0": 1, "no.": 1, ldfld: 1, pop: 1, ret: 1 }, + }); + expect( + managedBodyWithRegion({ + il, + tryOffset: 4, + tryLength: 5, + handlerOffset: 10, + }), + ).toMatchObject({ + status: "malformed", + issue: + "Exception clause try range does not align with CIL instruction boundaries", + }); + }); + it.each([ ["start", { tryOffset: 5, tryLength: 1, handlerOffset: 6 }], [ @@ -510,3 +535,41 @@ describe("managed exception section decoding", () => { }); }); }); + +describe("managed CIL truncated operands", () => { + const tinyBody = (il: readonly number[]) => { + const bytes = buildManagedPeFixture({ + ilBody: Buffer.from([(il.length << 2) | 0x02, ...il]), + }); + const inspection = inspectManagedMembersBytes( + bytes, + managedPeFixtureTarget(bytes), + ); + return { body: inspection.methods[0]?.body, coverage: inspection.coverage }; + }; + + it.each([ + [[0x20], 0, "Instruction ldc.i4 at IL offset 0 leaves method body"], + [ + [0x00, 0x20, 0x01], + 1, + "Instruction ldc.i4 at IL offset 1 leaves method body", + ], + [[0x2b], 0, "Instruction br.s at IL offset 0 leaves method body"], + [[0x45, 0x02], 0, "Instruction switch at IL offset 0 leaves method body"], + [ + [0xfe, 0x09, 0x00], + 0, + "Instruction ldarg at IL offset 0 leaves method body", + ], + [[0x00, 0xfe], 1, "Two-byte CIL opcode at IL offset 1 leaves method body"], + ])("names the instruction and IL offset for IL %j", (il, decoded, issue) => { + const { body, coverage } = tinyBody(il); + expect(body).toMatchObject({ + status: "malformed", + decoded_instruction_count: decoded, + issue, + }); + expect(coverage.state).toBe("partial"); + }); +}); diff --git a/src/dotnet/ManagedMemberInspector.test.ts b/src/dotnet/ManagedMemberInspector.test.ts index 81d64f4a1..9ada1bf55 100644 --- a/src/dotnet/ManagedMemberInspector.test.ts +++ b/src/dotnet/ManagedMemberInspector.test.ts @@ -110,6 +110,8 @@ describe("managed member inspection", () => { expect(result.member_refs).toEqual([ expect.objectContaining({ token: "0x0a000001", + parent_token: "0x01000001", + parent_name: "System.Runtime.Versioning.TargetFrameworkAttribute", name: ".ctor", signature: expect.objectContaining({ kind: "method", @@ -126,6 +128,8 @@ describe("managed member inspection", () => { target_token: "0x0a000001", target_kind: "member-ref", target_name: ".ctor", + target_declaring_type: + "System.Runtime.Versioning.TargetFrameworkAttribute", }, ]); expect(result.field_accesses).toEqual([ @@ -135,12 +139,366 @@ describe("managed member inspection", () => { opcode: "ldfld", field_token: "0x04000001", field_name: "counter", + field_declaring_type: "Fixture.Program", }, ]); expect(result.coverage).toMatchObject({ state: "complete", issues: [] }); }); }); +/** A tiny-header body of token-operand instructions followed by `ret`. */ +const tokenBody = ( + instructions: readonly (readonly [opcode: number, token: number])[], +): Buffer => { + const il = Buffer.concat([ + ...instructions.map(([opcode, token]) => { + const instruction = Buffer.alloc(5); + instruction.writeUInt8(opcode, 0); + instruction.writeUInt32LE(token, 1); + return instruction; + }), + Buffer.from([0x2a]), + ]); + return Buffer.concat([Buffer.from([(il.length << 2) | 2]), il]); +}; + +const attribute = "System.Runtime.Versioning.TargetFrameworkAttribute"; +const inspect = (options: Parameters[0]) => { + const bytes = buildManagedPeFixture(options); + return inspectManagedMembersBytes(bytes, managedPeFixtureTarget(bytes)); +}; +const call = 0x28; +const ldsfld = 0x7e; +const memberRefOnTypeSpec = (signature: Buffer) => + inspect({ + typeSpecSignatures: [signature], + memberRefParentRaw: (1 << 3) | 4, + }); + +// HASTHIS | GENERIC, one generic parameter, no parameters, void. +const genericRef = Buffer.from([0x30, 0x01, 0x00, 0x01]); +// GENERIC, one generic parameter, no parameters, void. +const genericDef = Buffer.from([0x10, 0x01, 0x00, 0x01]); + +describe("managed edge target declaring types", () => { + it("names a generic TypeSpec parent by its generic type and leaves other TypeSpecs unknown", () => { + // GENERICINST CLASS TypeRef(1) + const generic = memberRefOnTypeSpec( + Buffer.from([0x15, 0x12, (1 << 2) | 1, 0x01, 0x0e]), + ); + expect(generic.member_refs[0]?.parent_name).toBe(attribute); + expect(generic.call_edges[0]?.target_declaring_type).toBe(attribute); + + // SZARRAY i4 + const array = memberRefOnTypeSpec(Buffer.from([0x1d, 0x08])); + expect(array.member_refs[0]?.parent_name).toBeNull(); + expect(array.coverage).toEqual({ state: "complete", issues: [] }); + }); + + it.each([ + [ + "names a TypeSpec as its generic type", + [0x15, 0x12, (1 << 2) | 2, 0x01, 0x0e], + "invalid-row", + "metadata.TypeSpec:0x1b000001", + ], + [ + "has no type arguments", + [0x15, 0x12, (1 << 2) | 1], + "invalid-blob", + "metadata.TypeSpec:0x1b000001", + ], + [ + "names a missing TypeRef", + [0x15, 0x12, (9 << 2) | 1, 0x01, 0x0e], + "invalid-row", + "metadata.TypeSpec:0x1b000001", + ], + [ + "names a missing TypeDef", + [0x15, 0x12, (9 << 2) | 0, 0x01, 0x0e], + "invalid-row", + "metadata.TypeSpec:0x1b000001", + ], + [ + "has a type argument naming a missing TypeRef", + [0x15, 0x12, (1 << 2) | 1, 0x01, 0x12, (9 << 2) | 1], + "invalid-row", + "metadata.TypeSpec:0x1b000001", + ], + ])( + "reports a generic TypeSpec parent that %s and leaves it unknown", + (_case, signature, code, scope) => { + const result = memberRefOnTypeSpec(Buffer.from(signature)); + + expect(result.member_refs[0]?.parent_name).toBeNull(); + expect(result.coverage).toMatchObject({ + state: "partial", + issues: [{ code, scope }], + }); + }, + ); + + it("reports unsupported TypeSpec and MethodSpec forms instead of treating them as unknown", () => { + const typeSpec = memberRefOnTypeSpec(Buffer.from([0xff])); + const methodSpec = inspect({ + memberRefSignature: genericRef, + methodSpecMethods: [(1 << 1) | 1], + methodSpecInstantiation: Buffer.from([0x0a, 0x01, 0xff]), + ilBody: tokenBody([[call, 0x2b00_0001]]), + }); + + expect(typeSpec.member_refs[0]?.parent_name).toBeNull(); + expect(typeSpec.coverage).toMatchObject({ + state: "partial", + issues: [ + { + code: "unsupported-signature", + scope: "metadata.TypeSpec:0x1b000001", + }, + ], + }); + expect(methodSpec.coverage).toMatchObject({ + state: "partial", + issues: [ + { + code: "unsupported-signature", + scope: "metadata.MethodSpec:0x2b000001", + }, + ], + }); + }); + + it("names field MemberRefs as fields and leaves mismatched member kinds unnamed", () => { + const result = inspect({ + memberRefName: "Empty", + // FIELD string + memberRefSignature: Buffer.from([0x06, 0x0e]), + ilBody: tokenBody([ + [ldsfld, 0x0a00_0001], + [call, 0x0400_0001], + ]), + }); + const methodRef = inspect({ ilBody: tokenBody([[ldsfld, 0x0a00_0001]]) }); + const flaggedField = inspect({ + memberRefName: "Empty", + // GENERIC | FIELD string: FieldSig allows no flags. + memberRefSignature: Buffer.from([0x16, 0x0e]), + ilBody: tokenBody([[ldsfld, 0x0a00_0001]]), + }); + + expect(result.field_accesses).toMatchObject([ + { field_name: "Empty", field_declaring_type: attribute }, + ]); + expect(methodRef.field_accesses).toMatchObject([ + { field_name: null, field_declaring_type: null }, + ]); + expect(flaggedField.field_accesses).toMatchObject([ + { field_name: null, field_declaring_type: null }, + ]); + expect(flaggedField.coverage.state).toBe("partial"); + expect(result.call_edges).toMatchObject([ + { + target_kind: "unknown", + target_name: null, + target_declaring_type: null, + }, + ]); + }); + + it("reads Evidence recorded before declaring types were named as unknown", () => { + const current = inspect({}); + const parsed = managedMemberInspectionSchema.parse({ + ...current, + member_refs: current.member_refs.map( + ({ parent_name: _omitted, ...legacy }) => legacy, + ), + call_edges: current.call_edges.map( + ({ target_declaring_type: _omitted, ...legacy }) => legacy, + ), + field_accesses: current.field_accesses.map( + ({ field_declaring_type: _omitted, ...legacy }) => legacy, + ), + }); + + expect(parsed.member_refs[0]?.parent_name).toBeNull(); + expect(parsed.call_edges[0]?.target_declaring_type).toBeNull(); + expect(parsed.field_accesses[0]?.field_declaring_type).toBeNull(); + }); +}); + +describe("managed TypeRef resolution scopes", () => { + it.each([ + // AssemblyRef 9 does not exist. + ["an invalid", (9 << 2) | 2, "is invalid"], + // The TypeRef is its own enclosing type. + ["a cyclic", (1 << 2) | 3, "contains a cycle"], + ])( + "keeps a TypeRef name beside %s resolution scope", + (_case, typeRefScopeRaw, detail) => { + const result = inspect({ typeRefScopeRaw }); + + expect(result.member_refs[0]?.parent_name).toBe(attribute); + expect(result.call_edges[0]?.target_declaring_type).toBe(attribute); + expect(result.coverage).toMatchObject({ + state: "partial", + issues: [ + { + code: "invalid-row", + scope: "metadata.TypeRef:0x01000001", + detail: expect.stringContaining(detail), + }, + ], + }); + }, + ); +}); + +describe("managed MethodSpec call targets", () => { + it("names generic method calls after the method they instantiate", () => { + const result = inspect({ + memberRefSignature: genericRef, + methodSignature: genericDef, + methodSpecMethods: [(1 << 1) | 1, (1 << 1) | 0], + ilBody: tokenBody([ + [call, 0x2b00_0001], + [call, 0x2b00_0002], + ]), + }); + + expect( + result.call_edges.map((edge) => [ + edge.target_kind, + edge.target_name, + edge.target_declaring_type, + ]), + ).toEqual([ + ["method-spec", ".ctor", attribute], + ["method-spec", "Main", "Fixture.Program"], + ]); + expect(result.coverage).toEqual({ state: "complete", issues: [] }); + }); + + it.each([ + [ + "a non-generic method", + {}, + Buffer.from([0x0a, 0x01, 0x0e]), + "but its method declares 0 generic parameter(s)", + ], + [ + "the wrong number of type arguments", + { memberRefSignature: genericRef }, + Buffer.from([0x0a, 0x02, 0x0e, 0x08]), + "supplies 2 type argument(s), but its method declares 1", + ], + [ + "a field reference", + { memberRefSignature: Buffer.from([0x06, 0x0e]) }, + Buffer.from([0x0a, 0x01, 0x0e]), + "does not have a decodable method signature", + ], + [ + "a property signature", + // PROPERTY | GENERIC, one generic parameter, no parameters, void. + { memberRefSignature: Buffer.from([0x18, 0x01, 0x00, 0x01]) }, + Buffer.from([0x0a, 0x01, 0x0e]), + "does not have a decodable method signature", + ], + [ + "a truncated signature", + // HASTHIS | GENERIC, one generic parameter, no parameters, no return type. + { memberRefSignature: Buffer.from([0x30, 0x01, 0x00]) }, + Buffer.from([0x0a, 0x01, 0x0e]), + "does not have a decodable method signature", + ], + [ + "a type argument naming a missing TypeRef", + { memberRefSignature: genericRef }, + Buffer.from([0x0a, 0x01, 0x12, (9 << 2) | 1]), + "TypeDefOrRefOrSpec operand 0x25 is invalid", + ], + [ + "an unmanaged-convention signature", + // C | GENERIC: a function pointer convention, not a MethodRefSig one. + { memberRefSignature: Buffer.from([0x11, 0x01, 0x00, 0x01]) }, + Buffer.from([0x0a, 0x01, 0x0e]), + "does not have a decodable method signature", + ], + ])( + "reports a MethodSpec that instantiates %s", + (_case, options, methodSpecInstantiation, detail) => { + const result = inspect({ + ...options, + methodSpecMethods: [(1 << 1) | 1], + methodSpecInstantiation, + ilBody: tokenBody([[call, 0x2b00_0001]]), + }); + + expect(result.coverage).toMatchObject({ + state: "partial", + issues: [ + { + code: "invalid-row", + scope: "metadata.MethodSpec:0x2b000001", + detail: expect.stringContaining(detail), + }, + ], + }); + }, + ); + + it("never names a field through a MethodSpec", () => { + const result = inspect({ + memberRefName: "Empty", + // FIELD string: a MethodSpec on it is malformed. + memberRefSignature: Buffer.from([0x06, 0x0e]), + methodSpecMethods: [(1 << 1) | 1], + ilBody: tokenBody([ + [ldsfld, 0x2b00_0001], + [call, 0x2b00_0001], + ]), + }); + + expect(result.field_accesses).toMatchObject([ + { field_name: null, field_declaring_type: null }, + ]); + expect(result.call_edges).toMatchObject([ + { target_name: null, target_declaring_type: null }, + ]); + expect(result.coverage.state).toBe("partial"); + }); + + it("keeps a readable MethodSpec method beside a malformed instantiation", () => { + const result = inspect({ + methodSpecMethods: [(1 << 1) | 1, (9 << 1) | 1], + methodSpecInstantiation: Buffer.from([0x0a, 0x00]), + ilBody: tokenBody([ + [call, 0x2b00_0001], + [call, 0x2b00_0002], + ]), + }); + + expect( + result.call_edges.map(({ target_name, target_declaring_type }) => [ + target_name, + target_declaring_type, + ]), + ).toEqual([ + [".ctor", attribute], + [null, null], + ]); + expect(result.coverage.state).toBe("partial"); + expect( + result.coverage.issues.map(({ code, scope }) => [code, scope]), + ).toEqual([ + ["invalid-blob", "metadata.MethodSpec:0x2b000001"], + ["invalid-row", "metadata.MethodSpec:0x2b000002"], + ["invalid-blob", "metadata.MethodSpec:0x2b000002"], + ]); + }); +}); + describe("managed exported types", () => { it("reports type forwarders and nested exported types with their implementation scope", () => { const bytes = buildManagedPeFixture({ @@ -373,15 +731,9 @@ describe("managed derived edges", () => { body: { anchors: Array.from({ length: 40_000 }, () => anchor) }, } as unknown as ManagedMethod; - expect(() => - edges( - [method], - new Map(), - new Map(), - new Map(), - new ManagedDecodeBudget(), - ), - ).toThrow(ManagedReaderFailure); + expect(() => edges([method], new Map(), new ManagedDecodeBudget())).toThrow( + ManagedReaderFailure, + ); }); }); @@ -413,15 +765,9 @@ describe("managed derived caller names", () => { }, } as unknown as ManagedMethod; - expect(() => - edges( - [method], - new Map(), - new Map(), - new Map(), - new ManagedDecodeBudget(), - ), - ).toThrow(ManagedReaderFailure); + expect(() => edges([method], new Map(), new ManagedDecodeBudget())).toThrow( + ManagedReaderFailure, + ); expect(built).toBe(false); }); }); @@ -671,6 +1017,18 @@ it.each([ }); }); +it.each([ + ["a property header", [0x08, 0x00, 0x08]], + ["an explicit-this header without has-this", [0x40, 0x00, 0x01]], + ["a C calling convention header", [0x01, 0x00, 0x01]], + ["a generic vararg header", [0x15, 0x01, 0x00, 0x01]], +])("reports %s as a malformed member signature", (_name, blob) => { + expect(signature(Buffer.from(blob))).toMatchObject({ + kind: "unknown", + parse_status: "malformed", + }); +}); + it("rejects array ranks beyond the CLR limit before rendering dimensions", () => { expect( signature( diff --git a/src/dotnet/ManagedMemberInspector.ts b/src/dotnet/ManagedMemberInspector.ts index 24b97f9eb..dba20ac57 100644 --- a/src/dotnet/ManagedMemberInspector.ts +++ b/src/dotnet/ManagedMemberInspector.ts @@ -27,6 +27,8 @@ import { typeRanges, type TypeRange, } from "./ManagedMemberInspectorCore.js"; +import { edgeTargets } from "./ManagedMethodSpecs.js"; +import { typeNameLookup } from "./ManagedTypeNames.js"; import { edges, parseExportedTypes, @@ -171,7 +173,11 @@ export const inspectManagedMembersBytes = ( const exportedTypes = parseExportedTypes(bytes, layout); issues.push(...exportedTypes.issues); const fields = parseFields(bytes, layout, ranges, selected); - const memberRefs = parseMemberRefs(bytes, layout); + const memberRefs = parseMemberRefs( + bytes, + layout, + typeNameLookup(bytes, layout, ranges, issues), + ); issues.push(...memberRefs.issues); const methods = parseMethods({ bytes, @@ -181,6 +187,12 @@ export const inspectManagedMembersBytes = ( selected, }); issues.push(...fields.issues, ...methods.issues); + const targets = edgeTargets(bytes, layout, methods.methods, [ + fields.core, + memberRefs.core, + methods.core, + ]); + issues.push(...targets.issues); const coverageIssues = uniqueManagedParseIssues(issues); const incompleteMembers = [...fields.fields, ...memberRefs.refs, ...methods.methods].some( @@ -193,9 +205,7 @@ export const inspectManagedMembersBytes = ( ); const related = edges( methods.methods, - methods.core, - fields.core, - memberRefs.core, + targets.members, managedDecodeBudget(layout), ); return managedMemberInspectionSchema.parse( diff --git a/src/dotnet/ManagedMemberInspectorCore.ts b/src/dotnet/ManagedMemberInspectorCore.ts index f54a11769..7776c0090 100644 --- a/src/dotnet/ManagedMemberInspectorCore.ts +++ b/src/dotnet/ManagedMemberInspectorCore.ts @@ -7,6 +7,7 @@ import type { ManagedMemberInspection, ManagedParseIssue, } from "../domain/managed/managedArtifact.js"; +import { managedDecodeBudget } from "./ManagedDecodeBudget.js"; import type { ManagedMetadataLayout, MetadataTableLayout, @@ -49,20 +50,22 @@ export interface TypeRange { readonly methodEnd: number; } -export interface MethodCore { +/** + * Name and declaring type of a field, method, member reference, or method + * specification, keyed by its token so edges resolve any target uniformly. + */ +export interface MemberCore { readonly token: string; readonly name: string; readonly declaringType: string | null; -} - -export interface FieldCore { - readonly token: string; - readonly name: string; -} - -export interface MemberRefCore { - readonly token: string; - readonly name: string; + /** Fields and methods by table; member references by decoded signature. */ + readonly kind: ManagedSignature["kind"]; + /** + * Generic arity from the owner's decoded signature (see + * `signatureGenericArity`); absent when no owner decoded it, as for a + * MethodDef outside the type selection. + */ + readonly genericArity?: number | null; } export interface ParsedInstruction { @@ -88,7 +91,7 @@ const rowRange = ( return { first: start, last: end - 1, count: end - start }; }; -const fullName = (namespace: string, name: string): string => +export const fullName = (namespace: string, name: string): string => namespace.length === 0 ? name : `${namespace}.${name}`; const readTypeRange = ( @@ -242,7 +245,7 @@ export const parseTypes = ( return { types: items, issues }; }; -const readCompressed = ( +export const readCompressed = ( bytes: Buffer, offset: number, ): { readonly value: number; readonly next: number } => { @@ -341,13 +344,38 @@ interface MethodSignatureHeader { readonly next: number; } +// MethodDefSig and MethodRefSig headers (ECMA-335 II.23.2.1-2) without +// HASTHIS and EXPLICITTHIS: DEFAULT, VARARG, or GENERIC, which implies DEFAULT. +const MEMBER_METHOD_HEADERS = new Set([0x00, 0x05, 0x10]); + +// FNPTR also carries the unmanaged C, STDCALL, THISCALL, FASTCALL, and +// UNMANAGED conventions that compilers emit for unmanaged function pointers. +const FUNCTION_POINTER_CONVENTIONS = new Set([0, 1, 2, 3, 4, 5, 9]); + +/** + * Read a method signature's calling convention and counts. A header that is + * not a method signature in its context (a MethodDef or MemberRef row, or an + * FNPTR type), sets the reserved bit, or sets EXPLICITTHIS without HASTHIS is + * malformed, so no caller consumes its counts. FIELD, LOCAL_SIG, PROPERTY, + * and GENERICINST headers are never method signatures. + */ const readMethodSignatureHeader = ( blob: Buffer, offset: number, + context: "member" | "function-pointer", ): MethodSignatureHeader => { const first = blob[offset]; if (first === undefined) return malformedSignature("truncated method signature", offset); + const allowed = + context === "member" + ? MEMBER_METHOD_HEADERS.has(first & 0x9f) + : (first & 0x80) === 0 && FUNCTION_POINTER_CONVENTIONS.has(first & 0x0f); + if (!allowed || ((first & 0x40) !== 0 && (first & 0x20) === 0)) + return malformedSignature( + `calling convention 0x${first.toString(16)} does not head a ${context} method signature`, + offset, + ); let next = offset + 1; let genericParameterCount: number | null = null; if ((first & 0x10) !== 0) { @@ -366,12 +394,6 @@ const readMethodSignatureHeader = ( const MAX_ARRAY_RANK = 32; -// FNPTR carries a method signature: managed DEFAULT/VARARG, or the unmanaged -// C, STDCALL, THISCALL, FASTCALL, and UNMANAGED conventions that compilers -// emit for unmanaged function pointers. FIELD, LOCAL_SIG, PROPERTY, and -// GENERICINST headers are not method signatures. -const FUNCTION_POINTER_CONVENTIONS = new Set([0, 1, 2, 3, 4, 5, 9]); - const arrayShape = ( blob: Buffer, offset: number, @@ -449,10 +471,40 @@ const POSITION_ELEMENTS: ReadonlyMap = [0x10, ["return", "parameter", "field"]], ]); +/** Decode a type operand once; spec callers also validate its metadata coordinate. */ +const readTypeOperand = ( + blob: Buffer, + offset: number, + layout?: ManagedMetadataLayout, + tables: readonly number[] = [2, 1, 27], +): ReturnType => { + const token = readCompressed(blob, offset); + if (layout !== undefined) { + const reason = + token.value === 0 + ? "type operand is null" + : metadataCodedTokenInvalidReason( + token.value, + 2, + tables, + layout.rowCounts, + ); + if (reason !== null) + throw managedFailure( + "invalid-row", + "signature", + `TypeDefOrRefOrSpec operand 0x${token.value.toString(16)} is invalid: ${reason}`, + offset, + ); + } + return token; +}; + const readTypeSignature = ( blob: Buffer, offset: number, base: SignaturePosition, + layout?: ManagedMetadataLayout, ): { readonly value: string; readonly next: number } => { const stack: TypeContinuation[] = []; let cursor = offset; @@ -479,7 +531,7 @@ const readTypeSignature = ( stack.push({ kind: "suffix", suffix }); cursor += 1; } else if (kind === 0x1f || kind === 0x20) { - const token = readCompressed(blob, cursor + 1); + const token = readTypeOperand(blob, cursor + 1, layout); const modifier = `${kind === 0x1f ? "modreq" : "modopt"}:${String(token.value)}`; // A modifier frame on top was pushed by the directly preceding byte; // keep an adjacent run together and in source order. @@ -488,7 +540,7 @@ const readTypeSignature = ( else stack.push({ kind: "modifiers", modifiers: [modifier] }); cursor = token.next; } else if (kind === 0x11 || kind === 0x12) { - const token = readCompressed(blob, cursor + 1); + const token = readTypeOperand(blob, cursor + 1, layout); value = `${kind === 0x11 ? "valuetype" : "class"}:${String(token.value)}`; cursor = token.next; } else if (kind === 0x1e || kind === 0x13) { @@ -508,7 +560,7 @@ const readTypeSignature = ( "generic instantiation does not name a class or value type", cursor + 1, ); - const token = readCompressed(blob, cursor + 2); + const token = readTypeOperand(blob, cursor + 2, layout, [2, 1]); const argumentCount = readCompressed(blob, token.next); if (argumentCount.value === 0) return malformedSignature( @@ -525,16 +577,11 @@ const readTypeSignature = ( }); cursor = argumentCount.next; } else if (kind === 0x1b) { - const header = readMethodSignatureHeader(blob, cursor + 1); - if ( - !FUNCTION_POINTER_CONVENTIONS.has(header.first & 0x0f) || - (header.first & 0x80) !== 0 || - ((header.first & 0x40) !== 0 && (header.first & 0x20) === 0) - ) - return malformedSignature( - `function pointer calling convention 0x${header.first.toString(16)} does not head a method signature`, - cursor + 1, - ); + const header = readMethodSignatureHeader( + blob, + cursor + 1, + "function-pointer", + ); const convention = callingConvention(header.first); const head = header.genericParameterCount === null @@ -602,22 +649,81 @@ const callingConvention = (value: number): string => { return flags.length === 0 ? name : `${name} ${flags.join(" ")}`; }; -export const signature = ( +/** + * Decode a whole TypeSpec blob, or a MethodSpec instantiation (GENERICINST, + * argument count, Types), with the member-signature grammar and return the + * rendered types. A malformed or unsupported blob throws invalid-blob or + * unsupported-signature under the owning row, so coverage records it. + */ +export const specSignature = ( + layout: ManagedMetadataLayout, blob: Buffer, - rawSha256?: string, -): ManagedSignature => { - const raw = { - raw_length: blob.length, - raw_sha256: rawSha256 ?? sha256Bytes(blob), - }; + owner: { + readonly form: "type" | "instantiation"; + readonly scope: string; + readonly offset: number; + }, +): readonly string[] => { + const budget = managedDecodeBudget(layout); + // Rendering is transient, so charge it as member signatures are charged. + budget.admitTransient(blob.length * 256, owner.offset); + try { + let cursor = 0; + let count = 1; + if (owner.form === "instantiation") { + if (blob[0] !== 0x0a) + malformedSignature("MethodSpec instantiation is not GENERICINST", 0); + const header = readCompressed(blob, 1); + if (header.value === 0) + malformedSignature("MethodSpec instantiation has no type arguments", 1); + count = header.value; + cursor = header.next; + } + const types: string[] = []; + // Each Type consumes at least one byte, so a false count fails on truncation. + for (let index = 0; index < count; index += 1) { + const type = readTypeSignature(blob, cursor, "type", layout); + types.push(type.value); + cursor = type.next; + } + if (cursor !== blob.length) + malformedSignature("Trailing signature data", cursor); + return types; + } catch (cause: unknown) { + if ( + !(cause instanceof ManagedReaderFailure) || + (cause.issue.code !== "invalid-blob" && + cause.issue.code !== "invalid-row" && + cause.issue.code !== "unsupported-signature") + ) + throw cause; + throw managedFailure( + cause.issue.code, + owner.scope, + cause.issue.detail, + owner.offset, + ); + } finally { + budget.releaseTransient(); + } +}; + +type DecodedSignature = Omit; + +/** + * Decode a field or member method signature. A malformed or unsupported blob + * decodes as kind `unknown` with its reason. + */ +const decodeSignature = (blob: Buffer): DecodedSignature => { try { if (blob.length === 0) throw managedFailure("invalid-blob", "signature", "empty signature", 0); const first = blob[0] ?? 0; - if ((first & 0x0f) === 6) { + // FieldSig (II.23.2.4) is exactly FIELD; a flagged FIELD header is not a + // method header either, so it is malformed below. + if (first === 0x06) { const fieldType = readTypeSignature(blob, 1, "field"); return { - ...raw, kind: "field", parse_status: fieldType.next === blob.length ? "decoded" : "partial", calling_convention: callingConvention(first), @@ -630,7 +736,7 @@ export const signature = ( fieldType.next === blob.length ? null : "Trailing signature data", }; } - const header = readMethodSignatureHeader(blob, 0); + const header = readMethodSignatureHeader(blob, 0, "member"); const returnType = readTypeSignature(blob, header.next, "return"); let offset = returnType.next; const parameters: string[] = []; @@ -640,7 +746,6 @@ export const signature = ( offset = parameter.next; } return { - ...raw, kind: "method", parse_status: offset === blob.length ? "decoded" : "partial", calling_convention: callingConvention(first), @@ -659,7 +764,6 @@ export const signature = ( ) throw cause; return { - ...raw, kind: "unknown", parse_status: cause.issue.code === "unsupported-signature" @@ -675,3 +779,41 @@ export const signature = ( }; } }; + +/** + * Generic parameter count of a method signature that decodes completely, 0 + * when it is not generic, or null when the blob is not a decodable method + * signature. The rendered decode is transient. + */ +export const methodGenericArity = ( + layout: ManagedMetadataLayout, + blob: Buffer, +): number | null => { + const budget = managedDecodeBudget(layout); + budget.admitTransient(blob.length * 256, layout.blob.offset); + try { + return signatureGenericArity(decodeSignature(blob)); + } finally { + budget.releaseTransient(); + } +}; + +/** + * Generic parameter count of a completely decoded method signature, 0 when it + * is not generic, or null for any other signature. + */ +export const signatureGenericArity = ( + decoded: DecodedSignature, +): number | null => + decoded.kind === "method" && decoded.parse_status === "decoded" + ? (decoded.generic_parameter_count ?? 0) + : null; + +export const signature = ( + blob: Buffer, + rawSha256?: string, +): ManagedSignature => ({ + raw_length: blob.length, + raw_sha256: rawSha256 ?? sha256Bytes(blob), + ...decodeSignature(blob), +}); diff --git a/src/dotnet/ManagedMemberInstructionDecoder.ts b/src/dotnet/ManagedMemberInstructionDecoder.ts index 7ef518733..01c2b7814 100644 --- a/src/dotnet/ManagedMemberInstructionDecoder.ts +++ b/src/dotnet/ManagedMemberInstructionDecoder.ts @@ -12,6 +12,7 @@ const PREFIX_OPCODES = new Set([ "tail.", "constrained.", "readonly.", + "no.", ]); const OPCODES = new Map< @@ -232,6 +233,7 @@ const OPCODES = new Map< [0xfe16, { name: "constrained.", operand: "type" }], [0xfe17, { name: "cpblk", operand: "none" }], [0xfe18, { name: "initblk", operand: "none" }], + [0xfe19, { name: "no.", operand: "short-i" }], [0xfe1a, { name: "rethrow", operand: "none" }], [0xfe1c, { name: "sizeof", operand: "type" }], [0xfe1d, { name: "refanytype", operand: "none" }], @@ -384,6 +386,22 @@ const readOperand = ( return { next: offset, anchorKind: "unknown", value: null }; }; +/** An operand cut off by the end of the IL body has no decodable value. */ +const readOperandWithin = ( + il: Buffer, + offset: number, + kind: string, +): OperandResult | undefined => { + try { + return readOperand(il, offset, kind); + } catch (cause: unknown) { + // Buffer reads past the IL body throw RangeError; report it as the + // instruction leaving the method body rather than as a Buffer message. + if (cause instanceof RangeError) return undefined; + throw cause; + } +}; + export const decodeInstructions = ( il: Buffer, admitInstruction?: (offset: number) => void, @@ -404,14 +422,22 @@ export const decodeInstructions = ( const start = offset; const first = il.readUInt8(offset); offset += 1; + if (first === 0xfe && offset >= il.length) { + issue = `Two-byte CIL opcode at IL offset ${String(start)} leaves method body`; + break; + } const code = first === 0xfe ? 0xfe00 + il.readUInt8(offset++) : first; const descriptor = OPCODES.get(code); if (descriptor === undefined) { issue = `Unsupported CIL opcode 0x${code.toString(16)} at IL offset ${String(start)}`; break; } - const operand = readOperand(il, offset, descriptor.operand); - if (operand.next < offset || operand.next > il.length) { + const operand = readOperandWithin(il, offset, descriptor.operand); + if ( + operand === undefined || + operand.next < offset || + operand.next > il.length + ) { issue = `Instruction ${descriptor.name} at IL offset ${String(start)} leaves method body`; break; } diff --git a/src/dotnet/ManagedMemberRows.ts b/src/dotnet/ManagedMemberRows.ts index f4a63db5a..201ff1c45 100644 --- a/src/dotnet/ManagedMemberRows.ts +++ b/src/dotnet/ManagedMemberRows.ts @@ -9,15 +9,14 @@ import type { ManagedParseIssue } from "../domain/managed/managedArtifact.js"; import type { ManagedMetadataLayout } from "./ManagedMetadataLayout.js"; import { signature, - type FieldCore, + signatureGenericArity, type ManagedCallEdge, type ManagedExportedType, type ManagedField, type ManagedFieldAccess, type ManagedMemberRef, type ManagedMethod, - type MemberRefCore, - type MethodCore, + type MemberCore, type TypeRange, } from "./ManagedMemberInspectorCore.js"; import { @@ -51,11 +50,11 @@ export const parseFields = ( selected: SelectedTypeTokens = null, ): { readonly fields: readonly ManagedField[]; - readonly core: ReadonlyMap; + readonly core: ReadonlyMap; readonly issues: readonly ManagedParseIssue[]; } => { const fields: ManagedField[] = []; - const core = new Map(); + const core = new Map(); const issues: ManagedParseIssue[] = []; const table = layout.table(4); const ownerRow = readManagedValue( @@ -78,7 +77,12 @@ export const parseFields = ( : (readManagedValue(() => ownerRow(row), issues) ?? 0); const declared = ranges[owner - 1]; const token = metadataToken(4, row); - core.set(token, { token, name }); + core.set(token, { + token, + name, + declaringType: declared?.fullName ?? null, + kind: "field", + }); if (!isSelected(selected, declared)) return; const sig = readMetadataBlob( bytes, @@ -174,13 +178,14 @@ export const parseExportedTypes = ( export const parseMemberRefs = ( bytes: Buffer, layout: ManagedMetadataLayout, + typeName: (token: string | null) => string | null, ): { readonly refs: readonly ManagedMemberRef[]; - readonly core: ReadonlyMap; + readonly core: ReadonlyMap; readonly issues: readonly ManagedParseIssue[]; } => { const refs: ManagedMemberRef[] = []; - const core = new Map(); + const core = new Map(); const issues: ManagedParseIssue[] = []; const table = layout.table(10); for (let row = 1; row <= (table?.rowCount ?? 0); row += 1) { @@ -218,19 +223,29 @@ export const parseMemberRefs = ( ? "MemberRef parent coded index 0x0 is null, but the Class column must reference a row" : `MemberRef parent coded index 0x${parentRaw.toString(16)} is invalid: ${parentReason}`, }); + const parentToken = metadataCodedToken( + parentRaw, + 3, + [2, 1, 26, 6, 27], + layout.rowCounts, + ); + const parentName = typeName(parentToken); + const decoded = budgetedSignature(sig, layout); refs.push({ token, row_offset: cursor.start, - parent_token: metadataCodedToken( - parentRaw, - 3, - [2, 1, 26, 6, 27], - layout.rowCounts, - ), + parent_token: parentToken, + parent_name: parentName, name, - signature: budgetedSignature(sig, layout), + signature: decoded, + }); + core.set(token, { + token, + name, + declaringType: parentName, + kind: decoded.kind, + genericArity: signatureGenericArity(decoded), }); - core.set(token, { token, name }); }, issues); } return { refs, core, issues }; @@ -252,11 +267,11 @@ export const parseMethods = ({ selected = null, }: ParseMethodsInput): { readonly methods: readonly ManagedMethod[]; - readonly core: ReadonlyMap; + readonly core: ReadonlyMap; readonly issues: readonly ManagedParseIssue[]; } => { const methods: ManagedMethod[] = []; - const core = new Map(); + const core = new Map(); const issues: ManagedParseIssue[] = []; const table = layout.table(6); const ownerRow = readManagedValue( @@ -281,11 +296,13 @@ export const parseMethods = ({ : (readManagedValue(() => ownerRow(row), issues) ?? 0); const declared = ranges[owner - 1]; const token = metadataToken(6, row); - core.set(token, { + const member = { token, name, declaringType: declared?.fullName ?? null, - }); + kind: "method", + } as const; + core.set(token, member); if (!isSelected(selected, declared)) return; const sig = readMetadataBlob( bytes, @@ -294,6 +311,11 @@ export const parseMethods = ({ layout.blob.size, ); cursor.readIndex(layout.tableIndexSize(8)); + const decoded = budgetedSignature(sig, layout); + core.set(token, { + ...member, + genericArity: signatureGenericArity(decoded), + }); methods.push({ token, row_offset: cursor.start, @@ -303,7 +325,7 @@ export const parseMethods = ({ rva, impl_flags: implFlags, flags, - signature: budgetedSignature(sig, layout), + signature: decoded, body: methodBody( bytes, pe, @@ -317,6 +339,15 @@ export const parseMethods = ({ return { methods, core, issues }; }; +/** + * Member kinds each operand kind may name. A MemberRef whose signature does + * not decode keeps its name on calls, as before, but is never named a field. + */ +const NAMEABLE_KINDS = new Map>([ + ["method", new Set(["method", "unknown"])], + ["field", new Set(["field"])], +]); + const targetKind = (token: string): ManagedCallEdge["target_kind"] => token.startsWith("0x06") ? "method-def" @@ -326,11 +357,22 @@ const targetKind = (token: string): ManagedCallEdge["target_kind"] => ? "method-spec" : "unknown"; +/** The member an operand names, if the operand kind can name that member's kind. */ +const namedTarget = ( + members: ReadonlyMap, + operandKind: string, + operand: string, +): MemberCore | undefined => { + const member = members.get(operand); + return member !== undefined && + NAMEABLE_KINDS.get(operandKind)?.has(member.kind) === true + ? member + : undefined; +}; + export const edges = ( methods: readonly ManagedMethod[], - methodCore: ReadonlyMap, - fieldCore: ReadonlyMap, - refCore: ReadonlyMap, + members: ReadonlyMap, budget: ManagedDecodeBudget, ): { readonly callEdges: readonly ManagedCallEdge[]; @@ -350,11 +392,10 @@ export const edges = ( const caller = derives ? callerName(method) : method.name; for (const anchor of method.body.anchors) { if (anchor.opcode === "ldtoken") continue; - if (anchor.operand_kind === "method" && anchor.operand !== null) { - const named = - methodCore.get(anchor.operand)?.name ?? - refCore.get(anchor.operand)?.name ?? - null; + const { operand } = anchor; + if (operand === null) continue; + const target = namedTarget(members, anchor.operand_kind, operand); + if (anchor.operand_kind === "method") { // Each derived record is admitted before the graph grows further. callEdges.push( admitManagedProjection( @@ -362,23 +403,25 @@ export const edges = ( caller_token: method.token, caller, opcode: anchor.opcode, - target_token: anchor.operand, - target_kind: targetKind(anchor.operand), - target_name: named, + target_token: operand, + target_kind: targetKind(operand), + target_name: target?.name ?? null, + target_declaring_type: target?.declaringType ?? null, }, budget, ), ); } - if (anchor.operand_kind === "field" && anchor.operand !== null) { + if (anchor.operand_kind === "field") { fieldAccesses.push( admitManagedProjection( { method_token: method.token, method: caller, opcode: anchor.opcode, - field_token: anchor.operand, - field_name: fieldCore.get(anchor.operand)?.name ?? null, + field_token: operand, + field_name: target?.name ?? null, + field_declaring_type: target?.declaringType ?? null, }, budget, ), diff --git a/src/dotnet/ManagedMethodSpecs.ts b/src/dotnet/ManagedMethodSpecs.ts new file mode 100644 index 000000000..057090862 --- /dev/null +++ b/src/dotnet/ManagedMethodSpecs.ts @@ -0,0 +1,154 @@ +import type { ManagedParseIssue } from "../domain/managed/managedArtifact.js"; +import { + methodGenericArity, + specSignature, + type ManagedMethod, + type MemberCore, +} from "./ManagedMemberInspectorCore.js"; +import type { ManagedMetadataLayout } from "./ManagedMetadataLayout.js"; +import { + metadataCodedToken, + metadataCodedTokenInvalidReason, + metadataRowCursor, + readMetadataBlob, +} from "./ManagedMetadataHeaps.js"; +import { managedFailure, readManagedValue } from "./ManagedReaderFailure.js"; + +/** + * Index fields, member references, and methods by token, then name each + * MethodSpec an inspected body calls after the generic method it instantiates, + * so every edge target resolves through one lookup. + */ +export const edgeTargets = ( + bytes: Buffer, + layout: ManagedMetadataLayout, + methods: readonly ManagedMethod[], + cores: readonly ReadonlyMap[], +): { + readonly members: ReadonlyMap; + readonly issues: readonly ManagedParseIssue[]; +} => { + const members = new Map(cores.flatMap((core) => [...core])); + const issues: ManagedParseIssue[] = []; + const seen = new Set(); + // Generic arity by method token, so instantiations of one method decode it once. + const arities = new Map(); + for (const method of methods) + for (const { operand } of method.body.anchors) { + if (operand?.startsWith("0x2b") !== true || seen.has(operand)) continue; + seen.add(operand); + readManagedValue( + () => + readMethodSpec({ bytes, layout, members, issues, arities }, operand), + issues, + ); + } + return { members, issues }; +}; + +/** + * Map one MethodSpec to its generic method, then check its instantiation + * decodes and matches that method's generic arity. A readable Method column + * keeps its name when the rest is malformed; the failure makes coverage partial. + */ +const readMethodSpec = ( + { + bytes, + layout, + members, + issues, + arities, + }: { + readonly bytes: Buffer; + readonly layout: ManagedMetadataLayout; + readonly members: Map; + readonly issues: ManagedParseIssue[]; + readonly arities: Map; + }, + token: string, +): void => { + const scope = `metadata.MethodSpec:${token}`; + const cursor = metadataRowCursor( + bytes, + layout, + 0x2b, + Number.parseInt(token.slice(4), 16), + ); + const methodRaw = cursor.readIndex(layout.codedIndexSize("MethodDefOrRef")); + const instantiation = cursor.readIndex(layout.blobIndexSize); + const methodReason = metadataCodedTokenInvalidReason( + methodRaw, + 1, + [6, 10], + layout.rowCounts, + ); + if (methodReason !== null || methodRaw === 0) + issues.push({ + code: "invalid-row", + scope, + offset: cursor.start, + detail: + methodReason === null + ? "MethodSpec method coded index 0x0 is null, but the Method column must reference a row" + : `MethodSpec method coded index 0x${methodRaw.toString(16)} is invalid: ${methodReason}`, + }); + const generic = metadataCodedToken(methodRaw, 1, [6, 10], layout.rowCounts); + const target = generic === null ? undefined : members.get(generic); + // A MethodSpec instantiates a method, so a field target never lends it a + // name, even to a field operand. + if (target !== undefined && target.kind !== "field") + members.set(token, { ...target, token }); + const types = specSignature( + layout, + readMetadataBlob(bytes, layout, instantiation, layout.blob.size), + { form: "instantiation", scope, offset: cursor.start }, + ); + if (generic === null) return; + // Prefer the arity the target's owner decoded; read the raw signature only + // for a target nothing decoded, such as a MethodDef outside the selection. + if (target?.genericArity === undefined && !arities.has(generic)) + arities.set( + generic, + methodGenericArity(layout, methodSignatureBlob(bytes, layout, generic)), + ); + const arity = + target?.genericArity === undefined + ? (arities.get(generic) ?? null) + : target.genericArity; + if (arity !== types.length) + throw managedFailure( + "invalid-row", + scope, + arity === null + ? "MethodSpec method does not have a decodable method signature" + : `MethodSpec supplies ${String(types.length)} type argument(s), but its method declares ${String(arity)} generic parameter(s)`, + cursor.start, + ); +}; + +/** The Signature column of a MethodDef or MemberRef row. */ +const methodSignatureBlob = ( + bytes: Buffer, + layout: ManagedMetadataLayout, + token: string, +): Buffer => { + const table = Number.parseInt(token.slice(2, 4), 16); + const cursor = metadataRowCursor( + bytes, + layout, + table, + Number.parseInt(token.slice(4), 16), + ); + if (table === 6) { + cursor.readUInt32(); + cursor.readUInt16(); + cursor.readUInt16(); + } else cursor.readIndex(layout.codedIndexSize("MemberRefParent")); + cursor.readIndex(layout.stringIndexSize); + return readMetadataBlob( + bytes, + layout, + cursor.readIndex(layout.blobIndexSize), + layout.blob.size, + ); +}; diff --git a/src/dotnet/ManagedNativeBoundaryHelpers.ts b/src/dotnet/ManagedNativeBoundaryHelpers.ts index fb81e1d28..e4b1a5960 100644 --- a/src/dotnet/ManagedNativeBoundaryHelpers.ts +++ b/src/dotnet/ManagedNativeBoundaryHelpers.ts @@ -18,7 +18,10 @@ import { } from "./ManagedMetadataHeaps.js"; import { managedTableRowCounts } from "./ManagedMetadataInventory.js"; import type { ManagedMetadataInventory } from "./ManagedMetadataInventory.js"; -import type { ManagedPeLayout } from "./ManagedPeReader.js"; +import { + managedCli32BitRequirement, + type ManagedPeLayout, +} from "./ManagedPeReader.js"; import { readManagedValue } from "./ManagedReaderFailure.js"; type ModuleRef = ManagedNativeBoundaryInspection["module_refs"][number]; @@ -380,7 +383,7 @@ export const cliNative = ( if (pe.cli === null) return null; return { il_only: (pe.cli.flags & 0x0000_0001) !== 0, - requires_32bit: (pe.cli.flags & 0x0000_0002) !== 0, + requires_32bit: managedCli32BitRequirement(pe.cli.flags) === "required", strong_name_signed: (pe.cli.flags & 0x0000_0008) !== 0, native_entry_point: (pe.cli.flags & 0x0000_0010) !== 0, ready_to_run_signature: pe.cli.readyToRunSignature, diff --git a/src/dotnet/ManagedPe.fixture.ts b/src/dotnet/ManagedPe.fixture.ts index 25f819925..7ea2b06b4 100644 --- a/src/dotnet/ManagedPe.fixture.ts +++ b/src/dotnet/ManagedPe.fixture.ts @@ -29,6 +29,16 @@ interface ManagedPeFixtureOptions { readonly moduleRowCount?: number; readonly assemblyRowCount?: number; readonly memberRefParentRaw?: number; + readonly memberRefName?: string; + readonly memberRefSignature?: Buffer; + /** Raw MethodDefOrRef coded values, one MethodSpec row each. */ + readonly methodSpecMethods?: readonly number[]; + /** Instantiation blob shared by every MethodSpec row. */ + readonly methodSpecInstantiation?: Buffer; + /** TypeSpec signature blobs, one TypeSpec row each. */ + readonly typeSpecSignatures?: readonly Buffer[]; + /** Raw ResolutionScope coded value of the TypeRef row; null by default. */ + readonly typeRefScopeRaw?: number; readonly mvid?: Buffer; readonly pinvoke?: { readonly importName?: string; @@ -174,10 +184,10 @@ const typeRefRow = ( name: number, namespace: number, stringSize: MetadataIndexSize, - resolutionScopeSize: MetadataIndexSize, + resolutionScope: Buffer, ): Buffer => Buffer.concat([ - metadataIndex(0, resolutionScopeSize), + resolutionScope, metadataIndex(name, stringSize), metadataIndex(namespace, stringSize), ]); @@ -524,6 +534,8 @@ interface FixtureMetadataRowContext { readonly methodSignature: number; readonly constructorSignature: number; readonly attributeBlob: number; + readonly typeSpecSignatures: readonly number[]; + readonly methodSpecInstantiation: number; }; readonly heapSizes: { readonly strings: MetadataIndexSize; @@ -595,6 +607,37 @@ const exportedTypeTable = ( ], ]; +const specTables = ( + options: ManagedPeFixtureOptions, + indexes: FixtureMetadataRowContext["indexes"], + heapSizes: FixtureMetadataRowContext["heapSizes"], + widths: FixtureIndexWidths, +): readonly (readonly [number, readonly Buffer[]])[] => [ + ...(indexes.typeSpecSignatures.length === 0 + ? [] + : ([ + [ + 27, + indexes.typeSpecSignatures.map((signature) => + metadataIndex(signature, heapSizes.blobs), + ), + ], + ] as const)), + ...(options.methodSpecMethods === undefined + ? [] + : ([ + [ + 43, + options.methodSpecMethods.map((method) => + Buffer.concat([ + metadataIndex(method, widths.coded([6, 10], 1)), + metadataIndex(indexes.methodSpecInstantiation, heapSizes.blobs), + ]), + ), + ], + ] as const)), +]; + const fixtureIndexWidths = ({ options, referenceStringIndexes, @@ -616,10 +659,12 @@ const fixtureIndexWidths = ({ ], ...(indexes.pinvokeModuleName === null ? [] : ([[26, 1]] as const)), ...(indexes.pinvokeImportName === null ? [] : ([[28, 1]] as const)), + [27, indexes.typeSpecSignatures.length], [32, options.assemblyRowCount ?? 1], [35, referenceStringIndexes.length], [39, indexes.exportedTypes.length], [40, 1], + [43, options.methodSpecMethods?.length ?? 0], ]); const count = (table: number): number => rowCounts.get(table) ?? 0; const table = (index: number): MetadataIndexSize => indexSize(count(index)); @@ -673,7 +718,7 @@ const fixtureMetadataRows = ({ indexes.attributeName, indexes.attributeNamespace, heapSizes.strings, - widths.resolutionScope, + metadataIndex(options.typeRefScopeRaw ?? 0, widths.resolutionScope), ), ], ], @@ -780,6 +825,7 @@ const fixtureMetadataRows = ({ ), ], ...exportedTypeTable(options, indexes, heapSizes, widths), + ...specTables(options, indexes, heapSizes, widths), [ 40, [ @@ -795,6 +841,29 @@ const fixtureMetadataRows = ({ return rows; }; +/** Blob indexes for the MemberRef and optional spec rows; defaults leave the heap unchanged. */ +const referenceBlobIndexes = ( + blobs: BlobHeap, + options: ManagedPeFixtureOptions, +): Pick< + FixtureMetadataRowContext["indexes"], + "constructorSignature" | "typeSpecSignatures" | "methodSpecInstantiation" +> => ({ + constructorSignature: blobs.add( + options.memberRefSignature ?? Buffer.from([0x20, 0x01, 0x01, 0x0e]), + ), + typeSpecSignatures: (options.typeSpecSignatures ?? []).map((signature) => + blobs.add(signature), + ), + // Defaults to GENERICINST with one string type argument. + methodSpecInstantiation: + options.methodSpecMethods === undefined + ? 0 + : blobs.add( + options.methodSpecInstantiation ?? Buffer.from([0x0a, 0x01, 0x0e]), + ), +}); + const buildManagedFixtureMetadata = ( options: ManagedPeFixtureOptions, methodRva: number, @@ -805,7 +874,7 @@ const buildManagedFixtureMetadata = ( const assemblyName = strings.add("Fixture.Managed"); const attributeName = strings.add("TargetFrameworkAttribute"); const attributeNamespace = strings.add("System.Runtime.Versioning"); - const constructorName = strings.add(".ctor"); + const constructorName = strings.add(options.memberRefName ?? ".ctor"); const typeName = strings.add(options.typeName ?? "Program"); const additionalTypeNames = Array.from( { length: options.additionalEmptyTypes ?? 0 }, @@ -840,7 +909,7 @@ const buildManagedFixtureMetadata = ( const methodSignature = blobs.add( options.methodSignature ?? Buffer.from([0x00, 0x00, 0x01]), ); - const constructorSignature = blobs.add(Buffer.from([0x20, 0x01, 0x01, 0x0e])); + const referenceBlobs = referenceBlobIndexes(blobs, options); const attributeBlob = blobs.add( fixedStringAttributeBlob( options.targetFramework === null @@ -873,8 +942,8 @@ const buildManagedFixtureMetadata = ( tokenBlob, fieldSignature, methodSignature, - constructorSignature, attributeBlob, + ...referenceBlobs, }, heapSizes: { strings: stringSize, blobs: blobSize, guids: guidSize }, }); diff --git a/src/dotnet/ManagedPeReader.ts b/src/dotnet/ManagedPeReader.ts index 2f9bc3151..9a30c007b 100644 --- a/src/dotnet/ManagedPeReader.ts +++ b/src/dotnet/ManagedPeReader.ts @@ -31,6 +31,20 @@ export interface ManagedCliHeader { readonly readyToRunSignature: boolean; } +/** + * How the CLI header constrains process bitness. 32BITREQUIRED and + * 32BITPREFERRED are one two-bit field: required alone is x86-only, both set + * is platform-neutral code that prefers a 32-bit process, and preferred alone + * is reserved and constrains nothing. + */ +export const managedCli32BitRequirement = ( + flags: number, +): "required" | "preferred" | "none" => { + const pair = flags & 0x0002_0002; + if (pair === 0x0000_0002) return "required"; + return pair === 0x0002_0002 ? "preferred" : "none"; +}; + export interface ManagedPeLayout { readonly machine: number; readonly architecture: "x86" | "x86_64" | "arm" | "arm64"; diff --git a/src/dotnet/ManagedTypeNames.ts b/src/dotnet/ManagedTypeNames.ts new file mode 100644 index 000000000..ab3b34c8f --- /dev/null +++ b/src/dotnet/ManagedTypeNames.ts @@ -0,0 +1,137 @@ +import { managedDecodeBudget } from "./ManagedDecodeBudget.js"; +import type { ManagedParseIssue } from "../domain/managed/managedArtifact.js"; +import { + fullName, + readCompressed, + specSignature, + type TypeRange, +} from "./ManagedMemberInspectorCore.js"; +import { + metadataRowOffset, + type ManagedMetadataLayout, +} from "./ManagedMetadataLayout.js"; +import { + metadataCodedTokenInvalidReason, + metadataRowCursor, + metadataToken, + readMetadataBlob, + readMetadataString, +} from "./ManagedMetadataHeaps.js"; +import { managedFailure, readManagedValue } from "./ManagedReaderFailure.js"; + +/** + * Name the type a TypeDef, TypeRef, or generic-instantiation TypeSpec token + * denotes, in the namespace-qualified form TypeDef `full_name` uses, once per + * token. Other decoded TypeSpecs stay null: the type is unknown. Malformed or + * unsupported rows and blobs are reported to `issues` and also stay null, + * except that a TypeRef with an invalid ResolutionScope keeps its readable name. + */ +export const typeNameLookup = ( + bytes: Buffer, + layout: ManagedMetadataLayout, + ranges: readonly TypeRange[], + issues: ManagedParseIssue[], +): ((token: string | null) => string | null) => { + const names = new Map(); + const checkedScopes = new Set(); + const validateScopeChain = (row: number): void => { + const path = new Set(); + let current = row; + while (!checkedScopes.has(current)) { + const cursor = metadataRowCursor(bytes, layout, 1, current); + if (path.has(current)) { + issues.push({ + code: "invalid-row", + scope: `metadata.TypeRef:${metadataToken(1, current)}`, + offset: cursor.start, + detail: "TypeRef ResolutionScope chain contains a cycle", + }); + break; + } + // Admit the chain's visited entry and retained validation-cache entry. + managedDecodeBudget(layout).reserve(128, cursor.start); + path.add(current); + const raw = cursor.readIndex(layout.codedIndexSize("ResolutionScope")); + const reason = metadataCodedTokenInvalidReason( + raw, + 2, + [0, 26, 35, 1], + layout.rowCounts, + ); + if (reason !== null) { + issues.push({ + code: "invalid-row", + scope: `metadata.TypeRef:${metadataToken(1, current)}`, + offset: cursor.start, + detail: `TypeRef ResolutionScope coded index 0x${raw.toString(16)} is invalid: ${reason}`, + }); + break; + } + // Null scopes are legal; non-TypeRef scopes terminate local validation. + if (raw === 0 || (raw & 3) !== 3) break; + current = Math.floor(raw / 4); + } + for (const visited of path) checkedScopes.add(visited); + }; + const definitionName = (table: 1 | 2, row: number): string | null => { + if (table === 2) { + metadataRowOffset(layout, 2, row); + return ranges[row - 1]?.fullName ?? null; + } + const cursor = metadataRowCursor(bytes, layout, 1, row); + cursor.readIndex(layout.codedIndexSize("ResolutionScope")); + readManagedValue(() => validateScopeChain(row), issues); + const name = cursor.readIndex(layout.stringIndexSize); + const namespace = cursor.readIndex(layout.stringIndexSize); + return fullName( + readMetadataString(bytes, layout, namespace, layout.strings.size), + readMetadataString(bytes, layout, name, layout.strings.size), + ); + }; + const specName = (row: number): string | null => { + const cursor = metadataRowCursor(bytes, layout, 27, row); + const spec = readMetadataBlob( + bytes, + layout, + cursor.readIndex(layout.blobIndexSize), + layout.blob.size, + ); + const owner = { + form: "type", + scope: `metadata.TypeSpec:${metadataToken(27, row)}`, + offset: cursor.start, + } as const; + specSignature(layout, spec, owner); + if (spec[0] !== 0x15) return null; + // GENERICINST (CLASS | VALUETYPE) TypeDefOrRefEncoded: never a TypeSpec, + // so naming never recurses. + const encoded = readCompressed(spec, 2).value; + const table = ([2, 1] as const)[encoded & 3]; + if (table === undefined) + throw managedFailure( + "invalid-blob", + owner.scope, + "Generic instantiation does not name a TypeDef or TypeRef", + owner.offset, + ); + return definitionName(table, Math.floor(encoded / 4)); + }; + return (token) => { + if (token === null) return null; + if (names.has(token)) return names.get(token) ?? null; + const table = Number.parseInt(token.slice(2, 4), 16); + const row = Number.parseInt(token.slice(4), 16); + const name = + readManagedValue( + () => + table === 27 + ? specName(row) + : table === 1 || table === 2 + ? definitionName(table, row) + : null, + issues, + ) ?? null; + names.set(token, name); + return name; + }; +}; diff --git a/src/ghidra/GhidraAnalysisProfile.ts b/src/ghidra/GhidraAnalysisProfile.ts index fd7cdff11..81defabd3 100644 --- a/src/ghidra/GhidraAnalysisProfile.ts +++ b/src/ghidra/GhidraAnalysisProfile.ts @@ -127,7 +127,7 @@ export const resolveGhidraAnalysisProfile = async ( "address-first-unique-target-source-mapping-v2", instruction_flow_evidence: "decoded-return-pcode-v1", instruction_references: "exact-address-incoming-v1", - function_boundary_observations: "ghidra-terminal-call-limitations-v1", + function_boundary_observations: "ghidra-terminal-call-limitations-v2", no_return_repair: installation.platform === "win32" ? "disabled-windows-p0" @@ -136,7 +136,7 @@ export const resolveGhidraAnalysisProfile = async ( installation.platform === "win32" ? "official-headless-script-v1" : "inspected-jvm-launch-support-v1", - load_image_evidence: "independent-raw-mapping-context-v2", + load_image_evidence: "independent-raw-mapping-context-v4", jump_table_evidence: "typed-case-default-blocks-v1", decompiler_jump_loads: true, loader: "BinaryLoader", @@ -218,7 +218,7 @@ export const resolveGhidraAnalysisProfile = async ( "address-first-unique-target-source-mapping-v2", instruction_flow_evidence: "decoded-return-pcode-v1", instruction_references: "exact-address-incoming-v1", - function_boundary_observations: "ghidra-terminal-call-limitations-v1", + function_boundary_observations: "ghidra-terminal-call-limitations-v2", no_return_repair: installation.platform === "win32" ? "disabled-windows-p0" diff --git a/src/ghidra/GhidraFunctionValues.test.ts b/src/ghidra/GhidraFunctionValues.test.ts index 75d59c97f..e54692840 100644 --- a/src/ghidra/GhidraFunctionValues.test.ts +++ b/src/ghidra/GhidraFunctionValues.test.ts @@ -23,6 +23,8 @@ import { const observationOperations: ReadonlySet = new Set([ "analyze_function", "procedure_assembly", + "procedure_callees", + "procedure_callers", "procedure_info", "procedure_pseudo_code", "read_function_instructions", @@ -93,6 +95,29 @@ describe("Ghidra function-analysis result values", () => { }); }); + it.each(["procedure_callees", "procedure_callers"] as const)( + "requires the function-observation envelope to carry terminal-call limitations for %s", + (operation) => { + // A bare address list is the pre-fix bridge shape; without the envelope + // a truncated function body reports no warning, so it must be rejected. + expect(parseGhidraFunctionResult(operation, ["0x2000"])).toMatchObject({ + ok: false, + error: { _tag: "AnalysisOutputError" }, + }); + const limitation = + "Ghidra Listing reports a terminal call at 0x401000 to 0x402000 abort; fallthrough is excluded by Ghidra's flow model."; + expect( + parseGhidraFunctionResult(operation, { + value: ["0x2000"], + limitations: [limitation], + }), + ).toEqual({ + ok: true, + value: { value: ["0x2000"], limitations: [limitation] }, + }); + }, + ); + it.each(["reference_kinds_available", "unresolved_calls"])( "rejects omitted %s instead of inventing complete reference observations", (field) => { @@ -195,6 +220,51 @@ describe("Ghidra function-analysis result values", () => { }); }); +describe("Ghidra Swift String literals", () => { + it("keeps a small Swift literal without storage apart from a large literal's bytes", () => { + const dossier = ghidraFunctionDossier(); + if ( + typeof dossier !== "object" || + dossier === null || + Array.isArray(dossier) + ) + throw new TypeError("Ghidra dossier fixture is invalid"); + const literals = [ + { + source_address: "0x401004", + value: "The user name ", + representation: "small", + address: null, + registers: ["x0", "x1"], + words: ["0x7265737520656854", "0xee0020656d616e20"], + }, + { + source_address: "0x401006", + value: "", + representation: "small", + address: null, + registers: ["x0", "x1"], + words: ["0x0", "0xe000000000000000"], + }, + { + source_address: "0x401008", + value: " cannot be written into a sudoers rule safely.", + representation: "large", + address: "0x402270", + registers: ["x0", "x1"], + words: ["0xd00000000000002e", "0x8000000000402250"], + }, + ]; + expect( + parseFunctionValueFixture("analyze_function", { + ...dossier, + swift_string_literals: literals, + swift_string_literal_omissions: [], + }), + ).toMatchObject({ ok: true, value: { swift_string_literals: literals } }); + }); +}); + describe("Ghidra compact reference provenance", () => { it("distinguishes incoming references, examined emptiness and unavailable coverage at data addresses", () => { const instruction = { @@ -423,6 +493,34 @@ describe("Ghidra function-analysis malformed results", () => { ); }); +const malformedSwiftStringLiterals = ( + dossier: Record, +): Array<[string, GhidraFunctionOperation, JsonValue]> => + ( + [ + ["non-canonical Swift literal address", "large", "402240"], + ["small Swift literal with storage", "small", "0x402270"], + ["large Swift literal without storage", "large", null], + ] as const + ).map(([name, representation, address]) => [ + name, + "analyze_function", + { + ...dossier, + swift_string_literal_omissions: [], + swift_string_literals: [ + { + source_address: "0x401004", + value: "The power setting could not be changed.", + representation, + address, + registers: ["x0", "x1"], + words: ["0xd000000000000027", "0x8000000000402220"], + }, + ], + }, + ]); + const malformedOutputs = (): Array< [string, GhidraFunctionOperation, JsonValue] > => { @@ -437,6 +535,11 @@ const malformedOutputs = (): Array< throw new TypeError("Ghidra native API fixture is invalid"); return [ ["non-canonical address", "procedure_callers", ["401000"]], + [ + "Swift literals without their omission record", + "analyze_function", + { ...dossier, swift_string_literals: [] }, + ], [ "missing classification", "procedure_info", @@ -467,6 +570,26 @@ const malformedOutputs = (): Array< Object.entries(dossier).filter(([key]) => key !== "native_api"), ), ], + ...malformedSwiftStringLiterals(dossier), + [ + "non-canonical Swift literal omission address", + "analyze_function", + { + ...dossier, + swift_string_literals: [], + swift_string_literal_omissions: [ + { + source_address: "0x401004", + representation: "large", + address: "402240", + byte_length: 39, + reason: "capacity", + registers: ["x0", "x1"], + words: ["0xd000000000000027", "0x8000000000402220"], + }, + ], + }, + ], [ "non-canonical native API target", "analyze_function", diff --git a/src/ghidra/GhidraFunctionValues.ts b/src/ghidra/GhidraFunctionValues.ts index d45e41ba9..a6e62c93c 100644 --- a/src/ghidra/GhidraFunctionValues.ts +++ b/src/ghidra/GhidraFunctionValues.ts @@ -293,6 +293,15 @@ const ghidraFunctionDossier = functionDossierSchema }) .strict() .superRefine((value, context) => { + if ( + (value.swift_string_literals === undefined) !== + (value.swift_string_literal_omissions === undefined) + ) + context.addIssue({ + code: "custom", + message: + "Ghidra dossier must record Swift literals and their omissions together", + }); const addresses = [ value.procedure.address, ...value.comments.map(({ address }) => address), @@ -308,6 +317,14 @@ const ghidraFunctionDossier = functionDossierSchema address, source_address, ]), + ...(value.swift_string_literals ?? []).flatMap( + ({ address, source_address }) => + address === null ? [source_address] : [address, source_address], + ), + ...(value.swift_string_literal_omissions ?? []).flatMap( + ({ address, source_address }) => + address === null ? [source_address] : [address, source_address], + ), ...value.basic_blocks.flatMap(({ start, end, successors }) => [ start, end, @@ -392,6 +409,8 @@ const functionBoundaryObservationOperations: ReadonlySet ({ encoding: "utf8", env: environment, timeout: 5_000, - maxBuffer: Number.POSITIVE_INFINITY, + maxBuffer: JAVA_PROBE_MAX_OUTPUT_BYTES, windowsHide: true, }, ); diff --git a/src/ghidra/GhidraLanguageCatalog.ts b/src/ghidra/GhidraLanguageCatalog.ts index 5da5943dc..6373f187f 100644 --- a/src/ghidra/GhidraLanguageCatalog.ts +++ b/src/ghidra/GhidraLanguageCatalog.ts @@ -75,8 +75,10 @@ export const readGhidraLanguageCatalog = async ( if (!entry.endsWith(".ldefs")) continue; const bytes = await readFile(join(directory, entry)); if (bytes.length > MAX_LDEFS_BYTES) continue; + // XML 1.0 section 4.3.3: a leading UTF-8 BOM is an encoding signature, + // not document content; TextDecoder consumes it, Buffer#toString does not. definitions.push( - ...parseGhidraLanguageDefinitions(bytes.toString("utf8")), + ...parseGhidraLanguageDefinitions(new TextDecoder().decode(bytes)), ); } } diff --git a/src/ghidra/GhidraLanguageOverride.test.ts b/src/ghidra/GhidraLanguageOverride.test.ts index 5f90a1454..02c5bf245 100644 --- a/src/ghidra/GhidraLanguageOverride.test.ts +++ b/src/ghidra/GhidraLanguageOverride.test.ts @@ -2,6 +2,7 @@ import { mkdir, mkdtemp, rm, writeFile } from "node:fs/promises"; import { tmpdir } from "node:os"; import { join } from "node:path"; import { afterAll, beforeAll, describe, expect, it } from "vitest"; +import { createTestTempDirectory } from "../../tests/fixtures/temporaryDirectory.js"; import { parseConfig } from "../config/parseConfig.js"; import type { BinaryTarget } from "../domain/binaryTargetTypes.js"; import { projectAnalysisError } from "../domain/analysisErrorProjection.js"; @@ -190,6 +191,28 @@ describe("Ghidra language catalog", () => { ); }); + it("reads a UTF-8 signed definition beside an unsigned sibling", async () => { + const root = await createTestTempDirectory("rea-ghidra-bom-"); + const module = (name: string) => + join(root, "Ghidra/Processors", name, "data/languages"); + await mkdir(module("probe"), { recursive: true }); + await mkdir(module("x86"), { recursive: true }); + await writeFile( + join(module("probe"), "probe.ldefs"), + Buffer.concat([ + Buffer.from([0xef, 0xbb, 0xbf]), + Buffer.from( + '\n', + ), + ]), + ); + await writeFile(join(module("x86"), "x86.ldefs"), X86_LDEFS); + expect(await readGhidraLanguageCatalog(root)).toEqual([ + { id: "probe:LE:32:default", compilerSpecIds: [] }, + ...parseGhidraLanguageDefinitions(X86_LDEFS), + ]); + }); + it("admits only languages and compiler specs the installation defines", () => { const catalog = parseGhidraLanguageDefinitions(X86_LDEFS); expect( diff --git a/src/ghidra/GhidraLauncher.ts b/src/ghidra/GhidraLauncher.ts index 59c555606..e1527ea81 100644 --- a/src/ghidra/GhidraLauncher.ts +++ b/src/ghidra/GhidraLauncher.ts @@ -1,3 +1,4 @@ +import { parseGhidraJavaProperties } from "./GhidraJavaProperties.js"; import { GHIDRA_SEED_SCRIPT, snapshotGhidraAnalysisSeeds, @@ -5,7 +6,7 @@ import { } from "./GhidraAnalysisSeeds.js"; import type { GhidraLanguageOverride } from "../config/ghidraLanguageOverride.js"; import { snapshotEnvironment } from "../process/snapshotEnvironment.js"; -import { mkdir, realpath } from "node:fs/promises"; +import { mkdir, readFile, realpath } from "node:fs/promises"; import { basename, dirname, join, posix, win32 } from "node:path"; import writeFileAtomic from "write-file-atomic"; @@ -219,7 +220,11 @@ export class GhidraHeadlessLauncher implements GhidraLauncher { "Installed Allegrex content changed since profile resolution; reopen the target to resolve a fresh profile.", ); } - await createGhidraRuntimeDirectories(paths, platform); + await createGhidraRuntimeDirectories( + paths, + platform, + await ghidraConfigReleaseDirectory(this.options.analyzeHeadlessPath), + ); const seedPath = this.options.analysisSeeds === undefined ? undefined @@ -721,9 +726,44 @@ const ghidraRuntimePaths = (runtimeRoot: string) => ({ scriptLogPath: join(runtimeRoot, "script.log"), }); +/** + * Ghidra's launch support resolves its JDK before the headless run and then + * persists that choice under `/ghidra/`. Inside + * REA's Windows private runtime a child JVM cannot create those directories + * itself, so startup aborted with "Failed to create directory" and reported + * that no supported JDK was found. Create the release directory while this + * process still owns the runtime, leaving Ghidra only a file write inside it. + */ +const ghidraConfigReleaseDirectory = async ( + analyzeHeadlessPath: string, +): Promise => { + const installRoot = dirname(dirname(analyzeHeadlessPath)); + try { + const properties = await readFile( + join(installRoot, "Ghidra", "application.properties"), + "latin1", + ); + const parsed = parseGhidraJavaProperties(properties); + if (!parsed.ok) return basename(installRoot); + const version = parsed.value.get("application.version"); + const release = parsed.value.get("application.release.name"); + return version === undefined || + release === undefined || + release.length === 0 + ? basename(installRoot) + : `ghidra_${version}_${release}`; + } catch (cause: unknown) { + // An unreadable properties file leaves the release coordinate unknown; the + // conventional extraction directory carries the same name. + void cause; + return basename(installRoot); + } +}; + const createGhidraRuntimeDirectories = async ( paths: ReturnType, platform: NodeJS.Platform, + configReleaseDirectory: string, ): Promise => { if (platform === "win32") { const runtime = windowsPrivateRuntime(dirname(paths.projectRoot)); @@ -736,6 +776,11 @@ const createGhidraRuntimeDirectories = async ( paths.dataRoot, ]) runtime.mkdir(basename(path)); + // Each level needs its parent first: the native owner opens every existing + // component and creates only the final one. + const configRoot = basename(paths.configRoot); + runtime.mkdir(win32.join(configRoot, "ghidra")); + runtime.mkdir(win32.join(configRoot, "ghidra", configReleaseDirectory)); return; } await Promise.all( @@ -746,6 +791,7 @@ const createGhidraRuntimeDirectories = async ( paths.cacheRoot, paths.configRoot, paths.dataRoot, + join(paths.configRoot, "ghidra", configReleaseDirectory), ].map((path) => mkdir(path, { recursive: true, mode: 0o700 })), ); }; diff --git a/src/ghidra/GhidraLoadImageValues.ts b/src/ghidra/GhidraLoadImageValues.ts index 1c80bb635..f06f4fdaa 100644 --- a/src/ghidra/GhidraLoadImageValues.ts +++ b/src/ghidra/GhidraLoadImageValues.ts @@ -361,7 +361,10 @@ export const attestGhidraRawBinaryLoadImage = ( }); }; const source = observations.source_files[0]; - const mapping = observations.mappings[0]; + const fileMappings = observations.mappings.filter( + (item) => item.source_file_index !== null, + ); + const mapping = fileMappings[0]; check("executable_format", "Raw Binary", observations.executable_format); check("language_id", language.languageId, observations.language_id); check( @@ -371,11 +374,34 @@ export const attestGhidraRawBinaryLoadImage = ( ); // BinaryLoader's block base is independent of the Program image base. check("image_base", "0x0", observations.image_base); + check( + "default_address_space", + language.defaultAddressSpace, + observations.default_address_space, + ); check("source_file_count", 1, observations.source_files.length); check("source_file_size", snapshot.bytes, source?.size ?? null); check("original_file_sha256", target.sha256, source?.original_sha256 ?? null); check("modified_file_sha256", target.sha256, source?.modified_sha256 ?? null); - check("mapping_count", 1, observations.mappings.length); + // Language-defined uninitialized zero page/stack blocks are observations, not file bytes. + check("mapping_count", 1, fileMappings.length); + check( + "initialized_unbacked_mappings", + 0, + observations.mappings.filter( + (item) => item.initialized && item.source_file_index === null, + ).length, + ); + check("mapping.source_file_index", 0, mapping?.source_file_index ?? null); + // Matching bytes alone do not establish the admitted default-space load. + check("mapping.initialized", true, mapping?.initialized ?? null); + check("mapping.loaded", true, mapping?.loaded ?? null); + check("mapping.overlay", false, mapping?.overlay ?? null); + check( + "mapping.address_space", + observations.default_address_space, + mapping?.address_space ?? null, + ); check( "mapping.file_offset", 0, @@ -427,8 +453,9 @@ export const attestGhidraRawBinaryLoadImage = ( checks, limitations: [ "Raw binary is a headerless interpretation explicitly selected by the caller, not format identification from file bytes.", - "Verification covers complete original/modified source identity, mapped file bytes, the caller-supplied entry, and the bound little-endian Ghidra language. File offset 0 is mapped at the entry address.", + "Verification covers complete original/modified source identity, initialized and loaded non-overlay file bytes in the default address space, the caller-supplied entry, and the bound little-endian Ghidra language. File offset 0 is mapped at the entry address.", "Ghidra may report additional language-defined external entries outside the mapped image. They are preserved as observations; verification requires the caller-supplied entry and does not attest other entries.", + "Language-defined uninitialized memory, such as 6502 zero page and stack, is preserved as observations but is not attested as file-backed bytes.", "Reset vectors, memory-mapped I/O, big-endian and Thumb-only variants, and runtime boot behavior are not modeled or verified. No original executable bytes are changed.", "Mapping ends are inclusive; a mismatching digest identifies its source range, not the first differing byte.", ], diff --git a/src/ghidra/GhidraProvider.behavior.test.ts b/src/ghidra/GhidraProvider.behavior.test.ts index 3979222d9..c9084905b 100644 --- a/src/ghidra/GhidraProvider.behavior.test.ts +++ b/src/ghidra/GhidraProvider.behavior.test.ts @@ -8,7 +8,7 @@ import { tmpdir } from "node:os"; import { join } from "node:path"; import { fileURLToPath } from "node:url"; import { promisify } from "node:util"; -import { describe, expect, it } from "vitest"; +import { describe, expect, it, onTestFinished, vi } from "vitest"; import { fixtureDosLoadImage } from "./GhidraLoadImage.fixture.js"; import { fixtureRawBinaryLoadImage } from "./GhidraRawBinaryLoadImage.fixture.js"; import { @@ -30,6 +30,8 @@ import { err, ok } from "../domain/result.js"; import type { Result } from "../domain/result.js"; import { GhidraSessionError } from "./GhidraSessionError.js"; import { silentLogger } from "../logger.js"; +import * as WindowsNativeLoader from "../windows/WindowsNativeLoader.js"; +import { WINDOWS_NATIVE_AUTHORITY_UNAVAILABLE_REASON } from "../process/WindowsAuthority.js"; import { ProviderCleanupError } from "../domain/providerCleanupError.js"; import { projectAnalysisError } from "../domain/analysisErrorProjection.js"; import { ProviderStartupDeadline } from "../process/ProviderDeadline.js"; @@ -190,6 +192,36 @@ describe("Ghidra provider", () => { }); describe("Ghidra raw-binary profile", () => { + it("admits raw 6502 on Windows and binds its 16-bit language without mutation authority", async () => { + const ghidra = provider({ ...installationHost(), platform: "win32" }); + const target: BinaryTarget = { + path: "/tmp/6502.bin", + sha256: "a".repeat(64), + kind: "executable", + format: "raw-binary", + architecture: "6502", + availableArchitectures: ["6502"], + entryPoint: "0x8001", + entryPointSource: "0x008001", + }; + expect(ghidra.inspectTargetSupport(target).status).toBe("supported"); + const resolved = await ghidra.resolveAnalysisProfile(target); + if (!resolved.ok) throw resolved.error; + expect(resolved.value.profile?.parameters).toMatchObject({ + architecture: "6502", + loader: "BinaryLoader", + language_id: "6502:LE:16:default", + compiler_spec_id: "default", + entry_point: "0x8001", + language_selection: "caller-supplied-v1", + load_image_evidence: "independent-raw-mapping-context-v4", + }); + expect( + ghidra + .capabilities() + .some(({ available, effects }) => available && effects.mutatesArtifact), + ).toBe(false); + }); it("commits BinaryLoader language and caller-supplied entry", async () => { const ghidra = provider(); const target: BinaryTarget = { @@ -324,6 +356,13 @@ describe("Ghidra platform support", () => { }); it("keeps Windows P0 unavailable until native isolation authority exists", () => { + const native = vi + .spyOn(WindowsNativeLoader, "systemWindowsNativeAuthority") + .mockReturnValue({ + available: false, + reason: WINDOWS_NATIVE_AUTHORITY_UNAVAILABLE_REASON, + }); + onTestFinished(() => native.mockRestore()); const ghidra = provider({ ...installationHost(), platform: "win32" }); const nativeApplication = peTarget("x86_64"); @@ -789,7 +828,11 @@ describe("Ghidra relationship coverage", () => { start: () => Promise.resolve(ok(sessionInfo())), callTool: (operation) => Promise.resolve( - ok(operation === "procedure_callees" ? ["0x2000"] : []), + ok( + operation === "procedure_callees" + ? { value: ["0x2000"], limitations: [] } + : { value: [], limitations: [] }, + ), ), close: () => Promise.resolve(ok(null)), })); @@ -812,11 +855,22 @@ describe("Ghidra relationship coverage", () => { }); }); -describe("Ghidra measured load-image projection", () => { - it.each(["verified", "changed-snapshot", "cancelled"] as const)( +describe("Ghidra measured raw load-image projection", () => { + it.each([ + "verified", + "mapping-mismatch", + "changed-snapshot", + "cancelled", + ] as const)( "streams raw snapshot identity and preserves %s outcomes", async (state) => { const fixture = fixtureRawBinaryLoadImage(); + if (state === "mapping-mismatch") { + const mapping = fixture.observation.mappings[0]; + if (mapping === undefined) + throw new Error("Missing source mapping fixture"); + mapping.overlay = true; + } const controller = new AbortController(); const signals: (AbortSignal | undefined)[] = []; const ghidra = provider(installationHost(), () => ({ @@ -872,12 +926,23 @@ describe("Ghidra measured load-image projection", () => { { signal: controller.signal }, ); expect(signals).toEqual([controller.signal]); - if (state === "verified") { + if (state === "verified" || state === "mapping-mismatch") { if (!result.ok) throw result.error; expect(result.value.result).toMatchObject({ - status: "verified", + status: state === "verified" ? "verified" : "mismatch", observations: fixture.observation, }); + if (state === "mapping-mismatch") { + expect(result.value.result).toMatchObject({ + checks: expect.arrayContaining([ + expect.objectContaining({ + name: "mapping.overlay", + observed: true, + matched: false, + }), + ]), + }); + } } else { expect(result).toMatchObject({ ok: false, @@ -894,7 +959,9 @@ describe("Ghidra measured load-image projection", () => { } }, ); +}); +describe("Ghidra measured DOS load-image projection", () => { it.each([ "verified", "mismatch", diff --git a/src/ghidra/GhidraProvider.ts b/src/ghidra/GhidraProvider.ts index c48f58957..0a1cde0f0 100644 --- a/src/ghidra/GhidraProvider.ts +++ b/src/ghidra/GhidraProvider.ts @@ -66,6 +66,7 @@ const SUPPORTED_ARCHITECTURES = new Set([ "arm", "arm64", "mips", + "6502", "riscv", ]); @@ -184,7 +185,7 @@ export class GhidraProvider implements AnalysisProviderCandidate { status: "unsupported", code: "architecture_unsupported", reason: - "Ghidra v1 requires a supported x86, x86_64, arm, arm64, or MIPS target profile.", + "Ghidra v1 requires a supported x86, x86_64, arm, arm64, MIPS, RISC-V, or raw 6502 target profile.", diagnostics, }; return { @@ -304,11 +305,15 @@ const inspectWindowsP0TargetSupport = ( target: BinaryTarget, diagnostics: Readonly>, ): ProviderTargetSupport => { + // The raw 6502 lane uses the same immutable snapshot and native ownership boundary. + if (target.format === "raw-binary" && target.architecture === "6502") + return { status: "supported", code: null, reason: null, diagnostics }; if (target.format !== "pe") return { status: "unsupported", code: "target_format_unsupported", - reason: "Windows Ghidra P0 accepts PE targets only.", + reason: + "Windows Ghidra P0 accepts native PE and explicitly selected raw 6502 targets only.", diagnostics, }; if (target.architecture !== "x86" && target.architecture !== "x86_64") diff --git a/src/ghidra/GhidraProviderCapabilities.ts b/src/ghidra/GhidraProviderCapabilities.ts index ea863e8a7..68c3045ce 100644 --- a/src/ghidra/GhidraProviderCapabilities.ts +++ b/src/ghidra/GhidraProviderCapabilities.ts @@ -33,7 +33,7 @@ export const healthLimitations = Object.freeze([ /** Additional limitations applied to the experimental Windows x64 P0 boundary. */ export const windowsP0Limitations = Object.freeze([ - "Windows Ghidra P0 accepts approved native x86 and x86-64 PE applications and DLLs; managed, hostile, sensitive, and mutable-path targets are unsupported.", + "Windows Ghidra P0 accepts native x86 and x86-64 PE applications and DLLs, and explicitly selected raw 6502 images; managed, hostile, sensitive, and mutable-path targets are unsupported.", "The Windows bridge uses authenticated IPv4 loopback because Node path-based IPC does not expose Java AF_UNIX sockets; the endpoint file contains no bearer token.", "Windows sessions require the matching packaged Windows x64 native addon, local NTFS targets and runtimes, and Windows 10 or later. Native handles enforce path admission, private DACLs, and Job Object ownership automatically.", ]); diff --git a/src/ghidra/GhidraProviderClient.ts b/src/ghidra/GhidraProviderClient.ts index 5911985df..3a15dbc4e 100644 --- a/src/ghidra/GhidraProviderClient.ts +++ b/src/ghidra/GhidraProviderClient.ts @@ -516,7 +516,7 @@ export const createGhidraProviderClient = (input: { ...(target.format === "raw-binary" ? [ "Headerless raw-binary analysis uses the caller-supplied architecture and entry address; file offset 0 is mapped at that address. Headers are not discovered from file bytes.", - "Little-endian generic Ghidra languages are used for x86, x86_64, arm, and arm64. Big-endian, Thumb-only, Cortex-M, and other processor variants are unsupported in this increment.", + "Little-endian generic Ghidra languages are used for x86, x86_64, arm, arm64, and NMOS 6502. Big-endian, Thumb-only, Cortex-M, 65C02, 65816, and other processor variants are unsupported in this increment.", "Static analysis does not emulate reset vectors, memory-mapped I/O, or firmware boot sequences.", ] : []), diff --git a/src/ghidra/GhidraRawBinaryLanguage.ts b/src/ghidra/GhidraRawBinaryLanguage.ts index ea024bdba..e9046975d 100644 --- a/src/ghidra/GhidraRawBinaryLanguage.ts +++ b/src/ghidra/GhidraRawBinaryLanguage.ts @@ -1,28 +1,51 @@ -import type { HeaderArchitecture } from "../domain/binaryTargetTypes.js"; +import type { RawBinaryAdmission } from "../domain/rawBinary.js"; /** Bound Ghidra language for a caller-supplied raw-binary architecture. */ export interface GhidraRawBinaryLanguage { readonly languageId: string; readonly compilerSpecId: "default" | "gcc"; + readonly defaultAddressSpace: "ram" | "RAM"; } /** - * Map the four caller-visible CPU families to little-endian generic Ghidra + * Map caller-visible CPU families to little-endian generic Ghidra * languages with a supported generic compiler specification. These are * analysis conventions, not evidence of the target's original compiler. * Big-endian, Thumb-only, and Cortex-M variants are out of scope. */ export const ghidraRawBinaryLanguage = ( - architecture: HeaderArchitecture, + architecture: RawBinaryAdmission["architecture"], ): GhidraRawBinaryLanguage => { switch (architecture) { + case "6502": + return { + languageId: "6502:LE:16:default", + compilerSpecId: "default", + defaultAddressSpace: "RAM", + }; case "x86": - return { languageId: "x86:LE:32:default", compilerSpecId: "gcc" }; + return { + languageId: "x86:LE:32:default", + compilerSpecId: "gcc", + defaultAddressSpace: "ram", + }; case "x86_64": - return { languageId: "x86:LE:64:default", compilerSpecId: "gcc" }; + return { + languageId: "x86:LE:64:default", + compilerSpecId: "gcc", + defaultAddressSpace: "ram", + }; case "arm": - return { languageId: "ARM:LE:32:v7", compilerSpecId: "default" }; + return { + languageId: "ARM:LE:32:v7", + compilerSpecId: "default", + defaultAddressSpace: "ram", + }; case "arm64": - return { languageId: "AARCH64:LE:64:v8A", compilerSpecId: "default" }; + return { + languageId: "AARCH64:LE:64:v8A", + compilerSpecId: "default", + defaultAddressSpace: "ram", + }; } }; diff --git a/src/ghidra/GhidraRawBinaryLoadImage.test.ts b/src/ghidra/GhidraRawBinaryLoadImage.test.ts index 2fc11502e..d09b3a504 100644 --- a/src/ghidra/GhidraRawBinaryLoadImage.test.ts +++ b/src/ghidra/GhidraRawBinaryLoadImage.test.ts @@ -9,23 +9,161 @@ describe("raw-binary Ghidra language binding", () => { expect(ghidraRawBinaryLanguage("x86")).toEqual({ languageId: "x86:LE:32:default", compilerSpecId: "gcc", + defaultAddressSpace: "ram", }); expect(ghidraRawBinaryLanguage("x86_64")).toEqual({ languageId: "x86:LE:64:default", compilerSpecId: "gcc", + defaultAddressSpace: "ram", }); expect(ghidraRawBinaryLanguage("arm")).toEqual({ languageId: "ARM:LE:32:v7", compilerSpecId: "default", + defaultAddressSpace: "ram", }); expect(ghidraRawBinaryLanguage("arm64")).toEqual({ languageId: "AARCH64:LE:64:v8A", compilerSpecId: "default", + defaultAddressSpace: "ram", + }); + }); +}); + +describe("raw-binary mapped-block state", () => { + it.each(["RAM", "ram"])( + "requires the bound 6502 address space for %s observations", + (space) => { + const f = fixtureRawBinaryLoadImage(); + const target = { + ...f.target, + architecture: "6502" as const, + availableArchitectures: ["6502" as const], + entryPoint: "0x8001", + entryPointSource: "0x8001", + }; + f.observation.language_id = "6502:LE:16:default"; + f.observation.default_address_space = space; + f.observation.entry_points = [target.entryPoint]; + const mapping = f.observation.mappings[0]; + if (mapping === undefined) + throw new Error("Missing source mapping fixture"); + mapping.start = target.entryPoint; + mapping.end = "0x8004"; + mapping.address_space = space; + expect( + attestGhidraRawBinaryLoadImage(f.snapshot, target, f.observation), + ).toMatchObject({ + ok: true, + value: { + status: space === "RAM" ? "verified" : "mismatch", + checks: expect.arrayContaining([ + expect.objectContaining({ + name: "default_address_space", + expected: "RAM", + observed: space, + matched: space === "RAM", + }), + ]), + }, + }); + }, + ); + it.each([ + ["initialized", false], + ["loaded", false], + ["overlay", true], + ["address_space", "overlay"], + ] as const)( + "retains a mismatch when the file mapping declares %s=%s", + (property, value) => { + const f = fixtureRawBinaryLoadImage(); + const mapping = f.observation.mappings[0]; + if (mapping === undefined) + throw new Error("Missing source mapping fixture"); + Object.assign(mapping, { [property]: value }); + expect( + attestGhidraRawBinaryLoadImage(f.snapshot, f.target, f.observation), + ).toMatchObject({ + ok: true, + value: { + status: "mismatch", + observations: f.observation, + checks: expect.arrayContaining([ + expect.objectContaining({ + name: `mapping.${property}`, + matched: false, + observed: value, + }), + ]), + }, + }); + }, + ); + it("does not verify a foreign default address space even when its mapping agrees", () => { + const f = fixtureRawBinaryLoadImage(); + f.observation.default_address_space = "foreign"; + const mapping = f.observation.mappings[0]; + if (mapping === undefined) + throw new Error("Missing source mapping fixture"); + mapping.address_space = "foreign"; + expect( + attestGhidraRawBinaryLoadImage(f.snapshot, f.target, f.observation), + ).toMatchObject({ + ok: true, + value: { + status: "mismatch", + observations: f.observation, + checks: expect.arrayContaining([ + expect.objectContaining({ + name: "default_address_space", + matched: false, + observed: "foreign", + }), + ]), + }, }); }); }); describe("raw-binary independent load-image checks", () => { + it("retains uninitialized language memory without treating it as file-backed bytes", () => { + const f = fixtureRawBinaryLoadImage(); + const zeroPage = { + block: "ZERO_PAGE", + start: "0x0", + end: "0xff", + address_space: "ram", + initialized: false, + loaded: true, + overlay: false, + length: 256, + source_file_index: null, + file_offset: null, + sha256: null, + }; + f.observation.mappings.unshift(zeroPage); + expect( + attestGhidraRawBinaryLoadImage(f.snapshot, f.target, f.observation), + ).toMatchObject({ + ok: true, + value: { status: "verified", observations: f.observation }, + }); + zeroPage.initialized = true; + expect( + attestGhidraRawBinaryLoadImage(f.snapshot, f.target, f.observation), + ).toMatchObject({ + ok: true, + value: { + status: "mismatch", + checks: expect.arrayContaining([ + expect.objectContaining({ + name: "initialized_unbacked_mappings", + matched: false, + }), + ]), + }, + }); + }); it("verifies mapped bytes at the caller-supplied entry", () => { const f = fixtureRawBinaryLoadImage(); const result = attestGhidraRawBinaryLoadImage( diff --git a/src/ghidra/GhidraRiscvProfile.test.ts b/src/ghidra/GhidraRiscvProfile.test.ts index 881ba0078..718fe8d46 100644 --- a/src/ghidra/GhidraRiscvProfile.test.ts +++ b/src/ghidra/GhidraRiscvProfile.test.ts @@ -1,6 +1,6 @@ import { mkdir, mkdtemp, rm, writeFile } from "node:fs/promises"; import { tmpdir } from "node:os"; -import { join } from "node:path"; +import { join, parse, relative } from "node:path"; import { afterAll, beforeAll, describe, expect, it } from "vitest"; import { parseConfig } from "../config/parseConfig.js"; import type { @@ -92,7 +92,14 @@ const target = (change: Partial = {}): BinaryTarget => ({ let installDir = ""; beforeAll(async () => { - installDir = await mkdtemp(join(tmpdir(), "rea-ghidra-riscv-languages-")); + const root = + process.platform === "win32" ? join(process.cwd(), "build") : tmpdir(); + await mkdir(root, { recursive: true }); + installDir = await mkdtemp(join(root, "rea-ghidra-riscv-languages-")); + // The simulated Linux host needs a POSIX absolute path. A current-drive root + // path also reaches this real fixture on Windows without changing admission. + if (process.platform === "win32") + installDir = `/${relative(parse(installDir).root, installDir).replaceAll("\\", "/")}`; const languages = join(installDir, "Ghidra/Processors/RISCV/data/languages"); await mkdir(languages, { recursive: true }); await writeFile(join(languages, "riscv.ldefs"), RISCV_LDEFS); diff --git a/src/go/GoBinaryImage.ts b/src/go/GoBinaryImage.ts index 03d0866ab..4a845c197 100644 --- a/src/go/GoBinaryImage.ts +++ b/src/go/GoBinaryImage.ts @@ -8,6 +8,10 @@ import { type GoBinaryContainer, type GoFileMapping, } from "./GoBinaryContainer.js"; +import { + classifyFatContainerIdentity, + readFatContainerHeader, +} from "../domain/apple/machoContainer.js"; import { readGoElfImage } from "./GoElfImage.js"; import { readGoMachoImage } from "./GoMachoImage.js"; import { readGoPeImage } from "./GoPeImage.js"; @@ -187,11 +191,27 @@ const readContainer = (bytes: Buffer): GoBinaryContainer => { return readGoPeImage(bytes); if (bytes.length >= 4) { const magic = bytes.readUInt32BE(); - if ([0xcafebabe, 0xcafebabf, 0xbebafeca, 0xbfbafeca].includes(magic)) - throw new GoBinaryFormatFailure( - "unsupported", - "Universal/fat Mach-O containers require selecting a thin slice", - ); + if ([0xcafebabe, 0xcafebabf, 0xbebafeca, 0xbfbafeca].includes(magic)) { + const fatHeader = readFatContainerHeader(bytes, bytes.length); + if (fatHeader.state === "malformed") + throw new GoBinaryFormatFailure("malformed", fatHeader.reason); + const identity = + fatHeader.state === "parsed" + ? classifyFatContainerIdentity(fatHeader, (offset) => { + if (offset >= bytes.length) return undefined; + if (offset + 4 > bytes.length) return null; + return bytes.readUInt32BE(offset); + }) + : undefined; + // An invalid FAT-shaped table is not proof of a Mach-O container: JVM + // constant-pool bytes can share both the magic and a known CPU value. + const confirmedFatMachO = identity?.state === "mach-o"; + if (confirmedFatMachO) + throw new GoBinaryFormatFailure( + "unsupported", + "Universal/fat Mach-O containers require selecting a thin slice", + ); + } if ([0xfeedface, 0xfeedfacf, 0xcefaedfe, 0xcffaedfe].includes(magic)) return readGoMachoImage(bytes); } diff --git a/src/hopper/HopperAnalysisProfile.ts b/src/hopper/HopperAnalysisProfile.ts index 299862a97..a4848533f 100644 --- a/src/hopper/HopperAnalysisProfile.ts +++ b/src/hopper/HopperAnalysisProfile.ts @@ -123,6 +123,8 @@ export const hopperLoaderArgsForTarget = ( if (target.kind === "database") return ok([]); if (target.kind !== "executable") return err(new ProviderAdapterError("hopper", "resolve_analysis_profile")); + if (target.format === "raw-binary") + return err(new ProviderAdapterError("hopper", "resolve_analysis_profile")); const architecture = target.architecture; if (isElfMetadataArchitecture(architecture)) return err( @@ -155,7 +157,6 @@ export const hopperLoaderArgsForTarget = ( return ok(["-l", "WinPE", flag]); case "dos-mz": case "dos-com": - case "raw-binary": return err( new ProviderAdapterError("hopper", "resolve_analysis_profile"), ); diff --git a/src/hopper/HopperMachOImage.ts b/src/hopper/HopperMachOImage.ts index 20a53105c..ed81ea749 100644 --- a/src/hopper/HopperMachOImage.ts +++ b/src/hopper/HopperMachOImage.ts @@ -2,11 +2,7 @@ import { createHash } from "node:crypto"; import { open, type FileHandle } from "node:fs/promises"; import { z } from "zod"; -import { - isMachOFormat, - isElfMetadataArchitecture, - type BinaryTarget, -} from "../domain/binaryTargetTypes.js"; +import type { BinaryTarget } from "../domain/binaryTargetTypes.js"; import { BinaryTargetError } from "../domain/configurationErrors.js"; import { AnalysisCancelledError } from "../domain/analysisErrorCore.js"; import { @@ -48,10 +44,7 @@ export const resolveHopperMachOImage = async ( ), ); if (signal?.aborted) return err(new AnalysisCancelledError("open_binary")); - if ( - !isMachOFormat(target.format) || - isElfMetadataArchitecture(target.architecture) - ) + if (target.format !== "mach-o" && target.format !== "mach-o-universal") return invalid("Hopper FAT64 image preparation requires a Mach-O target"); let file: FileHandle; try { diff --git a/src/ida/IdaBusyHealth.test.ts b/src/ida/IdaBusyHealth.test.ts index 512cf78a6..b997b2591 100644 --- a/src/ida/IdaBusyHealth.test.ts +++ b/src/ida/IdaBusyHealth.test.ts @@ -18,6 +18,7 @@ const scripted = (answers: readonly JsonValue[]) => { connect: async () => [], serverInfo: () => null, close: async () => undefined, + detach: () => undefined, call: async () => { reads.push(reads.length); const answer = answers[Math.min(reads.length - 1, answers.length - 1)]; diff --git a/src/ida/IdaContainment.test.ts b/src/ida/IdaContainment.test.ts new file mode 100644 index 000000000..069388df8 --- /dev/null +++ b/src/ida/IdaContainment.test.ts @@ -0,0 +1,623 @@ +import { rm } from "node:fs/promises"; +import { afterEach, describe, expect, it } from "vitest"; +import type { JsonValue } from "../domain/jsonValue.js"; +import { + createIdaTarget, + RecordingIdaMcp, +} from "../../tests/fixtures/idaMcp.js"; +import { decodeIdaToolResult } from "./IdaMcpConnection.js"; +import { idaCapabilities } from "./IdaProviderCapabilities.js"; +import { IdaSessionClient } from "./IdaSessionClient.js"; + +const roots: string[] = []; +afterEach(async () => { + for (const root of roots.splice(0)) + await rm(root, { recursive: true, force: true }); +}); + +const segment = (name: string, start: string, end: string) => ({ + name, + start, + end, + size: `0x${(BigInt(end) - BigInt(start)).toString(16)}`, + permissions: "r--", +}); + +/** Adjacent .text and .data, a gap, then a segment above 2^53. */ +const SEGMENTS = [ + segment(".text", "0x1000", "0x2000"), + segment(".data", "0x2000", "0x3000"), + segment(".far", "0x5000", "0x6000"), + segment(".huge", "0x20000000000001", "0x20000000000002"), +]; + +/** The same segments in descending order, which a scan may not rely on. */ +const UNSORTED = [...SEGMENTS].reverse(); + +const survey = (segments: JsonValue = SEGMENTS) => ({ + metadata: { path: "/db/target.i64", module: "target", arch: "metapc" }, + statistics: { total_segments: Array.isArray(segments) ? segments.length : 0 }, + segments, + entrypoints: [], +}); + +const function_ = { addr: "0x1000", name: "main", size: "0x20" }; + +const MODERN_WITHOUT_SURVEY = [ + "idb_open", + "idb_list", + "idb_close", + "server_health", + "list_funcs", + "lookup_funcs", + "find_regex", + "decompile", + "disasm", + "callees", + "xrefs_to", +]; + +interface Upstream { + /** Answer for the lookup of the queried address; default is "Not a function". */ + readonly lookup?: (query: string) => JsonValue; + readonly survey?: JsonValue; + readonly tools?: readonly string[]; + readonly surveyFailure?: "error" | "preview"; + /** A distinct source alias of the admitted target, as for an app bundle. */ + readonly sourcePath?: string; + /** Report a different input from the second health read on. */ + readonly identityChangesAfterOperation?: boolean; +} + +const session = async ( + upstream: Upstream = {}, + mode: "headless" | "attached" = "headless", +) => { + const created = await createIdaTarget("x86_64"); + const { root } = created; + const target = + upstream.sourcePath === undefined + ? created.target + : { ...created.target, sourcePath: upstream.sourcePath }; + roots.push(root); + const producer = new RecordingIdaMcp(target, mode); + if (upstream.tools !== undefined) producer.tools = upstream.tools; + producer.beforeCall = async (name) => { + if (name !== "survey_binary") return; + if (upstream.surveyFailure === "error") + decodeIdaToolResult({ + content: [{ type: "text", text: "survey failed" }], + isError: true, + }); + if (upstream.surveyFailure === "preview") + decodeIdaToolResult({ + content: [{ type: "text", text: "preview" }], + _meta: { ida_mcp: { output_truncated: true } }, + }); + }; + let healthReads = 0; + producer.overrideCall = (name, args) => { + if (name === "server_health" && upstream.identityChangesAfterOperation) { + healthReads += 1; + if (healthReads === 2) + return { + status: "ok", + input_path: "/different/target.elf", + idb_path: "/different/target.elf.i64", + module: "target.elf", + imagebase: "0x1000", + auto_analysis_ready: true, + }; + } + if (name === "lookup_funcs") { + const queries = Array.isArray(args.queries) ? args.queries : []; + return queries.map((query) => + upstream.lookup === undefined + ? { query, fn: null, error: "Not a function" } + : upstream.lookup(String(query)), + ); + } + if (name === "survey_binary") return upstream.survey ?? survey(); + return undefined; + }; + const client = new IdaSessionClient( + { + command: "fixture", + args: [], + env: {}, + mode, + timeoutMs: 1000, + workspaceRoot: root, + }, + target, + producer, + ); + return { client, producer, target }; +}; + +const containing = async ( + client: IdaSessionClient, + address: string, + options?: Parameters[2], +) => client.execute("resolve_containing_procedure", { address }, options); + +const found = (query: string, fn: typeof function_) => ({ + query, + fn, + error: null, +}); + +describe("IDA containment capabilities", () => { + it("declares both operations for the headless profile only", () => { + const declared = (mode: "attached" | "headless", native = false) => + idaCapabilities(mode, false, native) + .map(({ operation }) => operation) + .filter( + (operation) => + operation === "resolve_containing_procedure" || + operation === "list_documents", + ); + + expect( + idaCapabilities("headless", false) + .filter( + ({ operation }) => + operation === "resolve_containing_procedure" || + operation === "list_documents", + ) + .map(({ available }) => available), + "both are declared available, not merely listed", + ).toEqual([true, true]); + expect(declared("headless")).toEqual([ + "resolve_containing_procedure", + "list_documents", + ]); + expect(declared("attached")).toEqual([]); + expect(declared("attached", true)).toEqual([]); + expect( + idaCapabilities("attached", false).map(({ operation }) => operation), + "attached profiles do not even declare them", + ).not.toContain("list_documents"); + }); + + it.each(["resolve_containing_procedure", "list_documents"] as const)( + "refuses %s on the legacy attached profile without calling upstream", + async (operation) => { + const { client, producer } = await session({}, "attached"); + try { + const result = await client.execute( + operation, + operation === "list_documents" ? {} : { address: "0x1000" }, + ); + + expect(result.ok).toBe(false); + if (!result.ok) + expect(result.error._tag).toBe("AnalysisCapabilityUnavailableError"); + expect(producer.calls.some(({ name }) => name === "lookup_funcs")).toBe( + false, + ); + } finally { + expect((await client.close()).ok).toBe(true); + } + }, + ); +}); + +describe("headless IDA containing procedure", () => { + it("returns the canonical entry for an interior address without reading segments", async () => { + const { client, producer } = await session({ + lookup: (query) => found(query, function_), + }); + try { + const result = await containing(client, "1ffe"); + + if (!result.ok) throw result.error; + expect(result.value.result).toEqual({ + query_address: "0x1ffe", + found: true, + procedure: { + address: "0x1000", + name: "main", + classification: null, + body: { + available: false, + reason: "IDA MCP does not report complete function body ranges.", + }, + }, + }); + expect( + producer.calls.filter(({ name }) => name === "lookup_funcs"), + "the normalized address is the only query", + ).toMatchObject([{ args: { queries: ["0x1ffe"] } }]); + expect( + producer.calls.some(({ name }) => name === "survey_binary"), + "a positive answer needs no segment facts", + ).toBe(false); + } finally { + expect((await client.close()).ok).toBe(true); + } + }); + + it.each([ + ["inside the first segment", "0x1000", "not_in_procedure"], + ["inside an adjacent segment", "0x2000", "not_in_procedure"], + [ + "at the exclusive end of the last adjacent segment", + "0x3000", + "outside_segments", + ], + ["in the gap between segments", "0x4000", "outside_segments"], + ["one below a segment start", "0xfff", "outside_segments"], + ["the last byte of a segment", "0x5fff", "not_in_procedure"], + ["above every segment", "0x6000", "outside_segments"], + [ + "inside a one-byte segment above 2^53", + "0x20000000000001", + "not_in_procedure", + ], + [ + "one below that segment, which a rounded number cannot tell apart", + "0x20000000000000", + "outside_segments", + ], + [ + "at the exclusive end of that segment", + "0x20000000000002", + "outside_segments", + ], + ] as const)( + "reports an address that is not a function %s as %s", + async (_description, address, reason) => { + const { client, producer } = await session(); + try { + const result = await containing(client, address); + + if (!result.ok) throw result.error; + expect(result.value.result).toEqual({ + query_address: address, + found: false, + procedure: null, + reason, + }); + expect( + producer.calls.find(({ name }) => name === "survey_binary")?.args, + "the minimal survey is the only segment source", + ).toMatchObject({ detail_level: "minimal" }); + expect( + producer.calls.filter(({ name }) => name === "survey_binary"), + ).toHaveLength(1); + } finally { + expect((await client.close()).ok).toBe(true); + } + }, + ); +}); + +describe("headless IDA containment failures", () => { + it.each([ + ["no segments field", { metadata: {}, statistics: {} }], + ["segments that are null", survey(null)], + ["an empty segment list", survey([])], + [ + "a row with a malformed address", + survey([ + { + name: ".x", + start: "0xzz", + end: "0x2000", + size: "0x1000", + permissions: "r--", + }, + ]), + ], + [ + "a reversed segment", + survey([ + { name: ".x", start: "0x2000", end: "0x1000", permissions: "r--" }, + ]), + ], + [ + "a size that disagrees with the bounds", + survey([{ ...segment(".x", "0x1000", "0x2000"), size: "0x10" }]), + ], + [ + "a total that disagrees with the rows", + { ...survey(), statistics: { total_segments: 9 } }, + ], + ["a malformed row after valid ones", survey([...SEGMENTS, { name: ".y" }])], + ] as const)( + "fails instead of guessing a reason when the survey has %s", + async (_description, reply) => { + const { client } = await session({ survey: reply as JsonValue }); + try { + const result = await containing(client, "0x4000"); + + expect(result.ok).toBe(false); + if (!result.ok) expect(result.error._tag).toBe("AnalysisOutputError"); + } finally { + expect((await client.close()).ok).toBe(true); + } + }, + ); + + it.each(["error", "preview"] as const)( + "fails when the survey tool reports %s", + async (surveyFailure) => { + const { client } = await session({ surveyFailure }); + try { + const result = await containing(client, "0x4000"); + + expect(result.ok).toBe(false); + } finally { + expect((await client.close()).ok).toBe(true); + } + }, + ); + + it("fails without a segment source when the server does not advertise the survey", async () => { + const { client, producer } = await session({ + tools: MODERN_WITHOUT_SURVEY, + }); + try { + const negative = await containing(client, "0x4000"); + + expect(negative.ok).toBe(false); + expect( + producer.calls.some(({ name }) => name === "survey_binary"), + "an unadvertised tool is never requested", + ).toBe(false); + } finally { + expect((await client.close()).ok).toBe(true); + } + }); + + it("still answers a positive lookup when the server does not advertise the survey", async () => { + const { client } = await session({ + lookup: (query) => found(query, function_), + tools: MODERN_WITHOUT_SURVEY, + }); + try { + const result = await containing(client, "0x1010"); + + expect(result.ok).toBe(true); + } finally { + expect((await client.close()).ok).toBe(true); + } + }); +}); + +describe("headless IDA containment lookup failures", () => { + it.each([ + ["an answer for another query", () => found("0x9999", function_)], + [ + "a function together with an error", + (query: string) => ({ query, fn: function_, error: "Not found" }), + ], + [ + "neither a function nor an error", + (query: string) => ({ query, fn: null, error: null }), + ], + [ + "an unknown error text", + (query: string) => ({ query, fn: null, error: "boom" }), + ], + [ + "a padded missing answer", + (query: string) => ({ query, fn: null, error: " Not a function" }), + ], + ] as const)( + "treats %s as a producer failure and reads no segments", + async (_description, lookup) => { + const { client, producer } = await session({ lookup }); + try { + const result = await containing(client, "0x4000"); + + expect(result.ok).toBe(false); + if (!result.ok) expect(result.error._tag).toBe("AnalysisProtocolError"); + expect( + producer.calls.some(({ name }) => name === "survey_binary"), + ).toBe(false); + } finally { + expect((await client.close()).ok).toBe(true); + } + }, + ); + + it('treats "Not found" for an address like any expected missing answer', async () => { + const { client } = await session({ + lookup: (query) => ({ query, fn: null, error: "Not found" }), + }); + try { + const result = await containing(client, "0x4000"); + + if (!result.ok) throw result.error; + expect(result.value.result).toMatchObject({ + found: false, + reason: "outside_segments", + }); + } finally { + expect((await client.close()).ok).toBe(true); + } + }); + + it("stops before the survey when the request is cancelled after the lookup", async () => { + const controller = new AbortController(); + const { client, producer } = await session(); + const inner = producer.beforeCall; + producer.beforeCall = async (name, args) => { + await inner?.(name, args); + if (name === "lookup_funcs") controller.abort(); + }; + try { + const result = await containing(client, "0x4000", { + signal: controller.signal, + }); + + expect(result.ok).toBe(false); + if (!result.ok) expect(result.error._tag).toBe("AnalysisCancelledError"); + expect( + producer.calls.some(({ name }) => name === "survey_binary"), + "nothing is dispatched after cancellation", + ).toBe(false); + } finally { + expect((await client.close()).ok).toBe(true); + } + }); + + it("does not depend on the order of the surveyed segments", async () => { + const { client } = await session({ survey: survey(UNSORTED) }); + try { + const inside = await containing(client, "0x1000"); + const gap = await containing(client, "0x4000"); + + if (!inside.ok) throw inside.error; + if (!gap.ok) throw gap.error; + expect(inside.value.result).toMatchObject({ reason: "not_in_procedure" }); + expect(gap.value.result).toMatchObject({ reason: "outside_segments" }); + } finally { + expect((await client.close()).ok).toBe(true); + } + }); + + it("retains the completed replies when the survey is rejected", async () => { + const { client } = await session({ + survey: survey([ + { name: ".x", start: "0x2000", end: "0x1000", permissions: "r--" }, + ]), + }); + try { + const result = await containing(client, "0x4000"); + + expect(result.ok).toBe(false); + if (!result.ok) + expect(result.error.partialObservation).toMatchObject({ + normalized_result: { + status: "partial", + observations: [ + expect.objectContaining({ tool: "lookup_funcs" }), + expect.objectContaining({ tool: "survey_binary" }), + ], + }, + }); + } finally { + expect((await client.close()).ok).toBe(true); + } + }); + + it("rejects an address that is not hexadecimal and a document that is not the admitted target", async () => { + const { client, producer } = await session(); + try { + const badAddress = await containing(client, "main"); + const badDocument = await client.execute("resolve_containing_procedure", { + address: "0x1000", + document: "/somewhere/else", + }); + + for (const result of [badAddress, badDocument]) { + expect(result.ok).toBe(false); + if (!result.ok) expect(result.error._tag).toBe("AnalysisInputError"); + } + expect( + producer.calls.some(({ name }) => name === "lookup_funcs"), + "invalid input never reaches upstream", + ).toBe(false); + } finally { + expect((await client.close()).ok).toBe(true); + } + }); +}); + +describe("headless IDA document identity", () => { + it("accepts an explicit admitted document and rejects an unknown one for the same operations", async () => { + const { client, target, producer } = await session(); + try { + const admitted = await client.execute("resolve_containing_procedure", { + address: "0x4000", + document: target.path, + }); + const unknownContainment = await client.execute( + "resolve_containing_procedure", + { address: "0x4000", document: "/somewhere/else" }, + ); + const unknownListing = await client.execute("list_documents", { + document: "/somewhere/else", + }); + + expect(admitted.ok, "the admitted program is accepted").toBe(true); + for (const rejected of [unknownContainment, unknownListing]) { + expect(rejected.ok).toBe(false); + if (!rejected.ok) + expect(rejected.error._tag).toBe("AnalysisInputError"); + } + expect( + producer.calls.filter(({ name }) => name === "lookup_funcs"), + "only the admitted request reached upstream", + ).toHaveLength(1); + } finally { + expect((await client.close()).ok).toBe(true); + } + }); + + it("lists the admitted path, not its source alias or the private copy", async () => { + const { client, target, producer } = await session({ + sourcePath: "/Applications/Fixture.app", + }); + try { + const result = await client.execute("list_documents", {}); + + if (!result.ok) throw result.error; + expect(result.value.result).toEqual([target.path]); + expect( + producer.calls.filter(({ name }) => name === "server_health"), + "the target identity is checked before and after the local answer", + ).toHaveLength(2); + } finally { + expect((await client.close()).ok).toBe(true); + } + }); + + it("fails when the target identity changes during the operation", async () => { + const { client } = await session({ identityChangesAfterOperation: true }); + try { + const result = await client.execute("list_documents", {}); + + expect(result.ok).toBe(false); + if (!result.ok) { + expect(result.error.message).toContain("identity changed"); + } + } finally { + expect((await client.close()).ok).toBe(true); + } + }); + + it("stops without dispatching when the request is already cancelled", async () => { + const controller = new AbortController(); + controller.abort(); + const { client, producer } = await session(); + try { + const result = await client.execute( + "list_documents", + {}, + { signal: controller.signal }, + ); + + expect(result.ok).toBe(false); + if (!result.ok) expect(result.error._tag).toBe("AnalysisCancelledError"); + expect(producer.calls).toEqual([]); + } finally { + expect((await client.close()).ok).toBe(true); + } + }); + + it("lists exactly the admitted program", async () => { + const { client, target } = await session(); + try { + const result = await client.execute("list_documents", {}); + + if (!result.ok) throw result.error; + expect(result.value.result).toEqual([target.path]); + } finally { + expect((await client.close()).ok).toBe(true); + } + }); +}); diff --git a/src/ida/IdaMcpConnection.ts b/src/ida/IdaMcpConnection.ts index ee7861389..d532a66e8 100644 --- a/src/ida/IdaMcpConnection.ts +++ b/src/ida/IdaMcpConnection.ts @@ -1,3 +1,5 @@ +import { ChildProcess } from "node:child_process"; +import { Socket } from "node:net"; import { snapshotEnvironment } from "../process/snapshotEnvironment.js"; import { Client, @@ -22,6 +24,12 @@ export interface IdaMcpConnection { readResource?(uri: string): Promise; serverInfo(): JsonValue; close(): Promise; + /** + * Keep a connection retained after failed cleanup from holding the host + * process open. A long-lived MCP server can still retry through it; a + * one-shot CLI exits, and the upstream proxy observes end of input. + */ + detach(): void; } const toolResultSchema = z.object({ @@ -205,5 +213,14 @@ export const createIdaMcpConnection = ( }, serverInfo: () => jsonValueSchema.parse(client.getServerVersion() ?? null), close: () => client.close().catch(transportFailure), + detach() { + if (!(transport instanceof StdioClientTransport)) return; + // The SDK exposes only the child's pid, not the handles it holds. + const child: unknown = Reflect.get(transport, "_process"); + if (!(child instanceof ChildProcess)) return; + child.unref(); + for (const stream of [child.stdin, child.stdout, child.stderr]) + if (stream instanceof Socket) stream.unref(); + }, }; }; diff --git a/src/ida/IdaNativeProtocol.test.ts b/src/ida/IdaNativeProtocol.test.ts index d59f598dd..dc95727f6 100644 --- a/src/ida/IdaNativeProtocol.test.ts +++ b/src/ida/IdaNativeProtocol.test.ts @@ -90,6 +90,7 @@ const fixture = async () => { }, serverInfo: () => producer.serverInfo(), close: () => producer.close(), + detach: () => producer.detach(), }; const client = new IdaSessionClient( { @@ -758,3 +759,33 @@ describe("native IDA malformed observations", () => { await client.close(); }); }); + +describe("native IDA attached operation admission", () => { + it("forwards the native producer's explicit reference cap flag", async () => { + const { client } = await fixture(); + const result = await client.execute("xrefs", { address: "0x1000" }); + + if (!result.ok) throw result.error; + expect(result.value.referencesTruncated).toBe(false); + await client.close(); + }); + + it("does not admit the headless-only containment and document operations", async () => { + const { client, producer } = await fixture(); + for (const [operation, parameters] of [ + ["resolve_containing_procedure", { address: "0x1000" }], + ["list_documents", {}], + ] as const) { + const result = await client.execute(operation, parameters); + + expect(result.ok).toBe(false); + if (!result.ok) + expect(result.error._tag).toBe("AnalysisCapabilityUnavailableError"); + } + expect( + producer.calls.some(({ name }) => name === "lookup_funcs"), + "nothing is dispatched upstream", + ).toBe(false); + await client.close(); + }); +}); diff --git a/src/ida/IdaOperationRunner.ts b/src/ida/IdaOperationRunner.ts index c30ec81bc..25a49c268 100644 --- a/src/ida/IdaOperationRunner.ts +++ b/src/ida/IdaOperationRunner.ts @@ -45,6 +45,7 @@ import { modernStringsSchema, modernXrefsSchema, nextOffset, + surveySegmentsSchema, type IdaDisassembly, type IdaFunction, } from "./IdaProtocolValues.js"; @@ -86,6 +87,8 @@ export class IdaOperationRunner { readonly locations: EvidenceLocation[] = []; readonly limitations: string[] = []; relationshipCoverage: RelationshipCoverage | undefined; + /** The upstream cap flag of the last `xrefs_to` read, when the producer reported one. */ + referencesTruncated: boolean | undefined; constructor( private readonly connection: IdaMcpConnection, private readonly protocol: "legacy" | "modern" | "native", @@ -97,6 +100,10 @@ export class IdaOperationRunner { readonly x86Only?: boolean; /** False when the upstream server does not advertise `callgraph`. */ readonly callGraphAvailable?: boolean; + /** False when the upstream server does not advertise `survey_binary`. */ + readonly surveyAvailable?: boolean; + /** The program identity REA admitted, reported by `list_documents`. */ + readonly admittedDocument?: string; } = {}, ) {} @@ -177,41 +184,89 @@ export class IdaOperationRunner { return entry; } - private lookupFunction( + /** Parse one lookup row: a function, an expected "missing" answer, or a producer failure. */ + private lookupOutcome( query: string, item: z.infer[number] | undefined, - requestedBy?: string, - ): IdaFunction { + ): { readonly fn: IdaFunction } | { readonly missing: string } { if (item?.query !== query) throw new AnalysisProtocolError( `IDA function lookup answered for a different query than ${JSON.stringify(query)}.`, ); if ( - requestedBy !== undefined && item.fn === null && item.error !== null && MISSING_PROCEDURE_ANSWERS.has(item.error) ) - throw new AnalysisInputError( - requestedBy, - { - cause: new AnalysisProtocolError( - `IDA function lookup failed for ${query}: ${item.error}`, - ), - }, - [ - { - path: [], - reason: "invalid_value", - message: `IDA could not resolve procedure ${JSON.stringify(query)}: ${item.error}.`, - }, - ], - ); + return { missing: item.error }; if (item.fn === null || item.error !== null) throw new AnalysisProtocolError( `IDA function lookup failed for ${query}: ${item.error ?? "missing function"}`, ); - return item.fn; + return { fn: item.fn }; + } + + private lookupFunction( + query: string, + item: z.infer[number] | undefined, + requestedBy?: string, + ): IdaFunction { + const outcome = this.lookupOutcome(query, item); + if ("fn" in outcome) return outcome.fn; + if (requestedBy === undefined) + throw new AnalysisProtocolError( + `IDA function lookup failed for ${query}: ${outcome.missing}`, + ); + throw new AnalysisInputError( + requestedBy, + { + cause: new AnalysisProtocolError( + `IDA function lookup failed for ${query}: ${outcome.missing}`, + ), + }, + [ + { + path: [], + reason: "invalid_value", + message: `IDA could not resolve procedure ${JSON.stringify(query)}: ${outcome.missing}.`, + }, + ], + ); + } + + /** The containing function of an address, or why none contains it. */ + private async containingProcedure(address: string): Promise { + const item = modernLookupSchema.parse( + await this.call("lookup_funcs", { queries: [address] }), + )[0]; + const outcome = this.lookupOutcome(address, item); + if ("fn" in outcome) + return { + query_address: address, + found: true, + procedure: identity(outcome.fn), + }; + return { + query_address: address, + found: false, + procedure: null, + reason: (await this.addressInSegment(address)) + ? "not_in_procedure" + : "outside_segments", + }; + } + + /** Segment membership from the database-scoped minimal survey; a missing fact is a failure. */ + private async addressInSegment(address: string): Promise { + if (this.options.surveyAvailable === false) + throw new AnalysisProtocolError( + "The upstream server does not advertise survey_binary, so segment membership cannot be established.", + ); + const { segments } = surveySegmentsSchema.parse( + await this.call("survey_binary", { detail_level: "minimal" }), + ); + const value = BigInt(address); + return segments.some(({ start, end }) => start <= value && value < end); } /** True only for modern callee reads on targets not proven x86-only; x86 reads never reach the graph code. */ @@ -390,6 +445,7 @@ export class IdaOperationRunner { throw new AnalysisProtocolError( `IDA xrefs failed: ${page?.error ?? "missing references"}`, ); + if (page.more !== undefined) this.referencesTruncated = page.more; if (page.more === true) this.limitations.push( "Upstream xrefs_to capped results at 1000 references; additional references remain unknown.", @@ -536,6 +592,16 @@ export class IdaOperationRunner { address === idaAddressSchema.parse(parameters.address), ); } + if (operation === "list_documents") { + if (this.options.admittedDocument === undefined) + throw new AnalysisProtocolError("The admitted IDA program is unknown."); + return [this.options.admittedDocument]; + } + if (operation === "resolve_containing_procedure") { + const queried = idaAddressSchema.parse(parameters.address); + this.locations.push({ kind: "address", address: queried }); + return this.containingProcedure(queried); + } if (operation === "xrefs") { const inspectedAddress = idaAddressSchema.parse(parameters.address); this.locations.push({ kind: "address", address: inspectedAddress }); diff --git a/src/ida/IdaProtocolValues.ts b/src/ida/IdaProtocolValues.ts index 93ddbef88..f18e85ee0 100644 --- a/src/ida/IdaProtocolValues.ts +++ b/src/ida/IdaProtocolValues.ts @@ -218,6 +218,14 @@ export const openDatabaseSchema = z .passthrough(), }) .passthrough(); +/** Only an explicit completed rejection preserves legacy no-worker cleanup. */ +export const rejectedOpenDatabaseSchema = z + .object({ + error: z.string().min(1), + success: z.never().optional(), + }) + .passthrough(); + export const databaseListSchema = z .object({ sessions: z.array( @@ -247,3 +255,53 @@ export const absentDatabaseCloseSchema = (sessionId: string) => z .object({ error: z.literal(`Session not found: ${sessionId}`) }) .passthrough(); + +// Piped, so a malformed coordinate is a schema issue and never reaches BigInt. +const hexCoordinate = z + .string() + .regex(/^0x[0-9a-f]+$/iu) + .pipe(z.transform((value) => BigInt(value))); + +/** + * The minimal `survey_binary` reply, reduced to validated segment intervals + * (exclusive ends). Any malformed, reversed or inconsistent fact rejects the + * whole reply, because a membership answer needs every segment. + */ +export const surveySegmentsSchema = z + .object({ + statistics: z + .object({ total_segments: z.number().int().nonnegative().optional() }) + .passthrough() + .optional(), + segments: z + .array( + z + .object({ + start: hexCoordinate, + end: hexCoordinate, + size: hexCoordinate.optional(), + }) + .passthrough(), + ) + .min(1), + }) + .passthrough() + .superRefine(({ statistics, segments }, context) => { + if ( + statistics?.total_segments !== undefined && + statistics.total_segments !== segments.length + ) + context.addIssue({ + code: "custom", + message: "survey segment count disagrees with its statistics", + }); + for (const { start, end, size } of segments) { + // A malformed row has already reported its own issue; refinements still run. + if (typeof start !== "bigint" || typeof end !== "bigint") continue; + if (start >= end || (size !== undefined && size !== end - start)) + context.addIssue({ + code: "custom", + message: "survey segment bounds are reversed or inconsistent", + }); + } + }); diff --git a/src/ida/IdaProviderCapabilities.ts b/src/ida/IdaProviderCapabilities.ts index d52c949e0..0ccc016d2 100644 --- a/src/ida/IdaProviderCapabilities.ts +++ b/src/ida/IdaProviderCapabilities.ts @@ -24,9 +24,23 @@ export const IDA_OPERATIONS = Object.freeze([ "read_function_instructions", "xrefs", "analyze_function", + "resolve_containing_procedure", + "list_documents", ] as const satisfies readonly Exclude[]); export type IdaOperation = (typeof IDA_OPERATIONS)[number]; +/** Operations only the modern headless profile implements. */ +const HEADLESS_ONLY_OPERATIONS: ReadonlySet = new Set([ + "resolve_containing_procedure", + "list_documents", +]); + +/** Whether the configured profile admits an operation. */ +export const idaProfileAdmits = ( + operation: IdaOperation, + mode: "attached" | "headless", +): boolean => mode === "headless" || !HEADLESS_ONLY_OPERATIONS.has(operation); + /** Shared limitations of observations from an externally mutable analysis database. */ export const IDA_LIMITATIONS = Object.freeze([ "Provider version identifies the REA IDA adapter. IDA engine and upstream distribution versions are unknown unless separately observed; MCP serverInfo is reported independently.", @@ -58,34 +72,37 @@ export const idaCapabilities = ( stdio: boolean, native = false, ): readonly CapabilityDescriptor[] => - IDA_OPERATIONS.map((operation) => ({ - provider: IDA_PROVIDER_IDENTITY, - operation, - ...((mode === "headless" || native) && operation === "procedure_callers" - ? ({ - available: false, - reason: native - ? "The native upstream profile reports code references without proving direct call edges." - : "The modern upstream profile reports code references without proving direct call edges.", - } as const) - : ({ available: true, reason: null } as const)), - cachePolicy: "live", - effects: { - mutatesArtifact: false, - launchesProcess: stdio || mode === "headless", - mayShowUi: - mode === "attached" && - (native - ? operation === "procedure_pseudo_code" || - operation === "analyze_function" - : ATTACHED_GUI_NAVIGATION_OPERATIONS.has(operation)), - mayAccessNetwork: true, - mayWriteFilesystem: stdio || mode === "headless", - changesPermissions: false, - requiresRoot: false, - }, - limitations: - native && (operation === "list_strings" || operation === "search_strings") - ? [...IDA_LIMITATIONS, IDA_NATIVE_STRING_LIMITATION] - : IDA_LIMITATIONS, - })); + IDA_OPERATIONS.filter((operation) => idaProfileAdmits(operation, mode)).map( + (operation) => ({ + provider: IDA_PROVIDER_IDENTITY, + operation, + ...((mode === "headless" || native) && operation === "procedure_callers" + ? ({ + available: false, + reason: native + ? "The native upstream profile reports code references without proving direct call edges." + : "The modern upstream profile reports code references without proving direct call edges.", + } as const) + : ({ available: true, reason: null } as const)), + cachePolicy: "live", + effects: { + mutatesArtifact: false, + launchesProcess: stdio || mode === "headless", + mayShowUi: + mode === "attached" && + (native + ? operation === "procedure_pseudo_code" || + operation === "analyze_function" + : ATTACHED_GUI_NAVIGATION_OPERATIONS.has(operation)), + mayAccessNetwork: true, + mayWriteFilesystem: stdio || mode === "headless", + changesPermissions: false, + requiresRoot: false, + }, + limitations: + native && + (operation === "list_strings" || operation === "search_strings") + ? [...IDA_LIMITATIONS, IDA_NATIVE_STRING_LIMITATION] + : IDA_LIMITATIONS, + }), + ); diff --git a/src/ida/IdaReferenceCoverage.test.ts b/src/ida/IdaReferenceCoverage.test.ts new file mode 100644 index 000000000..930c3a88a --- /dev/null +++ b/src/ida/IdaReferenceCoverage.test.ts @@ -0,0 +1,96 @@ +import { rm } from "node:fs/promises"; +import { afterEach, describe, expect, it } from "vitest"; +import { + createIdaTarget, + RecordingIdaMcp, +} from "../../tests/fixtures/idaMcp.js"; +import { + callGraphSession as session, + removeCallGraphSessions, +} from "../../tests/fixtures/idaCallGraph.js"; + +import { IdaSessionClient } from "./IdaSessionClient.js"; + +const attachedRoots: string[] = []; +afterEach(async () => { + await removeCallGraphSessions(); + for (const root of attachedRoots.splice(0)) + await rm(root, { recursive: true, force: true }); +}); + +const xrefs = async (more: boolean | undefined) => { + const { client, producer } = await session({ callees: [] }, "x86_64"); + const inner = producer.overrideCall; + producer.overrideCall = (name, args) => + name === "xrefs_to" + ? [{ xrefs: [], ...(more === undefined ? {} : { more }) }] + : inner?.(name, args); + try { + const result = await client.execute("xrefs", { address: "0x1000" }); + if (!result.ok) throw result.error; + return result.value; + } finally { + expect((await client.close()).ok).toBe(true); + } +}; + +describe("headless IDA reference coverage", () => { + it("reports a capped xrefs read as truncated", async () => { + const execution = await xrefs(true); + + expect(execution.referencesTruncated).toBe(true); + expect( + execution.limitations, + "the diagnostic limitation is still reported", + ).toContain( + "Upstream xrefs_to capped results at 1000 references; additional references remain unknown.", + ); + }); + + it("reports an explicitly uncapped xrefs read as not truncated", async () => { + expect((await xrefs(false)).referencesTruncated).toBe(false); + }); + + it("reports nothing when the producer does not say whether the read was capped", async () => { + expect((await xrefs(undefined)).referencesTruncated).toBeUndefined(); + }); + + it("does not report reference coverage for other operations", async () => { + const { client } = await session({ callees: [] }, "x86_64"); + try { + const result = await client.execute("procedure_callees", { + procedure: "main", + }); + + if (!result.ok) throw result.error; + expect(result.value.referencesTruncated).toBeUndefined(); + } finally { + expect((await client.close()).ok).toBe(true); + } + }); + + it("reports nothing for the legacy profile, whose producer has no cap flag", async () => { + const { root, target } = await createIdaTarget("x86_64"); + attachedRoots.push(root); + const client = new IdaSessionClient( + { + command: "fixture", + args: [], + env: {}, + mode: "attached", + timeoutMs: 1000, + workspaceRoot: root, + }, + target, + new RecordingIdaMcp(target, "attached"), + ); + try { + const result = await client.execute("xrefs", { address: "0x1000" }); + + if (!result.ok) throw result.error; + expect(result.value.referencesTruncated).toBeUndefined(); + } finally { + expect((await client.close()).ok).toBe(true); + } + }); +}); diff --git a/src/ida/IdaSessionClient.test.ts b/src/ida/IdaSessionClient.test.ts index e43f18cf7..7fa366bce 100644 --- a/src/ida/IdaSessionClient.test.ts +++ b/src/ida/IdaSessionClient.test.ts @@ -16,6 +16,18 @@ afterEach(async () => { for (const root of roots.splice(0)) await rm(root, { recursive: true, force: true }); }); +/** A failed close whose diagnostic names exactly these unverified sessions. */ +const reportsSessions = (...sessions: readonly object[]) => ({ + error: { + diagnostics: { upstream_sessions: { state: "unverified", sessions } }, + }, +}); +const LIVE_PEER = { + session_id: "unrelated-live-peer", + backend: "worker", + owned: false, +} as const; + const fixture = async ( mode: "attached" | "headless" = "attached", timeoutMs = 1000, @@ -515,7 +527,7 @@ describe("headless IDA transient busy health", () => { }); it.each([ - ["free-form text", "token=abc123 /private/home/user secret"], + ["free-form text", "token=abc123 /private secret"], ["an over-long name", "x".repeat(200)], ])( "reports %s from the worker as unknown instead of echoing it", @@ -792,10 +804,19 @@ describe("headless IDA session ownership and cleanup", () => { it("retains the private workspace and reports cleanup failure when worker release is unverified", async () => { const { producer, client, root } = await fixture("headless"); await client.execute("health", {}); + const sessionId = producer.database; producer.failClose = true; const result = await client.close(); expect(result.ok).toBe(false); if (!result.ok) expect(result.error.cleanupIncomplete).toBe(true); + expect(result).toMatchObject( + reportsSessions({ + session_id: sessionId, + backend: "worker", + owned: true, + input_path: producer.inputPath, + }), + ); expect( (await readdir(root)).some((name) => name.startsWith("rea-ida-")), ).toBe(true); @@ -843,10 +864,20 @@ describe("headless IDA reaped worker cleanup", () => { const result = await client.close(); expect(result.ok).toBe(false); if (!result.ok) expect(result.error.cleanupIncomplete).toBe(true); + // The released session must not be named as the remaining worker. + expect(result).toMatchObject(reportsSessions(LIVE_PEER)); expect(producer.closes).toBe(0); expect( (await readdir(root)).some((name) => name.startsWith("rea-ida-")), ).toBe(true); + + // A retry whose inventory is unavailable keeps the observed blocker. + producer.beforeCall = async (name) => { + if (name === "idb_list") throw new Error("inventory unavailable"); + }; + const retried = await client.close(); + expect(retried.ok).toBe(false); + expect(retried).toMatchObject(reportsSessions(LIVE_PEER)); }); it("retries independent inventory verification after an absent-session response", async () => { @@ -899,6 +930,305 @@ describe("headless IDA reaped worker cleanup", () => { }); }); +/** A rejected open response, with helpers for separately inventoried workers. */ +const rejectedOpenFixture = async () => { + const { producer, client } = await fixture("headless"); + producer.overrideCall = (name) => + name === "idb_open" ? { error: "open did not complete" } : undefined; + expect((await client.execute("health", {})).ok).toBe(false); + const open = producer.calls.find(({ name }) => name === "idb_open")?.args; + const entry = (session_id: string, owned: boolean) => ({ + session_id, + input_path: String(open?.input_path), + owned, + backend: "worker", + is_active: true, + }); + return { + producer, + client, + requested: String(open?.preferred_session_id), + entry, + }; +}; + +describe("headless IDA cleanup diagnostics", () => { + it("does not name a worker whose close was confirmed when inventory then fails", async () => { + const { producer, client } = await fixture("headless"); + expect((await client.execute("health", {})).ok).toBe(true); + producer.beforeCall = async (name) => { + if (name === "idb_list") throw new Error("inventory unavailable"); + }; + + const result = await client.close(); + expect(result.ok).toBe(false); + expect(producer.calls.map(({ name }) => name)).toContain("idb_close"); + expect(result).toMatchObject(reportsSessions()); + }); + + it("retains peers and confirmed ownership from successful startup when close fails", async () => { + const { producer, client } = await fixture("headless"); + producer.overrideCall = (name) => { + if (name === "idb_list") + return { + sessions: [ + { + session_id: String(producer.database), + input_path: producer.inputPath, + owned: true, + backend: "worker", + is_active: true, + }, + { + session_id: "startup-peer", + input_path: producer.inputPath, + owned: false, + backend: "worker", + is_active: false, + }, + ], + }; + if (name === "idb_close") return { error: "close rejected" }; + return undefined; + }; + expect((await client.execute("health", {})).ok).toBe(true); + const result = await client.close(); + expect(result).toMatchObject( + reportsSessions( + { + session_id: producer.database, + backend: "worker", + owned: true, + input_path: producer.inputPath, + }, + { + session_id: "startup-peer", + backend: "worker", + owned: false, + input_path: producer.inputPath, + }, + ), + ); + }); + + it("preserves an observed conflicting input binding after close", async () => { + const { producer, client } = await fixture("headless"); + expect((await client.execute("health", {})).ok).toBe(true); + const requested = String(producer.database); + producer.overrideCall = (name) => + name === "idb_list" + ? { + sessions: [ + { + session_id: requested, + input_path: "/different/input", + owned: false, + backend: "worker", + is_active: true, + }, + ], + } + : undefined; + const result = await client.close(); + expect(result).toMatchObject( + reportsSessions({ + session_id: requested, + backend: "worker", + owned: false, + input_path: "/different/input", + }), + ); + }); + + it("keeps peers seen beside the requested worker when its close fails", async () => { + const { producer, client, requested, entry } = await rejectedOpenFixture(); + producer.overrideCall = (name) => { + if (name === "idb_list") + return { + sessions: [ + entry(requested, true), + entry("unrelated-live-peer", false), + ], + }; + if (name === "idb_close") return { error: "close rejected" }; + return undefined; + }; + + const result = await client.close(); + expect(result.ok).toBe(false); + expect(result).toMatchObject( + reportsSessions( + { session_id: requested, backend: "worker", owned: true }, + LIVE_PEER, + ), + ); + }); +}); + +describe("headless IDA cleanup diagnostics for observed peers", () => { + it("drops the requested worker from blockers once its close is confirmed", async () => { + const { producer, client, requested, entry } = await rejectedOpenFixture(); + let inventories = 0; + producer.beforeCall = async (name) => { + if (name === "idb_list" && ++inventories > 1) + throw new Error("inventory unavailable"); + }; + producer.overrideCall = (name) => { + if (name === "idb_list") + return { + sessions: [ + entry(requested, true), + entry("unrelated-live-peer", false), + ], + }; + if (name === "idb_close") + return { + success: true, + session_id: requested, + saved: false, + owned: true, + backend: "worker", + }; + return undefined; + }; + + const result = await client.close(); + expect(result.ok).toBe(false); + expect(result).toMatchObject(reportsSessions(LIVE_PEER)); + }); + + it("reports a blocking GUI session by its backend", async () => { + const { producer, client } = await fixture("headless"); + expect((await client.execute("health", {})).ok).toBe(true); + const sessionId = producer.database; + producer.overrideCall = (name) => { + if (name === "idb_close") + return { error: `Session not found: ${sessionId}` }; + if (name === "idb_list") + return { + sessions: [ + { + session_id: "", + input_path: producer.inputPath, + owned: false, + backend: "gui", + is_active: true, + }, + ], + }; + return undefined; + }; + + const result = await client.close(); + expect(result.ok).toBe(false); + expect(result).toMatchObject( + reportsSessions({ session_id: "", backend: "gui", owned: false }), + ); + }); +}); + +/** A completed open response is not proof that its private Worker was ever released. */ +describe("headless IDA unresolved successful-open ownership", () => { + it.each(["matching", "foreign-input", "malformed"] as const)( + "retains the private input after %s success with incomplete initial discovery", + async (kind) => { + const { producer, client, root } = await fixture("headless"); + let hidden = true; + let ownedId: string | undefined; + let privateInput: string | undefined; + producer.beforeCall = async (name, args) => { + if (name !== "idb_open") return; + ownedId = z.string().parse(args.preferred_session_id); + privateInput = z.string().parse(args.input_path); + producer.database = ownedId; + producer.inputPath = privateInput; + }; + producer.overrideCall = (name) => { + if (name === "idb_open") { + const originalId = z.string().parse(ownedId); + const originalInput = z.string().parse(privateInput); + if (kind === "foreign-input") + return { + success: true, + session: { + session_id: "foreign-session", + input_path: "/foreign/input", + }, + }; + if (kind === "malformed") + return { success: true, session: { session_id: originalId } }; + return { + success: true, + session: { session_id: originalId, input_path: originalInput }, + }; + } + if (name === "idb_list" && hidden) return { sessions: [] }; + return undefined; + }; + + expect((await client.execute("health", {})).ok).toBe(false); + const incomplete = await client.close(); + expect(incomplete.ok).toBe(false); + if (!incomplete.ok) expect(incomplete.error.cleanupIncomplete).toBe(true); + expect(producer.closes).toBe(0); + expect(producer.database).toBe(ownedId); + expect( + (await readdir(root)).some((name) => name.startsWith("rea-ida-")), + ).toBe(true); + expect(producer.calls.some(({ name }) => name === "idb_close")).toBe( + false, + ); + + // A later positive private ownership record permits ordinary save=false + // cleanup, without ever adopting the foreign open-response identity. + hidden = false; + expect((await client.close()).ok).toBe(true); + expect(producer.calls.filter(({ name }) => name === "idb_close")).toEqual( + [{ name: "idb_close", args: { database: ownedId, save: false } }], + ); + expect(producer.closes).toBe(1); + expect(await readdir(root)).toEqual(["target.elf"]); + }, + ); + + it("preserves cleanup after a completed, explicit idb_open rejection", async () => { + const { producer, client, root } = await fixture("headless"); + producer.overrideCall = (name) => + name === "idb_open" + ? { error: "Input was rejected before creating a session" } + : undefined; + + expect((await client.execute("health", {})).ok).toBe(false); + expect((await client.close()).ok).toBe(true); + expect(producer.calls.some(({ name }) => name === "idb_close")).toBe(false); + expect(producer.closes).toBe(1); + expect(await readdir(root)).toEqual(["target.elf"]); + }); + + it("retains cleanup when a rejected open leaves the requested ID on another input", async () => { + const { producer, client, requested } = await rejectedOpenFixture(); + producer.overrideCall = (name) => + name === "idb_list" + ? { + sessions: [ + { + session_id: requested, + input_path: "/different/input", + owned: false, + backend: "worker", + is_active: true, + }, + ], + } + : undefined; + + const result = await client.close(); + expect(result.ok).toBe(false); + if (!result.ok) expect(result.error.cleanupIncomplete).toBe(true); + expect(producer.closes).toBe(0); + }); +}); + describe("headless IDA cleanup verification", () => { it("retries failed worker release through the retained connection before removing its workspace", async () => { const { producer, client, root } = await fixture("headless"); @@ -943,23 +1273,6 @@ describe("headless IDA cleanup verification", () => { ).toHaveLength(1); expect(await readdir(root)).toEqual(["target.elf"]); }); - it("never closes an unrelated database when startup returns a different input identity", async () => { - const { producer, client, root } = await fixture("headless"); - producer.overrideCall = (name) => - name === "idb_open" - ? { - success: true, - session: { - session_id: "user-database", - input_path: "/unrelated/input", - }, - } - : undefined; - expect((await client.execute("health", {})).ok).toBe(false); - expect((await client.close()).ok).toBe(true); - expect(producer.calls.some(({ name }) => name === "idb_close")).toBe(false); - expect(await readdir(root)).toEqual(["target.elf"]); - }); it("releases a privately owned worker after malformed open output using verified session inventory", async () => { const { producer, client, root } = await fixture("headless"); producer.beforeCall = async (name, args) => { diff --git a/src/ida/IdaSessionClient.ts b/src/ida/IdaSessionClient.ts index 7d3ec066c..dc0ae5d6f 100644 --- a/src/ida/IdaSessionClient.ts +++ b/src/ida/IdaSessionClient.ts @@ -33,6 +33,7 @@ import { IDA_LIMITATIONS, IDA_OPERATIONS, IDA_PROVIDER_IDENTITY, + idaProfileAdmits, type IdaOperation, } from "./IdaProviderCapabilities.js"; import { @@ -42,6 +43,7 @@ import { legacyMetadataSchema, modernHealthSchema, openDatabaseSchema, + rejectedOpenDatabaseSchema, } from "./IdaProtocolValues.js"; import { idaFileDigest, @@ -91,11 +93,25 @@ interface IdaOperationFailureContext { parameters: Readonly>; } +/** Upstream session ids of inventory entries, omitting unreported ones. */ +interface UpstreamSession { + readonly session_id: string; + readonly backend: "worker" | "gui"; + /** Upstream ownership; null when no inventory has reported it. */ + readonly owned: boolean | null; + /** Last inventoried input binding; null until upstream reports it. */ + readonly input_path: string | null; +} + /** A failed cleanup retains its private workspace and reports the remaining resource. */ class IdaCleanupError extends ProviderAdapterError { override readonly cleanupIncomplete = true; override readonly cleanupResources: readonly string[]; - constructor(path: string, cause: unknown) { + constructor( + path: string, + cause: unknown, + sessions?: readonly UpstreamSession[], + ) { super("ida", "close", { cause, diagnostics: { @@ -103,6 +119,23 @@ class IdaCleanupError extends ProviderAdapterError { "IDA cleanup could not be verified; the remaining connection or private workspace is retained for retry.", failure_reason: cause instanceof Error ? cause.message : String(cause), workspace: path, + ...(sessions === undefined + ? {} + : { + upstream_sessions: { + state: "unverified", + sessions: sessions.map( + ({ session_id, backend, owned, input_path }) => ({ + session_id, + backend, + owned, + input_path, + }), + ), + reason: + "Release of upstream sessions matching the requested session ID or private input was not confirmed; input_path preserves the last observed binding and is null when no inventory has reported it. A worker outlives its supervisor and may keep running until it finishes and reaches its idle timeout; exiting REA does not stop it. A gui session belongs to an IDA instance REA does not control. An empty session_id means upstream reported none; owned is null when no inventory has reported ownership; an empty list means the requested worker's close was confirmed but no inventory has identified any remaining session.", + }, + }), }, }); this.cleanupResources = [path]; @@ -121,7 +154,10 @@ export class IdaSessionClient implements AnalysisClient { #databaseCloseCompleted = false; #openAttempted = false; #openCompleted = false; + #blockingSessions: readonly UpstreamSession[] | undefined; + #openRejected = false; #callGraphAdvertised = false; + #surveyAdvertised = false; #initialIdentity: string | undefined; #metadata: JsonValue = null; #lifecycle: JsonValue[] = []; @@ -194,6 +230,17 @@ export class IdaSessionClient implements AnalysisClient { "This integration admits read-only analysis operations only.", ), ); + if ( + operation !== "health" && + !idaProfileAdmits(operation, this.config.mode) + ) + return err( + new AnalysisCapabilityUnavailableError( + "ida", + operation, + "This operation is available for the modern headless profile only.", + ), + ); if ( operation === "procedure_callers" && idaProtocol(this.config) !== "legacy" @@ -269,6 +316,9 @@ export class IdaSessionClient implements AnalysisClient { ...(runner.relationshipCoverage === undefined ? {} : { relationshipCoverage: runner.relationshipCoverage }), + ...(runner.referencesTruncated !== undefined + ? { referencesTruncated: runner.referencesTruncated } + : {}), locations: [ { kind: "artifact-path", path: this.target.path }, ...runner.locations, @@ -293,6 +343,8 @@ export class IdaSessionClient implements AnalysisClient { selectorOrigin: options?.selectorOrigin, x86Only: isX86OnlyTarget(this.target), callGraphAvailable: this.#callGraphAdvertised, + surveyAvailable: this.#surveyAdvertised, + admittedDocument: this.target.path, }, ); } @@ -423,6 +475,7 @@ export class IdaSessionClient implements AnalysisClient { async #initialize(): Promise { const tools = new Set(await this.connection.connect()); this.#callGraphAdvertised = tools.has("callgraph"); + this.#surveyAdvertised = tools.has("survey_binary"); const expected = this.config.protocol === "native" ? nativeTools @@ -450,6 +503,9 @@ export class IdaSessionClient implements AnalysisClient { preferred_session_id: this.#workspace.sessionId, }); this.#openCompleted = true; + // A completed but malformed/positive open may still own a live Worker. + // Only an explicit producer rejection keeps the old no-worker branch. + this.#openRejected = rejectedOpenDatabaseSchema.safeParse(raw).success; this.#lifecycle.push(raw); const opened = openDatabaseSchema.parse(raw); if ( @@ -466,6 +522,14 @@ export class IdaSessionClient implements AnalysisClient { await this.connection.call("idb_list", {}), ); this.#lifecycle.push(jsonValueSchema.parse(inventory)); + const workspace = this.#workspace; + this.#recordBlockingSessions( + inventory.sessions.filter( + (entry) => + entry.session_id === workspace.sessionId || + sameIdaHostPath(entry.input_path, workspace.inputPath), + ), + ); const entry = inventory.sessions.find( ({ session_id }) => session_id === opened.session.session_id, ); @@ -565,6 +629,52 @@ export class IdaSessionClient implements AnalysisClient { return shared; } + /** + * Upstream sessions whose release cleanup has not confirmed: the entries + * the latest successful inventory found blocking release; else the + * requested private worker until its close is confirmed, and then none + * known. + */ + #unreleasedSessions(): readonly UpstreamSession[] | undefined { + if (this.#blockingSessions !== undefined) return this.#blockingSessions; + if (!this.#openAttempted || this.#workspace === undefined) return undefined; + return this.#databaseCloseCompleted + ? [] + : [ + { + session_id: this.#workspace.sessionId, + backend: "worker", + owned: null, + input_path: null, + }, + ]; + } + + /** Drop a session whose release upstream confirmed, keeping other blockers. */ + #forgetReleasedSession(sessionId: string): void { + const remaining = this.#blockingSessions?.filter( + ({ session_id }) => session_id !== sessionId, + ); + this.#blockingSessions = + remaining !== undefined && remaining.length > 0 ? remaining : undefined; + } + + /** Replace blocker evidence from a successful inventory; true when blocked. */ + #recordBlockingSessions( + entries: readonly (UpstreamSession & { readonly owned: boolean })[], + ): boolean { + this.#blockingSessions = + entries.length > 0 + ? entries.map(({ session_id, backend, owned, input_path }) => ({ + session_id, + backend, + owned, + input_path, + })) + : undefined; + return entries.length > 0; + } + async #closeDatabase(database: string): Promise { const response = await this.connection.call("idb_close", { database, @@ -605,34 +715,47 @@ export class IdaSessionClient implements AnalysisClient { entry.session_id === workspace.sessionId && sameIdaHostPath(entry.input_path, workspace.inputPath), )?.session_id; - if ( - this.#database === undefined && - inventory.sessions.some((entry) => - sameIdaHostPath(entry.input_path, workspace.inputPath), - ) - ) + // Every same-input session and any entry under the requested ID, + // whatever its input, stays reported until a later successful + // inventory confirms release. + const blocked = this.#recordBlockingSessions( + inventory.sessions.filter( + (entry) => + entry.session_id === workspace.sessionId || + sameIdaHostPath(entry.input_path, workspace.inputPath), + ), + ); + if (this.#database === undefined && blocked) throw new AnalysisProtocolError( "An unowned worker still uses the private workspace; it was retained.", ); + if ( + this.#database === undefined && + this.#openCompleted && + !this.#openRejected + ) + throw new AnalysisProtocolError( + "IDA open returned without a verified private worker binding. An empty session inventory does not prove that its Worker exited; the private workspace was retained.", + ); } if (this.#database !== undefined) { if (!this.#databaseCloseCompleted) { await this.#closeDatabase(this.#database); this.#databaseCloseCompleted = true; + this.#forgetReleasedSession(this.#database); } const inventory = databaseListSchema.parse( await this.connection.call("idb_list", {}), ); - if ( - inventory.sessions.some( - (entry) => - entry.session_id === this.#database || - sameIdaHostPath( - entry.input_path, - this.#workspace?.inputPath ?? "", - ), - ) - ) + const remaining = inventory.sessions.filter( + (entry) => + entry.session_id === this.#database || + sameIdaHostPath( + entry.input_path, + this.#workspace?.inputPath ?? "", + ), + ); + if (this.#recordBlockingSessions(remaining)) throw new AnalysisProtocolError( "IDA worker remains registered after close.", ); @@ -649,10 +772,12 @@ export class IdaSessionClient implements AnalysisClient { this.#closed = true; return ok(null); } catch (cause: unknown) { + this.connection.detach(); return err( new IdaCleanupError( this.#workspace?.directory ?? "IDA MCP connection", cause, + this.#unreleasedSessions(), ), ); } diff --git a/src/ida/IdaWorkspaceOwnership.test.ts b/src/ida/IdaWorkspaceOwnership.test.ts index b39c6e916..1f4a10d65 100644 --- a/src/ida/IdaWorkspaceOwnership.test.ts +++ b/src/ida/IdaWorkspaceOwnership.test.ts @@ -98,11 +98,17 @@ describe("IDA workspace ownership before input preparation", () => { expect(closed.error.cleanupResources).toEqual([ expect.stringContaining(workspace ?? "missing workspace"), ]); + // No open was attempted, so there is no worker to report. + expect(closed.error).not.toHaveProperty( + "diagnostics.upstream_sessions", + ); } + expect(producer.detaches).toBe(1); } finally { await chmod(root, 0o700); } expect((await client.close()).ok).toBe(true); + expect(producer.detaches).toBe(1); expect(await readdir(root)).toEqual(["target.elf"]); }, ); diff --git a/src/native/NativeMacOSProvider.ts b/src/native/NativeMacOSProvider.ts index a59ec90a7..333569e47 100644 --- a/src/native/NativeMacOSProvider.ts +++ b/src/native/NativeMacOSProvider.ts @@ -73,7 +73,7 @@ import { } from "./SignatureTargetBinding.js"; import { parseCodeSignature } from "./parsers/codesign.js"; import { parseDemangledSymbols } from "./parsers/demangle.js"; -import { parseLipoArchitectures } from "./parsers/lipo.js"; +import { parseLipoOutput } from "./parsers/lipo.js"; import { parsePlistJson, parsePlistXml, @@ -507,7 +507,10 @@ class NativeMacOSClient implements AnalysisClient { ); if (!capture.ok) return capture; const provenance = [invocation(capture.value, this.target.path)]; - const architectures = parseLipoArchitectures(capture.value.stdout); + const { architectures } = parseLipoOutput( + capture.value.stdout, + this.target.path, + ); const result = listArchitecturesSchema.parse({ architectures: { items: architectures, @@ -561,6 +564,19 @@ class NativeMacOSClient implements AnalysisClient { async #inspectSignature( signal?: AbortSignal, ): Promise> { + // This provider's signature inspection covers Mach-O code signatures + // only. On another container, codesign's "not signed" verdict cannot + // see a format-level signature such as a xar pkg signature, so + // reporting it would claim unsigned for a layer that was never + // evaluated. + if (this.target.kind !== "executable" || !isMachOFormat(this.target.format)) + return err( + new AnalysisCapabilityUnavailableError( + IDENTITY.id, + "inspect_signature", + "Active artifact is not Mach-O; its container-level signature layer is not evaluated by codesign.", + ), + ); return withSignatureTarget( this.target, this.#signatureResources, diff --git a/src/native/NativeMachoInspection.ts b/src/native/NativeMachoInspection.ts index 5fca089cb..bb1966c3d 100644 --- a/src/native/NativeMachoInspection.ts +++ b/src/native/NativeMachoInspection.ts @@ -14,10 +14,7 @@ import { selectMachoArchitectureName, } from "./MachoArchitectureSelection.js"; import { parseDyldSymbols } from "./parsers/dyldInfo.js"; -import { - parseLipoArchitectures, - type LipoArchitecture, -} from "./parsers/lipo.js"; +import { parseLipoOutput, type LipoArchitecture } from "./parsers/lipo.js"; import { withoutEchoedPathHeader } from "./parsers/echoedPath.js"; import { parseOtoolLoadCommands } from "./parsers/otool.js"; import { reconcileMachoUuid } from "./parsers/machoUuid.js"; @@ -74,7 +71,10 @@ export const inspectNativeMacho = async ( } const lipo = captures.find(({ tool }) => tool === "lipo"); if (lipo === undefined) throw new TypeError("Missing lipo capture"); - const architectures = parseLipoArchitectures(lipo.stdout); + const { format, architectures } = parseLipoOutput( + lipo.stdout, + context.target.path, + ); const selection = selectArchitecture(context.target, architectures); if (selection.status === "absent") return err( @@ -145,7 +145,7 @@ export const inspectNativeMacho = async ( const containerOffset = selectedArchitectureName === null ? undefined - : /^Non-fat file:/mu.test(lipo.stdout) + : format === "thin" ? 0 : (architectures.find(({ name }) => name === selectedArchitectureName) ?.file_offset ?? undefined); @@ -185,7 +185,7 @@ const normalizeMacho = ( toInvocation: (capture: NativeCommandCapture) => NativeCommandInvocation, additionalLimitations: readonly string[], load: ReturnType, - architectures: ReturnType, + architectures: readonly LipoArchitecture[], artifactPath: string, selectedArchitecture: { readonly name: string | null; diff --git a/src/native/parsers/lipo.test.ts b/src/native/parsers/lipo.test.ts index 96f2afbbc..b5c97a415 100644 --- a/src/native/parsers/lipo.test.ts +++ b/src/native/parsers/lipo.test.ts @@ -1,10 +1,28 @@ import { describe, expect, it } from "vitest"; -import { parseLipoArchitectures } from "./lipo.js"; +import { parseLipoOutput } from "./lipo.js"; + +const fatRecords = `fat_magic 0xcafebabe +nfat_arch 2 +architecture arm64 + cputype CPU_TYPE_ARM64 + cpusubtype CPU_SUBTYPE_ARM64_ALL + capabilities 0x0 + offset 4096 + size 32 + align 2^12 (4096) +architecture x86_64 + cputype CPU_TYPE_X86_64 + cpusubtype CPU_SUBTYPE_X86_64_ALL + capabilities 0x0 + offset 8192 + size 32 + align 2^12 (4096) +`; describe("lipo architecture facts", () => { it("normalizes symbolic CPU fields and arm64e capability bits", () => { - const architectures = parseLipoArchitectures(`Fat header in: /bin/ls + const { format, architectures } = parseLipoOutput(`Fat header in: /bin/ls fat_magic 0xcafebabe nfat_arch 2 architecture x86_64 @@ -22,6 +40,7 @@ architecture arm64e size 89088 align 2^14 (16384) `); + expect(format).toBe("fat"); expect(architectures).toEqual([ expect.objectContaining({ name: "x86_64", @@ -50,9 +69,9 @@ architecture arm64e it("keeps unknown symbolic CPU values as source text without a code", () => { expect( - parseLipoArchitectures( + parseLipoOutput( "architecture future\n cputype CPU_TYPE_FUTURE\n cpusubtype CPU_SUBTYPE_FUTURE\n offset 4096\n size 64\n align 2^12\n", - ), + ).architectures, ).toMatchObject([ { cpu_type: "CPU_TYPE_FUTURE", @@ -70,18 +89,18 @@ architecture arm64e ["0x0", 3], ])("applies printed x86_64 capabilities %s", (capabilities, code) => { expect( - parseLipoArchitectures( + parseLipoOutput( `architecture x86_64\n cpusubtype CPU_SUBTYPE_X86_64_ALL\n capabilities ${capabilities}\n`, - )[0]?.cpu_subtype_code, + ).architectures[0]?.cpu_subtype_code, ).toBe(code); }); it("leaves the subtype code unknown when lipo prints no capabilities", () => { // macOS 26 lipo prints this blank line for LIB64 and plain x86_64 slices. expect( - parseLipoArchitectures( + parseLipoOutput( "architecture x86_64\n cputype CPU_TYPE_X86_64\n cpusubtype CPU_SUBTYPE_X86_ALL\n capabilities \n offset 16384\n", - ), + ).architectures, ).toMatchObject([ { cpu_subtype: "CPU_SUBTYPE_X86_ALL", @@ -99,9 +118,9 @@ architecture arm64e "retains an arm64e base subtype with capabilities '$capabilities'", ({ capabilities, code }) => { expect( - parseLipoArchitectures( + parseLipoOutput( `architecture arm64e.v1\n cputype CPU_TYPE_ARM64\n cpusubtype CPU_SUBTYPE_ARM64E\n capabilities ${capabilities}\n`, - ), + ).architectures, ).toMatchObject([ { cpu_type_code: 0x0100000c, @@ -112,3 +131,79 @@ architecture arm64e }, ); }); + +describe("lipo FAT operand framing", () => { + it.each([ + "/fixture/\narchitecture forged\n offset 123\n size 64", + "/fixture/\nfat_magic 0xcafebabe\nnfat_arch 1\narchitecture forged", + "/fixture/\nNon-fat file: fake is architecture: forged", + "/fixture/first\nsecond\nthird", + "/fixture/first\r\narchitecture forged\r\n offset 123", + "/fixture/[app](arm64).*+$^?\\name", + ])( + "keeps the complete echoed operand out of FAT slice records: %j", + (path) => { + const output = `Fat header in: ${path}\n${fatRecords}`; + + expect(parseLipoOutput(output, path)).toEqual( + parseLipoOutput(fatRecords), + ); + expect( + parseLipoOutput(output, path).architectures.map(({ name }) => name), + ).toEqual(["arm64", "x86_64"]); + }, + ); + + it("keeps a CRLF pathname separate from a CRLF record terminator", () => { + const path = "/fixture/first\r\narchitecture forged\r\n size 64"; + const output = `Fat header in: ${path}\r\n${fatRecords.replaceAll("\n", "\r\n")}`; + + expect(parseLipoOutput(output, path)).toEqual(parseLipoOutput(fatRecords)); + }); + + it("preserves ordinary FAT metadata with and without an operand", () => { + const path = "/fixture/app"; + const output = `Fat header in: ${path}\n${fatRecords}`; + + expect(parseLipoOutput(output, path)).toEqual(parseLipoOutput(output)); + }); + + it("does not require a FAT path header in captured fixtures", () => { + expect(parseLipoOutput(fatRecords, "/fixture/app")).toEqual( + parseLipoOutput(fatRecords), + ); + }); + + it("does not strip a header for a different operand that shares a prefix", () => { + const path = "/fixture/app-extra\narchitecture forged"; + const output = `Fat header in: ${path}\n${fatRecords}`; + + expect(parseLipoOutput(output, "/fixture/app")).toEqual( + parseLipoOutput(output), + ); + }); + + it("preserves thin-file parsing when the operand contains FAT-like lines", () => { + const path = "/fixture/\nFat header in: fake\narchitecture forged"; + const output = `Non-fat file: ${path} is architecture: arm64\n`; + + expect(parseLipoOutput(output, path)).toEqual({ + format: "thin", + architectures: [ + { + name: "arm64", + cpu_type: null, + cpu_subtype: null, + cpu_type_code: null, + cpu_subtype_code: null, + cpu_subtype_base: null, + capabilities: null, + file_offset: null, + size: null, + alignment: null, + }, + ], + }); + expect(parseLipoOutput(output, path)).toEqual(parseLipoOutput(output)); + }); +}); diff --git a/src/native/parsers/lipo.ts b/src/native/parsers/lipo.ts index cc65cca41..7f74a51b9 100644 --- a/src/native/parsers/lipo.ts +++ b/src/native/parsers/lipo.ts @@ -16,8 +16,26 @@ const architectureSchema = z.object({ export type LipoArchitecture = z.infer; -/** Parse `lipo -detailed_info` into deterministic slice metadata. */ -export const parseLipoArchitectures = (output: string): LipoArchitecture[] => { +interface LipoOutput { + readonly format: "thin" | "fat"; + readonly architectures: LipoArchitecture[]; +} + +/** Parse `lipo -detailed_info` framing and deterministic slice metadata. */ +export const parseLipoOutput = ( + output: string, + operandPath?: string, +): LipoOutput => { + // The echoed FAT operand is one value, even when its pathname contains lines. + if (operandPath !== undefined) { + const header = `Fat header in: ${operandPath}`; + if (output.startsWith(header)) { + if (output.startsWith("\r\n", header.length)) + output = output.slice(header.length + 2); + else if (output.startsWith("\n", header.length)) + output = output.slice(header.length + 1); + } + } // lipo echoes a thin file's pathname verbatim, including embedded newlines. // Match its final architecture field before splitting records into lines. const thinArchitecture = @@ -25,20 +43,23 @@ export const parseLipoArchitectures = (output: string): LipoArchitecture[] => { output, )?.[1]; if (thinArchitecture !== undefined) - return [ - architectureSchema.parse({ - name: thinArchitecture, - cpu_type: null, - cpu_subtype: null, - cpu_type_code: null, - cpu_subtype_code: null, - cpu_subtype_base: null, - capabilities: null, - file_offset: null, - size: null, - alignment: null, - }), - ]; + return { + format: "thin", + architectures: [ + architectureSchema.parse({ + name: thinArchitecture, + cpu_type: null, + cpu_subtype: null, + cpu_type_code: null, + cpu_subtype_code: null, + cpu_subtype_base: null, + capabilities: null, + file_offset: null, + size: null, + alignment: null, + }), + ], + }; const architectures: LipoArchitecture[] = []; let current: Record | undefined; const flush = (): void => { @@ -90,7 +111,7 @@ export const parseLipoArchitectures = (output: string): LipoArchitecture[] => { flush(); if (architectures.length === 0) throw new TypeError("lipo output contained no architectures"); - return architectures; + return { format: "fat", architectures }; }; const integer = (value: string | undefined): number | null => { diff --git a/src/native/pwntools/PwntoolsDecoder.ts b/src/native/pwntools/PwntoolsDecoder.ts index 169f25601..6aa9a3a96 100644 --- a/src/native/pwntools/PwntoolsDecoder.ts +++ b/src/native/pwntools/PwntoolsDecoder.ts @@ -32,7 +32,10 @@ import { analysisErrorWithCleanupFailure } from "../../domain/analysisErrorClean import { ProviderAdapterError } from "../../domain/providerAdapterError.js"; import { ProviderCleanupError } from "../../domain/providerCleanupError.js"; import { err, ok, type Result } from "../../domain/result.js"; -import { runOwnedCommand } from "../../process/OwnedCommand.js"; +import { + ownedCommandResources, + runOwnedCommand, +} from "../../process/OwnedCommand.js"; import type { ProviderProcessSupervisor, OwnedProviderProcessSpawnOptions, @@ -48,6 +51,7 @@ import { PWNTOOLS_PROVIDER_IDENTITY, PWNTOOLS_LIMITS, } from "./PwntoolsRelease.js"; +import { pwntoolsInputBytes } from "./PwntoolsInputLimit.js"; const HOST_REQUIREMENT = "The initial real-verified pwntools artifact profile supports Linux x64 only."; @@ -242,11 +246,23 @@ export class PwntoolsDecoder< undefined, context, ); + const inputLimit = pwntoolsInputBytes(this.environment); + if (!inputLimit.configured) + throw pwntoolsUnavailable( + inputLimit.error, + selectedPath, + undefined, + undefined, + { + ...context, + configurationKey: "REA_PWNTOOLS_MAX_INPUT_BYTES", + }, + ); phase = "artifact-read"; selectedPath = input.path; const snapshot = await readStableArtifact( input.path, - PWNTOOLS_LIMITS.inputBytes, + inputLimit.bytes, options?.signal, undefined, this.#resources, @@ -562,20 +578,16 @@ export class PwntoolsDecoder< const pending = this.#pendingCleanup; if (pending?.process === undefined) return undefined; const stopped = await pending.process.stop(); + const ownership = pending.process.launch.ownership; if (stopped.status === "incomplete") return new ProviderCleanupError( this.identity.id, - [ - pending.process.launch.ownership?.runId ?? "owned-decoder", - ...(pending.root === undefined ? [] : [pending.root.path]), - // The retained group may still be live; name it as OwnedCommand does. - ...(pending.process.launch.ownership === undefined - ? [] - : [ - `pid:${pending.process.launch.ownership.leaderPid}`, - `process-group:${pending.process.launch.ownership.processGroupId}`, - ]), - ], + ownership === undefined + ? [ + "owned-decoder", + ...(pending.root === undefined ? [] : [pending.root.path]), + ] + : ownedCommandResources(ownership, pending.root?.path), { reason: stopped.reason, captured_output: { diff --git a/src/native/pwntools/PwntoolsFailures.ts b/src/native/pwntools/PwntoolsFailures.ts index f5e1b2654..e36ae9503 100644 --- a/src/native/pwntools/PwntoolsFailures.ts +++ b/src/native/pwntools/PwntoolsFailures.ts @@ -30,6 +30,7 @@ export interface PwntoolsFailureContext { readonly operation: string; readonly providerId: string; readonly precedingOutput?: AnalysisCapturedOutput; + readonly configurationKey?: string; } const DEFAULT_CONTEXT = { operation: "inspect_binary_layout", @@ -274,7 +275,7 @@ export const pwntoolsUnavailable = ( code: "provider_unavailable", reason, diagnostics: { - configuration_key: "REA_PWNTOOLS_PYTHON", + configuration_key: context.configurationKey ?? "REA_PWNTOOLS_PYTHON", executable_path: path, ...(systemCode === undefined ? {} : { system_code: systemCode }), }, diff --git a/src/native/pwntools/PwntoolsInputLimit.test.ts b/src/native/pwntools/PwntoolsInputLimit.test.ts new file mode 100644 index 000000000..d60c27b80 --- /dev/null +++ b/src/native/pwntools/PwntoolsInputLimit.test.ts @@ -0,0 +1,96 @@ +import { describe, expect, it } from "vitest"; + +import { pwntoolsInputBytes } from "./PwntoolsInputLimit.js"; +import { pwntoolsUnavailable } from "./PwntoolsFailures.js"; +import { + PWNTOOLS_LIMITS, + PWNTOOLS_PROVIDER_IDENTITY, +} from "./PwntoolsRelease.js"; + +const CONSTRAINT = + "REA_PWNTOOLS_MAX_INPUT_BYTES must be a decimal integer from 1 to " + + "1073741824 bytes, and REA never installs Python or packages."; + +describe("pwntools input bound", () => { + it("keeps the shipped default when the override is absent", () => { + expect(pwntoolsInputBytes({})).toEqual({ + configured: true, + bytes: PWNTOOLS_LIMITS.inputBytes, + }); + expect(PWNTOOLS_LIMITS.inputBytes).toBe(32 * 1024 * 1024); + }); + + it.each([ + ["1", 1], + ["67108864", 64 * 1024 * 1024], + ["172879872", 172_879_872], + ["1073741824", 1024 * 1024 * 1024], + ] as const)("accepts %s as %s bytes", (value, expected) => { + expect(pwntoolsInputBytes({ REA_PWNTOOLS_MAX_INPUT_BYTES: value })).toEqual( + { configured: true, bytes: expected }, + ); + }); + + it.each([ + "", + "0", + "-1", + "33554432.0", + "1e8", + " 67108864", + "67_108_864", + "0x2000000", + "1073741825", + "99999999999999999999", + "not-a-number", + ])("rejects %s instead of widening the bound silently", (value) => { + expect(pwntoolsInputBytes({ REA_PWNTOOLS_MAX_INPUT_BYTES: value })).toEqual( + { + configured: false, + error: CONSTRAINT, + }, + ); + }); + + it("rejects a value above the ceiling that a float would round", () => { + const ceiling = pwntoolsInputBytes({ + REA_PWNTOOLS_MAX_INPUT_BYTES: "1073741824", + }); + const above = pwntoolsInputBytes({ + REA_PWNTOOLS_MAX_INPUT_BYTES: "10737418241", + }); + expect(ceiling.configured).toBe(true); + expect(above).toEqual({ configured: false, error: CONSTRAINT }); + }); + + it("never narrows the bound below the shipped default implicitly", () => { + const accepted = [1, 1024, PWNTOOLS_LIMITS.inputBytes].map((bytes) => + pwntoolsInputBytes({ REA_PWNTOOLS_MAX_INPUT_BYTES: String(bytes) }), + ); + expect(accepted.every((limit) => limit.configured)).toBe(true); + }); +}); + +describe("pwntools unavailability configuration key", () => { + const context = { + operation: "inspect_binary_layout", + providerId: PWNTOOLS_PROVIDER_IDENTITY.id, + }; + const reportedKey = (configurationKey?: string) => + pwntoolsUnavailable( + CONSTRAINT, + "/caller-selected/python", + undefined, + undefined, + configurationKey === undefined + ? context + : { ...context, configurationKey }, + ).rejections[0]?.diagnostics["configuration_key"]; + + it("keeps the engine key unless the failure is about the bound", () => { + expect(reportedKey()).toBe("REA_PWNTOOLS_PYTHON"); + expect(reportedKey("REA_PWNTOOLS_MAX_INPUT_BYTES")).toBe( + "REA_PWNTOOLS_MAX_INPUT_BYTES", + ); + }); +}); diff --git a/src/native/pwntools/PwntoolsInputLimit.ts b/src/native/pwntools/PwntoolsInputLimit.ts new file mode 100644 index 000000000..e6867e80f --- /dev/null +++ b/src/native/pwntools/PwntoolsInputLimit.ts @@ -0,0 +1,40 @@ +import { z } from "zod"; + +import { PWNTOOLS_LIMITS } from "./PwntoolsRelease.js"; + +// The artifact is read into one in-memory buffer and staged again as a private +// snapshot before the owned bridge parses it under a 3 GiB address-space +// reservation, so an override stays within a third of that reservation. +const MAXIMUM_INPUT_BYTES = 1024 * 1024 * 1024; + +/** Optional caller-supplied input bound, in bytes, without a decimal point. */ +export const pwntoolsInputBytesSchema = z + .string() + .regex(/^[1-9]\d*$/u, "must be a positive decimal integer") + .transform(Number) + .pipe(z.number().int().safe().max(MAXIMUM_INPUT_BYTES)); + +const CONSTRAINT = + `REA_PWNTOOLS_MAX_INPUT_BYTES must be a decimal integer from 1 to ` + + `${String(MAXIMUM_INPUT_BYTES)} bytes, and REA never installs Python or packages.`; + +/** Resolved input bound, or the reason the caller's override cannot be used. */ +export type PwntoolsInputBytes = + | { readonly configured: true; readonly bytes: number } + | { readonly configured: false; readonly error: string }; + +/** + * Resolve the admitted artifact size for the pwntools adapters. An unset value + * keeps the shipped default, so the bound is only ever widened deliberately. + */ +export const pwntoolsInputBytes = ( + environment: Readonly, +): PwntoolsInputBytes => { + const value = environment.REA_PWNTOOLS_MAX_INPUT_BYTES; + if (value === undefined) + return { configured: true, bytes: PWNTOOLS_LIMITS.inputBytes }; + const parsed = pwntoolsInputBytesSchema.safeParse(value); + return parsed.success + ? { configured: true, bytes: parsed.data } + : { configured: false, error: CONSTRAINT }; +}; diff --git a/src/process/OwnedCommand.ts b/src/process/OwnedCommand.ts index 02fb4defc..d375971ae 100644 --- a/src/process/OwnedCommand.ts +++ b/src/process/OwnedCommand.ts @@ -9,6 +9,7 @@ import { } from "./ProviderProcess.js"; import { cleanupOwnedProcessGroup, + type OwnedProcessGroup, type ProcessCleanupFailure, } from "./ProcessOwnership.js"; import { @@ -34,6 +35,17 @@ export class OwnedCommandFailure extends Error { } } +/** Report recorded launch identities; these labels do not authorize signaling. */ +export const ownedCommandResources = ( + ownership: OwnedProcessGroup, + directory?: string, +): readonly string[] => [ + ownership.runId, + ...(directory === undefined ? [] : [directory]), + `pid:${ownership.leaderPid}`, + `process-group:${ownership.processGroupId}`, +]; + /** Execute a single owned process with independent cleanup and bounded diagnostic retention. */ export const runOwnedCommand = async ( spawn: OwnedProviderProcessSpawnOptions, @@ -208,12 +220,7 @@ const collectOwnedCommand = async ( ? {} : { cleanupResult: stopped.cleanupResult }), }, - [ - spawn.runId, - ...(spawn.cwd === undefined ? [] : [spawn.cwd]), - `pid:${launched.ownership.leaderPid}`, - `process-group:${launched.ownership.processGroupId}`, - ], + ownedCommandResources(launched.ownership, spawn.cwd), supervisor, ); if (failure !== undefined) diff --git a/src/process/capture/ProcessScenarioRuntimeValidation.test.ts b/src/process/capture/ProcessScenarioRuntimeValidation.test.ts index 1277b753f..7e5cb7e30 100644 --- a/src/process/capture/ProcessScenarioRuntimeValidation.test.ts +++ b/src/process/capture/ProcessScenarioRuntimeValidation.test.ts @@ -17,6 +17,28 @@ import { resolveProcessScenarioRuntimePaths, } from "./ProcessScenarioRuntimeValidation.js"; import { parseProcessScenario } from "../../domain/process/processScenario.js"; +import { createTestTempDirectory } from "../../../tests/fixtures/temporaryDirectory.js"; + +it.skipIf(process.platform === "win32")( + "resolves a literal backslash in a POSIX executable basename through PATH", + async () => { + const root = await createTestTempDirectory("rea-process-posix-basename-"); + const bin = join(root, "bin"); + await mkdir(bin); + const command = "selected\\node"; + const selected = join(bin, command); + await symlink(process.execPath, selected); + await expect(resolveExecutable(command, root, bin)).resolves.toBe(selected); + const resolved = await resolveProcessScenarioRuntimePaths( + parseProcessScenario({ + executable: command, + working_directory: root, + environment: { PATH: bin }, + }), + ); + expect(resolved.executable).toBe(selected); + }, +); it("preserves the caller-selected executable symlink for process invocation", async () => { const root = await mkdtemp(join(tmpdir(), "rea-process-executable-alias-")); diff --git a/src/process/capture/ProcessScenarioRuntimeValidation.ts b/src/process/capture/ProcessScenarioRuntimeValidation.ts index 1ac9d2228..6d2978b5b 100644 --- a/src/process/capture/ProcessScenarioRuntimeValidation.ts +++ b/src/process/capture/ProcessScenarioRuntimeValidation.ts @@ -1,6 +1,6 @@ import { constants as fsConstants } from "node:fs"; import { access, realpath, stat } from "node:fs/promises"; -import { delimiter, isAbsolute, join, resolve } from "node:path"; +import { delimiter, isAbsolute, join, resolve, sep } from "node:path"; import type { ProcessScenario } from "../../domain/process/processScenario.js"; import { AnalysisInputError } from "../../domain/analysisErrorCore.js"; @@ -71,7 +71,9 @@ export const resolveExecutable = async ( : resolve(workingDirectory, directory), ); const candidates = - isAbsolute(command) || command.includes("/") || command.includes("\\") + isAbsolute(command) || + command.includes("/") || + (sep === "\\" && command.includes("\\")) ? [resolve(workingDirectory, command)] : directories.map((directory) => join(directory, command)); const failures: string[] = []; diff --git a/src/sqlite/SqliteDatabaseInspection.ts b/src/sqlite/SqliteDatabaseInspection.ts index f5f92bdf4..510fbaafc 100644 --- a/src/sqlite/SqliteDatabaseInspection.ts +++ b/src/sqlite/SqliteDatabaseInspection.ts @@ -394,7 +394,7 @@ const readRows = ( for (const name of columns) if (!available.has(name)) throw new SqliteInspectionFailure( - "selection", + "column-selection", `Selected column does not exist with that exact name in table ${input.table}: ${name}`, ); const projection = columns diff --git a/src/sqlite/SqliteDatabaseLimits.ts b/src/sqlite/SqliteDatabaseLimits.ts index 99df7ed9d..408fc4bc3 100644 --- a/src/sqlite/SqliteDatabaseLimits.ts +++ b/src/sqlite/SqliteDatabaseLimits.ts @@ -12,6 +12,7 @@ export class SqliteInspectionFailure extends Error { readonly reason: | "format" | "selection" + | "column-selection" | "unsupported" | "unavailable" | "output-limit" diff --git a/src/sqlite/SqliteDatabaseWorkerExecution.ts b/src/sqlite/SqliteDatabaseWorkerExecution.ts index 9474a6d27..cba180399 100644 --- a/src/sqlite/SqliteDatabaseWorkerExecution.ts +++ b/src/sqlite/SqliteDatabaseWorkerExecution.ts @@ -36,6 +36,7 @@ const replySchema = z.discriminatedUnion("ok", [ reason: z.enum([ "format", "selection", + "column-selection", "unsupported", "unavailable", "output-limit", @@ -49,6 +50,14 @@ export type SqliteWorkerLauncher = NonNullable< Parameters[2] >["launcher"]; +// Each input rejection names the caller argument that requires correction. +const rejectedArgument = { + format: "path", + unsupported: "path", + selection: "table", + "column-selection": "columns", +} as const; + const rejectedReply = ( reply: z.output & { ok: false }, capturedOutput: AnalysisCapturedOutput, @@ -83,7 +92,7 @@ const rejectedReply = ( ); return new AnalysisInputError(operation, { capturedOutput }, [ { - path: [reply.reason === "selection" ? "table" : "path"], + path: [rejectedArgument[reply.reason]], reason: "invalid_format", message: reply.message, }, diff --git a/src/windows/WindowsNativeLoader.test.ts b/src/windows/WindowsNativeLoader.test.ts index 1aa8d7375..c0fee8273 100644 --- a/src/windows/WindowsNativeLoader.test.ts +++ b/src/windows/WindowsNativeLoader.test.ts @@ -143,6 +143,22 @@ describe("packaged Windows native admission", () => { }, ); + it.each([ + { signature: "MZ", offset: 0 }, + { signature: "PE", offset: 64 }, + ])( + "rejects digest-valid high-bit $signature lookalikes before loading", + ({ offset }) => { + const value = fixture(); + value.bytes[offset] = (value.bytes[offset] ?? 0) | 0x80; + value.manifest.artifactSha256 = createHash("sha256") + .update(value.bytes) + .digest("hex"); + expect(loadWindowsNativeAuthority(value.host).available).toBe(false); + expect(value.loads()).toBe(0); + }, + ); + it("rejects a digest-valid artifact for another machine", () => { const value = fixture(); value.bytes.writeUInt16LE(0xaa64, 68); diff --git a/src/windows/WindowsNativeLoader.ts b/src/windows/WindowsNativeLoader.ts index 7719a0a85..2ab08e1ed 100644 --- a/src/windows/WindowsNativeLoader.ts +++ b/src/windows/WindowsNativeLoader.ts @@ -132,12 +132,12 @@ export const loadWindowsNativeAuthority = ( }; const assertWindowsX64Artifact = (bytes: Buffer): void => { - if (bytes.length < 64 || bytes.toString("ascii", 0, 2) !== "MZ") + if (bytes.length < 64 || bytes.toString("latin1", 0, 2) !== "MZ") throw new Error("Windows native artifact is not a PE image"); const offset = bytes.readUInt32LE(0x3c); if ( offset > bytes.length - 26 || - bytes.toString("ascii", offset, offset + 4) !== "PE\0\0" || + bytes.toString("latin1", offset, offset + 4) !== "PE\0\0" || bytes.readUInt16LE(offset + 4) !== 0x8664 || bytes.readUInt16LE(offset + 24) !== 0x20b ) diff --git a/tests/acceptance/applications/getterLocalAliasesCli.test.ts b/tests/acceptance/applications/getterLocalAliasesCli.test.ts new file mode 100644 index 000000000..2b2b0196c --- /dev/null +++ b/tests/acceptance/applications/getterLocalAliasesCli.test.ts @@ -0,0 +1,85 @@ +import { writeFile } from "node:fs/promises"; +import { join } from "node:path"; +import { expect } from "vitest"; +import { createTestTempDirectory } from "../../fixtures/temporaryDirectory.js"; +import { cliTest } from "../../support/cli/cliFixture.js"; +import { analyzeJavaScriptDirectory } from "../../support/cli/javascriptApplication.js"; + +cliTest.for(["CLI", "stdio MCP"] as const)( + "retains getter-local alias mutations in exported Evidence through %s", + async (surface, { cli }) => { + const root = await createTestTempDirectory("rea-getter-local-aliases-"); + const cases = [ + ["initialized", "const self = this; return self.s;", true], + ["assigned", "let self; self = this; return self.s;", true], + ["destructured", "const { s: value } = this; return value;", true], + ["readOnly", "const self = this; return self.s;", false], + [ + "cyclic", + "let self = this.start; while (flag) self = self.child; return self;", + true, + "start: { child: { child: shared } }", + ], + ] as const; + const privateCases = [ + ["privateProjection", "shared", true, "unknown"], + ["privateUnrelated", "{ peer: shared }", true, "literal"], + ["privateReadOnly", "shared", false, "literal"], + ] as const; + await writeFile( + join(root, "app.mjs"), + cases + .map( + ([ + name, + getter, + write, + members = "s: shared", + ]) => `export function ${name}() { + const shared = { x: 1 }; + const box = { ${members}, get v() { ${getter} } }; + ${write ? "box.v.x = 2;" : "const saved = box.v;"} + return shared.x; + }`, + ) + .join("\n") + + privateCases + .map( + ([name, value, write]) => `export function ${name}() { + const shared = { x: 1 }; + class Box { + #s = ${value}; + peer = this; + get v() { const { peer: self } = this; return self.#s; } + } + const box = new Box(); + ${write ? "box.v.x = 2;" : "const saved = box.v;"} + return shared.x; + }`, + ) + .join("\n"), + ); + const { graph } = await analyzeJavaScriptDirectory(surface, cli, root); + for (const [name, status] of [ + ...cases.map(([name, , write]) => [name, write ? "unknown" : "literal"]), + ...privateCases.map(([name, , , status]) => [name, status]), + ]) { + const observation = graph.nodes + .flatMap(({ observations }) => observations) + .find( + ({ properties }) => + properties.semantic_role === "export-return-shapes" && + properties.exported_name === name, + ); + expect(observation, name).toBeDefined(); + expect(observation?.properties.static_return_shapes).toMatchObject([ + { + value_status: status, + fields: [ + { path: "", state: status, value: status === "unknown" ? null : 1 }, + ], + }, + ]); + } + }, +); diff --git a/tests/acceptance/applications/getterReceiverMutationsCli.test.ts b/tests/acceptance/applications/getterReceiverMutationsCli.test.ts index a5ba0bfeb..95b6e4862 100644 --- a/tests/acceptance/applications/getterReceiverMutationsCli.test.ts +++ b/tests/acceptance/applications/getterReceiverMutationsCli.test.ts @@ -39,6 +39,47 @@ export function superclass() { new B().v.x = 2; return shared.x; } +export function arrayReceiver() { + const arr = [{ x: 1, get self() { return this; } }]; + arr[0].self.x = 2; + return arr[0].x; +} +export function nestedArrayReceiver() { + const outer = { arr: [{ x: 1, get self() { return this; } }] }; + outer.arr[0].self.x = 2; + return outer.arr[0].x; +} +export function assignedReceiver() { + let box; + box = { x: 1, get self() { return this; } }; + box.self.x = 2; + return box.x; +} +export function savedArrayReceiver() { + const arr = [{ x: 1, get self() { return this; } }]; + const saved = arr[0].self; + saved.x = 2; + return arr[0].x; +} +export function arrayReadOnly() { + const arr = [{ x: 1, get self() { return this; } }]; + const saved = arr[0].self; + return arr[0].x; +} +export function replacementKeepsOldObject() { + const old = { x: 1 }; + let box = old; + box = { x: 1, old: box, get self() { return this; } }; + box.self.x = 2; + return old.x; +} +export function replacementMutatesCapturedObject() { + const old = { x: 1 }; + let box = old; + box = { old: box, get self() { return this; } }; + box.self.old.x = 2; + return old.x; +} export function untouched() { const box = { x: 1, y: 1, get self() { return this; } }; box.self.y = 2; @@ -66,13 +107,20 @@ export function untouched() { const { graph } = z .object({ normalized_result: javascriptApplicationAnalysisResultSchema }) .parse(document).normalized_result; - for (const name of [ - "self", - "property", - "privateField", - "superclass", - "untouched", - ]) { + for (const [name, status] of [ + ["self", "unknown"], + ["property", "unknown"], + ["privateField", "unknown"], + ["superclass", "unknown"], + ["arrayReceiver", "unknown"], + ["nestedArrayReceiver", "unknown"], + ["assignedReceiver", "unknown"], + ["savedArrayReceiver", "unknown"], + ["arrayReadOnly", "literal"], + ["replacementKeepsOldObject", "literal"], + ["replacementMutatesCapturedObject", "unknown"], + ["untouched", "literal"], + ] as const) { const projection = graph.nodes .flatMap(({ observations }) => observations) .find( @@ -82,7 +130,16 @@ export function untouched() { ); expect(projection, name).toBeDefined(); expect(projection?.properties.static_return_shapes).toMatchObject([ - { value_status: name === "untouched" ? "literal" : "unknown" }, + { + value_status: status, + fields: [ + { + path: "", + state: status, + ...(status === "literal" ? { value: 1 } : {}), + }, + ], + }, ]); } }, diff --git a/tests/acceptance/applications/packageSubpathsCli.test.ts b/tests/acceptance/applications/packageSubpathsCli.test.ts index 15f734e22..869a8deba 100644 --- a/tests/acceptance/applications/packageSubpathsCli.test.ts +++ b/tests/acceptance/applications/packageSubpathsCli.test.ts @@ -141,6 +141,13 @@ cliTest.for(["CLI", "stdio MCP"] as const)( ).toBe("node_modules/fi%xture/actual.cjs"); const { graph } = await analyzeJavaScriptDirectory(surface, cli, root); + expect(graph.coverage.status).toBe("complete"); + const requireFileUrl = graph.edges.find( + ({ properties }) => + properties.module_link_kind === "require" && + properties.specifier === "file:///src/uri-dep.cjs", + ); + expect(requireFileUrl?.evidence.limitations.length).toBeGreaterThan(0); const imports = graph.edges.filter( ({ properties }) => properties.module_link_kind === "require", ); diff --git a/tests/acceptance/artifactClassification.test.ts b/tests/acceptance/artifactClassification.test.ts new file mode 100644 index 000000000..7436a4967 --- /dev/null +++ b/tests/acceptance/artifactClassification.test.ts @@ -0,0 +1,111 @@ +import { createHash } from "node:crypto"; +import { writeFile } from "node:fs/promises"; +import { join } from "node:path"; + +import { Uint8ArrayReader, Uint8ArrayWriter, ZipWriter } from "@zip.js/zip.js"; +import { expect } from "vitest"; + +import { + CPU, + fatImage, + machoImage, +} from "../../src/artifacts/apple/MachoImage.fixture.js"; +import { artifactInventoryResultSchema } from "../../src/domain/artifactGraph.js"; +import { artifactInspectionResultSchema } from "../../src/domain/artifactInspection.js"; +import { parseEvidence } from "../../src/domain/evidence.js"; +import { createTestTempDirectory } from "../fixtures/temporaryDirectory.js"; +import { cliTest } from "../support/cli/cliFixture.js"; + +cliTest.for(["jar", "zip"])( + "distinguishes JVM classes from universal Mach-O files in %s inventory", + async (extension, { cli }) => { + const root = await createTestTempDirectory("rea-class-inventory-"); + const path = join(root, `mixed.${extension}`); + // The large class can hold the entire 65-entry table suggested by its + // version bytes. Checking only the table length does not distinguish it. + const classes = [ + ["Hello.class", javaClass()], + ["Big.class", javaClass("x".repeat(4_000))], + ["renamed.bin", javaClass()], + ] as const; + const writer = new ZipWriter(new Uint8ArrayWriter()); + for (const [name, bytes] of classes) + await writer.add(name, new Uint8ArrayReader(bytes)); + const nativePaths: string[] = []; + for (const wide of [false, true]) { + for (const littleEndian of [false, true]) { + const name = `fat${wide ? 64 : 32}-${littleEndian ? "le" : "be"}`; + nativePaths.push(name); + await writer.add( + name, + new Uint8ArrayReader( + fatImage( + [{ cpu: CPU.arm64, bytes: machoImage({ cpu: CPU.arm64 }) }], + wide, + littleEndian, + ), + ), + ); + } + } + await writeFile(path, await writer.close()); + + const reply = await cli.run({ + arguments: ["inspect-artifact", path, "--json"], + }); + expect(reply.exitCode, reply.stdout + reply.stderr).toBe(0); + const inspection = artifactInspectionResultSchema.parse( + parseEvidence(reply.json).normalized_result, + ); + const inventory = artifactInventoryResultSchema.parse( + inspection.substeps[0]?.evidence.normalized_result, + ); + for (const name of nativePaths) { + expect(inventory.occurrences).toContainEqual( + expect.objectContaining({ + logical_path: name, + artifact_kind: "executable", + artifact_format: "mach-o-universal", + limitations: [], + }), + ); + } + for (const [name, bytes] of classes) { + const occurrence = inventory.occurrences.find( + (item) => item.logical_path === name, + ); + expect(occurrence).toBeDefined(); + expect.soft(occurrence?.artifact_kind, name).not.toBe("executable"); + expect.soft(occurrence?.artifact_format, name).not.toMatch(/^mach-o/u); + const sha256 = createHash("sha256").update(bytes).digest("hex"); + const node = inventory.nodes.find((item) => item.sha256 === sha256); + expect(node).toBeDefined(); + expect.soft(node?.format, name).not.toMatch(/^mach-o/u); + } + }, +); + +/** Minimal valid Java 21 class, with an optional unused UTF-8 constant. */ +const javaClass = (extra?: string): Buffer => { + const utf8 = (value: string): Buffer => { + const text = Buffer.from(value); + const header = Buffer.alloc(3); + header[0] = 1; + header.writeUInt16BE(text.length, 1); + return Buffer.concat([header, text]); + }; + const constants = [ + Buffer.from([7, 0, 2]), + utf8("Hello"), + Buffer.from([7, 0, 4]), + utf8("java/lang/Object"), + ...(extra === undefined ? [] : [utf8(extra)]), + ]; + const header = Buffer.from("cafebabe000000410000", "hex"); + header.writeUInt16BE(constants.length + 1, 8); + return Buffer.concat([ + header, + ...constants, + Buffer.from("0021000100030000000000000000", "hex"), + ]); +}; diff --git a/tests/acceptance/setup/dshClientSetup.test.ts b/tests/acceptance/setup/dshClientSetup.test.ts index 4010369cd..b544c5816 100644 --- a/tests/acceptance/setup/dshClientSetup.test.ts +++ b/tests/acceptance/setup/dshClientSetup.test.ts @@ -1,5 +1,5 @@ import { mkdir, readFile, writeFile } from "node:fs/promises"; -import { join } from "node:path"; +import { dirname, join } from "node:path"; import { Client } from "@modelcontextprotocol/client"; import { StdioClientTransport } from "@modelcontextprotocol/client/stdio"; import { expect } from "vitest"; @@ -8,6 +8,111 @@ import { z } from "zod"; import { createTestTempDirectory } from "../../fixtures/temporaryDirectory.js"; import { cliTest } from "../../support/cli/cliFixture.js"; +cliTest.for([ + { + label: "blank defaults", + harness: "", + agents: " \t ", + dshRoot: "home/.dsh", + agentsRoot: "home/.agents", + }, + { + label: "tilde roots", + harness: "~/selected harness", + agents: "~\\shared agents", + dshRoot: "home/selected harness", + agentsRoot: "home/shared agents", + }, + { + label: "relative roots", + harness: "./selected harness", + agents: "../shared agents", + dshRoot: "workspace/selected harness", + agentsRoot: "shared agents", + }, +])( + "DSH CLI uses Harness path semantics through setup, doctor and uninstall for $label", + async ({ harness, agents, dshRoot, agentsRoot }, { cli }) => { + const root = await createTestTempDirectory("rea-dsh-paths-"); + const home = join(root, "home"); + const cwd = join(root, "workspace"); + const config = join(root, dshRoot, "cordis.patch.yml"); + const skillRoot = join( + root, + agentsRoot, + "skills", + "reverse-engineer-anything", + ); + const skill = join(skillRoot, "SKILL.md"); + await mkdir(home); + await mkdir(cwd); + await mkdir(dirname(config), { recursive: true }); + const original = + "# caller patch\n- insert: [{id: caller, name: caller-plugin}]\n"; + await writeFile(config, original); + const run = (args: readonly string[]) => + cli.run({ + arguments: [...args, "--json"], + cwd, + environment: { + HOME: home, + USERPROFILE: home, + DSH_HOME: harness, + DSH_AGENTS_HOME: agents, + }, + }); + const planned = await run(["setup", "--client", "dsh", "--dry-run"]); + expect(planned.exitCode).toBe(0); + expect(planned.json).toMatchObject({ + status: "planned", + appliedActions: [], + plannedActions: expect.arrayContaining([ + expect.objectContaining({ id: "configure_client:dsh", target: config }), + expect.objectContaining({ kind: "install_skill", target: skillRoot }), + ]), + }); + expect(await readFile(config, "utf8")).toBe(original); + await expect(readFile(skill)).rejects.toMatchObject({ code: "ENOENT" }); + const configured = await run(["setup", "--client", "dsh", "--yes"]); + expect(configured.exitCode).toBe(0); + expect(configured.json).toMatchObject({ + status: "ready", + clients: { dsh: { status: "configured" } }, + }); + const text = await readFile(config, "utf8"); + expect(text).toContain("# caller patch"); + expect(text).toContain("rea-mcp"); + expect(await readFile(`${config}.rea.backup`, "utf8")).toBe(original); + expect(await readFile(skill, "utf8")).toContain( + "name: reverse-engineer-anything", + ); + const doctor = await run(["doctor", "--client", "dsh", "--skill"]); + expect(doctor.exitCode).toBe(0); + expect(doctor.json).toMatchObject({ healthy: true }); + const repeated = await run(["setup", "--client", "dsh", "--yes"]); + expect(repeated.exitCode).toBe(0); + expect(repeated.json).toMatchObject({ + status: "ready", + plannedActions: [], + appliedActions: [], + }); + expect(await readFile(config, "utf8")).toBe(text); + const removed = await run(["uninstall"]); + expect(removed.exitCode).toBe(0); + expect(removed.json).toMatchObject({ + status: "complete", + items: expect.arrayContaining([ + expect.objectContaining({ name: "dsh", status: "removed" }), + ]), + }); + expect(parse(await readFile(config, "utf8"))).toEqual([ + { insert: [{ id: "caller", name: "caller-plugin" }] }, + ]); + expect(await readFile(`${config}.rea.backup`, "utf8")).toBe(original); + await expect(readFile(skill)).rejects.toMatchObject({ code: "ENOENT" }); + }, +); + cliTest( "DSH CLI previews, approves, repairs, launches MCP, repeats and uninstalls while preserving caller YAML", async ({ cli }) => { diff --git a/tests/boundary/binaryDiagnostics/pwntoolsInputBoundAdmission.test.ts b/tests/boundary/binaryDiagnostics/pwntoolsInputBoundAdmission.test.ts new file mode 100644 index 000000000..ca9f0bc2a --- /dev/null +++ b/tests/boundary/binaryDiagnostics/pwntoolsInputBoundAdmission.test.ts @@ -0,0 +1,89 @@ +import { open } from "node:fs/promises"; +import { join } from "node:path"; + +import { expect, it, onTestFinished } from "vitest"; + +import { PwntoolsLayoutProvider } from "../../../src/native/pwntools/PwntoolsLayoutProvider.js"; +import { projectAnalysisError } from "../../../src/domain/analysisErrorProjection.js"; +import { spawnOwnedProviderProcess } from "../../../src/process/ProviderProcess.js"; +import { createTestTempDirectory } from "../../fixtures/temporaryDirectory.js"; + +const unsupportedHost = process.platform !== "linux" || process.arch !== "x64"; +const DEFAULT_INPUT_BYTES = 33_554_432; +// One byte above the shipped bound, so the admission decision is the only +// difference between the first two cases. +const ARTIFACT_BYTES = DEFAULT_INPUT_BYTES + 1; + +/** Sparse regular file just larger than the shipped input bound. */ +const oversizedArtifact = async (): Promise => { + const root = await createTestTempDirectory("rea-pwntools-input-bound-"); + const path = join(root, "recorded-core"); + const handle = await open(path, "w"); + await handle.truncate(ARTIFACT_BYTES); + await handle.close(); + return path; +}; + +/** One admission attempt, reported as the projected caller-visible failure. */ +const inspect = async (override?: string) => { + const path = await oversizedArtifact(); + const provider = new PwntoolsLayoutProvider( + { + REA_PWNTOOLS_PYTHON: process.execPath, + ...(override === undefined + ? {} + : { REA_PWNTOOLS_MAX_INPUT_BYTES: override }), + }, + async (spawn) => + spawnOwnedProviderProcess({ + ...spawn, + command: process.execPath, + arguments: ["-e", "process.stdout.write('no reply')"], + }), + ); + onTestFinished(async () => { + await provider.close(); + }); + const result = await provider.inspect({ path }); + if (result.ok) throw new Error("Expected the bounded decoder to fail"); + const projected = projectAnalysisError(result.error); + return { projected, text: JSON.stringify(projected) }; +}; + +it.runIf(!unsupportedHost)( + "refuses an artifact above the shipped bound when no override is set", + async () => { + const { projected, text } = await inspect(); + expect(projected.code).toBe("invalid_request"); + expect(text).toContain( + `Selected artifact exceeds ${String(DEFAULT_INPUT_BYTES)} bytes`, + ); + }, +); + +it.runIf(!unsupportedHost)( + "admits the same artifact once the override widens the bound", + async () => { + const { projected, text } = await inspect(String(ARTIFACT_BYTES)); + // Admission passed, so the only remaining failure is that the owned decoder + // never wrote a reply, which also proves the process really ran. + expect(text).not.toContain("Selected artifact exceeds"); + expect(projected.code).toBe("unreadable_output"); + expect(text).toContain("no reply"); + }, +); + +it.runIf(!unsupportedHost)( + "refuses an unusable override before reading the artifact", + async () => { + const { projected, text } = await inspect("0"); + expect(projected.code).toBe("provider_unavailable"); + expect(projected.details).toMatchObject({ + rejections: [ + { diagnostics: { configuration_key: "REA_PWNTOOLS_MAX_INPUT_BYTES" } }, + ], + }); + expect(text).toContain("REA_PWNTOOLS_MAX_INPUT_BYTES must be a decimal"); + expect(text).not.toContain("Selected artifact exceeds"); + }, +); diff --git a/tests/boundary/browser/cdpDomSvgIdentity.test.ts b/tests/boundary/browser/cdpDomSvgIdentity.test.ts new file mode 100644 index 000000000..7d42ae336 --- /dev/null +++ b/tests/boundary/browser/cdpDomSvgIdentity.test.ts @@ -0,0 +1,315 @@ +import { describe, expect, it } from "vitest"; + +import { CdpBrowserProvider } from "../../../src/browser/CdpBrowserProvider.js"; +import { inspectWebPageInputSchema } from "../../../src/domain/browserObservation.js"; +import { + startFakeCdpBrowser, + type FakeCdpBrowser, + type FakeOptions, +} from "../../fixtures/fakeCdpBrowser.js"; +import { trackBrowser } from "./cdpBrowserProvider.support.js"; + +const startBrowser = async (options: FakeOptions = {}) => { + const browser = await startFakeCdpBrowser({ + ...options, + commandResult(command, origin, frameReads) { + const override = options.commandResult?.(command, origin, frameReads); + if (override !== undefined) return override; + if (command.method === "DOMSnapshot.captureSnapshot") { + const nodes = { + nodeType: [9, 1, 1, 1, 1], + nodeName: [1, 2, 3, 3, 3], + nodeValue: [4, 4, 4, 4, 4], + parentIndex: [-1, 0, 1, 1, 1], + backendNodeId: [1, 2, 3, 4, 5], + attributes: [[], [], [5, 6], [5, 7], [5, 6, 8, 9]], + }; + return { + strings: [ + `${origin}/app`, + "#document", + "HTML", + "a", + "", + "xlink:href", + "/first", + "/second", + "href", + "/ordinary", + "https://excluded.example/page", + ], + documents: [ + { documentURL: 0, nodes }, + { + documentURL: 10, + nodes: { ...nodes, backendNodeId: [11, 12, 13, 14, 15] }, + }, + ], + }; + } + if (command.method === "DOM.describeNode") + return { + node: { + backendNodeId: command.params.backendNodeId, + nodeType: 1, + nodeName: "a", + isSVG: true, + }, + }; + return undefined; + }, + }); + trackBrowser(browser); + return browser; +}; + +const inspect = (browser: FakeCdpBrowser, signal?: AbortSignal) => + new CdpBrowserProvider().inspectPage( + inspectWebPageInputSchema.parse({ + cdp_endpoint: browser.endpoint, + target_id: "allowed-page", + allowed_origins: [browser.allowedOrigin], + observation_ms: 0, + }), + signal === undefined ? undefined : { signal }, + ); + +describe("CDP SVG identity at the snapshot boundary", () => { + it("describes only in-scope XLink candidates without a winning href", async () => { + const browser = await startBrowser(); + const result = await inspect(browser); + if (!result.ok) throw result.error; + expect(result.value.metadata.dom_urls.map(({ url }) => url)).toEqual( + ["/first", "/second", "/ordinary"].map( + (path) => browser.allowedOrigin + path, + ), + ); + expect( + browser.commands + .filter(({ method }) => method === "DOM.describeNode") + .map(({ params }) => params), + ).toEqual([ + { backendNodeId: 3, depth: 0 }, + { backendNodeId: 4, depth: 0 }, + ]); + }); + + it.each(["removed", "mismatched"])( + "retains partial metadata when a candidate is %s", + async (failure) => { + const browser = await startBrowser({ + commandError: (command) => + failure === "removed" && + command.method === "DOM.describeNode" && + command.params.backendNodeId === 3 + ? { code: -32_000, message: "No node found for given backend id" } + : undefined, + commandResult: (command) => + failure === "mismatched" && + command.method === "DOM.describeNode" && + command.params.backendNodeId === 3 + ? { + node: { + backendNodeId: 99, + nodeType: 1, + nodeName: "a", + isSVG: true, + }, + } + : undefined, + }); + const result = await inspect(browser); + if (!result.ok) throw result.error; + expect(result.value.metadata.dom_urls.map(({ url }) => url)).toEqual( + ["/second", "/ordinary"].map((path) => browser.allowedOrigin + path), + ); + expect(result.value.completeness.excluded).toContainEqual({ + section: "metadata", + reason: "provider_unavailable", + count: 1, + }); + expect(result.value.limitations).toContain( + "SVG identity was unavailable for 1 snapshotted XLink candidate nodes; their URL destinations remain unknown.", + ); + }, + ); + + it("stops querying after the first batch reports an unsupported method", async () => { + const browser = await startBrowser({ + commandResult: manyCandidates, + unsupportedMethods: ["DOM.describeNode"], + }); + const result = await inspect(browser); + if (!result.ok) throw result.error; + expect(result.value.metadata.dom_urls).toEqual([]); + expect(result.value.completeness.excluded).toContainEqual({ + section: "metadata", + reason: "provider_unavailable", + count: 25, + }); + expect(descriptions(browser).length).toBeGreaterThan(0); + expect(descriptions(browser).length).toBeLessThanOrEqual(8); + }); + + it.each([ + { options: { closeOnMethod: "DOM.describeNode" }, reason: "disconnected" }, + { + options: { + commandError: (command: { method: string }) => + command.method === "DOM.describeNode" + ? { code: -32_602, message: "Invalid parameters" } + : undefined, + }, + reason: "protocol_error", + }, + ])( + "propagates $reason during node description", + async ({ options, reason }) => { + const browser = await startBrowser(options); + expect(await inspect(browser)).toMatchObject({ + ok: false, + error: { _tag: "BrowserObservationError", reason }, + }); + }, + ); + + it("cancels a pending description and detaches without closing the target", async () => { + const browser = await startBrowser({ hangOnMethod: "DOM.describeNode" }); + const controller = new AbortController(); + const pending = inspect(browser, controller.signal); + await expect.poll(() => descriptions(browser).length).toBeGreaterThan(1); + controller.abort(); + expect(await pending).toMatchObject({ + ok: false, + error: { _tag: "AnalysisCancelledError", operation: "inspect_web_page" }, + }); + const methods = browser.commands.map(({ method }) => method); + expect(methods).toContain("Target.detachFromTarget"); + expect(methods).not.toContain("Target.closeTarget"); + expect(methods).not.toContain("Browser.close"); + }); +}); + +const manyCandidates: NonNullable = ( + command, + origin, +) => { + if (command.method !== "DOMSnapshot.captureSnapshot") return undefined; + const count = 25; + return { + strings: [ + origin, + "#document", + "HTML", + "a", + "", + "xlink:href", + ...Array.from({ length: count }, (_, index) => `/link-${String(index)}`), + ], + documents: [ + { + documentURL: 0, + nodes: { + nodeType: [9, 1, ...Array(count).fill(1)], + nodeName: [1, 2, ...Array(count).fill(3)], + nodeValue: Array(count + 2).fill(4), + parentIndex: [-1, 0, ...Array(count).fill(1)], + backendNodeId: Array.from( + { length: count + 2 }, + (_, index) => index + 1, + ), + attributes: [ + [], + [], + ...Array.from({ length: count }, (_, index) => [5, index + 6]), + ], + }, + }, + ], + }; +}; + +const descriptions = (browser: FakeCdpBrowser) => + browser.commands.filter(({ method }) => method === "DOM.describeNode"); + +describe("CDP SVG query concurrency", () => { + it("bounds in-flight reads and preserves metadata order across reversed replies", async () => { + const browser = await startBrowser({ + commandResult: manyCandidates, + hangOnMethod: "DOM.describeNode", + }); + const controller = new AbortController(); + const pending = inspect(browser, controller.signal); + try { + await expect.poll(() => descriptions(browser).length).toBeGreaterThan(1); + let replied = 0; + while (replied < 25) { + await expect + .poll(() => descriptions(browser).length) + .toBeGreaterThan(replied); + const batch = descriptions(browser).slice(replied); + expect(batch.length).toBeLessThanOrEqual(8); + for (const command of [...batch].reverse()) { + const id = Number(command.params.backendNodeId); + browser.emitRawMessage( + JSON.stringify({ + id: command.id, + result: { + node: { + backendNodeId: id, + nodeType: 1, + nodeName: "a", + isSVG: id % 2 === 1, + }, + }, + }), + ); + } + replied += batch.length; + } + const result = await pending; + if (!result.ok) throw result.error; + expect(result.value.metadata.dom_urls.map(({ url }) => url)).toEqual( + Array.from( + { length: 13 }, + (_, index) => `${browser.allowedOrigin}/link-${String(index * 2)}`, + ), + ); + expect(descriptions(browser)).toHaveLength(25); + } finally { + controller.abort(); + await pending; + } + }); + + it("cancels sibling reads on a fatal reply without waiting for hung nodes", async () => { + const browser = await startBrowser({ + commandResult: manyCandidates, + hangOnMethod: "DOM.describeNode", + }); + const controller = new AbortController(); + const pending = inspect(browser, controller.signal); + try { + await expect.poll(() => descriptions(browser).length).toBeGreaterThan(1); + const batch = descriptions(browser); + browser.emitRawMessage( + JSON.stringify({ + id: batch.at(-1)?.id, + error: { code: -32_602, message: "Invalid parameters" }, + }), + ); + await expect(pending).resolves.toMatchObject({ + ok: false, + error: { _tag: "BrowserObservationError", reason: "protocol_error" }, + }); + expect(descriptions(browser)).toHaveLength(batch.length); + const methods = browser.commands.map(({ method }) => method); + expect(methods).toContain("Target.detachFromTarget"); + expect(methods).not.toContain("Target.closeTarget"); + expect(methods).not.toContain("Browser.close"); + } finally { + controller.abort(); + await pending; + } + }); +}); diff --git a/tests/boundary/browser/cdpSvgXlinkHref.test.ts b/tests/boundary/browser/cdpSvgXlinkHref.test.ts index 56c3fc3cf..38048861b 100644 --- a/tests/boundary/browser/cdpSvgXlinkHref.test.ts +++ b/tests/boundary/browser/cdpSvgXlinkHref.test.ts @@ -45,8 +45,38 @@ const pageHtml = `x out + move desc + move title + + not an SVG link + HTML link + nested + + + <a xlink:href="/xl-html-title">not an SVG link</a> + <a href="/html-title">HTML link</a> + <svg><a xlink:href="/xl-nested-title"><text>nested</text></a></svg> + html + `; it.skipIf(chromeExecutable === undefined)( @@ -143,17 +173,36 @@ it.skipIf(chromeExecutable === undefined)( url: null, destination_scope: "outside_policy", }), + ...[ + "/html-desc", + "/html-title", + "/xl-nested-desc", + "/xl-nested-title", + "/xl-moved-desc", + "/xl-moved-title", + "/xl-created-desc", + "/xl-created-title", + "/xl-created-nested-desc", + "/xl-created-nested-title", + ].map((path) => + expect.objectContaining({ + attribute: "href", + url: `${origin}${path}`, + destination_scope: "approved", + }), + ), ]), ); expect( urls.some( (url) => - url.url?.endsWith("/xl-html") === true || + url.url?.includes("/xl-html") === true || + url.url?.includes("/xl-math") === true || url.url?.endsWith("/xlink-loses") === true, ), ).toBe(false); expect(urls.every((url) => url.attribute === "href")).toBe(true); - expect(urls).toHaveLength(5); + expect(urls).toHaveLength(15); } finally { child.kill("SIGTERM"); if (!(await waitForExit(child, 2_000))) child.kill("SIGKILL"); @@ -173,6 +222,8 @@ const xhtmlPage = ` p + desc + title no no @@ -202,6 +253,8 @@ for (const { format, page, contentType, nodeNames, link, destinations } of [ destinations: [ "/xl-pa1", "/xl-image", + "/xl-xml-desc", + "/xl-xml-title", "/xl-fo-svg", "/xl-orphan", "/xl-custom", diff --git a/tests/boundary/cli/cliOutput.test.ts b/tests/boundary/cli/cliOutput.test.ts index 43d517e16..44b3ac234 100644 --- a/tests/boundary/cli/cliOutput.test.ts +++ b/tests/boundary/cli/cliOutput.test.ts @@ -58,6 +58,95 @@ describe("CLI output argument boundary", () => { }); }); +describe("unclosed CLI output filter slices", () => { + it.each([ + "providers[", + "providers[0,1].name[", + "name,providers[", + "][,]", + "providers],name[0,1]", + ])("rejects %s before the filter parser runs", (filter) => { + expect( + validateCliOutputArguments([ + "providers", + "--json", + "--filter-output", + filter, + ]), + ).toMatchObject({ ok: false, code: "INVALID_OUTPUT_FILTER" }); + }); + + it("preserves effective-filter and earlier flag-error precedence", () => { + for (const arguments_ of [ + [ + "--filter-output", + "providers[", + "--filter-output", + "capabilities[0,1].provider.id,open", + ], + ["--filter-output", "capabilities[-1,].provider.id"], + ["--filter-output", "capabilities[[0,1]"], + ["--format", "invalid", "--filter-output", "providers["], + ["--token-limit", "invalid", "--filter-output", "providers["], + ]) + expect(validateCliOutputArguments(arguments_)).toEqual({ ok: true }); + }); + + workspaceCliTest( + "fails promptly across output formats without invoking the command", + async ({ cli }) => { + for (const format of ["json", "jsonl", "yaml", "toon", "md"]) { + const result = await cli.run({ + arguments: [ + "analyze", + "/absent/target", + "--format", + format, + "--filter-output", + "result[0,1].name[", + ], + timeoutMs: 3000, + }); + expect(result.exitCode).toBe(1); + expect(result.stdout).toContain("INVALID_OUTPUT_FILTER"); + expect(result.stdout).toContain("closing ]"); + expect(result.stderr).toBe(""); + } + const splitPath = await cli.run({ + arguments: ["providers", "--json", "--filter-output", "][,]"], + timeoutMs: 3000, + }); + expect(splitPath.exitCode).toBe(1); + expect(splitPath.json).toMatchObject({ + ok: false, + error: { code: "INVALID_OUTPUT_FILTER" }, + }); + }, + CLI_INTEGRATION_TIMEOUT_MS, + ); + + workspaceCliTest( + "retains closed slices and multiple filter paths", + async ({ cli }) => { + const result = await cli.run({ + arguments: [ + "providers", + "--json", + "--filter-output", + "capabilities[0,1].provider.id,open", + ], + timeoutMs: 3000, + }); + expect(result.exitCode).toBe(0); + expect(result.json).toEqual({ + capabilities: [{ provider: { id: expect.any(String) } }], + open: false, + }); + }, + CLI_INTEGRATION_TIMEOUT_MS, + ); +}); + describe("CLI output valued-flag parsing", () => { workspaceCliTest( "preserves complete JSON errors when native builtin flags follow unusual arguments", diff --git a/tests/boundary/cli/dylibResolutionCli.test.ts b/tests/boundary/cli/dylibResolutionCli.test.ts index 66fa21ba3..21ea99a9c 100644 --- a/tests/boundary/cli/dylibResolutionCli.test.ts +++ b/tests/boundary/cli/dylibResolutionCli.test.ts @@ -180,14 +180,17 @@ describe("trace-dylib-resolution CLI", () => { // A root below a symlinked directory that points outside the bundle. const outside = await createTestTempDirectory("rea-dylib-outside-"); await writeFiles(outside, { Tool: machoImage({}) }); - await symlink(outside, join(app, "Contents/Linked")); + await symlink(outside, join(app, "Contents/Linked"), "dir"); for (const [root, reason, message] of [ ["../outside", "invalid_format", "Use a normalized path"], ["Contents/Info.plist", "invalid_value", "is not a Mach-O image"], [ "Contents/Linked/Tool", "invalid_value", - "resolves outside the analyzed root", + // Windows link text is outside the POSIX dyld path representation. + process.platform === "win32" + ? "is not a regular file in the analyzed root" + : "resolves outside the analyzed root", ], ] as const) { await expectRootInputError(cli, { app, root, reason, message }); @@ -378,7 +381,8 @@ cliTest( }, ); -describe.skipIf(process.getuid?.() === 0)( +// This permission fixture requires POSIX mode bits and a non-root caller. +describe.skipIf(process.platform === "win32" || process.getuid?.() === 0)( "trace-dylib-resolution permissions", () => { cliTest( @@ -510,6 +514,8 @@ cliTest( app, "--root", "Contents/MacOS/App", + "--architecture", + "arm64e", "--json", ], environment: ENVIRONMENT, @@ -518,9 +524,7 @@ cliTest( const trace = dylibResolutionResultSchema.parse( parseEvidence(result.json).normalized_result, ); - expect( - trace.roots.filter(({ architecture }) => architecture === "arm64e"), - ).toHaveLength(2); + expect(trace.roots).toHaveLength(2); const environments = trace.findings .filter(({ kind }) => kind === "dyld-environment-present") .map(({ explanation }) => diff --git a/tests/boundary/cli/idaRetainedCleanupCli.test.ts b/tests/boundary/cli/idaRetainedCleanupCli.test.ts new file mode 100644 index 000000000..b31903489 --- /dev/null +++ b/tests/boundary/cli/idaRetainedCleanupCli.test.ts @@ -0,0 +1,92 @@ +import { describe, expect } from "vitest"; + +import { workspaceCliTest } from "../../support/cli/workspaceCliFixture.js"; + +import { elf } from "../../../src/domain/binaryTarget.fixture.js"; + +/** + * A headless IDA supervisor that completes the MCP handshake and then leaves + * every tool call unanswered, as an `idb_open` still running auto-analysis + * would. It exits when REA closes its input. + */ +const UNRESPONSIVE_SUPERVISOR = `import { createInterface } from "node:readline"; +const tools = ["idb_open", "idb_list", "idb_close", "server_health", "list_funcs", + "lookup_funcs", "find_regex", "decompile", "disasm", "callees", "xrefs_to"]; +const send = (message) => + process.stdout.write(JSON.stringify({ jsonrpc: "2.0", ...message }) + "\\n"); +for await (const line of createInterface({ input: process.stdin })) { + const request = JSON.parse(line); + if (request.method === "initialize") + send({ id: request.id, result: { + protocolVersion: request.params.protocolVersion, + capabilities: { tools: {} }, + serverInfo: { name: "unresponsive-ida-fixture", version: "1" }, + } }); + else if (request.method === "tools/list") + send({ id: request.id, result: { + tools: tools.map((name) => ({ name, inputSchema: { type: "object" } })), + } }); +} +`; + +describe("headless IDA timeout through the one-shot CLI", () => { + workspaceCliTest( + "exits after a retained-cleanup failure instead of waiting on the supervisor", + async ({ cli, workspace }) => { + const supervisor = await workspace.write( + "supervisor.mjs", + UNRESPONSIVE_SUPERVISOR, + ); + const workspaceRoot = await workspace.mkdir("ida-workspaces"); + const registration = await workspace.write( + "ida-mcp.json", + JSON.stringify({ + command: process.execPath, + args: [supervisor], + mode: "headless", + timeoutMs: 500, + workspaceRoot, + }), + ); + const target = await workspace.write("target.elf", elf(2, 1, 62)); + + const result = await cli.run({ + arguments: ["decompile", target, "main", "--json"], + environment: { + ...workspace.environment, + REA_ANALYSIS_PROVIDER: "ida", + REA_IDA_MCP_CONFIG: registration, + REA_LOG_LEVEL: "silent", + }, + timeoutMs: 30_000, + }); + + expect(result.exitCode).toBe(1); + expect(result.json).toMatchObject({ + code: "cleanup_incomplete", + details: { + provider_id: "ida", + cleanup: "incomplete", + resources: [expect.stringContaining(workspaceRoot)], + cleanup_error: { + details: { + diagnostics: { + upstream_sessions: { + state: "unverified", + sessions: [ + { + session_id: expect.stringMatching(/^rea-/), + backend: "worker", + owned: null, + input_path: null, + }, + ], + }, + }, + }, + }, + }, + }); + }, + ); +}); diff --git a/tests/boundary/cli/jsonOutput.test.ts b/tests/boundary/cli/jsonOutput.test.ts index 5e8e86e7f..0d1ee5a59 100644 --- a/tests/boundary/cli/jsonOutput.test.ts +++ b/tests/boundary/cli/jsonOutput.test.ts @@ -561,3 +561,35 @@ describe("CLI result output surface", () => { }); }, 60_000); }); + +describe("streamed CLI filtered selections", () => { + it.each([ + ["raw_result", "json", "null\n"], + ["raw_result", "jsonl", "null\n"], + ["nested.value", "json", '{"nested":{"value":null}}\n'], + ["missing", "json", "{}\n"], + ["count", "json", "0\n"], + ["enabled", "json", "false\n"], + ] as const)( + "keeps the selected value of --filter-output %s in %s", + async (filter, format, expected) => { + const chunks: Buffer[] = []; + const destination = new Writable({ + write(chunk: Buffer, _encoding, callback) { + chunks.push(chunk); + callback(); + }, + }); + const output = createCliResultOutput( + ["--format", format, "--filter-output", filter], + destination, + ); + if (output === undefined) throw new Error("Missing JSON output surface"); + await output.write( + { raw_result: null, nested: { value: null }, count: 0, enabled: false }, + { command: "fixture", duration: "1ms", format }, + ); + expect(Buffer.concat(chunks).toString("utf8")).toBe(expected); + }, + ); +}); diff --git a/tests/boundary/cli/nativeMachoEchoedPathCli.test.ts b/tests/boundary/cli/nativeMachoEchoedPathCli.test.ts index 0c362d263..a3118c1f0 100644 --- a/tests/boundary/cli/nativeMachoEchoedPathCli.test.ts +++ b/tests/boundary/cli/nativeMachoEchoedPathCli.test.ts @@ -1,9 +1,13 @@ import { copyFile } from "node:fs/promises"; -import { join } from "node:path"; +import { join, resolve } from "node:path"; import { describe, expect } from "vitest"; import { z } from "zod"; +import { + inspectMachoSchema, + listArchitecturesSchema, +} from "../../../src/domain/native/nativeInspection.js"; import { createTestTempDirectory } from "../../fixtures/temporaryDirectory.js"; import { cliTest } from "../../support/cli/cliFixture.js"; @@ -34,27 +38,68 @@ const resultSchema = z.object({ describe.skipIf(process.platform !== "darwin")( "native Mach-O CLI with echoed path text", () => { - cliTest( - "does not read load commands or symbols from the file name", - async ({ cli }) => { - const directory = await createTestTempDirectory("rea-macho-echo-"); - const inspect = async (name: string) => { - const path = join(directory, name); - await copyFile("/usr/bin/true", path); - const result = await cli.run({ - arguments: ["inspect-macho", path, "--json"], - environment, - }); - expect(result.exitCode).toBe(0); - const { provenance: _provenance, ...facts } = resultSchema.parse( - result.json, - ).normalized_result; - return facts; - }; - const forged = await inspect(FORGED_NAME); - expect(forged).toEqual(await inspect("tool")); - expect(JSON.stringify(forged)).not.toMatch(/forged|4242/u); - }, - ); + const lipoName = "tool\narchitecture forged\n offset 123\n size 32"; + for (const [facts, command, filename] of [ + ["load commands or symbols", "inspect-macho", FORGED_NAME], + ["FAT architecture records", "inspect-macho", lipoName], + ["FAT architecture inventory", "list-architectures", lipoName], + ["thin-file framing", "inspect-macho", "tool\nNon-fat file: forged"], + ] as const) { + cliTest( + `does not read ${facts} from the file name`, + async ({ cli, processes }) => { + const directory = await createTestTempDirectory( + "rea-native-cli-echo-", + ); + const fixture = join(directory, "fat-fixture"); + const compiled = await processes.run( + "/usr/bin/xcrun", + [ + "clang", + "-arch", + "arm64", + "-arch", + "x86_64", + resolve("tests/conformance/c/fixture.c"), + "-o", + fixture, + ], + { timeoutMs: 30_000 }, + ); + expect(compiled.exitCode, compiled.stderr).toBe(0); + const inspect = async (name: string) => { + const path = join(directory, name); + await copyFile(fixture, path); + const result = await cli.run({ + arguments: [command, path, "--json"], + environment, + }); + expect(result.exitCode).toBe(0); + const { provenance: _provenance, ...facts } = resultSchema.parse( + result.json, + ).normalized_result; + return facts; + }; + const ordinary = await inspect("tool"); + // Prove this comparison exercised FAT framing, independent of host binaries. + const architectures = + listArchitecturesSchema.shape.architectures.parse( + ordinary.architectures, + ); + expect(architectures.total).toBeGreaterThan(1); + if (command === "inspect-macho") { + const selected = + inspectMachoSchema.shape.selected_architecture.parse( + ordinary.selected_architecture, + ); + expect(selected.container_file_offset).toBeGreaterThan(0); + } + const forged = await inspect(filename); + expect(forged).toEqual(ordinary); + expect(JSON.stringify(forged)).not.toMatch(/forged|4242/u); + }, + 60_000, + ); + } }, ); diff --git a/tests/boundary/cli/rawBinaryCli.test.ts b/tests/boundary/cli/rawBinaryCli.test.ts index 1cff4cac6..f12312d02 100644 --- a/tests/boundary/cli/rawBinaryCli.test.ts +++ b/tests/boundary/cli/rawBinaryCli.test.ts @@ -38,6 +38,36 @@ cliTest( }, ); +cliTest( + "rejects a 6502 image crossing the 16-bit boundary before provider startup", + async ({ cli }) => { + const root = await createTestTempDirectory("rea-6502-cli-overflow-"); + const path = join(root, "image.bin"); + await writeFile(path, Buffer.from("a92a60", "hex")); + const result = await cli.run({ + arguments: [ + "inspect", + path, + "--target-format", + "raw-binary", + "--architecture", + "6502", + "--entry-point", + "0xffff", + "--provider", + "ghidra", + "--json", + ], + environment: { HOME: root, XDG_CONFIG_HOME: root, XDG_CACHE_HOME: root }, + }); + expect(result.exitCode).toBe(1); + expect(result.json).toMatchObject({ + code: "invalid_request", + details: { reason: expect.stringContaining("6502 address width") }, + }); + }, +); + cliTest( "rejects an unaligned AArch64 entry before provider startup", async ({ cli }) => { diff --git a/tests/boundary/cli/referenceSourceDirectoryImports.test.ts b/tests/boundary/cli/referenceSourceDirectoryImports.test.ts new file mode 100644 index 000000000..6270855cb --- /dev/null +++ b/tests/boundary/cli/referenceSourceDirectoryImports.test.ts @@ -0,0 +1,118 @@ +import { mkdir, writeFile } from "node:fs/promises"; +import { createRequire } from "node:module"; +import { dirname, join } from "node:path"; + +import { describe, expect } from "vitest"; + +import { historicalSourceGraphSchema } from "../../../src/domain/referenceSourceGraph.js"; +import { createTestTempDirectory } from "../../fixtures/temporaryDirectory.js"; +import { cliTest } from "../../support/cli/cliFixture.js"; + +// Reference-source reads require no-follow opens; Windows reports unsupported. +describe.skipIf(process.platform === "win32")( + "compiled reference-source directory imports", + () => { + cliTest( + "resolves directory spellings without selecting similarly named files", + async ({ cli }) => { + const workspace = await createTestTempDirectory( + "rea-reference-directory-imports-", + ); + const root = join(workspace, "source"); + const imports = [ + ["slash.cjs", "./lib/", "lib/index.js"], + ["dot.cjs", "./lib/.", "lib/index.js"], + ["self.cjs", ".", "index.js"], + ["self-slash.cjs", "./", "index.js"], + ["nested/parent.cjs", "..", "index.js"], + ["nested/parent-slash.cjs", "../", "index.js"], + ["direct.cjs", "./lib.js", "lib.js"], + ["fallback.cjs", "./fallback", "fallback/index.js"], + ["legacy-order.cjs", "./dep", "dep/index.js"], + ] as const; + const unresolved = [ + ["missing.cjs", "./missing/", "missing/"], + ["escape.cjs", "../outside/", "../outside/"], + ["main.ts", "./folder.js/.", "folder.js"], + ] as const; + const files = new Map([ + ["index.js", 'module.exports = "root index";\n'], + ["lib/index.js", 'module.exports = "library index";\n'], + ["lib.js", 'module.exports = "sibling JavaScript file";\n'], + ["lib.ts", 'export const value = "sibling TypeScript file";\n'], + ["fallback/index.js", 'module.exports = "fallback index";\n'], + ["dep/index.js", 'module.exports = "existing index priority";\n'], + ["dep.cjs", 'module.exports = "later CommonJS candidate";\n'], + [ + "folder.ts", + 'export const value = "emitted filename counterpart";\n', + ], + ["missing.js", 'module.exports = "not a directory";\n'], + ...[...imports, ...unresolved].map( + ([path, specifier]) => + [path, `require(${JSON.stringify(specifier)});\n`] as const, + ), + ]); + await Promise.all( + [...files].map(async ([path, contents]) => { + const target = join(root, path); + await mkdir(dirname(target), { recursive: true }); + await writeFile(target, contents); + }), + ); + const outside = join(workspace, "outside", "index.js"); + await mkdir(dirname(outside), { recursive: true }); + await writeFile(outside, 'module.exports = "outside import root";\n'); + + // Node's loader establishes each fixture's meaning independently. + for (const [path, specifier, target] of imports) + expect(createRequire(join(root, path)).resolve(specifier)).toBe( + join(root, target), + ); + expect(() => + createRequire(join(root, "missing.cjs")).resolve("./missing/"), + ).toThrow(); + expect(() => + createRequire(join(root, "main.ts")).resolve("./folder.js/."), + ).toThrow(); + expect( + createRequire(join(root, "escape.cjs")).resolve("../outside/"), + ).toBe(outside); + + const result = await cli.run({ + arguments: ["import-reference-source", root, "--json"], + environment: { + HOME: workspace, + XDG_CONFIG_HOME: workspace, + XDG_CACHE_HOME: workspace, + }, + }); + expect(result.exitCode).toBe(0); + const graph = historicalSourceGraphSchema.parse(result.json); + expect(graph.parse_failures).toEqual([]); + expect(graph.relationships).toHaveLength( + imports.length + unresolved.length, + ); + for (const [from, , to] of imports) + expect.soft(graph.relationships, from).toContainEqual({ + from_path: from, + to, + kind: "requires", + resolution: "internal", + parse_state: "parsed", + }); + for (const [from, , to] of unresolved) + expect.soft(graph.relationships, from).toContainEqual({ + from_path: from, + to, + kind: "requires", + resolution: "unresolved", + parse_state: "parsed", + }); + expect( + graph.entries.some((entry) => entry.path.includes("outside")), + ).toBe(false); + }, + ); + }, +); diff --git a/tests/boundary/cli/scenarioCliInputDiagnostics.test.ts b/tests/boundary/cli/scenarioCliInputDiagnostics.test.ts index bc98d9d9f..ae03568f4 100644 --- a/tests/boundary/cli/scenarioCliInputDiagnostics.test.ts +++ b/tests/boundary/cli/scenarioCliInputDiagnostics.test.ts @@ -1,3 +1,5 @@ +import { join } from "node:path"; +import { createTestTempDirectory } from "../../fixtures/temporaryDirectory.js"; import { describe, expect } from "vitest"; import { cliTest } from "../../support/cli/cliFixture.js"; @@ -62,3 +64,42 @@ describe("scenario CLI schema diagnostics", () => { }); }); }); + +for (const field of [ + "executable_path", + "application_path", + "application_root", +] as const) { + cliTest(`names the unresolved Electron ${field}`, async ({ cli }) => { + const root = await createTestTempDirectory("rea-electron-path-"); + const missing = join(root, "missing"); + const result = await cli.run({ + arguments: [ + "capture-electron-scenario", + JSON.stringify({ + executable_path: process.execPath, + application_path: root, + application_root: root, + actions: [], + [field]: missing, + }), + "--json", + ], + environment: { REA_LOG_LEVEL: "silent" }, + }); + expect(result.exitCode).toBe(1); + expect(result.json).toMatchObject({ + code: "invalid_request", + details: { + issues: [ + { + path: [field], + reason: "invalid_value", + message: expect.stringContaining("ENOENT"), + }, + ], + }, + }); + expect(JSON.stringify(result.json)).toContain(missing); + }); +} diff --git a/tests/boundary/filesystem/artifactZipIntegrity.test.ts b/tests/boundary/filesystem/artifactZipIntegrity.test.ts index 41397c520..cf7dcd595 100644 --- a/tests/boundary/filesystem/artifactZipIntegrity.test.ts +++ b/tests/boundary/filesystem/artifactZipIntegrity.test.ts @@ -366,16 +366,11 @@ it("rejects overlapping ZIP member data ranges through artifact inventory", asyn bytes.writeUInt32LE(firstLocalOffset, second + 42); await writeFile(path, bytes); - const target = await parseBinaryTarget(path); - if (!target.ok) throw target.error; - const result = await withArtifactClient(target.value, (client) => - client.execute("inventory_artifact", {}), + await expectInventoryFailure( + path, + "format", + "ZIP archive is malformed: Overlapping entry found", ); - - expect(result).toMatchObject({ - ok: false, - error: { _tag: "ArtifactOperationError", reason: "io" }, - }); }); it("preserves encrypted-member and corrupt-CRC observations", async () => { @@ -438,16 +433,11 @@ it("preserves encrypted-member and corrupt-CRC observations", async () => { bytes[dataOffset] = firstByte ^ 1; await writeFile(corruptPath, bytes); - const corruptTarget = await parseBinaryTarget(corruptPath); - if (!corruptTarget.ok) throw corruptTarget.error; - const corruptResult = await withArtifactClient( - corruptTarget.value, - (client) => client.execute("inventory_artifact", {}), + await expectInventoryFailure( + corruptPath, + "format", + "ZIP archive is malformed: Invalid compressed data", ); - expect(corruptResult).toMatchObject({ - ok: false, - error: { _tag: "ArtifactOperationError", reason: "io" }, - }); }); it("rejects a truncated ZIP through artifact inventory", async () => { @@ -455,17 +445,51 @@ it("rejects a truncated ZIP through artifact inventory", async () => { const path = join(root, "truncated.zip"); await writeFile(path, Buffer.from([0x50, 0x4b, 0x03, 0x04])); + await expectInventoryFailure( + path, + "format", + "ZIP archive is malformed: File format is not recognized", + ); +}); + +it("reports an unsupported ZIP compression method as unavailable", async () => { + const root = await createTestTempDirectory("rea-zip-method-"); + const path = join(root, "lzma.zip"); + const writer = new ZipWriter(new Uint8ArrayWriter()); + await writer.add("data.txt", new TextReader("member bytes"), { level: 0 }); + const bytes = Buffer.from(await writer.close()); + // Declare LZMA (14) in both headers; REA reads only stored and Deflate data. + const localHeader = bytes.indexOf(Buffer.from([0x50, 0x4b, 0x03, 0x04])); + const centralHeader = bytes.indexOf(Buffer.from([0x50, 0x4b, 0x01, 0x02])); + if (localHeader < 0 || centralHeader < 0) + throw new Error("Expected ZIP local and central headers"); + bytes.writeUInt16LE(14, localHeader + 8); + bytes.writeUInt16LE(14, centralHeader + 10); + await writeFile(path, bytes); + + await expectInventoryFailure( + path, + "unavailable", + "ZIP archive uses an unsupported feature: Compression method not supported", + ); +}); + +/** Inventory one archive through the provider and expect a typed failure. */ +const expectInventoryFailure = async ( + path: string, + reason: string, + detail: string, +): Promise => { const target = await parseBinaryTarget(path); if (!target.ok) throw target.error; const result = await withArtifactClient(target.value, (client) => client.execute("inventory_artifact", {}), ); - expect(result).toMatchObject({ ok: false, - error: { _tag: "ArtifactOperationError", reason: "io" }, + error: { _tag: "ArtifactOperationError", reason, detail }, }); -}); +}; const writeSingleEntryZip = async ( path: string, diff --git a/tests/boundary/filesystem/ciChangeScope.test.ts b/tests/boundary/filesystem/ciChangeScope.test.ts index 163219c39..321160a36 100644 --- a/tests/boundary/filesystem/ciChangeScope.test.ts +++ b/tests/boundary/filesystem/ciChangeScope.test.ts @@ -515,6 +515,10 @@ it.each([ "src/windows/Example.ts", { code: true, runtime: true, package: true, apple: false, evm: false }, ], + [ + "src/domain/javascript/javascriptRuntimeStaticCandidates.ts", + { code: true, runtime: true, package: true, javascript_recovery: true }, + ], ["install.sh", { code: true, runtime: true, package: true, evm: false }], [ ".agents/skills/reverse-engineer-anything/SKILL.md", diff --git a/tests/boundary/filesystem/ciRequiredLanes.test.ts b/tests/boundary/filesystem/ciRequiredLanes.test.ts index b6e02b54a..da6c43186 100644 --- a/tests/boundary/filesystem/ciRequiredLanes.test.ts +++ b/tests/boundary/filesystem/ciRequiredLanes.test.ts @@ -262,8 +262,8 @@ it("binds provider build reuse to the current run while keeping standalone dispa ), ), ); - const download = action.runs.steps.find( - (step) => step.uses === "actions/download-artifact@v4", + const download = action.runs.steps.find((step) => + step.uses?.startsWith("actions/download-artifact@"), ); expect(download?.with).toEqual({ name: "${{ inputs.build_cache_artifact }}", diff --git a/tests/boundary/filesystem/clientLifecycle.test.ts b/tests/boundary/filesystem/clientLifecycle.test.ts index 506a6d6df..63ca0667a 100644 --- a/tests/boundary/filesystem/clientLifecycle.test.ts +++ b/tests/boundary/filesystem/clientLifecycle.test.ts @@ -71,6 +71,7 @@ describe("client configuration filesystem lifecycle", () => { "pi", "hermes", "qoder", + "droid", "zcode", "dsh", "grok_bot", diff --git a/tests/boundary/filesystem/directAnalysisSnapshotReplay.test.ts b/tests/boundary/filesystem/directAnalysisSnapshotReplay.test.ts index 6687b0ef7..7adf193f4 100644 --- a/tests/boundary/filesystem/directAnalysisSnapshotReplay.test.ts +++ b/tests/boundary/filesystem/directAnalysisSnapshotReplay.test.ts @@ -1,5 +1,5 @@ import { parseConfig } from "../../../src/config/parseConfig.js"; -import { writeFile } from "node:fs/promises"; +import { symlink, writeFile } from "node:fs/promises"; import { basename, dirname, join } from "node:path"; import { describe, expect, it, vi } from "vitest"; @@ -14,7 +14,7 @@ import type { import type { DirectAnalysisDependencies } from "../../../src/application/DirectAnalysisDependencies.js"; import { createAnalysisProfile } from "../../../src/domain/analysisProfile.js"; import { createAnalysisExecution } from "../../../src/application/AnalysisProvider.js"; -import { ok } from "../../../src/domain/result.js"; +import { err, ok } from "../../../src/domain/result.js"; import { readAnalysisSnapshot, writeAnalysisSnapshot, @@ -40,6 +40,7 @@ import { } from "../../fixtures/binarySession.js"; import { BinarySession } from "../../../src/application/binary/BinarySession.js"; import { OwnedStagedOutput } from "../../../src/filesystem/OwnedStagedOutput.js"; +import { ProviderAdapterError } from "../../../src/domain/providerAdapterError.js"; import { createTestTempDirectory } from "../../fixtures/temporaryDirectory.js"; import { CAPABILITIES as GHIDRA_CAPABILITIES, @@ -396,7 +397,85 @@ describe("snapshot output cleanup lifecycle", () => { }); }); +const verifyWindowsSnapshotAdmission = async ( + selectedPath: "direct" | "junction", +): Promise => { + const fixture = await createSnapshotReplayFixture( + "rea-direct-snapshot-junction-", + ); + const rawPath = join(dirname(fixture.path), "image.bin"); + await writeFile(rawPath, Buffer.from("a92a60", "hex")); + const linked = join(dirname(fixture.path), "linked"); + const linkedPath = join(linked, basename(rawPath)); + const identity = { ...GHIDRA_PROVIDER_IDENTITY, version: "12.1.4" }; + const profile = createAnalysisProfile(identity, { + fixture: true, + }); + const baseProvider = fixture.createProvider({ + identity, + profile, + supportedOperations: ["search_strings"], + }); + let rejectAdmission = false; + const provider: AnalysisProvider = { + ...baseProvider, + createClient: (target, profile, context) => { + const client = baseProvider.createClient(target, profile, context); + return { + ...client, + execute: (operation, parameters, options) => + rejectAdmission && operation === "search_strings" + ? Promise.resolve( + err(new ProviderAdapterError("ghidra", operation)), + ) + : client.execute(operation, parameters, options), + }; + }, + }; + const dependencies = fixture.dependencies(provider); + const arguments_ = { pattern: "fixture", document: "fixture" }; + const options = { + snapshotPath: fixture.snapshotPath, + formatHint: "raw-binary" as const, + architecture: "6502" as const, + entryPoint: "0x8001", + }; + const first = await runDirectAnalysis( + dependencies, + rawPath, + "search_strings", + arguments_, + options, + ); + expect(first).toMatchObject({ operation: "search_strings" }); + expect(fixture.starts).toHaveLength(1); + if (selectedPath === "junction") + await symlink(dirname(fixture.path), linked, "junction"); + rejectAdmission = true; + const second = await runDirectAnalysis( + dependencies, + selectedPath === "junction" ? linkedPath : rawPath, + "search_strings", + arguments_, + options, + ); + expect(second).toMatchObject({ error: "Analysis failed" }); + expect(fixture.starts).toHaveLength(2); + expect( + fixture.calls.filter((call) => call === "search_strings"), + ).toHaveLength(1); +}; + describe("direct analysis composed snapshot replay", () => { + it.skipIf(process.platform !== "win32")( + "does not replay a CLI snapshot through a Windows junction", + () => verifyWindowsSnapshotAdmission("junction"), + ); + it.skipIf(process.platform !== "win32")( + "does not replay a CLI snapshot before native admission on a canonical path", + () => verifyWindowsSnapshotAdmission("direct"), + ); + it.each([ { tool: "search_strings", @@ -516,7 +595,7 @@ describe("Ghidra composed workflow snapshot replay", () => { { tool: "inspect_native_api", arguments: { procedure: "0x1000" } }, { tool: "trace_feature", arguments: { query: "fixture" } }, ] as const)( - "persists and replays $tool with production Ghidra capability policies", + "persists $tool with production Ghidra capability policies", async ({ tool, arguments: parameters }) => { const fixture = await createSnapshotReplayFixture("rea-ghidra-workflow-"); const profile = createAnalysisProfile( @@ -554,8 +633,14 @@ describe("Ghidra composed workflow snapshot replay", () => { { snapshotPath: fixture.snapshotPath }, ); expect(replay).toEqual(first); - expect(fixture.starts).toHaveLength(1); - expect(fixture.calls).toEqual(initialCalls); + if (process.platform === "win32") { + // The first provider call in a new CLI session admits its source. + expect(fixture.starts).toHaveLength(2); + expect(fixture.calls.length).toBeGreaterThan(initialCalls.length); + } else { + expect(fixture.starts).toHaveLength(1); + expect(fixture.calls).toEqual(initialCalls); + } }, ); }); diff --git a/tests/boundary/filesystem/droidClientConfiguration.test.ts b/tests/boundary/filesystem/droidClientConfiguration.test.ts new file mode 100644 index 000000000..6f9b26423 --- /dev/null +++ b/tests/boundary/filesystem/droidClientConfiguration.test.ts @@ -0,0 +1,167 @@ +import { mkdir, readFile, writeFile } from "node:fs/promises"; +import { join } from "node:path"; + +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; + +import { readClientRegistrationStatuses } from "../../../src/application/ClientRegistrationStatus.js"; +import { npxRegistrationCommand } from "../../../src/application/ClientRegistrationIdentity.js"; +import { configureClientConfiguration } from "../../../src/application/SetupClientConfiguration.js"; +import { skillDestinations } from "../../../src/application/SetupSkill.js"; +import { supportedClients } from "../../../src/application/SupportedClients.js"; +import { systemUninstallHost } from "../../../src/application/Uninstall.js"; +import { createTestTempDirectory } from "../../fixtures/temporaryDirectory.js"; +import { clearClientLocationEnvironment } from "../../fixtures/clientEnvironment.js"; + +/** + * The registration command setup writes on the host running the tests. Windows + * wraps it in `cmd.exe /d /c`, so the expected entry follows the host too. + */ +const command = npxRegistrationCommand(process.platform); + +beforeEach(clearClientLocationEnvironment); +afterEach(() => vi.unstubAllEnvs()); + +const droidPath = ( + home: string, + platform: NodeJS.Platform = "linux", + env: Parameters[2] = {}, +) => supportedClients(home, platform, env).find(({ name }) => name === "droid"); + +describe("Factory Droid configuration paths", () => { + it("resolves the user MCP config file and marker directory", () => { + expect(droidPath("/home/a")).toEqual({ + name: "droid", + displayName: "Factory Droid", + configPath: join("/home/a", ".factory", "mcp.json"), + markerPath: join("/home/a", ".factory"), + format: "json", + }); + expect(droidPath("/Users/a", "darwin")?.configPath).toBe( + join("/Users/a", ".factory", "mcp.json"), + ); + expect(droidPath("C:\\Users\\a", "win32")?.configPath).toBe( + join("C:\\Users\\a", ".factory", "mcp.json"), + ); + }); + + it("resolves the personal skills directory beside the config root", () => { + expect( + skillDestinations("/home/a", ["droid"]).map(({ path }) => path), + ).toEqual([ + join("/home/a", ".factory", "skills", "reverse-engineer-anything"), + ]); + }); +}); + +describe("Factory Droid JSON registration", () => { + const droidClient = async () => { + const home = await createTestTempDirectory("rea-droid-"); + const client = droidPath(home, process.platform); + if (client?.markerPath === undefined) + throw new Error("missing Factory Droid client"); + await mkdir(client.markerPath, { recursive: true }); + return { home, client }; + }; + + it("upserts rea into mcpServers while preserving unrelated servers and permissions", async () => { + const { home, client } = await droidClient(); + await writeFile( + client.configPath, + `{ + "mcpServers": { + "other": { + "type": "stdio", + "command": "other", + "args": [], + "disabled": false + } + }, + "persistentPermissions": { + "servers": { + "other": { "approvedAt": "2026-01-01T00:00:00.000Z" } + } + } +}`, + ); + + expect( + (await configureClientConfiguration(client, {}, command)).status, + ).toBe("configured"); + const parsed = JSON.parse(await readFile(client.configPath, "utf8")); + expect(parsed.mcpServers.other).toEqual({ + type: "stdio", + command: "other", + args: [], + disabled: false, + }); + expect(parsed.persistentPermissions).toEqual({ + servers: { other: { approvedAt: "2026-01-01T00:00:00.000Z" } }, + }); + expect(parsed.mcpServers.rea).toEqual({ + command: command[0], + args: command.slice(1), + }); + + expect(await configureClientConfiguration(client, {}, command)).toEqual({ + status: "unchanged", + }); + expect( + ( + await readClientRegistrationStatuses(home, undefined, { + platform: process.platform, + environment: {}, + }) + ).find(({ client }) => client === "droid"), + ).toMatchObject({ state: "aligned", command }); + }); + + it("creates the mcpServers table in an existing config file without servers", async () => { + const { client } = await droidClient(); + await writeFile( + client.configPath, + JSON.stringify({ persistentPermissions: { servers: {} } }, null, 2), + ); + expect( + (await configureClientConfiguration(client, {}, command)).status, + ).toBe("configured"); + const parsed = JSON.parse(await readFile(client.configPath, "utf8")); + expect(parsed.persistentPermissions).toEqual({ servers: {} }); + expect(parsed.mcpServers.rea).toEqual({ + command: command[0], + args: command.slice(1), + }); + }); + + it("registers into an empty config file", async () => { + const { client } = await droidClient(); + await writeFile(client.configPath, ""); + expect( + (await configureClientConfiguration(client, {}, command)).status, + ).toBe("configured"); + const parsed = JSON.parse(await readFile(client.configPath, "utf8")); + expect(parsed.mcpServers.rea).toEqual({ + command: command[0], + args: command.slice(1), + }); + }); + + it("removes only the rea entry on uninstall", async () => { + const { home, client } = await droidClient(); + await writeFile( + client.configPath, + JSON.stringify( + { mcpServers: { other: { command: "other-server" } } }, + null, + 2, + ), + ); + await configureClientConfiguration(client, {}, command); + + expect((await systemUninstallHost(home).removeClient(client)).status).toBe( + "removed", + ); + const parsed = JSON.parse(await readFile(client.configPath, "utf8")); + expect(parsed.mcpServers.other).toEqual({ command: "other-server" }); + expect(parsed.mcpServers.rea).toBeUndefined(); + }); +}); diff --git a/tests/boundary/filesystem/dshClientConfiguration.test.ts b/tests/boundary/filesystem/dshClientConfiguration.test.ts index 72dbded9e..c0315f67b 100644 --- a/tests/boundary/filesystem/dshClientConfiguration.test.ts +++ b/tests/boundary/filesystem/dshClientConfiguration.test.ts @@ -1,5 +1,5 @@ import { mkdir, readFile, writeFile } from "node:fs/promises"; -import { join } from "node:path"; +import { join, resolve } from "node:path"; import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; import { @@ -37,28 +37,76 @@ const dshClient = ( env: Parameters[2], platform: NodeJS.Platform = "linux", ) => - supportedClients(join("/home/a"), platform, env).find( + supportedClients(resolve("/home/a"), platform, env).find( ({ name }) => name === "dsh", ); describe("DeepSeek Harness configuration paths", () => { + it.each([ + [undefined, ".dsh", ".agents", "shared"], + ["", ".dsh", ".agents", "shared"], + [" \t ", ".dsh", ".agents", "shared"], + ["~", "", "", "dsh"], + ["~/selected home", "selected home", "selected home", "dsh"], + ["~\\selected home", "selected home", "selected home", "dsh"], + ] as const)( + "resolves Harness and agents roots for %j", + (override, dshRoot, agentsRoot, client) => { + const home = resolve("/home/a"); + const environment = { DSH_HOME: override, DSH_AGENTS_HOME: override }; + expect( + supportedClients(home, process.platform, environment).find( + ({ name }) => name === "dsh", + ), + ).toMatchObject({ + configPath: join(home, dshRoot, "cordis.patch.yml"), + markerPath: join(home, dshRoot), + }); + expect(clientSkillDirectories(home, ["dsh"], environment)).toEqual([ + { client, directory: join(home, agentsRoot, "skills") }, + ]); + }, + ); + + it.each(["relative home", " literal home ", "~someone"])( + "resolves a literal root %j relative to cwd without trimming it", + (override) => { + const home = resolve("/home/a"); + const root = resolve(override); + const environment = { DSH_HOME: override, DSH_AGENTS_HOME: override }; + expect( + supportedClients(home, process.platform, environment).find( + ({ name }) => name === "dsh", + ), + ).toMatchObject({ + configPath: join(root, "cordis.patch.yml"), + markerPath: root, + }); + expect(clientSkillDirectories(home, ["dsh"], environment)).toEqual([ + { client: "dsh", directory: join(root, "skills") }, + ]); + }, + ); + it("resolves the home Cordis patch under the Harness home", () => { expect(dshClient({})).toEqual({ name: "dsh", displayName: "DeepSeek Harness", - configPath: join("/home/a", ".dsh", "cordis.patch.yml"), - markerPath: join("/home/a", ".dsh"), + configPath: resolve("/home/a", ".dsh", "cordis.patch.yml"), + markerPath: resolve("/home/a", ".dsh"), format: "dsh", }); // The home patch applies to every profile; DSH_HOME relocates it. expect(dshClient({ DSH_HOME: "/custom/harness" })?.configPath).toBe( - join("/custom/harness", "cordis.patch.yml"), + resolve("/custom/harness", "cordis.patch.yml"), ); expect( clientSkillDirectories(join("/home/a"), ["dsh"], { DSH_AGENTS_HOME: "/custom/agents", }), - ).toEqual([{ client: "dsh", directory: join("/custom/agents", "skills") }]); + ).toEqual([ + { client: "dsh", directory: resolve("/custom/agents", "skills") }, + ]); }); }); diff --git a/tests/boundary/filesystem/ghidraJavaProbe.test.ts b/tests/boundary/filesystem/ghidraJavaProbe.test.ts index c0a6974d5..415fbf3dd 100644 --- a/tests/boundary/filesystem/ghidraJavaProbe.test.ts +++ b/tests/boundary/filesystem/ghidraJavaProbe.test.ts @@ -87,3 +87,43 @@ it.each([ ); }, ); + +it("bounds the Java version probe and reports an over-budget probe as a failed Java check", async () => { + const root = await createTestTempDirectory("rea-java-probe-bound-"); + const configuredHome = join(root, "selected-jdk"); + const installDir = join(root, "ghidra"); + await mkdir(join(installDir, "Ghidra"), { recursive: true }); + await writeFile( + join(installDir, "Ghidra", "application.properties"), + `application.version=${SUPPORTED_GHIDRA_VERSION}\napplication.java.min=17\n`, + ); + // A chatty or hostile JVM that overruns the capture budget: spawnSync reports + // ENOBUFS, which the probe already treats as a failed observation. + mockSpawnSync.mockReturnValue({ + stdout: "", + stderr: "java.version = 21.0.11\n", + status: null, + error: Object.assign(new Error("spawnSync /java ENOBUFS"), { + code: "ENOBUFS", + }), + }); + + const inspection = inspectGhidraInstallation({ + environment: {}, + installDir, + javaHome: configuredHome, + }); + + const [, , options] = mockSpawnSync.mock.calls[0] as [ + string, + readonly string[], + { readonly maxBuffer?: number }, + ]; + expect(options.maxBuffer).toBe(1024 * 1024); + expect(inspection.javaVersion).toBeNull(); + expect(inspection.javaHome).toBe(configuredHome); + expect(inspection.checks.find(({ name }) => name === "java")).toMatchObject({ + status: "failed", + code: "runtime_missing", + }); +}); diff --git a/tests/boundary/filesystem/hermesClientConfiguration.test.ts b/tests/boundary/filesystem/hermesClientConfiguration.test.ts index 43a5f841c..218573d5c 100644 --- a/tests/boundary/filesystem/hermesClientConfiguration.test.ts +++ b/tests/boundary/filesystem/hermesClientConfiguration.test.ts @@ -237,6 +237,38 @@ mcp_servers: expect(await readFile(client.configPath, "utf8")).toBe(original); }); + it("keeps unrelated numbers as written, including Discord IDs", async () => { + const { home, client } = await hermesClient(); + // Hermes reads these exactly: two IDs one apart, an octal mode, a float. + const original = `discord: + free_response_channels: + - 1187654321098765432 + channel_prompts: + 1187654321098765433: Build questions. + 1187654321098765434: Release questions. + mode: 0600 + min_score: 0.0000001 +mcp_servers: + other: + command: other-server +`; + await writeFile(client.configPath, original); + expect( + (await configureClientConfiguration(client, {}, command)).status, + ).toBe("configured"); + const text = await readFile(client.configPath, "utf8"); + expect(text.startsWith(original)).toBe(true); + expect(parseYaml(text, { intAsBigInt: true })).toMatchObject({ + mcp_servers: { + rea: { command: command[0], args: command.slice(1), enabled: true }, + }, + }); + expect((await systemUninstallHost(home).removeClient(client)).status).toBe( + "removed", + ); + expect(await readFile(client.configPath, "utf8")).toBe(original); + }); + it("creates config.yaml when Hermes has no configuration yet", async () => { const { client } = await hermesClient(); expect( diff --git a/tests/boundary/filesystem/interfaceBuilderBundle.test.ts b/tests/boundary/filesystem/interfaceBuilderBundle.test.ts index 4446c80dc..d584aaf3c 100644 --- a/tests/boundary/filesystem/interfaceBuilderBundle.test.ts +++ b/tests/boundary/filesystem/interfaceBuilderBundle.test.ts @@ -1060,6 +1060,142 @@ describe("NIBArchive UIKit event connections", () => { "UIRuntimeEventConnection", ); }); + + it("reports a runtime connection with no resolvable owner as an omission", async () => { + const root = await createTestTempDirectory("rea-ib-ownerless-"); + const bundle = join(root, "Example.app"); + await mkdir(bundle, { recursive: true }); + // The connector is recognized by class, but carries no UISource reference, + // so no connection can be projected from it. + await writeFile( + join(bundle, "Cell.nib"), + encodeNibArchiveFixture({ + classes: [ + "UIProxyObject", + "UIClassSwapper", + "UIButton", + "UIRuntimeEventConnection", + "NSString", + ], + objects: [ + { classIndex: 0, values: { UIProxiedObjectIdentifier: { ref: 5 } } }, + { classIndex: 1, values: { UIClassName: { ref: 6 } } }, + { classIndex: 2, values: {} }, + { + classIndex: 3, + values: { UIDestination: { ref: 1 }, UIEventMask: 64 }, + }, + { classIndex: 2, values: {} }, + { classIndex: 4, values: { "NS.bytes": "IBFilesOwner" } }, + { classIndex: 4, values: { "NS.bytes": "BuildCell" } }, + ], + }), + ); + + const analysis = await analyzeInterfaceBuilderBundle({ + bundlePath: bundle, + targetSha256: "e".repeat(64), + }); + + expect(analysis.graph.coverage).toContainEqual( + expect.objectContaining({ + facet: "connections:Cell.nib", + status: "partial", + reason: "connection_unresolved", + examined: 0, + omitted: 1, + }), + ); + // The keyed-archive producer counts a recognized connector it cannot emit, + // so this producer has to report the same document total for the same + // connector rather than zero. + expect(analysis.documents).toContainEqual( + expect.objectContaining({ + relative_path: "Cell.nib", + connection_count: 1, + }), + ); + expect(analysis.graph.truncated).toBe(true); + expect(analysis.limitations).toContain( + "Some recognized NIBArchive connectors declare no resolvable owner; they carry no connection evidence and are reported as omissions.", + ); + }); +}); + +describe("NIBArchive ownerless connector accounting", () => { + it("keeps a connection dropped by the limit reported as a limit", async () => { + const root = await createTestTempDirectory("rea-ib-ownerless-limit-"); + const bundle = join(root, "Example.app"); + await mkdir(bundle, { recursive: true }); + // Two resolvable connectors and one ownerless connector with the limit + // below the resolvable count, so this facet has both causes at once. Only a + // higher limit recovers the dropped connection, so that is the reason the + // caller needs, exactly as the keyed-archive producer reports it. + await writeFile( + join(bundle, "Cell.nib"), + encodeNibArchiveFixture({ + classes: [ + "UIProxyObject", + "UIClassSwapper", + "UIButton", + "UIRuntimeEventConnection", + "NSString", + ], + objects: [ + { classIndex: 0, values: { UIProxiedObjectIdentifier: { ref: 5 } } }, + { classIndex: 1, values: { UIClassName: { ref: 6 } } }, + { classIndex: 2, values: {} }, + { + classIndex: 3, + values: { + UISource: { ref: 2 }, + UIDestination: { ref: 1 }, + UILabel: { ref: 7 }, + UIEventMask: 64, + }, + }, + { classIndex: 2, values: {} }, + { classIndex: 4, values: { "NS.bytes": "IBFilesOwner" } }, + { classIndex: 4, values: { "NS.bytes": "BuildCell" } }, + { classIndex: 4, values: { "NS.bytes": "buildTapped:" } }, + { + classIndex: 3, + values: { + UISource: { ref: 4 }, + UIDestination: { ref: 1 }, + UILabel: { ref: 7 }, + }, + }, + { + classIndex: 3, + values: { UIDestination: { ref: 1 }, UIEventMask: 64 }, + }, + ], + }), + ); + + const analysis = await analyzeInterfaceBuilderBundle({ + bundlePath: bundle, + targetSha256: "e".repeat(64), + limits: { max_connections: 1 }, + }); + + expect(analysis.graph.coverage).toContainEqual( + expect.objectContaining({ + facet: "connections:Cell.nib", + status: "partial", + reason: "connection_limit_reached", + examined: 1, + omitted: 2, + }), + ); + expect(analysis.documents).toContainEqual( + expect.objectContaining({ + relative_path: "Cell.nib", + connection_count: 3, + }), + ); + }); }); // This exact file is already selected by the unchanged native Apple CI lane. diff --git a/tests/boundary/filesystem/javascript-applications/javascriptArtifactReconstruction.test.ts b/tests/boundary/filesystem/javascript-applications/javascriptArtifactReconstruction.test.ts index 6b7253b6d..416337097 100644 --- a/tests/boundary/filesystem/javascript-applications/javascriptArtifactReconstruction.test.ts +++ b/tests/boundary/filesystem/javascript-applications/javascriptArtifactReconstruction.test.ts @@ -388,7 +388,7 @@ it("produces deterministic ASAR graphs with unpacked native linkage and complete await mkdir(source); await writeJavaScriptArtifactFixture(source); const archive = join(root, "app.asar"); - await createPackageWithOptions(source, archive, { unpack: "**/*.node" }); + await createPackageWithOptions(source, archive, { unpack: "*.node" }); const first = await reconstructJavaScriptArtifact({ input_path: archive, @@ -487,7 +487,7 @@ it("keeps ASAR analysis usable when unpacked native companion bytes are absent", await mkdir(source); await writeJavaScriptArtifactFixture(source); const archive = join(root, "app.asar"); - await createPackageWithOptions(source, archive, { unpack: "**/*.node" }); + await createPackageWithOptions(source, archive, { unpack: "*.node" }); await rm(join(`${archive}.unpacked`, "native", "addon.node")); const snapshot = await scanArtifactInventory(archive); @@ -515,7 +515,7 @@ it("recurses into filesystem-backed ASAR containers without losing container-rel const resources = join(outer, "resources"); await mkdir(resources, { recursive: true }); await createPackageWithOptions(source, join(resources, "app.asar"), { - unpack: "**/*.node", + unpack: "*.node", }); const result = await reconstructJavaScriptArtifact({ diff --git a/tests/boundary/filesystem/javascriptApplicationWorkflows.test.ts b/tests/boundary/filesystem/javascriptApplicationWorkflows.test.ts index b79407821..8393bad54 100644 --- a/tests/boundary/filesystem/javascriptApplicationWorkflows.test.ts +++ b/tests/boundary/filesystem/javascriptApplicationWorkflows.test.ts @@ -444,73 +444,97 @@ describe("JavaScript application workflow error diagnostics", () => { }); }); +const RUNTIME_PATH_CASES = [ + { + name: "case-sensitive POSIX input root", + root: null, + path: "/applications/Example.app/Contents/Resources/app/renderer.js", + status: "unknown", + basis: "none", + }, + { + name: "case-sensitive POSIX mapping root", + root: "/runtime/App", + path: "/runtime/app/renderer.js", + status: "unknown", + basis: "none", + }, + { + name: "matching POSIX mapping root", + root: "/runtime/App", + path: "/runtime/App/renderer.js", + status: "matched", + basis: "operator-file-mapping", + }, + { + name: "case-insensitive Windows mapping root", + root: String.raw`C:\runtime\App`, + path: String.raw`c:\runtime\app\renderer.js`, + status: "matched", + basis: "operator-file-mapping", + }, +] as const; + describe("JavaScript runtime reconciliation local paths", () => { - it.each([ - { - name: "case-sensitive POSIX input root", - root: null, - path: "/applications/Example.app/Contents/Resources/app/renderer.js", - status: "unknown", - }, - { - name: "case-sensitive POSIX mapping root", - root: "/runtime/App", - path: "/runtime/app/renderer.js", - status: "unknown", - }, - { - name: "matching POSIX mapping root", - root: "/runtime/App", - path: "/runtime/App/renderer.js", - status: "matched", - }, - { - name: "case-insensitive Windows mapping root", - root: String.raw`C:\runtime\App`, - path: String.raw`c:\runtime\app\renderer.js`, - status: "matched", - }, - ])("uses $name on every host", ({ root, path, status }) => { - const base = JAVASCRIPT_RUNTIME_RECONCILIATION_EXAMPLE; - const original = base.runtime_observations[0]!; - const inspection = electronPageInspectionSchema.parse( - original.normalized_result, - ); - const runtime = createEvidence(undefined, original.provider, { - predicateType: original.predicate_type, - operation: original.operation, - parameters: { ...original.parameters, include_script_sources: false }, - result: { - ...inspection, - scripts: { - ...inspection.scripts, - items: inspection.scripts.items.map((script) => ({ - ...script, - file_path: path, - source: { included: false, reason: "not-requested" }, - })), + it.each(RUNTIME_PATH_CASES)( + "uses $name on every host", + ({ root, path, status, basis }) => { + const base = JAVASCRIPT_RUNTIME_RECONCILIATION_EXAMPLE; + const original = base.runtime_observations[0]!; + const inspection = electronPageInspectionSchema.parse( + original.normalized_result, + ); + // Keep target and script coordinates consistent so an unrelated target + // cannot activate the static layer after its script mapping was rejected. + const runtime = createEvidence(undefined, original.provider, { + predicateType: original.predicate_type, + operation: original.operation, + parameters: { ...original.parameters, include_script_sources: false }, + result: { + ...inspection, + target: { ...inspection.target, file_path: path }, + scripts: { + ...inspection.scripts, + items: inspection.scripts.items.map((script) => ({ + ...script, + file_path: path, + source: { included: false, reason: "not-requested" }, + })), + }, }, - }, - confidence: original.confidence, - authority: original.authority, - }); - const reconciled = reconcileJavaScriptRuntimeEvidence({ - static_layers: base.static_layers.map((layer) => ({ - ...layer, - runtime_mappings: root === null ? [] : [{ kind: "file-root", root }], - })), - runtime_observations: [runtime], - }); - if (!reconciled.ok) throw reconciled.error; - const result = javascriptRuntimeReconciliationResultSchema.parse( - reconciled.value.normalized_result, - ); - expect( - result.reconciliations.find( - ({ entity_kind }) => entity_kind === "script", - ), - ).toMatchObject({ status }); - }); + confidence: original.confidence, + authority: original.authority, + }); + const reconciled = reconcileJavaScriptRuntimeEvidence({ + static_layers: base.static_layers.map((layer) => ({ + ...layer, + runtime_mappings: root === null ? [] : [{ kind: "file-root", root }], + })), + runtime_observations: [runtime], + }); + if (!reconciled.ok) throw reconciled.error; + const result = javascriptRuntimeReconciliationResultSchema.parse( + reconciled.value.normalized_result, + ); + expect( + result.reconciliations.find( + ({ entity_kind }) => entity_kind === "script", + ), + ).toMatchObject({ status, basis }); + expect( + result.static_load_states.find( + ({ kind }) => kind === "javascript-asset", + ), + ).toMatchObject( + status === "matched" + ? { status: "loaded", reason: "runtime-script-correspondence" } + : { status: "unknown", reason: "layer-outside-runtime-scope" }, + ); + expect(result.static_layers[0]?.runtime_mappings).toEqual( + root === null ? [] : [{ kind: "file-root", root, artifact_prefix: "" }], + ); + }, + ); it("resolves relative file mapping roots before deriving result Evidence", () => { const absoluteRoot = resolve(process.cwd(), "runtime-cache-fixture"); diff --git a/tests/boundary/filesystem/javascriptIntegrityPolicy.test.ts b/tests/boundary/filesystem/javascriptIntegrityPolicy.test.ts index 47b77114a..67b167bdf 100644 --- a/tests/boundary/filesystem/javascriptIntegrityPolicy.test.ts +++ b/tests/boundary/filesystem/javascriptIntegrityPolicy.test.ts @@ -105,6 +105,34 @@ it("keeps a contradicted nested ASAR opaque during JavaScript reconstruction", a expect(result.graph.coverage.omitted_count).toBeNull(); }); +it("keeps an ASAR-named file without an ASAR header opaque during JavaScript reconstruction", async () => { + const root = await createTestTempDirectory("rea-opaque-asar-name-"); + await writeFile(join(root, "main.js"), "export const outer = true;\n"); + // Inventory records this file as ordinary bytes and does not expand it. + await writeFile(join(root, "ordinary.asar"), "ordinary bytes"); + + const snapshot = await scanCanonicalArtifactInventory(root, {}); + const reader = createJavaScriptArtifactReader(root, "directory"); + try { + const read = await readJavaScriptArtifactFiles(reader, snapshot); + expect(read.files.map(({ path }) => path)).toEqual(["main.js"]); + expect(read.containers).toEqual([]); + } finally { + await reader.close(); + } + + const analyzed = await analyzeJavaScriptApplication({ + input_path: root, + format: "directory", + }); + if (!analyzed.ok) throw analyzed.error; + const result = javascriptApplicationAnalysisResultSchema.parse( + analyzed.value.normalized_result, + ); + expect(result.statistics.nested_asar_containers).toBe(0); + expect(result.integrity_contradictions).toEqual([]); +}); + it("retains semantic resource limits alongside recorded integrity contradictions", async () => { const { archive } = await createStrippedAsarAddon([ { path: "app.js", contents: primitiveCandidateExpansionSource() }, diff --git a/tests/boundary/filesystem/rawBinaryTarget.test.ts b/tests/boundary/filesystem/rawBinaryTarget.test.ts index 5fd171d20..2757f0507 100644 --- a/tests/boundary/filesystem/rawBinaryTarget.test.ts +++ b/tests/boundary/filesystem/rawBinaryTarget.test.ts @@ -1,5 +1,11 @@ import { createHash } from "node:crypto"; -import { readFile, writeFile } from "node:fs/promises"; +import { + mkdir, + readFile, + realpath, + symlink, + writeFile, +} from "node:fs/promises"; import { join } from "node:path"; import { describe, expect, it } from "vitest"; @@ -7,6 +13,37 @@ import { parseBinaryTarget } from "../../../src/application/BinaryTargetResolver import { createTestTempDirectory } from "../../fixtures/temporaryDirectory.js"; describe("raw-binary admission", () => { + it("preserves the requested raw path for Windows reparse admission", async () => { + const directory = await createTestTempDirectory("rea-raw-linked-"); + const actual = join(directory, "actual"); + const linked = join(directory, "linked"); + await mkdir(actual); + const bytes = Buffer.from("a90160", "hex"); + await writeFile(join(actual, "fixture.bin"), bytes); + await symlink( + actual, + linked, + process.platform === "win32" ? "junction" : "dir", + ); + const requested = join(linked, "fixture.bin"); + const canonical = await realpath(requested); + const target = await parseBinaryTarget("linked/fixture.bin", { + cwd: directory, + formatHint: "raw-binary", + architecture: "6502", + entryPoint: "0x8000", + }); + expect(target).toMatchObject({ + ok: true, + value: { + path: canonical, + sourcePath: process.platform === "win32" ? requested : canonical, + sha256: createHash("sha256").update(bytes).digest("hex"), + format: "raw-binary", + architecture: "6502", + }, + }); + }); it("requires explicit architecture and entry even with a misleading suffix", async () => { const directory = await createTestTempDirectory("rea-raw-target-"); const path = join(directory, "fixture.js"); diff --git a/tests/boundary/filesystem/targetDiagnostics.test.ts b/tests/boundary/filesystem/targetDiagnostics.test.ts index 5d1953760..dacd21687 100644 --- a/tests/boundary/filesystem/targetDiagnostics.test.ts +++ b/tests/boundary/filesystem/targetDiagnostics.test.ts @@ -4,9 +4,35 @@ import { describe, expect, it } from "vitest"; import { systemDoctorHost } from "../../../src/application/Doctor.js"; import { parseBinaryTarget } from "../../../src/application/BinaryTargetResolver.js"; import { projectAnalysisError } from "../../../src/domain/analysisErrorProjection.js"; +import { pe } from "../../../src/domain/binaryTarget.fixture.js"; import { createTestTempDirectory } from "../../fixtures/temporaryDirectory.js"; describe("executable format refusal diagnostics", () => { + it.each([ + [0x01a6, "0x01a6"], + [0x0000, "0x0000"], + [0xffff, "0xffff"], + ] as const)( + "retains unsupported PE machine %i in the refusal", + async (machine, expectedMachine) => { + const directory = await createTestTempDirectory("rea-target-pe-machine-"); + const path = join(directory, "unsupported.exe"); + const bytes = pe(0x014c); + bytes.writeUInt16LE(machine, 0x44); + await writeFile(path, bytes); + + const result = await parseBinaryTarget(path); + if (result.ok) throw new Error("unsupported PE target was admitted"); + const reason = `unsupported PE architecture (COFF machine ${expectedMachine})`; + expect(projectAnalysisError(result.error)).toMatchObject({ + code: "target_unavailable", + category: "unsupported_target", + message: `Cannot open artifact: ${reason}.`, + details: { reason, constraint: "file_format_rejected" }, + }); + }, + ); + it.each(["NE", "LE", "LX"] as const)( "reports unsupported %s executables without path advice", async (signature) => { diff --git a/tests/boundary/go/goBinaryImage.test.ts b/tests/boundary/go/goBinaryImage.test.ts index 24321ecc8..29f0e952c 100644 --- a/tests/boundary/go/goBinaryImage.test.ts +++ b/tests/boundary/go/goBinaryImage.test.ts @@ -1,4 +1,10 @@ import { expect, it } from "vitest"; +import { javaClassFile } from "../../fixtures/javaClassFile.js"; +import { + CPU, + fatImage, + machoImage, +} from "../../../src/artifacts/apple/MachoImage.fixture.js"; import { readGoBinaryImage } from "../../../src/go/GoBinaryImage.js"; import { createGoBinaryFixture, @@ -221,6 +227,35 @@ it("distinguishes unsupported containers from malformed recognized images", () = expect(() => readGoBinaryImage(pe.bytes)).toThrowError(/range|truncated/i); }); +it("reports JVM class files as unsupported images instead of fat Mach-O containers", () => { + expect(() => readGoBinaryImage(javaClassFile())).toThrowError( + /Only ELF, PE, and thin Mach-O native images are supported/, + ); + expect(() => readGoBinaryImage(javaClassFile(4000))).toThrowError( + /Only ELF, PE, and thin Mach-O native images are supported/, + ); + expect(() => readGoBinaryImage(javaClassFile(4000, true))).toThrowError( + /Only ELF, PE, and thin Mach-O native images are supported/, + ); +}); + +it("keeps the thin-slice failure for confirmed universal Mach-O containers", () => { + const slices = [ + { cpu: CPU.x86_64, bytes: machoImage({ cpu: CPU.x86_64 }) }, + { cpu: CPU.arm64, bytes: machoImage({ cpu: CPU.arm64 }) }, + ]; + for (const universal of [ + fatImage(slices), + fatImage(slices, true), + fatImage(slices, false, true), + fatImage(slices, true, true), + ]) { + expect(() => readGoBinaryImage(Buffer.from(universal))).toThrowError( + /fat Mach-O containers require selecting a thin slice/, + ); + } +}); + it("rejects ELF header tables overlapping the native header", () => { const fixture = createGoBinaryFixture({ sectionless: true }); fixture.bytes.writeBigUInt64LE(0n, 32); diff --git a/tests/boundary/mcp/electronMcp.test.ts b/tests/boundary/mcp/electronMcp.test.ts index fef9839aa..72d9fdaf6 100644 --- a/tests/boundary/mcp/electronMcp.test.ts +++ b/tests/boundary/mcp/electronMcp.test.ts @@ -471,3 +471,49 @@ it("exposes the target-free static JavaScript application workflow", async () => const evidenceFor = (value: unknown) => { return parseEvidence(value); }; + +it("reports every unresolved Electron path without starting a provider", async () => { + const root = await createTestTempDirectory("rea-electron-invalid-paths-"); + const capture = vi.fn(); + const session = createTestBinarySession(() => { + throw new Error("must not start"); + }); + const server = createServer( + { kind: "session", session }, + { + electronActiveObservation: { + capture, + identity: () => ({ id: "test", name: "test", version: "1" }), + }, + }, + ); + const client = new Client({ name: "electron-path-test", version: "1" }); + const [clientTransport, serverTransport] = + InMemoryTransport.createLinkedPair(); + resources.push(client, server, session); + await server.connect(serverTransport); + await client.connect(clientTransport); + const fields = ["executable_path", "application_path", "application_root"]; + const result = await client.callTool({ + name: "capture_electron_scenario", + arguments: { + ...Object.fromEntries(fields.map((field) => [field, join(root, field)])), + actions: [], + }, + }); + expect(result.isError).toBe(true); + expect(parseMcpToolError(result)).toMatchObject({ + error: { + code: "invalid_request", + details: { + issues: fields.map((field) => ({ + path: [field], + reason: "invalid_value", + message: expect.stringContaining("ENOENT"), + })), + }, + }, + }); + expect(capture).not.toHaveBeenCalled(); + await client.ping(); +}); diff --git a/tests/boundary/mcp/enhancedTools.traceFeatureCoverage.test.ts b/tests/boundary/mcp/enhancedTools.traceFeatureCoverage.test.ts new file mode 100644 index 000000000..8c94d5de0 --- /dev/null +++ b/tests/boundary/mcp/enhancedTools.traceFeatureCoverage.test.ts @@ -0,0 +1,249 @@ +import { afterEach, describe, expect, it } from "vitest"; + +import type { AnalysisOperationPort } from "../../../src/application/AnalysisProvider.js"; +import { createAnalysisExecution } from "../../../src/application/AnalysisProvider.js"; +import { EnhancedTools } from "../../../src/application/EnhancedTools.js"; +import { AnalysisOutputError } from "../../../src/domain/analysisErrorCore.js"; +import { err, ok } from "../../../src/domain/result.js"; + +import { closeEnhancedToolResources, connect } from "./enhancedToolsHarness.js"; + +afterEach(closeEnhancedToolResources); + +const PROVIDER = { + id: "fixture", + name: "Fixture analysis provider", + version: "1", +} as const; + +type Coverage = boolean | undefined; + +interface Behavior { + /** What the provider reports for the references of each match address. */ + readonly coverage?: Readonly>; + readonly limitations?: readonly string[]; + /** Also report the first string match as a procedure match. */ + readonly duplicateProcedure?: boolean; + /** Fail the references read of this address. */ + readonly failXrefsOf?: string; + readonly calls?: string[]; +} + +const containing = (address: string) => ({ + query_address: address, + found: false, + procedure: null, + reason: "not_in_procedure", +}); + +/** + * A provider with two string matches (0x10 and 0x11), each referenced from + * one address (0x20 and 0x21). + */ +const provider = (behavior: Behavior = {}): AnalysisOperationPort => ({ + execute: async (name, arguments_) => { + const address = String(arguments_.address); + behavior.calls?.push(`${name}:${address}`); + switch (name) { + case "search_strings": + return ok( + createAnalysisExecution( + [ + { address: "0x10", value: "hello" }, + { address: "0x11", value: "hello again" }, + ], + PROVIDER, + ), + ); + case "search_procedures": + return ok( + createAnalysisExecution( + behavior.duplicateProcedure === true + ? [{ address: "0x10", value: "hello" }] + : [], + PROVIDER, + ), + ); + case "xrefs": { + if (address === behavior.failXrefsOf) + return err(new AnalysisOutputError("xrefs", "provider failed")); + const reported = behavior.coverage?.[address]; + return ok( + createAnalysisExecution( + [address === "0x10" ? "0x20" : "0x21"], + PROVIDER, + { + limitations: behavior.limitations ?? [], + ...(reported === undefined + ? {} + : { referencesTruncated: reported }), + }, + ), + ); + } + case "resolve_containing_procedure": + return ok(createAnalysisExecution(containing(address), PROVIDER)); + default: + throw new Error(`Unexpected operation: ${name}`); + } + }, +}); + +const expectedTrace = ( + residual: readonly string[], + extraMatches: readonly object[] = [], +) => ({ + query: "hello", + search_mode: "literal", + matches: [ + { type: "string", address: "0x10", value: "hello" }, + { type: "string", address: "0x11", value: "hello again" }, + ...extraMatches, + ], + references: [ + { + target_address: "0x10", + source_address: "0x20", + containing_procedure: containing("0x20"), + }, + { + target_address: "0x11", + source_address: "0x21", + containing_procedure: containing("0x21"), + }, + ...extraMatches.map(() => ({ + target_address: "0x10", + source_address: "0x20", + containing_procedure: containing("0x20"), + })), + ], + truncated: residual.length > 0, + residual_unknowns: residual, +}); + +const unknownFor = (address: string) => + `References to ${address} were truncated by the provider; additional references may exist.`; + +const trace = async (analysis: AnalysisOperationPort) => { + const client = await connect(analysis); + const result = await client.callTool({ + name: "trace_feature", + arguments: { query: "hello" }, + }); + expect(result.isError).not.toBe(true); + return result.structuredContent as { normalized_result: unknown }; +}; + +describe("trace_feature reference coverage", () => { + it("marks the trace truncated for the target whose references the provider capped", async () => { + const output = await trace( + provider({ coverage: { "0x10": true, "0x11": false } }), + ); + + expect( + output.normalized_result, + "the received references are kept and only the capped target is a residual unknown", + ).toEqual(expectedTrace([unknownFor("0x10")])); + }); + + it("does not carry a truncation over to the next target that reports nothing", async () => { + const output = await trace(provider({ coverage: { "0x10": true } })); + + expect(output.normalized_result).toEqual( + expectedTrace([unknownFor("0x10")]), + ); + }); + + it("reports a capped target once when it is matched twice", async () => { + const output = await trace( + provider({ coverage: { "0x10": true }, duplicateProcedure: true }), + ); + + expect(output.normalized_result).toEqual( + expectedTrace( + [unknownFor("0x10")], + [{ type: "procedure", address: "0x10", value: "hello" }], + ), + ); + }); + + it("resolves the containing procedure of a repeated reference source once", async () => { + const calls: string[] = []; + const output = await trace(provider({ duplicateProcedure: true, calls })); + + expect( + calls.filter((call) => call.startsWith("resolve_containing_procedure:")), + "the source 0x20 is referenced by two matches of the same address and resolved once", + ).toEqual([ + "resolve_containing_procedure:0x20", + "resolve_containing_procedure:0x21", + ]); + expect( + output.normalized_result, + "every reference is still reported with its containing procedure", + ).toEqual( + expectedTrace( + [], + [{ type: "procedure", address: "0x10", value: "hello" }], + ), + ); + }); + + it("keeps the trace complete when the provider explicitly reports no cap", async () => { + const output = await trace( + provider({ coverage: { "0x10": false, "0x11": false } }), + ); + + expect(output.normalized_result).toEqual(expectedTrace([])); + }); + + it("does not change the trace of a provider that reports no coverage", async () => { + const output = await trace( + provider({ + limitations: ["Upstream xrefs_to capped results at 1000 references."], + }), + ); + + expect( + output.normalized_result, + "a limitation text alone never truncates, and an absent field keeps the previous result", + ).toEqual(expectedTrace([])); + }); + + it("never exposes the coverage field in the tool output", async () => { + const output = await trace( + provider({ coverage: { "0x10": true, "0x11": false } }), + ); + + expect(JSON.stringify(output)).not.toMatch( + /referencesTruncated|references_truncated/, + ); + }); + + it("fails the trace when a references read fails and reads nothing further", async () => { + const calls: string[] = []; + const result = await new EnhancedTools( + provider({ failXrefsOf: "0x10", calls }), + ).execute("trace_feature", { query: "hello" }); + + expect(result.ok).toBe(false); + expect( + calls.filter((call) => call.startsWith("xrefs:")), + "the next target is not read after the failure", + ).toEqual(["xrefs:0x10"]); + }); + + it("stops without reading references when the trace is cancelled", async () => { + const calls: string[] = []; + const controller = new AbortController(); + controller.abort(); + const result = await new EnhancedTools(provider({ calls })).execute( + "trace_feature", + { query: "hello" }, + controller.signal, + ); + + expect(result.ok).toBe(false); + expect(calls.filter((call) => call.startsWith("xrefs:"))).toEqual([]); + }); +}); diff --git a/tests/boundary/mcp/ghidraEvidenceMcp.test.ts b/tests/boundary/mcp/ghidraEvidenceMcp.test.ts index c28fcca39..ed82400aa 100644 --- a/tests/boundary/mcp/ghidraEvidenceMcp.test.ts +++ b/tests/boundary/mcp/ghidraEvidenceMcp.test.ts @@ -46,6 +46,8 @@ it("retains terminal-call qualifications across function views and snapshot repl limitations: [limitation], }, ], + ["procedure_callees", ["0x402000"]], + ["procedure_callers", [identity.address]], ["analyze_function", jsonValueSchema.parse(dossier)], ]); const harness = await connectGhidraMcp( @@ -81,7 +83,7 @@ it("retains terminal-call qualifications across function views and snapshot repl }); expect(evidence.limitations, name).toContain(limitation); expect(evidence.analysis_profile?.parameters).toMatchObject({ - function_boundary_observations: "ghidra-terminal-call-limitations-v1", + function_boundary_observations: "ghidra-terminal-call-limitations-v2", }); const replay = await harness.session.execute(name, evidence.parameters); if (!replay.ok) throw replay.error; diff --git a/tests/boundary/mcp/ghidraMcpHarness.ts b/tests/boundary/mcp/ghidraMcpHarness.ts index ad3565992..a9b2248dc 100644 --- a/tests/boundary/mcp/ghidraMcpHarness.ts +++ b/tests/boundary/mcp/ghidraMcpHarness.ts @@ -263,6 +263,8 @@ const resultFor = ( const functionObservationOperations: ReadonlySet = new Set([ "analyze_function", "procedure_assembly", + "procedure_callees", + "procedure_callers", "procedure_info", "procedure_pseudo_code", "read_function_instructions", diff --git a/tests/boundary/mcp/rawBinaryMcp.test.ts b/tests/boundary/mcp/rawBinaryMcp.test.ts index c1d2029c2..26a0e6501 100644 --- a/tests/boundary/mcp/rawBinaryMcp.test.ts +++ b/tests/boundary/mcp/rawBinaryMcp.test.ts @@ -1,4 +1,4 @@ -import { writeFile } from "node:fs/promises"; +import { mkdir, symlink, writeFile } from "node:fs/promises"; import { join } from "node:path"; import { Client, InMemoryTransport } from "@modelcontextprotocol/client"; @@ -10,7 +10,8 @@ import { createServer } from "../../../src/server/createServer.js"; import { toolContract } from "../../../src/contracts/toolContracts.js"; import type { AnalysisProvider } from "../../../src/application/AnalysisProvider.js"; import { createAnalysisProfile } from "../../../src/domain/analysisProfile.js"; -import { ok } from "../../../src/domain/result.js"; +import { ProviderAdapterError } from "../../../src/domain/providerAdapterError.js"; +import { err, ok } from "../../../src/domain/result.js"; import { observed } from "../../fixtures/analysisExecution.js"; import { createCacheProvider, @@ -18,6 +19,7 @@ import { } from "../../fixtures/binarySession.js"; import { createTestTempDirectory } from "../../fixtures/temporaryDirectory.js"; import { silentLogger } from "../../../src/logger.js"; +import { GHIDRA_PROVIDER_IDENTITY } from "../../../src/ghidra/GhidraProviderCapabilities.js"; import type { CallToolResult } from "@modelcontextprotocol/server"; const connect = async ( @@ -50,29 +52,97 @@ const structured = (result: CallToolResult): Record => { return z.record(z.string(), z.unknown()).parse(result.structuredContent); }; -it("opens a headerless image when architecture and entry are supplied", async () => { - const root = await createTestTempDirectory("rea-raw-mcp-"); - const path = join(root, "ecu.bin"); - await writeFile(path, Buffer.from("1eff2fe1", "hex")); - const { client } = await connect(); - const opened = await client.callTool({ - name: "open_binary", - arguments: { +const withGhidraFixtureIdentity = ( + provider: AnalysisProvider, +): AnalysisProvider => { + const identity = { ...GHIDRA_PROVIDER_IDENTITY, version: "12.1.4" }; + const profile = createAnalysisProfile(identity, { fixture: true }); + return { + ...provider, + identity: () => identity, + capabilities: () => + provider.capabilities().map((capability) => ({ + ...capability, + provider: identity, + })), + resolveAnalysisProfile: () => Promise.resolve(ok({ profile })), + createClient: (target, _profile, context) => { + const client = provider.createClient(target, profile, context); + return { + ...client, + execute: async (operation, parameters, options) => { + const result = await client.execute(operation, parameters, options); + return result.ok + ? ok({ + ...result.value, + provider: identity, + analysisProfile: profile, + }) + : result; + }, + }; + }, + }; +}; + +it("keeps 6502 out of detected lifecycle output contracts", () => { + const schema = toolContract("open_binary").outputSchema; + const result = { + path: "/fixture/image.bin", + kind: "executable", + sha256: "a".repeat(64), + architecture: "6502", + }; + expect( + schema.safeParse({ result: { ...result, format: "pe" } }).success, + ).toBe(false); + expect( + schema.safeParse({ + result: { ...result, format: "raw-binary", entry_point: "0x8001" }, + }).success, + ).toBe(true); + const validateAdvertised = new Ajv2020({ strict: false }).compile( + z.toJSONSchema(schema), + ); + expect(validateAdvertised({ result: { ...result, format: "pe" } })).toBe( + false, + ); + expect( + validateAdvertised({ + result: { ...result, format: "raw-binary", entry_point: "0x8001" }, + }), + ).toBe(true); +}); + +it.each([ + ["arm", "0x08000000", "0x8000000", "1eff2fe1"], + ["6502", "0x008001", "0x8001", "a92a60"], +])( + "opens a headerless %s image when architecture and entry are supplied", + async (architecture, entryPoint, canonical, bytes) => { + const root = await createTestTempDirectory("rea-raw-mcp-"); + const path = join(root, "ecu.bin"); + await writeFile(path, Buffer.from(bytes, "hex")); + const { client } = await connect(); + const opened = await client.callTool({ + name: "open_binary", + arguments: { + path, + format: "raw-binary", + architecture, + entry_point: entryPoint, + }, + }); + expect(opened.isError).not.toBe(true); + expect(structured(opened).result).toMatchObject({ path, format: "raw-binary", - architecture: "arm", - entry_point: "0x08000000", - }, - }); - expect(opened.isError).not.toBe(true); - expect(structured(opened).result).toMatchObject({ - path, - format: "raw-binary", - kind: "executable", - architecture: "arm", - entry_point: "0x8000000", - }); -}); + kind: "executable", + architecture, + entry_point: canonical, + }); + }, +); it("rejects an unaligned AArch64 entry before selecting or starting a provider", async () => { const root = await createTestTempDirectory("rea-raw-mcp-alignment-"); @@ -217,3 +287,118 @@ it("reuses a canonical raw entry and replaces the session when the entry changes expect(created).toEqual(["0x8000000", "0x9000000"]); expect(closed).toEqual(["0x8000000"]); }); + +it.skipIf(process.platform !== "win32")( + "does not replay a saved read after switching to a Windows junction", + async () => { + const root = await createTestTempDirectory("rea-raw-mcp-linked-reopen-"); + const actual = join(root, "actual"); + const linked = join(root, "linked"); + await mkdir(actual); + const path = join(actual, "image.bin"); + await writeFile(path, Buffer.from("a92a60", "hex")); + await symlink(actual, linked, "junction"); + const created: string[] = []; + const executed: string[] = []; + const fixture = createCacheProvider([]); + const { client } = await connect( + withGhidraFixtureIdentity({ + ...fixture, + createClient: (target) => { + created.push(target.sourcePath ?? target.path); + return { + execute: (operation) => { + if (operation === "address_name") + executed.push(target.sourcePath ?? target.path); + if ( + operation === "address_name" && + target.sourcePath === join(linked, "image.bin") + ) + return Promise.resolve( + err(new ProviderAdapterError("fixture", operation)), + ); + return Promise.resolve( + observed(operation === "address_name" ? "entry" : null), + ); + }, + close: () => Promise.resolve(ok(null)), + }; + }, + }), + ); + const linkedPath = join(linked, "image.bin"); + for (const requested of [path, linkedPath]) { + const opened = await client.callTool({ + name: "open_binary", + arguments: { + path: requested, + format: "raw-binary", + architecture: "6502", + entry_point: "0x8001", + }, + }); + expect(opened.isError, JSON.stringify(opened)).not.toBe(true); + const address = await client.callTool({ + name: "address_name", + arguments: { address: "0x8001", document: "image.bin" }, + }); + expect(address.isError === true, JSON.stringify(address)).toBe( + requested === linkedPath, + ); + } + expect(created).toEqual([path, linkedPath]); + expect(executed).toEqual([path, linkedPath]); + }, +); + +it.skipIf(process.platform !== "win32")( + "reuses saved reads after a fixture admits a noncanonical Windows path", + async () => { + const root = await createTestTempDirectory("rea-raw-mcp-admitted-path-"); + const actual = join(root, "actual"); + const linked = join(root, "linked"); + await mkdir(actual); + const path = join(actual, "image.bin"); + await writeFile(path, Buffer.from("a92a60", "hex")); + await symlink(actual, linked, "junction"); + const calls: string[] = []; + const fixture = createCacheProvider([]); + // This fixture accepts the alias to isolate cache behavior after admission. + // The real Ghidra verifier checks that a junction itself is rejected. + const { client, session } = await connect( + withGhidraFixtureIdentity({ + ...fixture, + createClient: () => ({ + execute: (operation) => { + if (operation === "address_name") calls.push(operation); + return Promise.resolve( + observed(operation === "health" ? null : "entry"), + ); + }, + close: () => Promise.resolve(ok(null)), + }), + }), + ); + const linkedPath = join(linked, "image.bin"); + const opened = await client.callTool({ + name: "open_binary", + arguments: { + path: linkedPath, + format: "raw-binary", + architecture: "6502", + entry_point: "0x8001", + }, + }); + expect(opened.isError, JSON.stringify(opened)).not.toBe(true); + expect(session.activeTarget()?.sourcePath).toBe(linkedPath); + expect(session.activeTarget()?.path).toBe(path); + for (const _ of [0, 1]) { + const address = await client.callTool({ + name: "address_name", + arguments: { address: "0x8001", document: "image.bin" }, + }); + expect(address.isError, JSON.stringify(address)).not.toBe(true); + } + expect(calls).toEqual(["address_name"]); + }, +); diff --git a/tests/boundary/process/ownedCommand.test.ts b/tests/boundary/process/ownedCommand.test.ts index 34205af3d..ca0cb2db6 100644 --- a/tests/boundary/process/ownedCommand.test.ts +++ b/tests/boundary/process/ownedCommand.test.ts @@ -257,9 +257,11 @@ it.skipIf(process.platform === "win32").each(["cancelled", "timeout"] as const)( failures: [expect.objectContaining({ reason: "run-token-mismatch" })], }); expect(failure.cleanupResult).toEqual(cleanup); - expect(failure.resources).toEqual( - expect.arrayContaining([request.runId]), - ); + expect(failure.resources).toEqual([ + request.runId, + `pid:${launched.ownership.leaderPid}`, + `process-group:${launched.ownership.processGroupId}`, + ]); // An unresolved member does not grant authority to signal the shared group. const acquiredPid = launched.ownership.leaderPid; expect(() => process.kill(acquiredPid, 0)).not.toThrow(); diff --git a/tests/boundary/process/processCaptureLifecycle.test.ts b/tests/boundary/process/processCaptureLifecycle.test.ts index c3dc21542..01d8eba13 100644 --- a/tests/boundary/process/processCaptureLifecycle.test.ts +++ b/tests/boundary/process/processCaptureLifecycle.test.ts @@ -196,9 +196,9 @@ itWithCaptureCapability( result.value.frames.map(({ at_ms }) => at_ms), "capture-relative frame times stay ordered and nonnegative", ).toEqual( - [...result.value.frames.map(({ at_ms }) => at_ms)].sort( - (left, right) => left - right, - ), + result.value.frames + .map(({ at_ms }) => at_ms) + .sort((left, right) => left - right), ); expect(result.value.frames.every(({ at_ms }) => at_ms >= 0)).toBe(true); expect(result.value.exit.reason).toBe("exited"); diff --git a/tests/boundary/process/processCaptureNormalization.test.ts b/tests/boundary/process/processCaptureNormalization.test.ts index 2837ccf40..159f84c93 100644 --- a/tests/boundary/process/processCaptureNormalization.test.ts +++ b/tests/boundary/process/processCaptureNormalization.test.ts @@ -28,6 +28,35 @@ const identitySchema = z.object({ const itWithLinuxCaptureCapability = process.platform === "linux" ? itWithCaptureCapability : it.skip; +itWithCaptureCapability( + "captures a POSIX PATH executable with a literal backslash in its basename", + async () => { + const root = await createTestTempDirectory("rea-process-backslash-name-"); + const bin = join(root, "bin"); + await mkdir(bin); + const command = "selected\\node"; + const selected = join(bin, command); + await symlink(process.execPath, selected); + const scenario = parseProcessScenario({ + executable: command, + working_directory: root, + environment: { PATH: bin }, + arguments: ["-e", 'process.stdout.write("selected executable")'], + settle_ms: 100, + }); + const result = await captureProcessScenario(scenario); + if (!result.ok) throw result.error; + expect(result.value.frames.map(({ data }) => data).join("")).toBe( + "selected executable", + ); + expect(result.value.exit).toMatchObject({ code: 0, reason: "exited" }); + expect(result.value.cleanup).toMatchObject({ + owned_process_group: "verified", + temporary_root: "removed", + }); + }, +); + itWithCaptureCapability( "preserves numeric reports and original normalized PTY text in Evidence", async () => { diff --git a/tests/boundary/providers/ghidra/ghidraLauncher.test.ts b/tests/boundary/providers/ghidra/ghidraLauncher.test.ts index 270bf02b3..ea8fb964c 100644 --- a/tests/boundary/providers/ghidra/ghidraLauncher.test.ts +++ b/tests/boundary/providers/ghidra/ghidraLauncher.test.ts @@ -1,4 +1,11 @@ -import { access, chmod, readFile, stat } from "node:fs/promises"; +import { + access, + chmod, + mkdir, + readFile, + stat, + writeFile, +} from "node:fs/promises"; import { join, posix, win32 } from "node:path"; import { fileURLToPath } from "node:url"; @@ -480,6 +487,64 @@ describe("Ghidra Windows launcher", () => { ).toThrow(/metacharacters/u); }); + it.skipIf(process.platform !== "win32").each([ + { + properties: + "application.version=12.1.4\napplication.release.name=PUBLIC\n", + release: "ghidra_12.1.4_PUBLIC", + }, + { properties: undefined, release: "renamed-install" }, + { properties: "application.version=12.1.4\n", release: "renamed-install" }, + ])( + "prepares the release directory through the Windows runtime ($release)", + async ({ properties, release }) => { + const parent = await createTestTempDirectory("rea-launch-config-"); + const installRoot = join(parent, "renamed-install"); + if (properties !== undefined) { + await mkdir(join(installRoot, "Ghidra"), { recursive: true }); + await writeFile( + join(installRoot, "Ghidra", "application.properties"), + properties, + ); + } + const runtime = await createGhidraTestRuntime(parent); + runtimes.push(runtime); + const spawnProcess = vi.fn(async () => { + throw new Error("fixture stops before process creation"); + }); + const launcher = new GhidraHeadlessLauncher({ + environment: process.env, + platform: "win32", + analyzeHeadlessPath: join( + installRoot, + "support", + "analyzeHeadless.bat", + ), + javaHome: "C:\\Java\\jdk-21", + bridgeScriptPath: join(parent, "ReaGhidraBridge.java"), + spawnProcess, + }); + const launched = await launcher.launch({ + runtimeRoot: runtime.path, + transport: "unix-socket", + endpointPath: join(runtime.path, "bridge.sock"), + token: "fixture-token", + runId: "d6fcbb66-e829-4ff6-a535-0035aec63139", + targetPath: join(parent, "fixture.exe"), + targetSha256: "b".repeat(64), + providerVersion: "12.1.4", + profileDigest: "a".repeat(64), + }); + expect(launched.ok).toBe(false); + expect(spawnProcess).toHaveBeenCalledOnce(); + expect( + ( + await stat(join(runtime.path, "config", "ghidra", release)) + ).isDirectory(), + ).toBe(true); + }, + ); + it.skipIf(process.platform !== "win32")( "keeps authority in a private descriptor and isolates Ghidra state through the batch launcher", async () => { diff --git a/tests/boundary/providers/ida/containment.test.ts b/tests/boundary/providers/ida/containment.test.ts new file mode 100644 index 000000000..75eb78f42 --- /dev/null +++ b/tests/boundary/providers/ida/containment.test.ts @@ -0,0 +1,144 @@ +import { rm } from "node:fs/promises"; +import { afterEach, describe, expect, it } from "vitest"; +import { inspectAddressContext } from "../../../../src/application/AnalysisContextQueries.js"; +import { EnhancedTools } from "../../../../src/application/EnhancedTools.js"; +import { IdaSessionClient } from "../../../../src/ida/IdaSessionClient.js"; +import { createIdaTarget, RecordingIdaMcp } from "../../../fixtures/idaMcp.js"; + +const roots: string[] = []; +afterEach(async () => { + for (const root of roots.splice(0)) + await rm(root, { recursive: true, force: true }); +}); + +const session = async (cappedReferences = false) => { + const { root, target } = await createIdaTarget("arm64"); + roots.push(root); + const producer = new RecordingIdaMcp(target, "headless"); + if (cappedReferences) { + const inner = producer.overrideCall; + producer.overrideCall = (name, args) => + name === "xrefs_to" + ? [ + { + xrefs: [ + { + addr: "0x1004", + type: "code", + fn: { addr: "0x1000", name: "main", size: "0x20" }, + }, + ], + more: true, + }, + ] + : inner?.(name, args); + } + const client = new IdaSessionClient( + { + command: "fixture", + args: [], + env: {}, + mode: "headless", + timeoutMs: 1000, + workspaceRoot: root, + }, + target, + producer, + ); + return { client, target }; +}; + +const MAIN = { + found: true, + procedure: { address: "0x1000", name: "main" }, +}; + +describe("trace_feature on headless IDA", () => { + it("attributes a reference to its containing procedure", async () => { + const { client } = await session(); + try { + const result = await new EnhancedTools(client).execute("trace_feature", { + query: "a.b", + }); + + if (!result.ok) throw result.error; + expect(result.value).toMatchObject({ + matches: [{ type: "string", address: "0x3000" }], + references: [ + { + target_address: "0x3000", + source_address: "0x1004", + containing_procedure: MAIN, + }, + ], + truncated: false, + residual_unknowns: [], + }); + } finally { + expect((await client.close()).ok).toBe(true); + } + }); + + it("keeps an upstream reference cap visible once containment makes the trace run", async () => { + const { client } = await session(true); + try { + const result = await new EnhancedTools(client).execute("trace_feature", { + query: "a.b", + }); + + if (!result.ok) throw result.error; + expect(result.value).toMatchObject({ + references: [{ source_address: "0x1004", containing_procedure: MAIN }], + truncated: true, + residual_unknowns: [ + "References to 0x3000 were truncated by the provider; additional references may exist.", + ], + }); + } finally { + expect((await client.close()).ok).toBe(true); + } + }); +}); + +describe("inspect_address_context on headless IDA", () => { + it("resolves the admitted program as the document and reports the containing procedure", async () => { + const { client, target } = await session(); + try { + const result = await inspectAddressContext(client, { + address: "0x1004", + }); + + if (!result.ok) throw result.error; + expect(result.value).toMatchObject({ + address: "0x1004", + document: target.path, + procedure: { state: "available", value: MAIN }, + name: { state: "unavailable" }, + comment: { state: "unavailable" }, + inline_comment: { state: "unavailable" }, + bookmarks: { state: "unavailable" }, + }); + } finally { + expect((await client.close()).ok).toBe(true); + } + }); + + it("answers the same way for an explicit admitted document", async () => { + const { client, target } = await session(); + try { + const implicit = await inspectAddressContext(client, { + address: "0x1004", + }); + const explicit = await inspectAddressContext(client, { + address: "0x1004", + document: target.path, + }); + + if (!implicit.ok) throw implicit.error; + if (!explicit.ok) throw explicit.error; + expect(explicit.value).toEqual(implicit.value); + } finally { + expect((await client.close()).ok).toBe(true); + } + }); +}); diff --git a/tests/boundary/providers/ida/retainedConnectionRetry.test.ts b/tests/boundary/providers/ida/retainedConnectionRetry.test.ts new file mode 100644 index 000000000..4f9092842 --- /dev/null +++ b/tests/boundary/providers/ida/retainedConnectionRetry.test.ts @@ -0,0 +1,114 @@ +import { createHash } from "node:crypto"; +import { mkdir, readdir, writeFile } from "node:fs/promises"; +import { join } from "node:path"; + +import { describe, expect, it, onTestFinished } from "vitest"; + +import { createTestTempDirectory } from "../../../fixtures/temporaryDirectory.js"; + +import { elf } from "../../../../src/domain/binaryTarget.fixture.js"; +import type { BinaryTarget } from "../../../../src/domain/binaryTargetTypes.js"; +import { createIdaMcpConnection } from "../../../../src/ida/IdaMcpConnection.js"; +import { IdaSessionClient } from "../../../../src/ida/IdaSessionClient.js"; + +/** + * A headless IDA supervisor over real stdio that answers the session + * lifecycle, but rejects the first inventory after a close so cleanup fails + * once and must be retried through the retained connection. + */ +const FLAKY_INVENTORY_SUPERVISOR = `import { createInterface } from "node:readline"; +const tools = ["idb_open", "idb_list", "idb_close", "server_health", "list_funcs", + "lookup_funcs", "find_regex", "decompile", "disasm", "callees", "xrefs_to"]; +let session; +let inputPath; +let closed = false; +let inventoryFailed = false; +const send = (message) => + process.stdout.write(JSON.stringify({ jsonrpc: "2.0", ...message }) + "\\n"); +const result = (value) => ({ + content: [{ type: "text", text: JSON.stringify(value) }], + structuredContent: value, +}); +const call = ({ name, arguments: args }) => { + if (name === "idb_open") { + session = args.preferred_session_id; + inputPath = args.input_path; + return result({ success: true, session: { session_id: session, input_path: inputPath } }); + } + if (name === "server_health") + return result({ status: "ok", input_path: inputPath, idb_path: inputPath + ".i64", + module: "target.elf", imagebase: "0x1000", auto_analysis_ready: true }); + if (name === "idb_close") { + closed = true; + return result({ success: true, session_id: session, saved: null, owned: true, backend: "worker" }); + } + if (name === "idb_list") { + if (closed && !inventoryFailed) { + inventoryFailed = true; + return { isError: true, content: [{ type: "text", text: "inventory unavailable" }] }; + } + return result({ sessions: closed ? [] : [{ session_id: session, input_path: inputPath, + owned: true, backend: "worker", is_active: true }] }); + } + return { isError: true, content: [{ type: "text", text: "unsupported " + name }] }; +}; +for await (const line of createInterface({ input: process.stdin })) { + const request = JSON.parse(line); + if (request.method === "initialize") + send({ id: request.id, result: { + protocolVersion: request.params.protocolVersion, + capabilities: { tools: {} }, + serverInfo: { name: "flaky-inventory-ida-fixture", version: "1" }, + } }); + else if (request.method === "tools/list") + send({ id: request.id, result: { + tools: tools.map((name) => ({ name, inputSchema: { type: "object" } })), + } }); + else if (request.method === "tools/call") + send({ id: request.id, result: call(request.params) }); +} +`; + +describe("retained headless IDA connection in a long-lived host", () => { + it("retries cleanup through the detached stdio connection", async () => { + const root = await createTestTempDirectory("rea-ida-retry-"); + const supervisor = join(root, "supervisor.mjs"); + await writeFile(supervisor, FLAKY_INVENTORY_SUPERVISOR); + const workspaceRoot = join(root, "workspaces"); + await mkdir(workspaceRoot); + const bytes = elf(2, 1, 62); + const targetPath = join(root, "target.elf"); + await writeFile(targetPath, bytes); + const target: BinaryTarget = { + path: targetPath, + sha256: createHash("sha256").update(bytes).digest("hex"), + kind: "executable", + format: "elf", + architecture: "x86_64", + availableArchitectures: ["x86_64"], + }; + const config = { + command: process.execPath, + args: [supervisor], + env: {}, + mode: "headless" as const, + timeoutMs: 5000, + workspaceRoot, + }; + const connection = createIdaMcpConnection(config, process.env); + // Close the supervisor even when an assertion fails before cleanup does. + onTestFinished(() => connection.close().catch(() => undefined)); + const client = new IdaSessionClient(config, target, connection); + + expect((await client.execute("health", {})).ok).toBe(true); + const failed = await client.close(); + expect(failed.ok).toBe(false); + if (!failed.ok) expect(failed.error.cleanupIncomplete).toBe(true); + expect(await readdir(workspaceRoot)).toHaveLength(1); + + // The connection was detached when cleanup failed; it must still serve + // the retry that confirms release and removes the private workspace. + expect((await client.close()).ok).toBe(true); + expect(await readdir(workspaceRoot)).toEqual([]); + }); +}); diff --git a/tests/boundary/providers/ida/verifierFeatureTrace.test.ts b/tests/boundary/providers/ida/verifierFeatureTrace.test.ts new file mode 100644 index 000000000..77f654ffc --- /dev/null +++ b/tests/boundary/providers/ida/verifierFeatureTrace.test.ts @@ -0,0 +1,212 @@ +import { describe, expect, it } from "vitest"; +import { + assertFeatureTrace, + findMappedNonFunction, +} from "../../../../scripts/lib/ida-feature-trace.mjs"; + +const procedure = { address: "0x1000", name: "main" }; + +const containing = (found: boolean) => + found + ? { + query_address: "0x1004", + found: true, + procedure: { address: "0x1000", name: "main" }, + } + : { + query_address: "0x3010", + found: false, + procedure: null, + reason: "not_in_procedure", + }; + +const trace = ( + matches: readonly { type: string; address: string }[], + references: readonly { + target_address: string; + containing_procedure: unknown; + }[], +) => ({ + query: "main", + search_mode: "literal", + matches, + references, + truncated: false, + residual_unknowns: [], +}); + +const SELECTED = { type: "procedure", address: "0x1000" }; +const SUBSTRING = { type: "procedure", address: "0x2000" }; +const STRING = { type: "string", address: "0x3000" }; + +describe("the real IDA lane's feature trace assertions", () => { + it("accepts references to every matched address, whoever contains the source", () => { + expect(() => + assertFeatureTrace( + trace( + [SELECTED, SUBSTRING, STRING], + [ + { + target_address: "0x1000", + containing_procedure: containing(true), + }, + // A substring match such as main_helper, referenced from code. + { + target_address: "0x2000", + containing_procedure: containing(true), + }, + // A string containing the query, referenced from data outside functions. + { + target_address: "0x3000", + containing_procedure: containing(false), + }, + ], + ), + procedure, + ), + ).not.toThrow(); + }); + + it("accepts a reference to the selected procedure from data outside functions", () => { + expect(() => + assertFeatureTrace( + trace( + [SELECTED], + [ + { + target_address: "0x1000", + containing_procedure: containing(false), + }, + ], + ), + procedure, + ), + ).not.toThrow(); + }); + + it("rejects a reference whose target is not one of the matches", () => { + expect(() => + assertFeatureTrace( + trace( + [SELECTED], + [ + { + target_address: "0x9999", + containing_procedure: containing(true), + }, + ], + ), + procedure, + ), + ).toThrow(); + }); + + it("rejects a trace without the selected procedure's match", () => { + expect(() => assertFeatureTrace(trace([STRING], []), procedure)).toThrow(); + }); + + it("rejects a containment answer that is neither found nor a known reason", () => { + expect(() => + assertFeatureTrace( + trace( + [SELECTED], + [ + { + target_address: "0x1000", + containing_procedure: { + query_address: "0x1004", + found: false, + procedure: null, + reason: "unknown", + }, + }, + ], + ), + procedure, + ), + ).toThrow(); + }); + + it("rejects a trace that does not say whether it was truncated", () => { + expect(() => + assertFeatureTrace( + { ...trace([SELECTED], []), truncated: undefined }, + procedure, + ), + ).toThrow(); + }); +}); + +describe("the real IDA lane's mapped non-function probe", () => { + const inFunction = { + query_address: "0x1100", + found: true, + procedure: { address: "0x1000", name: "main" }, + }; + const data = { + query_address: "0x3000", + found: false, + procedure: null, + reason: "not_in_procedure", + }; + const resolver = + (answers: Readonly>, asked: string[] = []) => + async (address: string) => { + asked.push(address); + return answers[address]; + }; + + it("skips a string inside a function chunk and uses the next one", async () => { + const asked: string[] = []; + + await expect( + findMappedNonFunction( + [{ address: "0x1100" }, { address: "0x3000" }], + resolver({ "0x1100": inFunction, "0x3000": data }, asked), + ), + ).resolves.toBe("0x3000"); + expect(asked, "strings are tried in order until one qualifies").toEqual([ + "0x1100", + "0x3000", + ]); + }); + + it("stops at the first string that qualifies", async () => { + const asked: string[] = []; + + await expect( + findMappedNonFunction( + [{ address: "0x3000" }, { address: "0x3100" }], + resolver({ "0x3000": data, "0x3100": data }, asked), + ), + ).resolves.toBe("0x3000"); + expect(asked).toEqual(["0x3000"]); + }); + + it("reports no qualifying string when every string is inside a function", async () => { + await expect( + findMappedNonFunction( + [{ address: "0x1100" }, { address: "0x1200" }], + resolver({ "0x1100": inFunction, "0x1200": inFunction }), + ), + ).resolves.toBeUndefined(); + }); + + it("reports no qualifying string when the target has no strings", async () => { + await expect( + findMappedNonFunction([], resolver({})), + ).resolves.toBeUndefined(); + }); + + it("rejects a listed string that resolves as unmapped, since IDA cannot list an unmapped address", async () => { + await expect( + findMappedNonFunction( + [{ address: "0x4000" }, { address: "0x3000" }], + resolver({ + "0x4000": { ...data, reason: "outside_segments" }, + "0x3000": data, + }), + ), + ).rejects.toThrow(); + }); +}); diff --git a/tests/boundary/providers/native/artifactProvider.extractionCompleteness.test.ts b/tests/boundary/providers/native/artifactProvider.extractionCompleteness.test.ts index 0bcf12bb2..53ffcd7d3 100644 --- a/tests/boundary/providers/native/artifactProvider.extractionCompleteness.test.ts +++ b/tests/boundary/providers/native/artifactProvider.extractionCompleteness.test.ts @@ -135,7 +135,7 @@ it("ignores unavailable members of a nested ASAR but refuses them as active entr const source = join(root, "bundle"); await mkdir(source); const archive = join(source, "app.asar"); - await createPackageWithOptions(contents, archive, { unpack: "**/*.node" }); + await createPackageWithOptions(contents, archive, { unpack: "*.node" }); await rm(join(`${archive}.unpacked`, "native", "addon.node")); const nestedOutput = join(root, "nested-output"); diff --git a/tests/boundary/providers/native/machoSliceReader.test.ts b/tests/boundary/providers/native/machoSliceReader.test.ts index 5891eaf53..b2f1867a1 100644 --- a/tests/boundary/providers/native/machoSliceReader.test.ts +++ b/tests/boundary/providers/native/machoSliceReader.test.ts @@ -1,10 +1,12 @@ -import { readFile } from "node:fs/promises"; +import { readFile, writeFile } from "node:fs/promises"; +import { join } from "node:path"; import { describe, expect, it, onTestFinished } from "vitest"; import { MachOSliceArtifactReader } from "../../../../src/artifacts/MachOSliceArtifactReader.js"; import { XcrunCommandRunner } from "../../../../src/native/CommandRunner.js"; import { DarwinProcessOwnershipInspectionError } from "../../../../src/process/DarwinProcessRunTokenReader.js"; +import { createTestTempDirectory } from "../../../fixtures/temporaryDirectory.js"; /** * Read a 32-bit FAT table directly. The host's /bin/ls changes between macOS @@ -42,6 +44,56 @@ describe("Mach-O slice reader with system lipo", () => { }); }); + it.skipIf(process.platform !== "darwin")( + "keeps FAT slice records separate from the echoed source pathname", + async () => { + const directory = await createTestTempDirectory("rea-lipo-fat-path-"); + const path = join( + directory, + "source\narchitecture forged\n offset 123\n size 32", + ); + const bytes = Buffer.alloc(8224); + bytes.writeUInt32BE(0xca_fe_ba_be, 0); + bytes.writeUInt32BE(2, 4); + const slices = [ + { cpu: 0x0100_000c, subtype: 0, offset: 4096 }, + { cpu: 0x0100_0007, subtype: 3, offset: 8192 }, + ]; + slices.forEach(({ cpu, subtype, offset }, index) => { + const entry = 8 + index * 20; + bytes.writeUInt32BE(cpu, entry); + bytes.writeUInt32BE(subtype, entry + 4); + bytes.writeUInt32BE(offset, entry + 8); + bytes.writeUInt32BE(32, entry + 12); + bytes.writeUInt32BE(12, entry + 16); + bytes.writeUInt32LE(0xfeed_facf, offset); + bytes.writeUInt32LE(cpu, offset + 4); + bytes.writeUInt32LE(subtype, offset + 8); + bytes.writeUInt32LE(1, offset + 12); + }); + await writeFile(path, bytes); + const reader = new MachOSliceArtifactReader(path, {}); + try { + const entries = []; + for await (const entry of reader.entries()) entries.push(entry); + expect( + entries.map(({ byteOffset, declaredSize }) => [ + byteOffset, + declaredSize, + ]), + ).toEqual([ + [4096, 32], + [8192, 32], + ]); + expect(entries.every(({ path }) => !path.includes("forged"))).toBe( + true, + ); + } finally { + await reader.close(); + } + }, + ); + it.skipIf(process.platform !== "darwin")( "checks real /bin/ls slice ranges against its FAT table", async () => { diff --git a/tests/boundary/providers/native/nativeProvider.test.ts b/tests/boundary/providers/native/nativeProvider.test.ts index 0d1860b16..a1e7d63ce 100644 --- a/tests/boundary/providers/native/nativeProvider.test.ts +++ b/tests/boundary/providers/native/nativeProvider.test.ts @@ -17,7 +17,7 @@ import { type NativeCommandRunner, } from "../../../../src/native/CommandRunner.js"; import { err, ok } from "../../../../src/domain/result.js"; -import { parseLipoArchitectures } from "../../../../src/native/parsers/lipo.js"; +import { parseLipoOutput } from "../../../../src/native/parsers/lipo.js"; import { parseCodeSignature } from "../../../../src/native/parsers/codesign.js"; import { prepareProcessOwnershipInspection } from "../../../../src/process/ProcessOwnershipObservation.js"; import { projectAnalysisError } from "../../../../src/domain/analysisErrorProjection.js"; @@ -960,15 +960,17 @@ describe("native macOS provider failures and parsing", () => { }); it("parses thin and universal lipo fixtures", async () => { - expect( - parseLipoArchitectures(await fixture("lipo-thin.txt")), - ).toMatchObject([{ name: "arm64" }]); - expect(parseLipoArchitectures(await fixture("lipo-fat.txt"))).toMatchObject( - [ + expect(parseLipoOutput(await fixture("lipo-thin.txt"))).toMatchObject({ + format: "thin", + architectures: [{ name: "arm64" }], + }); + expect(parseLipoOutput(await fixture("lipo-fat.txt"))).toMatchObject({ + format: "fat", + architectures: [ { name: "x86_64", file_offset: 16384, size: 4096 }, { name: "arm64", file_offset: 32768, size: 8192 }, ], - ); + }); }); it("rejects trailing text in lipo numeric fields instead of truncating", () => { @@ -980,7 +982,7 @@ describe("native macOS provider failures and parsing", () => { " size 0x10", " align 2^14 (16384)", ].join("\n"); - expect(parseLipoArchitectures(output)).toMatchObject([ + expect(parseLipoOutput(output).architectures).toMatchObject([ { name: "arm64", file_offset: null, size: null, alignment: 16384 }, ]); }); @@ -993,10 +995,11 @@ describe("native macOS provider failures and parsing", () => { " align 2^14 (16384)", ].join("\n"); expect( - parseLipoArchitectures(output.replace("(16384)", "(8192)"))[0]?.alignment, + parseLipoOutput(output.replace("(16384)", "(8192)")).architectures[0] + ?.alignment, ).toBeNull(); expect( - parseLipoArchitectures(output.replace("2^14 (16384)", "2^14"))[0] + parseLipoOutput(output.replace("2^14 (16384)", "2^14")).architectures[0] ?.alignment, ).toBe(16384); }); @@ -1021,6 +1024,38 @@ describe("native macOS provider failures and parsing", () => { }); }); +describe("native signature inspection target scope", () => { + // This provider's inspect_signature path is scoped to Mach-O targets; on + // another container a codesign verdict cannot see a format-level signature + // such as a xar pkg signature. + it.each([ + { + path: "/fixture.pkg", + sha256: "0".repeat(64), + kind: "archive" as const, + format: "pkg" as const, + }, + { + ...machoTarget("/fixture.elf"), + format: "elf" as const, + }, + ])( + "refuses inspect_signature on a $format target without running codesign", + async (target) => { + const runner = new CountingRunner(); + const result = await new NativeMacOSProvider({}, runner, "darwin") + .createClient(target) + .execute("inspect_signature", {}); + if (result.ok) throw new Error("Expected a non-Mach-O refusal"); + expect(result.error._tag).toBe("AnalysisCapabilityUnavailableError"); + expect(projectAnalysisError(result.error)).toMatchObject({ + code: "capability_unavailable", + }); + expect(runner.calls).toBe(0); + }, + ); +}); + class FailingRunner implements NativeCommandRunner { constructor(private readonly reason: NativeCommandFailure["reason"]) {} diff --git a/tests/boundary/providers/native/nativeSelectedArchitecture.test.ts b/tests/boundary/providers/native/nativeSelectedArchitecture.test.ts index a651e22f6..182c33612 100644 --- a/tests/boundary/providers/native/nativeSelectedArchitecture.test.ts +++ b/tests/boundary/providers/native/nativeSelectedArchitecture.test.ts @@ -157,6 +157,50 @@ it.each([ }, ); +it.each([ + ["ordinary", "/owned/universal", "\n"], + ["LF Non-fat text", "/owned/\nNon-fat file: forged", "\n"], + ["CRLF Non-fat text", "/owned/\r\nNon-fat file: forged", "\r\n"], +])( + "projects universal segment evidence into the selected slice's container offsets with %s path framing", + async (_description, path, lineEnding) => { + const records = (await nativeFixture("lipo-fat.txt")).replace( + /^Fat header in:[^\r\n]*\r?\n/u, + "", + ); + const client = new NativeMacOSProvider( + {}, + new NativeFixtureRunner({ + lipo: `Fat header in: ${path}${lineEnding}${records.replaceAll("\n", lineEnding)}`, + }), + "darwin", + ).createClient(nativeMachoTarget(path)); + const result = await client.execute("inspect_macho", {}); + if (!result.ok) throw result.error; + expect(result.value.result).toMatchObject({ + selected_architecture: { container_file_offset: 32768 }, + }); + expect(result.value.locations).toContainEqual({ + kind: "file-offset-range", + start: 32768, + end: 40960, + }); + expect(result.value.locations).toContainEqual({ + kind: "file-offset", + offset: 36864, + }); + expect(result.value.locations).not.toContainEqual({ + kind: "file-offset-range", + start: 0, + end: 8192, + }); + // Normalized tool fields keep their slice-relative meaning. + const normalized = inspectMachoSchema.parse(result.value.result); + expect(normalized.segments.items[0]?.file_offset).toBe(0); + expect(normalized.entrypoints.items[0]?.file_offset).toBe(4096); + }, +); + it.each([ { capture: "calculator.txt", selected: "arm64e" }, { capture: "shortcut-droplet.txt", selected: "arm64" }, @@ -187,34 +231,6 @@ it.each([ }, ); -it("projects universal segment evidence into the selected slice's container offsets", async () => { - const client = new NativeMacOSProvider( - {}, - new NativeFixtureRunner(), - "darwin", - ).createClient(nativeMachoTarget("/owned/universal")); - const result = await client.execute("inspect_macho", {}); - if (!result.ok) throw result.error; - expect(result.value.locations).toContainEqual({ - kind: "file-offset-range", - start: 32768, - end: 40960, - }); - expect(result.value.locations).toContainEqual({ - kind: "file-offset", - offset: 36864, - }); - expect(result.value.locations).not.toContainEqual({ - kind: "file-offset-range", - start: 0, - end: 8192, - }); - // Normalized tool fields keep their slice-relative meaning. - const normalized = inspectMachoSchema.parse(result.value.result); - expect(normalized.segments.items[0]?.file_offset).toBe(0); - expect(normalized.entrypoints.items[0]?.file_offset).toBe(4096); -}); - it.each([ { description: "thin", diff --git a/tests/boundary/providers/native/nativeTextRobustness.test.ts b/tests/boundary/providers/native/nativeTextRobustness.test.ts index 256aa0a98..ce82bd520 100644 --- a/tests/boundary/providers/native/nativeTextRobustness.test.ts +++ b/tests/boundary/providers/native/nativeTextRobustness.test.ts @@ -2,7 +2,7 @@ import { expect, it } from "vitest"; import { parseOtoolLoadCommands } from "../../../../src/native/parsers/otool.js"; import { parsePlistJson } from "../../../../src/native/parsers/plist.js"; -import { parseLipoArchitectures } from "../../../../src/native/parsers/lipo.js"; +import { parseLipoOutput } from "../../../../src/native/parsers/lipo.js"; import { nativeFixture } from "../../../fixtures/nativeCommands.js"; const toCrlf = (value: string): string => value.replaceAll("\n", "\r\n"); @@ -15,7 +15,10 @@ it.each([ "retains the reported thin architecture when lipo echoes filename %j", (filename) => { const output = `input file /owned/${filename} is not a fat file\nNon-fat file: /owned/${filename} is architecture: arm64\n`; - expect(parseLipoArchitectures(output)).toMatchObject([{ name: "arm64" }]); + expect(parseLipoOutput(output)).toMatchObject({ + format: "thin", + architectures: [{ name: "arm64" }], + }); }, ); @@ -38,10 +41,13 @@ it("does not mistake fat slice output for a thin Non-fat file record", () => { " align 2^12 (4096)", "", ].join("\n"); - expect(parseLipoArchitectures(output)).toMatchObject([ - { name: "arm64", file_offset: 16384 }, - { name: "x86_64", file_offset: 16793600 }, - ]); + expect(parseLipoOutput(output)).toMatchObject({ + format: "fat", + architectures: [ + { name: "arm64", file_offset: 16384 }, + { name: "x86_64", file_offset: 16793600 }, + ], + }); }); it("keeps section blocks separate under CRLF captures", async () => { diff --git a/tests/boundary/sqlite/sqliteDatabaseProvider.test.ts b/tests/boundary/sqlite/sqliteDatabaseProvider.test.ts index 24dadc128..b5adf8297 100644 --- a/tests/boundary/sqlite/sqliteDatabaseProvider.test.ts +++ b/tests/boundary/sqlite/sqliteDatabaseProvider.test.ts @@ -5,6 +5,7 @@ import { DatabaseSync } from "node:sqlite"; import { promisify } from "node:util"; import { fileURLToPath } from "node:url"; import { expect, it as nativeTest, onTestFinished } from "vitest"; +import { projectAnalysisError } from "../../../src/domain/analysisErrorProjection.js"; import { SqliteDatabaseProvider } from "../../../src/sqlite/SqliteDatabaseProvider.js"; import { spawnOwnedProviderProcess } from "../../../src/process/ProviderProcess.js"; import { PrivateRuntimeRoot } from "../../../src/process/PrivateRuntimeRoot.js"; @@ -356,6 +357,49 @@ it("does not execute views when selecting rows", async () => { }); }); +it("attributes table and column selection failures to their own arguments", async () => { + const { database } = await createFixture(); + const provider = createProvider(); + onTestFinished(() => provider.close()); + const projection = async (selection: { + table: string; + columns?: string[]; + }) => { + const result = await provider.inspect({ + path: database.path, + row_limit: 1, + ...selection, + }); + if (result.ok) throw new Error("Expected a rejected selection"); + return projectAnalysisError(result.error); + }; + expect( + await projection({ table: "records", columns: ["nope"] }), + ).toMatchObject({ + details: { + issues: [ + { + path: ["columns"], + reason: "invalid_format", + message: + "Selected column does not exist with that exact name in table records: nope", + }, + ], + }, + }); + expect(await projection({ table: "absent" })).toMatchObject({ + details: { issues: [{ path: ["table"] }] }, + }); + const selected = await provider.inspect({ + path: database.path, + table: "records", + columns: ["id"], + row_limit: 1, + }); + if (!selected.ok) throw selected.error; + expect(selected.value.result).toMatchObject({ rows: { columns: ["id"] } }); +}); + it("rejects malformed databases without altering the selected file", async () => { const workspace = await createTestWorkspace("rea-sqlite-invalid-"); onTestFinished(() => removeTestWorkspace(workspace.root)); diff --git a/tests/composition/analysis-sessions/binarySession.test.ts b/tests/composition/analysis-sessions/binarySession.test.ts index 813b90004..092f869e2 100644 --- a/tests/composition/analysis-sessions/binarySession.test.ts +++ b/tests/composition/analysis-sessions/binarySession.test.ts @@ -1278,3 +1278,102 @@ describe("selector origin forwarding", () => { ).toEqual(["provider", "provider", undefined]); }); }); + +describe("reference truncation forwarding", () => { + const referenceProvider = ( + referencesTruncated: boolean | undefined, + calls: string[], + ): AnalysisProvider => { + const identity = { + id: "fixture", + name: "Fixture analysis provider", + version: "1", + } as const; + return { + identity: () => identity, + resolveAnalysisProfile: () => + Promise.resolve( + resultOk({ + profile: createAnalysisProfile(identity, { fixture: true }), + }), + ), + capabilities: () => [ + { + provider: identity, + operation: "xrefs", + available: true, + reason: null, + effects: { + mutatesArtifact: false, + launchesProcess: false, + mayShowUi: false, + mayAccessNetwork: false, + mayWriteFilesystem: false, + changesPermissions: false, + requiresRoot: false, + }, + limitations: [], + }, + ], + createClient: () => ({ + execute: (operation) => { + calls.push(operation); + return Promise.resolve( + resultOk( + createAnalysisExecution( + ["0x2000"], + identity, + referencesTruncated === undefined + ? {} + : { referencesTruncated }, + ), + ), + ); + }, + close: () => Promise.resolve(resultOk(null)), + }), + }; + }; + const parameters = { address: "0x1000", document: "fixture" }; + + it("forwards a truncation on both execution paths and never replays it from the snapshot", async () => { + const [first] = await createBinarySessionTargets(); + const calls: string[] = []; + const session = createTestBinarySession(referenceProvider(true, calls)); + expect((await session.open(first)).ok).toBe(true); + calls.length = 0; + + const direct = await session.execute("xrefs", parameters); + const admitted = await session.withAdmittedAnalysis( + "trace_feature", + undefined, + (analysis) => analysis.execute("xrefs", parameters), + ); + + expect(direct.ok && direct.value.referencesTruncated).toBe(true); + if (!admitted.ok) throw admitted.error; + expect(admitted.value.ok && admitted.value.value.referencesTruncated).toBe( + true, + ); + expect( + calls, + "a truncated read is not recorded, so every call reaches the provider", + ).toEqual(["xrefs", "xrefs"]); + }); + + it("keeps an explicit complete report out of the way of snapshot replay", async () => { + const [first] = await createBinarySessionTargets(); + const calls: string[] = []; + const session = createTestBinarySession(referenceProvider(false, calls)); + expect((await session.open(first)).ok).toBe(true); + calls.length = 0; + + const live = await session.execute("xrefs", parameters); + await session.execute("xrefs", parameters); + + expect(live.ok && live.value.referencesTruncated).toBe(false); + expect(calls, "the complete read is recorded and replayed").toEqual([ + "xrefs", + ]); + }); +}); diff --git a/tests/conformance/ghidra/6502.cfg b/tests/conformance/ghidra/6502.cfg new file mode 100644 index 000000000..cb85a4e76 --- /dev/null +++ b/tests/conformance/ghidra/6502.cfg @@ -0,0 +1,9 @@ +FEATURES { + STARTADDRESS: default = $8001; +} +MEMORY { + IMAGE: start = %S, size = $10000 - %S, file = %O; +} +SEGMENTS { + CODE: load = IMAGE, type = ro; +} diff --git a/tests/conformance/ghidra/6502.s b/tests/conformance/ghidra/6502.s new file mode 100644 index 000000000..201f8bcd2 --- /dev/null +++ b/tests/conformance/ghidra/6502.s @@ -0,0 +1,19 @@ +; Source-owned NMOS 6502 fixture: direct call, conditional branch, and arithmetic. +.setcpu "6502" +.segment "CODE" +.export entry, add_three + +entry: + lda #7 + jsr add_three + cmp #10 + bne mismatch + lda #42 + rts +mismatch: + lda #0 + rts +add_three: + clc + adc #3 + rts diff --git a/tests/conformance/ida/providerContracts.test.ts b/tests/conformance/ida/providerContracts.test.ts index 39c524637..4e4b04c3b 100644 --- a/tests/conformance/ida/providerContracts.test.ts +++ b/tests/conformance/ida/providerContracts.test.ts @@ -29,15 +29,25 @@ const parametersFor = ( operation: string, ): Readonly> => { if (operation.startsWith("search_")) return { pattern: "a.b" }; - if (operation === "xrefs") return { address: "0x1000" }; + if (operation === "xrefs" || operation === "resolve_containing_procedure") + return { address: "0x1000" }; if (operation.startsWith("list_")) return {}; return { procedure: "main" }; }; +/** Operations only the headless profile declares. */ +const HEADLESS_ONLY: ReadonlySet = new Set([ + "resolve_containing_procedure", + "list_documents", +]); + describe.each(["attached", "headless"] as const)( "IDA %s caller-visible contracts", (mode) => { - it.each(IDA_OPERATIONS)( + const operations = IDA_OPERATIONS.filter( + (operation) => mode === "headless" || !HEADLESS_ONLY.has(operation), + ); + it.each(operations)( "declares %s UI effects matching the upstream tools it reaches", async (operation) => { const { root, target } = await createIdaTarget(); @@ -69,7 +79,7 @@ describe.each(["attached", "headless"] as const)( }, ); - it.each(IDA_OPERATIONS)( + it.each(operations)( "serializes %s against its named advertised output contract", async (operation) => { const { root, target } = await createIdaTarget(); diff --git a/tests/conformance/providers/ghidra/realGhidraWindowsWorkflow.test.ts b/tests/conformance/providers/ghidra/realGhidraWindowsWorkflow.test.ts index 39eb8ff61..0e936b062 100644 --- a/tests/conformance/providers/ghidra/realGhidraWindowsWorkflow.test.ts +++ b/tests/conformance/providers/ghidra/realGhidraWindowsWorkflow.test.ts @@ -26,14 +26,18 @@ describe("real Windows Ghidra workflow trust boundary", () => { environment: "real-ghidra-windows", "runs-on": expect.arrayContaining(["self-hosted", "Windows", "x64"]), steps: expect.arrayContaining([ - expect.objectContaining({ uses: "actions/setup-node@v4" }), + expect.objectContaining({ + uses: expect.stringMatching(/^actions\/setup-node@[0-9a-f]{40}$/), + }), expect.objectContaining({ run: "npm ci" }), expect.objectContaining({ env: { REA_ANALYSIS_PROVIDER: "ghidra" }, run: "npm run verify:ghidra:windows | Tee-Object -FilePath windows-ghidra-proof.log", }), expect.objectContaining({ - uses: "actions/upload-artifact@v4", + uses: expect.stringMatching( + /^actions\/upload-artifact@[0-9a-f]{40}$/, + ), with: expect.objectContaining({ path: expect.stringContaining("windows-ghidra-proof.log"), }), diff --git a/tests/conformance/readmeConsistency.test.ts b/tests/conformance/readmeConsistency.test.ts index 79a1fc518..90f8e5886 100644 --- a/tests/conformance/readmeConsistency.test.ts +++ b/tests/conformance/readmeConsistency.test.ts @@ -35,6 +35,20 @@ const shellExamples = (content: string): string[] => ); describe("onboarding documentation product facts", () => { + it.each([ + ["docs/native-investigation.md", "raw-binary.md#nmos-6502"], + ["docs/ghidra-riscv.md", "raw-binary.md"], + ])( + "routes raw admission guidance in %s to the owning guide", + async (path, target) => { + const content = await readFile(resolve(path), "utf8"); + const links = [...content.matchAll(/\]\(([^)\s]+)\)/gu)].map( + (match) => match[1], + ); + expect(links, path).toContain(target); + }, + ); + it("keeps the installation MCP example aligned with the package registration command", async () => { const content = await readFile(resolve("docs/installation.md"), "utf8"); expect(jsonExamples(content)).toContainEqual( diff --git a/tests/conformance/swift/string-literals.swift b/tests/conformance/swift/string-literals.swift new file mode 100644 index 000000000..345c4cf74 --- /dev/null +++ b/tests/conformance/swift/string-literals.swift @@ -0,0 +1,23 @@ +@inline(never) +func consume(_ value: String) { print(value) } +@inline(never) +func smallLiteral() { consume("SleepDisabled") } +@inline(never) +func largeLiteral() { consume("The power setting could not be changed.") } +@inline(never) +func unicodeLiterals() { + consume("café") + consume("Résumé of the naïve décor, written twice") +} +@inline(never) +func emptyAndControlLiterals() { + consume("") + consume("tab\there") +} +@inline(never) +func returnedLiteral() -> String { "Holding awake" } +smallLiteral() +largeLiteral() +unicodeLiterals() +emptyAndControlLiterals() +print(returnedLiteral(), CommandLine.arguments.count) diff --git a/tests/fixtures/idaMcp.ts b/tests/fixtures/idaMcp.ts index 04bdb5f16..6be35a8bf 100644 --- a/tests/fixtures/idaMcp.ts +++ b/tests/fixtures/idaMcp.ts @@ -57,6 +57,7 @@ const modernNames = [ "callees", "xrefs_to", "callgraph", + "survey_binary", ]; /** Stateful producer fixture at the real connection seam; no module mocking. */ @@ -67,6 +68,7 @@ export class RecordingIdaMcp implements IdaMcpConnection { }[] = []; connects = 0; closes = 0; + detaches = 0; inputSha256: string; inputPath: string; imageBase = "0x1000"; @@ -103,6 +105,9 @@ export class RecordingIdaMcp implements IdaMcpConnection { async close() { this.closes += 1; } + detach() { + this.detaches += 1; + } async call( name: string, args: Readonly>, diff --git a/tests/fixtures/javaClassFile.ts b/tests/fixtures/javaClassFile.ts index 4f7727193..30f5ffc37 100644 --- a/tests/fixtures/javaClassFile.ts +++ b/tests/fixtures/javaClassFile.ts @@ -1,5 +1,5 @@ /** Build a valid minimal JVM class file, optionally with an unused UTF-8 constant. */ -export const javaClassFile = (extraUtf8Bytes = 0): Buffer => { +export const javaClassFile = (extraUtf8Bytes = 0, fatCpuLookalike = false): Buffer => { const utf8 = (value: string): Buffer => { const bytes = Buffer.from(value, "utf8"); const entry = Buffer.alloc(3 + bytes.length); @@ -13,6 +13,7 @@ export const javaClassFile = (extraUtf8Bytes = 0): Buffer => { utf8("Hello"), Buffer.from([7, 0, 4]), utf8("java/lang/Object"), + ...(fatCpuLookalike ? [utf8("x"), Buffer.from([3, 0, 0, 0, 7])] : []), ...(extraUtf8Bytes === 0 ? [] : [utf8("x".repeat(extraUtf8Bytes))]), ]; const headerAndPool = Buffer.alloc(10); diff --git a/tests/fixtures/strippedAsarAddon.ts b/tests/fixtures/strippedAsarAddon.ts index 5ff76356c..3422ac2ca 100644 --- a/tests/fixtures/strippedAsarAddon.ts +++ b/tests/fixtures/strippedAsarAddon.ts @@ -29,7 +29,7 @@ export const createStrippedAsarAddon = async ( await mkdir(dirname(path), { recursive: true }); await writeFile(path, file.contents); } - await createPackageWithOptions(source, archive, { unpack: "**/*.node" }); + await createPackageWithOptions(source, archive, { unpack: "*.node" }); await writeFile( join(`${archive}.unpacked`, "node_modules", "btime", "binding.node"), stripped, diff --git a/tests/process-global/cleanLinuxHopperRuntime.test.ts b/tests/process-global/cleanLinuxHopperRuntime.test.ts index d356f5abf..f6b9fc677 100644 --- a/tests/process-global/cleanLinuxHopperRuntime.test.ts +++ b/tests/process-global/cleanLinuxHopperRuntime.test.ts @@ -30,6 +30,9 @@ it await chmod(launcher, 0o700); await symlink(launcher, alternate); } + // A rejection must come from the fixture we are about to launch. An + // existing Xvfb would otherwise mask a broken executable-identity check. + await assertCleanLinuxHopperRuntime(alternateMode ? [launcher] : []); const command = alternateMode ? alternate : process.execPath; const arguments_ = alternateMode ? ["30"] : ["-e", childCode]; const parentCode = ` diff --git a/tests/process-global/pwntoolsStageRetirement.test.ts b/tests/process-global/pwntoolsStageRetirement.test.ts index 6ec062535..7cdf0b48a 100644 --- a/tests/process-global/pwntoolsStageRetirement.test.ts +++ b/tests/process-global/pwntoolsStageRetirement.test.ts @@ -18,7 +18,10 @@ import { PWNTOOLS_MEMORY_FAILURE_EXIT, PWNTOOLS_PROVIDER_IDENTITY, } from "../../src/native/pwntools/PwntoolsRelease.js"; -import { cleanupOwnedProcessGroup } from "../../src/process/ProcessOwnership.js"; +import { + cleanupOwnedProcessGroup, + type OwnedProcessGroup, +} from "../../src/process/ProcessOwnership.js"; import { prepareProcessOwnershipInspection } from "../../src/process/ProcessOwnershipObservation.js"; import { spawnOwnedProviderProcess } from "../../src/process/ProviderProcess.js"; import { PrivateRuntimeRoot } from "../../src/process/PrivateRuntimeRoot.js"; @@ -232,12 +235,13 @@ it.runIf(supportedHost).each([ const { path, root, environment, basic } = await fixture(); let launches = 0; let cleanupAllowed = false; + let retainedOwnership: OwnedProcessGroup | undefined; const read = vi.spyOn(artifacts, "readStableArtifact"); const provider = new PwntoolsRecordedCrashProvider( environment, async (spawn) => { if (stage === "debugger" && ++launches === 1) return basic(spawn); - return changingWorker( + const launched = await changingWorker( spawn, join(spawn.cwd ?? "", "resource.failure"), { @@ -247,6 +251,8 @@ it.runIf(supportedHost).each([ }, () => cleanupAllowed, ); + retainedOwnership = launched.ownership; + return launched; }, async () => root, ); @@ -261,14 +267,14 @@ it.runIf(supportedHost).each([ if (result.ok) throw new Error("Expected unresolved process ownership"); const projected = projectAnalysisError(result.error); expect(projected.code).toBe("cleanup_incomplete"); - expect(projected).toMatchObject({ - details: { - resources: expect.arrayContaining([ - expect.stringMatching(/^pid:\d+$/u), - expect.stringMatching(/^process-group:\d+$/u), - ]), - }, - }); + if (retainedOwnership === undefined) + throw new Error("Expected retained process ownership"); + expect(projected.details?.resources).toEqual([ + retainedOwnership.runId, + root.path, + `pid:${retainedOwnership.leaderPid}`, + `process-group:${retainedOwnership.processGroupId}`, + ]); if (stage === "debugger") expect(projected).toMatchObject({ details: {