diff --git a/.github/actions/portable-build-cache/action.yml b/.github/actions/portable-build-cache/action.yml index ad127ce3d..cee775ec5 100644 --- a/.github/actions/portable-build-cache/action.yml +++ b/.github/actions/portable-build-cache/action.yml @@ -1,14 +1,24 @@ name: Portable build cache -description: Reuse Turbo build outputs without serializing verification jobs. +description: Restore portable Turbo build outputs from the current run or cross-run cache. inputs: save: description: Save newly built outputs after the job succeeds. default: "false" + build_cache_artifact: + description: Reuse portable Turbo outputs uploaded by the current workflow run. + default: "" runs: using: composite steps: + - name: Download portable build outputs from this workflow run + if: inputs.build_cache_artifact != '' + uses: actions/download-artifact@v4 + with: + name: ${{ inputs.build_cache_artifact }} + path: .turbo/cache/ - name: Resolve portable build task hashes id: plan + if: inputs.build_cache_artifact == '' shell: bash env: BUILD_CACHE_PREFIX: rea-portable-build-v2-${{ runner.os }}-${{ runner.arch }}-${{ hashFiles('.npmrc', '.nvmrc', 'package-lock.json') }} @@ -28,14 +38,14 @@ runs: console.log(`prefix=${process.env.BUILD_CACHE_PREFIX}-`); ' >> "$GITHUB_OUTPUT" - name: Restore portable build outputs - if: inputs.save != 'true' + if: inputs.build_cache_artifact == '' && inputs.save != 'true' uses: actions/cache/restore@v4 with: path: .turbo/cache/ key: ${{ steps.plan.outputs.key }} restore-keys: ${{ steps.plan.outputs.prefix }} - name: Restore and save portable build outputs - if: inputs.save == 'true' + if: inputs.build_cache_artifact == '' && inputs.save == 'true' uses: actions/cache@v4 with: path: .turbo/cache/ diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index b1af43269..e0516da85 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -61,6 +61,7 @@ jobs: nativeaot: ${{ steps.plan.outputs.nativeaot }} website: ${{ steps.plan.outputs.website }} matrix: ${{ steps.plan.outputs.matrix }} + test_selection: ${{ steps.plan.outputs.test_selection }} steps: - uses: actions/checkout@v4 with: @@ -189,7 +190,18 @@ jobs: build: name: Build package needs: changes - if: needs.changes.outputs.code == 'true' + # Reusable provider workflows consume this run's portable build artifact, + # even when a workflow-only edit selects the lane without portable tests. + if: >- + needs.changes.outputs.code == 'true' || + needs.changes.outputs.browser == 'true' || + needs.changes.outputs.binary_layout == 'true' || + needs.changes.outputs.evm == 'true' || + needs.changes.outputs.javascript_recovery == 'true' || + needs.changes.outputs.recorded_crash == 'true' || + needs.changes.outputs.web_network == 'true' || + needs.changes.outputs.web_runtime == 'true' || + needs.changes.outputs.web_source_map == 'true' runs-on: ubuntu-latest timeout-minutes: 10 permissions: @@ -315,6 +327,8 @@ jobs: SHARD_RESULT: ${{ needs.test-shard.result }} MERGE_RESULT: ${{ needs.changes.outputs.tier == 'full' && steps.merge.outcome || steps.merge-fast.outcome }} CI_VERIFY_TEST_INVENTORY: "true" + CI_TEST_SELECTION: ${{ needs.changes.outputs.test_selection }} + CI_PR_HEAD_SHA: ${{ github.event.pull_request.head.sha }} run: node scripts/ci/test-report.mjs - uses: actions/upload-artifact@v4 if: needs.changes.outputs.code == 'true' && !cancelled() && hashFiles('test-results.json') != '' @@ -325,6 +339,8 @@ jobs: test-results.json test-timings.json coverage/ + .cache/ci-test-selection/report.json + include-hidden-files: true if-no-files-found: error retention-days: 30 - name: Publish successful timing history @@ -668,44 +684,60 @@ jobs: if-no-files-found: error browser: - needs: [changes, static] + needs: [changes, static, build] if: needs.changes.outputs.browser == 'true' uses: ./.github/workflows/real-browser.yml + with: + build_cache_artifact: rea-build-cache binary_layout: - needs: [changes, static] + needs: [changes, static, build] if: needs.changes.outputs.binary_layout == 'true' uses: ./.github/workflows/real-binary-layout.yml + with: + build_cache_artifact: rea-build-cache evm: - needs: [changes, static] + needs: [changes, static, build] if: needs.changes.outputs.evm == 'true' uses: ./.github/workflows/real-evm-interface.yml + with: + build_cache_artifact: rea-build-cache javascript_recovery: - needs: [changes, static] + needs: [changes, static, build] if: needs.changes.outputs.javascript_recovery == 'true' uses: ./.github/workflows/real-javascript-recovery.yml + with: + build_cache_artifact: rea-build-cache recorded_crash: - needs: [changes, static] + needs: [changes, static, build] if: needs.changes.outputs.recorded_crash == 'true' uses: ./.github/workflows/real-recorded-crash.yml + with: + build_cache_artifact: rea-build-cache web_network: - needs: [changes, static] + needs: [changes, static, build] if: needs.changes.outputs.web_network == 'true' uses: ./.github/workflows/real-web-network-captures.yml + with: + build_cache_artifact: rea-build-cache web_runtime: - needs: [changes, static] + needs: [changes, static, build] if: needs.changes.outputs.web_runtime == 'true' uses: ./.github/workflows/real-web-runtime.yml + with: + build_cache_artifact: rea-build-cache web_source_map: - needs: [changes, static] + needs: [changes, static, build] if: needs.changes.outputs.web_source_map == 'true' uses: ./.github/workflows/real-web-source-map.yml + with: + build_cache_artifact: rea-build-cache nativeaot: needs: [changes, static] diff --git a/.github/workflows/real-binary-layout.yml b/.github/workflows/real-binary-layout.yml index 8dd6f9866..5cec952ce 100644 --- a/.github/workflows/real-binary-layout.yml +++ b/.github/workflows/real-binary-layout.yml @@ -8,6 +8,12 @@ on: type: boolean default: false workflow_call: + inputs: + build_cache_artifact: + description: Portable build cache artifact from the calling CI run. + required: false + type: string + default: "" permissions: contents: read @@ -29,6 +35,8 @@ jobs: cache: npm - run: npm ci - uses: ./.github/actions/portable-build-cache + with: + build_cache_artifact: ${{ inputs.build_cache_artifact }} - name: Acquire exact upstream libraries in an isolated runner environment shell: bash run: | diff --git a/.github/workflows/real-browser.yml b/.github/workflows/real-browser.yml index 11a8b7a53..912fdafb1 100644 --- a/.github/workflows/real-browser.yml +++ b/.github/workflows/real-browser.yml @@ -2,6 +2,12 @@ name: Real browser verification on: workflow_call: + inputs: + build_cache_artifact: + description: Portable build cache artifact from the calling CI run. + required: false + type: string + default: "" workflow_dispatch: env: @@ -22,6 +28,8 @@ jobs: cache: npm - run: npm ci - uses: ./.github/actions/portable-build-cache + with: + build_cache_artifact: ${{ inputs.build_cache_artifact }} - run: npm run verify:browser env: REA_BROWSER_EXECUTABLE: /usr/bin/google-chrome diff --git a/.github/workflows/real-evm-interface.yml b/.github/workflows/real-evm-interface.yml index b335f1391..dfdd3970f 100644 --- a/.github/workflows/real-evm-interface.yml +++ b/.github/workflows/real-evm-interface.yml @@ -3,6 +3,12 @@ name: Real offline EVM interface on: workflow_dispatch: workflow_call: + inputs: + build_cache_artifact: + description: Portable build cache artifact from the calling CI run. + required: false + type: string + default: "" permissions: contents: read @@ -23,6 +29,8 @@ jobs: cache: npm - run: npm ci - uses: ./.github/actions/portable-build-cache + with: + build_cache_artifact: ${{ inputs.build_cache_artifact }} - name: Acquire unchanged fixture compiler in isolated runner storage shell: bash run: | diff --git a/.github/workflows/real-javascript-recovery.yml b/.github/workflows/real-javascript-recovery.yml index d94127d63..817dc4dc0 100644 --- a/.github/workflows/real-javascript-recovery.yml +++ b/.github/workflows/real-javascript-recovery.yml @@ -2,6 +2,12 @@ name: Real JavaScript recovery on: workflow_call: + inputs: + build_cache_artifact: + description: Portable build cache artifact from the calling CI run. + required: false + type: string + default: "" workflow_dispatch: permissions: @@ -24,6 +30,8 @@ jobs: cache: npm - run: npm ci - uses: ./.github/actions/portable-build-cache + with: + build_cache_artifact: ${{ inputs.build_cache_artifact }} - name: Fetch pinned Wakaru release into the job workspace shell: bash run: | diff --git a/.github/workflows/real-recorded-crash.yml b/.github/workflows/real-recorded-crash.yml index dc56034c8..04467a608 100644 --- a/.github/workflows/real-recorded-crash.yml +++ b/.github/workflows/real-recorded-crash.yml @@ -3,6 +3,12 @@ name: Real recorded crash evidence on: workflow_dispatch: workflow_call: + inputs: + build_cache_artifact: + description: Portable build cache artifact from the calling CI run. + required: false + type: string + default: "" permissions: contents: read @@ -26,6 +32,8 @@ jobs: cache: npm - run: npm ci - uses: ./.github/actions/portable-build-cache + with: + build_cache_artifact: ${{ inputs.build_cache_artifact }} - name: Install GDB on the disposable CI runner shell: bash run: | diff --git a/.github/workflows/real-web-network-captures.yml b/.github/workflows/real-web-network-captures.yml index df340520d..d41bb37a5 100644 --- a/.github/workflows/real-web-network-captures.yml +++ b/.github/workflows/real-web-network-captures.yml @@ -3,6 +3,12 @@ name: Real historical web network captures on: workflow_dispatch: workflow_call: + inputs: + build_cache_artifact: + description: Portable build cache artifact from the calling CI run. + required: false + type: string + default: "" permissions: contents: read @@ -25,6 +31,8 @@ jobs: run: npm install --global npm@11.16.0 - run: npm ci - uses: ./.github/actions/portable-build-cache + with: + build_cache_artifact: ${{ inputs.build_cache_artifact }} - name: Acquire fixed upstream native profile in runner temporary storage shell: bash run: | diff --git a/.github/workflows/real-web-runtime.yml b/.github/workflows/real-web-runtime.yml index 0ecdb92c6..2575c55f9 100644 --- a/.github/workflows/real-web-runtime.yml +++ b/.github/workflows/real-web-runtime.yml @@ -1,6 +1,12 @@ name: Real website runtime attribution on: workflow_call: + inputs: + build_cache_artifact: + description: Portable build cache artifact from the calling CI run. + required: false + type: string + default: "" workflow_dispatch: permissions: contents: read @@ -18,6 +24,8 @@ jobs: cache: npm - run: npm ci - uses: ./.github/actions/portable-build-cache + with: + build_cache_artifact: ${{ inputs.build_cache_artifact }} - run: npm run verify:browser:runtime env: REA_BROWSER_EXECUTABLE: /usr/bin/google-chrome diff --git a/.github/workflows/real-web-source-map.yml b/.github/workflows/real-web-source-map.yml index b94662129..8e0869eb7 100644 --- a/.github/workflows/real-web-source-map.yml +++ b/.github/workflows/real-web-source-map.yml @@ -1,6 +1,12 @@ name: Real website source-map verification on: workflow_call: + inputs: + build_cache_artifact: + description: Portable build cache artifact from the calling CI run. + required: false + type: string + default: "" workflow_dispatch: permissions: contents: read @@ -18,6 +24,8 @@ jobs: cache: npm - run: npm ci - uses: ./.github/actions/portable-build-cache + with: + build_cache_artifact: ${{ inputs.build_cache_artifact }} - name: Install the pinned compiler into an isolated fixture directory run: npm install --prefix "$RUNNER_TEMP/rea-web-source-map-compiler" --no-audit --no-fund --package-lock=false esbuild@0.25.10 - run: npm run verify:browser:source-maps diff --git a/bridge/mitmproxy/capture.py b/bridge/mitmproxy/capture.py index 5104130ab..419ecfdec 100644 --- a/bridge/mitmproxy/capture.py +++ b/bridge/mitmproxy/capture.py @@ -46,7 +46,7 @@ SCHEME_PREFIX = r"^[\x00-\x20]*([a-z][a-z0-9+.-]*):" URL_PREFIX = r"^([\x00-\x20]*(?:([a-z][a-z0-9+.-]*):)?//)" -def authority_start(text): +def authority_start(text, base_scheme=None): """Return where an authority begins and whether it is special, or None. Follows WHATWG URL parsing as src/domain/recordedTransportUrl.ts does: @@ -57,6 +57,11 @@ def authority_start(text): scheme = re.match(SCHEME_PREFIX, text, flags=re.I) name = scheme.group(1).lower() if scheme is not None else None if name is not None and name in SPECIAL_SCHEMES and name != "file": + # A same-scheme Location without two slash-equivalent separators is + # relative to the recorded request, so its @ belongs to the path. + following = text[scheme.end():scheme.end() + 2] + if name == base_scheme and (len(following) < 2 or any(char not in "/\\" for char in following)): + return None start = scheme.end() else: prefix = re.match(URL_PREFIX, text, flags=re.I) @@ -70,7 +75,7 @@ def authority_start(text): return start, True -def without_userinfo(value): +def without_userinfo(value, base_scheme=None): """Remove an authority's userinfo using the WHATWG authority boundary. Special schemes and scheme-relative references also end the authority @@ -78,7 +83,7 @@ def without_userinfo(value): """ binary = isinstance(value, bytes) text = value.decode("latin-1") if binary else value - located = authority_start(text) + located = authority_start(text, base_scheme) if located is None: return value start, special = located @@ -110,12 +115,12 @@ def project_record(state, secrets): return None return text - def transport_value(value, path, transport_url): + def transport_value(value, path, transport_url, base_scheme): if not (transport_url or re.fullmatch(r"/(?:backup/)*request/(?:path|authority)", path)): return value, False binary = isinstance(value, bytes) separator = b"@" if binary else "@" - safe_url = without_userinfo(value) + safe_url = without_userinfo(value, base_scheme if transport_url else None) if path.endswith("/authority") and separator in safe_url: safe_url = safe_url.rsplit(separator, 1)[1] changed = safe_url != value @@ -130,7 +135,7 @@ def project_record(state, secrets): del numbers[before[1]:] del redactions[before[2]:] - def visit(value, path, depth=0, credential=False, transport_url=False): + def visit(value, path, depth=0, credential=False, transport_url=False, base_scheme=None): if depth > 64: raise CaptureFailure("input-limit", "Native capture exceeds the 64-level evidence nesting budget.", path) if credential: @@ -140,7 +145,7 @@ def project_record(state, secrets): binaries.append({"pointer": path, "representation": "producer-bytes", "state": "redacted", "content_base64": None, "bytes": None, "sha256": None}) return None if isinstance(value, bytes): - safe_bytes, credential_url = transport_value(value, path, transport_url) + safe_bytes, credential_url = transport_value(value, path, transport_url, base_scheme) content_base64 = base64.b64encode(value).decode("ascii") digest = hashlib.sha256(value).hexdigest() hidden = any(secret in value or secret in safe_bytes for secret in byte_secrets) or any(secret in content_base64 or secret in digest or secret in str(len(value)) for secret in secrets) @@ -157,7 +162,7 @@ def project_record(state, secrets): return None return safe_text if isinstance(value, str): - safe_text, _ = transport_value(value, path, transport_url) + safe_text, _ = transport_value(value, path, transport_url, base_scheme) return redact(safe_text, path, value) if isinstance(value, bool) or value is None: return value @@ -175,10 +180,16 @@ def project_record(state, secrets): for index, field in enumerate(value): if not isinstance(field, (list, tuple)) or len(field) != 2 or not isinstance(field[0], bytes): raise CaptureFailure("format", "Native header/trailer is not an ordered byte pair.", pointer(path, index)) - result.append([visit(field[0], pointer(pointer(path, index), 0), depth + 2), visit(field[1], pointer(pointer(path, index), 1), depth + 2, field[0].lower() in CREDENTIALS, field[0].lower() in {b"location", b"referer", b"origin"})]) + result.append([visit(field[0], pointer(pointer(path, index), 0), depth + 2), visit(field[1], pointer(pointer(path, index), 1), depth + 2, field[0].lower() in CREDENTIALS, field[0].lower() in {b"location", b"referer", b"origin"}, base_scheme if field[0].lower() == b"location" else None)]) return result - return [visit(item, pointer(path, index), depth + 1) for index, item in enumerate(value)] + return [visit(item, pointer(path, index), depth + 1, base_scheme=base_scheme) for index, item in enumerate(value)] if isinstance(value, dict): + if path == "" or re.fullmatch(r"/backup(?:/backup)*", path): + request = value.get("request") + scheme = request.get("scheme") if isinstance(request, dict) else None + if isinstance(scheme, bytes): + scheme = scheme.decode("latin-1") + base_scheme = scheme.lower() if isinstance(scheme, str) and re.fullmatch(r"[a-z][a-z0-9+.-]*", scheme, flags=re.I) else None result = {} for key, item in value.items(): if not isinstance(key, str): @@ -189,7 +200,7 @@ def project_record(state, secrets): validate_omitted(item, pointer(path, key), depth + 1) redactions.append({"pointer": path, "reason": "explicit-sensitive-value", "scope": "property-name"}) continue - result[key] = visit(item, pointer(path, key), depth + 1) + result[key] = visit(item, pointer(path, key), depth + 1, base_scheme=base_scheme) return result raise CaptureFailure("decoder", "Native parser returned an unexpected value type: " + type(value).__name__ + ".", path) diff --git a/bridge/mitmproxy/test_capture.py b/bridge/mitmproxy/test_capture.py index 8b1217e59..e99ee9cd8 100644 --- a/bridge/mitmproxy/test_capture.py +++ b/bridge/mitmproxy/test_capture.py @@ -32,8 +32,11 @@ def load_capture(): capture = load_capture() -def location(value): - record = capture.project_record({"response": {"headers": [[b"Location", value]]}}, []) +def location(value, scheme=None): + state = {"response": {"headers": [[b"Location", value]]}} + if scheme is not None: + state["request"] = {"scheme": scheme} + record = capture.project_record(state, []) sidecar = next( field for field in record["binary_fields"] if field["pointer"] == "/response/headers/0/1" ) @@ -41,6 +44,37 @@ def location(value): class TransportUserinfoTests(unittest.TestCase): + def test_same_scheme_locations_preserve_relative_path_text_and_bytes(self): + for value in [b"https:invoice@customer", b"https:/invoice@customer", b"https:\\invoice@customer"]: + with self.subTest(value=value): + text, sidecar, redactions = location(value, b"https") + self.assertEqual(text, value.decode("ascii")) + self.assertEqual(sidecar["state"], "retained") + self.assertEqual(redactions, []) + + def test_location_authorities_still_remove_credentials_with_a_base_scheme(self): + for value, scheme, expected in [ + (b"https://user:pass@host/path", b"https", "https://host/path"), + (b"https:\\\\user:pass@host/path", b"https", "https:\\\\host/path"), + (b"https:user:pass@host/path", b"http", "https:host/path"), + ]: + with self.subTest(value=value, scheme=scheme): + text, sidecar, redactions = location(value, scheme) + self.assertEqual(text, expected) + self.assertEqual(sidecar["state"], "redacted") + self.assertEqual(len(redactions), 1) + + def test_backup_location_uses_its_own_request_scheme(self): + record = capture.project_record({ + "request": {"scheme": b"http"}, + "backup": { + "request": {"scheme": b"https"}, + "response": {"headers": [[b"Location", b"https:invoice@customer"]]}, + }, + }, []) + self.assertEqual(record["reported"]["backup"]["response"]["headers"][0][1], "https:invoice@customer") + self.assertEqual(record["redactions"], []) + def test_backslash_ends_a_special_scheme_authority(self): text, sidecar, redactions = location(b"https://example.test\\part@tail") self.assertEqual(text, "https://example.test\\part@tail") diff --git a/docs/testing.md b/docs/testing.md index 22b5c0c77..1288a954a 100644 --- a/docs/testing.md +++ b/docs/testing.md @@ -805,6 +805,27 @@ remain present and fail if classification fails. Every code change retains the complete six-shard Linux suite; individual test files are not selected by imports or filenames. +Ordinary PRs also produce an **observational** test-selection plan. The candidate +leaf-adapter ownership table lives in `scripts/ci/test-selection.mjs`, separately +from provider-job scopes. It proposes keeping affected owners plus every unowned +test, including all CLI/MCP, acceptance, composition, conformance, and process +tests. Shared inputs, unknown paths, missing owner tests, and removed tests fall +back to the complete inventory. Renames account for both paths. Main, scheduled, +manual, and `ci:full` runs always propose the full suite. + +All six shards still execute the complete suite. The existing report merge +compares the proposal with actual module outcomes and retains +`.cache/ci-test-selection/report.json` in `test-results`, including selected and +omitted paths, changed input paths, missed failures, unobserved files, scheduling weights, PR head and +executed merge SHA. This adds no test reruns or runner jobs. Missing, duplicate, +skipped, interrupted, or unhandled results cannot establish a complete comparison; +even a complete passing comparison is not proof of ownership completeness. +Before enabling omissions, review representative full-run comparisons and +cross-owner consumers, including failures, and measure completed CI feedback and +runner demand. There is no automatic promotion or switch to skip tests. The +initial proposal measures only leaf producer-test omissions; it does not promise +the savings of a broader affected-test planner. + `scripts/ci/plan.mjs` selects lanes using the ownership table in `scripts/ci/scopes.mjs`. Provider implementation changes run their real-provider verifiers and the Linux x64 installed-package/Inspector baseline. The `apple` @@ -826,8 +847,12 @@ scripts also select it. Unrelated code edits retain the complete portable suite on the pinned Node version without starting four additional compatibility jobs. Full qualification retains every SQLite version. Shared process/contract/CLI boundaries, dependencies, other packaging inputs, -unknown paths, and changes to the planner select the full baseline. Package -script-only edits select their owning verifier when its entrypoint is +unknown paths, and changes to the planner select the full baseline. +CI timing collection, timing history, and test-report helper edits retain static +checks and the complete portable suite without selecting unrelated host/provider +lanes. Classification, qualification, gate, release receipt, and unknown CI +scripts keep the broad fallback. +Package script-only edits select their owning verifier when its entrypoint is identifiable; unknown or shared script changes select everything. Android build-script edits retain the Linux suite and workflow validation; real Android verification stays manual. Renames account for both paths. @@ -874,7 +899,10 @@ summary. Representative Git-diff cases live in `tests/boundary/filesystem/ciChangeScope.test.ts`. `CI required` always runs and checks classification and every selected lane, -rejecting failed, cancelled, or unexpectedly skipped work. Branch protection +rejecting failed, cancelled, or unexpectedly skipped work. It rejects missing or +unknown tiers and requires every canonical scope to be explicitly `true` or +`false`; incomplete classification cannot silently omit verification lanes. +Branch protection should require this aggregate, `Static checks`, `Portable tests`, and `Conventional PR title`. Title validation runs separately on PR opening, reopening, commits, and edits; correcting a title refreshes only this lightweight @@ -954,6 +982,16 @@ timing data is reused; live provider results are never cached. Turbo outputs are restored by task hash, with trusted default-branch runs as cache writers and PRs as restore-only consumers. npm caches hold downloads, not `node_modules`. +The eight automatic browser, binary-layout, EVM, JavaScript-recovery, +recorded-crash, and web provider lanes download `rea-build-cache` from their own +run after the build and static jobs succeed. Their existing build commands still +validate task hashes and rebuild mismatched inputs; provider observations always +run. Provider-workflow-only edits also admit the build without enabling source +test shards. Direct provider dispatches retain the cross-run restore-only cache. +This removes repeated compilation on fresh source revisions, but adds the build +as a prerequisite: a delayed build can delay provider admission. Compare queue +and execution times before claiming an end-to-end speedup. + These native baseline checks complement the Linux source-test shards and the separate Apple-artifact and real-provider lanes. Actual Hopper, Ghidra, IDA, browser and managed-tool claims require their corresponding verification lanes. diff --git a/scripts/ci/plan.mjs b/scripts/ci/plan.mjs index 9384d9b05..86879216d 100644 --- a/scripts/ci/plan.mjs +++ b/scripts/ci/plan.mjs @@ -2,6 +2,7 @@ import { execFileSync } from "node:child_process"; import { appendFileSync } from "node:fs"; import { matchesGlob } from "node:path"; import { CI_FLAGS as flags, MANUAL_VERIFIERS, SCOPES } from "./scopes.mjs"; +import { planTestSelection } from "./test-selection.mjs"; // Test files select the host/runtime lanes that execute their focused suites. const HOST_TEST_OWNERS = ["apple", "package", "sqlite"]; @@ -25,6 +26,15 @@ const FIXTURE_SOURCES = [ "tests/conformance/versions/**", ]; const READINESS_SOURCES = ["tests/conformance/readiness/**"]; +// These reporters affect CI output only; they do not select a host or provider +// verification lane when changed on their own. +const PORTABLE_CI_REPORTERS = new Set([ + "scripts/ci/job-timings.mjs", + "scripts/ci/job-timings-batch.mjs", + "scripts/ci/test-report.mjs", + "scripts/ci/test-timing-reporter.mjs", + "scripts/ci/test-history.mjs", +]); const matchesAny = (file, patterns) => patterns.some((pattern) => matchesGlob(file, pattern)); const selected = Object.fromEntries(flags.map((flag) => [flag, false])); @@ -200,6 +210,7 @@ if (tier === "reused") { } if (matchesAny(file, READINESS_SOURCES)) selected.readiness = true; if (file.startsWith("tests/")) continue; + if (PORTABLE_CI_REPORTERS.has(file)) continue; if ( /^(src\/process\/|src\/filesystem\/|src\/contracts\/tool|src\/server\/createServer|src\/cli\.ts|scripts\/ci\/)/.test( file, @@ -240,11 +251,17 @@ const matrix = { ] : [linux], }; -const plan = { ...selected, tier, matrix, files, reasons }; +const testSelection = planTestSelection(files, event, tier); +const plan = { ...selected, tier, matrix, files, reasons, testSelection }; if (process.env.GITHUB_OUTPUT) appendFileSync( process.env.GITHUB_OUTPUT, - Object.entries({ ...selected, tier, matrix: JSON.stringify(matrix) }) + Object.entries({ + ...selected, + tier, + matrix: JSON.stringify(matrix), + test_selection: JSON.stringify(testSelection), + }) .map(([key, value]) => `${key}=${value}\n`) .join(""), ); diff --git a/scripts/ci/required.mjs b/scripts/ci/required.mjs index afe52c2b8..01907de37 100644 --- a/scripts/ci/required.mjs +++ b/scripts/ci/required.mjs @@ -2,22 +2,31 @@ import { CI_FLAGS } from "./scopes.mjs"; import { recordQualification } from "./qualification.mjs"; const jobs = JSON.parse(process.env.CI_JOBS); -const outputs = jobs.changes.outputs; +const outputs = jobs?.changes?.outputs; +if (!["fast", "full", "reused"].includes(outputs?.tier)) + throw new Error( + "CI classification requires a valid tier: fast, full, or reused", + ); +const invalidFlags = CI_FLAGS.filter( + (flag) => outputs[flag] !== "true" && outputs[flag] !== "false", +); +if (invalidFlags.length) + throw new Error( + `CI classification requires explicit true/false scope outputs: ${invalidFlags.join(", ")}`, + ); +const selectedScopes = CI_FLAGS.filter((flag) => outputs[flag] === "true"); const reused = outputs.tier === "reused"; if ( reused && (process.env.GITHUB_EVENT_NAME !== "schedule" || !/^[1-9]\d*$/.test(outputs.qualification_run_id ?? "") || !Number.isSafeInteger(Number(outputs.qualification_run_id)) || - CI_FLAGS.some((flag) => outputs[flag] !== "false")) + selectedScopes.length !== 0) ) throw new Error( "Reused qualification requires a scheduled admission receipt and no selected lanes", ); -if ( - outputs.tier === "full" && - CI_FLAGS.some((flag) => outputs[flag] !== "true") -) +if (outputs.tier === "full" && selectedScopes.length !== CI_FLAGS.length) throw new Error("Full qualification requires every canonical CI scope"); const required = reused ? ["changes"] : ["changes", "static", "test"]; const lanes = { @@ -31,9 +40,7 @@ const lanes = { docs: ["docs"], workflows: ["static"], }; -for (const [scope, value] of Object.entries(outputs)) { - if (value === "true") required.push(...(lanes[scope] ?? [scope])); -} +for (const scope of selectedScopes) required.push(...(lanes[scope] ?? [scope])); const failed = [...new Set(required)].filter( (name) => jobs[name]?.result !== "success", ); diff --git a/scripts/ci/test-report.mjs b/scripts/ci/test-report.mjs index 86419ae9f..035a01990 100644 --- a/scripts/ci/test-report.mjs +++ b/scripts/ci/test-report.mjs @@ -4,25 +4,33 @@ import { isAbsolute, join, relative, resolve, sep } from "node:path"; import { z } from "zod"; +import { compareTestSelection } from "./test-selection.mjs"; + const root = process.cwd(); const historyDirectory = join(root, ".cache/ci-test-history"); const historyPath = join(historyDirectory, "test-results.json"); +const selectionDirectory = join(root, ".cache/ci-test-selection"); +const selectionPath = join(selectionDirectory, "report.json"); // Never let a failed or partial rerun leave a previous successful history file. await rm(historyPath, { force: true }); +await rm(selectionPath, { force: true }); const report = z .object({ version: z.literal(1), success: z.boolean(), + reason: z.string().optional(), + unhandledErrors: z.number().int().nonnegative().optional(), files: z.array( z.object({ name: z.string().min(1), durationMs: z.number().finite().nonnegative(), + state: z.string().optional(), }), ), }) .parse(JSON.parse(await readFile("test-timings.json", "utf8"))); -const files = report.files.map(({ name, durationMs }) => { +const files = report.files.map(({ name, durationMs, state }) => { const path = relative(root, resolve(root, name)); if ( path === "" || @@ -31,21 +39,47 @@ const files = report.files.map(({ name, durationMs }) => { isAbsolute(path) ) throw new Error(`Test report path is outside the checkout: ${name}`); - return { name: path.split(sep).join("/"), durationMs }; + return { name: path.split(sep).join("/"), durationMs, state }; }); const complete = process.env.SHARD_RESULT === "success"; const passed = complete && process.env.MERGE_RESULT === "success" && report.success; // Successful shards and merged reports must account for the complete portable // inventory. Coverage thresholds alone cannot prove that every file ran. -if (passed && process.env.CI_VERIFY_TEST_INVENTORY === "true") { - const expected = execFileSync( - "git", - ["ls-files", "-z", "--", "src", "tests"], - { encoding: "utf8", maxBuffer: 8 * 1024 * 1024 }, - ) - .split("\0") - .filter((path) => path.endsWith(".test.ts")); +const verifyInventory = process.env.CI_VERIFY_TEST_INVENTORY === "true"; +const expected = + verifyInventory || process.env.CI_TEST_SELECTION + ? execFileSync("git", ["ls-files", "-z", "--", "src", "tests"], { + encoding: "utf8", + maxBuffer: 8 * 1024 * 1024, + }) + .split("\0") + .filter((path) => path.endsWith(".test.ts")) + : []; +let selection; +if (process.env.CI_TEST_SELECTION) { + const comparison = compareTestSelection( + JSON.parse(process.env.CI_TEST_SELECTION), + expected, + files, + ); + selection = { + ...comparison, + complete: + comparison.complete && + report.unhandledErrors === 0 && + ["passed", "failed"].includes(report.reason) && + ["success", "failure"].includes(process.env.SHARD_RESULT) && + ["success", "failure"].includes(process.env.MERGE_RESULT), + executionSha: process.env.GITHUB_SHA ?? null, + prHeadSha: process.env.CI_PR_HEAD_SHA ?? null, + runId: process.env.GITHUB_RUN_ID ?? null, + runAttempt: process.env.GITHUB_RUN_ATTEMPT ?? null, + }; + await mkdir(selectionDirectory, { recursive: true }); + await writeFile(selectionPath, JSON.stringify(selection)); +} +if (passed && verifyInventory) { const actual = files.map(({ name }) => name); const expectedSet = new Set(expected); const actualSet = new Set(actual); @@ -59,6 +93,16 @@ if (passed && process.env.CI_VERIFY_TEST_INVENTORY === "true") { const summary = [ complete ? "## Merged test diagnostics" : "## Partial test diagnostics", "", + ...(selection + ? [ + "### Observational PR test selection", + "", + `Plan: ${selection.plan.mode}. Proposed files: ${selection.selected.length}; omitted: ${selection.omitted.length}.`, + `Full-run comparison: ${selection.complete ? "complete" : "incomplete"}; failures outside proposed selection: ${selection.missedFailures.length}.`, + "The complete suite still ran. This report cannot enable omissions or qualify a revision. Skipped, missing, and interrupted observations do not establish parity.", + "", + ] + : []), `Shard result: ${process.env.SHARD_RESULT ?? "unknown"}; merge result: ${process.env.MERGE_RESULT ?? "unknown"}.`, passed ? "Successful timing history is available." @@ -85,6 +129,10 @@ if (passed) { await mkdir(historyDirectory, { recursive: true }); await writeFile( historyPath, - JSON.stringify({ version: 1, success: true, files }), + JSON.stringify({ + version: 1, + success: true, + files: files.map(({ name, durationMs }) => ({ name, durationMs })), + }), ); } diff --git a/scripts/ci/test-selection.d.mts b/scripts/ci/test-selection.d.mts new file mode 100644 index 000000000..c19414398 --- /dev/null +++ b/scripts/ci/test-selection.d.mts @@ -0,0 +1,35 @@ +export interface TestSelectionPlan { + readonly version: 1; + readonly mode: "full" | "candidate"; + readonly owners: readonly string[]; + readonly changedTests: readonly string[]; + readonly changedPaths: readonly string[]; + readonly reason: string; +} +export function planTestSelection( + files: readonly string[], + event: string, + tier: string, +): TestSelectionPlan; +export function compareTestSelection( + plan: unknown, + inventory: readonly string[], + results: readonly { + readonly name: string; + readonly durationMs: number; + readonly state?: string; + }[], +): { + readonly version: 1; + readonly observational: true; + readonly plan: TestSelectionPlan; + readonly selected: readonly string[]; + readonly omitted: readonly string[]; + readonly missing: readonly string[]; + readonly unexpected: readonly string[]; + readonly duplicateFiles: number; + readonly unobserved: readonly string[]; + readonly missedFailures: readonly string[]; + readonly complete: boolean; + readonly omittedDurationMs: number; +}; diff --git a/scripts/ci/test-selection.mjs b/scripts/ci/test-selection.mjs new file mode 100644 index 000000000..8165e3658 --- /dev/null +++ b/scripts/ci/test-selection.mjs @@ -0,0 +1,151 @@ +import { matchesGlob } from "node:path"; + +// Candidate ownership only: these leaf adapters and their producer tests can be +// compared with full-suite outcomes. This table never changes test execution. +// Shared application/domain code and dynamic CLI/MCP consumers are not leaf owners. +const OWNERS = { + adb: ["src/adb/**", "tests/boundary/providers/adb/**"], + apktool: ["src/apktool/**"], + dotnet: ["src/dotnet/**"], + evm: ["src/evm/**", "tests/boundary/evm/**"], + firmware: ["src/firmware/**", "tests/boundary/firmware/**"], + flutter: ["src/flutter/**"], + ghidra: ["src/ghidra/**", "tests/boundary/providers/ghidra/**"], + hopper: ["src/hopper/**", "tests/boundary/providers/hopper/**"], + ida: ["src/ida/**", "tests/boundary/providers/ida/**"], + jeb: ["src/jeb/**"], + pcap: ["src/pcap/**", "tests/boundary/pcap/**"], + sqlite: ["src/sqlite/**", "tests/boundary/sqlite/**"], + wasm: ["src/wasm/**"], +}; +const ownersOf = (path) => + Object.entries(OWNERS) + .filter(([, patterns]) => patterns.some((glob) => matchesGlob(path, glob))) + .map(([owner]) => owner); +const validPath = (path) => + typeof path === "string" && + /^(src|tests)\//.test(path) && + !/[\\\0\r\n]/.test(path) && + !path.split("/").some((part) => ["", ".", ".."].includes(part)); +const full = (reason, changedPaths = []) => ({ + version: 1, + mode: "full", + owners: [], + changedTests: [], + changedPaths, + reason, +}); + +/** Plan an observational PR subset from the classifier's existing Git diff. */ +export function planTestSelection(files, event, tier) { + if (event !== "pull_request" || tier !== "fast") + return full("Only ordinary PRs are candidates", files); + if (!files.length) return full("No changed paths available"); + const owners = new Set(); + const changedTests = []; + for (const path of files) { + if (!validPath(path)) + return full(`Shared or unknown input: ${path}`, files); + const matches = ownersOf(path); + if (!matches.length) return full(`Shared or unknown input: ${path}`, files); + for (const owner of matches) owners.add(owner); + if (path.endsWith(".test.ts")) changedTests.push(path); + } + return { + version: 1, + mode: "candidate", + owners: [...owners].sort(), + changedTests: [...new Set(changedTests)].sort(), + changedPaths: files, + reason: + "Observe affected leaf owners; retain integration and unowned tests", + }; +} + +/** Compare a proposal with the full run; missing observations never prove parity. */ +export function compareTestSelection(plan, inventory, results) { + if ( + plan?.version !== 1 || + !["full", "candidate"].includes(plan.mode) || + !Array.isArray(plan.owners) || + plan.owners.some((owner) => !Object.hasOwn(OWNERS, owner)) || + !Array.isArray(plan.changedTests) || + !Array.isArray(plan.changedPaths) || + plan.changedPaths.some((path) => typeof path !== "string") || + plan.changedTests.some( + (path) => !validPath(path) || !path.endsWith(".test.ts"), + ) || + typeof plan.reason !== "string" || + (plan.mode === "candidate" && !plan.owners.length) || + (plan.mode === "full" && (plan.owners.length || plan.changedTests.length)) + ) + throw new Error("Invalid observational test selection plan"); + const expected = new Set(inventory); + if (!expected.size || expected.size !== inventory.length) + throw new Error("Test selection requires a nonempty unique inventory"); + const ownersByFile = new Map( + plan.mode === "candidate" + ? inventory.map((path) => [path, ownersOf(path)]) + : [], + ); + // Removed/renamed tests have no current executable identity. Keep all tests. + const removed = plan.changedTests.filter((path) => !expected.has(path)); + const missingOwners = plan.owners.filter( + (owner) => + ![...ownersByFile.values()].some((owners) => owners.includes(owner)), + ); + const effective = removed.length + ? full( + `Changed tests absent from current inventory: ${removed.join(", ")}`, + plan.changedPaths, + ) + : missingOwners.length + ? full( + `Owners without executable tests: ${missingOwners.join(", ")}`, + plan.changedPaths, + ) + : plan; + const selected = []; + const omitted = []; + for (const path of inventory.toSorted()) { + const owners = ownersByFile.get(path) ?? []; + const keep = + effective.mode === "full" || + !owners.length || + effective.changedTests.includes(path) || + owners.some((owner) => effective.owners.includes(owner)); + (keep ? selected : omitted).push(path); + } + const actual = new Set(results.map((file) => file.name)); + const missing = inventory.filter((path) => !actual.has(path)); + const unexpected = [...actual].filter((path) => !expected.has(path)); + const duplicateFiles = results.length - actual.size; + const unobserved = results + .filter((file) => !["passed", "failed"].includes(file.state)) + .map((file) => file.name); + const omittedSet = new Set(omitted); + const missedFailures = results + .filter((file) => omittedSet.has(file.name) && file.state === "failed") + .map((file) => file.name); + return { + version: 1, + observational: true, + plan: effective, + selected, + omitted, + missing, + unexpected, + duplicateFiles, + unobserved, + missedFailures, + complete: + !missing.length && + !unexpected.length && + !duplicateFiles && + !unobserved.length, + // Phase totals are scheduling weights, not an estimate of wall-time savings. + omittedDurationMs: results + .filter((file) => omittedSet.has(file.name)) + .reduce((sum, file) => sum + file.durationMs, 0), + }; +} diff --git a/scripts/ci/test-timing-reporter.mjs b/scripts/ci/test-timing-reporter.mjs index 29f0cc582..ce0b1c5dc 100644 --- a/scripts/ci/test-timing-reporter.mjs +++ b/scripts/ci/test-timing-reporter.mjs @@ -21,6 +21,7 @@ export default class TestTimingReporter { }; return { name: module.moduleId, + state: module.state(), durationMs: Object.values(phases).reduce( (total, value) => total + value, 0, @@ -33,6 +34,8 @@ export default class TestTimingReporter { JSON.stringify({ version: 1, success: reason === "passed" && errors.length === 0, + reason, + unhandledErrors: errors.length, files, }), ); diff --git a/tests/boundary/cli/dylibResolutionCli.test.ts b/tests/boundary/cli/dylibResolutionCli.test.ts index f6482a2f3..66fa21ba3 100644 --- a/tests/boundary/cli/dylibResolutionCli.test.ts +++ b/tests/boundary/cli/dylibResolutionCli.test.ts @@ -477,6 +477,7 @@ cliTest( "reports each same-named slice's own embedded dyld environment", async ({ cli }) => { // arm64e and arm64e.x1 slices both report the architecture name arm64e. + // A plain x86_64 slice keeps the bundle openable on x64 hosts. for (const [settings, expected] of [ [[null, "DYLD_PRINT_RPATHS=1"], ["DYLD_PRINT_RPATHS=1"]], [ @@ -486,8 +487,8 @@ cliTest( ] as const) { const app = await fixtureApp(); await writeFiles(app, { - "Contents/MacOS/App": fatImage( - [CPU.arm64e, CPU.arm64eX1].map((cpu, index) => { + "Contents/MacOS/App": fatImage([ + ...[CPU.arm64e, CPU.arm64eX1].map((cpu, index) => { const setting = settings[index]; return { cpu, @@ -500,7 +501,8 @@ cliTest( }), }; }), - ), + { cpu: CPU.x86_64, bytes: machoImage({ cpu: CPU.x86_64 }) }, + ]), }); const result = await cli.run({ arguments: [ @@ -516,7 +518,9 @@ cliTest( const trace = dylibResolutionResultSchema.parse( parseEvidence(result.json).normalized_result, ); - expect(trace.roots).toHaveLength(2); + expect( + trace.roots.filter(({ architecture }) => architecture === "arm64e"), + ).toHaveLength(2); const environments = trace.findings .filter(({ kind }) => kind === "dyld-environment-present") .map(({ explanation }) => diff --git a/tests/boundary/filesystem/ciChangeScope.test.ts b/tests/boundary/filesystem/ciChangeScope.test.ts index 8ed397e84..163219c39 100644 --- a/tests/boundary/filesystem/ciChangeScope.test.ts +++ b/tests/boundary/filesystem/ciChangeScope.test.ts @@ -137,6 +137,64 @@ it("keeps every translated root README and authored guide on the documentation l }); }); +it.each([ + "job-timings.mjs", + "job-timings-batch.mjs", + "test-report.mjs", + "test-timing-reporter.mjs", + "test-history.mjs", +])( + "keeps portable CI reporter %s on static checks and portable tests", + async (name) => { + const { directory, base } = await fixture(); + const head = await change(directory, [`scripts/ci/${name}`]); + const plan = await classify(directory, base, head); + const { CI_FLAGS } = await loadScopes(); + expect(plan.code).toBe(true); + for (const flag of CI_FLAGS) + if (flag !== "code") expect(plan[flag]).toBe(false); + }, +); + +it("preserves a provider lane when a portable CI reporter changes with provider source", async () => { + const { directory, base } = await fixture(); + const head = await change(directory, [ + "scripts/ci/test-report.mjs", + "src/evm/Example.ts", + ]); + expect(await classify(directory, base, head)).toMatchObject({ + code: true, + evm: true, + browser: false, + package: false, + }); +}); + +it("keeps unknown CI scripts on complete qualification", async () => { + const { directory, base } = await fixture(); + const head = await change(directory, ["scripts/ci/new-qualification.mjs"]); + const plan = await classify(directory, base, head); + const { CI_FLAGS } = await loadScopes(); + for (const flag of CI_FLAGS) expect(plan[flag]).toBe(true); +}); + +it("emits an observational owner proposal without changing ordinary or full portable execution", async () => { + const { directory, base } = await fixture(); + const head = await change(directory, ["src/evm/Example.ts"]); + expect(await classify(directory, base, head)).toMatchObject({ + code: true, + testSelection: { mode: "candidate", owners: ["evm"] }, + }); + expect( + await classify(directory, base, head, "pull_request", true), + ).toMatchObject({ + code: true, + testSelection: { mode: "full", owners: [] }, + }); + const output = await readFile(join(directory, "scope-output"), "utf8"); + expect(output).toContain('test_selection={"version":1,"mode":"full"'); +}); + it("does not charge a documentation PR for unrelated implementation updates on main", async () => { const { directory, base } = await fixture(); await git(directory, ["switch", "-c", "docs/change"]); @@ -622,7 +680,9 @@ it.each([ async (scope, name) => { const workflow = z .object({ - on: z.object({ workflow_call: z.null() }).passthrough(), + on: z + .object({ workflow_call: z.object({}).passthrough().nullable() }) + .passthrough(), }) .parse( parse( @@ -662,8 +722,16 @@ it("rejects skipped selected work while allowing unselected lanes to skip", asyn const command = fileURLToPath( new URL("../../../scripts/ci/required.mjs", import.meta.url), ); + const { CI_FLAGS } = await loadScopes(); const jobs = { - changes: { result: "success", outputs: { evm: "true" } }, + changes: { + result: "success", + outputs: { + ...Object.fromEntries(CI_FLAGS.map((flag) => [flag, "false"])), + tier: "fast", + evm: "true", + }, + }, static: { result: "success" }, test: { result: "success" }, evm: { result: "success" }, diff --git a/tests/boundary/filesystem/ciRequiredLanes.test.ts b/tests/boundary/filesystem/ciRequiredLanes.test.ts index 1bac771a2..b6e02b54a 100644 --- a/tests/boundary/filesystem/ciRequiredLanes.test.ts +++ b/tests/boundary/filesystem/ciRequiredLanes.test.ts @@ -26,6 +26,15 @@ const workflow = async (filename = "ci.yml") => ), ); +const scopeFlags = async () => + z + .object({ CI_FLAGS: z.array(z.string()) }) + .parse( + await import( + new URL("../../../scripts/ci/scopes.mjs", import.meta.url).href + ), + ).CI_FLAGS; + it("binds SQLite compatibility jobs to the build's prepared runtime artifact and root layout", async () => { expect((await workflow()).jobs["sqlite-runtime"]?.if).toBe( "needs.changes.outputs.sqlite == 'true'", @@ -57,13 +66,7 @@ it("binds SQLite compatibility jobs to the build's prepared runtime artifact and it("exports every canonical scope with read authority for admission and keeps telemetry outside the gate", async () => { const { jobs } = await workflow(); - const { CI_FLAGS } = z - .object({ CI_FLAGS: z.array(z.string()) }) - .parse( - await import( - new URL("../../../scripts/ci/scopes.mjs", import.meta.url).href - ), - ); + const CI_FLAGS = await scopeFlags(); const outputs = z.record(z.string(), z.string()).parse(jobs.changes?.outputs); expect(jobs.changes?.permissions).toEqual({ contents: "read", @@ -165,6 +168,115 @@ it("holds native and provider fan-out behind static validation while portable fe expect(jobs["test-shard"]?.needs).not.toContain("static"); }); +it("binds provider build reuse to the current run while keeping standalone dispatches independent", async () => { + const { jobs } = await workflow(); + const consumers = [ + "browser", + "binary_layout", + "evm", + "javascript_recovery", + "recorded_crash", + "web_network", + "web_runtime", + "web_source_map", + ]; + const buildTerms = z + .string() + .parse(jobs.build?.if) + .split("||") + .map((term) => term.trim()); + // Provider workflow edits select their lane without code=true. The build + // producer must still run, without selecting the portable test shards. + expect(new Set(buildTerms)).toEqual( + new Set( + ["code", ...consumers].map( + (scope) => `needs.changes.outputs.${scope} == 'true'`, + ), + ), + ); + expect(jobs["test-shard"]?.if).toBe("needs.changes.outputs.code == 'true'"); + for (const name of consumers) { + const caller = jobs[name]; + expect(caller?.needs).toContain("build"); + expect(caller?.with).toEqual({ build_cache_artifact: "rea-build-cache" }); + const path = z.string().parse(caller?.uses); + const called = z + .object({ + on: z.object({ + workflow_call: z.object({ + inputs: z.record(z.string(), z.unknown()), + }), + workflow_dispatch: z.unknown(), + }), + jobs: z.record( + z.string(), + z.object({ + steps: z.array( + z.object({ + uses: z.string().optional(), + with: z.record(z.string(), z.unknown()).optional(), + }), + ), + }), + ), + }) + .parse( + parse( + await readFile(new URL(`../../../${path}`, import.meta.url), "utf8"), + ), + ); + expect(called.on.workflow_call.inputs.build_cache_artifact).toMatchObject({ + type: "string", + default: "", + }); + expect(Object.hasOwn(called.on, "workflow_dispatch")).toBe(true); + for (const job of Object.values(called.jobs)) + expect( + job.steps.find( + (step) => step.uses === "./.github/actions/portable-build-cache", + )?.with, + ).toEqual({ build_cache_artifact: "${{ inputs.build_cache_artifact }}" }); + } + const action = z + .object({ + runs: z.object({ + steps: z.array( + z.object({ + id: z.string().optional(), + uses: z.string().optional(), + if: z.string().optional(), + with: z.record(z.string(), z.unknown()).optional(), + "continue-on-error": z.boolean().optional(), + }), + ), + }), + }) + .parse( + parse( + await readFile( + new URL( + "../../../.github/actions/portable-build-cache/action.yml", + import.meta.url, + ), + "utf8", + ), + ), + ); + const download = action.runs.steps.find( + (step) => step.uses === "actions/download-artifact@v4", + ); + expect(download?.with).toEqual({ + name: "${{ inputs.build_cache_artifact }}", + path: ".turbo/cache/", + }); + expect(download?.if).toBe("inputs.build_cache_artifact != ''"); + expect(download?.["continue-on-error"]).not.toBe(true); + for (const step of action.runs.steps.filter( + (step) => step.id === "plan" || step.uses?.startsWith("actions/cache"), + )) + expect(step.if).toContain("inputs.build_cache_artifact == ''"); +}); + it("runs workflow validation inside required static checks when selected", async () => { const { jobs } = await workflow(); const steps = z @@ -294,7 +406,14 @@ it.each([ ); needs.changes = { result: "success", - outputs: { code: "true", [scope]: selected }, + outputs: { + ...Object.fromEntries( + (await scopeFlags()).map((flag) => [flag, "false"]), + ), + tier: "fast", + code: "true", + [scope]: selected, + }, }; if (result === "missing") delete needs[lane]; else needs[lane] = { result }; @@ -319,3 +438,69 @@ it.each([ } }, ); + +it("rejects incomplete or malformed classification even when available jobs succeeded", async () => { + const flags = await scopeFlags(); + const valid = { + ...Object.fromEntries(flags.map((flag) => [flag, "false"])), + tier: "fast", + }; + const run = (outputs: unknown) => + executeWorkflowFixture( + process.execPath, + [ + fileURLToPath( + new URL("../../../scripts/ci/required.mjs", import.meta.url), + ), + ], + { + env: { + ...process.env, + CI_JOBS: JSON.stringify({ + changes: { result: "success", outputs }, + static: { result: "success" }, + test: { result: "success" }, + }), + CI_QUALIFICATION_DIRECTORY: "", + }, + }, + ); + await expect(run(valid)).resolves.toMatchObject({ stderr: "" }); + // Metadata is not a scope, even when its string happens to be "true". + await expect( + run({ ...valid, qualification_run_id: "true" }), + ).resolves.toMatchObject({ stderr: "" }); + for (const outputs of [ + undefined, + null, + {}, + { ...valid, tier: undefined }, + { ...valid, tier: "" }, + { ...valid, tier: "unknown" }, + ]) { + await expect(run(outputs)).rejects.toMatchObject({ + code: 1, + stderr: expect.stringContaining("valid tier"), + }); + } + for (const flag of flags) { + await expect(run({ ...valid, [flag]: undefined })).rejects.toMatchObject({ + code: 1, + stderr: expect.stringContaining(`scope outputs: ${flag}`), + }); + } + for (const value of ["", "TRUE", "yes", true, false, 0, null, [], {}]) { + await expect(run({ ...valid, sqlite: value })).rejects.toMatchObject({ + code: 1, + stderr: expect.stringContaining("scope outputs: sqlite"), + }); + } + for (const tier of ["full", "reused"]) { + await expect( + run({ ...valid, tier, sqlite: undefined }), + ).rejects.toMatchObject({ + code: 1, + stderr: expect.stringContaining("scope outputs: sqlite"), + }); + } +}); diff --git a/tests/boundary/filesystem/ciTestReports.test.ts b/tests/boundary/filesystem/ciTestReports.test.ts index 333e93f08..06212ad07 100644 --- a/tests/boundary/filesystem/ciTestReports.test.ts +++ b/tests/boundary/filesystem/ciTestReports.test.ts @@ -8,6 +8,8 @@ import { expect, it } from "vitest"; import { parse } from "yaml"; import { z } from "zod"; +import { planTestSelection } from "../../../scripts/ci/test-selection.mjs"; + import { parseTestDurations } from "../../ciTestSequencer.js"; import { createTestTempDirectory } from "../../fixtures/temporaryDirectory.js"; import { workflowGit } from "../../support/workflowGit.js"; @@ -61,6 +63,16 @@ it("collects failed-shard diagnostics before enforcing the shard result", async "--reporter=./scripts/ci/test-timing-reporter.mjs", ); expect(steps[upload]?.with?.path).toContain("test-timings.json"); + expect(steps[upload]?.with).toMatchObject({ "include-hidden-files": true }); + expect(steps[upload]?.with?.path).toContain( + ".cache/ci-test-selection/report.json", + ); + const summary = steps.find((step) => + step.run?.includes("scripts/ci/test-report.mjs"), + ); + expect(summary?.env?.CI_TEST_SELECTION).toBe( + "${{ needs.changes.outputs.test_selection }}", + ); const command = steps[gate]?.run; if (command === undefined) throw new Error("Missing shard result gate"); for (const result of ["failure", "cancelled", "skipped", ""]) { @@ -304,6 +316,13 @@ it("recovers complete module costs from real blob reports without executing test ), }) .parse(JSON.parse(await readFile(join(root, "test-timings.json"), "utf8"))); + expect( + JSON.parse(await readFile(join(root, "test-timings.json"), "utf8")), + ).toMatchObject({ + reason: "passed", + unhandledErrors: 0, + files: [{ state: "passed" }], + }); expect(report.files).toHaveLength(1); const file = report.files[0]!; const json = z @@ -349,7 +368,13 @@ it("recovers complete module costs from real blob reports without executing test const failed = JSON.parse( await readFile(join(root, "test-timings.json"), "utf8"), ) as unknown; - expect(failed).toMatchObject({ version: 1, success: false }); + expect(failed).toMatchObject({ + version: 1, + success: false, + reason: "failed", + unhandledErrors: 0, + files: [{ state: "failed" }], + }); // Even a caller accidentally marking the merge successful cannot publish this report. await execute(process.execPath, [fileURLToPath(script)], { cwd: root, @@ -359,3 +384,76 @@ it("recovers complete module costs from real blob reports without executing test readFile(join(root, ".cache/ci-test-history/test-results.json")), ).rejects.toMatchObject({ code: "ENOENT" }); }); + +it("records failed omissions from full reports without treating partial or unhandled outcomes as parity", async () => { + const root = await createTestTempDirectory("rea-ci-selection-report-"); + await workflowGit(root, ["init"]); + const paths = [ + "src/sqlite/provider.test.ts", + "src/evm/provider.test.ts", + "tests/boundary/mcp/workflow.test.ts", + ]; + for (const path of paths) { + await mkdir(join(root, path, ".."), { recursive: true }); + await writeFile(join(root, path), "// fixture\n"); + } + await workflowGit(root, ["add", "src", "tests"]); + const timing = { + version: 1, + success: false, + reason: "failed", + unhandledErrors: 0, + files: paths.map((name, index) => ({ + name, + durationMs: 10, + state: index === 1 ? "failed" : "passed", + })), + }; + const run = async (report: unknown) => { + await writeFile(join(root, "test-timings.json"), JSON.stringify(report)); + await execute(process.execPath, [fileURLToPath(script)], { + cwd: root, + env: { + ...process.env, + SHARD_RESULT: "failure", + MERGE_RESULT: "failure", + CI_VERIFY_TEST_INVENTORY: "true", + CI_TEST_SELECTION: JSON.stringify( + planTestSelection(["src/sqlite/Provider.ts"], "pull_request", "fast"), + ), + GITHUB_SHA: "merge-sha", + CI_PR_HEAD_SHA: "pr-head-sha", + }, + }); + return JSON.parse( + await readFile( + join(root, ".cache/ci-test-selection/report.json"), + "utf8", + ), + ) as unknown; + }; + expect(await run(timing)).toMatchObject({ + observational: true, + complete: true, + selected: [paths[0], paths[2]], + omitted: [paths[1]], + missedFailures: [paths[1]], + executionSha: "merge-sha", + prHeadSha: "pr-head-sha", + }); + for (const report of [ + { ...timing, files: timing.files.slice(0, 1) }, + { ...timing, unhandledErrors: 1 }, + { ...timing, reason: "interrupted" }, + ]) + expect(await run(report)).toMatchObject({ complete: false }); + await expect(run({ ...timing, files: "invalid" })).rejects.toMatchObject({ + code: 1, + }); + await expect( + readFile(join(root, ".cache/ci-test-selection/report.json")), + ).rejects.toMatchObject({ code: "ENOENT" }); + await expect( + readFile(join(root, ".cache/ci-test-history/test-results.json")), + ).rejects.toMatchObject({ code: "ENOENT" }); +}); diff --git a/tests/boundary/filesystem/testTaskCache.test.ts b/tests/boundary/filesystem/testTaskCache.test.ts index 7e06ce1a1..5d877e829 100644 --- a/tests/boundary/filesystem/testTaskCache.test.ts +++ b/tests/boundary/filesystem/testTaskCache.test.ts @@ -1,6 +1,6 @@ import { execFile } from "node:child_process"; -import { readFile, writeFile } from "node:fs/promises"; -import { join } from "node:path"; +import { cp, readFile, writeFile } from "node:fs/promises"; +import { basename, join } from "node:path"; import { promisify } from "node:util"; import { expect, it } from "vitest"; @@ -78,7 +78,7 @@ it.skipIf(process.platform === "android")( await workflowGit(root, ["init", "--initial-branch=main"]); await workflowGit(root, ["add", "."]); await workflowGit(root, ["commit", "-m", "test: seed cache fixture"]); - const run = () => + const run = (cwd = root) => execute( process.execPath, [ @@ -87,7 +87,7 @@ it.skipIf(process.platform === "android")( "test:run", "--output-logs=errors-only", ], - { cwd: root, env: { ...process.env, TURBO_DAEMON: "false" } }, + { cwd, env: { ...process.env, TURBO_DAEMON: "false" } }, ); await run(); await writeFile(join(root, "README_zh.md"), "wrong command"); @@ -104,5 +104,32 @@ it.skipIf(process.platform === "android")( "prepared\n", ); } + // Provider jobs receive the producer's cache in a fresh checkout, not its + // outputs. Turbo must restore identical inputs and rebuild changed ones. + const consumer = await createTestTempDirectory("rea-cache-consumer-"); + await cp(root, consumer, { + recursive: true, + filter: (path) => + basename(path) !== ".turbo" && + !/^prepared-\d+\.txt$/u.test(basename(path)), + }); + await cp(join(root, ".turbo/cache"), join(consumer, ".turbo/cache"), { + recursive: true, + }); + const restored = await run(consumer); + expect(restored.stdout).toContain(`${dependencies.length} cached`); + for (const index of dependencies.keys()) + expect( + await readFile(join(consumer, `prepared-${index}.txt`), "utf8"), + ).toBe("prepared\n"); + await writeFile( + join(consumer, "artifact-input.txt"), + "changed producer input", + ); + await run(consumer); + for (const index of dependencies.keys()) + expect( + await readFile(join(consumer, `prepared-${index}.txt`), "utf8"), + ).toBe("prepared\nprepared\n"); }, ); diff --git a/tests/evaluation/ciTestSelection.test.ts b/tests/evaluation/ciTestSelection.test.ts new file mode 100644 index 000000000..1dff7f9e3 --- /dev/null +++ b/tests/evaluation/ciTestSelection.test.ts @@ -0,0 +1,240 @@ +import { expect, it } from "vitest"; + +import { + compareTestSelection, + planTestSelection, +} from "../../scripts/ci/test-selection.mjs"; + +const sqliteSource = "src/sqlite/SqliteDatabaseProvider.test.ts"; +const sqliteBoundary = "tests/boundary/sqlite/sqliteDatabaseProvider.test.ts"; +const adbSource = "src/adb/AdbProvider.test.ts"; +const adbBoundary = + "tests/boundary/providers/adb/adbBugreportPublication.test.ts"; +const cliBoundary = "tests/boundary/cli/apktoolCli.test.ts"; +const mcpBoundary = "tests/boundary/mcp/apktoolMcp.test.ts"; +const processBoundary = + "tests/boundary/process/processCaptureLifecycle.test.ts"; +const acceptance = "tests/acceptance/tools/localToolContracts.test.ts"; +const composition = "tests/composition/directAnalysisCleanup.test.ts"; +const evaluation = "tests/evaluation/testDiscovery.test.ts"; + +const passed = (name: string) => ({ name, durationMs: 10, state: "passed" }); + +it("unions affected leaf owners, keeps changed tests and retains unowned integration tests", () => { + const plan = planTestSelection( + ["src/sqlite/SqliteDatabaseProvider.ts", adbBoundary], + "pull_request", + "fast", + ); + const inventory = [ + sqliteSource, + sqliteBoundary, + adbSource, + adbBoundary, + cliBoundary, + mcpBoundary, + processBoundary, + acceptance, + composition, + evaluation, + ]; + const report = compareTestSelection(plan, inventory, inventory.map(passed)); + + expect(plan).toMatchObject({ + mode: "candidate", + owners: ["adb", "sqlite"], + }); + expect(report.selected).toEqual( + expect.arrayContaining([ + sqliteSource, + sqliteBoundary, + adbSource, + adbBoundary, + cliBoundary, + mcpBoundary, + processBoundary, + acceptance, + composition, + evaluation, + ]), + ); + expect(report.omitted).toEqual([]); +}); + +it("omits only unrelated owned tests while retaining dynamic boundary suites", () => { + const plan = planTestSelection( + ["src/sqlite/SqliteDatabaseProvider.ts"], + "pull_request", + "fast", + ); + const inventory = [ + sqliteSource, + sqliteBoundary, + adbSource, + adbBoundary, + cliBoundary, + mcpBoundary, + processBoundary, + acceptance, + composition, + evaluation, + ]; + const report = compareTestSelection(plan, inventory, inventory.map(passed)); + + expect(report.selected).toEqual( + expect.arrayContaining([ + sqliteSource, + sqliteBoundary, + cliBoundary, + mcpBoundary, + processBoundary, + acceptance, + composition, + evaluation, + ]), + ); + expect(report.omitted).toEqual([adbSource, adbBoundary]); + expect(report.complete).toBe(true); +}); + +it.each([ + ["unowned shared source", "src/application/Workflow.ts"], + ["fixture input", "tests/conformance/c/fixture.c"], + ["configuration", "vitest.config.ts"], + ["documentation", "README.md"], + ["CLI implementation", "src/cli/apktoolCommands.ts"], + ["MCP implementation", "src/server/registerSqliteTools.ts"], + ["process ownership", "src/process/ProcessOwnership.ts"], + ["acceptance test", acceptance], + ["composition test", composition], +])("uses full fallback for %s changes", (_description, path) => { + expect(planTestSelection([path], "pull_request", "fast")).toMatchObject({ + mode: "full", + owners: [], + changedTests: [], + }); +}); + +it("uses full fallback outside ordinary fast pull requests or without diff paths", () => { + expect( + planTestSelection(["src/sqlite/SqliteDatabaseProvider.ts"], "push", "fast") + .mode, + ).toBe("full"); + expect( + planTestSelection( + ["src/sqlite/SqliteDatabaseProvider.ts"], + "pull_request", + "full", + ).mode, + ).toBe("full"); + expect(planTestSelection([], "pull_request", "fast").mode).toBe("full"); +}); + +it("keeps new changed tests and falls back when a changed test was removed", () => { + const addedTest = "src/sqlite/NewDatabaseRule.test.ts"; + const candidate = planTestSelection([addedTest], "pull_request", "fast"); + expect(candidate.changedTests).toEqual([addedTest]); + const inventory = [addedTest, adbSource]; + expect( + compareTestSelection(candidate, inventory, inventory.map(passed)).selected, + ).toContain(addedTest); + + const removedTest = "src/sqlite/RemovedDatabaseRule.test.ts"; + const report = compareTestSelection( + planTestSelection([removedTest], "pull_request", "fast"), + [sqliteSource, adbSource], + [passed(sqliteSource), passed(adbSource)], + ); + expect(report.plan.mode).toBe("full"); + expect(report.selected).toEqual([adbSource, sqliteSource]); + expect(report.omitted).toEqual([]); +}); + +it("marks incomplete inventories and nonterminal outcomes as incomplete", () => { + const plan = planTestSelection( + ["src/sqlite/SqliteDatabaseProvider.ts"], + "pull_request", + "fast", + ); + const inventory = [sqliteSource, adbSource]; + const completeResults = inventory.map(passed); + + expect( + compareTestSelection(plan, inventory, [passed(sqliteSource)]).complete, + ).toBe(false); + expect( + compareTestSelection(plan, inventory, [ + ...completeResults, + passed("tests/evaluation/unexpected.test.ts"), + ]), + ).toMatchObject({ + complete: false, + unexpected: ["tests/evaluation/unexpected.test.ts"], + }); + expect( + compareTestSelection(plan, inventory, [ + ...completeResults, + passed(adbSource), + ]), + ).toMatchObject({ complete: false, duplicateFiles: 1 }); + for (const state of ["pending", "skipped"]) + expect( + compareTestSelection(plan, inventory, [ + passed(sqliteSource), + { name: adbSource, durationMs: 0, state }, + ]), + ).toMatchObject({ complete: false, unobserved: [adbSource] }); +}); + +it("reports a failure from an omitted test, including collection or module failures", () => { + const plan = planTestSelection( + ["src/sqlite/SqliteDatabaseProvider.ts"], + "pull_request", + "fast", + ); + const report = compareTestSelection( + plan, + [sqliteSource, adbSource], + [passed(sqliteSource), { name: adbSource, durationMs: 1, state: "failed" }], + ); + + expect(report.omitted).toContain(adbSource); + expect(report.missedFailures).toEqual([adbSource]); +}); + +it("rejects plans that name unowned owners or malformed changed tests", () => { + const plan = planTestSelection( + ["src/sqlite/SqliteDatabaseProvider.ts"], + "pull_request", + "fast", + ); + expect(() => + compareTestSelection( + { ...plan, owners: ["unknown"] }, + [sqliteSource], + [passed(sqliteSource)], + ), + ).toThrow("Invalid observational test selection plan"); + expect(() => + compareTestSelection( + { ...plan, changedTests: ["src/sqlite/not-a-test.ts"] }, + [sqliteSource], + [passed(sqliteSource)], + ), + ).toThrow("Invalid observational test selection plan"); +}); + +it("uses full fallback when a changed owner has no executable tests", () => { + const report = compareTestSelection( + planTestSelection( + ["src/sqlite/SqliteDatabaseProvider.ts"], + "pull_request", + "fast", + ), + [adbSource], + [passed(adbSource)], + ); + + expect(report.plan.mode).toBe("full"); + expect(report.omitted).toEqual([]); +});