From bd2c202dd930548194cdbc55c27a797f22279db9 Mon Sep 17 00:00:00 2001 From: morluto <76467478+morluto@users.noreply.github.com> Date: Tue, 13 Oct 2026 19:06:31 +0800 Subject: [PATCH 1/5] fix(ci): reject incomplete verification plans --- docs/testing.md | 5 +- scripts/ci/required.mjs | 25 +++-- .../boundary/filesystem/ciChangeScope.test.ts | 10 +- .../filesystem/ciRequiredLanes.test.ts | 92 +++++++++++++++++-- 4 files changed, 113 insertions(+), 19 deletions(-) diff --git a/docs/testing.md b/docs/testing.md index 22b5c0c77..6dddda64b 100644 --- a/docs/testing.md +++ b/docs/testing.md @@ -874,7 +874,10 @@ summary. Representative Git-diff cases live in `tests/boundary/filesystem/ciChangeScope.test.ts`. `CI required` always runs and checks classification and every selected lane, -rejecting failed, cancelled, or unexpectedly skipped work. Branch protection +rejecting failed, cancelled, or unexpectedly skipped work. It rejects missing or +unknown tiers and requires every canonical scope to be explicitly `true` or +`false`; incomplete classification cannot silently omit verification lanes. +Branch protection should require this aggregate, `Static checks`, `Portable tests`, and `Conventional PR title`. Title validation runs separately on PR opening, reopening, commits, and edits; correcting a title refreshes only this lightweight diff --git a/scripts/ci/required.mjs b/scripts/ci/required.mjs index afe52c2b8..01907de37 100644 --- a/scripts/ci/required.mjs +++ b/scripts/ci/required.mjs @@ -2,22 +2,31 @@ import { CI_FLAGS } from "./scopes.mjs"; import { recordQualification } from "./qualification.mjs"; const jobs = JSON.parse(process.env.CI_JOBS); -const outputs = jobs.changes.outputs; +const outputs = jobs?.changes?.outputs; +if (!["fast", "full", "reused"].includes(outputs?.tier)) + throw new Error( + "CI classification requires a valid tier: fast, full, or reused", + ); +const invalidFlags = CI_FLAGS.filter( + (flag) => outputs[flag] !== "true" && outputs[flag] !== "false", +); +if (invalidFlags.length) + throw new Error( + `CI classification requires explicit true/false scope outputs: ${invalidFlags.join(", ")}`, + ); +const selectedScopes = CI_FLAGS.filter((flag) => outputs[flag] === "true"); const reused = outputs.tier === "reused"; if ( reused && (process.env.GITHUB_EVENT_NAME !== "schedule" || !/^[1-9]\d*$/.test(outputs.qualification_run_id ?? "") || !Number.isSafeInteger(Number(outputs.qualification_run_id)) || - CI_FLAGS.some((flag) => outputs[flag] !== "false")) + selectedScopes.length !== 0) ) throw new Error( "Reused qualification requires a scheduled admission receipt and no selected lanes", ); -if ( - outputs.tier === "full" && - CI_FLAGS.some((flag) => outputs[flag] !== "true") -) +if (outputs.tier === "full" && selectedScopes.length !== CI_FLAGS.length) throw new Error("Full qualification requires every canonical CI scope"); const required = reused ? ["changes"] : ["changes", "static", "test"]; const lanes = { @@ -31,9 +40,7 @@ const lanes = { docs: ["docs"], workflows: ["static"], }; -for (const [scope, value] of Object.entries(outputs)) { - if (value === "true") required.push(...(lanes[scope] ?? [scope])); -} +for (const scope of selectedScopes) required.push(...(lanes[scope] ?? [scope])); const failed = [...new Set(required)].filter( (name) => jobs[name]?.result !== "success", ); diff --git a/tests/boundary/filesystem/ciChangeScope.test.ts b/tests/boundary/filesystem/ciChangeScope.test.ts index 8ed397e84..759f5c367 100644 --- a/tests/boundary/filesystem/ciChangeScope.test.ts +++ b/tests/boundary/filesystem/ciChangeScope.test.ts @@ -662,8 +662,16 @@ it("rejects skipped selected work while allowing unselected lanes to skip", asyn const command = fileURLToPath( new URL("../../../scripts/ci/required.mjs", import.meta.url), ); + const { CI_FLAGS } = await loadScopes(); const jobs = { - changes: { result: "success", outputs: { evm: "true" } }, + changes: { + result: "success", + outputs: { + ...Object.fromEntries(CI_FLAGS.map((flag) => [flag, "false"])), + tier: "fast", + evm: "true", + }, + }, static: { result: "success" }, test: { result: "success" }, evm: { result: "success" }, diff --git a/tests/boundary/filesystem/ciRequiredLanes.test.ts b/tests/boundary/filesystem/ciRequiredLanes.test.ts index 1bac771a2..e460bb52b 100644 --- a/tests/boundary/filesystem/ciRequiredLanes.test.ts +++ b/tests/boundary/filesystem/ciRequiredLanes.test.ts @@ -26,6 +26,15 @@ const workflow = async (filename = "ci.yml") => ), ); +const scopeFlags = async () => + z + .object({ CI_FLAGS: z.array(z.string()) }) + .parse( + await import( + new URL("../../../scripts/ci/scopes.mjs", import.meta.url).href + ), + ).CI_FLAGS; + it("binds SQLite compatibility jobs to the build's prepared runtime artifact and root layout", async () => { expect((await workflow()).jobs["sqlite-runtime"]?.if).toBe( "needs.changes.outputs.sqlite == 'true'", @@ -57,13 +66,7 @@ it("binds SQLite compatibility jobs to the build's prepared runtime artifact and it("exports every canonical scope with read authority for admission and keeps telemetry outside the gate", async () => { const { jobs } = await workflow(); - const { CI_FLAGS } = z - .object({ CI_FLAGS: z.array(z.string()) }) - .parse( - await import( - new URL("../../../scripts/ci/scopes.mjs", import.meta.url).href - ), - ); + const CI_FLAGS = await scopeFlags(); const outputs = z.record(z.string(), z.string()).parse(jobs.changes?.outputs); expect(jobs.changes?.permissions).toEqual({ contents: "read", @@ -294,7 +297,14 @@ it.each([ ); needs.changes = { result: "success", - outputs: { code: "true", [scope]: selected }, + outputs: { + ...Object.fromEntries( + (await scopeFlags()).map((flag) => [flag, "false"]), + ), + tier: "fast", + code: "true", + [scope]: selected, + }, }; if (result === "missing") delete needs[lane]; else needs[lane] = { result }; @@ -319,3 +329,69 @@ it.each([ } }, ); + +it("rejects incomplete or malformed classification even when available jobs succeeded", async () => { + const flags = await scopeFlags(); + const valid = { + ...Object.fromEntries(flags.map((flag) => [flag, "false"])), + tier: "fast", + }; + const run = (outputs: unknown) => + executeWorkflowFixture( + process.execPath, + [ + fileURLToPath( + new URL("../../../scripts/ci/required.mjs", import.meta.url), + ), + ], + { + env: { + ...process.env, + CI_JOBS: JSON.stringify({ + changes: { result: "success", outputs }, + static: { result: "success" }, + test: { result: "success" }, + }), + CI_QUALIFICATION_DIRECTORY: "", + }, + }, + ); + await expect(run(valid)).resolves.toMatchObject({ stderr: "" }); + // Metadata is not a scope, even when its string happens to be "true". + await expect( + run({ ...valid, qualification_run_id: "true" }), + ).resolves.toMatchObject({ stderr: "" }); + for (const outputs of [ + undefined, + null, + {}, + { ...valid, tier: undefined }, + { ...valid, tier: "" }, + { ...valid, tier: "unknown" }, + ]) { + await expect(run(outputs)).rejects.toMatchObject({ + code: 1, + stderr: expect.stringContaining("valid tier"), + }); + } + for (const flag of flags) { + await expect(run({ ...valid, [flag]: undefined })).rejects.toMatchObject({ + code: 1, + stderr: expect.stringContaining(`scope outputs: ${flag}`), + }); + } + for (const value of ["", "TRUE", "yes", true, false, 0, null, [], {}]) { + await expect(run({ ...valid, sqlite: value })).rejects.toMatchObject({ + code: 1, + stderr: expect.stringContaining("scope outputs: sqlite"), + }); + } + for (const tier of ["full", "reused"]) { + await expect( + run({ ...valid, tier, sqlite: undefined }), + ).rejects.toMatchObject({ + code: 1, + stderr: expect.stringContaining("scope outputs: sqlite"), + }); + } +}); From c8d861f9764f8c07f0925dae0c6451de85fffd6a Mon Sep 17 00:00:00 2001 From: morluto <76467478+morluto@users.noreply.github.com> Date: Tue, 13 Oct 2026 19:12:26 +0800 Subject: [PATCH 2/5] ci: scope timing helpers to portable verification --- docs/testing.md | 8 +++- scripts/ci/plan.mjs | 10 +++++ .../boundary/filesystem/ciChangeScope.test.ts | 41 +++++++++++++++++++ 3 files changed, 57 insertions(+), 2 deletions(-) diff --git a/docs/testing.md b/docs/testing.md index 6dddda64b..ac1a01f84 100644 --- a/docs/testing.md +++ b/docs/testing.md @@ -826,8 +826,12 @@ scripts also select it. Unrelated code edits retain the complete portable suite on the pinned Node version without starting four additional compatibility jobs. Full qualification retains every SQLite version. Shared process/contract/CLI boundaries, dependencies, other packaging inputs, -unknown paths, and changes to the planner select the full baseline. Package -script-only edits select their owning verifier when its entrypoint is +unknown paths, and changes to the planner select the full baseline. +CI timing collection, timing history, and test-report helper edits retain static +checks and the complete portable suite without selecting unrelated host/provider +lanes. Classification, qualification, gate, release receipt, and unknown CI +scripts keep the broad fallback. +Package script-only edits select their owning verifier when its entrypoint is identifiable; unknown or shared script changes select everything. Android build-script edits retain the Linux suite and workflow validation; real Android verification stays manual. Renames account for both paths. diff --git a/scripts/ci/plan.mjs b/scripts/ci/plan.mjs index 9384d9b05..3c711a1f9 100644 --- a/scripts/ci/plan.mjs +++ b/scripts/ci/plan.mjs @@ -25,6 +25,15 @@ const FIXTURE_SOURCES = [ "tests/conformance/versions/**", ]; const READINESS_SOURCES = ["tests/conformance/readiness/**"]; +// These reporters affect CI output only; they do not select a host or provider +// verification lane when changed on their own. +const PORTABLE_CI_REPORTERS = new Set([ + "scripts/ci/job-timings.mjs", + "scripts/ci/job-timings-batch.mjs", + "scripts/ci/test-report.mjs", + "scripts/ci/test-timing-reporter.mjs", + "scripts/ci/test-history.mjs", +]); const matchesAny = (file, patterns) => patterns.some((pattern) => matchesGlob(file, pattern)); const selected = Object.fromEntries(flags.map((flag) => [flag, false])); @@ -200,6 +209,7 @@ if (tier === "reused") { } if (matchesAny(file, READINESS_SOURCES)) selected.readiness = true; if (file.startsWith("tests/")) continue; + if (PORTABLE_CI_REPORTERS.has(file)) continue; if ( /^(src\/process\/|src\/filesystem\/|src\/contracts\/tool|src\/server\/createServer|src\/cli\.ts|scripts\/ci\/)/.test( file, diff --git a/tests/boundary/filesystem/ciChangeScope.test.ts b/tests/boundary/filesystem/ciChangeScope.test.ts index 759f5c367..714d2f980 100644 --- a/tests/boundary/filesystem/ciChangeScope.test.ts +++ b/tests/boundary/filesystem/ciChangeScope.test.ts @@ -137,6 +137,47 @@ it("keeps every translated root README and authored guide on the documentation l }); }); +it.each([ + "job-timings.mjs", + "job-timings-batch.mjs", + "test-report.mjs", + "test-timing-reporter.mjs", + "test-history.mjs", +])( + "keeps portable CI reporter %s on static checks and portable tests", + async (name) => { + const { directory, base } = await fixture(); + const head = await change(directory, [`scripts/ci/${name}`]); + const plan = await classify(directory, base, head); + const { CI_FLAGS } = await loadScopes(); + expect(plan.code).toBe(true); + for (const flag of CI_FLAGS) + if (flag !== "code") expect(plan[flag]).toBe(false); + }, +); + +it("preserves a provider lane when a portable CI reporter changes with provider source", async () => { + const { directory, base } = await fixture(); + const head = await change(directory, [ + "scripts/ci/test-report.mjs", + "src/evm/Example.ts", + ]); + expect(await classify(directory, base, head)).toMatchObject({ + code: true, + evm: true, + browser: false, + package: false, + }); +}); + +it("keeps unknown CI scripts on complete qualification", async () => { + const { directory, base } = await fixture(); + const head = await change(directory, ["scripts/ci/new-qualification.mjs"]); + const plan = await classify(directory, base, head); + const { CI_FLAGS } = await loadScopes(); + for (const flag of CI_FLAGS) expect(plan[flag]).toBe(true); +}); + it("does not charge a documentation PR for unrelated implementation updates on main", async () => { const { directory, base } = await fixture(); await git(directory, ["switch", "-c", "docs/change"]); From fa0a9b2d328a07d8ec2924dc224d51c620981dcc Mon Sep 17 00:00:00 2001 From: morluto <76467478+morluto@users.noreply.github.com> Date: Tue, 13 Oct 2026 19:21:09 +0800 Subject: [PATCH 3/5] ci: observe affected-owner test plans against full results --- .github/workflows/ci.yml | 5 + docs/testing.md | 21 ++ scripts/ci/plan.mjs | 11 +- scripts/ci/test-report.mjs | 70 ++++- scripts/ci/test-selection.d.mts | 35 +++ scripts/ci/test-selection.mjs | 151 +++++++++++ scripts/ci/test-timing-reporter.mjs | 3 + .../boundary/filesystem/ciChangeScope.test.ts | 17 ++ .../boundary/filesystem/ciTestReports.test.ts | 100 +++++++- tests/evaluation/ciTestSelection.test.ts | 240 ++++++++++++++++++ 10 files changed, 639 insertions(+), 14 deletions(-) create mode 100644 scripts/ci/test-selection.d.mts create mode 100644 scripts/ci/test-selection.mjs create mode 100644 tests/evaluation/ciTestSelection.test.ts diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index b1af43269..28373f89d 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -61,6 +61,7 @@ jobs: nativeaot: ${{ steps.plan.outputs.nativeaot }} website: ${{ steps.plan.outputs.website }} matrix: ${{ steps.plan.outputs.matrix }} + test_selection: ${{ steps.plan.outputs.test_selection }} steps: - uses: actions/checkout@v4 with: @@ -315,6 +316,8 @@ jobs: SHARD_RESULT: ${{ needs.test-shard.result }} MERGE_RESULT: ${{ needs.changes.outputs.tier == 'full' && steps.merge.outcome || steps.merge-fast.outcome }} CI_VERIFY_TEST_INVENTORY: "true" + CI_TEST_SELECTION: ${{ needs.changes.outputs.test_selection }} + CI_PR_HEAD_SHA: ${{ github.event.pull_request.head.sha }} run: node scripts/ci/test-report.mjs - uses: actions/upload-artifact@v4 if: needs.changes.outputs.code == 'true' && !cancelled() && hashFiles('test-results.json') != '' @@ -325,6 +328,8 @@ jobs: test-results.json test-timings.json coverage/ + .cache/ci-test-selection/report.json + include-hidden-files: true if-no-files-found: error retention-days: 30 - name: Publish successful timing history diff --git a/docs/testing.md b/docs/testing.md index ac1a01f84..03121e172 100644 --- a/docs/testing.md +++ b/docs/testing.md @@ -805,6 +805,27 @@ remain present and fail if classification fails. Every code change retains the complete six-shard Linux suite; individual test files are not selected by imports or filenames. +Ordinary PRs also produce an **observational** test-selection plan. The candidate +leaf-adapter ownership table lives in `scripts/ci/test-selection.mjs`, separately +from provider-job scopes. It proposes keeping affected owners plus every unowned +test, including all CLI/MCP, acceptance, composition, conformance, and process +tests. Shared inputs, unknown paths, missing owner tests, and removed tests fall +back to the complete inventory. Renames account for both paths. Main, scheduled, +manual, and `ci:full` runs always propose the full suite. + +All six shards still execute the complete suite. The existing report merge +compares the proposal with actual module outcomes and retains +`.cache/ci-test-selection/report.json` in `test-results`, including selected and +omitted paths, changed input paths, missed failures, unobserved files, scheduling weights, PR head and +executed merge SHA. This adds no test reruns or runner jobs. Missing, duplicate, +skipped, interrupted, or unhandled results cannot establish a complete comparison; +even a complete passing comparison is not proof of ownership completeness. +Before enabling omissions, review representative full-run comparisons and +cross-owner consumers, including failures, and measure completed CI feedback and +runner demand. There is no automatic promotion or switch to skip tests. The +initial proposal measures only leaf producer-test omissions; it does not promise +the savings of a broader affected-test planner. + `scripts/ci/plan.mjs` selects lanes using the ownership table in `scripts/ci/scopes.mjs`. Provider implementation changes run their real-provider verifiers and the Linux x64 installed-package/Inspector baseline. The `apple` diff --git a/scripts/ci/plan.mjs b/scripts/ci/plan.mjs index 3c711a1f9..86879216d 100644 --- a/scripts/ci/plan.mjs +++ b/scripts/ci/plan.mjs @@ -2,6 +2,7 @@ import { execFileSync } from "node:child_process"; import { appendFileSync } from "node:fs"; import { matchesGlob } from "node:path"; import { CI_FLAGS as flags, MANUAL_VERIFIERS, SCOPES } from "./scopes.mjs"; +import { planTestSelection } from "./test-selection.mjs"; // Test files select the host/runtime lanes that execute their focused suites. const HOST_TEST_OWNERS = ["apple", "package", "sqlite"]; @@ -250,11 +251,17 @@ const matrix = { ] : [linux], }; -const plan = { ...selected, tier, matrix, files, reasons }; +const testSelection = planTestSelection(files, event, tier); +const plan = { ...selected, tier, matrix, files, reasons, testSelection }; if (process.env.GITHUB_OUTPUT) appendFileSync( process.env.GITHUB_OUTPUT, - Object.entries({ ...selected, tier, matrix: JSON.stringify(matrix) }) + Object.entries({ + ...selected, + tier, + matrix: JSON.stringify(matrix), + test_selection: JSON.stringify(testSelection), + }) .map(([key, value]) => `${key}=${value}\n`) .join(""), ); diff --git a/scripts/ci/test-report.mjs b/scripts/ci/test-report.mjs index 86419ae9f..035a01990 100644 --- a/scripts/ci/test-report.mjs +++ b/scripts/ci/test-report.mjs @@ -4,25 +4,33 @@ import { isAbsolute, join, relative, resolve, sep } from "node:path"; import { z } from "zod"; +import { compareTestSelection } from "./test-selection.mjs"; + const root = process.cwd(); const historyDirectory = join(root, ".cache/ci-test-history"); const historyPath = join(historyDirectory, "test-results.json"); +const selectionDirectory = join(root, ".cache/ci-test-selection"); +const selectionPath = join(selectionDirectory, "report.json"); // Never let a failed or partial rerun leave a previous successful history file. await rm(historyPath, { force: true }); +await rm(selectionPath, { force: true }); const report = z .object({ version: z.literal(1), success: z.boolean(), + reason: z.string().optional(), + unhandledErrors: z.number().int().nonnegative().optional(), files: z.array( z.object({ name: z.string().min(1), durationMs: z.number().finite().nonnegative(), + state: z.string().optional(), }), ), }) .parse(JSON.parse(await readFile("test-timings.json", "utf8"))); -const files = report.files.map(({ name, durationMs }) => { +const files = report.files.map(({ name, durationMs, state }) => { const path = relative(root, resolve(root, name)); if ( path === "" || @@ -31,21 +39,47 @@ const files = report.files.map(({ name, durationMs }) => { isAbsolute(path) ) throw new Error(`Test report path is outside the checkout: ${name}`); - return { name: path.split(sep).join("/"), durationMs }; + return { name: path.split(sep).join("/"), durationMs, state }; }); const complete = process.env.SHARD_RESULT === "success"; const passed = complete && process.env.MERGE_RESULT === "success" && report.success; // Successful shards and merged reports must account for the complete portable // inventory. Coverage thresholds alone cannot prove that every file ran. -if (passed && process.env.CI_VERIFY_TEST_INVENTORY === "true") { - const expected = execFileSync( - "git", - ["ls-files", "-z", "--", "src", "tests"], - { encoding: "utf8", maxBuffer: 8 * 1024 * 1024 }, - ) - .split("\0") - .filter((path) => path.endsWith(".test.ts")); +const verifyInventory = process.env.CI_VERIFY_TEST_INVENTORY === "true"; +const expected = + verifyInventory || process.env.CI_TEST_SELECTION + ? execFileSync("git", ["ls-files", "-z", "--", "src", "tests"], { + encoding: "utf8", + maxBuffer: 8 * 1024 * 1024, + }) + .split("\0") + .filter((path) => path.endsWith(".test.ts")) + : []; +let selection; +if (process.env.CI_TEST_SELECTION) { + const comparison = compareTestSelection( + JSON.parse(process.env.CI_TEST_SELECTION), + expected, + files, + ); + selection = { + ...comparison, + complete: + comparison.complete && + report.unhandledErrors === 0 && + ["passed", "failed"].includes(report.reason) && + ["success", "failure"].includes(process.env.SHARD_RESULT) && + ["success", "failure"].includes(process.env.MERGE_RESULT), + executionSha: process.env.GITHUB_SHA ?? null, + prHeadSha: process.env.CI_PR_HEAD_SHA ?? null, + runId: process.env.GITHUB_RUN_ID ?? null, + runAttempt: process.env.GITHUB_RUN_ATTEMPT ?? null, + }; + await mkdir(selectionDirectory, { recursive: true }); + await writeFile(selectionPath, JSON.stringify(selection)); +} +if (passed && verifyInventory) { const actual = files.map(({ name }) => name); const expectedSet = new Set(expected); const actualSet = new Set(actual); @@ -59,6 +93,16 @@ if (passed && process.env.CI_VERIFY_TEST_INVENTORY === "true") { const summary = [ complete ? "## Merged test diagnostics" : "## Partial test diagnostics", "", + ...(selection + ? [ + "### Observational PR test selection", + "", + `Plan: ${selection.plan.mode}. Proposed files: ${selection.selected.length}; omitted: ${selection.omitted.length}.`, + `Full-run comparison: ${selection.complete ? "complete" : "incomplete"}; failures outside proposed selection: ${selection.missedFailures.length}.`, + "The complete suite still ran. This report cannot enable omissions or qualify a revision. Skipped, missing, and interrupted observations do not establish parity.", + "", + ] + : []), `Shard result: ${process.env.SHARD_RESULT ?? "unknown"}; merge result: ${process.env.MERGE_RESULT ?? "unknown"}.`, passed ? "Successful timing history is available." @@ -85,6 +129,10 @@ if (passed) { await mkdir(historyDirectory, { recursive: true }); await writeFile( historyPath, - JSON.stringify({ version: 1, success: true, files }), + JSON.stringify({ + version: 1, + success: true, + files: files.map(({ name, durationMs }) => ({ name, durationMs })), + }), ); } diff --git a/scripts/ci/test-selection.d.mts b/scripts/ci/test-selection.d.mts new file mode 100644 index 000000000..c19414398 --- /dev/null +++ b/scripts/ci/test-selection.d.mts @@ -0,0 +1,35 @@ +export interface TestSelectionPlan { + readonly version: 1; + readonly mode: "full" | "candidate"; + readonly owners: readonly string[]; + readonly changedTests: readonly string[]; + readonly changedPaths: readonly string[]; + readonly reason: string; +} +export function planTestSelection( + files: readonly string[], + event: string, + tier: string, +): TestSelectionPlan; +export function compareTestSelection( + plan: unknown, + inventory: readonly string[], + results: readonly { + readonly name: string; + readonly durationMs: number; + readonly state?: string; + }[], +): { + readonly version: 1; + readonly observational: true; + readonly plan: TestSelectionPlan; + readonly selected: readonly string[]; + readonly omitted: readonly string[]; + readonly missing: readonly string[]; + readonly unexpected: readonly string[]; + readonly duplicateFiles: number; + readonly unobserved: readonly string[]; + readonly missedFailures: readonly string[]; + readonly complete: boolean; + readonly omittedDurationMs: number; +}; diff --git a/scripts/ci/test-selection.mjs b/scripts/ci/test-selection.mjs new file mode 100644 index 000000000..8165e3658 --- /dev/null +++ b/scripts/ci/test-selection.mjs @@ -0,0 +1,151 @@ +import { matchesGlob } from "node:path"; + +// Candidate ownership only: these leaf adapters and their producer tests can be +// compared with full-suite outcomes. This table never changes test execution. +// Shared application/domain code and dynamic CLI/MCP consumers are not leaf owners. +const OWNERS = { + adb: ["src/adb/**", "tests/boundary/providers/adb/**"], + apktool: ["src/apktool/**"], + dotnet: ["src/dotnet/**"], + evm: ["src/evm/**", "tests/boundary/evm/**"], + firmware: ["src/firmware/**", "tests/boundary/firmware/**"], + flutter: ["src/flutter/**"], + ghidra: ["src/ghidra/**", "tests/boundary/providers/ghidra/**"], + hopper: ["src/hopper/**", "tests/boundary/providers/hopper/**"], + ida: ["src/ida/**", "tests/boundary/providers/ida/**"], + jeb: ["src/jeb/**"], + pcap: ["src/pcap/**", "tests/boundary/pcap/**"], + sqlite: ["src/sqlite/**", "tests/boundary/sqlite/**"], + wasm: ["src/wasm/**"], +}; +const ownersOf = (path) => + Object.entries(OWNERS) + .filter(([, patterns]) => patterns.some((glob) => matchesGlob(path, glob))) + .map(([owner]) => owner); +const validPath = (path) => + typeof path === "string" && + /^(src|tests)\//.test(path) && + !/[\\\0\r\n]/.test(path) && + !path.split("/").some((part) => ["", ".", ".."].includes(part)); +const full = (reason, changedPaths = []) => ({ + version: 1, + mode: "full", + owners: [], + changedTests: [], + changedPaths, + reason, +}); + +/** Plan an observational PR subset from the classifier's existing Git diff. */ +export function planTestSelection(files, event, tier) { + if (event !== "pull_request" || tier !== "fast") + return full("Only ordinary PRs are candidates", files); + if (!files.length) return full("No changed paths available"); + const owners = new Set(); + const changedTests = []; + for (const path of files) { + if (!validPath(path)) + return full(`Shared or unknown input: ${path}`, files); + const matches = ownersOf(path); + if (!matches.length) return full(`Shared or unknown input: ${path}`, files); + for (const owner of matches) owners.add(owner); + if (path.endsWith(".test.ts")) changedTests.push(path); + } + return { + version: 1, + mode: "candidate", + owners: [...owners].sort(), + changedTests: [...new Set(changedTests)].sort(), + changedPaths: files, + reason: + "Observe affected leaf owners; retain integration and unowned tests", + }; +} + +/** Compare a proposal with the full run; missing observations never prove parity. */ +export function compareTestSelection(plan, inventory, results) { + if ( + plan?.version !== 1 || + !["full", "candidate"].includes(plan.mode) || + !Array.isArray(plan.owners) || + plan.owners.some((owner) => !Object.hasOwn(OWNERS, owner)) || + !Array.isArray(plan.changedTests) || + !Array.isArray(plan.changedPaths) || + plan.changedPaths.some((path) => typeof path !== "string") || + plan.changedTests.some( + (path) => !validPath(path) || !path.endsWith(".test.ts"), + ) || + typeof plan.reason !== "string" || + (plan.mode === "candidate" && !plan.owners.length) || + (plan.mode === "full" && (plan.owners.length || plan.changedTests.length)) + ) + throw new Error("Invalid observational test selection plan"); + const expected = new Set(inventory); + if (!expected.size || expected.size !== inventory.length) + throw new Error("Test selection requires a nonempty unique inventory"); + const ownersByFile = new Map( + plan.mode === "candidate" + ? inventory.map((path) => [path, ownersOf(path)]) + : [], + ); + // Removed/renamed tests have no current executable identity. Keep all tests. + const removed = plan.changedTests.filter((path) => !expected.has(path)); + const missingOwners = plan.owners.filter( + (owner) => + ![...ownersByFile.values()].some((owners) => owners.includes(owner)), + ); + const effective = removed.length + ? full( + `Changed tests absent from current inventory: ${removed.join(", ")}`, + plan.changedPaths, + ) + : missingOwners.length + ? full( + `Owners without executable tests: ${missingOwners.join(", ")}`, + plan.changedPaths, + ) + : plan; + const selected = []; + const omitted = []; + for (const path of inventory.toSorted()) { + const owners = ownersByFile.get(path) ?? []; + const keep = + effective.mode === "full" || + !owners.length || + effective.changedTests.includes(path) || + owners.some((owner) => effective.owners.includes(owner)); + (keep ? selected : omitted).push(path); + } + const actual = new Set(results.map((file) => file.name)); + const missing = inventory.filter((path) => !actual.has(path)); + const unexpected = [...actual].filter((path) => !expected.has(path)); + const duplicateFiles = results.length - actual.size; + const unobserved = results + .filter((file) => !["passed", "failed"].includes(file.state)) + .map((file) => file.name); + const omittedSet = new Set(omitted); + const missedFailures = results + .filter((file) => omittedSet.has(file.name) && file.state === "failed") + .map((file) => file.name); + return { + version: 1, + observational: true, + plan: effective, + selected, + omitted, + missing, + unexpected, + duplicateFiles, + unobserved, + missedFailures, + complete: + !missing.length && + !unexpected.length && + !duplicateFiles && + !unobserved.length, + // Phase totals are scheduling weights, not an estimate of wall-time savings. + omittedDurationMs: results + .filter((file) => omittedSet.has(file.name)) + .reduce((sum, file) => sum + file.durationMs, 0), + }; +} diff --git a/scripts/ci/test-timing-reporter.mjs b/scripts/ci/test-timing-reporter.mjs index 29f0cc582..ce0b1c5dc 100644 --- a/scripts/ci/test-timing-reporter.mjs +++ b/scripts/ci/test-timing-reporter.mjs @@ -21,6 +21,7 @@ export default class TestTimingReporter { }; return { name: module.moduleId, + state: module.state(), durationMs: Object.values(phases).reduce( (total, value) => total + value, 0, @@ -33,6 +34,8 @@ export default class TestTimingReporter { JSON.stringify({ version: 1, success: reason === "passed" && errors.length === 0, + reason, + unhandledErrors: errors.length, files, }), ); diff --git a/tests/boundary/filesystem/ciChangeScope.test.ts b/tests/boundary/filesystem/ciChangeScope.test.ts index 714d2f980..fa4b71ebf 100644 --- a/tests/boundary/filesystem/ciChangeScope.test.ts +++ b/tests/boundary/filesystem/ciChangeScope.test.ts @@ -178,6 +178,23 @@ it("keeps unknown CI scripts on complete qualification", async () => { for (const flag of CI_FLAGS) expect(plan[flag]).toBe(true); }); +it("emits an observational owner proposal without changing ordinary or full portable execution", async () => { + const { directory, base } = await fixture(); + const head = await change(directory, ["src/evm/Example.ts"]); + expect(await classify(directory, base, head)).toMatchObject({ + code: true, + testSelection: { mode: "candidate", owners: ["evm"] }, + }); + expect( + await classify(directory, base, head, "pull_request", true), + ).toMatchObject({ + code: true, + testSelection: { mode: "full", owners: [] }, + }); + const output = await readFile(join(directory, "scope-output"), "utf8"); + expect(output).toContain('test_selection={"version":1,"mode":"full"'); +}); + it("does not charge a documentation PR for unrelated implementation updates on main", async () => { const { directory, base } = await fixture(); await git(directory, ["switch", "-c", "docs/change"]); diff --git a/tests/boundary/filesystem/ciTestReports.test.ts b/tests/boundary/filesystem/ciTestReports.test.ts index 333e93f08..06212ad07 100644 --- a/tests/boundary/filesystem/ciTestReports.test.ts +++ b/tests/boundary/filesystem/ciTestReports.test.ts @@ -8,6 +8,8 @@ import { expect, it } from "vitest"; import { parse } from "yaml"; import { z } from "zod"; +import { planTestSelection } from "../../../scripts/ci/test-selection.mjs"; + import { parseTestDurations } from "../../ciTestSequencer.js"; import { createTestTempDirectory } from "../../fixtures/temporaryDirectory.js"; import { workflowGit } from "../../support/workflowGit.js"; @@ -61,6 +63,16 @@ it("collects failed-shard diagnostics before enforcing the shard result", async "--reporter=./scripts/ci/test-timing-reporter.mjs", ); expect(steps[upload]?.with?.path).toContain("test-timings.json"); + expect(steps[upload]?.with).toMatchObject({ "include-hidden-files": true }); + expect(steps[upload]?.with?.path).toContain( + ".cache/ci-test-selection/report.json", + ); + const summary = steps.find((step) => + step.run?.includes("scripts/ci/test-report.mjs"), + ); + expect(summary?.env?.CI_TEST_SELECTION).toBe( + "${{ needs.changes.outputs.test_selection }}", + ); const command = steps[gate]?.run; if (command === undefined) throw new Error("Missing shard result gate"); for (const result of ["failure", "cancelled", "skipped", ""]) { @@ -304,6 +316,13 @@ it("recovers complete module costs from real blob reports without executing test ), }) .parse(JSON.parse(await readFile(join(root, "test-timings.json"), "utf8"))); + expect( + JSON.parse(await readFile(join(root, "test-timings.json"), "utf8")), + ).toMatchObject({ + reason: "passed", + unhandledErrors: 0, + files: [{ state: "passed" }], + }); expect(report.files).toHaveLength(1); const file = report.files[0]!; const json = z @@ -349,7 +368,13 @@ it("recovers complete module costs from real blob reports without executing test const failed = JSON.parse( await readFile(join(root, "test-timings.json"), "utf8"), ) as unknown; - expect(failed).toMatchObject({ version: 1, success: false }); + expect(failed).toMatchObject({ + version: 1, + success: false, + reason: "failed", + unhandledErrors: 0, + files: [{ state: "failed" }], + }); // Even a caller accidentally marking the merge successful cannot publish this report. await execute(process.execPath, [fileURLToPath(script)], { cwd: root, @@ -359,3 +384,76 @@ it("recovers complete module costs from real blob reports without executing test readFile(join(root, ".cache/ci-test-history/test-results.json")), ).rejects.toMatchObject({ code: "ENOENT" }); }); + +it("records failed omissions from full reports without treating partial or unhandled outcomes as parity", async () => { + const root = await createTestTempDirectory("rea-ci-selection-report-"); + await workflowGit(root, ["init"]); + const paths = [ + "src/sqlite/provider.test.ts", + "src/evm/provider.test.ts", + "tests/boundary/mcp/workflow.test.ts", + ]; + for (const path of paths) { + await mkdir(join(root, path, ".."), { recursive: true }); + await writeFile(join(root, path), "// fixture\n"); + } + await workflowGit(root, ["add", "src", "tests"]); + const timing = { + version: 1, + success: false, + reason: "failed", + unhandledErrors: 0, + files: paths.map((name, index) => ({ + name, + durationMs: 10, + state: index === 1 ? "failed" : "passed", + })), + }; + const run = async (report: unknown) => { + await writeFile(join(root, "test-timings.json"), JSON.stringify(report)); + await execute(process.execPath, [fileURLToPath(script)], { + cwd: root, + env: { + ...process.env, + SHARD_RESULT: "failure", + MERGE_RESULT: "failure", + CI_VERIFY_TEST_INVENTORY: "true", + CI_TEST_SELECTION: JSON.stringify( + planTestSelection(["src/sqlite/Provider.ts"], "pull_request", "fast"), + ), + GITHUB_SHA: "merge-sha", + CI_PR_HEAD_SHA: "pr-head-sha", + }, + }); + return JSON.parse( + await readFile( + join(root, ".cache/ci-test-selection/report.json"), + "utf8", + ), + ) as unknown; + }; + expect(await run(timing)).toMatchObject({ + observational: true, + complete: true, + selected: [paths[0], paths[2]], + omitted: [paths[1]], + missedFailures: [paths[1]], + executionSha: "merge-sha", + prHeadSha: "pr-head-sha", + }); + for (const report of [ + { ...timing, files: timing.files.slice(0, 1) }, + { ...timing, unhandledErrors: 1 }, + { ...timing, reason: "interrupted" }, + ]) + expect(await run(report)).toMatchObject({ complete: false }); + await expect(run({ ...timing, files: "invalid" })).rejects.toMatchObject({ + code: 1, + }); + await expect( + readFile(join(root, ".cache/ci-test-selection/report.json")), + ).rejects.toMatchObject({ code: "ENOENT" }); + await expect( + readFile(join(root, ".cache/ci-test-history/test-results.json")), + ).rejects.toMatchObject({ code: "ENOENT" }); +}); diff --git a/tests/evaluation/ciTestSelection.test.ts b/tests/evaluation/ciTestSelection.test.ts new file mode 100644 index 000000000..1dff7f9e3 --- /dev/null +++ b/tests/evaluation/ciTestSelection.test.ts @@ -0,0 +1,240 @@ +import { expect, it } from "vitest"; + +import { + compareTestSelection, + planTestSelection, +} from "../../scripts/ci/test-selection.mjs"; + +const sqliteSource = "src/sqlite/SqliteDatabaseProvider.test.ts"; +const sqliteBoundary = "tests/boundary/sqlite/sqliteDatabaseProvider.test.ts"; +const adbSource = "src/adb/AdbProvider.test.ts"; +const adbBoundary = + "tests/boundary/providers/adb/adbBugreportPublication.test.ts"; +const cliBoundary = "tests/boundary/cli/apktoolCli.test.ts"; +const mcpBoundary = "tests/boundary/mcp/apktoolMcp.test.ts"; +const processBoundary = + "tests/boundary/process/processCaptureLifecycle.test.ts"; +const acceptance = "tests/acceptance/tools/localToolContracts.test.ts"; +const composition = "tests/composition/directAnalysisCleanup.test.ts"; +const evaluation = "tests/evaluation/testDiscovery.test.ts"; + +const passed = (name: string) => ({ name, durationMs: 10, state: "passed" }); + +it("unions affected leaf owners, keeps changed tests and retains unowned integration tests", () => { + const plan = planTestSelection( + ["src/sqlite/SqliteDatabaseProvider.ts", adbBoundary], + "pull_request", + "fast", + ); + const inventory = [ + sqliteSource, + sqliteBoundary, + adbSource, + adbBoundary, + cliBoundary, + mcpBoundary, + processBoundary, + acceptance, + composition, + evaluation, + ]; + const report = compareTestSelection(plan, inventory, inventory.map(passed)); + + expect(plan).toMatchObject({ + mode: "candidate", + owners: ["adb", "sqlite"], + }); + expect(report.selected).toEqual( + expect.arrayContaining([ + sqliteSource, + sqliteBoundary, + adbSource, + adbBoundary, + cliBoundary, + mcpBoundary, + processBoundary, + acceptance, + composition, + evaluation, + ]), + ); + expect(report.omitted).toEqual([]); +}); + +it("omits only unrelated owned tests while retaining dynamic boundary suites", () => { + const plan = planTestSelection( + ["src/sqlite/SqliteDatabaseProvider.ts"], + "pull_request", + "fast", + ); + const inventory = [ + sqliteSource, + sqliteBoundary, + adbSource, + adbBoundary, + cliBoundary, + mcpBoundary, + processBoundary, + acceptance, + composition, + evaluation, + ]; + const report = compareTestSelection(plan, inventory, inventory.map(passed)); + + expect(report.selected).toEqual( + expect.arrayContaining([ + sqliteSource, + sqliteBoundary, + cliBoundary, + mcpBoundary, + processBoundary, + acceptance, + composition, + evaluation, + ]), + ); + expect(report.omitted).toEqual([adbSource, adbBoundary]); + expect(report.complete).toBe(true); +}); + +it.each([ + ["unowned shared source", "src/application/Workflow.ts"], + ["fixture input", "tests/conformance/c/fixture.c"], + ["configuration", "vitest.config.ts"], + ["documentation", "README.md"], + ["CLI implementation", "src/cli/apktoolCommands.ts"], + ["MCP implementation", "src/server/registerSqliteTools.ts"], + ["process ownership", "src/process/ProcessOwnership.ts"], + ["acceptance test", acceptance], + ["composition test", composition], +])("uses full fallback for %s changes", (_description, path) => { + expect(planTestSelection([path], "pull_request", "fast")).toMatchObject({ + mode: "full", + owners: [], + changedTests: [], + }); +}); + +it("uses full fallback outside ordinary fast pull requests or without diff paths", () => { + expect( + planTestSelection(["src/sqlite/SqliteDatabaseProvider.ts"], "push", "fast") + .mode, + ).toBe("full"); + expect( + planTestSelection( + ["src/sqlite/SqliteDatabaseProvider.ts"], + "pull_request", + "full", + ).mode, + ).toBe("full"); + expect(planTestSelection([], "pull_request", "fast").mode).toBe("full"); +}); + +it("keeps new changed tests and falls back when a changed test was removed", () => { + const addedTest = "src/sqlite/NewDatabaseRule.test.ts"; + const candidate = planTestSelection([addedTest], "pull_request", "fast"); + expect(candidate.changedTests).toEqual([addedTest]); + const inventory = [addedTest, adbSource]; + expect( + compareTestSelection(candidate, inventory, inventory.map(passed)).selected, + ).toContain(addedTest); + + const removedTest = "src/sqlite/RemovedDatabaseRule.test.ts"; + const report = compareTestSelection( + planTestSelection([removedTest], "pull_request", "fast"), + [sqliteSource, adbSource], + [passed(sqliteSource), passed(adbSource)], + ); + expect(report.plan.mode).toBe("full"); + expect(report.selected).toEqual([adbSource, sqliteSource]); + expect(report.omitted).toEqual([]); +}); + +it("marks incomplete inventories and nonterminal outcomes as incomplete", () => { + const plan = planTestSelection( + ["src/sqlite/SqliteDatabaseProvider.ts"], + "pull_request", + "fast", + ); + const inventory = [sqliteSource, adbSource]; + const completeResults = inventory.map(passed); + + expect( + compareTestSelection(plan, inventory, [passed(sqliteSource)]).complete, + ).toBe(false); + expect( + compareTestSelection(plan, inventory, [ + ...completeResults, + passed("tests/evaluation/unexpected.test.ts"), + ]), + ).toMatchObject({ + complete: false, + unexpected: ["tests/evaluation/unexpected.test.ts"], + }); + expect( + compareTestSelection(plan, inventory, [ + ...completeResults, + passed(adbSource), + ]), + ).toMatchObject({ complete: false, duplicateFiles: 1 }); + for (const state of ["pending", "skipped"]) + expect( + compareTestSelection(plan, inventory, [ + passed(sqliteSource), + { name: adbSource, durationMs: 0, state }, + ]), + ).toMatchObject({ complete: false, unobserved: [adbSource] }); +}); + +it("reports a failure from an omitted test, including collection or module failures", () => { + const plan = planTestSelection( + ["src/sqlite/SqliteDatabaseProvider.ts"], + "pull_request", + "fast", + ); + const report = compareTestSelection( + plan, + [sqliteSource, adbSource], + [passed(sqliteSource), { name: adbSource, durationMs: 1, state: "failed" }], + ); + + expect(report.omitted).toContain(adbSource); + expect(report.missedFailures).toEqual([adbSource]); +}); + +it("rejects plans that name unowned owners or malformed changed tests", () => { + const plan = planTestSelection( + ["src/sqlite/SqliteDatabaseProvider.ts"], + "pull_request", + "fast", + ); + expect(() => + compareTestSelection( + { ...plan, owners: ["unknown"] }, + [sqliteSource], + [passed(sqliteSource)], + ), + ).toThrow("Invalid observational test selection plan"); + expect(() => + compareTestSelection( + { ...plan, changedTests: ["src/sqlite/not-a-test.ts"] }, + [sqliteSource], + [passed(sqliteSource)], + ), + ).toThrow("Invalid observational test selection plan"); +}); + +it("uses full fallback when a changed owner has no executable tests", () => { + const report = compareTestSelection( + planTestSelection( + ["src/sqlite/SqliteDatabaseProvider.ts"], + "pull_request", + "fast", + ), + [adbSource], + [passed(adbSource)], + ); + + expect(report.plan.mode).toBe("full"); + expect(report.omitted).toEqual([]); +}); From 490a285070268aee0ecd50001b9a9a83c1e313f0 Mon Sep 17 00:00:00 2001 From: morluto <76467478+morluto@users.noreply.github.com> Date: Tue, 13 Oct 2026 19:30:50 +0800 Subject: [PATCH 4/5] ci: reuse current-run builds across provider lanes --- .../actions/portable-build-cache/action.yml | 16 ++- .github/workflows/ci.yml | 45 ++++++-- .github/workflows/real-binary-layout.yml | 8 ++ .github/workflows/real-browser.yml | 8 ++ .github/workflows/real-evm-interface.yml | 8 ++ .../workflows/real-javascript-recovery.yml | 8 ++ .github/workflows/real-recorded-crash.yml | 8 ++ .../workflows/real-web-network-captures.yml | 8 ++ .github/workflows/real-web-runtime.yml | 8 ++ .github/workflows/real-web-source-map.yml | 8 ++ docs/testing.md | 10 ++ .../boundary/filesystem/ciChangeScope.test.ts | 4 +- .../filesystem/ciRequiredLanes.test.ts | 109 ++++++++++++++++++ .../boundary/filesystem/testTaskCache.test.ts | 35 +++++- 14 files changed, 266 insertions(+), 17 deletions(-) diff --git a/.github/actions/portable-build-cache/action.yml b/.github/actions/portable-build-cache/action.yml index ad127ce3d..cee775ec5 100644 --- a/.github/actions/portable-build-cache/action.yml +++ b/.github/actions/portable-build-cache/action.yml @@ -1,14 +1,24 @@ name: Portable build cache -description: Reuse Turbo build outputs without serializing verification jobs. +description: Restore portable Turbo build outputs from the current run or cross-run cache. inputs: save: description: Save newly built outputs after the job succeeds. default: "false" + build_cache_artifact: + description: Reuse portable Turbo outputs uploaded by the current workflow run. + default: "" runs: using: composite steps: + - name: Download portable build outputs from this workflow run + if: inputs.build_cache_artifact != '' + uses: actions/download-artifact@v4 + with: + name: ${{ inputs.build_cache_artifact }} + path: .turbo/cache/ - name: Resolve portable build task hashes id: plan + if: inputs.build_cache_artifact == '' shell: bash env: BUILD_CACHE_PREFIX: rea-portable-build-v2-${{ runner.os }}-${{ runner.arch }}-${{ hashFiles('.npmrc', '.nvmrc', 'package-lock.json') }} @@ -28,14 +38,14 @@ runs: console.log(`prefix=${process.env.BUILD_CACHE_PREFIX}-`); ' >> "$GITHUB_OUTPUT" - name: Restore portable build outputs - if: inputs.save != 'true' + if: inputs.build_cache_artifact == '' && inputs.save != 'true' uses: actions/cache/restore@v4 with: path: .turbo/cache/ key: ${{ steps.plan.outputs.key }} restore-keys: ${{ steps.plan.outputs.prefix }} - name: Restore and save portable build outputs - if: inputs.save == 'true' + if: inputs.build_cache_artifact == '' && inputs.save == 'true' uses: actions/cache@v4 with: path: .turbo/cache/ diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 28373f89d..e0516da85 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -190,7 +190,18 @@ jobs: build: name: Build package needs: changes - if: needs.changes.outputs.code == 'true' + # Reusable provider workflows consume this run's portable build artifact, + # even when a workflow-only edit selects the lane without portable tests. + if: >- + needs.changes.outputs.code == 'true' || + needs.changes.outputs.browser == 'true' || + needs.changes.outputs.binary_layout == 'true' || + needs.changes.outputs.evm == 'true' || + needs.changes.outputs.javascript_recovery == 'true' || + needs.changes.outputs.recorded_crash == 'true' || + needs.changes.outputs.web_network == 'true' || + needs.changes.outputs.web_runtime == 'true' || + needs.changes.outputs.web_source_map == 'true' runs-on: ubuntu-latest timeout-minutes: 10 permissions: @@ -673,44 +684,60 @@ jobs: if-no-files-found: error browser: - needs: [changes, static] + needs: [changes, static, build] if: needs.changes.outputs.browser == 'true' uses: ./.github/workflows/real-browser.yml + with: + build_cache_artifact: rea-build-cache binary_layout: - needs: [changes, static] + needs: [changes, static, build] if: needs.changes.outputs.binary_layout == 'true' uses: ./.github/workflows/real-binary-layout.yml + with: + build_cache_artifact: rea-build-cache evm: - needs: [changes, static] + needs: [changes, static, build] if: needs.changes.outputs.evm == 'true' uses: ./.github/workflows/real-evm-interface.yml + with: + build_cache_artifact: rea-build-cache javascript_recovery: - needs: [changes, static] + needs: [changes, static, build] if: needs.changes.outputs.javascript_recovery == 'true' uses: ./.github/workflows/real-javascript-recovery.yml + with: + build_cache_artifact: rea-build-cache recorded_crash: - needs: [changes, static] + needs: [changes, static, build] if: needs.changes.outputs.recorded_crash == 'true' uses: ./.github/workflows/real-recorded-crash.yml + with: + build_cache_artifact: rea-build-cache web_network: - needs: [changes, static] + needs: [changes, static, build] if: needs.changes.outputs.web_network == 'true' uses: ./.github/workflows/real-web-network-captures.yml + with: + build_cache_artifact: rea-build-cache web_runtime: - needs: [changes, static] + needs: [changes, static, build] if: needs.changes.outputs.web_runtime == 'true' uses: ./.github/workflows/real-web-runtime.yml + with: + build_cache_artifact: rea-build-cache web_source_map: - needs: [changes, static] + needs: [changes, static, build] if: needs.changes.outputs.web_source_map == 'true' uses: ./.github/workflows/real-web-source-map.yml + with: + build_cache_artifact: rea-build-cache nativeaot: needs: [changes, static] diff --git a/.github/workflows/real-binary-layout.yml b/.github/workflows/real-binary-layout.yml index 8dd6f9866..5cec952ce 100644 --- a/.github/workflows/real-binary-layout.yml +++ b/.github/workflows/real-binary-layout.yml @@ -8,6 +8,12 @@ on: type: boolean default: false workflow_call: + inputs: + build_cache_artifact: + description: Portable build cache artifact from the calling CI run. + required: false + type: string + default: "" permissions: contents: read @@ -29,6 +35,8 @@ jobs: cache: npm - run: npm ci - uses: ./.github/actions/portable-build-cache + with: + build_cache_artifact: ${{ inputs.build_cache_artifact }} - name: Acquire exact upstream libraries in an isolated runner environment shell: bash run: | diff --git a/.github/workflows/real-browser.yml b/.github/workflows/real-browser.yml index 11a8b7a53..912fdafb1 100644 --- a/.github/workflows/real-browser.yml +++ b/.github/workflows/real-browser.yml @@ -2,6 +2,12 @@ name: Real browser verification on: workflow_call: + inputs: + build_cache_artifact: + description: Portable build cache artifact from the calling CI run. + required: false + type: string + default: "" workflow_dispatch: env: @@ -22,6 +28,8 @@ jobs: cache: npm - run: npm ci - uses: ./.github/actions/portable-build-cache + with: + build_cache_artifact: ${{ inputs.build_cache_artifact }} - run: npm run verify:browser env: REA_BROWSER_EXECUTABLE: /usr/bin/google-chrome diff --git a/.github/workflows/real-evm-interface.yml b/.github/workflows/real-evm-interface.yml index b335f1391..dfdd3970f 100644 --- a/.github/workflows/real-evm-interface.yml +++ b/.github/workflows/real-evm-interface.yml @@ -3,6 +3,12 @@ name: Real offline EVM interface on: workflow_dispatch: workflow_call: + inputs: + build_cache_artifact: + description: Portable build cache artifact from the calling CI run. + required: false + type: string + default: "" permissions: contents: read @@ -23,6 +29,8 @@ jobs: cache: npm - run: npm ci - uses: ./.github/actions/portable-build-cache + with: + build_cache_artifact: ${{ inputs.build_cache_artifact }} - name: Acquire unchanged fixture compiler in isolated runner storage shell: bash run: | diff --git a/.github/workflows/real-javascript-recovery.yml b/.github/workflows/real-javascript-recovery.yml index d94127d63..817dc4dc0 100644 --- a/.github/workflows/real-javascript-recovery.yml +++ b/.github/workflows/real-javascript-recovery.yml @@ -2,6 +2,12 @@ name: Real JavaScript recovery on: workflow_call: + inputs: + build_cache_artifact: + description: Portable build cache artifact from the calling CI run. + required: false + type: string + default: "" workflow_dispatch: permissions: @@ -24,6 +30,8 @@ jobs: cache: npm - run: npm ci - uses: ./.github/actions/portable-build-cache + with: + build_cache_artifact: ${{ inputs.build_cache_artifact }} - name: Fetch pinned Wakaru release into the job workspace shell: bash run: | diff --git a/.github/workflows/real-recorded-crash.yml b/.github/workflows/real-recorded-crash.yml index dc56034c8..04467a608 100644 --- a/.github/workflows/real-recorded-crash.yml +++ b/.github/workflows/real-recorded-crash.yml @@ -3,6 +3,12 @@ name: Real recorded crash evidence on: workflow_dispatch: workflow_call: + inputs: + build_cache_artifact: + description: Portable build cache artifact from the calling CI run. + required: false + type: string + default: "" permissions: contents: read @@ -26,6 +32,8 @@ jobs: cache: npm - run: npm ci - uses: ./.github/actions/portable-build-cache + with: + build_cache_artifact: ${{ inputs.build_cache_artifact }} - name: Install GDB on the disposable CI runner shell: bash run: | diff --git a/.github/workflows/real-web-network-captures.yml b/.github/workflows/real-web-network-captures.yml index df340520d..d41bb37a5 100644 --- a/.github/workflows/real-web-network-captures.yml +++ b/.github/workflows/real-web-network-captures.yml @@ -3,6 +3,12 @@ name: Real historical web network captures on: workflow_dispatch: workflow_call: + inputs: + build_cache_artifact: + description: Portable build cache artifact from the calling CI run. + required: false + type: string + default: "" permissions: contents: read @@ -25,6 +31,8 @@ jobs: run: npm install --global npm@11.16.0 - run: npm ci - uses: ./.github/actions/portable-build-cache + with: + build_cache_artifact: ${{ inputs.build_cache_artifact }} - name: Acquire fixed upstream native profile in runner temporary storage shell: bash run: | diff --git a/.github/workflows/real-web-runtime.yml b/.github/workflows/real-web-runtime.yml index 0ecdb92c6..2575c55f9 100644 --- a/.github/workflows/real-web-runtime.yml +++ b/.github/workflows/real-web-runtime.yml @@ -1,6 +1,12 @@ name: Real website runtime attribution on: workflow_call: + inputs: + build_cache_artifact: + description: Portable build cache artifact from the calling CI run. + required: false + type: string + default: "" workflow_dispatch: permissions: contents: read @@ -18,6 +24,8 @@ jobs: cache: npm - run: npm ci - uses: ./.github/actions/portable-build-cache + with: + build_cache_artifact: ${{ inputs.build_cache_artifact }} - run: npm run verify:browser:runtime env: REA_BROWSER_EXECUTABLE: /usr/bin/google-chrome diff --git a/.github/workflows/real-web-source-map.yml b/.github/workflows/real-web-source-map.yml index b94662129..8e0869eb7 100644 --- a/.github/workflows/real-web-source-map.yml +++ b/.github/workflows/real-web-source-map.yml @@ -1,6 +1,12 @@ name: Real website source-map verification on: workflow_call: + inputs: + build_cache_artifact: + description: Portable build cache artifact from the calling CI run. + required: false + type: string + default: "" workflow_dispatch: permissions: contents: read @@ -18,6 +24,8 @@ jobs: cache: npm - run: npm ci - uses: ./.github/actions/portable-build-cache + with: + build_cache_artifact: ${{ inputs.build_cache_artifact }} - name: Install the pinned compiler into an isolated fixture directory run: npm install --prefix "$RUNNER_TEMP/rea-web-source-map-compiler" --no-audit --no-fund --package-lock=false esbuild@0.25.10 - run: npm run verify:browser:source-maps diff --git a/docs/testing.md b/docs/testing.md index 03121e172..1288a954a 100644 --- a/docs/testing.md +++ b/docs/testing.md @@ -982,6 +982,16 @@ timing data is reused; live provider results are never cached. Turbo outputs are restored by task hash, with trusted default-branch runs as cache writers and PRs as restore-only consumers. npm caches hold downloads, not `node_modules`. +The eight automatic browser, binary-layout, EVM, JavaScript-recovery, +recorded-crash, and web provider lanes download `rea-build-cache` from their own +run after the build and static jobs succeed. Their existing build commands still +validate task hashes and rebuild mismatched inputs; provider observations always +run. Provider-workflow-only edits also admit the build without enabling source +test shards. Direct provider dispatches retain the cross-run restore-only cache. +This removes repeated compilation on fresh source revisions, but adds the build +as a prerequisite: a delayed build can delay provider admission. Compare queue +and execution times before claiming an end-to-end speedup. + These native baseline checks complement the Linux source-test shards and the separate Apple-artifact and real-provider lanes. Actual Hopper, Ghidra, IDA, browser and managed-tool claims require their corresponding verification lanes. diff --git a/tests/boundary/filesystem/ciChangeScope.test.ts b/tests/boundary/filesystem/ciChangeScope.test.ts index fa4b71ebf..163219c39 100644 --- a/tests/boundary/filesystem/ciChangeScope.test.ts +++ b/tests/boundary/filesystem/ciChangeScope.test.ts @@ -680,7 +680,9 @@ it.each([ async (scope, name) => { const workflow = z .object({ - on: z.object({ workflow_call: z.null() }).passthrough(), + on: z + .object({ workflow_call: z.object({}).passthrough().nullable() }) + .passthrough(), }) .parse( parse( diff --git a/tests/boundary/filesystem/ciRequiredLanes.test.ts b/tests/boundary/filesystem/ciRequiredLanes.test.ts index e460bb52b..b6e02b54a 100644 --- a/tests/boundary/filesystem/ciRequiredLanes.test.ts +++ b/tests/boundary/filesystem/ciRequiredLanes.test.ts @@ -168,6 +168,115 @@ it("holds native and provider fan-out behind static validation while portable fe expect(jobs["test-shard"]?.needs).not.toContain("static"); }); +it("binds provider build reuse to the current run while keeping standalone dispatches independent", async () => { + const { jobs } = await workflow(); + const consumers = [ + "browser", + "binary_layout", + "evm", + "javascript_recovery", + "recorded_crash", + "web_network", + "web_runtime", + "web_source_map", + ]; + const buildTerms = z + .string() + .parse(jobs.build?.if) + .split("||") + .map((term) => term.trim()); + // Provider workflow edits select their lane without code=true. The build + // producer must still run, without selecting the portable test shards. + expect(new Set(buildTerms)).toEqual( + new Set( + ["code", ...consumers].map( + (scope) => `needs.changes.outputs.${scope} == 'true'`, + ), + ), + ); + expect(jobs["test-shard"]?.if).toBe("needs.changes.outputs.code == 'true'"); + for (const name of consumers) { + const caller = jobs[name]; + expect(caller?.needs).toContain("build"); + expect(caller?.with).toEqual({ build_cache_artifact: "rea-build-cache" }); + const path = z.string().parse(caller?.uses); + const called = z + .object({ + on: z.object({ + workflow_call: z.object({ + inputs: z.record(z.string(), z.unknown()), + }), + workflow_dispatch: z.unknown(), + }), + jobs: z.record( + z.string(), + z.object({ + steps: z.array( + z.object({ + uses: z.string().optional(), + with: z.record(z.string(), z.unknown()).optional(), + }), + ), + }), + ), + }) + .parse( + parse( + await readFile(new URL(`../../../${path}`, import.meta.url), "utf8"), + ), + ); + expect(called.on.workflow_call.inputs.build_cache_artifact).toMatchObject({ + type: "string", + default: "", + }); + expect(Object.hasOwn(called.on, "workflow_dispatch")).toBe(true); + for (const job of Object.values(called.jobs)) + expect( + job.steps.find( + (step) => step.uses === "./.github/actions/portable-build-cache", + )?.with, + ).toEqual({ build_cache_artifact: "${{ inputs.build_cache_artifact }}" }); + } + const action = z + .object({ + runs: z.object({ + steps: z.array( + z.object({ + id: z.string().optional(), + uses: z.string().optional(), + if: z.string().optional(), + with: z.record(z.string(), z.unknown()).optional(), + "continue-on-error": z.boolean().optional(), + }), + ), + }), + }) + .parse( + parse( + await readFile( + new URL( + "../../../.github/actions/portable-build-cache/action.yml", + import.meta.url, + ), + "utf8", + ), + ), + ); + const download = action.runs.steps.find( + (step) => step.uses === "actions/download-artifact@v4", + ); + expect(download?.with).toEqual({ + name: "${{ inputs.build_cache_artifact }}", + path: ".turbo/cache/", + }); + expect(download?.if).toBe("inputs.build_cache_artifact != ''"); + expect(download?.["continue-on-error"]).not.toBe(true); + for (const step of action.runs.steps.filter( + (step) => step.id === "plan" || step.uses?.startsWith("actions/cache"), + )) + expect(step.if).toContain("inputs.build_cache_artifact == ''"); +}); + it("runs workflow validation inside required static checks when selected", async () => { const { jobs } = await workflow(); const steps = z diff --git a/tests/boundary/filesystem/testTaskCache.test.ts b/tests/boundary/filesystem/testTaskCache.test.ts index 7e06ce1a1..5d877e829 100644 --- a/tests/boundary/filesystem/testTaskCache.test.ts +++ b/tests/boundary/filesystem/testTaskCache.test.ts @@ -1,6 +1,6 @@ import { execFile } from "node:child_process"; -import { readFile, writeFile } from "node:fs/promises"; -import { join } from "node:path"; +import { cp, readFile, writeFile } from "node:fs/promises"; +import { basename, join } from "node:path"; import { promisify } from "node:util"; import { expect, it } from "vitest"; @@ -78,7 +78,7 @@ it.skipIf(process.platform === "android")( await workflowGit(root, ["init", "--initial-branch=main"]); await workflowGit(root, ["add", "."]); await workflowGit(root, ["commit", "-m", "test: seed cache fixture"]); - const run = () => + const run = (cwd = root) => execute( process.execPath, [ @@ -87,7 +87,7 @@ it.skipIf(process.platform === "android")( "test:run", "--output-logs=errors-only", ], - { cwd: root, env: { ...process.env, TURBO_DAEMON: "false" } }, + { cwd, env: { ...process.env, TURBO_DAEMON: "false" } }, ); await run(); await writeFile(join(root, "README_zh.md"), "wrong command"); @@ -104,5 +104,32 @@ it.skipIf(process.platform === "android")( "prepared\n", ); } + // Provider jobs receive the producer's cache in a fresh checkout, not its + // outputs. Turbo must restore identical inputs and rebuild changed ones. + const consumer = await createTestTempDirectory("rea-cache-consumer-"); + await cp(root, consumer, { + recursive: true, + filter: (path) => + basename(path) !== ".turbo" && + !/^prepared-\d+\.txt$/u.test(basename(path)), + }); + await cp(join(root, ".turbo/cache"), join(consumer, ".turbo/cache"), { + recursive: true, + }); + const restored = await run(consumer); + expect(restored.stdout).toContain(`${dependencies.length} cached`); + for (const index of dependencies.keys()) + expect( + await readFile(join(consumer, `prepared-${index}.txt`), "utf8"), + ).toBe("prepared\n"); + await writeFile( + join(consumer, "artifact-input.txt"), + "changed producer input", + ); + await run(consumer); + for (const index of dependencies.keys()) + expect( + await readFile(join(consumer, `prepared-${index}.txt`), "utf8"), + ).toBe("prepared\nprepared\n"); }, ); From fb28ec1537e2d02aaca96325483c849c5c8ade6f Mon Sep 17 00:00:00 2001 From: Kaoru0822 <208281322+kaoru0822-kitauji@users.noreply.github.com> Date: Tue, 13 Oct 2026 04:31:18 -0700 Subject: [PATCH 5/5] test(apple): keep the same-named slice dyld fixture openable on x64 hosts (#2755) Add a plain x86_64 slice to the dyld fixture while retaining both same-named arm64e slices and their independent environment assertions. The original fixture was rejected by real x64 CLI admission before tracing. The contributing PR's complete Linux CI run passes all 30 jobs, including the portable suite. Codex and Greptile reviews completed without findings; no unresolved threads remain. This restores the main test baseline needed for #2635. The change is limited to test data and preserves the original author's contribution. Co-authored-by: Kaoru0822 <208281322+kaoru0822-kitauji@users.noreply.github.com> --- tests/boundary/cli/dylibResolutionCli.test.ts | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/tests/boundary/cli/dylibResolutionCli.test.ts b/tests/boundary/cli/dylibResolutionCli.test.ts index f6482a2f3..66fa21ba3 100644 --- a/tests/boundary/cli/dylibResolutionCli.test.ts +++ b/tests/boundary/cli/dylibResolutionCli.test.ts @@ -477,6 +477,7 @@ cliTest( "reports each same-named slice's own embedded dyld environment", async ({ cli }) => { // arm64e and arm64e.x1 slices both report the architecture name arm64e. + // A plain x86_64 slice keeps the bundle openable on x64 hosts. for (const [settings, expected] of [ [[null, "DYLD_PRINT_RPATHS=1"], ["DYLD_PRINT_RPATHS=1"]], [ @@ -486,8 +487,8 @@ cliTest( ] as const) { const app = await fixtureApp(); await writeFiles(app, { - "Contents/MacOS/App": fatImage( - [CPU.arm64e, CPU.arm64eX1].map((cpu, index) => { + "Contents/MacOS/App": fatImage([ + ...[CPU.arm64e, CPU.arm64eX1].map((cpu, index) => { const setting = settings[index]; return { cpu, @@ -500,7 +501,8 @@ cliTest( }), }; }), - ), + { cpu: CPU.x86_64, bytes: machoImage({ cpu: CPU.x86_64 }) }, + ]), }); const result = await cli.run({ arguments: [ @@ -516,7 +518,9 @@ cliTest( const trace = dylibResolutionResultSchema.parse( parseEvidence(result.json).normalized_result, ); - expect(trace.roots).toHaveLength(2); + expect( + trace.roots.filter(({ architecture }) => architecture === "arm64e"), + ).toHaveLength(2); const environments = trace.findings .filter(({ kind }) => kind === "dyld-environment-present") .map(({ explanation }) =>