diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 6e6521b41..fe79048ba 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -5,16 +5,25 @@ on: branches: [main] pull_request: branches: [main, "release/*"] + schedule: + - cron: "23 * * * *" workflow_dispatch: + workflow_call: + inputs: + source_sha: + description: Exact release source; must match the caller revision. + required: true + type: string permissions: contents: read concurrency: - group: ci-${{ github.event.pull_request.number || github.ref }} - # The latest main push verifies the full tree, including earlier changes. - # Publication uses its own release workflow and concurrency group. - cancel-in-progress: true + # Complete qualification finishes even while main receives more pushes. + # Scheduled/manual main runs coalesce to one active and the latest pending; + # release qualification has its own exact-revision group. + group: ci-${{ (github.event_name == 'schedule' || github.event_name == 'workflow_dispatch' || inputs.source_sha != '' || contains(github.event.pull_request.labels.*.name, 'ci:full')) && 'qualification' || 'feedback' }}-${{ inputs.source_sha || github.event.pull_request.number || github.ref }} + cancel-in-progress: ${{ github.event_name != 'schedule' && github.event_name != 'workflow_dispatch' && inputs.source_sha == '' && !contains(github.event.pull_request.labels.*.name, 'ci:full') }} env: HUSKY: "0" @@ -22,7 +31,7 @@ env: jobs: pr-title: name: Conventional PR title - if: github.event_name == 'pull_request' + if: github.event_name == 'pull_request' && inputs.source_sha == '' runs-on: ubuntu-latest steps: - name: Validate title for Release Please @@ -39,7 +48,11 @@ jobs: changes: name: Classify changes runs-on: ubuntu-latest + permissions: + contents: read + actions: read outputs: + tier: ${{ steps.plan.outputs.tier }} code: ${{ steps.plan.outputs.code }} runtime: ${{ steps.plan.outputs.runtime }} package: ${{ steps.plan.outputs.package }} @@ -62,16 +75,34 @@ jobs: steps: - uses: actions/checkout@v4 with: + ref: ${{ inputs.source_sha || github.sha }} fetch-depth: 0 - uses: actions/setup-node@v4 with: node-version-file: .nvmrc + - name: Bind release qualification to the caller revision + if: inputs.source_sha != '' + env: + SOURCE_SHA: ${{ inputs.source_sha }} + run: | + test "${SOURCE_SHA}" = "${GITHUB_SHA}" + test "${SOURCE_SHA}" = "$(git rev-parse HEAD)" + - name: Find a verified ancestor for push selection + id: baseline + if: github.event_name == 'push' && inputs.source_sha == '' + env: + GH_TOKEN: ${{ github.token }} + HEAD_SHA: ${{ github.sha }} + CI_REPOSITORY: ${{ github.repository }} + CI_DEFAULT_BRANCH: ${{ github.event.repository.default_branch }} + run: node scripts/ci/baseline.mjs - name: Select verification lanes id: plan env: - BASE_SHA: ${{ github.event.pull_request.base.sha }} - HEAD_SHA: ${{ github.event.pull_request.head.sha }} - CI_FULL: ${{ contains(github.event.pull_request.labels.*.name, 'ci:full') }} + BASE_SHA: ${{ steps.baseline.outputs.base || github.event.pull_request.base.sha }} + BASELINE_REASON: ${{ steps.baseline.outputs.reason }} + HEAD_SHA: ${{ inputs.source_sha || github.event.pull_request.head.sha || github.sha }} + CI_FULL: ${{ inputs.source_sha != '' || contains(github.event.pull_request.labels.*.name, 'ci:full') }} run: node scripts/ci/plan.mjs static: @@ -129,7 +160,14 @@ jobs: with: name: rea-test-runtime path: . - - run: npm run test:ci:shard:run -- --shard=${{ matrix.shard }} + - name: Run complete portable suite with coverage + if: needs.changes.outputs.tier == 'full' + run: npm run test:ci:shard:run -- --shard=${{ matrix.shard }} + env: + REA_CI_TEST_HISTORY: .cache/ci-test-history/test-results.json + - name: Run complete portable suite for fast feedback + if: needs.changes.outputs.tier == 'fast' + run: npm run test:run -- --reporter=default --reporter=blob --shard=${{ matrix.shard }} env: REA_CI_TEST_HISTORY: .cache/ci-test-history/test-results.json - uses: actions/upload-artifact@v4 @@ -160,7 +198,7 @@ jobs: - name: Reuse portable build outputs across runs uses: ./.github/actions/portable-build-cache with: - save: "true" + save: ${{ github.event_name != 'pull_request' && github.ref == format('refs/heads/{0}', github.event.repository.default_branch) }} - run: npm run test:prepare - name: Find successful default-branch timing history id: history @@ -172,7 +210,7 @@ jobs: run: | run_id="$(gh api --method GET \ "repos/${CI_REPOSITORY}/actions/workflows/ci.yml/runs" \ - -f branch="${CI_DEFAULT_BRANCH}" -f event=push -f status=success -f per_page=1 \ + -f branch="${CI_DEFAULT_BRANCH}" -f event=schedule -f status=success -f per_page=1 \ --jq '.workflow_runs[0].id // empty')" echo "run-id=${run_id}" >> "${GITHUB_OUTPUT}" - name: Download previous timing report @@ -215,7 +253,7 @@ jobs: uses: ./.github/workflows/sqlite-runtime.yml test: - name: Test (coverage + timing) + name: Portable tests needs: [changes, test-shard] if: ${{ !cancelled() }} runs-on: ubuntu-latest @@ -241,7 +279,7 @@ jobs: merge-multiple: true - name: Merge coverage, JUnit, and timing reports id: merge - if: needs.changes.outputs.code == 'true' && success() + if: needs.changes.outputs.code == 'true' && needs.changes.outputs.tier == 'full' && success() run: >- npm run test:ci:merge -- --coverage.reportsDirectory=coverage @@ -253,11 +291,22 @@ jobs: --reporter=json --outputFile.junit=test-results.xml --outputFile.json=test-results.json + - name: Merge fast-suite JUnit and timing reports + id: merge-fast + if: needs.changes.outputs.code == 'true' && needs.changes.outputs.tier == 'fast' && success() + run: >- + npm exec -- vitest --merge-reports + --reporter=default + --reporter=junit + --reporter=json + --outputFile.junit=test-results.xml + --outputFile.json=test-results.json - name: Summarize slowest test files if: needs.changes.outputs.code == 'true' && !cancelled() && hashFiles('test-results.json') != '' env: SHARD_RESULT: ${{ needs.test-shard.result }} - MERGE_RESULT: ${{ steps.merge.outcome }} + MERGE_RESULT: ${{ needs.changes.outputs.tier == 'full' && steps.merge.outcome || steps.merge-fast.outcome }} + CI_VERIFY_TEST_INVENTORY: "true" run: node scripts/ci/test-report.mjs - uses: actions/upload-artifact@v4 if: needs.changes.outputs.code == 'true' && !cancelled() && hashFiles('test-results.json') != '' @@ -270,7 +319,7 @@ jobs: if-no-files-found: error retention-days: 30 - name: Publish successful timing history - if: needs.changes.outputs.code == 'true' && success() && needs.test-shard.result == 'success' + if: needs.changes.outputs.code == 'true' && needs.changes.outputs.tier == 'full' && success() && needs.test-shard.result == 'success' uses: actions/upload-artifact@v4 with: name: test-timings @@ -381,9 +430,20 @@ jobs: - run: npm run verify:package env: npm_config_cache: ${{ steps.npm-cache.outputs.path }} + REA_VERIFIED_PACKAGE_DIRECTORY: ${{ needs.changes.outputs.tier == 'full' && matrix.platform == 'linux' && matrix.architecture == 'x64' && '.cache/verified-package' || '' }} - name: Verify real Node Inspector discovery and observation timeout-minutes: 10 run: npm run verify:inspector + - name: Retain the exact qualified release package + if: needs.changes.outputs.tier == 'full' && matrix.platform == 'linux' && matrix.architecture == 'x64' + uses: actions/upload-artifact@v4 + with: + name: verified-release-package + path: .cache/verified-package/ + include-hidden-files: true + if-no-files-found: error + overwrite: true + retention-days: 7 apple-artifacts: name: Real Apple artifact CLI/MCP E2E @@ -657,7 +717,7 @@ jobs: required: name: CI required - if: ${{ !cancelled() }} + if: ${{ always() }} needs: [ changes, @@ -687,6 +747,7 @@ jobs: website, ] runs-on: ubuntu-latest + timeout-minutes: 5 steps: - uses: actions/checkout@v4 - uses: actions/setup-node@v4 diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index c60f32a2c..7cc798cfa 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -290,13 +290,25 @@ jobs: if: inputs.phase == 'prepare' && steps.release.outputs.prs_created == 'true' run: npm run docs:check && git diff --exit-code - publish: - name: Publish to npm + qualification: + name: Qualify exact release revision needs: [merged-release, release-please] if: >- always() && !cancelled() && ((needs.merged-release.result == 'success' && needs.merged-release.outputs.release_created == 'true') || (needs.release-please.result == 'success' && inputs.phase == 'publish' && needs.release-please.outputs.release_created == 'true')) + permissions: + contents: read + actions: read + uses: ./.github/workflows/ci.yml + with: + source_sha: ${{ needs.merged-release.outputs.sha || needs.release-please.outputs.sha }} + + publish: + name: Publish to npm + needs: [merged-release, release-please, qualification] + if: >- + always() && !cancelled() && needs.qualification.result == 'success' runs-on: ubuntu-latest permissions: contents: read @@ -315,17 +327,20 @@ jobs: - run: npm ci - - name: Prepare Windows native build toolchain - run: sudo apt-get update && sudo apt-get install --no-install-recommends -y g++-mingw-w64-x86-64 + - name: Download the exact qualified package + uses: actions/download-artifact@v4 + with: + name: verified-release-package + path: .cache/verified-package/ - - name: Build and verify bundled Windows native controls - run: npm run build:windows-native && npm run verify:windows-native:artifact - - - name: Verify packaged release before publishing - run: npm run verify:package + - name: Verify qualified package provenance and digest + id: package + run: node scripts/ci/verified-package.mjs validate .cache/verified-package - name: Publish shell: bash + env: + VERIFIED_PACKAGE_TARBALL: ${{ steps.package.outputs.tarball }} run: | version="$(node --input-type=module -e 'import packageJson from "./package.json" with { type: "json" }; process.stdout.write(packageJson.version)')" published_version="$(npm view "rea-agents@${version}" version --json 2>/dev/null || true)" @@ -333,7 +348,7 @@ jobs: echo "rea-agents@${version} is already published; verifying the existing package." else tag="$(node scripts/release-npm-tag.mjs "${version}")" - npm publish --access public --tag "${tag}" + npm publish "${VERIFIED_PACKAGE_TARBALL}" --access public --tag "${tag}" fi - name: Verify published CLI from npm diff --git a/docs/releasing.md b/docs/releasing.md index dd3a98be3..c532d0663 100644 --- a/docs/releasing.md +++ b/docs/releasing.md @@ -6,7 +6,10 @@ checks to pass. Merging that PR into main starts publication automatically. The Release workflow validates the merged release metadata and ancestry before creating a tag. npm and MCP Registry publication use the exact reviewed merge -SHA returned by Release Please. Later main commits cannot change that source. +SHA returned by Release Please. Before either registry is published, the release +run calls full CI qualification at that exact revision, including coverage, +portable providers, and every native package host. Later main commits cannot +change or cancel that proof. Ordinary main pushes and closed, unmerged PRs cannot publish packages. Check both publication jobs and the published-package canary; a merged PR or GitHub tag alone does not establish that publication finished. Retry failed jobs in @@ -196,8 +199,14 @@ both registry publishes. See [npm's provenance implementation](https://github.com/npm/cli/blob/v11.16.0/workspaces/libnpmpublish/lib/provenance.js) for the use of GitHub's workflow ref and commit SHA. -The workflow builds the bundled Windows controls and verifies the packaged -artifact before npm publication. It then verifies the published CLI, the +Full qualification builds the bundled Windows controls and verifies the packaged +artifact on its native hosts. The Linux x64 verifier retains the exact tarball +it installed and tested, after its workspace cleanup completes. The npm job +downloads that archive from the same workflow run and checks its source SHA, +run identity/attempt, package and Node versions, host identity, and SHA-256. +It publishes that archive without rebuilding or repacking it. Re-running failed +publisher jobs can reuse a prior successful attempt from the same run. +The workflow then verifies the published CLI, the capability-scoped MCP catalog, and the isolated package update path before publishing MCP Registry metadata. diff --git a/docs/testing.md b/docs/testing.md index a36eff6ec..b1d0031f6 100644 --- a/docs/testing.md +++ b/docs/testing.md @@ -263,12 +263,13 @@ CI owns coverage; aggregate and domain/contract thresholds are maintained in Coverage does not replace named boundary or real-provider scenarios. CI runs six native Vitest shards without retries. Each shard emits a blob -report; the merge job produces aggregate coverage plus JUnit and JSON timing +report; the merge job produces JUnit and JSON timing reports, plus aggregate +coverage during full qualification. It retains available reports even when a shard fails, uploads available diagnostics, and writes the slowest files to the workflow summary. Partial reports are marked diagnostic only. Failed, cancelled, or missing selected shards still fail the required gate, -and aggregate coverage thresholds remain enforced by the merge command. -Only successful complete merges publish the small `test-timings` artifact, with +and aggregate coverage thresholds remain enforced in full qualification. +Only successful complete full-tier merges publish the small `test-timings` artifact, with checkout-relative paths; the build job downloads that artifact for the sequencer instead of the coverage/JUnit bundle. Missing history falls back to native sharding. Static checks, documentation, build, package verification, and @@ -717,7 +718,7 @@ or the GitHub issue and pull-request templates run formatting and generated-document validation, including the small onboarding example consistency suite, without the source-test shards or native package lanes. Classification compares the PR head with its merge base, so later -base-branch changes do not expand that scope. Static and coverage aggregate jobs +base-branch changes do not expand that scope. Static and portable-test aggregate jobs remain present and fail if classification fails. Every code change retains the complete six-shard Linux suite; individual test files are not selected by imports or filenames. @@ -746,38 +747,58 @@ verification stays manual. Renames account for both paths. This narrower selection trades PR-time coverage for runner capacity. A change to unowned shared runtime code that fails only on arm64, macOS, Windows, the Apple artifact lane, or a provider lane that does not own the file is not caught on -the PR; the main-push full baseline reports it after merge. Ownership globs do +the PR; scheduled full qualification reports it after merge. Ownership globs do not follow imports. Use the `ci:full` label when a change may have native-host or provider effects outside its owners. -Main pushes and manual dispatch retain the full baseline, including all portable -provider lanes and the native matrix below. A `ci:full` PR label selects the full -baseline on the next PR run; manual dispatch can request it immediately. Each +Ordinary main pushes use the same owner scopes as PRs. Their diff starts at a +successful ancestor push run of this CI workflow, rather than the preceding +push: changes from failed or cancelled intermediate runs remain in scope. +Missing, untrusted, or non-ancestor history selects every lane. The lookup is +bounded to the last 100 successful push runs; lookup failures broaden selection. +Every code change still runs all six portable shards, without coverage +instrumentation in this fast tier. This deliberately avoids relying on an +import graph for generated files, filesystem inputs, subprocesses, or bridges. + +Full qualification runs hourly at minute 23, on manual dispatch, and for the +exact release revision before publication. It includes aggregate coverage, +every portable provider lane, and all native hosts below. A `ci:full` PR label +selects full qualification on the next PR run; manual dispatch can request it +immediately. Fast-tier broad fallbacks retain all owner lanes and the three-host +package matrix; Intel macOS and coverage remain full-tier obligations. Each classification job records its selection and broad-fallback reasons in the run summary. Representative Git-diff cases live in `tests/boundary/filesystem/ciChangeScope.test.ts`. -`CI required` checks classification and every selected lane, rejecting failed, -cancelled, or unexpectedly skipped work. Existing required check names are -preserved; changing branch protection to require the new aggregate is a separate -repository-settings operation. Newer commits cancel superseded CI for the same -PR or main branch. Release publication has its own concurrency group and retains -its package verification and public-registry canary. Licensed/self-hosted +`CI required` always runs and checks classification and every selected lane, +rejecting failed, cancelled, or unexpectedly skipped work. Branch protection +should require this aggregate, `Static checks`, and `Portable tests`. Fast runs +cancel superseded feedback for the same PR or main branch. Scheduled/manual +qualification has a separate concurrency group: active work finishes and only +the latest pending run is retained. Release qualification is grouped by its +exact source SHA and cannot be cancelled by newer main pushes. Publication +requires that qualification to succeed and consumes its exact verified tarball, +checking source SHA, workflow run, run attempt, package/Node version and SHA-256 +before npm publication. Earlier attempts from the same workflow run are accepted +when rerunning failed publisher jobs; future attempts and other runs are rejected. +The public-registry canary still verifies the published package. Licensed/self-hosted provider workflows and the Android smoke test retain their explicit triggers. CI exercises the pinned Node.js runtime on native hosted runners: -| Host | Runner | Baseline checks | -| --------------------- | ------------------ | ---------------------------------------------------------------------------------------- | -| Linux x64 | `ubuntu-latest` | Installed package and real Node Inspector CLI/MCP | -| Linux arm64 (aarch64) | `ubuntu-24.04-arm` | Installed package and real Node Inspector CLI/MCP | -| macOS 15 arm64 | `macos-15` | Installed package and real Node Inspector CLI/MCP | -| macOS 15 x64 | `macos-15-intel` | Installed package and real Node Inspector CLI/MCP (main, dispatch and `ci:full` only) | -| Windows x64 | `windows-latest` | Curated capabilities, native controls, installed package and real Node Inspector CLI/MCP | +| Host | Runner | Baseline checks | +| --------------------- | ------------------ | ------------------------------------------------------------------------------------------- | +| Linux x64 | `ubuntu-latest` | Installed package and real Node Inspector CLI/MCP | +| Linux arm64 (aarch64) | `ubuntu-24.04-arm` | Installed package and real Node Inspector CLI/MCP | +| macOS 15 arm64 | `macos-15` | Installed package and real Node Inspector CLI/MCP | +| macOS 15 x64 | `macos-15-intel` | Installed package and real Node Inspector CLI/MCP (hourly, dispatch, release and `ci:full`) | +| Windows x64 | `windows-latest` | Curated capabilities, native controls, installed package and real Node Inspector CLI/MCP | -macOS x64 is the slowest package host. Pull requests without the `ci:full` label -omit it, so darwin x64 installed-package and Inspector regressions surface on -the main-push full baseline rather than on the PR. +macOS x64 is the slowest package host. Ordinary PRs and main pushes omit it, +so darwin x64 installed-package and Inspector regressions surface in full +qualification. Schedule admission can be delayed by GitHub; the hourly trigger +does not guarantee an hourly completed result. Release publication waits for +its own complete result at the exact released revision. Package lanes assert the actual Node platform/architecture before verification and record those values with the Node version. The package matrix runs up to @@ -803,17 +824,21 @@ The Linux source-test shards download a shared runtime and generated test metadata snapshot produced by the build job with `npm run test:prepare`. The snapshot includes `dist/`, generated skills, the MCP tool catalog, product catalog, and managed-conformance metadata. Each shard still installs the locked -dependencies and runs its assigned coverage-enabled tests; it no longer +dependencies and runs its assigned tests; full qualification enables coverage. It no longer regenerates the same metadata independently. The build job also retrieves the merged timing report from the last successful -default-branch push and includes that same report in each shard's snapshot. +default-branch scheduled qualification and includes that same report in each shard's snapshot. CI assigns discovered files by historical duration, with a deterministic fallback weight for new files, rather than dividing only by file count. Every -discovered file still belongs to exactly one shard; worker limits, project -ordering, coverage instrumentation, and aggregate thresholds remain unchanged. +discovered file still belongs to exactly one shard; worker limits and project +ordering remain unchanged. The merge job rejects successful reports with missing, +unexpected, or duplicate portable files. Coverage instrumentation and aggregate +thresholds remain enforced in full qualification. Missing or unusable history falls back to Vitest's default assignment. Only -timing data is reused; persistent compile caches remain disabled. +timing data is reused; live provider results are never cached. Turbo outputs are +restored by task hash, with trusted default-branch runs as cache writers and PRs +as restore-only consumers. npm caches hold downloads, not `node_modules`. These native baseline checks complement the Linux source-test shards and the separate Apple-artifact and real-provider lanes. Actual Hopper, Ghidra, IDA, diff --git a/scripts/ci/baseline.mjs b/scripts/ci/baseline.mjs new file mode 100644 index 000000000..171bd7dd3 --- /dev/null +++ b/scripts/ci/baseline.mjs @@ -0,0 +1,76 @@ +import { execFileSync } from "node:child_process"; +import { appendFileSync } from "node:fs"; + +const repository = process.env.CI_REPOSITORY; +const branch = process.env.CI_DEFAULT_BRANCH; +const head = process.env.HEAD_SHA; +const shaPattern = /^[0-9a-f]{40}$/i; + +function ancestor(base) { + try { + execFileSync("git", ["merge-base", "--is-ancestor", base, head], { + stdio: "ignore", + }); + return true; + } catch { + return false; + } +} + +function write(base, reason) { + const output = { base, reason }; + if (process.env.GITHUB_OUTPUT) { + appendFileSync( + process.env.GITHUB_OUTPUT, + `base=${base}\nreason=${reason.replace(/[\r\n]/g, " ")}\n`, + ); + } + console.log(JSON.stringify(output)); +} + +if (!repository || !branch || !head || !shaPattern.test(head)) { + write("", "Repository, default branch, or checked-out head is invalid"); +} else { + try { + const response = execFileSync( + "gh", + [ + "api", + "--method", + "GET", + `repos/${repository}/actions/workflows/ci.yml/runs`, + "-f", + `branch=${branch}`, + "-f", + "event=push", + "-f", + "status=success", + "-F", + "per_page=100", + ], + { encoding: "utf8", timeout: 15_000, maxBuffer: 4 * 1024 * 1024 }, + ); + const runs = JSON.parse(response).workflow_runs; + if (!Array.isArray(runs)) + throw new Error("GitHub returned no workflow run list"); + const base = runs.find( + (run) => + run?.event === "push" && + run?.conclusion === "success" && + run?.head_branch === branch && + run?.head_repository?.full_name === repository && + typeof run?.head_sha === "string" && + shaPattern.test(run.head_sha) && + run.head_sha.toLowerCase() !== head.toLowerCase() && + ancestor(run.head_sha), + )?.head_sha; + if (base) write(base, ""); + else + write( + "", + "No trusted successful CI push baseline is an ancestor of this head", + ); + } catch { + write("", "Could not read a successful CI push baseline from GitHub"); + } +} diff --git a/scripts/ci/plan.mjs b/scripts/ci/plan.mjs index 8b99efce5..7ffec2c85 100644 --- a/scripts/ci/plan.mjs +++ b/scripts/ci/plan.mjs @@ -108,98 +108,123 @@ function classifyPackage(base, head) { } const event = process.env.GITHUB_EVENT_NAME; -let files = []; -if ( - event === "push" || +if (!["pull_request", "push", "schedule", "workflow_dispatch"].includes(event)) + throw new Error(`Unsupported CI event: ${event}`); +const tier = event === "schedule" || event === "workflow_dispatch" || process.env.CI_FULL === "true" -) { + ? "full" + : "fast"; +let files = []; +if (tier === "full") { full("Full baseline requested by event or ci:full label"); -} else if (event === "pull_request") { - const base = git("merge-base", process.env.BASE_SHA, process.env.HEAD_SHA); +} else if (event === "pull_request" || event === "push") { const head = process.env.HEAD_SHA; - // Disable rename detection so both the removed and added paths participate. - files = git("diff", "--name-only", "--no-renames", "-z", base, head) - .split("\0") - .filter(Boolean); - for (const file of files) { - if ( - /^(README(?:_[^/]+)?\.md|AGENTS\.md|CONTRIBUTING\.md|CHANGELOG\.md|SECURITY\.md|native\/windows\/README\.md|\.github\/pull_request_template\.md)$/.test( - file, - ) || - file.startsWith("docs/") || - file.startsWith(".github/ISSUE_TEMPLATE/") - ) { - selected.docs = true; - selected.website ||= matchesAny(file, SCOPES.website); - continue; - } - if (/^(src|tests)\/.*\.(?:test|spec)\.[cm]?[jt]sx?$/.test(file)) { - selected.code = true; - for (const owner of NATIVE_TEST_OWNERS) - selected[owner] ||= matchesAny(file, SCOPES[owner]); - continue; - } - if (matchesAny(file, MANUAL_VERIFIERS)) { - selected.code = true; - continue; - } - const owners = Object.entries(SCOPES) - .filter(([, patterns]) => - patterns.some((pattern) => matchesGlob(file, pattern)), - ) - .map(([owner]) => owner); - for (const owner of owners) selected[owner] = true; - if ( - file.startsWith(".github/workflows/") || - file === ".github/actionlint.yaml" - ) { - selected.workflows = true; - if (GUARDED_WORKFLOWS.has(file)) selected.code = true; - if (file !== ".github/workflows/ci.yml") continue; - } - if (owners.length === 1 && owners[0] === "website") continue; - if (/^src\/(contracts|domain)\//.test(file)) selected.docs = true; - if (file === "package.json") { - classifyPackage(base, head); - continue; - } - selected.code = true; - if (matchesAny(file, FIXTURE_SOURCES)) { - selected.fixtures = true; - selected.readiness = true; - } - if (matchesAny(file, READINESS_SOURCES)) selected.readiness = true; - if (file.startsWith("tests/")) continue; - if ( - /^(src\/process\/|src\/filesystem\/|src\/contracts\/tool|src\/server\/createServer|src\/cli\.ts|scripts\/ci\/)/.test( - file, - ) - ) { - full(`Shared boundary: ${file}`); - } else if (owners.length) { - if (file.startsWith("src/") || file.startsWith("bridge/")) - selected.runtime = true; - } else if (/^(src|bridge)\//.test(file)) { - // Unowned runtime sources keep the Linux x64 installed-package baseline; - // ci:full, main pushes and manual dispatch retain every native host. - selected.runtime = true; - reasons.push(`Unowned runtime source: ${file}`); - } else if (file.startsWith(".agents/skills/")) { - selected.runtime = true; - selected.docs = true; - } else { - full(`Unclassified input: ${file}`); + let base; + if (event === "pull_request") { + base = git("merge-base", process.env.BASE_SHA, head); + } else { + const candidate = process.env.BASE_SHA; + const baselineReason = process.env.BASELINE_REASON; + try { + if (!candidate || !/^[0-9a-f]{40}$/i.test(candidate)) + throw new Error( + baselineReason || "No valid successful push baseline was found", + ); + execFileSync("git", ["merge-base", "--is-ancestor", candidate, head], { + stdio: "ignore", + }); + base = candidate; + } catch { + full( + baselineReason || + "No valid successful push baseline ancestor; selecting all lanes", + ); } } -} else throw new Error(`Unsupported CI event: ${event}`); + if (base) { + // Disable rename detection so both the removed and added paths participate. + files = git("diff", "--name-only", "--no-renames", "-z", base, head) + .split("\0") + .filter(Boolean); + for (const file of files) { + if ( + /^(README(?:_[^/]+)?\.md|AGENTS\.md|CONTRIBUTING\.md|CHANGELOG\.md|SECURITY\.md|native\/windows\/README\.md|\.github\/pull_request_template\.md)$/.test( + file, + ) || + file.startsWith("docs/") || + file.startsWith(".github/ISSUE_TEMPLATE/") + ) { + selected.docs = true; + selected.website ||= matchesAny(file, SCOPES.website); + continue; + } + if (/^(src|tests)\/.*\.(?:test|spec)\.[cm]?[jt]sx?$/.test(file)) { + selected.code = true; + for (const owner of NATIVE_TEST_OWNERS) + selected[owner] ||= matchesAny(file, SCOPES[owner]); + continue; + } + if (matchesAny(file, MANUAL_VERIFIERS)) { + selected.code = true; + continue; + } + const owners = Object.entries(SCOPES) + .filter(([, patterns]) => + patterns.some((pattern) => matchesGlob(file, pattern)), + ) + .map(([owner]) => owner); + for (const owner of owners) selected[owner] = true; + if ( + file.startsWith(".github/workflows/") || + file === ".github/actionlint.yaml" + ) { + selected.workflows = true; + if (GUARDED_WORKFLOWS.has(file)) selected.code = true; + if (file !== ".github/workflows/ci.yml") continue; + } + if (owners.length === 1 && owners[0] === "website") continue; + if (/^src\/(contracts|domain)\//.test(file)) selected.docs = true; + if (file === "package.json") { + classifyPackage(base, head); + continue; + } + selected.code = true; + if (matchesAny(file, FIXTURE_SOURCES)) { + selected.fixtures = true; + selected.readiness = true; + } + if (matchesAny(file, READINESS_SOURCES)) selected.readiness = true; + if (file.startsWith("tests/")) continue; + if ( + /^(src\/process\/|src\/filesystem\/|src\/contracts\/tool|src\/server\/createServer|src\/cli\.ts|scripts\/ci\/)/.test( + file, + ) + ) { + full(`Shared boundary: ${file}`); + } else if (owners.length) { + if (file.startsWith("src/") || file.startsWith("bridge/")) + selected.runtime = true; + } else if (/^(src|bridge)\//.test(file)) { + // Unowned runtime sources keep the Linux x64 installed-package baseline; + // full qualification retains every native host. + selected.runtime = true; + reasons.push(`Unowned runtime source: ${file}`); + } else if (file.startsWith(".agents/skills/")) { + selected.runtime = true; + selected.docs = true; + } else { + full(`Unclassified input: ${file}`); + } + } + } +} selected.runtime ||= selected.package; const linux = { os: "ubuntu-latest", platform: "linux", architecture: "x64" }; -// macOS x64 is the slowest package host. Pull requests run it only with ci:full; -// main pushes, schedules and manual dispatch always include it. +// macOS x64 is the slowest package host. Fast qualification omits it. const intel = - event !== "pull_request" || process.env.CI_FULL === "true" + tier === "full" ? [{ os: "macos-15-intel", platform: "darwin", architecture: "x64" }] : []; const matrix = { @@ -212,17 +237,17 @@ const matrix = { ] : [linux], }; -const plan = { ...selected, matrix, files, reasons }; +const plan = { ...selected, tier, matrix, files, reasons }; if (process.env.GITHUB_OUTPUT) appendFileSync( process.env.GITHUB_OUTPUT, - Object.entries({ ...selected, matrix: JSON.stringify(matrix) }) + Object.entries({ ...selected, tier, matrix: JSON.stringify(matrix) }) .map(([key, value]) => `${key}=${value}\n`) .join(""), ); if (process.env.GITHUB_STEP_SUMMARY) appendFileSync( process.env.GITHUB_STEP_SUMMARY, - `### Selected CI lanes\n\n${flags.filter((flag) => selected[flag]).join(", ") || "Formatting only"}\n\n${reasons.join("\n\n")}\n`, + `### Selected CI lanes (${tier})\n\n${flags.filter((flag) => selected[flag]).join(", ") || "Formatting only"}\n\n${reasons.join("\n\n")}\n`, ); console.log(JSON.stringify(plan)); diff --git a/scripts/ci/test-report.mjs b/scripts/ci/test-report.mjs index a3686d6ed..9f2d979aa 100644 --- a/scripts/ci/test-report.mjs +++ b/scripts/ci/test-report.mjs @@ -1,3 +1,4 @@ +import { execFileSync } from "node:child_process"; import { appendFile, mkdir, readFile, rm, writeFile } from "node:fs/promises"; import { isAbsolute, join, relative, resolve, sep } from "node:path"; @@ -42,6 +43,26 @@ const passed = process.env.MERGE_RESULT === "success" && report.success && report.numFailedTests === 0; +// Successful shards and merged reports must account for the complete portable +// inventory. Coverage thresholds alone cannot prove that every file ran. +if (passed && process.env.CI_VERIFY_TEST_INVENTORY === "true") { + const expected = execFileSync( + "git", + ["ls-files", "-z", "--", "src", "tests"], + { encoding: "utf8", maxBuffer: 8 * 1024 * 1024 }, + ) + .split("\0") + .filter((path) => path.endsWith(".test.ts")); + const actual = files.map(({ name }) => name); + const expectedSet = new Set(expected); + const actualSet = new Set(actual); + const missing = expected.filter((path) => !actualSet.has(path)); + const unexpected = actual.filter((path) => !expectedSet.has(path)); + if (missing.length || unexpected.length || actualSet.size !== actual.length) + throw new Error( + `Incomplete portable test inventory: ${JSON.stringify({ missing, unexpected, duplicateFiles: actual.length - actualSet.size })}`, + ); +} const summary = [ complete ? "## Merged test diagnostics" : "## Partial test diagnostics", "", diff --git a/scripts/ci/verified-package.mjs b/scripts/ci/verified-package.mjs new file mode 100644 index 000000000..1053760b2 --- /dev/null +++ b/scripts/ci/verified-package.mjs @@ -0,0 +1,235 @@ +import { createHash } from "node:crypto"; +import { createReadStream } from "node:fs"; +import { execFile } from "node:child_process"; +import { copyFile, readFile, writeFile } from "node:fs/promises"; +import { join, resolve } from "node:path"; +import { fileURLToPath } from "node:url"; +import { promisify } from "node:util"; + +const MANIFEST_NAME = "manifest.json"; +const TARBALL_NAME = "package.tgz"; +const MANIFEST_KEYS = [ + "schemaVersion", + "sourceSha", + "runId", + "runAttempt", + "platform", + "architecture", + "packageVersion", + "nodeVersion", + "sha256", +].sort(); +const execFileAsync = promisify(execFile); + +/** Normalize workflow identity and host facts into the manifest-owned fields. */ +export function createVerifiedPackageMetadata({ + sourceSha, + runId, + runAttempt, + platform, + architecture, + packageVersion, + nodeVersion, +}) { + if (!/^[a-f0-9]{40}$/u.test(sourceSha)) + throw new Error("Source SHA must be a full commit SHA"); + return { + sourceSha, + runId: parsePositiveInteger(runId, "GITHUB_RUN_ID"), + runAttempt: parsePositiveInteger(runAttempt, "GITHUB_RUN_ATTEMPT"), + platform, + architecture, + packageVersion, + nodeVersion, + }; +} + +/** Build the single supported manifest representation for a verified tarball. */ +function createVerifiedPackageManifest(metadata) { + const manifest = { schemaVersion: 1, ...metadata }; + validateManifestShape(manifest); + return manifest; +} + +/** Write a manifest after the caller has completed qualification and cleanup. */ +export async function writeVerifiedPackageManifest(directory, metadata) { + const manifest = createVerifiedPackageManifest(metadata); + await writeFile( + join(directory, MANIFEST_NAME), + `${JSON.stringify(manifest, null, 2)}\n`, + { encoding: "utf8" }, + ); + return manifest; +} + +/** Copy the qualified archive under its fixed name and prepare its manifest. */ +export async function retainVerifiedPackage({ + sourceTarball, + directory, + metadata, +}) { + const tarball = join(directory, TARBALL_NAME); + await copyFile(sourceTarball, tarball); + const sha256 = await sha256File(tarball); + return { + directory, + metadata: createVerifiedPackageManifest({ ...metadata, sha256 }), + }; +} + +/** Validate a retained package against the current checkout and workflow attempt. */ +async function validateVerifiedPackage({ + directory, + sourceSha, + runId, + runAttempt, + packageVersion, + nodeVersion, +}) { + if (!/^[a-f0-9]{40}$/u.test(sourceSha)) + throw new Error("Expected source SHA must be a full commit SHA"); + if (!Number.isSafeInteger(runId) || runId <= 0) + throw new Error("Expected workflow run ID must be a positive integer"); + if (!Number.isSafeInteger(runAttempt) || runAttempt <= 0) + throw new Error("Expected workflow run attempt must be a positive integer"); + const manifestPath = join(directory, MANIFEST_NAME); + let manifest; + try { + manifest = JSON.parse(await readFile(manifestPath, "utf8")); + } catch (cause) { + throw new Error(`Could not read verified package manifest: ${cause}`, { + cause, + }); + } + validateManifestShape(manifest); + assertEqual(manifest.sourceSha, sourceSha, "source SHA"); + assertEqual(manifest.runId, runId, "workflow run ID"); + // A successful producer may belong to an earlier attempt when only failed + // publisher jobs are rerun. The workflow still requires same-run qualification. + if (manifest.runAttempt > runAttempt) + throw new Error("Verified package workflow run attempt is from the future"); + assertEqual(manifest.platform, "linux", "verifier platform"); + assertEqual(manifest.architecture, "x64", "verifier architecture"); + assertEqual(manifest.packageVersion, packageVersion, "package version"); + assertEqual(manifest.nodeVersion, nodeVersion, "Node.js version"); + + // The artifact format deliberately fixes the name; no manifest value can + // redirect validation to another path. + const tarball = resolve(directory, TARBALL_NAME); + const digest = await sha256File(tarball); + assertEqual(manifest.sha256, digest, "tarball SHA-256"); + return tarball; +} + +function validateManifestShape(manifest) { + if ( + typeof manifest !== "object" || + manifest === null || + Array.isArray(manifest) + ) + throw new Error("Verified package manifest must be a JSON object"); + const keys = Object.keys(manifest).sort(); + if ( + keys.length !== MANIFEST_KEYS.length || + keys.some((key, index) => key !== MANIFEST_KEYS[index]) + ) + throw new Error("Verified package manifest has missing or unknown fields"); + if (manifest.schemaVersion !== 1) + throw new Error("Unsupported verified package manifest schema version"); + if ( + typeof manifest.sourceSha !== "string" || + !/^[a-f0-9]{40}$/u.test(manifest.sourceSha) + ) + throw new Error("Verified package manifest has an invalid source SHA"); + if (!Number.isSafeInteger(manifest.runId) || manifest.runId <= 0) + throw new Error("Verified package manifest has an invalid workflow run ID"); + if (!Number.isSafeInteger(manifest.runAttempt) || manifest.runAttempt <= 0) + throw new Error( + "Verified package manifest has an invalid workflow run attempt", + ); + for (const key of [ + "platform", + "architecture", + "packageVersion", + "nodeVersion", + ]) + if (typeof manifest[key] !== "string" || manifest[key].length === 0) + throw new Error(`Verified package manifest has an invalid ${key}`); + if ( + typeof manifest.sha256 !== "string" || + !/^[a-f0-9]{64}$/u.test(manifest.sha256) + ) + throw new Error("Verified package manifest has an invalid tarball SHA-256"); +} + +function assertEqual(actual, expected, label) { + if (actual !== expected) + throw new Error(`Verified package ${label} mismatch`); +} + +async function sha256File(path) { + const hash = createHash("sha256"); + for await (const chunk of createReadStream(path)) hash.update(chunk); + return hash.digest("hex"); +} + +async function validateFromCli(directory) { + const sourceSha = process.env.GITHUB_SHA; + const runId = parsePositiveInteger( + process.env.GITHUB_RUN_ID, + "GITHUB_RUN_ID", + ); + const runAttempt = parsePositiveInteger( + process.env.GITHUB_RUN_ATTEMPT, + "GITHUB_RUN_ATTEMPT", + ); + if (sourceSha === undefined || !/^[a-f0-9]{40}$/u.test(sourceSha)) + throw new Error("GITHUB_SHA must be a full commit SHA"); + const checkoutSha = ( + await execFileAsync("git", ["rev-parse", "HEAD"]) + ).stdout.trim(); + assertEqual(checkoutSha, sourceSha, "checked out source SHA"); + const packageJson = JSON.parse(await readFile("package.json", "utf8")); + const tarball = await validateVerifiedPackage({ + directory, + sourceSha, + runId, + runAttempt, + packageVersion: packageJson.version, + nodeVersion: process.version, + }); + if (process.env.GITHUB_OUTPUT === undefined) + throw new Error( + "GITHUB_OUTPUT is required to publish the validated tarball path", + ); + await writeFile(process.env.GITHUB_OUTPUT, `tarball=${tarball}\n`, { + encoding: "utf8", + flag: "a", + }); + process.stdout.write(`${tarball}\n`); +} + +function parsePositiveInteger(value, name) { + if (value === undefined || !/^[1-9]\d*$/u.test(value)) + throw new Error(`${name} must be a positive integer`); + const parsed = Number(value); + if (!Number.isSafeInteger(parsed)) + throw new Error(`${name} must be a safe positive integer`); + return parsed; +} + +const invokedPath = process.argv[1]; +if ( + invokedPath !== undefined && + resolve(invokedPath) === fileURLToPath(import.meta.url) +) { + const [command, directory, ...extra] = process.argv.slice(2); + if (command !== "validate" || directory === undefined || extra.length > 0) { + process.stderr.write( + "Usage: node scripts/ci/verified-package.mjs validate \n", + ); + process.exitCode = 2; + } else { + await validateFromCli(directory); + } +} diff --git a/scripts/verify-package.mjs b/scripts/verify-package.mjs index fff3a07c6..32bad0a62 100644 --- a/scripts/verify-package.mjs +++ b/scripts/verify-package.mjs @@ -1,6 +1,8 @@ -import { mkdtemp, rm } from "node:fs/promises"; +import { mkdir, mkdtemp, rm, unlink } from "node:fs/promises"; +import { execFile } from "node:child_process"; import { tmpdir } from "node:os"; import { join } from "node:path"; +import { promisify } from "node:util"; import { TOOL_CONTRACTS } from "../dist/contracts/toolContracts.js"; import { PRODUCT_IDENTITY } from "../dist/identity.js"; @@ -21,12 +23,19 @@ import { verifyRepositorySkill } from "./verify-repository-skill.mjs"; import { verifyManaged } from "./verify-package-managed.mjs"; import { verifyUnknownProvider } from "./verify-package-unknown-provider.mjs"; import { completeVerifierRun, createVerifierRun } from "./lib/verifier-run.mjs"; +import { + createVerifiedPackageMetadata, + retainVerifiedPackage, + writeVerifiedPackageManifest, +} from "./ci/verified-package.mjs"; import { measureMcpStartup, profileMcpModuleLoading, } from "./lib/mcp-startup-probe.mjs"; const root = process.cwd(); +const execFileAsync = promisify(execFile); +const retentionDirectory = process.env.REA_VERIFIED_PACKAGE_DIRECTORY; const verifierRun = createVerifierRun(); const workspace = await mkdtemp(join(tmpdir(), "rea-package-")); const evidenceRoot = join(workspace, "evidence"); @@ -37,6 +46,8 @@ let mcpModuleLoading; let update; let repositorySkill; let report; +let retainedPackage; +let retentionMetadata; const runStage = async (name, action) => { const started = performance.now(); @@ -49,6 +60,27 @@ const runStage = async (name, action) => { }; try { + if (retentionDirectory !== undefined && retentionDirectory !== "") { + const sourceSha = ( + await execFileAsync("git", ["rev-parse", "HEAD"], { cwd: root }) + ).stdout.trim(); + retentionMetadata = createVerifiedPackageMetadata({ + sourceSha, + runId: process.env.GITHUB_RUN_ID, + runAttempt: process.env.GITHUB_RUN_ATTEMPT, + platform: process.platform, + architecture: process.arch, + packageVersion: PRODUCT_IDENTITY.packageVersion, + nodeVersion: process.version, + }); + await mkdir(retentionDirectory, { recursive: true }); + await unlink(join(retentionDirectory, "manifest.json")).catch((cause) => { + if ( + !(cause instanceof Error && "code" in cause && cause.code === "ENOENT") + ) + throw cause; + }); + } repositorySkill = await runStage("repository skill", () => verifyRepositorySkill({ root, workspace }), ); @@ -215,6 +247,13 @@ try { mcpStartup, mcpModuleLoading, }; + if (retentionMetadata !== undefined) { + retainedPackage = await retainVerifiedPackage({ + sourceTarball: tarball, + directory: retentionDirectory, + metadata: retentionMetadata, + }); + } } finally { await runStage("workspace cleanup", () => rm(workspace, { recursive: true, force: true }), @@ -223,6 +262,11 @@ try { const verifierLineage = await runStage("verifier lineage", () => completeVerifierRun(verifierRun), ); +if (retainedPackage !== undefined) + await writeVerifiedPackageManifest( + retainedPackage.directory, + retainedPackage.metadata, + ); process.stdout.write( `${JSON.stringify({ verifier_run: verifierLineage, ...report })}\n`, ); diff --git a/tests/boundary/filesystem/ciChangeScope.test.ts b/tests/boundary/filesystem/ciChangeScope.test.ts index 9176dec34..48fc19624 100644 --- a/tests/boundary/filesystem/ciChangeScope.test.ts +++ b/tests/boundary/filesystem/ciChangeScope.test.ts @@ -1,4 +1,5 @@ import { + chmod, cp, mkdir, readFile, @@ -161,13 +162,158 @@ it("retains complete checks when implementation is renamed into documentation", }); }); -it("retains the complete main-push baseline even for documentation", async () => { +it("keeps main pushes scoped when a successful ancestor baseline is available", async () => { const { directory, base } = await fixture(); const head = await change(directory, ["README_ja.md"]); expect(await classify(directory, base, head, "push")).toMatchObject({ + tier: "fast", + code: false, + package: false, + docs: true, + matrix: { + include: [ + { os: "ubuntu-latest", platform: "linux", architecture: "x64" }, + ], + }, + }); +}); + +it("includes changes from earlier cancelled pushes since the last successful baseline", async () => { + const { directory, base } = await fixture(); + await change(directory, ["src/evm/client.ts"]); + const head = await change(directory, ["README.md"]); + expect(await classify(directory, base, head, "push")).toMatchObject({ + tier: "fast", + code: true, + evm: true, + runtime: true, + package: false, + }); +}); + +it.each(["", "not-a-commit"])( + "fails closed to all lanes for a missing or invalid push baseline (%s)", + async (base) => { + const { directory } = await fixture(); + const head = await change(directory, ["README.md"]); + const plan = await classify(directory, base, head, "push"); + expect(plan).toMatchObject({ + tier: "fast", + code: true, + package: true, + evm: true, + browser: true, + reasons: expect.arrayContaining([expect.stringContaining("baseline")]), + }); + const { SCOPES } = await loadScopes(); + for (const flag of [ + "code", + "runtime", + "package", + "fixtures", + "readiness", + "docs", + "workflows", + ...Object.keys(SCOPES), + ]) + expect(plan[flag]).toBe(true); + }, +); + +it("selects only a trusted successful ancestor from recent CI push runs", async () => { + const { directory, base } = await fixture(); + const head = await change(directory, ["src/evm/client.ts"]); + const bin = join(directory, "fake-bin"); + await mkdir(bin); + const gh = join(bin, "gh"); + await writeFile(gh, '#!/bin/sh\ncat "$GH_FIXTURE_RESPONSE"\n'); + await chmod(gh, 0o755); + const response = join(directory, "runs.json"); + await writeFile( + response, + JSON.stringify({ + workflow_runs: [ + { + event: "push", + conclusion: "success", + head_branch: "main", + head_repository: { full_name: "untrusted/fork" }, + head_sha: base, + }, + { + event: "push", + conclusion: "success", + head_branch: "main", + head_repository: { full_name: "owner/repo" }, + head_sha: base, + }, + ], + }), + ); + const output = join(directory, "baseline-output"); + const result = await execFileAsync( + process.execPath, + [new URL("../../../scripts/ci/baseline.mjs", import.meta.url).pathname], + { + cwd: directory, + env: { + ...process.env, + PATH: `${bin}:${process.env.PATH}`, + GH_FIXTURE_RESPONSE: response, + CI_REPOSITORY: "owner/repo", + CI_DEFAULT_BRANCH: "main", + HEAD_SHA: head, + GITHUB_OUTPUT: output, + }, + }, + ); + expect(JSON.parse(result.stdout)).toEqual({ base, reason: "" }); + expect(await readFile(output, "utf8")).toBe(`base=${base}\nreason=\n`); +}); + +it("returns an empty baseline when GitHub lookup fails", async () => { + const { directory, base } = await fixture(); + const head = await change(directory, ["src/evm/client.ts"]); + const bin = join(directory, "fake-bin"); + await mkdir(bin); + const gh = join(bin, "gh"); + await writeFile(gh, "#!/bin/sh\nexit 1\n"); + await chmod(gh, 0o755); + const result = await execFileAsync( + process.execPath, + [new URL("../../../scripts/ci/baseline.mjs", import.meta.url).pathname], + { + cwd: directory, + env: { + ...process.env, + PATH: `${bin}:${process.env.PATH}`, + CI_REPOSITORY: "owner/repo", + CI_DEFAULT_BRANCH: "main", + HEAD_SHA: head, + }, + }, + ); + expect(JSON.parse(result.stdout)).toMatchObject({ + base: "", + reason: expect.stringContaining("Could not read"), + }); + expect(head).not.toBe(base); +}); + +it("fails closed when the successful push baseline is not an ancestor", async () => { + const { directory, base } = await fixture(); + await git(directory, ["switch", "-c", "unrelated"]); + const unrelated = await change(directory, ["other.txt"]); + await git(directory, ["switch", "main"]); + const head = await change(directory, ["README.md"]); + expect(await classify(directory, unrelated, head, "push")).toMatchObject({ + tier: "fast", code: true, package: true, + evm: true, + browser: true, }); + expect(head).not.toBe(base); }); it("fails classification rather than treating an invalid Git comparison as a scope", async () => { @@ -207,7 +353,8 @@ it("retains native Inspector verification on every documented package host", asy ); expect(packageInspector.if).toBeUndefined(); expect( - (await classify((await fixture()).directory, "", "", "push")).matrix, + (await classify((await fixture()).directory, "", "", "workflow_dispatch")) + .matrix, ).toEqual({ include: [ { os: "ubuntu-latest", platform: "linux", architecture: "x64" }, @@ -595,6 +742,7 @@ it.each(["workflow_dispatch", "schedule"])( async (event) => { const { directory, base } = await fixture(); expect(await classify(directory, base, base, event)).toMatchObject({ + tier: "full", code: true, package: true, evm: true, @@ -610,7 +758,13 @@ it("honors ci:full for successive PR revisions", async () => { for (const head of [first, second]) { expect( await classify(directory, base, head, "pull_request", true), - ).toMatchObject({ code: true, package: true, evm: true, browser: true }); + ).toMatchObject({ + tier: "full", + code: true, + package: true, + evm: true, + browser: true, + }); } }); @@ -650,7 +804,7 @@ const INTEL_HOST = { architecture: "x64", }; -it("keeps macOS x64 package verification for ci:full and non-PR events", async () => { +it("keeps macOS x64 package verification only for full qualification", async () => { const { directory, base } = await fixture(); const head = await change(directory, ["src/windows/Example.ts"]); const matrix = z.object({ include: z.array(z.unknown()) }); @@ -661,7 +815,8 @@ it("keeps macOS x64 package verification for ci:full and non-PR events", async ( expect(pullRequest).toHaveLength(3); expect(pullRequest).not.toContainEqual(INTEL_HOST); expect(await hosts("pull_request", true)).toContainEqual(INTEL_HOST); - for (const event of ["push", "schedule", "workflow_dispatch"]) + expect(await hosts("push", false)).not.toContainEqual(INTEL_HOST); + for (const event of ["schedule", "workflow_dispatch"]) expect(await hosts(event, false)).toContainEqual(INTEL_HOST); }); diff --git a/tests/boundary/filesystem/ciRequiredLanes.test.ts b/tests/boundary/filesystem/ciRequiredLanes.test.ts index 4cff44e96..6e30d8784 100644 --- a/tests/boundary/filesystem/ciRequiredLanes.test.ts +++ b/tests/boundary/filesystem/ciRequiredLanes.test.ts @@ -64,6 +64,76 @@ it("makes the required gate wait for every CI lane, including future additions", ); }); +it("keeps fast feedback separate from complete qualification and release proof", async () => { + const ci = z + .object({ + on: z.object({ + push: z.unknown(), + pull_request: z.unknown(), + schedule: z.array(z.object({ cron: z.string() })), + workflow_call: z.object({ + inputs: z.object({ + source_sha: z.object({ + required: z.literal(true), + type: z.literal("string"), + }), + }), + }), + }), + concurrency: z.object({ + group: z.string(), + "cancel-in-progress": z.string(), + }), + jobs: z.record( + z.string(), + z.object({ + if: z.string().optional(), + steps: z + .array( + z.object({ + run: z.string().optional(), + if: z.string().optional(), + with: z.record(z.string(), z.unknown()).optional(), + }), + ) + .optional(), + }), + ), + }) + .parse( + parse( + await readFile( + new URL("../../../.github/workflows/ci.yml", import.meta.url), + "utf8", + ), + ), + ); + expect(ci.on.schedule).toEqual([{ cron: "23 * * * *" }]); + expect(ci.concurrency.group).toContain("'qualification' || 'feedback'"); + expect(ci.concurrency.group).toContain("inputs.source_sha ||"); + for (const fullEvent of ["schedule", "workflow_dispatch"]) + expect(ci.concurrency["cancel-in-progress"]).toContain( + `github.event_name != '${fullEvent}'`, + ); + expect(ci.concurrency["cancel-in-progress"]).toContain( + "inputs.source_sha == ''", + ); + expect(ci.jobs.required?.if).toBe("${{ always() }}"); + const shards = ci.jobs["test-shard"]?.steps ?? []; + const fast = shards.find((step) => step.run?.includes("npm run test:run")); + const full = shards.find((step) => step.run?.includes("test:ci:shard:run")); + expect(fast?.if).toBe("needs.changes.outputs.tier == 'fast'"); + expect(fast?.run).toContain("--reporter=blob"); + expect(fast?.run).not.toContain("--coverage"); + expect(full?.if).toBe("needs.changes.outputs.tier == 'full'"); + const proof = ci.jobs["package-e2e"]?.steps?.find( + (step) => step.with?.name === "verified-release-package", + ); + expect(proof?.if).toContain("needs.changes.outputs.tier == 'full'"); + expect(proof?.if).toContain("matrix.platform == 'linux'"); + expect(proof?.if).toContain("matrix.architecture == 'x64'"); +}); + it.each([ ["true", "success", true], ["true", "failure", false], diff --git a/tests/boundary/filesystem/ciTestReports.test.ts b/tests/boundary/filesystem/ciTestReports.test.ts index 0c9bb4ccf..3ab042718 100644 --- a/tests/boundary/filesystem/ciTestReports.test.ts +++ b/tests/boundary/filesystem/ciTestReports.test.ts @@ -1,5 +1,5 @@ import { execFile } from "node:child_process"; -import { readFile, writeFile } from "node:fs/promises"; +import { mkdir, readFile, writeFile } from "node:fs/promises"; import { join, resolve } from "node:path"; import { fileURLToPath } from "node:url"; import { promisify } from "node:util"; @@ -10,6 +10,7 @@ import { z } from "zod"; import { parseTestDurations } from "../../ciTestSequencer.js"; import { createTestTempDirectory } from "../../fixtures/temporaryDirectory.js"; +import { workflowGit } from "../../support/workflowGit.js"; const execute = promisify(execFile); const script = new URL("../../../scripts/ci/test-report.mjs", import.meta.url); @@ -77,6 +78,7 @@ it("keeps diagnosis available after merge failure and publishes only complete su const history = steps.find((step) => step.with?.name === "test-timings"); expect(history?.if).toContain("success()"); expect(history?.if).toContain("needs.test-shard.result == 'success'"); + expect(history?.if).toContain("needs.changes.outputs.tier == 'full'"); expect(history?.with?.path).toBe(".cache/ci-test-history/test-results.json"); const download = (await workflow()).build!.steps.find( (step) => step.name === "Download previous timing report", @@ -84,6 +86,49 @@ it("keeps diagnosis available after merge failure and publishes only complete su expect(download?.with?.name).toBe(history?.with?.name); }); +it("rejects successful shard reports with missing or duplicate portable files", async () => { + const root = await createTestTempDirectory("rea-ci-report-inventory-"); + await workflowGit(root, ["init"]); + await mkdir(join(root, "tests")); + for (const name of ["first", "second"]) + await writeFile(join(root, `tests/${name}.test.ts`), "// fixture\n"); + await workflowGit(root, ["add", "tests"]); + const run = async (names: readonly string[]) => { + await writeFile( + join(root, "test-results.json"), + JSON.stringify({ + success: true, + numFailedTests: 0, + testResults: names.map((name) => ({ name, startTime: 0, endTime: 1 })), + }), + ); + return execute(process.execPath, [fileURLToPath(script)], { + cwd: root, + env: { + ...process.env, + SHARD_RESULT: "success", + MERGE_RESULT: "success", + CI_VERIFY_TEST_INVENTORY: "true", + }, + }); + }; + await expect( + run(["tests/first.test.ts", "tests/second.test.ts"]), + ).resolves.toMatchObject({ stderr: "" }); + for (const names of [ + ["tests/first.test.ts"], + ["tests/first.test.ts", "tests/second.test.ts", "tests/unplanned.test.ts"], + ["tests/first.test.ts", "tests/first.test.ts", "tests/second.test.ts"], + ]) { + await expect(run(names)).rejects.toMatchObject({ + stderr: expect.stringContaining("Incomplete portable test inventory"), + }); + await expect( + readFile(join(root, ".cache/ci-test-history/test-results.json")), + ).rejects.toMatchObject({ code: "ENOENT" }); + } +}); + it("runs the existing onboarding and Python producer guards from their selected CI owners", async () => { const jobs = await workflow(); const scripts = z diff --git a/tests/boundary/filesystem/ciVerifiedPackage.test.ts b/tests/boundary/filesystem/ciVerifiedPackage.test.ts new file mode 100644 index 000000000..ff5045024 --- /dev/null +++ b/tests/boundary/filesystem/ciVerifiedPackage.test.ts @@ -0,0 +1,137 @@ +import { createHash } from "node:crypto"; +import { mkdir, readFile, writeFile } from "node:fs/promises"; +import { join } from "node:path"; +import { fileURLToPath } from "node:url"; + +import { expect, it } from "vitest"; + +import { createTestTempDirectory } from "../../fixtures/temporaryDirectory.js"; +import { + executeWorkflowFixture, + workflowGit, +} from "../../support/workflowGit.js"; + +const validator = fileURLToPath( + new URL("../../../scripts/ci/verified-package.mjs", import.meta.url), +); +type Fixture = Awaited>; +type FixtureOptions = { + override?: Record; + manifestContents?: string; + writeManifest?: boolean; +}; + +it("validates a tarball against the exact checkout and current workflow attempt", async () => { + const fixture = await createFixture(); + const result = await validate(fixture); + + expect(result.stdout.trim()).toBe( + join(fixture.artifactDirectory, "package.tgz"), + ); + expect(await readFile(fixture.output, "utf8")).toBe( + `tarball=${join(fixture.artifactDirectory, "package.tgz")}\n`, + ); +}); + +it("rejects a workflow SHA that differs from the checked out HEAD", async () => { + const fixture = await createFixture(); + + await expect(validate(fixture, "c".repeat(40))).rejects.toThrow(); +}); + +it.each([ + ["wrong checkout SHA", { sourceSha: "c".repeat(40) }], + ["wrong workflow run", { runId: 124 }], + ["future workflow attempt", { runAttempt: 3 }], + ["wrong verifier platform", { platform: "darwin" }], + ["wrong verifier architecture", { architecture: "arm64" }], + ["wrong package digest", { sha256: "c".repeat(64) }], +])("rejects %s", async (_label, override) => { + const fixture = await createFixture({ override }); + + await expect(validate(fixture)).rejects.toThrow(); +}); + +it("accepts an earlier successful attempt from the same workflow run", async () => { + const fixture = await createFixture({ override: { runAttempt: 1 } }); + + await expect(validate(fixture)).resolves.toMatchObject({ + stdout: expect.any(String), + }); +}); + +it.each([ + ["malformed manifest", "{"], + ["manifest with missing fields", JSON.stringify({ schemaVersion: 1 })], +])("rejects %s", async (_label, contents) => { + const fixture = await createFixture({ manifestContents: contents }); + + await expect(validate(fixture)).rejects.toThrow(); +}); + +it("rejects an absent manifest", async () => { + const fixture = await createFixture({ writeManifest: false }); + + await expect(validate(fixture)).rejects.toThrow(); +}); + +async function createFixture({ + override = {}, + manifestContents, + writeManifest = true, +}: FixtureOptions = {}) { + const directory = await createTestTempDirectory("rea-verified-package-"); + await workflowGit(directory, ["init", "--initial-branch=main"]); + await writeFile( + join(directory, "package.json"), + '{"name":"fixture","version":"1.2.3"}\n', + ); + await workflowGit(directory, ["add", "package.json"]); + await workflowGit(directory, [ + "commit", + "-m", + "test: verified package fixture", + ]); + const sourceSha = await workflowGit(directory, ["rev-parse", "HEAD"]); + const artifactDirectory = join(directory, "artifact"); + await mkdir(artifactDirectory); + const archive = "verified tarball fixture"; + await writeFile(join(artifactDirectory, "package.tgz"), archive); + const manifest = { + schemaVersion: 1, + sourceSha, + runId: 123, + runAttempt: 2, + platform: "linux", + architecture: "x64", + packageVersion: "1.2.3", + nodeVersion: process.version, + sha256: createHash("sha256").update(archive).digest("hex"), + ...override, + }; + if (writeManifest) + await writeFile( + join(artifactDirectory, "manifest.json"), + manifestContents ?? JSON.stringify(manifest), + ); + const output = join(directory, "github-output"); + await writeFile(output, ""); + return { directory, sourceSha, artifactDirectory, output }; +} + +function validate(fixture: Fixture, sourceSha = fixture.sourceSha) { + return executeWorkflowFixture( + process.execPath, + [validator, "validate", fixture.artifactDirectory], + { + cwd: fixture.directory, + env: { + ...process.env, + GITHUB_SHA: sourceSha, + GITHUB_RUN_ID: "123", + GITHUB_RUN_ATTEMPT: "2", + GITHUB_OUTPUT: fixture.output, + }, + }, + ); +} diff --git a/tests/boundary/filesystem/releaseWorkflow.test.ts b/tests/boundary/filesystem/releaseWorkflow.test.ts index 0bfebd1d2..4d3859433 100644 --- a/tests/boundary/filesystem/releaseWorkflow.test.ts +++ b/tests/boundary/filesystem/releaseWorkflow.test.ts @@ -34,6 +34,12 @@ async function readReleaseWorkflow() { "release-proposal": jobSchema, "release-please": jobSchema, "merged-release": jobSchema, + qualification: z.object({ + if: z.string(), + needs: z.array(z.string()), + uses: z.string(), + with: z.record(z.string(), z.string()), + }), publish: jobSchema, "publish-mcp": jobSchema, }), @@ -271,10 +277,19 @@ it("binds npm and MCP publication to the same immutable release SHA", async () = release_created: "${{ steps.release.outputs.release_created }}", sha: "${{ steps.release.outputs.sha }}", }); - expect(workflow.jobs.publish.if).toContain("inputs.phase == 'publish'"); - expect(workflow.jobs.publish.if).toContain( + expect(workflow.jobs.qualification.if).toContain("inputs.phase == 'publish'"); + expect(workflow.jobs.qualification.if).toContain( "needs.release-please.outputs.release_created", ); + expect(workflow.jobs.qualification.uses).toBe("./.github/workflows/ci.yml"); + expect(workflow.jobs.qualification.with).toEqual({ + source_sha: + "${{ needs.merged-release.outputs.sha || needs.release-please.outputs.sha }}", + }); + expect(workflow.jobs.publish.needs).toContain("qualification"); + expect(workflow.jobs.publish.if).toContain( + "needs.qualification.result == 'success'", + ); expect(workflow.jobs["publish-mcp"].if).toContain( "needs.publish.result == 'success'", ); @@ -309,7 +324,7 @@ it("binds npm and MCP publication to the same immutable release SHA", async () = )?.run; expect(publishCommand).toContain("scripts/release-npm-tag.mjs"); expect(publishCommand).toContain( - 'npm publish --access public --tag "${tag}"', + 'npm publish "${VERIFIED_PACKAGE_TARBALL}" --access public --tag "${tag}"', ); for (const [version, tag] of [ ["6.1.0", "latest"], @@ -644,7 +659,7 @@ it("routes website verification through the central PR policy", async () => { // Publishing is irreversible. Keep the release authority invariant as a static // boundary check; it is not proof that package installation or publishing works. -it("requires package verification before publishing and retains the registry canary", async () => { +it("requires qualified package validation before publishing and retains the registry canary", async () => { const workflow = z .object({ jobs: z.object({ @@ -663,7 +678,7 @@ it("requires package verification before publishing and retains the registry can ); const commands = workflow.jobs.publish.steps.map(({ run }) => run ?? ""); const verify = commands.findIndex((command) => - command.includes("npm run verify:package"), + command.includes("scripts/ci/verified-package.mjs validate"), ); const publish = commands.findIndex((command) => command.includes("npm publish"),