diff --git a/.github/workflows/ci-timings.yml b/.github/workflows/ci-timings.yml index 73a9ba4b9..74f27af32 100644 --- a/.github/workflows/ci-timings.yml +++ b/.github/workflows/ci-timings.yml @@ -12,12 +12,13 @@ concurrency: permissions: contents: read actions: read + pull-requests: read jobs: report: name: Collect recent completed workflow attempts runs-on: ubuntu-latest - timeout-minutes: 5 + timeout-minutes: 10 steps: # Consume only API data and execute batch code from the trusted default branch. - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0 @@ -27,6 +28,18 @@ jobs: - uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4.4.0 with: node-version-file: .nvmrc + - name: Cache npm package downloads + id: npm-cache + continue-on-error: true + uses: ./.github/actions/npm-cache + - name: Diagnose missing and startup-failed PR CI + continue-on-error: true + env: + GH_TOKEN: ${{ github.token }} + CI_REPOSITORY: ${{ github.repository }} + CI_REPOSITORY_ID: ${{ github.repository_id }} + CI_DEFAULT_BRANCH: ${{ github.event.repository.default_branch }} + run: node scripts/ci/pr-ci-health.mjs - name: Record queue, execution, and matched step timing continue-on-error: true env: @@ -44,3 +57,28 @@ jobs: if-no-files-found: error overwrite: true retention-days: 14 + - name: Install dependencies for static failure attribution + id: failure-deps + if: ${{ !cancelled() && hashFiles('.cache/ci-job-timings/batch-manifest.json') != '' }} + continue-on-error: true + run: npm ci + - name: Attribute supported static failures against main + if: ${{ !cancelled() && steps.failure-deps.outcome == 'success' && hashFiles('.cache/ci-job-timings/batch-manifest.json') != '' }} + continue-on-error: true + env: + GH_TOKEN: ${{ github.token }} + CI_REPOSITORY: ${{ github.repository }} + CI_DEFAULT_BRANCH: ${{ github.event.repository.default_branch }} + run: node scripts/ci/failure-report.mjs + - name: Upload CI health and failure diagnostics + if: ${{ !cancelled() }} + continue-on-error: true + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 + with: + name: ci-health-diagnostics + path: | + .cache/ci-pr-health/ + .cache/ci-failures/ + include-hidden-files: true + if-no-files-found: warn + retention-days: 14 diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 8da32ee27..b35b6a6e1 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -139,6 +139,12 @@ jobs: - name: Validate workflow syntax and expressions if: needs.changes.outputs.workflows == 'true' run: go run github.com/rhysd/actionlint/cmd/actionlint@v1.7.7 -shellcheck= -pyflakes= + - name: Record static task source and execution identity + continue-on-error: true + if: needs.changes.outputs.code == 'true' + env: + CI_PR_HEAD_SHA: ${{ github.event.pull_request.head.sha }} + run: npm run ci:static-context - if: needs.changes.outputs.code == 'true' run: npm run check:ci - if: needs.changes.outputs.code == 'true' @@ -167,6 +173,34 @@ jobs: if-no-files-found: ignore retention-days: 7 + test-source: + name: Source tests + needs: changes + if: needs.changes.outputs.code == 'true' && needs.changes.outputs.tier == 'fast' + runs-on: ubuntu-latest + timeout-minutes: 10 + steps: + - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0 + - uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4.4.0 + with: + node-version-file: .nvmrc + - name: Cache npm package downloads + id: npm-cache + uses: ./.github/actions/npm-cache + - run: npm ci + - name: Run source tests without waiting for build + run: npm run test:fast -- --reporter=default --reporter=blob + env: + VITEST_BLOB_LABEL: source + - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 + if: ${{ !cancelled() }} + with: + name: vitest-report-source + path: .vitest/blob/* + include-hidden-files: true + if-no-files-found: error + retention-days: 7 + test-shard: name: Test shard (${{ matrix.shard }}) needs: [changes, build] @@ -178,7 +212,9 @@ jobs: # every shard's diagnostics and coverage. Passing PRs run the full suite. fail-fast: ${{ github.event_name == 'pull_request' && needs.changes.outputs.tier == 'fast' }} matrix: - shard: [1/6, 2/6, 3/6, 4/6, 5/6, 6/6] + # Fast feedback spends the same six test jobs on one source group and + # five built groups; full qualification retains six complete shards. + shard: ${{ fromJSON(needs.changes.outputs.tier == 'fast' && '["1/5","2/5","3/5","4/5","5/5"]' || '["1/6","2/6","3/6","4/6","5/6","6/6"]') }} steps: - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0 - uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4.4.0 @@ -197,15 +233,16 @@ jobs: run: npm run test:ci:shard:run -- --shard=${{ matrix.shard }} env: REA_CI_TEST_HISTORY: .cache/ci-test-history/test-results.json - - name: Run complete portable suite for fast feedback + - name: Run built-dependent tests for fast feedback if: needs.changes.outputs.tier == 'fast' - run: npm run test:run -- --reporter=default --reporter=blob --shard=${{ matrix.shard }} + run: npm run test:built -- --reporter=default --reporter=blob --shard=${{ matrix.shard }} env: + VITEST_BLOB_LABEL: built REA_CI_TEST_HISTORY: .cache/ci-test-history/test-results.json - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 if: ${{ !cancelled() }} with: - name: vitest-report-${{ strategy.job-index }} + name: vitest-report-built-${{ strategy.job-index }} path: .vitest/blob/* include-hidden-files: true if-no-files-found: error @@ -219,11 +256,13 @@ jobs: runs-on: ubuntu-latest timeout-minutes: 60 steps: - - uses: actions/checkout@v4 - - uses: actions/setup-node@v4 + - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0 + - uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4.4.0 with: node-version-file: .nvmrc - cache: npm + - name: Cache npm package downloads + id: npm-cache + uses: ./.github/actions/npm-cache - run: npm ci - name: Run reverse tools tests with coverage run: npm run test:coverage:reverse-tools @@ -316,7 +355,7 @@ jobs: test: name: Portable tests - needs: [changes, test-shard] + needs: [changes, test-source, test-shard] if: ${{ !cancelled() && needs.changes.outputs.tier != 'reused' }} runs-on: ubuntu-latest timeout-minutes: 10 @@ -340,7 +379,9 @@ jobs: - uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0 if: >- !cancelled() && needs.changes.outputs.code == 'true' && - (needs.test-shard.result == 'success' || needs.test-shard.result == 'failure') + (needs.test-shard.result == 'success' || needs.test-shard.result == 'failure' || + (needs.changes.outputs.tier == 'fast' && + (needs.test-source.result == 'success' || needs.test-source.result == 'failure'))) with: path: .vitest/blob pattern: vitest-report-* @@ -368,7 +409,8 @@ jobs: if: >- !cancelled() && needs.changes.outputs.code == 'true' && needs.changes.outputs.tier == 'fast' && - (needs.test-shard.result == 'success' || needs.test-shard.result == 'failure') + (needs.test-shard.result == 'success' || needs.test-shard.result == 'failure' || + needs.test-source.result == 'success' || needs.test-source.result == 'failure') run: >- npm exec -- vitest --merge-reports --reporter=default @@ -381,6 +423,8 @@ jobs: if: needs.changes.outputs.code == 'true' && !cancelled() && hashFiles('test-timings.json') != '' env: SHARD_RESULT: ${{ needs.test-shard.result }} + SOURCE_REQUIRED: ${{ needs.changes.outputs.tier == 'fast' }} + SOURCE_RESULT: ${{ needs.test-source.result }} MERGE_RESULT: ${{ needs.changes.outputs.tier == 'full' && steps.merge.outcome || steps.merge-fast.outcome }} CI_VERIFY_TEST_INVENTORY: "true" CI_TEST_SELECTION: ${{ needs.changes.outputs.test_selection }} @@ -413,7 +457,13 @@ jobs: env: CODE_REQUIRED: ${{ needs.changes.outputs.code }} SHARD_RESULT: ${{ needs.test-shard.result }} - run: test "${CODE_REQUIRED}" != "true" || test "${SHARD_RESULT}" = "success" + SOURCE_REQUIRED: ${{ needs.changes.outputs.tier == 'fast' }} + SOURCE_RESULT: ${{ needs.test-source.result }} + run: | + if [ "${CODE_REQUIRED}" = "true" ]; then + test "${SHARD_RESULT}" = "success" || exit 1 + test "${SOURCE_REQUIRED}" != "true" || test "${SOURCE_RESULT}" = "success" + fi conformance-fixtures: name: Source-built conformance fixtures @@ -847,6 +897,7 @@ jobs: build, reverse-tools-quality, test-shard, + test-source, sqlite-runtime, test, conformance-fixtures, diff --git a/.github/workflows/real-cutter.yml b/.github/workflows/real-cutter.yml index 856777999..91121f207 100644 --- a/.github/workflows/real-cutter.yml +++ b/.github/workflows/real-cutter.yml @@ -13,8 +13,8 @@ jobs: name: Build Windows bridge security boundary runs-on: ubuntu-latest steps: - - uses: actions/checkout@v4 - - uses: actions/setup-node@v4 + - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0 + - uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4.4.0 with: node-version-file: .nvmrc - name: Install the cross compiler used by the Windows native lane @@ -44,11 +44,13 @@ jobs: - platform: Windows runner: [self-hosted, cutter-upstream, Windows] steps: - - uses: actions/checkout@v4 - - uses: actions/setup-node@v4 + - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0 + - uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4.4.0 with: node-version-file: .nvmrc - cache: npm + - name: Cache npm package downloads + id: npm-cache + uses: ./.github/actions/npm-cache - run: npm ci - if: matrix.platform == 'Windows' uses: actions/download-artifact@v4 diff --git a/docs/testing.md b/docs/testing.md index b543fe89f..aa8d2163f 100644 --- a/docs/testing.md +++ b/docs/testing.md @@ -280,8 +280,15 @@ focused suite also exercises affected MCP routes and the Python Cutter bridge. Contract, integration, and real-provider lanes complement these focused tests; simulated providers do not establish real-provider behavior. -CI runs six native Vitest shards without retries. Each shard emits a blob -report; the merge job produces JUnit and JSON diagnostics, a module timing +Fast CI starts the source-only projects after classification, independently of +build and static checks, and runs five built-dependent shards after the build. +The six test jobs still select every portable test. Full qualification keeps six +complete Vitest shards with coverage. Neither tier retries tests. Canonical +project ownership lives in `scripts/lib/test-projects.mjs`, shared by Vitest, +CI group execution, and development-test preparation. A test importing compiled +verifier code belongs to the built group even if its directory is conformance. +Source and built blobs have separate labels to prevent merge collisions. +Each test job emits a blob report; the merge job produces JUnit and JSON diagnostics, a module timing report, plus aggregate coverage during full qualification. It retains available reports even when a shard fails, uploads available diagnostics, and writes the slowest files to the workflow summary. Partial reports are marked diagnostic @@ -316,6 +323,30 @@ portable test shard. These limits leave headroom for cold setup and full coverag while preventing a hung job from holding the active main run for hours. They do not bound runner queue waiting. Timeouts still fail the required gate. +The separate `CI timings` workflow also diagnoses missing, startup-failed, +queued/running, approval-required, canceled, and failed PR CI through bounded, +read-only API scans. Incomplete pagination or changing identity stays unknown. +It never closes/reopens PRs, reruns workflows, posts comments, or changes gates. +The timing manifest retains each run's event and head branch, including explicit +unknowns, so PR runs cannot consume the eligible-main comparison budget. +Its static failure report compares supported lint/duplication diagnostics with +failed main runs using exact source, ancestry, task hashes, and toolchain proof. +A match describes those diagnostics only; it does not establish that every +failure in a job came from main. Duplication reports retain the actual checker +exit result, so passing duplication checks preserve lint failures and caller +thresholds remain owned by the producer. Missing checker outcomes leave +those duplication diagnostics unclassified. Missing historical receipts remain unknown. +The reporter executes only trusted default-branch code and parses whitelisted +JSON artifact members without executing artifact content. Reports and limits +are retained in `ci-health-diagnostics`; its ten-minute job limit covers the +existing four-minute timing collection plus the bounded diagnostic stages. + +Shard-count reduction remains deferred: the available successful timing sample +covers an older inventory, and file phase weights are not job wall time. The +one-source/five-built split retains six test jobs; it makes no hosted speedup +claim. Current complete timings and completed candidate hosted runs are needed +before enabling a smaller adaptive matrix. + The separate `CI timings` workflow batches recent completed CI and Release runs hourly or on manual dispatch. It uploads `ci-job-timings-batch` with raw API pages and a normalized point-in-time report for each collected workflow attempt, plus a diff --git a/package.json b/package.json index dff275eb4..e09e51366 100644 --- a/package.json +++ b/package.json @@ -118,7 +118,9 @@ "precheck:pr": "npm run deps:check", "check:pr": "node scripts/run-exclusive.mjs artifacts turbo run typecheck lint:built format:check knip metadata:check test:run && npm run test:python && npm run docs:check && npm run jscpd && npm run scan:todos", "precheck:ci": "npm run deps:check", - "check:ci": "node scripts/run-exclusive.mjs artifacts turbo run typecheck lint:built format:check knip metadata:check jscpd scan:todos && npm run verify:architecture-guards", + "check:ci": "node scripts/run-exclusive.mjs artifacts npm run check:ci:tasks && npm run verify:architecture-guards", + "check:ci:tasks": "turbo run typecheck lint:built format:check knip metadata:check jscpd scan:todos", + "ci:static-context": "node scripts/ci/static-task-context.mjs", "build:cached": "node scripts/check-dependency-install.mjs && node scripts/run-exclusive.mjs artifacts turbo run artifacts:skills", "build:termux": "node scripts/run-exclusive.mjs artifacts npm run build:termux:run", "build:termux:run": "npm run compile:termux && npm run artifacts:skills", @@ -138,7 +140,8 @@ "test:run": "vitest run", "test:python": "python3 -m unittest bridge/native/test_rea_lldb_tracer.py -v && python3 -m unittest discover -s experiments/analysis-efficiency -p 'test_*.py' -v", "test:local": "node scripts/run-development-tests.mjs local", - "test:fast": "vitest run --project domain --project services --project adapters --project composition --project conformance --project evaluation", + "test:fast": "node scripts/ci/test-group.mjs source", + "test:built": "node scripts/ci/test-group.mjs built", "pretest:boundary": "npm run test:prepare", "test:boundary": "vitest run --project boundary --project cli-boundary --project process-boundary --project mcp-boundary --project process-global", "pretest:mcp": "node scripts/check-dependency-install.mjs && node scripts/run-exclusive.mjs artifacts turbo run artifacts:skills artifacts:mcp-catalog", diff --git a/scripts/ci/artifacts.d.mts b/scripts/ci/artifacts.d.mts new file mode 100644 index 000000000..93cf2f701 --- /dev/null +++ b/scripts/ci/artifacts.d.mts @@ -0,0 +1,67 @@ +import type { NormalizedWorkflowRun } from "./workflow-run.mjs"; +import type { StaticTaskContext } from "./static-task-context.mjs"; +export type { StaticTaskContext } from "./static-task-context.mjs"; + +export interface StaticDiagnostic { + readonly kind: "lint" | "jscpd"; + readonly path?: string; + readonly paths?: readonly string[]; + readonly code?: string; + readonly message?: string; + readonly severity?: string; +} + +export function classifyFailure(input: { + repository: string; + defaultBranch: string; + run: Record & NormalizedWorkflowRun; + job: Record; + diagnostics: readonly StaticDiagnostic[]; + context: StaticTaskContext; + mainRun: Record & NormalizedWorkflowRun; + mainJob: Record; + mainDiagnostics: readonly StaticDiagnostic[]; + mainContext: StaticTaskContext; + executionSourcesVerified: boolean; + ancestry: { status: string; mergeBaseSha: string | null }; + blobs: Readonly>; +}): { + classification: "matching-main-diagnostics" | "unknown"; + reason: string; +}; + +export function diagnosticPaths(item: StaticDiagnostic): readonly string[]; +export function isFailedConclusion( + value: unknown, +): value is "failure" | "timed_out"; +export function isEligibleMainRun( + run: Record, + defaultBranch: string, + repository: string, +): boolean; +export function pullRequestBaseSha( + run: Record, + repository: string, +): string | null; +export function safeSha(value: unknown): string | null; +export function validRepoPath(value: unknown): value is string; +export function selectStaticCheckArtifact( + page: unknown, + run: { id?: string | number; head_sha?: string }, + runAttempt: number, +): unknown | null; + +export function readStaticCheckArtifact(options: { + repository: string; + artifactId: number | string; + token: string; + fetchImpl?: typeof fetch; +}): Promise<{ + context: StaticTaskContext; + diagnostics: readonly StaticDiagnostic[]; +}>; + +export function parseStaticArchive(archivePath: string): Promise<{ + context: StaticTaskContext; + diagnostics: readonly StaticDiagnostic[]; +}>; diff --git a/scripts/ci/artifacts.mjs b/scripts/ci/artifacts.mjs new file mode 100644 index 000000000..9a7cc1c21 --- /dev/null +++ b/scripts/ci/artifacts.mjs @@ -0,0 +1,436 @@ +import { execFile } from "node:child_process"; +import { mkdtemp, rm, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { isAbsolute, join } from "node:path"; +import { promisify } from "node:util"; + +import { parseStaticTaskContext } from "./static-task-context.mjs"; + +const execute = promisify(execFile); +const MAX_ARCHIVE_BYTES = 32 * 1024 * 1024; +const MAX_MEMBER_BYTES = 4 * 1024 * 1024; +const MAX_DIAGNOSTICS = 10_000; +const API_ROOT = "https://api.github.com/repos"; +const SHA = /^[0-9a-f]{40}$/iu; +const TASK_HASH = /^[a-f0-9]{16}$/u; + +/** Compare parsed static diagnostics only after source, task, host, and ancestry proof. */ +export function classifyFailure({ + repository, + defaultBranch, + run, + job, + diagnostics, + context, + mainRun, + mainJob, + mainDiagnostics, + mainContext, + executionSourcesVerified, + ancestry, + blobs, +}) { + const failedGate = firstFailure([ + () => [ + !isCompletedFailure(run) || + job?.status !== "completed" || + !isFailedConclusion(job?.conclusion), + "run_or_job_not_a_completed_failure", + ], + () => [ + !isEligiblePullRun(run, repository) || + !isEligibleMainRun(mainRun, defaultBranch, repository), + "run_event_or_repository_identity_unverified", + ], + () => [ + job?.name !== mainJob?.name || job?.name !== "Static checks", + "failed_job_not_the_same_static_gate", + ], + () => [ + !Array.isArray(diagnostics) || + diagnostics.length === 0 || + !Array.isArray(mainDiagnostics), + "structured_diagnostics_missing_or_invalid", + ], + () => [ + !validContext(context) || !validContext(mainContext), + "static_task_context_missing_or_invalid", + ], + () => [ + executionSourcesVerified !== true, + "checkout_sources_not_verified_against_api_commits", + ], + () => [ + !sameExecutionContext(context, mainContext), + "task_or_toolchain_identity_differs", + ], + () => [ + ancestry?.status !== "ahead" && ancestry?.status !== "identical", + "main_head_not_proven_ancestor_of_tested_base", + ], + () => [ + ancestry?.mergeBaseSha !== mainRun?.head_sha, + "main_head_is_not_the_verified_merge_base", + ], + () => [ + !sameTaskHashes(diagnostics, context, mainContext), + "owning_task_hash_differs", + ], + ]); + if (failedGate) return outcome("unknown", failedGate.reason); + const keys = diagnostics.map(diagnosticKey); + const mainKeys = new Set(mainDiagnostics.map(diagnosticKey).filter(Boolean)); + if (keys.some((key) => key === null)) + return outcome("unknown", "diagnostic_shape_not_supported"); + if (!keys.every((key) => mainKeys.has(key))) + return outcome( + "unknown", + "main_run_does_not_contain_the_same_structured_diagnostic", + ); + if (!sameBlobEvidence(diagnostics, blobs)) + return outcome( + "unknown", + "diagnostic_source_content_not_identical_at_main_and_tested_base", + ); + return outcome( + "matching-main-diagnostics", + "supported_diagnostics_match_failed_task_and_source_on_verified_main_ancestor", + ); +} + +export function diagnosticPaths(item) { + return item.kind === "jscpd" ? item.paths : [item.path]; +} +export function isFailedConclusion(value) { + return value === "failure" || value === "timed_out"; +} +export function isEligibleMainRun(run, defaultBranch, repository) { + return ( + isEligibleStaticRun(run) && + run.repository.full_name === repository && + run.head_branch === defaultBranch && + ["push", "schedule", "workflow_dispatch"].includes(run.event) + ); +} +export function pullRequestBaseSha(run, repository) { + if ( + run?.event !== "pull_request" || + run.repository?.full_name !== repository || + !Array.isArray(run.pull_requests) || + run.pull_requests.length !== 1 + ) + return null; + const base = run.pull_requests[0].base; + return base?.repo?.full_name === repository && SHA.test(base?.sha ?? "") + ? base.sha + : null; +} + +export function safeSha(value) { + return typeof value === "string" && SHA.test(value) + ? value.toLowerCase() + : null; +} +export function selectStaticCheckArtifact(page, run, runAttempt) { + if (runAttempt !== 1) return null; + if ( + !Array.isArray(page?.artifacts) || + !Number.isSafeInteger(page.total_count) || + page.total_count !== page.artifacts.length || + page.artifacts.length >= 100 + ) + return null; + const matches = page.artifacts.filter( + (artifact) => artifact.name === "static-check-reports", + ); + if (matches.length !== 1) return null; + const [artifact] = matches; + if ( + artifact.expired !== false || + String(artifact.workflow_run?.id) !== String(run.id) || + artifact.workflow_run?.head_sha !== run.head_sha + ) + return null; + return artifact; +} +export function validRepoPath(value) { + return ( + typeof value === "string" && + value.length > 0 && + !isAbsolute(value) && + !value.includes("\\") && + value + .split("/") + .every((part) => part !== "" && part !== "." && part !== "..") && + !value.startsWith("/") + ); +} + +/** Download one named static report archive and parse only whitelisted JSON members. */ +export async function readStaticCheckArtifact({ + repository, + artifactId, + token, + fetchImpl = fetch, +}) { + if (!/^[A-Za-z0-9_.-]+\/[A-Za-z0-9_.-]+$/u.test(repository ?? "")) + throw new Error("artifact repository identity is invalid"); + if (!token) throw new Error("artifact download requires a GitHub token"); + const url = `${API_ROOT}/${repository}/actions/artifacts/${positiveId(artifactId)}/zip`; + let response = await fetchImpl(url, { + headers: { + Accept: "application/vnd.github+json", + Authorization: `Bearer ${token}`, + "X-GitHub-Api-Version": "2022-11-28", + }, + signal: AbortSignal.timeout(10_000), + redirect: "manual", + }); + if ([301, 302, 303, 307, 308].includes(response.status)) { + const location = response.headers.get("location"); + if (!location || new URL(location).protocol !== "https:") + throw new Error("static_artifact_redirect_invalid"); + response = await fetchImpl(location, { + signal: AbortSignal.timeout(10_000), + redirect: "error", + }); + } + if (!response.ok) throw new Error(`github_http_${response.status}`); + const bytes = await readLimitedBytes(response, MAX_ARCHIVE_BYTES); + const directory = await mkdtemp(join(tmpdir(), "rea-ci-static-artifact-")); + const archive = join(directory, "artifact.zip"); + try { + await writeFile(archive, bytes, { flag: "wx" }); + return await parseStaticArchive(archive); + } finally { + await rm(directory, { recursive: true, force: true }); + } +} + +/** Accept only the three named producer outputs; no archive paths are written to disk. */ +export async function parseStaticArchive(archivePath) { + const { stdout: listing } = await execute("unzip", ["-Z1", archivePath], { + encoding: "utf8", + maxBuffer: 128 * 1024, + timeout: 5_000, + }); + const allowed = new Set([ + "oxlint.json", + "jscpd/jscpd-report.json", + "static-task-context.json", + ]); + const members = new Map(); + for (const name of listing.split(/\r?\n/u).filter(Boolean)) { + const prefix = ".cache/static-checks/"; + const member = name.startsWith(prefix) ? name.slice(prefix.length) : name; + if (!allowed.has(member)) continue; + if (members.has(member)) + throw new Error("static artifact contains duplicate report members"); + const { stdout } = await execute("unzip", ["-p", archivePath, name], { + encoding: "buffer", + maxBuffer: MAX_MEMBER_BYTES, + timeout: 5_000, + }); + members.set(member, JSON.parse(stdout.toString("utf8"))); + } + const context = parseStaticTaskContext( + members.get("static-task-context.json"), + ); + const diagnostics = normalizeDiagnostics( + members.get("oxlint.json"), + members.get("jscpd/jscpd-report.json"), + ); + if (diagnostics.length > MAX_DIAGNOSTICS) + throw new Error("static diagnostic count exceeds the reporting limit"); + return { context, diagnostics }; +} + +function normalizeDiagnostics(lint, duplicate) { + return [...normalizeLint(lint), ...normalizeDuplicates(duplicate)]; +} + +function normalizeLint(report) { + if (report === undefined) return []; + if (!Array.isArray(report?.diagnostics)) + throw new Error("oxlint diagnostics are malformed"); + return report.diagnostics + .filter((item) => item.severity === "error") + .map((item) => { + if ( + typeof item.filename !== "string" || + !repoPath(item.filename) || + typeof item.code !== "string" || + typeof item.message !== "string" + ) + throw new Error("oxlint error diagnostic is malformed"); + return { + kind: "lint", + path: item.filename, + code: item.code, + message: item.message, + severity: "error", + }; + }); +} + +function normalizeDuplicates(report) { + if (report === undefined) return []; + if ( + !Array.isArray(report?.duplicates) || + !Number.isFinite(report.statistics?.total?.percentage) || + report.statistics.total.percentage < 0 + ) + throw new Error("jscpd report is malformed"); + // The producer owns the checker outcome; do not copy or infer its threshold. + const exitCode = report.reaCheck?.exitCode; + if (exitCode === undefined || exitCode === 0) return []; + if (!Number.isSafeInteger(exitCode) || exitCode < 0) + throw new Error("jscpd checker outcome is malformed"); + if (exitCode !== 1) return []; + return report.duplicates.map((item) => { + const paths = [item.firstFile?.name, item.secondFile?.name].sort(); + if (paths.some((path) => !repoPath(path))) + throw new Error("jscpd duplicate paths are malformed"); + return { kind: "jscpd", paths }; + }); +} + +async function readLimitedBytes(response, limit) { + const reader = response.body?.getReader(); + if (!reader) throw new Error("static artifact response has no body"); + const chunks = []; + let size = 0; + try { + for (;;) { + const { done, value } = await reader.read(); + if (done) break; + size += value.byteLength; + if (size > limit) { + await reader.cancel(); + throw new Error("static artifact exceeds the download limit"); + } + chunks.push(value); + } + } finally { + reader.releaseLock(); + } + const result = new Uint8Array(size); + let offset = 0; + for (const chunk of chunks) { + result.set(chunk, offset); + offset += chunk.byteLength; + } + return result; +} + +function repoPath(value) { + return ( + typeof value === "string" && + value.length > 0 && + !value.startsWith("/") && + !value.includes("\\") && + value + .split("/") + .every((part) => part !== "" && part !== "." && part !== "..") + ); +} + +function positiveId(value) { + const id = Number(value); + if (!Number.isSafeInteger(id) || id < 1) + throw new Error("artifact ID must be a positive integer"); + return String(id); +} + +function diagnosticKey(item) { + if ( + item?.kind === "lint" && + validRepoPath(item.path) && + typeof item.code === "string" && + typeof item.message === "string" && + item.severity === "error" + ) + return JSON.stringify([ + item.kind, + item.path, + item.code, + item.message, + item.severity, + ]); + if ( + item?.kind === "jscpd" && + Array.isArray(item.paths) && + item.paths.length === 2 && + item.paths.every(validRepoPath) + ) + return JSON.stringify([item.kind, ...[...item.paths].sort()]); + return null; +} +function sameBlobEvidence(diagnostics, blobs) { + const paths = [...new Set(diagnostics.flatMap(diagnosticPaths))]; + return paths.length > 0 && paths.every((path) => blobs?.[path] === true); +} +function sameTaskHashes(diagnostics, left, right) { + return diagnostics.every((diagnostic) => { + const task = diagnostic.kind === "jscpd" ? "jscpd" : "lint:built"; + return ( + TASK_HASH.test(left.taskHashes[task] ?? "") && + left.taskHashes[task] === right.taskHashes[task] + ); + }); +} +function validContext(value) { + return ( + value?.version === 1 && + SHA.test(value.sourceSha ?? "") && + value.sourceClean === true && + typeof value.nodeVersion === "string" && + typeof value.npmVersion === "string" && + typeof value.turboVersion === "string" && + typeof value.platform === "string" && + typeof value.architecture === "string" && + value.taskHashes && + typeof value.taskHashes === "object" && + !Array.isArray(value.taskHashes) + ); +} +function sameExecutionContext(left, right) { + return [ + "nodeVersion", + "npmVersion", + "turboVersion", + "platform", + "architecture", + ].every((key) => left[key] === right[key]); +} +function isEligiblePullRun(run, repository) { + return ( + isEligibleStaticRun(run) && pullRequestBaseSha(run, repository) !== null + ); +} +function isEligibleStaticRun(run) { + return ( + isRecord(run) && + run.status === "completed" && + isFailedConclusion(run.conclusion) && + SHA.test(run.head_sha ?? "") && + typeof run.repository?.full_name === "string" && + run.workflowPath?.file === ".github/workflows/ci.yml" + ); +} +function isCompletedFailure(run) { + return run?.status === "completed" && isFailedConclusion(run.conclusion); +} +function isRecord(value) { + return typeof value === "object" && value !== null && !Array.isArray(value); +} +function firstFailure(checks) { + for (const check of checks) { + const [failed, reason] = check(); + if (failed) return { reason }; + } + return null; +} +function outcome(classification, reason) { + return { classification, reason }; +} diff --git a/scripts/ci/failure-report.d.mts b/scripts/ci/failure-report.d.mts new file mode 100644 index 000000000..47ac65745 --- /dev/null +++ b/scripts/ci/failure-report.d.mts @@ -0,0 +1,159 @@ +import type { NormalizedWorkflowRun } from "./workflow-run.mjs"; +import type { StaticDiagnostic } from "./artifacts.mjs"; +export type { StaticDiagnostic } from "./artifacts.mjs"; +import type { StaticTaskContext } from "./static-task-context.mjs"; +export type { StaticTaskContext } from "./static-task-context.mjs"; +export interface CiRun { + readonly id?: string | number; + readonly run_attempt?: number; + readonly status?: string; + readonly conclusion?: string | null; + readonly event?: string; + readonly path?: string; + readonly head_sha?: string; + readonly head_branch?: string; + readonly repository?: { readonly full_name?: string }; + readonly pull_requests?: readonly { + readonly base?: { + readonly sha?: string; + readonly repo?: { readonly full_name?: string }; + }; + }[]; +} + +export interface CiJob { + readonly name?: string; + readonly status?: string; + readonly conclusion?: string | null; + readonly run_id?: number; + readonly run_attempt?: number; +} + +export interface StaticEvidence { + readonly context: StaticTaskContext; + readonly diagnostics: readonly StaticDiagnostic[]; +} + +export interface CiFailureReport { + readonly version: 1; + readonly observedAt: string; + readonly repository: string; + readonly defaultBranch: string; + readonly informationalOnly: true; + readonly gateEffect: string; + readonly boundedInput: { + readonly candidateRuns: number; + readonly limit: number; + readonly mainCandidateLimit: number; + readonly mainCandidateRuns: number; + readonly mainRunLookupLimit: number; + readonly manifestTruncated: boolean; + }; + readonly counts: Readonly< + Record<"matching-main-diagnostics" | "unknown", number> + >; + readonly failures: readonly { + readonly runId: number | string; + readonly runAttempt: number; + readonly classification: "matching-main-diagnostics" | "unknown"; + readonly reason: string; + readonly candidateFailures?: readonly { + readonly runId: number | string; + readonly runAttempt: number; + readonly reason: string; + }[]; + readonly runHead?: string | null; + readonly event?: string | null; + readonly job?: string; + readonly conclusion?: string | null; + }[]; +} + +export interface FailureReportApi { + getRun(repository: string, id: string | number): Promise; + getJobs( + repository: string, + id: string | number, + attempt: number, + ): Promise; + getStaticEvidence( + repository: string, + run: CiRun, + job: CiJob, + ): Promise; + getAncestry( + repository: string, + ancestor: string, + descendant: string, + ): Promise<{ status: string; mergeBaseSha: string | null }>; + verifyExecutionSource( + repository: string, + run: CiRun, + context: StaticTaskContext, + ): Promise; + getBlobIdentity( + repository: string, + ancestor: string, + descendant: string, + paths: readonly string[], + ): Promise>>; +} + +export function classifyFailure(input: { + repository: string; + defaultBranch: string; + run: CiRun & NormalizedWorkflowRun; + job: CiJob; + diagnostics: readonly StaticDiagnostic[]; + context: StaticTaskContext; + mainRun: CiRun & NormalizedWorkflowRun; + mainJob: CiJob; + mainDiagnostics: readonly StaticDiagnostic[]; + mainContext: StaticTaskContext; + executionSourcesVerified: boolean; + ancestry: { status: string; mergeBaseSha: string | null }; + blobs: Readonly>; +}): { + classification: "matching-main-diagnostics" | "unknown"; + reason: string; +}; + +export function reportCiFailures(options: { + repository: string; + defaultBranch: string; + batchManifest: { + readonly repository: string; + readonly runs: readonly { + readonly id: number | string; + readonly runAttempt: number; + readonly workflow: string; + readonly event?: string | null; + readonly headBranch?: string | null; + readonly conclusion: string | null; + readonly status: string; + }[]; + readonly truncation?: readonly unknown[]; + readonly coverage?: { readonly exhaustive?: boolean }; + readonly selection?: { readonly exhaustive?: boolean }; + }; + api: FailureReportApi; + outputDirectory?: string; + summaryPath?: string; + observedAt?: string; +}): Promise; + +export function createGitHubApi(options?: { + token?: string; + fetchImpl?: typeof fetch; + apiRequest?: ( + method: string, + endpoint: string, + budget: { requests: number; startedAt: number; limitMs: number }, + ) => Promise; +}): FailureReportApi; + +export function writeFailureReport( + report: CiFailureReport, + outputDirectory?: string, + summaryPath?: string, +): Promise<{ jsonPath: string; markdownPath: string }>; diff --git a/scripts/ci/failure-report.mjs b/scripts/ci/failure-report.mjs new file mode 100644 index 000000000..2ce6d813c --- /dev/null +++ b/scripts/ci/failure-report.mjs @@ -0,0 +1,529 @@ +import { appendFile, mkdir, readFile, writeFile } from "node:fs/promises"; +import { join, resolve } from "node:path"; +import { fileURLToPath } from "node:url"; + +import { + classifyFailure, + diagnosticPaths, + isEligibleMainRun, + isFailedConclusion, + pullRequestBaseSha, + readStaticCheckArtifact, + safeSha, + selectStaticCheckArtifact, + validRepoPath, +} from "./artifacts.mjs"; +import { createGhApiRequest } from "./github-api.mjs"; +import { normalizeWorkflowRun } from "./workflow-run.mjs"; + +const OUTPUT_DIRECTORY = join(".cache", "ci-failures"); +const BATCH_DIRECTORY = join(".cache", "ci-job-timings"); +const MAX_JSON_BYTES = 8 * 1024 * 1024; +const MAX_FAILED_RUNS = 20; +const MAX_MAIN_CANDIDATES = 5; +const MAX_MAIN_RUN_LOOKUPS = 100; +const MAX_PATHS_PER_COMPARISON = 50; +const MAX_API_REQUESTS = 250; + +export { classifyFailure } from "./artifacts.mjs"; + +/** Uses only failed CI candidates present in the supplied bounded timing manifest. */ +export async function reportCiFailures({ + repository, + defaultBranch, + batchManifest, + api, + outputDirectory = OUTPUT_DIRECTORY, + summaryPath, + observedAt = new Date().toISOString(), +}) { + validateRepository(repository); + if (typeof defaultBranch !== "string" || defaultBranch.length === 0) + throw new Error("defaultBranch is required"); + if ( + !batchManifest || + batchManifest.repository !== repository || + !Array.isArray(batchManifest.runs) + ) + throw new Error("timing manifest repository or runs are invalid"); + + const failedRuns = batchManifest.runs.filter( + (entry) => + entry.workflow === "CI" && + entry.status === "collected" && + isFailedConclusion(entry.conclusion), + ); + const prRuns = failedRuns.filter((entry) => entry.event === "pull_request"); + const candidates = prRuns.slice(0, MAX_FAILED_RUNS); + const mainEntries = failedRuns.filter( + (entry) => + entry.headBranch === defaultBranch && + ["push", "schedule", "workflow_dispatch"].includes(entry.event), + ); + const lookupRun = verifiedRunLookup(repository, api); + const mainCandidates = []; + for (const entry of mainEntries.slice(0, MAX_MAIN_RUN_LOOKUPS)) { + try { + const run = await lookupRun(entry); + if (isEligibleMainRun(run, defaultBranch, repository)) + mainCandidates.push({ + run, + getEvidence: mainEvidenceLookup(repository, api, entry, run), + }); + if (mainCandidates.length === MAX_MAIN_CANDIDATES) break; + } catch { + // Unverified metadata cannot consume the eligible-main candidate budget. + } + } + const entries = []; + for (const entry of candidates) { + try { + entries.push( + ...(await analyzeFailedRun({ + repository, + defaultBranch, + entry, + run: await lookupRun(entry), + mainCandidates, + api, + })), + ); + } catch (error) { + entries.push({ + runId: entry.id, + runAttempt: entry.runAttempt, + classification: "unknown", + reason: errorCode(error), + }); + } + } + const report = { + version: 1, + observedAt, + repository, + defaultBranch, + informationalOnly: true, + gateEffect: "none; this report never changes CI conclusions", + boundedInput: { + candidateRuns: candidates.length, + limit: MAX_FAILED_RUNS, + mainCandidateLimit: MAX_MAIN_CANDIDATES, + mainCandidateRuns: mainCandidates.length, + mainRunLookupLimit: MAX_MAIN_RUN_LOOKUPS, + manifestTruncated: + prRuns.length > MAX_FAILED_RUNS || + batchManifest.truncation?.length > 0 || + batchManifest.selection?.exhaustive === false || + batchManifest.coverage?.exhaustive === false, + }, + counts: Object.fromEntries( + ["matching-main-diagnostics", "unknown"].map((category) => [ + category, + entries.filter((item) => item.classification === category).length, + ]), + ), + failures: entries, + }; + await writeFailureReport(report, outputDirectory, summaryPath); + return report; +} + +async function analyzeFailedRun({ + repository, + defaultBranch, + entry, + run, + mainCandidates, + api, +}) { + const jobs = await api.getJobs(repository, entry.id, entry.runAttempt); + if (jobs.some((job) => !hasJobIdentity(job, entry))) + throw new Error("workflow_job_identity_mismatch"); + const failedJobs = jobs.filter( + (job) => job.status === "completed" && isFailedConclusion(job.conclusion), + ); + if (failedJobs.length === 0) + return [ + { + runId: entry.id, + runAttempt: entry.runAttempt, + classification: "unknown", + reason: "failed_jobs_not_present_in_api_snapshot", + }, + ]; + const failures = []; + for (const job of failedJobs) { + const evidence = await api.getStaticEvidence(repository, run, job); + const classification = await attributeToMain({ + repository, + defaultBranch, + run, + job, + evidence, + mainCandidates, + api, + }); + failures.push({ + runId: entry.id, + runAttempt: entry.runAttempt, + runHead: safeSha(run.head_sha), + event: run.event ?? null, + job: job.name, + conclusion: job.conclusion, + ...classification, + }); + } + return failures; +} + +async function attributeToMain({ + repository, + defaultBranch, + run, + job, + evidence, + mainCandidates, + api, +}) { + const baseSha = pullRequestBaseSha(run, repository); + if (baseSha === null || evidence === null) + return outcome("unknown", "no_matching_verified_main_failure"); + let result = outcome("unknown", "no_matching_verified_main_failure"); + const candidateFailures = []; + for (const candidate of mainCandidates) { + try { + const main = await compareCandidate({ + repository, + defaultBranch, + run, + job, + evidence, + baseSha, + candidate, + api, + }); + if (main !== null) result = main; + if (main?.classification === "matching-main-diagnostics") break; + } catch (error) { + candidateFailures.push({ + runId: candidate.run.id, + runAttempt: candidate.run.run_attempt, + reason: errorCode(error), + }); + } + } + return { + ...result, + ...(candidateFailures.length > 0 ? { candidateFailures } : {}), + }; +} + +async function compareCandidate({ + repository, + defaultBranch, + run, + job, + evidence, + baseSha, + candidate, + api, +}) { + const mainRun = candidate.run; + const main = await candidate.getEvidence(job.name); + if (!main?.evidence) return null; + const { job: mainJob, evidence: mainEvidence } = main; + const ancestry = await api.getAncestry(repository, mainRun.head_sha, baseSha); + const paths = [ + ...new Set(evidence.diagnostics.flatMap(diagnosticPaths)), + ].slice(0, MAX_PATHS_PER_COMPARISON); + const blobs = await api.getBlobIdentity( + repository, + mainRun.head_sha, + baseSha, + paths, + ); + const executionSourcesVerified = + (await api.verifyExecutionSource(repository, run, evidence.context)) && + (await api.verifyExecutionSource( + repository, + mainRun, + mainEvidence.context, + )); + return classifyFailure({ + repository, + defaultBranch, + run, + job, + diagnostics: evidence.diagnostics, + context: evidence.context, + mainRun, + mainJob, + mainDiagnostics: mainEvidence.diagnostics, + mainContext: mainEvidence.context, + executionSourcesVerified, + ancestry, + blobs, + }); +} + +// Each verified main run/attempt owns its lazy jobs and parsed evidence for this +// report only. Cache nulls and rejected reads too; retries cannot spend the budget +// repeatedly on an immutable observation that was unavailable or invalid. +function mainEvidenceLookup(repository, api, entry, run) { + let jobs; + const evidence = new Map(); + return (name) => { + jobs ??= Promise.resolve().then(async () => { + const items = await api.getJobs(repository, entry.id, entry.runAttempt); + return items.some((item) => !hasJobIdentity(item, entry)) ? [] : items; + }); + if (!evidence.has(name)) + evidence.set( + name, + jobs.then(async (items) => { + const job = items.find( + (item) => + item.name === name && + item.status === "completed" && + isFailedConclusion(item.conclusion), + ); + if (!job) return null; + return { + job, + evidence: await api.getStaticEvidence(repository, run, job), + }; + }), + ); + return evidence.get(name); + }; +} + +// Selection and attribution share one immutable API observation per run/attempt. +function verifiedRunLookup(repository, api) { + const runs = new Map(); + return (entry) => { + const key = `${positiveId(entry.id)}:${positiveId(entry.runAttempt)}`; + if (!runs.has(key)) + runs.set( + key, + Promise.resolve().then(async () => { + const run = normalizeWorkflowRun( + await api.getRun(repository, entry.id), + ); + if (!hasRunIdentity(run, entry)) + throw new Error("workflow_run_identity_mismatch"); + return run; + }), + ); + return runs.get(key); + }; +} + +function hasRunIdentity(run, entry) { + return ( + String(run?.id) === String(entry.id) && + Number(run.run_attempt) === Number(entry.runAttempt) && + run.workflowPath?.file === ".github/workflows/ci.yml" + ); +} +function hasJobIdentity(job, entry) { + return ( + String(job.run_id) === String(entry.id) && + Number(job.run_attempt) === Number(entry.runAttempt) + ); +} + +export function createGitHubApi({ + token = process.env.GH_TOKEN, + fetchImpl = fetch, + apiRequest = createGhApiRequest(), +} = {}) { + if (!token) throw new Error("GH_TOKEN is required"); + const budget = { requests: 0, startedAt: Date.now(), limitMs: 60_000 }; + const request = async (endpoint) => { + if (budget.requests >= MAX_API_REQUESTS) + throw new Error("github_api_request_budget_exhausted"); + budget.requests += 1; + return await apiRequest("GET", endpoint, budget); + }; + return { + async getRun(repository, id) { + return await request( + `repos/${repository}/actions/runs/${positiveId(id)}`, + ); + }, + async getJobs(repository, id, attempt) { + const page = await request( + `repos/${repository}/actions/runs/${positiveId(id)}/attempts/${positiveId(attempt)}/jobs?per_page=100&page=1`, + ); + if (!Array.isArray(page.jobs) || page.jobs.length >= 100) + throw new Error("job_list_incomplete_or_malformed"); + return page.jobs; + }, + async getStaticEvidence(repository, run, job) { + if (job.name !== "Static checks") return null; + if (run.run_attempt !== 1) return null; + const page = await request( + `repos/${repository}/actions/runs/${positiveId(run.id)}/artifacts?per_page=100&page=1`, + ); + const artifact = selectStaticCheckArtifact(page, run, run.run_attempt); + if (!artifact) return null; + return await readStaticCheckArtifact({ + repository, + artifactId: artifact.id, + token, + fetchImpl, + }); + }, + async getAncestry(repository, ancestor, descendant) { + const comparison = await request( + `repos/${repository}/compare/${safeSha(ancestor)}...${safeSha(descendant)}`, + ); + return { + status: comparison.status, + mergeBaseSha: safeSha(comparison.merge_base_commit?.sha), + }; + }, + async verifyExecutionSource(repository, run, context) { + const sourceSha = safeSha(context.sourceSha); + if (!sourceSha) return false; + const pr = + run.event === "pull_request" + ? pullRequestBaseSha(run, repository) + : null; + if ( + run.event === "pull_request" && + (pr === null || context.prHeadSha !== run.head_sha) + ) + return false; + const commit = await request(`repos/${repository}/commits/${sourceSha}`); + if (safeSha(commit.sha) !== sourceSha) return false; + if (sourceSha === run.head_sha) return true; + if ( + pr === null || + !Array.isArray(commit.parents) || + commit.parents.length !== 2 + ) + return false; + return ( + safeSha(commit.parents[0]?.sha) === pr && + safeSha(commit.parents[1]?.sha) === run.head_sha + ); + }, + async getBlobIdentity(repository, ancestor, descendant, paths) { + const result = {}; + for (const path of paths) { + if (!validRepoPath(path)) continue; + const [left, right] = await Promise.all([ + getContent(repository, ancestor, path, request), + getContent(repository, descendant, path, request), + ]); + result[path] = left !== null && left === right; + } + return result; + }, + }; +} + +async function getContent(repository, ref, path, request) { + try { + const encodedPath = path.split("/").map(encodeURIComponent).join("/"); + const value = await request( + `repos/${repository}/contents/${encodedPath}?ref=${encodeURIComponent(ref)}`, + ); + return typeof value.sha === "string" ? value.sha : null; + } catch { + return null; + } +} + +function outcome(classification, reason) { + return { classification, reason }; +} +function validateRepository(value) { + if ( + typeof value !== "string" || + !/^[A-Za-z0-9_.-]+\/[A-Za-z0-9_.-]+$/u.test(value) + ) + throw new Error("repository must be owner/name"); +} +function positiveId(value) { + if (!Number.isSafeInteger(Number(value)) || Number(value) < 1) + throw new Error( + "GitHub run and attempt IDs must be positive safe integers", + ); + return String(Number(value)); +} +function errorCode(error) { + return error instanceof Error ? error.message.slice(0, 160) : "unknown_error"; +} + +export async function writeFailureReport( + report, + outputDirectory = OUTPUT_DIRECTORY, + summaryPath, +) { + await mkdir(outputDirectory, { recursive: true }); + const jsonPath = join(outputDirectory, "report.json"); + const markdownPath = join(outputDirectory, "report.md"); + const matchedDiagnostics = report.failures.filter( + (item) => item.classification === "matching-main-diagnostics", + ); + const lines = [ + "# CI failure attribution", + "", + "Informational only. This report never changes a CI conclusion or required gate.", + "", + `Observed ${report.observedAt}; ${String(matchedDiagnostics.length)} of ${String(report.failures.length)} failure reports contain supported diagnostics matching a failed static task on an ancestor main revision. This does not attribute other failures in the same job. Unknown means evidence was incomplete or did not match.`, + "", + ]; + for (const item of report.failures) + lines.push( + `- Run ${item.runId} attempt ${item.runAttempt}${item.job ? `, ${item.job}` : ""}: **${item.classification}** — ${item.reason}.`, + ); + if (report.failures.length === 0) + lines.push( + "- No completed failed CI attempts were present in the supplied timing snapshot.", + ); + await writeFile(jsonPath, `${JSON.stringify(report, null, 2)}\n`, "utf8"); + await writeFile(markdownPath, `${lines.join("\n")}\n`, "utf8"); + if (summaryPath) + await appendFile(summaryPath, `${lines.join("\n")}\n`, "utf8"); + return { jsonPath, markdownPath }; +} + +async function main() { + const repository = process.env.CI_REPOSITORY; + const defaultBranch = process.env.CI_DEFAULT_BRANCH; + const batchPath = + process.env.CI_TIMING_MANIFEST ?? + join(BATCH_DIRECTORY, "batch-manifest.json"); + const batchManifest = JSON.parse(await readBoundedFile(batchPath)); + const api = createGitHubApi(); + await reportCiFailures({ + repository, + defaultBranch, + batchManifest, + api, + outputDirectory: process.env.CI_FAILURES_DIRECTORY ?? OUTPUT_DIRECTORY, + summaryPath: process.env.GITHUB_STEP_SUMMARY, + }); +} +async function readBoundedFile(path) { + const bytes = await readFile(path); + if (bytes.byteLength > MAX_JSON_BYTES) + throw new Error("timing manifest exceeds size limit"); + return bytes.toString("utf8"); +} + +if ( + process.argv[1] && + resolve(process.argv[1]) === fileURLToPath(import.meta.url) +) { + try { + await main(); + } catch (error) { + console.error(error instanceof Error ? error.message : String(error)); + process.exitCode = 1; + } +} diff --git a/scripts/ci/github-api.d.mts b/scripts/ci/github-api.d.mts new file mode 100644 index 000000000..67e157e6d --- /dev/null +++ b/scripts/ci/github-api.d.mts @@ -0,0 +1,16 @@ +export interface GitHubApiBudget { + requests: number; + startedAt: number; + limitMs: number; +} + +export type GitHubApiRequest = ( + method: string, + endpoint: string, + budget: GitHubApiBudget, +) => Promise; + +export function createGhApiRequest(options?: { + env?: NodeJS.ProcessEnv; + now?: () => number; +}): GitHubApiRequest; diff --git a/scripts/ci/github-api.mjs b/scripts/ci/github-api.mjs new file mode 100644 index 000000000..9567bf93b --- /dev/null +++ b/scripts/ci/github-api.mjs @@ -0,0 +1,33 @@ +import { execFile } from "node:child_process"; +import { promisify } from "node:util"; + +const execFileAsync = promisify(execFile); +const MAX_BUFFER = 8 * 1024 * 1024; +const REQUEST_TIMEOUT_MS = 10_000; + +/** Create a bounded GitHub CLI JSON transport with GET-only authority. */ +export function createGhApiRequest({ env = process.env, now = Date.now } = {}) { + return async (method, endpoint, budget) => { + if (method !== "GET") + throw new Error( + "GitHub API transport supports read-only GET requests only", + ); + const remaining = budget.limitMs - (now() - budget.startedAt); + if (remaining <= 0) throw new Error("lookup_time_budget_exhausted"); + const { stdout } = await execFileAsync( + "gh", + ["api", "--method", "GET", endpoint], + { + env, + encoding: "utf8", + maxBuffer: MAX_BUFFER, + timeout: Math.min(REQUEST_TIMEOUT_MS, remaining), + }, + ); + try { + return JSON.parse(stdout); + } catch (cause) { + throw new Error("github_api_malformed_json", { cause }); + } + }; +} diff --git a/scripts/ci/job-timings-batch.mjs b/scripts/ci/job-timings-batch.mjs index 842e884f0..f2dca7143 100644 --- a/scripts/ci/job-timings-batch.mjs +++ b/scripts/ci/job-timings-batch.mjs @@ -103,6 +103,9 @@ export async function collectJobTimingsBatch({ id: run.id, runAttempt: run.run_attempt, workflow: workflow.name, + event: typeof run.event === "string" ? run.event : null, + headBranch: + typeof run.head_branch === "string" ? run.head_branch : null, conclusion: run.conclusion ?? null, createdAt: run.created_at ?? null, }); diff --git a/scripts/ci/pr-ci-health.d.mts b/scripts/ci/pr-ci-health.d.mts new file mode 100644 index 000000000..0db2454bd --- /dev/null +++ b/scripts/ci/pr-ci-health.d.mts @@ -0,0 +1,95 @@ +export interface PrCiHead { + repositoryId: number | null; + repository?: string | null; + ref: string | null; + sha: string | null; +} + +export interface PrCiPullRequest { + number: number | null; + draft?: boolean; + identityValid: boolean; + mergeable?: boolean; + head: PrCiHead; + [key: string]: unknown; +} + +export interface PrCiWorkflowRun { + id?: number; + run_attempt?: number; + event?: string; + head_branch?: string; + head_repository?: { id?: number | null } | null; + head_sha?: string; + status?: string; + conclusion?: string | null; + created_at?: string; + html_url?: string; + pull_requests?: Array<{ + number?: number; + head?: { sha?: string; repo?: { id?: number | null } } | null; + }>; + [key: string]: unknown; +} + +export interface PrCiClassification { + category: string; + reason: string; + runs: Array>; +} + +export interface PrCiHealthReport { + schemaVersion: 1; + observedAt: string; + repository: string; + repositoryId: number | null; + defaultBranch: string; + lookbackDays: number; + scan: { + status: "complete" | "partial"; + cutoff: string; + pullRequestPages: number; + workflowRunPages: number; + pullRequestInventoryComplete: boolean; + workflowRunInventoryComplete: boolean; + pullRequestDetailsFetched: number; + pullRequestDetailsLimit: number; + apiRequests: number; + apiRequestLimit: number; + errors: string[]; + }; + counts: Record; + pullRequests: Array; +} + +export function collectPrCiHealth(options: { + repository: string; + repositoryId?: number | string | null; + defaultBranch: string; + lookbackDays?: number; + now?: number; + apiRequest?: GitHubApiRequest; +}): Promise; + +export function classifyPrCi(input: { + pull: PrCiPullRequest; + runs: Array<{ run: PrCiWorkflowRun; basis: string }>; + runsComplete: boolean; + pullRequestInventoryComplete: boolean; + ambiguousRuns: Array<{ run: PrCiWorkflowRun; basis: string }>; +}): PrCiClassification; + +export function matchRunToPullRequest(input: { + run: PrCiWorkflowRun; + pull: PrCiPullRequest; + identityCounts: Map; + inventoryComplete?: boolean; +}): { kind: "none" } | { kind: "ambiguous" } | { kind: "match"; basis: string }; + +export function writePrCiHealthReport( + report: PrCiHealthReport, + outputDirectory?: string, +): Promise<{ jsonPath: string; markdownPath: string }>; + +export function renderMarkdownSummary(report: PrCiHealthReport): string; +import type { GitHubApiRequest } from "./github-api.mjs"; diff --git a/scripts/ci/pr-ci-health.mjs b/scripts/ci/pr-ci-health.mjs new file mode 100644 index 000000000..d312dcecd --- /dev/null +++ b/scripts/ci/pr-ci-health.mjs @@ -0,0 +1,718 @@ +import { appendFile, mkdir, writeFile } from "node:fs/promises"; +import { join, resolve } from "node:path"; +import { fileURLToPath } from "node:url"; + +import { createGhApiRequest } from "./github-api.mjs"; + +const OUTPUT_DIRECTORY = join(".cache", "ci-pr-health"); +const PAGE_SIZE = 100; +const MAX_PULL_REQUEST_PAGES = 3; +const MAX_WORKFLOW_RUN_PAGES = 5; +const MAX_PULL_REQUEST_DETAILS = 40; +const MAX_API_REQUESTS = + MAX_PULL_REQUEST_PAGES + MAX_WORKFLOW_RUN_PAGES + MAX_PULL_REQUEST_DETAILS; +const LOOKBACK_DAYS = 14; +const MAX_LOOKBACK_DAYS = 30; +const MAX_TOTAL_TIME_MS = 60_000; +const SHA_PATTERN = /^[0-9a-f]{40}$/iu; +const COMPLETED_CATEGORIES = new Map([ + ["failure", "completed_failure"], + ["timed_out", "completed_failure"], + ["cancelled", "completed_canceled"], + ["success", "completed_success"], +]); + +/** Read workflow/PR state and conservatively distinguish absent CI from unknown. */ +export async function collectPrCiHealth({ + repository, + repositoryId, + defaultBranch, + lookbackDays = LOOKBACK_DAYS, + now = Date.now(), + apiRequest = createGhApiRequest(), +}) { + const identity = validateInputs({ + repository, + repositoryId, + defaultBranch, + lookbackDays, + now, + }); + const observedAt = new Date(now).toISOString(); + const cutoff = now - identity.lookbackDays * 24 * 60 * 60 * 1_000; + const snapshot = await collectApiSnapshot({ apiRequest, identity, cutoff }); + const classified = classifyPullRequests(snapshot); + const { pulls, runs, budget } = snapshot; + const errors = [pulls.error, runs.error, ...snapshot.detailsErrors].filter( + Boolean, + ); + return { + schemaVersion: 1, + observedAt, + repository: identity.repository, + repositoryId: identity.repositoryId, + defaultBranch: identity.defaultBranch, + lookbackDays: identity.lookbackDays, + scan: { + status: + errors.length === 0 && pulls.complete && runs.complete + ? "complete" + : "partial", + cutoff: new Date(cutoff).toISOString(), + pullRequestPages: pulls.pages, + workflowRunPages: runs.pages, + pullRequestInventoryComplete: pulls.complete, + workflowRunInventoryComplete: runs.complete, + pullRequestDetailsFetched: snapshot.detailsFetched, + pullRequestDetailsLimit: MAX_PULL_REQUEST_DETAILS, + apiRequests: budget.requests, + apiRequestLimit: MAX_API_REQUESTS, + errors, + }, + counts: countCategories(classified), + pullRequests: classified, + }; +} + +async function collectApiSnapshot({ apiRequest, identity, cutoff }) { + const budget = { + requests: 0, + startedAt: Date.now(), + limitMs: MAX_TOTAL_TIME_MS, + }; + const requestGet = async (endpoint) => { + if (budget.requests >= MAX_API_REQUESTS) + throw new Error("lookup_request_budget_exhausted"); + if (Date.now() - budget.startedAt >= budget.limitMs) + throw new Error("lookup_time_budget_exhausted"); + budget.requests += 1; + return await apiRequest("GET", endpoint, budget); + }; + const pulls = await listOpenPullRequests({ + requestGet, + identity, + }); + const candidates = pulls.items.filter( + (pull) => + !pull.draft && + Number.isFinite(Date.parse(pull.createdAt ?? "")) && + Date.parse(pull.createdAt ?? "") >= cutoff && + pull.base.ref === identity.defaultBranch && + (identity.repositoryId === null || + pull.base.repositoryId === identity.repositoryId), + ); + const runs = await listRecentWorkflowRuns({ + requestGet, + identity, + cutoff, + }); + + const detailResult = await resolveMergeability({ + candidates, + requestGet, + identity, + }); + return { + budget, + pulls, + runs, + candidates, + detailsFetched: detailResult.fetched, + detailsErrors: detailResult.errors, + }; +} + +function classifyPullRequests(snapshot) { + const { candidates, pulls, runs } = snapshot; + const identityCounts = countHeadIdentities(pulls.items); + return candidates.map((pull) => { + const relevant = []; + const ambiguousRuns = []; + for (const run of runs.items) { + const match = matchRunToPullRequest({ + run, + pull, + identityCounts, + inventoryComplete: pulls.complete, + }); + if (match.kind === "match") relevant.push({ run, basis: match.basis }); + if (match.kind === "ambiguous") + ambiguousRuns.push({ run, basis: "ambiguous_head_identity" }); + } + return { + ...pull, + ci: classifyPrCi({ + pull, + runs: relevant, + runsComplete: runs.complete, + pullRequestInventoryComplete: pulls.complete, + ambiguousRuns, + }), + }; + }); +} + +async function resolveMergeability({ candidates, requestGet, identity }) { + let detailsFetched = 0; + const errors = new Set(); + for (const pull of candidates) { + if (pull.mergeable !== undefined) continue; + if (detailsFetched >= MAX_PULL_REQUEST_DETAILS) { + errors.add("pull_request_mergeability_lookup_budget_exhausted"); + continue; + } + const result = await getPullRequestDetails({ requestGet, identity, pull }); + detailsFetched += 1; + if (result.error !== null) { + errors.add(result.error); + continue; + } + const { detail } = result; + const identityChanged = + detail.headSha !== pull.head.sha || + detail.headRef !== pull.head.ref || + detail.headRepositoryId !== pull.head.repositoryId || + detail.baseRef !== pull.base.ref || + detail.baseRepositoryId !== pull.base.repositoryId; + if (identityChanged) { + pull.identityValid = false; + pull.identityChangedDuringScan = true; + errors.add("pull_request_identity_changed_during_scan"); + } else { + pull.mergeable = detail.mergeable; + pull.detailFetched = true; + if (detail.mergeable === undefined) + errors.add("pull_request_mergeability_unknown"); + } + } + return { fetched: detailsFetched, errors: [...errors] }; +} + +function countCategories(pulls) { + const categories = [ + "missing", + "startup_failure", + "queued_pending_executing", + "approval_required", + "completed_success", + "completed_failure", + "completed_canceled", + "completed_other", + "merge_conflict", + "unknown", + "deferred", + ]; + return Object.fromEntries( + categories.map((category) => [ + category, + pulls.filter(({ ci }) => ci.category === category).length, + ]), + ); +} + +/** Pure status classifier; missing requires complete, identity-safe inventories. */ +export function classifyPrCi({ + pull, + runs, + runsComplete, + pullRequestInventoryComplete, + ambiguousRuns, +}) { + if (!pull.identityValid) + return result("unknown", "pull_request_head_identity_invalid", []); + const evidence = [...runs, ...ambiguousRuns]; + if (pull.mergeable === false) + return result("merge_conflict", "github_reports_merge_conflict", evidence); + // The newest collected run is not proven current until pagination completes: + // an older-created run outside this snapshot may have been retried recently. + if (!runsComplete) + return result( + evidence.length > 0 ? "unknown" : "deferred", + evidence.length > 0 + ? "workflow_run_inventory_incomplete" + : "lookup_inventory_incomplete", + evidence, + ); + const selection = newestRun(runs); + if (!selection.known) + return result("unknown", "workflow_run_order_unknown", evidence); + const latest = selection.latest; + if ( + ambiguousRuns.some((candidate) => { + const order = latest ? compareRunEvidence(candidate, latest) : null; + return order === null || order >= 0; + }) + ) + return result("unknown", "workflow_run_identity_ambiguous", evidence); + if (latest !== undefined) { + const run = latest.run; + if (run.conclusion === "startup_failure") + return result("startup_failure", "workflow_startup_failure", evidence); + if (run.status === "waiting" || run.conclusion === "action_required") + return result( + "approval_required", + "workflow_waiting_for_approval_or_gate", + evidence, + ); + if (["queued", "requested", "pending", "in_progress"].includes(run.status)) + return result( + "queued_pending_executing", + `workflow_${run.status}`, + evidence, + ); + if (run.status === "completed") + return result( + COMPLETED_CATEGORIES.get(run.conclusion) ?? "completed_other", + run.conclusion === "success" + ? "workflow_succeeded" + : `workflow_${run.conclusion ?? "unknown_conclusion"}`, + evidence, + ); + return result("unknown", "workflow_status_unrecognized", evidence); + } + if (!pullRequestInventoryComplete) + return result("deferred", "lookup_inventory_incomplete", []); + if (pull.mergeable !== true) + return result("unknown", "pull_request_mergeability_unknown", []); + return result("missing", "no_pull_request_event_run_for_current_head", []); +} + +/** Match only a pull_request event proven to belong to this immutable PR head. */ +export function matchRunToPullRequest({ + run, + pull, + identityCounts, + inventoryComplete = true, +}) { + if (run?.event !== "pull_request") return { kind: "none" }; + if (!pull.identityValid || !isRecord(run)) return { kind: "none" }; + const association = matchExplicitPullAssociation(run, pull); + if (association === "match") + return { kind: "match", basis: "event_pr_number_and_head_sha" }; + if (association === "ambiguous") return { kind: "ambiguous" }; + if (association === "other" || !sameHeadRepositoryAndRef(run, pull)) + return { kind: "none" }; + if (!uniqueHeadRepositoryAndRef(pull, identityCounts, inventoryComplete)) + return { kind: "ambiguous" }; + if (isSha(run.head_sha) && run.head_sha.toLowerCase() === pull.head.sha) + return { kind: "match", basis: "event_unique_head_ref_and_repository_id" }; + return { kind: "none" }; +} + +function matchExplicitPullAssociation(run, pull) { + const associations = Array.isArray(run.pull_requests) + ? run.pull_requests + : []; + if (associations.length === 0) return "absent"; + const matchingNumber = associations.filter( + (item) => item?.number === pull.number, + ); + if (matchingNumber.length === 0) return "other"; + const exactHead = matchingNumber.find( + (item) => item?.head?.sha === pull.head.sha, + ); + if (exactHead === undefined) { + return matchingNumber.some( + (item) => + typeof item?.head?.sha === "string" && item.head.sha !== pull.head.sha, + ) + ? "other" + : "absent"; + } + const associatedHeadRepositoryId = positiveInteger(exactHead.head?.repo?.id); + const runHeadRepositoryId = positiveInteger(run.head_repository?.id); + if (associatedHeadRepositoryId === null || runHeadRepositoryId === null) + return "ambiguous"; + if ( + associatedHeadRepositoryId !== pull.head.repositoryId || + runHeadRepositoryId !== pull.head.repositoryId + ) + return "other"; + const baseRefMatches = + typeof exactHead.base?.ref !== "string" || + exactHead.base.ref === pull.base.ref; + const baseRepositoryId = positiveInteger(exactHead.base?.repo?.id); + const baseRepositoryMatches = + baseRepositoryId === null || baseRepositoryId === pull.base.repositoryId; + return baseRefMatches && baseRepositoryMatches ? "match" : "other"; +} + +function sameHeadRepositoryAndRef(run, pull) { + return ( + run.head_branch === pull.head.ref && + run.head_repository?.id === pull.head.repositoryId + ); +} + +function uniqueHeadRepositoryAndRef(pull, identityCounts, inventoryComplete) { + if (!inventoryComplete) return false; + const key = headIdentityKey(pull.head.repositoryId, pull.head.ref); + const unknownRepositoryKey = headIdentityKey(null, pull.head.ref); + return ( + identityCounts.get(key) === 1 && + (identityCounts.get(unknownRepositoryKey) ?? 0) === 0 + ); +} + +function normalizePullRequest(raw, identity) { + const number = positiveInteger(raw?.number); + const base = normalizeBase(raw?.base); + const head = normalizeHead(raw?.head); + const draft = raw?.draft === true; + return { + number, + draft, + createdAt: typeof raw?.created_at === "string" ? raw.created_at : null, + url: typeof raw?.html_url === "string" ? raw.html_url : null, + base, + head, + mergeable: typeof raw?.mergeable === "boolean" ? raw.mergeable : undefined, + identityValid: hasValidPullIdentity({ raw, identity, number, base, head }), + }; +} + +function normalizeBase(raw) { + return { + ref: typeof raw?.ref === "string" ? raw.ref : null, + repositoryId: positiveInteger(raw?.repo?.id), + }; +} + +function normalizeHead(raw) { + return { + repositoryId: positiveInteger(raw?.repo?.id), + repository: + typeof raw?.repo?.full_name === "string" ? raw.repo.full_name : null, + ref: typeof raw?.ref === "string" ? raw.ref : null, + sha: + typeof raw?.sha === "string" && isSha(raw.sha) + ? raw.sha.toLowerCase() + : null, + }; +} + +function hasValidPullIdentity({ raw, identity, number, base, head }) { + return ( + number !== null && + typeof raw?.draft === "boolean" && + base.ref === identity.defaultBranch && + (identity.repositoryId === null || + base.repositoryId === identity.repositoryId) && + head.repositoryId !== null && + head.sha !== null && + head.ref !== null + ); +} + +function countHeadIdentities(pulls) { + const counts = new Map(); + for (const pull of pulls) { + if (typeof pull.head.ref !== "string") continue; + const key = headIdentityKey(pull.head.repositoryId, pull.head.ref); + counts.set(key, (counts.get(key) ?? 0) + 1); + } + return counts; +} + +function headIdentityKey(repositoryId, ref) { + return `${String(repositoryId)}\u0000${ref}`; +} + +function newestRun(runs) { + let latest = runs[0]; + for (const candidate of runs) { + if (latest && (compareRunEvidence(candidate, latest) ?? 0) > 0) + latest = candidate; + } + const known = runs.every((candidate) => { + if (candidate === latest) return true; + const order = compareRunEvidence(candidate, latest); + return order !== null && order <= 0; + }); + return { known, latest }; +} + +// Attempt ordering is meaningful within one run; different run IDs are not clocks. +function compareRunEvidence(left, right) { + const leftRun = left.run; + const rightRun = right.run; + const id = positiveInteger(leftRun.id); + if (id !== null && id === positiveInteger(rightRun.id)) { + const leftAttempt = positiveInteger(leftRun.run_attempt); + const rightAttempt = positiveInteger(rightRun.run_attempt); + if (leftAttempt === null || rightAttempt === null) return null; + if (leftAttempt !== rightAttempt) return leftAttempt - rightAttempt; + return leftRun.status === rightRun.status && + leftRun.conclusion === rightRun.conclusion + ? 0 + : null; + } + // A retry may start long after its original creation; without an attempt + // clock, creation dates only order first attempts from different runs. + if ( + positiveInteger(leftRun.run_attempt) !== 1 || + positiveInteger(rightRun.run_attempt) !== 1 + ) + return null; + const time = + Date.parse(leftRun.created_at ?? "") - + Date.parse(rightRun.created_at ?? ""); + return Number.isFinite(time) && time !== 0 ? time : null; +} + +function result(category, reason, runs) { + return { + category, + reason, + runs: runs.map(({ run, basis }) => ({ + id: positiveInteger(run.id), + attempt: positiveInteger(run.run_attempt), + status: typeof run.status === "string" ? run.status : null, + conclusion: typeof run.conclusion === "string" ? run.conclusion : null, + headSha: isSha(run.head_sha) ? run.head_sha.toLowerCase() : null, + createdAt: typeof run.created_at === "string" ? run.created_at : null, + url: typeof run.html_url === "string" ? run.html_url : null, + identityBasis: basis, + })), + }; +} + +async function listOpenPullRequests({ requestGet, identity }) { + const items = []; + let complete = false; + let pages = 0; + let error = null; + for (let page = 1; page <= MAX_PULL_REQUEST_PAGES; page += 1) { + try { + const endpoint = `repos/${identity.repository}/pulls?state=open&sort=created&direction=desc&per_page=${PAGE_SIZE}&page=${page}`; + const response = await requestGet(endpoint); + if (!Array.isArray(response)) + throw new Error("pull_request_page_malformed"); + pages += 1; + const normalized = response.map((raw) => + normalizePullRequest(raw, identity), + ); + if ( + normalized.some( + (pull) => !Number.isFinite(Date.parse(pull.createdAt ?? "")), + ) + ) + throw new Error("pull_request_page_malformed"); + items.push(...normalized); + if (response.length < PAGE_SIZE) { + complete = true; + break; + } + } catch (cause) { + error = safeErrorCode(cause); + break; + } + } + return { items, pages, complete, error }; +} + +async function listRecentWorkflowRuns({ requestGet, identity, cutoff }) { + const items = []; + let complete = false; + let pages = 0; + let error = null; + for (let page = 1; page <= MAX_WORKFLOW_RUN_PAGES; page += 1) { + try { + const cutoffDate = new Date(cutoff).toISOString().slice(0, 10); + const endpoint = `repos/${identity.repository}/actions/workflows/ci.yml/runs?event=pull_request&created=%3E%3D${cutoffDate}&per_page=${PAGE_SIZE}&page=${page}`; + const response = await requestGet(endpoint); + if (!isRecord(response) || !Array.isArray(response.workflow_runs)) + throw new Error("workflow_run_page_malformed"); + pages += 1; + if (!response.workflow_runs.every(isRecord)) + throw new Error("workflow_run_page_malformed"); + items.push(...response.workflow_runs); + if (response.workflow_runs.length < PAGE_SIZE) { + complete = true; + break; + } + } catch (cause) { + error = safeErrorCode(cause); + break; + } + } + return { items, pages, complete, error }; +} + +async function getPullRequestDetails({ requestGet, identity, pull }) { + if (pull.number === null) + return { detail: null, error: "pull_request_detail_identity_missing" }; + try { + const response = await requestGet( + `repos/${identity.repository}/pulls/${pull.number}`, + ); + if (!isRecord(response)) + return { detail: null, error: "pull_request_detail_malformed" }; + return { + detail: { + mergeable: + typeof response.mergeable === "boolean" + ? response.mergeable + : undefined, + headSha: isSha(response.head?.sha) + ? response.head.sha.toLowerCase() + : null, + headRef: + typeof response.head?.ref === "string" ? response.head.ref : null, + headRepositoryId: positiveInteger(response.head?.repo?.id), + baseRef: + typeof response.base?.ref === "string" ? response.base.ref : null, + baseRepositoryId: positiveInteger(response.base?.repo?.id), + }, + error: null, + }; + } catch (cause) { + return { detail: null, error: safeErrorCode(cause) }; + } +} + +function validateInputs({ + repository, + repositoryId, + defaultBranch, + lookbackDays, + now, +}) { + if ( + typeof repository !== "string" || + !/^[A-Za-z0-9_.-]+\/[A-Za-z0-9_.-]+$/u.test(repository) + ) + throw new Error( + "CI_REPOSITORY must be an owner/name repository identifier", + ); + if ( + typeof defaultBranch !== "string" || + defaultBranch.length === 0 || + /[\r\n]/u.test(defaultBranch) + ) + throw new Error("CI_DEFAULT_BRANCH must be a nonempty branch name"); + const parsedRepositoryId = + repositoryId === undefined || repositoryId === null || repositoryId === "" + ? null + : positiveInteger(repositoryId); + if ( + repositoryId !== undefined && + repositoryId !== null && + repositoryId !== "" && + parsedRepositoryId === null + ) + throw new Error("CI_REPOSITORY_ID must be a positive safe integer"); + if ( + !Number.isInteger(lookbackDays) || + lookbackDays < 1 || + lookbackDays > MAX_LOOKBACK_DAYS + ) + throw new Error( + `lookbackDays must be an integer from 1 to ${String(MAX_LOOKBACK_DAYS)}`, + ); + if (!Number.isFinite(now)) throw new Error("now must be a finite timestamp"); + return { + repository, + repositoryId: parsedRepositoryId, + defaultBranch, + lookbackDays, + }; +} + +function positiveInteger(value) { + const number = typeof value === "number" ? value : Number(value); + return Number.isSafeInteger(number) && number > 0 ? number : null; +} + +function isSha(value) { + return typeof value === "string" && SHA_PATTERN.test(value); +} + +function isRecord(value) { + return typeof value === "object" && value !== null && !Array.isArray(value); +} + +function safeErrorCode(cause) { + if (cause?.message === "lookup_time_budget_exhausted") + return "lookup_time_budget_exhausted"; + if (cause?.message === "lookup_request_budget_exhausted") + return "lookup_request_budget_exhausted"; + if (cause?.message === "github_api_malformed_json") + return "github_api_malformed_json"; + if (cause?.message?.endsWith("_malformed")) return cause.message; + if (cause?.code === "ETIMEDOUT" || cause?.killed) return "github_api_timeout"; + return "github_api_request_failed"; +} + +export async function writePrCiHealthReport( + report, + outputDirectory = OUTPUT_DIRECTORY, +) { + const jsonPath = join(outputDirectory, "report.json"); + const markdownPath = join(outputDirectory, "summary.md"); + await mkdir(outputDirectory, { recursive: true }); + await writeFile(jsonPath, `${JSON.stringify(report, null, 2)}\n`, "utf8"); + await writeFile(markdownPath, renderMarkdownSummary(report), "utf8"); + return { jsonPath, markdownPath }; +} + +export function renderMarkdownSummary(report) { + const rows = report.pullRequests.map( + (pull) => + `| #${String(pull.number ?? "?")} | ${escapeCell(pull.head.repository ?? "unknown")} | ${escapeCell(pull.head.ref ?? "unknown")} | \`${pull.head.sha ?? "unknown"}\` | ${pull.ci.category} | ${escapeCell(pull.ci.reason)} |`, + ); + return [ + "# Pull request CI health snapshot", + "", + `Observed ${report.observedAt} for \`${report.repository}\`, target branch \`${escapeCell(report.defaultBranch)}\`; recent window ${String(report.lookbackDays)} days. Scan: **${report.scan.status}**.`, + "", + "This is read-only point-in-time telemetry. Missing is reported only when PR and workflow-run inventories are complete and the current immutable PR head identity is unambiguous. Incomplete lookups remain deferred or unknown.", + "", + `API pages: ${String(report.scan.pullRequestPages)} PR, ${String(report.scan.workflowRunPages)} workflow runs; ${String(report.scan.apiRequests)} bounded requests. Errors: ${report.scan.errors.length === 0 ? "none" : report.scan.errors.join(", ")}.`, + "", + "| PR | Head repository | Head ref | Head SHA | CI classification | Evidence |", + "| ---: | --- | --- | --- | --- | --- |", + ...rows, + "", + "Categories: missing, startup failure, queued/pending/executing, approval required, completed outcomes, merge conflict, unknown, and deferred.", + "", + ].join("\n"); +} + +function escapeCell(value) { + return String(value) + .replaceAll("|", "\\|") + .replaceAll("\n", " ") + .replaceAll("\r", " "); +} + +if ( + process.argv[1] !== undefined && + resolve(process.argv[1]) === fileURLToPath(import.meta.url) +) { + try { + const report = await collectPrCiHealth({ + repository: process.env.CI_REPOSITORY, + repositoryId: process.env.CI_REPOSITORY_ID, + defaultBranch: process.env.CI_DEFAULT_BRANCH, + }); + const paths = await writePrCiHealthReport( + report, + process.env.CI_PR_HEALTH_DIRECTORY ?? OUTPUT_DIRECTORY, + ); + if (process.env.GITHUB_STEP_SUMMARY) + await appendFile( + process.env.GITHUB_STEP_SUMMARY, + renderMarkdownSummary(report), + "utf8", + ); + process.stdout.write( + `Wrote ${report.pullRequests.length} recent PR CI records to ${paths.jsonPath}\n`, + ); + } catch (cause) { + process.stderr.write( + `PR CI health telemetry failed: ${cause instanceof Error ? cause.message : "unknown_error"}\n`, + ); + process.exitCode = 1; + } +} diff --git a/scripts/ci/qualification.mjs b/scripts/ci/qualification.mjs index ab47c168f..8dfb3f20c 100644 --- a/scripts/ci/qualification.mjs +++ b/scripts/ci/qualification.mjs @@ -2,6 +2,7 @@ import { execFileSync } from "node:child_process"; import { appendFileSync, mkdirSync, writeFileSync } from "node:fs"; import { resolve } from "node:path"; import { fileURLToPath } from "node:url"; +import { normalizeWorkflowRun } from "./workflow-run.mjs"; const ARTIFACT_NAME = "ci-full-qualification-v1"; const MAX_AGE_MS = 24 * 60 * 60 * 1000; @@ -119,8 +120,8 @@ function admit() { // Artifact metadata narrows the search before run lookups. Never scan an // unbounded history or treat a missing receipt as a successful qualification. for (const artifact of candidates.slice(0, 3)) { - const run = api( - `repos/${repository}/actions/runs/${artifact.workflow_run.id}`, + const run = normalizeWorkflowRun( + api(`repos/${repository}/actions/runs/${artifact.workflow_run.id}`), ); // An old run can upload a new artifact when only its gate is rerun. // Bound the original run age as well so retries cannot renew stale proof. @@ -133,7 +134,7 @@ function admit() { run.id === artifact.workflow_run.id && run.status === "completed" && run.conclusion === "success" && - run.path === ".github/workflows/ci.yml" && + run.workflowPath?.file === ".github/workflows/ci.yml" && ["schedule", "workflow_dispatch"].includes(run.event) && run.head_sha === head && run.head_branch === branch && diff --git a/scripts/ci/required.mjs b/scripts/ci/required.mjs index 20fbac23f..1839a1149 100644 --- a/scripts/ci/required.mjs +++ b/scripts/ci/required.mjs @@ -30,7 +30,11 @@ if (outputs.tier === "full" && selectedScopes.length !== CI_FLAGS.length) throw new Error("Full qualification requires every canonical CI scope"); const required = reused ? ["changes"] : ["changes", "static", "test"]; const lanes = { - code: ["build", "test-shard"], + code: [ + "build", + "test-shard", + ...(outputs.tier === "fast" ? ["test-source"] : []), + ], reverse_tools: ["reverse-tools-quality"], runtime: ["windows-native-build", "package-e2e"], package: ["windows-curated"], diff --git a/scripts/ci/static-check-report.mjs b/scripts/ci/static-check-report.mjs index 04460b2c2..eecaf750a 100644 --- a/scripts/ci/static-check-report.mjs +++ b/scripts/ci/static-check-report.mjs @@ -133,6 +133,8 @@ try { for (const clone of report.duplicates) for (const file of [clone.firstFile, clone.secondFile]) file.name = relative(process.cwd(), file.name).split(sep).join("/"); + // Record the actual checker result, including caller-selected thresholds. + report.reaCheck = { exitCode: result.status }; await writeFile(reportPath, JSON.stringify(report, null, 2)); const { sources, clones, percentage } = report.statistics.total; console.log( diff --git a/scripts/ci/static-task-context.d.mts b/scripts/ci/static-task-context.d.mts new file mode 100644 index 000000000..164756b15 --- /dev/null +++ b/scripts/ci/static-task-context.d.mts @@ -0,0 +1,18 @@ +export interface StaticTaskContext { + readonly version: 1; + readonly sourceSha: string; + readonly sourceClean: boolean; + readonly prHeadSha?: string; + readonly nodeVersion: string; + readonly npmVersion: string; + readonly turboVersion: string; + readonly platform: string; + readonly architecture: string; + readonly taskHashes: Readonly>; +} +export function parseStaticTaskContext(value: unknown): StaticTaskContext; +export function recordStaticTaskContext(options?: { + root?: string; + npmCli?: string; + outputDirectory?: string; +}): Promise; diff --git a/scripts/ci/static-task-context.mjs b/scripts/ci/static-task-context.mjs new file mode 100644 index 000000000..879c568d1 --- /dev/null +++ b/scripts/ci/static-task-context.mjs @@ -0,0 +1,104 @@ +import { execFile } from "node:child_process"; +import { mkdir, readFile, writeFile } from "node:fs/promises"; +import { join, resolve } from "node:path"; +import { fileURLToPath } from "node:url"; +import { promisify } from "node:util"; +import { z } from "zod"; + +const execute = promisify(execFile); +const contextSchema = z.object({ + version: z.literal(1), + sourceSha: z.string().regex(/^[a-f0-9]{40}$/u), + sourceClean: z.boolean(), + prHeadSha: z + .string() + .regex(/^[a-f0-9]{40}$/u) + .optional(), + nodeVersion: z.string().min(1), + npmVersion: z.string().min(1), + turboVersion: z.string().min(1), + platform: z.string().min(1), + architecture: z.string().min(1), + taskHashes: z.record(z.string(), z.string().regex(/^[a-f0-9]{16}$/u)), +}); + +/** Parse the producing check graph's receipt; incomplete provenance is unknown. */ +export const parseStaticTaskContext = (value) => contextSchema.parse(value); + +/** Retain only named task hashes and tool/source identity, never the environment. */ +export async function recordStaticTaskContext({ + root = process.cwd(), + npmCli = process.env.npm_execpath, + outputDirectory = join(root, ".cache/static-checks"), +} = {}) { + if (typeof npmCli !== "string" || !npmCli) + throw new Error( + "Run static context recording through npm run ci:static-context", + ); + const options = { + cwd: root, + encoding: "utf8", + timeout: 30_000, + maxBuffer: 32 * 1024 * 1024, + }; + const [graph, source, status, npmVersion, turbo] = await Promise.all([ + execute( + process.execPath, + [npmCli, "run", "--silent", "check:ci:tasks", "--", "--dry=json"], + options, + ), + execute("git", ["rev-parse", "HEAD"], options), + execute( + "git", + ["status", "--porcelain", "--untracked-files=normal"], + options, + ), + execute(process.execPath, [npmCli, "--version"], options), + readFile(join(root, "node_modules/turbo/package.json"), "utf8"), + ]); + const tasks = z + .object({ + tasks: z + .array( + z.object({ + task: z.string().min(1), + hash: z.string().regex(/^[a-f0-9]{16}$/u), + }), + ) + .min(1), + }) + .parse(JSON.parse(graph.stdout)).tasks; + if (new Set(tasks.map(({ task }) => task)).size !== tasks.length) + throw new Error("Static graph contains duplicate task identities"); + const context = parseStaticTaskContext({ + version: 1, + sourceSha: source.stdout.trim(), + sourceClean: status.stdout.length === 0, + ...(process.env.CI_PR_HEAD_SHA + ? { prHeadSha: process.env.CI_PR_HEAD_SHA } + : {}), + nodeVersion: process.version, + npmVersion: npmVersion.stdout.trim(), + turboVersion: z.object({ version: z.string() }).parse(JSON.parse(turbo)) + .version, + platform: process.platform, + architecture: process.arch, + taskHashes: Object.fromEntries(tasks.map(({ task, hash }) => [task, hash])), + }); + if (process.env.GITHUB_SHA && process.env.GITHUB_SHA !== context.sourceSha) + throw new Error( + "Static context source does not match the workflow revision", + ); + await mkdir(outputDirectory, { recursive: true }); + await writeFile( + join(outputDirectory, "static-task-context.json"), + JSON.stringify(context, null, 2) + "\n", + ); + return context; +} + +if ( + process.argv[1] && + resolve(process.argv[1]) === fileURLToPath(import.meta.url) +) + await recordStaticTaskContext(); diff --git a/scripts/ci/test-group.mjs b/scripts/ci/test-group.mjs new file mode 100644 index 000000000..7d99175b4 --- /dev/null +++ b/scripts/ci/test-group.mjs @@ -0,0 +1,46 @@ +import { spawn } from "node:child_process"; +import { resolve } from "node:path"; + +import { + BUILT_TEST_PROJECTS, + SOURCE_TEST_PROJECTS, +} from "../lib/test-projects.mjs"; + +const [group, ...arguments_] = process.argv.slice(2); +const projects = { source: SOURCE_TEST_PROJECTS, built: BUILT_TEST_PROJECTS }[ + group +]; +if (!projects) throw new Error("Expected source or built test group"); +// The group owns project selection; callers can request reporters or shards. +if ( + arguments_.some((argument) => /^--(?:project|exclude)(?:=|$)/u.test(argument)) +) + throw new Error("Test groups cannot override their project inventory"); +const child = spawn( + process.execPath, + [ + resolve("node_modules/vitest/vitest.mjs"), + "run", + ...projects.flatMap(({ name }) => ["--project", name]), + ...arguments_, + ], + { stdio: "inherit" }, +); +let shutdownSignal; +for (const signal of ["SIGINT", "SIGTERM"]) + process.on(signal, () => { + shutdownSignal ??= signal; + child.kill(signal); + }); +child.once("error", (error) => { + console.error(error.message); + process.exitCode = 1; +}); +child.once("close", (code) => { + process.exitCode = + shutdownSignal === "SIGINT" + ? 130 + : shutdownSignal === "SIGTERM" + ? 143 + : (code ?? 1); +}); diff --git a/scripts/ci/test-history.mjs b/scripts/ci/test-history.mjs index 4386f48af..14dbb582b 100644 --- a/scripts/ci/test-history.mjs +++ b/scripts/ci/test-history.mjs @@ -2,6 +2,7 @@ import { execFileSync } from "node:child_process"; import { appendFileSync } from "node:fs"; import { resolve } from "node:path"; import { fileURLToPath } from "node:url"; +import { normalizeWorkflowRun } from "./workflow-run.mjs"; const ARTIFACT_NAME = "test-timings"; const MAX_AGE_MS = 30 * 24 * 60 * 60 * 1000; @@ -68,8 +69,8 @@ function findTimingRun({ repository, branch, repositoryId, now = Date.now() }) { .slice(0, MAX_CANDIDATES); for (const artifact of candidates) { - const run = api( - `repos/${repository}/actions/runs/${artifact.workflow_run.id}`, + const run = normalizeWorkflowRun( + api(`repos/${repository}/actions/runs/${artifact.workflow_run.id}`), ); const runAge = now - Date.parse(run?.created_at); if ( @@ -80,7 +81,7 @@ function findTimingRun({ repository, branch, repositoryId, now = Date.now() }) { run.id === artifact.workflow_run.id && run.status === "completed" && run.conclusion === "success" && - run.path === ".github/workflows/ci.yml" && + run.workflowPath?.file === ".github/workflows/ci.yml" && ["schedule", "workflow_dispatch"].includes(run.event) && run.head_branch === branch && run.head_repository?.full_name === repository diff --git a/scripts/ci/test-report.mjs b/scripts/ci/test-report.mjs index 035a01990..7f85c6ec5 100644 --- a/scripts/ci/test-report.mjs +++ b/scripts/ci/test-report.mjs @@ -41,9 +41,20 @@ const files = report.files.map(({ name, durationMs, state }) => { throw new Error(`Test report path is outside the checkout: ${name}`); return { name: path.split(sep).join("/"), durationMs, state }; }); -const complete = process.env.SHARD_RESULT === "success"; +// Keep group outcomes intact: an unavailable group is not a completed failure. +const groupResults = [ + process.env.SHARD_RESULT, + ...(process.env.SOURCE_REQUIRED === "true" + ? [process.env.SOURCE_RESULT] + : []), +]; +const groupsCompleted = groupResults.every((state) => + ["success", "failure"].includes(state), +); const passed = - complete && process.env.MERGE_RESULT === "success" && report.success; + groupResults.every((state) => state === "success") && + process.env.MERGE_RESULT === "success" && + report.success; // Successful shards and merged reports must account for the complete portable // inventory. Coverage thresholds alone cannot prove that every file ran. const verifyInventory = process.env.CI_VERIFY_TEST_INVENTORY === "true"; @@ -69,7 +80,7 @@ if (process.env.CI_TEST_SELECTION) { comparison.complete && report.unhandledErrors === 0 && ["passed", "failed"].includes(report.reason) && - ["success", "failure"].includes(process.env.SHARD_RESULT) && + groupsCompleted && ["success", "failure"].includes(process.env.MERGE_RESULT), executionSha: process.env.GITHUB_SHA ?? null, prHeadSha: process.env.CI_PR_HEAD_SHA ?? null, @@ -91,7 +102,9 @@ if (passed && verifyInventory) { ); } const summary = [ - complete ? "## Merged test diagnostics" : "## Partial test diagnostics", + groupsCompleted + ? "## Merged test diagnostics" + : "## Partial test diagnostics", "", ...(selection ? [ @@ -99,11 +112,14 @@ const summary = [ "", `Plan: ${selection.plan.mode}. Proposed files: ${selection.selected.length}; omitted: ${selection.omitted.length}.`, `Full-run comparison: ${selection.complete ? "complete" : "incomplete"}; failures outside proposed selection: ${selection.missedFailures.length}.`, - "The complete suite still ran. This report cannot enable omissions or qualify a revision. Skipped, missing, and interrupted observations do not establish parity.", + "This observational report cannot enable omissions or qualify a revision. Skipped, missing, and interrupted observations do not establish parity.", "", ] : []), `Shard result: ${process.env.SHARD_RESULT ?? "unknown"}; merge result: ${process.env.MERGE_RESULT ?? "unknown"}.`, + ...(process.env.SOURCE_REQUIRED === "true" + ? [`Source result: ${process.env.SOURCE_RESULT ?? "unknown"}.`] + : []), passed ? "Successful timing history is available." : "This report is diagnostic only and is not a successful timing baseline.", diff --git a/scripts/ci/workflow-run.d.mts b/scripts/ci/workflow-run.d.mts new file mode 100644 index 000000000..b0e626378 --- /dev/null +++ b/scripts/ci/workflow-run.d.mts @@ -0,0 +1,10 @@ +export interface WorkflowPath { + readonly file: string; + readonly ref: string | null; +} +export interface NormalizedWorkflowRun { + readonly workflowPath: WorkflowPath | null; +} +export function normalizeWorkflowRun>( + value: T, +): Omit & NormalizedWorkflowRun; diff --git a/scripts/ci/workflow-run.mjs b/scripts/ci/workflow-run.mjs new file mode 100644 index 000000000..8c33214bf --- /dev/null +++ b/scripts/ci/workflow-run.mjs @@ -0,0 +1,15 @@ +/** Keep the API path as evidence and parse its optional ref at the boundary. */ +export function normalizeWorkflowRun(value) { + if (typeof value !== "object" || value === null || Array.isArray(value)) + throw new Error("workflow_run_malformed"); + const match = + typeof value.path === "string" && !/[\u0000-\u001f\u007f]/u.test(value.path) + ? /^(\.github\/workflows\/[^/\\@\s]+\.ya?ml)(?:@([^\s]+))?$/u.exec( + value.path, + ) + : null; + return { + ...value, + workflowPath: match ? { file: match[1], ref: match[2] ?? null } : null, + }; +} diff --git a/scripts/lib/development-tests.mjs b/scripts/lib/development-tests.mjs index 8ddebe115..6aaf10768 100644 --- a/scripts/lib/development-tests.mjs +++ b/scripts/lib/development-tests.mjs @@ -1,17 +1,4 @@ -const sourceProjects = [ - "domain", - "services", - "adapters", - "composition", - "conformance", - "evaluation", -]; - -const sourceTest = (path) => - path.startsWith("src/") || - ["composition", "conformance", "evaluation"].some((project) => - path.startsWith(`tests/${project}/`), - ); +import { SOURCE_TEST_PROJECTS, projectForTest } from "./test-projects.mjs"; /** Parse explicit test paths and an optional branch-comparison base. */ export const parseDevelopmentTestRequest = (mode, arguments_) => { @@ -35,10 +22,9 @@ export const parseDevelopmentTestRequest = (mode, arguments_) => { path.split("/").some((part) => part === ".." || part === ".") ) throw new Error(`Expected an exact repository test path: ${argument}`); - if (mode !== "focused" && !sourceTest(path)) - throw new Error( - `Use test:focused for compiled boundary tests: ${path}`, - ); + const project = projectForTest(path); + if (mode !== "focused" && project.group !== "source") + throw new Error(`Use test:focused for built test projects: ${path}`); paths.push(path); } } @@ -72,9 +58,14 @@ export const developmentTestPlan = (request, baseCommit) => { !/^[a-f0-9]{40,64}$/u.test(baseCommit ?? "") ) throw new Error("Changed tests require a resolved Git merge base"); - const projects = request.mode === "focused" ? [] : sourceProjects; + const projects = + request.mode === "focused" + ? [] + : SOURCE_TEST_PROJECTS.map(({ name }) => name); return { - needsBuild: request.paths.some((path) => !sourceTest(path)), + needsBuild: request.paths.some( + (path) => projectForTest(path).group === "built", + ), artifactTasks: Object.entries(artifactConsumers).flatMap( ([task, consumers]) => request.paths.some((path) => consumers.includes(path)) ? [task] : [], diff --git a/scripts/lib/test-projects.d.mts b/scripts/lib/test-projects.d.mts new file mode 100644 index 000000000..a9d6ae356 --- /dev/null +++ b/scripts/lib/test-projects.d.mts @@ -0,0 +1,10 @@ +export interface TestProjectDefinition { + readonly name: string; + readonly group: "source" | "built"; + readonly include: readonly string[]; + readonly exclude?: readonly string[]; +} + +export const SOURCE_TEST_PROJECTS: readonly TestProjectDefinition[]; +export const BUILT_TEST_PROJECTS: readonly TestProjectDefinition[]; +export function projectForTest(testPath: string): TestProjectDefinition; diff --git a/scripts/lib/test-projects.mjs b/scripts/lib/test-projects.mjs new file mode 100644 index 000000000..68fb90dbd --- /dev/null +++ b/scripts/lib/test-projects.mjs @@ -0,0 +1,107 @@ +import { matchesGlob } from "node:path"; + +/** Canonical file ownership shared by Vitest and development-test planning. */ +export const SOURCE_TEST_PROJECTS = [ + { + name: "domain", + group: "source", + include: ["src/{contracts,domain}/**/*.test.ts"], + }, + { + name: "services", + group: "source", + include: ["src/application/**/*.test.ts"], + }, + { + name: "adapters", + group: "source", + include: [ + "src/*.test.ts", + "src/{apktool,jeb,adb,artifacts,browser,dotnet,evm,firmware,flutter,pcap,ghidra,go,hopper,ida,inspector,javascript,native,process,reference,rust,server,wasm,windows}/**/*.test.ts", + ], + }, + { + name: "composition", + group: "source", + include: ["tests/composition/**/*.test.ts"], + }, + { + name: "conformance", + group: "source", + include: ["tests/conformance/**/*.test.ts"], + exclude: ["tests/conformance/providers/hopper/realHopperSearch.test.ts"], + }, + { + name: "evaluation", + group: "source", + include: ["tests/evaluation/**/*.test.ts"], + }, +]; + +export const BUILT_TEST_PROJECTS = [ + { + name: "boundary", + group: "built", + include: [ + "tests/boundary/**/*.test.ts", + "tests/conformance/providers/hopper/realHopperSearch.test.ts", + ], + exclude: [ + "tests/boundary/cli/**/*.test.ts", + "tests/boundary/mcp/**/*.test.ts", + "tests/boundary/process/**/*.test.ts", + ], + }, + { + name: "cli-boundary", + group: "built", + include: ["tests/boundary/cli/**/*.test.ts"], + }, + { + name: "process-boundary", + group: "built", + include: ["tests/boundary/process/**/*.test.ts"], + }, + { + name: "mcp-boundary", + group: "built", + include: ["tests/boundary/mcp/**/*.test.ts"], + }, + { + name: "acceptance", + group: "built", + include: ["tests/acceptance/**/*.test.ts"], + }, + { + name: "process-global", + group: "built", + include: ["tests/process-global/**/*.test.ts"], + }, +]; + +const ALL_TEST_PROJECTS = [...SOURCE_TEST_PROJECTS, ...BUILT_TEST_PROJECTS]; + +/** Return the unique Vitest owner for an exact repository-relative test path. */ +export const projectForTest = (testPath) => { + if ( + typeof testPath !== "string" || + testPath.startsWith("/") || + testPath.includes("\\") || + testPath.split("/").some((part) => part === ".." || part === ".") || + !testPath.endsWith(".test.ts") + ) + throw new Error(`Expected a normalized repository test path: ${testPath}`); + + const owners = ALL_TEST_PROJECTS.filter( + ({ include, exclude = [] }) => + include.some((pattern) => matchesGlob(testPath, pattern)) && + !exclude.some((pattern) => matchesGlob(testPath, pattern)), + ); + if (owners.length !== 1) + throw new Error( + owners.length === 0 + ? `No test project owns ${testPath}` + : `Multiple test projects own ${testPath}: ${owners.map(({ name }) => name).join(", ")}`, + ); + return owners[0]; +}; diff --git a/src/domain/analysisView/analysisView.test.ts b/src/domain/analysisView/analysisView.test.ts index 3c14636d7..5939d458a 100644 --- a/src/domain/analysisView/analysisView.test.ts +++ b/src/domain/analysisView/analysisView.test.ts @@ -961,7 +961,7 @@ it("does not impose a message-count cutoff below the byte budget", () => { ); expect( boundedViewMessages(messages, `ev_${"a".repeat(64)}`, "limitations"), - ).toEqual(messages); + ).toEqual(messages.toSorted()); }); it("bounds repeated view messages even when fewer than 64 source strings are huge", () => { diff --git a/tests/boundary/cli/developmentTests.test.ts b/tests/boundary/cli/developmentTests.test.ts index 2987206c8..7becdd4e7 100644 --- a/tests/boundary/cli/developmentTests.test.ts +++ b/tests/boundary/cli/developmentTests.test.ts @@ -45,6 +45,7 @@ const fixture = async () => { for (const path of [ "scripts/run-development-tests.mjs", "scripts/lib/development-tests.mjs", + "scripts/lib/test-projects.mjs", ]) await copyFile(join(process.cwd(), path), join(root, path)); await symlink( diff --git a/tests/boundary/filesystem/ciFailureReport.test.ts b/tests/boundary/filesystem/ciFailureReport.test.ts new file mode 100644 index 000000000..26feb3ee2 --- /dev/null +++ b/tests/boundary/filesystem/ciFailureReport.test.ts @@ -0,0 +1,419 @@ +import { readFile } from "node:fs/promises"; +import { join } from "node:path"; + +import { expect, it } from "vitest"; +import { createTestTempDirectory } from "../../fixtures/temporaryDirectory.js"; +import { normalizeWorkflowRun } from "../../../scripts/ci/workflow-run.mjs"; + +import { + readStaticCheckArtifact, + selectStaticCheckArtifact, +} from "../../../scripts/ci/artifacts.mjs"; +import { + classifyFailure, + createGitHubApi, + reportCiFailures, + type CiRun, + type CiJob, + type FailureReportApi, +} from "../../../scripts/ci/failure-report.mjs"; + +const sha = (digit: string) => digit.repeat(40); +const diagnostic = { + kind: "lint", + path: "src/example.ts", + code: "no-debugger", + message: "Unexpected debugger statement.", + severity: "error", +} as const; +const context = ( + sourceSha: string, + taskHash = "a".repeat(16), + prHeadSha?: string, +) => ({ + version: 1 as const, + sourceSha, + ...(prHeadSha ? { prHeadSha } : {}), + sourceClean: true, + nodeVersion: "v24.18.0", + npmVersion: "11.16.0", + turboVersion: "2.11.7", + platform: "linux", + architecture: "x64", + taskHashes: { "lint:built": taskHash }, +}); +const run = (overrides: Partial = {}) => + normalizeWorkflowRun({ + id: 10, + run_attempt: 1, + status: "completed", + conclusion: "failure", + event: "pull_request", + path: ".github/workflows/ci.yml@main", + head_sha: sha("a"), + head_branch: "feature", + repository: { full_name: "owner/rea" }, + pull_requests: [ + { base: { sha: sha("b"), repo: { full_name: "owner/rea" } } }, + ], + ...overrides, + }); +const job = (overrides: Partial = {}) => ({ + name: "Static checks", + status: "completed", + conclusion: "failure", + ...overrides, +}); + +const comparison = (): Parameters[0] => ({ + repository: "owner/rea", + defaultBranch: "main", + run: run(), + job: job(), + diagnostics: [diagnostic], + context: context(sha("a")), + mainRun: run({ + id: 20, + event: "push", + head_sha: sha("c"), + head_branch: "main", + pull_requests: [], + }), + mainJob: job(), + mainDiagnostics: [diagnostic], + mainContext: context(sha("c")), + executionSourcesVerified: true, + ancestry: { status: "ahead", mergeBaseSha: sha("c") }, + blobs: { "src/example.ts": true }, +}); + +it("attributes only an identical structured lint failure with matching task, context, ancestry, and source blob", () => { + const result = classifyFailure(comparison()); + expect(result).toEqual({ + classification: "matching-main-diagnostics", + reason: + "supported_diagnostics_match_failed_task_and_source_on_verified_main_ancestor", + }); +}); + +it.each([ + ["task hash", { mainContext: context(sha("c"), "b".repeat(16)) }], + ["dirty source", { context: { ...context(sha("a")), sourceClean: false } }], + [ + "toolchain", + { mainContext: { ...context(sha("c")), turboVersion: "other" } }, + ], + [ + "source sha", + { context: context(sha("d")), executionSourcesVerified: false }, + ], + [ + "diagnostic", + { mainDiagnostics: [{ ...diagnostic, message: "different message" }] }, + ], + ["content", { blobs: { "src/example.ts": false } }], + ["ancestry", { ancestry: { status: "diverged", mergeBaseSha: sha("e") } }], + ["canceled job", { job: job({ conclusion: "cancelled" }) }], + ["not a static job", { job: job({ name: "Test shard (1)" }) }], +] as const)("does not attribute on mismatched %s", (_label, overrides) => { + const result = classifyFailure({ ...comparison(), ...overrides }); + expect(result.classification).toBe("unknown"); +}); + +it("writes informational JSON and Markdown from the supplied manifest using only read API calls", async () => { + const directory = await createTestTempDirectory("rea-ci-failure-test-"); + const calls: string[] = []; + const manifest = { + repository: "owner/rea", + truncation: [], + coverage: { exhaustive: false }, + runs: [ + { + id: 10, + runAttempt: 1, + workflow: "CI", + event: "pull_request", + conclusion: "failure", + status: "collected", + }, + { + id: 11, + runAttempt: 1, + workflow: "CI", + event: "pull_request", + conclusion: "success", + status: "collected", + }, + ], + }; + const failedRun = run(); + const api: FailureReportApi = { + async getRun(_repository, id) { + calls.push(`run:${id}`); + return failedRun; + }, + async getJobs(_repository, id) { + calls.push(`jobs:${id}`); + return [{ ...job(), run_id: Number(id), run_attempt: 1 }]; + }, + async getStaticEvidence() { + calls.push("artifact"); + return null; + }, + async getAncestry() { + throw new Error("must not compare without evidence"); + }, + async verifyExecutionSource() { + throw new Error("must not verify source without evidence"); + }, + async getBlobIdentity() { + throw new Error("must not inspect blobs without evidence"); + }, + }; + const report = await reportCiFailures({ + repository: "owner/rea", + defaultBranch: "main", + batchManifest: manifest, + api, + outputDirectory: directory, + }); + expect(calls).toEqual(["run:10", "jobs:10", "artifact"]); + expect(report).toMatchObject({ + informationalOnly: true, + gateEffect: "none; this report never changes CI conclusions", + counts: { unknown: 1 }, + }); + const json = JSON.parse( + await readFile(join(directory, "report.json"), "utf8"), + ); + const markdown = await readFile(join(directory, "report.md"), "utf8"); + expect(json.failures[0]).toMatchObject({ + classification: "unknown", + reason: "no_matching_verified_main_failure", + }); + expect(markdown).toContain("Informational only"); + expect(markdown).toContain("Unknown means evidence was incomplete"); +}); + +it.each([ + [22, false], + [6, true], +] as const)( + "finds verified main evidence after %s newer PR runs (extra incorrect main hint: %s)", + async (prCount, incorrectMainHints) => { + const directory = await createTestTempDirectory("rea-ci-main-proof-"); + const mainSha = sha("c"); + const prHeadSha = sha("a"); + const mergeSha = sha("d"); + const mainRun = run({ + id: 20, + event: "push", + head_sha: mainSha, + head_branch: "main", + pull_requests: [], + }); + const prRun = run({ + head_sha: prHeadSha, + pull_requests: [ + { base: { sha: mainSha, repo: { full_name: "owner/rea" } } }, + ], + }); + const evidence = (sourceSha: string, prHead?: string) => ({ + context: context(sourceSha, undefined, prHead), + diagnostics: [diagnostic], + }); + const lookups: number[] = []; + const api: FailureReportApi = { + async getRun(_repository, id) { + lookups.push(Number(id)); + return Number(id) === 20 ? mainRun : { ...prRun, id }; + }, + async getJobs(_repository, id) { + return [{ ...job(), run_id: Number(id), run_attempt: 1 }]; + }, + async getStaticEvidence(_repository, selectedRun) { + return Number(selectedRun.id) === 20 + ? evidence(mainSha) + : evidence(mergeSha, prHeadSha); + }, + async verifyExecutionSource(_repository, selectedRun, sourceContext) { + return selectedRun.event === "pull_request" + ? sourceContext.sourceSha === mergeSha && + sourceContext.prHeadSha === selectedRun.head_sha + : sourceContext.sourceSha === selectedRun.head_sha; + }, + async getAncestry() { + return { status: "ahead", mergeBaseSha: mainSha }; + }, + async getBlobIdentity() { + return { "src/example.ts": true }; + }, + }; + const result = await reportCiFailures({ + repository: "owner/rea", + defaultBranch: "main", + batchManifest: { + repository: "owner/rea", + runs: [ + ...(incorrectMainHints + ? [ + { + id: 30, + runAttempt: 1, + workflow: "CI", + event: "push", + headBranch: "main", + conclusion: "failure", + status: "collected", + }, + ] + : []), + ...Array.from({ length: prCount }, (_, index) => ({ + id: 100 + index, + runAttempt: 1, + workflow: "CI", + event: "pull_request", + headBranch: "feature", + conclusion: "failure", + status: "collected", + })), + { + id: 20, + runAttempt: 1, + workflow: "CI", + event: "push", + headBranch: "main", + conclusion: "failure", + status: "collected", + }, + ], + }, + api, + outputDirectory: directory, + }); + expect(result.failures[0]).toMatchObject({ + runId: 100, + classification: "matching-main-diagnostics", + reason: + "supported_diagnostics_match_failed_task_and_source_on_verified_main_ancestor", + }); + expect(lookups).toHaveLength( + Math.min(prCount, 20) + 1 + Number(incorrectMainHints), + ); + expect(lookups[0]).toBe(incorrectMainHints ? 30 : 20); + expect(new Set(lookups).size).toBe(lookups.length); + expect(result.boundedInput.mainCandidateRuns).toBe(1); + expect(result.failures).toHaveLength(Math.min(prCount, 20)); + }, +); + +it("proves checkout merge parents in base-then-PR-head order and rejects false identities", async () => { + const baseSha = sha("b"); + const headSha = sha("a"); + const mergeSha = sha("c"); + const prRun = run({ + head_sha: headSha, + pull_requests: [ + { base: { sha: baseSha, repo: { full_name: "owner/rea" } } }, + ], + }); + let parents = [baseSha, headSha]; + const calls: string[] = []; + const api = createGitHubApi({ + token: "fixture-token", + apiRequest: async (_method, endpoint) => { + calls.push(endpoint); + return { + sha: mergeSha, + parents: parents.map((parentSha) => ({ sha: parentSha })), + }; + }, + }); + const mergeContext = context(mergeSha, undefined, headSha); + expect( + await api.verifyExecutionSource("owner/rea", prRun, mergeContext), + ).toBe(true); + expect(calls).toEqual([`repos/owner/rea/commits/${mergeSha}`]); + + parents = [headSha, baseSha]; + expect( + await api.verifyExecutionSource("owner/rea", prRun, mergeContext), + ).toBe(false); + expect( + await api.verifyExecutionSource("owner/rea", prRun, { + ...mergeContext, + prHeadSha: sha("e"), + }), + ).toBe(false); +}); + +it("uses static artifacts only when listing is complete, unique, and tied to this run attempt", () => { + const artifact = { + id: 12, + name: "static-check-reports", + expired: false, + workflow_run: { id: 10, head_sha: sha("a") }, + }; + const runRecord = { id: 10, head_sha: sha("a") }; + expect( + selectStaticCheckArtifact( + { total_count: 1, artifacts: [artifact] }, + runRecord, + 1, + ), + ).toBe(artifact); + expect( + selectStaticCheckArtifact( + { total_count: 2, artifacts: [artifact] }, + runRecord, + 1, + ), + ).toBeNull(); + expect( + selectStaticCheckArtifact( + { total_count: 2, artifacts: [artifact, artifact] }, + runRecord, + 1, + ), + ).toBeNull(); + expect( + selectStaticCheckArtifact( + { + total_count: 1, + artifacts: [ + { ...artifact, workflow_run: { id: 11, head_sha: sha("a") } }, + ], + }, + runRecord, + 1, + ), + ).toBeNull(); + expect( + selectStaticCheckArtifact( + { total_count: 1, artifacts: [artifact] }, + runRecord, + 2, + ), + ).toBeNull(); +}); + +it("rejects artifact redirects that could send the bearer token to a non-HTTPS host", async () => { + const calls: string[] = []; + await expect( + readStaticCheckArtifact({ + repository: "owner/rea", + artifactId: 12, + token: "fixture-token", + fetchImpl: (async (url: string | URL | Request) => { + calls.push(String(url)); + return new Response(null, { + status: 302, + headers: { location: "http://untrusted.example/artifact.zip" }, + }); + }) as typeof fetch, + }), + ).rejects.toThrow("static_artifact_redirect_invalid"); + expect(calls).toEqual([ + "https://api.github.com/repos/owner/rea/actions/artifacts/12/zip", + ]); +}); diff --git a/tests/boundary/filesystem/ciFailureReportBudget.test.ts b/tests/boundary/filesystem/ciFailureReportBudget.test.ts new file mode 100644 index 000000000..92fe3e4ae --- /dev/null +++ b/tests/boundary/filesystem/ciFailureReportBudget.test.ts @@ -0,0 +1,290 @@ +import { expect, it } from "vitest"; +import { createTestTempDirectory } from "../../fixtures/temporaryDirectory.js"; +import { + reportCiFailures, + type CiRun, + type FailureReportApi, + type StaticDiagnostic, + type StaticEvidence, +} from "../../../scripts/ci/failure-report.mjs"; + +const repository = "owner/rea"; +const mainSha = "c".repeat(40); +const prSha = "a".repeat(40); +const diagnostic: StaticDiagnostic = { + kind: "lint", + path: "src/example.ts", + code: "no-debugger", + message: "Unexpected debugger statement.", + severity: "error", +}; +const evidence = ( + run: CiRun, + diagnostics: readonly StaticDiagnostic[], +): StaticEvidence => ({ + context: { + version: 1, + sourceSha: run.head_sha!, + sourceClean: true, + nodeVersion: "v24.18.0", + npmVersion: "11.16.0", + turboVersion: "2.11.7", + platform: "linux", + architecture: "x64", + taskHashes: { "lint:built": "a".repeat(16) }, + }, + diagnostics, +}); +const entry = (id: number, event = "pull_request", runAttempt = 1) => ({ + id, + runAttempt, + workflow: "CI", + event, + headBranch: event === "pull_request" ? "feature" : "main", + status: "collected", + conclusion: "failure", +}); + +function fixture(mainIds: readonly number[], prCount: number) { + const jobs: number[] = []; + const artifacts: number[] = []; + const paths: (readonly string[])[] = []; + const runs: Array< + Omit, "event"> & { event?: string } + > = [ + ...mainIds.map((id) => entry(id, "schedule")), + ...Array.from({ length: prCount }, (_, index) => entry(100 + index)), + ]; + const api: FailureReportApi = { + async getRun(_repository, id) { + const selected = runs.find((run) => run.id === Number(id))!; + return { + id, + run_attempt: selected.runAttempt, + status: "completed", + conclusion: "failure", + ...(selected.event ? { event: selected.event } : {}), + head_branch: selected.headBranch, + path: ".github/workflows/ci.yml@main", + head_sha: mainIds.includes(Number(id)) ? mainSha : prSha, + repository: { full_name: repository }, + pull_requests: + selected.event === "pull_request" + ? [{ base: { sha: mainSha, repo: { full_name: repository } } }] + : [], + }; + }, + async getJobs(_repository, id, attempt) { + jobs.push(Number(id)); + return [ + { + name: "Static checks", + status: "completed", + conclusion: "failure", + run_id: Number(id), + run_attempt: attempt, + }, + ]; + }, + async getStaticEvidence(_repository, run) { + artifacts.push(Number(run.id)); + return evidence(run, [diagnostic]); + }, + async getAncestry() { + return { status: "identical", mergeBaseSha: mainSha }; + }, + async getBlobIdentity(_repository, _ancestor, _descendant, requested) { + paths.push(requested); + return Object.fromEntries(requested.map((path) => [path, true])); + }, + async verifyExecutionSource() { + return true; + }, + }; + return { api, runs, jobs, artifacts, paths }; +} + +async function report(input: ReturnType) { + return await reportCiFailures({ + repository, + defaultBranch: "main", + batchManifest: { repository, runs: input.runs }, + api: input.api, + outputDirectory: await createTestTempDirectory("rea-ci-budget-"), + }); +} + +it("selects PR failures before the cap even after a day of scheduled main failures", async () => { + const input = fixture( + Array.from({ length: 24 }, (_, index) => 20 + index), + 1, + ); + input.runs.push({ + id: 200, + runAttempt: 1, + workflow: "CI", + headBranch: "feature", + status: "collected", + conclusion: "failure", + }); + const result = await report(input); + expect(result.failures).toMatchObject([ + { runId: 100, classification: "matching-main-diagnostics" }, + ]); + expect(result.boundedInput).toMatchObject({ + candidateRuns: 1, + mainCandidateRuns: 5, + manifestTruncated: false, + }); + expect(input.jobs).toEqual([100, 20]); +}); + +it("reuses five main job/artifact observations across twenty completed PR comparisons", async () => { + const input = fixture([20, 21, 22, 23, 24], 20); + const load = input.api.getStaticEvidence; + input.api.getStaticEvidence = async (...args) => { + const result = await load(...args); + return Number(args[1].id) < 24 + ? evidence(args[1], [{ ...diagnostic, message: "different" }]) + : result; + }; + const result = await report(input); + expect(result.counts).toEqual({ + "matching-main-diagnostics": 20, + unknown: 0, + }); + expect(input.paths).toHaveLength(100); + expect(input.jobs).toHaveLength(25); + expect(input.artifacts).toHaveLength(25); + for (const id of [20, 21, 22, 23, 24]) { + expect(input.jobs.filter((item) => item === id)).toHaveLength(1); + expect(input.artifacts.filter((item) => item === id)).toHaveLength(1); + } +}); + +it.each(["missing", "artifact error", "job error", "wrong job attempt"])( + "reuses unavailable main evidence without retries: %s", + async (failure) => { + const input = fixture([20], 3); + const load = input.api.getStaticEvidence; + const jobs = input.api.getJobs; + input.api.getJobs = async (...args) => { + const result = await jobs(...args); + if (Number(args[1]) !== 20) return result; + if (failure === "job error") throw new Error("job_snapshot_unavailable"); + return failure === "wrong job attempt" + ? result.map((job) => ({ ...job, run_attempt: 2 })) + : result; + }; + input.api.getStaticEvidence = async (...args) => { + const result = await load(...args); + if (Number(args[1].id) !== 20) return result; + if (failure === "artifact error") throw new Error("artifact_unavailable"); + return null; + }; + const result = await report(input); + expect(result.counts.unknown).toBe(3); + expect(input.jobs.filter((id) => id === 20)).toHaveLength(1); + expect(input.artifacts.filter((id) => id === 20)).toHaveLength( + failure === "job error" || failure === "wrong job attempt" ? 0 : 1, + ); + }, +); + +it.each([1, 2])( + "refreshes unavailable main evidence in the next report (attempt %s)", + async (attempt) => { + const input = fixture([20], 1); + expect((await report(input)).counts["matching-main-diagnostics"]).toBe(1); + input.runs[0]!.runAttempt = attempt; + const load = input.api.getStaticEvidence; + input.api.getStaticEvidence = async (...args) => { + const result = await load(...args); + return Number(args[1].id) === 20 ? null : result; + }; + expect((await report(input)).counts.unknown).toBe(1); + expect(input.jobs.filter((id) => id === 20)).toHaveLength(2); + expect(input.artifacts.filter((id) => id === 20)).toHaveLength(2); + }, +); + +it.each([1, 51])( + "fetches only PR paths with %s PR diagnostics and noisy main evidence", + async (count) => { + const input = fixture([20], 1); + const load = input.api.getStaticEvidence; + const diagnostics = Array.from({ length: count }, (_, index) => ({ + ...diagnostic, + path: `src/pr-${index}.ts`, + })); + input.api.getStaticEvidence = async (...args) => { + await load(...args); + const noise = + Number(args[1].id) === 20 + ? Array.from({ length: 60 }, (_, index) => ({ + ...diagnostic, + path: `src/main-${index}.ts`, + })) + : []; + return evidence(args[1], [...noise, ...diagnostics]); + }; + const result = await report(input); + expect(input.paths).toEqual([ + diagnostics.slice(0, 50).map((item) => item.path), + ]); + expect(result.failures[0]!.classification).toBe( + count === 1 ? "matching-main-diagnostics" : "unknown", + ); + }, +); + +it.each(["jobs", "artifact", "comparison"])( + "continues after an isolated main %s failure and preserves its provenance", + async (failure) => { + const input = fixture([20, 21], 3); + const jobs = input.api.getJobs; + const load = input.api.getStaticEvidence; + input.api.getJobs = async (...args) => { + const result = await jobs(...args); + if (Number(args[1]) === 20 && failure === "jobs") + throw new Error("main_jobs_unavailable"); + return result; + }; + input.api.getStaticEvidence = async (...args) => { + const result = await load(...args); + if (Number(args[1].id) === 20 && failure === "artifact") + throw new Error("main_artifact_unavailable"); + return Number(args[1].id) === 20 && failure === "comparison" + ? { + ...result!, + context: { ...result!.context, sourceSha: "d".repeat(40) }, + } + : result; + }; + input.api.verifyExecutionSource = async (_repository, run, context) => { + if (Number(run.id) === 20 && context.sourceSha !== run.head_sha) + throw new Error("main_comparison_unavailable"); + return true; + }; + const result = await report(input); + expect(result.counts).toEqual({ + "matching-main-diagnostics": 3, + unknown: 0, + }); + for (const item of result.failures) { + expect(item.candidateFailures).toEqual([ + { + runId: 20, + runAttempt: 1, + reason: `main_${failure}_unavailable`, + }, + ]); + } + expect(input.jobs.filter((id) => id === 20)).toHaveLength(1); + expect(input.jobs.filter((id) => id === 21)).toHaveLength(1); + expect(input.artifacts.filter((id) => id === 20)).toHaveLength( + failure === "jobs" ? 0 : 1, + ); + expect(input.artifacts.filter((id) => id === 21)).toHaveLength(1); + }, +); diff --git a/tests/boundary/filesystem/ciFeedback.test.ts b/tests/boundary/filesystem/ciFeedback.test.ts new file mode 100644 index 000000000..c61cb0d92 --- /dev/null +++ b/tests/boundary/filesystem/ciFeedback.test.ts @@ -0,0 +1,313 @@ +import { execFile } from "node:child_process"; +import { mkdir, readFile, readdir, rm, writeFile } from "node:fs/promises"; +import { join } from "node:path"; +import { fileURLToPath } from "node:url"; +import { promisify } from "node:util"; +import { runInNewContext } from "node:vm"; +import { expect, it } from "vitest"; +import { parse } from "yaml"; +import { z } from "zod"; + +import { + parseStaticTaskContext, + recordStaticTaskContext, +} from "../../../scripts/ci/static-task-context.mjs"; +import { createGhApiRequest } from "../../../scripts/ci/github-api.mjs"; +import { createTestTempDirectory } from "../../fixtures/temporaryDirectory.js"; + +const execute = promisify(execFile); +const scopeFlags = async () => + z + .object({ CI_FLAGS: z.array(z.string()) }) + .parse( + await import( + new URL("../../../scripts/ci/scopes.mjs", import.meta.url).href + ), + ).CI_FLAGS; +const workflow = async () => + z + .object({ + jobs: z.record( + z.string(), + z + .object({ + needs: z.union([z.string(), z.array(z.string())]).optional(), + if: z.string().optional(), + steps: z + .array( + z + .object({ + run: z.string().optional(), + env: z.record(z.string(), z.string()).optional(), + }) + .passthrough(), + ) + .optional(), + }) + .passthrough(), + ), + }) + .parse( + parse( + await readFile( + new URL("../../../.github/workflows/ci.yml", import.meta.url), + "utf8", + ), + ), + ).jobs; + +it("starts source feedback before build and requires both groups without waiving missing jobs", async () => { + const jobs = await workflow(); + expect( + jobs.static?.steps?.find( + (step) => step.run === "npm run ci:static-context", + )?.["continue-on-error"], + ).toBe(true); + expect(jobs["test-source"]?.needs).toBe("changes"); + expect(jobs["test-shard"]?.needs).toEqual(["changes", "build"]); + expect( + jobs["test-source"]?.steps?.some( + (step) => + step.run === "npm run test:fast -- --reporter=default --reporter=blob", + ), + ).toBe(true); + const required = fileURLToPath( + new URL("../../../scripts/ci/required.mjs", import.meta.url), + ); + const flags = await scopeFlags(); + for (const tier of ["fast", "full"]) { + for (const result of [ + "success", + "failure", + "cancelled", + "skipped", + "missing", + ]) { + const needs = Object.fromEntries( + Object.keys(jobs) + .filter((name) => name !== "required") + .map((name) => [name, { result: "success" }]), + ); + const outputs = { + ...Object.fromEntries( + flags.map((flag) => [flag, tier === "full" ? "true" : "false"]), + ), + tier, + code: "true", + }; + const context: Record< + string, + { result: string; outputs?: Record } + > = { ...needs, changes: { result: "success", outputs } }; + if (result === "missing") delete context["test-source"]; + else context["test-source"] = { result }; + const run = execute(process.execPath, [required], { + env: { + ...process.env, + CI_JOBS: JSON.stringify(context), + CI_QUALIFICATION_DIRECTORY: "", + }, + }); + if ( + result === "success" || + (tier === "full" && ["skipped", "missing"].includes(result)) + ) + await expect(run).resolves.toMatchObject({ stderr: "" }); + else + await expect(run).rejects.toMatchObject({ + code: 1, + stderr: expect.stringContaining("test-source"), + }); + } + } +}); + +it("records the actual static task graph with explicit source identity and no ambient environment", async () => { + const directory = await createTestTempDirectory("rea-ci-context-"); + const context = await recordStaticTaskContext({ outputDirectory: directory }); + const head = await execute("git", ["rev-parse", "HEAD"]); + expect(context.sourceSha).toBe(head.stdout.trim()); + expect(context.taskHashes["lint:built"]).toMatch(/^[a-f0-9]{16}$/u); + expect(context.taskHashes.jscpd).toMatch(/^[a-f0-9]{16}$/u); + expect(context).not.toHaveProperty("environmentVariables"); + expect(context).not.toHaveProperty("env"); + expect( + JSON.parse(await readFile(`${directory}/static-task-context.json`, "utf8")), + ).toEqual(context); + for (const invalid of [ + undefined, + {}, + { ...context, sourceSha: "bad" }, + { ...context, prHeadSha: "bad" }, + { ...context, taskHashes: { jscpd: "stale" } }, + ]) + expect(() => parseStaticTaskContext(invalid)).toThrow(); +}); + +it("rejects mutation methods at the shared GitHub transport before invoking the CLI", async () => { + const request = createGhApiRequest({ env: { PATH: "" } }); + await expect( + request("POST", "repos/owner/repo/actions/runs/1/rerun", { + requests: 0, + startedAt: Date.now(), + limitMs: 1000, + }), + ).rejects.toThrow("read-only GET"); +}); + +it("collects basic CI telemetry before optional dependency installation can fail", async () => { + const { jobs } = z + .object({ + jobs: z.object({ + report: z.object({ + steps: z.array( + z.object({ + id: z.string().optional(), + run: z.string().optional(), + if: z.string().optional(), + "continue-on-error": z.boolean().optional(), + }), + ), + }), + }), + }) + .parse( + parse( + await readFile( + new URL("../../../.github/workflows/ci-timings.yml", import.meta.url), + "utf8", + ), + ), + ); + const steps = jobs.report.steps; + const installation = steps.findIndex((step) => step.id === "failure-deps"); + for (const script of ["pr-ci-health.mjs", "job-timings-batch.mjs"]) { + const collection = steps.findIndex( + (step) => step.run === `node scripts/ci/${script}`, + ); + expect(collection).toBeGreaterThanOrEqual(0); + expect(installation).toBeGreaterThan(collection); + } + expect(steps[installation]).toMatchObject({ + run: "npm ci", + "continue-on-error": true, + }); + expect( + steps.find((step) => step.id === "npm-cache")?.["continue-on-error"], + ).toBe(true); + expect( + steps.find((step) => step.run === "node scripts/ci/failure-report.mjs")?.if, + ).toContain("steps.failure-deps.outcome == 'success'"); +}); + +it("merges source and built reports exactly once using the workflow's shard matrix", async () => { + const jobs = await workflow(); + const matrix = z + .object({ + matrix: z.object({ shard: z.string().regex(/^\$\{\{.*\}\}$/u) }), + }) + .parse(jobs["test-shard"]?.strategy).matrix.shard; + const shards = z + .array(z.string().regex(/^\d+\/\d+$/u)) + .min(1) + .parse( + runInNewContext( + matrix.slice(3, -2), + { + fromJSON: JSON.parse, + needs: { changes: { outputs: { tier: "fast" } } }, + }, + { timeout: 1_000 }, + ), + ); + const labels = Object.fromEntries( + ["source", "built"].map((group) => [ + group, + z + .string() + .min(1) + .parse( + jobs[group === "source" ? "test-source" : "test-shard"]?.steps?.find( + (step) => step.env?.VITEST_BLOB_LABEL, + )?.env?.VITEST_BLOB_LABEL, + ), + ]), + ); + const root = await createTestTempDirectory("rea-ci-group-merge-"); + const config = join(root, "vitest.config.mjs"); + await writeFile( + config, + `export default { test: { maxWorkers: 1, projects: [${["source", "built"].map((name) => `{ test: { name: "${name}", globals: true, include: ["${name}*.test.js"] } }`).join(",")}] } };`, + ); + const files = [ + "source.test.js", + ...Array.from( + { length: shards.length }, + (_, index) => `built-${index}.test.js`, + ), + ]; + for (const file of files) + await writeFile( + join(root, file), + 'it("runs its selected group", () => expect(true).toBe(true));', + ); + const vitest = fileURLToPath( + new URL("../../../node_modules/vitest/vitest.mjs", import.meta.url), + ); + const blobs = new Map(); + const runs = [ + { group: "source", shard: [] }, + ...shards.map((shard) => ({ group: "built", shard: [`--shard=${shard}`] })), + ]; + for (const { group, shard } of runs) { + await execute( + process.execPath, + [ + vitest, + "run", + "--config", + config, + "--project", + group, + "--reporter=blob", + ...shard, + ], + { + cwd: root, + env: { ...process.env, VITEST_BLOB_LABEL: labels[group]! }, + timeout: 20_000, + }, + ); + for (const filename of await readdir(join(root, ".vitest/blob"))) { + expect(blobs.has(filename)).toBe(false); + blobs.set(filename, await readFile(join(root, ".vitest/blob", filename))); + } + // Separate hosted jobs start with independent report directories. + await rm(join(root, ".vitest/blob"), { recursive: true }); + } + expect(blobs.size).toBe(runs.length); + await mkdir(join(root, ".vitest/blob"), { recursive: true }); + for (const [filename, bytes] of blobs) + await writeFile(join(root, ".vitest/blob", filename), bytes); + await execute( + process.execPath, + [ + vitest, + "--config", + config, + "--merge-reports", + "--reporter=json", + "--outputFile.json=merged.json", + ], + { cwd: root, timeout: 20_000 }, + ); + const merged = z + .object({ + success: z.literal(true), + testResults: z.array(z.object({ name: z.string() })), + }) + .parse(JSON.parse(await readFile(join(root, "merged.json"), "utf8"))); + expect( + merged.testResults.map(({ name }) => name.split("/").at(-1)).sort(), + ).toEqual(files.toSorted()); +}); diff --git a/tests/boundary/filesystem/ciJobTimingsBatch.test.ts b/tests/boundary/filesystem/ciJobTimingsBatch.test.ts index 6c2658c00..e0dcecc82 100644 --- a/tests/boundary/filesystem/ciJobTimingsBatch.test.ts +++ b/tests/boundary/filesystem/ciJobTimingsBatch.test.ts @@ -18,6 +18,8 @@ it("collects the latest attempt for recent completed CI and Release runs into a workflow_runs: [ { id: 42, + event: "push", + head_branch: "main", run_attempt: 3, status: "completed", conclusion: "failure", @@ -43,51 +45,9 @@ it("collects the latest attempt for recent completed CI and Release runs into a }, ], }, - [ - { - jobs: [ - { - name: "Release lane", - status: "completed", - conclusion: "success", - created_at: recent, - started_at: recent, - completed_at: new Date(Date.parse(recent) + 50_000).toISOString(), - steps: [], - }, - ], - }, - ], - [ - { - jobs: [ - { - name: "Failed lane", - status: "completed", - conclusion: "failure", - created_at: recent, - started_at: recent, - completed_at: new Date(Date.parse(recent) + 55_000).toISOString(), - steps: [], - }, - ], - }, - ], - [ - { - jobs: [ - { - name: "Cancelled lane", - status: "completed", - conclusion: "cancelled", - created_at: recent, - started_at: recent, - completed_at: new Date(Date.parse(recent) + 30_000).toISOString(), - steps: [], - }, - ], - }, - ], + completedJobs("Release lane", "success", recent, 50), + completedJobs("Failed lane", "failure", recent, 55), + completedJobs("Cancelled lane", "cancelled", recent, 30), ]); const result = await runBatch(fixture); const manifest = JSON.parse( @@ -116,8 +76,20 @@ it("collects the latest attempt for recent completed CI and Release runs into a expect(manifest.errors).toEqual([]); expect(manifest.runs).toMatchObject([ { id: 43, runAttempt: 2, conclusion: "success" }, - { id: 42, runAttempt: 3, conclusion: "failure" }, - { id: 41, runAttempt: 1, conclusion: "cancelled" }, + { + id: 42, + runAttempt: 3, + conclusion: "failure", + event: "push", + headBranch: "main", + }, + { + id: 41, + runAttempt: 1, + conclusion: "cancelled", + event: null, + headBranch: null, + }, ]); expect(manifest.runs).toEqual( expect.arrayContaining([ @@ -234,6 +206,31 @@ it("retains a partial manifest when its collection budget is exhausted", async ( expect(manifest.truncation.join(" ")).toContain("Time budget"); }); +function completedJobs( + name: string, + conclusion: string, + startedAt: string, + seconds: number, +) { + return [ + { + jobs: [ + { + name, + status: "completed", + conclusion, + created_at: startedAt, + started_at: startedAt, + completed_at: new Date( + Date.parse(startedAt) + seconds * 1000, + ).toISOString(), + steps: [], + }, + ], + }, + ]; +} + async function createFixture(responses: unknown[]) { const directory = await createTestTempDirectory("rea-ci-job-timings-batch-"); const fakeBin = join(directory, "bin"); diff --git a/tests/boundary/filesystem/ciPrHealth.test.ts b/tests/boundary/filesystem/ciPrHealth.test.ts new file mode 100644 index 000000000..756fb4748 --- /dev/null +++ b/tests/boundary/filesystem/ciPrHealth.test.ts @@ -0,0 +1,617 @@ +import { chmod, mkdir, readFile, writeFile } from "node:fs/promises"; +import { delimiter, join } from "node:path"; +import { fileURLToPath } from "node:url"; + +import { expect, it } from "vitest"; + +import { createTestTempDirectory } from "../../fixtures/temporaryDirectory.js"; +import { + classifyPrCi, + collectPrCiHealth, + matchRunToPullRequest, + writePrCiHealthReport, +} from "../../../scripts/ci/pr-ci-health.mjs"; +import { createGhApiRequest } from "../../../scripts/ci/github-api.mjs"; +import type { + PrCiPullRequest, + PrCiWorkflowRun, +} from "../../../scripts/ci/pr-ci-health.mjs"; +import { executeWorkflowFixture } from "../../support/workflowGit.js"; + +const reporter = fileURLToPath( + new URL("../../../scripts/ci/pr-ci-health.mjs", import.meta.url), +); +const observedAt = Date.parse("2026-10-14T00:00:00Z"); +const headSha = "a".repeat(40); +const mergeSha = "b".repeat(40); + +it("classifies workflow states without confusing absent, pending, and completed CI", () => { + const pull = validPull(); + const cases = [ + [ + "startup_failure", + run({ status: "completed", conclusion: "startup_failure" }), + ], + [ + "queued_pending_executing", + run({ status: "in_progress", conclusion: null }), + ], + ["approval_required", run({ status: "waiting", conclusion: null })], + ["completed_failure", run({ status: "completed", conclusion: "failure" })], + [ + "completed_canceled", + run({ status: "completed", conclusion: "cancelled" }), + ], + ["completed_success", run({ status: "completed", conclusion: "success" })], + ["completed_other", run({ status: "completed", conclusion: "neutral" })], + [ + "completed_other", + run({ status: "completed", conclusion: "constructor" }), + ], + ] as const; + for (const [category, evidence] of cases) { + for (const runsComplete of [true, false]) { + const result = classifyPrCi({ + pull, + runs: [ + { + run: { ...evidence, run_attempt: 1 }, + basis: "event_pr_number_and_head_sha", + }, + ], + runsComplete, + pullRequestInventoryComplete: true, + ambiguousRuns: [], + }); + expect(result).toMatchObject({ + category: runsComplete ? category : "unknown", + ...(runsComplete + ? {} + : { reason: "workflow_run_inventory_incomplete" }), + runs: [ + { + id: 77, + attempt: 1, + status: evidence.status, + conclusion: evidence.conclusion, + }, + ], + }); + } + } + for (const runsComplete of [true, false]) { + expect( + classifyPrCi({ + pull: { ...pull, mergeable: false }, + runs: [], + runsComplete, + pullRequestInventoryComplete: true, + ambiguousRuns: [], + }).category, + ).toBe("merge_conflict"); + } +}); + +it("requires complete inventories, known mergeability, and immutable head identity to report missing", () => { + const pull = validPull({ mergeable: true }); + const classify = (overrides = {}) => + classifyPrCi({ + pull, + runs: [], + runsComplete: true, + pullRequestInventoryComplete: true, + ambiguousRuns: [], + ...overrides, + }); + + expect(classify().category).toBe("missing"); + expect(classify({ runsComplete: false })).toMatchObject({ + category: "deferred", + reason: "lookup_inventory_incomplete", + }); + expect(classify({ pullRequestInventoryComplete: false }).category).toBe( + "deferred", + ); + expect( + classify({ + ambiguousRuns: [{ run: run(), basis: "ambiguous_head_identity" }], + }).category, + ).toBe("unknown"); + expect(classify({ pull: { ...pull, mergeable: undefined } }).category).toBe( + "unknown", + ); + expect(classify({ pull: { ...pull, identityValid: false } }).category).toBe( + "unknown", + ); +}); + +it("accepts event PR/head evidence for merge-SHA runs and refuses ambiguous branch identities", () => { + const pull = validPull(); + const associatedMergeRun = run({ + head_sha: mergeSha, + pull_requests: [ + { + number: pull.number!, + head: { sha: headSha, repo: { id: pull.head.repositoryId } }, + }, + ], + }); + expect( + matchRunToPullRequest({ + run: associatedMergeRun, + pull, + identityCounts: new Map([["73\u0000feature", 1]]), + }), + ).toEqual({ kind: "match", basis: "event_pr_number_and_head_sha" }); + expect( + matchRunToPullRequest({ + run: run({ + pull_requests: [ + { number: pull.number!, head: { sha: "c".repeat(40) } }, + ], + }), + pull, + identityCounts: new Map([["73\u0000feature", 1]]), + }).kind, + ).toBe("none"); + + const branchRun = run({ pull_requests: [] }); + expect( + matchRunToPullRequest({ + run: branchRun, + pull, + identityCounts: new Map([["73\u0000feature", 1]]), + inventoryComplete: false, + }).kind, + ).toBe("ambiguous"); + expect( + matchRunToPullRequest({ + run: branchRun, + pull, + identityCounts: new Map([["73\u0000feature", 1]]), + }), + ).toEqual({ + kind: "match", + basis: "event_unique_head_ref_and_repository_id", + }); + expect( + matchRunToPullRequest({ + run: run({ head_sha: mergeSha, pull_requests: [] }), + pull, + identityCounts: new Map([["73\u0000feature", 1]]), + }).kind, + ).toBe("none"); + expect( + matchRunToPullRequest({ + run: run({ + pull_requests: [{ number: pull.number!, head: { sha: headSha } }], + }), + pull, + identityCounts: new Map([["73\u0000feature", 1]]), + }).kind, + ).toBe("ambiguous"); + expect( + matchRunToPullRequest({ + run: branchRun, + pull, + identityCounts: new Map([["73\u0000feature", 2]]), + }).kind, + ).toBe("ambiguous"); + expect( + matchRunToPullRequest({ + run: run({ event: "workflow_dispatch" }), + pull, + identityCounts: new Map([["73\u0000feature", 1]]), + }).kind, + ).toBe("none"); +}); + +it("scans target-branch PRs through public gh GET requests and writes JSON and Markdown", async () => { + const fixture = await createFixture(); + const calls: string[] = []; + const apiRequest: NonNullable< + Parameters[0]["apiRequest"] + > = async (method, endpoint) => { + calls.push(`${method} ${endpoint}`); + if (method !== "GET") throw new Error("fixture rejected a non-GET request"); + if (endpoint.includes("/pulls?")) + return [ + rawPull(42, { mergeable: true }), + rawPull(43, { mergeable: false }), + rawPull(44, { + base: { ref: "release", repo: { id: 11 } }, + mergeable: true, + }), + rawPull(45, { draft: true, mergeable: true }), + ]; + if (endpoint.includes("/actions/workflows/ci.yml/runs?")) + return { + total_count: 1, + workflow_runs: [ + run({ + id: 9001, + event: "pull_request", + status: "completed", + conclusion: "failure", + head_sha: mergeSha, + pull_requests: [ + { + number: 42, + head: { sha: headSha, repo: { id: 73 } }, + }, + ], + }), + ], + }; + throw new Error(`unexpected GET endpoint: ${endpoint}`); + }; + const report = await collectPrCiHealth({ + repository: "owner/rea", + repositoryId: 11, + defaultBranch: "main", + now: observedAt, + apiRequest, + }); + expect(calls).toHaveLength(2); + expect(calls.every((call) => call.startsWith("GET "))).toBe(true); + expect(calls[0]).toContain("state=open"); + expect(calls[0]).not.toContain("base="); + expect(calls[1]).toContain("event=pull_request"); + expect(report.pullRequests).toHaveLength(2); + expect(report.pullRequests.map(({ number }) => number)).toEqual([42, 43]); + expect(report.pullRequests.map(({ ci }) => ci.category)).toEqual([ + "completed_failure", + "merge_conflict", + ]); + expect(report.scan).toMatchObject({ + status: "complete", + pullRequestInventoryComplete: true, + workflowRunInventoryComplete: true, + apiRequests: 2, + }); + + const output = join(fixture.directory, ".cache", "ci-pr-health"); + const paths = await writePrCiHealthReport(report, output); + const written = JSON.parse(await readFile(paths.jsonPath, "utf8")); + const markdown = await readFile(paths.markdownPath, "utf8"); + expect(written.counts.completed_failure).toBe(1); + expect(markdown).toContain("# Pull request CI health snapshot"); + expect(markdown).toContain("#42"); + expect(markdown).toContain("completed_failure"); +}); + +it("counts older open PRs when deciding whether a branch/ref fallback is unique", async () => { + const olderPull = rawPull(63, { + created_at: "2026-09-01T00:00:00Z", + head: { ...rawPull(63).head, ref: "shared-feature" }, + }); + const currentPull = rawPull(64, { + mergeable: true, + head: { ...olderPull.head }, + }); + const apiRequest: NonNullable< + Parameters[0]["apiRequest"] + > = async (_method, endpoint) => { + if (endpoint.includes("/pulls?")) return [currentPull, olderPull]; + if (endpoint.includes("/actions/workflows/ci.yml/runs?")) + return { + workflow_runs: [ + run({ + head_branch: "shared-feature", + pull_requests: [], + }), + ], + }; + throw new Error(`unexpected GET endpoint: ${endpoint}`); + }; + const report = await collectPrCiHealth({ + repository: "owner/rea", + repositoryId: 11, + defaultBranch: "main", + now: observedAt, + apiRequest, + }); + expect(report.scan.pullRequestInventoryComplete).toBe(true); + expect(report.pullRequests.map(({ number }) => number)).toEqual([64]); + expect(report.pullRequests[0]?.ci).toMatchObject({ + category: "unknown", + reason: "workflow_run_identity_ambiguous", + }); +}); + +it.each([false, true])( + "keeps page-capped scans uncertain (observed success: %s)", + async (observedSuccess) => { + const pull = rawPull(62, { mergeable: true }); + const apiRequest: NonNullable< + Parameters[0]["apiRequest"] + > = async (_method, endpoint) => { + if (endpoint.includes("/pulls?")) return [pull]; + const page = Number( + new URL(`https://fixture.invalid/${endpoint}`).searchParams.get("page"), + ); + return { + workflow_runs: Array.from({ length: 100 }, (_item, index) => + observedSuccess && page === 1 && index === 0 + ? run({ + run_attempt: 1, + pull_requests: [ + { number: 62, head: { sha: headSha, repo: { id: 73 } } }, + ], + }) + : run({ + id: page * 100 + index, + head_branch: "different-branch", + head_repository: { id: 999 }, + pull_requests: [], + }), + ), + }; + }; + const report = await collectPrCiHealth({ + repository: "owner/rea", + repositoryId: 11, + defaultBranch: "main", + now: observedAt, + apiRequest, + }); + expect(report.scan.workflowRunPages).toBe(5); + expect(report.scan.workflowRunInventoryComplete).toBe(false); + const firstPull = report.pullRequests[0]; + if (firstPull === undefined) throw new Error("fixture PR was not scanned"); + expect(firstPull.ci).toMatchObject({ + category: observedSuccess ? "unknown" : "deferred", + reason: observedSuccess + ? "workflow_run_inventory_incomplete" + : "lookup_inventory_incomplete", + }); + expect(firstPull.ci.runs).toHaveLength(observedSuccess ? 1 : 0); + }, +); + +it("keeps the result unknown when the head changes between API snapshots", async () => { + const apiRequest: NonNullable< + Parameters[0]["apiRequest"] + > = async (_method, endpoint) => { + if (endpoint.includes("/pulls?")) return [rawPull(63)]; + if (endpoint.includes("/actions/workflows/ci.yml/runs?")) + return { workflow_runs: [] }; + return { + ...rawPull(63, { mergeable: true }), + head: { + ...rawPull(63).head, + sha: mergeSha, + }, + }; + }; + const report = await collectPrCiHealth({ + repository: "owner/rea", + repositoryId: 11, + defaultBranch: "main", + now: observedAt, + apiRequest, + }); + expect(report.pullRequests[0]).toMatchObject({ + identityChangedDuringScan: true, + ci: { category: "unknown", reason: "pull_request_head_identity_invalid" }, + }); + expect(report.scan).toMatchObject({ + status: "partial", + errors: ["pull_request_identity_changed_during_scan"], + }); +}); + +it("public CLI uses only GET and produces a report from fixture GitHub responses", async () => { + const fixture = await createFixture(); + const ghCalls = join(fixture.directory, "gh-calls.jsonl"); + const ghResponse = join(fixture.directory, "gh-response.json"); + const fakeGh = join(fixture.fakeBin, "gh"); + await writeFile( + ghResponse, + JSON.stringify([ + [rawPull(51, { mergeable: true })], + { total_count: 0, workflow_runs: [] }, + ]), + ); + await writeFile( + fakeGh, + `#!/usr/bin/env node\nconst fs=require("node:fs");\nconst args=process.argv.slice(2);\nfs.appendFileSync(process.env.CI_GH_CALLS, JSON.stringify(args)+"\\n");\nif(args[0]!=="api"||args[1]!=="--method"||args[2]!=="GET")process.exit(41);\nconst page=Number(new URL("https://fixture.invalid/"+args.at(-1)).searchParams.get("page"));\nconst responses=JSON.parse(fs.readFileSync(process.env.CI_GH_RESPONSE,"utf8"));\nprocess.stdout.write(JSON.stringify(responses[page===1&&args.at(-1).includes("/pulls?")?0:1]));\n`, + ); + await chmod(fakeGh, 0o755); + const outputDirectory = join(fixture.directory, ".cache", "ci-pr-health"); + await executeWorkflowFixture(process.execPath, [reporter], { + cwd: fixture.directory, + env: { + ...process.env, + PATH: `${fixture.fakeBin}${delimiter}${process.env.PATH ?? ""}`, + CI_REPOSITORY: "owner/rea", + CI_REPOSITORY_ID: "11", + CI_DEFAULT_BRANCH: "main", + CI_PR_HEALTH_DIRECTORY: outputDirectory, + CI_GH_CALLS: ghCalls, + CI_GH_RESPONSE: ghResponse, + }, + }); + const calls = (await readFile(ghCalls, "utf8")) + .trim() + .split("\n") + .map((line) => JSON.parse(line)); + expect(calls).toHaveLength(2); + expect( + calls.every( + (args: string[]) => args[1] === "--method" && args[2] === "GET", + ), + ).toBe(true); + const report = JSON.parse( + await readFile(join(outputDirectory, "report.json"), "utf8"), + ); + expect(report.pullRequests[0].ci.category).toBe("missing"); + await expect( + readFile(join(outputDirectory, "summary.md"), "utf8"), + ).resolves.toContain("missing"); +}); + +it("rejects non-GET methods before launching the GitHub client", async () => { + const fixture = await createFixture(); + const request = createGhApiRequest({ + env: { + ...process.env, + PATH: `${fixture.fakeBin}${delimiter}${process.env.PATH ?? ""}`, + CI_GH_CALLS: join(fixture.directory, "should-not-exist"), + }, + }); + await expect( + request("POST", "repos/owner/rea/pulls", { + requests: 0, + startedAt: observedAt, + limitMs: 1_000, + }), + ).rejects.toThrow("read-only GET"); + await expect( + readFile(join(fixture.directory, "should-not-exist")), + ).rejects.toMatchObject({ code: "ENOENT" }); +}); + +function validPull(overrides: Partial = {}): PrCiPullRequest { + return { + number: 42, + identityValid: true, + mergeable: true, + head: { repositoryId: 73, ref: "feature", sha: headSha }, + ...overrides, + }; +} + +it.each([ + ["newer run", { id: 78, created_at: "2026-10-13T01:00:00Z" }, "unknown"], + [ + "older run", + { id: 78, created_at: "2026-10-12T23:00:00Z" }, + "completed_success", + ], + ["equal timestamps", { id: 78 }, "unknown"], + [ + "retry of an older run", + { id: 78, created_at: "2026-10-12T23:00:00Z", run_attempt: 2 }, + "unknown", + ], + ["unknown timestamp", { id: 78, created_at: "invalid" }, "unknown"], + ["newer attempt", { run_attempt: 3 }, "unknown"], + ["older attempt", { run_attempt: 1 }, "completed_success"], +] as const)( + "retains ambiguity evidence for a %s without reporting a stale completion", + (_label, overrides, category) => { + const currentAttempt = "id" in overrides ? 1 : 2; + const result = classifyPrCi({ + pull: validPull(), + runs: [ + { + run: run({ run_attempt: currentAttempt }), + basis: "event_pr_number_and_head_sha", + }, + ], + ambiguousRuns: [ + { + run: run({ run_attempt: currentAttempt, ...overrides }), + basis: "ambiguous_head_identity", + }, + ], + runsComplete: true, + pullRequestInventoryComplete: true, + }); + expect(result.category).toBe(category); + expect(result.runs).toHaveLength(2); + }, +); + +it("does not let an older proven run hide a newer run with an unverified repository ID", async () => { + const pull = rawPull(42); + const older = run({ + id: 70, + created_at: "2026-10-13T00:00:00Z", + pull_requests: [{ number: 42, head: { sha: headSha, repo: { id: 73 } } }], + }); + const newer = run({ + id: 71, + created_at: "2026-10-13T01:00:00Z", + status: "in_progress", + conclusion: null, + pull_requests: [{ number: 42, head: { sha: headSha } }], + }); + const report = await collectPrCiHealth({ + repository: "owner/rea", + repositoryId: 11, + defaultBranch: "main", + now: observedAt, + apiRequest: async (_method, endpoint) => { + if (endpoint.includes("/pulls?")) return [pull]; + if (endpoint.endsWith("/pulls/42")) return { ...pull, mergeable: true }; + return { workflow_runs: [newer, older] }; + }, + }); + expect(report.pullRequests[0]?.ci).toMatchObject({ + category: "unknown", + reason: "workflow_run_identity_ambiguous", + }); + expect(report.pullRequests[0]?.ci.runs).toHaveLength(2); +}); + +it("keeps a proven newest run even when two older matched runs have equal timestamps", () => { + const result = classifyPrCi({ + pull: validPull(), + runs: [ + run({ id: 70, run_attempt: 1 }), + run({ id: 71, run_attempt: 1, conclusion: "failure" }), + run({ + id: 72, + run_attempt: 1, + created_at: "2026-10-13T01:00:00Z", + status: "in_progress", + conclusion: null, + }), + ].map((run) => ({ run, basis: "event_pr_number_and_head_sha" })), + ambiguousRuns: [], + runsComplete: true, + pullRequestInventoryComplete: true, + }); + expect(result.category).toBe("queued_pending_executing"); + expect(result.runs).toHaveLength(3); +}); + +function run(overrides: Partial = {}): PrCiWorkflowRun { + return { + id: 77, + event: "pull_request", + head_branch: "feature", + head_repository: { id: 73 }, + head_sha: headSha, + status: "completed", + conclusion: "success", + created_at: "2026-10-13T00:00:00Z", + pull_requests: [], + ...overrides, + }; +} + +function rawPull(number: number, overrides: Record = {}) { + return { + number, + draft: false, + created_at: "2026-10-13T00:00:00Z", + html_url: `https://github.com/owner/rea/pull/${String(number)}`, + base: { ref: "main", repo: { id: 11 } }, + head: { + ref: `feature-${String(number)}`, + sha: headSha, + repo: { id: 73, full_name: "contributor/rea" }, + }, + ...overrides, + }; +} + +async function createFixture() { + const directory = await createTestTempDirectory("rea-ci-pr-health-"); + const fakeBin = join(directory, "bin"); + await mkdir(fakeBin); + return { directory, fakeBin }; +} diff --git a/tests/boundary/filesystem/ciQualification.test.ts b/tests/boundary/filesystem/ciQualification.test.ts index b6a5a1e1a..673b09d34 100644 --- a/tests/boundary/filesystem/ciQualification.test.ts +++ b/tests/boundary/filesystem/ciQualification.test.ts @@ -44,7 +44,7 @@ function run(overrides: Record = {}) { id: 7, status: "completed", conclusion: "success", - path: ".github/workflows/ci.yml", + path: ".github/workflows/ci.yml@main", event: "schedule", created_at: new Date(now - 120_000).toISOString(), head_sha: validSha, diff --git a/tests/boundary/filesystem/ciRequiredLanes.test.ts b/tests/boundary/filesystem/ciRequiredLanes.test.ts index d7ac1418e..d2191eb01 100644 --- a/tests/boundary/filesystem/ciRequiredLanes.test.ts +++ b/tests/boundary/filesystem/ciRequiredLanes.test.ts @@ -121,7 +121,11 @@ it("exports every canonical scope with read authority for admission and keeps te ); expect(timings.on.workflow_run).toBeUndefined(); expect(timings.on.schedule).toHaveLength(1); - expect(timings.permissions).toEqual({ contents: "read", actions: "read" }); + expect(timings.permissions).toEqual({ + contents: "read", + actions: "read", + "pull-requests": "read", + }); const checkout = timings.jobs.report.steps.find((step) => step.uses?.startsWith("actions/checkout@"), ); @@ -154,6 +158,7 @@ it("holds native and provider fan-out behind static validation while portable fe "static", "build", "test-shard", + "test-source", "test", "docs", "required", @@ -369,7 +374,7 @@ it("keeps fast feedback separate from complete qualification and release proof", // release their concurrency slot without waiting for another runner. expect(ci.jobs.required?.if).toBe("${{ !cancelled() }}"); const shards = ci.jobs["test-shard"]?.steps ?? []; - const fast = shards.find((step) => step.run?.includes("npm run test:run")); + const fast = shards.find((step) => step.run?.includes("npm run test:built")); const full = shards.find((step) => step.run?.includes("test:ci:shard:run")); expect(fast?.if).toBe("needs.changes.outputs.tier == 'fast'"); expect(fast?.run).toContain("--reporter=blob"); diff --git a/tests/boundary/filesystem/ciStaticArtifacts.test.ts b/tests/boundary/filesystem/ciStaticArtifacts.test.ts new file mode 100644 index 000000000..45915f450 --- /dev/null +++ b/tests/boundary/filesystem/ciStaticArtifacts.test.ts @@ -0,0 +1,99 @@ +import { execFile } from "node:child_process"; +import { mkdir, readFile, writeFile } from "node:fs/promises"; +import { join } from "node:path"; +import { fileURLToPath } from "node:url"; +import { promisify } from "node:util"; +import { expect, it } from "vitest"; + +import { parseStaticArchive } from "../../../scripts/ci/artifacts.mjs"; +import { createTestTempDirectory } from "../../fixtures/temporaryDirectory.js"; + +const execute = promisify(execFile); +const producer = fileURLToPath( + new URL("../../../scripts/ci/static-check-report.mjs", import.meta.url), +); + +it.each([3.5, 100, 0])( + "retains real lint failures when the duplication threshold is %s", + async (threshold) => { + const root = await createTestTempDirectory("rea-ci-static-archive-"); + await mkdir(join(root, "src")); + await mkdir(join(root, "tests")); + await writeFile( + join(root, ".oxlintrc.json"), + JSON.stringify({ rules: { "no-debugger": "error" } }), + ); + await writeFile( + join(root, ".jscpd.json"), + JSON.stringify({ threshold, minLines: 5, minTokens: 50 }), + ); + await writeFile( + join(root, "src/example.ts"), + "debugger;\nexport const example = true;\n", + ); + const duplicate = + "export function compute(value: number) {\n" + + Array.from({ length: 20 }, (_, index) => ` value += ${index};\n`).join( + "", + ) + + " return value;\n}\n"; + for (const name of threshold === 3.5 ? [] : ["first", "second"]) + await writeFile(join(root, `src/${name}.ts`), duplicate); + await expect( + execute(process.execPath, [producer, "lint", "src/example.ts"], { + cwd: root, + }), + ).rejects.toMatchObject({ code: 1 }); + const duplication = execute(process.execPath, [producer, "duplicates"], { + cwd: root, + }); + if (threshold === 0) + await expect(duplication).rejects.toMatchObject({ code: 1 }); + else await duplication; + const reports = join(root, ".cache/static-checks"); + expect( + JSON.parse( + await readFile(join(reports, "jscpd/jscpd-report.json"), "utf8"), + ), + ).toMatchObject({ reaCheck: { exitCode: threshold === 0 ? 1 : 0 } }); + await writeFile( + join(reports, "static-task-context.json"), + JSON.stringify({ + version: 1, + sourceSha: "a".repeat(40), + sourceClean: true, + nodeVersion: process.version, + npmVersion: "11.16.0", + turboVersion: "2.11.7", + platform: process.platform, + architecture: process.arch, + taskHashes: { "lint:built": "a".repeat(16), jscpd: "b".repeat(16) }, + }), + ); + const archive = join(root, "reports.zip"); + await execute( + "zip", + [ + "-q", + archive, + "oxlint.json", + "jscpd/jscpd-report.json", + "static-task-context.json", + ], + { cwd: reports }, + ); + const evidence = await parseStaticArchive(archive); + expect(evidence.diagnostics.filter(({ kind }) => kind === "lint")).toEqual([ + expect.objectContaining({ + kind: "lint", + path: "src/example.ts", + severity: "error", + }), + ]); + const duplicates = evidence.diagnostics.filter( + ({ kind }) => kind === "jscpd", + ); + if (threshold === 0) expect(duplicates.length).toBeGreaterThan(0); + else expect(duplicates).toEqual([]); + }, +); diff --git a/tests/boundary/filesystem/ciTestHistory.test.ts b/tests/boundary/filesystem/ciTestHistory.test.ts index 411e9ece9..84950d49c 100644 --- a/tests/boundary/filesystem/ciTestHistory.test.ts +++ b/tests/boundary/filesystem/ciTestHistory.test.ts @@ -35,7 +35,7 @@ function run(id: number, overrides: Record = {}) { id, status: "completed", conclusion: "success", - path: ".github/workflows/ci.yml", + path: ".github/workflows/ci.yml@main", event: "schedule", created_at: new Date(now - 120_000).toISOString(), head_branch: "main", diff --git a/tests/boundary/filesystem/ciTestProjects.test.ts b/tests/boundary/filesystem/ciTestProjects.test.ts new file mode 100644 index 000000000..f0394c5bb --- /dev/null +++ b/tests/boundary/filesystem/ciTestProjects.test.ts @@ -0,0 +1,75 @@ +import { execFileSync } from "node:child_process"; +import { readFile } from "node:fs/promises"; +import { join } from "node:path"; + +import { describe, expect, it } from "vitest"; +import { + BUILT_TEST_PROJECTS, + SOURCE_TEST_PROJECTS, + projectForTest, +} from "../../../scripts/lib/test-projects.mjs"; +import { + developmentTestPlan, + parseDevelopmentTestRequest, +} from "../../../scripts/lib/development-tests.mjs"; + +const repository = process.cwd(); + +describe("canonical test-project ownership", () => { + it("assigns every tracked test file exactly once to source or built", () => { + const files = execFileSync("git", ["ls-files", "*.test.ts"], { + cwd: repository, + encoding: "utf8", + }) + .trim() + .split("\n") + .filter(Boolean); + const grouped = new Map(); + for (const file of files) { + const project = projectForTest(file); + grouped.set(project.group, [...(grouped.get(project.group) ?? []), file]); + } + + expect(grouped.get("source")).not.toBeUndefined(); + expect(grouped.get("built")).not.toBeUndefined(); + expect( + (grouped.get("source")?.length ?? 0) + + (grouped.get("built")?.length ?? 0), + ).toBe(files.length); + expect( + SOURCE_TEST_PROJECTS.reduce( + (count, project) => count + project.include.length, + 0, + ), + ).toBeGreaterThan(0); + expect( + BUILT_TEST_PROJECTS.every((project) => project.group === "built"), + ).toBe(true); + expect(() => projectForTest("tests/unowned/example.test.ts")).toThrow( + "No test project owns", + ); + }); + + it("owns the Hopper real-provider conformance test as built and requires preparation", () => { + const testPath = + "tests/conformance/providers/hopper/realHopperSearch.test.ts"; + expect(projectForTest(testPath)).toMatchObject({ + name: "boundary", + group: "built", + }); + expect(() => parseDevelopmentTestRequest("local", [testPath])).toThrow( + `Use test:focused for built test projects: ${testPath}`, + ); + const request = parseDevelopmentTestRequest("focused", [testPath]); + expect(developmentTestPlan(request)).toMatchObject({ needsBuild: true }); + }); + + it("uses canonical selectors in Vitest while retaining independent runner policy", async () => { + const config = await readFile(join(repository, "vitest.config.ts"), "utf8"); + expect(config).toContain( + "[...SOURCE_TEST_PROJECTS, ...BUILT_TEST_PROJECTS]", + ); + expect(config).toContain("PROJECT_POLICIES"); + expect(config).not.toContain('"src/{contracts,domain}/**/*.test.ts"'); + }); +}); diff --git a/tests/boundary/filesystem/ciTestReports.test.ts b/tests/boundary/filesystem/ciTestReports.test.ts index 3e2aadcd4..6a5942448 100644 --- a/tests/boundary/filesystem/ciTestReports.test.ts +++ b/tests/boundary/filesystem/ciTestReports.test.ts @@ -3,6 +3,7 @@ import { mkdir, readFile, rm, writeFile } from "node:fs/promises"; import { join, resolve } from "node:path"; import { fileURLToPath } from "node:url"; import { promisify } from "node:util"; +import { runInNewContext } from "node:vm"; import { expect, it } from "vitest"; import { parse } from "yaml"; @@ -48,9 +49,7 @@ const workflow = async () => it("merges available failed-shard diagnostics and skips unavailable shard reports", async () => { const steps = (await workflow()).test!.steps; const download = steps.find( - (step) => - step.uses?.includes("actions/download-artifact") && - step.with?.pattern === "vitest-report-*", + (step) => step.with?.pattern === "vitest-report-*", ); const merge = steps.findIndex((step) => step.run?.includes("test:ci:merge")); const upload = steps.findIndex((step) => step.with?.name === "test-results"); @@ -59,8 +58,7 @@ it("merges available failed-shard diagnostics and skips unavailable shard report ); expect(merge).toBeGreaterThan(0); expect(download?.if).toContain("!cancelled()"); - expect(download?.if).toContain("needs.test-shard.result == 'success'"); - expect(download?.if).toContain("needs.test-shard.result == 'failure'"); + expect(upload).toBeGreaterThan(merge); expect(gate).toBeGreaterThan(upload); const merges = steps.filter((step) => step.run?.includes("--reporter=json")); @@ -70,8 +68,6 @@ it("merges available failed-shard diagnostics and skips unavailable shard report "--reporter=./scripts/ci/test-timing-reporter.mjs", ); expect(step.if).toContain("!cancelled()"); - expect(step.if).toContain("needs.test-shard.result == 'success'"); - expect(step.if).toContain("needs.test-shard.result == 'failure'"); } expect(steps[upload]?.with?.path).toContain("test-timings.json"); expect(steps[upload]?.with).toMatchObject({ "include-hidden-files": true }); @@ -86,12 +82,23 @@ it("merges available failed-shard diagnostics and skips unavailable shard report ); const command = steps[gate]?.run; if (command === undefined) throw new Error("Missing shard result gate"); - for (const result of ["failure", "cancelled", "skipped", ""]) { - await expect( - execute("bash", ["-c", command], { - env: { ...process.env, CODE_REQUIRED: "true", SHARD_RESULT: result }, - }), - ).rejects.toMatchObject({ code: 1 }); + for (const tier of ["fast", "full"]) { + for (const shard of ["success", "failure", "cancelled", "skipped", ""]) { + for (const source of ["success", "failure", "cancelled", "skipped", ""]) { + const check = execute("bash", ["-c", command], { + env: { + ...process.env, + CODE_REQUIRED: "true", + SHARD_RESULT: shard, + SOURCE_REQUIRED: String(tier === "fast"), + SOURCE_RESULT: source, + }, + }); + if (shard === "success" && (tier !== "fast" || source === "success")) + await expect(check).resolves.toMatchObject({ stderr: "" }); + else await expect(check).rejects.toMatchObject({ code: 1 }); + } + } } await expect( execute("bash", ["-c", command], { @@ -100,6 +107,80 @@ it("merges available failed-shard diagnostics and skips unavailable shard report ).resolves.toMatchObject({ stderr: "" }); }); +// These workflow conditions use the shared boolean/string expression subset. +// Evaluate the checked-in expressions so failure-state coverage cannot drift +// into a separate test implementation of the collection policy. +function conditionValue( + condition: string | undefined, + needs: Record, + canceled: boolean, +) { + if (!condition) throw new Error("Missing report collection condition"); + return runInNewContext( + condition + .replaceAll("needs.test-shard", 'needs["test-shard"]') + .replaceAll("needs.test-source", 'needs["test-source"]'), + { needs, cancelled: () => canceled }, + { timeout: 1_000 }, + ); +} + +it("collects either fast test group while keeping full reports tied to full shards", async () => { + const steps = (await workflow()).test.steps; + const download = steps.find( + (step) => step.with?.pattern === "vitest-report-*", + ); + const fast = steps.find( + (step) => step.name === "Merge fast-suite JUnit and timing reports", + ); + const full = steps.find( + (step) => step.name === "Merge coverage, JUnit, and timing reports", + ); + for (const tier of ["fast", "full"]) { + for (const shard of ["success", "failure", "cancelled", "skipped", ""]) { + for (const source of ["success", "failure", "cancelled", "skipped", ""]) { + for (const [code, canceled] of [ + ["true", false], + ["true", true], + ["false", false], + ["false", true], + ] as const) { + const needs = { + changes: { outputs: { code, tier } }, + "test-shard": { result: shard }, + "test-source": { result: source }, + }; + const builtAvailable = ["success", "failure"].includes(shard); + const sourceAvailable = ["success", "failure"].includes(source); + const enabled = code === "true" && !canceled; + const available = + builtAvailable || (tier === "fast" && sourceAvailable); + const expected = enabled + ? { + download: available, + fast: tier === "fast" && available, + full: tier === "full" && builtAvailable, + } + : { download: false, fast: false, full: false }; + expect(conditionValue(download?.if, needs, canceled)).toBe( + expected.download, + ); + expect(conditionValue(fast?.if, needs, canceled)).toBe(expected.fast); + expect(conditionValue(full?.if, needs, canceled)).toBe(expected.full); + } + } + } + } + const summary = steps.find((step) => + step.run?.includes("scripts/ci/test-report.mjs"), + ); + expect(summary?.env?.SHARD_RESULT).toBe("${{ needs.test-shard.result }}"); + expect(summary?.env?.SOURCE_RESULT).toBe("${{ needs.test-source.result }}"); + expect(summary?.env?.SOURCE_REQUIRED).toBe( + "${{ needs.changes.outputs.tier == 'fast' }}", + ); +}); + it("keeps diagnosis available after merge failure and publishes only complete successful history", async () => { const steps = (await workflow()).test!.steps; const diagnosis = steps.find((step) => step.with?.name === "test-results"); @@ -220,7 +301,12 @@ it("exports portable timing history and removes it on a failed, partial, or inva { name: join(root, "tests/fast.test.ts"), durationMs: 10 }, ], }; - const run = async (value: unknown, shards = "success", merge = "success") => { + const run = async ( + value: unknown, + shards = "success", + merge = "success", + source?: string, + ) => { await writeFile(join(root, "test-timings.json"), JSON.stringify(value)); await writeFile(summary, ""); return execute(process.execPath, [fileURLToPath(script)], { @@ -229,6 +315,8 @@ it("exports portable timing history and removes it on a failed, partial, or inva ...process.env, GITHUB_STEP_SUMMARY: summary, SHARD_RESULT: shards, + SOURCE_REQUIRED: String(source !== undefined), + SOURCE_RESULT: source ?? "", MERGE_RESULT: merge, }, }); @@ -256,6 +344,24 @@ it("exports portable timing history and removes it on a failed, partial, or inva await expect(readFile(history)).rejects.toMatchObject({ code: "ENOENT" }); expect(await readFile(summary, "utf8")).toContain("diagnostic only"); } + await run(report, "success", "success", "success"); + expect(await readFile(history, "utf8")).toContain('"success":true'); + for (const [shards, source] of [ + ["skipped", "success"], + ["success", "skipped"], + ["success", "cancelled"], + ["success", "failure"], + ["success", ""], + ]) { + await run(report); + await run(report, shards, "success", source); + await expect(readFile(history)).rejects.toMatchObject({ code: "ENOENT" }); + const text = await readFile(summary, "utf8"); + expect(text).toContain(`Source result: ${source}.`); + expect(text).toContain("diagnostic only"); + if (shards === "skipped" || source !== "failure") + expect(text).toContain("Partial test diagnostics"); + } await run(report); await expect( run({ @@ -352,6 +458,21 @@ it("recovers complete module costs from real blob reports without executing test expect(file.durationMs).toBeCloseTo( Object.values(file.phases).reduce((sum, value) => sum + value, 0), ); + const partial = await execute(process.execPath, [fileURLToPath(script)], { + cwd: root, + env: { + ...process.env, + SHARD_RESULT: "skipped", + SOURCE_REQUIRED: "true", + SOURCE_RESULT: "success", + MERGE_RESULT: "success", + }, + }); + expect(partial.stdout).toContain("Partial test diagnostics"); + expect(partial.stdout).toContain("sample.test.js"); + await expect( + readFile(join(root, ".cache/ci-test-history/test-results.json")), + ).rejects.toMatchObject({ code: "ENOENT" }); await execute(process.execPath, [fileURLToPath(script)], { cwd: root, env: { ...process.env, SHARD_RESULT: "success", MERGE_RESULT: "success" }, @@ -420,13 +541,15 @@ it("records failed omissions from full reports without treating partial or unhan state: index === 1 ? "failed" : "passed", })), }; - const run = async (report: unknown) => { + const run = async (report: unknown, source = "failure") => { await writeFile(join(root, "test-timings.json"), JSON.stringify(report)); await execute(process.execPath, [fileURLToPath(script)], { cwd: root, env: { ...process.env, SHARD_RESULT: "failure", + SOURCE_REQUIRED: "true", + SOURCE_RESULT: source, MERGE_RESULT: "failure", CI_VERIFY_TEST_INVENTORY: "true", CI_TEST_SELECTION: JSON.stringify( @@ -458,6 +581,8 @@ it("records failed omissions from full reports without treating partial or unhan { ...timing, reason: "interrupted" }, ]) expect(await run(report)).toMatchObject({ complete: false }); + for (const source of ["skipped", "cancelled", ""]) + expect(await run(timing, source)).toMatchObject({ complete: false }); await expect(run({ ...timing, files: "invalid" })).rejects.toMatchObject({ code: 1, }); diff --git a/tests/boundary/filesystem/ciWorkflowRun.test.ts b/tests/boundary/filesystem/ciWorkflowRun.test.ts new file mode 100644 index 000000000..cbebb5c4d --- /dev/null +++ b/tests/boundary/filesystem/ciWorkflowRun.test.ts @@ -0,0 +1,54 @@ +import { expect, it } from "vitest"; +import { normalizeWorkflowRun } from "../../../scripts/ci/workflow-run.mjs"; + +it.each([ + [".github/workflows/ci.yml", ".github/workflows/ci.yml", null], + [".github/workflows/ci.yml@main", ".github/workflows/ci.yml", "main"], + [ + ".github/workflows/ci.yml@refs/heads/release/v1", + ".github/workflows/ci.yml", + "refs/heads/release/v1", + ], + [ + ".github/workflows/ci.yml@feature@v1", + ".github/workflows/ci.yml", + "feature@v1", + ], + [ + ".github/workflows/release.yaml@main", + ".github/workflows/release.yaml", + "main", + ], +])( + "preserves API path %s while separating its file identity and ref", + (path, file, ref) => { + const raw = { id: 10, path, head_sha: "a".repeat(40) }; + const parsed = normalizeWorkflowRun(raw); + expect(parsed).toEqual({ ...raw, workflowPath: { file, ref } }); + expect(raw).not.toHaveProperty("workflowPath"); + }, +); + +it.each([ + undefined, + null, + 42, + "", + ".github/workflows/ci.yml@", + ".github/workflows/ci.yml@main\n", + ".github/workflows/ci.yml@main\0", + "../.github/workflows/ci.yml", + ".github/workflows/../ci.yml@main", + ".github/workflows/ci.yml-extra@main", + "https://example.test/ci.yml@main", +])( + "keeps malformed workflow path %s unknown instead of inventing a file identity", + (path) => { + const parsed = normalizeWorkflowRun({ + path, + workflowPath: { file: ".github/workflows/ci.yml", ref: "main" }, + }); + expect(parsed.path).toBe(path); + expect(parsed.workflowPath).toBeNull(); + }, +); diff --git a/turbo.json b/turbo.json index ffee2a070..35e7ece04 100644 --- a/turbo.json +++ b/turbo.json @@ -63,9 +63,7 @@ "!docs/.vitepress/dist/**", "!website/_site/**", "!experiments/analysis-efficiency/results/**", - "vitest.reverse-tools.config.ts", - "vitest.reverse-tools-semantics.config.ts", - "vitest.config.ts" + "*.{cjs,cts,js,jsx,mjs,mts,ts,tsx}" ] }, "lint:built": { @@ -86,9 +84,7 @@ "!docs/.vitepress/dist/**", "!website/_site/**", "!experiments/analysis-efficiency/results/**", - "vitest.reverse-tools.config.ts", - "vitest.reverse-tools-semantics.config.ts", - "vitest.config.ts" + "*.{cjs,cts,js,jsx,mjs,mts,ts,tsx}" ] }, "format:check": {}, @@ -104,8 +100,8 @@ "tests/**/*.ts", "tsconfig.json", "tsconfig.build.json", - "vitest.config.ts", - "docs/.vitepress/config.ts" + "docs/.vitepress/config.ts", + "*.{cjs,cts,js,jsx,mjs,mts,ts,tsx}" ] }, "jscpd": { diff --git a/vitest.config.ts b/vitest.config.ts index a8bf6b045..2c034beb2 100644 --- a/vitest.config.ts +++ b/vitest.config.ts @@ -3,6 +3,10 @@ import { availableParallelism, tmpdir } from "node:os"; import { join } from "node:path"; import { defineConfig } from "vitest/config"; +import { + BUILT_TEST_PROJECTS, + SOURCE_TEST_PROJECTS, +} from "./scripts/lib/test-projects.mjs"; import { CiTestSequencer } from "./tests/ciTestSequencer.js"; @@ -28,57 +32,32 @@ const COVERAGE_SHARD = process.argv.some((argument) => // Keep local and CI concurrency bounded by the runner rather than command locks. const MAX_TEST_WORKERS = Math.min(2, availableParallelism()); -const TEST_PROJECTS = [ - { - name: "domain", - include: ["src/{contracts,domain}/**/*.test.ts"], +const TEST_PROJECTS = [...SOURCE_TEST_PROJECTS, ...BUILT_TEST_PROJECTS]; +const PROJECT_POLICIES = { + domain: { pool: "threads" as const, maxWorkers: MAX_TEST_WORKERS, // Domain and contract modules are pure and own no process-global state, so // per-file module isolation adds startup cost without protecting a seam. isolate: false, }, - { - name: "services", - include: ["src/application/**/*.test.ts"], + services: { pool: "threads" as const, maxWorkers: MAX_TEST_WORKERS, // Service tests use recording ports and own no process-global state. Share // their module graph so file startup does not dominate the lane. isolate: false, }, - { - name: "adapters", - include: [ - "src/*.test.ts", - "src/{apktool,jeb,adb,artifacts,browser,dotnet,evm,firmware,flutter,pcap,ghidra,go,hopper,ida,inspector,javascript,native,process,reference,rust,server,wasm,windows}/**/*.test.ts", - ], - pool: "forks" as const, - maxWorkers: MAX_TEST_WORKERS, - }, - { - name: "composition", - include: ["tests/composition/**/*.test.ts"], + adapters: { pool: "forks" as const, maxWorkers: MAX_TEST_WORKERS }, + composition: { pool: "threads" as const, maxWorkers: MAX_TEST_WORKERS, // Composition tests use provider-neutral sessions and test-scoped cleanup; // sharing their immutable module graph avoids per-file startup work. isolate: false, }, - { - name: "boundary", - include: ["tests/boundary/**/*.test.ts"], - exclude: [ - "tests/boundary/cli/**/*.test.ts", - "tests/boundary/mcp/**/*.test.ts", - "tests/boundary/process/**/*.test.ts", - ], - pool: "forks" as const, - maxWorkers: MAX_TEST_WORKERS, - }, - { - name: "cli-boundary", - include: ["tests/boundary/cli/**/*.test.ts"], + boundary: { pool: "forks" as const, maxWorkers: MAX_TEST_WORKERS }, + "cli-boundary": { pool: "forks" as const, maxWorkers: MAX_TEST_WORKERS, // CLI tests create fresh command instances or subprocesses and own their @@ -88,54 +67,36 @@ const TEST_PROJECTS = [ // before their two heaps compete with the real CLI child processes. execArgv: ["--max-old-space-size=384"], }, - { - // Process-tree observations share host sampling resources; keep this small - // lane serial while the independent projects run together. - name: "process-boundary", - include: ["tests/boundary/process/**/*.test.ts"], + "process-boundary": { pool: "forks" as const, maxWorkers: MAX_TEST_WORKERS, + // Process-tree observations share host sampling resources; keep this small + // lane serial while the independent projects run together. fileParallelism: false, sequence: { groupOrder: 1 }, }, - { - name: "mcp-boundary", - include: ["tests/boundary/mcp/**/*.test.ts"], + "mcp-boundary": { pool: "threads" as const, maxWorkers: MAX_TEST_WORKERS, // Each file creates and closes an explicit in-memory MCP session. Reuse the // immutable server graph so module startup does not outweigh transport work. isolate: false, }, - { - name: "acceptance", - include: ["tests/acceptance/**/*.test.ts"], - pool: "forks" as const, - maxWorkers: MAX_TEST_WORKERS, - }, - { - name: "process-global", - include: ["tests/process-global/**/*.test.ts"], - pool: "forks" as const, - maxWorkers: MAX_TEST_WORKERS, - }, - { - name: "conformance", - include: ["tests/conformance/**/*.test.ts"], + acceptance: { pool: "forks" as const, maxWorkers: MAX_TEST_WORKERS }, + "process-global": { pool: "forks" as const, maxWorkers: MAX_TEST_WORKERS }, + conformance: { pool: "threads" as const, maxWorkers: MAX_TEST_WORKERS, // Contract cases use recording providers and own no process-global state. isolate: false, }, - { - name: "evaluation", - include: ["tests/evaluation/**/*.test.ts"], + evaluation: { pool: "threads" as const, maxWorkers: MAX_TEST_WORKERS, // Evaluator tests inspect deterministic inputs without mutating shared state. isolate: false, }, -]; +}; const ZERO_COVERAGE_THRESHOLDS = { statements: 0, @@ -156,10 +117,21 @@ const ZERO_COVERAGE_THRESHOLDS = { }, }; -const projects = TEST_PROJECTS.map((project) => ({ - extends: true as const, - test: project, -})); +const projects = TEST_PROJECTS.map((project) => { + const policy = + PROJECT_POLICIES[project.name as keyof typeof PROJECT_POLICIES]; + if (!policy) + throw new Error(`Missing test runner policy for ${project.name}`); + return { + extends: true as const, + test: { + name: project.name, + include: [...project.include], + ...(project.exclude ? { exclude: [...project.exclude] } : {}), + ...policy, + }, + }; +}); export default defineConfig({ test: {