From ab02f3c8d665ead35391c94db74cd0fb0cfed0ed Mon Sep 17 00:00:00 2001 From: morluto <76467478+morluto@users.noreply.github.com> Date: Tue, 13 Oct 2026 18:16:10 +0800 Subject: [PATCH] ci: reduce reporter fan-out and preserve shard history --- .github/workflows/ci-timings.yml | 21 +- .github/workflows/ci.yml | 24 +- docs/testing.md | 29 +- scripts/ci/job-timings-batch.mjs | 268 ++++++++++++++++ scripts/ci/job-timings.mjs | 5 +- scripts/ci/required.mjs | 1 + scripts/ci/test-history.mjs | 114 +++++++ .../filesystem/ciJobTimingsBatch.test.ts | 288 ++++++++++++++++++ .../filesystem/ciRequiredLanes.test.ts | 98 +++--- .../boundary/filesystem/ciTestHistory.test.ts | 188 ++++++++++++ .../boundary/filesystem/ciTestReports.test.ts | 14 +- 11 files changed, 968 insertions(+), 82 deletions(-) create mode 100644 scripts/ci/job-timings-batch.mjs create mode 100644 scripts/ci/test-history.mjs create mode 100644 tests/boundary/filesystem/ciJobTimingsBatch.test.ts create mode 100644 tests/boundary/filesystem/ciTestHistory.test.ts diff --git a/.github/workflows/ci-timings.yml b/.github/workflows/ci-timings.yml index 70ef1ebc7..30fa19464 100644 --- a/.github/workflows/ci-timings.yml +++ b/.github/workflows/ci-timings.yml @@ -1,9 +1,13 @@ name: CI timings on: - workflow_run: - workflows: [CI, Release] - types: [completed] + schedule: + - cron: "17 * * * *" + workflow_dispatch: + +concurrency: + group: ci-timings-${{ github.repository }} + cancel-in-progress: false permissions: contents: read @@ -11,12 +15,11 @@ permissions: jobs: report: - name: Report completed workflow attempt + name: Collect recent completed workflow attempts runs-on: ubuntu-latest timeout-minutes: 5 steps: - # Completed PR runs can contain untrusted changes. Execute only the - # reporter from the trusted default branch; consume jobs as API data. + # Consume only API data and execute batch code from the trusted default branch. - uses: actions/checkout@v4 with: ref: ${{ github.event.repository.default_branch }} @@ -29,15 +32,13 @@ jobs: env: GH_TOKEN: ${{ github.token }} CI_REPOSITORY: ${{ github.repository }} - CI_TIMING_RUN_ID: ${{ github.event.workflow_run.id }} - CI_TIMING_RUN_ATTEMPT: ${{ github.event.workflow_run.run_attempt }} - run: node scripts/ci/job-timings.mjs + run: node scripts/ci/job-timings-batch.mjs - name: Upload CI timing artifacts if: always() && hashFiles('.cache/ci-job-timings/*') != '' continue-on-error: true uses: actions/upload-artifact@v4 with: - name: ci-job-timings + name: ci-job-timings-batch path: .cache/ci-job-timings/ include-hidden-files: true if-no-files-found: error diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 39ebf2e86..7faf9e4ee 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -136,6 +136,9 @@ jobs: node-version-file: .nvmrc cache: npm - run: npm ci + - name: Validate workflow syntax and expressions + if: needs.changes.outputs.workflows == 'true' + run: go run github.com/rhysd/actionlint/cmd/actionlint@v1.7.7 -shellcheck= -pyflakes= - if: needs.changes.outputs.code == 'true' run: npm run check:ci - if: needs.changes.outputs.code == 'true' @@ -215,19 +218,15 @@ jobs: with: save: ${{ github.event_name != 'pull_request' && github.ref == format('refs/heads/{0}', github.event.repository.default_branch) }} - run: npm run test:prepare - - name: Find successful default-branch timing history + - name: Find available trusted full-suite timing history id: history continue-on-error: true env: GH_TOKEN: ${{ github.token }} CI_REPOSITORY: ${{ github.repository }} + CI_REPOSITORY_ID: ${{ github.repository_id }} CI_DEFAULT_BRANCH: ${{ github.event.repository.default_branch }} - run: | - run_id="$(gh api --method GET \ - "repos/${CI_REPOSITORY}/actions/workflows/ci.yml/runs" \ - -f branch="${CI_DEFAULT_BRANCH}" -f event=schedule -f status=success -f per_page=1 \ - --jq '.workflow_runs[0].id // empty')" - echo "run-id=${run_id}" >> "${GITHUB_OUTPUT}" + run: node scripts/ci/test-history.mjs - name: Download previous timing report if: steps.history.outputs.run-id != '' continue-on-error: true @@ -670,16 +669,6 @@ jobs: skills/reverse-engineer-anything if-no-files-found: error - workflows: - name: Validate workflows - needs: changes - if: needs.changes.outputs.workflows == 'true' - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - - name: Validate workflow syntax and expressions - run: go run github.com/rhysd/actionlint/cmd/actionlint@v1.7.7 -shellcheck= -pyflakes= - browser: needs: changes if: needs.changes.outputs.browser == 'true' @@ -749,7 +738,6 @@ jobs: reconstruction-readiness, windows-curated, docs, - workflows, browser, binary_layout, evm, diff --git a/docs/testing.md b/docs/testing.md index 0203abade..4e79c3fe7 100644 --- a/docs/testing.md +++ b/docs/testing.md @@ -274,18 +274,26 @@ checkout-relative paths; the build job downloads that artifact for the sequencer instead of the coverage/JUnit bundle. Missing history falls back to native sharding. Static checks, documentation, build, package verification, and real-system lanes remain separate jobs so one kind of evidence cannot stand in -for another. +for another. Workflow syntax and expression validation runs inside the required +static-check job when workflow changes select it, avoiding a separate runner. -The separate `CI timings` workflow runs after CI or Release completes and uploads -`ci-job-timings` with raw API pages and a normalized point-in-time report for that -workflow attempt. It runs the reporter from the trusted default branch with -read-only permissions, treating the triggering run as API data. Its summary separates queue -waiting (`created_at` to `started_at`) from completed execution and reports +The separate `CI timings` workflow batches recent completed CI and Release runs +hourly or on manual dispatch. It uploads `ci-job-timings-batch` with raw API pages and +a normalized point-in-time report for each collected workflow attempt, plus a +batch manifest describing examined coverage, failures, and limits. Overlapping +lookback windows may collect an attempt again; identify records by run ID and +attempt. Each batch examines at most 100 completed runs per workflow created in +the preceding 24 hours, with a four-minute API collection budget. It collects the +latest attempt of each selected run and does not claim an exhaustive archive. +It runs the reporter from the trusted default branch with read-only permissions, +treating workflow runs as API data. It does not allocate a new reporter job for +every completed, cancelled, or skipped workflow. Reports separate queue +waiting (`created_at` to `started_at`) from completed execution and report percentiles plus matched `npm ci` and artifact-step durations. Queue excludes dependency waiting before job creation and does not establish whether account concurrency or runner availability caused the wait. Queued jobs with placeholder start times, skipped jobs, and missing or invalid timestamps remain -unknown. The reporter targets the triggering run and attempt explicitly. +unknown. The reporter targets each selected run and attempt explicitly. Summed job elapsed time is parallel work, not workflow wall time, CPU time, memory usage, or billing. Step names identify samples, not complete transfer costs. Its queue, execution, and failures are independent of the required gate @@ -863,8 +871,11 @@ catalog, and managed-conformance metadata. Each shard still installs the locked dependencies and runs its assigned tests; full qualification enables coverage. It no longer regenerates the same metadata independently. -The build job also retrieves the merged timing report from the last successful -default-branch scheduled qualification and includes that same report in each shard's snapshot. +The build job also finds an available `test-timings` artifact from a successful +trusted default-branch scheduled or manual qualification and includes that same +report in each shard's snapshot. The bounded lookup verifies the artifact's +producer; newer schedules that reuse qualification without producing timings do +not hide the last full run's artifact. CI assigns discovered files by historical duration, with a deterministic fallback weight for new files, rather than dividing only by file count. Every discovered file still belongs to exactly one shard; worker limits and project diff --git a/scripts/ci/job-timings-batch.mjs b/scripts/ci/job-timings-batch.mjs new file mode 100644 index 000000000..842e884f0 --- /dev/null +++ b/scripts/ci/job-timings-batch.mjs @@ -0,0 +1,268 @@ +import { execFile } from "node:child_process"; +import { appendFile, mkdir, writeFile } from "node:fs/promises"; +import { join, resolve } from "node:path"; +import { promisify } from "node:util"; +import { fileURLToPath } from "node:url"; + +import { collectJobTimings } from "./job-timings.mjs"; + +const execFileAsync = promisify(execFile); +const OUTPUT_DIRECTORY = join(".cache", "ci-job-timings"); +const MAX_BUFFER = 16 * 1024 * 1024; +const API_TIMEOUT_MS = 30_000; +const WORKFLOWS = [ + { id: "ci.yml", name: "CI" }, + { id: "release.yml", name: "Release" }, +]; +const LOOKBACK_MS = 24 * 60 * 60 * 1_000; +const MAX_RUNS_PER_WORKFLOW = 100; +const TIME_BUDGET_MS = 240_000; + +/** Collect a bounded recent sample of completed CI and Release attempts. */ +export async function collectJobTimingsBatch({ + repository, + outputDirectory = OUTPUT_DIRECTORY, + observedAt = new Date().toISOString(), + env = process.env, + now = Date.now, + collect = collectJobTimings, +} = {}) { + const identity = validateRepository(repository); + const started = now(); + const cutoff = new Date(started - LOOKBACK_MS).toISOString(); + const manifest = { + schemaVersion: 1, + observedAt, + repository: identity, + selection: { + workflows: WORKFLOWS, + status: "completed", + createdAfter: cutoff, + lookbackHours: 24, + overlap: "Each hourly run re-reads the preceding 24 hours.", + pagination: + "Run listings use the first 100; jobs are paginated within the remaining time and 16 MiB response limits.", + maxRunsPerWorkflow: MAX_RUNS_PER_WORKFLOW, + timeBudgetMs: TIME_BUDGET_MS, + exhaustive: false, + }, + coverage: { + workflowsQueried: [], + runsListed: 0, + selectedRuns: 0, + collectedRuns: 0, + }, + truncation: [], + errors: [], + runs: [], + elapsedMs: 0, + }; + const budgetExpired = () => now() - started >= TIME_BUDGET_MS; + + const selected = []; + for (const workflow of WORKFLOWS) { + if (budgetExpired()) { + manifest.truncation.push("Time budget reached while listing workflows."); + break; + } + try { + const response = await apiJson( + `repos/${identity}/actions/workflows/${workflow.id}/runs?status=completed&created=%3E%3D${encodeURIComponent(cutoff)}&per_page=${String(MAX_RUNS_PER_WORKFLOW)}&page=1`, + env, + started, + now, + ); + if (!Array.isArray(response.workflow_runs)) + throw new Error("run listing must contain a workflow_runs array"); + manifest.coverage.workflowsQueried.push(workflow.name); + manifest.coverage.runsListed += response.workflow_runs.length; + if (response.workflow_runs.length >= MAX_RUNS_PER_WORKFLOW) + manifest.truncation.push( + `${workflow.name} run listing reached the 100-run first-page limit.`, + ); + for (const run of response.workflow_runs.slice( + 0, + MAX_RUNS_PER_WORKFLOW, + )) { + if ( + run.status !== "completed" || + !Number.isSafeInteger(run.id) || + !Number.isSafeInteger(run.run_attempt) + ) { + manifest.errors.push({ + scope: `workflow:${workflow.name}`, + runId: run.id ?? null, + message: + "Run record missing completed status, numeric id, or numeric latest attempt.", + }); + continue; + } + if (run.created_at && Date.parse(run.created_at) < Date.parse(cutoff)) + continue; + selected.push({ + id: run.id, + runAttempt: run.run_attempt, + workflow: workflow.name, + conclusion: run.conclusion ?? null, + createdAt: run.created_at ?? null, + }); + } + } catch (cause) { + manifest.errors.push({ + scope: `workflow:${workflow.name}`, + message: errorMessage(cause), + }); + } + } + + selected.sort( + (left, right) => + (Date.parse(right.createdAt ?? "") || 0) - + (Date.parse(left.createdAt ?? "") || 0), + ); + manifest.coverage.selectedRuns = selected.length; + for (const run of selected) { + if (budgetExpired()) { + manifest.truncation.push( + "Time budget reached before all selected runs were collected.", + ); + break; + } + const runRecord = { + ...run, + status: "pending", + rawPath: null, + reportPath: null, + error: null, + }; + manifest.runs.push(runRecord); + const runDirectory = join( + outputDirectory, + "runs", + `${String(run.id)}-${String(run.runAttempt)}`, + ); + try { + const result = await collect({ + repository: identity, + runId: String(run.id), + runAttempt: String(run.runAttempt), + outputDirectory: runDirectory, + env, + timeoutMs: Math.max( + 1, + Math.min(API_TIMEOUT_MS, TIME_BUDGET_MS - (now() - started)), + ), + }); + runRecord.status = "collected"; + runRecord.rawPath = result.rawPath; + runRecord.reportPath = result.reportPath; + manifest.coverage.collectedRuns += 1; + } catch (cause) { + runRecord.status = "error"; + runRecord.error = errorMessage(cause); + manifest.errors.push({ + scope: "run", + runId: run.id, + runAttempt: run.runAttempt, + message: runRecord.error, + }); + } + } + if (budgetExpired() && manifest.runs.length < manifest.coverage.selectedRuns) + manifest.truncation.push( + "Time budget reached; selected runs remain unattempted.", + ); + return await writeManifest(outputDirectory, manifest, started, now, env); +} + +function validateRepository(repository) { + if ( + typeof repository !== "string" || + !/^[A-Za-z0-9_.-]+\/[A-Za-z0-9_.-]+$/u.test(repository) + ) + throw new Error( + "CI_REPOSITORY must be an owner/name repository identifier", + ); + return repository; +} + +async function apiJson(endpoint, env, started, now) { + const timeout = Math.max( + 1, + Math.min(API_TIMEOUT_MS, TIME_BUDGET_MS - (now() - started)), + ); + const { stdout } = await execFileAsync("gh", ["api", endpoint], { + env, + encoding: "utf8", + maxBuffer: MAX_BUFFER, + timeout, + }); + try { + return JSON.parse(stdout); + } catch (cause) { + throw new Error("GitHub Actions API returned malformed JSON", { cause }); + } +} + +async function writeManifest(outputDirectory, manifest, started, now, env) { + manifest.elapsedMs = Math.max(0, now() - started); + if ( + manifest.elapsedMs >= TIME_BUDGET_MS && + !manifest.truncation.some((item) => item.includes("Time budget")) + ) + manifest.truncation.push("Time budget reached."); + await mkdir(outputDirectory, { recursive: true }); + const manifestPath = join(outputDirectory, "batch-manifest.json"); + await writeFile( + manifestPath, + `${JSON.stringify(manifest, null, 2)}\n`, + "utf8", + ); + if (env.GITHUB_STEP_SUMMARY) + await appendBatchSummary(env.GITHUB_STEP_SUMMARY, manifest, manifestPath); + return { manifestPath, manifest }; +} + +async function appendBatchSummary(summaryPath, manifest, manifestPath) { + const collected = manifest.runs.filter( + ({ status }) => status === "collected", + ).length; + const failed = manifest.runs.filter( + ({ status }) => status === "error", + ).length; + const lines = [ + "### Batched CI timing snapshot", + "", + `Observed ${manifest.observedAt}; collected ${String(collected)} of ${String(manifest.coverage.selectedRuns)} selected recent completed runs (${String(failed)} run errors).`, + `Coverage uses the first ${String(MAX_RUNS_PER_WORKFLOW)} runs per workflow created in the preceding 24 hours and a ${String(TIME_BUDGET_MS / 1_000)} second total budget. This sample is bounded and is not exhaustive.`, + `Manifest: \`${manifestPath}\`.`, + ...manifest.truncation.map((message) => `- Truncated: ${message}`), + ...manifest.errors.map( + ({ scope, message }) => `- Error (${scope}): ${message}`, + ), + "", + ]; + await appendFile(summaryPath, lines.join("\n"), "utf8"); +} + +function errorMessage(cause) { + return cause instanceof Error ? cause.message : String(cause); +} + +if ( + process.argv[1] !== undefined && + resolve(process.argv[1]) === fileURLToPath(import.meta.url) +) { + try { + const result = await collectJobTimingsBatch({ + repository: process.env.CI_REPOSITORY, + outputDirectory: process.env.CI_JOB_TIMINGS_DIRECTORY ?? OUTPUT_DIRECTORY, + }); + process.stdout.write( + `Wrote batch manifest for ${String(result.manifest.coverage.collectedRuns)} collected runs to ${result.manifestPath}\n`, + ); + } catch (cause) { + process.stderr.write(`CI timing batch failed: ${errorMessage(cause)}\n`); + process.exitCode = 1; + } +} diff --git a/scripts/ci/job-timings.mjs b/scripts/ci/job-timings.mjs index 9d22f0d41..4d96f91b0 100644 --- a/scripts/ci/job-timings.mjs +++ b/scripts/ci/job-timings.mjs @@ -15,7 +15,7 @@ const STEP_COSTS = [ ]; /** Fetch and summarize one workflow attempt without turning telemetry into a gate. */ -async function collectJobTimings({ +export async function collectJobTimings({ repository, runId, runAttempt, @@ -23,13 +23,14 @@ async function collectJobTimings({ summaryPath, observedAt = new Date().toISOString(), env = process.env, + timeoutMs = TIMEOUT_MS, }) { const identity = validateIdentity({ repository, runId, runAttempt }); const endpoint = `repos/${identity.repository}/actions/runs/${identity.runId}/attempts/${identity.runAttempt}/jobs?per_page=100`; const { stdout } = await execFileAsync( "gh", ["api", "--paginate", "--slurp", endpoint], - { env, encoding: "utf8", maxBuffer: MAX_BUFFER, timeout: TIMEOUT_MS }, + { env, encoding: "utf8", maxBuffer: MAX_BUFFER, timeout: timeoutMs }, ); const rawPath = join(outputDirectory, "jobs.json"); const reportPath = join(outputDirectory, "report.json"); diff --git a/scripts/ci/required.mjs b/scripts/ci/required.mjs index d48a5ec2c..afe52c2b8 100644 --- a/scripts/ci/required.mjs +++ b/scripts/ci/required.mjs @@ -29,6 +29,7 @@ const lanes = { fixtures: ["conformance-fixtures"], readiness: ["reconstruction-readiness"], docs: ["docs"], + workflows: ["static"], }; for (const [scope, value] of Object.entries(outputs)) { if (value === "true") required.push(...(lanes[scope] ?? [scope])); diff --git a/scripts/ci/test-history.mjs b/scripts/ci/test-history.mjs new file mode 100644 index 000000000..4386f48af --- /dev/null +++ b/scripts/ci/test-history.mjs @@ -0,0 +1,114 @@ +import { execFileSync } from "node:child_process"; +import { appendFileSync } from "node:fs"; +import { resolve } from "node:path"; +import { fileURLToPath } from "node:url"; + +const ARTIFACT_NAME = "test-timings"; +const MAX_AGE_MS = 30 * 24 * 60 * 60 * 1000; +const MAX_CANDIDATES = 5; +const positiveId = (value) => + (typeof value === "string" || typeof value === "number") && + /^[1-9]\d*$/.test(String(value)) && + Number.isSafeInteger(Number(value)); + +function findTimingRun({ repository, branch, repositoryId, now = Date.now() }) { + if ( + !/^[A-Za-z0-9_.-]+\/[A-Za-z0-9_.-]+$/.test(repository ?? "") || + !branch || + !positiveId(repositoryId) + ) + return ""; + const deadline = Date.now() + 20_000; + const api = (...arguments_) => { + const remaining = deadline - Date.now(); + if (remaining <= 0) + throw new Error("Timing history lookup budget exhausted"); + return JSON.parse( + execFileSync("gh", ["api", "--method", "GET", ...arguments_], { + encoding: "utf8", + timeout: Math.min(10_000, remaining), + maxBuffer: 4 * 1024 * 1024, + stdio: ["ignore", "pipe", "pipe"], + }), + ); + }; + + const { artifacts } = api( + `repos/${repository}/actions/artifacts`, + "-f", + `name=${ARTIFACT_NAME}`, + "-F", + "per_page=100", + ); + if (!Array.isArray(artifacts)) + throw new Error("Missing timing artifact inventory"); + const candidates = artifacts + .filter((artifact) => { + const lineage = artifact?.workflow_run; + const age = now - Date.parse(artifact?.created_at); + return ( + artifact?.name === ARTIFACT_NAME && + artifact?.expired === false && + typeof artifact?.created_at === "string" && + Number.isFinite(age) && + age >= 0 && + age < MAX_AGE_MS && + positiveId(lineage?.id) && + positiveId(lineage?.repository_id) && + positiveId(lineage?.head_repository_id) && + lineage.repository_id === lineage.head_repository_id && + lineage.repository_id === Number(repositoryId) && + lineage.head_branch === branch + ); + }) + .sort( + (left, right) => + Date.parse(right.created_at) - Date.parse(left.created_at), + ) + .slice(0, MAX_CANDIDATES); + + for (const artifact of candidates) { + const run = api( + `repos/${repository}/actions/runs/${artifact.workflow_run.id}`, + ); + const runAge = now - Date.parse(run?.created_at); + if ( + typeof run?.created_at === "string" && + Number.isFinite(runAge) && + runAge >= 0 && + runAge < MAX_AGE_MS && + run.id === artifact.workflow_run.id && + run.status === "completed" && + run.conclusion === "success" && + run.path === ".github/workflows/ci.yml" && + ["schedule", "workflow_dispatch"].includes(run.event) && + run.head_branch === branch && + run.head_repository?.full_name === repository + ) + return String(run.id); + } + return ""; +} + +function write(runId) { + if (process.env.GITHUB_OUTPUT) + appendFileSync(process.env.GITHUB_OUTPUT, `run-id=${runId}\n`); + console.log(JSON.stringify({ run_id: runId })); +} + +if ( + process.argv[1] && + fileURLToPath(import.meta.url) === resolve(process.argv[1]) +) { + let runId = ""; + try { + runId = findTimingRun({ + repository: process.env.CI_REPOSITORY, + branch: process.env.CI_DEFAULT_BRANCH, + repositoryId: process.env.CI_REPOSITORY_ID, + }); + } catch { + // History improves shard ordering but must never block CI. + } + write(runId); +} diff --git a/tests/boundary/filesystem/ciJobTimingsBatch.test.ts b/tests/boundary/filesystem/ciJobTimingsBatch.test.ts new file mode 100644 index 000000000..6c2658c00 --- /dev/null +++ b/tests/boundary/filesystem/ciJobTimingsBatch.test.ts @@ -0,0 +1,288 @@ +import { chmod, mkdir, readFile, writeFile } from "node:fs/promises"; +import { delimiter, join } from "node:path"; +import { fileURLToPath, pathToFileURL } from "node:url"; + +import { expect, it } from "vitest"; + +import { createTestTempDirectory } from "../../fixtures/temporaryDirectory.js"; +import { executeWorkflowFixture } from "../../support/workflowGit.js"; + +const batchScript = fileURLToPath( + new URL("../../../scripts/ci/job-timings-batch.mjs", import.meta.url), +); + +it("collects the latest attempt for recent completed CI and Release runs into a bounded manifest", async () => { + const recent = new Date().toISOString(); + const fixture = await createFixture([ + { + workflow_runs: [ + { + id: 42, + run_attempt: 3, + status: "completed", + conclusion: "failure", + created_at: new Date(Date.parse(recent) + 1_000).toISOString(), + }, + { + id: 41, + run_attempt: 1, + status: "completed", + conclusion: "cancelled", + created_at: recent, + }, + ], + }, + { + workflow_runs: [ + { + id: 43, + run_attempt: 2, + status: "completed", + conclusion: "success", + created_at: new Date(Date.parse(recent) + 2_000).toISOString(), + }, + ], + }, + [ + { + jobs: [ + { + name: "Release lane", + status: "completed", + conclusion: "success", + created_at: recent, + started_at: recent, + completed_at: new Date(Date.parse(recent) + 50_000).toISOString(), + steps: [], + }, + ], + }, + ], + [ + { + jobs: [ + { + name: "Failed lane", + status: "completed", + conclusion: "failure", + created_at: recent, + started_at: recent, + completed_at: new Date(Date.parse(recent) + 55_000).toISOString(), + steps: [], + }, + ], + }, + ], + [ + { + jobs: [ + { + name: "Cancelled lane", + status: "completed", + conclusion: "cancelled", + created_at: recent, + started_at: recent, + completed_at: new Date(Date.parse(recent) + 30_000).toISOString(), + steps: [], + }, + ], + }, + ], + ]); + const result = await runBatch(fixture); + const manifest = JSON.parse( + await readFile( + join(fixture.outputDirectory, "batch-manifest.json"), + "utf8", + ), + ); + const calls = JSON.parse(await readFile(fixture.calls, "utf8")); + + expect(result.stdout).toContain("3 collected runs"); + expect( + await readFile(join(fixture.directory, "summary.md"), "utf8"), + ).toContain("collected 3 of 3"); + expect(manifest.selection).toMatchObject({ + lookbackHours: 24, + maxRunsPerWorkflow: 100, + exhaustive: false, + }); + expect(manifest.coverage).toMatchObject({ + workflowsQueried: ["CI", "Release"], + selectedRuns: 3, + collectedRuns: 3, + }); + expect(manifest.truncation).toEqual([]); + expect(manifest.errors).toEqual([]); + expect(manifest.runs).toMatchObject([ + { id: 43, runAttempt: 2, conclusion: "success" }, + { id: 42, runAttempt: 3, conclusion: "failure" }, + { id: 41, runAttempt: 1, conclusion: "cancelled" }, + ]); + expect(manifest.runs).toEqual( + expect.arrayContaining([ + expect.objectContaining({ + status: "collected", + rawPath: expect.any(String), + reportPath: expect.any(String), + }), + ]), + ); + expect(calls).toContainEqual([ + "api", + "--paginate", + "--slurp", + "repos/example/rea/actions/runs/42/attempts/3/jobs?per_page=100", + ]); + expect(calls).toContainEqual([ + "api", + "--paginate", + "--slurp", + "repos/example/rea/actions/runs/41/attempts/1/jobs?per_page=100", + ]); + const failedReport = JSON.parse( + await readFile( + join(fixture.outputDirectory, "runs/42-3/report.json"), + "utf8", + ), + ); + expect(failedReport.jobs[0].conclusion).toBe("failure"); + expect(failedReport.jobs[0].execution.seconds).toBe(55); +}); + +it("records listing and per-run errors without claiming complete coverage", async () => { + const recent = new Date().toISOString(); + const fixture = await createFixture([ + { + workflow_runs: [ + { + id: 7, + run_attempt: 2, + status: "completed", + conclusion: "success", + created_at: recent, + }, + ], + }, + { error: true }, + { jobs: "malformed" }, + ]); + await runBatch(fixture); + const manifest = JSON.parse( + await readFile( + join(fixture.outputDirectory, "batch-manifest.json"), + "utf8", + ), + ); + expect(manifest.selection.exhaustive).toBe(false); + expect(manifest.errors).toMatchObject([ + { scope: "workflow:Release" }, + { scope: "run" }, + ]); + expect(manifest.runs[0]).toMatchObject({ + id: 7, + runAttempt: 2, + status: "error", + }); +}); + +it("retains a partial manifest when its collection budget is exhausted", async () => { + const fixture = await createFixture([ + { + workflow_runs: [8, 9].map((id) => ({ + id, + run_attempt: 1, + status: "completed", + created_at: new Date().toISOString(), + })), + }, + { workflow_runs: [] }, + ]); + await executeWorkflowFixture( + process.execPath, + [ + "--input-type=module", + "-e", + ` + const { collectJobTimingsBatch } = await import(${JSON.stringify(pathToFileURL(batchScript).href)}); + let clock = Date.now(); + await collectJobTimingsBatch({ + repository: "example/rea", + outputDirectory: process.env.CI_JOB_TIMINGS_DIRECTORY, + now: () => clock, + collect: async () => { + clock += 240001; + throw new Error("fixture request exhausted the remaining budget"); + }, + }); + `, + ], + { cwd: fixture.directory, env: fixtureEnvironment(fixture) }, + ); + const manifest = JSON.parse( + await readFile( + join(fixture.outputDirectory, "batch-manifest.json"), + "utf8", + ), + ); + expect(manifest.coverage).toMatchObject({ + selectedRuns: 2, + collectedRuns: 0, + }); + expect(manifest.runs).toHaveLength(1); + expect(manifest.runs[0]).toMatchObject({ status: "error" }); + expect(manifest.truncation.join(" ")).toContain("Time budget"); +}); + +async function createFixture(responses: unknown[]) { + const directory = await createTestTempDirectory("rea-ci-job-timings-batch-"); + const fakeBin = join(directory, "bin"); + await mkdir(fakeBin); + const responsePath = join(directory, "responses.json"); + const calls = join(directory, "calls.json"); + await writeFile(responsePath, JSON.stringify(responses)); + await writeFile(calls, "[]"); + const fakeGh = join(fakeBin, "gh"); + await writeFile( + fakeGh, + `#!/usr/bin/env node +const fs = require("node:fs"); +const calls = JSON.parse(fs.readFileSync(process.env.CI_GH_CALLS, "utf8")); +calls.push(process.argv.slice(2)); +fs.writeFileSync(process.env.CI_GH_CALLS, JSON.stringify(calls)); +const responses = JSON.parse(fs.readFileSync(process.env.CI_GH_RESPONSES, "utf8")); +const response = responses[calls.length - 1]; +if (response && response.error) { process.stderr.write("fixture API error"); process.exitCode = 1; } +else process.stdout.write(JSON.stringify(response)); +`, + ); + await chmod(fakeGh, 0o755); + return { + directory, + fakeBin, + responses: responsePath, + calls, + outputDirectory: join(directory, ".cache", "ci-job-timings"), + }; +} + +function runBatch(fixture: Awaited>) { + return executeWorkflowFixture(process.execPath, [batchScript], { + cwd: fixture.directory, + env: fixtureEnvironment(fixture), + }); +} + +function fixtureEnvironment( + fixture: Awaited>, +) { + return { + ...process.env, + PATH: `${fixture.fakeBin}${delimiter}${process.env.PATH ?? ""}`, + CI_REPOSITORY: "example/rea", + CI_JOB_TIMINGS_DIRECTORY: fixture.outputDirectory, + CI_GH_CALLS: fixture.calls, + CI_GH_RESPONSES: fixture.responses, + GITHUB_STEP_SUMMARY: join(fixture.directory, "summary.md"), + }; +} diff --git a/tests/boundary/filesystem/ciRequiredLanes.test.ts b/tests/boundary/filesystem/ciRequiredLanes.test.ts index a86392e89..bc2e5eb06 100644 --- a/tests/boundary/filesystem/ciRequiredLanes.test.ts +++ b/tests/boundary/filesystem/ciRequiredLanes.test.ts @@ -90,10 +90,9 @@ it("exports every canonical scope with read authority for admission and keeps te const timings = z .object({ on: z.object({ - workflow_run: z.object({ - workflows: z.array(z.string()), - types: z.array(z.string()), - }), + schedule: z.array(z.object({ cron: z.string() })), + workflow_dispatch: z.unknown(), + workflow_run: z.unknown().optional(), }), permissions: z.record(z.string(), z.string()), jobs: z.object({ @@ -117,10 +116,8 @@ it("exports every canonical scope with read authority for admission and keeps te ), ), ); - expect(timings.on.workflow_run).toEqual({ - workflows: ["CI", "Release"], - types: ["completed"], - }); + expect(timings.on.workflow_run).toBeUndefined(); + expect(timings.on.schedule).toHaveLength(1); expect(timings.permissions).toEqual({ contents: "read", actions: "read" }); const checkout = timings.jobs.report.steps.find((step) => step.uses?.startsWith("actions/checkout@"), @@ -130,14 +127,9 @@ it("exports every canonical scope with read authority for admission and keeps te "persist-credentials": false, }); const report = timings.jobs.report.steps.find((step) => - step.run?.includes("scripts/ci/job-timings.mjs"), - ); - expect(report?.env?.CI_TIMING_RUN_ID).toBe( - "${{ github.event.workflow_run.id }}", - ); - expect(report?.env?.CI_TIMING_RUN_ATTEMPT).toBe( - "${{ github.event.workflow_run.run_attempt }}", + step.run?.includes("scripts/ci/job-timings-batch.mjs"), ); + expect(report?.env?.CI_REPOSITORY).toBe("${{ github.repository }}"); }); const requiredNeeds = async () => { @@ -152,6 +144,17 @@ it("makes the required gate wait for every CI lane, including future additions", ); }); +it("runs workflow validation inside required static checks when selected", async () => { + const { jobs } = await workflow(); + const steps = z + .array(z.object({ run: z.string().optional(), if: z.string().optional() })) + .parse(jobs.static?.steps); + const validation = steps.filter((step) => step.run?.includes("actionlint")); + expect(validation).toHaveLength(1); + expect(validation[0]?.if).toBe("needs.changes.outputs.workflows == 'true'"); + expect(jobs.workflows).toBeUndefined(); +}); + it("leaves cancellation ownership with CI so child lanes cannot cancel another tier", async () => { const { jobs } = await workflow(); const children = Object.values(jobs).flatMap((job) => @@ -257,33 +260,44 @@ it.each([ ["false", "skipped", true], ["false", "failure", false], ] as const)( - "enforces the SQLite required gate for sqlite=%s and result=%s", - async (sqlite, result, passes) => { + "enforces mapped required scopes for selected=%s and result=%s", + async (selected, result, passes) => { const workflow = await requiredNeeds(); - // GitHub exposes only direct dependencies in the required job's needs context. - const needs: Record< - string, - { result: string; outputs?: Record } - > = Object.fromEntries( - workflow.needs.map((name) => [name, { result: "success" }]), - ); - needs.changes = { result: "success", outputs: { code: "true", sqlite } }; - if (result === "missing") delete needs["sqlite-runtime"]; - else if ("sqlite-runtime" in needs) needs["sqlite-runtime"] = { result }; - const execution = executeWorkflowFixture( - process.execPath, - [ - fileURLToPath( - new URL("../../../scripts/ci/required.mjs", import.meta.url), - ), - ], - { env: { ...process.env, CI_JOBS: JSON.stringify(needs) } }, - ); - if (passes) await expect(execution).resolves.toMatchObject({ stderr: "" }); - else - await expect(execution).rejects.toMatchObject({ - code: 1, - stderr: expect.stringContaining("sqlite-runtime"), - }); + for (const [scope, lane] of [ + ["sqlite", "sqlite-runtime"], + ["workflows", "static"], + ] as const) { + // GitHub exposes only direct dependencies in the required job's needs context. + const needs: Record< + string, + { result: string; outputs?: Record } + > = Object.fromEntries( + workflow.needs.map((name) => [name, { result: "success" }]), + ); + needs.changes = { + result: "success", + outputs: { code: "true", [scope]: selected }, + }; + if (result === "missing") delete needs[lane]; + else needs[lane] = { result }; + const execution = executeWorkflowFixture( + process.execPath, + [ + fileURLToPath( + new URL("../../../scripts/ci/required.mjs", import.meta.url), + ), + ], + { env: { ...process.env, CI_JOBS: JSON.stringify(needs) } }, + ); + // Static checks remain required even without workflow edits. + const expected = passes && !(lane === "static" && result !== "success"); + if (expected) + await expect(execution).resolves.toMatchObject({ stderr: "" }); + else + await expect(execution).rejects.toMatchObject({ + code: 1, + stderr: expect.stringContaining(lane), + }); + } }, ); diff --git a/tests/boundary/filesystem/ciTestHistory.test.ts b/tests/boundary/filesystem/ciTestHistory.test.ts new file mode 100644 index 000000000..411e9ece9 --- /dev/null +++ b/tests/boundary/filesystem/ciTestHistory.test.ts @@ -0,0 +1,188 @@ +import { chmod, mkdir, readFile, writeFile } from "node:fs/promises"; +import { delimiter, dirname, join } from "node:path"; +import { fileURLToPath } from "node:url"; + +import { expect, it } from "vitest"; + +import { createTestTempDirectory } from "../../fixtures/temporaryDirectory.js"; +import { executeWorkflowFixture } from "../../support/workflowGit.js"; + +const script = fileURLToPath( + new URL("../../../scripts/ci/test-history.mjs", import.meta.url), +); +const repository = "example/rea"; +const repositoryId = 1; +const now = Date.now(); + +function artifact(id: number, overrides: Record = {}) { + return { + id: id + 100, + name: "test-timings", + expired: false, + created_at: new Date(now - 60_000).toISOString(), + workflow_run: { + id, + repository_id: repositoryId, + head_repository_id: repositoryId, + head_branch: "main", + }, + ...overrides, + }; +} + +function run(id: number, overrides: Record = {}) { + return { + id, + status: "completed", + conclusion: "success", + path: ".github/workflows/ci.yml", + event: "schedule", + created_at: new Date(now - 120_000).toISOString(), + head_branch: "main", + head_repository: { full_name: repository }, + ...overrides, + }; +} + +async function invoke( + artifacts: unknown[], + runs: Record, + options: { + apiError?: boolean; + repository?: string; + repositoryId?: string; + } = {}, +) { + const directory = await createTestTempDirectory("rea-ci-test-history-"); + const fakeBin = join(directory, "bin"); + await mkdir(fakeBin); + const fixture = join(directory, "api.json"); + const calls = join(directory, "api-calls"); + const output = join(directory, "github-output"); + await writeFile(fixture, JSON.stringify({ artifacts, runs })); + await writeFile(output, ""); + await writeFile(calls, ""); + const fakeGh = join(fakeBin, "gh"); + await writeFile( + fakeGh, + [ + "#!/bin/sh", + 'printf \'%s\\n\' "$*" >> "$GH_API_CALLS"', + 'if [ "$GH_API_ERROR" = "true" ]; then exit 1; fi', + 'if [ "$4" = "repos/example/rea/actions/artifacts" ]; then', + ' node -e \'const fs=require("node:fs"); const x=JSON.parse(fs.readFileSync(process.env.GH_FIXTURE_RESPONSE,"utf8")); process.stdout.write(JSON.stringify({artifacts:x.artifacts}));\'', + "else", + ' run_id="${4##*/}"', + ' node -e \'const fs=require("node:fs"); const x=JSON.parse(fs.readFileSync(process.env.GH_FIXTURE_RESPONSE,"utf8")); process.stdout.write(JSON.stringify(x.runs[Number(process.argv[1])] ?? {}));\' "$run_id"', + "fi", + "", + ].join("\n"), + ); + await chmod(fakeGh, 0o755); + const node24 = dirname(process.execPath); + const result = await executeWorkflowFixture(process.execPath, [script], { + cwd: directory, + env: { + ...process.env, + PATH: [fakeBin, node24, process.env.PATH ?? ""].join(delimiter), + GH_FIXTURE_RESPONSE: fixture, + GH_API_CALLS: calls, + GH_API_ERROR: String(options.apiError ?? false), + GH_TOKEN: "fixture-token", + CI_REPOSITORY: options.repository ?? repository, + CI_REPOSITORY_ID: options.repositoryId ?? String(repositoryId), + CI_DEFAULT_BRANCH: "main", + GITHUB_OUTPUT: output, + }, + }); + return { + result, + output: await readFile(output, "utf8"), + calls: await readFile(calls, "utf8"), + }; +} + +it("selects the latest trusted full run that actually published timing history", async () => { + const { result, output, calls } = await invoke( + [ + artifact(15), + artifact(14), + artifact(13, { + workflow_run: { ...artifact(13).workflow_run, head_repository_id: 2 }, + }), + artifact(12, { + workflow_run: { ...artifact(12).workflow_run, head_branch: "feature" }, + }), + artifact(11), + ], + { + 15: run(15, { conclusion: "failure" }), + 14: run(14, { path: ".github/workflows/release.yml" }), + 11: run(11, { event: "workflow_dispatch" }), + }, + ); + + expect(JSON.parse(result.stdout)).toEqual({ run_id: "11" }); + expect(output).toBe("run-id=11\n"); + expect(calls).toContain("name=test-timings"); + expect(calls).toContain("per_page=100"); + expect(calls.match(/actions\/runs\//gu)).toHaveLength(3); + expect(calls).not.toContain("actions/workflows"); +}); + +it("rejects expired, foreign, old, and non-full run candidates", async () => { + const { result, output } = await invoke( + [ + artifact(15, { expired: true }), + artifact(14, { + workflow_run: { ...artifact(14).workflow_run, head_repository_id: 2 }, + }), + artifact(13, { + created_at: new Date(now - 31 * 24 * 60 * 60 * 1000).toISOString(), + }), + artifact(12), + ], + { + 12: run(12, { event: "pull_request" }), + }, + ); + + expect(JSON.parse(result.stdout)).toEqual({ run_id: "" }); + expect(output).toBe("run-id=\n"); +}); + +it("writes an empty run id when GitHub history is unavailable", async () => { + const { result, output } = await invoke([], {}, { apiError: true }); + expect(JSON.parse(result.stdout)).toEqual({ run_id: "" }); + expect(output).toBe("run-id=\n"); +}); + +it("bounds run lookups to five artifacts even when inventory has more", async () => { + const ids = [25, 24, 23, 22, 21, 20, 19]; + const { result, calls } = await invoke( + ids.map((id) => artifact(id)), + Object.fromEntries( + ids.map((id, index) => [ + id, + run(id, index < 5 ? { conclusion: "failure" } : {}), + ]), + ), + ); + + expect(JSON.parse(result.stdout)).toEqual({ run_id: "" }); + expect(calls.match(/actions\/runs\//gu)).toHaveLength(5); +}); + +it("returns empty history without querying for an invalid repository identity", async () => { + const { result, output, calls } = await invoke( + [], + {}, + { + repository: "owner/repo?per_page=100", + repositoryId: "not-an-id", + }, + ); + expect(JSON.parse(result.stdout)).toEqual({ run_id: "" }); + expect(output).toBe("run-id=\n"); + expect(calls).toBe(""); +}); diff --git a/tests/boundary/filesystem/ciTestReports.test.ts b/tests/boundary/filesystem/ciTestReports.test.ts index 3ab042718..2c44878de 100644 --- a/tests/boundary/filesystem/ciTestReports.test.ts +++ b/tests/boundary/filesystem/ciTestReports.test.ts @@ -80,10 +80,22 @@ it("keeps diagnosis available after merge failure and publishes only complete su expect(history?.if).toContain("needs.test-shard.result == 'success'"); expect(history?.if).toContain("needs.changes.outputs.tier == 'full'"); expect(history?.with?.path).toBe(".cache/ci-test-history/test-results.json"); - const download = (await workflow()).build!.steps.find( + const buildSteps = (await workflow()).build!.steps; + const lookup = buildSteps.find((step) => + step.run?.includes("test-history.mjs"), + ); + expect(lookup?.env).toMatchObject({ + CI_REPOSITORY: "${{ github.repository }}", + CI_REPOSITORY_ID: "${{ github.repository_id }}", + CI_DEFAULT_BRANCH: "${{ github.event.repository.default_branch }}", + }); + const download = buildSteps.find( (step) => step.name === "Download previous timing report", ); expect(download?.with?.name).toBe(history?.with?.name); + expect(download?.with?.["run-id"]).toBe( + "${{ steps.history.outputs.run-id }}", + ); }); it("rejects successful shard reports with missing or duplicate portable files", async () => {