import { chmod, mkdir, readFile, writeFile } from "node:fs/promises"; import { delimiter, join } from "node:path"; import { fileURLToPath } from "node:url"; import { expect, it } from "vitest"; import { createTestTempDirectory } from "../../fixtures/temporaryDirectory.js"; import { classifyPrCi, collectPrCiHealth, matchRunToPullRequest, writePrCiHealthReport, } from "../../../scripts/ci/pr-ci-health.mjs"; import { createGhApiRequest } from "../../../scripts/ci/github-api.mjs"; import type { PrCiPullRequest, PrCiWorkflowRun, } from "../../../scripts/ci/pr-ci-health.mjs"; import { executeWorkflowFixture } from "../../support/workflowGit.js"; const reporter = fileURLToPath( new URL("../../../scripts/ci/pr-ci-health.mjs", import.meta.url), ); const observedAt = Date.parse("2026-10-14T00:00:00Z"); const headSha = "a".repeat(40); const mergeSha = "b".repeat(40); it("classifies workflow states without confusing absent, pending, and completed CI", () => { const pull = validPull(); const cases = [ [ "startup_failure", run({ status: "completed", conclusion: "startup_failure" }), ], [ "queued_pending_executing", run({ status: "in_progress", conclusion: null }), ], ["approval_required", run({ status: "waiting", conclusion: null })], ["completed_failure", run({ status: "completed", conclusion: "failure" })], [ "completed_canceled", run({ status: "completed", conclusion: "cancelled" }), ], ["completed_success", run({ status: "completed", conclusion: "success" })], ["completed_other", run({ status: "completed", conclusion: "neutral" })], [ "completed_other", run({ status: "completed", conclusion: "constructor" }), ], ] as const; for (const [category, evidence] of cases) { for (const runsComplete of [true, false]) { const result = classifyPrCi({ pull, runs: [ { run: { ...evidence, run_attempt: 1 }, basis: "event_pr_number_and_head_sha", }, ], runsComplete, pullRequestInventoryComplete: true, ambiguousRuns: [], }); expect(result).toMatchObject({ category: runsComplete ? category : "unknown", ...(runsComplete ? {} : { reason: "workflow_run_inventory_incomplete" }), runs: [ { id: 77, attempt: 1, status: evidence.status, conclusion: evidence.conclusion, }, ], }); } } for (const runsComplete of [true, false]) { expect( classifyPrCi({ pull: { ...pull, mergeable: false }, runs: [], runsComplete, pullRequestInventoryComplete: true, ambiguousRuns: [], }).category, ).toBe("merge_conflict"); } }); it("requires complete inventories, known mergeability, and immutable head identity to report missing", () => { const pull = validPull({ mergeable: true }); const classify = (overrides = {}) => classifyPrCi({ pull, runs: [], runsComplete: true, pullRequestInventoryComplete: true, ambiguousRuns: [], ...overrides, }); expect(classify().category).toBe("missing"); expect(classify({ runsComplete: false })).toMatchObject({ category: "deferred", reason: "lookup_inventory_incomplete", }); expect(classify({ pullRequestInventoryComplete: false }).category).toBe( "deferred", ); expect( classify({ ambiguousRuns: [{ run: run(), basis: "ambiguous_head_identity" }], }).category, ).toBe("unknown"); expect(classify({ pull: { ...pull, mergeable: undefined } }).category).toBe( "unknown", ); expect(classify({ pull: { ...pull, identityValid: false } }).category).toBe( "unknown", ); }); it("accepts event PR/head evidence for merge-SHA runs and refuses ambiguous branch identities", () => { const pull = validPull(); const associatedMergeRun = run({ head_sha: mergeSha, pull_requests: [ { number: pull.number!, head: { sha: headSha, repo: { id: pull.head.repositoryId } }, }, ], }); expect( matchRunToPullRequest({ run: associatedMergeRun, pull, identityCounts: new Map([["73\u0000feature", 1]]), }), ).toEqual({ kind: "match", basis: "event_pr_number_and_head_sha" }); expect( matchRunToPullRequest({ run: run({ pull_requests: [ { number: pull.number!, head: { sha: "c".repeat(40) } }, ], }), pull, identityCounts: new Map([["73\u0000feature", 1]]), }).kind, ).toBe("none"); const branchRun = run({ pull_requests: [] }); expect( matchRunToPullRequest({ run: branchRun, pull, identityCounts: new Map([["73\u0000feature", 1]]), inventoryComplete: false, }).kind, ).toBe("ambiguous"); expect( matchRunToPullRequest({ run: branchRun, pull, identityCounts: new Map([["73\u0000feature", 1]]), }), ).toEqual({ kind: "match", basis: "event_unique_head_ref_and_repository_id", }); expect( matchRunToPullRequest({ run: run({ head_sha: mergeSha, pull_requests: [] }), pull, identityCounts: new Map([["73\u0000feature", 1]]), }).kind, ).toBe("none"); expect( matchRunToPullRequest({ run: run({ pull_requests: [{ number: pull.number!, head: { sha: headSha } }], }), pull, identityCounts: new Map([["73\u0000feature", 1]]), }).kind, ).toBe("ambiguous"); expect( matchRunToPullRequest({ run: branchRun, pull, identityCounts: new Map([["73\u0000feature", 2]]), }).kind, ).toBe("ambiguous"); expect( matchRunToPullRequest({ run: run({ event: "workflow_dispatch" }), pull, identityCounts: new Map([["73\u0000feature", 1]]), }).kind, ).toBe("none"); }); it("scans target-branch PRs through public gh GET requests and writes JSON and Markdown", async () => { const fixture = await createFixture(); const calls: string[] = []; const apiRequest: NonNullable< Parameters[0]["apiRequest"] > = async (method, endpoint) => { calls.push(`${method} ${endpoint}`); if (method !== "GET") throw new Error("fixture rejected a non-GET request"); if (endpoint.includes("/pulls?")) return [ rawPull(42, { mergeable: true }), rawPull(43, { mergeable: false }), rawPull(44, { base: { ref: "release", repo: { id: 11 } }, mergeable: true, }), rawPull(45, { draft: true, mergeable: true }), ]; if (endpoint.includes("/actions/workflows/ci.yml/runs?")) return { total_count: 1, workflow_runs: [ run({ id: 9001, event: "pull_request", status: "completed", conclusion: "failure", head_sha: mergeSha, pull_requests: [ { number: 42, head: { sha: headSha, repo: { id: 73 } }, }, ], }), ], }; throw new Error(`unexpected GET endpoint: ${endpoint}`); }; const report = await collectPrCiHealth({ repository: "owner/rea", repositoryId: 11, defaultBranch: "main", now: observedAt, apiRequest, }); expect(calls).toHaveLength(2); expect(calls.every((call) => call.startsWith("GET "))).toBe(true); expect(calls[0]).toContain("state=open"); expect(calls[0]).not.toContain("base="); expect(calls[1]).toContain("event=pull_request"); expect(report.pullRequests).toHaveLength(2); expect(report.pullRequests.map(({ number }) => number)).toEqual([42, 43]); expect(report.pullRequests.map(({ ci }) => ci.category)).toEqual([ "completed_failure", "merge_conflict", ]); expect(report.scan).toMatchObject({ status: "complete", pullRequestInventoryComplete: true, workflowRunInventoryComplete: true, apiRequests: 2, }); const output = join(fixture.directory, ".cache", "ci-pr-health"); const paths = await writePrCiHealthReport(report, output); const written = JSON.parse(await readFile(paths.jsonPath, "utf8")); const markdown = await readFile(paths.markdownPath, "utf8"); expect(written.counts.completed_failure).toBe(1); expect(markdown).toContain("# Pull request CI health snapshot"); expect(markdown).toContain("#42"); expect(markdown).toContain("completed_failure"); }); it("counts older open PRs when deciding whether a branch/ref fallback is unique", async () => { const olderPull = rawPull(63, { created_at: "2026-09-01T00:00:00Z", head: { ...rawPull(63).head, ref: "shared-feature" }, }); const currentPull = rawPull(64, { mergeable: true, head: { ...olderPull.head }, }); const apiRequest: NonNullable< Parameters[0]["apiRequest"] > = async (_method, endpoint) => { if (endpoint.includes("/pulls?")) return [currentPull, olderPull]; if (endpoint.includes("/actions/workflows/ci.yml/runs?")) return { workflow_runs: [ run({ head_branch: "shared-feature", pull_requests: [], }), ], }; throw new Error(`unexpected GET endpoint: ${endpoint}`); }; const report = await collectPrCiHealth({ repository: "owner/rea", repositoryId: 11, defaultBranch: "main", now: observedAt, apiRequest, }); expect(report.scan.pullRequestInventoryComplete).toBe(true); expect(report.pullRequests.map(({ number }) => number)).toEqual([64]); expect(report.pullRequests[0]?.ci).toMatchObject({ category: "unknown", reason: "workflow_run_identity_ambiguous", }); }); it.each([false, true])( "keeps page-capped scans uncertain (observed success: %s)", async (observedSuccess) => { const pull = rawPull(62, { mergeable: true }); const apiRequest: NonNullable< Parameters[0]["apiRequest"] > = async (_method, endpoint) => { if (endpoint.includes("/pulls?")) return [pull]; const page = Number( new URL(`https://fixture.invalid/${endpoint}`).searchParams.get("page"), ); return { workflow_runs: Array.from({ length: 100 }, (_item, index) => observedSuccess && page === 1 && index === 0 ? run({ run_attempt: 1, pull_requests: [ { number: 62, head: { sha: headSha, repo: { id: 73 } } }, ], }) : run({ id: page * 100 + index, head_branch: "different-branch", head_repository: { id: 999 }, pull_requests: [], }), ), }; }; const report = await collectPrCiHealth({ repository: "owner/rea", repositoryId: 11, defaultBranch: "main", now: observedAt, apiRequest, }); expect(report.scan.workflowRunPages).toBe(5); expect(report.scan.workflowRunInventoryComplete).toBe(false); const firstPull = report.pullRequests[0]; if (firstPull === undefined) throw new Error("fixture PR was not scanned"); expect(firstPull.ci).toMatchObject({ category: observedSuccess ? "unknown" : "deferred", reason: observedSuccess ? "workflow_run_inventory_incomplete" : "lookup_inventory_incomplete", }); expect(firstPull.ci.runs).toHaveLength(observedSuccess ? 1 : 0); }, ); it("keeps the result unknown when the head changes between API snapshots", async () => { const apiRequest: NonNullable< Parameters[0]["apiRequest"] > = async (_method, endpoint) => { if (endpoint.includes("/pulls?")) return [rawPull(63)]; if (endpoint.includes("/actions/workflows/ci.yml/runs?")) return { workflow_runs: [] }; return { ...rawPull(63, { mergeable: true }), head: { ...rawPull(63).head, sha: mergeSha, }, }; }; const report = await collectPrCiHealth({ repository: "owner/rea", repositoryId: 11, defaultBranch: "main", now: observedAt, apiRequest, }); expect(report.pullRequests[0]).toMatchObject({ identityChangedDuringScan: true, ci: { category: "unknown", reason: "pull_request_head_identity_invalid" }, }); expect(report.scan).toMatchObject({ status: "partial", errors: ["pull_request_identity_changed_during_scan"], }); }); it("public CLI uses only GET and produces a report from fixture GitHub responses", async () => { const fixture = await createFixture(); const ghCalls = join(fixture.directory, "gh-calls.jsonl"); const ghResponse = join(fixture.directory, "gh-response.json"); const fakeGh = join(fixture.fakeBin, "gh"); await writeFile( ghResponse, JSON.stringify([ [rawPull(51, { mergeable: true })], { total_count: 0, workflow_runs: [] }, ]), ); await writeFile( fakeGh, `#!/usr/bin/env node\nconst fs=require("node:fs");\nconst args=process.argv.slice(2);\nfs.appendFileSync(process.env.CI_GH_CALLS, JSON.stringify(args)+"\\n");\nif(args[0]!=="api"||args[1]!=="--method"||args[2]!=="GET")process.exit(41);\nconst page=Number(new URL("https://fixture.invalid/"+args.at(-1)).searchParams.get("page"));\nconst responses=JSON.parse(fs.readFileSync(process.env.CI_GH_RESPONSE,"utf8"));\nprocess.stdout.write(JSON.stringify(responses[page===1&&args.at(-1).includes("/pulls?")?0:1]));\n`, ); await chmod(fakeGh, 0o755); const outputDirectory = join(fixture.directory, ".cache", "ci-pr-health"); await executeWorkflowFixture(process.execPath, [reporter], { cwd: fixture.directory, env: { ...process.env, PATH: `${fixture.fakeBin}${delimiter}${process.env.PATH ?? ""}`, CI_REPOSITORY: "owner/rea", CI_REPOSITORY_ID: "11", CI_DEFAULT_BRANCH: "main", CI_PR_HEALTH_DIRECTORY: outputDirectory, CI_GH_CALLS: ghCalls, CI_GH_RESPONSE: ghResponse, }, }); const calls = (await readFile(ghCalls, "utf8")) .trim() .split("\n") .map((line) => JSON.parse(line)); expect(calls).toHaveLength(2); expect( calls.every( (args: string[]) => args[1] === "--method" && args[2] === "GET", ), ).toBe(true); const report = JSON.parse( await readFile(join(outputDirectory, "report.json"), "utf8"), ); expect(report.pullRequests[0].ci.category).toBe("missing"); await expect( readFile(join(outputDirectory, "summary.md"), "utf8"), ).resolves.toContain("missing"); }); it("rejects non-GET methods before launching the GitHub client", async () => { const fixture = await createFixture(); const request = createGhApiRequest({ env: { ...process.env, PATH: `${fixture.fakeBin}${delimiter}${process.env.PATH ?? ""}`, CI_GH_CALLS: join(fixture.directory, "should-not-exist"), }, }); await expect( request("POST", "repos/owner/rea/pulls", { requests: 0, startedAt: observedAt, limitMs: 1_000, }), ).rejects.toThrow("read-only GET"); await expect( readFile(join(fixture.directory, "should-not-exist")), ).rejects.toMatchObject({ code: "ENOENT" }); }); function validPull(overrides: Partial = {}): PrCiPullRequest { return { number: 42, identityValid: true, mergeable: true, head: { repositoryId: 73, ref: "feature", sha: headSha }, ...overrides, }; } it.each([ ["newer run", { id: 78, created_at: "2026-10-13T01:00:00Z" }, "unknown"], [ "older run", { id: 78, created_at: "2026-10-12T23:00:00Z" }, "completed_success", ], ["equal timestamps", { id: 78 }, "unknown"], [ "retry of an older run", { id: 78, created_at: "2026-10-12T23:00:00Z", run_attempt: 2 }, "unknown", ], ["unknown timestamp", { id: 78, created_at: "invalid" }, "unknown"], ["newer attempt", { run_attempt: 3 }, "unknown"], ["older attempt", { run_attempt: 1 }, "completed_success"], ] as const)( "retains ambiguity evidence for a %s without reporting a stale completion", (_label, overrides, category) => { const currentAttempt = "id" in overrides ? 1 : 2; const result = classifyPrCi({ pull: validPull(), runs: [ { run: run({ run_attempt: currentAttempt }), basis: "event_pr_number_and_head_sha", }, ], ambiguousRuns: [ { run: run({ run_attempt: currentAttempt, ...overrides }), basis: "ambiguous_head_identity", }, ], runsComplete: true, pullRequestInventoryComplete: true, }); expect(result.category).toBe(category); expect(result.runs).toHaveLength(2); }, ); it("does not let an older proven run hide a newer run with an unverified repository ID", async () => { const pull = rawPull(42); const older = run({ id: 70, created_at: "2026-10-13T00:00:00Z", pull_requests: [{ number: 42, head: { sha: headSha, repo: { id: 73 } } }], }); const newer = run({ id: 71, created_at: "2026-10-13T01:00:00Z", status: "in_progress", conclusion: null, pull_requests: [{ number: 42, head: { sha: headSha } }], }); const report = await collectPrCiHealth({ repository: "owner/rea", repositoryId: 11, defaultBranch: "main", now: observedAt, apiRequest: async (_method, endpoint) => { if (endpoint.includes("/pulls?")) return [pull]; if (endpoint.endsWith("/pulls/42")) return { ...pull, mergeable: true }; return { workflow_runs: [newer, older] }; }, }); expect(report.pullRequests[0]?.ci).toMatchObject({ category: "unknown", reason: "workflow_run_identity_ambiguous", }); expect(report.pullRequests[0]?.ci.runs).toHaveLength(2); }); it("keeps a proven newest run even when two older matched runs have equal timestamps", () => { const result = classifyPrCi({ pull: validPull(), runs: [ run({ id: 70, run_attempt: 1 }), run({ id: 71, run_attempt: 1, conclusion: "failure" }), run({ id: 72, run_attempt: 1, created_at: "2026-10-13T01:00:00Z", status: "in_progress", conclusion: null, }), ].map((run) => ({ run, basis: "event_pr_number_and_head_sha" })), ambiguousRuns: [], runsComplete: true, pullRequestInventoryComplete: true, }); expect(result.category).toBe("queued_pending_executing"); expect(result.runs).toHaveLength(3); }); function run(overrides: Partial = {}): PrCiWorkflowRun { return { id: 77, event: "pull_request", head_branch: "feature", head_repository: { id: 73 }, head_sha: headSha, status: "completed", conclusion: "success", created_at: "2026-10-13T00:00:00Z", pull_requests: [], ...overrides, }; } function rawPull(number: number, overrides: Record = {}) { return { number, draft: false, created_at: "2026-10-13T00:00:00Z", html_url: `https://github.com/owner/rea/pull/${String(number)}`, base: { ref: "main", repo: { id: 11 } }, head: { ref: `feature-${String(number)}`, sha: headSha, repo: { id: 73, full_name: "contributor/rea" }, }, ...overrides, }; } async function createFixture() { const directory = await createTestTempDirectory("rea-ci-pr-health-"); const fakeBin = join(directory, "bin"); await mkdir(fakeBin); return { directory, fakeBin }; }