diff --git a/scripts/ci/artifacts.mjs b/scripts/ci/artifacts.mjs index 9a7cc1c21..250a15872 100644 --- a/scripts/ci/artifacts.mjs +++ b/scripts/ci/artifacts.mjs @@ -87,10 +87,13 @@ export function classifyFailure({ "unknown", "main_run_does_not_contain_the_same_structured_diagnostic", ); - if (!sameBlobEvidence(diagnostics, blobs)) + const blobStatus = blobIdentityStatus(diagnostics, blobs); + if (blobStatus !== "identical") return outcome( "unknown", - "diagnostic_source_content_not_identical_at_main_and_tested_base", + blobStatus === "different" + ? "diagnostic_source_content_not_identical_at_main_and_tested_base" + : "diagnostic_source_blob_identity_unavailable", ); return outcome( "matching-main-diagnostics", @@ -366,9 +369,12 @@ function diagnosticKey(item) { return JSON.stringify([item.kind, ...[...item.paths].sort()]); return null; } -function sameBlobEvidence(diagnostics, blobs) { +function blobIdentityStatus(diagnostics, blobs) { const paths = [...new Set(diagnostics.flatMap(diagnosticPaths))]; - return paths.length > 0 && paths.every((path) => blobs?.[path] === true); + if (paths.some((path) => blobs?.[path] === false)) return "different"; + return paths.length > 0 && paths.every((path) => blobs?.[path] === true) + ? "identical" + : "unavailable"; } function sameTaskHashes(diagnostics, left, right) { return diagnostics.every((diagnostic) => { diff --git a/scripts/ci/failure-report.mjs b/scripts/ci/failure-report.mjs index 2ce6d813c..12364c59f 100644 --- a/scripts/ci/failure-report.mjs +++ b/scripts/ci/failure-report.mjs @@ -153,16 +153,21 @@ async function analyzeFailedRun({ ]; const failures = []; for (const job of failedJobs) { - const evidence = await api.getStaticEvidence(repository, run, job); - const classification = await attributeToMain({ - repository, - defaultBranch, - run, - job, - evidence, - mainCandidates, - api, - }); + let classification; + try { + const evidence = await api.getStaticEvidence(repository, run, job); + classification = await attributeToMain({ + repository, + defaultBranch, + run, + job, + evidence, + mainCandidates, + api, + }); + } catch (error) { + classification = outcome("unknown", errorCode(error)); + } failures.push({ runId: entry.id, runAttempt: entry.runAttempt, @@ -415,26 +420,25 @@ export function createGitHubApi({ for (const path of paths) { if (!validRepoPath(path)) continue; const [left, right] = await Promise.all([ - getContent(repository, ancestor, path, request), - getContent(repository, descendant, path, request), + getContentBlobSha(repository, ancestor, path, request), + getContentBlobSha(repository, descendant, path, request), ]); - result[path] = left !== null && left === right; + result[path] = left === right; } return result; }, }; } -async function getContent(repository, ref, path, request) { - try { - const encodedPath = path.split("/").map(encodeURIComponent).join("/"); - const value = await request( - `repos/${repository}/contents/${encodedPath}?ref=${encodeURIComponent(ref)}`, - ); - return typeof value.sha === "string" ? value.sha : null; - } catch { - return null; - } +async function getContentBlobSha(repository, ref, path, request) { + const encodedPath = path.split("/").map(encodeURIComponent).join("/"); + const value = await request( + `repos/${repository}/contents/${encodedPath}?ref=${encodeURIComponent(ref)}`, + ); + const sha = safeSha(value?.sha); + if (value?.type !== "file" || sha === null) + throw new Error("github_content_blob_identity_unavailable"); + return sha; } function outcome(classification, reason) { diff --git a/tests/boundary/filesystem/ciFailureReport.test.ts b/tests/boundary/filesystem/ciFailureReport.test.ts index 26feb3ee2..c2a282906 100644 --- a/tests/boundary/filesystem/ciFailureReport.test.ts +++ b/tests/boundary/filesystem/ciFailureReport.test.ts @@ -120,6 +120,22 @@ it.each([ expect(result.classification).toBe("unknown"); }); +it.each([ + [{}, "diagnostic_source_blob_identity_unavailable"], + [ + { "src/example.ts": false }, + "diagnostic_source_content_not_identical_at_main_and_tested_base", + ], +] as const)( + "distinguishes unavailable blob identity from verified differences (%s)", + (blobs, reason) => { + expect(classifyFailure({ ...comparison(), blobs })).toEqual({ + classification: "unknown", + reason, + }); + }, +); + it("writes informational JSON and Markdown from the supplied manifest using only read API calls", async () => { const directory = await createTestTempDirectory("rea-ci-failure-test-"); const calls: string[] = []; diff --git a/tests/boundary/filesystem/ciFailureReportBudget.test.ts b/tests/boundary/filesystem/ciFailureReportBudget.test.ts index 92fe3e4ae..1caacde16 100644 --- a/tests/boundary/filesystem/ciFailureReportBudget.test.ts +++ b/tests/boundary/filesystem/ciFailureReportBudget.test.ts @@ -2,6 +2,7 @@ import { expect, it } from "vitest"; import { createTestTempDirectory } from "../../fixtures/temporaryDirectory.js"; import { reportCiFailures, + createGitHubApi, type CiRun, type FailureReportApi, type StaticDiagnostic, @@ -235,6 +236,10 @@ it.each([1, 51])( expect(result.failures[0]!.classification).toBe( count === 1 ? "matching-main-diagnostics" : "unknown", ); + if (count > 50) + expect(result.failures[0]!.reason).toBe( + "diagnostic_source_blob_identity_unavailable", + ); }, ); @@ -288,3 +293,115 @@ it.each(["jobs", "artifact", "comparison"])( expect(input.artifacts.filter((id) => id === 21)).toHaveLength(1); }, ); + +it.each([0, 1, 2])( + "preserves every failed job when artifact analysis fails at position %s", + async (position) => { + const input = fixture([20], 1); + const jobs = [ + { name: "Python producer", conclusion: "failure" }, + { name: "Test shard (1)", conclusion: "timed_out" }, + ]; + jobs.splice(position, 0, { name: "Static checks", conclusion: "failure" }); + input.api.getJobs = async (_repository, id, attempt) => + jobs.map((job) => ({ + ...job, + status: "completed", + run_id: Number(id), + run_attempt: attempt, + })); + input.api.getStaticEvidence = async (_repository, _run, job) => { + if (job.name === "Static checks") + throw new Error("static_artifact_unavailable"); + return null; + }; + const result = await report(input); + expect(result.counts.unknown).toBe(3); + expect(result.failures).toEqual( + jobs.map((job) => ({ + runId: 100, + runAttempt: 1, + runHead: prSha, + event: "pull_request", + job: job.name, + conclusion: job.conclusion, + classification: "unknown", + reason: + job.name === "Static checks" + ? "static_artifact_unavailable" + : "no_matching_verified_main_failure", + })), + ); + }, +); + +it("keeps unavailable job inventories at run scope", async () => { + const input = fixture([], 1); + input.api.getJobs = async () => { + throw new Error("job_inventory_unavailable"); + }; + expect((await report(input)).failures).toEqual([ + { + runId: 100, + runAttempt: 1, + classification: "unknown", + reason: "job_inventory_unavailable", + }, + ]); +}); + +it.each([ + ["request", "github_api_temporarily_unavailable"], + ["budget", "github_api_request_budget_exhausted"], + ["missing sha", "github_content_blob_identity_unavailable"], + ["invalid sha", "github_content_blob_identity_unavailable"], + ["directory", "github_content_blob_identity_unavailable"], +] as const)( + "retains unavailable blob evidence as a comparison error: %s", + async (failure, reason) => { + const input = fixture([20], 1); + const contentApi = createGitHubApi({ + token: "fixture-token", + apiRequest: async () => { + if (failure === "request") throw new Error(reason); + if (failure === "missing sha") return { type: "file" }; + if (failure === "invalid sha") + return { type: "file", sha: "not-a-sha" }; + if (failure === "directory") return []; + return { type: "file", sha: "e".repeat(40) }; + }, + }); + if (failure === "budget") { + for (let id = 1; id <= 250; id++) await contentApi.getRun(repository, id); + } + input.api.getBlobIdentity = contentApi.getBlobIdentity; + const result = await report(input); + expect(result.failures).toMatchObject([ + { + job: "Static checks", + classification: "unknown", + reason: "no_matching_verified_main_failure", + candidateFailures: [{ runId: 20, runAttempt: 1, reason }], + }, + ]); + }, +); + +it.each([true, false])( + "reports blob equality only from verified file SHAs (equal: %s)", + async (equal) => { + const api = createGitHubApi({ + token: "fixture-token", + apiRequest: async (_method, endpoint) => ({ + type: "file", + sha: + equal || endpoint.includes(`ref=${mainSha}`) + ? "e".repeat(40) + : "f".repeat(40), + }), + }); + expect( + await api.getBlobIdentity(repository, mainSha, prSha, ["src/example.ts"]), + ).toEqual({ "src/example.ts": equal }); + }, +);