fix(ci): preserve per-job findings and unavailable blob evidence

scripts/ci/artifacts.mjs

@@ -87,10 +87,13 @@ export function classifyFailure({
       "unknown",
       "main_run_does_not_contain_the_same_structured_diagnostic",
     );
-  if (!sameBlobEvidence(diagnostics, blobs))
+  const blobStatus = blobIdentityStatus(diagnostics, blobs);
+  if (blobStatus !== "identical")
     return outcome(
       "unknown",
-      "diagnostic_source_content_not_identical_at_main_and_tested_base",
+      blobStatus === "different"
+        ? "diagnostic_source_content_not_identical_at_main_and_tested_base"
+        : "diagnostic_source_blob_identity_unavailable",
     );
   return outcome(
     "matching-main-diagnostics",
@@ -366,9 +369,12 @@ function diagnosticKey(item) {
     return JSON.stringify([item.kind, ...[...item.paths].sort()]);
   return null;
 }
-function sameBlobEvidence(diagnostics, blobs) {
+function blobIdentityStatus(diagnostics, blobs) {
   const paths = [...new Set(diagnostics.flatMap(diagnosticPaths))];
-  return paths.length > 0 && paths.every((path) => blobs?.[path] === true);
+  if (paths.some((path) => blobs?.[path] === false)) return "different";
+  return paths.length > 0 && paths.every((path) => blobs?.[path] === true)
+    ? "identical"
+    : "unavailable";
 }
 function sameTaskHashes(diagnostics, left, right) {
   return diagnostics.every((diagnostic) => {

scripts/ci/failure-report.mjs

@@ -153,16 +153,21 @@ async function analyzeFailedRun({
     ];
   const failures = [];
   for (const job of failedJobs) {
-    const evidence = await api.getStaticEvidence(repository, run, job);
-    const classification = await attributeToMain({
-      repository,
-      defaultBranch,
-      run,
-      job,
-      evidence,
-      mainCandidates,
-      api,
-    });
+    let classification;
+    try {
+      const evidence = await api.getStaticEvidence(repository, run, job);
+      classification = await attributeToMain({
+        repository,
+        defaultBranch,
+        run,
+        job,
+        evidence,
+        mainCandidates,
+        api,
+      });
+    } catch (error) {
+      classification = outcome("unknown", errorCode(error));
+    }
     failures.push({
       runId: entry.id,
       runAttempt: entry.runAttempt,
@@ -415,26 +420,25 @@ export function createGitHubApi({
       for (const path of paths) {
         if (!validRepoPath(path)) continue;
         const [left, right] = await Promise.all([
-          getContent(repository, ancestor, path, request),
-          getContent(repository, descendant, path, request),
+          getContentBlobSha(repository, ancestor, path, request),
+          getContentBlobSha(repository, descendant, path, request),
         ]);
-        result[path] = left !== null && left === right;
+        result[path] = left === right;
       }
       return result;
     },
   };
 }
 
-async function getContent(repository, ref, path, request) {
-  try {
-    const encodedPath = path.split("/").map(encodeURIComponent).join("/");
-    const value = await request(
-      `repos/${repository}/contents/${encodedPath}?ref=${encodeURIComponent(ref)}`,
-    );
-    return typeof value.sha === "string" ? value.sha : null;
-  } catch {
-    return null;
-  }
+async function getContentBlobSha(repository, ref, path, request) {
+  const encodedPath = path.split("/").map(encodeURIComponent).join("/");
+  const value = await request(
+    `repos/${repository}/contents/${encodedPath}?ref=${encodeURIComponent(ref)}`,
+  );
+  const sha = safeSha(value?.sha);
+  if (value?.type !== "file" || sha === null)
+    throw new Error("github_content_blob_identity_unavailable");
+  return sha;
 }
 
 function outcome(classification, reason) {

tests/boundary/filesystem/ciFailureReport.test.ts

@@ -120,6 +120,22 @@ it.each([
   expect(result.classification).toBe("unknown");
 });
 
+it.each([
+  [{}, "diagnostic_source_blob_identity_unavailable"],
+  [
+    { "src/example.ts": false },
+    "diagnostic_source_content_not_identical_at_main_and_tested_base",
+  ],
+] as const)(
+  "distinguishes unavailable blob identity from verified differences (%s)",
+  (blobs, reason) => {
+    expect(classifyFailure({ ...comparison(), blobs })).toEqual({
+      classification: "unknown",
+      reason,
+    });
+  },
+);
+
 it("writes informational JSON and Markdown from the supplied manifest using only read API calls", async () => {
   const directory = await createTestTempDirectory("rea-ci-failure-test-");
   const calls: string[] = [];

tests/boundary/filesystem/ciFailureReportBudget.test.ts

@@ -2,6 +2,7 @@ import { expect, it } from "vitest";
 import { createTestTempDirectory } from "../../fixtures/temporaryDirectory.js";
 import {
   reportCiFailures,
+  createGitHubApi,
   type CiRun,
   type FailureReportApi,
   type StaticDiagnostic,
@@ -235,6 +236,10 @@ it.each([1, 51])(
     expect(result.failures[0]!.classification).toBe(
       count === 1 ? "matching-main-diagnostics" : "unknown",
     );
+    if (count > 50)
+      expect(result.failures[0]!.reason).toBe(
+        "diagnostic_source_blob_identity_unavailable",
+      );
   },
 );
 
@@ -288,3 +293,115 @@ it.each(["jobs", "artifact", "comparison"])(
     expect(input.artifacts.filter((id) => id === 21)).toHaveLength(1);
   },
 );
+
+it.each([0, 1, 2])(
+  "preserves every failed job when artifact analysis fails at position %s",
+  async (position) => {
+    const input = fixture([20], 1);
+    const jobs = [
+      { name: "Python producer", conclusion: "failure" },
+      { name: "Test shard (1)", conclusion: "timed_out" },
+    ];
+    jobs.splice(position, 0, { name: "Static checks", conclusion: "failure" });
+    input.api.getJobs = async (_repository, id, attempt) =>
+      jobs.map((job) => ({
+        ...job,
+        status: "completed",
+        run_id: Number(id),
+        run_attempt: attempt,
+      }));
+    input.api.getStaticEvidence = async (_repository, _run, job) => {
+      if (job.name === "Static checks")
+        throw new Error("static_artifact_unavailable");
+      return null;
+    };
+    const result = await report(input);
+    expect(result.counts.unknown).toBe(3);
+    expect(result.failures).toEqual(
+      jobs.map((job) => ({
+        runId: 100,
+        runAttempt: 1,
+        runHead: prSha,
+        event: "pull_request",
+        job: job.name,
+        conclusion: job.conclusion,
+        classification: "unknown",
+        reason:
+          job.name === "Static checks"
+            ? "static_artifact_unavailable"
+            : "no_matching_verified_main_failure",
+      })),
+    );
+  },
+);
+
+it("keeps unavailable job inventories at run scope", async () => {
+  const input = fixture([], 1);
+  input.api.getJobs = async () => {
+    throw new Error("job_inventory_unavailable");
+  };
+  expect((await report(input)).failures).toEqual([
+    {
+      runId: 100,
+      runAttempt: 1,
+      classification: "unknown",
+      reason: "job_inventory_unavailable",
+    },
+  ]);
+});
+
+it.each([
+  ["request", "github_api_temporarily_unavailable"],
+  ["budget", "github_api_request_budget_exhausted"],
+  ["missing sha", "github_content_blob_identity_unavailable"],
+  ["invalid sha", "github_content_blob_identity_unavailable"],
+  ["directory", "github_content_blob_identity_unavailable"],
+] as const)(
+  "retains unavailable blob evidence as a comparison error: %s",
+  async (failure, reason) => {
+    const input = fixture([20], 1);
+    const contentApi = createGitHubApi({
+      token: "fixture-token",
+      apiRequest: async () => {
+        if (failure === "request") throw new Error(reason);
+        if (failure === "missing sha") return { type: "file" };
+        if (failure === "invalid sha")
+          return { type: "file", sha: "not-a-sha" };
+        if (failure === "directory") return [];
+        return { type: "file", sha: "e".repeat(40) };
+      },
+    });
+    if (failure === "budget") {
+      for (let id = 1; id <= 250; id++) await contentApi.getRun(repository, id);
+    }
+    input.api.getBlobIdentity = contentApi.getBlobIdentity;
+    const result = await report(input);
+    expect(result.failures).toMatchObject([
+      {
+        job: "Static checks",
+        classification: "unknown",
+        reason: "no_matching_verified_main_failure",
+        candidateFailures: [{ runId: 20, runAttempt: 1, reason }],
+      },
+    ]);
+  },
+);
+
+it.each([true, false])(
+  "reports blob equality only from verified file SHAs (equal: %s)",
+  async (equal) => {
+    const api = createGitHubApi({
+      token: "fixture-token",
+      apiRequest: async (_method, endpoint) => ({
+        type: "file",
+        sha:
+          equal || endpoint.includes(`ref=${mainSha}`)
+            ? "e".repeat(40)
+            : "f".repeat(40),
+      }),
+    });
+    expect(
+      await api.getBlobIdentity(repository, mainSha, prSha, ["src/example.ts"]),
+    ).toEqual({ "src/example.ts": equal });
+  },
+);