530 lines
16 KiB
JavaScript
530 lines
16 KiB
JavaScript
import { appendFile, mkdir, readFile, writeFile } from "node:fs/promises";
|
|
import { join, resolve } from "node:path";
|
|
import { fileURLToPath } from "node:url";
|
|
|
|
import {
|
|
classifyFailure,
|
|
diagnosticPaths,
|
|
isEligibleMainRun,
|
|
isFailedConclusion,
|
|
pullRequestBaseSha,
|
|
readStaticCheckArtifact,
|
|
safeSha,
|
|
selectStaticCheckArtifact,
|
|
validRepoPath,
|
|
} from "./artifacts.mjs";
|
|
import { createGhApiRequest } from "./github-api.mjs";
|
|
import { normalizeWorkflowRun } from "./workflow-run.mjs";
|
|
|
|
const OUTPUT_DIRECTORY = join(".cache", "ci-failures");
|
|
const BATCH_DIRECTORY = join(".cache", "ci-job-timings");
|
|
const MAX_JSON_BYTES = 8 * 1024 * 1024;
|
|
const MAX_FAILED_RUNS = 20;
|
|
const MAX_MAIN_CANDIDATES = 5;
|
|
const MAX_MAIN_RUN_LOOKUPS = 100;
|
|
const MAX_PATHS_PER_COMPARISON = 50;
|
|
const MAX_API_REQUESTS = 250;
|
|
|
|
export { classifyFailure } from "./artifacts.mjs";
|
|
|
|
/** Uses only failed CI candidates present in the supplied bounded timing manifest. */
|
|
export async function reportCiFailures({
|
|
repository,
|
|
defaultBranch,
|
|
batchManifest,
|
|
api,
|
|
outputDirectory = OUTPUT_DIRECTORY,
|
|
summaryPath,
|
|
observedAt = new Date().toISOString(),
|
|
}) {
|
|
validateRepository(repository);
|
|
if (typeof defaultBranch !== "string" || defaultBranch.length === 0)
|
|
throw new Error("defaultBranch is required");
|
|
if (
|
|
!batchManifest ||
|
|
batchManifest.repository !== repository ||
|
|
!Array.isArray(batchManifest.runs)
|
|
)
|
|
throw new Error("timing manifest repository or runs are invalid");
|
|
|
|
const failedRuns = batchManifest.runs.filter(
|
|
(entry) =>
|
|
entry.workflow === "CI" &&
|
|
entry.status === "collected" &&
|
|
isFailedConclusion(entry.conclusion),
|
|
);
|
|
const prRuns = failedRuns.filter((entry) => entry.event === "pull_request");
|
|
const candidates = prRuns.slice(0, MAX_FAILED_RUNS);
|
|
const mainEntries = failedRuns.filter(
|
|
(entry) =>
|
|
entry.headBranch === defaultBranch &&
|
|
["push", "schedule", "workflow_dispatch"].includes(entry.event),
|
|
);
|
|
const lookupRun = verifiedRunLookup(repository, api);
|
|
const mainCandidates = [];
|
|
for (const entry of mainEntries.slice(0, MAX_MAIN_RUN_LOOKUPS)) {
|
|
try {
|
|
const run = await lookupRun(entry);
|
|
if (isEligibleMainRun(run, defaultBranch, repository))
|
|
mainCandidates.push({
|
|
run,
|
|
getEvidence: mainEvidenceLookup(repository, api, entry, run),
|
|
});
|
|
if (mainCandidates.length === MAX_MAIN_CANDIDATES) break;
|
|
} catch {
|
|
// Unverified metadata cannot consume the eligible-main candidate budget.
|
|
}
|
|
}
|
|
const entries = [];
|
|
for (const entry of candidates) {
|
|
try {
|
|
entries.push(
|
|
...(await analyzeFailedRun({
|
|
repository,
|
|
defaultBranch,
|
|
entry,
|
|
run: await lookupRun(entry),
|
|
mainCandidates,
|
|
api,
|
|
})),
|
|
);
|
|
} catch (error) {
|
|
entries.push({
|
|
runId: entry.id,
|
|
runAttempt: entry.runAttempt,
|
|
classification: "unknown",
|
|
reason: errorCode(error),
|
|
});
|
|
}
|
|
}
|
|
const report = {
|
|
version: 1,
|
|
observedAt,
|
|
repository,
|
|
defaultBranch,
|
|
informationalOnly: true,
|
|
gateEffect: "none; this report never changes CI conclusions",
|
|
boundedInput: {
|
|
candidateRuns: candidates.length,
|
|
limit: MAX_FAILED_RUNS,
|
|
mainCandidateLimit: MAX_MAIN_CANDIDATES,
|
|
mainCandidateRuns: mainCandidates.length,
|
|
mainRunLookupLimit: MAX_MAIN_RUN_LOOKUPS,
|
|
manifestTruncated:
|
|
prRuns.length > MAX_FAILED_RUNS ||
|
|
batchManifest.truncation?.length > 0 ||
|
|
batchManifest.selection?.exhaustive === false ||
|
|
batchManifest.coverage?.exhaustive === false,
|
|
},
|
|
counts: Object.fromEntries(
|
|
["matching-main-diagnostics", "unknown"].map((category) => [
|
|
category,
|
|
entries.filter((item) => item.classification === category).length,
|
|
]),
|
|
),
|
|
failures: entries,
|
|
};
|
|
await writeFailureReport(report, outputDirectory, summaryPath);
|
|
return report;
|
|
}
|
|
|
|
async function analyzeFailedRun({
|
|
repository,
|
|
defaultBranch,
|
|
entry,
|
|
run,
|
|
mainCandidates,
|
|
api,
|
|
}) {
|
|
const jobs = await api.getJobs(repository, entry.id, entry.runAttempt);
|
|
if (jobs.some((job) => !hasJobIdentity(job, entry)))
|
|
throw new Error("workflow_job_identity_mismatch");
|
|
const failedJobs = jobs.filter(
|
|
(job) => job.status === "completed" && isFailedConclusion(job.conclusion),
|
|
);
|
|
if (failedJobs.length === 0)
|
|
return [
|
|
{
|
|
runId: entry.id,
|
|
runAttempt: entry.runAttempt,
|
|
classification: "unknown",
|
|
reason: "failed_jobs_not_present_in_api_snapshot",
|
|
},
|
|
];
|
|
const failures = [];
|
|
for (const job of failedJobs) {
|
|
const evidence = await api.getStaticEvidence(repository, run, job);
|
|
const classification = await attributeToMain({
|
|
repository,
|
|
defaultBranch,
|
|
run,
|
|
job,
|
|
evidence,
|
|
mainCandidates,
|
|
api,
|
|
});
|
|
failures.push({
|
|
runId: entry.id,
|
|
runAttempt: entry.runAttempt,
|
|
runHead: safeSha(run.head_sha),
|
|
event: run.event ?? null,
|
|
job: job.name,
|
|
conclusion: job.conclusion,
|
|
...classification,
|
|
});
|
|
}
|
|
return failures;
|
|
}
|
|
|
|
async function attributeToMain({
|
|
repository,
|
|
defaultBranch,
|
|
run,
|
|
job,
|
|
evidence,
|
|
mainCandidates,
|
|
api,
|
|
}) {
|
|
const baseSha = pullRequestBaseSha(run, repository);
|
|
if (baseSha === null || evidence === null)
|
|
return outcome("unknown", "no_matching_verified_main_failure");
|
|
let result = outcome("unknown", "no_matching_verified_main_failure");
|
|
const candidateFailures = [];
|
|
for (const candidate of mainCandidates) {
|
|
try {
|
|
const main = await compareCandidate({
|
|
repository,
|
|
defaultBranch,
|
|
run,
|
|
job,
|
|
evidence,
|
|
baseSha,
|
|
candidate,
|
|
api,
|
|
});
|
|
if (main !== null) result = main;
|
|
if (main?.classification === "matching-main-diagnostics") break;
|
|
} catch (error) {
|
|
candidateFailures.push({
|
|
runId: candidate.run.id,
|
|
runAttempt: candidate.run.run_attempt,
|
|
reason: errorCode(error),
|
|
});
|
|
}
|
|
}
|
|
return {
|
|
...result,
|
|
...(candidateFailures.length > 0 ? { candidateFailures } : {}),
|
|
};
|
|
}
|
|
|
|
async function compareCandidate({
|
|
repository,
|
|
defaultBranch,
|
|
run,
|
|
job,
|
|
evidence,
|
|
baseSha,
|
|
candidate,
|
|
api,
|
|
}) {
|
|
const mainRun = candidate.run;
|
|
const main = await candidate.getEvidence(job.name);
|
|
if (!main?.evidence) return null;
|
|
const { job: mainJob, evidence: mainEvidence } = main;
|
|
const ancestry = await api.getAncestry(repository, mainRun.head_sha, baseSha);
|
|
const paths = [
|
|
...new Set(evidence.diagnostics.flatMap(diagnosticPaths)),
|
|
].slice(0, MAX_PATHS_PER_COMPARISON);
|
|
const blobs = await api.getBlobIdentity(
|
|
repository,
|
|
mainRun.head_sha,
|
|
baseSha,
|
|
paths,
|
|
);
|
|
const executionSourcesVerified =
|
|
(await api.verifyExecutionSource(repository, run, evidence.context)) &&
|
|
(await api.verifyExecutionSource(
|
|
repository,
|
|
mainRun,
|
|
mainEvidence.context,
|
|
));
|
|
return classifyFailure({
|
|
repository,
|
|
defaultBranch,
|
|
run,
|
|
job,
|
|
diagnostics: evidence.diagnostics,
|
|
context: evidence.context,
|
|
mainRun,
|
|
mainJob,
|
|
mainDiagnostics: mainEvidence.diagnostics,
|
|
mainContext: mainEvidence.context,
|
|
executionSourcesVerified,
|
|
ancestry,
|
|
blobs,
|
|
});
|
|
}
|
|
|
|
// Each verified main run/attempt owns its lazy jobs and parsed evidence for this
|
|
// report only. Cache nulls and rejected reads too; retries cannot spend the budget
|
|
// repeatedly on an immutable observation that was unavailable or invalid.
|
|
function mainEvidenceLookup(repository, api, entry, run) {
|
|
let jobs;
|
|
const evidence = new Map();
|
|
return (name) => {
|
|
jobs ??= Promise.resolve().then(async () => {
|
|
const items = await api.getJobs(repository, entry.id, entry.runAttempt);
|
|
return items.some((item) => !hasJobIdentity(item, entry)) ? [] : items;
|
|
});
|
|
if (!evidence.has(name))
|
|
evidence.set(
|
|
name,
|
|
jobs.then(async (items) => {
|
|
const job = items.find(
|
|
(item) =>
|
|
item.name === name &&
|
|
item.status === "completed" &&
|
|
isFailedConclusion(item.conclusion),
|
|
);
|
|
if (!job) return null;
|
|
return {
|
|
job,
|
|
evidence: await api.getStaticEvidence(repository, run, job),
|
|
};
|
|
}),
|
|
);
|
|
return evidence.get(name);
|
|
};
|
|
}
|
|
|
|
// Selection and attribution share one immutable API observation per run/attempt.
|
|
function verifiedRunLookup(repository, api) {
|
|
const runs = new Map();
|
|
return (entry) => {
|
|
const key = `${positiveId(entry.id)}:${positiveId(entry.runAttempt)}`;
|
|
if (!runs.has(key))
|
|
runs.set(
|
|
key,
|
|
Promise.resolve().then(async () => {
|
|
const run = normalizeWorkflowRun(
|
|
await api.getRun(repository, entry.id),
|
|
);
|
|
if (!hasRunIdentity(run, entry))
|
|
throw new Error("workflow_run_identity_mismatch");
|
|
return run;
|
|
}),
|
|
);
|
|
return runs.get(key);
|
|
};
|
|
}
|
|
|
|
function hasRunIdentity(run, entry) {
|
|
return (
|
|
String(run?.id) === String(entry.id) &&
|
|
Number(run.run_attempt) === Number(entry.runAttempt) &&
|
|
run.workflowPath?.file === ".github/workflows/ci.yml"
|
|
);
|
|
}
|
|
function hasJobIdentity(job, entry) {
|
|
return (
|
|
String(job.run_id) === String(entry.id) &&
|
|
Number(job.run_attempt) === Number(entry.runAttempt)
|
|
);
|
|
}
|
|
|
|
export function createGitHubApi({
|
|
token = process.env.GH_TOKEN,
|
|
fetchImpl = fetch,
|
|
apiRequest = createGhApiRequest(),
|
|
} = {}) {
|
|
if (!token) throw new Error("GH_TOKEN is required");
|
|
const budget = { requests: 0, startedAt: Date.now(), limitMs: 60_000 };
|
|
const request = async (endpoint) => {
|
|
if (budget.requests >= MAX_API_REQUESTS)
|
|
throw new Error("github_api_request_budget_exhausted");
|
|
budget.requests += 1;
|
|
return await apiRequest("GET", endpoint, budget);
|
|
};
|
|
return {
|
|
async getRun(repository, id) {
|
|
return await request(
|
|
`repos/${repository}/actions/runs/${positiveId(id)}`,
|
|
);
|
|
},
|
|
async getJobs(repository, id, attempt) {
|
|
const page = await request(
|
|
`repos/${repository}/actions/runs/${positiveId(id)}/attempts/${positiveId(attempt)}/jobs?per_page=100&page=1`,
|
|
);
|
|
if (!Array.isArray(page.jobs) || page.jobs.length >= 100)
|
|
throw new Error("job_list_incomplete_or_malformed");
|
|
return page.jobs;
|
|
},
|
|
async getStaticEvidence(repository, run, job) {
|
|
if (job.name !== "Static checks") return null;
|
|
if (run.run_attempt !== 1) return null;
|
|
const page = await request(
|
|
`repos/${repository}/actions/runs/${positiveId(run.id)}/artifacts?per_page=100&page=1`,
|
|
);
|
|
const artifact = selectStaticCheckArtifact(page, run, run.run_attempt);
|
|
if (!artifact) return null;
|
|
return await readStaticCheckArtifact({
|
|
repository,
|
|
artifactId: artifact.id,
|
|
token,
|
|
fetchImpl,
|
|
});
|
|
},
|
|
async getAncestry(repository, ancestor, descendant) {
|
|
const comparison = await request(
|
|
`repos/${repository}/compare/${safeSha(ancestor)}...${safeSha(descendant)}`,
|
|
);
|
|
return {
|
|
status: comparison.status,
|
|
mergeBaseSha: safeSha(comparison.merge_base_commit?.sha),
|
|
};
|
|
},
|
|
async verifyExecutionSource(repository, run, context) {
|
|
const sourceSha = safeSha(context.sourceSha);
|
|
if (!sourceSha) return false;
|
|
const pr =
|
|
run.event === "pull_request"
|
|
? pullRequestBaseSha(run, repository)
|
|
: null;
|
|
if (
|
|
run.event === "pull_request" &&
|
|
(pr === null || context.prHeadSha !== run.head_sha)
|
|
)
|
|
return false;
|
|
const commit = await request(`repos/${repository}/commits/${sourceSha}`);
|
|
if (safeSha(commit.sha) !== sourceSha) return false;
|
|
if (sourceSha === run.head_sha) return true;
|
|
if (
|
|
pr === null ||
|
|
!Array.isArray(commit.parents) ||
|
|
commit.parents.length !== 2
|
|
)
|
|
return false;
|
|
return (
|
|
safeSha(commit.parents[0]?.sha) === pr &&
|
|
safeSha(commit.parents[1]?.sha) === run.head_sha
|
|
);
|
|
},
|
|
async getBlobIdentity(repository, ancestor, descendant, paths) {
|
|
const result = {};
|
|
for (const path of paths) {
|
|
if (!validRepoPath(path)) continue;
|
|
const [left, right] = await Promise.all([
|
|
getContent(repository, ancestor, path, request),
|
|
getContent(repository, descendant, path, request),
|
|
]);
|
|
result[path] = left !== null && left === right;
|
|
}
|
|
return result;
|
|
},
|
|
};
|
|
}
|
|
|
|
async function getContent(repository, ref, path, request) {
|
|
try {
|
|
const encodedPath = path.split("/").map(encodeURIComponent).join("/");
|
|
const value = await request(
|
|
`repos/${repository}/contents/${encodedPath}?ref=${encodeURIComponent(ref)}`,
|
|
);
|
|
return typeof value.sha === "string" ? value.sha : null;
|
|
} catch {
|
|
return null;
|
|
}
|
|
}
|
|
|
|
function outcome(classification, reason) {
|
|
return { classification, reason };
|
|
}
|
|
function validateRepository(value) {
|
|
if (
|
|
typeof value !== "string" ||
|
|
!/^[A-Za-z0-9_.-]+\/[A-Za-z0-9_.-]+$/u.test(value)
|
|
)
|
|
throw new Error("repository must be owner/name");
|
|
}
|
|
function positiveId(value) {
|
|
if (!Number.isSafeInteger(Number(value)) || Number(value) < 1)
|
|
throw new Error(
|
|
"GitHub run and attempt IDs must be positive safe integers",
|
|
);
|
|
return String(Number(value));
|
|
}
|
|
function errorCode(error) {
|
|
return error instanceof Error ? error.message.slice(0, 160) : "unknown_error";
|
|
}
|
|
|
|
export async function writeFailureReport(
|
|
report,
|
|
outputDirectory = OUTPUT_DIRECTORY,
|
|
summaryPath,
|
|
) {
|
|
await mkdir(outputDirectory, { recursive: true });
|
|
const jsonPath = join(outputDirectory, "report.json");
|
|
const markdownPath = join(outputDirectory, "report.md");
|
|
const matchedDiagnostics = report.failures.filter(
|
|
(item) => item.classification === "matching-main-diagnostics",
|
|
);
|
|
const lines = [
|
|
"# CI failure attribution",
|
|
"",
|
|
"Informational only. This report never changes a CI conclusion or required gate.",
|
|
"",
|
|
`Observed ${report.observedAt}; ${String(matchedDiagnostics.length)} of ${String(report.failures.length)} failure reports contain supported diagnostics matching a failed static task on an ancestor main revision. This does not attribute other failures in the same job. Unknown means evidence was incomplete or did not match.`,
|
|
"",
|
|
];
|
|
for (const item of report.failures)
|
|
lines.push(
|
|
`- Run ${item.runId} attempt ${item.runAttempt}${item.job ? `, ${item.job}` : ""}: **${item.classification}** — ${item.reason}.`,
|
|
);
|
|
if (report.failures.length === 0)
|
|
lines.push(
|
|
"- No completed failed CI attempts were present in the supplied timing snapshot.",
|
|
);
|
|
await writeFile(jsonPath, `${JSON.stringify(report, null, 2)}\n`, "utf8");
|
|
await writeFile(markdownPath, `${lines.join("\n")}\n`, "utf8");
|
|
if (summaryPath)
|
|
await appendFile(summaryPath, `${lines.join("\n")}\n`, "utf8");
|
|
return { jsonPath, markdownPath };
|
|
}
|
|
|
|
async function main() {
|
|
const repository = process.env.CI_REPOSITORY;
|
|
const defaultBranch = process.env.CI_DEFAULT_BRANCH;
|
|
const batchPath =
|
|
process.env.CI_TIMING_MANIFEST ??
|
|
join(BATCH_DIRECTORY, "batch-manifest.json");
|
|
const batchManifest = JSON.parse(await readBoundedFile(batchPath));
|
|
const api = createGitHubApi();
|
|
await reportCiFailures({
|
|
repository,
|
|
defaultBranch,
|
|
batchManifest,
|
|
api,
|
|
outputDirectory: process.env.CI_FAILURES_DIRECTORY ?? OUTPUT_DIRECTORY,
|
|
summaryPath: process.env.GITHUB_STEP_SUMMARY,
|
|
});
|
|
}
|
|
async function readBoundedFile(path) {
|
|
const bytes = await readFile(path);
|
|
if (bytes.byteLength > MAX_JSON_BYTES)
|
|
throw new Error("timing manifest exceeds size limit");
|
|
return bytes.toString("utf8");
|
|
}
|
|
|
|
if (
|
|
process.argv[1] &&
|
|
resolve(process.argv[1]) === fileURLToPath(import.meta.url)
|
|
) {
|
|
try {
|
|
await main();
|
|
} catch (error) {
|
|
console.error(error instanceof Error ? error.message : String(error));
|
|
process.exitCode = 1;
|
|
}
|
|
}
|