perf(ci): start source feedback early and diagnose stalled runs (#3013)
* perf(ci): start source feedback early and diagnose stalled runs * fix(ci): verify workflow identities before main comparison limits * fix(ci): preserve diagnostics across passing checks and install failures * fix(ci): preserve ambiguous workflow run ordering * fix(ci): preserve failure-report candidate and request budgets * fix(ci): retain candidate errors while checking later main evidence * fix(ci): retain partial reports and require complete health scans * test(ci): prune stale fixtures and duplicated assumptions
This commit is contained in:
morluto
GitHub
parent
aa6f74af04
commit
6510d0d57f
|
|
@ -12,12 +12,13 @@ concurrency:
|
|||
permissions:
|
||||
contents: read
|
||||
actions: read
|
||||
pull-requests: read
|
||||
|
||||
jobs:
|
||||
report:
|
||||
name: Collect recent completed workflow attempts
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 5
|
||||
timeout-minutes: 10
|
||||
steps:
|
||||
# Consume only API data and execute batch code from the trusted default branch.
|
||||
- uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0
|
||||
|
|
@ -27,6 +28,18 @@ jobs:
|
|||
- uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4.4.0
|
||||
with:
|
||||
node-version-file: .nvmrc
|
||||
- name: Cache npm package downloads
|
||||
id: npm-cache
|
||||
continue-on-error: true
|
||||
uses: ./.github/actions/npm-cache
|
||||
- name: Diagnose missing and startup-failed PR CI
|
||||
continue-on-error: true
|
||||
env:
|
||||
GH_TOKEN: ${{ github.token }}
|
||||
CI_REPOSITORY: ${{ github.repository }}
|
||||
CI_REPOSITORY_ID: ${{ github.repository_id }}
|
||||
CI_DEFAULT_BRANCH: ${{ github.event.repository.default_branch }}
|
||||
run: node scripts/ci/pr-ci-health.mjs
|
||||
- name: Record queue, execution, and matched step timing
|
||||
continue-on-error: true
|
||||
env:
|
||||
|
|
@ -44,3 +57,28 @@ jobs:
|
|||
if-no-files-found: error
|
||||
overwrite: true
|
||||
retention-days: 14
|
||||
- name: Install dependencies for static failure attribution
|
||||
id: failure-deps
|
||||
if: ${{ !cancelled() && hashFiles('.cache/ci-job-timings/batch-manifest.json') != '' }}
|
||||
continue-on-error: true
|
||||
run: npm ci
|
||||
- name: Attribute supported static failures against main
|
||||
if: ${{ !cancelled() && steps.failure-deps.outcome == 'success' && hashFiles('.cache/ci-job-timings/batch-manifest.json') != '' }}
|
||||
continue-on-error: true
|
||||
env:
|
||||
GH_TOKEN: ${{ github.token }}
|
||||
CI_REPOSITORY: ${{ github.repository }}
|
||||
CI_DEFAULT_BRANCH: ${{ github.event.repository.default_branch }}
|
||||
run: node scripts/ci/failure-report.mjs
|
||||
- name: Upload CI health and failure diagnostics
|
||||
if: ${{ !cancelled() }}
|
||||
continue-on-error: true
|
||||
uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2
|
||||
with:
|
||||
name: ci-health-diagnostics
|
||||
path: |
|
||||
.cache/ci-pr-health/
|
||||
.cache/ci-failures/
|
||||
include-hidden-files: true
|
||||
if-no-files-found: warn
|
||||
retention-days: 14
|
||||
|
|
|
|||
|
|
@ -139,6 +139,12 @@ jobs:
|
|||
- name: Validate workflow syntax and expressions
|
||||
if: needs.changes.outputs.workflows == 'true'
|
||||
run: go run github.com/rhysd/actionlint/cmd/actionlint@v1.7.7 -shellcheck= -pyflakes=
|
||||
- name: Record static task source and execution identity
|
||||
continue-on-error: true
|
||||
if: needs.changes.outputs.code == 'true'
|
||||
env:
|
||||
CI_PR_HEAD_SHA: ${{ github.event.pull_request.head.sha }}
|
||||
run: npm run ci:static-context
|
||||
- if: needs.changes.outputs.code == 'true'
|
||||
run: npm run check:ci
|
||||
- if: needs.changes.outputs.code == 'true'
|
||||
|
|
@ -167,6 +173,34 @@ jobs:
|
|||
if-no-files-found: ignore
|
||||
retention-days: 7
|
||||
|
||||
test-source:
|
||||
name: Source tests
|
||||
needs: changes
|
||||
if: needs.changes.outputs.code == 'true' && needs.changes.outputs.tier == 'fast'
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 10
|
||||
steps:
|
||||
- uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0
|
||||
- uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4.4.0
|
||||
with:
|
||||
node-version-file: .nvmrc
|
||||
- name: Cache npm package downloads
|
||||
id: npm-cache
|
||||
uses: ./.github/actions/npm-cache
|
||||
- run: npm ci
|
||||
- name: Run source tests without waiting for build
|
||||
run: npm run test:fast -- --reporter=default --reporter=blob
|
||||
env:
|
||||
VITEST_BLOB_LABEL: source
|
||||
- uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2
|
||||
if: ${{ !cancelled() }}
|
||||
with:
|
||||
name: vitest-report-source
|
||||
path: .vitest/blob/*
|
||||
include-hidden-files: true
|
||||
if-no-files-found: error
|
||||
retention-days: 7
|
||||
|
||||
test-shard:
|
||||
name: Test shard (${{ matrix.shard }})
|
||||
needs: [changes, build]
|
||||
|
|
@ -178,7 +212,9 @@ jobs:
|
|||
# every shard's diagnostics and coverage. Passing PRs run the full suite.
|
||||
fail-fast: ${{ github.event_name == 'pull_request' && needs.changes.outputs.tier == 'fast' }}
|
||||
matrix:
|
||||
shard: [1/6, 2/6, 3/6, 4/6, 5/6, 6/6]
|
||||
# Fast feedback spends the same six test jobs on one source group and
|
||||
# five built groups; full qualification retains six complete shards.
|
||||
shard: ${{ fromJSON(needs.changes.outputs.tier == 'fast' && '["1/5","2/5","3/5","4/5","5/5"]' || '["1/6","2/6","3/6","4/6","5/6","6/6"]') }}
|
||||
steps:
|
||||
- uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0
|
||||
- uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4.4.0
|
||||
|
|
@ -197,15 +233,16 @@ jobs:
|
|||
run: npm run test:ci:shard:run -- --shard=${{ matrix.shard }}
|
||||
env:
|
||||
REA_CI_TEST_HISTORY: .cache/ci-test-history/test-results.json
|
||||
- name: Run complete portable suite for fast feedback
|
||||
- name: Run built-dependent tests for fast feedback
|
||||
if: needs.changes.outputs.tier == 'fast'
|
||||
run: npm run test:run -- --reporter=default --reporter=blob --shard=${{ matrix.shard }}
|
||||
run: npm run test:built -- --reporter=default --reporter=blob --shard=${{ matrix.shard }}
|
||||
env:
|
||||
VITEST_BLOB_LABEL: built
|
||||
REA_CI_TEST_HISTORY: .cache/ci-test-history/test-results.json
|
||||
- uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2
|
||||
if: ${{ !cancelled() }}
|
||||
with:
|
||||
name: vitest-report-${{ strategy.job-index }}
|
||||
name: vitest-report-built-${{ strategy.job-index }}
|
||||
path: .vitest/blob/*
|
||||
include-hidden-files: true
|
||||
if-no-files-found: error
|
||||
|
|
@ -219,11 +256,13 @@ jobs:
|
|||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 60
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
- uses: actions/setup-node@v4
|
||||
- uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0
|
||||
- uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4.4.0
|
||||
with:
|
||||
node-version-file: .nvmrc
|
||||
cache: npm
|
||||
- name: Cache npm package downloads
|
||||
id: npm-cache
|
||||
uses: ./.github/actions/npm-cache
|
||||
- run: npm ci
|
||||
- name: Run reverse tools tests with coverage
|
||||
run: npm run test:coverage:reverse-tools
|
||||
|
|
@ -316,7 +355,7 @@ jobs:
|
|||
|
||||
test:
|
||||
name: Portable tests
|
||||
needs: [changes, test-shard]
|
||||
needs: [changes, test-source, test-shard]
|
||||
if: ${{ !cancelled() && needs.changes.outputs.tier != 'reused' }}
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 10
|
||||
|
|
@ -340,7 +379,9 @@ jobs:
|
|||
- uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0
|
||||
if: >-
|
||||
!cancelled() && needs.changes.outputs.code == 'true' &&
|
||||
(needs.test-shard.result == 'success' || needs.test-shard.result == 'failure')
|
||||
(needs.test-shard.result == 'success' || needs.test-shard.result == 'failure' ||
|
||||
(needs.changes.outputs.tier == 'fast' &&
|
||||
(needs.test-source.result == 'success' || needs.test-source.result == 'failure')))
|
||||
with:
|
||||
path: .vitest/blob
|
||||
pattern: vitest-report-*
|
||||
|
|
@ -368,7 +409,8 @@ jobs:
|
|||
if: >-
|
||||
!cancelled() && needs.changes.outputs.code == 'true' &&
|
||||
needs.changes.outputs.tier == 'fast' &&
|
||||
(needs.test-shard.result == 'success' || needs.test-shard.result == 'failure')
|
||||
(needs.test-shard.result == 'success' || needs.test-shard.result == 'failure' ||
|
||||
needs.test-source.result == 'success' || needs.test-source.result == 'failure')
|
||||
run: >-
|
||||
npm exec -- vitest --merge-reports
|
||||
--reporter=default
|
||||
|
|
@ -381,6 +423,8 @@ jobs:
|
|||
if: needs.changes.outputs.code == 'true' && !cancelled() && hashFiles('test-timings.json') != ''
|
||||
env:
|
||||
SHARD_RESULT: ${{ needs.test-shard.result }}
|
||||
SOURCE_REQUIRED: ${{ needs.changes.outputs.tier == 'fast' }}
|
||||
SOURCE_RESULT: ${{ needs.test-source.result }}
|
||||
MERGE_RESULT: ${{ needs.changes.outputs.tier == 'full' && steps.merge.outcome || steps.merge-fast.outcome }}
|
||||
CI_VERIFY_TEST_INVENTORY: "true"
|
||||
CI_TEST_SELECTION: ${{ needs.changes.outputs.test_selection }}
|
||||
|
|
@ -413,7 +457,13 @@ jobs:
|
|||
env:
|
||||
CODE_REQUIRED: ${{ needs.changes.outputs.code }}
|
||||
SHARD_RESULT: ${{ needs.test-shard.result }}
|
||||
run: test "${CODE_REQUIRED}" != "true" || test "${SHARD_RESULT}" = "success"
|
||||
SOURCE_REQUIRED: ${{ needs.changes.outputs.tier == 'fast' }}
|
||||
SOURCE_RESULT: ${{ needs.test-source.result }}
|
||||
run: |
|
||||
if [ "${CODE_REQUIRED}" = "true" ]; then
|
||||
test "${SHARD_RESULT}" = "success" || exit 1
|
||||
test "${SOURCE_REQUIRED}" != "true" || test "${SOURCE_RESULT}" = "success"
|
||||
fi
|
||||
|
||||
conformance-fixtures:
|
||||
name: Source-built conformance fixtures
|
||||
|
|
@ -847,6 +897,7 @@ jobs:
|
|||
build,
|
||||
reverse-tools-quality,
|
||||
test-shard,
|
||||
test-source,
|
||||
sqlite-runtime,
|
||||
test,
|
||||
conformance-fixtures,
|
||||
|
|
|
|||
|
|
@ -13,8 +13,8 @@ jobs:
|
|||
name: Build Windows bridge security boundary
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
- uses: actions/setup-node@v4
|
||||
- uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0
|
||||
- uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4.4.0
|
||||
with:
|
||||
node-version-file: .nvmrc
|
||||
- name: Install the cross compiler used by the Windows native lane
|
||||
|
|
@ -44,11 +44,13 @@ jobs:
|
|||
- platform: Windows
|
||||
runner: [self-hosted, cutter-upstream, Windows]
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
- uses: actions/setup-node@v4
|
||||
- uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0
|
||||
- uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4.4.0
|
||||
with:
|
||||
node-version-file: .nvmrc
|
||||
cache: npm
|
||||
- name: Cache npm package downloads
|
||||
id: npm-cache
|
||||
uses: ./.github/actions/npm-cache
|
||||
- run: npm ci
|
||||
- if: matrix.platform == 'Windows'
|
||||
uses: actions/download-artifact@v4
|
||||
|
|
|
|||
|
|
@ -280,8 +280,15 @@ focused suite also exercises affected MCP routes and the Python Cutter bridge.
|
|||
Contract, integration, and real-provider lanes complement these focused tests;
|
||||
simulated providers do not establish real-provider behavior.
|
||||
|
||||
CI runs six native Vitest shards without retries. Each shard emits a blob
|
||||
report; the merge job produces JUnit and JSON diagnostics, a module timing
|
||||
Fast CI starts the source-only projects after classification, independently of
|
||||
build and static checks, and runs five built-dependent shards after the build.
|
||||
The six test jobs still select every portable test. Full qualification keeps six
|
||||
complete Vitest shards with coverage. Neither tier retries tests. Canonical
|
||||
project ownership lives in `scripts/lib/test-projects.mjs`, shared by Vitest,
|
||||
CI group execution, and development-test preparation. A test importing compiled
|
||||
verifier code belongs to the built group even if its directory is conformance.
|
||||
Source and built blobs have separate labels to prevent merge collisions.
|
||||
Each test job emits a blob report; the merge job produces JUnit and JSON diagnostics, a module timing
|
||||
report, plus aggregate coverage during full qualification. It retains available
|
||||
reports even when a shard fails, uploads available diagnostics, and writes the
|
||||
slowest files to the workflow summary. Partial reports are marked diagnostic
|
||||
|
|
@ -316,6 +323,30 @@ portable test shard. These limits leave headroom for cold setup and full coverag
|
|||
while preventing a hung job from holding the active main run for hours. They do
|
||||
not bound runner queue waiting. Timeouts still fail the required gate.
|
||||
|
||||
The separate `CI timings` workflow also diagnoses missing, startup-failed,
|
||||
queued/running, approval-required, canceled, and failed PR CI through bounded,
|
||||
read-only API scans. Incomplete pagination or changing identity stays unknown.
|
||||
It never closes/reopens PRs, reruns workflows, posts comments, or changes gates.
|
||||
The timing manifest retains each run's event and head branch, including explicit
|
||||
unknowns, so PR runs cannot consume the eligible-main comparison budget.
|
||||
Its static failure report compares supported lint/duplication diagnostics with
|
||||
failed main runs using exact source, ancestry, task hashes, and toolchain proof.
|
||||
A match describes those diagnostics only; it does not establish that every
|
||||
failure in a job came from main. Duplication reports retain the actual checker
|
||||
exit result, so passing duplication checks preserve lint failures and caller
|
||||
thresholds remain owned by the producer. Missing checker outcomes leave
|
||||
those duplication diagnostics unclassified. Missing historical receipts remain unknown.
|
||||
The reporter executes only trusted default-branch code and parses whitelisted
|
||||
JSON artifact members without executing artifact content. Reports and limits
|
||||
are retained in `ci-health-diagnostics`; its ten-minute job limit covers the
|
||||
existing four-minute timing collection plus the bounded diagnostic stages.
|
||||
|
||||
Shard-count reduction remains deferred: the available successful timing sample
|
||||
covers an older inventory, and file phase weights are not job wall time. The
|
||||
one-source/five-built split retains six test jobs; it makes no hosted speedup
|
||||
claim. Current complete timings and completed candidate hosted runs are needed
|
||||
before enabling a smaller adaptive matrix.
|
||||
|
||||
The separate `CI timings` workflow batches recent completed CI and Release runs
|
||||
hourly or on manual dispatch. It uploads `ci-job-timings-batch` with raw API pages and
|
||||
a normalized point-in-time report for each collected workflow attempt, plus a
|
||||
|
|
|
|||
|
|
@ -118,7 +118,9 @@
|
|||
"precheck:pr": "npm run deps:check",
|
||||
"check:pr": "node scripts/run-exclusive.mjs artifacts turbo run typecheck lint:built format:check knip metadata:check test:run && npm run test:python && npm run docs:check && npm run jscpd && npm run scan:todos",
|
||||
"precheck:ci": "npm run deps:check",
|
||||
"check:ci": "node scripts/run-exclusive.mjs artifacts turbo run typecheck lint:built format:check knip metadata:check jscpd scan:todos && npm run verify:architecture-guards",
|
||||
"check:ci": "node scripts/run-exclusive.mjs artifacts npm run check:ci:tasks && npm run verify:architecture-guards",
|
||||
"check:ci:tasks": "turbo run typecheck lint:built format:check knip metadata:check jscpd scan:todos",
|
||||
"ci:static-context": "node scripts/ci/static-task-context.mjs",
|
||||
"build:cached": "node scripts/check-dependency-install.mjs && node scripts/run-exclusive.mjs artifacts turbo run artifacts:skills",
|
||||
"build:termux": "node scripts/run-exclusive.mjs artifacts npm run build:termux:run",
|
||||
"build:termux:run": "npm run compile:termux && npm run artifacts:skills",
|
||||
|
|
@ -138,7 +140,8 @@
|
|||
"test:run": "vitest run",
|
||||
"test:python": "python3 -m unittest bridge/native/test_rea_lldb_tracer.py -v && python3 -m unittest discover -s experiments/analysis-efficiency -p 'test_*.py' -v",
|
||||
"test:local": "node scripts/run-development-tests.mjs local",
|
||||
"test:fast": "vitest run --project domain --project services --project adapters --project composition --project conformance --project evaluation",
|
||||
"test:fast": "node scripts/ci/test-group.mjs source",
|
||||
"test:built": "node scripts/ci/test-group.mjs built",
|
||||
"pretest:boundary": "npm run test:prepare",
|
||||
"test:boundary": "vitest run --project boundary --project cli-boundary --project process-boundary --project mcp-boundary --project process-global",
|
||||
"pretest:mcp": "node scripts/check-dependency-install.mjs && node scripts/run-exclusive.mjs artifacts turbo run artifacts:skills artifacts:mcp-catalog",
|
||||
|
|
|
|||
|
|
@ -0,0 +1,67 @@
|
|||
import type { NormalizedWorkflowRun } from "./workflow-run.mjs";
|
||||
import type { StaticTaskContext } from "./static-task-context.mjs";
|
||||
export type { StaticTaskContext } from "./static-task-context.mjs";
|
||||
|
||||
export interface StaticDiagnostic {
|
||||
readonly kind: "lint" | "jscpd";
|
||||
readonly path?: string;
|
||||
readonly paths?: readonly string[];
|
||||
readonly code?: string;
|
||||
readonly message?: string;
|
||||
readonly severity?: string;
|
||||
}
|
||||
|
||||
export function classifyFailure(input: {
|
||||
repository: string;
|
||||
defaultBranch: string;
|
||||
run: Record<string, unknown> & NormalizedWorkflowRun;
|
||||
job: Record<string, unknown>;
|
||||
diagnostics: readonly StaticDiagnostic[];
|
||||
context: StaticTaskContext;
|
||||
mainRun: Record<string, unknown> & NormalizedWorkflowRun;
|
||||
mainJob: Record<string, unknown>;
|
||||
mainDiagnostics: readonly StaticDiagnostic[];
|
||||
mainContext: StaticTaskContext;
|
||||
executionSourcesVerified: boolean;
|
||||
ancestry: { status: string; mergeBaseSha: string | null };
|
||||
blobs: Readonly<Record<string, boolean>>;
|
||||
}): {
|
||||
classification: "matching-main-diagnostics" | "unknown";
|
||||
reason: string;
|
||||
};
|
||||
|
||||
export function diagnosticPaths(item: StaticDiagnostic): readonly string[];
|
||||
export function isFailedConclusion(
|
||||
value: unknown,
|
||||
): value is "failure" | "timed_out";
|
||||
export function isEligibleMainRun(
|
||||
run: Record<string, unknown>,
|
||||
defaultBranch: string,
|
||||
repository: string,
|
||||
): boolean;
|
||||
export function pullRequestBaseSha(
|
||||
run: Record<string, unknown>,
|
||||
repository: string,
|
||||
): string | null;
|
||||
export function safeSha(value: unknown): string | null;
|
||||
export function validRepoPath(value: unknown): value is string;
|
||||
export function selectStaticCheckArtifact(
|
||||
page: unknown,
|
||||
run: { id?: string | number; head_sha?: string },
|
||||
runAttempt: number,
|
||||
): unknown | null;
|
||||
|
||||
export function readStaticCheckArtifact(options: {
|
||||
repository: string;
|
||||
artifactId: number | string;
|
||||
token: string;
|
||||
fetchImpl?: typeof fetch;
|
||||
}): Promise<{
|
||||
context: StaticTaskContext;
|
||||
diagnostics: readonly StaticDiagnostic[];
|
||||
}>;
|
||||
|
||||
export function parseStaticArchive(archivePath: string): Promise<{
|
||||
context: StaticTaskContext;
|
||||
diagnostics: readonly StaticDiagnostic[];
|
||||
}>;
|
||||
|
|
@ -0,0 +1,436 @@
|
|||
import { execFile } from "node:child_process";
|
||||
import { mkdtemp, rm, writeFile } from "node:fs/promises";
|
||||
import { tmpdir } from "node:os";
|
||||
import { isAbsolute, join } from "node:path";
|
||||
import { promisify } from "node:util";
|
||||
|
||||
import { parseStaticTaskContext } from "./static-task-context.mjs";
|
||||
|
||||
const execute = promisify(execFile);
|
||||
const MAX_ARCHIVE_BYTES = 32 * 1024 * 1024;
|
||||
const MAX_MEMBER_BYTES = 4 * 1024 * 1024;
|
||||
const MAX_DIAGNOSTICS = 10_000;
|
||||
const API_ROOT = "https://api.github.com/repos";
|
||||
const SHA = /^[0-9a-f]{40}$/iu;
|
||||
const TASK_HASH = /^[a-f0-9]{16}$/u;
|
||||
|
||||
/** Compare parsed static diagnostics only after source, task, host, and ancestry proof. */
|
||||
export function classifyFailure({
|
||||
repository,
|
||||
defaultBranch,
|
||||
run,
|
||||
job,
|
||||
diagnostics,
|
||||
context,
|
||||
mainRun,
|
||||
mainJob,
|
||||
mainDiagnostics,
|
||||
mainContext,
|
||||
executionSourcesVerified,
|
||||
ancestry,
|
||||
blobs,
|
||||
}) {
|
||||
const failedGate = firstFailure([
|
||||
() => [
|
||||
!isCompletedFailure(run) ||
|
||||
job?.status !== "completed" ||
|
||||
!isFailedConclusion(job?.conclusion),
|
||||
"run_or_job_not_a_completed_failure",
|
||||
],
|
||||
() => [
|
||||
!isEligiblePullRun(run, repository) ||
|
||||
!isEligibleMainRun(mainRun, defaultBranch, repository),
|
||||
"run_event_or_repository_identity_unverified",
|
||||
],
|
||||
() => [
|
||||
job?.name !== mainJob?.name || job?.name !== "Static checks",
|
||||
"failed_job_not_the_same_static_gate",
|
||||
],
|
||||
() => [
|
||||
!Array.isArray(diagnostics) ||
|
||||
diagnostics.length === 0 ||
|
||||
!Array.isArray(mainDiagnostics),
|
||||
"structured_diagnostics_missing_or_invalid",
|
||||
],
|
||||
() => [
|
||||
!validContext(context) || !validContext(mainContext),
|
||||
"static_task_context_missing_or_invalid",
|
||||
],
|
||||
() => [
|
||||
executionSourcesVerified !== true,
|
||||
"checkout_sources_not_verified_against_api_commits",
|
||||
],
|
||||
() => [
|
||||
!sameExecutionContext(context, mainContext),
|
||||
"task_or_toolchain_identity_differs",
|
||||
],
|
||||
() => [
|
||||
ancestry?.status !== "ahead" && ancestry?.status !== "identical",
|
||||
"main_head_not_proven_ancestor_of_tested_base",
|
||||
],
|
||||
() => [
|
||||
ancestry?.mergeBaseSha !== mainRun?.head_sha,
|
||||
"main_head_is_not_the_verified_merge_base",
|
||||
],
|
||||
() => [
|
||||
!sameTaskHashes(diagnostics, context, mainContext),
|
||||
"owning_task_hash_differs",
|
||||
],
|
||||
]);
|
||||
if (failedGate) return outcome("unknown", failedGate.reason);
|
||||
const keys = diagnostics.map(diagnosticKey);
|
||||
const mainKeys = new Set(mainDiagnostics.map(diagnosticKey).filter(Boolean));
|
||||
if (keys.some((key) => key === null))
|
||||
return outcome("unknown", "diagnostic_shape_not_supported");
|
||||
if (!keys.every((key) => mainKeys.has(key)))
|
||||
return outcome(
|
||||
"unknown",
|
||||
"main_run_does_not_contain_the_same_structured_diagnostic",
|
||||
);
|
||||
if (!sameBlobEvidence(diagnostics, blobs))
|
||||
return outcome(
|
||||
"unknown",
|
||||
"diagnostic_source_content_not_identical_at_main_and_tested_base",
|
||||
);
|
||||
return outcome(
|
||||
"matching-main-diagnostics",
|
||||
"supported_diagnostics_match_failed_task_and_source_on_verified_main_ancestor",
|
||||
);
|
||||
}
|
||||
|
||||
export function diagnosticPaths(item) {
|
||||
return item.kind === "jscpd" ? item.paths : [item.path];
|
||||
}
|
||||
export function isFailedConclusion(value) {
|
||||
return value === "failure" || value === "timed_out";
|
||||
}
|
||||
export function isEligibleMainRun(run, defaultBranch, repository) {
|
||||
return (
|
||||
isEligibleStaticRun(run) &&
|
||||
run.repository.full_name === repository &&
|
||||
run.head_branch === defaultBranch &&
|
||||
["push", "schedule", "workflow_dispatch"].includes(run.event)
|
||||
);
|
||||
}
|
||||
export function pullRequestBaseSha(run, repository) {
|
||||
if (
|
||||
run?.event !== "pull_request" ||
|
||||
run.repository?.full_name !== repository ||
|
||||
!Array.isArray(run.pull_requests) ||
|
||||
run.pull_requests.length !== 1
|
||||
)
|
||||
return null;
|
||||
const base = run.pull_requests[0].base;
|
||||
return base?.repo?.full_name === repository && SHA.test(base?.sha ?? "")
|
||||
? base.sha
|
||||
: null;
|
||||
}
|
||||
|
||||
export function safeSha(value) {
|
||||
return typeof value === "string" && SHA.test(value)
|
||||
? value.toLowerCase()
|
||||
: null;
|
||||
}
|
||||
export function selectStaticCheckArtifact(page, run, runAttempt) {
|
||||
if (runAttempt !== 1) return null;
|
||||
if (
|
||||
!Array.isArray(page?.artifacts) ||
|
||||
!Number.isSafeInteger(page.total_count) ||
|
||||
page.total_count !== page.artifacts.length ||
|
||||
page.artifacts.length >= 100
|
||||
)
|
||||
return null;
|
||||
const matches = page.artifacts.filter(
|
||||
(artifact) => artifact.name === "static-check-reports",
|
||||
);
|
||||
if (matches.length !== 1) return null;
|
||||
const [artifact] = matches;
|
||||
if (
|
||||
artifact.expired !== false ||
|
||||
String(artifact.workflow_run?.id) !== String(run.id) ||
|
||||
artifact.workflow_run?.head_sha !== run.head_sha
|
||||
)
|
||||
return null;
|
||||
return artifact;
|
||||
}
|
||||
export function validRepoPath(value) {
|
||||
return (
|
||||
typeof value === "string" &&
|
||||
value.length > 0 &&
|
||||
!isAbsolute(value) &&
|
||||
!value.includes("\\") &&
|
||||
value
|
||||
.split("/")
|
||||
.every((part) => part !== "" && part !== "." && part !== "..") &&
|
||||
!value.startsWith("/")
|
||||
);
|
||||
}
|
||||
|
||||
/** Download one named static report archive and parse only whitelisted JSON members. */
|
||||
export async function readStaticCheckArtifact({
|
||||
repository,
|
||||
artifactId,
|
||||
token,
|
||||
fetchImpl = fetch,
|
||||
}) {
|
||||
if (!/^[A-Za-z0-9_.-]+\/[A-Za-z0-9_.-]+$/u.test(repository ?? ""))
|
||||
throw new Error("artifact repository identity is invalid");
|
||||
if (!token) throw new Error("artifact download requires a GitHub token");
|
||||
const url = `${API_ROOT}/${repository}/actions/artifacts/${positiveId(artifactId)}/zip`;
|
||||
let response = await fetchImpl(url, {
|
||||
headers: {
|
||||
Accept: "application/vnd.github+json",
|
||||
Authorization: `Bearer ${token}`,
|
||||
"X-GitHub-Api-Version": "2022-11-28",
|
||||
},
|
||||
signal: AbortSignal.timeout(10_000),
|
||||
redirect: "manual",
|
||||
});
|
||||
if ([301, 302, 303, 307, 308].includes(response.status)) {
|
||||
const location = response.headers.get("location");
|
||||
if (!location || new URL(location).protocol !== "https:")
|
||||
throw new Error("static_artifact_redirect_invalid");
|
||||
response = await fetchImpl(location, {
|
||||
signal: AbortSignal.timeout(10_000),
|
||||
redirect: "error",
|
||||
});
|
||||
}
|
||||
if (!response.ok) throw new Error(`github_http_${response.status}`);
|
||||
const bytes = await readLimitedBytes(response, MAX_ARCHIVE_BYTES);
|
||||
const directory = await mkdtemp(join(tmpdir(), "rea-ci-static-artifact-"));
|
||||
const archive = join(directory, "artifact.zip");
|
||||
try {
|
||||
await writeFile(archive, bytes, { flag: "wx" });
|
||||
return await parseStaticArchive(archive);
|
||||
} finally {
|
||||
await rm(directory, { recursive: true, force: true });
|
||||
}
|
||||
}
|
||||
|
||||
/** Accept only the three named producer outputs; no archive paths are written to disk. */
|
||||
export async function parseStaticArchive(archivePath) {
|
||||
const { stdout: listing } = await execute("unzip", ["-Z1", archivePath], {
|
||||
encoding: "utf8",
|
||||
maxBuffer: 128 * 1024,
|
||||
timeout: 5_000,
|
||||
});
|
||||
const allowed = new Set([
|
||||
"oxlint.json",
|
||||
"jscpd/jscpd-report.json",
|
||||
"static-task-context.json",
|
||||
]);
|
||||
const members = new Map();
|
||||
for (const name of listing.split(/\r?\n/u).filter(Boolean)) {
|
||||
const prefix = ".cache/static-checks/";
|
||||
const member = name.startsWith(prefix) ? name.slice(prefix.length) : name;
|
||||
if (!allowed.has(member)) continue;
|
||||
if (members.has(member))
|
||||
throw new Error("static artifact contains duplicate report members");
|
||||
const { stdout } = await execute("unzip", ["-p", archivePath, name], {
|
||||
encoding: "buffer",
|
||||
maxBuffer: MAX_MEMBER_BYTES,
|
||||
timeout: 5_000,
|
||||
});
|
||||
members.set(member, JSON.parse(stdout.toString("utf8")));
|
||||
}
|
||||
const context = parseStaticTaskContext(
|
||||
members.get("static-task-context.json"),
|
||||
);
|
||||
const diagnostics = normalizeDiagnostics(
|
||||
members.get("oxlint.json"),
|
||||
members.get("jscpd/jscpd-report.json"),
|
||||
);
|
||||
if (diagnostics.length > MAX_DIAGNOSTICS)
|
||||
throw new Error("static diagnostic count exceeds the reporting limit");
|
||||
return { context, diagnostics };
|
||||
}
|
||||
|
||||
function normalizeDiagnostics(lint, duplicate) {
|
||||
return [...normalizeLint(lint), ...normalizeDuplicates(duplicate)];
|
||||
}
|
||||
|
||||
function normalizeLint(report) {
|
||||
if (report === undefined) return [];
|
||||
if (!Array.isArray(report?.diagnostics))
|
||||
throw new Error("oxlint diagnostics are malformed");
|
||||
return report.diagnostics
|
||||
.filter((item) => item.severity === "error")
|
||||
.map((item) => {
|
||||
if (
|
||||
typeof item.filename !== "string" ||
|
||||
!repoPath(item.filename) ||
|
||||
typeof item.code !== "string" ||
|
||||
typeof item.message !== "string"
|
||||
)
|
||||
throw new Error("oxlint error diagnostic is malformed");
|
||||
return {
|
||||
kind: "lint",
|
||||
path: item.filename,
|
||||
code: item.code,
|
||||
message: item.message,
|
||||
severity: "error",
|
||||
};
|
||||
});
|
||||
}
|
||||
|
||||
function normalizeDuplicates(report) {
|
||||
if (report === undefined) return [];
|
||||
if (
|
||||
!Array.isArray(report?.duplicates) ||
|
||||
!Number.isFinite(report.statistics?.total?.percentage) ||
|
||||
report.statistics.total.percentage < 0
|
||||
)
|
||||
throw new Error("jscpd report is malformed");
|
||||
// The producer owns the checker outcome; do not copy or infer its threshold.
|
||||
const exitCode = report.reaCheck?.exitCode;
|
||||
if (exitCode === undefined || exitCode === 0) return [];
|
||||
if (!Number.isSafeInteger(exitCode) || exitCode < 0)
|
||||
throw new Error("jscpd checker outcome is malformed");
|
||||
if (exitCode !== 1) return [];
|
||||
return report.duplicates.map((item) => {
|
||||
const paths = [item.firstFile?.name, item.secondFile?.name].sort();
|
||||
if (paths.some((path) => !repoPath(path)))
|
||||
throw new Error("jscpd duplicate paths are malformed");
|
||||
return { kind: "jscpd", paths };
|
||||
});
|
||||
}
|
||||
|
||||
async function readLimitedBytes(response, limit) {
|
||||
const reader = response.body?.getReader();
|
||||
if (!reader) throw new Error("static artifact response has no body");
|
||||
const chunks = [];
|
||||
let size = 0;
|
||||
try {
|
||||
for (;;) {
|
||||
const { done, value } = await reader.read();
|
||||
if (done) break;
|
||||
size += value.byteLength;
|
||||
if (size > limit) {
|
||||
await reader.cancel();
|
||||
throw new Error("static artifact exceeds the download limit");
|
||||
}
|
||||
chunks.push(value);
|
||||
}
|
||||
} finally {
|
||||
reader.releaseLock();
|
||||
}
|
||||
const result = new Uint8Array(size);
|
||||
let offset = 0;
|
||||
for (const chunk of chunks) {
|
||||
result.set(chunk, offset);
|
||||
offset += chunk.byteLength;
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
function repoPath(value) {
|
||||
return (
|
||||
typeof value === "string" &&
|
||||
value.length > 0 &&
|
||||
!value.startsWith("/") &&
|
||||
!value.includes("\\") &&
|
||||
value
|
||||
.split("/")
|
||||
.every((part) => part !== "" && part !== "." && part !== "..")
|
||||
);
|
||||
}
|
||||
|
||||
function positiveId(value) {
|
||||
const id = Number(value);
|
||||
if (!Number.isSafeInteger(id) || id < 1)
|
||||
throw new Error("artifact ID must be a positive integer");
|
||||
return String(id);
|
||||
}
|
||||
|
||||
function diagnosticKey(item) {
|
||||
if (
|
||||
item?.kind === "lint" &&
|
||||
validRepoPath(item.path) &&
|
||||
typeof item.code === "string" &&
|
||||
typeof item.message === "string" &&
|
||||
item.severity === "error"
|
||||
)
|
||||
return JSON.stringify([
|
||||
item.kind,
|
||||
item.path,
|
||||
item.code,
|
||||
item.message,
|
||||
item.severity,
|
||||
]);
|
||||
if (
|
||||
item?.kind === "jscpd" &&
|
||||
Array.isArray(item.paths) &&
|
||||
item.paths.length === 2 &&
|
||||
item.paths.every(validRepoPath)
|
||||
)
|
||||
return JSON.stringify([item.kind, ...[...item.paths].sort()]);
|
||||
return null;
|
||||
}
|
||||
function sameBlobEvidence(diagnostics, blobs) {
|
||||
const paths = [...new Set(diagnostics.flatMap(diagnosticPaths))];
|
||||
return paths.length > 0 && paths.every((path) => blobs?.[path] === true);
|
||||
}
|
||||
function sameTaskHashes(diagnostics, left, right) {
|
||||
return diagnostics.every((diagnostic) => {
|
||||
const task = diagnostic.kind === "jscpd" ? "jscpd" : "lint:built";
|
||||
return (
|
||||
TASK_HASH.test(left.taskHashes[task] ?? "") &&
|
||||
left.taskHashes[task] === right.taskHashes[task]
|
||||
);
|
||||
});
|
||||
}
|
||||
function validContext(value) {
|
||||
return (
|
||||
value?.version === 1 &&
|
||||
SHA.test(value.sourceSha ?? "") &&
|
||||
value.sourceClean === true &&
|
||||
typeof value.nodeVersion === "string" &&
|
||||
typeof value.npmVersion === "string" &&
|
||||
typeof value.turboVersion === "string" &&
|
||||
typeof value.platform === "string" &&
|
||||
typeof value.architecture === "string" &&
|
||||
value.taskHashes &&
|
||||
typeof value.taskHashes === "object" &&
|
||||
!Array.isArray(value.taskHashes)
|
||||
);
|
||||
}
|
||||
function sameExecutionContext(left, right) {
|
||||
return [
|
||||
"nodeVersion",
|
||||
"npmVersion",
|
||||
"turboVersion",
|
||||
"platform",
|
||||
"architecture",
|
||||
].every((key) => left[key] === right[key]);
|
||||
}
|
||||
function isEligiblePullRun(run, repository) {
|
||||
return (
|
||||
isEligibleStaticRun(run) && pullRequestBaseSha(run, repository) !== null
|
||||
);
|
||||
}
|
||||
function isEligibleStaticRun(run) {
|
||||
return (
|
||||
isRecord(run) &&
|
||||
run.status === "completed" &&
|
||||
isFailedConclusion(run.conclusion) &&
|
||||
SHA.test(run.head_sha ?? "") &&
|
||||
typeof run.repository?.full_name === "string" &&
|
||||
run.workflowPath?.file === ".github/workflows/ci.yml"
|
||||
);
|
||||
}
|
||||
function isCompletedFailure(run) {
|
||||
return run?.status === "completed" && isFailedConclusion(run.conclusion);
|
||||
}
|
||||
function isRecord(value) {
|
||||
return typeof value === "object" && value !== null && !Array.isArray(value);
|
||||
}
|
||||
function firstFailure(checks) {
|
||||
for (const check of checks) {
|
||||
const [failed, reason] = check();
|
||||
if (failed) return { reason };
|
||||
}
|
||||
return null;
|
||||
}
|
||||
function outcome(classification, reason) {
|
||||
return { classification, reason };
|
||||
}
|
||||
|
|
@ -0,0 +1,159 @@
|
|||
import type { NormalizedWorkflowRun } from "./workflow-run.mjs";
|
||||
import type { StaticDiagnostic } from "./artifacts.mjs";
|
||||
export type { StaticDiagnostic } from "./artifacts.mjs";
|
||||
import type { StaticTaskContext } from "./static-task-context.mjs";
|
||||
export type { StaticTaskContext } from "./static-task-context.mjs";
|
||||
export interface CiRun {
|
||||
readonly id?: string | number;
|
||||
readonly run_attempt?: number;
|
||||
readonly status?: string;
|
||||
readonly conclusion?: string | null;
|
||||
readonly event?: string;
|
||||
readonly path?: string;
|
||||
readonly head_sha?: string;
|
||||
readonly head_branch?: string;
|
||||
readonly repository?: { readonly full_name?: string };
|
||||
readonly pull_requests?: readonly {
|
||||
readonly base?: {
|
||||
readonly sha?: string;
|
||||
readonly repo?: { readonly full_name?: string };
|
||||
};
|
||||
}[];
|
||||
}
|
||||
|
||||
export interface CiJob {
|
||||
readonly name?: string;
|
||||
readonly status?: string;
|
||||
readonly conclusion?: string | null;
|
||||
readonly run_id?: number;
|
||||
readonly run_attempt?: number;
|
||||
}
|
||||
|
||||
export interface StaticEvidence {
|
||||
readonly context: StaticTaskContext;
|
||||
readonly diagnostics: readonly StaticDiagnostic[];
|
||||
}
|
||||
|
||||
export interface CiFailureReport {
|
||||
readonly version: 1;
|
||||
readonly observedAt: string;
|
||||
readonly repository: string;
|
||||
readonly defaultBranch: string;
|
||||
readonly informationalOnly: true;
|
||||
readonly gateEffect: string;
|
||||
readonly boundedInput: {
|
||||
readonly candidateRuns: number;
|
||||
readonly limit: number;
|
||||
readonly mainCandidateLimit: number;
|
||||
readonly mainCandidateRuns: number;
|
||||
readonly mainRunLookupLimit: number;
|
||||
readonly manifestTruncated: boolean;
|
||||
};
|
||||
readonly counts: Readonly<
|
||||
Record<"matching-main-diagnostics" | "unknown", number>
|
||||
>;
|
||||
readonly failures: readonly {
|
||||
readonly runId: number | string;
|
||||
readonly runAttempt: number;
|
||||
readonly classification: "matching-main-diagnostics" | "unknown";
|
||||
readonly reason: string;
|
||||
readonly candidateFailures?: readonly {
|
||||
readonly runId: number | string;
|
||||
readonly runAttempt: number;
|
||||
readonly reason: string;
|
||||
}[];
|
||||
readonly runHead?: string | null;
|
||||
readonly event?: string | null;
|
||||
readonly job?: string;
|
||||
readonly conclusion?: string | null;
|
||||
}[];
|
||||
}
|
||||
|
||||
export interface FailureReportApi {
|
||||
getRun(repository: string, id: string | number): Promise<CiRun>;
|
||||
getJobs(
|
||||
repository: string,
|
||||
id: string | number,
|
||||
attempt: number,
|
||||
): Promise<readonly CiJob[]>;
|
||||
getStaticEvidence(
|
||||
repository: string,
|
||||
run: CiRun,
|
||||
job: CiJob,
|
||||
): Promise<StaticEvidence | null>;
|
||||
getAncestry(
|
||||
repository: string,
|
||||
ancestor: string,
|
||||
descendant: string,
|
||||
): Promise<{ status: string; mergeBaseSha: string | null }>;
|
||||
verifyExecutionSource(
|
||||
repository: string,
|
||||
run: CiRun,
|
||||
context: StaticTaskContext,
|
||||
): Promise<boolean>;
|
||||
getBlobIdentity(
|
||||
repository: string,
|
||||
ancestor: string,
|
||||
descendant: string,
|
||||
paths: readonly string[],
|
||||
): Promise<Readonly<Record<string, boolean>>>;
|
||||
}
|
||||
|
||||
export function classifyFailure(input: {
|
||||
repository: string;
|
||||
defaultBranch: string;
|
||||
run: CiRun & NormalizedWorkflowRun;
|
||||
job: CiJob;
|
||||
diagnostics: readonly StaticDiagnostic[];
|
||||
context: StaticTaskContext;
|
||||
mainRun: CiRun & NormalizedWorkflowRun;
|
||||
mainJob: CiJob;
|
||||
mainDiagnostics: readonly StaticDiagnostic[];
|
||||
mainContext: StaticTaskContext;
|
||||
executionSourcesVerified: boolean;
|
||||
ancestry: { status: string; mergeBaseSha: string | null };
|
||||
blobs: Readonly<Record<string, boolean>>;
|
||||
}): {
|
||||
classification: "matching-main-diagnostics" | "unknown";
|
||||
reason: string;
|
||||
};
|
||||
|
||||
export function reportCiFailures(options: {
|
||||
repository: string;
|
||||
defaultBranch: string;
|
||||
batchManifest: {
|
||||
readonly repository: string;
|
||||
readonly runs: readonly {
|
||||
readonly id: number | string;
|
||||
readonly runAttempt: number;
|
||||
readonly workflow: string;
|
||||
readonly event?: string | null;
|
||||
readonly headBranch?: string | null;
|
||||
readonly conclusion: string | null;
|
||||
readonly status: string;
|
||||
}[];
|
||||
readonly truncation?: readonly unknown[];
|
||||
readonly coverage?: { readonly exhaustive?: boolean };
|
||||
readonly selection?: { readonly exhaustive?: boolean };
|
||||
};
|
||||
api: FailureReportApi;
|
||||
outputDirectory?: string;
|
||||
summaryPath?: string;
|
||||
observedAt?: string;
|
||||
}): Promise<CiFailureReport>;
|
||||
|
||||
export function createGitHubApi(options?: {
|
||||
token?: string;
|
||||
fetchImpl?: typeof fetch;
|
||||
apiRequest?: (
|
||||
method: string,
|
||||
endpoint: string,
|
||||
budget: { requests: number; startedAt: number; limitMs: number },
|
||||
) => Promise<unknown>;
|
||||
}): FailureReportApi;
|
||||
|
||||
export function writeFailureReport(
|
||||
report: CiFailureReport,
|
||||
outputDirectory?: string,
|
||||
summaryPath?: string,
|
||||
): Promise<{ jsonPath: string; markdownPath: string }>;
|
||||
|
|
@ -0,0 +1,529 @@
|
|||
import { appendFile, mkdir, readFile, writeFile } from "node:fs/promises";
|
||||
import { join, resolve } from "node:path";
|
||||
import { fileURLToPath } from "node:url";
|
||||
|
||||
import {
|
||||
classifyFailure,
|
||||
diagnosticPaths,
|
||||
isEligibleMainRun,
|
||||
isFailedConclusion,
|
||||
pullRequestBaseSha,
|
||||
readStaticCheckArtifact,
|
||||
safeSha,
|
||||
selectStaticCheckArtifact,
|
||||
validRepoPath,
|
||||
} from "./artifacts.mjs";
|
||||
import { createGhApiRequest } from "./github-api.mjs";
|
||||
import { normalizeWorkflowRun } from "./workflow-run.mjs";
|
||||
|
||||
const OUTPUT_DIRECTORY = join(".cache", "ci-failures");
|
||||
const BATCH_DIRECTORY = join(".cache", "ci-job-timings");
|
||||
const MAX_JSON_BYTES = 8 * 1024 * 1024;
|
||||
const MAX_FAILED_RUNS = 20;
|
||||
const MAX_MAIN_CANDIDATES = 5;
|
||||
const MAX_MAIN_RUN_LOOKUPS = 100;
|
||||
const MAX_PATHS_PER_COMPARISON = 50;
|
||||
const MAX_API_REQUESTS = 250;
|
||||
|
||||
export { classifyFailure } from "./artifacts.mjs";
|
||||
|
||||
/** Uses only failed CI candidates present in the supplied bounded timing manifest. */
|
||||
export async function reportCiFailures({
|
||||
repository,
|
||||
defaultBranch,
|
||||
batchManifest,
|
||||
api,
|
||||
outputDirectory = OUTPUT_DIRECTORY,
|
||||
summaryPath,
|
||||
observedAt = new Date().toISOString(),
|
||||
}) {
|
||||
validateRepository(repository);
|
||||
if (typeof defaultBranch !== "string" || defaultBranch.length === 0)
|
||||
throw new Error("defaultBranch is required");
|
||||
if (
|
||||
!batchManifest ||
|
||||
batchManifest.repository !== repository ||
|
||||
!Array.isArray(batchManifest.runs)
|
||||
)
|
||||
throw new Error("timing manifest repository or runs are invalid");
|
||||
|
||||
const failedRuns = batchManifest.runs.filter(
|
||||
(entry) =>
|
||||
entry.workflow === "CI" &&
|
||||
entry.status === "collected" &&
|
||||
isFailedConclusion(entry.conclusion),
|
||||
);
|
||||
const prRuns = failedRuns.filter((entry) => entry.event === "pull_request");
|
||||
const candidates = prRuns.slice(0, MAX_FAILED_RUNS);
|
||||
const mainEntries = failedRuns.filter(
|
||||
(entry) =>
|
||||
entry.headBranch === defaultBranch &&
|
||||
["push", "schedule", "workflow_dispatch"].includes(entry.event),
|
||||
);
|
||||
const lookupRun = verifiedRunLookup(repository, api);
|
||||
const mainCandidates = [];
|
||||
for (const entry of mainEntries.slice(0, MAX_MAIN_RUN_LOOKUPS)) {
|
||||
try {
|
||||
const run = await lookupRun(entry);
|
||||
if (isEligibleMainRun(run, defaultBranch, repository))
|
||||
mainCandidates.push({
|
||||
run,
|
||||
getEvidence: mainEvidenceLookup(repository, api, entry, run),
|
||||
});
|
||||
if (mainCandidates.length === MAX_MAIN_CANDIDATES) break;
|
||||
} catch {
|
||||
// Unverified metadata cannot consume the eligible-main candidate budget.
|
||||
}
|
||||
}
|
||||
const entries = [];
|
||||
for (const entry of candidates) {
|
||||
try {
|
||||
entries.push(
|
||||
...(await analyzeFailedRun({
|
||||
repository,
|
||||
defaultBranch,
|
||||
entry,
|
||||
run: await lookupRun(entry),
|
||||
mainCandidates,
|
||||
api,
|
||||
})),
|
||||
);
|
||||
} catch (error) {
|
||||
entries.push({
|
||||
runId: entry.id,
|
||||
runAttempt: entry.runAttempt,
|
||||
classification: "unknown",
|
||||
reason: errorCode(error),
|
||||
});
|
||||
}
|
||||
}
|
||||
const report = {
|
||||
version: 1,
|
||||
observedAt,
|
||||
repository,
|
||||
defaultBranch,
|
||||
informationalOnly: true,
|
||||
gateEffect: "none; this report never changes CI conclusions",
|
||||
boundedInput: {
|
||||
candidateRuns: candidates.length,
|
||||
limit: MAX_FAILED_RUNS,
|
||||
mainCandidateLimit: MAX_MAIN_CANDIDATES,
|
||||
mainCandidateRuns: mainCandidates.length,
|
||||
mainRunLookupLimit: MAX_MAIN_RUN_LOOKUPS,
|
||||
manifestTruncated:
|
||||
prRuns.length > MAX_FAILED_RUNS ||
|
||||
batchManifest.truncation?.length > 0 ||
|
||||
batchManifest.selection?.exhaustive === false ||
|
||||
batchManifest.coverage?.exhaustive === false,
|
||||
},
|
||||
counts: Object.fromEntries(
|
||||
["matching-main-diagnostics", "unknown"].map((category) => [
|
||||
category,
|
||||
entries.filter((item) => item.classification === category).length,
|
||||
]),
|
||||
),
|
||||
failures: entries,
|
||||
};
|
||||
await writeFailureReport(report, outputDirectory, summaryPath);
|
||||
return report;
|
||||
}
|
||||
|
||||
async function analyzeFailedRun({
|
||||
repository,
|
||||
defaultBranch,
|
||||
entry,
|
||||
run,
|
||||
mainCandidates,
|
||||
api,
|
||||
}) {
|
||||
const jobs = await api.getJobs(repository, entry.id, entry.runAttempt);
|
||||
if (jobs.some((job) => !hasJobIdentity(job, entry)))
|
||||
throw new Error("workflow_job_identity_mismatch");
|
||||
const failedJobs = jobs.filter(
|
||||
(job) => job.status === "completed" && isFailedConclusion(job.conclusion),
|
||||
);
|
||||
if (failedJobs.length === 0)
|
||||
return [
|
||||
{
|
||||
runId: entry.id,
|
||||
runAttempt: entry.runAttempt,
|
||||
classification: "unknown",
|
||||
reason: "failed_jobs_not_present_in_api_snapshot",
|
||||
},
|
||||
];
|
||||
const failures = [];
|
||||
for (const job of failedJobs) {
|
||||
const evidence = await api.getStaticEvidence(repository, run, job);
|
||||
const classification = await attributeToMain({
|
||||
repository,
|
||||
defaultBranch,
|
||||
run,
|
||||
job,
|
||||
evidence,
|
||||
mainCandidates,
|
||||
api,
|
||||
});
|
||||
failures.push({
|
||||
runId: entry.id,
|
||||
runAttempt: entry.runAttempt,
|
||||
runHead: safeSha(run.head_sha),
|
||||
event: run.event ?? null,
|
||||
job: job.name,
|
||||
conclusion: job.conclusion,
|
||||
...classification,
|
||||
});
|
||||
}
|
||||
return failures;
|
||||
}
|
||||
|
||||
async function attributeToMain({
|
||||
repository,
|
||||
defaultBranch,
|
||||
run,
|
||||
job,
|
||||
evidence,
|
||||
mainCandidates,
|
||||
api,
|
||||
}) {
|
||||
const baseSha = pullRequestBaseSha(run, repository);
|
||||
if (baseSha === null || evidence === null)
|
||||
return outcome("unknown", "no_matching_verified_main_failure");
|
||||
let result = outcome("unknown", "no_matching_verified_main_failure");
|
||||
const candidateFailures = [];
|
||||
for (const candidate of mainCandidates) {
|
||||
try {
|
||||
const main = await compareCandidate({
|
||||
repository,
|
||||
defaultBranch,
|
||||
run,
|
||||
job,
|
||||
evidence,
|
||||
baseSha,
|
||||
candidate,
|
||||
api,
|
||||
});
|
||||
if (main !== null) result = main;
|
||||
if (main?.classification === "matching-main-diagnostics") break;
|
||||
} catch (error) {
|
||||
candidateFailures.push({
|
||||
runId: candidate.run.id,
|
||||
runAttempt: candidate.run.run_attempt,
|
||||
reason: errorCode(error),
|
||||
});
|
||||
}
|
||||
}
|
||||
return {
|
||||
...result,
|
||||
...(candidateFailures.length > 0 ? { candidateFailures } : {}),
|
||||
};
|
||||
}
|
||||
|
||||
async function compareCandidate({
|
||||
repository,
|
||||
defaultBranch,
|
||||
run,
|
||||
job,
|
||||
evidence,
|
||||
baseSha,
|
||||
candidate,
|
||||
api,
|
||||
}) {
|
||||
const mainRun = candidate.run;
|
||||
const main = await candidate.getEvidence(job.name);
|
||||
if (!main?.evidence) return null;
|
||||
const { job: mainJob, evidence: mainEvidence } = main;
|
||||
const ancestry = await api.getAncestry(repository, mainRun.head_sha, baseSha);
|
||||
const paths = [
|
||||
...new Set(evidence.diagnostics.flatMap(diagnosticPaths)),
|
||||
].slice(0, MAX_PATHS_PER_COMPARISON);
|
||||
const blobs = await api.getBlobIdentity(
|
||||
repository,
|
||||
mainRun.head_sha,
|
||||
baseSha,
|
||||
paths,
|
||||
);
|
||||
const executionSourcesVerified =
|
||||
(await api.verifyExecutionSource(repository, run, evidence.context)) &&
|
||||
(await api.verifyExecutionSource(
|
||||
repository,
|
||||
mainRun,
|
||||
mainEvidence.context,
|
||||
));
|
||||
return classifyFailure({
|
||||
repository,
|
||||
defaultBranch,
|
||||
run,
|
||||
job,
|
||||
diagnostics: evidence.diagnostics,
|
||||
context: evidence.context,
|
||||
mainRun,
|
||||
mainJob,
|
||||
mainDiagnostics: mainEvidence.diagnostics,
|
||||
mainContext: mainEvidence.context,
|
||||
executionSourcesVerified,
|
||||
ancestry,
|
||||
blobs,
|
||||
});
|
||||
}
|
||||
|
||||
// Each verified main run/attempt owns its lazy jobs and parsed evidence for this
|
||||
// report only. Cache nulls and rejected reads too; retries cannot spend the budget
|
||||
// repeatedly on an immutable observation that was unavailable or invalid.
|
||||
function mainEvidenceLookup(repository, api, entry, run) {
|
||||
let jobs;
|
||||
const evidence = new Map();
|
||||
return (name) => {
|
||||
jobs ??= Promise.resolve().then(async () => {
|
||||
const items = await api.getJobs(repository, entry.id, entry.runAttempt);
|
||||
return items.some((item) => !hasJobIdentity(item, entry)) ? [] : items;
|
||||
});
|
||||
if (!evidence.has(name))
|
||||
evidence.set(
|
||||
name,
|
||||
jobs.then(async (items) => {
|
||||
const job = items.find(
|
||||
(item) =>
|
||||
item.name === name &&
|
||||
item.status === "completed" &&
|
||||
isFailedConclusion(item.conclusion),
|
||||
);
|
||||
if (!job) return null;
|
||||
return {
|
||||
job,
|
||||
evidence: await api.getStaticEvidence(repository, run, job),
|
||||
};
|
||||
}),
|
||||
);
|
||||
return evidence.get(name);
|
||||
};
|
||||
}
|
||||
|
||||
// Selection and attribution share one immutable API observation per run/attempt.
|
||||
function verifiedRunLookup(repository, api) {
|
||||
const runs = new Map();
|
||||
return (entry) => {
|
||||
const key = `${positiveId(entry.id)}:${positiveId(entry.runAttempt)}`;
|
||||
if (!runs.has(key))
|
||||
runs.set(
|
||||
key,
|
||||
Promise.resolve().then(async () => {
|
||||
const run = normalizeWorkflowRun(
|
||||
await api.getRun(repository, entry.id),
|
||||
);
|
||||
if (!hasRunIdentity(run, entry))
|
||||
throw new Error("workflow_run_identity_mismatch");
|
||||
return run;
|
||||
}),
|
||||
);
|
||||
return runs.get(key);
|
||||
};
|
||||
}
|
||||
|
||||
function hasRunIdentity(run, entry) {
|
||||
return (
|
||||
String(run?.id) === String(entry.id) &&
|
||||
Number(run.run_attempt) === Number(entry.runAttempt) &&
|
||||
run.workflowPath?.file === ".github/workflows/ci.yml"
|
||||
);
|
||||
}
|
||||
function hasJobIdentity(job, entry) {
|
||||
return (
|
||||
String(job.run_id) === String(entry.id) &&
|
||||
Number(job.run_attempt) === Number(entry.runAttempt)
|
||||
);
|
||||
}
|
||||
|
||||
export function createGitHubApi({
|
||||
token = process.env.GH_TOKEN,
|
||||
fetchImpl = fetch,
|
||||
apiRequest = createGhApiRequest(),
|
||||
} = {}) {
|
||||
if (!token) throw new Error("GH_TOKEN is required");
|
||||
const budget = { requests: 0, startedAt: Date.now(), limitMs: 60_000 };
|
||||
const request = async (endpoint) => {
|
||||
if (budget.requests >= MAX_API_REQUESTS)
|
||||
throw new Error("github_api_request_budget_exhausted");
|
||||
budget.requests += 1;
|
||||
return await apiRequest("GET", endpoint, budget);
|
||||
};
|
||||
return {
|
||||
async getRun(repository, id) {
|
||||
return await request(
|
||||
`repos/${repository}/actions/runs/${positiveId(id)}`,
|
||||
);
|
||||
},
|
||||
async getJobs(repository, id, attempt) {
|
||||
const page = await request(
|
||||
`repos/${repository}/actions/runs/${positiveId(id)}/attempts/${positiveId(attempt)}/jobs?per_page=100&page=1`,
|
||||
);
|
||||
if (!Array.isArray(page.jobs) || page.jobs.length >= 100)
|
||||
throw new Error("job_list_incomplete_or_malformed");
|
||||
return page.jobs;
|
||||
},
|
||||
async getStaticEvidence(repository, run, job) {
|
||||
if (job.name !== "Static checks") return null;
|
||||
if (run.run_attempt !== 1) return null;
|
||||
const page = await request(
|
||||
`repos/${repository}/actions/runs/${positiveId(run.id)}/artifacts?per_page=100&page=1`,
|
||||
);
|
||||
const artifact = selectStaticCheckArtifact(page, run, run.run_attempt);
|
||||
if (!artifact) return null;
|
||||
return await readStaticCheckArtifact({
|
||||
repository,
|
||||
artifactId: artifact.id,
|
||||
token,
|
||||
fetchImpl,
|
||||
});
|
||||
},
|
||||
async getAncestry(repository, ancestor, descendant) {
|
||||
const comparison = await request(
|
||||
`repos/${repository}/compare/${safeSha(ancestor)}...${safeSha(descendant)}`,
|
||||
);
|
||||
return {
|
||||
status: comparison.status,
|
||||
mergeBaseSha: safeSha(comparison.merge_base_commit?.sha),
|
||||
};
|
||||
},
|
||||
async verifyExecutionSource(repository, run, context) {
|
||||
const sourceSha = safeSha(context.sourceSha);
|
||||
if (!sourceSha) return false;
|
||||
const pr =
|
||||
run.event === "pull_request"
|
||||
? pullRequestBaseSha(run, repository)
|
||||
: null;
|
||||
if (
|
||||
run.event === "pull_request" &&
|
||||
(pr === null || context.prHeadSha !== run.head_sha)
|
||||
)
|
||||
return false;
|
||||
const commit = await request(`repos/${repository}/commits/${sourceSha}`);
|
||||
if (safeSha(commit.sha) !== sourceSha) return false;
|
||||
if (sourceSha === run.head_sha) return true;
|
||||
if (
|
||||
pr === null ||
|
||||
!Array.isArray(commit.parents) ||
|
||||
commit.parents.length !== 2
|
||||
)
|
||||
return false;
|
||||
return (
|
||||
safeSha(commit.parents[0]?.sha) === pr &&
|
||||
safeSha(commit.parents[1]?.sha) === run.head_sha
|
||||
);
|
||||
},
|
||||
async getBlobIdentity(repository, ancestor, descendant, paths) {
|
||||
const result = {};
|
||||
for (const path of paths) {
|
||||
if (!validRepoPath(path)) continue;
|
||||
const [left, right] = await Promise.all([
|
||||
getContent(repository, ancestor, path, request),
|
||||
getContent(repository, descendant, path, request),
|
||||
]);
|
||||
result[path] = left !== null && left === right;
|
||||
}
|
||||
return result;
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
async function getContent(repository, ref, path, request) {
|
||||
try {
|
||||
const encodedPath = path.split("/").map(encodeURIComponent).join("/");
|
||||
const value = await request(
|
||||
`repos/${repository}/contents/${encodedPath}?ref=${encodeURIComponent(ref)}`,
|
||||
);
|
||||
return typeof value.sha === "string" ? value.sha : null;
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
function outcome(classification, reason) {
|
||||
return { classification, reason };
|
||||
}
|
||||
function validateRepository(value) {
|
||||
if (
|
||||
typeof value !== "string" ||
|
||||
!/^[A-Za-z0-9_.-]+\/[A-Za-z0-9_.-]+$/u.test(value)
|
||||
)
|
||||
throw new Error("repository must be owner/name");
|
||||
}
|
||||
function positiveId(value) {
|
||||
if (!Number.isSafeInteger(Number(value)) || Number(value) < 1)
|
||||
throw new Error(
|
||||
"GitHub run and attempt IDs must be positive safe integers",
|
||||
);
|
||||
return String(Number(value));
|
||||
}
|
||||
function errorCode(error) {
|
||||
return error instanceof Error ? error.message.slice(0, 160) : "unknown_error";
|
||||
}
|
||||
|
||||
export async function writeFailureReport(
|
||||
report,
|
||||
outputDirectory = OUTPUT_DIRECTORY,
|
||||
summaryPath,
|
||||
) {
|
||||
await mkdir(outputDirectory, { recursive: true });
|
||||
const jsonPath = join(outputDirectory, "report.json");
|
||||
const markdownPath = join(outputDirectory, "report.md");
|
||||
const matchedDiagnostics = report.failures.filter(
|
||||
(item) => item.classification === "matching-main-diagnostics",
|
||||
);
|
||||
const lines = [
|
||||
"# CI failure attribution",
|
||||
"",
|
||||
"Informational only. This report never changes a CI conclusion or required gate.",
|
||||
"",
|
||||
`Observed ${report.observedAt}; ${String(matchedDiagnostics.length)} of ${String(report.failures.length)} failure reports contain supported diagnostics matching a failed static task on an ancestor main revision. This does not attribute other failures in the same job. Unknown means evidence was incomplete or did not match.`,
|
||||
"",
|
||||
];
|
||||
for (const item of report.failures)
|
||||
lines.push(
|
||||
`- Run ${item.runId} attempt ${item.runAttempt}${item.job ? `, ${item.job}` : ""}: **${item.classification}** — ${item.reason}.`,
|
||||
);
|
||||
if (report.failures.length === 0)
|
||||
lines.push(
|
||||
"- No completed failed CI attempts were present in the supplied timing snapshot.",
|
||||
);
|
||||
await writeFile(jsonPath, `${JSON.stringify(report, null, 2)}\n`, "utf8");
|
||||
await writeFile(markdownPath, `${lines.join("\n")}\n`, "utf8");
|
||||
if (summaryPath)
|
||||
await appendFile(summaryPath, `${lines.join("\n")}\n`, "utf8");
|
||||
return { jsonPath, markdownPath };
|
||||
}
|
||||
|
||||
async function main() {
|
||||
const repository = process.env.CI_REPOSITORY;
|
||||
const defaultBranch = process.env.CI_DEFAULT_BRANCH;
|
||||
const batchPath =
|
||||
process.env.CI_TIMING_MANIFEST ??
|
||||
join(BATCH_DIRECTORY, "batch-manifest.json");
|
||||
const batchManifest = JSON.parse(await readBoundedFile(batchPath));
|
||||
const api = createGitHubApi();
|
||||
await reportCiFailures({
|
||||
repository,
|
||||
defaultBranch,
|
||||
batchManifest,
|
||||
api,
|
||||
outputDirectory: process.env.CI_FAILURES_DIRECTORY ?? OUTPUT_DIRECTORY,
|
||||
summaryPath: process.env.GITHUB_STEP_SUMMARY,
|
||||
});
|
||||
}
|
||||
async function readBoundedFile(path) {
|
||||
const bytes = await readFile(path);
|
||||
if (bytes.byteLength > MAX_JSON_BYTES)
|
||||
throw new Error("timing manifest exceeds size limit");
|
||||
return bytes.toString("utf8");
|
||||
}
|
||||
|
||||
if (
|
||||
process.argv[1] &&
|
||||
resolve(process.argv[1]) === fileURLToPath(import.meta.url)
|
||||
) {
|
||||
try {
|
||||
await main();
|
||||
} catch (error) {
|
||||
console.error(error instanceof Error ? error.message : String(error));
|
||||
process.exitCode = 1;
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,16 @@
|
|||
export interface GitHubApiBudget {
|
||||
requests: number;
|
||||
startedAt: number;
|
||||
limitMs: number;
|
||||
}
|
||||
|
||||
export type GitHubApiRequest = (
|
||||
method: string,
|
||||
endpoint: string,
|
||||
budget: GitHubApiBudget,
|
||||
) => Promise<unknown>;
|
||||
|
||||
export function createGhApiRequest(options?: {
|
||||
env?: NodeJS.ProcessEnv;
|
||||
now?: () => number;
|
||||
}): GitHubApiRequest;
|
||||
|
|
@ -0,0 +1,33 @@
|
|||
import { execFile } from "node:child_process";
|
||||
import { promisify } from "node:util";
|
||||
|
||||
const execFileAsync = promisify(execFile);
|
||||
const MAX_BUFFER = 8 * 1024 * 1024;
|
||||
const REQUEST_TIMEOUT_MS = 10_000;
|
||||
|
||||
/** Create a bounded GitHub CLI JSON transport with GET-only authority. */
|
||||
export function createGhApiRequest({ env = process.env, now = Date.now } = {}) {
|
||||
return async (method, endpoint, budget) => {
|
||||
if (method !== "GET")
|
||||
throw new Error(
|
||||
"GitHub API transport supports read-only GET requests only",
|
||||
);
|
||||
const remaining = budget.limitMs - (now() - budget.startedAt);
|
||||
if (remaining <= 0) throw new Error("lookup_time_budget_exhausted");
|
||||
const { stdout } = await execFileAsync(
|
||||
"gh",
|
||||
["api", "--method", "GET", endpoint],
|
||||
{
|
||||
env,
|
||||
encoding: "utf8",
|
||||
maxBuffer: MAX_BUFFER,
|
||||
timeout: Math.min(REQUEST_TIMEOUT_MS, remaining),
|
||||
},
|
||||
);
|
||||
try {
|
||||
return JSON.parse(stdout);
|
||||
} catch (cause) {
|
||||
throw new Error("github_api_malformed_json", { cause });
|
||||
}
|
||||
};
|
||||
}
|
||||
|
|
@ -103,6 +103,9 @@ export async function collectJobTimingsBatch({
|
|||
id: run.id,
|
||||
runAttempt: run.run_attempt,
|
||||
workflow: workflow.name,
|
||||
event: typeof run.event === "string" ? run.event : null,
|
||||
headBranch:
|
||||
typeof run.head_branch === "string" ? run.head_branch : null,
|
||||
conclusion: run.conclusion ?? null,
|
||||
createdAt: run.created_at ?? null,
|
||||
});
|
||||
|
|
|
|||
|
|
@ -0,0 +1,95 @@
|
|||
export interface PrCiHead {
|
||||
repositoryId: number | null;
|
||||
repository?: string | null;
|
||||
ref: string | null;
|
||||
sha: string | null;
|
||||
}
|
||||
|
||||
export interface PrCiPullRequest {
|
||||
number: number | null;
|
||||
draft?: boolean;
|
||||
identityValid: boolean;
|
||||
mergeable?: boolean;
|
||||
head: PrCiHead;
|
||||
[key: string]: unknown;
|
||||
}
|
||||
|
||||
export interface PrCiWorkflowRun {
|
||||
id?: number;
|
||||
run_attempt?: number;
|
||||
event?: string;
|
||||
head_branch?: string;
|
||||
head_repository?: { id?: number | null } | null;
|
||||
head_sha?: string;
|
||||
status?: string;
|
||||
conclusion?: string | null;
|
||||
created_at?: string;
|
||||
html_url?: string;
|
||||
pull_requests?: Array<{
|
||||
number?: number;
|
||||
head?: { sha?: string; repo?: { id?: number | null } } | null;
|
||||
}>;
|
||||
[key: string]: unknown;
|
||||
}
|
||||
|
||||
export interface PrCiClassification {
|
||||
category: string;
|
||||
reason: string;
|
||||
runs: Array<Record<string, unknown>>;
|
||||
}
|
||||
|
||||
export interface PrCiHealthReport {
|
||||
schemaVersion: 1;
|
||||
observedAt: string;
|
||||
repository: string;
|
||||
repositoryId: number | null;
|
||||
defaultBranch: string;
|
||||
lookbackDays: number;
|
||||
scan: {
|
||||
status: "complete" | "partial";
|
||||
cutoff: string;
|
||||
pullRequestPages: number;
|
||||
workflowRunPages: number;
|
||||
pullRequestInventoryComplete: boolean;
|
||||
workflowRunInventoryComplete: boolean;
|
||||
pullRequestDetailsFetched: number;
|
||||
pullRequestDetailsLimit: number;
|
||||
apiRequests: number;
|
||||
apiRequestLimit: number;
|
||||
errors: string[];
|
||||
};
|
||||
counts: Record<string, number>;
|
||||
pullRequests: Array<PrCiPullRequest & { ci: PrCiClassification }>;
|
||||
}
|
||||
|
||||
export function collectPrCiHealth(options: {
|
||||
repository: string;
|
||||
repositoryId?: number | string | null;
|
||||
defaultBranch: string;
|
||||
lookbackDays?: number;
|
||||
now?: number;
|
||||
apiRequest?: GitHubApiRequest;
|
||||
}): Promise<PrCiHealthReport>;
|
||||
|
||||
export function classifyPrCi(input: {
|
||||
pull: PrCiPullRequest;
|
||||
runs: Array<{ run: PrCiWorkflowRun; basis: string }>;
|
||||
runsComplete: boolean;
|
||||
pullRequestInventoryComplete: boolean;
|
||||
ambiguousRuns: Array<{ run: PrCiWorkflowRun; basis: string }>;
|
||||
}): PrCiClassification;
|
||||
|
||||
export function matchRunToPullRequest(input: {
|
||||
run: PrCiWorkflowRun;
|
||||
pull: PrCiPullRequest;
|
||||
identityCounts: Map<string, number>;
|
||||
inventoryComplete?: boolean;
|
||||
}): { kind: "none" } | { kind: "ambiguous" } | { kind: "match"; basis: string };
|
||||
|
||||
export function writePrCiHealthReport(
|
||||
report: PrCiHealthReport,
|
||||
outputDirectory?: string,
|
||||
): Promise<{ jsonPath: string; markdownPath: string }>;
|
||||
|
||||
export function renderMarkdownSummary(report: PrCiHealthReport): string;
|
||||
import type { GitHubApiRequest } from "./github-api.mjs";
|
||||
|
|
@ -0,0 +1,718 @@
|
|||
import { appendFile, mkdir, writeFile } from "node:fs/promises";
|
||||
import { join, resolve } from "node:path";
|
||||
import { fileURLToPath } from "node:url";
|
||||
|
||||
import { createGhApiRequest } from "./github-api.mjs";
|
||||
|
||||
const OUTPUT_DIRECTORY = join(".cache", "ci-pr-health");
|
||||
const PAGE_SIZE = 100;
|
||||
const MAX_PULL_REQUEST_PAGES = 3;
|
||||
const MAX_WORKFLOW_RUN_PAGES = 5;
|
||||
const MAX_PULL_REQUEST_DETAILS = 40;
|
||||
const MAX_API_REQUESTS =
|
||||
MAX_PULL_REQUEST_PAGES + MAX_WORKFLOW_RUN_PAGES + MAX_PULL_REQUEST_DETAILS;
|
||||
const LOOKBACK_DAYS = 14;
|
||||
const MAX_LOOKBACK_DAYS = 30;
|
||||
const MAX_TOTAL_TIME_MS = 60_000;
|
||||
const SHA_PATTERN = /^[0-9a-f]{40}$/iu;
|
||||
const COMPLETED_CATEGORIES = new Map([
|
||||
["failure", "completed_failure"],
|
||||
["timed_out", "completed_failure"],
|
||||
["cancelled", "completed_canceled"],
|
||||
["success", "completed_success"],
|
||||
]);
|
||||
|
||||
/** Read workflow/PR state and conservatively distinguish absent CI from unknown. */
|
||||
export async function collectPrCiHealth({
|
||||
repository,
|
||||
repositoryId,
|
||||
defaultBranch,
|
||||
lookbackDays = LOOKBACK_DAYS,
|
||||
now = Date.now(),
|
||||
apiRequest = createGhApiRequest(),
|
||||
}) {
|
||||
const identity = validateInputs({
|
||||
repository,
|
||||
repositoryId,
|
||||
defaultBranch,
|
||||
lookbackDays,
|
||||
now,
|
||||
});
|
||||
const observedAt = new Date(now).toISOString();
|
||||
const cutoff = now - identity.lookbackDays * 24 * 60 * 60 * 1_000;
|
||||
const snapshot = await collectApiSnapshot({ apiRequest, identity, cutoff });
|
||||
const classified = classifyPullRequests(snapshot);
|
||||
const { pulls, runs, budget } = snapshot;
|
||||
const errors = [pulls.error, runs.error, ...snapshot.detailsErrors].filter(
|
||||
Boolean,
|
||||
);
|
||||
return {
|
||||
schemaVersion: 1,
|
||||
observedAt,
|
||||
repository: identity.repository,
|
||||
repositoryId: identity.repositoryId,
|
||||
defaultBranch: identity.defaultBranch,
|
||||
lookbackDays: identity.lookbackDays,
|
||||
scan: {
|
||||
status:
|
||||
errors.length === 0 && pulls.complete && runs.complete
|
||||
? "complete"
|
||||
: "partial",
|
||||
cutoff: new Date(cutoff).toISOString(),
|
||||
pullRequestPages: pulls.pages,
|
||||
workflowRunPages: runs.pages,
|
||||
pullRequestInventoryComplete: pulls.complete,
|
||||
workflowRunInventoryComplete: runs.complete,
|
||||
pullRequestDetailsFetched: snapshot.detailsFetched,
|
||||
pullRequestDetailsLimit: MAX_PULL_REQUEST_DETAILS,
|
||||
apiRequests: budget.requests,
|
||||
apiRequestLimit: MAX_API_REQUESTS,
|
||||
errors,
|
||||
},
|
||||
counts: countCategories(classified),
|
||||
pullRequests: classified,
|
||||
};
|
||||
}
|
||||
|
||||
async function collectApiSnapshot({ apiRequest, identity, cutoff }) {
|
||||
const budget = {
|
||||
requests: 0,
|
||||
startedAt: Date.now(),
|
||||
limitMs: MAX_TOTAL_TIME_MS,
|
||||
};
|
||||
const requestGet = async (endpoint) => {
|
||||
if (budget.requests >= MAX_API_REQUESTS)
|
||||
throw new Error("lookup_request_budget_exhausted");
|
||||
if (Date.now() - budget.startedAt >= budget.limitMs)
|
||||
throw new Error("lookup_time_budget_exhausted");
|
||||
budget.requests += 1;
|
||||
return await apiRequest("GET", endpoint, budget);
|
||||
};
|
||||
const pulls = await listOpenPullRequests({
|
||||
requestGet,
|
||||
identity,
|
||||
});
|
||||
const candidates = pulls.items.filter(
|
||||
(pull) =>
|
||||
!pull.draft &&
|
||||
Number.isFinite(Date.parse(pull.createdAt ?? "")) &&
|
||||
Date.parse(pull.createdAt ?? "") >= cutoff &&
|
||||
pull.base.ref === identity.defaultBranch &&
|
||||
(identity.repositoryId === null ||
|
||||
pull.base.repositoryId === identity.repositoryId),
|
||||
);
|
||||
const runs = await listRecentWorkflowRuns({
|
||||
requestGet,
|
||||
identity,
|
||||
cutoff,
|
||||
});
|
||||
|
||||
const detailResult = await resolveMergeability({
|
||||
candidates,
|
||||
requestGet,
|
||||
identity,
|
||||
});
|
||||
return {
|
||||
budget,
|
||||
pulls,
|
||||
runs,
|
||||
candidates,
|
||||
detailsFetched: detailResult.fetched,
|
||||
detailsErrors: detailResult.errors,
|
||||
};
|
||||
}
|
||||
|
||||
function classifyPullRequests(snapshot) {
|
||||
const { candidates, pulls, runs } = snapshot;
|
||||
const identityCounts = countHeadIdentities(pulls.items);
|
||||
return candidates.map((pull) => {
|
||||
const relevant = [];
|
||||
const ambiguousRuns = [];
|
||||
for (const run of runs.items) {
|
||||
const match = matchRunToPullRequest({
|
||||
run,
|
||||
pull,
|
||||
identityCounts,
|
||||
inventoryComplete: pulls.complete,
|
||||
});
|
||||
if (match.kind === "match") relevant.push({ run, basis: match.basis });
|
||||
if (match.kind === "ambiguous")
|
||||
ambiguousRuns.push({ run, basis: "ambiguous_head_identity" });
|
||||
}
|
||||
return {
|
||||
...pull,
|
||||
ci: classifyPrCi({
|
||||
pull,
|
||||
runs: relevant,
|
||||
runsComplete: runs.complete,
|
||||
pullRequestInventoryComplete: pulls.complete,
|
||||
ambiguousRuns,
|
||||
}),
|
||||
};
|
||||
});
|
||||
}
|
||||
|
||||
async function resolveMergeability({ candidates, requestGet, identity }) {
|
||||
let detailsFetched = 0;
|
||||
const errors = new Set();
|
||||
for (const pull of candidates) {
|
||||
if (pull.mergeable !== undefined) continue;
|
||||
if (detailsFetched >= MAX_PULL_REQUEST_DETAILS) {
|
||||
errors.add("pull_request_mergeability_lookup_budget_exhausted");
|
||||
continue;
|
||||
}
|
||||
const result = await getPullRequestDetails({ requestGet, identity, pull });
|
||||
detailsFetched += 1;
|
||||
if (result.error !== null) {
|
||||
errors.add(result.error);
|
||||
continue;
|
||||
}
|
||||
const { detail } = result;
|
||||
const identityChanged =
|
||||
detail.headSha !== pull.head.sha ||
|
||||
detail.headRef !== pull.head.ref ||
|
||||
detail.headRepositoryId !== pull.head.repositoryId ||
|
||||
detail.baseRef !== pull.base.ref ||
|
||||
detail.baseRepositoryId !== pull.base.repositoryId;
|
||||
if (identityChanged) {
|
||||
pull.identityValid = false;
|
||||
pull.identityChangedDuringScan = true;
|
||||
errors.add("pull_request_identity_changed_during_scan");
|
||||
} else {
|
||||
pull.mergeable = detail.mergeable;
|
||||
pull.detailFetched = true;
|
||||
if (detail.mergeable === undefined)
|
||||
errors.add("pull_request_mergeability_unknown");
|
||||
}
|
||||
}
|
||||
return { fetched: detailsFetched, errors: [...errors] };
|
||||
}
|
||||
|
||||
function countCategories(pulls) {
|
||||
const categories = [
|
||||
"missing",
|
||||
"startup_failure",
|
||||
"queued_pending_executing",
|
||||
"approval_required",
|
||||
"completed_success",
|
||||
"completed_failure",
|
||||
"completed_canceled",
|
||||
"completed_other",
|
||||
"merge_conflict",
|
||||
"unknown",
|
||||
"deferred",
|
||||
];
|
||||
return Object.fromEntries(
|
||||
categories.map((category) => [
|
||||
category,
|
||||
pulls.filter(({ ci }) => ci.category === category).length,
|
||||
]),
|
||||
);
|
||||
}
|
||||
|
||||
/** Pure status classifier; missing requires complete, identity-safe inventories. */
|
||||
export function classifyPrCi({
|
||||
pull,
|
||||
runs,
|
||||
runsComplete,
|
||||
pullRequestInventoryComplete,
|
||||
ambiguousRuns,
|
||||
}) {
|
||||
if (!pull.identityValid)
|
||||
return result("unknown", "pull_request_head_identity_invalid", []);
|
||||
const evidence = [...runs, ...ambiguousRuns];
|
||||
if (pull.mergeable === false)
|
||||
return result("merge_conflict", "github_reports_merge_conflict", evidence);
|
||||
// The newest collected run is not proven current until pagination completes:
|
||||
// an older-created run outside this snapshot may have been retried recently.
|
||||
if (!runsComplete)
|
||||
return result(
|
||||
evidence.length > 0 ? "unknown" : "deferred",
|
||||
evidence.length > 0
|
||||
? "workflow_run_inventory_incomplete"
|
||||
: "lookup_inventory_incomplete",
|
||||
evidence,
|
||||
);
|
||||
const selection = newestRun(runs);
|
||||
if (!selection.known)
|
||||
return result("unknown", "workflow_run_order_unknown", evidence);
|
||||
const latest = selection.latest;
|
||||
if (
|
||||
ambiguousRuns.some((candidate) => {
|
||||
const order = latest ? compareRunEvidence(candidate, latest) : null;
|
||||
return order === null || order >= 0;
|
||||
})
|
||||
)
|
||||
return result("unknown", "workflow_run_identity_ambiguous", evidence);
|
||||
if (latest !== undefined) {
|
||||
const run = latest.run;
|
||||
if (run.conclusion === "startup_failure")
|
||||
return result("startup_failure", "workflow_startup_failure", evidence);
|
||||
if (run.status === "waiting" || run.conclusion === "action_required")
|
||||
return result(
|
||||
"approval_required",
|
||||
"workflow_waiting_for_approval_or_gate",
|
||||
evidence,
|
||||
);
|
||||
if (["queued", "requested", "pending", "in_progress"].includes(run.status))
|
||||
return result(
|
||||
"queued_pending_executing",
|
||||
`workflow_${run.status}`,
|
||||
evidence,
|
||||
);
|
||||
if (run.status === "completed")
|
||||
return result(
|
||||
COMPLETED_CATEGORIES.get(run.conclusion) ?? "completed_other",
|
||||
run.conclusion === "success"
|
||||
? "workflow_succeeded"
|
||||
: `workflow_${run.conclusion ?? "unknown_conclusion"}`,
|
||||
evidence,
|
||||
);
|
||||
return result("unknown", "workflow_status_unrecognized", evidence);
|
||||
}
|
||||
if (!pullRequestInventoryComplete)
|
||||
return result("deferred", "lookup_inventory_incomplete", []);
|
||||
if (pull.mergeable !== true)
|
||||
return result("unknown", "pull_request_mergeability_unknown", []);
|
||||
return result("missing", "no_pull_request_event_run_for_current_head", []);
|
||||
}
|
||||
|
||||
/** Match only a pull_request event proven to belong to this immutable PR head. */
|
||||
export function matchRunToPullRequest({
|
||||
run,
|
||||
pull,
|
||||
identityCounts,
|
||||
inventoryComplete = true,
|
||||
}) {
|
||||
if (run?.event !== "pull_request") return { kind: "none" };
|
||||
if (!pull.identityValid || !isRecord(run)) return { kind: "none" };
|
||||
const association = matchExplicitPullAssociation(run, pull);
|
||||
if (association === "match")
|
||||
return { kind: "match", basis: "event_pr_number_and_head_sha" };
|
||||
if (association === "ambiguous") return { kind: "ambiguous" };
|
||||
if (association === "other" || !sameHeadRepositoryAndRef(run, pull))
|
||||
return { kind: "none" };
|
||||
if (!uniqueHeadRepositoryAndRef(pull, identityCounts, inventoryComplete))
|
||||
return { kind: "ambiguous" };
|
||||
if (isSha(run.head_sha) && run.head_sha.toLowerCase() === pull.head.sha)
|
||||
return { kind: "match", basis: "event_unique_head_ref_and_repository_id" };
|
||||
return { kind: "none" };
|
||||
}
|
||||
|
||||
function matchExplicitPullAssociation(run, pull) {
|
||||
const associations = Array.isArray(run.pull_requests)
|
||||
? run.pull_requests
|
||||
: [];
|
||||
if (associations.length === 0) return "absent";
|
||||
const matchingNumber = associations.filter(
|
||||
(item) => item?.number === pull.number,
|
||||
);
|
||||
if (matchingNumber.length === 0) return "other";
|
||||
const exactHead = matchingNumber.find(
|
||||
(item) => item?.head?.sha === pull.head.sha,
|
||||
);
|
||||
if (exactHead === undefined) {
|
||||
return matchingNumber.some(
|
||||
(item) =>
|
||||
typeof item?.head?.sha === "string" && item.head.sha !== pull.head.sha,
|
||||
)
|
||||
? "other"
|
||||
: "absent";
|
||||
}
|
||||
const associatedHeadRepositoryId = positiveInteger(exactHead.head?.repo?.id);
|
||||
const runHeadRepositoryId = positiveInteger(run.head_repository?.id);
|
||||
if (associatedHeadRepositoryId === null || runHeadRepositoryId === null)
|
||||
return "ambiguous";
|
||||
if (
|
||||
associatedHeadRepositoryId !== pull.head.repositoryId ||
|
||||
runHeadRepositoryId !== pull.head.repositoryId
|
||||
)
|
||||
return "other";
|
||||
const baseRefMatches =
|
||||
typeof exactHead.base?.ref !== "string" ||
|
||||
exactHead.base.ref === pull.base.ref;
|
||||
const baseRepositoryId = positiveInteger(exactHead.base?.repo?.id);
|
||||
const baseRepositoryMatches =
|
||||
baseRepositoryId === null || baseRepositoryId === pull.base.repositoryId;
|
||||
return baseRefMatches && baseRepositoryMatches ? "match" : "other";
|
||||
}
|
||||
|
||||
function sameHeadRepositoryAndRef(run, pull) {
|
||||
return (
|
||||
run.head_branch === pull.head.ref &&
|
||||
run.head_repository?.id === pull.head.repositoryId
|
||||
);
|
||||
}
|
||||
|
||||
function uniqueHeadRepositoryAndRef(pull, identityCounts, inventoryComplete) {
|
||||
if (!inventoryComplete) return false;
|
||||
const key = headIdentityKey(pull.head.repositoryId, pull.head.ref);
|
||||
const unknownRepositoryKey = headIdentityKey(null, pull.head.ref);
|
||||
return (
|
||||
identityCounts.get(key) === 1 &&
|
||||
(identityCounts.get(unknownRepositoryKey) ?? 0) === 0
|
||||
);
|
||||
}
|
||||
|
||||
function normalizePullRequest(raw, identity) {
|
||||
const number = positiveInteger(raw?.number);
|
||||
const base = normalizeBase(raw?.base);
|
||||
const head = normalizeHead(raw?.head);
|
||||
const draft = raw?.draft === true;
|
||||
return {
|
||||
number,
|
||||
draft,
|
||||
createdAt: typeof raw?.created_at === "string" ? raw.created_at : null,
|
||||
url: typeof raw?.html_url === "string" ? raw.html_url : null,
|
||||
base,
|
||||
head,
|
||||
mergeable: typeof raw?.mergeable === "boolean" ? raw.mergeable : undefined,
|
||||
identityValid: hasValidPullIdentity({ raw, identity, number, base, head }),
|
||||
};
|
||||
}
|
||||
|
||||
function normalizeBase(raw) {
|
||||
return {
|
||||
ref: typeof raw?.ref === "string" ? raw.ref : null,
|
||||
repositoryId: positiveInteger(raw?.repo?.id),
|
||||
};
|
||||
}
|
||||
|
||||
function normalizeHead(raw) {
|
||||
return {
|
||||
repositoryId: positiveInteger(raw?.repo?.id),
|
||||
repository:
|
||||
typeof raw?.repo?.full_name === "string" ? raw.repo.full_name : null,
|
||||
ref: typeof raw?.ref === "string" ? raw.ref : null,
|
||||
sha:
|
||||
typeof raw?.sha === "string" && isSha(raw.sha)
|
||||
? raw.sha.toLowerCase()
|
||||
: null,
|
||||
};
|
||||
}
|
||||
|
||||
function hasValidPullIdentity({ raw, identity, number, base, head }) {
|
||||
return (
|
||||
number !== null &&
|
||||
typeof raw?.draft === "boolean" &&
|
||||
base.ref === identity.defaultBranch &&
|
||||
(identity.repositoryId === null ||
|
||||
base.repositoryId === identity.repositoryId) &&
|
||||
head.repositoryId !== null &&
|
||||
head.sha !== null &&
|
||||
head.ref !== null
|
||||
);
|
||||
}
|
||||
|
||||
function countHeadIdentities(pulls) {
|
||||
const counts = new Map();
|
||||
for (const pull of pulls) {
|
||||
if (typeof pull.head.ref !== "string") continue;
|
||||
const key = headIdentityKey(pull.head.repositoryId, pull.head.ref);
|
||||
counts.set(key, (counts.get(key) ?? 0) + 1);
|
||||
}
|
||||
return counts;
|
||||
}
|
||||
|
||||
function headIdentityKey(repositoryId, ref) {
|
||||
return `${String(repositoryId)}\u0000${ref}`;
|
||||
}
|
||||
|
||||
function newestRun(runs) {
|
||||
let latest = runs[0];
|
||||
for (const candidate of runs) {
|
||||
if (latest && (compareRunEvidence(candidate, latest) ?? 0) > 0)
|
||||
latest = candidate;
|
||||
}
|
||||
const known = runs.every((candidate) => {
|
||||
if (candidate === latest) return true;
|
||||
const order = compareRunEvidence(candidate, latest);
|
||||
return order !== null && order <= 0;
|
||||
});
|
||||
return { known, latest };
|
||||
}
|
||||
|
||||
// Attempt ordering is meaningful within one run; different run IDs are not clocks.
|
||||
function compareRunEvidence(left, right) {
|
||||
const leftRun = left.run;
|
||||
const rightRun = right.run;
|
||||
const id = positiveInteger(leftRun.id);
|
||||
if (id !== null && id === positiveInteger(rightRun.id)) {
|
||||
const leftAttempt = positiveInteger(leftRun.run_attempt);
|
||||
const rightAttempt = positiveInteger(rightRun.run_attempt);
|
||||
if (leftAttempt === null || rightAttempt === null) return null;
|
||||
if (leftAttempt !== rightAttempt) return leftAttempt - rightAttempt;
|
||||
return leftRun.status === rightRun.status &&
|
||||
leftRun.conclusion === rightRun.conclusion
|
||||
? 0
|
||||
: null;
|
||||
}
|
||||
// A retry may start long after its original creation; without an attempt
|
||||
// clock, creation dates only order first attempts from different runs.
|
||||
if (
|
||||
positiveInteger(leftRun.run_attempt) !== 1 ||
|
||||
positiveInteger(rightRun.run_attempt) !== 1
|
||||
)
|
||||
return null;
|
||||
const time =
|
||||
Date.parse(leftRun.created_at ?? "") -
|
||||
Date.parse(rightRun.created_at ?? "");
|
||||
return Number.isFinite(time) && time !== 0 ? time : null;
|
||||
}
|
||||
|
||||
function result(category, reason, runs) {
|
||||
return {
|
||||
category,
|
||||
reason,
|
||||
runs: runs.map(({ run, basis }) => ({
|
||||
id: positiveInteger(run.id),
|
||||
attempt: positiveInteger(run.run_attempt),
|
||||
status: typeof run.status === "string" ? run.status : null,
|
||||
conclusion: typeof run.conclusion === "string" ? run.conclusion : null,
|
||||
headSha: isSha(run.head_sha) ? run.head_sha.toLowerCase() : null,
|
||||
createdAt: typeof run.created_at === "string" ? run.created_at : null,
|
||||
url: typeof run.html_url === "string" ? run.html_url : null,
|
||||
identityBasis: basis,
|
||||
})),
|
||||
};
|
||||
}
|
||||
|
||||
async function listOpenPullRequests({ requestGet, identity }) {
|
||||
const items = [];
|
||||
let complete = false;
|
||||
let pages = 0;
|
||||
let error = null;
|
||||
for (let page = 1; page <= MAX_PULL_REQUEST_PAGES; page += 1) {
|
||||
try {
|
||||
const endpoint = `repos/${identity.repository}/pulls?state=open&sort=created&direction=desc&per_page=${PAGE_SIZE}&page=${page}`;
|
||||
const response = await requestGet(endpoint);
|
||||
if (!Array.isArray(response))
|
||||
throw new Error("pull_request_page_malformed");
|
||||
pages += 1;
|
||||
const normalized = response.map((raw) =>
|
||||
normalizePullRequest(raw, identity),
|
||||
);
|
||||
if (
|
||||
normalized.some(
|
||||
(pull) => !Number.isFinite(Date.parse(pull.createdAt ?? "")),
|
||||
)
|
||||
)
|
||||
throw new Error("pull_request_page_malformed");
|
||||
items.push(...normalized);
|
||||
if (response.length < PAGE_SIZE) {
|
||||
complete = true;
|
||||
break;
|
||||
}
|
||||
} catch (cause) {
|
||||
error = safeErrorCode(cause);
|
||||
break;
|
||||
}
|
||||
}
|
||||
return { items, pages, complete, error };
|
||||
}
|
||||
|
||||
async function listRecentWorkflowRuns({ requestGet, identity, cutoff }) {
|
||||
const items = [];
|
||||
let complete = false;
|
||||
let pages = 0;
|
||||
let error = null;
|
||||
for (let page = 1; page <= MAX_WORKFLOW_RUN_PAGES; page += 1) {
|
||||
try {
|
||||
const cutoffDate = new Date(cutoff).toISOString().slice(0, 10);
|
||||
const endpoint = `repos/${identity.repository}/actions/workflows/ci.yml/runs?event=pull_request&created=%3E%3D${cutoffDate}&per_page=${PAGE_SIZE}&page=${page}`;
|
||||
const response = await requestGet(endpoint);
|
||||
if (!isRecord(response) || !Array.isArray(response.workflow_runs))
|
||||
throw new Error("workflow_run_page_malformed");
|
||||
pages += 1;
|
||||
if (!response.workflow_runs.every(isRecord))
|
||||
throw new Error("workflow_run_page_malformed");
|
||||
items.push(...response.workflow_runs);
|
||||
if (response.workflow_runs.length < PAGE_SIZE) {
|
||||
complete = true;
|
||||
break;
|
||||
}
|
||||
} catch (cause) {
|
||||
error = safeErrorCode(cause);
|
||||
break;
|
||||
}
|
||||
}
|
||||
return { items, pages, complete, error };
|
||||
}
|
||||
|
||||
async function getPullRequestDetails({ requestGet, identity, pull }) {
|
||||
if (pull.number === null)
|
||||
return { detail: null, error: "pull_request_detail_identity_missing" };
|
||||
try {
|
||||
const response = await requestGet(
|
||||
`repos/${identity.repository}/pulls/${pull.number}`,
|
||||
);
|
||||
if (!isRecord(response))
|
||||
return { detail: null, error: "pull_request_detail_malformed" };
|
||||
return {
|
||||
detail: {
|
||||
mergeable:
|
||||
typeof response.mergeable === "boolean"
|
||||
? response.mergeable
|
||||
: undefined,
|
||||
headSha: isSha(response.head?.sha)
|
||||
? response.head.sha.toLowerCase()
|
||||
: null,
|
||||
headRef:
|
||||
typeof response.head?.ref === "string" ? response.head.ref : null,
|
||||
headRepositoryId: positiveInteger(response.head?.repo?.id),
|
||||
baseRef:
|
||||
typeof response.base?.ref === "string" ? response.base.ref : null,
|
||||
baseRepositoryId: positiveInteger(response.base?.repo?.id),
|
||||
},
|
||||
error: null,
|
||||
};
|
||||
} catch (cause) {
|
||||
return { detail: null, error: safeErrorCode(cause) };
|
||||
}
|
||||
}
|
||||
|
||||
function validateInputs({
|
||||
repository,
|
||||
repositoryId,
|
||||
defaultBranch,
|
||||
lookbackDays,
|
||||
now,
|
||||
}) {
|
||||
if (
|
||||
typeof repository !== "string" ||
|
||||
!/^[A-Za-z0-9_.-]+\/[A-Za-z0-9_.-]+$/u.test(repository)
|
||||
)
|
||||
throw new Error(
|
||||
"CI_REPOSITORY must be an owner/name repository identifier",
|
||||
);
|
||||
if (
|
||||
typeof defaultBranch !== "string" ||
|
||||
defaultBranch.length === 0 ||
|
||||
/[\r\n]/u.test(defaultBranch)
|
||||
)
|
||||
throw new Error("CI_DEFAULT_BRANCH must be a nonempty branch name");
|
||||
const parsedRepositoryId =
|
||||
repositoryId === undefined || repositoryId === null || repositoryId === ""
|
||||
? null
|
||||
: positiveInteger(repositoryId);
|
||||
if (
|
||||
repositoryId !== undefined &&
|
||||
repositoryId !== null &&
|
||||
repositoryId !== "" &&
|
||||
parsedRepositoryId === null
|
||||
)
|
||||
throw new Error("CI_REPOSITORY_ID must be a positive safe integer");
|
||||
if (
|
||||
!Number.isInteger(lookbackDays) ||
|
||||
lookbackDays < 1 ||
|
||||
lookbackDays > MAX_LOOKBACK_DAYS
|
||||
)
|
||||
throw new Error(
|
||||
`lookbackDays must be an integer from 1 to ${String(MAX_LOOKBACK_DAYS)}`,
|
||||
);
|
||||
if (!Number.isFinite(now)) throw new Error("now must be a finite timestamp");
|
||||
return {
|
||||
repository,
|
||||
repositoryId: parsedRepositoryId,
|
||||
defaultBranch,
|
||||
lookbackDays,
|
||||
};
|
||||
}
|
||||
|
||||
function positiveInteger(value) {
|
||||
const number = typeof value === "number" ? value : Number(value);
|
||||
return Number.isSafeInteger(number) && number > 0 ? number : null;
|
||||
}
|
||||
|
||||
function isSha(value) {
|
||||
return typeof value === "string" && SHA_PATTERN.test(value);
|
||||
}
|
||||
|
||||
function isRecord(value) {
|
||||
return typeof value === "object" && value !== null && !Array.isArray(value);
|
||||
}
|
||||
|
||||
function safeErrorCode(cause) {
|
||||
if (cause?.message === "lookup_time_budget_exhausted")
|
||||
return "lookup_time_budget_exhausted";
|
||||
if (cause?.message === "lookup_request_budget_exhausted")
|
||||
return "lookup_request_budget_exhausted";
|
||||
if (cause?.message === "github_api_malformed_json")
|
||||
return "github_api_malformed_json";
|
||||
if (cause?.message?.endsWith("_malformed")) return cause.message;
|
||||
if (cause?.code === "ETIMEDOUT" || cause?.killed) return "github_api_timeout";
|
||||
return "github_api_request_failed";
|
||||
}
|
||||
|
||||
export async function writePrCiHealthReport(
|
||||
report,
|
||||
outputDirectory = OUTPUT_DIRECTORY,
|
||||
) {
|
||||
const jsonPath = join(outputDirectory, "report.json");
|
||||
const markdownPath = join(outputDirectory, "summary.md");
|
||||
await mkdir(outputDirectory, { recursive: true });
|
||||
await writeFile(jsonPath, `${JSON.stringify(report, null, 2)}\n`, "utf8");
|
||||
await writeFile(markdownPath, renderMarkdownSummary(report), "utf8");
|
||||
return { jsonPath, markdownPath };
|
||||
}
|
||||
|
||||
export function renderMarkdownSummary(report) {
|
||||
const rows = report.pullRequests.map(
|
||||
(pull) =>
|
||||
`| #${String(pull.number ?? "?")} | ${escapeCell(pull.head.repository ?? "unknown")} | ${escapeCell(pull.head.ref ?? "unknown")} | \`${pull.head.sha ?? "unknown"}\` | ${pull.ci.category} | ${escapeCell(pull.ci.reason)} |`,
|
||||
);
|
||||
return [
|
||||
"# Pull request CI health snapshot",
|
||||
"",
|
||||
`Observed ${report.observedAt} for \`${report.repository}\`, target branch \`${escapeCell(report.defaultBranch)}\`; recent window ${String(report.lookbackDays)} days. Scan: **${report.scan.status}**.`,
|
||||
"",
|
||||
"This is read-only point-in-time telemetry. Missing is reported only when PR and workflow-run inventories are complete and the current immutable PR head identity is unambiguous. Incomplete lookups remain deferred or unknown.",
|
||||
"",
|
||||
`API pages: ${String(report.scan.pullRequestPages)} PR, ${String(report.scan.workflowRunPages)} workflow runs; ${String(report.scan.apiRequests)} bounded requests. Errors: ${report.scan.errors.length === 0 ? "none" : report.scan.errors.join(", ")}.`,
|
||||
"",
|
||||
"| PR | Head repository | Head ref | Head SHA | CI classification | Evidence |",
|
||||
"| ---: | --- | --- | --- | --- | --- |",
|
||||
...rows,
|
||||
"",
|
||||
"Categories: missing, startup failure, queued/pending/executing, approval required, completed outcomes, merge conflict, unknown, and deferred.",
|
||||
"",
|
||||
].join("\n");
|
||||
}
|
||||
|
||||
function escapeCell(value) {
|
||||
return String(value)
|
||||
.replaceAll("|", "\\|")
|
||||
.replaceAll("\n", " ")
|
||||
.replaceAll("\r", " ");
|
||||
}
|
||||
|
||||
if (
|
||||
process.argv[1] !== undefined &&
|
||||
resolve(process.argv[1]) === fileURLToPath(import.meta.url)
|
||||
) {
|
||||
try {
|
||||
const report = await collectPrCiHealth({
|
||||
repository: process.env.CI_REPOSITORY,
|
||||
repositoryId: process.env.CI_REPOSITORY_ID,
|
||||
defaultBranch: process.env.CI_DEFAULT_BRANCH,
|
||||
});
|
||||
const paths = await writePrCiHealthReport(
|
||||
report,
|
||||
process.env.CI_PR_HEALTH_DIRECTORY ?? OUTPUT_DIRECTORY,
|
||||
);
|
||||
if (process.env.GITHUB_STEP_SUMMARY)
|
||||
await appendFile(
|
||||
process.env.GITHUB_STEP_SUMMARY,
|
||||
renderMarkdownSummary(report),
|
||||
"utf8",
|
||||
);
|
||||
process.stdout.write(
|
||||
`Wrote ${report.pullRequests.length} recent PR CI records to ${paths.jsonPath}\n`,
|
||||
);
|
||||
} catch (cause) {
|
||||
process.stderr.write(
|
||||
`PR CI health telemetry failed: ${cause instanceof Error ? cause.message : "unknown_error"}\n`,
|
||||
);
|
||||
process.exitCode = 1;
|
||||
}
|
||||
}
|
||||
|
|
@ -2,6 +2,7 @@ import { execFileSync } from "node:child_process";
|
|||
import { appendFileSync, mkdirSync, writeFileSync } from "node:fs";
|
||||
import { resolve } from "node:path";
|
||||
import { fileURLToPath } from "node:url";
|
||||
import { normalizeWorkflowRun } from "./workflow-run.mjs";
|
||||
|
||||
const ARTIFACT_NAME = "ci-full-qualification-v1";
|
||||
const MAX_AGE_MS = 24 * 60 * 60 * 1000;
|
||||
|
|
@ -119,8 +120,8 @@ function admit() {
|
|||
// Artifact metadata narrows the search before run lookups. Never scan an
|
||||
// unbounded history or treat a missing receipt as a successful qualification.
|
||||
for (const artifact of candidates.slice(0, 3)) {
|
||||
const run = api(
|
||||
`repos/${repository}/actions/runs/${artifact.workflow_run.id}`,
|
||||
const run = normalizeWorkflowRun(
|
||||
api(`repos/${repository}/actions/runs/${artifact.workflow_run.id}`),
|
||||
);
|
||||
// An old run can upload a new artifact when only its gate is rerun.
|
||||
// Bound the original run age as well so retries cannot renew stale proof.
|
||||
|
|
@ -133,7 +134,7 @@ function admit() {
|
|||
run.id === artifact.workflow_run.id &&
|
||||
run.status === "completed" &&
|
||||
run.conclusion === "success" &&
|
||||
run.path === ".github/workflows/ci.yml" &&
|
||||
run.workflowPath?.file === ".github/workflows/ci.yml" &&
|
||||
["schedule", "workflow_dispatch"].includes(run.event) &&
|
||||
run.head_sha === head &&
|
||||
run.head_branch === branch &&
|
||||
|
|
|
|||
|
|
@ -30,7 +30,11 @@ if (outputs.tier === "full" && selectedScopes.length !== CI_FLAGS.length)
|
|||
throw new Error("Full qualification requires every canonical CI scope");
|
||||
const required = reused ? ["changes"] : ["changes", "static", "test"];
|
||||
const lanes = {
|
||||
code: ["build", "test-shard"],
|
||||
code: [
|
||||
"build",
|
||||
"test-shard",
|
||||
...(outputs.tier === "fast" ? ["test-source"] : []),
|
||||
],
|
||||
reverse_tools: ["reverse-tools-quality"],
|
||||
runtime: ["windows-native-build", "package-e2e"],
|
||||
package: ["windows-curated"],
|
||||
|
|
|
|||
|
|
@ -133,6 +133,8 @@ try {
|
|||
for (const clone of report.duplicates)
|
||||
for (const file of [clone.firstFile, clone.secondFile])
|
||||
file.name = relative(process.cwd(), file.name).split(sep).join("/");
|
||||
// Record the actual checker result, including caller-selected thresholds.
|
||||
report.reaCheck = { exitCode: result.status };
|
||||
await writeFile(reportPath, JSON.stringify(report, null, 2));
|
||||
const { sources, clones, percentage } = report.statistics.total;
|
||||
console.log(
|
||||
|
|
|
|||
|
|
@ -0,0 +1,18 @@
|
|||
export interface StaticTaskContext {
|
||||
readonly version: 1;
|
||||
readonly sourceSha: string;
|
||||
readonly sourceClean: boolean;
|
||||
readonly prHeadSha?: string;
|
||||
readonly nodeVersion: string;
|
||||
readonly npmVersion: string;
|
||||
readonly turboVersion: string;
|
||||
readonly platform: string;
|
||||
readonly architecture: string;
|
||||
readonly taskHashes: Readonly<Record<string, string>>;
|
||||
}
|
||||
export function parseStaticTaskContext(value: unknown): StaticTaskContext;
|
||||
export function recordStaticTaskContext(options?: {
|
||||
root?: string;
|
||||
npmCli?: string;
|
||||
outputDirectory?: string;
|
||||
}): Promise<StaticTaskContext>;
|
||||
|
|
@ -0,0 +1,104 @@
|
|||
import { execFile } from "node:child_process";
|
||||
import { mkdir, readFile, writeFile } from "node:fs/promises";
|
||||
import { join, resolve } from "node:path";
|
||||
import { fileURLToPath } from "node:url";
|
||||
import { promisify } from "node:util";
|
||||
import { z } from "zod";
|
||||
|
||||
const execute = promisify(execFile);
|
||||
const contextSchema = z.object({
|
||||
version: z.literal(1),
|
||||
sourceSha: z.string().regex(/^[a-f0-9]{40}$/u),
|
||||
sourceClean: z.boolean(),
|
||||
prHeadSha: z
|
||||
.string()
|
||||
.regex(/^[a-f0-9]{40}$/u)
|
||||
.optional(),
|
||||
nodeVersion: z.string().min(1),
|
||||
npmVersion: z.string().min(1),
|
||||
turboVersion: z.string().min(1),
|
||||
platform: z.string().min(1),
|
||||
architecture: z.string().min(1),
|
||||
taskHashes: z.record(z.string(), z.string().regex(/^[a-f0-9]{16}$/u)),
|
||||
});
|
||||
|
||||
/** Parse the producing check graph's receipt; incomplete provenance is unknown. */
|
||||
export const parseStaticTaskContext = (value) => contextSchema.parse(value);
|
||||
|
||||
/** Retain only named task hashes and tool/source identity, never the environment. */
|
||||
export async function recordStaticTaskContext({
|
||||
root = process.cwd(),
|
||||
npmCli = process.env.npm_execpath,
|
||||
outputDirectory = join(root, ".cache/static-checks"),
|
||||
} = {}) {
|
||||
if (typeof npmCli !== "string" || !npmCli)
|
||||
throw new Error(
|
||||
"Run static context recording through npm run ci:static-context",
|
||||
);
|
||||
const options = {
|
||||
cwd: root,
|
||||
encoding: "utf8",
|
||||
timeout: 30_000,
|
||||
maxBuffer: 32 * 1024 * 1024,
|
||||
};
|
||||
const [graph, source, status, npmVersion, turbo] = await Promise.all([
|
||||
execute(
|
||||
process.execPath,
|
||||
[npmCli, "run", "--silent", "check:ci:tasks", "--", "--dry=json"],
|
||||
options,
|
||||
),
|
||||
execute("git", ["rev-parse", "HEAD"], options),
|
||||
execute(
|
||||
"git",
|
||||
["status", "--porcelain", "--untracked-files=normal"],
|
||||
options,
|
||||
),
|
||||
execute(process.execPath, [npmCli, "--version"], options),
|
||||
readFile(join(root, "node_modules/turbo/package.json"), "utf8"),
|
||||
]);
|
||||
const tasks = z
|
||||
.object({
|
||||
tasks: z
|
||||
.array(
|
||||
z.object({
|
||||
task: z.string().min(1),
|
||||
hash: z.string().regex(/^[a-f0-9]{16}$/u),
|
||||
}),
|
||||
)
|
||||
.min(1),
|
||||
})
|
||||
.parse(JSON.parse(graph.stdout)).tasks;
|
||||
if (new Set(tasks.map(({ task }) => task)).size !== tasks.length)
|
||||
throw new Error("Static graph contains duplicate task identities");
|
||||
const context = parseStaticTaskContext({
|
||||
version: 1,
|
||||
sourceSha: source.stdout.trim(),
|
||||
sourceClean: status.stdout.length === 0,
|
||||
...(process.env.CI_PR_HEAD_SHA
|
||||
? { prHeadSha: process.env.CI_PR_HEAD_SHA }
|
||||
: {}),
|
||||
nodeVersion: process.version,
|
||||
npmVersion: npmVersion.stdout.trim(),
|
||||
turboVersion: z.object({ version: z.string() }).parse(JSON.parse(turbo))
|
||||
.version,
|
||||
platform: process.platform,
|
||||
architecture: process.arch,
|
||||
taskHashes: Object.fromEntries(tasks.map(({ task, hash }) => [task, hash])),
|
||||
});
|
||||
if (process.env.GITHUB_SHA && process.env.GITHUB_SHA !== context.sourceSha)
|
||||
throw new Error(
|
||||
"Static context source does not match the workflow revision",
|
||||
);
|
||||
await mkdir(outputDirectory, { recursive: true });
|
||||
await writeFile(
|
||||
join(outputDirectory, "static-task-context.json"),
|
||||
JSON.stringify(context, null, 2) + "\n",
|
||||
);
|
||||
return context;
|
||||
}
|
||||
|
||||
if (
|
||||
process.argv[1] &&
|
||||
resolve(process.argv[1]) === fileURLToPath(import.meta.url)
|
||||
)
|
||||
await recordStaticTaskContext();
|
||||
|
|
@ -0,0 +1,46 @@
|
|||
import { spawn } from "node:child_process";
|
||||
import { resolve } from "node:path";
|
||||
|
||||
import {
|
||||
BUILT_TEST_PROJECTS,
|
||||
SOURCE_TEST_PROJECTS,
|
||||
} from "../lib/test-projects.mjs";
|
||||
|
||||
const [group, ...arguments_] = process.argv.slice(2);
|
||||
const projects = { source: SOURCE_TEST_PROJECTS, built: BUILT_TEST_PROJECTS }[
|
||||
group
|
||||
];
|
||||
if (!projects) throw new Error("Expected source or built test group");
|
||||
// The group owns project selection; callers can request reporters or shards.
|
||||
if (
|
||||
arguments_.some((argument) => /^--(?:project|exclude)(?:=|$)/u.test(argument))
|
||||
)
|
||||
throw new Error("Test groups cannot override their project inventory");
|
||||
const child = spawn(
|
||||
process.execPath,
|
||||
[
|
||||
resolve("node_modules/vitest/vitest.mjs"),
|
||||
"run",
|
||||
...projects.flatMap(({ name }) => ["--project", name]),
|
||||
...arguments_,
|
||||
],
|
||||
{ stdio: "inherit" },
|
||||
);
|
||||
let shutdownSignal;
|
||||
for (const signal of ["SIGINT", "SIGTERM"])
|
||||
process.on(signal, () => {
|
||||
shutdownSignal ??= signal;
|
||||
child.kill(signal);
|
||||
});
|
||||
child.once("error", (error) => {
|
||||
console.error(error.message);
|
||||
process.exitCode = 1;
|
||||
});
|
||||
child.once("close", (code) => {
|
||||
process.exitCode =
|
||||
shutdownSignal === "SIGINT"
|
||||
? 130
|
||||
: shutdownSignal === "SIGTERM"
|
||||
? 143
|
||||
: (code ?? 1);
|
||||
});
|
||||
|
|
@ -2,6 +2,7 @@ import { execFileSync } from "node:child_process";
|
|||
import { appendFileSync } from "node:fs";
|
||||
import { resolve } from "node:path";
|
||||
import { fileURLToPath } from "node:url";
|
||||
import { normalizeWorkflowRun } from "./workflow-run.mjs";
|
||||
|
||||
const ARTIFACT_NAME = "test-timings";
|
||||
const MAX_AGE_MS = 30 * 24 * 60 * 60 * 1000;
|
||||
|
|
@ -68,8 +69,8 @@ function findTimingRun({ repository, branch, repositoryId, now = Date.now() }) {
|
|||
.slice(0, MAX_CANDIDATES);
|
||||
|
||||
for (const artifact of candidates) {
|
||||
const run = api(
|
||||
`repos/${repository}/actions/runs/${artifact.workflow_run.id}`,
|
||||
const run = normalizeWorkflowRun(
|
||||
api(`repos/${repository}/actions/runs/${artifact.workflow_run.id}`),
|
||||
);
|
||||
const runAge = now - Date.parse(run?.created_at);
|
||||
if (
|
||||
|
|
@ -80,7 +81,7 @@ function findTimingRun({ repository, branch, repositoryId, now = Date.now() }) {
|
|||
run.id === artifact.workflow_run.id &&
|
||||
run.status === "completed" &&
|
||||
run.conclusion === "success" &&
|
||||
run.path === ".github/workflows/ci.yml" &&
|
||||
run.workflowPath?.file === ".github/workflows/ci.yml" &&
|
||||
["schedule", "workflow_dispatch"].includes(run.event) &&
|
||||
run.head_branch === branch &&
|
||||
run.head_repository?.full_name === repository
|
||||
|
|
|
|||
|
|
@ -41,9 +41,20 @@ const files = report.files.map(({ name, durationMs, state }) => {
|
|||
throw new Error(`Test report path is outside the checkout: ${name}`);
|
||||
return { name: path.split(sep).join("/"), durationMs, state };
|
||||
});
|
||||
const complete = process.env.SHARD_RESULT === "success";
|
||||
// Keep group outcomes intact: an unavailable group is not a completed failure.
|
||||
const groupResults = [
|
||||
process.env.SHARD_RESULT,
|
||||
...(process.env.SOURCE_REQUIRED === "true"
|
||||
? [process.env.SOURCE_RESULT]
|
||||
: []),
|
||||
];
|
||||
const groupsCompleted = groupResults.every((state) =>
|
||||
["success", "failure"].includes(state),
|
||||
);
|
||||
const passed =
|
||||
complete && process.env.MERGE_RESULT === "success" && report.success;
|
||||
groupResults.every((state) => state === "success") &&
|
||||
process.env.MERGE_RESULT === "success" &&
|
||||
report.success;
|
||||
// Successful shards and merged reports must account for the complete portable
|
||||
// inventory. Coverage thresholds alone cannot prove that every file ran.
|
||||
const verifyInventory = process.env.CI_VERIFY_TEST_INVENTORY === "true";
|
||||
|
|
@ -69,7 +80,7 @@ if (process.env.CI_TEST_SELECTION) {
|
|||
comparison.complete &&
|
||||
report.unhandledErrors === 0 &&
|
||||
["passed", "failed"].includes(report.reason) &&
|
||||
["success", "failure"].includes(process.env.SHARD_RESULT) &&
|
||||
groupsCompleted &&
|
||||
["success", "failure"].includes(process.env.MERGE_RESULT),
|
||||
executionSha: process.env.GITHUB_SHA ?? null,
|
||||
prHeadSha: process.env.CI_PR_HEAD_SHA ?? null,
|
||||
|
|
@ -91,7 +102,9 @@ if (passed && verifyInventory) {
|
|||
);
|
||||
}
|
||||
const summary = [
|
||||
complete ? "## Merged test diagnostics" : "## Partial test diagnostics",
|
||||
groupsCompleted
|
||||
? "## Merged test diagnostics"
|
||||
: "## Partial test diagnostics",
|
||||
"",
|
||||
...(selection
|
||||
? [
|
||||
|
|
@ -99,11 +112,14 @@ const summary = [
|
|||
"",
|
||||
`Plan: ${selection.plan.mode}. Proposed files: ${selection.selected.length}; omitted: ${selection.omitted.length}.`,
|
||||
`Full-run comparison: ${selection.complete ? "complete" : "incomplete"}; failures outside proposed selection: ${selection.missedFailures.length}.`,
|
||||
"The complete suite still ran. This report cannot enable omissions or qualify a revision. Skipped, missing, and interrupted observations do not establish parity.",
|
||||
"This observational report cannot enable omissions or qualify a revision. Skipped, missing, and interrupted observations do not establish parity.",
|
||||
"",
|
||||
]
|
||||
: []),
|
||||
`Shard result: ${process.env.SHARD_RESULT ?? "unknown"}; merge result: ${process.env.MERGE_RESULT ?? "unknown"}.`,
|
||||
...(process.env.SOURCE_REQUIRED === "true"
|
||||
? [`Source result: ${process.env.SOURCE_RESULT ?? "unknown"}.`]
|
||||
: []),
|
||||
passed
|
||||
? "Successful timing history is available."
|
||||
: "This report is diagnostic only and is not a successful timing baseline.",
|
||||
|
|
|
|||
|
|
@ -0,0 +1,10 @@
|
|||
export interface WorkflowPath {
|
||||
readonly file: string;
|
||||
readonly ref: string | null;
|
||||
}
|
||||
export interface NormalizedWorkflowRun {
|
||||
readonly workflowPath: WorkflowPath | null;
|
||||
}
|
||||
export function normalizeWorkflowRun<T extends Record<string, unknown>>(
|
||||
value: T,
|
||||
): Omit<T, "workflowPath"> & NormalizedWorkflowRun;
|
||||
|
|
@ -0,0 +1,15 @@
|
|||
/** Keep the API path as evidence and parse its optional ref at the boundary. */
|
||||
export function normalizeWorkflowRun(value) {
|
||||
if (typeof value !== "object" || value === null || Array.isArray(value))
|
||||
throw new Error("workflow_run_malformed");
|
||||
const match =
|
||||
typeof value.path === "string" && !/[\u0000-\u001f\u007f]/u.test(value.path)
|
||||
? /^(\.github\/workflows\/[^/\\@\s]+\.ya?ml)(?:@([^\s]+))?$/u.exec(
|
||||
value.path,
|
||||
)
|
||||
: null;
|
||||
return {
|
||||
...value,
|
||||
workflowPath: match ? { file: match[1], ref: match[2] ?? null } : null,
|
||||
};
|
||||
}
|
||||
|
|
@ -1,17 +1,4 @@
|
|||
const sourceProjects = [
|
||||
"domain",
|
||||
"services",
|
||||
"adapters",
|
||||
"composition",
|
||||
"conformance",
|
||||
"evaluation",
|
||||
];
|
||||
|
||||
const sourceTest = (path) =>
|
||||
path.startsWith("src/") ||
|
||||
["composition", "conformance", "evaluation"].some((project) =>
|
||||
path.startsWith(`tests/${project}/`),
|
||||
);
|
||||
import { SOURCE_TEST_PROJECTS, projectForTest } from "./test-projects.mjs";
|
||||
|
||||
/** Parse explicit test paths and an optional branch-comparison base. */
|
||||
export const parseDevelopmentTestRequest = (mode, arguments_) => {
|
||||
|
|
@ -35,10 +22,9 @@ export const parseDevelopmentTestRequest = (mode, arguments_) => {
|
|||
path.split("/").some((part) => part === ".." || part === ".")
|
||||
)
|
||||
throw new Error(`Expected an exact repository test path: ${argument}`);
|
||||
if (mode !== "focused" && !sourceTest(path))
|
||||
throw new Error(
|
||||
`Use test:focused for compiled boundary tests: ${path}`,
|
||||
);
|
||||
const project = projectForTest(path);
|
||||
if (mode !== "focused" && project.group !== "source")
|
||||
throw new Error(`Use test:focused for built test projects: ${path}`);
|
||||
paths.push(path);
|
||||
}
|
||||
}
|
||||
|
|
@ -72,9 +58,14 @@ export const developmentTestPlan = (request, baseCommit) => {
|
|||
!/^[a-f0-9]{40,64}$/u.test(baseCommit ?? "")
|
||||
)
|
||||
throw new Error("Changed tests require a resolved Git merge base");
|
||||
const projects = request.mode === "focused" ? [] : sourceProjects;
|
||||
const projects =
|
||||
request.mode === "focused"
|
||||
? []
|
||||
: SOURCE_TEST_PROJECTS.map(({ name }) => name);
|
||||
return {
|
||||
needsBuild: request.paths.some((path) => !sourceTest(path)),
|
||||
needsBuild: request.paths.some(
|
||||
(path) => projectForTest(path).group === "built",
|
||||
),
|
||||
artifactTasks: Object.entries(artifactConsumers).flatMap(
|
||||
([task, consumers]) =>
|
||||
request.paths.some((path) => consumers.includes(path)) ? [task] : [],
|
||||
|
|
|
|||
|
|
@ -0,0 +1,10 @@
|
|||
export interface TestProjectDefinition {
|
||||
readonly name: string;
|
||||
readonly group: "source" | "built";
|
||||
readonly include: readonly string[];
|
||||
readonly exclude?: readonly string[];
|
||||
}
|
||||
|
||||
export const SOURCE_TEST_PROJECTS: readonly TestProjectDefinition[];
|
||||
export const BUILT_TEST_PROJECTS: readonly TestProjectDefinition[];
|
||||
export function projectForTest(testPath: string): TestProjectDefinition;
|
||||
|
|
@ -0,0 +1,107 @@
|
|||
import { matchesGlob } from "node:path";
|
||||
|
||||
/** Canonical file ownership shared by Vitest and development-test planning. */
|
||||
export const SOURCE_TEST_PROJECTS = [
|
||||
{
|
||||
name: "domain",
|
||||
group: "source",
|
||||
include: ["src/{contracts,domain}/**/*.test.ts"],
|
||||
},
|
||||
{
|
||||
name: "services",
|
||||
group: "source",
|
||||
include: ["src/application/**/*.test.ts"],
|
||||
},
|
||||
{
|
||||
name: "adapters",
|
||||
group: "source",
|
||||
include: [
|
||||
"src/*.test.ts",
|
||||
"src/{apktool,jeb,adb,artifacts,browser,dotnet,evm,firmware,flutter,pcap,ghidra,go,hopper,ida,inspector,javascript,native,process,reference,rust,server,wasm,windows}/**/*.test.ts",
|
||||
],
|
||||
},
|
||||
{
|
||||
name: "composition",
|
||||
group: "source",
|
||||
include: ["tests/composition/**/*.test.ts"],
|
||||
},
|
||||
{
|
||||
name: "conformance",
|
||||
group: "source",
|
||||
include: ["tests/conformance/**/*.test.ts"],
|
||||
exclude: ["tests/conformance/providers/hopper/realHopperSearch.test.ts"],
|
||||
},
|
||||
{
|
||||
name: "evaluation",
|
||||
group: "source",
|
||||
include: ["tests/evaluation/**/*.test.ts"],
|
||||
},
|
||||
];
|
||||
|
||||
export const BUILT_TEST_PROJECTS = [
|
||||
{
|
||||
name: "boundary",
|
||||
group: "built",
|
||||
include: [
|
||||
"tests/boundary/**/*.test.ts",
|
||||
"tests/conformance/providers/hopper/realHopperSearch.test.ts",
|
||||
],
|
||||
exclude: [
|
||||
"tests/boundary/cli/**/*.test.ts",
|
||||
"tests/boundary/mcp/**/*.test.ts",
|
||||
"tests/boundary/process/**/*.test.ts",
|
||||
],
|
||||
},
|
||||
{
|
||||
name: "cli-boundary",
|
||||
group: "built",
|
||||
include: ["tests/boundary/cli/**/*.test.ts"],
|
||||
},
|
||||
{
|
||||
name: "process-boundary",
|
||||
group: "built",
|
||||
include: ["tests/boundary/process/**/*.test.ts"],
|
||||
},
|
||||
{
|
||||
name: "mcp-boundary",
|
||||
group: "built",
|
||||
include: ["tests/boundary/mcp/**/*.test.ts"],
|
||||
},
|
||||
{
|
||||
name: "acceptance",
|
||||
group: "built",
|
||||
include: ["tests/acceptance/**/*.test.ts"],
|
||||
},
|
||||
{
|
||||
name: "process-global",
|
||||
group: "built",
|
||||
include: ["tests/process-global/**/*.test.ts"],
|
||||
},
|
||||
];
|
||||
|
||||
const ALL_TEST_PROJECTS = [...SOURCE_TEST_PROJECTS, ...BUILT_TEST_PROJECTS];
|
||||
|
||||
/** Return the unique Vitest owner for an exact repository-relative test path. */
|
||||
export const projectForTest = (testPath) => {
|
||||
if (
|
||||
typeof testPath !== "string" ||
|
||||
testPath.startsWith("/") ||
|
||||
testPath.includes("\\") ||
|
||||
testPath.split("/").some((part) => part === ".." || part === ".") ||
|
||||
!testPath.endsWith(".test.ts")
|
||||
)
|
||||
throw new Error(`Expected a normalized repository test path: ${testPath}`);
|
||||
|
||||
const owners = ALL_TEST_PROJECTS.filter(
|
||||
({ include, exclude = [] }) =>
|
||||
include.some((pattern) => matchesGlob(testPath, pattern)) &&
|
||||
!exclude.some((pattern) => matchesGlob(testPath, pattern)),
|
||||
);
|
||||
if (owners.length !== 1)
|
||||
throw new Error(
|
||||
owners.length === 0
|
||||
? `No test project owns ${testPath}`
|
||||
: `Multiple test projects own ${testPath}: ${owners.map(({ name }) => name).join(", ")}`,
|
||||
);
|
||||
return owners[0];
|
||||
};
|
||||
|
|
@ -961,7 +961,7 @@ it("does not impose a message-count cutoff below the byte budget", () => {
|
|||
);
|
||||
expect(
|
||||
boundedViewMessages(messages, `ev_${"a".repeat(64)}`, "limitations"),
|
||||
).toEqual(messages);
|
||||
).toEqual(messages.toSorted());
|
||||
});
|
||||
|
||||
it("bounds repeated view messages even when fewer than 64 source strings are huge", () => {
|
||||
|
|
|
|||
|
|
@ -45,6 +45,7 @@ const fixture = async () => {
|
|||
for (const path of [
|
||||
"scripts/run-development-tests.mjs",
|
||||
"scripts/lib/development-tests.mjs",
|
||||
"scripts/lib/test-projects.mjs",
|
||||
])
|
||||
await copyFile(join(process.cwd(), path), join(root, path));
|
||||
await symlink(
|
||||
|
|
|
|||
|
|
@ -0,0 +1,419 @@
|
|||
import { readFile } from "node:fs/promises";
|
||||
import { join } from "node:path";
|
||||
|
||||
import { expect, it } from "vitest";
|
||||
import { createTestTempDirectory } from "../../fixtures/temporaryDirectory.js";
|
||||
import { normalizeWorkflowRun } from "../../../scripts/ci/workflow-run.mjs";
|
||||
|
||||
import {
|
||||
readStaticCheckArtifact,
|
||||
selectStaticCheckArtifact,
|
||||
} from "../../../scripts/ci/artifacts.mjs";
|
||||
import {
|
||||
classifyFailure,
|
||||
createGitHubApi,
|
||||
reportCiFailures,
|
||||
type CiRun,
|
||||
type CiJob,
|
||||
type FailureReportApi,
|
||||
} from "../../../scripts/ci/failure-report.mjs";
|
||||
|
||||
const sha = (digit: string) => digit.repeat(40);
|
||||
const diagnostic = {
|
||||
kind: "lint",
|
||||
path: "src/example.ts",
|
||||
code: "no-debugger",
|
||||
message: "Unexpected debugger statement.",
|
||||
severity: "error",
|
||||
} as const;
|
||||
const context = (
|
||||
sourceSha: string,
|
||||
taskHash = "a".repeat(16),
|
||||
prHeadSha?: string,
|
||||
) => ({
|
||||
version: 1 as const,
|
||||
sourceSha,
|
||||
...(prHeadSha ? { prHeadSha } : {}),
|
||||
sourceClean: true,
|
||||
nodeVersion: "v24.18.0",
|
||||
npmVersion: "11.16.0",
|
||||
turboVersion: "2.11.7",
|
||||
platform: "linux",
|
||||
architecture: "x64",
|
||||
taskHashes: { "lint:built": taskHash },
|
||||
});
|
||||
const run = (overrides: Partial<CiRun> = {}) =>
|
||||
normalizeWorkflowRun({
|
||||
id: 10,
|
||||
run_attempt: 1,
|
||||
status: "completed",
|
||||
conclusion: "failure",
|
||||
event: "pull_request",
|
||||
path: ".github/workflows/ci.yml@main",
|
||||
head_sha: sha("a"),
|
||||
head_branch: "feature",
|
||||
repository: { full_name: "owner/rea" },
|
||||
pull_requests: [
|
||||
{ base: { sha: sha("b"), repo: { full_name: "owner/rea" } } },
|
||||
],
|
||||
...overrides,
|
||||
});
|
||||
const job = (overrides: Partial<CiJob> = {}) => ({
|
||||
name: "Static checks",
|
||||
status: "completed",
|
||||
conclusion: "failure",
|
||||
...overrides,
|
||||
});
|
||||
|
||||
const comparison = (): Parameters<typeof classifyFailure>[0] => ({
|
||||
repository: "owner/rea",
|
||||
defaultBranch: "main",
|
||||
run: run(),
|
||||
job: job(),
|
||||
diagnostics: [diagnostic],
|
||||
context: context(sha("a")),
|
||||
mainRun: run({
|
||||
id: 20,
|
||||
event: "push",
|
||||
head_sha: sha("c"),
|
||||
head_branch: "main",
|
||||
pull_requests: [],
|
||||
}),
|
||||
mainJob: job(),
|
||||
mainDiagnostics: [diagnostic],
|
||||
mainContext: context(sha("c")),
|
||||
executionSourcesVerified: true,
|
||||
ancestry: { status: "ahead", mergeBaseSha: sha("c") },
|
||||
blobs: { "src/example.ts": true },
|
||||
});
|
||||
|
||||
it("attributes only an identical structured lint failure with matching task, context, ancestry, and source blob", () => {
|
||||
const result = classifyFailure(comparison());
|
||||
expect(result).toEqual({
|
||||
classification: "matching-main-diagnostics",
|
||||
reason:
|
||||
"supported_diagnostics_match_failed_task_and_source_on_verified_main_ancestor",
|
||||
});
|
||||
});
|
||||
|
||||
it.each([
|
||||
["task hash", { mainContext: context(sha("c"), "b".repeat(16)) }],
|
||||
["dirty source", { context: { ...context(sha("a")), sourceClean: false } }],
|
||||
[
|
||||
"toolchain",
|
||||
{ mainContext: { ...context(sha("c")), turboVersion: "other" } },
|
||||
],
|
||||
[
|
||||
"source sha",
|
||||
{ context: context(sha("d")), executionSourcesVerified: false },
|
||||
],
|
||||
[
|
||||
"diagnostic",
|
||||
{ mainDiagnostics: [{ ...diagnostic, message: "different message" }] },
|
||||
],
|
||||
["content", { blobs: { "src/example.ts": false } }],
|
||||
["ancestry", { ancestry: { status: "diverged", mergeBaseSha: sha("e") } }],
|
||||
["canceled job", { job: job({ conclusion: "cancelled" }) }],
|
||||
["not a static job", { job: job({ name: "Test shard (1)" }) }],
|
||||
] as const)("does not attribute on mismatched %s", (_label, overrides) => {
|
||||
const result = classifyFailure({ ...comparison(), ...overrides });
|
||||
expect(result.classification).toBe("unknown");
|
||||
});
|
||||
|
||||
it("writes informational JSON and Markdown from the supplied manifest using only read API calls", async () => {
|
||||
const directory = await createTestTempDirectory("rea-ci-failure-test-");
|
||||
const calls: string[] = [];
|
||||
const manifest = {
|
||||
repository: "owner/rea",
|
||||
truncation: [],
|
||||
coverage: { exhaustive: false },
|
||||
runs: [
|
||||
{
|
||||
id: 10,
|
||||
runAttempt: 1,
|
||||
workflow: "CI",
|
||||
event: "pull_request",
|
||||
conclusion: "failure",
|
||||
status: "collected",
|
||||
},
|
||||
{
|
||||
id: 11,
|
||||
runAttempt: 1,
|
||||
workflow: "CI",
|
||||
event: "pull_request",
|
||||
conclusion: "success",
|
||||
status: "collected",
|
||||
},
|
||||
],
|
||||
};
|
||||
const failedRun = run();
|
||||
const api: FailureReportApi = {
|
||||
async getRun(_repository, id) {
|
||||
calls.push(`run:${id}`);
|
||||
return failedRun;
|
||||
},
|
||||
async getJobs(_repository, id) {
|
||||
calls.push(`jobs:${id}`);
|
||||
return [{ ...job(), run_id: Number(id), run_attempt: 1 }];
|
||||
},
|
||||
async getStaticEvidence() {
|
||||
calls.push("artifact");
|
||||
return null;
|
||||
},
|
||||
async getAncestry() {
|
||||
throw new Error("must not compare without evidence");
|
||||
},
|
||||
async verifyExecutionSource() {
|
||||
throw new Error("must not verify source without evidence");
|
||||
},
|
||||
async getBlobIdentity() {
|
||||
throw new Error("must not inspect blobs without evidence");
|
||||
},
|
||||
};
|
||||
const report = await reportCiFailures({
|
||||
repository: "owner/rea",
|
||||
defaultBranch: "main",
|
||||
batchManifest: manifest,
|
||||
api,
|
||||
outputDirectory: directory,
|
||||
});
|
||||
expect(calls).toEqual(["run:10", "jobs:10", "artifact"]);
|
||||
expect(report).toMatchObject({
|
||||
informationalOnly: true,
|
||||
gateEffect: "none; this report never changes CI conclusions",
|
||||
counts: { unknown: 1 },
|
||||
});
|
||||
const json = JSON.parse(
|
||||
await readFile(join(directory, "report.json"), "utf8"),
|
||||
);
|
||||
const markdown = await readFile(join(directory, "report.md"), "utf8");
|
||||
expect(json.failures[0]).toMatchObject({
|
||||
classification: "unknown",
|
||||
reason: "no_matching_verified_main_failure",
|
||||
});
|
||||
expect(markdown).toContain("Informational only");
|
||||
expect(markdown).toContain("Unknown means evidence was incomplete");
|
||||
});
|
||||
|
||||
it.each([
|
||||
[22, false],
|
||||
[6, true],
|
||||
] as const)(
|
||||
"finds verified main evidence after %s newer PR runs (extra incorrect main hint: %s)",
|
||||
async (prCount, incorrectMainHints) => {
|
||||
const directory = await createTestTempDirectory("rea-ci-main-proof-");
|
||||
const mainSha = sha("c");
|
||||
const prHeadSha = sha("a");
|
||||
const mergeSha = sha("d");
|
||||
const mainRun = run({
|
||||
id: 20,
|
||||
event: "push",
|
||||
head_sha: mainSha,
|
||||
head_branch: "main",
|
||||
pull_requests: [],
|
||||
});
|
||||
const prRun = run({
|
||||
head_sha: prHeadSha,
|
||||
pull_requests: [
|
||||
{ base: { sha: mainSha, repo: { full_name: "owner/rea" } } },
|
||||
],
|
||||
});
|
||||
const evidence = (sourceSha: string, prHead?: string) => ({
|
||||
context: context(sourceSha, undefined, prHead),
|
||||
diagnostics: [diagnostic],
|
||||
});
|
||||
const lookups: number[] = [];
|
||||
const api: FailureReportApi = {
|
||||
async getRun(_repository, id) {
|
||||
lookups.push(Number(id));
|
||||
return Number(id) === 20 ? mainRun : { ...prRun, id };
|
||||
},
|
||||
async getJobs(_repository, id) {
|
||||
return [{ ...job(), run_id: Number(id), run_attempt: 1 }];
|
||||
},
|
||||
async getStaticEvidence(_repository, selectedRun) {
|
||||
return Number(selectedRun.id) === 20
|
||||
? evidence(mainSha)
|
||||
: evidence(mergeSha, prHeadSha);
|
||||
},
|
||||
async verifyExecutionSource(_repository, selectedRun, sourceContext) {
|
||||
return selectedRun.event === "pull_request"
|
||||
? sourceContext.sourceSha === mergeSha &&
|
||||
sourceContext.prHeadSha === selectedRun.head_sha
|
||||
: sourceContext.sourceSha === selectedRun.head_sha;
|
||||
},
|
||||
async getAncestry() {
|
||||
return { status: "ahead", mergeBaseSha: mainSha };
|
||||
},
|
||||
async getBlobIdentity() {
|
||||
return { "src/example.ts": true };
|
||||
},
|
||||
};
|
||||
const result = await reportCiFailures({
|
||||
repository: "owner/rea",
|
||||
defaultBranch: "main",
|
||||
batchManifest: {
|
||||
repository: "owner/rea",
|
||||
runs: [
|
||||
...(incorrectMainHints
|
||||
? [
|
||||
{
|
||||
id: 30,
|
||||
runAttempt: 1,
|
||||
workflow: "CI",
|
||||
event: "push",
|
||||
headBranch: "main",
|
||||
conclusion: "failure",
|
||||
status: "collected",
|
||||
},
|
||||
]
|
||||
: []),
|
||||
...Array.from({ length: prCount }, (_, index) => ({
|
||||
id: 100 + index,
|
||||
runAttempt: 1,
|
||||
workflow: "CI",
|
||||
event: "pull_request",
|
||||
headBranch: "feature",
|
||||
conclusion: "failure",
|
||||
status: "collected",
|
||||
})),
|
||||
{
|
||||
id: 20,
|
||||
runAttempt: 1,
|
||||
workflow: "CI",
|
||||
event: "push",
|
||||
headBranch: "main",
|
||||
conclusion: "failure",
|
||||
status: "collected",
|
||||
},
|
||||
],
|
||||
},
|
||||
api,
|
||||
outputDirectory: directory,
|
||||
});
|
||||
expect(result.failures[0]).toMatchObject({
|
||||
runId: 100,
|
||||
classification: "matching-main-diagnostics",
|
||||
reason:
|
||||
"supported_diagnostics_match_failed_task_and_source_on_verified_main_ancestor",
|
||||
});
|
||||
expect(lookups).toHaveLength(
|
||||
Math.min(prCount, 20) + 1 + Number(incorrectMainHints),
|
||||
);
|
||||
expect(lookups[0]).toBe(incorrectMainHints ? 30 : 20);
|
||||
expect(new Set(lookups).size).toBe(lookups.length);
|
||||
expect(result.boundedInput.mainCandidateRuns).toBe(1);
|
||||
expect(result.failures).toHaveLength(Math.min(prCount, 20));
|
||||
},
|
||||
);
|
||||
|
||||
it("proves checkout merge parents in base-then-PR-head order and rejects false identities", async () => {
|
||||
const baseSha = sha("b");
|
||||
const headSha = sha("a");
|
||||
const mergeSha = sha("c");
|
||||
const prRun = run({
|
||||
head_sha: headSha,
|
||||
pull_requests: [
|
||||
{ base: { sha: baseSha, repo: { full_name: "owner/rea" } } },
|
||||
],
|
||||
});
|
||||
let parents = [baseSha, headSha];
|
||||
const calls: string[] = [];
|
||||
const api = createGitHubApi({
|
||||
token: "fixture-token",
|
||||
apiRequest: async (_method, endpoint) => {
|
||||
calls.push(endpoint);
|
||||
return {
|
||||
sha: mergeSha,
|
||||
parents: parents.map((parentSha) => ({ sha: parentSha })),
|
||||
};
|
||||
},
|
||||
});
|
||||
const mergeContext = context(mergeSha, undefined, headSha);
|
||||
expect(
|
||||
await api.verifyExecutionSource("owner/rea", prRun, mergeContext),
|
||||
).toBe(true);
|
||||
expect(calls).toEqual([`repos/owner/rea/commits/${mergeSha}`]);
|
||||
|
||||
parents = [headSha, baseSha];
|
||||
expect(
|
||||
await api.verifyExecutionSource("owner/rea", prRun, mergeContext),
|
||||
).toBe(false);
|
||||
expect(
|
||||
await api.verifyExecutionSource("owner/rea", prRun, {
|
||||
...mergeContext,
|
||||
prHeadSha: sha("e"),
|
||||
}),
|
||||
).toBe(false);
|
||||
});
|
||||
|
||||
it("uses static artifacts only when listing is complete, unique, and tied to this run attempt", () => {
|
||||
const artifact = {
|
||||
id: 12,
|
||||
name: "static-check-reports",
|
||||
expired: false,
|
||||
workflow_run: { id: 10, head_sha: sha("a") },
|
||||
};
|
||||
const runRecord = { id: 10, head_sha: sha("a") };
|
||||
expect(
|
||||
selectStaticCheckArtifact(
|
||||
{ total_count: 1, artifacts: [artifact] },
|
||||
runRecord,
|
||||
1,
|
||||
),
|
||||
).toBe(artifact);
|
||||
expect(
|
||||
selectStaticCheckArtifact(
|
||||
{ total_count: 2, artifacts: [artifact] },
|
||||
runRecord,
|
||||
1,
|
||||
),
|
||||
).toBeNull();
|
||||
expect(
|
||||
selectStaticCheckArtifact(
|
||||
{ total_count: 2, artifacts: [artifact, artifact] },
|
||||
runRecord,
|
||||
1,
|
||||
),
|
||||
).toBeNull();
|
||||
expect(
|
||||
selectStaticCheckArtifact(
|
||||
{
|
||||
total_count: 1,
|
||||
artifacts: [
|
||||
{ ...artifact, workflow_run: { id: 11, head_sha: sha("a") } },
|
||||
],
|
||||
},
|
||||
runRecord,
|
||||
1,
|
||||
),
|
||||
).toBeNull();
|
||||
expect(
|
||||
selectStaticCheckArtifact(
|
||||
{ total_count: 1, artifacts: [artifact] },
|
||||
runRecord,
|
||||
2,
|
||||
),
|
||||
).toBeNull();
|
||||
});
|
||||
|
||||
it("rejects artifact redirects that could send the bearer token to a non-HTTPS host", async () => {
|
||||
const calls: string[] = [];
|
||||
await expect(
|
||||
readStaticCheckArtifact({
|
||||
repository: "owner/rea",
|
||||
artifactId: 12,
|
||||
token: "fixture-token",
|
||||
fetchImpl: (async (url: string | URL | Request) => {
|
||||
calls.push(String(url));
|
||||
return new Response(null, {
|
||||
status: 302,
|
||||
headers: { location: "http://untrusted.example/artifact.zip" },
|
||||
});
|
||||
}) as typeof fetch,
|
||||
}),
|
||||
).rejects.toThrow("static_artifact_redirect_invalid");
|
||||
expect(calls).toEqual([
|
||||
"https://api.github.com/repos/owner/rea/actions/artifacts/12/zip",
|
||||
]);
|
||||
});
|
||||
|
|
@ -0,0 +1,290 @@
|
|||
import { expect, it } from "vitest";
|
||||
import { createTestTempDirectory } from "../../fixtures/temporaryDirectory.js";
|
||||
import {
|
||||
reportCiFailures,
|
||||
type CiRun,
|
||||
type FailureReportApi,
|
||||
type StaticDiagnostic,
|
||||
type StaticEvidence,
|
||||
} from "../../../scripts/ci/failure-report.mjs";
|
||||
|
||||
const repository = "owner/rea";
|
||||
const mainSha = "c".repeat(40);
|
||||
const prSha = "a".repeat(40);
|
||||
const diagnostic: StaticDiagnostic = {
|
||||
kind: "lint",
|
||||
path: "src/example.ts",
|
||||
code: "no-debugger",
|
||||
message: "Unexpected debugger statement.",
|
||||
severity: "error",
|
||||
};
|
||||
const evidence = (
|
||||
run: CiRun,
|
||||
diagnostics: readonly StaticDiagnostic[],
|
||||
): StaticEvidence => ({
|
||||
context: {
|
||||
version: 1,
|
||||
sourceSha: run.head_sha!,
|
||||
sourceClean: true,
|
||||
nodeVersion: "v24.18.0",
|
||||
npmVersion: "11.16.0",
|
||||
turboVersion: "2.11.7",
|
||||
platform: "linux",
|
||||
architecture: "x64",
|
||||
taskHashes: { "lint:built": "a".repeat(16) },
|
||||
},
|
||||
diagnostics,
|
||||
});
|
||||
const entry = (id: number, event = "pull_request", runAttempt = 1) => ({
|
||||
id,
|
||||
runAttempt,
|
||||
workflow: "CI",
|
||||
event,
|
||||
headBranch: event === "pull_request" ? "feature" : "main",
|
||||
status: "collected",
|
||||
conclusion: "failure",
|
||||
});
|
||||
|
||||
function fixture(mainIds: readonly number[], prCount: number) {
|
||||
const jobs: number[] = [];
|
||||
const artifacts: number[] = [];
|
||||
const paths: (readonly string[])[] = [];
|
||||
const runs: Array<
|
||||
Omit<ReturnType<typeof entry>, "event"> & { event?: string }
|
||||
> = [
|
||||
...mainIds.map((id) => entry(id, "schedule")),
|
||||
...Array.from({ length: prCount }, (_, index) => entry(100 + index)),
|
||||
];
|
||||
const api: FailureReportApi = {
|
||||
async getRun(_repository, id) {
|
||||
const selected = runs.find((run) => run.id === Number(id))!;
|
||||
return {
|
||||
id,
|
||||
run_attempt: selected.runAttempt,
|
||||
status: "completed",
|
||||
conclusion: "failure",
|
||||
...(selected.event ? { event: selected.event } : {}),
|
||||
head_branch: selected.headBranch,
|
||||
path: ".github/workflows/ci.yml@main",
|
||||
head_sha: mainIds.includes(Number(id)) ? mainSha : prSha,
|
||||
repository: { full_name: repository },
|
||||
pull_requests:
|
||||
selected.event === "pull_request"
|
||||
? [{ base: { sha: mainSha, repo: { full_name: repository } } }]
|
||||
: [],
|
||||
};
|
||||
},
|
||||
async getJobs(_repository, id, attempt) {
|
||||
jobs.push(Number(id));
|
||||
return [
|
||||
{
|
||||
name: "Static checks",
|
||||
status: "completed",
|
||||
conclusion: "failure",
|
||||
run_id: Number(id),
|
||||
run_attempt: attempt,
|
||||
},
|
||||
];
|
||||
},
|
||||
async getStaticEvidence(_repository, run) {
|
||||
artifacts.push(Number(run.id));
|
||||
return evidence(run, [diagnostic]);
|
||||
},
|
||||
async getAncestry() {
|
||||
return { status: "identical", mergeBaseSha: mainSha };
|
||||
},
|
||||
async getBlobIdentity(_repository, _ancestor, _descendant, requested) {
|
||||
paths.push(requested);
|
||||
return Object.fromEntries(requested.map((path) => [path, true]));
|
||||
},
|
||||
async verifyExecutionSource() {
|
||||
return true;
|
||||
},
|
||||
};
|
||||
return { api, runs, jobs, artifacts, paths };
|
||||
}
|
||||
|
||||
async function report(input: ReturnType<typeof fixture>) {
|
||||
return await reportCiFailures({
|
||||
repository,
|
||||
defaultBranch: "main",
|
||||
batchManifest: { repository, runs: input.runs },
|
||||
api: input.api,
|
||||
outputDirectory: await createTestTempDirectory("rea-ci-budget-"),
|
||||
});
|
||||
}
|
||||
|
||||
it("selects PR failures before the cap even after a day of scheduled main failures", async () => {
|
||||
const input = fixture(
|
||||
Array.from({ length: 24 }, (_, index) => 20 + index),
|
||||
1,
|
||||
);
|
||||
input.runs.push({
|
||||
id: 200,
|
||||
runAttempt: 1,
|
||||
workflow: "CI",
|
||||
headBranch: "feature",
|
||||
status: "collected",
|
||||
conclusion: "failure",
|
||||
});
|
||||
const result = await report(input);
|
||||
expect(result.failures).toMatchObject([
|
||||
{ runId: 100, classification: "matching-main-diagnostics" },
|
||||
]);
|
||||
expect(result.boundedInput).toMatchObject({
|
||||
candidateRuns: 1,
|
||||
mainCandidateRuns: 5,
|
||||
manifestTruncated: false,
|
||||
});
|
||||
expect(input.jobs).toEqual([100, 20]);
|
||||
});
|
||||
|
||||
it("reuses five main job/artifact observations across twenty completed PR comparisons", async () => {
|
||||
const input = fixture([20, 21, 22, 23, 24], 20);
|
||||
const load = input.api.getStaticEvidence;
|
||||
input.api.getStaticEvidence = async (...args) => {
|
||||
const result = await load(...args);
|
||||
return Number(args[1].id) < 24
|
||||
? evidence(args[1], [{ ...diagnostic, message: "different" }])
|
||||
: result;
|
||||
};
|
||||
const result = await report(input);
|
||||
expect(result.counts).toEqual({
|
||||
"matching-main-diagnostics": 20,
|
||||
unknown: 0,
|
||||
});
|
||||
expect(input.paths).toHaveLength(100);
|
||||
expect(input.jobs).toHaveLength(25);
|
||||
expect(input.artifacts).toHaveLength(25);
|
||||
for (const id of [20, 21, 22, 23, 24]) {
|
||||
expect(input.jobs.filter((item) => item === id)).toHaveLength(1);
|
||||
expect(input.artifacts.filter((item) => item === id)).toHaveLength(1);
|
||||
}
|
||||
});
|
||||
|
||||
it.each(["missing", "artifact error", "job error", "wrong job attempt"])(
|
||||
"reuses unavailable main evidence without retries: %s",
|
||||
async (failure) => {
|
||||
const input = fixture([20], 3);
|
||||
const load = input.api.getStaticEvidence;
|
||||
const jobs = input.api.getJobs;
|
||||
input.api.getJobs = async (...args) => {
|
||||
const result = await jobs(...args);
|
||||
if (Number(args[1]) !== 20) return result;
|
||||
if (failure === "job error") throw new Error("job_snapshot_unavailable");
|
||||
return failure === "wrong job attempt"
|
||||
? result.map((job) => ({ ...job, run_attempt: 2 }))
|
||||
: result;
|
||||
};
|
||||
input.api.getStaticEvidence = async (...args) => {
|
||||
const result = await load(...args);
|
||||
if (Number(args[1].id) !== 20) return result;
|
||||
if (failure === "artifact error") throw new Error("artifact_unavailable");
|
||||
return null;
|
||||
};
|
||||
const result = await report(input);
|
||||
expect(result.counts.unknown).toBe(3);
|
||||
expect(input.jobs.filter((id) => id === 20)).toHaveLength(1);
|
||||
expect(input.artifacts.filter((id) => id === 20)).toHaveLength(
|
||||
failure === "job error" || failure === "wrong job attempt" ? 0 : 1,
|
||||
);
|
||||
},
|
||||
);
|
||||
|
||||
it.each([1, 2])(
|
||||
"refreshes unavailable main evidence in the next report (attempt %s)",
|
||||
async (attempt) => {
|
||||
const input = fixture([20], 1);
|
||||
expect((await report(input)).counts["matching-main-diagnostics"]).toBe(1);
|
||||
input.runs[0]!.runAttempt = attempt;
|
||||
const load = input.api.getStaticEvidence;
|
||||
input.api.getStaticEvidence = async (...args) => {
|
||||
const result = await load(...args);
|
||||
return Number(args[1].id) === 20 ? null : result;
|
||||
};
|
||||
expect((await report(input)).counts.unknown).toBe(1);
|
||||
expect(input.jobs.filter((id) => id === 20)).toHaveLength(2);
|
||||
expect(input.artifacts.filter((id) => id === 20)).toHaveLength(2);
|
||||
},
|
||||
);
|
||||
|
||||
it.each([1, 51])(
|
||||
"fetches only PR paths with %s PR diagnostics and noisy main evidence",
|
||||
async (count) => {
|
||||
const input = fixture([20], 1);
|
||||
const load = input.api.getStaticEvidence;
|
||||
const diagnostics = Array.from({ length: count }, (_, index) => ({
|
||||
...diagnostic,
|
||||
path: `src/pr-${index}.ts`,
|
||||
}));
|
||||
input.api.getStaticEvidence = async (...args) => {
|
||||
await load(...args);
|
||||
const noise =
|
||||
Number(args[1].id) === 20
|
||||
? Array.from({ length: 60 }, (_, index) => ({
|
||||
...diagnostic,
|
||||
path: `src/main-${index}.ts`,
|
||||
}))
|
||||
: [];
|
||||
return evidence(args[1], [...noise, ...diagnostics]);
|
||||
};
|
||||
const result = await report(input);
|
||||
expect(input.paths).toEqual([
|
||||
diagnostics.slice(0, 50).map((item) => item.path),
|
||||
]);
|
||||
expect(result.failures[0]!.classification).toBe(
|
||||
count === 1 ? "matching-main-diagnostics" : "unknown",
|
||||
);
|
||||
},
|
||||
);
|
||||
|
||||
it.each(["jobs", "artifact", "comparison"])(
|
||||
"continues after an isolated main %s failure and preserves its provenance",
|
||||
async (failure) => {
|
||||
const input = fixture([20, 21], 3);
|
||||
const jobs = input.api.getJobs;
|
||||
const load = input.api.getStaticEvidence;
|
||||
input.api.getJobs = async (...args) => {
|
||||
const result = await jobs(...args);
|
||||
if (Number(args[1]) === 20 && failure === "jobs")
|
||||
throw new Error("main_jobs_unavailable");
|
||||
return result;
|
||||
};
|
||||
input.api.getStaticEvidence = async (...args) => {
|
||||
const result = await load(...args);
|
||||
if (Number(args[1].id) === 20 && failure === "artifact")
|
||||
throw new Error("main_artifact_unavailable");
|
||||
return Number(args[1].id) === 20 && failure === "comparison"
|
||||
? {
|
||||
...result!,
|
||||
context: { ...result!.context, sourceSha: "d".repeat(40) },
|
||||
}
|
||||
: result;
|
||||
};
|
||||
input.api.verifyExecutionSource = async (_repository, run, context) => {
|
||||
if (Number(run.id) === 20 && context.sourceSha !== run.head_sha)
|
||||
throw new Error("main_comparison_unavailable");
|
||||
return true;
|
||||
};
|
||||
const result = await report(input);
|
||||
expect(result.counts).toEqual({
|
||||
"matching-main-diagnostics": 3,
|
||||
unknown: 0,
|
||||
});
|
||||
for (const item of result.failures) {
|
||||
expect(item.candidateFailures).toEqual([
|
||||
{
|
||||
runId: 20,
|
||||
runAttempt: 1,
|
||||
reason: `main_${failure}_unavailable`,
|
||||
},
|
||||
]);
|
||||
}
|
||||
expect(input.jobs.filter((id) => id === 20)).toHaveLength(1);
|
||||
expect(input.jobs.filter((id) => id === 21)).toHaveLength(1);
|
||||
expect(input.artifacts.filter((id) => id === 20)).toHaveLength(
|
||||
failure === "jobs" ? 0 : 1,
|
||||
);
|
||||
expect(input.artifacts.filter((id) => id === 21)).toHaveLength(1);
|
||||
},
|
||||
);
|
||||
|
|
@ -0,0 +1,313 @@
|
|||
import { execFile } from "node:child_process";
|
||||
import { mkdir, readFile, readdir, rm, writeFile } from "node:fs/promises";
|
||||
import { join } from "node:path";
|
||||
import { fileURLToPath } from "node:url";
|
||||
import { promisify } from "node:util";
|
||||
import { runInNewContext } from "node:vm";
|
||||
import { expect, it } from "vitest";
|
||||
import { parse } from "yaml";
|
||||
import { z } from "zod";
|
||||
|
||||
import {
|
||||
parseStaticTaskContext,
|
||||
recordStaticTaskContext,
|
||||
} from "../../../scripts/ci/static-task-context.mjs";
|
||||
import { createGhApiRequest } from "../../../scripts/ci/github-api.mjs";
|
||||
import { createTestTempDirectory } from "../../fixtures/temporaryDirectory.js";
|
||||
|
||||
const execute = promisify(execFile);
|
||||
const scopeFlags = async () =>
|
||||
z
|
||||
.object({ CI_FLAGS: z.array(z.string()) })
|
||||
.parse(
|
||||
await import(
|
||||
new URL("../../../scripts/ci/scopes.mjs", import.meta.url).href
|
||||
),
|
||||
).CI_FLAGS;
|
||||
const workflow = async () =>
|
||||
z
|
||||
.object({
|
||||
jobs: z.record(
|
||||
z.string(),
|
||||
z
|
||||
.object({
|
||||
needs: z.union([z.string(), z.array(z.string())]).optional(),
|
||||
if: z.string().optional(),
|
||||
steps: z
|
||||
.array(
|
||||
z
|
||||
.object({
|
||||
run: z.string().optional(),
|
||||
env: z.record(z.string(), z.string()).optional(),
|
||||
})
|
||||
.passthrough(),
|
||||
)
|
||||
.optional(),
|
||||
})
|
||||
.passthrough(),
|
||||
),
|
||||
})
|
||||
.parse(
|
||||
parse(
|
||||
await readFile(
|
||||
new URL("../../../.github/workflows/ci.yml", import.meta.url),
|
||||
"utf8",
|
||||
),
|
||||
),
|
||||
).jobs;
|
||||
|
||||
it("starts source feedback before build and requires both groups without waiving missing jobs", async () => {
|
||||
const jobs = await workflow();
|
||||
expect(
|
||||
jobs.static?.steps?.find(
|
||||
(step) => step.run === "npm run ci:static-context",
|
||||
)?.["continue-on-error"],
|
||||
).toBe(true);
|
||||
expect(jobs["test-source"]?.needs).toBe("changes");
|
||||
expect(jobs["test-shard"]?.needs).toEqual(["changes", "build"]);
|
||||
expect(
|
||||
jobs["test-source"]?.steps?.some(
|
||||
(step) =>
|
||||
step.run === "npm run test:fast -- --reporter=default --reporter=blob",
|
||||
),
|
||||
).toBe(true);
|
||||
const required = fileURLToPath(
|
||||
new URL("../../../scripts/ci/required.mjs", import.meta.url),
|
||||
);
|
||||
const flags = await scopeFlags();
|
||||
for (const tier of ["fast", "full"]) {
|
||||
for (const result of [
|
||||
"success",
|
||||
"failure",
|
||||
"cancelled",
|
||||
"skipped",
|
||||
"missing",
|
||||
]) {
|
||||
const needs = Object.fromEntries(
|
||||
Object.keys(jobs)
|
||||
.filter((name) => name !== "required")
|
||||
.map((name) => [name, { result: "success" }]),
|
||||
);
|
||||
const outputs = {
|
||||
...Object.fromEntries(
|
||||
flags.map((flag) => [flag, tier === "full" ? "true" : "false"]),
|
||||
),
|
||||
tier,
|
||||
code: "true",
|
||||
};
|
||||
const context: Record<
|
||||
string,
|
||||
{ result: string; outputs?: Record<string, string> }
|
||||
> = { ...needs, changes: { result: "success", outputs } };
|
||||
if (result === "missing") delete context["test-source"];
|
||||
else context["test-source"] = { result };
|
||||
const run = execute(process.execPath, [required], {
|
||||
env: {
|
||||
...process.env,
|
||||
CI_JOBS: JSON.stringify(context),
|
||||
CI_QUALIFICATION_DIRECTORY: "",
|
||||
},
|
||||
});
|
||||
if (
|
||||
result === "success" ||
|
||||
(tier === "full" && ["skipped", "missing"].includes(result))
|
||||
)
|
||||
await expect(run).resolves.toMatchObject({ stderr: "" });
|
||||
else
|
||||
await expect(run).rejects.toMatchObject({
|
||||
code: 1,
|
||||
stderr: expect.stringContaining("test-source"),
|
||||
});
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
it("records the actual static task graph with explicit source identity and no ambient environment", async () => {
|
||||
const directory = await createTestTempDirectory("rea-ci-context-");
|
||||
const context = await recordStaticTaskContext({ outputDirectory: directory });
|
||||
const head = await execute("git", ["rev-parse", "HEAD"]);
|
||||
expect(context.sourceSha).toBe(head.stdout.trim());
|
||||
expect(context.taskHashes["lint:built"]).toMatch(/^[a-f0-9]{16}$/u);
|
||||
expect(context.taskHashes.jscpd).toMatch(/^[a-f0-9]{16}$/u);
|
||||
expect(context).not.toHaveProperty("environmentVariables");
|
||||
expect(context).not.toHaveProperty("env");
|
||||
expect(
|
||||
JSON.parse(await readFile(`${directory}/static-task-context.json`, "utf8")),
|
||||
).toEqual(context);
|
||||
for (const invalid of [
|
||||
undefined,
|
||||
{},
|
||||
{ ...context, sourceSha: "bad" },
|
||||
{ ...context, prHeadSha: "bad" },
|
||||
{ ...context, taskHashes: { jscpd: "stale" } },
|
||||
])
|
||||
expect(() => parseStaticTaskContext(invalid)).toThrow();
|
||||
});
|
||||
|
||||
it("rejects mutation methods at the shared GitHub transport before invoking the CLI", async () => {
|
||||
const request = createGhApiRequest({ env: { PATH: "" } });
|
||||
await expect(
|
||||
request("POST", "repos/owner/repo/actions/runs/1/rerun", {
|
||||
requests: 0,
|
||||
startedAt: Date.now(),
|
||||
limitMs: 1000,
|
||||
}),
|
||||
).rejects.toThrow("read-only GET");
|
||||
});
|
||||
|
||||
it("collects basic CI telemetry before optional dependency installation can fail", async () => {
|
||||
const { jobs } = z
|
||||
.object({
|
||||
jobs: z.object({
|
||||
report: z.object({
|
||||
steps: z.array(
|
||||
z.object({
|
||||
id: z.string().optional(),
|
||||
run: z.string().optional(),
|
||||
if: z.string().optional(),
|
||||
"continue-on-error": z.boolean().optional(),
|
||||
}),
|
||||
),
|
||||
}),
|
||||
}),
|
||||
})
|
||||
.parse(
|
||||
parse(
|
||||
await readFile(
|
||||
new URL("../../../.github/workflows/ci-timings.yml", import.meta.url),
|
||||
"utf8",
|
||||
),
|
||||
),
|
||||
);
|
||||
const steps = jobs.report.steps;
|
||||
const installation = steps.findIndex((step) => step.id === "failure-deps");
|
||||
for (const script of ["pr-ci-health.mjs", "job-timings-batch.mjs"]) {
|
||||
const collection = steps.findIndex(
|
||||
(step) => step.run === `node scripts/ci/${script}`,
|
||||
);
|
||||
expect(collection).toBeGreaterThanOrEqual(0);
|
||||
expect(installation).toBeGreaterThan(collection);
|
||||
}
|
||||
expect(steps[installation]).toMatchObject({
|
||||
run: "npm ci",
|
||||
"continue-on-error": true,
|
||||
});
|
||||
expect(
|
||||
steps.find((step) => step.id === "npm-cache")?.["continue-on-error"],
|
||||
).toBe(true);
|
||||
expect(
|
||||
steps.find((step) => step.run === "node scripts/ci/failure-report.mjs")?.if,
|
||||
).toContain("steps.failure-deps.outcome == 'success'");
|
||||
});
|
||||
|
||||
it("merges source and built reports exactly once using the workflow's shard matrix", async () => {
|
||||
const jobs = await workflow();
|
||||
const matrix = z
|
||||
.object({
|
||||
matrix: z.object({ shard: z.string().regex(/^\$\{\{.*\}\}$/u) }),
|
||||
})
|
||||
.parse(jobs["test-shard"]?.strategy).matrix.shard;
|
||||
const shards = z
|
||||
.array(z.string().regex(/^\d+\/\d+$/u))
|
||||
.min(1)
|
||||
.parse(
|
||||
runInNewContext(
|
||||
matrix.slice(3, -2),
|
||||
{
|
||||
fromJSON: JSON.parse,
|
||||
needs: { changes: { outputs: { tier: "fast" } } },
|
||||
},
|
||||
{ timeout: 1_000 },
|
||||
),
|
||||
);
|
||||
const labels = Object.fromEntries(
|
||||
["source", "built"].map((group) => [
|
||||
group,
|
||||
z
|
||||
.string()
|
||||
.min(1)
|
||||
.parse(
|
||||
jobs[group === "source" ? "test-source" : "test-shard"]?.steps?.find(
|
||||
(step) => step.env?.VITEST_BLOB_LABEL,
|
||||
)?.env?.VITEST_BLOB_LABEL,
|
||||
),
|
||||
]),
|
||||
);
|
||||
const root = await createTestTempDirectory("rea-ci-group-merge-");
|
||||
const config = join(root, "vitest.config.mjs");
|
||||
await writeFile(
|
||||
config,
|
||||
`export default { test: { maxWorkers: 1, projects: [${["source", "built"].map((name) => `{ test: { name: "${name}", globals: true, include: ["${name}*.test.js"] } }`).join(",")}] } };`,
|
||||
);
|
||||
const files = [
|
||||
"source.test.js",
|
||||
...Array.from(
|
||||
{ length: shards.length },
|
||||
(_, index) => `built-${index}.test.js`,
|
||||
),
|
||||
];
|
||||
for (const file of files)
|
||||
await writeFile(
|
||||
join(root, file),
|
||||
'it("runs its selected group", () => expect(true).toBe(true));',
|
||||
);
|
||||
const vitest = fileURLToPath(
|
||||
new URL("../../../node_modules/vitest/vitest.mjs", import.meta.url),
|
||||
);
|
||||
const blobs = new Map<string, Buffer>();
|
||||
const runs = [
|
||||
{ group: "source", shard: [] },
|
||||
...shards.map((shard) => ({ group: "built", shard: [`--shard=${shard}`] })),
|
||||
];
|
||||
for (const { group, shard } of runs) {
|
||||
await execute(
|
||||
process.execPath,
|
||||
[
|
||||
vitest,
|
||||
"run",
|
||||
"--config",
|
||||
config,
|
||||
"--project",
|
||||
group,
|
||||
"--reporter=blob",
|
||||
...shard,
|
||||
],
|
||||
{
|
||||
cwd: root,
|
||||
env: { ...process.env, VITEST_BLOB_LABEL: labels[group]! },
|
||||
timeout: 20_000,
|
||||
},
|
||||
);
|
||||
for (const filename of await readdir(join(root, ".vitest/blob"))) {
|
||||
expect(blobs.has(filename)).toBe(false);
|
||||
blobs.set(filename, await readFile(join(root, ".vitest/blob", filename)));
|
||||
}
|
||||
// Separate hosted jobs start with independent report directories.
|
||||
await rm(join(root, ".vitest/blob"), { recursive: true });
|
||||
}
|
||||
expect(blobs.size).toBe(runs.length);
|
||||
await mkdir(join(root, ".vitest/blob"), { recursive: true });
|
||||
for (const [filename, bytes] of blobs)
|
||||
await writeFile(join(root, ".vitest/blob", filename), bytes);
|
||||
await execute(
|
||||
process.execPath,
|
||||
[
|
||||
vitest,
|
||||
"--config",
|
||||
config,
|
||||
"--merge-reports",
|
||||
"--reporter=json",
|
||||
"--outputFile.json=merged.json",
|
||||
],
|
||||
{ cwd: root, timeout: 20_000 },
|
||||
);
|
||||
const merged = z
|
||||
.object({
|
||||
success: z.literal(true),
|
||||
testResults: z.array(z.object({ name: z.string() })),
|
||||
})
|
||||
.parse(JSON.parse(await readFile(join(root, "merged.json"), "utf8")));
|
||||
expect(
|
||||
merged.testResults.map(({ name }) => name.split("/").at(-1)).sort(),
|
||||
).toEqual(files.toSorted());
|
||||
});
|
||||
|
|
@ -18,6 +18,8 @@ it("collects the latest attempt for recent completed CI and Release runs into a
|
|||
workflow_runs: [
|
||||
{
|
||||
id: 42,
|
||||
event: "push",
|
||||
head_branch: "main",
|
||||
run_attempt: 3,
|
||||
status: "completed",
|
||||
conclusion: "failure",
|
||||
|
|
@ -43,51 +45,9 @@ it("collects the latest attempt for recent completed CI and Release runs into a
|
|||
},
|
||||
],
|
||||
},
|
||||
[
|
||||
{
|
||||
jobs: [
|
||||
{
|
||||
name: "Release lane",
|
||||
status: "completed",
|
||||
conclusion: "success",
|
||||
created_at: recent,
|
||||
started_at: recent,
|
||||
completed_at: new Date(Date.parse(recent) + 50_000).toISOString(),
|
||||
steps: [],
|
||||
},
|
||||
],
|
||||
},
|
||||
],
|
||||
[
|
||||
{
|
||||
jobs: [
|
||||
{
|
||||
name: "Failed lane",
|
||||
status: "completed",
|
||||
conclusion: "failure",
|
||||
created_at: recent,
|
||||
started_at: recent,
|
||||
completed_at: new Date(Date.parse(recent) + 55_000).toISOString(),
|
||||
steps: [],
|
||||
},
|
||||
],
|
||||
},
|
||||
],
|
||||
[
|
||||
{
|
||||
jobs: [
|
||||
{
|
||||
name: "Cancelled lane",
|
||||
status: "completed",
|
||||
conclusion: "cancelled",
|
||||
created_at: recent,
|
||||
started_at: recent,
|
||||
completed_at: new Date(Date.parse(recent) + 30_000).toISOString(),
|
||||
steps: [],
|
||||
},
|
||||
],
|
||||
},
|
||||
],
|
||||
completedJobs("Release lane", "success", recent, 50),
|
||||
completedJobs("Failed lane", "failure", recent, 55),
|
||||
completedJobs("Cancelled lane", "cancelled", recent, 30),
|
||||
]);
|
||||
const result = await runBatch(fixture);
|
||||
const manifest = JSON.parse(
|
||||
|
|
@ -116,8 +76,20 @@ it("collects the latest attempt for recent completed CI and Release runs into a
|
|||
expect(manifest.errors).toEqual([]);
|
||||
expect(manifest.runs).toMatchObject([
|
||||
{ id: 43, runAttempt: 2, conclusion: "success" },
|
||||
{ id: 42, runAttempt: 3, conclusion: "failure" },
|
||||
{ id: 41, runAttempt: 1, conclusion: "cancelled" },
|
||||
{
|
||||
id: 42,
|
||||
runAttempt: 3,
|
||||
conclusion: "failure",
|
||||
event: "push",
|
||||
headBranch: "main",
|
||||
},
|
||||
{
|
||||
id: 41,
|
||||
runAttempt: 1,
|
||||
conclusion: "cancelled",
|
||||
event: null,
|
||||
headBranch: null,
|
||||
},
|
||||
]);
|
||||
expect(manifest.runs).toEqual(
|
||||
expect.arrayContaining([
|
||||
|
|
@ -234,6 +206,31 @@ it("retains a partial manifest when its collection budget is exhausted", async (
|
|||
expect(manifest.truncation.join(" ")).toContain("Time budget");
|
||||
});
|
||||
|
||||
function completedJobs(
|
||||
name: string,
|
||||
conclusion: string,
|
||||
startedAt: string,
|
||||
seconds: number,
|
||||
) {
|
||||
return [
|
||||
{
|
||||
jobs: [
|
||||
{
|
||||
name,
|
||||
status: "completed",
|
||||
conclusion,
|
||||
created_at: startedAt,
|
||||
started_at: startedAt,
|
||||
completed_at: new Date(
|
||||
Date.parse(startedAt) + seconds * 1000,
|
||||
).toISOString(),
|
||||
steps: [],
|
||||
},
|
||||
],
|
||||
},
|
||||
];
|
||||
}
|
||||
|
||||
async function createFixture(responses: unknown[]) {
|
||||
const directory = await createTestTempDirectory("rea-ci-job-timings-batch-");
|
||||
const fakeBin = join(directory, "bin");
|
||||
|
|
|
|||
|
|
@ -0,0 +1,617 @@
|
|||
import { chmod, mkdir, readFile, writeFile } from "node:fs/promises";
|
||||
import { delimiter, join } from "node:path";
|
||||
import { fileURLToPath } from "node:url";
|
||||
|
||||
import { expect, it } from "vitest";
|
||||
|
||||
import { createTestTempDirectory } from "../../fixtures/temporaryDirectory.js";
|
||||
import {
|
||||
classifyPrCi,
|
||||
collectPrCiHealth,
|
||||
matchRunToPullRequest,
|
||||
writePrCiHealthReport,
|
||||
} from "../../../scripts/ci/pr-ci-health.mjs";
|
||||
import { createGhApiRequest } from "../../../scripts/ci/github-api.mjs";
|
||||
import type {
|
||||
PrCiPullRequest,
|
||||
PrCiWorkflowRun,
|
||||
} from "../../../scripts/ci/pr-ci-health.mjs";
|
||||
import { executeWorkflowFixture } from "../../support/workflowGit.js";
|
||||
|
||||
const reporter = fileURLToPath(
|
||||
new URL("../../../scripts/ci/pr-ci-health.mjs", import.meta.url),
|
||||
);
|
||||
const observedAt = Date.parse("2026-10-14T00:00:00Z");
|
||||
const headSha = "a".repeat(40);
|
||||
const mergeSha = "b".repeat(40);
|
||||
|
||||
it("classifies workflow states without confusing absent, pending, and completed CI", () => {
|
||||
const pull = validPull();
|
||||
const cases = [
|
||||
[
|
||||
"startup_failure",
|
||||
run({ status: "completed", conclusion: "startup_failure" }),
|
||||
],
|
||||
[
|
||||
"queued_pending_executing",
|
||||
run({ status: "in_progress", conclusion: null }),
|
||||
],
|
||||
["approval_required", run({ status: "waiting", conclusion: null })],
|
||||
["completed_failure", run({ status: "completed", conclusion: "failure" })],
|
||||
[
|
||||
"completed_canceled",
|
||||
run({ status: "completed", conclusion: "cancelled" }),
|
||||
],
|
||||
["completed_success", run({ status: "completed", conclusion: "success" })],
|
||||
["completed_other", run({ status: "completed", conclusion: "neutral" })],
|
||||
[
|
||||
"completed_other",
|
||||
run({ status: "completed", conclusion: "constructor" }),
|
||||
],
|
||||
] as const;
|
||||
for (const [category, evidence] of cases) {
|
||||
for (const runsComplete of [true, false]) {
|
||||
const result = classifyPrCi({
|
||||
pull,
|
||||
runs: [
|
||||
{
|
||||
run: { ...evidence, run_attempt: 1 },
|
||||
basis: "event_pr_number_and_head_sha",
|
||||
},
|
||||
],
|
||||
runsComplete,
|
||||
pullRequestInventoryComplete: true,
|
||||
ambiguousRuns: [],
|
||||
});
|
||||
expect(result).toMatchObject({
|
||||
category: runsComplete ? category : "unknown",
|
||||
...(runsComplete
|
||||
? {}
|
||||
: { reason: "workflow_run_inventory_incomplete" }),
|
||||
runs: [
|
||||
{
|
||||
id: 77,
|
||||
attempt: 1,
|
||||
status: evidence.status,
|
||||
conclusion: evidence.conclusion,
|
||||
},
|
||||
],
|
||||
});
|
||||
}
|
||||
}
|
||||
for (const runsComplete of [true, false]) {
|
||||
expect(
|
||||
classifyPrCi({
|
||||
pull: { ...pull, mergeable: false },
|
||||
runs: [],
|
||||
runsComplete,
|
||||
pullRequestInventoryComplete: true,
|
||||
ambiguousRuns: [],
|
||||
}).category,
|
||||
).toBe("merge_conflict");
|
||||
}
|
||||
});
|
||||
|
||||
it("requires complete inventories, known mergeability, and immutable head identity to report missing", () => {
|
||||
const pull = validPull({ mergeable: true });
|
||||
const classify = (overrides = {}) =>
|
||||
classifyPrCi({
|
||||
pull,
|
||||
runs: [],
|
||||
runsComplete: true,
|
||||
pullRequestInventoryComplete: true,
|
||||
ambiguousRuns: [],
|
||||
...overrides,
|
||||
});
|
||||
|
||||
expect(classify().category).toBe("missing");
|
||||
expect(classify({ runsComplete: false })).toMatchObject({
|
||||
category: "deferred",
|
||||
reason: "lookup_inventory_incomplete",
|
||||
});
|
||||
expect(classify({ pullRequestInventoryComplete: false }).category).toBe(
|
||||
"deferred",
|
||||
);
|
||||
expect(
|
||||
classify({
|
||||
ambiguousRuns: [{ run: run(), basis: "ambiguous_head_identity" }],
|
||||
}).category,
|
||||
).toBe("unknown");
|
||||
expect(classify({ pull: { ...pull, mergeable: undefined } }).category).toBe(
|
||||
"unknown",
|
||||
);
|
||||
expect(classify({ pull: { ...pull, identityValid: false } }).category).toBe(
|
||||
"unknown",
|
||||
);
|
||||
});
|
||||
|
||||
it("accepts event PR/head evidence for merge-SHA runs and refuses ambiguous branch identities", () => {
|
||||
const pull = validPull();
|
||||
const associatedMergeRun = run({
|
||||
head_sha: mergeSha,
|
||||
pull_requests: [
|
||||
{
|
||||
number: pull.number!,
|
||||
head: { sha: headSha, repo: { id: pull.head.repositoryId } },
|
||||
},
|
||||
],
|
||||
});
|
||||
expect(
|
||||
matchRunToPullRequest({
|
||||
run: associatedMergeRun,
|
||||
pull,
|
||||
identityCounts: new Map([["73\u0000feature", 1]]),
|
||||
}),
|
||||
).toEqual({ kind: "match", basis: "event_pr_number_and_head_sha" });
|
||||
expect(
|
||||
matchRunToPullRequest({
|
||||
run: run({
|
||||
pull_requests: [
|
||||
{ number: pull.number!, head: { sha: "c".repeat(40) } },
|
||||
],
|
||||
}),
|
||||
pull,
|
||||
identityCounts: new Map([["73\u0000feature", 1]]),
|
||||
}).kind,
|
||||
).toBe("none");
|
||||
|
||||
const branchRun = run({ pull_requests: [] });
|
||||
expect(
|
||||
matchRunToPullRequest({
|
||||
run: branchRun,
|
||||
pull,
|
||||
identityCounts: new Map([["73\u0000feature", 1]]),
|
||||
inventoryComplete: false,
|
||||
}).kind,
|
||||
).toBe("ambiguous");
|
||||
expect(
|
||||
matchRunToPullRequest({
|
||||
run: branchRun,
|
||||
pull,
|
||||
identityCounts: new Map([["73\u0000feature", 1]]),
|
||||
}),
|
||||
).toEqual({
|
||||
kind: "match",
|
||||
basis: "event_unique_head_ref_and_repository_id",
|
||||
});
|
||||
expect(
|
||||
matchRunToPullRequest({
|
||||
run: run({ head_sha: mergeSha, pull_requests: [] }),
|
||||
pull,
|
||||
identityCounts: new Map([["73\u0000feature", 1]]),
|
||||
}).kind,
|
||||
).toBe("none");
|
||||
expect(
|
||||
matchRunToPullRequest({
|
||||
run: run({
|
||||
pull_requests: [{ number: pull.number!, head: { sha: headSha } }],
|
||||
}),
|
||||
pull,
|
||||
identityCounts: new Map([["73\u0000feature", 1]]),
|
||||
}).kind,
|
||||
).toBe("ambiguous");
|
||||
expect(
|
||||
matchRunToPullRequest({
|
||||
run: branchRun,
|
||||
pull,
|
||||
identityCounts: new Map([["73\u0000feature", 2]]),
|
||||
}).kind,
|
||||
).toBe("ambiguous");
|
||||
expect(
|
||||
matchRunToPullRequest({
|
||||
run: run({ event: "workflow_dispatch" }),
|
||||
pull,
|
||||
identityCounts: new Map([["73\u0000feature", 1]]),
|
||||
}).kind,
|
||||
).toBe("none");
|
||||
});
|
||||
|
||||
it("scans target-branch PRs through public gh GET requests and writes JSON and Markdown", async () => {
|
||||
const fixture = await createFixture();
|
||||
const calls: string[] = [];
|
||||
const apiRequest: NonNullable<
|
||||
Parameters<typeof collectPrCiHealth>[0]["apiRequest"]
|
||||
> = async (method, endpoint) => {
|
||||
calls.push(`${method} ${endpoint}`);
|
||||
if (method !== "GET") throw new Error("fixture rejected a non-GET request");
|
||||
if (endpoint.includes("/pulls?"))
|
||||
return [
|
||||
rawPull(42, { mergeable: true }),
|
||||
rawPull(43, { mergeable: false }),
|
||||
rawPull(44, {
|
||||
base: { ref: "release", repo: { id: 11 } },
|
||||
mergeable: true,
|
||||
}),
|
||||
rawPull(45, { draft: true, mergeable: true }),
|
||||
];
|
||||
if (endpoint.includes("/actions/workflows/ci.yml/runs?"))
|
||||
return {
|
||||
total_count: 1,
|
||||
workflow_runs: [
|
||||
run({
|
||||
id: 9001,
|
||||
event: "pull_request",
|
||||
status: "completed",
|
||||
conclusion: "failure",
|
||||
head_sha: mergeSha,
|
||||
pull_requests: [
|
||||
{
|
||||
number: 42,
|
||||
head: { sha: headSha, repo: { id: 73 } },
|
||||
},
|
||||
],
|
||||
}),
|
||||
],
|
||||
};
|
||||
throw new Error(`unexpected GET endpoint: ${endpoint}`);
|
||||
};
|
||||
const report = await collectPrCiHealth({
|
||||
repository: "owner/rea",
|
||||
repositoryId: 11,
|
||||
defaultBranch: "main",
|
||||
now: observedAt,
|
||||
apiRequest,
|
||||
});
|
||||
expect(calls).toHaveLength(2);
|
||||
expect(calls.every((call) => call.startsWith("GET "))).toBe(true);
|
||||
expect(calls[0]).toContain("state=open");
|
||||
expect(calls[0]).not.toContain("base=");
|
||||
expect(calls[1]).toContain("event=pull_request");
|
||||
expect(report.pullRequests).toHaveLength(2);
|
||||
expect(report.pullRequests.map(({ number }) => number)).toEqual([42, 43]);
|
||||
expect(report.pullRequests.map(({ ci }) => ci.category)).toEqual([
|
||||
"completed_failure",
|
||||
"merge_conflict",
|
||||
]);
|
||||
expect(report.scan).toMatchObject({
|
||||
status: "complete",
|
||||
pullRequestInventoryComplete: true,
|
||||
workflowRunInventoryComplete: true,
|
||||
apiRequests: 2,
|
||||
});
|
||||
|
||||
const output = join(fixture.directory, ".cache", "ci-pr-health");
|
||||
const paths = await writePrCiHealthReport(report, output);
|
||||
const written = JSON.parse(await readFile(paths.jsonPath, "utf8"));
|
||||
const markdown = await readFile(paths.markdownPath, "utf8");
|
||||
expect(written.counts.completed_failure).toBe(1);
|
||||
expect(markdown).toContain("# Pull request CI health snapshot");
|
||||
expect(markdown).toContain("#42");
|
||||
expect(markdown).toContain("completed_failure");
|
||||
});
|
||||
|
||||
it("counts older open PRs when deciding whether a branch/ref fallback is unique", async () => {
|
||||
const olderPull = rawPull(63, {
|
||||
created_at: "2026-09-01T00:00:00Z",
|
||||
head: { ...rawPull(63).head, ref: "shared-feature" },
|
||||
});
|
||||
const currentPull = rawPull(64, {
|
||||
mergeable: true,
|
||||
head: { ...olderPull.head },
|
||||
});
|
||||
const apiRequest: NonNullable<
|
||||
Parameters<typeof collectPrCiHealth>[0]["apiRequest"]
|
||||
> = async (_method, endpoint) => {
|
||||
if (endpoint.includes("/pulls?")) return [currentPull, olderPull];
|
||||
if (endpoint.includes("/actions/workflows/ci.yml/runs?"))
|
||||
return {
|
||||
workflow_runs: [
|
||||
run({
|
||||
head_branch: "shared-feature",
|
||||
pull_requests: [],
|
||||
}),
|
||||
],
|
||||
};
|
||||
throw new Error(`unexpected GET endpoint: ${endpoint}`);
|
||||
};
|
||||
const report = await collectPrCiHealth({
|
||||
repository: "owner/rea",
|
||||
repositoryId: 11,
|
||||
defaultBranch: "main",
|
||||
now: observedAt,
|
||||
apiRequest,
|
||||
});
|
||||
expect(report.scan.pullRequestInventoryComplete).toBe(true);
|
||||
expect(report.pullRequests.map(({ number }) => number)).toEqual([64]);
|
||||
expect(report.pullRequests[0]?.ci).toMatchObject({
|
||||
category: "unknown",
|
||||
reason: "workflow_run_identity_ambiguous",
|
||||
});
|
||||
});
|
||||
|
||||
it.each([false, true])(
|
||||
"keeps page-capped scans uncertain (observed success: %s)",
|
||||
async (observedSuccess) => {
|
||||
const pull = rawPull(62, { mergeable: true });
|
||||
const apiRequest: NonNullable<
|
||||
Parameters<typeof collectPrCiHealth>[0]["apiRequest"]
|
||||
> = async (_method, endpoint) => {
|
||||
if (endpoint.includes("/pulls?")) return [pull];
|
||||
const page = Number(
|
||||
new URL(`https://fixture.invalid/${endpoint}`).searchParams.get("page"),
|
||||
);
|
||||
return {
|
||||
workflow_runs: Array.from({ length: 100 }, (_item, index) =>
|
||||
observedSuccess && page === 1 && index === 0
|
||||
? run({
|
||||
run_attempt: 1,
|
||||
pull_requests: [
|
||||
{ number: 62, head: { sha: headSha, repo: { id: 73 } } },
|
||||
],
|
||||
})
|
||||
: run({
|
||||
id: page * 100 + index,
|
||||
head_branch: "different-branch",
|
||||
head_repository: { id: 999 },
|
||||
pull_requests: [],
|
||||
}),
|
||||
),
|
||||
};
|
||||
};
|
||||
const report = await collectPrCiHealth({
|
||||
repository: "owner/rea",
|
||||
repositoryId: 11,
|
||||
defaultBranch: "main",
|
||||
now: observedAt,
|
||||
apiRequest,
|
||||
});
|
||||
expect(report.scan.workflowRunPages).toBe(5);
|
||||
expect(report.scan.workflowRunInventoryComplete).toBe(false);
|
||||
const firstPull = report.pullRequests[0];
|
||||
if (firstPull === undefined) throw new Error("fixture PR was not scanned");
|
||||
expect(firstPull.ci).toMatchObject({
|
||||
category: observedSuccess ? "unknown" : "deferred",
|
||||
reason: observedSuccess
|
||||
? "workflow_run_inventory_incomplete"
|
||||
: "lookup_inventory_incomplete",
|
||||
});
|
||||
expect(firstPull.ci.runs).toHaveLength(observedSuccess ? 1 : 0);
|
||||
},
|
||||
);
|
||||
|
||||
it("keeps the result unknown when the head changes between API snapshots", async () => {
|
||||
const apiRequest: NonNullable<
|
||||
Parameters<typeof collectPrCiHealth>[0]["apiRequest"]
|
||||
> = async (_method, endpoint) => {
|
||||
if (endpoint.includes("/pulls?")) return [rawPull(63)];
|
||||
if (endpoint.includes("/actions/workflows/ci.yml/runs?"))
|
||||
return { workflow_runs: [] };
|
||||
return {
|
||||
...rawPull(63, { mergeable: true }),
|
||||
head: {
|
||||
...rawPull(63).head,
|
||||
sha: mergeSha,
|
||||
},
|
||||
};
|
||||
};
|
||||
const report = await collectPrCiHealth({
|
||||
repository: "owner/rea",
|
||||
repositoryId: 11,
|
||||
defaultBranch: "main",
|
||||
now: observedAt,
|
||||
apiRequest,
|
||||
});
|
||||
expect(report.pullRequests[0]).toMatchObject({
|
||||
identityChangedDuringScan: true,
|
||||
ci: { category: "unknown", reason: "pull_request_head_identity_invalid" },
|
||||
});
|
||||
expect(report.scan).toMatchObject({
|
||||
status: "partial",
|
||||
errors: ["pull_request_identity_changed_during_scan"],
|
||||
});
|
||||
});
|
||||
|
||||
it("public CLI uses only GET and produces a report from fixture GitHub responses", async () => {
|
||||
const fixture = await createFixture();
|
||||
const ghCalls = join(fixture.directory, "gh-calls.jsonl");
|
||||
const ghResponse = join(fixture.directory, "gh-response.json");
|
||||
const fakeGh = join(fixture.fakeBin, "gh");
|
||||
await writeFile(
|
||||
ghResponse,
|
||||
JSON.stringify([
|
||||
[rawPull(51, { mergeable: true })],
|
||||
{ total_count: 0, workflow_runs: [] },
|
||||
]),
|
||||
);
|
||||
await writeFile(
|
||||
fakeGh,
|
||||
`#!/usr/bin/env node\nconst fs=require("node:fs");\nconst args=process.argv.slice(2);\nfs.appendFileSync(process.env.CI_GH_CALLS, JSON.stringify(args)+"\\n");\nif(args[0]!=="api"||args[1]!=="--method"||args[2]!=="GET")process.exit(41);\nconst page=Number(new URL("https://fixture.invalid/"+args.at(-1)).searchParams.get("page"));\nconst responses=JSON.parse(fs.readFileSync(process.env.CI_GH_RESPONSE,"utf8"));\nprocess.stdout.write(JSON.stringify(responses[page===1&&args.at(-1).includes("/pulls?")?0:1]));\n`,
|
||||
);
|
||||
await chmod(fakeGh, 0o755);
|
||||
const outputDirectory = join(fixture.directory, ".cache", "ci-pr-health");
|
||||
await executeWorkflowFixture(process.execPath, [reporter], {
|
||||
cwd: fixture.directory,
|
||||
env: {
|
||||
...process.env,
|
||||
PATH: `${fixture.fakeBin}${delimiter}${process.env.PATH ?? ""}`,
|
||||
CI_REPOSITORY: "owner/rea",
|
||||
CI_REPOSITORY_ID: "11",
|
||||
CI_DEFAULT_BRANCH: "main",
|
||||
CI_PR_HEALTH_DIRECTORY: outputDirectory,
|
||||
CI_GH_CALLS: ghCalls,
|
||||
CI_GH_RESPONSE: ghResponse,
|
||||
},
|
||||
});
|
||||
const calls = (await readFile(ghCalls, "utf8"))
|
||||
.trim()
|
||||
.split("\n")
|
||||
.map((line) => JSON.parse(line));
|
||||
expect(calls).toHaveLength(2);
|
||||
expect(
|
||||
calls.every(
|
||||
(args: string[]) => args[1] === "--method" && args[2] === "GET",
|
||||
),
|
||||
).toBe(true);
|
||||
const report = JSON.parse(
|
||||
await readFile(join(outputDirectory, "report.json"), "utf8"),
|
||||
);
|
||||
expect(report.pullRequests[0].ci.category).toBe("missing");
|
||||
await expect(
|
||||
readFile(join(outputDirectory, "summary.md"), "utf8"),
|
||||
).resolves.toContain("missing");
|
||||
});
|
||||
|
||||
it("rejects non-GET methods before launching the GitHub client", async () => {
|
||||
const fixture = await createFixture();
|
||||
const request = createGhApiRequest({
|
||||
env: {
|
||||
...process.env,
|
||||
PATH: `${fixture.fakeBin}${delimiter}${process.env.PATH ?? ""}`,
|
||||
CI_GH_CALLS: join(fixture.directory, "should-not-exist"),
|
||||
},
|
||||
});
|
||||
await expect(
|
||||
request("POST", "repos/owner/rea/pulls", {
|
||||
requests: 0,
|
||||
startedAt: observedAt,
|
||||
limitMs: 1_000,
|
||||
}),
|
||||
).rejects.toThrow("read-only GET");
|
||||
await expect(
|
||||
readFile(join(fixture.directory, "should-not-exist")),
|
||||
).rejects.toMatchObject({ code: "ENOENT" });
|
||||
});
|
||||
|
||||
function validPull(overrides: Partial<PrCiPullRequest> = {}): PrCiPullRequest {
|
||||
return {
|
||||
number: 42,
|
||||
identityValid: true,
|
||||
mergeable: true,
|
||||
head: { repositoryId: 73, ref: "feature", sha: headSha },
|
||||
...overrides,
|
||||
};
|
||||
}
|
||||
|
||||
it.each([
|
||||
["newer run", { id: 78, created_at: "2026-10-13T01:00:00Z" }, "unknown"],
|
||||
[
|
||||
"older run",
|
||||
{ id: 78, created_at: "2026-10-12T23:00:00Z" },
|
||||
"completed_success",
|
||||
],
|
||||
["equal timestamps", { id: 78 }, "unknown"],
|
||||
[
|
||||
"retry of an older run",
|
||||
{ id: 78, created_at: "2026-10-12T23:00:00Z", run_attempt: 2 },
|
||||
"unknown",
|
||||
],
|
||||
["unknown timestamp", { id: 78, created_at: "invalid" }, "unknown"],
|
||||
["newer attempt", { run_attempt: 3 }, "unknown"],
|
||||
["older attempt", { run_attempt: 1 }, "completed_success"],
|
||||
] as const)(
|
||||
"retains ambiguity evidence for a %s without reporting a stale completion",
|
||||
(_label, overrides, category) => {
|
||||
const currentAttempt = "id" in overrides ? 1 : 2;
|
||||
const result = classifyPrCi({
|
||||
pull: validPull(),
|
||||
runs: [
|
||||
{
|
||||
run: run({ run_attempt: currentAttempt }),
|
||||
basis: "event_pr_number_and_head_sha",
|
||||
},
|
||||
],
|
||||
ambiguousRuns: [
|
||||
{
|
||||
run: run({ run_attempt: currentAttempt, ...overrides }),
|
||||
basis: "ambiguous_head_identity",
|
||||
},
|
||||
],
|
||||
runsComplete: true,
|
||||
pullRequestInventoryComplete: true,
|
||||
});
|
||||
expect(result.category).toBe(category);
|
||||
expect(result.runs).toHaveLength(2);
|
||||
},
|
||||
);
|
||||
|
||||
it("does not let an older proven run hide a newer run with an unverified repository ID", async () => {
|
||||
const pull = rawPull(42);
|
||||
const older = run({
|
||||
id: 70,
|
||||
created_at: "2026-10-13T00:00:00Z",
|
||||
pull_requests: [{ number: 42, head: { sha: headSha, repo: { id: 73 } } }],
|
||||
});
|
||||
const newer = run({
|
||||
id: 71,
|
||||
created_at: "2026-10-13T01:00:00Z",
|
||||
status: "in_progress",
|
||||
conclusion: null,
|
||||
pull_requests: [{ number: 42, head: { sha: headSha } }],
|
||||
});
|
||||
const report = await collectPrCiHealth({
|
||||
repository: "owner/rea",
|
||||
repositoryId: 11,
|
||||
defaultBranch: "main",
|
||||
now: observedAt,
|
||||
apiRequest: async (_method, endpoint) => {
|
||||
if (endpoint.includes("/pulls?")) return [pull];
|
||||
if (endpoint.endsWith("/pulls/42")) return { ...pull, mergeable: true };
|
||||
return { workflow_runs: [newer, older] };
|
||||
},
|
||||
});
|
||||
expect(report.pullRequests[0]?.ci).toMatchObject({
|
||||
category: "unknown",
|
||||
reason: "workflow_run_identity_ambiguous",
|
||||
});
|
||||
expect(report.pullRequests[0]?.ci.runs).toHaveLength(2);
|
||||
});
|
||||
|
||||
it("keeps a proven newest run even when two older matched runs have equal timestamps", () => {
|
||||
const result = classifyPrCi({
|
||||
pull: validPull(),
|
||||
runs: [
|
||||
run({ id: 70, run_attempt: 1 }),
|
||||
run({ id: 71, run_attempt: 1, conclusion: "failure" }),
|
||||
run({
|
||||
id: 72,
|
||||
run_attempt: 1,
|
||||
created_at: "2026-10-13T01:00:00Z",
|
||||
status: "in_progress",
|
||||
conclusion: null,
|
||||
}),
|
||||
].map((run) => ({ run, basis: "event_pr_number_and_head_sha" })),
|
||||
ambiguousRuns: [],
|
||||
runsComplete: true,
|
||||
pullRequestInventoryComplete: true,
|
||||
});
|
||||
expect(result.category).toBe("queued_pending_executing");
|
||||
expect(result.runs).toHaveLength(3);
|
||||
});
|
||||
|
||||
function run(overrides: Partial<PrCiWorkflowRun> = {}): PrCiWorkflowRun {
|
||||
return {
|
||||
id: 77,
|
||||
event: "pull_request",
|
||||
head_branch: "feature",
|
||||
head_repository: { id: 73 },
|
||||
head_sha: headSha,
|
||||
status: "completed",
|
||||
conclusion: "success",
|
||||
created_at: "2026-10-13T00:00:00Z",
|
||||
pull_requests: [],
|
||||
...overrides,
|
||||
};
|
||||
}
|
||||
|
||||
function rawPull(number: number, overrides: Record<string, unknown> = {}) {
|
||||
return {
|
||||
number,
|
||||
draft: false,
|
||||
created_at: "2026-10-13T00:00:00Z",
|
||||
html_url: `https://github.com/owner/rea/pull/${String(number)}`,
|
||||
base: { ref: "main", repo: { id: 11 } },
|
||||
head: {
|
||||
ref: `feature-${String(number)}`,
|
||||
sha: headSha,
|
||||
repo: { id: 73, full_name: "contributor/rea" },
|
||||
},
|
||||
...overrides,
|
||||
};
|
||||
}
|
||||
|
||||
async function createFixture() {
|
||||
const directory = await createTestTempDirectory("rea-ci-pr-health-");
|
||||
const fakeBin = join(directory, "bin");
|
||||
await mkdir(fakeBin);
|
||||
return { directory, fakeBin };
|
||||
}
|
||||
|
|
@ -44,7 +44,7 @@ function run(overrides: Record<string, unknown> = {}) {
|
|||
id: 7,
|
||||
status: "completed",
|
||||
conclusion: "success",
|
||||
path: ".github/workflows/ci.yml",
|
||||
path: ".github/workflows/ci.yml@main",
|
||||
event: "schedule",
|
||||
created_at: new Date(now - 120_000).toISOString(),
|
||||
head_sha: validSha,
|
||||
|
|
|
|||
|
|
@ -121,7 +121,11 @@ it("exports every canonical scope with read authority for admission and keeps te
|
|||
);
|
||||
expect(timings.on.workflow_run).toBeUndefined();
|
||||
expect(timings.on.schedule).toHaveLength(1);
|
||||
expect(timings.permissions).toEqual({ contents: "read", actions: "read" });
|
||||
expect(timings.permissions).toEqual({
|
||||
contents: "read",
|
||||
actions: "read",
|
||||
"pull-requests": "read",
|
||||
});
|
||||
const checkout = timings.jobs.report.steps.find((step) =>
|
||||
step.uses?.startsWith("actions/checkout@"),
|
||||
);
|
||||
|
|
@ -154,6 +158,7 @@ it("holds native and provider fan-out behind static validation while portable fe
|
|||
"static",
|
||||
"build",
|
||||
"test-shard",
|
||||
"test-source",
|
||||
"test",
|
||||
"docs",
|
||||
"required",
|
||||
|
|
@ -369,7 +374,7 @@ it("keeps fast feedback separate from complete qualification and release proof",
|
|||
// release their concurrency slot without waiting for another runner.
|
||||
expect(ci.jobs.required?.if).toBe("${{ !cancelled() }}");
|
||||
const shards = ci.jobs["test-shard"]?.steps ?? [];
|
||||
const fast = shards.find((step) => step.run?.includes("npm run test:run"));
|
||||
const fast = shards.find((step) => step.run?.includes("npm run test:built"));
|
||||
const full = shards.find((step) => step.run?.includes("test:ci:shard:run"));
|
||||
expect(fast?.if).toBe("needs.changes.outputs.tier == 'fast'");
|
||||
expect(fast?.run).toContain("--reporter=blob");
|
||||
|
|
|
|||
|
|
@ -0,0 +1,99 @@
|
|||
import { execFile } from "node:child_process";
|
||||
import { mkdir, readFile, writeFile } from "node:fs/promises";
|
||||
import { join } from "node:path";
|
||||
import { fileURLToPath } from "node:url";
|
||||
import { promisify } from "node:util";
|
||||
import { expect, it } from "vitest";
|
||||
|
||||
import { parseStaticArchive } from "../../../scripts/ci/artifacts.mjs";
|
||||
import { createTestTempDirectory } from "../../fixtures/temporaryDirectory.js";
|
||||
|
||||
const execute = promisify(execFile);
|
||||
const producer = fileURLToPath(
|
||||
new URL("../../../scripts/ci/static-check-report.mjs", import.meta.url),
|
||||
);
|
||||
|
||||
it.each([3.5, 100, 0])(
|
||||
"retains real lint failures when the duplication threshold is %s",
|
||||
async (threshold) => {
|
||||
const root = await createTestTempDirectory("rea-ci-static-archive-");
|
||||
await mkdir(join(root, "src"));
|
||||
await mkdir(join(root, "tests"));
|
||||
await writeFile(
|
||||
join(root, ".oxlintrc.json"),
|
||||
JSON.stringify({ rules: { "no-debugger": "error" } }),
|
||||
);
|
||||
await writeFile(
|
||||
join(root, ".jscpd.json"),
|
||||
JSON.stringify({ threshold, minLines: 5, minTokens: 50 }),
|
||||
);
|
||||
await writeFile(
|
||||
join(root, "src/example.ts"),
|
||||
"debugger;\nexport const example = true;\n",
|
||||
);
|
||||
const duplicate =
|
||||
"export function compute(value: number) {\n" +
|
||||
Array.from({ length: 20 }, (_, index) => ` value += ${index};\n`).join(
|
||||
"",
|
||||
) +
|
||||
" return value;\n}\n";
|
||||
for (const name of threshold === 3.5 ? [] : ["first", "second"])
|
||||
await writeFile(join(root, `src/${name}.ts`), duplicate);
|
||||
await expect(
|
||||
execute(process.execPath, [producer, "lint", "src/example.ts"], {
|
||||
cwd: root,
|
||||
}),
|
||||
).rejects.toMatchObject({ code: 1 });
|
||||
const duplication = execute(process.execPath, [producer, "duplicates"], {
|
||||
cwd: root,
|
||||
});
|
||||
if (threshold === 0)
|
||||
await expect(duplication).rejects.toMatchObject({ code: 1 });
|
||||
else await duplication;
|
||||
const reports = join(root, ".cache/static-checks");
|
||||
expect(
|
||||
JSON.parse(
|
||||
await readFile(join(reports, "jscpd/jscpd-report.json"), "utf8"),
|
||||
),
|
||||
).toMatchObject({ reaCheck: { exitCode: threshold === 0 ? 1 : 0 } });
|
||||
await writeFile(
|
||||
join(reports, "static-task-context.json"),
|
||||
JSON.stringify({
|
||||
version: 1,
|
||||
sourceSha: "a".repeat(40),
|
||||
sourceClean: true,
|
||||
nodeVersion: process.version,
|
||||
npmVersion: "11.16.0",
|
||||
turboVersion: "2.11.7",
|
||||
platform: process.platform,
|
||||
architecture: process.arch,
|
||||
taskHashes: { "lint:built": "a".repeat(16), jscpd: "b".repeat(16) },
|
||||
}),
|
||||
);
|
||||
const archive = join(root, "reports.zip");
|
||||
await execute(
|
||||
"zip",
|
||||
[
|
||||
"-q",
|
||||
archive,
|
||||
"oxlint.json",
|
||||
"jscpd/jscpd-report.json",
|
||||
"static-task-context.json",
|
||||
],
|
||||
{ cwd: reports },
|
||||
);
|
||||
const evidence = await parseStaticArchive(archive);
|
||||
expect(evidence.diagnostics.filter(({ kind }) => kind === "lint")).toEqual([
|
||||
expect.objectContaining({
|
||||
kind: "lint",
|
||||
path: "src/example.ts",
|
||||
severity: "error",
|
||||
}),
|
||||
]);
|
||||
const duplicates = evidence.diagnostics.filter(
|
||||
({ kind }) => kind === "jscpd",
|
||||
);
|
||||
if (threshold === 0) expect(duplicates.length).toBeGreaterThan(0);
|
||||
else expect(duplicates).toEqual([]);
|
||||
},
|
||||
);
|
||||
|
|
@ -35,7 +35,7 @@ function run(id: number, overrides: Record<string, unknown> = {}) {
|
|||
id,
|
||||
status: "completed",
|
||||
conclusion: "success",
|
||||
path: ".github/workflows/ci.yml",
|
||||
path: ".github/workflows/ci.yml@main",
|
||||
event: "schedule",
|
||||
created_at: new Date(now - 120_000).toISOString(),
|
||||
head_branch: "main",
|
||||
|
|
|
|||
|
|
@ -0,0 +1,75 @@
|
|||
import { execFileSync } from "node:child_process";
|
||||
import { readFile } from "node:fs/promises";
|
||||
import { join } from "node:path";
|
||||
|
||||
import { describe, expect, it } from "vitest";
|
||||
import {
|
||||
BUILT_TEST_PROJECTS,
|
||||
SOURCE_TEST_PROJECTS,
|
||||
projectForTest,
|
||||
} from "../../../scripts/lib/test-projects.mjs";
|
||||
import {
|
||||
developmentTestPlan,
|
||||
parseDevelopmentTestRequest,
|
||||
} from "../../../scripts/lib/development-tests.mjs";
|
||||
|
||||
const repository = process.cwd();
|
||||
|
||||
describe("canonical test-project ownership", () => {
|
||||
it("assigns every tracked test file exactly once to source or built", () => {
|
||||
const files = execFileSync("git", ["ls-files", "*.test.ts"], {
|
||||
cwd: repository,
|
||||
encoding: "utf8",
|
||||
})
|
||||
.trim()
|
||||
.split("\n")
|
||||
.filter(Boolean);
|
||||
const grouped = new Map<string, string[]>();
|
||||
for (const file of files) {
|
||||
const project = projectForTest(file);
|
||||
grouped.set(project.group, [...(grouped.get(project.group) ?? []), file]);
|
||||
}
|
||||
|
||||
expect(grouped.get("source")).not.toBeUndefined();
|
||||
expect(grouped.get("built")).not.toBeUndefined();
|
||||
expect(
|
||||
(grouped.get("source")?.length ?? 0) +
|
||||
(grouped.get("built")?.length ?? 0),
|
||||
).toBe(files.length);
|
||||
expect(
|
||||
SOURCE_TEST_PROJECTS.reduce(
|
||||
(count, project) => count + project.include.length,
|
||||
0,
|
||||
),
|
||||
).toBeGreaterThan(0);
|
||||
expect(
|
||||
BUILT_TEST_PROJECTS.every((project) => project.group === "built"),
|
||||
).toBe(true);
|
||||
expect(() => projectForTest("tests/unowned/example.test.ts")).toThrow(
|
||||
"No test project owns",
|
||||
);
|
||||
});
|
||||
|
||||
it("owns the Hopper real-provider conformance test as built and requires preparation", () => {
|
||||
const testPath =
|
||||
"tests/conformance/providers/hopper/realHopperSearch.test.ts";
|
||||
expect(projectForTest(testPath)).toMatchObject({
|
||||
name: "boundary",
|
||||
group: "built",
|
||||
});
|
||||
expect(() => parseDevelopmentTestRequest("local", [testPath])).toThrow(
|
||||
`Use test:focused for built test projects: ${testPath}`,
|
||||
);
|
||||
const request = parseDevelopmentTestRequest("focused", [testPath]);
|
||||
expect(developmentTestPlan(request)).toMatchObject({ needsBuild: true });
|
||||
});
|
||||
|
||||
it("uses canonical selectors in Vitest while retaining independent runner policy", async () => {
|
||||
const config = await readFile(join(repository, "vitest.config.ts"), "utf8");
|
||||
expect(config).toContain(
|
||||
"[...SOURCE_TEST_PROJECTS, ...BUILT_TEST_PROJECTS]",
|
||||
);
|
||||
expect(config).toContain("PROJECT_POLICIES");
|
||||
expect(config).not.toContain('"src/{contracts,domain}/**/*.test.ts"');
|
||||
});
|
||||
});
|
||||
|
|
@ -3,6 +3,7 @@ import { mkdir, readFile, rm, writeFile } from "node:fs/promises";
|
|||
import { join, resolve } from "node:path";
|
||||
import { fileURLToPath } from "node:url";
|
||||
import { promisify } from "node:util";
|
||||
import { runInNewContext } from "node:vm";
|
||||
|
||||
import { expect, it } from "vitest";
|
||||
import { parse } from "yaml";
|
||||
|
|
@ -48,9 +49,7 @@ const workflow = async () =>
|
|||
it("merges available failed-shard diagnostics and skips unavailable shard reports", async () => {
|
||||
const steps = (await workflow()).test!.steps;
|
||||
const download = steps.find(
|
||||
(step) =>
|
||||
step.uses?.includes("actions/download-artifact") &&
|
||||
step.with?.pattern === "vitest-report-*",
|
||||
(step) => step.with?.pattern === "vitest-report-*",
|
||||
);
|
||||
const merge = steps.findIndex((step) => step.run?.includes("test:ci:merge"));
|
||||
const upload = steps.findIndex((step) => step.with?.name === "test-results");
|
||||
|
|
@ -59,8 +58,7 @@ it("merges available failed-shard diagnostics and skips unavailable shard report
|
|||
);
|
||||
expect(merge).toBeGreaterThan(0);
|
||||
expect(download?.if).toContain("!cancelled()");
|
||||
expect(download?.if).toContain("needs.test-shard.result == 'success'");
|
||||
expect(download?.if).toContain("needs.test-shard.result == 'failure'");
|
||||
|
||||
expect(upload).toBeGreaterThan(merge);
|
||||
expect(gate).toBeGreaterThan(upload);
|
||||
const merges = steps.filter((step) => step.run?.includes("--reporter=json"));
|
||||
|
|
@ -70,8 +68,6 @@ it("merges available failed-shard diagnostics and skips unavailable shard report
|
|||
"--reporter=./scripts/ci/test-timing-reporter.mjs",
|
||||
);
|
||||
expect(step.if).toContain("!cancelled()");
|
||||
expect(step.if).toContain("needs.test-shard.result == 'success'");
|
||||
expect(step.if).toContain("needs.test-shard.result == 'failure'");
|
||||
}
|
||||
expect(steps[upload]?.with?.path).toContain("test-timings.json");
|
||||
expect(steps[upload]?.with).toMatchObject({ "include-hidden-files": true });
|
||||
|
|
@ -86,12 +82,23 @@ it("merges available failed-shard diagnostics and skips unavailable shard report
|
|||
);
|
||||
const command = steps[gate]?.run;
|
||||
if (command === undefined) throw new Error("Missing shard result gate");
|
||||
for (const result of ["failure", "cancelled", "skipped", ""]) {
|
||||
await expect(
|
||||
execute("bash", ["-c", command], {
|
||||
env: { ...process.env, CODE_REQUIRED: "true", SHARD_RESULT: result },
|
||||
}),
|
||||
).rejects.toMatchObject({ code: 1 });
|
||||
for (const tier of ["fast", "full"]) {
|
||||
for (const shard of ["success", "failure", "cancelled", "skipped", ""]) {
|
||||
for (const source of ["success", "failure", "cancelled", "skipped", ""]) {
|
||||
const check = execute("bash", ["-c", command], {
|
||||
env: {
|
||||
...process.env,
|
||||
CODE_REQUIRED: "true",
|
||||
SHARD_RESULT: shard,
|
||||
SOURCE_REQUIRED: String(tier === "fast"),
|
||||
SOURCE_RESULT: source,
|
||||
},
|
||||
});
|
||||
if (shard === "success" && (tier !== "fast" || source === "success"))
|
||||
await expect(check).resolves.toMatchObject({ stderr: "" });
|
||||
else await expect(check).rejects.toMatchObject({ code: 1 });
|
||||
}
|
||||
}
|
||||
}
|
||||
await expect(
|
||||
execute("bash", ["-c", command], {
|
||||
|
|
@ -100,6 +107,80 @@ it("merges available failed-shard diagnostics and skips unavailable shard report
|
|||
).resolves.toMatchObject({ stderr: "" });
|
||||
});
|
||||
|
||||
// These workflow conditions use the shared boolean/string expression subset.
|
||||
// Evaluate the checked-in expressions so failure-state coverage cannot drift
|
||||
// into a separate test implementation of the collection policy.
|
||||
function conditionValue(
|
||||
condition: string | undefined,
|
||||
needs: Record<string, unknown>,
|
||||
canceled: boolean,
|
||||
) {
|
||||
if (!condition) throw new Error("Missing report collection condition");
|
||||
return runInNewContext(
|
||||
condition
|
||||
.replaceAll("needs.test-shard", 'needs["test-shard"]')
|
||||
.replaceAll("needs.test-source", 'needs["test-source"]'),
|
||||
{ needs, cancelled: () => canceled },
|
||||
{ timeout: 1_000 },
|
||||
);
|
||||
}
|
||||
|
||||
it("collects either fast test group while keeping full reports tied to full shards", async () => {
|
||||
const steps = (await workflow()).test.steps;
|
||||
const download = steps.find(
|
||||
(step) => step.with?.pattern === "vitest-report-*",
|
||||
);
|
||||
const fast = steps.find(
|
||||
(step) => step.name === "Merge fast-suite JUnit and timing reports",
|
||||
);
|
||||
const full = steps.find(
|
||||
(step) => step.name === "Merge coverage, JUnit, and timing reports",
|
||||
);
|
||||
for (const tier of ["fast", "full"]) {
|
||||
for (const shard of ["success", "failure", "cancelled", "skipped", ""]) {
|
||||
for (const source of ["success", "failure", "cancelled", "skipped", ""]) {
|
||||
for (const [code, canceled] of [
|
||||
["true", false],
|
||||
["true", true],
|
||||
["false", false],
|
||||
["false", true],
|
||||
] as const) {
|
||||
const needs = {
|
||||
changes: { outputs: { code, tier } },
|
||||
"test-shard": { result: shard },
|
||||
"test-source": { result: source },
|
||||
};
|
||||
const builtAvailable = ["success", "failure"].includes(shard);
|
||||
const sourceAvailable = ["success", "failure"].includes(source);
|
||||
const enabled = code === "true" && !canceled;
|
||||
const available =
|
||||
builtAvailable || (tier === "fast" && sourceAvailable);
|
||||
const expected = enabled
|
||||
? {
|
||||
download: available,
|
||||
fast: tier === "fast" && available,
|
||||
full: tier === "full" && builtAvailable,
|
||||
}
|
||||
: { download: false, fast: false, full: false };
|
||||
expect(conditionValue(download?.if, needs, canceled)).toBe(
|
||||
expected.download,
|
||||
);
|
||||
expect(conditionValue(fast?.if, needs, canceled)).toBe(expected.fast);
|
||||
expect(conditionValue(full?.if, needs, canceled)).toBe(expected.full);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
const summary = steps.find((step) =>
|
||||
step.run?.includes("scripts/ci/test-report.mjs"),
|
||||
);
|
||||
expect(summary?.env?.SHARD_RESULT).toBe("${{ needs.test-shard.result }}");
|
||||
expect(summary?.env?.SOURCE_RESULT).toBe("${{ needs.test-source.result }}");
|
||||
expect(summary?.env?.SOURCE_REQUIRED).toBe(
|
||||
"${{ needs.changes.outputs.tier == 'fast' }}",
|
||||
);
|
||||
});
|
||||
|
||||
it("keeps diagnosis available after merge failure and publishes only complete successful history", async () => {
|
||||
const steps = (await workflow()).test!.steps;
|
||||
const diagnosis = steps.find((step) => step.with?.name === "test-results");
|
||||
|
|
@ -220,7 +301,12 @@ it("exports portable timing history and removes it on a failed, partial, or inva
|
|||
{ name: join(root, "tests/fast.test.ts"), durationMs: 10 },
|
||||
],
|
||||
};
|
||||
const run = async (value: unknown, shards = "success", merge = "success") => {
|
||||
const run = async (
|
||||
value: unknown,
|
||||
shards = "success",
|
||||
merge = "success",
|
||||
source?: string,
|
||||
) => {
|
||||
await writeFile(join(root, "test-timings.json"), JSON.stringify(value));
|
||||
await writeFile(summary, "");
|
||||
return execute(process.execPath, [fileURLToPath(script)], {
|
||||
|
|
@ -229,6 +315,8 @@ it("exports portable timing history and removes it on a failed, partial, or inva
|
|||
...process.env,
|
||||
GITHUB_STEP_SUMMARY: summary,
|
||||
SHARD_RESULT: shards,
|
||||
SOURCE_REQUIRED: String(source !== undefined),
|
||||
SOURCE_RESULT: source ?? "",
|
||||
MERGE_RESULT: merge,
|
||||
},
|
||||
});
|
||||
|
|
@ -256,6 +344,24 @@ it("exports portable timing history and removes it on a failed, partial, or inva
|
|||
await expect(readFile(history)).rejects.toMatchObject({ code: "ENOENT" });
|
||||
expect(await readFile(summary, "utf8")).toContain("diagnostic only");
|
||||
}
|
||||
await run(report, "success", "success", "success");
|
||||
expect(await readFile(history, "utf8")).toContain('"success":true');
|
||||
for (const [shards, source] of [
|
||||
["skipped", "success"],
|
||||
["success", "skipped"],
|
||||
["success", "cancelled"],
|
||||
["success", "failure"],
|
||||
["success", ""],
|
||||
]) {
|
||||
await run(report);
|
||||
await run(report, shards, "success", source);
|
||||
await expect(readFile(history)).rejects.toMatchObject({ code: "ENOENT" });
|
||||
const text = await readFile(summary, "utf8");
|
||||
expect(text).toContain(`Source result: ${source}.`);
|
||||
expect(text).toContain("diagnostic only");
|
||||
if (shards === "skipped" || source !== "failure")
|
||||
expect(text).toContain("Partial test diagnostics");
|
||||
}
|
||||
await run(report);
|
||||
await expect(
|
||||
run({
|
||||
|
|
@ -352,6 +458,21 @@ it("recovers complete module costs from real blob reports without executing test
|
|||
expect(file.durationMs).toBeCloseTo(
|
||||
Object.values(file.phases).reduce((sum, value) => sum + value, 0),
|
||||
);
|
||||
const partial = await execute(process.execPath, [fileURLToPath(script)], {
|
||||
cwd: root,
|
||||
env: {
|
||||
...process.env,
|
||||
SHARD_RESULT: "skipped",
|
||||
SOURCE_REQUIRED: "true",
|
||||
SOURCE_RESULT: "success",
|
||||
MERGE_RESULT: "success",
|
||||
},
|
||||
});
|
||||
expect(partial.stdout).toContain("Partial test diagnostics");
|
||||
expect(partial.stdout).toContain("sample.test.js");
|
||||
await expect(
|
||||
readFile(join(root, ".cache/ci-test-history/test-results.json")),
|
||||
).rejects.toMatchObject({ code: "ENOENT" });
|
||||
await execute(process.execPath, [fileURLToPath(script)], {
|
||||
cwd: root,
|
||||
env: { ...process.env, SHARD_RESULT: "success", MERGE_RESULT: "success" },
|
||||
|
|
@ -420,13 +541,15 @@ it("records failed omissions from full reports without treating partial or unhan
|
|||
state: index === 1 ? "failed" : "passed",
|
||||
})),
|
||||
};
|
||||
const run = async (report: unknown) => {
|
||||
const run = async (report: unknown, source = "failure") => {
|
||||
await writeFile(join(root, "test-timings.json"), JSON.stringify(report));
|
||||
await execute(process.execPath, [fileURLToPath(script)], {
|
||||
cwd: root,
|
||||
env: {
|
||||
...process.env,
|
||||
SHARD_RESULT: "failure",
|
||||
SOURCE_REQUIRED: "true",
|
||||
SOURCE_RESULT: source,
|
||||
MERGE_RESULT: "failure",
|
||||
CI_VERIFY_TEST_INVENTORY: "true",
|
||||
CI_TEST_SELECTION: JSON.stringify(
|
||||
|
|
@ -458,6 +581,8 @@ it("records failed omissions from full reports without treating partial or unhan
|
|||
{ ...timing, reason: "interrupted" },
|
||||
])
|
||||
expect(await run(report)).toMatchObject({ complete: false });
|
||||
for (const source of ["skipped", "cancelled", ""])
|
||||
expect(await run(timing, source)).toMatchObject({ complete: false });
|
||||
await expect(run({ ...timing, files: "invalid" })).rejects.toMatchObject({
|
||||
code: 1,
|
||||
});
|
||||
|
|
|
|||
|
|
@ -0,0 +1,54 @@
|
|||
import { expect, it } from "vitest";
|
||||
import { normalizeWorkflowRun } from "../../../scripts/ci/workflow-run.mjs";
|
||||
|
||||
it.each([
|
||||
[".github/workflows/ci.yml", ".github/workflows/ci.yml", null],
|
||||
[".github/workflows/ci.yml@main", ".github/workflows/ci.yml", "main"],
|
||||
[
|
||||
".github/workflows/ci.yml@refs/heads/release/v1",
|
||||
".github/workflows/ci.yml",
|
||||
"refs/heads/release/v1",
|
||||
],
|
||||
[
|
||||
".github/workflows/ci.yml@feature@v1",
|
||||
".github/workflows/ci.yml",
|
||||
"feature@v1",
|
||||
],
|
||||
[
|
||||
".github/workflows/release.yaml@main",
|
||||
".github/workflows/release.yaml",
|
||||
"main",
|
||||
],
|
||||
])(
|
||||
"preserves API path %s while separating its file identity and ref",
|
||||
(path, file, ref) => {
|
||||
const raw = { id: 10, path, head_sha: "a".repeat(40) };
|
||||
const parsed = normalizeWorkflowRun(raw);
|
||||
expect(parsed).toEqual({ ...raw, workflowPath: { file, ref } });
|
||||
expect(raw).not.toHaveProperty("workflowPath");
|
||||
},
|
||||
);
|
||||
|
||||
it.each([
|
||||
undefined,
|
||||
null,
|
||||
42,
|
||||
"",
|
||||
".github/workflows/ci.yml@",
|
||||
".github/workflows/ci.yml@main\n",
|
||||
".github/workflows/ci.yml@main\0",
|
||||
"../.github/workflows/ci.yml",
|
||||
".github/workflows/../ci.yml@main",
|
||||
".github/workflows/ci.yml-extra@main",
|
||||
"https://example.test/ci.yml@main",
|
||||
])(
|
||||
"keeps malformed workflow path %s unknown instead of inventing a file identity",
|
||||
(path) => {
|
||||
const parsed = normalizeWorkflowRun({
|
||||
path,
|
||||
workflowPath: { file: ".github/workflows/ci.yml", ref: "main" },
|
||||
});
|
||||
expect(parsed.path).toBe(path);
|
||||
expect(parsed.workflowPath).toBeNull();
|
||||
},
|
||||
);
|
||||
12
turbo.json
12
turbo.json
|
|
@ -63,9 +63,7 @@
|
|||
"!docs/.vitepress/dist/**",
|
||||
"!website/_site/**",
|
||||
"!experiments/analysis-efficiency/results/**",
|
||||
"vitest.reverse-tools.config.ts",
|
||||
"vitest.reverse-tools-semantics.config.ts",
|
||||
"vitest.config.ts"
|
||||
"*.{cjs,cts,js,jsx,mjs,mts,ts,tsx}"
|
||||
]
|
||||
},
|
||||
"lint:built": {
|
||||
|
|
@ -86,9 +84,7 @@
|
|||
"!docs/.vitepress/dist/**",
|
||||
"!website/_site/**",
|
||||
"!experiments/analysis-efficiency/results/**",
|
||||
"vitest.reverse-tools.config.ts",
|
||||
"vitest.reverse-tools-semantics.config.ts",
|
||||
"vitest.config.ts"
|
||||
"*.{cjs,cts,js,jsx,mjs,mts,ts,tsx}"
|
||||
]
|
||||
},
|
||||
"format:check": {},
|
||||
|
|
@ -104,8 +100,8 @@
|
|||
"tests/**/*.ts",
|
||||
"tsconfig.json",
|
||||
"tsconfig.build.json",
|
||||
"vitest.config.ts",
|
||||
"docs/.vitepress/config.ts"
|
||||
"docs/.vitepress/config.ts",
|
||||
"*.{cjs,cts,js,jsx,mjs,mts,ts,tsx}"
|
||||
]
|
||||
},
|
||||
"jscpd": {
|
||||
|
|
|
|||
100
vitest.config.ts
100
vitest.config.ts
|
|
@ -3,6 +3,10 @@ import { availableParallelism, tmpdir } from "node:os";
|
|||
import { join } from "node:path";
|
||||
|
||||
import { defineConfig } from "vitest/config";
|
||||
import {
|
||||
BUILT_TEST_PROJECTS,
|
||||
SOURCE_TEST_PROJECTS,
|
||||
} from "./scripts/lib/test-projects.mjs";
|
||||
|
||||
import { CiTestSequencer } from "./tests/ciTestSequencer.js";
|
||||
|
||||
|
|
@ -28,57 +32,32 @@ const COVERAGE_SHARD = process.argv.some((argument) =>
|
|||
// Keep local and CI concurrency bounded by the runner rather than command locks.
|
||||
const MAX_TEST_WORKERS = Math.min(2, availableParallelism());
|
||||
|
||||
const TEST_PROJECTS = [
|
||||
{
|
||||
name: "domain",
|
||||
include: ["src/{contracts,domain}/**/*.test.ts"],
|
||||
const TEST_PROJECTS = [...SOURCE_TEST_PROJECTS, ...BUILT_TEST_PROJECTS];
|
||||
const PROJECT_POLICIES = {
|
||||
domain: {
|
||||
pool: "threads" as const,
|
||||
maxWorkers: MAX_TEST_WORKERS,
|
||||
// Domain and contract modules are pure and own no process-global state, so
|
||||
// per-file module isolation adds startup cost without protecting a seam.
|
||||
isolate: false,
|
||||
},
|
||||
{
|
||||
name: "services",
|
||||
include: ["src/application/**/*.test.ts"],
|
||||
services: {
|
||||
pool: "threads" as const,
|
||||
maxWorkers: MAX_TEST_WORKERS,
|
||||
// Service tests use recording ports and own no process-global state. Share
|
||||
// their module graph so file startup does not dominate the lane.
|
||||
isolate: false,
|
||||
},
|
||||
{
|
||||
name: "adapters",
|
||||
include: [
|
||||
"src/*.test.ts",
|
||||
"src/{apktool,jeb,adb,artifacts,browser,dotnet,evm,firmware,flutter,pcap,ghidra,go,hopper,ida,inspector,javascript,native,process,reference,rust,server,wasm,windows}/**/*.test.ts",
|
||||
],
|
||||
pool: "forks" as const,
|
||||
maxWorkers: MAX_TEST_WORKERS,
|
||||
},
|
||||
{
|
||||
name: "composition",
|
||||
include: ["tests/composition/**/*.test.ts"],
|
||||
adapters: { pool: "forks" as const, maxWorkers: MAX_TEST_WORKERS },
|
||||
composition: {
|
||||
pool: "threads" as const,
|
||||
maxWorkers: MAX_TEST_WORKERS,
|
||||
// Composition tests use provider-neutral sessions and test-scoped cleanup;
|
||||
// sharing their immutable module graph avoids per-file startup work.
|
||||
isolate: false,
|
||||
},
|
||||
{
|
||||
name: "boundary",
|
||||
include: ["tests/boundary/**/*.test.ts"],
|
||||
exclude: [
|
||||
"tests/boundary/cli/**/*.test.ts",
|
||||
"tests/boundary/mcp/**/*.test.ts",
|
||||
"tests/boundary/process/**/*.test.ts",
|
||||
],
|
||||
pool: "forks" as const,
|
||||
maxWorkers: MAX_TEST_WORKERS,
|
||||
},
|
||||
{
|
||||
name: "cli-boundary",
|
||||
include: ["tests/boundary/cli/**/*.test.ts"],
|
||||
boundary: { pool: "forks" as const, maxWorkers: MAX_TEST_WORKERS },
|
||||
"cli-boundary": {
|
||||
pool: "forks" as const,
|
||||
maxWorkers: MAX_TEST_WORKERS,
|
||||
// CLI tests create fresh command instances or subprocesses and own their
|
||||
|
|
@ -88,54 +67,36 @@ const TEST_PROJECTS = [
|
|||
// before their two heaps compete with the real CLI child processes.
|
||||
execArgv: ["--max-old-space-size=384"],
|
||||
},
|
||||
{
|
||||
// Process-tree observations share host sampling resources; keep this small
|
||||
// lane serial while the independent projects run together.
|
||||
name: "process-boundary",
|
||||
include: ["tests/boundary/process/**/*.test.ts"],
|
||||
"process-boundary": {
|
||||
pool: "forks" as const,
|
||||
maxWorkers: MAX_TEST_WORKERS,
|
||||
// Process-tree observations share host sampling resources; keep this small
|
||||
// lane serial while the independent projects run together.
|
||||
fileParallelism: false,
|
||||
sequence: { groupOrder: 1 },
|
||||
},
|
||||
{
|
||||
name: "mcp-boundary",
|
||||
include: ["tests/boundary/mcp/**/*.test.ts"],
|
||||
"mcp-boundary": {
|
||||
pool: "threads" as const,
|
||||
maxWorkers: MAX_TEST_WORKERS,
|
||||
// Each file creates and closes an explicit in-memory MCP session. Reuse the
|
||||
// immutable server graph so module startup does not outweigh transport work.
|
||||
isolate: false,
|
||||
},
|
||||
{
|
||||
name: "acceptance",
|
||||
include: ["tests/acceptance/**/*.test.ts"],
|
||||
pool: "forks" as const,
|
||||
maxWorkers: MAX_TEST_WORKERS,
|
||||
},
|
||||
{
|
||||
name: "process-global",
|
||||
include: ["tests/process-global/**/*.test.ts"],
|
||||
pool: "forks" as const,
|
||||
maxWorkers: MAX_TEST_WORKERS,
|
||||
},
|
||||
{
|
||||
name: "conformance",
|
||||
include: ["tests/conformance/**/*.test.ts"],
|
||||
acceptance: { pool: "forks" as const, maxWorkers: MAX_TEST_WORKERS },
|
||||
"process-global": { pool: "forks" as const, maxWorkers: MAX_TEST_WORKERS },
|
||||
conformance: {
|
||||
pool: "threads" as const,
|
||||
maxWorkers: MAX_TEST_WORKERS,
|
||||
// Contract cases use recording providers and own no process-global state.
|
||||
isolate: false,
|
||||
},
|
||||
{
|
||||
name: "evaluation",
|
||||
include: ["tests/evaluation/**/*.test.ts"],
|
||||
evaluation: {
|
||||
pool: "threads" as const,
|
||||
maxWorkers: MAX_TEST_WORKERS,
|
||||
// Evaluator tests inspect deterministic inputs without mutating shared state.
|
||||
isolate: false,
|
||||
},
|
||||
];
|
||||
};
|
||||
|
||||
const ZERO_COVERAGE_THRESHOLDS = {
|
||||
statements: 0,
|
||||
|
|
@ -156,10 +117,21 @@ const ZERO_COVERAGE_THRESHOLDS = {
|
|||
},
|
||||
};
|
||||
|
||||
const projects = TEST_PROJECTS.map((project) => ({
|
||||
extends: true as const,
|
||||
test: project,
|
||||
}));
|
||||
const projects = TEST_PROJECTS.map((project) => {
|
||||
const policy =
|
||||
PROJECT_POLICIES[project.name as keyof typeof PROJECT_POLICIES];
|
||||
if (!policy)
|
||||
throw new Error(`Missing test runner policy for ${project.name}`);
|
||||
return {
|
||||
extends: true as const,
|
||||
test: {
|
||||
name: project.name,
|
||||
include: [...project.include],
|
||||
...(project.exclude ? { exclude: [...project.exclude] } : {}),
|
||||
...policy,
|
||||
},
|
||||
};
|
||||
});
|
||||
|
||||
export default defineConfig({
|
||||
test: {
|
||||
|
|
|
|||
Loading…
Reference in New Issue