ci: reduce redundant compatibility and qualification work
Select SQLite compatibility by explicit source, fixture, lifecycle, and script ownership while retaining every version in full qualification. Share the canonical scope inventory between planning and the required gate. Reuse recent exact-SHA full qualification only for hourly schedules, with trusted successful-run metadata and receipts. Force a daily refresh, reject stale original runs even after retries, and never renew proof from reused validation. Report attempt-specific queue, execution, and matched step timing in a separate completion workflow using trusted default-branch code and read-only API access. Preserve raw metadata and unknown timings without delaying validation or publication. Validation: 177 focused tests; typecheck/lint; knip; actionlint 1.7.7; all 1046 tests discovered; live 37-job API report smoke. Remote workflow scheduling and real-provider matrices remain CI verification.
This commit is contained in:
morluto
parent
0cf224e762
commit
d13c9f840d
|
|
@ -0,0 +1,45 @@
|
|||
name: CI timings
|
||||
|
||||
on:
|
||||
workflow_run:
|
||||
workflows: [CI, Release]
|
||||
types: [completed]
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
actions: read
|
||||
|
||||
jobs:
|
||||
report:
|
||||
name: Report completed workflow attempt
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 5
|
||||
steps:
|
||||
# Completed PR runs can contain untrusted changes. Execute only the
|
||||
# reporter from the trusted default branch; consume jobs as API data.
|
||||
- uses: actions/checkout@v4
|
||||
with:
|
||||
ref: ${{ github.event.repository.default_branch }}
|
||||
persist-credentials: false
|
||||
- uses: actions/setup-node@v4
|
||||
with:
|
||||
node-version-file: .nvmrc
|
||||
- name: Record queue, execution, and matched step timing
|
||||
continue-on-error: true
|
||||
env:
|
||||
GH_TOKEN: ${{ github.token }}
|
||||
CI_REPOSITORY: ${{ github.repository }}
|
||||
CI_TIMING_RUN_ID: ${{ github.event.workflow_run.id }}
|
||||
CI_TIMING_RUN_ATTEMPT: ${{ github.event.workflow_run.run_attempt }}
|
||||
run: node scripts/ci/job-timings.mjs
|
||||
- name: Upload CI timing artifacts
|
||||
if: always() && hashFiles('.cache/ci-job-timings/*') != ''
|
||||
continue-on-error: true
|
||||
uses: actions/upload-artifact@v4
|
||||
with:
|
||||
name: ci-job-timings
|
||||
path: .cache/ci-job-timings/
|
||||
include-hidden-files: true
|
||||
if-no-files-found: error
|
||||
overwrite: true
|
||||
retention-days: 14
|
||||
|
|
@ -6,7 +6,8 @@ on:
|
|||
pull_request:
|
||||
branches: [main, "release/*"]
|
||||
schedule:
|
||||
- cron: "23 * * * *"
|
||||
- cron: "23 0-2,4-23 * * *"
|
||||
- cron: "23 3 * * *"
|
||||
workflow_dispatch:
|
||||
workflow_call:
|
||||
inputs:
|
||||
|
|
@ -53,7 +54,9 @@ jobs:
|
|||
actions: read
|
||||
outputs:
|
||||
tier: ${{ steps.plan.outputs.tier }}
|
||||
qualification_run_id: ${{ steps.qualification.outputs.run_id }}
|
||||
code: ${{ steps.plan.outputs.code }}
|
||||
sqlite: ${{ steps.plan.outputs.sqlite }}
|
||||
runtime: ${{ steps.plan.outputs.runtime }}
|
||||
package: ${{ steps.plan.outputs.package }}
|
||||
apple: ${{ steps.plan.outputs.apple }}
|
||||
|
|
@ -96,6 +99,17 @@ jobs:
|
|||
CI_REPOSITORY: ${{ github.repository }}
|
||||
CI_DEFAULT_BRANCH: ${{ github.event.repository.default_branch }}
|
||||
run: node scripts/ci/baseline.mjs
|
||||
- name: Admit scheduled full qualification
|
||||
id: qualification
|
||||
if: github.event_name == 'schedule' && inputs.source_sha == ''
|
||||
env:
|
||||
GH_TOKEN: ${{ github.token }}
|
||||
HEAD_SHA: ${{ github.sha }}
|
||||
CI_REPOSITORY: ${{ github.repository }}
|
||||
CI_REPOSITORY_ID: ${{ github.repository_id }}
|
||||
CI_DEFAULT_BRANCH: ${{ github.event.repository.default_branch }}
|
||||
CI_FORCE_FULL: ${{ github.event.schedule == '23 3 * * *' }}
|
||||
run: node scripts/ci/qualification.mjs
|
||||
- name: Select verification lanes
|
||||
id: plan
|
||||
env:
|
||||
|
|
@ -103,12 +117,13 @@ jobs:
|
|||
BASELINE_REASON: ${{ steps.baseline.outputs.reason }}
|
||||
HEAD_SHA: ${{ inputs.source_sha || github.event.pull_request.head.sha || github.sha }}
|
||||
CI_FULL: ${{ inputs.source_sha != '' || contains(github.event.pull_request.labels.*.name, 'ci:full') }}
|
||||
CI_SKIP_FULL: ${{ steps.qualification.outputs.run_full == 'false' }}
|
||||
run: node scripts/ci/plan.mjs
|
||||
|
||||
static:
|
||||
name: Static checks
|
||||
needs: changes
|
||||
if: ${{ !cancelled() }}
|
||||
if: ${{ !cancelled() && needs.changes.outputs.tier != 'reused' }}
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: Require successful change classification
|
||||
|
|
@ -249,13 +264,13 @@ jobs:
|
|||
|
||||
sqlite-runtime:
|
||||
needs: [changes, build]
|
||||
if: needs.changes.outputs.code == 'true'
|
||||
if: needs.changes.outputs.sqlite == 'true'
|
||||
uses: ./.github/workflows/sqlite-runtime.yml
|
||||
|
||||
test:
|
||||
name: Portable tests
|
||||
needs: [changes, test-shard]
|
||||
if: ${{ !cancelled() }}
|
||||
if: ${{ !cancelled() && needs.changes.outputs.tier != 'reused' }}
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: Require successful change classification
|
||||
|
|
@ -754,6 +769,17 @@ jobs:
|
|||
with:
|
||||
node-version-file: .nvmrc
|
||||
- name: Require every selected lane to pass
|
||||
id: gate
|
||||
env:
|
||||
CI_JOBS: ${{ toJSON(needs) }}
|
||||
CI_QUALIFICATION_DIRECTORY: .cache/ci-qualification
|
||||
run: node scripts/ci/required.mjs
|
||||
- name: Upload full qualification receipt artifact
|
||||
if: needs.changes.outputs.tier == 'full' && success()
|
||||
uses: actions/upload-artifact@v4
|
||||
with:
|
||||
name: ${{ steps.gate.outputs.receipt_name }}
|
||||
path: ${{ steps.gate.outputs.receipt_path }}
|
||||
if-no-files-found: error
|
||||
overwrite: true
|
||||
retention-days: 7
|
||||
|
|
|
|||
|
|
@ -276,6 +276,21 @@ sharding. Static checks, documentation, build, package verification, and
|
|||
real-system lanes remain separate jobs so one kind of evidence cannot stand in
|
||||
for another.
|
||||
|
||||
The separate `CI timings` workflow runs after CI or Release completes and uploads
|
||||
`ci-job-timings` with raw API pages and a normalized point-in-time report for that
|
||||
workflow attempt. It runs the reporter from the trusted default branch with
|
||||
read-only permissions, treating the triggering run as API data. Its summary separates queue
|
||||
waiting (`created_at` to `started_at`) from completed execution and reports
|
||||
percentiles plus matched `npm ci` and artifact-step durations.
|
||||
Queue excludes dependency waiting before job creation and does not establish
|
||||
whether account concurrency or runner availability caused the wait. Queued jobs with
|
||||
placeholder start times, skipped jobs, and missing or invalid timestamps remain
|
||||
unknown. The reporter targets the triggering run and attempt explicitly.
|
||||
Summed job elapsed time is parallel work, not workflow wall time, CPU time,
|
||||
memory usage, or billing. Step names identify samples, not complete transfer
|
||||
costs. Its queue, execution, and failures are independent of the required gate
|
||||
and publication. CPU time and peak process-family memory are not measured by this API.
|
||||
|
||||
The build job saves portable Turbo outputs across runs. Eight Linux real-system
|
||||
lanes restore this cache without waiting for the build job. The shared action
|
||||
derives its key from `test:prepare --dry=json` task hashes, runner OS and
|
||||
|
|
@ -737,6 +752,12 @@ lane. Edits to verifier entrypoints run only by manual, licensed or self-hosted
|
|||
workflows run static checks and the Linux suite. Website changes run website
|
||||
checks. Provider workflow changes validate the workflow and run that provider's
|
||||
verifier. Manual-provider/Android workflow changes run workflow validation only.
|
||||
The four-version SQLite compatibility matrix has its own `sqlite` scope for
|
||||
SQLite implementation, contracts, CLI/MCP registration, fixtures, and shared
|
||||
runtime boundaries. SQLite and server-lifecycle test edits and the SQLite npm
|
||||
scripts also select it. Unrelated code edits retain the complete portable suite
|
||||
on the pinned Node version without starting four additional compatibility jobs.
|
||||
Full qualification retains every SQLite version.
|
||||
Shared process/contract/CLI boundaries, dependencies, other packaging inputs,
|
||||
unknown paths, and changes to the planner select the full baseline. Package
|
||||
script-only edits select their owning verifier when its entrypoint is
|
||||
|
|
@ -760,8 +781,20 @@ Every code change still runs all six portable shards, without coverage
|
|||
instrumentation in this fast tier. This deliberately avoids relying on an
|
||||
import graph for generated files, filesystem inputs, subprocesses, or bridges.
|
||||
|
||||
Full qualification runs hourly at minute 23, on manual dispatch, and for the
|
||||
exact release revision before publication. It includes aggregate coverage,
|
||||
Scheduled qualification is admitted hourly at minute 23. A forced full run at
|
||||
03:23 UTC refreshes runner and dependency drift every day. Other hourly runs
|
||||
can reuse a successful full-qualification receipt for the exact source SHA
|
||||
from the same repository's trusted CI workflow on its default branch. Both the
|
||||
receipt and original workflow run must be less than 24 hours old, so retrying an
|
||||
old gate cannot renew stale proof. Missing, expired, failed, foreign, or uncertain proof runs full
|
||||
validation; API lookup failure also runs full. Lookups examine at most 100 named
|
||||
receipts and three candidate runs within a 20-second budget. Reused runs select
|
||||
no validation lanes, identify the original run, and produce no new receipt, so
|
||||
reuse cannot indefinitely extend proof freshness. They make no fresh provider
|
||||
observation. Manual dispatch, `ci:full`, and exact release qualification always
|
||||
run full validation.
|
||||
|
||||
Full qualification includes aggregate coverage,
|
||||
every portable provider lane, and all native hosts below. A `ci:full` PR label
|
||||
selects full qualification on the next PR run; manual dispatch can request it
|
||||
immediately. Fast-tier broad fallbacks retain all owner lanes and the three-host
|
||||
|
|
|
|||
|
|
@ -0,0 +1,353 @@
|
|||
import { execFile } from "node:child_process";
|
||||
import { mkdir, readFile, writeFile } from "node:fs/promises";
|
||||
import { dirname, join, resolve } from "node:path";
|
||||
import { promisify } from "node:util";
|
||||
import { fileURLToPath } from "node:url";
|
||||
|
||||
const execFileAsync = promisify(execFile);
|
||||
const MAX_BUFFER = 16 * 1024 * 1024;
|
||||
const TIMEOUT_MS = 30_000;
|
||||
const OUTPUT_DIRECTORY = join(".cache", "ci-job-timings");
|
||||
const STEP_COSTS = [
|
||||
{ key: "npmCi", pattern: /\bnpm ci\b/iu },
|
||||
{ key: "artifactUpload", pattern: /upload.*artifact/iu },
|
||||
{ key: "artifactDownload", pattern: /download.*artifact/iu },
|
||||
];
|
||||
|
||||
/** Fetch and summarize one workflow attempt without turning telemetry into a gate. */
|
||||
async function collectJobTimings({
|
||||
repository,
|
||||
runId,
|
||||
runAttempt,
|
||||
outputDirectory = OUTPUT_DIRECTORY,
|
||||
summaryPath,
|
||||
observedAt = new Date().toISOString(),
|
||||
env = process.env,
|
||||
}) {
|
||||
const identity = validateIdentity({ repository, runId, runAttempt });
|
||||
const endpoint = `repos/${identity.repository}/actions/runs/${identity.runId}/attempts/${identity.runAttempt}/jobs?per_page=100`;
|
||||
const { stdout } = await execFileAsync(
|
||||
"gh",
|
||||
["api", "--paginate", "--slurp", endpoint],
|
||||
{ env, encoding: "utf8", maxBuffer: MAX_BUFFER, timeout: TIMEOUT_MS },
|
||||
);
|
||||
const rawPath = join(outputDirectory, "jobs.json");
|
||||
const reportPath = join(outputDirectory, "report.json");
|
||||
await mkdir(outputDirectory, { recursive: true });
|
||||
await writeFile(rawPath, stdout, "utf8");
|
||||
const pages = parsePages(stdout);
|
||||
const jobs = pages.flatMap(({ jobs: pageJobs }, index) =>
|
||||
validateJobs(pageJobs, index),
|
||||
);
|
||||
const report = normalizeReport({ identity, jobs, observedAt });
|
||||
|
||||
await writeFile(reportPath, `${JSON.stringify(report, null, 2)}\n`, "utf8");
|
||||
if (summaryPath !== undefined)
|
||||
await appendSummary(summaryPath, report, rawPath, reportPath);
|
||||
return { rawPath, reportPath, report };
|
||||
}
|
||||
|
||||
function validateIdentity({ repository, runId, runAttempt }) {
|
||||
if (
|
||||
typeof repository !== "string" ||
|
||||
!/^[A-Za-z0-9_.-]+\/[A-Za-z0-9_.-]+$/u.test(repository)
|
||||
)
|
||||
throw new Error(
|
||||
"CI_REPOSITORY must be an owner/name repository identifier",
|
||||
);
|
||||
return {
|
||||
repository,
|
||||
runId: parsePositiveInteger(runId, "CI_TIMING_RUN_ID"),
|
||||
runAttempt: parsePositiveInteger(runAttempt, "CI_TIMING_RUN_ATTEMPT"),
|
||||
};
|
||||
}
|
||||
|
||||
function parsePositiveInteger(value, name) {
|
||||
if (typeof value === "string" && /^[1-9]\d*$/u.test(value)) {
|
||||
const parsed = Number(value);
|
||||
if (Number.isSafeInteger(parsed)) return parsed;
|
||||
}
|
||||
throw new Error(`${name} must be a positive safe integer`);
|
||||
}
|
||||
|
||||
function parsePages(stdout) {
|
||||
let pages;
|
||||
try {
|
||||
pages = JSON.parse(stdout);
|
||||
} catch (cause) {
|
||||
throw new Error("GitHub jobs API returned malformed JSON", { cause });
|
||||
}
|
||||
if (!Array.isArray(pages) || pages.length === 0)
|
||||
throw new Error("GitHub jobs API slurp must contain a nonempty page array");
|
||||
for (const page of pages)
|
||||
if (
|
||||
typeof page !== "object" ||
|
||||
page === null ||
|
||||
Array.isArray(page) ||
|
||||
!Array.isArray(page.jobs)
|
||||
)
|
||||
throw new Error("GitHub jobs API page must contain a jobs array");
|
||||
return pages;
|
||||
}
|
||||
|
||||
function validateJobs(jobs, pageIndex) {
|
||||
return jobs.map((job, jobIndex) => {
|
||||
if (
|
||||
typeof job !== "object" ||
|
||||
job === null ||
|
||||
Array.isArray(job) ||
|
||||
typeof job.name !== "string" ||
|
||||
job.name.length === 0 ||
|
||||
typeof job.status !== "string"
|
||||
)
|
||||
throw new Error(
|
||||
`GitHub jobs API page ${String(pageIndex + 1)} job ${String(jobIndex + 1)} is malformed`,
|
||||
);
|
||||
return job;
|
||||
});
|
||||
}
|
||||
|
||||
function normalizeReport({ identity, jobs, observedAt }) {
|
||||
const normalizedJobs = jobs.map(normalizeJob);
|
||||
const queues = normalizedJobs
|
||||
.map(({ queue }) => queue.seconds)
|
||||
.filter((value) => value !== null);
|
||||
const completedExecutions = normalizedJobs
|
||||
.filter(
|
||||
({ status, conclusion }) =>
|
||||
status === "completed" && conclusion !== "skipped",
|
||||
)
|
||||
.map(({ execution }) => execution.seconds)
|
||||
.filter((value) => value !== null);
|
||||
const completedDurationSumSeconds = completedExecutions.reduce(
|
||||
(sum, seconds) => sum + seconds,
|
||||
0,
|
||||
);
|
||||
const completedExecutionUnknown = normalizedJobs.filter(
|
||||
({ status, conclusion, execution }) =>
|
||||
status === "completed" &&
|
||||
conclusion !== "skipped" &&
|
||||
execution.seconds === null,
|
||||
).length;
|
||||
const skipped = normalizedJobs.filter(
|
||||
({ conclusion }) => conclusion === "skipped",
|
||||
).length;
|
||||
const costs = Object.fromEntries(STEP_COSTS.map(({ key }) => [key, []]));
|
||||
for (const job of normalizedJobs)
|
||||
for (const step of job.steps)
|
||||
if (step.costKey !== null)
|
||||
costs[step.costKey].push({ job: job.name, ...step });
|
||||
|
||||
return {
|
||||
schemaVersion: 1,
|
||||
run: {
|
||||
...identity,
|
||||
observedAt,
|
||||
snapshot:
|
||||
normalizedJobs.length > 0 &&
|
||||
normalizedJobs.every(({ status }) => status === "completed")
|
||||
? "completed job status snapshot; unknown timestamps remain unmeasured"
|
||||
: "partial; job statuses may still be queued or in progress",
|
||||
},
|
||||
counts: {
|
||||
jobs: normalizedJobs.length,
|
||||
completed: normalizedJobs.filter(({ status }) => status === "completed")
|
||||
.length,
|
||||
skipped,
|
||||
queueUnknown: normalizedJobs.filter(({ queue }) => queue.seconds === null)
|
||||
.length,
|
||||
executionUnknown: normalizedJobs.filter(
|
||||
({ execution }) => execution.seconds === null,
|
||||
).length,
|
||||
completedExecutionUnknown,
|
||||
},
|
||||
aggregates: {
|
||||
queueSeconds: percentileSummary(queues),
|
||||
completedExecutionSeconds: percentileSummary(completedExecutions),
|
||||
summedCompletedJobElapsedSeconds: {
|
||||
value:
|
||||
completedExecutions.length === 0
|
||||
? null
|
||||
: round(completedDurationSumSeconds),
|
||||
knownCount: completedExecutions.length,
|
||||
unknownCount: completedExecutionUnknown,
|
||||
label:
|
||||
"sum of known completed job elapsed durations; incomplete when timestamps are unknown; not CPU time, billing, or workflow wall time",
|
||||
},
|
||||
},
|
||||
jobs: normalizedJobs,
|
||||
stepCostInterpretation:
|
||||
"Matched install/artifact steps by step name; these are not total artifact transfer costs.",
|
||||
stepCosts: costs,
|
||||
};
|
||||
}
|
||||
|
||||
function normalizeJob(job) {
|
||||
const skipped = job.conclusion === "skipped";
|
||||
const hasStarted = ["in_progress", "completed"].includes(job.status);
|
||||
const queue = skipped
|
||||
? unknownDuration("job was skipped")
|
||||
: !hasStarted
|
||||
? unknownDuration("job start is not established by its status")
|
||||
: duration(job.created_at, job.started_at);
|
||||
const execution = skipped
|
||||
? unknownDuration("job was skipped")
|
||||
: job.status === "completed"
|
||||
? duration(job.started_at, job.completed_at)
|
||||
: unknownDuration("job is not completed");
|
||||
const steps = Array.isArray(job.steps) ? job.steps.map(normalizeStep) : [];
|
||||
return {
|
||||
id: Number.isSafeInteger(job.id) && job.id > 0 ? job.id : null,
|
||||
url: stringOrNull(job.html_url),
|
||||
name: job.name,
|
||||
workflowName: stringOrNull(job.workflow_name),
|
||||
status: job.status,
|
||||
conclusion: stringOrNull(job.conclusion),
|
||||
createdAt: stringOrNull(job.created_at),
|
||||
startedAt: stringOrNull(job.started_at),
|
||||
completedAt: stringOrNull(job.completed_at),
|
||||
queue,
|
||||
execution,
|
||||
steps,
|
||||
};
|
||||
}
|
||||
|
||||
function normalizeStep(step) {
|
||||
if (
|
||||
typeof step !== "object" ||
|
||||
step === null ||
|
||||
Array.isArray(step) ||
|
||||
typeof step.name !== "string"
|
||||
)
|
||||
throw new Error("GitHub jobs API contains a malformed job step");
|
||||
const cost = STEP_COSTS.find(({ pattern }) => pattern.test(step.name));
|
||||
return {
|
||||
name: step.name,
|
||||
costKey: cost?.key ?? null,
|
||||
duration: duration(step.started_at, step.completed_at),
|
||||
};
|
||||
}
|
||||
|
||||
function duration(startValue, endValue) {
|
||||
if (typeof startValue !== "string" || typeof endValue !== "string")
|
||||
return unknownDuration("timestamp unavailable");
|
||||
const start = Date.parse(startValue);
|
||||
const end = Date.parse(endValue);
|
||||
if (!Number.isFinite(start) || !Number.isFinite(end))
|
||||
return unknownDuration("invalid timestamp");
|
||||
const milliseconds = end - start;
|
||||
if (!Number.isFinite(milliseconds) || milliseconds < 0)
|
||||
return unknownDuration("timestamps out of order");
|
||||
return { seconds: round(milliseconds / 1_000), reason: null };
|
||||
}
|
||||
|
||||
function unknownDuration(reason) {
|
||||
return { seconds: null, reason };
|
||||
}
|
||||
|
||||
function percentileSummary(values) {
|
||||
return {
|
||||
count: values.length,
|
||||
p50: nearestRank(values, 0.5),
|
||||
p95: nearestRank(values, 0.95),
|
||||
unit: "seconds",
|
||||
};
|
||||
}
|
||||
|
||||
function nearestRank(values, percentile) {
|
||||
if (values.length === 0) return null;
|
||||
const sorted = [...values].sort((left, right) => left - right);
|
||||
return round(sorted[Math.ceil(percentile * sorted.length) - 1]);
|
||||
}
|
||||
|
||||
function round(value) {
|
||||
return Math.round(value * 1_000) / 1_000;
|
||||
}
|
||||
|
||||
function stringOrNull(value) {
|
||||
return typeof value === "string" ? value : null;
|
||||
}
|
||||
|
||||
async function appendSummary(summaryPath, report, rawPath, reportPath) {
|
||||
await mkdir(dirname(summaryPath), { recursive: true });
|
||||
const rows = [
|
||||
["Queue", report.aggregates.queueSeconds],
|
||||
["Completed execution", report.aggregates.completedExecutionSeconds],
|
||||
]
|
||||
.map(
|
||||
([label, item]) =>
|
||||
`| ${label} | ${display(item.p50)} | ${display(item.p95)} | ${String(item.count)} |`,
|
||||
)
|
||||
.join("\n");
|
||||
const jobRows = report.jobs
|
||||
.map(
|
||||
(job) =>
|
||||
`| ${escapeCell(job.name)} | ${job.status} | ${job.conclusion ?? "unknown"} | ${display(job.queue.seconds)} | ${job.queue.seconds === null ? job.queue.reason : ""} | ${display(job.execution.seconds)} | ${job.execution.seconds === null ? job.execution.reason : ""} |`,
|
||||
)
|
||||
.join("\n");
|
||||
const costRows = STEP_COSTS.map(({ key }) => {
|
||||
const samples = report.stepCosts[key];
|
||||
const values = samples
|
||||
.map(({ duration: item }) => item.seconds)
|
||||
.filter((value) => value !== null);
|
||||
return `| ${key} | ${String(samples.length)} | ${display(nearestRank(values, 0.5))} | ${samples.filter(({ duration: item }) => item.seconds === null).length} |`;
|
||||
}).join("\n");
|
||||
const text = [
|
||||
"### CI job timing snapshot",
|
||||
"",
|
||||
`Observed at ${report.run.observedAt} for run ${String(report.run.runId)}, attempt ${String(report.run.runAttempt)}. This is a point-in-time snapshot of fetched jobs. Queue starts at job creation and excludes preceding dependency waiting. Missing, queued, in-progress, invalid, or out-of-order timestamps are reported as unknown. Elapsed durations are not CPU time or billing data.`,
|
||||
"",
|
||||
`Fetched ${String(report.counts.jobs)} jobs; ${String(report.counts.completed)} are completed (${String(report.counts.skipped)} skipped); ${String(report.counts.queueUnknown)} queue durations and ${String(report.counts.executionUnknown)} execution durations (${String(report.counts.completedExecutionUnknown)} for non-skipped completed jobs) are unknown.`,
|
||||
"",
|
||||
"| Metric | p50 (s) | p95 (s) | Samples |",
|
||||
"| --- | ---: | ---: | ---: |",
|
||||
rows,
|
||||
"",
|
||||
"| Job | Status | Conclusion | Queue (s) | Queue detail | Execution (s) | Execution detail |",
|
||||
"| --- | --- | --- | ---: | --- | ---: | --- |",
|
||||
jobRows,
|
||||
"",
|
||||
`Sum of known completed job elapsed durations: **${display(report.aggregates.summedCompletedJobElapsedSeconds.value)} s** across ${String(report.aggregates.summedCompletedJobElapsedSeconds.knownCount)} known and ${String(report.aggregates.summedCompletedJobElapsedSeconds.unknownCount)} unknown jobs; this is not CPU time, billing, or workflow wall time.`,
|
||||
"",
|
||||
"Matched install/artifact steps by name; these are not total artifact transfer costs.",
|
||||
"",
|
||||
"| Matched step group | Samples | p50 (s) | Unknown |",
|
||||
"| --- | ---: | ---: | ---: |",
|
||||
costRows,
|
||||
"",
|
||||
`Reports: \`${rawPath}\`, \`${reportPath}\`.`,
|
||||
"",
|
||||
].join("\n");
|
||||
await writeFile(summaryPath, text, { encoding: "utf8", flag: "a" });
|
||||
}
|
||||
|
||||
function display(value) {
|
||||
return value === null ? "unknown" : String(value);
|
||||
}
|
||||
|
||||
function escapeCell(value) {
|
||||
return value.replaceAll("|", "\\|").replaceAll("\n", " ");
|
||||
}
|
||||
|
||||
if (
|
||||
process.argv[1] !== undefined &&
|
||||
resolve(process.argv[1]) === fileURLToPath(import.meta.url)
|
||||
) {
|
||||
try {
|
||||
const result = await collectJobTimings({
|
||||
repository: process.env.CI_REPOSITORY,
|
||||
runId: process.env.CI_TIMING_RUN_ID,
|
||||
runAttempt: process.env.CI_TIMING_RUN_ATTEMPT,
|
||||
outputDirectory: process.env.CI_JOB_TIMINGS_DIRECTORY ?? OUTPUT_DIRECTORY,
|
||||
summaryPath: process.env.GITHUB_STEP_SUMMARY,
|
||||
});
|
||||
process.stdout.write(
|
||||
`Wrote ${result.report.jobs.length} job timing records to ${result.reportPath}\n`,
|
||||
);
|
||||
} catch (cause) {
|
||||
process.stderr.write(
|
||||
`CI timing telemetry failed: ${cause instanceof Error ? cause.message : String(cause)}\n`,
|
||||
);
|
||||
process.exitCode = 1;
|
||||
}
|
||||
}
|
||||
|
|
@ -1,30 +1,17 @@
|
|||
import { execFileSync } from "node:child_process";
|
||||
import { appendFileSync } from "node:fs";
|
||||
import { matchesGlob } from "node:path";
|
||||
import { MANUAL_VERIFIERS, SCOPES } from "./scopes.mjs";
|
||||
import { CI_FLAGS as flags, MANUAL_VERIFIERS, SCOPES } from "./scopes.mjs";
|
||||
|
||||
const flags = [
|
||||
...new Set([
|
||||
"code",
|
||||
"runtime",
|
||||
"package",
|
||||
"apple",
|
||||
"fixtures",
|
||||
"readiness",
|
||||
"docs",
|
||||
"workflows",
|
||||
...Object.keys(SCOPES),
|
||||
]),
|
||||
];
|
||||
// Test files select the native-host lanes that run them explicitly. Provider
|
||||
// scopes remain implementation-owned, so test-only edits stay off those lanes.
|
||||
const NATIVE_TEST_OWNERS = ["apple", "package"];
|
||||
// Test files select the host/runtime lanes that execute their focused suites.
|
||||
const HOST_TEST_OWNERS = ["apple", "package", "sqlite"];
|
||||
// Actionlint validates workflow syntax only. The semantic suite asserts these
|
||||
// workflows' release, publication and self-hosted runner safeguards.
|
||||
const GUARDED_WORKFLOWS = new Set([
|
||||
".github/workflows/release.yml",
|
||||
".github/workflows/real-ghidra-windows.yml",
|
||||
".github/workflows/sqlite-runtime.yml",
|
||||
".github/workflows/ci-timings.yml",
|
||||
]);
|
||||
// Sources compiled by build-conformance-fixtures.mjs. The readiness journey
|
||||
// analyzes the built C fixture as well as its own applications.
|
||||
|
|
@ -63,6 +50,10 @@ function classifyPackage(base, head) {
|
|||
]),
|
||||
].filter((key) => beforeScripts?.[key] !== afterScripts?.[key]);
|
||||
for (const script of changed) {
|
||||
if (script === "test:sqlite" || script === "test:sqlite:run") {
|
||||
selected.sqlite = true;
|
||||
continue;
|
||||
}
|
||||
if (
|
||||
/^(test(?::|$)|check(?::|$)|lint(?::|$)|format(?::|$)|knip$|jscpd$)/.test(
|
||||
script,
|
||||
|
|
@ -110,14 +101,24 @@ function classifyPackage(base, head) {
|
|||
const event = process.env.GITHUB_EVENT_NAME;
|
||||
if (!["pull_request", "push", "schedule", "workflow_dispatch"].includes(event))
|
||||
throw new Error(`Unsupported CI event: ${event}`);
|
||||
const tier =
|
||||
const requestedFull =
|
||||
event === "schedule" ||
|
||||
event === "workflow_dispatch" ||
|
||||
process.env.CI_FULL === "true"
|
||||
? "full"
|
||||
: "fast";
|
||||
process.env.CI_FULL === "true";
|
||||
const tier =
|
||||
event === "schedule" &&
|
||||
process.env.CI_SKIP_FULL === "true" &&
|
||||
process.env.CI_FULL !== "true"
|
||||
? "reused"
|
||||
: requestedFull
|
||||
? "full"
|
||||
: "fast";
|
||||
let files = [];
|
||||
if (tier === "full") {
|
||||
if (tier === "reused") {
|
||||
reasons.push(
|
||||
"Recent exact-SHA full qualification reused; no fresh provider observation",
|
||||
);
|
||||
} else if (tier === "full") {
|
||||
full("Full baseline requested by event or ci:full label");
|
||||
} else if (event === "pull_request" || event === "push") {
|
||||
const head = process.env.HEAD_SHA;
|
||||
|
|
@ -162,7 +163,7 @@ if (tier === "full") {
|
|||
}
|
||||
if (/^(src|tests)\/.*\.(?:test|spec)\.[cm]?[jt]sx?$/.test(file)) {
|
||||
selected.code = true;
|
||||
for (const owner of NATIVE_TEST_OWNERS)
|
||||
for (const owner of HOST_TEST_OWNERS)
|
||||
selected[owner] ||= matchesAny(file, SCOPES[owner]);
|
||||
continue;
|
||||
}
|
||||
|
|
|
|||
|
|
@ -0,0 +1,176 @@
|
|||
import { execFileSync } from "node:child_process";
|
||||
import { appendFileSync, mkdirSync, writeFileSync } from "node:fs";
|
||||
import { resolve } from "node:path";
|
||||
import { fileURLToPath } from "node:url";
|
||||
|
||||
const ARTIFACT_NAME = "ci-full-qualification-v1";
|
||||
const MAX_AGE_MS = 24 * 60 * 60 * 1000;
|
||||
const positiveId = (value) =>
|
||||
(typeof value === "string" || typeof value === "number") &&
|
||||
/^[1-9]\d*$/.test(String(value)) &&
|
||||
Number.isSafeInteger(Number(value));
|
||||
const commitSha = (value) =>
|
||||
typeof value === "string" && /^[0-9a-f]{40}$/.test(value);
|
||||
|
||||
/** Record only after the required gate has verified every full-tier obligation. */
|
||||
export function recordQualification(directory) {
|
||||
const sourceSha = process.env.GITHUB_SHA;
|
||||
const runId = process.env.GITHUB_RUN_ID;
|
||||
const runAttempt = process.env.GITHUB_RUN_ATTEMPT;
|
||||
const repository = process.env.GITHUB_REPOSITORY;
|
||||
if (
|
||||
!commitSha(sourceSha) ||
|
||||
!positiveId(runId) ||
|
||||
!positiveId(runAttempt) ||
|
||||
!repository
|
||||
)
|
||||
throw new Error(
|
||||
"Full qualification receipt requires source and workflow identity",
|
||||
);
|
||||
const checkoutSha = execFileSync("git", ["rev-parse", "HEAD"], {
|
||||
encoding: "utf8",
|
||||
}).trim();
|
||||
if (checkoutSha !== sourceSha)
|
||||
throw new Error("Qualification checkout differs from workflow source");
|
||||
mkdirSync(directory, { recursive: true });
|
||||
const path = resolve(directory, "receipt.json");
|
||||
writeFileSync(
|
||||
path,
|
||||
`${JSON.stringify(
|
||||
{
|
||||
schemaVersion: 1,
|
||||
tier: "full",
|
||||
sourceSha,
|
||||
repository,
|
||||
runId: Number(runId),
|
||||
runAttempt: Number(runAttempt),
|
||||
qualifiedAt: new Date().toISOString(),
|
||||
},
|
||||
null,
|
||||
2,
|
||||
)}\n`,
|
||||
);
|
||||
if (process.env.GITHUB_OUTPUT)
|
||||
appendFileSync(
|
||||
process.env.GITHUB_OUTPUT,
|
||||
`receipt_name=${ARTIFACT_NAME}\nreceipt_path=${path}\n`,
|
||||
);
|
||||
}
|
||||
|
||||
function admit() {
|
||||
if (process.env.CI_FORCE_FULL === "true")
|
||||
return {
|
||||
run_full: true,
|
||||
reason: "Forced daily runner and dependency refresh",
|
||||
};
|
||||
const repository = process.env.CI_REPOSITORY;
|
||||
const repositoryId = Number(process.env.CI_REPOSITORY_ID);
|
||||
const branch = process.env.CI_DEFAULT_BRANCH;
|
||||
const head = process.env.HEAD_SHA;
|
||||
if (!repository || !positiveId(repositoryId) || !branch || !commitSha(head))
|
||||
return {
|
||||
run_full: true,
|
||||
reason: "Qualification lookup identity is incomplete",
|
||||
};
|
||||
|
||||
const now = Date.now();
|
||||
const deadline = now + 20_000;
|
||||
const api = (...arguments_) => {
|
||||
const remaining = deadline - Date.now();
|
||||
if (remaining <= 0)
|
||||
throw new Error("Qualification lookup budget exhausted");
|
||||
return JSON.parse(
|
||||
execFileSync("gh", ["api", "--method", "GET", ...arguments_], {
|
||||
encoding: "utf8",
|
||||
timeout: Math.min(15_000, remaining),
|
||||
maxBuffer: 8 * 1024 * 1024,
|
||||
stdio: ["ignore", "pipe", "pipe"],
|
||||
}),
|
||||
);
|
||||
};
|
||||
try {
|
||||
const { artifacts } = api(
|
||||
`repos/${repository}/actions/artifacts`,
|
||||
"-f",
|
||||
`name=${ARTIFACT_NAME}`,
|
||||
"-F",
|
||||
"per_page=100",
|
||||
);
|
||||
if (!Array.isArray(artifacts))
|
||||
throw new Error("Missing qualification artifact inventory");
|
||||
const candidates = artifacts.filter((artifact) => {
|
||||
const lineage = artifact?.workflow_run;
|
||||
const age = now - Date.parse(artifact?.created_at);
|
||||
return (
|
||||
artifact?.name === ARTIFACT_NAME &&
|
||||
artifact?.expired === false &&
|
||||
typeof artifact?.created_at === "string" &&
|
||||
Number.isFinite(age) &&
|
||||
age >= 0 &&
|
||||
age < MAX_AGE_MS &&
|
||||
positiveId(lineage?.id) &&
|
||||
String(lineage.id) !== process.env.GITHUB_RUN_ID &&
|
||||
lineage?.repository_id === repositoryId &&
|
||||
lineage?.head_repository_id === repositoryId &&
|
||||
lineage?.head_branch === branch &&
|
||||
lineage?.head_sha === head
|
||||
);
|
||||
});
|
||||
// Artifact metadata narrows the search before run lookups. Never scan an
|
||||
// unbounded history or treat a missing receipt as a successful qualification.
|
||||
for (const artifact of candidates.slice(0, 3)) {
|
||||
const run = api(
|
||||
`repos/${repository}/actions/runs/${artifact.workflow_run.id}`,
|
||||
);
|
||||
// An old run can upload a new artifact when only its gate is rerun.
|
||||
// Bound the original run age as well so retries cannot renew stale proof.
|
||||
const runAge = Date.now() - Date.parse(run.created_at);
|
||||
if (
|
||||
typeof run.created_at === "string" &&
|
||||
Number.isFinite(runAge) &&
|
||||
runAge >= 0 &&
|
||||
runAge < MAX_AGE_MS &&
|
||||
run.id === artifact.workflow_run.id &&
|
||||
run.status === "completed" &&
|
||||
run.conclusion === "success" &&
|
||||
run.path === ".github/workflows/ci.yml" &&
|
||||
["schedule", "workflow_dispatch"].includes(run.event) &&
|
||||
run.head_sha === head &&
|
||||
run.head_branch === branch &&
|
||||
run.head_repository?.full_name === repository
|
||||
) {
|
||||
return {
|
||||
run_full: false,
|
||||
run_id: run.id,
|
||||
reason: `Reusing full qualification from run ${run.id}; no fresh provider observation`,
|
||||
};
|
||||
}
|
||||
}
|
||||
return {
|
||||
run_full: true,
|
||||
reason: "No trusted full qualification for this SHA within 24 hours",
|
||||
};
|
||||
} catch {
|
||||
return {
|
||||
run_full: true,
|
||||
reason: "Qualification lookup unavailable; running full validation",
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
if (resolve(process.argv[1] ?? "") === fileURLToPath(import.meta.url)) {
|
||||
const decision = admit();
|
||||
if (process.env.GITHUB_OUTPUT)
|
||||
appendFileSync(
|
||||
process.env.GITHUB_OUTPUT,
|
||||
Object.entries(decision)
|
||||
.map(([key, value]) => `${key}=${value}\n`)
|
||||
.join(""),
|
||||
);
|
||||
if (process.env.GITHUB_STEP_SUMMARY)
|
||||
appendFileSync(
|
||||
process.env.GITHUB_STEP_SUMMARY,
|
||||
`### Scheduled qualification\n\n${decision.reason}.\n`,
|
||||
);
|
||||
console.log(JSON.stringify(decision));
|
||||
}
|
||||
|
|
@ -1,10 +1,30 @@
|
|||
import { CI_FLAGS } from "./scopes.mjs";
|
||||
import { recordQualification } from "./qualification.mjs";
|
||||
|
||||
const jobs = JSON.parse(process.env.CI_JOBS);
|
||||
const outputs = jobs.changes.outputs;
|
||||
const required = ["changes", "static", "test"];
|
||||
const reused = outputs.tier === "reused";
|
||||
if (
|
||||
reused &&
|
||||
(process.env.GITHUB_EVENT_NAME !== "schedule" ||
|
||||
!/^[1-9]\d*$/.test(outputs.qualification_run_id ?? "") ||
|
||||
!Number.isSafeInteger(Number(outputs.qualification_run_id)) ||
|
||||
CI_FLAGS.some((flag) => outputs[flag] !== "false"))
|
||||
)
|
||||
throw new Error(
|
||||
"Reused qualification requires a scheduled admission receipt and no selected lanes",
|
||||
);
|
||||
if (
|
||||
outputs.tier === "full" &&
|
||||
CI_FLAGS.some((flag) => outputs[flag] !== "true")
|
||||
)
|
||||
throw new Error("Full qualification requires every canonical CI scope");
|
||||
const required = reused ? ["changes"] : ["changes", "static", "test"];
|
||||
const lanes = {
|
||||
code: ["build", "test-shard", "sqlite-runtime"],
|
||||
code: ["build", "test-shard"],
|
||||
runtime: ["windows-native-build", "package-e2e"],
|
||||
package: ["windows-curated"],
|
||||
sqlite: ["sqlite-runtime"],
|
||||
apple: ["apple-artifacts"],
|
||||
fixtures: ["conformance-fixtures"],
|
||||
readiness: ["reconstruction-readiness"],
|
||||
|
|
@ -23,3 +43,9 @@ for (const [name, job] of Object.entries(jobs)) {
|
|||
if (failed.length)
|
||||
throw new Error(`CI lanes did not pass: ${failed.join(", ")}`);
|
||||
console.log("Every selected CI lane passed");
|
||||
if (reused)
|
||||
console.log(
|
||||
`Qualification reused from run ${outputs.qualification_run_id}; no fresh provider observation`,
|
||||
);
|
||||
if (outputs.tier === "full" && process.env.CI_QUALIFICATION_DIRECTORY)
|
||||
recordQualification(process.env.CI_QUALIFICATION_DIRECTORY);
|
||||
|
|
|
|||
|
|
@ -85,6 +85,55 @@ export const SCOPES = {
|
|||
"tests/boundary/filesystem/javascriptArtifactContentIdentity.test.ts",
|
||||
"tests/boundary/process/sourceMapUncHosts.test.ts",
|
||||
],
|
||||
// Native SQLite runtime compatibility for the producer, public adapters,
|
||||
// fixtures, shared lifecycle, and the package script that defines its suite.
|
||||
sqlite: [
|
||||
"src/sqlite/**",
|
||||
"src/domain/sqlite/**",
|
||||
"src/contracts/sqlite/**",
|
||||
"src/application/sqlite/**",
|
||||
"src/composition/sqlite.ts",
|
||||
"src/cli/sqliteCommands.ts",
|
||||
"src/server/registerSqliteTools.ts",
|
||||
"src/application/DirectAnalysisStatus*.ts",
|
||||
"src/application/CapabilityInventory*.ts",
|
||||
"src/application/AnalysisProvider*.ts",
|
||||
"src/contracts/toolContracts.ts",
|
||||
"src/contracts/toolContractTypes.ts",
|
||||
"src/contracts/toolEffects*.ts",
|
||||
"src/contracts/toolOutputSchemaPrimitives.ts",
|
||||
"src/domain/analysisError*.ts",
|
||||
"src/domain/evidence*.ts",
|
||||
"src/domain/inputIssueProjection*.ts",
|
||||
"src/domain/jsonSerialization*.ts",
|
||||
"src/domain/jsonValue*.ts",
|
||||
"src/domain/localPath*.ts",
|
||||
"src/domain/providerAdapterError*.ts",
|
||||
"src/domain/providerCleanupError*.ts",
|
||||
"src/domain/result*.ts",
|
||||
"src/process/OwnedCommand*.ts",
|
||||
"src/process/ProviderProcess*.ts",
|
||||
"src/process/PrivateRuntimeRoot*.ts",
|
||||
"src/artifacts/ArtifactReader*.ts",
|
||||
"src/artifacts/ArtifactResourceScope*.ts",
|
||||
"src/artifacts/ArtifactPaths*.ts",
|
||||
"src/artifacts/readStableArtifact*.ts",
|
||||
"src/filesystem/OwnedFileHandle*.ts",
|
||||
"src/server/EvidenceMcpServer*.ts",
|
||||
"src/server/analysisAdmission*.ts",
|
||||
"src/server/contractToolHandler*.ts",
|
||||
"src/server/toolRegistrationOptions*.ts",
|
||||
"src/server/createServer*.ts",
|
||||
"src/cli.ts",
|
||||
"src/cliCommandNames.ts",
|
||||
"src/cliLogging.ts",
|
||||
"src/cli/commandCancellation.ts",
|
||||
"tests/boundary/sqlite/**",
|
||||
"tests/fixtures/sqlite/**",
|
||||
"tests/acceptance/sqliteDatabaseCli.test.ts",
|
||||
"tests/boundary/mcp/serverLifecycleMcp.test.ts",
|
||||
".github/workflows/sqlite-runtime.yml",
|
||||
],
|
||||
browser: [
|
||||
".github/workflows/real-browser.yml",
|
||||
"scripts/verify-real-browser.mjs",
|
||||
|
|
@ -312,6 +361,20 @@ export const SCOPES = {
|
|||
],
|
||||
};
|
||||
|
||||
export const CI_FLAGS = [
|
||||
...new Set([
|
||||
"code",
|
||||
"runtime",
|
||||
"package",
|
||||
"apple",
|
||||
"fixtures",
|
||||
"readiness",
|
||||
"docs",
|
||||
"workflows",
|
||||
...Object.keys(SCOPES),
|
||||
]),
|
||||
];
|
||||
|
||||
// Verifier entrypoints that only manual, licensed, or self-hosted workflows
|
||||
// run. No selected CI lane executes them, so their edits need only the static
|
||||
// checks and the source-test suite. Shared helpers under scripts/lib remain
|
||||
|
|
|
|||
|
|
@ -79,6 +79,7 @@ async function classify(
|
|||
head: string,
|
||||
event = "pull_request",
|
||||
full = false,
|
||||
skipFull = false,
|
||||
) {
|
||||
const output = join(directory, "scope-output");
|
||||
await writeFile(output, "");
|
||||
|
|
@ -93,6 +94,7 @@ async function classify(
|
|||
HEAD_SHA: head,
|
||||
GITHUB_EVENT_NAME: event,
|
||||
CI_FULL: String(full),
|
||||
CI_SKIP_FULL: String(skipFull),
|
||||
GITHUB_OUTPUT: output,
|
||||
},
|
||||
},
|
||||
|
|
@ -100,6 +102,21 @@ async function classify(
|
|||
return z.record(z.string(), z.unknown()).parse(JSON.parse(result.stdout));
|
||||
}
|
||||
|
||||
it("reuses only scheduled qualification and preserves every explicit full request", async () => {
|
||||
const { directory, base } = await fixture();
|
||||
const reused = await classify(directory, base, base, "schedule", false, true);
|
||||
expect(reused.tier).toBe("reused");
|
||||
const { CI_FLAGS } = await loadScopes();
|
||||
for (const flag of CI_FLAGS) expect(reused[flag]).toBe(false);
|
||||
for (const event of ["workflow_dispatch", "pull_request", "schedule"])
|
||||
expect(
|
||||
(await classify(directory, base, base, event, true, true)).tier,
|
||||
).toBe("full");
|
||||
expect(
|
||||
(await classify(directory, base, base, "push", false, true)).tier,
|
||||
).toBe("fast");
|
||||
});
|
||||
|
||||
it("keeps every translated root README and authored guide on the documentation lane", async () => {
|
||||
const { directory, base } = await fixture();
|
||||
const readmes = (await readdir(new URL("../../../", import.meta.url))).filter(
|
||||
|
|
@ -205,18 +222,8 @@ it.each(["", "not-a-commit"])(
|
|||
browser: true,
|
||||
reasons: expect.arrayContaining([expect.stringContaining("baseline")]),
|
||||
});
|
||||
const { SCOPES } = await loadScopes();
|
||||
for (const flag of [
|
||||
"code",
|
||||
"runtime",
|
||||
"package",
|
||||
"fixtures",
|
||||
"readiness",
|
||||
"docs",
|
||||
"workflows",
|
||||
...Object.keys(SCOPES),
|
||||
])
|
||||
expect(plan[flag]).toBe(true);
|
||||
const { CI_FLAGS } = await loadScopes();
|
||||
for (const flag of CI_FLAGS) expect(plan[flag]).toBe(true);
|
||||
},
|
||||
);
|
||||
|
||||
|
|
@ -374,7 +381,14 @@ it("retains native Inspector verification on every documented package host", asy
|
|||
it.each([
|
||||
[
|
||||
"src/evm/client.ts",
|
||||
{ code: true, runtime: true, evm: true, browser: false, package: false },
|
||||
{
|
||||
code: true,
|
||||
runtime: true,
|
||||
evm: true,
|
||||
browser: false,
|
||||
package: false,
|
||||
sqlite: false,
|
||||
},
|
||||
],
|
||||
[
|
||||
"src/browser/execution/evaluate.ts",
|
||||
|
|
@ -471,6 +485,38 @@ it.each([
|
|||
"src/process/ProcessOwnership.test.ts",
|
||||
{ code: true, package: true, apple: false, evm: false, browser: false },
|
||||
],
|
||||
[
|
||||
"src/sqlite/SqliteDatabaseProvider.ts",
|
||||
{ code: true, sqlite: true, runtime: true, package: false },
|
||||
],
|
||||
[
|
||||
"src/application/sqlite/SqliteDatabaseService.test.ts",
|
||||
{ code: true, sqlite: true, runtime: false },
|
||||
],
|
||||
[
|
||||
"tests/boundary/sqlite/sqliteDatabaseProvider.test.ts",
|
||||
{ code: true, sqlite: true, runtime: false },
|
||||
],
|
||||
[
|
||||
"tests/fixtures/sqlite/database.ts",
|
||||
{ code: true, sqlite: true, runtime: false },
|
||||
],
|
||||
["tests/acceptance/sqliteDatabaseCli.test.ts", { code: true, sqlite: true }],
|
||||
[
|
||||
"tests/boundary/mcp/serverLifecycleMcp.test.ts",
|
||||
{ code: true, sqlite: true },
|
||||
],
|
||||
["src/process/OwnedCommand.ts", { code: true, sqlite: true, runtime: true }],
|
||||
["src/server/createServer.ts", { code: true, sqlite: true, runtime: true }],
|
||||
[
|
||||
"src/contracts/toolContracts.ts",
|
||||
{ code: true, sqlite: true, runtime: true },
|
||||
],
|
||||
[
|
||||
"src/domain/analysisErrorProjection.ts",
|
||||
{ code: true, sqlite: true, runtime: true },
|
||||
],
|
||||
["src/domain/javascript/example.ts", { code: true, sqlite: false }],
|
||||
])("selects the appropriate lanes for %s", async (path, expected) => {
|
||||
const { directory, base } = await fixture();
|
||||
const head = await change(directory, [path]);
|
||||
|
|
@ -513,6 +559,52 @@ it("scopes package script edits while retaining broad dependency checks", async
|
|||
});
|
||||
});
|
||||
|
||||
it.each(["test:sqlite", "test:sqlite:run"])(
|
||||
"routes %s script changes to the SQLite runtime matrix",
|
||||
async (script) => {
|
||||
const { directory } = await fixture();
|
||||
const base = await change(
|
||||
directory,
|
||||
["package.json"],
|
||||
JSON.stringify({ scripts: { [script]: "vitest run" } }),
|
||||
);
|
||||
const head = await change(
|
||||
directory,
|
||||
["package.json"],
|
||||
JSON.stringify({
|
||||
scripts: { [script]: "vitest run --reporter=default" },
|
||||
}),
|
||||
);
|
||||
expect(await classify(directory, base, head)).toMatchObject({
|
||||
code: true,
|
||||
sqlite: true,
|
||||
evm: false,
|
||||
package: false,
|
||||
});
|
||||
},
|
||||
);
|
||||
|
||||
it("keeps unrelated test script changes off the SQLite runtime matrix", async () => {
|
||||
const { directory } = await fixture();
|
||||
const base = await change(
|
||||
directory,
|
||||
["package.json"],
|
||||
JSON.stringify({ scripts: { "test:domain": "vitest run src/domain" } }),
|
||||
);
|
||||
const head = await change(
|
||||
directory,
|
||||
["package.json"],
|
||||
JSON.stringify({
|
||||
scripts: { "test:domain": "vitest run src/domain --reporter=default" },
|
||||
}),
|
||||
);
|
||||
expect(await classify(directory, base, head)).toMatchObject({
|
||||
code: true,
|
||||
sqlite: false,
|
||||
evm: false,
|
||||
});
|
||||
});
|
||||
|
||||
it.each([
|
||||
["browser", "real-browser"],
|
||||
["binary_layout", "real-binary-layout"],
|
||||
|
|
@ -605,7 +697,10 @@ it.each([
|
|||
"src/process/capture/example.test.ts",
|
||||
{ code: true, runtime: false, browser: false, package: false },
|
||||
],
|
||||
[".nvmrc", { code: true, package: true, browser: true, evm: true }],
|
||||
[
|
||||
".nvmrc",
|
||||
{ code: true, package: true, browser: true, evm: true, sqlite: true },
|
||||
],
|
||||
[
|
||||
"tsconfig.build.json",
|
||||
{ code: true, package: true, browser: true, evm: true },
|
||||
|
|
@ -745,6 +840,7 @@ it.each(["workflow_dispatch", "schedule"])(
|
|||
tier: "full",
|
||||
code: true,
|
||||
package: true,
|
||||
sqlite: true,
|
||||
evm: true,
|
||||
browser: true,
|
||||
});
|
||||
|
|
@ -762,6 +858,7 @@ it("honors ci:full for successive PR revisions", async () => {
|
|||
tier: "full",
|
||||
code: true,
|
||||
package: true,
|
||||
sqlite: true,
|
||||
evm: true,
|
||||
browser: true,
|
||||
});
|
||||
|
|
@ -829,6 +926,7 @@ const workflowFile = async (name: string): Promise<unknown> =>
|
|||
);
|
||||
|
||||
const scopesSchema = z.object({
|
||||
CI_FLAGS: z.array(z.string()),
|
||||
SCOPES: z.record(z.string(), z.array(z.string())),
|
||||
MANUAL_VERIFIERS: z.array(z.string()),
|
||||
});
|
||||
|
|
|
|||
|
|
@ -0,0 +1,243 @@
|
|||
import { chmod, mkdir, readFile, writeFile } from "node:fs/promises";
|
||||
import { delimiter, join } from "node:path";
|
||||
import { fileURLToPath } from "node:url";
|
||||
|
||||
import { expect, it } from "vitest";
|
||||
|
||||
import { createTestTempDirectory } from "../../fixtures/temporaryDirectory.js";
|
||||
import {
|
||||
executeWorkflowFixture,
|
||||
workflowGit,
|
||||
} from "../../support/workflowGit.js";
|
||||
|
||||
const reporter = fileURLToPath(
|
||||
new URL("../../../scripts/ci/job-timings.mjs", import.meta.url),
|
||||
);
|
||||
|
||||
it("merges paginated jobs, reports elapsed metrics, and preserves raw responses", async () => {
|
||||
const fixture = await createFixture([
|
||||
{
|
||||
total_count: 4,
|
||||
jobs: [
|
||||
{
|
||||
name: "Build package",
|
||||
workflow_name: "CI",
|
||||
status: "completed",
|
||||
conclusion: "success",
|
||||
created_at: "2026-10-13T00:00:00Z",
|
||||
started_at: "2026-10-13T00:00:10Z",
|
||||
completed_at: "2026-10-13T00:01:10Z",
|
||||
steps: [
|
||||
{
|
||||
name: "Install dependencies (npm ci)",
|
||||
started_at: "2026-10-13T00:00:10Z",
|
||||
completed_at: "2026-10-13T00:00:40Z",
|
||||
},
|
||||
{
|
||||
name: "Upload artifact",
|
||||
started_at: "2026-10-13T00:00:40Z",
|
||||
completed_at: "2026-10-13T00:00:41Z",
|
||||
},
|
||||
],
|
||||
},
|
||||
],
|
||||
},
|
||||
{
|
||||
total_count: 4,
|
||||
jobs: [
|
||||
{
|
||||
name: "Queued lane",
|
||||
status: "queued",
|
||||
created_at: "2026-10-13T00:00:00Z",
|
||||
started_at: "2026-10-13T00:00:00Z",
|
||||
completed_at: null,
|
||||
steps: [],
|
||||
},
|
||||
{
|
||||
name: "Running lane",
|
||||
status: "in_progress",
|
||||
created_at: "2026-10-13T00:00:00Z",
|
||||
started_at: "2026-10-13T00:00:15Z",
|
||||
completed_at: null,
|
||||
steps: [
|
||||
{
|
||||
name: "Download artifacts",
|
||||
started_at: "2026-10-13T00:00:20Z",
|
||||
completed_at: "2026-10-13T00:00:22Z",
|
||||
},
|
||||
],
|
||||
},
|
||||
{
|
||||
name: "Skipped lane",
|
||||
status: "completed",
|
||||
conclusion: "skipped",
|
||||
created_at: "2026-10-13T00:00:00Z",
|
||||
started_at: "2026-10-13T00:00:00Z",
|
||||
completed_at: "2026-10-13T00:00:00Z",
|
||||
steps: [],
|
||||
},
|
||||
],
|
||||
},
|
||||
]);
|
||||
const result = await runReporter(fixture);
|
||||
const report = JSON.parse(
|
||||
await readFile(join(fixture.outputDirectory, "report.json"), "utf8"),
|
||||
);
|
||||
|
||||
expect(report.run.snapshot).toContain("partial");
|
||||
expect(report.counts).toMatchObject({
|
||||
jobs: 4,
|
||||
completed: 2,
|
||||
skipped: 1,
|
||||
queueUnknown: 2,
|
||||
executionUnknown: 3,
|
||||
completedExecutionUnknown: 0,
|
||||
});
|
||||
expect(report.aggregates.queueSeconds).toMatchObject({
|
||||
count: 2,
|
||||
p50: 10,
|
||||
p95: 15,
|
||||
});
|
||||
expect(report.aggregates.completedExecutionSeconds).toMatchObject({
|
||||
count: 1,
|
||||
p50: 60,
|
||||
p95: 60,
|
||||
});
|
||||
expect(report.aggregates.summedCompletedJobElapsedSeconds).toMatchObject({
|
||||
value: 60,
|
||||
knownCount: 1,
|
||||
unknownCount: 0,
|
||||
label: expect.stringContaining("not CPU time"),
|
||||
});
|
||||
expect(report.jobs[1].queue).toEqual({
|
||||
seconds: null,
|
||||
reason: "job start is not established by its status",
|
||||
});
|
||||
expect(report.jobs[3].queue).toEqual({
|
||||
seconds: null,
|
||||
reason: "job was skipped",
|
||||
});
|
||||
expect(report.jobs[3].execution).toEqual({
|
||||
seconds: null,
|
||||
reason: "job was skipped",
|
||||
});
|
||||
expect(report.stepCosts.npmCi[0].duration.seconds).toBe(30);
|
||||
expect(report.stepCosts.artifactUpload[0].duration.seconds).toBe(1);
|
||||
expect(report.stepCosts.artifactDownload[0].duration.seconds).toBe(2);
|
||||
expect(report.stepCostInterpretation).toContain(
|
||||
"not total artifact transfer costs",
|
||||
);
|
||||
expect(report.cpuSeconds).toBeUndefined();
|
||||
expect(result.stdout).toContain("4 job timing records");
|
||||
expect(
|
||||
await readFile(join(fixture.outputDirectory, "jobs.json"), "utf8"),
|
||||
).toBe(fixture.raw);
|
||||
const summary = await readFile(fixture.summary, "utf8");
|
||||
expect(summary).toContain("point-in-time snapshot");
|
||||
expect(summary).toContain("unknown");
|
||||
expect(summary).toContain("not CPU time");
|
||||
expect(summary).toContain("60 s");
|
||||
expect(summary).toContain("not total artifact transfer costs");
|
||||
const args = JSON.parse(await readFile(fixture.args, "utf8"));
|
||||
expect(args).toEqual([
|
||||
"api",
|
||||
"--paginate",
|
||||
"--slurp",
|
||||
"repos/example/rea/actions/runs/42/attempts/2/jobs?per_page=100",
|
||||
]);
|
||||
});
|
||||
|
||||
it("marks invalid and out-of-order timestamps unknown instead of emitting negative durations", async () => {
|
||||
const fixture = await createFixture([
|
||||
{
|
||||
jobs: [
|
||||
{
|
||||
name: "Invalid dates",
|
||||
status: "completed",
|
||||
created_at: "not-a-date",
|
||||
started_at: "2026-10-13T00:00:10Z",
|
||||
completed_at: "2026-10-13T00:00:05Z",
|
||||
steps: [],
|
||||
},
|
||||
],
|
||||
},
|
||||
]);
|
||||
await runReporter(fixture);
|
||||
const report = JSON.parse(
|
||||
await readFile(join(fixture.outputDirectory, "report.json"), "utf8"),
|
||||
);
|
||||
|
||||
expect(report.jobs[0].queue).toEqual({
|
||||
seconds: null,
|
||||
reason: "invalid timestamp",
|
||||
});
|
||||
expect(report.jobs[0].execution).toEqual({
|
||||
seconds: null,
|
||||
reason: "timestamps out of order",
|
||||
});
|
||||
expect(report.counts.completedExecutionUnknown).toBe(1);
|
||||
expect(report.aggregates.summedCompletedJobElapsedSeconds).toMatchObject({
|
||||
value: null,
|
||||
unknownCount: 1,
|
||||
});
|
||||
});
|
||||
|
||||
it("fails on malformed pages or a failed GitHub API request", async () => {
|
||||
const malformed = await createFixture([{ jobs: "not-an-array" }]);
|
||||
await expect(runReporter(malformed)).rejects.toThrow();
|
||||
await expect(
|
||||
readFile(join(malformed.outputDirectory, "jobs.json"), "utf8"),
|
||||
).resolves.toBe(malformed.raw);
|
||||
|
||||
const failedApi = await createFixture([], { exitCode: 1 });
|
||||
await expect(runReporter(failedApi)).rejects.toThrow();
|
||||
});
|
||||
|
||||
async function createFixture(
|
||||
pages: unknown[],
|
||||
{ exitCode = 0 }: { exitCode?: number } = {},
|
||||
) {
|
||||
const directory = await createTestTempDirectory("rea-ci-job-timings-");
|
||||
await workflowGit(directory, ["init", "--initial-branch=main"]);
|
||||
const fakeBin = join(directory, "bin");
|
||||
await mkdir(fakeBin);
|
||||
const raw = JSON.stringify(pages);
|
||||
const rawPath = join(directory, "gh-response.json");
|
||||
const args = join(directory, "gh-args.json");
|
||||
await writeFile(rawPath, raw);
|
||||
const fakeGh = join(fakeBin, "gh");
|
||||
await writeFile(
|
||||
fakeGh,
|
||||
`#!/usr/bin/env node\nconst fs = require("node:fs");\nfs.writeFileSync(process.env.CI_GH_ARGS, JSON.stringify(process.argv.slice(2)));\nprocess.stdout.write(fs.readFileSync(process.env.CI_GH_RESPONSE, "utf8"));\nprocess.exitCode = ${String(exitCode)};\n`,
|
||||
);
|
||||
await chmod(fakeGh, 0o755);
|
||||
const outputDirectory = join(directory, ".cache", "ci-job-timings");
|
||||
const summary = join(directory, "summary.md");
|
||||
await writeFile(summary, "");
|
||||
return {
|
||||
directory,
|
||||
fakeBin,
|
||||
raw,
|
||||
rawPath,
|
||||
args,
|
||||
outputDirectory,
|
||||
summary,
|
||||
};
|
||||
}
|
||||
|
||||
function runReporter(fixture: Awaited<ReturnType<typeof createFixture>>) {
|
||||
return executeWorkflowFixture(process.execPath, [reporter], {
|
||||
cwd: fixture.directory,
|
||||
env: {
|
||||
...process.env,
|
||||
PATH: `${fixture.fakeBin}${delimiter}${process.env.PATH ?? ""}`,
|
||||
CI_REPOSITORY: "example/rea",
|
||||
CI_JOB_TIMINGS_DIRECTORY: fixture.outputDirectory,
|
||||
CI_GH_ARGS: fixture.args,
|
||||
CI_GH_RESPONSE: fixture.rawPath,
|
||||
CI_TIMING_RUN_ID: "42",
|
||||
CI_TIMING_RUN_ATTEMPT: "2",
|
||||
GITHUB_STEP_SUMMARY: fixture.summary,
|
||||
},
|
||||
});
|
||||
}
|
||||
|
|
@ -0,0 +1,375 @@
|
|||
import { chmod, mkdir, readFile, writeFile } from "node:fs/promises";
|
||||
import { join } from "node:path";
|
||||
import { fileURLToPath } from "node:url";
|
||||
|
||||
import { expect, it } from "vitest";
|
||||
import { parse } from "yaml";
|
||||
import { z } from "zod";
|
||||
|
||||
import { createTestTempDirectory } from "../../fixtures/temporaryDirectory.js";
|
||||
import {
|
||||
executeWorkflowFixture,
|
||||
workflowGit,
|
||||
} from "../../support/workflowGit.js";
|
||||
|
||||
const qualificationScript = fileURLToPath(
|
||||
new URL("../../../scripts/ci/qualification.mjs", import.meta.url),
|
||||
);
|
||||
const requiredScript = fileURLToPath(
|
||||
new URL("../../../scripts/ci/required.mjs", import.meta.url),
|
||||
);
|
||||
const now = Date.now();
|
||||
const validSha = "a".repeat(40);
|
||||
const repository = "example/rea";
|
||||
|
||||
function artifact(overrides: Record<string, unknown> = {}) {
|
||||
return {
|
||||
id: 42,
|
||||
name: "ci-full-qualification-v1",
|
||||
expired: false,
|
||||
created_at: new Date(now - 60_000).toISOString(),
|
||||
workflow_run: {
|
||||
id: 7,
|
||||
repository_id: 1,
|
||||
head_repository_id: 1,
|
||||
head_branch: "main",
|
||||
head_sha: validSha,
|
||||
},
|
||||
...overrides,
|
||||
};
|
||||
}
|
||||
|
||||
function run(overrides: Record<string, unknown> = {}) {
|
||||
return {
|
||||
id: 7,
|
||||
status: "completed",
|
||||
conclusion: "success",
|
||||
path: ".github/workflows/ci.yml",
|
||||
event: "schedule",
|
||||
created_at: new Date(now - 120_000).toISOString(),
|
||||
head_sha: validSha,
|
||||
head_branch: "main",
|
||||
head_repository: { full_name: repository },
|
||||
...overrides,
|
||||
};
|
||||
}
|
||||
|
||||
async function admissionFixture(
|
||||
candidate: Record<string, unknown> = artifact(),
|
||||
candidateRun: Record<string, unknown> = run(),
|
||||
options: { apiError?: boolean; malformed?: boolean; force?: boolean } = {},
|
||||
) {
|
||||
const directory = await createTestTempDirectory("rea-ci-qualification-");
|
||||
const bin = join(directory, "bin");
|
||||
await mkdir(bin);
|
||||
const fixturePath = join(directory, "api.json");
|
||||
await writeFile(
|
||||
fixturePath,
|
||||
options.malformed
|
||||
? "not-json"
|
||||
: JSON.stringify({ artifacts: [candidate], run: candidateRun }),
|
||||
);
|
||||
const gh = join(bin, "gh");
|
||||
await writeFile(
|
||||
gh,
|
||||
[
|
||||
"#!/bin/sh",
|
||||
'if [ "$GH_API_ERROR" = "true" ]; then exit 1; fi',
|
||||
'case "$*" in',
|
||||
' *actions/artifacts*) cat "$GH_FIXTURE_RESPONSE" ;;',
|
||||
" *actions/runs/7*) node -e 'const x=require(process.env.GH_FIXTURE_RESPONSE); process.stdout.write(JSON.stringify(x.run))' ;;",
|
||||
" *) exit 2 ;;",
|
||||
"esac",
|
||||
"",
|
||||
].join("\n"),
|
||||
);
|
||||
await chmod(gh, 0o755);
|
||||
const output = join(directory, "github-output");
|
||||
await writeFile(output, "");
|
||||
const env = {
|
||||
...process.env,
|
||||
PATH: `${bin}:${process.env.PATH ?? ""}`,
|
||||
GH_FIXTURE_RESPONSE: fixturePath,
|
||||
GH_API_ERROR: String(options.apiError ?? false),
|
||||
CI_REPOSITORY: repository,
|
||||
CI_REPOSITORY_ID: "1",
|
||||
CI_DEFAULT_BRANCH: "main",
|
||||
HEAD_SHA: validSha,
|
||||
GITHUB_RUN_ID: "100",
|
||||
GITHUB_OUTPUT: output,
|
||||
...(options.force ? { CI_FORCE_FULL: "true" } : {}),
|
||||
};
|
||||
return { directory, output, env };
|
||||
}
|
||||
|
||||
async function decide(
|
||||
candidate?: Record<string, unknown>,
|
||||
candidateRun?: Record<string, unknown>,
|
||||
options?: { apiError?: boolean; malformed?: boolean; force?: boolean },
|
||||
) {
|
||||
const fixture = await admissionFixture(candidate, candidateRun, options);
|
||||
const result = await executeWorkflowFixture(
|
||||
process.execPath,
|
||||
[qualificationScript],
|
||||
{
|
||||
cwd: fixture.directory,
|
||||
env: fixture.env,
|
||||
},
|
||||
);
|
||||
return {
|
||||
...fixture,
|
||||
decision: z
|
||||
.record(z.string(), z.unknown())
|
||||
.parse(JSON.parse(result.stdout)),
|
||||
outputText: await readFile(fixture.output, "utf8"),
|
||||
};
|
||||
}
|
||||
|
||||
it("reuses only a recent trusted full qualification for the exact default-branch SHA", async () => {
|
||||
const { decision, outputText } = await decide();
|
||||
expect(decision).toMatchObject({ run_full: false, run_id: 7 });
|
||||
expect(Object.keys(decision).sort()).toEqual([
|
||||
"reason",
|
||||
"run_full",
|
||||
"run_id",
|
||||
]);
|
||||
expect(outputText).toContain("run_full=false\n");
|
||||
expect(outputText).toContain("run_id=7\n");
|
||||
});
|
||||
|
||||
const rejectedCandidates: Array<
|
||||
[string, () => Record<string, unknown>, () => Record<string, unknown>]
|
||||
> = [
|
||||
[
|
||||
"old",
|
||||
() =>
|
||||
artifact({
|
||||
created_at: new Date(now - 24 * 60 * 60 * 1000 - 1).toISOString(),
|
||||
}),
|
||||
run,
|
||||
],
|
||||
[
|
||||
"future",
|
||||
() => artifact({ created_at: new Date(now + 60_000).toISOString() }),
|
||||
run,
|
||||
],
|
||||
["expired", () => artifact({ expired: true }), run],
|
||||
[
|
||||
"foreign repository",
|
||||
() =>
|
||||
artifact({
|
||||
workflow_run: { ...artifact().workflow_run, repository_id: 2 },
|
||||
}),
|
||||
run,
|
||||
],
|
||||
[
|
||||
"foreign branch",
|
||||
() =>
|
||||
artifact({
|
||||
workflow_run: { ...artifact().workflow_run, head_branch: "other" },
|
||||
}),
|
||||
run,
|
||||
],
|
||||
[
|
||||
"foreign SHA",
|
||||
() =>
|
||||
artifact({
|
||||
workflow_run: { ...artifact().workflow_run, head_sha: "b".repeat(40) },
|
||||
}),
|
||||
run,
|
||||
],
|
||||
[
|
||||
"untrusted workflow",
|
||||
artifact,
|
||||
() => run({ path: ".github/workflows/release.yml" }),
|
||||
],
|
||||
["failed run", artifact, () => run({ conclusion: "failure" })],
|
||||
[
|
||||
"old producer rerun with a fresh artifact",
|
||||
artifact,
|
||||
() =>
|
||||
run({ created_at: new Date(now - 25 * 60 * 60 * 1000).toISOString() }),
|
||||
],
|
||||
["wrong workflow event", artifact, () => run({ event: "push" })],
|
||||
];
|
||||
|
||||
it.each(rejectedCandidates)(
|
||||
"runs full qualification for a %s candidate",
|
||||
async (_name, makeArtifact, makeRun) => {
|
||||
const { decision } = await decide(makeArtifact(), makeRun());
|
||||
expect(decision.run_full).toBe(true);
|
||||
},
|
||||
);
|
||||
|
||||
it.each([
|
||||
["API error", { apiError: true }],
|
||||
["malformed query response", { malformed: true }],
|
||||
] as const)("fails closed on %s", async (_name, options) => {
|
||||
const { decision } = await decide(undefined, undefined, options);
|
||||
expect(decision.run_full).toBe(true);
|
||||
});
|
||||
|
||||
it("lets the daily force-full setting bypass the GitHub API", async () => {
|
||||
const { decision } = await decide(undefined, undefined, {
|
||||
force: true,
|
||||
apiError: true,
|
||||
});
|
||||
expect(decision).toMatchObject({ run_full: true });
|
||||
expect(decision.reason).toContain("Forced");
|
||||
});
|
||||
|
||||
async function workflowContract() {
|
||||
const workflowPath = new URL(
|
||||
"../../../.github/workflows/ci.yml",
|
||||
import.meta.url,
|
||||
);
|
||||
const parsed = parse(await readFile(workflowPath, "utf8"));
|
||||
return z
|
||||
.object({
|
||||
jobs: z.record(
|
||||
z.string(),
|
||||
z.object({ needs: z.unknown().optional() }).passthrough(),
|
||||
),
|
||||
})
|
||||
.parse(parsed).jobs;
|
||||
}
|
||||
|
||||
async function requiredRun(
|
||||
changes: Record<string, string>,
|
||||
overrides: { failed?: string; runAttempt?: string; event?: string } = {},
|
||||
) {
|
||||
const directory = await createTestTempDirectory("rea-ci-receipt-");
|
||||
await workflowGit(directory, ["init", "--initial-branch=main"]);
|
||||
await writeFile(join(directory, "fixture.txt"), "qualified source\n");
|
||||
await workflowGit(directory, ["add", "."]);
|
||||
await workflowGit(directory, ["commit", "-m", "fixture source"]);
|
||||
const head = await workflowGit(directory, ["rev-parse", "HEAD"]);
|
||||
const jobs = await workflowContract();
|
||||
const results: Record<
|
||||
string,
|
||||
{ result: string; outputs?: Record<string, string> }
|
||||
> = {};
|
||||
for (const name of z.array(z.string()).parse(jobs.required?.needs)) {
|
||||
results[name] = {
|
||||
result:
|
||||
name === overrides.failed
|
||||
? "failure"
|
||||
: changes.tier === "reused"
|
||||
? "skipped"
|
||||
: "success",
|
||||
};
|
||||
}
|
||||
results.changes = { result: "success", outputs: changes };
|
||||
const receiptDirectory = join(directory, ".cache", "receipt");
|
||||
const output = join(directory, "github-output");
|
||||
await writeFile(output, "");
|
||||
const env = {
|
||||
...process.env,
|
||||
CI_JOBS: JSON.stringify(results),
|
||||
CI_QUALIFICATION_DIRECTORY: receiptDirectory,
|
||||
GITHUB_EVENT_NAME: overrides.event ?? "schedule",
|
||||
GITHUB_SHA: head,
|
||||
GITHUB_RUN_ID: "100",
|
||||
GITHUB_RUN_ATTEMPT: overrides.runAttempt ?? "1",
|
||||
GITHUB_REPOSITORY: repository,
|
||||
GITHUB_OUTPUT: output,
|
||||
};
|
||||
const result = executeWorkflowFixture(process.execPath, [requiredScript], {
|
||||
cwd: directory,
|
||||
env,
|
||||
});
|
||||
return { directory, head, output, result };
|
||||
}
|
||||
|
||||
it("writes a full receipt only after every canonical flag and workflow need pass", async () => {
|
||||
const scopesPath = new URL("../../../scripts/ci/scopes.mjs", import.meta.url);
|
||||
const { CI_FLAGS } = await import(scopesPath.href);
|
||||
const changes = Object.fromEntries(
|
||||
CI_FLAGS.map((flag: string) => [flag, "true"]),
|
||||
);
|
||||
changes.tier = "full";
|
||||
const { directory, head, output, result } = await requiredRun(changes);
|
||||
await expect(result).resolves.toMatchObject({ stderr: "" });
|
||||
const receiptPath = join(directory, ".cache", "receipt", "receipt.json");
|
||||
const receipt = z
|
||||
.object({
|
||||
sourceSha: z.string(),
|
||||
runId: z.number(),
|
||||
runAttempt: z.number(),
|
||||
})
|
||||
.parse(JSON.parse(await readFile(receiptPath, "utf8")));
|
||||
expect(receipt).toMatchObject({ sourceSha: head, runId: 100, runAttempt: 1 });
|
||||
expect(await readFile(output, "utf8")).toContain(
|
||||
`receipt_path=${receiptPath}\n`,
|
||||
);
|
||||
expect(await readFile(output, "utf8")).toContain(
|
||||
"receipt_name=ci-full-qualification-v1\n",
|
||||
);
|
||||
});
|
||||
|
||||
it.each([
|
||||
["failed selected lane", { failed: "sqlite-runtime" }],
|
||||
["partial full scope", {}],
|
||||
] as const)(
|
||||
"does not issue a receipt after %s",
|
||||
async (caseName, overrides) => {
|
||||
const scopesPath = new URL(
|
||||
"../../../scripts/ci/scopes.mjs",
|
||||
import.meta.url,
|
||||
);
|
||||
const { CI_FLAGS } = await import(scopesPath.href);
|
||||
const changes = Object.fromEntries(
|
||||
CI_FLAGS.map((flag: string) => [flag, "true"]),
|
||||
);
|
||||
changes.tier = "full";
|
||||
if (caseName === "partial full scope") changes.sqlite = "false";
|
||||
const outcome = await requiredRun(changes, overrides);
|
||||
await expect(outcome.result).rejects.toMatchObject({ code: 1 });
|
||||
await expect(
|
||||
readFile(join(outcome.directory, ".cache", "receipt", "receipt.json")),
|
||||
).rejects.toMatchObject({ code: "ENOENT" });
|
||||
expect(await readFile(outcome.output, "utf8")).not.toContain("receipt_");
|
||||
},
|
||||
);
|
||||
|
||||
it("never writes a fresh receipt for a reused scheduled qualification", async () => {
|
||||
const scopesPath = new URL("../../../scripts/ci/scopes.mjs", import.meta.url);
|
||||
const { CI_FLAGS } = await import(scopesPath.href);
|
||||
const changes = Object.fromEntries(
|
||||
CI_FLAGS.map((flag: string) => [flag, "false"]),
|
||||
);
|
||||
changes.tier = "reused";
|
||||
changes.qualification_run_id = "7";
|
||||
const outcome = await requiredRun(changes);
|
||||
const result = await outcome.result;
|
||||
await expect(
|
||||
readFile(join(outcome.directory, ".cache", "receipt", "receipt.json")),
|
||||
).rejects.toMatchObject({ code: "ENOENT" });
|
||||
expect(result.stdout).toContain("Qualification reused from run 7");
|
||||
expect(await readFile(outcome.output, "utf8")).not.toContain("receipt_");
|
||||
});
|
||||
|
||||
it.each([
|
||||
["missing receipt run", { qualification_run_id: "" }, "schedule"],
|
||||
["selected validation lane", { sqlite: "true" }, "schedule"],
|
||||
["manual dispatch", {}, "workflow_dispatch"],
|
||||
] as const)(
|
||||
"rejects qualification reuse with %s",
|
||||
async (_name, override, event) => {
|
||||
const { CI_FLAGS } = await import(
|
||||
new URL("../../../scripts/ci/scopes.mjs", import.meta.url).href
|
||||
);
|
||||
const changes: Record<string, string> = {
|
||||
...Object.fromEntries(CI_FLAGS.map((flag: string) => [flag, "false"])),
|
||||
tier: "reused",
|
||||
qualification_run_id: "7",
|
||||
...override,
|
||||
};
|
||||
const outcome = await requiredRun(changes, { event });
|
||||
await expect(outcome.result).rejects.toMatchObject({
|
||||
code: 1,
|
||||
stderr: expect.stringContaining("scheduled admission receipt"),
|
||||
});
|
||||
expect(await readFile(outcome.output, "utf8")).not.toContain("receipt_");
|
||||
},
|
||||
);
|
||||
|
|
@ -27,6 +27,9 @@ const workflow = async (filename = "ci.yml") =>
|
|||
);
|
||||
|
||||
it("binds SQLite compatibility jobs to the build's prepared runtime artifact and root layout", async () => {
|
||||
expect((await workflow()).jobs["sqlite-runtime"]?.if).toBe(
|
||||
"needs.changes.outputs.sqlite == 'true'",
|
||||
);
|
||||
const steps = z.array(
|
||||
z.object({
|
||||
uses: z.string().optional(),
|
||||
|
|
@ -52,6 +55,91 @@ it("binds SQLite compatibility jobs to the build's prepared runtime artifact and
|
|||
expect(downloads[0]?.with).toEqual({ name, path: "." });
|
||||
});
|
||||
|
||||
it("exports every canonical scope with read authority for admission and keeps telemetry outside the gate", async () => {
|
||||
const { jobs } = await workflow();
|
||||
const { CI_FLAGS } = z
|
||||
.object({ CI_FLAGS: z.array(z.string()) })
|
||||
.parse(
|
||||
await import(
|
||||
new URL("../../../scripts/ci/scopes.mjs", import.meta.url).href
|
||||
),
|
||||
);
|
||||
const outputs = z.record(z.string(), z.string()).parse(jobs.changes?.outputs);
|
||||
expect(jobs.changes?.permissions).toEqual({
|
||||
contents: "read",
|
||||
actions: "read",
|
||||
});
|
||||
expect(jobs.build?.permissions).toEqual({
|
||||
contents: "read",
|
||||
actions: "read",
|
||||
});
|
||||
for (const flag of CI_FLAGS)
|
||||
expect(outputs[flag]).toBe(`\${{ steps.plan.outputs.${flag} }}`);
|
||||
const steps = z
|
||||
.array(
|
||||
z.object({
|
||||
run: z.string().optional(),
|
||||
if: z.string().optional(),
|
||||
}),
|
||||
)
|
||||
.parse(jobs.required?.steps);
|
||||
const telemetry = steps.find((step) =>
|
||||
step.run?.includes("scripts/ci/job-timings.mjs"),
|
||||
);
|
||||
expect(telemetry).toBeUndefined();
|
||||
const timings = z
|
||||
.object({
|
||||
on: z.object({
|
||||
workflow_run: z.object({
|
||||
workflows: z.array(z.string()),
|
||||
types: z.array(z.string()),
|
||||
}),
|
||||
}),
|
||||
permissions: z.record(z.string(), z.string()),
|
||||
jobs: z.object({
|
||||
report: z.object({
|
||||
steps: z.array(
|
||||
z.object({
|
||||
uses: z.string().optional(),
|
||||
run: z.string().optional(),
|
||||
with: z.record(z.string(), z.unknown()).optional(),
|
||||
env: z.record(z.string(), z.unknown()).optional(),
|
||||
}),
|
||||
),
|
||||
}),
|
||||
}),
|
||||
})
|
||||
.parse(
|
||||
parse(
|
||||
await readFile(
|
||||
new URL("../../../.github/workflows/ci-timings.yml", import.meta.url),
|
||||
"utf8",
|
||||
),
|
||||
),
|
||||
);
|
||||
expect(timings.on.workflow_run).toEqual({
|
||||
workflows: ["CI", "Release"],
|
||||
types: ["completed"],
|
||||
});
|
||||
expect(timings.permissions).toEqual({ contents: "read", actions: "read" });
|
||||
const checkout = timings.jobs.report.steps.find((step) =>
|
||||
step.uses?.startsWith("actions/checkout@"),
|
||||
);
|
||||
expect(checkout?.with).toEqual({
|
||||
ref: "${{ github.event.repository.default_branch }}",
|
||||
"persist-credentials": false,
|
||||
});
|
||||
const report = timings.jobs.report.steps.find((step) =>
|
||||
step.run?.includes("scripts/ci/job-timings.mjs"),
|
||||
);
|
||||
expect(report?.env?.CI_TIMING_RUN_ID).toBe(
|
||||
"${{ github.event.workflow_run.id }}",
|
||||
);
|
||||
expect(report?.env?.CI_TIMING_RUN_ATTEMPT).toBe(
|
||||
"${{ github.event.workflow_run.run_attempt }}",
|
||||
);
|
||||
});
|
||||
|
||||
const requiredNeeds = async () => {
|
||||
const { jobs } = await workflow();
|
||||
return { jobs, needs: z.array(z.string()).parse(jobs.required?.needs) };
|
||||
|
|
@ -131,7 +219,10 @@ it("keeps fast feedback separate from complete qualification and release proof",
|
|||
),
|
||||
),
|
||||
);
|
||||
expect(ci.on.schedule).toEqual([{ cron: "23 * * * *" }]);
|
||||
expect(ci.on.schedule).toEqual([
|
||||
{ cron: "23 0-2,4-23 * * *" },
|
||||
{ cron: "23 3 * * *" },
|
||||
]);
|
||||
expect(ci.concurrency.group).toContain("'qualification' || 'feedback'");
|
||||
expect(ci.concurrency.group).toContain("inputs.source_sha ||");
|
||||
for (const fullEvent of ["schedule", "workflow_dispatch"])
|
||||
|
|
@ -164,9 +255,10 @@ it.each([
|
|||
["true", "skipped", false],
|
||||
["true", "missing", false],
|
||||
["false", "skipped", true],
|
||||
["false", "failure", false],
|
||||
] as const)(
|
||||
"enforces the SQLite required gate for code=%s and result=%s",
|
||||
async (code, result, passes) => {
|
||||
"enforces the SQLite required gate for sqlite=%s and result=%s",
|
||||
async (sqlite, result, passes) => {
|
||||
const workflow = await requiredNeeds();
|
||||
// GitHub exposes only direct dependencies in the required job's needs context.
|
||||
const needs: Record<
|
||||
|
|
@ -175,7 +267,7 @@ it.each([
|
|||
> = Object.fromEntries(
|
||||
workflow.needs.map((name) => [name, { result: "success" }]),
|
||||
);
|
||||
needs.changes = { result: "success", outputs: { code } };
|
||||
needs.changes = { result: "success", outputs: { code: "true", sqlite } };
|
||||
if (result === "missing") delete needs["sqlite-runtime"];
|
||||
else if ("sqlite-runtime" in needs) needs["sqlite-runtime"] = { result };
|
||||
const execution = executeWorkflowFixture(
|
||||
|
|
|
|||
Loading…
Reference in New Issue