ci: document deterministic suite ownership
This commit is contained in:
morluto
parent
a8161f0e55
commit
3c42403b78
|
|
@ -88,7 +88,7 @@ jobs:
|
|||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
shard: [1/2, 2/2]
|
||||
shard: [1/4, 2/4, 3/4, 4/4]
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
- uses: actions/setup-node@v4
|
||||
|
|
@ -160,13 +160,47 @@ jobs:
|
|||
path: .vitest-reports
|
||||
pattern: vitest-report-*
|
||||
merge-multiple: true
|
||||
- if: needs.changes.outputs.code == 'true' && success()
|
||||
run: npm run test:ci:merge
|
||||
- name: Merge coverage, JUnit, and timing reports
|
||||
if: needs.changes.outputs.code == 'true' && success()
|
||||
run: >-
|
||||
npm run test:ci:merge --
|
||||
--coverage.reportsDirectory=coverage
|
||||
--coverage.reporter=text
|
||||
--coverage.reporter=json-summary
|
||||
--coverage.reporter=lcov
|
||||
--reporter=default
|
||||
--reporter=junit
|
||||
--reporter=json
|
||||
--outputFile.junit=test-results.xml
|
||||
--outputFile.json=test-results.json
|
||||
- name: Summarize slowest test files
|
||||
if: needs.changes.outputs.code == 'true' && success()
|
||||
shell: bash
|
||||
run: |
|
||||
node <<'NODE' >> "${GITHUB_STEP_SUMMARY}"
|
||||
const report = JSON.parse(require("node:fs").readFileSync("test-results.json", "utf8"));
|
||||
const files = (report.testResults ?? [])
|
||||
.map((result) => ({
|
||||
name: result.name,
|
||||
duration: Math.max(0, (result.endTime ?? result.startTime) - result.startTime),
|
||||
}))
|
||||
.sort((left, right) => right.duration - left.duration)
|
||||
.slice(0, 20);
|
||||
console.log("## Slowest deterministic test files\n");
|
||||
console.log("| File | Duration (ms) |");
|
||||
console.log("| --- | ---: |");
|
||||
for (const file of files) {
|
||||
console.log(`| \`${file.name}\` | ${file.duration} |`);
|
||||
}
|
||||
NODE
|
||||
- uses: actions/upload-artifact@v4
|
||||
if: needs.changes.outputs.code == 'true' && always()
|
||||
with:
|
||||
name: test-results
|
||||
path: test-results.xml
|
||||
path: |
|
||||
test-results.xml
|
||||
test-results.json
|
||||
coverage/
|
||||
if-no-files-found: error
|
||||
retention-days: 30
|
||||
|
||||
|
|
@ -258,17 +292,17 @@ jobs:
|
|||
- name: Run curated Windows P0 tests
|
||||
run: >-
|
||||
npx vitest run --coverage.enabled=false
|
||||
tests/binaryTarget.test.ts
|
||||
tests/windowsCapabilities.test.ts
|
||||
tests/ghidraWindowsFixture.test.ts
|
||||
tests/ghidraInstallation.test.ts
|
||||
tests/ghidraProvider.test.ts
|
||||
tests/ghidraTransport.test.ts
|
||||
tests/ghidraTargetSnapshot.test.ts
|
||||
tests/ghidraClient.test.ts
|
||||
tests/ghidraLauncher.test.ts
|
||||
tests/processOwnership.test.ts
|
||||
tests/realGhidraWindowsWorkflow.test.ts
|
||||
tests/composition/analysis-sessions/binaryTarget.test.ts
|
||||
tests/conformance/providers/ghidra/windowsCapabilities.test.ts
|
||||
tests/conformance/providers/ghidra/ghidraWindowsFixture.test.ts
|
||||
tests/boundary/providers/ghidra/ghidraInstallation.test.ts
|
||||
tests/boundary/providers/ghidra/ghidraProvider.test.ts
|
||||
tests/boundary/providers/ghidra/ghidraTransport.test.ts
|
||||
tests/boundary/providers/ghidra/ghidraTargetSnapshot.test.ts
|
||||
tests/boundary/providers/ghidra/ghidraClient.test.ts
|
||||
tests/boundary/providers/ghidra/ghidraLauncher.test.ts
|
||||
tests/boundary/process/processOwnership.test.ts
|
||||
tests/conformance/providers/ghidra/realGhidraWindowsWorkflow.test.ts
|
||||
- run: npm run verify:package:windows
|
||||
- uses: actions/upload-artifact@v4
|
||||
if: always()
|
||||
|
|
|
|||
|
|
@ -38,15 +38,13 @@ provider and replay execution remains uncached, but their deterministic build
|
|||
prerequisite uses Turbo. CI uploads the rendered TypeDoc site as an `api-docs`
|
||||
artifact; the generated HTML is not committed.
|
||||
|
||||
Local `npm test` runs without coverage, retries, or verbose output. Use `npm run
|
||||
test:fast` for pure and subprocess groups, `npm run test:integration` for serial
|
||||
filesystem/process/CLI cases. `npm run test:changed` runs changed and related
|
||||
non-serial tests once; `npm run test:watch` keeps that selection live. Pure tests
|
||||
use Vitest threads, while subprocess tests stay in isolated forks. CI splits the
|
||||
complete suite across two native Vitest
|
||||
shards, then merges their coverage and JUnit reports. `npm run test:ci` runs the
|
||||
equivalent unsharded gate locally. Coverage thresholds remain in
|
||||
`vitest.config.ts`.
|
||||
Local `npm test` runs every deterministic Vitest project without coverage,
|
||||
retries, or verbose output. Focused boundary, acceptance, changed-test, and
|
||||
watch commands are documented in [docs/testing.md](docs/testing.md), together
|
||||
with the path-based behavioral-depth rules. Changed-test selection follows the
|
||||
import graph and is feedback, not the release gate. CI splits the complete suite
|
||||
across four native Vitest shards, then merges coverage, JUnit, and JSON timing
|
||||
reports. Coverage thresholds remain in `vitest.config.ts`.
|
||||
|
||||
CI installs dependencies once for all static checks, cancels superseded PR
|
||||
runs, and skips package, Windows, and full test lanes for documentation-only
|
||||
|
|
|
|||
|
|
@ -794,7 +794,10 @@ Any agent that can run a local MCP server can use the manual configuration. Setu
|
|||
|
||||
## Development
|
||||
|
||||
See [CONTRIBUTING.md](CONTRIBUTING.md) for setup, architecture, tests, and release instructions. PR CI publishes generated API documentation as the `api-docs` workflow artifact.
|
||||
See [CONTRIBUTING.md](CONTRIBUTING.md) for setup, architecture, and release
|
||||
instructions, and [docs/testing.md](docs/testing.md) for the behavioral test
|
||||
depths, focused developer commands, coverage floors, and CI evidence. PR CI
|
||||
publishes generated API documentation as the `api-docs` workflow artifact.
|
||||
|
||||
`npm run verify:agent` runs brandless native, JavaScript-application, managed,
|
||||
and browser prompts through a real local Codex CLI. Its JSON report measures
|
||||
|
|
|
|||
|
|
@ -0,0 +1,71 @@
|
|||
# Test architecture migration ledger
|
||||
|
||||
This ledger records how the flat test suite was classified during the
|
||||
coordinated architecture cutover. It is an evidence index, not a declaration
|
||||
that every test became independent merely because its path changed.
|
||||
|
||||
## Classification batches
|
||||
|
||||
The following mappings are grounded in the current moved paths. They describe
|
||||
ownership only; decomposition and evidence replacement must be recorded in the
|
||||
per-suite table below.
|
||||
|
||||
| Previous path | Current path | Classification |
|
||||
| --------------------------------------------------------------------------------------------------- | -------------------------------------------------------------------------------- | ----------------------------------- |
|
||||
| `tests/applicationWorkflowCli.test.ts` | `tests/acceptance/applications/applicationWorkflowCli.test.ts` | Acceptance application workflow |
|
||||
| `tests/setupLifecycle.test.ts` | `tests/acceptance/setup/setupLifecycle.test.ts` | Acceptance setup workflow |
|
||||
| `tests/cdpBrowserProvider.test.ts` | `tests/boundary/browser/cdpBrowserProvider*.test.ts` | Browser boundary |
|
||||
| `tests/ghidraClient.test.ts` | `tests/boundary/providers/ghidra/ghidraClient.test.ts` | Ghidra provider boundary |
|
||||
| `tests/processOwnership.test.ts` | `tests/boundary/process/processOwnership*.test.ts` | Process boundary |
|
||||
| `tests/binarySession.test.ts` | `tests/composition/analysis-sessions/binarySession.*.test.ts` | Analysis-session composition |
|
||||
| `tests/javascriptSemanticAnalysis.test.ts` | `tests/composition/javascript-applications/javascriptSemanticAnalysis.*.test.ts` | JavaScript-application composition |
|
||||
| `tests/config.test.ts` | `tests/composition/setup-lifecycle/config.test.ts` | Setup-lifecycle composition |
|
||||
| `tests/conformanceFixtures.test.ts` | `tests/conformance/providers/conformanceFixtures.test.ts` | Provider conformance |
|
||||
| `tests/codexAgentEval.test.ts` | `tests/evaluation/model-evals/codexAgentEval.test.ts` | Deterministic model-evaluator logic |
|
||||
| `tests/composition/analysis-sessions/{boundedCartesianProjection,symbolAnalysis,jsonShape}.test.ts` | `src/domain/{boundedCartesianProjection,symbolAnalysis,jsonShape}.test.ts` | Colocated pure-domain behavior |
|
||||
|
||||
The same directory mapping applies to the other files moved in each batch:
|
||||
`tests/acceptance/{analysis,applications,investigations,setup}`,
|
||||
`tests/boundary/{browser,cli,filesystem,mcp,process,providers}`,
|
||||
`tests/composition/{analysis-sessions,evidence-investigations,javascript-applications,process-capture,runtime-observation,setup-lifecycle}`,
|
||||
`tests/conformance`, and `tests/evaluation`. Git rename detection remains the
|
||||
canonical source for the complete old-to-new path list.
|
||||
|
||||
## Per-suite evidence record
|
||||
|
||||
Add a row when a legacy suite is materially decomposed, consolidated, replaced,
|
||||
or removed. “Replacement evidence” must identify executable behavior, a
|
||||
generated artifact comparison, a protocol probe, or an explicit verifier; a new
|
||||
path alone is not replacement evidence.
|
||||
|
||||
| Legacy suite or claim | Disposition | Replacement evidence | Validation | Remaining gap |
|
||||
| ----------------------------------------------- | ------------ | ------------------------------------------------------------------------------------------------------------------------------------- | ---------------------------------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------- |
|
||||
| Flat path classification | Moved | Vitest configuration contract proves every deterministic test belongs to exactly one project | Configuration contract test | Record any exceptions discovered during cutover |
|
||||
| Direct session construction outside owner tests | Consolidated | `composeBinarySession` and `composeBinarySessionFromProvider` are the shared production/test seams | Repository search plus session-composition and acceptance tests | None |
|
||||
| Repeated temporary-directory setup | Consolidated | `createTestTempDirectory` delegates every test-owned root to the typed workspace fixture with HOME/XDG projection and awaited cleanup | Repository search plus filesystem, process, runtime suites, and the Vitest topology hygiene contract | None; direct `mkdtemp` remains only in the workspace fixture and package-canary source text |
|
||||
| Repeated CLI/MCP connection helpers | Consolidated | Typed CLI and MCP fixtures under `tests/support`; CLI output, dispatcher, and server tests use them | Acceptance startup, typed failure, lifecycle, and teardown tests | Provider-specific protocol harnesses remain local |
|
||||
| Implementation-shaped assertions | Replaced | Recording adapters, executable bridge/protocol probes, generated-artifact comparisons, and supported configuration contracts | Focused boundary/conformance tests and repository search | None identified in the final audit |
|
||||
| Module mocks and spies | Replaced | Recording adapters and caller-visible outcomes | Repository search plus focused reload, Ghidra, and enhanced-tool tests | None |
|
||||
| `processCapture.test.ts` | Consolidated | Ten lifecycle, observation, reactive, replay, terminal, validation, and trace suites | 42 focused boundary tests | None |
|
||||
| `cdpBrowserProvider.test.ts` | Consolidated | Five discovery, document/script, network, lifecycle, and sensitive-data suites | 36 focused browser-boundary tests | None |
|
||||
| `javascriptSemanticAnalysis.test.ts` | Consolidated | Six structure, call, dataflow, runtime, native-binding, and rejection suites | 29 focused composition tests | None |
|
||||
| `binarySession.test.ts` | Consolidated | Nine provider binding, lifecycle, cache, Evidence, permissions, and shutdown suites | Focused composition tests and shared composition-seam test | None |
|
||||
| MCP mega-suites | Consolidated | Tool/resource-family suites plus catalog-wide identity and lifecycle coverage | Focused MCP tests and complete deterministic run | None |
|
||||
| Provider client mega-suites | Consolidated | Capability-named protocol, timeout, ownership, error, and cleanup suites | 85 focused provider tests | Real systems remain in `verify:*` lanes |
|
||||
| Pure domain proofs | Moved | Colocated bounded projection, symbol classification, and JSON-shape tests | Domain Vitest project | Continue moving only tests with no external boundary dependency |
|
||||
|
||||
Use these dispositions consistently:
|
||||
|
||||
- **Moved**: the same material claim now has depth-appropriate ownership.
|
||||
- **Replaced**: stronger executable evidence supersedes the old assertion.
|
||||
- **Consolidated**: one retained test proves a claim previously duplicated.
|
||||
- **Intentionally removed**: the claim was not public or independent; the row
|
||||
must still name the retained evidence or explain why none is warranted.
|
||||
- **In progress**: classification exists but the replacement audit is not
|
||||
complete. No final-cutover row may retain this disposition.
|
||||
|
||||
The exact old-to-new inventory is mechanically enforced by the project
|
||||
classification contract and retained in Git rename metadata. Real-system lanes,
|
||||
coverage, repeated-run stability, and benchmark timing are release evidence;
|
||||
they are reported separately because they depend on the execution host and
|
||||
installed external systems.
|
||||
|
|
@ -0,0 +1,94 @@
|
|||
# Testing REA
|
||||
|
||||
REA's test suite follows one rule: use the lowest behavioral depth that can
|
||||
prove a claim. Tests are grouped into native Vitest projects so ownership,
|
||||
allowed dependencies, and runtime cost are visible from their paths.
|
||||
|
||||
## Behavioral depths
|
||||
|
||||
| Depth | Location | What it proves |
|
||||
| ----------- | ---------------------- | ----------------------------------------------------------------------------------------------------- |
|
||||
| Module | `src/**/*.test.ts` | One domain, service, or adapter through its narrow public surface |
|
||||
| Composition | `tests/composition/**` | Provider-neutral session and registry wiring; filesystem access is limited to fixture materialization |
|
||||
| Boundary | `tests/boundary/**` | Exactly one production filesystem, process, network, browser, CLI, MCP, or provider boundary |
|
||||
| Acceptance | `tests/acceptance/**` | A complete compiled CLI or MCP workflow |
|
||||
| Conformance | `tests/conformance/**` | Shared provider contracts, parameterized by declared capabilities and explicit opt-outs |
|
||||
| Evaluation | `tests/evaluation/**` | Deterministic evaluator parsing, scoring, and report generation |
|
||||
|
||||
Colocated domain tests may import only their owning domain and inward
|
||||
dependencies. They do not construct sessions or servers and do not start
|
||||
subprocesses or network listeners. Colocated application tests exercise one
|
||||
service through explicit ports and recording adapters, never a CLI or MCP
|
||||
entrypoint. Composition tests may assemble provider-neutral sessions and
|
||||
registries but do not cross production filesystem, process, socket, or browser boundaries.
|
||||
Boundary tests cross one production boundary. Only acceptance tests assemble
|
||||
the complete runtime or invoke the compiled product surface.
|
||||
|
||||
`tests/process-global/**` is reserved for cases with a demonstrated dependency
|
||||
on process-global state. Those tests run without file parallelism. Reusable,
|
||||
test-scoped fixtures live under `tests/support/**`; immutable source artifacts
|
||||
remain under `tests/fixtures/**`.
|
||||
The process-global Vitest configuration contract rejects new direct temporary-root
|
||||
creation outside the workspace seam and its narrowly documented boundary/package
|
||||
exceptions.
|
||||
|
||||
Real Hopper, Ghidra, browser, package, managed-code, and controlled-replay
|
||||
claims belong to their explicit `npm run verify:*` lanes. They are not inferred
|
||||
from mocks or folded into the deterministic local gate. Real model trials are
|
||||
manual; Vitest covers only deterministic evaluator logic.
|
||||
|
||||
## Developer commands
|
||||
|
||||
`npm test` runs every deterministic project once. The narrower feedback loops
|
||||
are:
|
||||
|
||||
| Command | Scope |
|
||||
| ------------------------- | -------------------------------------------------------------- |
|
||||
| `npm run test:fast` | Domain, service, and composition projects |
|
||||
| `npm run test:boundary` | Boundary and process-global projects |
|
||||
| `npm run test:acceptance` | Complete CLI and MCP acceptance workflows |
|
||||
| `npm run test:changed` | Tests related through Vitest's import graph |
|
||||
| `npm run test:watch` | Changed domain and service tests in watch mode |
|
||||
| `npm run test:watch:all` | Changed tests from every deterministic project in watch mode |
|
||||
| `npm run check:changed` | Cached static checks followed by changed tests |
|
||||
| `npm run check:pr` | Static, generated-document, and complete deterministic PR gate |
|
||||
|
||||
Changed-test selection is a fast feedback aid, not release evidence. It can
|
||||
miss behavior connected through runtime registration, generated data, shell
|
||||
entrypoints, or other relationships that are absent from the import graph.
|
||||
Use `npm run check:pr` before handing off a contribution.
|
||||
|
||||
Vitest and Node persistent compile caches are deliberately not enabled by
|
||||
default. To evaluate repeated local runs, opt in for both cold and warm
|
||||
measurements with an isolated cache:
|
||||
|
||||
```bash
|
||||
NODE_COMPILE_CACHE=.cache/node-compile npm test
|
||||
```
|
||||
|
||||
Do not report the warm result as a cold-suite improvement, and do not enable
|
||||
the cache in coverage or benchmark CI without first showing that its
|
||||
instrumentation remains equivalent.
|
||||
|
||||
## Coverage and timing
|
||||
|
||||
CI owns coverage. The aggregate floors are 65% statements, 60% branches, 60%
|
||||
functions, and 68% lines. `src/domain/**` must reach 80% statements, 75%
|
||||
branches, 75% functions, and 80% lines. `src/contracts/**` must reach 85%
|
||||
statements, 80% branches, 80% functions, and 85% lines. Thresholds are
|
||||
glob-specific rather than per-file and are never updated automatically.
|
||||
Coverage does not replace named boundary, acceptance, or real-provider scenario
|
||||
matrices.
|
||||
|
||||
CI runs four native Vitest shards without retries. Each shard emits a blob
|
||||
report; the merge job produces aggregate coverage plus JUnit and JSON timing
|
||||
reports, uploads them together, and writes the slowest files to the workflow
|
||||
summary. Static checks, documentation, build, package verification, and
|
||||
real-system lanes remain separate jobs so one kind of evidence cannot stand in
|
||||
for another.
|
||||
|
||||
The PR acceptance target is a median `npm run check:pr` wall time below three
|
||||
minutes across three warm-build runs on the benchmark host. Keep Vitest caches
|
||||
cold unless separately identified. The final cutover also requires three
|
||||
consecutive deterministic passes without retries, `npm run verify:package`, and
|
||||
the applicable real-system lanes.
|
||||
16
package.json
16
package.json
|
|
@ -63,6 +63,8 @@
|
|||
"check": "turbo run typecheck lint format:check knip metadata:check",
|
||||
"precheck:fast": "npm run deps:check",
|
||||
"check:fast": "turbo run typecheck lint",
|
||||
"precheck:changed": "npm run deps:check",
|
||||
"check:changed": "turbo run typecheck lint && npm run test:changed",
|
||||
"precheck:test": "npm run deps:check",
|
||||
"check:test": "turbo run typecheck lint format:check knip metadata:check test:run",
|
||||
"precheck:pr": "npm run deps:check",
|
||||
|
|
@ -83,11 +85,15 @@
|
|||
"test": "vitest run",
|
||||
"test:run": "vitest run",
|
||||
"pretest:fast": "npm run build:cached",
|
||||
"test:fast": "vitest run --project parallel --project subprocess",
|
||||
"test:fast": "vitest run --project domain --project services --project composition",
|
||||
"pretest:boundary": "npm run build:cached",
|
||||
"test:boundary": "vitest run --project boundary --project process-global",
|
||||
"pretest:acceptance": "npm run build:cached",
|
||||
"test:acceptance": "vitest run --project acceptance",
|
||||
"pretest:changed": "npm run build:cached",
|
||||
"test:changed": "vitest run --project parallel --project subprocess --changed",
|
||||
"test:changed": "vitest run --changed",
|
||||
"pretest:integration": "npm run build:cached",
|
||||
"test:integration": "vitest run --project serial-integration",
|
||||
"test:integration": "vitest run --project boundary --project process-global",
|
||||
"pretest:coverage": "npm run build:cached",
|
||||
"test:coverage": "vitest run --coverage.enabled --coverage.provider=v8",
|
||||
"test:ci": "npm run test:coverage -- --reporter=default --reporter=junit --outputFile.junit=test-results.xml",
|
||||
|
|
@ -98,7 +104,9 @@
|
|||
"pretest:verbose": "npm run build:cached",
|
||||
"test:verbose": "vitest run --reporter=verbose",
|
||||
"pretest:watch": "npm run build:cached",
|
||||
"test:watch": "vitest --project parallel --project subprocess --changed",
|
||||
"test:watch": "vitest --project domain --project services --changed",
|
||||
"pretest:watch:all": "npm run build:cached",
|
||||
"test:watch:all": "vitest --changed",
|
||||
"typecheck": "tsc --noEmit",
|
||||
"verify:test-temp-hygiene": "npm run build:cached && node scripts/verify-test-temp-hygiene.mjs",
|
||||
"verify:hopper": "npm run verify:fixtures && npm run build:cached && node scripts/verify-real-hopper.mjs",
|
||||
|
|
|
|||
Loading…
Reference in New Issue