From 3c42403b78b53e7b5358c2bca9bc156ef2c32efd Mon Sep 17 00:00:00 2001 From: morluto <76467478+morluto@users.noreply.github.com> Date: Wed, 29 Jul 2026 22:02:24 +0000 Subject: [PATCH] ci: document deterministic suite ownership --- .github/workflows/ci.yml | 64 ++++++++++++++++++------ CONTRIBUTING.md | 16 +++--- README.md | 5 +- docs/test-migration-ledger.md | 71 ++++++++++++++++++++++++++ docs/testing.md | 94 +++++++++++++++++++++++++++++++++++ package.json | 16 ++++-- 6 files changed, 237 insertions(+), 29 deletions(-) create mode 100644 docs/test-migration-ledger.md create mode 100644 docs/testing.md diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index a82e459b9..d368ababb 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -88,7 +88,7 @@ jobs: strategy: fail-fast: false matrix: - shard: [1/2, 2/2] + shard: [1/4, 2/4, 3/4, 4/4] steps: - uses: actions/checkout@v4 - uses: actions/setup-node@v4 @@ -160,13 +160,47 @@ jobs: path: .vitest-reports pattern: vitest-report-* merge-multiple: true - - if: needs.changes.outputs.code == 'true' && success() - run: npm run test:ci:merge + - name: Merge coverage, JUnit, and timing reports + if: needs.changes.outputs.code == 'true' && success() + run: >- + npm run test:ci:merge -- + --coverage.reportsDirectory=coverage + --coverage.reporter=text + --coverage.reporter=json-summary + --coverage.reporter=lcov + --reporter=default + --reporter=junit + --reporter=json + --outputFile.junit=test-results.xml + --outputFile.json=test-results.json + - name: Summarize slowest test files + if: needs.changes.outputs.code == 'true' && success() + shell: bash + run: | + node <<'NODE' >> "${GITHUB_STEP_SUMMARY}" + const report = JSON.parse(require("node:fs").readFileSync("test-results.json", "utf8")); + const files = (report.testResults ?? []) + .map((result) => ({ + name: result.name, + duration: Math.max(0, (result.endTime ?? result.startTime) - result.startTime), + })) + .sort((left, right) => right.duration - left.duration) + .slice(0, 20); + console.log("## Slowest deterministic test files\n"); + console.log("| File | Duration (ms) |"); + console.log("| --- | ---: |"); + for (const file of files) { + console.log(`| \`${file.name}\` | ${file.duration} |`); + } + NODE - uses: actions/upload-artifact@v4 if: needs.changes.outputs.code == 'true' && always() with: name: test-results - path: test-results.xml + path: | + test-results.xml + test-results.json + coverage/ if-no-files-found: error retention-days: 30 @@ -258,17 +292,17 @@ jobs: - name: Run curated Windows P0 tests run: >- npx vitest run --coverage.enabled=false - tests/binaryTarget.test.ts - tests/windowsCapabilities.test.ts - tests/ghidraWindowsFixture.test.ts - tests/ghidraInstallation.test.ts - tests/ghidraProvider.test.ts - tests/ghidraTransport.test.ts - tests/ghidraTargetSnapshot.test.ts - tests/ghidraClient.test.ts - tests/ghidraLauncher.test.ts - tests/processOwnership.test.ts - tests/realGhidraWindowsWorkflow.test.ts + tests/composition/analysis-sessions/binaryTarget.test.ts + tests/conformance/providers/ghidra/windowsCapabilities.test.ts + tests/conformance/providers/ghidra/ghidraWindowsFixture.test.ts + tests/boundary/providers/ghidra/ghidraInstallation.test.ts + tests/boundary/providers/ghidra/ghidraProvider.test.ts + tests/boundary/providers/ghidra/ghidraTransport.test.ts + tests/boundary/providers/ghidra/ghidraTargetSnapshot.test.ts + tests/boundary/providers/ghidra/ghidraClient.test.ts + tests/boundary/providers/ghidra/ghidraLauncher.test.ts + tests/boundary/process/processOwnership.test.ts + tests/conformance/providers/ghidra/realGhidraWindowsWorkflow.test.ts - run: npm run verify:package:windows - uses: actions/upload-artifact@v4 if: always() diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 9a4c84717..1a5bcbd45 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -38,15 +38,13 @@ provider and replay execution remains uncached, but their deterministic build prerequisite uses Turbo. CI uploads the rendered TypeDoc site as an `api-docs` artifact; the generated HTML is not committed. -Local `npm test` runs without coverage, retries, or verbose output. Use `npm run -test:fast` for pure and subprocess groups, `npm run test:integration` for serial -filesystem/process/CLI cases. `npm run test:changed` runs changed and related -non-serial tests once; `npm run test:watch` keeps that selection live. Pure tests -use Vitest threads, while subprocess tests stay in isolated forks. CI splits the -complete suite across two native Vitest -shards, then merges their coverage and JUnit reports. `npm run test:ci` runs the -equivalent unsharded gate locally. Coverage thresholds remain in -`vitest.config.ts`. +Local `npm test` runs every deterministic Vitest project without coverage, +retries, or verbose output. Focused boundary, acceptance, changed-test, and +watch commands are documented in [docs/testing.md](docs/testing.md), together +with the path-based behavioral-depth rules. Changed-test selection follows the +import graph and is feedback, not the release gate. CI splits the complete suite +across four native Vitest shards, then merges coverage, JUnit, and JSON timing +reports. Coverage thresholds remain in `vitest.config.ts`. CI installs dependencies once for all static checks, cancels superseded PR runs, and skips package, Windows, and full test lanes for documentation-only diff --git a/README.md b/README.md index 0ef2707a6..14e7e5677 100644 --- a/README.md +++ b/README.md @@ -794,7 +794,10 @@ Any agent that can run a local MCP server can use the manual configuration. Setu ## Development -See [CONTRIBUTING.md](CONTRIBUTING.md) for setup, architecture, tests, and release instructions. PR CI publishes generated API documentation as the `api-docs` workflow artifact. +See [CONTRIBUTING.md](CONTRIBUTING.md) for setup, architecture, and release +instructions, and [docs/testing.md](docs/testing.md) for the behavioral test +depths, focused developer commands, coverage floors, and CI evidence. PR CI +publishes generated API documentation as the `api-docs` workflow artifact. `npm run verify:agent` runs brandless native, JavaScript-application, managed, and browser prompts through a real local Codex CLI. Its JSON report measures diff --git a/docs/test-migration-ledger.md b/docs/test-migration-ledger.md new file mode 100644 index 000000000..763b39cc7 --- /dev/null +++ b/docs/test-migration-ledger.md @@ -0,0 +1,71 @@ +# Test architecture migration ledger + +This ledger records how the flat test suite was classified during the +coordinated architecture cutover. It is an evidence index, not a declaration +that every test became independent merely because its path changed. + +## Classification batches + +The following mappings are grounded in the current moved paths. They describe +ownership only; decomposition and evidence replacement must be recorded in the +per-suite table below. + +| Previous path | Current path | Classification | +| --------------------------------------------------------------------------------------------------- | -------------------------------------------------------------------------------- | ----------------------------------- | +| `tests/applicationWorkflowCli.test.ts` | `tests/acceptance/applications/applicationWorkflowCli.test.ts` | Acceptance application workflow | +| `tests/setupLifecycle.test.ts` | `tests/acceptance/setup/setupLifecycle.test.ts` | Acceptance setup workflow | +| `tests/cdpBrowserProvider.test.ts` | `tests/boundary/browser/cdpBrowserProvider*.test.ts` | Browser boundary | +| `tests/ghidraClient.test.ts` | `tests/boundary/providers/ghidra/ghidraClient.test.ts` | Ghidra provider boundary | +| `tests/processOwnership.test.ts` | `tests/boundary/process/processOwnership*.test.ts` | Process boundary | +| `tests/binarySession.test.ts` | `tests/composition/analysis-sessions/binarySession.*.test.ts` | Analysis-session composition | +| `tests/javascriptSemanticAnalysis.test.ts` | `tests/composition/javascript-applications/javascriptSemanticAnalysis.*.test.ts` | JavaScript-application composition | +| `tests/config.test.ts` | `tests/composition/setup-lifecycle/config.test.ts` | Setup-lifecycle composition | +| `tests/conformanceFixtures.test.ts` | `tests/conformance/providers/conformanceFixtures.test.ts` | Provider conformance | +| `tests/codexAgentEval.test.ts` | `tests/evaluation/model-evals/codexAgentEval.test.ts` | Deterministic model-evaluator logic | +| `tests/composition/analysis-sessions/{boundedCartesianProjection,symbolAnalysis,jsonShape}.test.ts` | `src/domain/{boundedCartesianProjection,symbolAnalysis,jsonShape}.test.ts` | Colocated pure-domain behavior | + +The same directory mapping applies to the other files moved in each batch: +`tests/acceptance/{analysis,applications,investigations,setup}`, +`tests/boundary/{browser,cli,filesystem,mcp,process,providers}`, +`tests/composition/{analysis-sessions,evidence-investigations,javascript-applications,process-capture,runtime-observation,setup-lifecycle}`, +`tests/conformance`, and `tests/evaluation`. Git rename detection remains the +canonical source for the complete old-to-new path list. + +## Per-suite evidence record + +Add a row when a legacy suite is materially decomposed, consolidated, replaced, +or removed. “Replacement evidence” must identify executable behavior, a +generated artifact comparison, a protocol probe, or an explicit verifier; a new +path alone is not replacement evidence. + +| Legacy suite or claim | Disposition | Replacement evidence | Validation | Remaining gap | +| ----------------------------------------------- | ------------ | ------------------------------------------------------------------------------------------------------------------------------------- | ---------------------------------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------- | +| Flat path classification | Moved | Vitest configuration contract proves every deterministic test belongs to exactly one project | Configuration contract test | Record any exceptions discovered during cutover | +| Direct session construction outside owner tests | Consolidated | `composeBinarySession` and `composeBinarySessionFromProvider` are the shared production/test seams | Repository search plus session-composition and acceptance tests | None | +| Repeated temporary-directory setup | Consolidated | `createTestTempDirectory` delegates every test-owned root to the typed workspace fixture with HOME/XDG projection and awaited cleanup | Repository search plus filesystem, process, runtime suites, and the Vitest topology hygiene contract | None; direct `mkdtemp` remains only in the workspace fixture and package-canary source text | +| Repeated CLI/MCP connection helpers | Consolidated | Typed CLI and MCP fixtures under `tests/support`; CLI output, dispatcher, and server tests use them | Acceptance startup, typed failure, lifecycle, and teardown tests | Provider-specific protocol harnesses remain local | +| Implementation-shaped assertions | Replaced | Recording adapters, executable bridge/protocol probes, generated-artifact comparisons, and supported configuration contracts | Focused boundary/conformance tests and repository search | None identified in the final audit | +| Module mocks and spies | Replaced | Recording adapters and caller-visible outcomes | Repository search plus focused reload, Ghidra, and enhanced-tool tests | None | +| `processCapture.test.ts` | Consolidated | Ten lifecycle, observation, reactive, replay, terminal, validation, and trace suites | 42 focused boundary tests | None | +| `cdpBrowserProvider.test.ts` | Consolidated | Five discovery, document/script, network, lifecycle, and sensitive-data suites | 36 focused browser-boundary tests | None | +| `javascriptSemanticAnalysis.test.ts` | Consolidated | Six structure, call, dataflow, runtime, native-binding, and rejection suites | 29 focused composition tests | None | +| `binarySession.test.ts` | Consolidated | Nine provider binding, lifecycle, cache, Evidence, permissions, and shutdown suites | Focused composition tests and shared composition-seam test | None | +| MCP mega-suites | Consolidated | Tool/resource-family suites plus catalog-wide identity and lifecycle coverage | Focused MCP tests and complete deterministic run | None | +| Provider client mega-suites | Consolidated | Capability-named protocol, timeout, ownership, error, and cleanup suites | 85 focused provider tests | Real systems remain in `verify:*` lanes | +| Pure domain proofs | Moved | Colocated bounded projection, symbol classification, and JSON-shape tests | Domain Vitest project | Continue moving only tests with no external boundary dependency | + +Use these dispositions consistently: + +- **Moved**: the same material claim now has depth-appropriate ownership. +- **Replaced**: stronger executable evidence supersedes the old assertion. +- **Consolidated**: one retained test proves a claim previously duplicated. +- **Intentionally removed**: the claim was not public or independent; the row + must still name the retained evidence or explain why none is warranted. +- **In progress**: classification exists but the replacement audit is not + complete. No final-cutover row may retain this disposition. + +The exact old-to-new inventory is mechanically enforced by the project +classification contract and retained in Git rename metadata. Real-system lanes, +coverage, repeated-run stability, and benchmark timing are release evidence; +they are reported separately because they depend on the execution host and +installed external systems. diff --git a/docs/testing.md b/docs/testing.md new file mode 100644 index 000000000..5b61cd876 --- /dev/null +++ b/docs/testing.md @@ -0,0 +1,94 @@ +# Testing REA + +REA's test suite follows one rule: use the lowest behavioral depth that can +prove a claim. Tests are grouped into native Vitest projects so ownership, +allowed dependencies, and runtime cost are visible from their paths. + +## Behavioral depths + +| Depth | Location | What it proves | +| ----------- | ---------------------- | ----------------------------------------------------------------------------------------------------- | +| Module | `src/**/*.test.ts` | One domain, service, or adapter through its narrow public surface | +| Composition | `tests/composition/**` | Provider-neutral session and registry wiring; filesystem access is limited to fixture materialization | +| Boundary | `tests/boundary/**` | Exactly one production filesystem, process, network, browser, CLI, MCP, or provider boundary | +| Acceptance | `tests/acceptance/**` | A complete compiled CLI or MCP workflow | +| Conformance | `tests/conformance/**` | Shared provider contracts, parameterized by declared capabilities and explicit opt-outs | +| Evaluation | `tests/evaluation/**` | Deterministic evaluator parsing, scoring, and report generation | + +Colocated domain tests may import only their owning domain and inward +dependencies. They do not construct sessions or servers and do not start +subprocesses or network listeners. Colocated application tests exercise one +service through explicit ports and recording adapters, never a CLI or MCP +entrypoint. Composition tests may assemble provider-neutral sessions and +registries but do not cross production filesystem, process, socket, or browser boundaries. +Boundary tests cross one production boundary. Only acceptance tests assemble +the complete runtime or invoke the compiled product surface. + +`tests/process-global/**` is reserved for cases with a demonstrated dependency +on process-global state. Those tests run without file parallelism. Reusable, +test-scoped fixtures live under `tests/support/**`; immutable source artifacts +remain under `tests/fixtures/**`. +The process-global Vitest configuration contract rejects new direct temporary-root +creation outside the workspace seam and its narrowly documented boundary/package +exceptions. + +Real Hopper, Ghidra, browser, package, managed-code, and controlled-replay +claims belong to their explicit `npm run verify:*` lanes. They are not inferred +from mocks or folded into the deterministic local gate. Real model trials are +manual; Vitest covers only deterministic evaluator logic. + +## Developer commands + +`npm test` runs every deterministic project once. The narrower feedback loops +are: + +| Command | Scope | +| ------------------------- | -------------------------------------------------------------- | +| `npm run test:fast` | Domain, service, and composition projects | +| `npm run test:boundary` | Boundary and process-global projects | +| `npm run test:acceptance` | Complete CLI and MCP acceptance workflows | +| `npm run test:changed` | Tests related through Vitest's import graph | +| `npm run test:watch` | Changed domain and service tests in watch mode | +| `npm run test:watch:all` | Changed tests from every deterministic project in watch mode | +| `npm run check:changed` | Cached static checks followed by changed tests | +| `npm run check:pr` | Static, generated-document, and complete deterministic PR gate | + +Changed-test selection is a fast feedback aid, not release evidence. It can +miss behavior connected through runtime registration, generated data, shell +entrypoints, or other relationships that are absent from the import graph. +Use `npm run check:pr` before handing off a contribution. + +Vitest and Node persistent compile caches are deliberately not enabled by +default. To evaluate repeated local runs, opt in for both cold and warm +measurements with an isolated cache: + +```bash +NODE_COMPILE_CACHE=.cache/node-compile npm test +``` + +Do not report the warm result as a cold-suite improvement, and do not enable +the cache in coverage or benchmark CI without first showing that its +instrumentation remains equivalent. + +## Coverage and timing + +CI owns coverage. The aggregate floors are 65% statements, 60% branches, 60% +functions, and 68% lines. `src/domain/**` must reach 80% statements, 75% +branches, 75% functions, and 80% lines. `src/contracts/**` must reach 85% +statements, 80% branches, 80% functions, and 85% lines. Thresholds are +glob-specific rather than per-file and are never updated automatically. +Coverage does not replace named boundary, acceptance, or real-provider scenario +matrices. + +CI runs four native Vitest shards without retries. Each shard emits a blob +report; the merge job produces aggregate coverage plus JUnit and JSON timing +reports, uploads them together, and writes the slowest files to the workflow +summary. Static checks, documentation, build, package verification, and +real-system lanes remain separate jobs so one kind of evidence cannot stand in +for another. + +The PR acceptance target is a median `npm run check:pr` wall time below three +minutes across three warm-build runs on the benchmark host. Keep Vitest caches +cold unless separately identified. The final cutover also requires three +consecutive deterministic passes without retries, `npm run verify:package`, and +the applicable real-system lanes. diff --git a/package.json b/package.json index f4d5704e4..899880e13 100644 --- a/package.json +++ b/package.json @@ -63,6 +63,8 @@ "check": "turbo run typecheck lint format:check knip metadata:check", "precheck:fast": "npm run deps:check", "check:fast": "turbo run typecheck lint", + "precheck:changed": "npm run deps:check", + "check:changed": "turbo run typecheck lint && npm run test:changed", "precheck:test": "npm run deps:check", "check:test": "turbo run typecheck lint format:check knip metadata:check test:run", "precheck:pr": "npm run deps:check", @@ -83,11 +85,15 @@ "test": "vitest run", "test:run": "vitest run", "pretest:fast": "npm run build:cached", - "test:fast": "vitest run --project parallel --project subprocess", + "test:fast": "vitest run --project domain --project services --project composition", + "pretest:boundary": "npm run build:cached", + "test:boundary": "vitest run --project boundary --project process-global", + "pretest:acceptance": "npm run build:cached", + "test:acceptance": "vitest run --project acceptance", "pretest:changed": "npm run build:cached", - "test:changed": "vitest run --project parallel --project subprocess --changed", + "test:changed": "vitest run --changed", "pretest:integration": "npm run build:cached", - "test:integration": "vitest run --project serial-integration", + "test:integration": "vitest run --project boundary --project process-global", "pretest:coverage": "npm run build:cached", "test:coverage": "vitest run --coverage.enabled --coverage.provider=v8", "test:ci": "npm run test:coverage -- --reporter=default --reporter=junit --outputFile.junit=test-results.xml", @@ -98,7 +104,9 @@ "pretest:verbose": "npm run build:cached", "test:verbose": "vitest run --reporter=verbose", "pretest:watch": "npm run build:cached", - "test:watch": "vitest --project parallel --project subprocess --changed", + "test:watch": "vitest --project domain --project services --changed", + "pretest:watch:all": "npm run build:cached", + "test:watch:all": "vitest --changed", "typecheck": "tsc --noEmit", "verify:test-temp-hygiene": "npm run build:cached && node scripts/verify-test-temp-hygiene.mjs", "verify:hopper": "npm run verify:fixtures && npm run build:cached && node scripts/verify-real-hopper.mjs",