From 36cbe31e1932797c0ef3685634948f97a4100fd9 Mon Sep 17 00:00:00 2001 From: morluto <76467478+morluto@users.noreply.github.com> Date: Fri, 24 Jul 2026 03:02:11 +0000 Subject: [PATCH] perf(test): speed up local and CI validation --- .github/workflows/ci.yml | 28 +++++++++++++-- .gitignore | 1 + AGENTS.md | 5 +-- CONTRIBUTING.md | 8 +++-- package.json | 9 +++-- tests/cdpBrowserProvider.test.ts | 4 ++- tests/packageInstallWorkflow.test.ts | 4 +++ tests/processCapture.test.ts | 8 +++-- tests/processCaptureElicitation.test.ts | 2 +- tests/vitestConfiguration.test.ts | 13 ++++--- tsconfig.build.json | 2 ++ tsconfig.json | 2 ++ turbo.json | 33 +++++++++++++++--- vitest.config.ts | 45 +++++++++++++++++++++++-- 14 files changed, 139 insertions(+), 25 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 846de4735..77b3825c6 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -82,7 +82,7 @@ jobs: test-shard: name: Test shard (${{ matrix.shard }}) - needs: changes + needs: [changes, build] if: github.event_name == 'push' || needs.changes.outputs.code == 'true' runs-on: ubuntu-latest strategy: @@ -96,7 +96,11 @@ jobs: node-version-file: .nvmrc cache: npm - run: npm ci - - run: npm run test:ci:shard -- --shard=${{ matrix.shard }} + - uses: actions/download-artifact@v4 + with: + name: rea-dist + path: dist + - run: npm run test:ci:shard:run -- --shard=${{ matrix.shard }} - uses: actions/upload-artifact@v4 if: ${{ !cancelled() }} with: @@ -107,6 +111,26 @@ jobs: overwrite: true retention-days: 1 + build: + name: Build package + needs: changes + if: github.event_name == 'push' || needs.changes.outputs.code == 'true' + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-node@v4 + with: + node-version-file: .nvmrc + cache: npm + - run: npm ci + - run: npm run build:cached + - uses: actions/upload-artifact@v4 + with: + name: rea-dist + path: dist + if-no-files-found: error + retention-days: 1 + test: name: Test (coverage + timing) needs: [changes, test-shard] diff --git a/.gitignore b/.gitignore index 114a6eb6f..34f732095 100644 --- a/.gitignore +++ b/.gitignore @@ -37,6 +37,7 @@ coverage/ .vitest-reports/ test-results.xml .turbo/ +.cache/ # OS .DS_Store diff --git a/AGENTS.md b/AGENTS.md index 09580a6d7..262217f08 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -54,9 +54,10 @@ See [docs/architecture.mermaid](docs/architecture.mermaid) for a visual architec - `npm run build`: compile `src/` into `dist/`. - `npm run build:cached`: compile through Turbo and reuse a valid local build cache entry. - `npm test`: restore or build `dist/`, then run the Vitest suite once without coverage or retries locally. -- `npm run test:fast`: run the parallel test project without the serial integration group. +- `npm run test:fast`: run pure and subprocess test projects without the serial integration group. +- `npm run test:changed`: run changed and related non-serial tests once. - `npm run test:integration`: run the serial filesystem, process, and CLI integration group. -- `npm run test:watch`: watch tests affected by working-tree changes in the parallel project, without coverage. +- `npm run test:watch`: watch changed and related non-serial tests, without coverage. - `npm run test:coverage`: run the complete suite with coverage; CI shards the suite and owns JUnit reporting. - `npm run typecheck`: run strict TypeScript checks without emitting files. - `npm run lint`: apply oxlint rules (complexity, max-lines, unused vars, and TypeScript-specific checks). diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 40bc35c6b..9a4c84717 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -39,9 +39,11 @@ prerequisite uses Turbo. CI uploads the rendered TypeDoc site as an `api-docs` artifact; the generated HTML is not committed. Local `npm test` runs without coverage, retries, or verbose output. Use `npm run -test:fast` for the parallel group, `npm run test:integration` for serial -filesystem/process/CLI cases, and `npm run test:watch` to watch tests affected by -working-tree changes. CI splits the complete suite across two native Vitest +test:fast` for pure and subprocess groups, `npm run test:integration` for serial +filesystem/process/CLI cases. `npm run test:changed` runs changed and related +non-serial tests once; `npm run test:watch` keeps that selection live. Pure tests +use Vitest threads, while subprocess tests stay in isolated forks. CI splits the +complete suite across two native Vitest shards, then merges their coverage and JUnit reports. `npm run test:ci` runs the equivalent unsharded gate locally. Coverage thresholds remain in `vitest.config.ts`. diff --git a/package.json b/package.json index 7fe6a8f74..f6c4aadd4 100644 --- a/package.json +++ b/package.json @@ -80,19 +80,22 @@ "test": "vitest run", "test:run": "vitest run", "pretest:fast": "npm run build:cached", - "test:fast": "vitest run --project parallel", + "test:fast": "vitest run --project parallel --project subprocess", + "pretest:changed": "npm run build:cached", + "test:changed": "vitest run --project parallel --project subprocess --changed", "pretest:integration": "npm run build:cached", "test:integration": "vitest run --project serial-integration", "pretest:coverage": "npm run build:cached", "test:coverage": "vitest run --coverage.enabled --coverage.provider=v8", "test:ci": "npm run test:coverage -- --reporter=default --reporter=junit --outputFile.junit=test-results.xml", "pretest:ci:shard": "npm run build:cached", - "test:ci:shard": "vitest run --coverage.enabled --coverage.provider=v8 --coverage.thresholds.lines=0 --coverage.thresholds.functions=0 --coverage.thresholds.branches=0 --coverage.thresholds.statements=0 --reporter=default --reporter=blob", + "test:ci:shard": "npm run test:ci:shard:run", + "test:ci:shard:run": "vitest run --coverage.enabled --coverage.provider=v8 --coverage.thresholds.lines=0 --coverage.thresholds.functions=0 --coverage.thresholds.branches=0 --coverage.thresholds.statements=0 --reporter=default --reporter=blob", "test:ci:merge": "vitest --merge-reports --coverage.enabled --coverage.provider=v8 --reporter=default --reporter=junit --outputFile.junit=test-results.xml", "pretest:verbose": "npm run build:cached", "test:verbose": "vitest run --reporter=verbose", "pretest:watch": "npm run build:cached", - "test:watch": "vitest --project parallel --changed", + "test:watch": "vitest --project parallel --project subprocess --changed", "typecheck": "tsc --noEmit", "verify:test-temp-hygiene": "npm run build:cached && node scripts/verify-test-temp-hygiene.mjs", "verify:hopper": "npm run verify:fixtures && npm run build:cached && node scripts/verify-real-hopper.mjs", diff --git a/tests/cdpBrowserProvider.test.ts b/tests/cdpBrowserProvider.test.ts index 6c4f682ec..60f8031ea 100644 --- a/tests/cdpBrowserProvider.test.ts +++ b/tests/cdpBrowserProvider.test.ts @@ -1361,7 +1361,9 @@ describe("CdpBrowserProvider", () => { }), { signal: controller.signal }, ); - await waitForCommand(browser, "Target.attachToTarget"); + // Wait until the attach response has been consumed and capture has begun. + // Seeing the attach request alone races its response under worker load. + await waitForCommand(browser, "Page.getFrameTree"); controller.abort(); const result = await pending; expect(result).toMatchObject({ diff --git a/tests/packageInstallWorkflow.test.ts b/tests/packageInstallWorkflow.test.ts index 9b48d3303..c386b0dfc 100644 --- a/tests/packageInstallWorkflow.test.ts +++ b/tests/packageInstallWorkflow.test.ts @@ -21,6 +21,10 @@ describe("package installation workflows", () => { expect(continuousIntegration).toContain("name: Static checks"); expect(continuousIntegration).toContain("npm run check:ci"); expect(continuousIntegration).toContain("shard: [1/2, 2/2]"); + expect(continuousIntegration).toContain("name: Build package"); + expect(continuousIntegration).toContain("npm run build:cached"); + expect(continuousIntegration).toContain("name: rea-dist"); + expect(continuousIntegration).toContain("npm run test:ci:shard:run"); expect(continuousIntegration).toContain("needs: [changes, test-shard]"); expect(continuousIntegration).toContain("if-no-files-found: error"); expect(continuousIntegration).toContain("overwrite: true"); diff --git a/tests/processCapture.test.ts b/tests/processCapture.test.ts index 94d7fd4bd..2b5a1f142 100644 --- a/tests/processCapture.test.ts +++ b/tests/processCapture.test.ts @@ -1809,12 +1809,14 @@ describe("process capture adapter", () => { reactive: { version: 1, initial_state: "waiting", - deadline_ms: 2_000, + deadline_ms: 5_000, states: [ { id: "waiting", max_visits: 1, - deadline_ms: 300, + // Leave process startup scheduling headroom; this test targets + // post-exit settlement ordering, not a 300ms startup deadline. + deadline_ms: 2_000, on: [ { id: "unreachable", @@ -1913,7 +1915,7 @@ describe("process capture adapter", () => { } finally { await rm(root, { recursive: true, force: true }); } - }); + }, 10_000); it("renders terminal state, records shim invocations, and captures literal checkpoints", async () => { const root = await createTestTempDirectory("rea-v3-test-"); diff --git a/tests/processCaptureElicitation.test.ts b/tests/processCaptureElicitation.test.ts index 09bc86f34..163fce251 100644 --- a/tests/processCaptureElicitation.test.ts +++ b/tests/processCaptureElicitation.test.ts @@ -185,7 +185,7 @@ describe("process-capture MCP elicitation", () => { ); const echoedState = failure === "tampered" - ? `${requestState.slice(0, -1)}${requestState.endsWith("A") ? "B" : "A"}` + ? `${requestState.startsWith("A") ? "B" : "A"}${requestState.slice(1)}` : requestState; const retryMethod: string = "tools/call"; await expect( diff --git a/tests/vitestConfiguration.test.ts b/tests/vitestConfiguration.test.ts index ea0d6943f..29fb50edb 100644 --- a/tests/vitestConfiguration.test.ts +++ b/tests/vitestConfiguration.test.ts @@ -8,6 +8,7 @@ import { describe, expect, it } from "vitest"; import vitestConfiguration, { SERIAL_INTEGRATION_TESTS, + SUBPROCESS_TESTS, } from "../vitest.config.js"; const execute = promisify(execFile); @@ -24,7 +25,7 @@ describe("Vitest project configuration", () => { ); }); - it("collects exact disjoint parallel and serial test sets", async () => { + it("collects exact disjoint unit, subprocess, and serial test sets", async () => { const { stdout } = await execute( process.execPath, [ @@ -37,14 +38,18 @@ describe("Vitest project configuration", () => { ); const projects = parseProjects(stdout); const parallel = projects.get("parallel") ?? []; + const subprocess = projects.get("subprocess") ?? []; const serial = projects.get("serial-integration") ?? []; const repositoryTests = await testFiles("tests"); expect(serial).toEqual([...SERIAL_INTEGRATION_TESTS].sort()); expect(parallel.filter((path) => serial.includes(path))).toEqual([]); - expect([...new Set([...parallel, ...serial])].sort()).toEqual( - repositoryTests, - ); + expect(subprocess).toEqual([...SUBPROCESS_TESTS].sort()); + expect(subprocess.filter((path) => serial.includes(path))).toEqual([]); + expect(parallel.filter((path) => subprocess.includes(path))).toEqual([]); + expect( + [...new Set([...parallel, ...subprocess, ...serial])].sort(), + ).toEqual(repositoryTests); }, 20_000); }); diff --git a/tsconfig.build.json b/tsconfig.build.json index 49b57cbfe..a07edf91b 100644 --- a/tsconfig.build.json +++ b/tsconfig.build.json @@ -1,7 +1,9 @@ { "extends": "./tsconfig.json", "compilerOptions": { + "incremental": true, "rootDir": "src", + "tsBuildInfoFile": ".cache/tsconfig.build.tsbuildinfo", "types": ["node"] }, "include": ["src/**/*.ts"], diff --git a/tsconfig.json b/tsconfig.json index da6b70239..3b0d6c682 100644 --- a/tsconfig.json +++ b/tsconfig.json @@ -15,6 +15,8 @@ "rootDir": ".", "skipLibCheck": true, "strict": true, + "incremental": true, + "tsBuildInfoFile": ".cache/tsconfig.typecheck.tsbuildinfo", "target": "ES2023", "types": ["node"], "verbatimModuleSyntax": true diff --git a/turbo.json b/turbo.json index 1556c8413..55ce436b0 100644 --- a/turbo.json +++ b/turbo.json @@ -14,7 +14,7 @@ "tsconfig.json", "tsconfig.build.json" ], - "outputs": ["dist/**"] + "outputs": ["dist/**", ".cache/tsconfig.build.tsbuildinfo"] }, "typecheck": { "inputs": [ @@ -26,7 +26,8 @@ "tsconfig.json", "tsconfig.build.json", "vitest.config.ts" - ] + ], + "outputs": [".cache/tsconfig.typecheck.tsbuildinfo"] }, "metadata:check": { "inputs": [ @@ -38,9 +39,33 @@ "src/generatedPackageMetadata.ts" ] }, - "lint": {}, + "lint": { + "inputs": [ + ".nvmrc", + ".oxlintrc.json", + "eslint.config.js", + "package.json", + "package-lock.json", + "scripts/**/*.{cjs,cts,js,jsx,mjs,mts,ts,tsx}", + "src/**/*.{cjs,cts,js,jsx,mjs,mts,ts,tsx}", + "tests/**/*.{cjs,cts,js,jsx,mjs,mts,ts,tsx}", + "vitest.config.ts" + ] + }, "format:check": {}, - "knip": {}, + "knip": { + "inputs": [ + ".nvmrc", + "knip.json", + "package.json", + "package-lock.json", + "scripts/**/*.{cjs,cts,js,jsx,mjs,mts,ts,tsx}", + "src/**/*.ts", + "tests/**/*.ts", + "tsconfig.json", + "tsconfig.build.json" + ] + }, "jscpd": { "inputs": [ ".gitignore", diff --git a/vitest.config.ts b/vitest.config.ts index f7c53f04b..e703ddfa1 100644 --- a/vitest.config.ts +++ b/vitest.config.ts @@ -8,6 +8,7 @@ export const TEST_FILES = ["tests/**/*.test.ts"]; const IS_CONTINUOUS_INTEGRATION = process.env.CI === "true"; // Subprocess-heavy parallel tests exceed their existing deadlines above two workers. const MAX_TEST_WORKERS = 2; +const MAX_UNIT_TEST_WORKERS = 4; const PROCESS_CAPTURE_TEST = "tests/processCapture.test.ts"; const CANONICAL_TEMPORARY_DIRECTORY = realpathSync(tmpdir()); export const SERIAL_INTEGRATION_TESTS = [ @@ -19,6 +20,33 @@ export const SERIAL_INTEGRATION_TESTS = [ PROCESS_CAPTURE_TEST, "tests/runtime.test.ts", ]; +// Keep tests that launch child processes or production runtimes in the forks pool. +// In-memory MCP and filesystem-seam tests remain safe in the faster threads pool. +export const SUBPROCESS_TESTS = [ + "tests/applicationWorkflowVerifier.test.ts", + "tests/bridgeLauncher.test.ts", + "tests/bridgeRegex.test.ts", + "tests/browserProcessStartup.test.ts", + "tests/cliPolicyCommands.test.ts", + "tests/cliSetup.test.ts", + "tests/dependencyInstall.test.ts", + "tests/dispatcher.test.ts", + "tests/ghidraClient.test.ts", + "tests/ghidraWindowsFixture.test.ts", + "tests/hopperClient.test.ts", + "tests/installScript.test.ts", + "tests/javascriptReplayWorker.test.ts", + "tests/javascriptRuntimeReconciliation.test.ts", + "tests/linuxPrivateDisplay.test.ts", + "tests/mcpDoctor.test.ts", + "tests/processCli.test.ts", + "tests/providerProcess.test.ts", + "tests/runtimeExecutableDiagnostics.test.ts", + "tests/scanTodos.test.ts", + "tests/sessionMcp.test.ts", + "tests/verifierRun.test.ts", + "tests/vitestConfiguration.test.ts", +]; const isolatedProjects = { maxWorkers: Math.min(MAX_TEST_WORKERS, availableParallelism()), projects: [ @@ -27,17 +55,30 @@ const isolatedProjects = { test: { name: "parallel", include: TEST_FILES, - exclude: SERIAL_INTEGRATION_TESTS, + exclude: [...SERIAL_INTEGRATION_TESTS, ...SUBPROCESS_TESTS], + pool: "threads", + maxWorkers: Math.min(MAX_UNIT_TEST_WORKERS, availableParallelism()), sequence: { groupOrder: 0 }, }, }, + { + extends: true as const, + test: { + name: "subprocess", + include: SUBPROCESS_TESTS, + pool: "forks", + maxWorkers: Math.min(MAX_TEST_WORKERS, availableParallelism()), + sequence: { groupOrder: 1 }, + }, + }, { extends: true as const, test: { name: "serial-integration", include: SERIAL_INTEGRATION_TESTS, fileParallelism: false, - sequence: { groupOrder: 1 }, + pool: "forks", + sequence: { groupOrder: 2 }, }, }, ],