diff --git a/src/application/ProcessPairedExperiment.test.ts b/src/application/ProcessPairedExperiment.test.ts deleted file mode 100644 index 03f2da707..000000000 --- a/src/application/ProcessPairedExperiment.test.ts +++ /dev/null @@ -1,260 +0,0 @@ -import { describe, expect, it } from "vitest"; - -import { - runPairedProcessExperiment, - type ProcessCapturePort, -} from "./ProcessPairedExperiment.js"; -import { EMPTY_PROCESS_CAPTURE_EXAMPLE } from "../domain/processCapture.fixture.js"; -import { - parseProcessCapture, - type EnabledProcessExecutionPolicy, -} from "../domain/processCapture.js"; -import { - analyzeProcessRepeatability, - bindProcessScenario, - processPairedExperimentSchema, -} from "../domain/processPairedExperiment.js"; -import { ok } from "../domain/result.js"; - -const experiment = () => - processPairedExperimentSchema.parse({ - shared_scenario: { - executable: "/bin/template", - arguments: ["shared"], - working_directory: "/tmp", - environment: { SHARED: "yes" }, - }, - authority: { - id: "authority", - executable: "/bin/authority", - environment: { SIDE: "authority" }, - }, - candidate: { - id: "candidate", - executable: "/bin/candidate", - arguments: ["candidate"], - environment: { SIDE: "candidate" }, - }, - repeat_count: 3, - freshness_policy: { max_capture_age_ms: 60_000 }, - required_dimensions: [ - "terminal", - "interaction", - "exit", - "filesystem", - "protocol", - "process", - "shim", - ], - }); - -const policy: EnabledProcessExecutionPolicy = { - status: "enabled", - executableRoots: ["/bin"], - workingRoots: ["/tmp"], - allowedEnvironment: ["SHARED", "SIDE"], - networkAccess: "none", -}; - -const cloneCapture = ( - capture: ReturnType, -): ReturnType => - parseProcessCapture(structuredClone(capture)); - -describe("paired process experiments", () => { - it("binds only declared side inputs onto the shared observation contract", () => { - const parsed = experiment(); - const authority = bindProcessScenario( - parsed.shared_scenario, - parsed.authority, - ); - const candidate = bindProcessScenario( - parsed.shared_scenario, - parsed.candidate, - ); - expect(authority).toMatchObject({ - executable: "/bin/authority", - arguments: ["shared"], - environment: { SHARED: "yes", SIDE: "authority" }, - }); - expect(candidate).toMatchObject({ - executable: "/bin/candidate", - arguments: ["candidate"], - environment: { SHARED: "yes", SIDE: "candidate" }, - }); - }); - - it("runs same-side comparisons first and compares stable sides", async () => { - const observedExecutables: string[] = []; - const capture = parseProcessCapture(EMPTY_PROCESS_CAPTURE_EXAMPLE); - const capturePort: ProcessCapturePort = { - capture: (scenario) => { - observedExecutables.push(scenario.executable); - return Promise.resolve(ok(cloneCapture(capture))); - }, - }; - const result = await runPairedProcessExperiment(experiment(), policy, { - capturePort, - now: () => Date.parse("2026-01-01T00:00:01.000Z"), - }); - expect(result.ok).toBe(true); - if (!result.ok) return; - expect(observedExecutables).toEqual([ - "/bin/authority", - "/bin/authority", - "/bin/authority", - "/bin/candidate", - "/bin/candidate", - "/bin/candidate", - ]); - expect(result.value.authority_repeatability.stable).toBe(true); - expect(result.value.candidate_repeatability.stable).toBe(true); - expect(result.value.cross_side?.status).toBe("unchanged"); - expect(result.value.cross_side_blocked_reason).toBeNull(); - }); - - it("blocks cross-side claims when a candidate has a flaky exit", async () => { - const base = parseProcessCapture(EMPTY_PROCESS_CAPTURE_EXAMPLE); - let candidateRun = 0; - const capturePort: ProcessCapturePort = { - capture: (scenario) => { - let capture = cloneCapture(base); - if (scenario.executable === "/bin/candidate") { - candidateRun += 1; - if (candidateRun === 2) - capture = parseProcessCapture({ - ...capture, - exit: { code: 1, signal: null, reason: "exited" }, - }); - } - return Promise.resolve(ok(capture)); - }, - }; - const result = await runPairedProcessExperiment(experiment(), policy, { - capturePort, - now: () => Date.parse("2026-01-01T00:00:01.000Z"), - }); - expect(result.ok).toBe(true); - if (!result.ok) return; - expect(result.value.candidate_repeatability).toMatchObject({ - stable: false, - classification: "flaky_exit", - unstable_dimensions: ["exit"], - }); - expect(result.value.cross_side).toBeNull(); - expect(result.value.cross_side_blocked_reason).toMatch(/unstable/u); - }); -}); - -describe("paired process experiment comparisons", () => { - it("distinguishes normalized timing variance from semantic output drift", () => { - const base = parseProcessCapture(EMPTY_PROCESS_CAPTURE_EXAMPLE); - const early = parseProcessCapture({ - ...base, - frames: [{ sequence: 0, at_ms: 10, data: "ready" }], - }); - const late = parseProcessCapture({ - ...base, - frames: [{ sequence: 0, at_ms: 20, data: "ready" }], - }); - expect( - analyzeProcessRepeatability([early, late], ["terminal"]), - ).toMatchObject({ - stable: false, - classification: "timing_variance", - unstable_dimensions: ["terminal"], - }); - const changed = parseProcessCapture({ - ...base, - frames: [{ sequence: 0, at_ms: 20, data: "failed" }], - }); - expect( - analyzeProcessRepeatability([early, changed], ["terminal"]), - ).toMatchObject({ classification: "nondeterministic_output" }); - }); - - it("stops before capture when the experiment is cancelled", async () => { - const controller = new AbortController(); - controller.abort(); - const capturePort: ProcessCapturePort = { - capture: () => { - throw new Error("capture must not start after cancellation"); - }, - }; - - const result = await runPairedProcessExperiment(experiment(), policy, { - capturePort, - signal: controller.signal, - }); - - expect(result).toMatchObject({ - ok: false, - error: { _tag: "ProcessCaptureError", reason: "cancelled" }, - }); - }); - - it("blocks full cross-side output when only a subset was repeatable", async () => { - const capture = parseProcessCapture(EMPTY_PROCESS_CAPTURE_EXAMPLE); - const capturePort: ProcessCapturePort = { - capture: () => Promise.resolve(ok(cloneCapture(capture))), - }; - const subset = processPairedExperimentSchema.parse({ - ...experiment(), - required_dimensions: ["exit"], - }); - - const result = await runPairedProcessExperiment(subset, policy, { - capturePort, - }); - - expect(result).toMatchObject({ - ok: true, - value: { - cross_side: null, - cross_side_blocked_reason: expect.stringMatching(/every returned/u), - }, - }); - }); -}); - -describe("paired process experiment freshness", () => { - it("compares the newest stable captures under the freshness policy", async () => { - const base = parseProcessCapture(EMPTY_PROCESS_CAPTURE_EXAMPLE); - let captureIndex = 0; - const capturePort: ProcessCapturePort = { - capture: () => { - const completedAt = - captureIndex === 0 - ? "2026-01-01T00:00:00.000Z" - : "2026-01-01T00:00:09.500Z"; - captureIndex += 1; - return Promise.resolve( - ok( - parseProcessCapture({ - ...structuredClone(base), - manifest: { - ...base.manifest, - started_at: completedAt, - completed_at: completedAt, - }, - }), - ), - ); - }, - }; - const shortFreshness = processPairedExperimentSchema.parse({ - ...experiment(), - freshness_policy: { max_capture_age_ms: 1_000 }, - }); - - const result = await runPairedProcessExperiment(shortFreshness, policy, { - capturePort, - now: () => Date.parse("2026-01-01T00:00:10.000Z"), - }); - - expect(result).toMatchObject({ - ok: true, - value: { cross_side: { status: "unchanged" } }, - }); - }); -}); diff --git a/src/application/ProcessPairedExperiment.ts b/src/application/ProcessPairedExperiment.ts deleted file mode 100644 index 3705cec78..000000000 --- a/src/application/ProcessPairedExperiment.ts +++ /dev/null @@ -1,152 +0,0 @@ -import { - processCaptureCancelled, - type ProcessCaptureError, -} from "./ProcessCaptureError.js"; -import { captureProcessScenario } from "./ProcessHarness.js"; -import type { - ProcessCapture, - ProcessExecutionPolicy, - ProcessScenario, -} from "../domain/processCapture.js"; -import { compareProcessCaptures } from "../domain/processComparison.js"; -import { PROCESS_COMPARISON_DIMENSIONS } from "../domain/processComparison.js"; -import { - analyzeProcessRepeatability, - bindProcessScenario, - type ProcessPairedExperiment, -} from "../domain/processPairedExperiment.js"; -import { err, ok, type Result } from "../domain/result.js"; - -/** Capture seam used by paired process experiments. */ -export interface ProcessCapturePort { - capture( - scenario: ProcessScenario, - policy: ProcessExecutionPolicy, - signal?: AbortSignal, - ): Promise>; -} - -interface PairedProcessExperimentEvidence { - readonly authority_runs: readonly ProcessCapture[]; - readonly candidate_runs: readonly ProcessCapture[]; - readonly authority_repeatability: ReturnType< - typeof analyzeProcessRepeatability - >; - readonly candidate_repeatability: ReturnType< - typeof analyzeProcessRepeatability - >; -} - -/** Repeatability evidence and either a stable comparison or its blocking reason. */ -export type PairedProcessExperimentResult = PairedProcessExperimentEvidence & - ( - | { - readonly cross_side: ReturnType; - readonly cross_side_blocked_reason: null; - } - | { - readonly cross_side: null; - readonly cross_side_blocked_reason: string; - } - ); - -const productionCapturePort: ProcessCapturePort = { - capture: captureProcessScenario, -}; - -const captureRepeats = async (input: { - readonly scenario: ProcessScenario; - readonly repeatCount: number; - readonly policy: ProcessExecutionPolicy; - readonly port: ProcessCapturePort; - readonly signal?: AbortSignal; -}): Promise> => { - const captures: ProcessCapture[] = []; - for (let index = 0; index < input.repeatCount; index += 1) { - if (input.signal?.aborted === true) return err(processCaptureCancelled()); - const capture = await input.port.capture( - input.scenario, - input.policy, - input.signal, - ); - if (!capture.ok) return err(capture.error); - captures.push(capture.value); - } - return ok(captures); -}; - -/** Execute repeatability-first authority/candidate captures under one contract. */ -export const runPairedProcessExperiment = async ( - experiment: ProcessPairedExperiment, - policy: ProcessExecutionPolicy, - options: { - readonly signal?: AbortSignal; - readonly capturePort?: ProcessCapturePort; - readonly now?: () => number; - } = {}, -): Promise> => { - const port = options.capturePort ?? productionCapturePort; - const authorityScenario = bindProcessScenario( - experiment.shared_scenario, - experiment.authority, - ); - const candidateScenario = bindProcessScenario( - experiment.shared_scenario, - experiment.candidate, - ); - const authorityRuns = await captureRepeats({ - scenario: authorityScenario, - repeatCount: experiment.repeat_count, - policy, - port, - ...(options.signal === undefined ? {} : { signal: options.signal }), - }); - if (!authorityRuns.ok) return authorityRuns; - const candidateRuns = await captureRepeats({ - scenario: candidateScenario, - repeatCount: experiment.repeat_count, - policy, - port, - ...(options.signal === undefined ? {} : { signal: options.signal }), - }); - if (!candidateRuns.ok) return candidateRuns; - const authorityRepeatability = analyzeProcessRepeatability( - authorityRuns.value, - experiment.required_dimensions, - ); - const candidateRepeatability = analyzeProcessRepeatability( - candidateRuns.value, - experiment.required_dimensions, - ); - const stable = authorityRepeatability.stable && candidateRepeatability.stable; - const allDimensionsRequired = PROCESS_COMPARISON_DIMENSIONS.every( - (dimension) => experiment.required_dimensions.includes(dimension), - ); - const authorityLatest = authorityRuns.value.at(-1); - const candidateLatest = candidateRuns.value.at(-1); - const crossSideOutcome = - stable && - allDimensionsRequired && - authorityLatest !== undefined && - candidateLatest !== undefined - ? { - cross_side: compareProcessCaptures(authorityLatest, candidateLatest, { - maxCaptureAgeMs: experiment.freshness_policy.max_capture_age_ms, - ...(options.now === undefined ? {} : { now: options.now }), - }), - cross_side_blocked_reason: null, - } - : { - cross_side: null, - cross_side_blocked_reason: !allDimensionsRequired - ? "Cross-side comparison is blocked until every returned dimension is required and repeatable." - : "Cross-side comparison is blocked because a required same-side dimension is unstable.", - }; - return ok({ - authority_runs: authorityRuns.value, - candidate_runs: candidateRuns.value, - authority_repeatability: authorityRepeatability, - candidate_repeatability: candidateRepeatability, - ...crossSideOutcome, - }); -}; diff --git a/src/domain/processPairedExperiment.ts b/src/domain/processPairedExperiment.ts deleted file mode 100644 index 4d71d1308..000000000 --- a/src/domain/processPairedExperiment.ts +++ /dev/null @@ -1,205 +0,0 @@ -import { z } from "zod"; - -import type { ProcessCapture } from "./processCapture.js"; -import { - PROCESS_COMPARISON_DIMENSIONS, - compareProcessCaptures, - processDimensionObservations, - type ProcessCaptureComparison, -} from "./processComparison.js"; -import { - digestProcessCommitment, - processScenarioSchema, - type ProcessScenario, -} from "./processScenario.js"; - -const sideIdentifierSchema = z - .string() - .regex(/^[A-Za-z][A-Za-z0-9._-]{0,127}$/u); -const environmentNameSchema = z.string().regex(/^[A-Za-z_][A-Za-z0-9_]*$/u); -const processDimensionSchema = z.enum(PROCESS_COMPARISON_DIMENSIONS); - -const sideBindingSchema = z - .object({ - id: sideIdentifierSchema, - executable: z.string().startsWith("/"), - arguments: z.array(z.string()).max(256).optional(), - environment: z.record(environmentNameSchema, z.string()).default({}), - }) - .strict(); - -/** Boundary contract for one bounded authority/candidate process experiment. */ -export const processPairedExperimentSchema = z - .object({ - shared_scenario: processScenarioSchema, - authority: sideBindingSchema, - candidate: sideBindingSchema, - repeat_count: z.number().int().min(2).max(10).default(3), - freshness_policy: z - .object({ - max_capture_age_ms: z.number().int().positive().max(86_400_000), - }) - .strict(), - required_dimensions: z - .array(processDimensionSchema) - .min(1) - .max(PROCESS_COMPARISON_DIMENSIONS.length) - .default([...PROCESS_COMPARISON_DIMENSIONS]), - }) - .strict() - .superRefine((experiment, context) => { - if (experiment.authority.id === experiment.candidate.id) - context.addIssue({ - code: "custom", - message: "authority and candidate identities must differ", - path: ["candidate", "id"], - }); - if ( - new Set(experiment.required_dimensions).size !== - experiment.required_dimensions.length - ) - context.addIssue({ - code: "custom", - message: "required process dimensions must be unique", - path: ["required_dimensions"], - }); - }); - -export type ProcessPairedExperiment = z.infer< - typeof processPairedExperimentSchema ->; -/** Executable and environment values applied to one experiment side. */ -export type ProcessSideBinding = ProcessPairedExperiment[ - | "authority" - | "candidate"]; - -type ProcessRepeatabilityClassification = - | "stable" - | "timing_variance" - | "nondeterministic_output" - | "flaky_exit" - | "environmental_drift" - | "insufficient_samples"; - -/** Stability classification for repeated captures of one experiment side. */ -export interface ProcessSideRepeatability { - readonly classification: ProcessRepeatabilityClassification; - readonly stable: boolean; - readonly digest: string; - readonly comparisons: readonly ProcessCaptureComparison[]; - readonly unstable_dimensions: readonly (typeof PROCESS_COMPARISON_DIMENSIONS)[number][]; -} - -/** Apply a side binding without changing the shared observation contract. */ -export const bindProcessScenario = ( - shared: ProcessScenario, - side: ProcessSideBinding, -): ProcessScenario => - processScenarioSchema.parse({ - ...shared, - executable: side.executable, - arguments: side.arguments ?? shared.arguments, - environment: { ...shared.environment, ...side.environment }, - }); - -const classifyInstability = ( - dimensions: ReadonlySet<(typeof PROCESS_COMPARISON_DIMENSIONS)[number]>, -): ProcessRepeatabilityClassification => { - if (dimensions.has("exit")) return "flaky_exit"; - if (dimensions.has("filesystem") || dimensions.has("process")) - return "environmental_drift"; - return "nondeterministic_output"; -}; - -const timingKeys = new Set([ - "at_ms", - "scheduled_at_ms", - "dispatched_at_ms", - "elapsed_ms", -]); - -const withoutTiming = (value: unknown): unknown => - typeof value !== "object" || value === null || Array.isArray(value) - ? value - : Object.fromEntries( - Object.entries(value).filter(([key]) => !timingKeys.has(key)), - ); - -const observationsWithoutTiming = ( - capture: ProcessCapture, - dimension: (typeof PROCESS_COMPARISON_DIMENSIONS)[number], -): readonly unknown[] => - dimension === "exit" - ? [ - { - ...capture.exit, - settlement: { - state: capture.settlement.state, - cleanup_outcome: capture.settlement.cleanup_outcome, - }, - }, - ] - : processDimensionObservations(capture, dimension).map(withoutTiming); - -const hasOnlyTimingVariance = ( - captures: readonly ProcessCapture[], - dimensions: readonly (typeof PROCESS_COMPARISON_DIMENSIONS)[number][], -): boolean => - dimensions.every((dimension) => { - const first = captures[0]; - if (first === undefined) return false; - const baseline = JSON.stringify( - observationsWithoutTiming(first, dimension), - ); - return captures - .slice(1) - .every( - (capture) => - JSON.stringify(observationsWithoutTiming(capture, dimension)) === - baseline, - ); - }); - -/** Compare same-side repeats before permitting a cross-side claim. */ -export const analyzeProcessRepeatability = ( - captures: readonly ProcessCapture[], - requiredDimensions: readonly (typeof PROCESS_COMPARISON_DIMENSIONS)[number][], -): ProcessSideRepeatability => { - if (captures.length < 2) - return { - classification: "insufficient_samples", - stable: false, - digest: digestProcessCommitment({ captures: [], requiredDimensions }), - comparisons: [], - unstable_dimensions: [], - }; - const baseline = captures[0]; - if (baseline === undefined) - throw new TypeError("Repeatability analysis lost its baseline capture"); - const comparisons = captures - .slice(1) - .map((capture) => compareProcessCaptures(baseline, capture)); - const unstable = new Set( - requiredDimensions.filter((dimension) => - comparisons.some((comparison) => comparison[dimension] !== "unchanged"), - ), - ); - const unstableDimensions = PROCESS_COMPARISON_DIMENSIONS.filter((dimension) => - unstable.has(dimension), - ); - return { - classification: - unstable.size === 0 - ? "stable" - : hasOnlyTimingVariance(captures, unstableDimensions) - ? "timing_variance" - : classifyInstability(unstable), - stable: unstable.size === 0, - digest: digestProcessCommitment({ - requiredDimensions, - comparisons, - }), - comparisons, - unstable_dimensions: unstableDimensions, - }; -};