rea/tests/processPairedExperiment.test.ts

import { describe, expect, it } from "vitest";

import {
  runPairedProcessExperiment,
  type ProcessCapturePort,
} from "../src/application/ProcessPairedExperiment.js";
import { EMPTY_PROCESS_CAPTURE_EXAMPLE } from "../src/contracts/processCaptureExample.js";
import { parseProcessCapture } from "../src/domain/processCapture.js";
import {
  analyzeProcessRepeatability,
  bindProcessScenario,
  processPairedExperimentSchema,
} from "../src/domain/processPairedExperiment.js";
import { ok } from "../src/domain/result.js";

const experiment = () =>
  processPairedExperimentSchema.parse({
    approved: true,
    shared_scenario: {
      approved: true,
      executable: "/bin/template",
      arguments: ["shared"],
      working_directory: "/tmp",
      environment: { SHARED: "yes" },
    },
    authority: {
      id: "authority",
      executable: "/bin/authority",
      environment: { SIDE: "authority" },
    },
    candidate: {
      id: "candidate",
      executable: "/bin/candidate",
      arguments: ["candidate"],
      environment: { SIDE: "candidate" },
    },
    repeat_count: 3,
    freshness_policy: { max_capture_age_ms: 60_000 },
    required_dimensions: [
      "terminal",
      "interaction",
      "exit",
      "filesystem",
      "protocol",
      "process",
      "shim",
    ],
  });

const policy = {
  enabled: true,
  executableRoots: ["/bin"],
  workingRoots: ["/tmp"],
  allowedEnvironment: ["SHARED", "SIDE"],
  allowExternalNetwork: false,
};

describe("paired process experiments", () => {
  it("binds only declared side inputs onto the shared observation contract", () => {
    const parsed = experiment();
    const authority = bindProcessScenario(
      parsed.shared_scenario,
      parsed.authority,
    );
    const candidate = bindProcessScenario(
      parsed.shared_scenario,
      parsed.candidate,
    );
    expect(authority).toMatchObject({
      executable: "/bin/authority",
      arguments: ["shared"],
      environment: { SHARED: "yes", SIDE: "authority" },
    });
    expect(candidate).toMatchObject({
      executable: "/bin/candidate",
      arguments: ["candidate"],
      environment: { SHARED: "yes", SIDE: "candidate" },
    });
  });

  it("runs same-side comparisons first and compares stable sides", async () => {
    const observedExecutables: string[] = [];
    const capture = parseProcessCapture(EMPTY_PROCESS_CAPTURE_EXAMPLE);
    const capturePort: ProcessCapturePort = {
      capture: (scenario) => {
        observedExecutables.push(scenario.executable);
        return Promise.resolve(ok(structuredClone(capture)));
      },
    };
    const result = await runPairedProcessExperiment(experiment(), policy, {
      capturePort,
      now: () => Date.parse("2026-01-01T00:00:01.000Z"),
    });
    expect(result.ok).toBe(true);
    if (!result.ok) return;
    expect(observedExecutables).toEqual([
      "/bin/authority",
      "/bin/authority",
      "/bin/authority",
      "/bin/candidate",
      "/bin/candidate",
      "/bin/candidate",
    ]);
    expect(result.value.authority_repeatability.stable).toBe(true);
    expect(result.value.candidate_repeatability.stable).toBe(true);
    expect(result.value.cross_side?.status).toBe("unchanged");
  });

  it("blocks cross-side claims when a candidate has a flaky exit", async () => {
    const base = parseProcessCapture(EMPTY_PROCESS_CAPTURE_EXAMPLE);
    let candidateRun = 0;
    const capturePort: ProcessCapturePort = {
      capture: (scenario) => {
        let capture = structuredClone(base);
        if (scenario.executable === "/bin/candidate") {
          candidateRun += 1;
          if (candidateRun === 2)
            capture = {
              ...capture,
              exit: { code: 1, signal: null, reason: "exited" },
            };
        }
        return Promise.resolve(ok(capture));
      },
    };
    const result = await runPairedProcessExperiment(experiment(), policy, {
      capturePort,
      now: () => Date.parse("2026-01-01T00:00:01.000Z"),
    });
    expect(result.ok).toBe(true);
    if (!result.ok) return;
    expect(result.value.candidate_repeatability).toMatchObject({
      stable: false,
      classification: "flaky_exit",
      unstable_dimensions: ["exit"],
    });
    expect(result.value.cross_side).toBeNull();
    expect(result.value.cross_side_blocked_reason).toMatch(/unstable/u);
  });

  it("distinguishes normalized timing variance from semantic output drift", () => {
    const base = parseProcessCapture(EMPTY_PROCESS_CAPTURE_EXAMPLE);
    const early = {
      ...base,
      frames: [{ sequence: 0, at_ms: 10, data: "ready" }],
    };
    const late = {
      ...base,
      frames: [{ sequence: 0, at_ms: 20, data: "ready" }],
    };
    expect(
      analyzeProcessRepeatability([early, late], ["terminal"]),
    ).toMatchObject({
      stable: false,
      classification: "timing_variance",
      unstable_dimensions: ["terminal"],
    });
    const changed = {
      ...base,
      frames: [{ sequence: 0, at_ms: 20, data: "failed" }],
    };
    expect(
      analyzeProcessRepeatability([early, changed], ["terminal"]),
    ).toMatchObject({ classification: "nondeterministic_output" });
  });

  it("stops before capture when the experiment is cancelled", async () => {
    const controller = new AbortController();
    controller.abort();
    const capturePort: ProcessCapturePort = {
      capture: () => {
        throw new Error("capture must not start after cancellation");
      },
    };

    const result = await runPairedProcessExperiment(experiment(), policy, {
      capturePort,
      signal: controller.signal,
    });

    expect(result).toMatchObject({
      ok: false,
      error: { _tag: "ProcessCaptureError", reason: "cancelled" },
    });
  });

  it("blocks full cross-side output when only a subset was repeatable", async () => {
    const capture = parseProcessCapture(EMPTY_PROCESS_CAPTURE_EXAMPLE);
    const capturePort: ProcessCapturePort = {
      capture: () => Promise.resolve(ok(structuredClone(capture))),
    };
    const subset = processPairedExperimentSchema.parse({
      ...experiment(),
      required_dimensions: ["exit"],
    });

    const result = await runPairedProcessExperiment(subset, policy, {
      capturePort,
    });

    expect(result).toMatchObject({
      ok: true,
      value: {
        cross_side: null,
        cross_side_blocked_reason: expect.stringMatching(/every returned/u),
      },
    });
  });

  it("compares the newest stable captures under the freshness policy", async () => {
    const base = parseProcessCapture(EMPTY_PROCESS_CAPTURE_EXAMPLE);
    let captureIndex = 0;
    const capturePort: ProcessCapturePort = {
      capture: () => {
        const completedAt =
          captureIndex === 0
            ? "2026-01-01T00:00:00.000Z"
            : "2026-01-01T00:00:09.500Z";
        captureIndex += 1;
        return Promise.resolve(
          ok({
            ...structuredClone(base),
            manifest: {
              ...base.manifest,
              started_at: completedAt,
              completed_at: completedAt,
            },
          }),
        );
      },
    };
    const shortFreshness = processPairedExperimentSchema.parse({
      ...experiment(),
      freshness_policy: { max_capture_age_ms: 1_000 },
    });

    const result = await runPairedProcessExperiment(shortFreshness, policy, {
      capturePort,
      now: () => Date.parse("2026-01-01T00:00:10.000Z"),
    });

    expect(result).toMatchObject({
      ok: true,
      value: { cross_side: { status: "unchanged" } },
    });
  });
});