rea/tests/evaluation/analysisEfficiencyComparison.test.ts

import assert from "node:assert/strict";
import { spawnSync } from "node:child_process";
import { mkdtempSync, mkdirSync, rmSync, writeFileSync } from "node:fs";
import { tmpdir } from "node:os";
import { join } from "node:path";
import { test } from "vitest";
import { z } from "zod";
import { fileURLToPath } from "node:url";

type Delivery = "complete" | "summary";
type Observation = {
  evidence_id: string;
  parent_evidence_id?: string;
  limitations: string[];
  kind?: string;
  view?: { kind: string; facet?: string; offset?: number; limit?: number };
  coverage?:
    | string
    | {
        examined: number;
        total: number;
        next_offset: number | null;
        exhausted: boolean;
      };
  unknowns?: string[];
};
type Fixture = {
  summary: {
    artifact: string;
    delivery: Delivery;
    provider_search_path_sha256: string;
    calls: unknown[];
  };
  answer: {
    capture: { coverage: string; capture_start: null };
    http_request_uris_first2: string[];
    delivery_observations: Observation[];
  };
};
const reportSchema = z.object({
  facts_match: z.boolean(),
  settings_match: z.boolean(),
  fact_differences: z.array(z.string()),
  preserved_run_specific_values: z.array(z.object({ value: z.unknown() })),
});

// Synthetic measurement directories exercise the comparator CLI, not providers.
const observation = (kind: string): Observation => ({
  evidence_id: `ev_${kind}`,
  parent_evidence_id: "ev_parent",
  limitations: ["saved bytes only"],
  kind,
  view:
    kind === "summary"
      ? { kind }
      : { kind, facet: "http-uris", offset: 0, limit: 2 },
  coverage: { examined: 2, total: 3, next_offset: 2, exhausted: false },
  unknowns: ["unrecorded traffic"],
});
const fixture = (delivery: Delivery, artifact: string): Fixture => ({
  summary: {
    artifact,
    delivery,
    provider_search_path_sha256: "a".repeat(64),
    calls: [],
  },
  answer: {
    capture: { coverage: "complete", capture_start: null },
    http_request_uris_first2: [
      "http://example.test/one",
      "http://example.test/two",
    ],
    delivery_observations:
      delivery === "summary"
        ? [observation("summary"), observation("pcap")]
        : [
            {
              evidence_id: "ev_complete",
              limitations: ["saved bytes only"],
              coverage: "complete",
            },
          ],
  },
});
function compare(left: Fixture, right: Fixture) {
  const root = mkdtempSync(join(tmpdir(), "rea-compare-test-"));
  try {
    const directories = [join(root, "left"), join(root, "right")] as const;
    for (const [index, value] of [left, right].entries()) {
      const directory = directories[index];
      assert.ok(directory);
      mkdirSync(directory);
      for (const name of ["summary", "answer"] as const)
        writeFileSync(
          join(directory, `${name}.json`),
          JSON.stringify(value[name]),
        );
    }
    const result = spawnSync(
      process.execPath,
      [
        fileURLToPath(
          new URL(
            "../../experiments/analysis-efficiency/compare.mjs",
            import.meta.url,
          ),
        ),
        "--left",
        directories[0],
        "--right",
        directories[1],
      ],
      { encoding: "utf8" },
    );
    assert.ifError(result.error);
    assert.equal(result.stderr, "");
    return {
      status: result.status,
      report: reportSchema.parse(JSON.parse(result.stdout)),
    };
  } finally {
    rmSync(root, { recursive: true, force: true });
  }
}

for (const artifact of [
  "wireshark-http.cap",
  "wireshark-http_redirects.pcapng",
]) {
  test(`${artifact}: repeated modes ignore only receipt IDs`, () => {
    for (const delivery of ["complete", "summary"] as const) {
      const left = fixture(delivery, artifact);
      const right = structuredClone(left);
      for (const value of right.answer.delivery_observations) {
        value.evidence_id += "_second";
        if (typeof value.parent_evidence_id === "string")
          value.parent_evidence_id += "_second";
      }
      const { status, report } = compare(left, right);
      assert.equal(status, 0);
      assert.equal(report.facts_match, true);
      assert.deepEqual(
        report.preserved_run_specific_values.map((value) => value.value),
        [left.answer.delivery_observations, right.answer.delivery_observations],
      );
    }
  });

  test(`${artifact}: repeated modes reject stable observation changes`, () => {
    for (const delivery of ["complete", "summary"] as const) {
      const left = fixture(delivery, artifact);
      const mutations: ((values: Observation[]) => void)[] = [
        (values) => values[0]!.limitations.push("new limitation"),
        (values) => {
          values[0]!.coverage = "partial";
        },
        (values) => {
          delete values[0]!.coverage;
        },
      ];
      if (delivery === "summary")
        mutations.push(
          (values) => {
            values[1]!.view!.offset = 1;
          },
          (values) => {
            const coverage = values[1]!.coverage;
            assert.ok(typeof coverage === "object" && coverage !== null);
            coverage.next_offset = null;
          },
          (values) => values[1]!.unknowns!.push("extra unknown"),
          (values) => values.reverse(),
        );
      for (const mutate of mutations) {
        const right = structuredClone(left);
        mutate(right.answer.delivery_observations);
        const { status, report } = compare(left, right);
        assert.equal(status, 1);
        assert.equal(report.settings_match, true);
        assert.equal(report.facts_match, false);
        assert.deepEqual(report.fact_differences, ["delivery_observations"]);
      }
    }
  });

  test(`${artifact}: different modes preserve their different receipt scopes`, () => {
    const left = fixture("complete", artifact);
    const right = fixture("summary", artifact);
    const { status, report } = compare(left, right);
    assert.equal(status, 0);
    assert.equal(report.facts_match, true);
    assert.deepEqual(
      report.preserved_run_specific_values.map((value) => value.value),
      [left.answer.delivery_observations, right.answer.delivery_observations],
    );
    right.answer.capture.coverage = "partial";
    assert.equal(compare(left, right).report.facts_match, false);
  });
}