From b4df565b779c4f38ebf1c784dad73bf7f3de68eb Mon Sep 17 00:00:00 2001 From: morluto <76467478+morluto@users.noreply.github.com> Date: Wed, 22 Jul 2026 21:04:07 +0000 Subject: [PATCH] fix(process): bound trace comparison semantics --- docs/process-capture.md | 3 + docs/product-catalog.json | 4 +- .../managed-conformance-manifest.json | 4 +- skills/reverse-engineer-anything/SKILL.md | 2 +- src/domain/processTraceComparison.ts | 21 +- src/domain/processTraceEvaluation.ts | 50 +++-- src/domain/processTraceSpecification.ts | 55 +++++- tests/processTraceAdapters.test.ts | 2 +- tests/processTraceComparison.test.ts | 182 +++++++++++++----- 9 files changed, 238 insertions(+), 85 deletions(-) diff --git a/docs/process-capture.md b/docs/process-capture.md index cf6004ce9..58eba6c32 100644 --- a/docs/process-capture.md +++ b/docs/process-capture.md @@ -198,6 +198,9 @@ each side's raw journal locations and identifies either the matched finite variant or the complete satisfied constraint set. A failure returns the first declaration-ordered predicate, cardinality, edge, prefix, suffix, or language violation with its relevant event locations. +If both captures have the same invalid declared trace, the nested verdict is +`nonconforming` while the top-level capture comparison remains unchanged. Side +statuses and the diagnostic distinguish conformance from pairwise difference. The result classifies terminal, interaction, exit, filesystem, protocol, process, and shim behavior. `first_divergence` points to the earliest observed diff --git a/docs/product-catalog.json b/docs/product-catalog.json index db39b51a7..9526ab20e 100644 --- a/docs/product-catalog.json +++ b/docs/product-catalog.json @@ -579,10 +579,10 @@ "mcp_resource_templates": 9 }, "digests": { - "tools_sha256": "cca8f06f796f8a95d2f2e46c0fb4176e0ae442081fe7bf992ee4526071fb0aa5", + "tools_sha256": "073b2d48443034a05ba56b649057f9e5f0ab21afcb39be2c380e5a05bc0cecae", "prompts_sha256": "f6da3ad38a1000c11364d1710f122ac4990551f04efd8c53ffb2e819b5cf5707", "resources_sha256": "94bebbab1a1b3cf47644c38d3de6469de568caa7bd7b842f529ead8ad320a912", - "combined_sha256": "3ce063a71e1e53c18094aca45df31840eff75f1eaded7cbd84a7ef48db279817" + "combined_sha256": "a555f03ab0881d6e140b15d59ba8d75ba596a16a9690d26a8341d6b32395da5f" } } } diff --git a/docs/verification/managed-conformance-manifest.json b/docs/verification/managed-conformance-manifest.json index 7b4bc6cd7..2ea26ef25 100644 --- a/docs/verification/managed-conformance-manifest.json +++ b/docs/verification/managed-conformance-manifest.json @@ -1,6 +1,6 @@ { "schema_version": 1, - "manifest_id": "ecm_8ecf7db7dddcf68278e18291d8d5a4d352b807a55f8b10cd20ad893b3bf9a2ea", + "manifest_id": "ecm_a77e4a40338b12af5108a59c7323c22a0f2367ff2cd925c8c3fb4213b5eba480", "verifier": { "id": "rea-managed-conformance", "version": "1" @@ -14,7 +14,7 @@ "skill_digests": [ { "skill_id": "reverse-engineer-anything", - "sha256": "718b929b39749da6416a45645db6ed4d0775548cfaa1443509066b5c8201015f" + "sha256": "6308c7735c8b1f16f58f9d7020102fdf0a2492d3f78d979f4537d49e8e83fdc9" } ], "claims": [ diff --git a/skills/reverse-engineer-anything/SKILL.md b/skills/reverse-engineer-anything/SKILL.md index 268f0c446..04ccbda24 100644 --- a/skills/reverse-engineer-anything/SKILL.md +++ b/skills/reverse-engineer-anything/SKILL.md @@ -4,7 +4,7 @@ description: Reverse engineer native, managed, Electron/JavaScript, packaged, an metadata: version: "23" tool_count: 97 - catalog_digest: "3ce063a71e1e53c18094aca45df31840eff75f1eaded7cbd84a7ef48db279817" + catalog_digest: "a555f03ab0881d6e140b15d59ba8d75ba596a16a9690d26a8341d6b32395da5f" --- # REA diff --git a/src/domain/processTraceComparison.ts b/src/domain/processTraceComparison.ts index 1b30bee6a..c545295f3 100644 --- a/src/domain/processTraceComparison.ts +++ b/src/domain/processTraceComparison.ts @@ -1,4 +1,5 @@ import type { ProcessCapture } from "./processCapture.js"; +import { validateProcessCapture } from "./processCaptureValidation.js"; import { evaluateProcessTraceSide, processTraceComparisonResultSchema, @@ -27,6 +28,14 @@ export const compareProcessTraces = ( right: ProcessCapture, specificationInput: ProcessTraceSpecification, ): ProcessTraceComparisonResult => { + const captureIssues = [ + ...validateProcessCapture(left), + ...validateProcessCapture(right), + ]; + if (captureIssues.length > 0) + throw new TypeError( + `Invalid Process Capture v4: ${captureIssues[0]!.path}`, + ); const specification = processTraceSpecificationSchema.parse(specificationInput); const leftEvaluation = evaluateProcessTraceSide(left, specification); @@ -37,6 +46,14 @@ export const compareProcessTraces = ( ? null : { side: "right" as const, ...rightEvaluation.diagnostic } : { side: "left" as const, ...leftEvaluation.diagnostic }; + const outcomesDiffer = + leftEvaluation.result.status !== rightEvaluation.result.status || + JSON.stringify( + leftEvaluation.result.raw_trace.map(({ event_id }) => event_id), + ) !== + JSON.stringify( + rightEvaluation.result.raw_trace.map(({ event_id }) => event_id), + ); return processTraceComparisonResultSchema.parse({ verdict: leftEvaluation.result.status === "unknown" || @@ -45,7 +62,9 @@ export const compareProcessTraces = ( : leftEvaluation.result.status === "pass" && rightEvaluation.result.status === "pass" ? "equivalent" - : "different", + : outcomesDiffer + ? "different" + : "nonconforming", left: leftEvaluation.result, right: rightEvaluation.result, diagnostic, diff --git a/src/domain/processTraceEvaluation.ts b/src/domain/processTraceEvaluation.ts index b181b722a..10539e83b 100644 --- a/src/domain/processTraceEvaluation.ts +++ b/src/domain/processTraceEvaluation.ts @@ -73,7 +73,7 @@ const diagnosticSchema = z.strictObject({ /** Structured verdict for one declared process trace language. */ export const processTraceComparisonResultSchema = z.strictObject({ - verdict: z.enum(["equivalent", "different", "unknown"]), + verdict: z.enum(["equivalent", "different", "nonconforming", "unknown"]), left: sideResultSchema, right: sideResultSchema, diagnostic: diagnosticSchema.nullable(), @@ -257,31 +257,41 @@ const matchRecords = ( ): MatchedTrace | EvaluatedSide => { const rawTrace: z.infer[] = []; const locationsById = new Map(); - for (const record of records) { - const matches = specification.events.filter( - (event) => - event.source === record.source && - canonicalTraceJson( - comparableTracePayload(event.exact, event.ignore_fields), - ) === - canonicalTraceJson( - comparableTracePayload(record.payload, event.ignore_fields), - ), + const ignoredFieldsBySource = new Map< + ProcessTraceSource, + readonly string[] + >(); + const eventsByKey = new Map< + string, + ProcessTraceSpecification["events"][number] + >(); + for (const event of specification.events) { + const ignoredFields = event.ignore_fields ?? []; + ignoredFieldsBySource.set(event.source, ignoredFields); + eventsByKey.set( + `${event.source}\0${canonicalTraceJson( + comparableTracePayload(event.exact, ignoredFields), + )}`, + event, ); - if (matches.length !== 1) + } + for (const record of records) { + const match = eventsByKey.get( + `${record.source}\0${canonicalTraceJson( + comparableTracePayload( + record.payload, + ignoredFieldsBySource.get(record.source), + ), + )}`, + ); + if (match === undefined) return failure({ kind: "predicate", - message: - matches.length === 0 - ? "Observed event does not match any declared exact predicate." - : "Observed event matches multiple predicates.", - eventIds: matches.map(({ id }) => id), + message: "Observed event does not match any declared exact predicate.", + eventIds: [], locations: [record.location], rawTrace, }); - const match = matches[0]; - if (match === undefined) - throw new TypeError("Unique trace match is missing"); rawTrace.push({ event_id: match.id, location: record.location }); const locations = locationsById.get(match.id) ?? []; locations.push(record.location); diff --git a/src/domain/processTraceSpecification.ts b/src/domain/processTraceSpecification.ts index b597c62dc..b15b8a017 100644 --- a/src/domain/processTraceSpecification.ts +++ b/src/domain/processTraceSpecification.ts @@ -197,6 +197,16 @@ const buildAdjacency = ( }); adjacency.get(edge.before)?.add(edge.after); } + return adjacency; +}; + +const validateNotBefore = ( + language: PartialOrderLanguage, + context: z.RefinementCtx, + known: ReadonlySet, + reachable: ReadonlyMap>, +): ReadonlySet => { + const pairs = new Set(); for (const [index, constraint] of language.not_before.entries()) { const path = ["language", "not_before", index]; addReferenceIssue(context, known, constraint.event, [...path, "event"]); @@ -207,9 +217,22 @@ const buildAdjacency = ( message: "not-before constraints must reference distinct events", path, }); - adjacency.get(constraint.anchor)?.add(constraint.event); + const key = pairKey(constraint.event, constraint.anchor); + if (pairs.has(key)) + context.addIssue({ + code: "custom", + message: "not-before event pairs must be declared once", + path, + }); + pairs.add(key); + if (reachable.get(constraint.event)?.has(constraint.anchor) === true) + context.addIssue({ + code: "custom", + message: "not-before constraint conflicts with happens-before order", + path, + }); } - return adjacency; + return pairs; }; const graphHasCycle = ( @@ -257,11 +280,16 @@ const isOrdered = ( reachable.get(left)?.has(right) === true || reachable.get(right)?.has(left) === true; +type OrderingRelations = { + readonly reachable: ReadonlyMap>; + readonly notBeforePairs: ReadonlySet; +}; + const validateUnorderedGroups = ( language: PartialOrderLanguage, context: z.RefinementCtx, known: ReadonlySet, - reachable: ReadonlyMap>, + relations: OrderingRelations, ): ReadonlySet => { const pairs = new Set(); for (const [groupIndex, group] of language.unordered_groups.entries()) { @@ -284,7 +312,10 @@ const validateUnorderedGroups = ( path, }); pairs.add(key); - if (isOrdered(reachable, leftId, rightId)) + if ( + isOrdered(relations.reachable, leftId, rightId) || + relations.notBeforePairs.has(key) + ) context.addIssue({ code: "custom", message: "ordered events cannot also be declared unordered", @@ -298,14 +329,15 @@ const validateUnorderedGroups = ( const validateCompletePairDeclarations = ( ids: readonly string[], - reachable: ReadonlyMap>, + relations: OrderingRelations, unorderedPairs: ReadonlySet, context: z.RefinementCtx, ): void => { for (const [leftIndex, leftId] of ids.entries()) for (const rightId of ids.slice(leftIndex + 1)) if ( - !isOrdered(reachable, leftId, rightId) && + !isOrdered(relations.reachable, leftId, rightId) && + !relations.notBeforePairs.has(pairKey(leftId, rightId)) && !unorderedPairs.has(pairKey(leftId, rightId)) ) context.addIssue({ @@ -330,13 +362,20 @@ const validatePartialOrder = ( path: ["language"], }); const reachable = computeReachability(ids, adjacency); - const unorderedPairs = validateUnorderedGroups( + const notBeforePairs = validateNotBefore( specification.language, context, known, reachable, ); - validateCompletePairDeclarations(ids, reachable, unorderedPairs, context); + const relations = { reachable, notBeforePairs }; + const unorderedPairs = validateUnorderedGroups( + specification.language, + context, + known, + relations, + ); + validateCompletePairDeclarations(ids, relations, unorderedPairs, context); for (const [kind, values] of [ ["prefix", specification.language.prefix], ["suffix", specification.language.suffix], diff --git a/tests/processTraceAdapters.test.ts b/tests/processTraceAdapters.test.ts index 019ea8335..37d1c8176 100644 --- a/tests/processTraceAdapters.test.ts +++ b/tests/processTraceAdapters.test.ts @@ -290,7 +290,7 @@ describe("declared trace comparison adapters", () => { }), ).toMatchObject({ status: "unchanged", - trace: { verdict: "different" }, + trace: { verdict: "nonconforming" }, }); }); diff --git a/tests/processTraceComparison.test.ts b/tests/processTraceComparison.test.ts index 8099e0d78..d4b577555 100644 --- a/tests/processTraceComparison.test.ts +++ b/tests/processTraceComparison.test.ts @@ -1,12 +1,18 @@ import { describe, expect, it } from "vitest"; -import type { ProcessCapture } from "../src/domain/processCapture.js"; +import { EMPTY_PROCESS_CAPTURE_EXAMPLE } from "../src/contracts/processCaptureExample.js"; +import { + processCaptureSchema, + type ProcessCapture, +} from "../src/domain/processCapture.js"; import { compareProcessTraces, processTraceSpecificationSchema, type ProcessTraceSpecification, } from "../src/domain/processTraceComparison.js"; +const emptyCapture = processCaptureSchema.parse(EMPTY_PROCESS_CAPTURE_EXAMPLE); + const capture = ( values: Pick< ProcessCapture, @@ -23,58 +29,40 @@ const capture = ( readonly residualUnknowns?: ProcessCapture["residual_unknowns"]; } = {}, ): ProcessCapture => ({ - schema_version: 4, - manifest: { - rea_version: "test", - provider_version: "3", - platform: "linux", - architecture: "x64", - pty_backend: "node-pty", - started_at: "2026-01-01T00:00:00.000Z", - completed_at: "2026-01-01T00:00:01.000Z", - scenario: {}, - comparison_contract: {}, - shim_plan: [], - replay_plan: {}, - full_scenario_sha256: "0".repeat(64), - comparison_contract_sha256: "1".repeat(64), - executable_sha256: "2".repeat(64), - normalization_sha256: "3".repeat(64), - shim_plan_sha256: "4".repeat(64), - replay_plan_sha256: "5".repeat(64), - }, - normalization: { - paths: true, - pids: true, - ports: true, - time_bucket_ms: 10, - patterns: [], - }, + ...emptyCapture, frames: values.frames, - rendered_frames: [], - interaction_events: [], - exit: { code: 0, signal: null, reason: "exited" }, - settlement: { - state: "quiesced", - elapsed_ms: 10, - cleanup_outcome: "not_required", - }, process_samples: values.process_samples, - filesystem_checkpoints: values.filesystem_checkpoints, + filesystem_checkpoints: emptyCapture.filesystem_checkpoints, shim_events: values.shim_events, protocol_events: values.protocol_events, - replay_transitions: [], - event_journal: values.event_journal, - files_before: [], - files_after: [], - filesystem_effects: [], + event_journal: [ + { + capture_order: 0, + collection: "filesystem_checkpoints", + index: 0, + }, + ...values.event_journal.map((entry) => ({ + ...entry, + capture_order: entry.capture_order + 1, + })), + { + capture_order: values.event_journal.length + 1, + collection: "lifecycle", + index: 0, + }, + { + capture_order: values.event_journal.length + 2, + collection: "lifecycle", + index: 1, + }, + { + capture_order: values.event_journal.length + 3, + collection: "filesystem_checkpoints", + index: 1, + }, + ], truncated: options.truncated ?? false, - limitations: [], residual_unknowns: options.residualUnknowns ?? [], - cleanup: { - owned_process_group: "verified", - temporary_root: "removed", - }, }); const terminal = { sequence: 0, at_ms: 900, data: "Ready" }; @@ -94,7 +82,7 @@ const http = { method: "GET", path: "/status", data: "", - outcome: "matched" as const, + outcome: "unmatched" as const, }; const websocket = { sequence: 1, @@ -186,6 +174,18 @@ describe("process trace specification", () => { }, }).success, ).toBe(false); + expect( + processTraceSpecificationSchema.safeParse({ + version: 1, + events: base.events.slice(0, 3), + language: { + kind: "partial_order", + happens_before: [{ before: "ready", after: "status" }], + not_before: [{ event: "worker", anchor: "ready" }], + unordered_groups: [], + }, + }).success, + ).toBe(false); expect( processTraceSpecificationSchema.safeParse({ version: 1, @@ -315,7 +315,7 @@ describe("process trace comparison", () => { kind: "edge", side: "right", event_ids: ["ready", "worker"], - locations: [{ capture_order: 1 }, { capture_order: 0 }], + locations: [{ capture_order: 2 }, { capture_order: 1 }], }, }); }); @@ -417,6 +417,80 @@ describe("process trace comparison", () => { }); }); + it("enforces exact and range cardinality for one repeated predicate", () => { + const repeated = (count: number) => + capture({ + frames: Array.from({ length: count }, (_, sequence) => ({ + sequence, + at_ms: sequence, + data: "tick", + })), + process_samples: [], + filesystem_checkpoints: [], + protocol_events: [], + shim_events: [], + event_journal: Array.from({ length: count }, (_, index) => ({ + capture_order: index, + collection: "frames" as const, + index, + })), + }); + const event: Omit< + ProcessTraceSpecification["events"][number], + "cardinality" + > = { + id: "tick", + source: "terminal_raw", + exact: { data: "tick" }, + ignore_fields: ["sequence", "at_ms"], + }; + const rangeSpecification: ProcessTraceSpecification = { + version: 1, + events: [{ ...event, cardinality: { kind: "range", min: 2, max: 3 } }], + language: { + kind: "partial_order", + happens_before: [], + not_before: [], + unordered_groups: [], + prefix: [], + suffix: [], + }, + }; + expect( + compareProcessTraces(repeated(2), repeated(3), rangeSpecification), + ).toMatchObject({ verdict: "equivalent" }); + expect( + compareProcessTraces(repeated(1), repeated(2), rangeSpecification), + ).toMatchObject({ + verdict: "different", + diagnostic: { kind: "cardinality", side: "left" }, + }); + expect( + compareProcessTraces(repeated(2), repeated(4), rangeSpecification), + ).toMatchObject({ + verdict: "different", + diagnostic: { kind: "cardinality", side: "right" }, + }); + + const exactSpecification: ProcessTraceSpecification = { + version: 1, + events: [{ ...event, cardinality: { kind: "exact", count: 2 } }], + language: { + kind: "finite_traces", + variants: [{ id: "two", trace: ["tick", "tick"] }], + }, + }; + expect( + compareProcessTraces(repeated(2), repeated(2), exactSpecification), + ).toMatchObject({ verdict: "equivalent" }); + expect( + compareProcessTraces(repeated(3), repeated(2), exactSpecification), + ).toMatchObject({ + verdict: "different", + diagnostic: { kind: "cardinality", side: "left" }, + }); + }); + it("never proves equivalence from truncated, unknown, or journal-free evidence", () => { const complete = capture( values(["terminal", "process", "http", "websocket"]), @@ -437,7 +511,7 @@ describe("process trace comparison", () => { compareProcessTraces(complete, unknown, partialSpecification()).verdict, ).toBe("unknown"); - const noJournal = capture({ ...values([]), event_journal: [] }); + const noJournal = { ...complete, event_journal: [] }; const comparison = compareProcessTraces( complete, noJournal, @@ -447,5 +521,13 @@ describe("process trace comparison", () => { verdict: "unknown", diagnostic: { kind: "journal", side: "right" }, }); + + const incompleteJournal = { + ...complete, + event_journal: (complete.event_journal ?? []).slice(1), + }; + expect(() => + compareProcessTraces(complete, incompleteJournal, partialSpecification()), + ).toThrow("Invalid Process Capture v4: event_journal.0.capture_order"); }); });