From 00d8e44cbf82caa51709c2e0857fbaa69ea329b7 Mon Sep 17 00:00:00 2001 From: N0zoM1z0 <161784452+N0zoM1z0@users.noreply.github.com> Date: Fri, 9 Oct 2026 17:55:28 +0800 Subject: [PATCH] fix(process): preserve numeric output and original PTY evidence --- docs/process-capture.md | 17 +++++++ .../process/processCapture.comparison.test.ts | 22 +++++++++ src/domain/process/processCapture.ts | 3 ++ src/domain/process/processComparison.ts | 6 ++- src/domain/process/processObservation.ts | 19 ++++++-- src/domain/process/processScenario.ts | 3 ++ .../capture/ProcessCaptureLifecycle.ts | 1 + .../capture/ProcessNormalization.test.ts | 33 ++++++++++++- src/process/capture/ProcessNormalization.ts | 25 +++++++++- .../processCaptureNormalization.test.ts | 46 +++++++++++++++++++ 10 files changed, 168 insertions(+), 7 deletions(-) diff --git a/docs/process-capture.md b/docs/process-capture.md index 2b4438139..f7a117b4a 100644 --- a/docs/process-capture.md +++ b/docs/process-capture.md @@ -94,6 +94,23 @@ deleted, modified, or unchanged. These are bounded snapshots, not a syscall trace; short-lived changes between snapshots may be missed. With no selected paths, filesystem effects remain unknown. +Terminal frame `data` is the comparison text after the selected normalization. +New captures also preserve `raw_data` when normalization changes a PTY chunk; +otherwise `data` is already the original text. Older captures may lack the +original text for changed chunks. Both values participate in Evidence identity, +while comparisons and trace assertions use the normalized text. Retaining the +original adds at most `limits.output_bytes` of text, because it comes from the +same admitted PTY chunks rather than a second unbounded stream. + +Port normalization recognizes explicit `port`, `tcp_port`, `udp_port`, and +`listen` fields, URL authorities, IP endpoints, and `localhost` endpoints, with +ports from 0 through 65535. It preserves ordinary counters, dimensions, file +line numbers, and ambiguous bare host labels. Use explicit literal `patterns` +for a different application-specific spelling. Set `ports: false` to preserve +all endpoint numbers in comparison text. Captures commit the port-normalization +version so older broad numeric normalization cannot silently compare as the +same contract. + Output, file count, file size, process sampling, filesystem depth, total runtime, idle time, and post-exit settlement are bounded by the scenario's limits. The result marks truncated observations and residual unknowns rather diff --git a/src/domain/process/processCapture.comparison.test.ts b/src/domain/process/processCapture.comparison.test.ts index 93d724627..85fd98369 100644 --- a/src/domain/process/processCapture.comparison.test.ts +++ b/src/domain/process/processCapture.comparison.test.ts @@ -5,6 +5,28 @@ import { emptyProcessCapture as emptyCapture, } from "./processCapture.fixture.js"; +it("compares normalized terminal facts while retaining different source text for audit", () => { + const capture = emptyCapture(); + const frame = { sequence: 0, at_ms: 0, data: "listen: score=20" }; + const left = { + ...capture, + frames: [{ ...frame, raw_data: "listen:8080 score=20" }], + }; + const right = { + ...capture, + frames: [{ ...frame, raw_data: "listen:9090 score=20" }], + }; + expect(compareProcessCaptures(left, right)).toMatchObject({ + terminal: "unchanged", + }); + expect( + compareProcessCaptures(left, { + ...right, + frames: [{ ...frame, data: "listen: score=21" }], + }), + ).toMatchObject({ terminal: "changed" }); +}); + it("distinguishes added evidence from unknown observations", () => { const capture = emptyCapture(); const added = compareProcessCaptures(capture, { diff --git a/src/domain/process/processCapture.ts b/src/domain/process/processCapture.ts index 9739cba66..45803af00 100644 --- a/src/domain/process/processCapture.ts +++ b/src/domain/process/processCapture.ts @@ -11,6 +11,8 @@ export interface TerminalFrame { readonly sequence: number; readonly at_ms: number; readonly data: string; + /** Original PTY text when normalization changed it; older captures may omit it. */ + readonly raw_data?: string | undefined; } /** Serialized terminal state after interpreting control and resize sequences. */ @@ -402,6 +404,7 @@ const processCaptureShapeSchema = z.strictObject({ sequence: z.number().int().nonnegative(), at_ms: z.number().int().nonnegative(), data: z.string(), + raw_data: z.string().optional(), }), ), rendered_frames: z.array( diff --git a/src/domain/process/processComparison.ts b/src/domain/process/processComparison.ts index 1aa052bc4..04542631a 100644 --- a/src/domain/process/processComparison.ts +++ b/src/domain/process/processComparison.ts @@ -2,6 +2,7 @@ import canonicalize from "canonicalize"; import { z } from "zod"; import type { ProcessCapture } from "./processCapture.js"; +import { comparableTerminalFrame } from "./processObservation.js"; import { AnalysisInputError } from "../analysisErrorCore.js"; import { jsonValueSchema } from "../jsonValue.js"; import { @@ -100,7 +101,10 @@ export type ProcessCaptureComparison = z.infer< const terminalObservations = (capture: ProcessCapture): readonly unknown[] => [ - ...capture.frames.map((frame) => ({ kind: "raw" as const, ...frame })), + ...capture.frames.map((frame) => ({ + kind: "raw" as const, + ...comparableTerminalFrame(frame), + })), ...capture.rendered_frames.map((frame) => ({ kind: "rendered" as const, ...frame, diff --git a/src/domain/process/processObservation.ts b/src/domain/process/processObservation.ts index 167d36ec5..f8ad26130 100644 --- a/src/domain/process/processObservation.ts +++ b/src/domain/process/processObservation.ts @@ -1,7 +1,17 @@ import { z } from "zod"; import { jsonValueSchema, type JsonValue } from "../jsonValue.js"; -import type { UnverifiedProcessCapture } from "./processCapture.js"; +import type { + TerminalFrame, + UnverifiedProcessCapture, +} from "./processCapture.js"; + +/** Project normalized PTY facts without source text used only for audit. */ +export const comparableTerminalFrame = (frame: TerminalFrame) => ({ + sequence: frame.sequence, + at_ms: frame.at_ms, + data: frame.data, +}); /** Process-capture observation families shared by live drivers and trace assertions. */ export const processObservationSourceSchema = z.enum([ @@ -126,11 +136,14 @@ const projectedRecord = ( ): ProjectedRecord | null => { const projected = (() => { switch (location.collection) { - case "frames": + case "frames": { + const frame = capture.frames[location.index]; return { source: "terminal_raw" as const, - value: capture.frames[location.index], + value: + frame === undefined ? undefined : comparableTerminalFrame(frame), }; + } case "rendered_frames": return { source: "terminal_rendered" as const, diff --git a/src/domain/process/processScenario.ts b/src/domain/process/processScenario.ts index 586775032..08672cdaf 100644 --- a/src/domain/process/processScenario.ts +++ b/src/domain/process/processScenario.ts @@ -84,6 +84,9 @@ export const processComparisonContract = ( settle_ms: scenario.settle_ms, limits: scenario.limits, normalization: scenario.normalization, + ...(scenario.normalization.ports + ? { port_normalization_version: "contextual-endpoints-v1" } + : {}), }); /** diff --git a/src/process/capture/ProcessCaptureLifecycle.ts b/src/process/capture/ProcessCaptureLifecycle.ts index d0b22abc9..0e413beb3 100644 --- a/src/process/capture/ProcessCaptureLifecycle.ts +++ b/src/process/capture/ProcessCaptureLifecycle.ts @@ -828,6 +828,7 @@ export const captureTerminalFrames = (options: { sequence, at_ms: atMs, data: normalized, + ...(normalized === data ? {} : { raw_data: data }), }); options.renderer.write(data, atMs); options.recordEvent("frames", sequence); diff --git a/src/process/capture/ProcessNormalization.test.ts b/src/process/capture/ProcessNormalization.test.ts index 7a15b8b96..c28d1e42d 100644 --- a/src/process/capture/ProcessNormalization.test.ts +++ b/src/process/capture/ProcessNormalization.test.ts @@ -1,6 +1,9 @@ import { expect, it } from "vitest"; -import { normalizeProcessSamples } from "./ProcessNormalization.js"; +import { + normalizeProcessSamples, + normalizeProcessText, +} from "./ProcessNormalization.js"; import { parseProcessScenario } from "../../domain/process/processCapture.js"; const rootPid = 41_010; @@ -68,6 +71,34 @@ const scenario = (normalization: { }, }); +it("preserves compact JSON, counters, line numbers, and ambiguous endpoint spellings", () => { + const input = [ + '{"pid":32,"size":108,"height":480,"width":640,"score":20}', + "score=20 size:108 elapsed=3600 file.js:480 server:8080", + "port=65536 port:8080ms localhost:70000 999.1.2.3:8080", + "https://user:32@example.test/path http://user:20@localhost/path", + ].join("\n"); + expect(normalizeProcessText(input, scenario({}), "/temporary", rootPid)).toBe( + input, + ); +}); + +it("normalizes only contextual ports before overlapping PID values and honors disabled rules", () => { + const input = + 'http://example.test:8080/x https://user:password@example.test:443/x 127.0.0.1:9 [::1]:65535 localhost:41010 "port":80 tcp_port=123 udp_port:0 listen:8080'; + expect(normalizeProcessText(input, scenario({}), "/temporary", rootPid)).toBe( + 'http://example.test:/x https://user:password@example.test:/x 127.0.0.1: [::1]: localhost: "port": tcp_port= udp_port: listen:', + ); + expect( + normalizeProcessText( + input, + scenario({ ports: false, pids: false }), + "/temporary", + rootPid, + ), + ).toBe(input); +}); + it("normalizes representative process identities and preserves null, zero, and unrelated numbers", () => { const before = JSON.stringify(samples); const normalized = normalizeProcessSamples(samples, scenario({}), rootPid); diff --git a/src/process/capture/ProcessNormalization.ts b/src/process/capture/ProcessNormalization.ts index 347c0e5cf..d9dd68385 100644 --- a/src/process/capture/ProcessNormalization.ts +++ b/src/process/capture/ProcessNormalization.ts @@ -1,3 +1,5 @@ +import { isIP } from "node:net"; + import type { ProcessSample, ProcessScenario, @@ -30,15 +32,34 @@ export const normalizeProcessText = ( ``, ); } + if (scenario.normalization.ports) + normalized = normalizePortTokens(normalized); if (scenario.normalization.pids) normalized = normalizePidTokens(normalized, [pid]); - if (scenario.normalization.ports) - normalized = normalized.replaceAll(/(?<=[:=])\d{2,5}\b/g, ""); for (const pattern of scenario.normalization.patterns) normalized = normalized.replaceAll(pattern.pattern, pattern.replacement); return normalized; }; +const normalizePortTokens = (value: string): string => { + const endpointNormalized = value.replaceAll( + /(\b[a-z][a-z0-9+.-]*:\/\/[^\s/?#"'<>]+|\blocalhost|\b(?:\d{1,3}\.){3}\d{1,3}|\[[\da-f:.%]+\]):(\d{1,5})(?![\w.@:])/giu, + (match: string, endpoint: string, port: string) => { + if (Number(port) > 65_535) return match; + const isEndpoint = endpoint.includes("://") + ? URL.canParse(`${endpoint}:${port}`) + : endpoint.toLowerCase() === "localhost" || + isIP(endpoint.replace(/^\[|\]$/gu, "")) !== 0; + return isEndpoint ? `${endpoint}:` : match; + }, + ); + return endpointNormalized.replaceAll( + /(\b(?:port|tcp_port|udp_port|listen)\b["']?\s*[:=]\s*)(\d{1,5})(?![\w.])/giu, + (match: string, prefix: string, port: string) => + Number(port) <= 65_535 ? `${prefix}` : match, + ); +}; + /** Project sampled observations under the caller-selected normalization rules. */ export const normalizeProcessSamples = ( samples: readonly ProcessSample[], diff --git a/tests/boundary/process/processCaptureNormalization.test.ts b/tests/boundary/process/processCaptureNormalization.test.ts index dbaef7c28..468339c86 100644 --- a/tests/boundary/process/processCaptureNormalization.test.ts +++ b/tests/boundary/process/processCaptureNormalization.test.ts @@ -27,6 +27,52 @@ const identitySchema = z.object({ const itWithLinuxCaptureCapability = process.platform === "linux" ? itWithCaptureCapability : it.skip; +itWithCaptureCapability( + "preserves numeric reports and original normalized PTY text in Evidence", + async () => { + const root = await createTestTempDirectory("rea-process-numeric-report-"); + const report = JSON.stringify({ + pid: 32, + size: 108, + height: 480, + width: 640, + score: 20, + }); + const original = `${report}\nlisten:8080\n`; + const scenario = parseProcessScenario({ + executable: process.execPath, + working_directory: root, + arguments: ["-e", `process.stdout.write(${JSON.stringify(original)})`], + settle_ms: 100, + }); + const result = await captureProcessScenario(scenario); + if (!result.ok) throw result.error; + const capture = parseProcessCapture( + createProcessCaptureEvidence(scenario, result.value).normalized_result, + ); + expect(capture.frames.map(({ data }) => data).join("")).toContain(report); + expect(capture.frames.map(({ data }) => data).join("")).toContain( + "listen:", + ); + expect( + capture.frames.map(({ data, raw_data }) => raw_data ?? data).join(""), + ).toBe(original.replaceAll("\n", "\r\n")); + expect( + capture.rendered_frames.some(({ serialized_state }) => + serialized_state.includes(report), + ), + ).toBe(true); + expect(capture.manifest.comparison_contract).toMatchObject({ + port_normalization_version: "contextual-endpoints-v1", + }); + expect(capture.exit).toMatchObject({ code: 0, reason: "exited" }); + expect(capture.cleanup).toMatchObject({ + owned_process_group: "verified", + temporary_root: "removed", + }); + }, +); + itWithLinuxCaptureCapability.each([false, true])( "honors pids=%s in owned Linux capture samples and committed Evidence", async (pids) => {