fix(process): report single empty settlement observation as unverifiable (#2232)

observeSettlement classified a group as alive_at_deadline when the settlement
budget expired after one empty observation, though no live member was ever
seen. Track observed liveness and return unverifiable when the deadline ends
observation with only empty rounds. The two-consecutive-empty rule for
quiesced is unchanged, and the cleanup_outcome projection for non-quiesced
settlements stays cleaned.

Fixes #2170.
This commit is contained in:
Saim Shafique 2026-10-12 21:15:12 +05:00 committed by GitHub
parent 0baad58e60
commit 7724edbdd3
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
2 changed files with 83 additions and 2 deletions

View File

@ -662,6 +662,7 @@ export const observeSettlement = async (
}
const started = Date.now();
let consecutiveEmpty = 0;
let sawLive = false;
let deadlineReached = false;
while (!deadlineReached) {
onLiveProgress?.();
@ -685,13 +686,20 @@ export const observeSettlement = async (
recordEvent("lifecycle", 1);
return { state: "quiesced", elapsed_ms: Date.now() - started };
}
} else consecutiveEmpty = 0;
} else {
consecutiveEmpty = 0;
sawLive = true;
}
deadlineReached = Date.now() - started >= settleMs;
if (deadlineReached) break;
await delay(50, undefined, { signal });
}
recordEvent("lifecycle", 1);
return { state: "alive_at_deadline", elapsed_ms: Date.now() - started };
// A lone empty observation proves neither liveness nor quiescence; only an
// observed live member justifies alive_at_deadline.
return sawLive
? { state: "alive_at_deadline", elapsed_ms: Date.now() - started }
: { state: "unverifiable", elapsed_ms: Date.now() - started };
};
/** Name why an attempt is unverified; a blank message never reaches the capture schema. */

View File

@ -0,0 +1,73 @@
import { spawn } from "node:child_process";
import { describe, expect, it } from "vitest";
import { observeSettlement } from "../../../src/process/capture/ProcessCaptureLifecycle.js";
import { observeOwnedProcessGroups } from "../../../src/process/ProcessOwnershipObservation.js";
/** Process group id that cannot exist on any supported host. */
const ABSENT_GROUP_ID = 4_294_967_295;
const observeGroup = async (groupId: number) =>
observeOwnedProcessGroups("settlement-deadline-probe", [groupId]);
describe("settlement deadline classification", () => {
it("does not report a single empty observation as alive_at_deadline", async () => {
expect(
(await observeGroup(ABSENT_GROUP_ID)).get(ABSENT_GROUP_ID)?.state,
).toBe("empty");
const settlement = await observeSettlement(
"settlement-single-empty",
[ABSENT_GROUP_ID],
0,
() => undefined,
"linux",
);
expect(settlement.state).toBe("unverifiable");
});
it("still reports observed liveness as alive_at_deadline", async () => {
const runId = "settlement-live-group";
// A detached child leads its own process group and carries the run token,
// so the group observes as a genuinely owned live member.
const child = spawn("sleep", ["30"], {
detached: true,
stdio: "ignore",
env: { ...process.env, REA_PROCESS_RUN_ID: runId },
});
child.unref();
try {
// Wait until the child is actually listed before asserting on it.
const started = Date.now();
while (
(await observeOwnedProcessGroups(runId, [child.pid!])).get(child.pid!)
?.state !== "alive"
) {
if (Date.now() - started > 5_000)
throw new Error("live fixture group never observed as alive");
await new Promise((resolve) => setTimeout(resolve, 50));
}
const settlement = await observeSettlement(
runId,
[child.pid!],
0,
() => undefined,
"linux",
);
expect(settlement.state).toBe("alive_at_deadline");
} finally {
child.kill("SIGKILL");
}
});
it("still reports two consecutive empty observations as quiesced", async () => {
const settlement = await observeSettlement(
"settlement-two-empty",
[ABSENT_GROUP_ID],
1_000,
() => undefined,
"linux",
);
expect(settlement.state).toBe("quiesced");
});
});