diff --git a/src/evaluation/CodexAgentEval.ts b/src/evaluation/CodexAgentEval.ts index ba5c500d0..bac8d8191 100644 --- a/src/evaluation/CodexAgentEval.ts +++ b/src/evaluation/CodexAgentEval.ts @@ -1,4 +1,5 @@ import { + deliveredCodexEvidence, evaluateKnownAnswers, type FixtureClaimExpectation, type KnownAnswerAssessment, @@ -67,12 +68,6 @@ const canonicalValue = (value: unknown): unknown => { ); }; -const evidenceIdsFrom = (value: unknown): readonly string[] => { - const encoded = JSON.stringify(value); - if (encoded === undefined) return []; - return [...new Set(encoded.match(/ev_[a-f0-9]{64}/gu) ?? [])]; -}; - const toolResultError = ( item: Record, ): Record | undefined => { @@ -118,8 +113,16 @@ const toolResultFailed = ( ); }; -const callFromItem = (value: unknown): CodexMcpCall | undefined => { - const item = record(value); +const callFromEvent = ( + event: Record, +): CodexMcpCall | undefined => { + const item = record( + event.type === "item.completed" + ? event.item + : event.type === "mcp_tool_call" + ? event + : undefined, + ); if (item?.type !== "mcp_tool_call") return undefined; const tool = textValue(item.tool) ?? textValue(item.name); if (tool === undefined) return undefined; @@ -129,7 +132,9 @@ const callFromItem = (value: unknown): CodexMcpCall | undefined => { server: textValue(item.server) ?? textValue(item.server_name) ?? null, tool, arguments: item.arguments ?? item.input ?? {}, - evidenceIds: evidenceIdsFrom(item.result ?? item.output), + evidenceIds: deliveredCodexEvidence(event).map( + ({ evidence_id }) => evidence_id, + ), error: toolResultFailed(item, error), errorCode: textValue(error?.code) ?? null, }; @@ -163,9 +168,7 @@ const consumeEvent = (state: EvaluationState, value: unknown): void => { if (event === undefined) return; const completedItem = event.type === "item.completed" ? event.item : undefined; - const call = - callFromItem(completedItem) ?? - (event.type === "mcp_tool_call" ? callFromItem(event) : undefined); + const call = callFromEvent(event); if (call !== undefined) { const identity = call.id ?? diff --git a/src/evaluation/KnownAnswerEvaluation.ts b/src/evaluation/KnownAnswerEvaluation.ts index 77fbd9718..807308b59 100644 --- a/src/evaluation/KnownAnswerEvaluation.ts +++ b/src/evaluation/KnownAnswerEvaluation.ts @@ -124,7 +124,7 @@ const sameJson = (left: unknown, right: unknown): boolean => { ); }; -const parseClosedAnswer = (text: string): unknown => { +const parseClosedJson = (text: string): unknown => { const objects: Set[] = []; let invalid = false; try { @@ -156,7 +156,7 @@ const parseClosedAnswer = (text: string): unknown => { // The JSON visitor checked syntax and duplicate members before JSON.parse can fold them. return JSON.parse(text); } catch (error) { - // Syntax and nesting failures are malformed final-answer input, not an evaluation crash. + // Syntax and nesting failures are malformed transcript input, not an evaluation crash. if (error instanceof SyntaxError || error instanceof RangeError) return undefined; throw error; @@ -224,20 +224,29 @@ const evidenceFromCall = ( (result.error !== undefined && result.error !== null) ) return undefined; - const structured = record( - result.structuredContent ?? result.structured_content, - ); - if ( - structured === undefined || - (structured.error !== undefined && structured.error !== null) - ) - return undefined; if ( result.structuredContent !== undefined && result.structured_content !== undefined && !sameJson(result.structuredContent, result.structured_content) ) return undefined; + let value = result.structuredContent ?? result.structured_content; + if (value === undefined || value === null) { + // Some Codex transcripts retain only the complete MCP JSON text block. + // Prose, multiple blocks, and truncated previews cannot establish delivery. + if (!Array.isArray(result.content) || result.content.length !== 1) + return undefined; + const block = record(result.content[0]); + if (block?.type !== "text" || typeof block.text !== "string") + return undefined; + value = parseClosedJson(block.text); + } + const structured = record(value); + if ( + structured === undefined || + (structured.error !== undefined && structured.error !== null) + ) + return undefined; let evidence: Evidence; try { evidence = parseEvidence(structured); @@ -249,20 +258,53 @@ const evidenceFromCall = ( return evidence; }; +/** Authenticate completed REA deliveries for both citations and fixture grading. */ +export const deliveredCodexEvidence = (event: unknown): readonly Evidence[] => { + const item = completedCall(event); + if (item === undefined) return []; + const evidence = evidenceFromCall(item); + if (evidence === undefined) return []; + const records = [evidence]; + // Artifact inspection delivers its inventory Evidence inline. Bare links and + // arbitrary IDs in normalized facts are references, not delivered records. + if (evidence.operation !== "inspect_artifact") return records; + const substeps = record(evidence.normalized_result)?.substeps; + if (!Array.isArray(substeps)) return records; + for (const value of substeps) { + const substep = record(value); + if ( + substep?.status !== "completed" || + substep.operation !== "inventory_artifact" || + typeof substep.evidence_id !== "string" || + !evidence.evidence_links.includes(substep.evidence_id) + ) + continue; + try { + const nested = parseEvidence(substep.evidence); + if ( + nested.evidence_id === substep.evidence_id && + nested.operation === substep.operation + ) + records.push(nested); + } catch { + // An authenticated parent does not authenticate an embedded record's ID. + } + } + return records; +}; + const collectDeliveredEvidence = ( events: readonly unknown[], ): DeliveredEvidence => { const records = new Map(); const conflicts = new Set(); for (const event of events) { - const item = completedCall(event); - if (item === undefined) continue; - const evidence = evidenceFromCall(item); - if (evidence === undefined) continue; - const previous = records.get(evidence.evidence_id); - if (previous !== undefined && !sameJson(previous, evidence)) - conflicts.add(evidence.evidence_id); - else records.set(evidence.evidence_id, evidence); + for (const evidence of deliveredCodexEvidence(event)) { + const previous = records.get(evidence.evidence_id); + if (previous !== undefined && !sameJson(previous, evidence)) + conflicts.add(evidence.evidence_id); + else records.set(evidence.evidence_id, evidence); + } } return { records, conflicts }; }; @@ -382,7 +424,7 @@ export const evaluateKnownAnswers = ( manifestFailures: ["duplicate_fixture_claim_ids"], claims: [], }; - const manifest = finalAnswerSchema.safeParse(parseClosedAnswer(finalMessage)); + const manifest = finalAnswerSchema.safeParse(parseClosedJson(finalMessage)); if (!manifest.success) return { scope, diff --git a/tests/evaluation/model-evals/codexAgentEval.test.ts b/tests/evaluation/model-evals/codexAgentEval.test.ts index f802d68d2..d96265202 100644 --- a/tests/evaluation/model-evals/codexAgentEval.test.ts +++ b/tests/evaluation/model-evals/codexAgentEval.test.ts @@ -1,7 +1,56 @@ import { describe, expect, it } from "vitest"; +import { createEvidence } from "../../../src/domain/evidence.js"; +import type { JsonValue } from "../../../src/domain/jsonValue.js"; import { evaluateCodexEvents } from "../../../src/evaluation/CodexAgentEval.js"; +const observation = ( + operation = "inspect_managed_artifact", + result: JsonValue = { observed: true }, + evidenceLinks: readonly string[] = [], +) => + createEvidence( + { path: "/tmp/app.dll", sha256: "1".repeat(64), format: "pe" }, + { id: "managed", name: "Managed static analysis", version: null }, + { + operation, + parameters: {}, + result, + evidenceLinks, + authority: "shipped-artifact", + }, + ); + +const citationMetrics = ( + result: unknown, + evidenceId: string, + tool = "inspect_managed_artifact", +) => + evaluateCodexEvents( + [ + { + type: "item.completed", + item: { + type: "mcp_tool_call", + status: "completed", + server: "rea", + tool, + arguments: {}, + result, + }, + }, + { + type: "item.completed", + item: { + type: "agent_message", + text: `Observed artifact evidence ${evidenceId} is cited here; runtime behavior remains unknown because no runtime authority was supplied.`, + }, + }, + ], + tool, + { requireEvidence: true }, + ); + describe("Codex agent release evaluation", () => { it("measures natural routing, repeated calls, tokens, and answer heuristics", () => { const call = { @@ -50,7 +99,8 @@ describe("Codex agent release evaluation", () => { }); it("requires the final answer to cite produced Evidence when requested", () => { - const evidenceId = `ev_${"a".repeat(64)}`; + const evidence = observation(); + const evidenceId = evidence.evidence_id; const events = [ { type: "item.completed", @@ -60,7 +110,7 @@ describe("Codex agent release evaluation", () => { server: "rea", tool: "inspect_managed_artifact", arguments: { path: "/tmp/app.dll" }, - result: { evidence_id: evidenceId }, + result: { structuredContent: evidence }, }, }, { @@ -82,7 +132,126 @@ describe("Codex agent release evaluation", () => { answerHeuristicsMet: true, }); }); +}); +describe("Codex Evidence delivery", () => { + it("rejects diagnostic IDs and failed results as delivered Evidence", () => { + const evidenceId = `ev_${"a".repeat(64)}`; + for (const result of [ + { content: [{ type: "text", text: `diagnostic: ${evidenceId}` }] }, + { evidence_id: evidenceId }, + { structuredContent: { metadata: { evidence_id: evidenceId } } }, + { + isError: true, + content: [ + { + type: "text", + text: JSON.stringify({ error: { message: evidenceId } }), + }, + ], + }, + ]) { + expect(citationMetrics(result, evidenceId)).toMatchObject({ + evidenceIds: [], + finalCitesEvidence: false, + answerHeuristicsMet: false, + }); + } + }); + + it("accepts complete text-only Evidence while rejecting truncated and tampered copies", () => { + const evidence = observation(); + const text = JSON.stringify(evidence); + const textResult = (value: string) => ({ + content: [{ type: "text", text: value }], + structured_content: null, + }); + expect( + citationMetrics(textResult(text), evidence.evidence_id), + ).toMatchObject({ + evidenceIds: [evidence.evidence_id], + finalCitesEvidence: true, + answerHeuristicsMet: true, + }); + for (const result of [ + textResult(text.slice(0, -1)), + textResult( + JSON.stringify({ + ...evidence, + normalized_result: { fabricated: true }, + }), + ), + textResult( + text.replace( + `"evidence_id":`, + `"evidence_id":"${evidence.evidence_id}","evidence_id":`, + ), + ), + ]) { + expect(citationMetrics(result, evidence.evidence_id).evidenceIds).toEqual( + [], + ); + } + }); + + it("retains authenticated nested inspection Evidence without accepting bare or tampered links", () => { + const nested = observation("inventory_artifact"); + const substep = { + status: "completed", + operation: nested.operation, + evidence_id: nested.evidence_id, + evidence: nested, + }; + const inspection = observation( + "inspect_artifact", + { substeps: [substep] }, + [nested.evidence_id], + ); + expect( + citationMetrics( + { structuredContent: inspection }, + nested.evidence_id, + "inspect_artifact", + ), + ).toMatchObject({ + evidenceIds: [inspection.evidence_id, nested.evidence_id], + finalCitesEvidence: true, + }); + for (const [step, links] of [ + [substep, []], + [ + { + ...substep, + evidence: { ...nested, normalized_result: { fabricated: true } }, + }, + [nested.evidence_id], + ], + [{ ...substep, operation: "other_tool" }, [nested.evidence_id]], + [ + { ...substep, evidence_id: `ev_${"b".repeat(64)}` }, + [nested.evidence_id], + ], + ] as const) { + const parent = observation( + "inspect_artifact", + { substeps: [step] }, + links, + ); + expect( + citationMetrics( + { structuredContent: parent }, + nested.evidence_id, + "inspect_artifact", + ), + ).toMatchObject({ + evidenceIds: [parent.evidence_id], + finalCitesEvidence: false, + }); + } + }); +}); + +describe("Codex request accounting", () => { it("flags identical calls with distinct item identities", () => { const event = (id: string) => ({ type: "item.completed", @@ -269,7 +438,8 @@ describe("Codex agent routing and error evaluation", () => { describe("Codex agent completion evaluation", () => { it("does not present matching answer terms as factual correctness or honesty", () => { - const evidenceId = `ev_${"a".repeat(64)}`; + const evidence = observation("analyze_javascript_application"); + const evidenceId = evidence.evidence_id; const metrics = evaluateCodexEvents( [ { @@ -280,7 +450,7 @@ describe("Codex agent completion evaluation", () => { server: "rea", tool: "analyze_javascript_application", arguments: { input_path: "/tmp/app.asar" }, - result: { evidence_id: evidenceId }, + result: { structuredContent: evidence }, }, }, { @@ -380,11 +550,15 @@ describe("Codex agent completion evaluation", () => { describe("Codex agent workflow evaluation", () => { it("requires an ordered tool subsequence and rejects input validation failures", () => { - const evidenceId = `ev_${"b".repeat(64)}`; + const comparison = observation("compare_javascript_export_shapes"); + const evidenceId = comparison.evidence_id; const call = ( id: string, tool: string, - result: unknown = { evidence_id: evidenceId }, + result: unknown = { + structuredContent: + tool === comparison.operation ? comparison : observation(tool), + }, ) => ({ type: "item.completed", item: { diff --git a/tests/evaluation/model-evals/knownAnswerEvaluation.test.ts b/tests/evaluation/model-evals/knownAnswerEvaluation.test.ts index eeb1b9b5d..cbe59dd7b 100644 --- a/tests/evaluation/model-evals/knownAnswerEvaluation.test.ts +++ b/tests/evaluation/model-evals/knownAnswerEvaluation.test.ts @@ -351,6 +351,19 @@ describe("successful Evidence grounding", () => { ).toBe("passed"); }); + it("grades a complete text-only production Evidence response", () => { + const evidence = createObservation(); + const event = delivered(evidence, { + result: { + content: resultFor(evidence).content, + structured_content: null, + }, + }); + expect( + assess([event], answer([claimFor(evidence)])).factualCorrectness, + ).toBe("passed"); + }); + it("rejects authentic unrelated Evidence containing the same value elsewhere", () => { const evidence = createObservation({ unrelated: expected().expectedValue }); expect(