diff --git a/src/evaluation/CodexAgentEval.ts b/src/evaluation/CodexAgentEval.ts index 55ec8333e..744524067 100644 --- a/src/evaluation/CodexAgentEval.ts +++ b/src/evaluation/CodexAgentEval.ts @@ -205,7 +205,7 @@ export const evaluateCodexEvents = ( ); const reaCalls = state.calls.filter( - ({ server }) => server === null || server.toLowerCase() === "rea", + ({ server }) => server?.toLowerCase() === "rea", ); const signatures = new Map(); let targetRevision = 0; diff --git a/tests/evaluation/model-evals/codexAgentEval.test.ts b/tests/evaluation/model-evals/codexAgentEval.test.ts index 6b46d9b1d..910fd7ae6 100644 --- a/tests/evaluation/model-evals/codexAgentEval.test.ts +++ b/tests/evaluation/model-evals/codexAgentEval.test.ts @@ -218,7 +218,23 @@ describe("Codex completed tool call parsing", () => { }, ); - it("preserves legacy completed calls with no server field", () => { + it.each([ + { + label: "missing server identity", + identity: {}, + routed: false, + }, + { + label: "another server", + identity: { server: "other" }, + routed: false, + }, + { + label: "verified REA alias", + identity: { server_name: "rea" }, + routed: true, + }, + ])("routes only $label calls", ({ identity, routed }) => { const metrics = evaluateCodexEvents( [ { @@ -228,16 +244,20 @@ describe("Codex completed tool call parsing", () => { status: "completed", tool: "inspect_managed_artifact", arguments: {}, + ...identity, }, }, ], "inspect_managed_artifact", + { requiredToolSubsequence: ["inspect_managed_artifact"] }, ); expect(metrics).toMatchObject({ - naturalUse: true, - correctFirstTool: true, - reaCalls: [{ server: null }], + naturalUse: routed, + correctFirstTool: routed, + firstTool: routed ? "inspect_managed_artifact" : null, + requiredToolSubsequenceMet: routed, + reaCalls: routed ? [{ server: "rea" }] : [], }); }); });