shifathosn
/
rea
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

test(perf): distinguish Codex tool context and token accounting (#2416) 

Add optional owned Codex tool/usage capture and exact-name discovery experiments while preserving default ephemeral trials and all REA public APIs. Retain scoped managed and actual Android results, mixed outcomes, source identities and resource unknowns.

Six semantic Python tests, full checked pre-push, byte-identical own-file rebase verification, manual review and all 39 exact-head CI checks pass. Historical main fixture failures and their independent repair remain documented; measured source/runtime pins are unchanged.

Co-authored-by: factory-droid[bot] <138933559+factory-droid[bot]@users.noreply.github.com>
This commit is contained in:
Ysopwnable 2026-10-13 09:01:17 +08:00 committed by GitHub
parent 37b7550ecc
commit ee87111009
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
7 changed files with 582 additions and 28 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
experiments/analysis-efficiency/README.md
View File

@ -32,6 +32,11 @@ records two shipped assembly checks, previous-runtime portable Evidence, and fou
actual Codex trials. Selected replies shrink substantially; observed model input
tokens increase and latency is mixed, so model-token savings are not claimed.
Follow-up: [model tool context and per-response usage](model-context.md) separates
raw MCP replies from observed client execution/discovery and usage. Optional
trial capture retains tool/usage items while excluding account and ambient state;
existing ephemeral trials keep their default behavior.
The 12 detailed checkpoint receipts were archived byte-for-byte before removal
from the final PR tree. The archive `review-receipts-00d51e0fc-20261012` is in the
private experiment lab outside the checkout; its `archive-index.json` records

79
experiments/analysis-efficiency/agent-trial.py
View File

@ -11,6 +11,8 @@ import subprocess
import tempfile
import time
from codex_context import capture_tool_context
def file_digest(path):
digest = hashlib.sha256()
@ -48,7 +50,20 @@ def main():
parser.add_argument('--providers', type=Path)
parser.add_argument('--enabled-tools', type=Path)
parser.add_argument('--authority', choices=['static', 'browser'], default='static')
parser.add_argument('--capture-tool-context', action='store_true',
help='Retain owned tool/usage items; excludes account/ambient state')
parser.add_argument('--guide-enabled-tools', action='store_true',
help='Opt-in exact-name discovery guidance for selected REA tools')
args = parser.parse_args()
if args.guide_enabled_tools and args.enabled_tools is None:
parser.error('--guide-enabled-tools requires --enabled-tools')
enabled_tools = None
if args.enabled_tools:
enabled_tools = json.loads(args.enabled_tools.read_text())
if not isinstance(enabled_tools, list) or not all(isinstance(name, str) and name for name in enabled_tools) or len(enabled_tools) != len(set(enabled_tools)):
raise ValueError('enabled-tools must be a unique list of exact tool names')
if args.guide_enabled_tools and not enabled_tools:
parser.error('--guide-enabled-tools requires a nonempty selected tool list')
def terminate(signum, _frame):
# The outer resource scope terminates descendants together. Ensure the
# runner's copied credentials are also removed on a resource abort.
@ -130,11 +145,14 @@ def main():
'possible and select the requested normalized_result fields before returning '
'intermediate output to model context. Keep real Evidence IDs and limitations.'
)
enabled_tools = None
if args.enabled_tools:
enabled_tools = json.loads(args.enabled_tools.read_text())
if not isinstance(enabled_tools, list) or not all(isinstance(name, str) and name for name in enabled_tools) or len(enabled_tools) != len(set(enabled_tools)):
raise ValueError('enabled-tools must be a unique list of exact tool names')
if args.guide_enabled_tools:
instruction += (
' Configured REA tool function names for this trial: '
+ json.dumps(['mcp__rea__' + name for name in enabled_tools])
+ '. For discovery select only these exact names from ALL_TOOLS and '
'return only their metadata. Preserve the original analyst question, '
'dependency order and source Evidence.'
)
quote = json.dumps
config = '\n'.join([
'approval_policy = "never"', 'sandbox_mode = "read-only"',
@ -183,6 +201,8 @@ def main():
(args.output / 'prompt.txt').write_text(prompt)
(args.output / 'trial-settings.json').write_text(json.dumps({
'mode': args.mode, 'model': args.model, 'reasoning': args.reasoning,
'capture_tool_context': args.capture_tool_context,
'guide_enabled_tools': args.guide_enabled_tools,
'driver_commit': subprocess.check_output(
['git', 'rev-parse', 'HEAD'], cwd=driver_repository, text=True).strip(),
'runtime_revision': args.revision,
@ -196,7 +216,7 @@ def main():
'package_lock_sha256': file_digest(repository / 'package-lock.json'),
'experiment_sources_sha256': {
name: hashlib.sha256((experiment / name).read_bytes()).hexdigest()
for name in ['agent-trial.py', 'provider-environment.json']
for name in ['agent-trial.py', 'codex_context.py', 'provider-environment.json']
},
'developer_instructions': instruction,
'developer_instructions_sha256': hashlib.sha256(instruction.encode()).hexdigest(),
@ -212,7 +232,8 @@ def main():
'multi_agent': False, 'code_mode': args.mode == 'client-code',
'code_mode_only': args.mode == 'client-code'},
}, indent=2) + '\n')
invocation = [args.codex, '--no-daemon', 'exec', '--strict-config', '--ephemeral',
invocation = [args.codex, '--no-daemon', 'exec', '--strict-config',
*([] if args.capture_tool_context else ['--ephemeral']),
'--skip-git-repo-check', '--ignore-rules', '--json',
'--color', 'never', '-C', str(workspace),
'--output-last-message', str(args.output.resolve() / 'answer.txt'), '-']
@ -234,26 +255,28 @@ def main():
usage = []
tools = []
event_types = {}
for line in (args.output / 'events.jsonl').open():
if not line.strip():
continue
event = json.loads(line)
kind = event.get('type', 'unknown')
event_types[kind] = event_types.get(kind, 0) + 1
if 'usage' in event:
usage.append(event['usage'])
item = event.get('item', {})
if item.get('type') == 'mcp_tool_call' and kind == 'item.completed':
tools.append({
**{k: item.get(k) for k in ['server', 'tool', 'arguments', 'status', 'error']},
'codex_event_result_json_utf8_bytes': len(json.dumps(
item.get('result'), separators=(',', ':'), ensure_ascii=False
).encode()),
})
with (args.output / 'events.jsonl').open() as event_file:
for line in event_file:
if not line.strip():
continue
event = json.loads(line)
kind = event.get('type', 'unknown')
event_types[kind] = event_types.get(kind, 0) + 1
if 'usage' in event:
usage.append(event['usage'])
item = event.get('item', {})
if item.get('type') == 'mcp_tool_call' and kind == 'item.completed':
tools.append({
**{k: item.get(k) for k in ['server', 'tool', 'arguments', 'status', 'error']},
'codex_event_result_json_utf8_bytes': len(json.dumps(
item.get('result'), separators=(',', ':'), ensure_ascii=False
).encode()),
})
final_target_digest = file_digest(Path(args.artifact_path)) if args.artifact_path else None
summary = {
'mode': args.mode, 'requested_model': args.model, 'reasoning': args.reasoning,
'runtime_revision': args.revision,
'capture_tool_context_requested': args.capture_tool_context,
'selected_artifact_sha256_before': target_digest,
'selected_artifact_sha256_after': final_target_digest,
'selected_artifact_bytes_unchanged': target_digest == final_target_digest if target_digest else None,
@ -268,9 +291,13 @@ def main():
(args.output / 'summary.json').write_text(json.dumps(summary, indent=2) + '\n')
print(json.dumps(summary))
finally:
# Always remove copied credentials, config, caches and profile state.
shutil.rmtree(profile)
(args.output / 'cleanup.json').write_text(json.dumps({'profile_removed': not profile.exists()}) + '\n')
try:
if args.capture_tool_context:
capture_tool_context(trial_codex_home, args.output)
finally:
# Capture failure must never prevent copied-credential cleanup.
shutil.rmtree(profile)
(args.output / 'cleanup.json').write_text(json.dumps({'profile_removed': not profile.exists()}) + '\n')
raise SystemExit(124 if timeout else exit_code)

117
experiments/analysis-efficiency/codex_context.py Normal file
View File

@ -0,0 +1,117 @@
"""Retain owned Codex tool/usage observations without account or ambient state."""
import hashlib
import json
import os
def token_counts(value):
if not isinstance(value, dict):
raise ValueError('Expected a token-count object')
result = {}
for key in ['input_tokens', 'cached_input_tokens', 'cache_write_input_tokens',
'output_tokens', 'reasoning_output_tokens', 'total_tokens']:
if key in value:
count = value[key]
if not isinstance(count, int) or isinstance(count, bool) or count < 0:
raise ValueError('Expected nonnegative integer token counts')
result[key] = count
return result
def selected_event(event):
"""Select observed items; these are not the exact submitted model request."""
if not isinstance(event, dict) or not isinstance(event.get('payload'), dict):
raise ValueError('Expected an event object with an object payload')
if not isinstance(event.get('type'), str):
raise ValueError('Expected an event type string')
if event.get('timestamp') is not None and not isinstance(event['timestamp'], str):
raise ValueError('Expected a timestamp string or omission')
payload = event['payload']
kind = event.get('type')
if kind == 'response_item' and payload.get('type') in [
'custom_tool_call', 'custom_tool_call_output',
'function_call', 'function_call_output',
]:
keys = ['type', 'call_id', 'name', 'input', 'arguments', 'output']
elif kind == 'token_usage_record':
keys = ['response_id', 'usage', 'turn_token_usage', 'thread_token_usage']
payload = {key: token_counts(payload[key]) if key != 'response_id'
else payload[key] for key in keys if key in payload}
elif kind == 'event_msg' and payload.get('type') == 'token_count':
keys = ['type', 'info']
info = payload.get('info')
if info is not None:
if not isinstance(info, dict):
raise ValueError('Expected token-count info')
selected_info = {key: token_counts(info[key]) for key in
['total_token_usage', 'last_token_usage'] if key in info}
window = info.get('model_context_window')
if 'model_context_window' in info:
if window is not None and (not isinstance(window, int) or isinstance(window, bool) or window <= 0):
raise ValueError('Expected a positive model context window')
selected_info['model_context_window'] = window
payload = {'type': 'token_count', 'info': selected_info}
else:
payload = {'type': 'token_count', 'info': None}
else:
return None
return {
'timestamp': event.get('timestamp'),
'type': kind,
'payload': {key: payload[key] for key in keys if key in payload},
}
def capture_tool_context(codex_home, output):
"""Capture only an explicitly selected trial's rollouts; never its auth/config."""
sessions = codex_home / 'sessions'
blocked = sessions.is_symlink()
sources = [] if blocked else sorted(sessions.rglob('*.jsonl'))
target = output / 'codex-tool-context.jsonl'
source_receipts = []
errors = [{'source': 'sessions', 'reason': 'not_an_owned_directory'}] if blocked else []
retained = 0
with os.fdopen(os.open(target, os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o600),
'w', encoding='utf-8') as result:
for source in sources:
name = source.relative_to(codex_home).as_posix()
if source.is_symlink() or not source.is_file():
errors.append({'source': name, 'reason': 'not_an_owned_regular_file'})
continue
digest = hashlib.sha256()
counts = {}
with source.open('rb') as lines:
for ordinal, line in enumerate(lines, 1):
digest.update(line)
try:
event = json.loads(line)
selected = selected_event(event)
except ValueError as error:
errors.append({'source': name, 'line': ordinal,
'reason': type(error).__name__})
continue
kind = event.get('type')
counts[str(kind)] = counts.get(str(kind), 0) + 1
if selected is not None:
result.write(json.dumps(selected, ensure_ascii=False) + '\n')
retained += 1
source_receipts.append({'source': name, 'sha256': digest.hexdigest(),
'event_counts': counts})
retained_digest = hashlib.sha256()
with target.open('rb') as captured:
for chunk in iter(lambda: captured.read(1024 * 1024), b''):
retained_digest.update(chunk)
receipt = {
'status': 'partial' if errors else 'captured' if source_receipts else 'unavailable',
'retained_file': target.name,
'retained_sha256': retained_digest.hexdigest(),
'retained_events': retained, 'sources': source_receipts, 'errors': errors,
'scope': 'Owned client tool-call/reply and per-response usage items, not '
'the exact submitted model request. Session/account metadata, '
'world state, user/developer messages, encrypted reasoning and '
'internal passthrough are omitted. Authentication, configuration '
'and ambient environment are never copied by this capture.',
}
(output / 'context-receipt.json').write_text(json.dumps(receipt, indent=2) + '\n')
return receipt

209
experiments/analysis-efficiency/model-context.md Normal file
View File

@ -0,0 +1,209 @@
# Observe the client boundary before claiming token savings
Continues [#2016](https://github.com/morluto/rea/issues/2016) after the managed
summary [PR #2413](https://github.com/morluto/rea/pull/2413) merged as
`82cfcee19f18b0eeae9a330ba04e0411cdf85225`. That opt-in view has verified smaller
delivery and complete-default migration, while its original real-model cohort
does not establish token savings. This follow-up measures why raw reply bytes
and aggregate model usage cannot be treated as interchangeable observations.
## Concrete diagnostic observation
An individual real Codex CLI 0.161.0 control uses frozen runtime
`ae8051302b5834322fa00cfa31c968ae74ab6ed9`, the unchanged ShareX identity question,
requested `gpt-5.6-terra` high and the same two enabled REA tools. A private
diagnostic copy removes the original runner's explicit `--ephemeral` flag.
Authentication/profile cleanup still runs; only selected tool/usage items are
retained after filtering the owned rollout. Its source rollout SHA-256 is
`d4df41d6187880d177f00b7533d85ad032c9c4ec502a39e7d5a566180e10461b` and the retained
filtered-item SHA-256 is
`0d5b4fb67c5674d1ba14ac32ae37e9a72b0368f4d9f5f9f81eb4d13193cda260`.
The original whole-copy is removed; account metadata, world state, messages and
internal passthrough are not retained in the filtered receipt.
Despite the requested `direct` configuration label, the observed client uses
three JavaScript `exec` calls. First it filters `ALL_TOOLS` with a broad
`/REA|reverse|artifact|managed|assembly|metadata/i` expression and prints matching
metadata. The client reports truncation of a 164,005-token original output.
This is the client's output measurement, not billed input or a complete inventory.
Next it invokes complete managed inspection but selects requested normalized
identity/classification fields before printing. Finally it invokes native-boundary
inspection and prints selected fields. The actual original complete MCP replies
remain available in the separate public-event transcript.
The persisted custom-tool output values are 41,704 / 2,154 / 13,489 UTF-8 bytes.
These are the serialized recorded output **values**, not raw wire bytes or exact
submitted model requests. The trace reports four per-response usage observations:
| Response | Input | Cached input | Output |
| -------- | -----: | -----------: | -----: |
| 1 | 10,400 | 0 | 132 |
| 2 | 19,992 | 9,984 | 456 |
| 3 | 21,242 | 19,200 | 251 |
| 4 | 24,799 | 20,224 | 1,535 |
Their totals match public turn usage: 76,433 input, 49,408 cached, 27,025 uncached
and 2,374 output, with 61.436 seconds observed wall time. Two MCP calls alone do
not describe the client's discovery/execution rounds. This proves the observed
control can project a complete result before printing it; it does not establish
that every earlier control/candidate did the same, or attribute every token
difference to discovery. The earlier `direct` label specifies requested client
configuration, not a verified absence of an executor.
Initial transcript-capture attempts retained empty source lists because the
original runner intentionally passed `--ephemeral`. A first private flag-removal
edit missed its newline and failed; that candidate remains an unsuccessful
capture attempt. The later corrected control is an individual diagnostic,
not a comparable pair with that candidate. Other exploratory aggregates remain
private and are not silently folded into the original four-trial cohort.
## Reproducible optional capture
`agent-trial.py --capture-tool-context` omits `--ephemeral` only for that explicitly
selected temporary trial. Its default is unchanged. The profile still disappears
in `finally`, including when capture fails. The capture reads only its owned
`sessions/*.jsonl` records, rejects symlink sources, and retains:
- Custom/function tool calls and replies, preserving call IDs and exact values.
- Per-response token counts and cumulative counts, with original response IDs.
- Client token-count events, retaining explicit unknown `info: null`.
It excludes session/account metadata, world state, user/developer messages,
encrypted reasoning, internal passthrough and unknown nested usage fields. It
does not copy authentication, configuration or the ambient environment. Receipts
record source/retained digests, event counts, malformed-line failures and missing
rollouts as unavailable. Capture is an observation of persisted client items,
not proof of the exact request accepted by a remote model service.
Run the same frozen runtime, question, tool selection and capture mode on both
sides. Preserve source/settings hashes and separate requested versus observed
behavior. Use the existing resource wrapper with unchanged one CPU, 4 GiB hard
memory, 3 GiB early-stop, no swap and task/host safeguards. Final physical counters
remain unknown when transient cgroups disappear; this instrumentation does not
repair that measurement limitation.
The focused Python tests run through the existing CI `test:python` gate.
They cover account/ambient exclusion while preserving caller data, malformed and
missing records, foreign symlinks, and capture failure without copied-credential
leakage. The cleanup test simulates a CLI and establishes only cleanup semantics;
it is not real-model/provider evidence.
## An explicit discovery-guidance experiment
`--guide-enabled-tools` is another opt-in experiment flag. It requires a nonempty
`--enabled-tools` list and adds the exact corresponding `mcp__rea__...` function
names to the temporary developer instructions, requesting only exact-name
metadata discovery. It preserves the analyst question, operation order, tool
selection and provider inputs. The default instruction is unchanged. Missing or
invalid selected tools fail before creating an output/profile. The test gate
also verifies the new prerequisite without acquiring a profile.
Compare guided and unguided runs on the **same** frozen REA runtime with matching
capture mode and requested facts. A smaller discovery response or additional
hint is not itself a task/token success: grade answers, count all response usage,
preserve failed/truncated runs and repeat both orders before recommending this
guidance. This is a caller-side experiment, not a new REA tool or provider preset.
## First committed capture pair
Checkpoint `9ebdcc458c399b9bdecaaca2ef47422888231181` runs the unchanged question
against frozen control `ae8051302b5834322fa00cfa31c968ae74ab6ed9` and candidate
`8bc1601ca51e417ca472d90ea0367e9418a1a0f2`, both with capture enabled and no tool
guidance. Both captures succeed with 10 retained events, unchanged source bytes,
removed profiles, two successful MCP calls and passing selected factual checks.
Full grader gates remain false because final physical resource counters are
unavailable; factual subsets do not override that limitation.
| Side | Input | Cached | Uncached | Output | Wall seconds |
| ----------------- | -----: | -----: | -------: | -----: | -----------: |
| Complete control | 52,656 | 27,136 | 25,520 | 2,015 | 57.736 |
| Summary candidate | 56,655 | 29,184 | 27,471 | 1,534 | 55.055 |
Both clients make two executor calls and three response-usage observations.
First they print broad discovered metadata, with truncation. Then each executor
sequentially performs both MCP operations, preserving selected-target dependency
order. The control prints a 2,086-byte output value; the candidate prints 22,770
bytes. Their discovery output values are 41,723 and 41,675 bytes. These are
recorded serialized values and remain distinct from raw MCP envelopes, billed
tokens and exact submitted requests. Both requested modes are `direct`; the
actual executor use is separately observed.
The candidate's smaller raw metadata reply does not result in a smaller printed
client value in this pair. Candidate input/uncached tokens remain higher. The
observed projection and round counts are useful constraints for the next
experiment; this one pair does not attribute all usage differences to either
formatting or discovery, or establish a reliable speedup.
## Same-runtime guided discovery, repeated in both orders
Checkpoint `e20bd0e2a905f06bc8c56fe04b0e61f3da88636a` runs four actual Codex trials
in unguided/guided/guided/unguided order. All use the **same** frozen candidate
runtime, artifact, original question, requested model/reasoning, two selected
tools, provider settings and capture mode. Only the explicit discovery-guidance
treatment changes. Every trial selects artifact summary and makes the same two
successful MCP calls in dependency order; source bytes stay unchanged, captures
succeed and profiles are removed.
| Trial | Input | Cached | Uncached | Output | Seconds | Observed responses |
| ---------- | ------: | -----: | -------: | -----: | ------: | -----------------: |
| Unguided 1 | 77,251 | 50,432 | 26,819 | 1,931 | 62.066 | 4 |
| Guided 1 | 39,175 | 23,040 | 16,135 | 1,763 | 55.658 | 3 |
| Guided 2 | 58,336 | 38,144 | 20,192 | 1,626 | 50.685 | 4 |
| Unguided 2 | 103,301 | 72,704 | 30,597 | 1,983 | 60.336 | 5 |
Both guided discovery output values repeat at 12,336 bytes without truncation,
versus 41,709 bytes with truncation on both unguided trials. The recorded
executor/output usage explains observable discovery and round differences; it
does not certify exact remote request contents or attribute every token to the
hint. CPU samples are 6.303–6.554 seconds and peak family memory 684,519,424–
712,028,160 bytes, with no guard abort. Final resource counters remain unknown.
These samples do not establish a memory or CPU improvement.
Unguided 1 omits `classification.container` and fails the selected factual gate;
it is not a fully acceptable comparison answer. Guided 1 and Unguided 2 pass
that gate. Guided 2 initially fails exact coverage-object equality because it
adds `module_ref_count: 0` and `pinvoke_import_count: 0` to the explicitly named
native scope. Both numbers exactly match the producer's returned arrays. The
grader now explicitly verifies only these two known additional counts, including
integer type, while still requiring all original coverage facts. Unknown extra
claims, conflicting aliases, wrong counts, boolean/string counts and omitted
issues continue to fail. Source coverage remains unchanged; returned-row counts
do not establish global absence. Original failure receipts remain preserved;
the regrade separately records its new source hash. Guided 2 then passes selected
facts, but every **full** gate still fails unknown final physical resource checks.
The second comparison has two factually acceptable answers: observed guided input
falls about 43.5%, uncached input about 34.0%, and wall time about 16.0%. The first
comparison has an incomplete unguided answer and is reported without counting it
as equivalent completed work. Two trials per treatment on one question/application
remain a preliminary signal, not a general default recommendation. The hint stays
opt-in; wider questions/domains, count/coverage representation and client round
behavior need further verification before promoting it into general guidance.
## Real Android follow-up: the primary outcomes are mixed
Checkpoint `22834c7d73056e2480b426848f58c31af0c307bc` checks the pinned real
NewPipe 0.29.1 APK using actual configured JADX 0.7.1/Java, the original Android
question and the same frozen candidate runtime. First a fresh SDK workflow
validates three live advertised operations and produces the comparison facts:
package/version identity, every case-sensitive MainActivity match and all
reported onCreate entries, preserving signatures, overload indices and null DEX
descriptors. It takes 26.961 seconds; the owned scope samples 1,234,735,104 bytes.
Both subsequent actual Codex trials use the same three selected tool names,
capture mode, model/reasoning, question and provider settings. Only discovery
guidance changes. Each makes three successful MCP calls and five observed model
responses, passes selected facts against the fresh actual producer, preserves
source bytes and removes its temporary profile. No APK execution is requested.
| Trial | Input | Cached | Uncached | Output | Wall seconds | Sampled peak bytes |
| -------- | -----: | -----: | -------: | -----: | -----------: | -----------------: |
| Unguided | 99,913 | 80,640 | 19,273 | 1,527 | 66.797 | 1,405,829,120 |
| Guided | 89,220 | 62,464 | 26,756 | 2,142 | 76.936 | 1,345,576,960 |
Total input falls about 10.7%, but uncached input increases and wall time is
longer. One pair does not demonstrate improved Android performance or memory.
No guard abort occurs; final counters remain unknown and full resource gates
remain false. This second domain is a concrete counterexample to generalizing
the favorable managed comparison. Keep both flags optional and continue
task-specific experiments before changing installed/default agent guidance.

23
experiments/analysis-efficiency/review-model.py
View File

@ -52,6 +52,21 @@ def named_scope(value, names):
return {'conflicting_reported_scopes': selected}
def scope_with_verified_counts(value, expected_scope, returned_counts):
"""Validate explicit extra row counts; never erase unknown or conflicting facts."""
if not isinstance(value, dict):
return value, {}
scope = dict(value)
checks = {}
for name, expected in returned_counts.items():
if name in scope:
reported = scope[name]
checks[name] = type(reported) is int and reported == expected
if name not in expected_scope:
del scope[name]
return scope, checks
def main():
parser = argparse.ArgumentParser()
parser.add_argument('--domain', choices=['wasm', 'evm', 'android', 'managed', 'managed-nonempty', 'firmware', 'inventory', 'javascript', 'native', 'browser'], required=True)
@ -134,7 +149,13 @@ def main():
for key in ['module_refs', 'pinvoke_imports']:
check(key, answer.get('native_boundaries', {}).get(key), reference['boundaries'][key])
check('artifact_coverage', named_scope(answer.get('coverage'), ['artifact', 'managed_artifact', 'managed_metadata']), n['coverage'])
check('native_boundaries_coverage', named_scope(answer.get('coverage'), ['native_boundaries', 'native_boundary_declarations']), reference['boundaries']['coverage'])
boundary_scope, counts = scope_with_verified_counts(
named_scope(answer.get('coverage'), ['native_boundaries', 'native_boundary_declarations']),
reference['boundaries']['coverage'],
{'module_ref_count': len(reference['boundaries']['module_refs']),
'pinvoke_import_count': len(reference['boundaries']['pinvoke_imports'])})
check('native_boundaries_coverage', boundary_scope, reference['boundaries']['coverage'])
checks.update({'native_coverage_' + name: valid for name, valid in counts.items()})
elif domain == 'managed-nonempty':
n = reference['boundaries']
check('module_mvid', answer.get('module_mvid'), n['module']['mvid'])

175
experiments/analysis-efficiency/test_codex_context.py Normal file
View File

@ -0,0 +1,175 @@
"""Distinct capture/cleanup failures; simulated CLI is not real-model evidence."""
import importlib.util
import io
import json
import os
import runpy
import sys
from pathlib import Path
import tempfile
import unittest
from unittest.mock import Mock, patch
from codex_context import capture_tool_context, selected_event
class ManagedCoverageTests(unittest.TestCase):
def test_verifies_extra_counts_and_preserves_unknown_fields(self):
review = runpy.run_path(str(Path(__file__).with_name('review-model.py')))
project = review['scope_with_verified_counts']
expected = {'state': 'partial', 'issues': ['unreturned rows remain unknown']}
original = {**expected, 'module_ref_count': 0, 'pinvoke_import_count': 2}
scope, checks = project(original, expected, {'module_ref_count': 0, 'pinvoke_import_count': 2})
self.assertEqual(scope, expected)
self.assertTrue(all(checks.values()))
self.assertEqual(original['pinvoke_import_count'], 2)
for count in [3, '2', True]:
scope, checks = project({**expected, 'pinvoke_import_count': count}, expected, {'pinvoke_import_count': 2})
self.assertFalse(checks['pinvoke_import_count'])
_, checks = project({**expected, 'module_ref_count': False}, {**expected, 'module_ref_count': 0}, {'module_ref_count': 0})
self.assertFalse(checks['module_ref_count'])
scope, _ = project({**expected, 'unsupported_claim': True}, expected, {})
self.assertNotEqual(scope, expected)
scope, _ = project({'state': 'partial', 'module_ref_count': 0}, expected, {'module_ref_count': 0})
self.assertNotEqual(scope, expected)
ambiguous = review['named_scope']({'native_boundaries': expected, 'native_boundary_declarations': {'state': 'complete'}}, ['native_boundaries', 'native_boundary_declarations'])
scope, _ = project(ambiguous, expected, {'module_ref_count': 0})
self.assertNotEqual(scope, expected)
class ContextCaptureTests(unittest.TestCase):
def test_filters_ambient_account_and_nested_usage_state(self):
with tempfile.TemporaryDirectory() as directory:
root = Path(directory)
home = root / 'home'
sessions = home / 'sessions'
sessions.mkdir(parents=True)
output = root / 'output'
output.mkdir()
events = [
{'type': 'world_state', 'payload': {'state': {'env': 'ambient-marker'}}},
{'type': 'session_meta', 'payload': {'creator_account_id': 'account-marker'}},
{'type': 'response_item', 'payload': {
'type': 'custom_tool_call_output', 'call_id': 'original-call',
'output': 'caller-selected evidence and paths',
'internal_chat_message_metadata_passthrough': 'private-marker'}},
{'type': 'token_usage_record', 'payload': {
'response_id': 'original-response', 'usage': {
'input_tokens': 12, 'cached_input_tokens': 4,
'environment': 'nested-ambient-marker'},
'transport_auth': 'auth-marker'}},
]
(sessions / 'rollout.jsonl').write_text(''.join(json.dumps(x) + '\n' for x in events))
(home / 'auth.json').write_text('copied-credential-marker')
receipt = capture_tool_context(home, output)
text = (output / 'codex-tool-context.jsonl').read_text()
self.assertEqual(receipt['status'], 'captured')
self.assertEqual(receipt['retained_events'], 2)
for marker in ['ambient-marker', 'account-marker', 'private-marker',
'auth-marker', 'copied-credential-marker']:
self.assertNotIn(marker, text)
self.assertIn('caller-selected evidence and paths', text)
self.assertIn('original-response', text)
self.assertEqual((output / 'codex-tool-context.jsonl').stat().st_mode & 0o777, 0o600)
def test_missing_rollouts_are_unknown_and_malformed_lines_remain_partial(self):
with tempfile.TemporaryDirectory() as directory:
root = Path(directory)
output = root / 'missing'
output.mkdir()
self.assertEqual(capture_tool_context(root / 'home', output)['status'], 'unavailable')
sessions = root / 'home' / 'sessions'
sessions.mkdir(parents=True)
(sessions / 'rollout.jsonl').write_text('not-json\n' + json.dumps({
'type': 'event_msg', 'payload': {'type': 'token_count', 'info': None}}) + '\n')
partial = root / 'partial'
partial.mkdir()
receipt = capture_tool_context(root / 'home', partial)
self.assertEqual(receipt['status'], 'partial')
self.assertEqual(receipt['errors'][0]['line'], 1)
self.assertEqual(receipt['retained_events'], 1)
with self.assertRaises(ValueError):
selected_event({'type': 'token_usage_record', 'payload': {'usage': {'input_tokens': True}}})
with self.assertRaises(ValueError):
selected_event({'type': {'environment': 'unexpected'}, 'payload': {}})
with self.assertRaises(ValueError):
selected_event({'type': 'response_item', 'timestamp': {'environment': 'unexpected'}, 'payload': {}})
unknown = selected_event({'type': 'event_msg', 'payload': {'type': 'token_count', 'info': {'model_context_window': None}}})
self.assertIsNone(unknown['payload']['info']['model_context_window'])
def test_capture_failure_still_removes_trial_credentials_and_profile(self):
spec = importlib.util.spec_from_file_location('agent_trial', Path(__file__).with_name('agent-trial.py'))
agent = importlib.util.module_from_spec(spec)
spec.loader.exec_module(agent)
with tempfile.TemporaryDirectory() as directory:
root = Path(directory)
runtime = root / 'runtime'
(runtime / 'scripts').mkdir(parents=True)
(runtime / 'scripts' / 'rea.mjs').write_text('')
(runtime / 'package-lock.json').write_text('{}')
auth_home = root / 'source-auth'
auth_home.mkdir()
(auth_home / 'auth.json').write_text('synthetic-credential')
question = root / 'question.txt'
question.write_text('Observe the declared fixture.')
output = root / 'trial'
child = Mock(stdin=io.StringIO())
child.wait.return_value = 0
arguments = ['agent-trial.py', '--runtime-root', str(runtime), '--revision', 'fixture',
'--prompt', str(question), '--output', str(output), '--capture-tool-context']
with patch.dict(os.environ, {'CODEX_HOME': str(auth_home)}, clear=True), \
patch.object(sys, 'argv', arguments), \
patch.object(agent.shutil, 'which', return_value='/fixture/node'), \
patch.object(agent.subprocess, 'check_output', return_value='v24.18.0\n'), \
patch.object(agent.subprocess, 'Popen', return_value=child), \
patch.object(agent, 'capture_tool_context', side_effect=OSError('fixture capture failure')), \
patch('builtins.print'):
with self.assertRaisesRegex(OSError, 'fixture capture failure'):
agent.main()
self.assertFalse(list(output.glob('profile-*')))
self.assertTrue(json.loads((output / 'cleanup.json').read_text())['profile_removed'])
self.assertEqual((auth_home / 'auth.json').read_text(), 'synthetic-credential')
def test_refuses_foreign_session_directory_and_symlink_records(self):
with tempfile.TemporaryDirectory() as directory:
root = Path(directory)
home = root / 'home'
home.mkdir()
foreign = root / 'foreign'
foreign.mkdir()
(foreign / 'rollout.jsonl').write_text('private marker')
(home / 'sessions').symlink_to(foreign, target_is_directory=True)
output = root / 'directory-result'
output.mkdir()
receipt = capture_tool_context(home, output)
self.assertEqual(receipt['status'], 'partial')
self.assertEqual(receipt['retained_events'], 0)
(home / 'sessions').unlink()
(home / 'sessions').mkdir()
(home / 'sessions' / 'linked.jsonl').symlink_to(foreign / 'rollout.jsonl')
output = root / 'file-result'
output.mkdir()
receipt = capture_tool_context(home, output)
self.assertEqual(receipt['status'], 'partial')
self.assertEqual(receipt['errors'][0]['reason'], 'not_an_owned_regular_file')
self.assertEqual((foreign / 'rollout.jsonl').read_text(), 'private marker')
def test_guidance_requires_selected_tools_before_acquiring_a_profile(self):
spec = importlib.util.spec_from_file_location('agent_trial', Path(__file__).with_name('agent-trial.py'))
agent = importlib.util.module_from_spec(spec)
spec.loader.exec_module(agent)
with tempfile.TemporaryDirectory() as directory:
root = Path(directory)
output = root / 'trial'
arguments = ['agent-trial.py', '--runtime-root', str(root), '--revision', 'fixture',
'--prompt', str(root / 'question'), '--output', str(output), '--guide-enabled-tools']
with patch.object(sys, 'argv', arguments), patch.object(sys, 'stderr', io.StringIO()):
with self.assertRaises(SystemExit) as raised:
agent.main()
self.assertEqual(raised.exception.code, 2)
self.assertFalse(output.exists())
if __name__ == '__main__':
unittest.main()

2
package.json
View File

@ -130,7 +130,7 @@
"test": "npm run test:with-build",
"test:with-build": "npm run test:prepare && vitest run",
"test:run": "vitest run",
"test:python": "python3 -m unittest bridge/native/test_rea_lldb_tracer.py -v",
"test:python": "python3 -m unittest bridge/native/test_rea_lldb_tracer.py -v && python3 -m unittest discover -s experiments/analysis-efficiency -p test_codex_context.py -v",
"test:local": "node scripts/run-development-tests.mjs local",
"test:fast": "vitest run --project domain --project services --project adapters --project composition --project conformance --project evaluation",
"pretest:boundary": "npm run test:prepare",