From ee87111009ace2ca4ef82c9aef09c82a37ca55f7 Mon Sep 17 00:00:00 2001 From: Ysopwnable <161784452+N0zoM1z0@users.noreply.github.com> Date: Tue, 13 Oct 2026 09:01:17 +0800 Subject: [PATCH] test(perf): distinguish Codex tool context and token accounting (#2416) Add optional owned Codex tool/usage capture and exact-name discovery experiments while preserving default ephemeral trials and all REA public APIs. Retain scoped managed and actual Android results, mixed outcomes, source identities and resource unknowns. Six semantic Python tests, full checked pre-push, byte-identical own-file rebase verification, manual review and all 39 exact-head CI checks pass. Historical main fixture failures and their independent repair remain documented; measured source/runtime pins are unchanged. Co-authored-by: factory-droid[bot] <138933559+factory-droid[bot]@users.noreply.github.com> --- experiments/analysis-efficiency/README.md | 5 + .../analysis-efficiency/agent-trial.py | 79 ++++--- .../analysis-efficiency/codex_context.py | 117 ++++++++++ .../analysis-efficiency/model-context.md | 209 ++++++++++++++++++ .../analysis-efficiency/review-model.py | 23 +- .../analysis-efficiency/test_codex_context.py | 175 +++++++++++++++ package.json | 2 +- 7 files changed, 582 insertions(+), 28 deletions(-) create mode 100644 experiments/analysis-efficiency/codex_context.py create mode 100644 experiments/analysis-efficiency/model-context.md create mode 100644 experiments/analysis-efficiency/test_codex_context.py diff --git a/experiments/analysis-efficiency/README.md b/experiments/analysis-efficiency/README.md index e41e6b7df..fb645b1e0 100644 --- a/experiments/analysis-efficiency/README.md +++ b/experiments/analysis-efficiency/README.md @@ -32,6 +32,11 @@ records two shipped assembly checks, previous-runtime portable Evidence, and fou actual Codex trials. Selected replies shrink substantially; observed model input tokens increase and latency is mixed, so model-token savings are not claimed. +Follow-up: [model tool context and per-response usage](model-context.md) separates +raw MCP replies from observed client execution/discovery and usage. Optional +trial capture retains tool/usage items while excluding account and ambient state; +existing ephemeral trials keep their default behavior. + The 12 detailed checkpoint receipts were archived byte-for-byte before removal from the final PR tree. The archive `review-receipts-00d51e0fc-20261012` is in the private experiment lab outside the checkout; its `archive-index.json` records diff --git a/experiments/analysis-efficiency/agent-trial.py b/experiments/analysis-efficiency/agent-trial.py index 8392000ca..682be27ec 100644 --- a/experiments/analysis-efficiency/agent-trial.py +++ b/experiments/analysis-efficiency/agent-trial.py @@ -11,6 +11,8 @@ import subprocess import tempfile import time +from codex_context import capture_tool_context + def file_digest(path): digest = hashlib.sha256() @@ -48,7 +50,20 @@ def main(): parser.add_argument('--providers', type=Path) parser.add_argument('--enabled-tools', type=Path) parser.add_argument('--authority', choices=['static', 'browser'], default='static') + parser.add_argument('--capture-tool-context', action='store_true', + help='Retain owned tool/usage items; excludes account/ambient state') + parser.add_argument('--guide-enabled-tools', action='store_true', + help='Opt-in exact-name discovery guidance for selected REA tools') args = parser.parse_args() + if args.guide_enabled_tools and args.enabled_tools is None: + parser.error('--guide-enabled-tools requires --enabled-tools') + enabled_tools = None + if args.enabled_tools: + enabled_tools = json.loads(args.enabled_tools.read_text()) + if not isinstance(enabled_tools, list) or not all(isinstance(name, str) and name for name in enabled_tools) or len(enabled_tools) != len(set(enabled_tools)): + raise ValueError('enabled-tools must be a unique list of exact tool names') + if args.guide_enabled_tools and not enabled_tools: + parser.error('--guide-enabled-tools requires a nonempty selected tool list') def terminate(signum, _frame): # The outer resource scope terminates descendants together. Ensure the # runner's copied credentials are also removed on a resource abort. @@ -130,11 +145,14 @@ def main(): 'possible and select the requested normalized_result fields before returning ' 'intermediate output to model context. Keep real Evidence IDs and limitations.' ) - enabled_tools = None - if args.enabled_tools: - enabled_tools = json.loads(args.enabled_tools.read_text()) - if not isinstance(enabled_tools, list) or not all(isinstance(name, str) and name for name in enabled_tools) or len(enabled_tools) != len(set(enabled_tools)): - raise ValueError('enabled-tools must be a unique list of exact tool names') + if args.guide_enabled_tools: + instruction += ( + ' Configured REA tool function names for this trial: ' + + json.dumps(['mcp__rea__' + name for name in enabled_tools]) + + '. For discovery select only these exact names from ALL_TOOLS and ' + 'return only their metadata. Preserve the original analyst question, ' + 'dependency order and source Evidence.' + ) quote = json.dumps config = '\n'.join([ 'approval_policy = "never"', 'sandbox_mode = "read-only"', @@ -183,6 +201,8 @@ def main(): (args.output / 'prompt.txt').write_text(prompt) (args.output / 'trial-settings.json').write_text(json.dumps({ 'mode': args.mode, 'model': args.model, 'reasoning': args.reasoning, + 'capture_tool_context': args.capture_tool_context, + 'guide_enabled_tools': args.guide_enabled_tools, 'driver_commit': subprocess.check_output( ['git', 'rev-parse', 'HEAD'], cwd=driver_repository, text=True).strip(), 'runtime_revision': args.revision, @@ -196,7 +216,7 @@ def main(): 'package_lock_sha256': file_digest(repository / 'package-lock.json'), 'experiment_sources_sha256': { name: hashlib.sha256((experiment / name).read_bytes()).hexdigest() - for name in ['agent-trial.py', 'provider-environment.json'] + for name in ['agent-trial.py', 'codex_context.py', 'provider-environment.json'] }, 'developer_instructions': instruction, 'developer_instructions_sha256': hashlib.sha256(instruction.encode()).hexdigest(), @@ -212,7 +232,8 @@ def main(): 'multi_agent': False, 'code_mode': args.mode == 'client-code', 'code_mode_only': args.mode == 'client-code'}, }, indent=2) + '\n') - invocation = [args.codex, '--no-daemon', 'exec', '--strict-config', '--ephemeral', + invocation = [args.codex, '--no-daemon', 'exec', '--strict-config', + *([] if args.capture_tool_context else ['--ephemeral']), '--skip-git-repo-check', '--ignore-rules', '--json', '--color', 'never', '-C', str(workspace), '--output-last-message', str(args.output.resolve() / 'answer.txt'), '-'] @@ -234,26 +255,28 @@ def main(): usage = [] tools = [] event_types = {} - for line in (args.output / 'events.jsonl').open(): - if not line.strip(): - continue - event = json.loads(line) - kind = event.get('type', 'unknown') - event_types[kind] = event_types.get(kind, 0) + 1 - if 'usage' in event: - usage.append(event['usage']) - item = event.get('item', {}) - if item.get('type') == 'mcp_tool_call' and kind == 'item.completed': - tools.append({ - **{k: item.get(k) for k in ['server', 'tool', 'arguments', 'status', 'error']}, - 'codex_event_result_json_utf8_bytes': len(json.dumps( - item.get('result'), separators=(',', ':'), ensure_ascii=False - ).encode()), - }) + with (args.output / 'events.jsonl').open() as event_file: + for line in event_file: + if not line.strip(): + continue + event = json.loads(line) + kind = event.get('type', 'unknown') + event_types[kind] = event_types.get(kind, 0) + 1 + if 'usage' in event: + usage.append(event['usage']) + item = event.get('item', {}) + if item.get('type') == 'mcp_tool_call' and kind == 'item.completed': + tools.append({ + **{k: item.get(k) for k in ['server', 'tool', 'arguments', 'status', 'error']}, + 'codex_event_result_json_utf8_bytes': len(json.dumps( + item.get('result'), separators=(',', ':'), ensure_ascii=False + ).encode()), + }) final_target_digest = file_digest(Path(args.artifact_path)) if args.artifact_path else None summary = { 'mode': args.mode, 'requested_model': args.model, 'reasoning': args.reasoning, 'runtime_revision': args.revision, + 'capture_tool_context_requested': args.capture_tool_context, 'selected_artifact_sha256_before': target_digest, 'selected_artifact_sha256_after': final_target_digest, 'selected_artifact_bytes_unchanged': target_digest == final_target_digest if target_digest else None, @@ -268,9 +291,13 @@ def main(): (args.output / 'summary.json').write_text(json.dumps(summary, indent=2) + '\n') print(json.dumps(summary)) finally: - # Always remove copied credentials, config, caches and profile state. - shutil.rmtree(profile) - (args.output / 'cleanup.json').write_text(json.dumps({'profile_removed': not profile.exists()}) + '\n') + try: + if args.capture_tool_context: + capture_tool_context(trial_codex_home, args.output) + finally: + # Capture failure must never prevent copied-credential cleanup. + shutil.rmtree(profile) + (args.output / 'cleanup.json').write_text(json.dumps({'profile_removed': not profile.exists()}) + '\n') raise SystemExit(124 if timeout else exit_code) diff --git a/experiments/analysis-efficiency/codex_context.py b/experiments/analysis-efficiency/codex_context.py new file mode 100644 index 000000000..cce8bf9cf --- /dev/null +++ b/experiments/analysis-efficiency/codex_context.py @@ -0,0 +1,117 @@ +"""Retain owned Codex tool/usage observations without account or ambient state.""" + +import hashlib +import json +import os + + +def token_counts(value): + if not isinstance(value, dict): + raise ValueError('Expected a token-count object') + result = {} + for key in ['input_tokens', 'cached_input_tokens', 'cache_write_input_tokens', + 'output_tokens', 'reasoning_output_tokens', 'total_tokens']: + if key in value: + count = value[key] + if not isinstance(count, int) or isinstance(count, bool) or count < 0: + raise ValueError('Expected nonnegative integer token counts') + result[key] = count + return result + + +def selected_event(event): + """Select observed items; these are not the exact submitted model request.""" + if not isinstance(event, dict) or not isinstance(event.get('payload'), dict): + raise ValueError('Expected an event object with an object payload') + if not isinstance(event.get('type'), str): + raise ValueError('Expected an event type string') + if event.get('timestamp') is not None and not isinstance(event['timestamp'], str): + raise ValueError('Expected a timestamp string or omission') + payload = event['payload'] + kind = event.get('type') + if kind == 'response_item' and payload.get('type') in [ + 'custom_tool_call', 'custom_tool_call_output', + 'function_call', 'function_call_output', + ]: + keys = ['type', 'call_id', 'name', 'input', 'arguments', 'output'] + elif kind == 'token_usage_record': + keys = ['response_id', 'usage', 'turn_token_usage', 'thread_token_usage'] + payload = {key: token_counts(payload[key]) if key != 'response_id' + else payload[key] for key in keys if key in payload} + elif kind == 'event_msg' and payload.get('type') == 'token_count': + keys = ['type', 'info'] + info = payload.get('info') + if info is not None: + if not isinstance(info, dict): + raise ValueError('Expected token-count info') + selected_info = {key: token_counts(info[key]) for key in + ['total_token_usage', 'last_token_usage'] if key in info} + window = info.get('model_context_window') + if 'model_context_window' in info: + if window is not None and (not isinstance(window, int) or isinstance(window, bool) or window <= 0): + raise ValueError('Expected a positive model context window') + selected_info['model_context_window'] = window + payload = {'type': 'token_count', 'info': selected_info} + else: + payload = {'type': 'token_count', 'info': None} + else: + return None + return { + 'timestamp': event.get('timestamp'), + 'type': kind, + 'payload': {key: payload[key] for key in keys if key in payload}, + } + + +def capture_tool_context(codex_home, output): + """Capture only an explicitly selected trial's rollouts; never its auth/config.""" + sessions = codex_home / 'sessions' + blocked = sessions.is_symlink() + sources = [] if blocked else sorted(sessions.rglob('*.jsonl')) + target = output / 'codex-tool-context.jsonl' + source_receipts = [] + errors = [{'source': 'sessions', 'reason': 'not_an_owned_directory'}] if blocked else [] + retained = 0 + with os.fdopen(os.open(target, os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o600), + 'w', encoding='utf-8') as result: + for source in sources: + name = source.relative_to(codex_home).as_posix() + if source.is_symlink() or not source.is_file(): + errors.append({'source': name, 'reason': 'not_an_owned_regular_file'}) + continue + digest = hashlib.sha256() + counts = {} + with source.open('rb') as lines: + for ordinal, line in enumerate(lines, 1): + digest.update(line) + try: + event = json.loads(line) + selected = selected_event(event) + except ValueError as error: + errors.append({'source': name, 'line': ordinal, + 'reason': type(error).__name__}) + continue + kind = event.get('type') + counts[str(kind)] = counts.get(str(kind), 0) + 1 + if selected is not None: + result.write(json.dumps(selected, ensure_ascii=False) + '\n') + retained += 1 + source_receipts.append({'source': name, 'sha256': digest.hexdigest(), + 'event_counts': counts}) + retained_digest = hashlib.sha256() + with target.open('rb') as captured: + for chunk in iter(lambda: captured.read(1024 * 1024), b''): + retained_digest.update(chunk) + receipt = { + 'status': 'partial' if errors else 'captured' if source_receipts else 'unavailable', + 'retained_file': target.name, + 'retained_sha256': retained_digest.hexdigest(), + 'retained_events': retained, 'sources': source_receipts, 'errors': errors, + 'scope': 'Owned client tool-call/reply and per-response usage items, not ' + 'the exact submitted model request. Session/account metadata, ' + 'world state, user/developer messages, encrypted reasoning and ' + 'internal passthrough are omitted. Authentication, configuration ' + 'and ambient environment are never copied by this capture.', + } + (output / 'context-receipt.json').write_text(json.dumps(receipt, indent=2) + '\n') + return receipt diff --git a/experiments/analysis-efficiency/model-context.md b/experiments/analysis-efficiency/model-context.md new file mode 100644 index 000000000..3d878beed --- /dev/null +++ b/experiments/analysis-efficiency/model-context.md @@ -0,0 +1,209 @@ +# Observe the client boundary before claiming token savings + +Continues [#2016](https://github.com/morluto/rea/issues/2016) after the managed +summary [PR #2413](https://github.com/morluto/rea/pull/2413) merged as +`82cfcee19f18b0eeae9a330ba04e0411cdf85225`. That opt-in view has verified smaller +delivery and complete-default migration, while its original real-model cohort +does not establish token savings. This follow-up measures why raw reply bytes +and aggregate model usage cannot be treated as interchangeable observations. + +## Concrete diagnostic observation + +An individual real Codex CLI 0.161.0 control uses frozen runtime +`ae8051302b5834322fa00cfa31c968ae74ab6ed9`, the unchanged ShareX identity question, +requested `gpt-5.6-terra` high and the same two enabled REA tools. A private +diagnostic copy removes the original runner's explicit `--ephemeral` flag. +Authentication/profile cleanup still runs; only selected tool/usage items are +retained after filtering the owned rollout. Its source rollout SHA-256 is +`d4df41d6187880d177f00b7533d85ad032c9c4ec502a39e7d5a566180e10461b` and the retained +filtered-item SHA-256 is +`0d5b4fb67c5674d1ba14ac32ae37e9a72b0368f4d9f5f9f81eb4d13193cda260`. +The original whole-copy is removed; account metadata, world state, messages and +internal passthrough are not retained in the filtered receipt. + +Despite the requested `direct` configuration label, the observed client uses +three JavaScript `exec` calls. First it filters `ALL_TOOLS` with a broad +`/REA|reverse|artifact|managed|assembly|metadata/i` expression and prints matching +metadata. The client reports truncation of a 164,005-token original output. +This is the client's output measurement, not billed input or a complete inventory. +Next it invokes complete managed inspection but selects requested normalized +identity/classification fields before printing. Finally it invokes native-boundary +inspection and prints selected fields. The actual original complete MCP replies +remain available in the separate public-event transcript. + +The persisted custom-tool output values are 41,704 / 2,154 / 13,489 UTF-8 bytes. +These are the serialized recorded output **values**, not raw wire bytes or exact +submitted model requests. The trace reports four per-response usage observations: + +| Response | Input | Cached input | Output | +| -------- | -----: | -----------: | -----: | +| 1 | 10,400 | 0 | 132 | +| 2 | 19,992 | 9,984 | 456 | +| 3 | 21,242 | 19,200 | 251 | +| 4 | 24,799 | 20,224 | 1,535 | + +Their totals match public turn usage: 76,433 input, 49,408 cached, 27,025 uncached +and 2,374 output, with 61.436 seconds observed wall time. Two MCP calls alone do +not describe the client's discovery/execution rounds. This proves the observed +control can project a complete result before printing it; it does not establish +that every earlier control/candidate did the same, or attribute every token +difference to discovery. The earlier `direct` label specifies requested client +configuration, not a verified absence of an executor. + +Initial transcript-capture attempts retained empty source lists because the +original runner intentionally passed `--ephemeral`. A first private flag-removal +edit missed its newline and failed; that candidate remains an unsuccessful +capture attempt. The later corrected control is an individual diagnostic, +not a comparable pair with that candidate. Other exploratory aggregates remain +private and are not silently folded into the original four-trial cohort. + +## Reproducible optional capture + +`agent-trial.py --capture-tool-context` omits `--ephemeral` only for that explicitly +selected temporary trial. Its default is unchanged. The profile still disappears +in `finally`, including when capture fails. The capture reads only its owned +`sessions/*.jsonl` records, rejects symlink sources, and retains: + +- Custom/function tool calls and replies, preserving call IDs and exact values. +- Per-response token counts and cumulative counts, with original response IDs. +- Client token-count events, retaining explicit unknown `info: null`. + +It excludes session/account metadata, world state, user/developer messages, +encrypted reasoning, internal passthrough and unknown nested usage fields. It +does not copy authentication, configuration or the ambient environment. Receipts +record source/retained digests, event counts, malformed-line failures and missing +rollouts as unavailable. Capture is an observation of persisted client items, +not proof of the exact request accepted by a remote model service. + +Run the same frozen runtime, question, tool selection and capture mode on both +sides. Preserve source/settings hashes and separate requested versus observed +behavior. Use the existing resource wrapper with unchanged one CPU, 4 GiB hard +memory, 3 GiB early-stop, no swap and task/host safeguards. Final physical counters +remain unknown when transient cgroups disappear; this instrumentation does not +repair that measurement limitation. + +The focused Python tests run through the existing CI `test:python` gate. +They cover account/ambient exclusion while preserving caller data, malformed and +missing records, foreign symlinks, and capture failure without copied-credential +leakage. The cleanup test simulates a CLI and establishes only cleanup semantics; +it is not real-model/provider evidence. + +## An explicit discovery-guidance experiment + +`--guide-enabled-tools` is another opt-in experiment flag. It requires a nonempty +`--enabled-tools` list and adds the exact corresponding `mcp__rea__...` function +names to the temporary developer instructions, requesting only exact-name +metadata discovery. It preserves the analyst question, operation order, tool +selection and provider inputs. The default instruction is unchanged. Missing or +invalid selected tools fail before creating an output/profile. The test gate +also verifies the new prerequisite without acquiring a profile. + +Compare guided and unguided runs on the **same** frozen REA runtime with matching +capture mode and requested facts. A smaller discovery response or additional +hint is not itself a task/token success: grade answers, count all response usage, +preserve failed/truncated runs and repeat both orders before recommending this +guidance. This is a caller-side experiment, not a new REA tool or provider preset. + +## First committed capture pair + +Checkpoint `9ebdcc458c399b9bdecaaca2ef47422888231181` runs the unchanged question +against frozen control `ae8051302b5834322fa00cfa31c968ae74ab6ed9` and candidate +`8bc1601ca51e417ca472d90ea0367e9418a1a0f2`, both with capture enabled and no tool +guidance. Both captures succeed with 10 retained events, unchanged source bytes, +removed profiles, two successful MCP calls and passing selected factual checks. +Full grader gates remain false because final physical resource counters are +unavailable; factual subsets do not override that limitation. + +| Side | Input | Cached | Uncached | Output | Wall seconds | +| ----------------- | -----: | -----: | -------: | -----: | -----------: | +| Complete control | 52,656 | 27,136 | 25,520 | 2,015 | 57.736 | +| Summary candidate | 56,655 | 29,184 | 27,471 | 1,534 | 55.055 | + +Both clients make two executor calls and three response-usage observations. +First they print broad discovered metadata, with truncation. Then each executor +sequentially performs both MCP operations, preserving selected-target dependency +order. The control prints a 2,086-byte output value; the candidate prints 22,770 +bytes. Their discovery output values are 41,723 and 41,675 bytes. These are +recorded serialized values and remain distinct from raw MCP envelopes, billed +tokens and exact submitted requests. Both requested modes are `direct`; the +actual executor use is separately observed. + +The candidate's smaller raw metadata reply does not result in a smaller printed +client value in this pair. Candidate input/uncached tokens remain higher. The +observed projection and round counts are useful constraints for the next +experiment; this one pair does not attribute all usage differences to either +formatting or discovery, or establish a reliable speedup. + +## Same-runtime guided discovery, repeated in both orders + +Checkpoint `e20bd0e2a905f06bc8c56fe04b0e61f3da88636a` runs four actual Codex trials +in unguided/guided/guided/unguided order. All use the **same** frozen candidate +runtime, artifact, original question, requested model/reasoning, two selected +tools, provider settings and capture mode. Only the explicit discovery-guidance +treatment changes. Every trial selects artifact summary and makes the same two +successful MCP calls in dependency order; source bytes stay unchanged, captures +succeed and profiles are removed. + +| Trial | Input | Cached | Uncached | Output | Seconds | Observed responses | +| ---------- | ------: | -----: | -------: | -----: | ------: | -----------------: | +| Unguided 1 | 77,251 | 50,432 | 26,819 | 1,931 | 62.066 | 4 | +| Guided 1 | 39,175 | 23,040 | 16,135 | 1,763 | 55.658 | 3 | +| Guided 2 | 58,336 | 38,144 | 20,192 | 1,626 | 50.685 | 4 | +| Unguided 2 | 103,301 | 72,704 | 30,597 | 1,983 | 60.336 | 5 | + +Both guided discovery output values repeat at 12,336 bytes without truncation, +versus 41,709 bytes with truncation on both unguided trials. The recorded +executor/output usage explains observable discovery and round differences; it +does not certify exact remote request contents or attribute every token to the +hint. CPU samples are 6.303–6.554 seconds and peak family memory 684,519,424– +712,028,160 bytes, with no guard abort. Final resource counters remain unknown. +These samples do not establish a memory or CPU improvement. + +Unguided 1 omits `classification.container` and fails the selected factual gate; +it is not a fully acceptable comparison answer. Guided 1 and Unguided 2 pass +that gate. Guided 2 initially fails exact coverage-object equality because it +adds `module_ref_count: 0` and `pinvoke_import_count: 0` to the explicitly named +native scope. Both numbers exactly match the producer's returned arrays. The +grader now explicitly verifies only these two known additional counts, including +integer type, while still requiring all original coverage facts. Unknown extra +claims, conflicting aliases, wrong counts, boolean/string counts and omitted +issues continue to fail. Source coverage remains unchanged; returned-row counts +do not establish global absence. Original failure receipts remain preserved; +the regrade separately records its new source hash. Guided 2 then passes selected +facts, but every **full** gate still fails unknown final physical resource checks. + +The second comparison has two factually acceptable answers: observed guided input +falls about 43.5%, uncached input about 34.0%, and wall time about 16.0%. The first +comparison has an incomplete unguided answer and is reported without counting it +as equivalent completed work. Two trials per treatment on one question/application +remain a preliminary signal, not a general default recommendation. The hint stays +opt-in; wider questions/domains, count/coverage representation and client round +behavior need further verification before promoting it into general guidance. + +## Real Android follow-up: the primary outcomes are mixed + +Checkpoint `22834c7d73056e2480b426848f58c31af0c307bc` checks the pinned real +NewPipe 0.29.1 APK using actual configured JADX 0.7.1/Java, the original Android +question and the same frozen candidate runtime. First a fresh SDK workflow +validates three live advertised operations and produces the comparison facts: +package/version identity, every case-sensitive MainActivity match and all +reported onCreate entries, preserving signatures, overload indices and null DEX +descriptors. It takes 26.961 seconds; the owned scope samples 1,234,735,104 bytes. + +Both subsequent actual Codex trials use the same three selected tool names, +capture mode, model/reasoning, question and provider settings. Only discovery +guidance changes. Each makes three successful MCP calls and five observed model +responses, passes selected facts against the fresh actual producer, preserves +source bytes and removes its temporary profile. No APK execution is requested. + +| Trial | Input | Cached | Uncached | Output | Wall seconds | Sampled peak bytes | +| -------- | -----: | -----: | -------: | -----: | -----------: | -----------------: | +| Unguided | 99,913 | 80,640 | 19,273 | 1,527 | 66.797 | 1,405,829,120 | +| Guided | 89,220 | 62,464 | 26,756 | 2,142 | 76.936 | 1,345,576,960 | + +Total input falls about 10.7%, but uncached input increases and wall time is +longer. One pair does not demonstrate improved Android performance or memory. +No guard abort occurs; final counters remain unknown and full resource gates +remain false. This second domain is a concrete counterexample to generalizing +the favorable managed comparison. Keep both flags optional and continue +task-specific experiments before changing installed/default agent guidance. diff --git a/experiments/analysis-efficiency/review-model.py b/experiments/analysis-efficiency/review-model.py index 3c56aabb2..a61bb0984 100644 --- a/experiments/analysis-efficiency/review-model.py +++ b/experiments/analysis-efficiency/review-model.py @@ -52,6 +52,21 @@ def named_scope(value, names): return {'conflicting_reported_scopes': selected} +def scope_with_verified_counts(value, expected_scope, returned_counts): + """Validate explicit extra row counts; never erase unknown or conflicting facts.""" + if not isinstance(value, dict): + return value, {} + scope = dict(value) + checks = {} + for name, expected in returned_counts.items(): + if name in scope: + reported = scope[name] + checks[name] = type(reported) is int and reported == expected + if name not in expected_scope: + del scope[name] + return scope, checks + + def main(): parser = argparse.ArgumentParser() parser.add_argument('--domain', choices=['wasm', 'evm', 'android', 'managed', 'managed-nonempty', 'firmware', 'inventory', 'javascript', 'native', 'browser'], required=True) @@ -134,7 +149,13 @@ def main(): for key in ['module_refs', 'pinvoke_imports']: check(key, answer.get('native_boundaries', {}).get(key), reference['boundaries'][key]) check('artifact_coverage', named_scope(answer.get('coverage'), ['artifact', 'managed_artifact', 'managed_metadata']), n['coverage']) - check('native_boundaries_coverage', named_scope(answer.get('coverage'), ['native_boundaries', 'native_boundary_declarations']), reference['boundaries']['coverage']) + boundary_scope, counts = scope_with_verified_counts( + named_scope(answer.get('coverage'), ['native_boundaries', 'native_boundary_declarations']), + reference['boundaries']['coverage'], + {'module_ref_count': len(reference['boundaries']['module_refs']), + 'pinvoke_import_count': len(reference['boundaries']['pinvoke_imports'])}) + check('native_boundaries_coverage', boundary_scope, reference['boundaries']['coverage']) + checks.update({'native_coverage_' + name: valid for name, valid in counts.items()}) elif domain == 'managed-nonempty': n = reference['boundaries'] check('module_mvid', answer.get('module_mvid'), n['module']['mvid']) diff --git a/experiments/analysis-efficiency/test_codex_context.py b/experiments/analysis-efficiency/test_codex_context.py new file mode 100644 index 000000000..37df83a7c --- /dev/null +++ b/experiments/analysis-efficiency/test_codex_context.py @@ -0,0 +1,175 @@ +"""Distinct capture/cleanup failures; simulated CLI is not real-model evidence.""" + +import importlib.util +import io +import json +import os +import runpy +import sys +from pathlib import Path +import tempfile +import unittest +from unittest.mock import Mock, patch + +from codex_context import capture_tool_context, selected_event + + +class ManagedCoverageTests(unittest.TestCase): + def test_verifies_extra_counts_and_preserves_unknown_fields(self): + review = runpy.run_path(str(Path(__file__).with_name('review-model.py'))) + project = review['scope_with_verified_counts'] + expected = {'state': 'partial', 'issues': ['unreturned rows remain unknown']} + original = {**expected, 'module_ref_count': 0, 'pinvoke_import_count': 2} + scope, checks = project(original, expected, {'module_ref_count': 0, 'pinvoke_import_count': 2}) + self.assertEqual(scope, expected) + self.assertTrue(all(checks.values())) + self.assertEqual(original['pinvoke_import_count'], 2) + for count in [3, '2', True]: + scope, checks = project({**expected, 'pinvoke_import_count': count}, expected, {'pinvoke_import_count': 2}) + self.assertFalse(checks['pinvoke_import_count']) + _, checks = project({**expected, 'module_ref_count': False}, {**expected, 'module_ref_count': 0}, {'module_ref_count': 0}) + self.assertFalse(checks['module_ref_count']) + scope, _ = project({**expected, 'unsupported_claim': True}, expected, {}) + self.assertNotEqual(scope, expected) + scope, _ = project({'state': 'partial', 'module_ref_count': 0}, expected, {'module_ref_count': 0}) + self.assertNotEqual(scope, expected) + ambiguous = review['named_scope']({'native_boundaries': expected, 'native_boundary_declarations': {'state': 'complete'}}, ['native_boundaries', 'native_boundary_declarations']) + scope, _ = project(ambiguous, expected, {'module_ref_count': 0}) + self.assertNotEqual(scope, expected) + + +class ContextCaptureTests(unittest.TestCase): + def test_filters_ambient_account_and_nested_usage_state(self): + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + home = root / 'home' + sessions = home / 'sessions' + sessions.mkdir(parents=True) + output = root / 'output' + output.mkdir() + events = [ + {'type': 'world_state', 'payload': {'state': {'env': 'ambient-marker'}}}, + {'type': 'session_meta', 'payload': {'creator_account_id': 'account-marker'}}, + {'type': 'response_item', 'payload': { + 'type': 'custom_tool_call_output', 'call_id': 'original-call', + 'output': 'caller-selected evidence and paths', + 'internal_chat_message_metadata_passthrough': 'private-marker'}}, + {'type': 'token_usage_record', 'payload': { + 'response_id': 'original-response', 'usage': { + 'input_tokens': 12, 'cached_input_tokens': 4, + 'environment': 'nested-ambient-marker'}, + 'transport_auth': 'auth-marker'}}, + ] + (sessions / 'rollout.jsonl').write_text(''.join(json.dumps(x) + '\n' for x in events)) + (home / 'auth.json').write_text('copied-credential-marker') + receipt = capture_tool_context(home, output) + text = (output / 'codex-tool-context.jsonl').read_text() + self.assertEqual(receipt['status'], 'captured') + self.assertEqual(receipt['retained_events'], 2) + for marker in ['ambient-marker', 'account-marker', 'private-marker', + 'auth-marker', 'copied-credential-marker']: + self.assertNotIn(marker, text) + self.assertIn('caller-selected evidence and paths', text) + self.assertIn('original-response', text) + self.assertEqual((output / 'codex-tool-context.jsonl').stat().st_mode & 0o777, 0o600) + + def test_missing_rollouts_are_unknown_and_malformed_lines_remain_partial(self): + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + output = root / 'missing' + output.mkdir() + self.assertEqual(capture_tool_context(root / 'home', output)['status'], 'unavailable') + sessions = root / 'home' / 'sessions' + sessions.mkdir(parents=True) + (sessions / 'rollout.jsonl').write_text('not-json\n' + json.dumps({ + 'type': 'event_msg', 'payload': {'type': 'token_count', 'info': None}}) + '\n') + partial = root / 'partial' + partial.mkdir() + receipt = capture_tool_context(root / 'home', partial) + self.assertEqual(receipt['status'], 'partial') + self.assertEqual(receipt['errors'][0]['line'], 1) + self.assertEqual(receipt['retained_events'], 1) + with self.assertRaises(ValueError): + selected_event({'type': 'token_usage_record', 'payload': {'usage': {'input_tokens': True}}}) + with self.assertRaises(ValueError): + selected_event({'type': {'environment': 'unexpected'}, 'payload': {}}) + with self.assertRaises(ValueError): + selected_event({'type': 'response_item', 'timestamp': {'environment': 'unexpected'}, 'payload': {}}) + unknown = selected_event({'type': 'event_msg', 'payload': {'type': 'token_count', 'info': {'model_context_window': None}}}) + self.assertIsNone(unknown['payload']['info']['model_context_window']) + + def test_capture_failure_still_removes_trial_credentials_and_profile(self): + spec = importlib.util.spec_from_file_location('agent_trial', Path(__file__).with_name('agent-trial.py')) + agent = importlib.util.module_from_spec(spec) + spec.loader.exec_module(agent) + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + runtime = root / 'runtime' + (runtime / 'scripts').mkdir(parents=True) + (runtime / 'scripts' / 'rea.mjs').write_text('') + (runtime / 'package-lock.json').write_text('{}') + auth_home = root / 'source-auth' + auth_home.mkdir() + (auth_home / 'auth.json').write_text('synthetic-credential') + question = root / 'question.txt' + question.write_text('Observe the declared fixture.') + output = root / 'trial' + child = Mock(stdin=io.StringIO()) + child.wait.return_value = 0 + arguments = ['agent-trial.py', '--runtime-root', str(runtime), '--revision', 'fixture', + '--prompt', str(question), '--output', str(output), '--capture-tool-context'] + with patch.dict(os.environ, {'CODEX_HOME': str(auth_home)}, clear=True), \ + patch.object(sys, 'argv', arguments), \ + patch.object(agent.shutil, 'which', return_value='/fixture/node'), \ + patch.object(agent.subprocess, 'check_output', return_value='v24.18.0\n'), \ + patch.object(agent.subprocess, 'Popen', return_value=child), \ + patch.object(agent, 'capture_tool_context', side_effect=OSError('fixture capture failure')), \ + patch('builtins.print'): + with self.assertRaisesRegex(OSError, 'fixture capture failure'): + agent.main() + self.assertFalse(list(output.glob('profile-*'))) + self.assertTrue(json.loads((output / 'cleanup.json').read_text())['profile_removed']) + self.assertEqual((auth_home / 'auth.json').read_text(), 'synthetic-credential') + + def test_refuses_foreign_session_directory_and_symlink_records(self): + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + home = root / 'home' + home.mkdir() + foreign = root / 'foreign' + foreign.mkdir() + (foreign / 'rollout.jsonl').write_text('private marker') + (home / 'sessions').symlink_to(foreign, target_is_directory=True) + output = root / 'directory-result' + output.mkdir() + receipt = capture_tool_context(home, output) + self.assertEqual(receipt['status'], 'partial') + self.assertEqual(receipt['retained_events'], 0) + (home / 'sessions').unlink() + (home / 'sessions').mkdir() + (home / 'sessions' / 'linked.jsonl').symlink_to(foreign / 'rollout.jsonl') + output = root / 'file-result' + output.mkdir() + receipt = capture_tool_context(home, output) + self.assertEqual(receipt['status'], 'partial') + self.assertEqual(receipt['errors'][0]['reason'], 'not_an_owned_regular_file') + self.assertEqual((foreign / 'rollout.jsonl').read_text(), 'private marker') + + def test_guidance_requires_selected_tools_before_acquiring_a_profile(self): + spec = importlib.util.spec_from_file_location('agent_trial', Path(__file__).with_name('agent-trial.py')) + agent = importlib.util.module_from_spec(spec) + spec.loader.exec_module(agent) + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + output = root / 'trial' + arguments = ['agent-trial.py', '--runtime-root', str(root), '--revision', 'fixture', + '--prompt', str(root / 'question'), '--output', str(output), '--guide-enabled-tools'] + with patch.object(sys, 'argv', arguments), patch.object(sys, 'stderr', io.StringIO()): + with self.assertRaises(SystemExit) as raised: + agent.main() + self.assertEqual(raised.exception.code, 2) + self.assertFalse(output.exists()) + + +if __name__ == '__main__': + unittest.main() diff --git a/package.json b/package.json index 5635c6d3c..327c14315 100644 --- a/package.json +++ b/package.json @@ -130,7 +130,7 @@ "test": "npm run test:with-build", "test:with-build": "npm run test:prepare && vitest run", "test:run": "vitest run", - "test:python": "python3 -m unittest bridge/native/test_rea_lldb_tracer.py -v", + "test:python": "python3 -m unittest bridge/native/test_rea_lldb_tracer.py -v && python3 -m unittest discover -s experiments/analysis-efficiency -p test_codex_context.py -v", "test:local": "node scripts/run-development-tests.mjs local", "test:fast": "vitest run --project domain --project services --project adapters --project composition --project conformance --project evaluation", "pretest:boundary": "npm run test:prepare",