diff --git a/experiments/analysis-efficiency/README.md b/experiments/analysis-efficiency/README.md index e41e6b7df..fb645b1e0 100644 --- a/experiments/analysis-efficiency/README.md +++ b/experiments/analysis-efficiency/README.md @@ -32,6 +32,11 @@ records two shipped assembly checks, previous-runtime portable Evidence, and fou actual Codex trials. Selected replies shrink substantially; observed model input tokens increase and latency is mixed, so model-token savings are not claimed. +Follow-up: [model tool context and per-response usage](model-context.md) separates +raw MCP replies from observed client execution/discovery and usage. Optional +trial capture retains tool/usage items while excluding account and ambient state; +existing ephemeral trials keep their default behavior. + The 12 detailed checkpoint receipts were archived byte-for-byte before removal from the final PR tree. The archive `review-receipts-00d51e0fc-20261012` is in the private experiment lab outside the checkout; its `archive-index.json` records diff --git a/experiments/analysis-efficiency/agent-trial.py b/experiments/analysis-efficiency/agent-trial.py index 8392000ca..682be27ec 100644 --- a/experiments/analysis-efficiency/agent-trial.py +++ b/experiments/analysis-efficiency/agent-trial.py @@ -11,6 +11,8 @@ import subprocess import tempfile import time +from codex_context import capture_tool_context + def file_digest(path): digest = hashlib.sha256() @@ -48,7 +50,20 @@ def main(): parser.add_argument('--providers', type=Path) parser.add_argument('--enabled-tools', type=Path) parser.add_argument('--authority', choices=['static', 'browser'], default='static') + parser.add_argument('--capture-tool-context', action='store_true', + help='Retain owned tool/usage items; excludes account/ambient state') + parser.add_argument('--guide-enabled-tools', action='store_true', + help='Opt-in exact-name discovery guidance for selected REA tools') args = parser.parse_args() + if args.guide_enabled_tools and args.enabled_tools is None: + parser.error('--guide-enabled-tools requires --enabled-tools') + enabled_tools = None + if args.enabled_tools: + enabled_tools = json.loads(args.enabled_tools.read_text()) + if not isinstance(enabled_tools, list) or not all(isinstance(name, str) and name for name in enabled_tools) or len(enabled_tools) != len(set(enabled_tools)): + raise ValueError('enabled-tools must be a unique list of exact tool names') + if args.guide_enabled_tools and not enabled_tools: + parser.error('--guide-enabled-tools requires a nonempty selected tool list') def terminate(signum, _frame): # The outer resource scope terminates descendants together. Ensure the # runner's copied credentials are also removed on a resource abort. @@ -130,11 +145,14 @@ def main(): 'possible and select the requested normalized_result fields before returning ' 'intermediate output to model context. Keep real Evidence IDs and limitations.' ) - enabled_tools = None - if args.enabled_tools: - enabled_tools = json.loads(args.enabled_tools.read_text()) - if not isinstance(enabled_tools, list) or not all(isinstance(name, str) and name for name in enabled_tools) or len(enabled_tools) != len(set(enabled_tools)): - raise ValueError('enabled-tools must be a unique list of exact tool names') + if args.guide_enabled_tools: + instruction += ( + ' Configured REA tool function names for this trial: ' + + json.dumps(['mcp__rea__' + name for name in enabled_tools]) + + '. For discovery select only these exact names from ALL_TOOLS and ' + 'return only their metadata. Preserve the original analyst question, ' + 'dependency order and source Evidence.' + ) quote = json.dumps config = '\n'.join([ 'approval_policy = "never"', 'sandbox_mode = "read-only"', @@ -183,6 +201,8 @@ def main(): (args.output / 'prompt.txt').write_text(prompt) (args.output / 'trial-settings.json').write_text(json.dumps({ 'mode': args.mode, 'model': args.model, 'reasoning': args.reasoning, + 'capture_tool_context': args.capture_tool_context, + 'guide_enabled_tools': args.guide_enabled_tools, 'driver_commit': subprocess.check_output( ['git', 'rev-parse', 'HEAD'], cwd=driver_repository, text=True).strip(), 'runtime_revision': args.revision, @@ -196,7 +216,7 @@ def main(): 'package_lock_sha256': file_digest(repository / 'package-lock.json'), 'experiment_sources_sha256': { name: hashlib.sha256((experiment / name).read_bytes()).hexdigest() - for name in ['agent-trial.py', 'provider-environment.json'] + for name in ['agent-trial.py', 'codex_context.py', 'provider-environment.json'] }, 'developer_instructions': instruction, 'developer_instructions_sha256': hashlib.sha256(instruction.encode()).hexdigest(), @@ -212,7 +232,8 @@ def main(): 'multi_agent': False, 'code_mode': args.mode == 'client-code', 'code_mode_only': args.mode == 'client-code'}, }, indent=2) + '\n') - invocation = [args.codex, '--no-daemon', 'exec', '--strict-config', '--ephemeral', + invocation = [args.codex, '--no-daemon', 'exec', '--strict-config', + *([] if args.capture_tool_context else ['--ephemeral']), '--skip-git-repo-check', '--ignore-rules', '--json', '--color', 'never', '-C', str(workspace), '--output-last-message', str(args.output.resolve() / 'answer.txt'), '-'] @@ -234,26 +255,28 @@ def main(): usage = [] tools = [] event_types = {} - for line in (args.output / 'events.jsonl').open(): - if not line.strip(): - continue - event = json.loads(line) - kind = event.get('type', 'unknown') - event_types[kind] = event_types.get(kind, 0) + 1 - if 'usage' in event: - usage.append(event['usage']) - item = event.get('item', {}) - if item.get('type') == 'mcp_tool_call' and kind == 'item.completed': - tools.append({ - **{k: item.get(k) for k in ['server', 'tool', 'arguments', 'status', 'error']}, - 'codex_event_result_json_utf8_bytes': len(json.dumps( - item.get('result'), separators=(',', ':'), ensure_ascii=False - ).encode()), - }) + with (args.output / 'events.jsonl').open() as event_file: + for line in event_file: + if not line.strip(): + continue + event = json.loads(line) + kind = event.get('type', 'unknown') + event_types[kind] = event_types.get(kind, 0) + 1 + if 'usage' in event: + usage.append(event['usage']) + item = event.get('item', {}) + if item.get('type') == 'mcp_tool_call' and kind == 'item.completed': + tools.append({ + **{k: item.get(k) for k in ['server', 'tool', 'arguments', 'status', 'error']}, + 'codex_event_result_json_utf8_bytes': len(json.dumps( + item.get('result'), separators=(',', ':'), ensure_ascii=False + ).encode()), + }) final_target_digest = file_digest(Path(args.artifact_path)) if args.artifact_path else None summary = { 'mode': args.mode, 'requested_model': args.model, 'reasoning': args.reasoning, 'runtime_revision': args.revision, + 'capture_tool_context_requested': args.capture_tool_context, 'selected_artifact_sha256_before': target_digest, 'selected_artifact_sha256_after': final_target_digest, 'selected_artifact_bytes_unchanged': target_digest == final_target_digest if target_digest else None, @@ -268,9 +291,13 @@ def main(): (args.output / 'summary.json').write_text(json.dumps(summary, indent=2) + '\n') print(json.dumps(summary)) finally: - # Always remove copied credentials, config, caches and profile state. - shutil.rmtree(profile) - (args.output / 'cleanup.json').write_text(json.dumps({'profile_removed': not profile.exists()}) + '\n') + try: + if args.capture_tool_context: + capture_tool_context(trial_codex_home, args.output) + finally: + # Capture failure must never prevent copied-credential cleanup. + shutil.rmtree(profile) + (args.output / 'cleanup.json').write_text(json.dumps({'profile_removed': not profile.exists()}) + '\n') raise SystemExit(124 if timeout else exit_code) diff --git a/experiments/analysis-efficiency/codex_context.py b/experiments/analysis-efficiency/codex_context.py new file mode 100644 index 000000000..cce8bf9cf --- /dev/null +++ b/experiments/analysis-efficiency/codex_context.py @@ -0,0 +1,117 @@ +"""Retain owned Codex tool/usage observations without account or ambient state.""" + +import hashlib +import json +import os + + +def token_counts(value): + if not isinstance(value, dict): + raise ValueError('Expected a token-count object') + result = {} + for key in ['input_tokens', 'cached_input_tokens', 'cache_write_input_tokens', + 'output_tokens', 'reasoning_output_tokens', 'total_tokens']: + if key in value: + count = value[key] + if not isinstance(count, int) or isinstance(count, bool) or count < 0: + raise ValueError('Expected nonnegative integer token counts') + result[key] = count + return result + + +def selected_event(event): + """Select observed items; these are not the exact submitted model request.""" + if not isinstance(event, dict) or not isinstance(event.get('payload'), dict): + raise ValueError('Expected an event object with an object payload') + if not isinstance(event.get('type'), str): + raise ValueError('Expected an event type string') + if event.get('timestamp') is not None and not isinstance(event['timestamp'], str): + raise ValueError('Expected a timestamp string or omission') + payload = event['payload'] + kind = event.get('type') + if kind == 'response_item' and payload.get('type') in [ + 'custom_tool_call', 'custom_tool_call_output', + 'function_call', 'function_call_output', + ]: + keys = ['type', 'call_id', 'name', 'input', 'arguments', 'output'] + elif kind == 'token_usage_record': + keys = ['response_id', 'usage', 'turn_token_usage', 'thread_token_usage'] + payload = {key: token_counts(payload[key]) if key != 'response_id' + else payload[key] for key in keys if key in payload} + elif kind == 'event_msg' and payload.get('type') == 'token_count': + keys = ['type', 'info'] + info = payload.get('info') + if info is not None: + if not isinstance(info, dict): + raise ValueError('Expected token-count info') + selected_info = {key: token_counts(info[key]) for key in + ['total_token_usage', 'last_token_usage'] if key in info} + window = info.get('model_context_window') + if 'model_context_window' in info: + if window is not None and (not isinstance(window, int) or isinstance(window, bool) or window <= 0): + raise ValueError('Expected a positive model context window') + selected_info['model_context_window'] = window + payload = {'type': 'token_count', 'info': selected_info} + else: + payload = {'type': 'token_count', 'info': None} + else: + return None + return { + 'timestamp': event.get('timestamp'), + 'type': kind, + 'payload': {key: payload[key] for key in keys if key in payload}, + } + + +def capture_tool_context(codex_home, output): + """Capture only an explicitly selected trial's rollouts; never its auth/config.""" + sessions = codex_home / 'sessions' + blocked = sessions.is_symlink() + sources = [] if blocked else sorted(sessions.rglob('*.jsonl')) + target = output / 'codex-tool-context.jsonl' + source_receipts = [] + errors = [{'source': 'sessions', 'reason': 'not_an_owned_directory'}] if blocked else [] + retained = 0 + with os.fdopen(os.open(target, os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o600), + 'w', encoding='utf-8') as result: + for source in sources: + name = source.relative_to(codex_home).as_posix() + if source.is_symlink() or not source.is_file(): + errors.append({'source': name, 'reason': 'not_an_owned_regular_file'}) + continue + digest = hashlib.sha256() + counts = {} + with source.open('rb') as lines: + for ordinal, line in enumerate(lines, 1): + digest.update(line) + try: + event = json.loads(line) + selected = selected_event(event) + except ValueError as error: + errors.append({'source': name, 'line': ordinal, + 'reason': type(error).__name__}) + continue + kind = event.get('type') + counts[str(kind)] = counts.get(str(kind), 0) + 1 + if selected is not None: + result.write(json.dumps(selected, ensure_ascii=False) + '\n') + retained += 1 + source_receipts.append({'source': name, 'sha256': digest.hexdigest(), + 'event_counts': counts}) + retained_digest = hashlib.sha256() + with target.open('rb') as captured: + for chunk in iter(lambda: captured.read(1024 * 1024), b''): + retained_digest.update(chunk) + receipt = { + 'status': 'partial' if errors else 'captured' if source_receipts else 'unavailable', + 'retained_file': target.name, + 'retained_sha256': retained_digest.hexdigest(), + 'retained_events': retained, 'sources': source_receipts, 'errors': errors, + 'scope': 'Owned client tool-call/reply and per-response usage items, not ' + 'the exact submitted model request. Session/account metadata, ' + 'world state, user/developer messages, encrypted reasoning and ' + 'internal passthrough are omitted. Authentication, configuration ' + 'and ambient environment are never copied by this capture.', + } + (output / 'context-receipt.json').write_text(json.dumps(receipt, indent=2) + '\n') + return receipt diff --git a/experiments/analysis-efficiency/model-context.md b/experiments/analysis-efficiency/model-context.md new file mode 100644 index 000000000..3d878beed --- /dev/null +++ b/experiments/analysis-efficiency/model-context.md @@ -0,0 +1,209 @@ +# Observe the client boundary before claiming token savings + +Continues [#2016](https://github.com/morluto/rea/issues/2016) after the managed +summary [PR #2413](https://github.com/morluto/rea/pull/2413) merged as +`82cfcee19f18b0eeae9a330ba04e0411cdf85225`. That opt-in view has verified smaller +delivery and complete-default migration, while its original real-model cohort +does not establish token savings. This follow-up measures why raw reply bytes +and aggregate model usage cannot be treated as interchangeable observations. + +## Concrete diagnostic observation + +An individual real Codex CLI 0.161.0 control uses frozen runtime +`ae8051302b5834322fa00cfa31c968ae74ab6ed9`, the unchanged ShareX identity question, +requested `gpt-5.6-terra` high and the same two enabled REA tools. A private +diagnostic copy removes the original runner's explicit `--ephemeral` flag. +Authentication/profile cleanup still runs; only selected tool/usage items are +retained after filtering the owned rollout. Its source rollout SHA-256 is +`d4df41d6187880d177f00b7533d85ad032c9c4ec502a39e7d5a566180e10461b` and the retained +filtered-item SHA-256 is +`0d5b4fb67c5674d1ba14ac32ae37e9a72b0368f4d9f5f9f81eb4d13193cda260`. +The original whole-copy is removed; account metadata, world state, messages and +internal passthrough are not retained in the filtered receipt. + +Despite the requested `direct` configuration label, the observed client uses +three JavaScript `exec` calls. First it filters `ALL_TOOLS` with a broad +`/REA|reverse|artifact|managed|assembly|metadata/i` expression and prints matching +metadata. The client reports truncation of a 164,005-token original output. +This is the client's output measurement, not billed input or a complete inventory. +Next it invokes complete managed inspection but selects requested normalized +identity/classification fields before printing. Finally it invokes native-boundary +inspection and prints selected fields. The actual original complete MCP replies +remain available in the separate public-event transcript. + +The persisted custom-tool output values are 41,704 / 2,154 / 13,489 UTF-8 bytes. +These are the serialized recorded output **values**, not raw wire bytes or exact +submitted model requests. The trace reports four per-response usage observations: + +| Response | Input | Cached input | Output | +| -------- | -----: | -----------: | -----: | +| 1 | 10,400 | 0 | 132 | +| 2 | 19,992 | 9,984 | 456 | +| 3 | 21,242 | 19,200 | 251 | +| 4 | 24,799 | 20,224 | 1,535 | + +Their totals match public turn usage: 76,433 input, 49,408 cached, 27,025 uncached +and 2,374 output, with 61.436 seconds observed wall time. Two MCP calls alone do +not describe the client's discovery/execution rounds. This proves the observed +control can project a complete result before printing it; it does not establish +that every earlier control/candidate did the same, or attribute every token +difference to discovery. The earlier `direct` label specifies requested client +configuration, not a verified absence of an executor. + +Initial transcript-capture attempts retained empty source lists because the +original runner intentionally passed `--ephemeral`. A first private flag-removal +edit missed its newline and failed; that candidate remains an unsuccessful +capture attempt. The later corrected control is an individual diagnostic, +not a comparable pair with that candidate. Other exploratory aggregates remain +private and are not silently folded into the original four-trial cohort. + +## Reproducible optional capture + +`agent-trial.py --capture-tool-context` omits `--ephemeral` only for that explicitly +selected temporary trial. Its default is unchanged. The profile still disappears +in `finally`, including when capture fails. The capture reads only its owned +`sessions/*.jsonl` records, rejects symlink sources, and retains: + +- Custom/function tool calls and replies, preserving call IDs and exact values. +- Per-response token counts and cumulative counts, with original response IDs. +- Client token-count events, retaining explicit unknown `info: null`. + +It excludes session/account metadata, world state, user/developer messages, +encrypted reasoning, internal passthrough and unknown nested usage fields. It +does not copy authentication, configuration or the ambient environment. Receipts +record source/retained digests, event counts, malformed-line failures and missing +rollouts as unavailable. Capture is an observation of persisted client items, +not proof of the exact request accepted by a remote model service. + +Run the same frozen runtime, question, tool selection and capture mode on both +sides. Preserve source/settings hashes and separate requested versus observed +behavior. Use the existing resource wrapper with unchanged one CPU, 4 GiB hard +memory, 3 GiB early-stop, no swap and task/host safeguards. Final physical counters +remain unknown when transient cgroups disappear; this instrumentation does not +repair that measurement limitation. + +The focused Python tests run through the existing CI `test:python` gate. +They cover account/ambient exclusion while preserving caller data, malformed and +missing records, foreign symlinks, and capture failure without copied-credential +leakage. The cleanup test simulates a CLI and establishes only cleanup semantics; +it is not real-model/provider evidence. + +## An explicit discovery-guidance experiment + +`--guide-enabled-tools` is another opt-in experiment flag. It requires a nonempty +`--enabled-tools` list and adds the exact corresponding `mcp__rea__...` function +names to the temporary developer instructions, requesting only exact-name +metadata discovery. It preserves the analyst question, operation order, tool +selection and provider inputs. The default instruction is unchanged. Missing or +invalid selected tools fail before creating an output/profile. The test gate +also verifies the new prerequisite without acquiring a profile. + +Compare guided and unguided runs on the **same** frozen REA runtime with matching +capture mode and requested facts. A smaller discovery response or additional +hint is not itself a task/token success: grade answers, count all response usage, +preserve failed/truncated runs and repeat both orders before recommending this +guidance. This is a caller-side experiment, not a new REA tool or provider preset. + +## First committed capture pair + +Checkpoint `9ebdcc458c399b9bdecaaca2ef47422888231181` runs the unchanged question +against frozen control `ae8051302b5834322fa00cfa31c968ae74ab6ed9` and candidate +`8bc1601ca51e417ca472d90ea0367e9418a1a0f2`, both with capture enabled and no tool +guidance. Both captures succeed with 10 retained events, unchanged source bytes, +removed profiles, two successful MCP calls and passing selected factual checks. +Full grader gates remain false because final physical resource counters are +unavailable; factual subsets do not override that limitation. + +| Side | Input | Cached | Uncached | Output | Wall seconds | +| ----------------- | -----: | -----: | -------: | -----: | -----------: | +| Complete control | 52,656 | 27,136 | 25,520 | 2,015 | 57.736 | +| Summary candidate | 56,655 | 29,184 | 27,471 | 1,534 | 55.055 | + +Both clients make two executor calls and three response-usage observations. +First they print broad discovered metadata, with truncation. Then each executor +sequentially performs both MCP operations, preserving selected-target dependency +order. The control prints a 2,086-byte output value; the candidate prints 22,770 +bytes. Their discovery output values are 41,723 and 41,675 bytes. These are +recorded serialized values and remain distinct from raw MCP envelopes, billed +tokens and exact submitted requests. Both requested modes are `direct`; the +actual executor use is separately observed. + +The candidate's smaller raw metadata reply does not result in a smaller printed +client value in this pair. Candidate input/uncached tokens remain higher. The +observed projection and round counts are useful constraints for the next +experiment; this one pair does not attribute all usage differences to either +formatting or discovery, or establish a reliable speedup. + +## Same-runtime guided discovery, repeated in both orders + +Checkpoint `e20bd0e2a905f06bc8c56fe04b0e61f3da88636a` runs four actual Codex trials +in unguided/guided/guided/unguided order. All use the **same** frozen candidate +runtime, artifact, original question, requested model/reasoning, two selected +tools, provider settings and capture mode. Only the explicit discovery-guidance +treatment changes. Every trial selects artifact summary and makes the same two +successful MCP calls in dependency order; source bytes stay unchanged, captures +succeed and profiles are removed. + +| Trial | Input | Cached | Uncached | Output | Seconds | Observed responses | +| ---------- | ------: | -----: | -------: | -----: | ------: | -----------------: | +| Unguided 1 | 77,251 | 50,432 | 26,819 | 1,931 | 62.066 | 4 | +| Guided 1 | 39,175 | 23,040 | 16,135 | 1,763 | 55.658 | 3 | +| Guided 2 | 58,336 | 38,144 | 20,192 | 1,626 | 50.685 | 4 | +| Unguided 2 | 103,301 | 72,704 | 30,597 | 1,983 | 60.336 | 5 | + +Both guided discovery output values repeat at 12,336 bytes without truncation, +versus 41,709 bytes with truncation on both unguided trials. The recorded +executor/output usage explains observable discovery and round differences; it +does not certify exact remote request contents or attribute every token to the +hint. CPU samples are 6.303–6.554 seconds and peak family memory 684,519,424– +712,028,160 bytes, with no guard abort. Final resource counters remain unknown. +These samples do not establish a memory or CPU improvement. + +Unguided 1 omits `classification.container` and fails the selected factual gate; +it is not a fully acceptable comparison answer. Guided 1 and Unguided 2 pass +that gate. Guided 2 initially fails exact coverage-object equality because it +adds `module_ref_count: 0` and `pinvoke_import_count: 0` to the explicitly named +native scope. Both numbers exactly match the producer's returned arrays. The +grader now explicitly verifies only these two known additional counts, including +integer type, while still requiring all original coverage facts. Unknown extra +claims, conflicting aliases, wrong counts, boolean/string counts and omitted +issues continue to fail. Source coverage remains unchanged; returned-row counts +do not establish global absence. Original failure receipts remain preserved; +the regrade separately records its new source hash. Guided 2 then passes selected +facts, but every **full** gate still fails unknown final physical resource checks. + +The second comparison has two factually acceptable answers: observed guided input +falls about 43.5%, uncached input about 34.0%, and wall time about 16.0%. The first +comparison has an incomplete unguided answer and is reported without counting it +as equivalent completed work. Two trials per treatment on one question/application +remain a preliminary signal, not a general default recommendation. The hint stays +opt-in; wider questions/domains, count/coverage representation and client round +behavior need further verification before promoting it into general guidance. + +## Real Android follow-up: the primary outcomes are mixed + +Checkpoint `22834c7d73056e2480b426848f58c31af0c307bc` checks the pinned real +NewPipe 0.29.1 APK using actual configured JADX 0.7.1/Java, the original Android +question and the same frozen candidate runtime. First a fresh SDK workflow +validates three live advertised operations and produces the comparison facts: +package/version identity, every case-sensitive MainActivity match and all +reported onCreate entries, preserving signatures, overload indices and null DEX +descriptors. It takes 26.961 seconds; the owned scope samples 1,234,735,104 bytes. + +Both subsequent actual Codex trials use the same three selected tool names, +capture mode, model/reasoning, question and provider settings. Only discovery +guidance changes. Each makes three successful MCP calls and five observed model +responses, passes selected facts against the fresh actual producer, preserves +source bytes and removes its temporary profile. No APK execution is requested. + +| Trial | Input | Cached | Uncached | Output | Wall seconds | Sampled peak bytes | +| -------- | -----: | -----: | -------: | -----: | -----------: | -----------------: | +| Unguided | 99,913 | 80,640 | 19,273 | 1,527 | 66.797 | 1,405,829,120 | +| Guided | 89,220 | 62,464 | 26,756 | 2,142 | 76.936 | 1,345,576,960 | + +Total input falls about 10.7%, but uncached input increases and wall time is +longer. One pair does not demonstrate improved Android performance or memory. +No guard abort occurs; final counters remain unknown and full resource gates +remain false. This second domain is a concrete counterexample to generalizing +the favorable managed comparison. Keep both flags optional and continue +task-specific experiments before changing installed/default agent guidance. diff --git a/experiments/analysis-efficiency/review-model.py b/experiments/analysis-efficiency/review-model.py index 3c56aabb2..a61bb0984 100644 --- a/experiments/analysis-efficiency/review-model.py +++ b/experiments/analysis-efficiency/review-model.py @@ -52,6 +52,21 @@ def named_scope(value, names): return {'conflicting_reported_scopes': selected} +def scope_with_verified_counts(value, expected_scope, returned_counts): + """Validate explicit extra row counts; never erase unknown or conflicting facts.""" + if not isinstance(value, dict): + return value, {} + scope = dict(value) + checks = {} + for name, expected in returned_counts.items(): + if name in scope: + reported = scope[name] + checks[name] = type(reported) is int and reported == expected + if name not in expected_scope: + del scope[name] + return scope, checks + + def main(): parser = argparse.ArgumentParser() parser.add_argument('--domain', choices=['wasm', 'evm', 'android', 'managed', 'managed-nonempty', 'firmware', 'inventory', 'javascript', 'native', 'browser'], required=True) @@ -134,7 +149,13 @@ def main(): for key in ['module_refs', 'pinvoke_imports']: check(key, answer.get('native_boundaries', {}).get(key), reference['boundaries'][key]) check('artifact_coverage', named_scope(answer.get('coverage'), ['artifact', 'managed_artifact', 'managed_metadata']), n['coverage']) - check('native_boundaries_coverage', named_scope(answer.get('coverage'), ['native_boundaries', 'native_boundary_declarations']), reference['boundaries']['coverage']) + boundary_scope, counts = scope_with_verified_counts( + named_scope(answer.get('coverage'), ['native_boundaries', 'native_boundary_declarations']), + reference['boundaries']['coverage'], + {'module_ref_count': len(reference['boundaries']['module_refs']), + 'pinvoke_import_count': len(reference['boundaries']['pinvoke_imports'])}) + check('native_boundaries_coverage', boundary_scope, reference['boundaries']['coverage']) + checks.update({'native_coverage_' + name: valid for name, valid in counts.items()}) elif domain == 'managed-nonempty': n = reference['boundaries'] check('module_mvid', answer.get('module_mvid'), n['module']['mvid']) diff --git a/experiments/analysis-efficiency/test_codex_context.py b/experiments/analysis-efficiency/test_codex_context.py new file mode 100644 index 000000000..37df83a7c --- /dev/null +++ b/experiments/analysis-efficiency/test_codex_context.py @@ -0,0 +1,175 @@ +"""Distinct capture/cleanup failures; simulated CLI is not real-model evidence.""" + +import importlib.util +import io +import json +import os +import runpy +import sys +from pathlib import Path +import tempfile +import unittest +from unittest.mock import Mock, patch + +from codex_context import capture_tool_context, selected_event + + +class ManagedCoverageTests(unittest.TestCase): + def test_verifies_extra_counts_and_preserves_unknown_fields(self): + review = runpy.run_path(str(Path(__file__).with_name('review-model.py'))) + project = review['scope_with_verified_counts'] + expected = {'state': 'partial', 'issues': ['unreturned rows remain unknown']} + original = {**expected, 'module_ref_count': 0, 'pinvoke_import_count': 2} + scope, checks = project(original, expected, {'module_ref_count': 0, 'pinvoke_import_count': 2}) + self.assertEqual(scope, expected) + self.assertTrue(all(checks.values())) + self.assertEqual(original['pinvoke_import_count'], 2) + for count in [3, '2', True]: + scope, checks = project({**expected, 'pinvoke_import_count': count}, expected, {'pinvoke_import_count': 2}) + self.assertFalse(checks['pinvoke_import_count']) + _, checks = project({**expected, 'module_ref_count': False}, {**expected, 'module_ref_count': 0}, {'module_ref_count': 0}) + self.assertFalse(checks['module_ref_count']) + scope, _ = project({**expected, 'unsupported_claim': True}, expected, {}) + self.assertNotEqual(scope, expected) + scope, _ = project({'state': 'partial', 'module_ref_count': 0}, expected, {'module_ref_count': 0}) + self.assertNotEqual(scope, expected) + ambiguous = review['named_scope']({'native_boundaries': expected, 'native_boundary_declarations': {'state': 'complete'}}, ['native_boundaries', 'native_boundary_declarations']) + scope, _ = project(ambiguous, expected, {'module_ref_count': 0}) + self.assertNotEqual(scope, expected) + + +class ContextCaptureTests(unittest.TestCase): + def test_filters_ambient_account_and_nested_usage_state(self): + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + home = root / 'home' + sessions = home / 'sessions' + sessions.mkdir(parents=True) + output = root / 'output' + output.mkdir() + events = [ + {'type': 'world_state', 'payload': {'state': {'env': 'ambient-marker'}}}, + {'type': 'session_meta', 'payload': {'creator_account_id': 'account-marker'}}, + {'type': 'response_item', 'payload': { + 'type': 'custom_tool_call_output', 'call_id': 'original-call', + 'output': 'caller-selected evidence and paths', + 'internal_chat_message_metadata_passthrough': 'private-marker'}}, + {'type': 'token_usage_record', 'payload': { + 'response_id': 'original-response', 'usage': { + 'input_tokens': 12, 'cached_input_tokens': 4, + 'environment': 'nested-ambient-marker'}, + 'transport_auth': 'auth-marker'}}, + ] + (sessions / 'rollout.jsonl').write_text(''.join(json.dumps(x) + '\n' for x in events)) + (home / 'auth.json').write_text('copied-credential-marker') + receipt = capture_tool_context(home, output) + text = (output / 'codex-tool-context.jsonl').read_text() + self.assertEqual(receipt['status'], 'captured') + self.assertEqual(receipt['retained_events'], 2) + for marker in ['ambient-marker', 'account-marker', 'private-marker', + 'auth-marker', 'copied-credential-marker']: + self.assertNotIn(marker, text) + self.assertIn('caller-selected evidence and paths', text) + self.assertIn('original-response', text) + self.assertEqual((output / 'codex-tool-context.jsonl').stat().st_mode & 0o777, 0o600) + + def test_missing_rollouts_are_unknown_and_malformed_lines_remain_partial(self): + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + output = root / 'missing' + output.mkdir() + self.assertEqual(capture_tool_context(root / 'home', output)['status'], 'unavailable') + sessions = root / 'home' / 'sessions' + sessions.mkdir(parents=True) + (sessions / 'rollout.jsonl').write_text('not-json\n' + json.dumps({ + 'type': 'event_msg', 'payload': {'type': 'token_count', 'info': None}}) + '\n') + partial = root / 'partial' + partial.mkdir() + receipt = capture_tool_context(root / 'home', partial) + self.assertEqual(receipt['status'], 'partial') + self.assertEqual(receipt['errors'][0]['line'], 1) + self.assertEqual(receipt['retained_events'], 1) + with self.assertRaises(ValueError): + selected_event({'type': 'token_usage_record', 'payload': {'usage': {'input_tokens': True}}}) + with self.assertRaises(ValueError): + selected_event({'type': {'environment': 'unexpected'}, 'payload': {}}) + with self.assertRaises(ValueError): + selected_event({'type': 'response_item', 'timestamp': {'environment': 'unexpected'}, 'payload': {}}) + unknown = selected_event({'type': 'event_msg', 'payload': {'type': 'token_count', 'info': {'model_context_window': None}}}) + self.assertIsNone(unknown['payload']['info']['model_context_window']) + + def test_capture_failure_still_removes_trial_credentials_and_profile(self): + spec = importlib.util.spec_from_file_location('agent_trial', Path(__file__).with_name('agent-trial.py')) + agent = importlib.util.module_from_spec(spec) + spec.loader.exec_module(agent) + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + runtime = root / 'runtime' + (runtime / 'scripts').mkdir(parents=True) + (runtime / 'scripts' / 'rea.mjs').write_text('') + (runtime / 'package-lock.json').write_text('{}') + auth_home = root / 'source-auth' + auth_home.mkdir() + (auth_home / 'auth.json').write_text('synthetic-credential') + question = root / 'question.txt' + question.write_text('Observe the declared fixture.') + output = root / 'trial' + child = Mock(stdin=io.StringIO()) + child.wait.return_value = 0 + arguments = ['agent-trial.py', '--runtime-root', str(runtime), '--revision', 'fixture', + '--prompt', str(question), '--output', str(output), '--capture-tool-context'] + with patch.dict(os.environ, {'CODEX_HOME': str(auth_home)}, clear=True), \ + patch.object(sys, 'argv', arguments), \ + patch.object(agent.shutil, 'which', return_value='/fixture/node'), \ + patch.object(agent.subprocess, 'check_output', return_value='v24.18.0\n'), \ + patch.object(agent.subprocess, 'Popen', return_value=child), \ + patch.object(agent, 'capture_tool_context', side_effect=OSError('fixture capture failure')), \ + patch('builtins.print'): + with self.assertRaisesRegex(OSError, 'fixture capture failure'): + agent.main() + self.assertFalse(list(output.glob('profile-*'))) + self.assertTrue(json.loads((output / 'cleanup.json').read_text())['profile_removed']) + self.assertEqual((auth_home / 'auth.json').read_text(), 'synthetic-credential') + + def test_refuses_foreign_session_directory_and_symlink_records(self): + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + home = root / 'home' + home.mkdir() + foreign = root / 'foreign' + foreign.mkdir() + (foreign / 'rollout.jsonl').write_text('private marker') + (home / 'sessions').symlink_to(foreign, target_is_directory=True) + output = root / 'directory-result' + output.mkdir() + receipt = capture_tool_context(home, output) + self.assertEqual(receipt['status'], 'partial') + self.assertEqual(receipt['retained_events'], 0) + (home / 'sessions').unlink() + (home / 'sessions').mkdir() + (home / 'sessions' / 'linked.jsonl').symlink_to(foreign / 'rollout.jsonl') + output = root / 'file-result' + output.mkdir() + receipt = capture_tool_context(home, output) + self.assertEqual(receipt['status'], 'partial') + self.assertEqual(receipt['errors'][0]['reason'], 'not_an_owned_regular_file') + self.assertEqual((foreign / 'rollout.jsonl').read_text(), 'private marker') + + def test_guidance_requires_selected_tools_before_acquiring_a_profile(self): + spec = importlib.util.spec_from_file_location('agent_trial', Path(__file__).with_name('agent-trial.py')) + agent = importlib.util.module_from_spec(spec) + spec.loader.exec_module(agent) + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + output = root / 'trial' + arguments = ['agent-trial.py', '--runtime-root', str(root), '--revision', 'fixture', + '--prompt', str(root / 'question'), '--output', str(output), '--guide-enabled-tools'] + with patch.object(sys, 'argv', arguments), patch.object(sys, 'stderr', io.StringIO()): + with self.assertRaises(SystemExit) as raised: + agent.main() + self.assertEqual(raised.exception.code, 2) + self.assertFalse(output.exists()) + + +if __name__ == '__main__': + unittest.main() diff --git a/package.json b/package.json index 5635c6d3c..327c14315 100644 --- a/package.json +++ b/package.json @@ -130,7 +130,7 @@ "test": "npm run test:with-build", "test:with-build": "npm run test:prepare && vitest run", "test:run": "vitest run", - "test:python": "python3 -m unittest bridge/native/test_rea_lldb_tracer.py -v", + "test:python": "python3 -m unittest bridge/native/test_rea_lldb_tracer.py -v && python3 -m unittest discover -s experiments/analysis-efficiency -p test_codex_context.py -v", "test:local": "node scripts/run-development-tests.mjs local", "test:fast": "vitest run --project domain --project services --project adapters --project composition --project conformance --project evaluation", "pretest:boundary": "npm run test:prepare",