fix(native)!: preserve incomplete native UI captures (#991)

* fix(native): preserve unknown accessibility child counts

* fix(native): align helper buffer with capture budget

* fix(native): align Swift helper JSON wire size

* fix(native): bound raw helper output to scenario budget

bridge/native/NativeUIChildren.swift

@@ -1,8 +1,24 @@
+import Foundation
+
 struct ChildBatch<Element> {
   let values: [Element]
   let complete: Bool
 }
 
+struct ChildCount {
+  let value: Int?
+  let error: Int32?
+}
+
+func serializeHelperOutput(_ value: Any) throws -> Data {
+  try JSONSerialization.data(withJSONObject: value, options: [.sortedKeys, .withoutEscapingSlashes])
+}
+
+func captureChildCount(status: Int32, success: Int32, value: Int) -> ChildCount {
+  guard status == success else { return ChildCount(value: nil, error: status) }
+  return ChildCount(value: value, error: nil)
+}
+
 func captureChildValues<Element>(requestedCount: Int, copy: () -> [Element]?) -> ChildBatch<Element> {
   guard requestedCount > 0 else { return ChildBatch(values: [], complete: true) }
   let returned = copy() ?? []

bridge/native/ReaNativeUI.swift

@@ -22,9 +22,10 @@ func attribute(_ element: AXUIElement, _ key: String) -> CFTypeRef? {
   var value: CFTypeRef?
   return AXUIElementCopyAttributeValue(element, key as CFString, &value) == .success ? value : nil
 }
-func childCount(_ element: AXUIElement) -> Int {
+func childCount(_ element: AXUIElement) -> ChildCount {
   var count = 0
-  return AXUIElementGetAttributeValueCount(element, kAXChildrenAttribute as CFString, &count) == .success ? count : 0
+  let status = AXUIElementGetAttributeValueCount(element, kAXChildrenAttribute as CFString, &count)
+  return captureChildCount(status: status.rawValue, success: AXError.success.rawValue, value: count)
 }
 func children(_ element: AXUIElement, count: Int) -> ChildBatch<AXUIElement> {
   captureChildValues(requestedCount: count) {
@@ -88,7 +89,11 @@ func observe(_ request: Request) async throws -> [String: Any] {
     var element = root
     for index in action.path ?? [] {
       var selectedChild: CFArray?
-      guard index >= 0 && index < childCount(element),
+      let count = childCount(element)
+      guard let available = count.value else {
+        try fail("element-count-unavailable", "Cannot validate selected accessibility path because child count failed with AXError \(count.error ?? -1)")
+      }
+      guard index >= 0 && index < available,
         AXUIElementCopyAttributeValues(element, kAXChildrenAttribute as CFString, index, 1, &selectedChild) == .success,
         let item = (selectedChild as? [AXUIElement])?.first else { try fail("element-missing", "Selected accessibility path no longer exists") }
       element = item
@@ -104,15 +109,20 @@ func observe(_ request: Request) async throws -> [String: Any] {
     }
     guard outcome == .success else { try fail("action-failed", "Accessibility action failed with AXError \(outcome.rawValue)") }
   }
-  var nodes: [[String: Any]] = []; var truncated = false
+  var nodes: [[String: Any]] = []; var truncated = false; var gaps: [String] = []
   if request.accessibility, let root = selected {
     var pending: [(AXUIElement, [Int])] = [(root, [])]
     while let (element, path) = pending.popLast() {
       if nodes.count >= request.max_nodes { truncated = true; break }
-      let totalChildren = childCount(element)
+      let childCountResult = childCount(element)
       var actions: CFArray?
       AXUIElementCopyActionNames(element, &actions)
-      nodes.append(["path": path, "role": text(element, kAXRoleAttribute), "title": text(element, kAXTitleAttribute), "value": text(element, kAXValueAttribute), "actions": actions as? [String] ?? [], "children_count": totalChildren])
+      nodes.append(["path": path, "role": text(element, kAXRoleAttribute), "title": text(element, kAXTitleAttribute), "value": text(element, kAXValueAttribute), "actions": actions as? [String] ?? [], "children_count": childCountResult.value.map { $0 as Any } ?? NSNull()])
+      guard let totalChildren = childCountResult.value else {
+        truncated = true
+        gaps.append("AX child count unavailable at path \(path): AXError \(childCountResult.error ?? -1)")
+        continue
+      }
       if path.count >= 32 { if totalChildren > 0 { truncated = true }; continue }
       let available = max(0, request.max_nodes - nodes.count - pending.count)
       let count = min(available, totalChildren)
@@ -143,18 +153,18 @@ func observe(_ request: Request) async throws -> [String: Any] {
     }
     screenshot = ["mime_type": "image/png", "base64": png.base64EncodedString(), "sha256": SHA256.hash(data: png).map({ String(format: "%02x", $0) }).joined(), "width": image.width, "height": image.height]
   }
-  return ["window": ["pid": request.pid, "window_id": request.window_id, "executable": request.executable, "launch_time": launch, "title": window[kCGWindowName as String] as? String ?? ""], "nodes": nodes, "truncated": truncated, "screenshot": screenshot, "gaps": request.accessibility ? [] : ["Accessibility capture was disabled"]]
+  return ["window": ["pid": request.pid, "window_id": request.window_id, "executable": request.executable, "launch_time": launch, "title": window[kCGWindowName as String] as? String ?? ""], "nodes": nodes, "truncated": truncated, "screenshot": screenshot, "gaps": request.accessibility ? gaps : ["Accessibility capture was disabled"]]
 }
 Task { @MainActor in
   do {
     let request = try JSONDecoder().decode(Request.self, from: Data(CommandLine.arguments[1].utf8))
     let result = try await observe(request)
-    let output = try JSONSerialization.data(withJSONObject: ["ok": true, "result": result], options: [.sortedKeys])
+    let output = try serializeHelperOutput(["ok": true, "result": result])
     FileHandle.standardOutput.write(output)
   } catch {
     let failure = error as? BoundaryFailure
     let output: [String: Any] = ["ok": false, "code": failure?.code ?? "capture-failed", "message": failure?.message ?? error.localizedDescription]
-    if let data = try? JSONSerialization.data(withJSONObject: output, options: [.sortedKeys]) { FileHandle.standardOutput.write(data) }
+    if let data = try? serializeHelperOutput(output) { FileHandle.standardOutput.write(data) }
   }
   exit(0)
 }

docs/product-catalog.json

@@ -675,9 +675,9 @@
       "mcp_prompts": 6
     },
     "digests": {
-      "tools_sha256": "4e80c4f1c50b30bd69b521b19f442d2c2ffb4211e8948d555f7e9450ae2b6958",
+      "tools_sha256": "4b37a03a97667f5660ab945a88d62b80b3fe81c859b0f0c6e979024d1735f1aa",
       "prompts_sha256": "ef2de8814390b340623c32001610aa2709d90bed51537b6e562ab3abf144ab4d",
-      "combined_sha256": "f2d8d03be076d9b0483e6a167cc695f91302b6155ef6bba040d33917062c866c",
+      "combined_sha256": "9ccd0ea584d29355bebff66556ef18322dfcc632a1e2d01eae174051d1cb60af",
       "providers_sha256": "aea15f517c822d80918ccaba57a4fa06557f484cf27b8acbaed4e1a660fac896"
     }
   }

docs/verification/managed-conformance-manifest.json

@@ -1,5 +1,5 @@
 {
-  "manifest_id": "ecm_f7875d899da42bc030f7bdf8c688d6df097a1cd8c02a179f9911f1cad89d86b0",
+  "manifest_id": "ecm_21a14df000707932724e7346d8d1826444d10f5ec63619977e84e22970bb357a",
   "verifier": {
     "id": "rea-managed-conformance",
     "version": "1"
@@ -13,7 +13,7 @@
   "skill_digests": [
     {
       "skill_id": "reverse-engineer-anything",
-      "sha256": "317c4116deb414b40955ab1dd00012950d78eaa7a2de2cbf8ae0f7ecc6dfb5a9"
+      "sha256": "12b79d17ea55072977b8dd1f185a48a924e1663c66ccf4e286f49074ab329f0f"
     }
   ],
   "claims": [

scripts/verify-native-ui.mjs

@@ -6,6 +6,7 @@ import { fileURLToPath } from "node:url";
 import { promisify } from "node:util";
 import { parseBinaryTarget } from "../dist/application/BinaryTargetResolver.js";
 import { observeNativeUi } from "../dist/native/NativeUiObservation.js";
+import { NATIVE_UI_HELPER_MAX_BUFFER } from "../dist/native/NativeUiOutputBudget.js";
 
 if (process.platform !== "darwin")
   throw new Error(
@@ -30,7 +31,38 @@ try {
     "-o",
     childRetrievalTest,
   ]);
-  await promisify(execFile)(childRetrievalTest, []);
+  const childRetrieval = await promisify(execFile)(childRetrievalTest, []);
+  const [slashRichProducerJson, numericProducerJson] = childRetrieval.stdout
+    .trimEnd()
+    .split("\n");
+  if (
+    slashRichProducerJson === undefined ||
+    numericProducerJson === undefined
+  ) {
+    throw new Error(
+      "Native UI serializer seam did not emit its producer bytes",
+    );
+  }
+  const slashRichConsumerJson = JSON.stringify(
+    JSON.parse(slashRichProducerJson),
+  );
+  if (slashRichProducerJson !== slashRichConsumerJson) {
+    throw new Error(
+      "Swift helper JSON bytes differ from the normalized JavaScript consumer representation",
+    );
+  }
+  const numericConsumerJson = JSON.stringify(JSON.parse(numericProducerJson));
+  if (
+    Buffer.byteLength(numericProducerJson) <=
+    Buffer.byteLength(numericConsumerJson)
+  ) {
+    throw new Error(
+      "Foundation numeric fixture no longer exercises larger producer JSON bytes",
+    );
+  }
+  if (Buffer.byteLength(numericProducerJson) > NATIVE_UI_HELPER_MAX_BUFFER) {
+    throw new Error("Foundation numeric fixture exceeds the raw helper budget");
+  }
 
   const contents = join(root, "Fixture.app", "Contents");
   await mkdir(join(contents, "MacOS"), { recursive: true });

skills/reverse-engineer-anything/SKILL.md

@@ -4,7 +4,7 @@ description: Reverse engineer native, managed, Electron/JavaScript, packaged, fi
 metadata:
   version: "27"
   tool_count: 133
-  catalog_digest: "f2d8d03be076d9b0483e6a167cc695f91302b6155ef6bba040d33917062c866c"
+  catalog_digest: "9ccd0ea584d29355bebff66556ef18322dfcc632a1e2d01eae174051d1cb60af"
 ---
 
 # REA

src/domain/native/nativeUiObservation.ts

@@ -117,7 +117,7 @@ export const nativeUiSnapshotSchema = z.strictObject({
       title: z.string().nullable(),
       value: z.string().nullable(),
       actions: z.array(z.string()),
-      children_count: z.number().int().nonnegative(),
+      children_count: z.number().int().nonnegative().nullable(),
     }),
   ),
   truncated: z.boolean(),

src/native/NativeUiHelperRuntime.ts

@@ -5,6 +5,7 @@ import { fileURLToPath } from "node:url";
 import { execFileOutput } from "../process/ExecFileOutput.js";
 import { safeParseJson } from "../domain/safeJson.js";
 import type { NativeUiHelper } from "./NativeUiObservation.js";
+import { NATIVE_UI_HELPER_MAX_BUFFER } from "./NativeUiOutputBudget.js";
 
 /** Lazily compile one owned helper per observation/scenario and remove its compiler cache. */
 export const createNativeUiHelperRuntime = () => {
@@ -50,7 +51,7 @@ export const createNativeUiHelperRuntime = () => {
       [JSON.stringify(parameters)],
       {
         timeout: 30_000,
-        maxBuffer: 16 * 1024 * 1024,
+        maxBuffer: NATIVE_UI_HELPER_MAX_BUFFER,
         ...(signal === undefined ? {} : { signal }),
       },
     );

src/native/NativeUiObservation.test.ts

@@ -111,6 +111,37 @@ describe("native UI screenshot validation", () => {
     );
     expect(result.ok).toBe(true);
   });
+
+  it("preserves an unknown AX child count and its truncation gap", async () => {
+    const partial = {
+      ...snapshot,
+      nodes: [
+        {
+          path: [],
+          role: "AXWindow",
+          title: "Fixture",
+          value: null,
+          actions: [],
+          children_count: null,
+        },
+      ],
+      truncated: true,
+      gaps: ["AX child count unavailable at path []: AXError -25204"],
+    };
+    const result = await observeNativeUi(
+      target,
+      "observe_native_ui",
+      { ...scope, accessibility: true },
+      { invoke: async () => ({ ok: true, result: partial }) },
+    );
+    expect(result.ok).toBe(true);
+    if (result.ok)
+      expect(result.value.initial).toMatchObject({
+        truncated: true,
+        gaps: ["AX child count unavailable at path []: AXError -25204"],
+        nodes: [{ children_count: null }],
+      });
+  });
 });
 
 describe("native UI capture selection and budgets", () => {
@@ -205,6 +236,27 @@ describe("native UI capture selection and budgets", () => {
         outcome: "cancelled",
       });
   });
+  it("reports aggregate output exhaustion after individually valid captures", async () => {
+    const largeSnapshot = {
+      ...snapshot,
+      window: { ...snapshot.window, title: "x".repeat(17 * 1024 * 1024) },
+    };
+    const result = await observeNativeUi(
+      target,
+      "capture_native_ui_scenario",
+      {
+        ...scope,
+        steps: [{ kind: "wait", milliseconds: 0 }],
+      },
+      { invoke: async () => ({ ok: true, result: largeSnapshot }) },
+    );
+    expect(result.ok).toBe(true);
+    if (result.ok)
+      expect(result.value.steps[0]).toMatchObject({
+        outcome: "failed",
+        reason: expect.stringContaining("64 MiB output budget"),
+      });
+  });
 });
 
 describe("native UI target and cancellation failures", () => {

src/native/NativeUiObservation.ts

@@ -16,6 +16,10 @@ import { ProviderAdapterError } from "../domain/providerAdapterError.js";
 import { type AnalysisError } from "../domain/analysisErrorBase.js";
 import { err, ok, type Result } from "../domain/result.js";
 import { createNativeUiHelperRuntime } from "./NativeUiHelperRuntime.js";
+import {
+  NATIVE_UI_OUTPUT_BUDGET_BYTES,
+  NATIVE_UI_OUTPUT_WEIGHT,
+} from "./NativeUiOutputBudget.js";
 
 const helper = fileURLToPath(
   new URL("../../bridge/native/ReaNativeUI.swift", import.meta.url),
@@ -135,8 +139,10 @@ const observeWithHelper = async (
           ),
         );
       launchTime = response.result.window.launch_time;
-      outputBytes += 2 * Buffer.byteLength(JSON.stringify(response.result));
-      if (outputBytes > 64 * 1024 * 1024)
+      outputBytes +=
+        NATIVE_UI_OUTPUT_WEIGHT *
+        Buffer.byteLength(JSON.stringify(response.result));
+      if (outputBytes > NATIVE_UI_OUTPUT_BUDGET_BYTES)
         return err(
           new AnalysisCapabilityUnavailableError(
             "native-macos",

src/native/NativeUiOutputBudget.ts

@@ -0,0 +1,15 @@
+const HELPER_SUCCESS_WRAPPER = Buffer.from(
+  JSON.stringify({ ok: true, result: {} }),
+);
+
+/** Scenario output is charged at twice the encoded result size before return. */
+export const NATIVE_UI_OUTPUT_BUDGET_BYTES = 64 * 1024 * 1024;
+export const NATIVE_UI_OUTPUT_WEIGHT = 2;
+
+/** Bytes added around a result by the helper's success JSON response. */
+export const NATIVE_UI_HELPER_ENVELOPE_BYTES =
+  HELPER_SUCCESS_WRAPPER.byteLength - Buffer.byteLength("{}");
+
+/** Raw helper output is bounded by the scenario's existing unweighted byte budget. */
+export const NATIVE_UI_HELPER_MAX_BUFFER =
+  NATIVE_UI_OUTPUT_BUDGET_BYTES + NATIVE_UI_HELPER_ENVELOPE_BYTES;

tests/conformance/native/native-ui-children/main.swift

@@ -1,3 +1,28 @@
+import Foundation
+
+let zero = captureChildCount(status: 0, success: 0, value: 0)
+precondition(zero.value == 0, "A successful empty child list must remain a known zero")
+precondition(zero.error == nil, "A successful child count must not report an error")
+
+let failedCount = captureChildCount(status: -25204, success: 0, value: 0)
+precondition(failedCount.value == nil, "A failed count must not be reported as zero")
+precondition(failedCount.error == -25204, "A failed count must retain its AX error")
+
+let slashRich = try! serializeHelperOutput([
+  "ok": true,
+  "result": [
+    "control": "\u{0001}",
+    "slash": String(repeating: "/", count: 4096),
+    "unicode": "café 😀\u{2028}",
+  ],
+])
+FileHandle.standardOutput.write(slashRich)
+FileHandle.standardOutput.write(Data("\n".utf8))
+
+let numericFormat = try! serializeHelperOutput(["launch_time": 1_791_385_671.621])
+FileHandle.standardOutput.write(numericFormat)
+FileHandle.standardOutput.write(Data("\n".utf8))
+
 let failed: ChildBatch<Int> = captureChildValues(requestedCount: 2) { nil }
 precondition(failed.values.isEmpty, "AX retrieval failure must return no values")
 precondition(!failed.complete, "AX retrieval failure must mark traversal incomplete")
@@ -15,4 +40,4 @@ let complete = captureChildValues(requestedCount: 2) { [10, 20, 30] }
 precondition(complete.values == [10, 20], "Retrieval must respect the requested node budget")
 precondition(complete.complete, "A full bounded result must remain complete")
 
-print("Native UI child retrieval seam passed")
+print("Native UI child retrieval and serialization seams passed")