perf(frida): evict captured messages without copying the whole queue (#2628)

Once a Frida session reached its 16 MiB capture budget, every new message
evicted the oldest one with Array#shift, re-serialized it to recover its
byte charge, and searched and spliced its per-script array. Each of
those is linear in the retained messages, so with about 188k small
messages retained the cost per message rose from 0.14 µs to 32 µs and
capped the main thread at roughly 31k messages per second, which a
hooked hot function can exceed.

Keep both queues as head-offset FIFOs that compact once half their slots
are evicted, and record each message's charge when it is admitted.
Arrival order already makes the globally oldest message the oldest one
of its script, so per-script eviction becomes a head check. Eviction is
now amortized O(1) at about 0.14 µs per message; retained messages,
their order and the truncation flag are unchanged.

Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>

src/application/frida/FridaInstrumentationManager.test.ts

@@ -809,6 +809,42 @@ it("keeps an active script message bucket after its last message is evicted", as
   await manager.closeAll();
 });
 
+it("retains the newest contiguous messages across repeated budget evictions", async () => {
+  const { bindings, manager, started } = await startLocalAttachedSession();
+  expect(started.ok).toBe(true);
+  if (!started.ok) return;
+  bindings.device.session.nextPayload = { index: -1 };
+  const loaded = await manager.loadScript(started.value.sessionId, {
+    sourceKind: "inline",
+    source: "send({index: -1});",
+  });
+  expect(loaded.ok).toBe(true);
+  const script = bindings.device.session.scripts[0];
+  expect(script).toBeDefined();
+  if (script === undefined) return;
+
+  // About 500 of these fit the 16 MiB budget, so 2,000 sends evict and
+  // compact the queue several times.
+  const filler = "x".repeat(32_000);
+  for (let index = 0; index < 2_000; index += 1)
+    script.message.emit({ type: "send", payload: { index, filler } }, null);
+
+  const status = manager.status(started.value.sessionId);
+  expect(status?.messagesTruncated).toBe(true);
+  const indexes = (status?.messages ?? []).map(
+    (message) => (message as { payload: { index: number } }).payload.index,
+  );
+  expect(indexes.length).toBeGreaterThan(400);
+  expect(indexes.at(-1)).toBe(1_999);
+  expect(indexes).toEqual(
+    Array.from(
+      { length: indexes.length },
+      (_, offset) => 2_000 - indexes.length + offset,
+    ),
+  );
+  await manager.closeAll();
+});
+
 it("serializes a final remote release against a new same-endpoint session", async () => {
   const bindings = new FakeDeviceManager();
   let enteredRemoval: (() => void) | undefined;

src/frida/FridaInstrumentationManager.ts

@@ -83,8 +83,8 @@ interface ManagedSession extends ManagedRemoteConnection {
   readonly pid: number;
   readonly mode: "attach" | "spawn";
   readonly scripts: Map<string, ManagedScript>;
-  readonly messages: JsonValue[];
-  readonly messagesByScript: Map<string, JsonValue[]>;
+  readonly messages: MessageFifo<CapturedMessage>;
+  readonly messagesByScript: Map<string, MessageFifo<JsonValue>>;
   state: "paused" | "running" | "detached";
   targetResumeObserved: boolean;
   resumeAfterDetachPending: boolean;
@@ -277,7 +277,7 @@ export class FridaInstrumentationManager implements FridaInstrumentationPort {
         pid,
         mode: input.mode,
         scripts: new Map(),
-        messages: [],
+        messages: new MessageFifo(),
         messagesByScript: new Map(),
         state: input.mode === "spawn" ? "paused" : "running",
         targetResumeObserved: false,
@@ -373,7 +373,7 @@ export class FridaInstrumentationManager implements FridaInstrumentationPort {
       .digest("hex");
     const scriptId = randomUUID();
     const scriptName = `rea-${scriptId}`;
-    managed.messagesByScript.set(scriptId, []);
+    managed.messagesByScript.set(scriptId, new MessageFifo());
     let nativeScript: Script | undefined;
     const onMessage = (message: FridaMessage, data: Buffer | null): void => {
       this.#recordMessage(managed, scriptId, message, data);
@@ -397,7 +397,7 @@ export class FridaInstrumentationManager implements FridaInstrumentationPort {
         sourceKind: source.sourceKind,
         sourcePath: source.sourceKind === "file" ? source.path : null,
         sourceSha256,
-        messages: [...(managed.messagesByScript.get(scriptId) ?? [])],
+        messages: managed.messagesByScript.get(scriptId)?.toArray() ?? [],
         messagesTruncated: managed.messagesTruncated,
       });
     } catch (cause: unknown) {
@@ -517,7 +517,9 @@ export class FridaInstrumentationManager implements FridaInstrumentationPort {
         scriptId: id,
         name,
       })),
-      messages: [...managed.messages],
+      messages: managed.messages
+        .toArray()
+        .map(({ observation }) => observation),
       messagesTruncated: managed.messagesTruncated,
     };
   }
@@ -851,30 +853,21 @@ export class FridaInstrumentationManager implements FridaInstrumentationPort {
       return;
     }
     while (
-      managed.messages.length > 0 &&
+      managed.messages.size > 0 &&
       managed.capturedBytes + charge > MAX_SESSION_CAPTURE_BYTES
     ) {
       const removed = managed.messages.shift();
       if (removed !== undefined) {
-        managed.capturedBytes -= Buffer.byteLength(JSON.stringify(removed));
-        if (
-          typeof removed === "object" &&
-          removed !== null &&
-          !Array.isArray(removed) &&
-          typeof removed.script_id === "string"
-        ) {
-          const scriptMessages = managed.messagesByScript.get(
-            removed.script_id,
-          );
-          const index = scriptMessages?.indexOf(removed) ?? -1;
-          if (scriptMessages !== undefined && index >= 0) {
-            scriptMessages.splice(index, 1);
-          }
-        }
+        managed.capturedBytes -= removed.charge;
+        // Both queues are in arrival order, so the globally oldest message is
+        // also the oldest one retained for its script.
+        const scriptMessages = managed.messagesByScript.get(removed.scriptId);
+        if (scriptMessages?.peek() === removed.observation)
+          scriptMessages.shift();
       }
       managed.messagesTruncated = true;
     }
-    managed.messages.push(observation);
+    managed.messages.push({ observation, scriptId, charge });
     managed.messagesByScript.get(scriptId)?.push(observation);
     managed.capturedBytes += charge;
   }
@@ -1240,3 +1233,49 @@ const providerError = (
 
 const errorMessage = (cause: unknown): string =>
   cause instanceof Error ? cause.message : String(cause);
+
+interface CapturedMessage {
+  readonly observation: JsonValue;
+  readonly scriptId: string;
+  /** Serialized bytes charged against the session capture budget. */
+  readonly charge: number;
+}
+
+/**
+ * Arrival-ordered queue whose eviction advances a head offset. A full capture
+ * budget evicts on every message, and `Array#shift` would copy every retained
+ * message each time.
+ */
+class MessageFifo<T> {
+  #items: (T | undefined)[] = [];
+  #head = 0;
+
+  get size(): number {
+    return this.#items.length - this.#head;
+  }
+
+  push(item: T): void {
+    this.#items.push(item);
+  }
+
+  peek(): T | undefined {
+    return this.#items[this.#head];
+  }
+
+  shift(): T | undefined {
+    if (this.#head >= this.#items.length) return undefined;
+    const item = this.#items[this.#head];
+    this.#items[this.#head] = undefined;
+    this.#head += 1;
+    // Compacting once half the slots are evicted keeps eviction amortized O(1).
+    if (this.#head * 2 >= this.#items.length) {
+      this.#items = this.#items.slice(this.#head);
+      this.#head = 0;
+    }
+    return item;
+  }
+
+  toArray(): T[] {
+    return this.#items.slice(this.#head) as T[];
+  }
+}