From f201b04fbbb29591b0d2bb3f3b24636526e4ec4e Mon Sep 17 00:00:00 2001 From: morluto <76467478+morluto@users.noreply.github.com> Date: Fri, 9 Oct 2026 05:06:48 +0800 Subject: [PATCH] fix(hopper)!: isolate regex matching from the native API thread BREAKING CHANGE: Hopper regex mode uses ECMAScript Unicode syntax instead of Python re. Matching runs in a cancellable worker with a five-second deadline; literal mode keeps Unicode casefold matching. --- bridge/hopper_bridge.py | 7 +- docs/installation.md | 3 + docs/testing.md | 3 +- scripts/lib/real-hopper-boundaries.mjs | 10 +- scripts/lib/real-hopper-search.mjs | 57 ++++++ src/contracts/officialToolContracts.ts | 4 +- src/hopper/HopperProvider.ts | 19 +- src/hopper/HopperProviderCapabilities.ts | 5 + src/hopper/HopperRegexSearch.ts | 244 +++++++++++++++++++++++ src/hopper/HopperRegexWorker.ts | 27 +++ 10 files changed, 365 insertions(+), 14 deletions(-) create mode 100644 src/hopper/HopperRegexSearch.ts create mode 100644 src/hopper/HopperRegexWorker.ts diff --git a/bridge/hopper_bridge.py b/bridge/hopper_bridge.py index 635b27b10..d99669a58 100644 --- a/bridge/hopper_bridge.py +++ b/bridge/hopper_bridge.py @@ -8,7 +8,6 @@ API access on this thread: moving dispatch to a worker can deadlock Hopper. import json import hmac import os -import re import socket import struct from typing import Any, Optional, Protocol, Sequence @@ -585,11 +584,7 @@ def _search_results(document, kind, params): needle = pattern if case_sensitive else pattern.casefold() matches = lambda value: needle in (value if case_sensitive else value.casefold()) else: - try: - expression = re.compile(pattern, 0 if case_sensitive else re.IGNORECASE) - except (re.error, OverflowError) as error: - raise InvalidRequestError("Invalid regex pattern") from error - matches = lambda value: expression.search(value) is not None + raise CapabilityUnavailableError("Regex matching requires REA's supervised worker; the native bridge only executes literal searches") selected = [] for item in _search_inventory(document, kind): diff --git a/docs/installation.md b/docs/installation.md index 4553b2c47..078b29013 100644 --- a/docs/installation.md +++ b/docs/installation.md @@ -358,6 +358,9 @@ and a reason, so one uncertain object does not block unrelated inspections. Function dossiers retain this same string evidence. Native call edges retain Hopper's partial `CallReference` classification and exact endpoints; detailed reference flags remain unavailable rather than being invented. +Regex searches use ECMAScript Unicode syntax in a cancellable worker with a +five-second matching deadline. Deadline or cancellation stops matching while +leaving the Hopper API available. Literal mode retains Unicode casefold matching. Closing or switching a target closes its bound Hopper document, shuts down REA's bridge and removes its temporary socket directory while preserving the Hopper diff --git a/docs/testing.md b/docs/testing.md index 3485fa3de..b86be7054 100644 --- a/docs/testing.md +++ b/docs/testing.md @@ -208,7 +208,8 @@ modes, annotation cache invalidation, complete native fragments of long literals checked against byte reads, escaped UTF-8/control text, byte-preserving Latin-1 decoding, and Hopper's UTF-16 symbol-name truncation boundary. It verifies native CallReference classifications across reference and dossier results, retains long -string fragment metadata in dossiers. When the macOS +string fragment metadata in dossiers, and exercises pathological regex deadline +and cancellation followed by successful requests in the same native session. When the macOS Objective-C fixture is present, the lane also verifies native UTF-16 string objects and their inferred encodings against their actual bytes. Unmapped annotation destinations and diff --git a/scripts/lib/real-hopper-boundaries.mjs b/scripts/lib/real-hopper-boundaries.mjs index c53044bd4..e54e71810 100644 --- a/scripts/lib/real-hopper-boundaries.mjs +++ b/scripts/lib/real-hopper-boundaries.mjs @@ -9,7 +9,10 @@ import addFormats from "ajv-formats"; import { mcpTextValue, requireMcpResult } from "./mcp-verifier-results.mjs"; import { verifyHopperNavigationAndText } from "./real-hopper-navigation.mjs"; import { verifyHopperWorkflows } from "./real-hopper-workflows.mjs"; -import { verifyHopperSearch } from "./real-hopper-search.mjs"; +import { + verifyHopperSearch, + verifyHopperRegexIsolation, +} from "./real-hopper-search.mjs"; /** Exercise real Hopper annotation and malformed-input boundaries over MCP. */ export async function verifyHopperBoundaryContracts( @@ -75,7 +78,10 @@ export async function verifyHopperBoundaryContracts( address, ); const workflows = await verifyHopperWorkflows(call, invalid, procedure); - const search = await verifyHopperSearch(call, invalid, procedure); + const search = { + ...(await verifyHopperSearch(call, invalid, procedure)), + ...(await verifyHopperRegexIsolation(client, options, call, address)), + }; try { if (foreignDocument !== undefined) { await invalid( diff --git a/scripts/lib/real-hopper-search.mjs b/scripts/lib/real-hopper-search.mjs index 887128d7c..0d796f3d2 100644 --- a/scripts/lib/real-hopper-search.mjs +++ b/scripts/lib/real-hopper-search.mjs @@ -1,4 +1,5 @@ import assert from "node:assert/strict"; +import { mcpTextValue } from "./mcp-verifier-results.mjs"; /** Exercise literal/regex search, Unicode identities and cache invalidation in Hopper. */ export async function verifyHopperSearch(call, invalid, procedure) { @@ -228,3 +229,59 @@ export async function verifyHopperStringObjects(call, expectations) { displayEscapesExcludedFromMatching: true, }; } + +/** A failed or cancelled pathological regex must leave the real Hopper API usable. */ +export async function verifyHopperRegexIsolation( + client, + options, + call, + address, +) { + const original = await call("address_name", { address }); + const pathological = "a".repeat(40) + "!"; + try { + await call("set_address_name", { address, name: pathological }); + const reply = await client.callTool( + { + name: "search_procedures", + arguments: { pattern: "(a+)+$", mode: "regex", case_sensitive: true }, + }, + options, + ); + assert.equal(reply.isError, true); + const { error } = JSON.parse(mcpTextValue(reply)); + assert.equal(error.code, "resource_constraint"); + assert.equal(error.details.operation, "search_procedures"); + assert.equal(error.details.resource, "cpu"); + assert.equal(await call("address_name", { address }), pathological); + const controller = new AbortController(); + const pending = client.callTool( + { + name: "search_procedures", + arguments: { pattern: "(a+)+$", mode: "regex", case_sensitive: true }, + }, + { ...options, signal: controller.signal }, + ); + const cancelled = pending.catch((cause) => cause); + // The same small inventory has already completed once; leave the pathological + // matching worker running before cancelling the second caller. + await new Promise((resolve) => setTimeout(resolve, 750)); + controller.abort(); + await cancelled; + assert.equal(await call("address_name", { address }), pathological); + assert.deepEqual( + await call("search_procedures", { + pattern: pathological, + mode: "literal", + case_sensitive: true, + }), + [{ address, value: pathological }], + ); + } finally { + await call("set_address_name", { address, name: original }); + } + return { + regexDeadlinePreservesSession: true, + regexCancellationPreservesSession: true, + }; +} diff --git a/src/contracts/officialToolContracts.ts b/src/contracts/officialToolContracts.ts index 542b66b7b..7d4d0715f 100644 --- a/src/contracts/officialToolContracts.ts +++ b/src/contracts/officialToolContracts.ts @@ -193,12 +193,12 @@ export const OFFICIAL_TOOL_CONTRACTS = [ ), official( "search_procedures", - "Search every analyzed procedure name using literal matching by default or regex when requested. Results are deterministic and complete.", + "Search every analyzed procedure name using literal matching by default or regex when requested. Results are deterministic and complete. Hopper regex mode uses ECMAScript Unicode syntax in a cancellable worker; a matching deadline leaves its API usable.", z.object(analysisSearchInput), ), official( "search_strings", - "Search every analyzed string object using literal matching by default or regex when requested. Results are deterministic and complete over the native inventory. Hopper retains typed byte decoding, native display, and explicit decoding unknowns. Adjacent long-literal fragments match independently.", + "Search every analyzed string object using literal matching by default or regex when requested. Results are deterministic and complete over the native inventory. Hopper retains typed byte decoding, native display, and explicit decoding unknowns. Adjacent long-literal fragments match independently. Hopper regex mode uses ECMAScript Unicode syntax in a cancellable worker; a matching deadline leaves its API usable.", z.object(analysisSearchInput), ), official( diff --git a/src/hopper/HopperProvider.ts b/src/hopper/HopperProvider.ts index 01c89aa0d..093f26335 100644 --- a/src/hopper/HopperProvider.ts +++ b/src/hopper/HopperProvider.ts @@ -23,6 +23,7 @@ import { hopperLoaderArgsForTarget, resolveHopperAnalysisProfile, } from "./HopperAnalysisProfile.js"; +import { HopperRegexSearch } from "./HopperRegexSearch.js"; import { HopperClient } from "./HopperClient.js"; import { hopperMachOImageSchema } from "./HopperMachOImage.js"; @@ -229,9 +230,15 @@ export class HopperProvider implements AnalysisProviderCandidate { onDiagnostic: (diagnostic) => this.logger.info(diagnostic, "Hopper bridge reported a diagnostic"), }); + const regexSearch = new HopperRegexSearch(client); return { execute: async (operation, parameters, options) => { - const result = await client.callTool(operation, parameters, options); + const result = + (operation === "search_strings" || + operation === "search_procedures") && + parameters.mode === "regex" + ? await regexSearch.execute(operation, parameters, options) + : await client.callTool(operation, parameters, options); return result.ok ? { ok: true, @@ -286,8 +293,14 @@ export class HopperProvider implements AnalysisProviderCandidate { ]; }, operationHealthSnapshot: () => client.operationHealth(), - closeWithOutcome: (options) => client.closeWithOutcome(options), - close: () => client.close(), + closeWithOutcome: async (options) => { + await regexSearch.close(); + return client.closeWithOutcome(options); + }, + close: async () => { + await regexSearch.close(); + await client.close(); + }, }; } } diff --git a/src/hopper/HopperProviderCapabilities.ts b/src/hopper/HopperProviderCapabilities.ts index 88a4d5287..1d2373191 100644 --- a/src/hopper/HopperProviderCapabilities.ts +++ b/src/hopper/HopperProviderCapabilities.ts @@ -79,6 +79,11 @@ export const CAPABILITIES: readonly CapabilityDescriptor[] = Object.freeze( }), limitations: Object.freeze([ "Results depend on Hopper's completed static analysis.", + ...(operation === "search_strings" || operation === "search_procedures" + ? [ + "Regex mode uses ECMAScript Unicode semantics in a supervised worker with a five-second matching deadline. Cancellation or deadline failure stops matching without blocking Hopper; literal mode retains native casefold semantics.", + ] + : []), ...(operation === "list_strings" || operation === "search_strings" || operation === "analyze_function" diff --git a/src/hopper/HopperRegexSearch.ts b/src/hopper/HopperRegexSearch.ts new file mode 100644 index 000000000..585fb3631 --- /dev/null +++ b/src/hopper/HopperRegexSearch.ts @@ -0,0 +1,244 @@ +import { Worker } from "node:worker_threads"; +import { z } from "zod"; + +import type { ExecutionOptions } from "../application/AnalysisProvider.js"; +import type { AnalysisError } from "../domain/analysisErrorBase.js"; +import { + AnalysisCancelledError, + AnalysisOutputError, + AnalysisResourceConstraintError, +} from "../domain/analysisErrorCore.js"; +import { analysisStringSchema } from "../domain/hopperValues.js"; +import { HopperRemoteError } from "../domain/hopperErrors.js"; +import type { JsonValue } from "../domain/jsonValue.js"; +import { err, ok, type Result } from "../domain/result.js"; +import { analysisSearchInput } from "../contracts/analysisSearchContract.js"; +import type { HopperClient } from "./HopperClient.js"; + +const MATCHING_DEADLINE_MS = 5_000; +const inputSchema = z.strictObject({ + ...analysisSearchInput, + mode: z.literal("regex"), + document: z.string().nullish(), + case_sensitive: z + .boolean() + .nullish() + .transform((value) => value ?? false), +}); +const replySchema = z.discriminatedUnion("status", [ + z.strictObject({ + status: z.literal("matched"), + indices: z.array(z.number().int().nonnegative()), + }), + z.strictObject({ status: z.literal("invalid_pattern"), reason: z.string() }), +]); + +/** Match collected native facts off Hopper's API thread, with cancellable lifetime. */ +export class HopperRegexSearch { + readonly #active = new Map< + AbortController, + Promise> + >(); + + constructor(private readonly client: HopperClient) {} + + execute( + operation: "search_strings" | "search_procedures", + parameters: Readonly>, + options?: ExecutionOptions, + ): Promise> { + const controller = new AbortController(); + const signal = + options?.signal === undefined + ? controller.signal + : AbortSignal.any([controller.signal, options.signal]); + const pending = this.#execute(operation, parameters, { + ...options, + signal, + }); + this.#active.set(controller, pending); + void pending.then( + () => this.#active.delete(controller), + () => this.#active.delete(controller), + ); + return pending; + } + + /** Abort matching workers before their provider session closes. */ + async close(): Promise { + for (const controller of this.#active.keys()) controller.abort(); + await Promise.all(this.#active.values()); + } + + async #execute( + operation: "search_strings" | "search_procedures", + parameters: Readonly>, + options: ExecutionOptions, + ): Promise> { + const input = inputSchema.safeParse(parameters); + if (!input.success) + return err( + new HopperRemoteError( + -32000, + `Invalid search arguments: ${input.error.message}`, + { diagnosticType: "invalid_request", operation }, + ), + ); + const inventoryOptions: ExecutionOptions = { + ...options, + ...(options.progress === undefined + ? {} + : { + progress: { + report: (update) => + options.progress?.report({ + ...update, + completed: + update.total === null || update.total === 0 + ? 0 + : Math.min(0.5, update.completed / update.total / 2), + total: 1, + }) ?? Promise.resolve(), + }, + }), + }; + const inventory = await this.client.callTool( + operation === "search_strings" ? "list_strings" : "list_procedures", + input.data.document == null ? {} : { document: input.data.document }, + inventoryOptions, + ); + if (!inventory.ok) return inventory; + const parsed = z.array(analysisStringSchema).safeParse(inventory.value); + if (!parsed.success) + return err( + new AnalysisOutputError( + operation, + "Hopper search inventory is malformed", + ), + ); + if (options.signal?.aborted) + return err(new AnalysisCancelledError(operation)); + await options.progress + ?.report({ + phase: "hopper_regex", + completed: 0.5, + total: 1, + message: "Matching inventory in cancellable regex worker", + }) + .catch(() => undefined); + if (options.signal?.aborted) + return err(new AnalysisCancelledError(operation)); + let worker: Worker; + try { + worker = new Worker(new URL("./HopperRegexWorker.js", import.meta.url), { + workerData: { + pattern: input.data.pattern, + caseSensitive: input.data.case_sensitive, + values: parsed.data.map((item) => item.value), + }, + }); + } catch (cause: unknown) { + return err( + new AnalysisOutputError( + operation, + `Cannot start regex worker: ${cause instanceof Error ? cause.message : String(cause)}`, + { cause }, + ), + ); + } + try { + return await new Promise>((resolve) => { + const finish = (result: Result): void => { + clearTimeout(timer); + options.signal?.removeEventListener("abort", onAbort); + resolve(result); + }; + const onAbort = (): void => + finish(err(new AnalysisCancelledError(operation))); + const timer = setTimeout( + () => + finish( + err( + new AnalysisResourceConstraintError( + operation, + "cpu", + "Regex matching exceeded its supervised deadline; the Hopper API thread remains available.", + { matching_deadline_ms: MATCHING_DEADLINE_MS }, + { + remediationAction: + "Simplify the ECMAScript Unicode regex or use literal mode, then retry in the same session.", + }, + ), + ), + ), + MATCHING_DEADLINE_MS, + ); + options.signal?.addEventListener("abort", onAbort, { once: true }); + if (options.signal?.aborted) onAbort(); + worker.once("message", (message: unknown) => { + const reply = replySchema.safeParse(message); + if (!reply.success) + return finish( + err( + new AnalysisOutputError( + operation, + "Regex worker reply is malformed", + ), + ), + ); + if (reply.data.status === "invalid_pattern") + return finish( + err( + new HopperRemoteError( + -32000, + `Invalid regex pattern: ${reply.data.reason}`, + { diagnosticType: "invalid_request", operation }, + ), + ), + ); + const indices = reply.data.indices; + if ( + indices.some( + (index, position) => + index >= parsed.data.length || + (position > 0 && index <= (indices[position - 1] ?? -1)), + ) + ) + return finish( + err( + new AnalysisOutputError( + operation, + "Regex worker returned invalid inventory indices", + ), + ), + ); + const selected = new Set(indices); + finish(ok(parsed.data.filter((_, index) => selected.has(index)))); + }); + worker.once("error", (cause: Error) => + finish( + err( + new AnalysisOutputError( + operation, + `Regex worker failed: ${cause.message}`, + { cause }, + ), + ), + ), + ); + worker.once("exit", (code) => + finish( + err( + new AnalysisOutputError( + operation, + `Regex worker exited without a result (${code})`, + ), + ), + ), + ); + }); + } finally { + await worker.terminate(); + } + } +} diff --git a/src/hopper/HopperRegexWorker.ts b/src/hopper/HopperRegexWorker.ts new file mode 100644 index 000000000..71fd79b50 --- /dev/null +++ b/src/hopper/HopperRegexWorker.ts @@ -0,0 +1,27 @@ +import { parentPort, workerData } from "node:worker_threads"; +import { z } from "zod"; + +const input = z + .strictObject({ + pattern: z.string().min(1), + caseSensitive: z.boolean(), + values: z.array(z.string()), + }) + .parse(workerData); + +let expression: RegExp | undefined; +try { + expression = new RegExp(input.pattern, input.caseSensitive ? "u" : "iu"); +} catch (cause: unknown) { + parentPort?.postMessage({ + status: "invalid_pattern", + reason: cause instanceof Error ? cause.message : String(cause), + }); +} +if (expression !== undefined) { + const indices = []; + for (const [index, value] of input.values.entries()) { + if (expression.test(value)) indices.push(index); + } + parentPort?.postMessage({ status: "matched", indices }); +}