fix(javascript): yield inside semantic analysis and large projection passes (#1542)

* fix(javascript): yield inside semantic analysis and large projection passes

After #1525 the longest event-loop blocks were single stages on very large
files: semantic analysis of Obsidian's 3.7 MB app.js ran 5.8 s in one piece,
and its object projection 1.8 s.

- analyzeParsedJavaScriptSemanticsSteps yields between the analysis phases
  and between the eight derived collectors; the synchronous function drains
  it, so results are unchanged.
- Per-file analysis yields between open-receiver classification and static
  analysis and drives the semantic steps.
- Semantic node creation, definitions and references, and the object and
  frontier passes yield every 64 items. Node creation keeps source order,
  which the per-file node budget depends on.

On obsidian.asar the longest block falls from 5,355 ms to 907 ms with the
same total time (40.6 s to 39.5 s). Graph IDs and complete result digests
match main on five inputs. Remaining blocks up to about 0.9 s are single
AST traversal passes inside the analyzers.

Refs #1462

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

* refactor(javascript): drop the unused synchronous derived-semantics wrapper

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

---------

Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>

src/application/javascript/JavaScriptArtifactAnalysis.ts

@@ -4,7 +4,7 @@ import { setImmediate } from "node:timers/promises";
 import { resolveJavaScriptSourceMapReference } from "../../domain/javascript/javascriptSourceMapPaths.js";
 import { analyzeParsedJavaScriptStaticSource } from "../../domain/javascript/javascriptStaticAnalysis.js";
 import {
-  analyzeParsedJavaScriptSemantics,
+  analyzeParsedJavaScriptSemanticsSteps,
   classifyParsedJavaScriptOpenReceivers,
 } from "../../domain/javascript/javascriptSemanticAnalysis.js";
 import type { JavaScriptSemanticIr } from "../../domain/javascript/javascriptSemanticIr.js";
@@ -189,6 +189,7 @@ function* analyzeArtifactFileSteps<
     return;
   }
   const openReceiverFacts = classifyParsedJavaScriptOpenReceivers(parsed);
+  yield;
   const analysis = analyzeParsedJavaScriptStaticSource(
     file.text.value,
     parsed,
@@ -198,7 +199,7 @@ function* analyzeArtifactFileSteps<
   yield;
   const semantics =
     analysis.parse_status === "complete" || analysis.parse_status === "partial"
-      ? analyzeParsedJavaScriptSemantics(parsed)
+      ? yield* analyzeParsedJavaScriptSemanticsSteps(parsed)
       : null;
   const projected =
     semantics === null

src/application/javascript/JavaScriptSemanticGraphBuilder.ts

@@ -287,38 +287,50 @@ function* projectFileSteps(
   });
   if (moduleNode === null) return [];
   state.roots.add(moduleNode.node_id);
-  const callableNodes = new Map(
-    ir.callables.flatMap((callable) => {
-      const node = retainNode(state, file, {
-        kind: "function",
-        roleKey: `callable:${callable.callableId}`,
-        location: callable.location,
-        label: callable.name,
-        functionNodeId: null,
-        // The label is display text; keep the exact name, which may be "".
-        properties: { name: callable.name },
-      });
-      return node === null ? [] : [[callable.callableId, node] as const];
-    }),
-  );
+  // Node creation keeps source order, which the per-file node budget relies
+  // on; yields between nodes do not change it.
+  const callableNodes = new Map<string, JavaScriptSemanticGraphNode>();
+  for (const [index, callable] of ir.callables.entries()) {
+    if (index % 64 === 0) yield;
+    const node = retainNode(state, file, {
+      kind: "function",
+      roleKey: `callable:${callable.callableId}`,
+      location: callable.location,
+      label: callable.name,
+      functionNodeId: null,
+      // The label is display text; keep the exact name, which may be "".
+      properties: { name: callable.name },
+    });
+    if (node !== null) callableNodes.set(callable.callableId, node);
+  }
   const callableOwnerAt = createSemanticCallableOwnerLookup(ir, callableNodes);
-  const bindingNodes = new Map(
-    ir.bindings.flatMap((binding) => {
-      const location = binding.definitions[0]?.location ?? null;
-      const kind = binding.kind === "parameter" ? "parameter" : "binding";
-      const owner = callableOwnerAt(location);
-      const node = retainNode(state, file, {
-        kind,
-        roleKey: `binding:${binding.bindingId}`,
-        location,
-        label: binding.name,
-        functionNodeId: owner?.node_id ?? null,
-      });
-      return node === null ? [] : [[binding.bindingId, node] as const];
-    }),
+  const bindingNodes = new Map<string, JavaScriptSemanticGraphNode>();
+  for (const [index, binding] of ir.bindings.entries()) {
+    if (index % 64 === 0) yield;
+    const location = binding.definitions[0]?.location ?? null;
+    const kind = binding.kind === "parameter" ? "parameter" : "binding";
+    const owner = callableOwnerAt(location);
+    const node = retainNode(state, file, {
+      kind,
+      roleKey: `binding:${binding.bindingId}`,
+      location,
+      label: binding.name,
+      functionNodeId: owner?.node_id ?? null,
+    });
+    if (node !== null) bindingNodes.set(binding.bindingId, node);
+  }
+  const returnSiteNodes = yield* createReturnSiteNodes(
+    file,
+    ir,
+    callableNodes,
+    state,
+  );
+  const callSiteNodes = yield* createCallSiteNodes(
+    file,
+    ir,
+    callableNodes,
+    state,
   );
-  const returnSiteNodes = createReturnSiteNodes(file, ir, callableNodes, state);
-  const callSiteNodes = createCallSiteNodes(file, ir, callableNodes, state);
   const definitions: Omit<FileContext, "referenceNodesWithin"> = {
     file,
     ir,
@@ -339,8 +351,7 @@ function* projectFileSteps(
     callableOwnerAt,
     callSiteAt: createSemanticCallSiteLookup(ir, callSiteNodes),
   };
-  yield;
-  projectDefinitionsAndReferences(definitions);
+  yield* projectDefinitionsAndReferences(definitions);
   const context: FileContext = {
     ...definitions,
     referenceNodesWithin: createSemanticNodeRangeLookup(
@@ -351,7 +362,8 @@ function* projectFileSteps(
   // keeps projection order and results unchanged.
   for (const project of FILE_PROJECTION_PASSES) {
     yield;
-    project(context);
+    const steps = project(context);
+    if (steps !== undefined) while (steps.next().done !== true) yield;
   }
   yield;
   return projectSemanticFunctionFingerprints(
@@ -362,14 +374,15 @@ function* projectFileSteps(
   );
 }
 
-const createReturnSiteNodes = (
+function* createReturnSiteNodes(
   file: JavaScriptArtifactFile,
   ir: JavaScriptSemanticIr,
   callables: ReadonlyMap<string, JavaScriptSemanticGraphNode>,
   state: BuilderState,
-): Map<string, JavaScriptSemanticGraphNode> => {
+): Generator<void, Map<string, JavaScriptSemanticGraphNode>> {
   const result = new Map<string, JavaScriptSemanticGraphNode>();
-  for (const callable of ir.callables) {
+  for (const [index, callable] of ir.callables.entries()) {
+    if (index % 64 === 0) yield;
     const owner = callables.get(callable.callableId);
     if (owner === undefined) continue;
     for (const site of callable.returnSites) {
@@ -385,16 +398,17 @@ const createReturnSiteNodes = (
     }
   }
   return result;
-};
+}
 
-const createCallSiteNodes = (
+function* createCallSiteNodes(
   file: JavaScriptArtifactFile,
   ir: JavaScriptSemanticIr,
   callables: ReadonlyMap<string, JavaScriptSemanticGraphNode>,
   state: BuilderState,
-): Map<string, JavaScriptSemanticGraphNode> => {
+): Generator<void, Map<string, JavaScriptSemanticGraphNode>> {
   const result = new Map<string, JavaScriptSemanticGraphNode>();
-  for (const call of ir.callSites) {
+  for (const [index, call] of ir.callSites.entries()) {
+    if (index % 64 === 0) yield;
     const owner =
       call.callerCallableId === null
         ? null
@@ -409,12 +423,13 @@ const createCallSiteNodes = (
     if (node !== null) result.set(call.callSiteId, node);
   }
   return result;
-};
+}
 
-const projectDefinitionsAndReferences = (
+function* projectDefinitionsAndReferences(
   context: Omit<FileContext, "referenceNodesWithin">,
-): void => {
-  for (const binding of context.ir.bindings) {
+): Generator<void, void> {
+  for (const [bindingIndex, binding] of context.ir.bindings.entries()) {
+    if (bindingIndex % 64 === 0) yield;
     const bindingNode = context.bindingNodes.get(binding.bindingId);
     if (bindingNode === undefined) continue;
     for (const [index, definition] of binding.definitions.entries()) {
@@ -434,6 +449,7 @@ const projectDefinitionsAndReferences = (
     }
   }
   for (const [index, reference] of context.ir.references.entries()) {
+    if (index % 64 === 0) yield;
     const expression = retainNode(context.state, context.file, {
       kind: "expression",
       roleKey: `reference:${String(index)}:${reference.role}:${reference.name}`,
@@ -464,7 +480,7 @@ const projectDefinitionsAndReferences = (
         resolution: "resolved",
       });
   }
-};
+}
 
 const projectCalls = (context: FileContext): void => {
   for (const call of context.ir.callSites) {
@@ -551,7 +567,9 @@ const createArgumentNode = (
   });
 
 /** Per-file projection passes, run in this order with a yield between each. */
-const FILE_PROJECTION_PASSES: readonly ((context: FileContext) => void)[] = [
+const FILE_PROJECTION_PASSES: readonly ((
+  context: FileContext,
+) => Iterator<void, void> | void)[] = [
   projectSemanticValues,
   projectSemanticObjects,
   projectCalls,

src/application/javascript/JavaScriptSemanticGraphFlowProjection.ts

@@ -240,10 +240,11 @@ export const projectSemanticClosureCaptures = (
 };
 
 /** Retain bounded unresolved dynamic-call, dynamic-property and dynamic-scope frontiers. */
-export const projectSemanticFrontiers = (
+export function* projectSemanticFrontiers(
   context: SemanticFlowProjectionContext,
-): void => {
-  for (const frontier of context.ir.frontiers) {
+): Generator<void, void> {
+  for (const [index, frontier] of context.ir.frontiers.entries()) {
+    if (index % 64 === 0) yield;
     // Dynamic environments leave identifier reads and writes unresolved.
     const family =
       frontier.kind === "dynamic-call"
@@ -275,4 +276,4 @@ export const projectSemanticFrontiers = (
     );
     addSemanticGraphUnknown(context.state, unknown);
   }
-};
+}

src/application/javascript/JavaScriptSemanticGraphObjectProjection.ts

@@ -20,13 +20,14 @@ import type { SemanticFlowProjectionContext } from "./JavaScriptSemanticGraphFlo
 import { semanticPropertySlot } from "./JavaScriptSemanticGraphValueProjection.js";
 
 /** Project static object reads, writes, spreads, and destructuring. */
-export const projectSemanticObjects = (
+export function* projectSemanticObjects(
   context: SemanticFlowProjectionContext,
-): void => {
+): Generator<void, void> {
   const values = new Map(
     context.ir.bindings.map((binding) => [binding.bindingId, binding.value]),
   );
-  for (const operation of context.ir.objectOperations) {
+  for (const [index, operation] of context.ir.objectOperations.entries()) {
+    if (index % 64 === 0) yield;
     const occurrence = addObjectOccurrence(context, operation);
     const target = objectTarget(context, values, operation);
     const ends = objectRelationEnds(context, operation, occurrence, target);
@@ -43,7 +44,7 @@ export const projectSemanticObjects = (
     if (!resolved)
       addObjectUnknown(context, operation, target.slot ?? occurrence, target);
   }
-};
+}
 
 const OBJECT_RELATIONS = {
   read: "reads-property",

src/domain/javascript/javascriptSemanticAnalysis.ts

@@ -38,7 +38,7 @@ import {
   collectSemanticReturns,
   resolveSemanticModuleCallables,
 } from "./javascriptSemanticReturns.js";
-import { collectJavaScriptDerivedSemantics } from "./javascriptSemanticDerivedAnalysis.js";
+import { collectJavaScriptDerivedSemanticsSteps } from "./javascriptSemanticDerivedAnalysis.js";
 import { range } from "./javascriptStaticAnalysisHelpers.js";
 import { propertyName } from "./javascriptAstValues.js";
 import { semanticCoverage } from "./javascriptSemanticCoverage.js";
@@ -107,18 +107,40 @@ export const analyzeParsedJavaScriptReferences = (
 export const analyzeParsedJavaScriptSemantics = (
   file: ParsedJavaScriptSource,
 ): JavaScriptSemanticIr => {
+  const steps = analyzeParsedJavaScriptSemanticsSteps(file);
+  for (;;) {
+    const step = steps.next();
+    if (step.done === true) return step.value;
+  }
+};
+
+/**
+ * Recover semantics from a parsed artifact with a yield between analysis
+ * phases, so a caller can serve control messages within one large file.
+ * Each phase completes before the next starts; results equal the synchronous
+ * form.
+ */
+export function* analyzeParsedJavaScriptSemanticsSteps(
+  file: ParsedJavaScriptSource,
+): Generator<void, JavaScriptSemanticIr> {
   const state = createState(file.program);
   collectDefinitions(file.program, state);
+  yield;
   collectSemanticMemberMutations(file.program, state);
+  yield;
   traverseJavaScriptAst(file.program, {
     enter: (node) => collectSemanticModuleLink(node, state),
   });
+  yield;
   const references = collectSemanticReferences(file.program, state);
+  yield;
   const parserPartial = file.errors.length > 0;
   const bindings = immutableSemanticBindings(state);
+  yield;
   const callables = collectSemanticReturns(file.program, state, parserPartial);
+  yield;
   const moduleLinks = resolveSemanticModuleCallables(state, callables);
-  const derived = collectJavaScriptDerivedSemantics(
+  const derived = yield* collectJavaScriptDerivedSemanticsSteps(
     file.program,
     state,
     callables,
@@ -154,7 +176,7 @@ export const analyzeParsedJavaScriptSemantics = (
       "Cross-function mutation and dynamic property resolution remain unknown.",
     ],
   };
-};
+}
 
 /** Receiver identity facts for the overloaded `.open` syntax only. */
 export type JavaScriptOpenReceiverFact =

src/domain/javascript/javascriptSemanticAnalysisSteps.test.ts

@@ -0,0 +1,34 @@
+import { expect, it } from "vitest";
+
+import {
+  analyzeParsedJavaScriptSemantics,
+  analyzeParsedJavaScriptSemanticsSteps,
+} from "./javascriptSemanticAnalysis.js";
+import { parseJavaScriptSource } from "./javascriptSourceParser.js";
+
+const source = `
+import { readFile } from "node:fs";
+const shared = { token: "TOKEN" };
+export function load(path) {
+  return new Promise((resolve) => setTimeout(() => resolve(readFile(path)), 1));
+}
+export const box = { get() { return shared; } };
+box.get().value = process.env.MODE ?? "default";
+`;
+
+it("yields between semantic phases and returns the synchronous result (#1462)", () => {
+  const parsed = parseJavaScriptSource(source, "main.js");
+  if (parsed === null) throw new Error("Expected parsed source");
+  const steps = analyzeParsedJavaScriptSemanticsSteps(parsed);
+  let yields = 0;
+  let step = steps.next();
+  while (step.done !== true) {
+    yields += 1;
+    step = steps.next();
+  }
+  // Every top-level phase and derived collector is a separate step.
+  expect(yields).toBeGreaterThanOrEqual(13);
+  const reparsed = parseJavaScriptSource(source, "main.js");
+  if (reparsed === null) throw new Error("Expected parsed source");
+  expect(step.value).toEqual(analyzeParsedJavaScriptSemantics(reparsed));
+});

src/domain/javascript/javascriptSemanticDerivedAnalysis.ts

@@ -36,43 +36,54 @@ type DerivedSemanticAnalysis = Pick<
 >;
 
 /** Compose bounded semantic passes after lexical definitions are available. */
-export const collectJavaScriptDerivedSemantics = (
+/**
+ * Collect derived semantics with a yield between collectors. Each collector
+ * completes before the next starts, so results equal the synchronous form.
+ */
+export function* collectJavaScriptDerivedSemanticsSteps(
   program: t.Program,
   state: JavaScriptSemanticAnalysisState,
   callables: readonly JavaScriptSemanticCallable[],
   parserPartial: boolean,
-): DerivedSemanticAnalysis => {
+): Generator<void, DerivedSemanticAnalysis> {
   const calls = collectJavaScriptSemanticCalls(program, state, callables);
+  yield;
   const promiseOperations = collectJavaScriptSemanticPromises(
     program,
     state,
     callables,
   );
+  yield;
   const asyncEffects = collectJavaScriptSemanticAsyncEffects(
     program,
     state,
     callables,
   );
+  yield;
   const childProcesses = collectJavaScriptSemanticChildProcesses(
     program,
     state,
     callables,
   );
+  yield;
   const dataEffects = collectJavaScriptSemanticDataEffects(
     program,
     state,
     callables,
   );
+  yield;
   const resourceOperations = collectJavaScriptSemanticResources(
     program,
     state,
     callables,
   );
+  yield;
   const objectOperations = collectJavaScriptSemanticObjects(
     program,
     state,
     callables,
   );
+  yield;
   const functionFingerprints = collectJavaScriptSemanticFingerprints({
     state,
     callables,
@@ -96,4 +107,4 @@ export const collectJavaScriptDerivedSemantics = (
     objectOperations,
     functionFingerprints,
   };
-};
+}