From ca7ef69c0ceed367470cb2d8e867a33e61706b02 Mon Sep 17 00:00:00 2001 From: Kaoru0822 Date: Sat, 10 Oct 2026 04:20:50 -0700 Subject: [PATCH] fix(javascript): yield inside semantic analysis and large projection passes (#1542) * fix(javascript): yield inside semantic analysis and large projection passes After #1525 the longest event-loop blocks were single stages on very large files: semantic analysis of Obsidian's 3.7 MB app.js ran 5.8 s in one piece, and its object projection 1.8 s. - analyzeParsedJavaScriptSemanticsSteps yields between the analysis phases and between the eight derived collectors; the synchronous function drains it, so results are unchanged. - Per-file analysis yields between open-receiver classification and static analysis and drives the semantic steps. - Semantic node creation, definitions and references, and the object and frontier passes yield every 64 items. Node creation keeps source order, which the per-file node budget depends on. On obsidian.asar the longest block falls from 5,355 ms to 907 ms with the same total time (40.6 s to 39.5 s). Graph IDs and complete result digests match main on five inputs. Remaining blocks up to about 0.9 s are single AST traversal passes inside the analyzers. Refs #1462 Co-Authored-By: Claude Opus 5.5 * refactor(javascript): drop the unused synchronous derived-semantics wrapper Co-Authored-By: Claude Opus 5.5 --------- Co-authored-by: Claude Opus 5.5 --- .../javascript/JavaScriptArtifactAnalysis.ts | 5 +- .../JavaScriptSemanticGraphBuilder.ts | 110 ++++++++++-------- .../JavaScriptSemanticGraphFlowProjection.ts | 9 +- ...JavaScriptSemanticGraphObjectProjection.ts | 9 +- .../javascript/javascriptSemanticAnalysis.ts | 28 ++++- .../javascriptSemanticAnalysisSteps.test.ts | 34 ++++++ .../javascriptSemanticDerivedAnalysis.ts | 17 ++- 7 files changed, 150 insertions(+), 62 deletions(-) create mode 100644 src/domain/javascript/javascriptSemanticAnalysisSteps.test.ts diff --git a/src/application/javascript/JavaScriptArtifactAnalysis.ts b/src/application/javascript/JavaScriptArtifactAnalysis.ts index 3fac39dbb..72fb67edd 100644 --- a/src/application/javascript/JavaScriptArtifactAnalysis.ts +++ b/src/application/javascript/JavaScriptArtifactAnalysis.ts @@ -4,7 +4,7 @@ import { setImmediate } from "node:timers/promises"; import { resolveJavaScriptSourceMapReference } from "../../domain/javascript/javascriptSourceMapPaths.js"; import { analyzeParsedJavaScriptStaticSource } from "../../domain/javascript/javascriptStaticAnalysis.js"; import { - analyzeParsedJavaScriptSemantics, + analyzeParsedJavaScriptSemanticsSteps, classifyParsedJavaScriptOpenReceivers, } from "../../domain/javascript/javascriptSemanticAnalysis.js"; import type { JavaScriptSemanticIr } from "../../domain/javascript/javascriptSemanticIr.js"; @@ -189,6 +189,7 @@ function* analyzeArtifactFileSteps< return; } const openReceiverFacts = classifyParsedJavaScriptOpenReceivers(parsed); + yield; const analysis = analyzeParsedJavaScriptStaticSource( file.text.value, parsed, @@ -198,7 +199,7 @@ function* analyzeArtifactFileSteps< yield; const semantics = analysis.parse_status === "complete" || analysis.parse_status === "partial" - ? analyzeParsedJavaScriptSemantics(parsed) + ? yield* analyzeParsedJavaScriptSemanticsSteps(parsed) : null; const projected = semantics === null diff --git a/src/application/javascript/JavaScriptSemanticGraphBuilder.ts b/src/application/javascript/JavaScriptSemanticGraphBuilder.ts index 770930de7..a0cc8ccf8 100644 --- a/src/application/javascript/JavaScriptSemanticGraphBuilder.ts +++ b/src/application/javascript/JavaScriptSemanticGraphBuilder.ts @@ -287,38 +287,50 @@ function* projectFileSteps( }); if (moduleNode === null) return []; state.roots.add(moduleNode.node_id); - const callableNodes = new Map( - ir.callables.flatMap((callable) => { - const node = retainNode(state, file, { - kind: "function", - roleKey: `callable:${callable.callableId}`, - location: callable.location, - label: callable.name, - functionNodeId: null, - // The label is display text; keep the exact name, which may be "". - properties: { name: callable.name }, - }); - return node === null ? [] : [[callable.callableId, node] as const]; - }), - ); + // Node creation keeps source order, which the per-file node budget relies + // on; yields between nodes do not change it. + const callableNodes = new Map(); + for (const [index, callable] of ir.callables.entries()) { + if (index % 64 === 0) yield; + const node = retainNode(state, file, { + kind: "function", + roleKey: `callable:${callable.callableId}`, + location: callable.location, + label: callable.name, + functionNodeId: null, + // The label is display text; keep the exact name, which may be "". + properties: { name: callable.name }, + }); + if (node !== null) callableNodes.set(callable.callableId, node); + } const callableOwnerAt = createSemanticCallableOwnerLookup(ir, callableNodes); - const bindingNodes = new Map( - ir.bindings.flatMap((binding) => { - const location = binding.definitions[0]?.location ?? null; - const kind = binding.kind === "parameter" ? "parameter" : "binding"; - const owner = callableOwnerAt(location); - const node = retainNode(state, file, { - kind, - roleKey: `binding:${binding.bindingId}`, - location, - label: binding.name, - functionNodeId: owner?.node_id ?? null, - }); - return node === null ? [] : [[binding.bindingId, node] as const]; - }), + const bindingNodes = new Map(); + for (const [index, binding] of ir.bindings.entries()) { + if (index % 64 === 0) yield; + const location = binding.definitions[0]?.location ?? null; + const kind = binding.kind === "parameter" ? "parameter" : "binding"; + const owner = callableOwnerAt(location); + const node = retainNode(state, file, { + kind, + roleKey: `binding:${binding.bindingId}`, + location, + label: binding.name, + functionNodeId: owner?.node_id ?? null, + }); + if (node !== null) bindingNodes.set(binding.bindingId, node); + } + const returnSiteNodes = yield* createReturnSiteNodes( + file, + ir, + callableNodes, + state, + ); + const callSiteNodes = yield* createCallSiteNodes( + file, + ir, + callableNodes, + state, ); - const returnSiteNodes = createReturnSiteNodes(file, ir, callableNodes, state); - const callSiteNodes = createCallSiteNodes(file, ir, callableNodes, state); const definitions: Omit = { file, ir, @@ -339,8 +351,7 @@ function* projectFileSteps( callableOwnerAt, callSiteAt: createSemanticCallSiteLookup(ir, callSiteNodes), }; - yield; - projectDefinitionsAndReferences(definitions); + yield* projectDefinitionsAndReferences(definitions); const context: FileContext = { ...definitions, referenceNodesWithin: createSemanticNodeRangeLookup( @@ -351,7 +362,8 @@ function* projectFileSteps( // keeps projection order and results unchanged. for (const project of FILE_PROJECTION_PASSES) { yield; - project(context); + const steps = project(context); + if (steps !== undefined) while (steps.next().done !== true) yield; } yield; return projectSemanticFunctionFingerprints( @@ -362,14 +374,15 @@ function* projectFileSteps( ); } -const createReturnSiteNodes = ( +function* createReturnSiteNodes( file: JavaScriptArtifactFile, ir: JavaScriptSemanticIr, callables: ReadonlyMap, state: BuilderState, -): Map => { +): Generator> { const result = new Map(); - for (const callable of ir.callables) { + for (const [index, callable] of ir.callables.entries()) { + if (index % 64 === 0) yield; const owner = callables.get(callable.callableId); if (owner === undefined) continue; for (const site of callable.returnSites) { @@ -385,16 +398,17 @@ const createReturnSiteNodes = ( } } return result; -}; +} -const createCallSiteNodes = ( +function* createCallSiteNodes( file: JavaScriptArtifactFile, ir: JavaScriptSemanticIr, callables: ReadonlyMap, state: BuilderState, -): Map => { +): Generator> { const result = new Map(); - for (const call of ir.callSites) { + for (const [index, call] of ir.callSites.entries()) { + if (index % 64 === 0) yield; const owner = call.callerCallableId === null ? null @@ -409,12 +423,13 @@ const createCallSiteNodes = ( if (node !== null) result.set(call.callSiteId, node); } return result; -}; +} -const projectDefinitionsAndReferences = ( +function* projectDefinitionsAndReferences( context: Omit, -): void => { - for (const binding of context.ir.bindings) { +): Generator { + for (const [bindingIndex, binding] of context.ir.bindings.entries()) { + if (bindingIndex % 64 === 0) yield; const bindingNode = context.bindingNodes.get(binding.bindingId); if (bindingNode === undefined) continue; for (const [index, definition] of binding.definitions.entries()) { @@ -434,6 +449,7 @@ const projectDefinitionsAndReferences = ( } } for (const [index, reference] of context.ir.references.entries()) { + if (index % 64 === 0) yield; const expression = retainNode(context.state, context.file, { kind: "expression", roleKey: `reference:${String(index)}:${reference.role}:${reference.name}`, @@ -464,7 +480,7 @@ const projectDefinitionsAndReferences = ( resolution: "resolved", }); } -}; +} const projectCalls = (context: FileContext): void => { for (const call of context.ir.callSites) { @@ -551,7 +567,9 @@ const createArgumentNode = ( }); /** Per-file projection passes, run in this order with a yield between each. */ -const FILE_PROJECTION_PASSES: readonly ((context: FileContext) => void)[] = [ +const FILE_PROJECTION_PASSES: readonly (( + context: FileContext, +) => Iterator | void)[] = [ projectSemanticValues, projectSemanticObjects, projectCalls, diff --git a/src/application/javascript/JavaScriptSemanticGraphFlowProjection.ts b/src/application/javascript/JavaScriptSemanticGraphFlowProjection.ts index be801ab86..e98526160 100644 --- a/src/application/javascript/JavaScriptSemanticGraphFlowProjection.ts +++ b/src/application/javascript/JavaScriptSemanticGraphFlowProjection.ts @@ -240,10 +240,11 @@ export const projectSemanticClosureCaptures = ( }; /** Retain bounded unresolved dynamic-call, dynamic-property and dynamic-scope frontiers. */ -export const projectSemanticFrontiers = ( +export function* projectSemanticFrontiers( context: SemanticFlowProjectionContext, -): void => { - for (const frontier of context.ir.frontiers) { +): Generator { + for (const [index, frontier] of context.ir.frontiers.entries()) { + if (index % 64 === 0) yield; // Dynamic environments leave identifier reads and writes unresolved. const family = frontier.kind === "dynamic-call" @@ -275,4 +276,4 @@ export const projectSemanticFrontiers = ( ); addSemanticGraphUnknown(context.state, unknown); } -}; +} diff --git a/src/application/javascript/JavaScriptSemanticGraphObjectProjection.ts b/src/application/javascript/JavaScriptSemanticGraphObjectProjection.ts index 1b71b2784..337899f83 100644 --- a/src/application/javascript/JavaScriptSemanticGraphObjectProjection.ts +++ b/src/application/javascript/JavaScriptSemanticGraphObjectProjection.ts @@ -20,13 +20,14 @@ import type { SemanticFlowProjectionContext } from "./JavaScriptSemanticGraphFlo import { semanticPropertySlot } from "./JavaScriptSemanticGraphValueProjection.js"; /** Project static object reads, writes, spreads, and destructuring. */ -export const projectSemanticObjects = ( +export function* projectSemanticObjects( context: SemanticFlowProjectionContext, -): void => { +): Generator { const values = new Map( context.ir.bindings.map((binding) => [binding.bindingId, binding.value]), ); - for (const operation of context.ir.objectOperations) { + for (const [index, operation] of context.ir.objectOperations.entries()) { + if (index % 64 === 0) yield; const occurrence = addObjectOccurrence(context, operation); const target = objectTarget(context, values, operation); const ends = objectRelationEnds(context, operation, occurrence, target); @@ -43,7 +44,7 @@ export const projectSemanticObjects = ( if (!resolved) addObjectUnknown(context, operation, target.slot ?? occurrence, target); } -}; +} const OBJECT_RELATIONS = { read: "reads-property", diff --git a/src/domain/javascript/javascriptSemanticAnalysis.ts b/src/domain/javascript/javascriptSemanticAnalysis.ts index 4b5ad0f72..50af0f2cb 100644 --- a/src/domain/javascript/javascriptSemanticAnalysis.ts +++ b/src/domain/javascript/javascriptSemanticAnalysis.ts @@ -38,7 +38,7 @@ import { collectSemanticReturns, resolveSemanticModuleCallables, } from "./javascriptSemanticReturns.js"; -import { collectJavaScriptDerivedSemantics } from "./javascriptSemanticDerivedAnalysis.js"; +import { collectJavaScriptDerivedSemanticsSteps } from "./javascriptSemanticDerivedAnalysis.js"; import { range } from "./javascriptStaticAnalysisHelpers.js"; import { propertyName } from "./javascriptAstValues.js"; import { semanticCoverage } from "./javascriptSemanticCoverage.js"; @@ -107,18 +107,40 @@ export const analyzeParsedJavaScriptReferences = ( export const analyzeParsedJavaScriptSemantics = ( file: ParsedJavaScriptSource, ): JavaScriptSemanticIr => { + const steps = analyzeParsedJavaScriptSemanticsSteps(file); + for (;;) { + const step = steps.next(); + if (step.done === true) return step.value; + } +}; + +/** + * Recover semantics from a parsed artifact with a yield between analysis + * phases, so a caller can serve control messages within one large file. + * Each phase completes before the next starts; results equal the synchronous + * form. + */ +export function* analyzeParsedJavaScriptSemanticsSteps( + file: ParsedJavaScriptSource, +): Generator { const state = createState(file.program); collectDefinitions(file.program, state); + yield; collectSemanticMemberMutations(file.program, state); + yield; traverseJavaScriptAst(file.program, { enter: (node) => collectSemanticModuleLink(node, state), }); + yield; const references = collectSemanticReferences(file.program, state); + yield; const parserPartial = file.errors.length > 0; const bindings = immutableSemanticBindings(state); + yield; const callables = collectSemanticReturns(file.program, state, parserPartial); + yield; const moduleLinks = resolveSemanticModuleCallables(state, callables); - const derived = collectJavaScriptDerivedSemantics( + const derived = yield* collectJavaScriptDerivedSemanticsSteps( file.program, state, callables, @@ -154,7 +176,7 @@ export const analyzeParsedJavaScriptSemantics = ( "Cross-function mutation and dynamic property resolution remain unknown.", ], }; -}; +} /** Receiver identity facts for the overloaded `.open` syntax only. */ export type JavaScriptOpenReceiverFact = diff --git a/src/domain/javascript/javascriptSemanticAnalysisSteps.test.ts b/src/domain/javascript/javascriptSemanticAnalysisSteps.test.ts new file mode 100644 index 000000000..9aaa71d93 --- /dev/null +++ b/src/domain/javascript/javascriptSemanticAnalysisSteps.test.ts @@ -0,0 +1,34 @@ +import { expect, it } from "vitest"; + +import { + analyzeParsedJavaScriptSemantics, + analyzeParsedJavaScriptSemanticsSteps, +} from "./javascriptSemanticAnalysis.js"; +import { parseJavaScriptSource } from "./javascriptSourceParser.js"; + +const source = ` +import { readFile } from "node:fs"; +const shared = { token: "TOKEN" }; +export function load(path) { + return new Promise((resolve) => setTimeout(() => resolve(readFile(path)), 1)); +} +export const box = { get() { return shared; } }; +box.get().value = process.env.MODE ?? "default"; +`; + +it("yields between semantic phases and returns the synchronous result (#1462)", () => { + const parsed = parseJavaScriptSource(source, "main.js"); + if (parsed === null) throw new Error("Expected parsed source"); + const steps = analyzeParsedJavaScriptSemanticsSteps(parsed); + let yields = 0; + let step = steps.next(); + while (step.done !== true) { + yields += 1; + step = steps.next(); + } + // Every top-level phase and derived collector is a separate step. + expect(yields).toBeGreaterThanOrEqual(13); + const reparsed = parseJavaScriptSource(source, "main.js"); + if (reparsed === null) throw new Error("Expected parsed source"); + expect(step.value).toEqual(analyzeParsedJavaScriptSemantics(reparsed)); +}); diff --git a/src/domain/javascript/javascriptSemanticDerivedAnalysis.ts b/src/domain/javascript/javascriptSemanticDerivedAnalysis.ts index d3e630ac9..545652a74 100644 --- a/src/domain/javascript/javascriptSemanticDerivedAnalysis.ts +++ b/src/domain/javascript/javascriptSemanticDerivedAnalysis.ts @@ -36,43 +36,54 @@ type DerivedSemanticAnalysis = Pick< >; /** Compose bounded semantic passes after lexical definitions are available. */ -export const collectJavaScriptDerivedSemantics = ( +/** + * Collect derived semantics with a yield between collectors. Each collector + * completes before the next starts, so results equal the synchronous form. + */ +export function* collectJavaScriptDerivedSemanticsSteps( program: t.Program, state: JavaScriptSemanticAnalysisState, callables: readonly JavaScriptSemanticCallable[], parserPartial: boolean, -): DerivedSemanticAnalysis => { +): Generator { const calls = collectJavaScriptSemanticCalls(program, state, callables); + yield; const promiseOperations = collectJavaScriptSemanticPromises( program, state, callables, ); + yield; const asyncEffects = collectJavaScriptSemanticAsyncEffects( program, state, callables, ); + yield; const childProcesses = collectJavaScriptSemanticChildProcesses( program, state, callables, ); + yield; const dataEffects = collectJavaScriptSemanticDataEffects( program, state, callables, ); + yield; const resourceOperations = collectJavaScriptSemanticResources( program, state, callables, ); + yield; const objectOperations = collectJavaScriptSemanticObjects( program, state, callables, ); + yield; const functionFingerprints = collectJavaScriptSemanticFingerprints({ state, callables, @@ -96,4 +107,4 @@ export const collectJavaScriptDerivedSemantics = ( objectOperations, functionFingerprints, }; -}; +}