From a26a1c73df019e060a5ca50208e71dad1eb55f1e Mon Sep 17 00:00:00 2001 From: morluto <76467478+morluto@users.noreply.github.com> Date: Wed, 22 Jul 2026 21:40:23 +0000 Subject: [PATCH 1/4] feat(javascript): add local semantic call flow --- src/domain/javascriptSemanticAnalysis.ts | 7 +- .../javascriptSemanticCallResolution.ts | 229 +++++++++++ src/domain/javascriptSemanticCalls.ts | 386 ++++++++++++++++++ src/domain/javascriptSemanticIr.ts | 81 +++- src/domain/javascriptSemanticProjection.ts | 10 +- src/domain/javascriptSemanticReturns.ts | 10 + src/domain/javascriptSemanticState.ts | 1 + tests/javascriptSemanticAnalysis.test.ts | 147 +++++++ 8 files changed, 864 insertions(+), 7 deletions(-) create mode 100644 src/domain/javascriptSemanticCallResolution.ts create mode 100644 src/domain/javascriptSemanticCalls.ts diff --git a/src/domain/javascriptSemanticAnalysis.ts b/src/domain/javascriptSemanticAnalysis.ts index 15818eba2..6e39f758d 100644 --- a/src/domain/javascriptSemanticAnalysis.ts +++ b/src/domain/javascriptSemanticAnalysis.ts @@ -35,6 +35,7 @@ import { collectSemanticReturns, resolveSemanticModuleCallables, } from "./javascriptSemanticReturns.js"; +import { collectJavaScriptSemanticCalls } from "./javascriptSemanticCalls.js"; import { compareCodePoints, propertyName, @@ -86,14 +87,16 @@ export const analyzeJavaScriptSemantics = ( const bindings = immutableSemanticBindings(state); const callables = collectSemanticReturns(file.program, state, parserPartial); const moduleLinks = resolveSemanticModuleCallables(state, callables); + const calls = collectJavaScriptSemanticCalls(file.program, state, callables); return { schema: "JavaScriptSemanticIR", - schemaVersion: 2, + schemaVersion: 3, scopes: immutableSemanticScopes(state), bindings, callables, references, moduleLinks, + ...calls, coverage: { status: state.limitsReached.size > 0 @@ -115,6 +118,7 @@ export const analyzeJavaScriptSemantics = ( : ["Semantic recovery stopped retaining facts at explicit limits."]), "Values and aliases were recovered from inert syntax only; no JavaScript was executed.", "Return sites include only direct callable returns; nested callable returns remain separate.", + "Local call, argument, return, and closure relations are static candidates and do not prove runtime invocation.", "Cross-function mutation and dynamic property resolution remain unknown.", ], }; @@ -139,6 +143,7 @@ const createState = ( scopeByNode: new WeakMap([[program, root]]), bindingsById: new Map(), callables: [], + callableNodesById: new Map(), moduleLinks: [], limitsReached: new Set(), omittedCount: 0, diff --git a/src/domain/javascriptSemanticCallResolution.ts b/src/domain/javascriptSemanticCallResolution.ts new file mode 100644 index 000000000..62873dc22 --- /dev/null +++ b/src/domain/javascriptSemanticCallResolution.ts @@ -0,0 +1,229 @@ +import * as t from "@babel/types"; + +import type { + JavaScriptSemanticArgumentFlow, + JavaScriptSemanticCallable, +} from "./javascriptSemanticIr.js"; +import { + reachSemanticLimit, + semanticCallableIdForNode, +} from "./javascriptSemanticProjection.js"; +import { + resolveSemanticBindingState, + type JavaScriptSemanticAnalysisState, + type JavaScriptSemanticBindingState, +} from "./javascriptSemanticState.js"; +import { compareCodePoints, range } from "./javascriptStaticAnalysisHelpers.js"; + +/** Resolution of one call target within the currently analyzed source. */ +export interface LocalCallableResolution { + readonly callableIds: readonly string[]; + readonly complete: boolean; + readonly reason: string; +} + +interface CallResolutionContext { + readonly state: JavaScriptSemanticAnalysisState; + readonly callableById: ReadonlyMap; + readonly seenBindings: ReadonlySet; + readonly depth: number; + readonly bindingCache: Map; +} + +/** Inputs needed to resolve one local call target. */ +export interface ResolveLocalCallablesInput { + readonly node: t.Node; + readonly state: JavaScriptSemanticAnalysisState; + readonly callableById: ReadonlyMap; + readonly seenBindings: ReadonlySet; + readonly depth: number; + readonly bindingCache: Map; +} + +/** Resolve bounded local callable candidates without following dynamic properties. */ +export const resolveLocalCallables = ( + input: ResolveLocalCallablesInput, +): LocalCallableResolution => + resolveCallables(input.node, { + state: input.state, + callableById: input.callableById, + seenBindings: input.seenBindings, + depth: input.depth, + bindingCache: input.bindingCache, + }); + +const resolveCallables = ( + node: t.Node, + context: CallResolutionContext, +): LocalCallableResolution => { + if (context.depth >= context.state.limits.maxValueDepth) { + reachSemanticLimit(context.state, "maxValueDepth"); + return { + callableIds: [], + complete: false, + reason: "Call target depth limit reached.", + }; + } + const direct = semanticCallableIdForNode(node); + if (direct !== null && context.callableById.has(direct)) + return { + callableIds: [direct], + complete: true, + reason: "Direct local callable.", + }; + if (t.isIdentifier(node)) return resolveIdentifier(node, context); + if ( + t.isTSAsExpression(node) || + t.isTSTypeAssertion(node) || + t.isTSNonNullExpression(node) + ) + return resolveCallables(node.expression, nestedContext(context)); + if (t.isConditionalExpression(node) || t.isLogicalExpression(node)) + return resolveAlternatives( + t.isConditionalExpression(node) ? node.consequent : node.left, + t.isConditionalExpression(node) ? node.alternate : node.right, + context, + ); + return { + callableIds: [], + complete: false, + reason: `Unsupported ${node.type} call target.`, + }; +}; + +const resolveIdentifier = ( + node: t.Identifier, + context: CallResolutionContext, +): LocalCallableResolution => { + const binding = resolveSemanticBindingState(context.state, node, node.name); + return binding === undefined + ? { + callableIds: [], + complete: false, + reason: `Unresolved call target ${node.name}.`, + } + : resolveBindingCallables(binding, nestedContext(context)); +}; + +const resolveAlternatives = ( + left: t.Node, + right: t.Node, + context: CallResolutionContext, +): LocalCallableResolution => { + const candidates = [left, right].map((candidate) => + resolveCallables(candidate, nestedContext(context)), + ); + return { + callableIds: uniqueCallableIds( + candidates.flatMap(({ callableIds }) => callableIds), + ), + complete: candidates.every(({ complete }) => complete), + reason: "Conditional or logical call target has multiple candidates.", + }; +}; + +const resolveBindingCallables = ( + binding: JavaScriptSemanticBindingState, + context: CallResolutionContext, +): LocalCallableResolution => { + const cached = context.bindingCache.get(binding.bindingId); + if (cached !== undefined) return cached; + if (context.seenBindings.has(binding.bindingId)) + return { + callableIds: [], + complete: false, + reason: `Call target alias cycle at ${binding.name}.`, + }; + if (binding.initializers.length === 0) + return missingBindingInitializer(binding); + const candidateContext = { + ...nestedContext(context), + seenBindings: new Set([...context.seenBindings, binding.bindingId]), + }; + const candidates = binding.initializers.map(({ node, projection }) => + projection.length === 0 + ? resolveCallables(node, candidateContext) + : { + callableIds: [], + complete: false, + reason: `Projected call target ${binding.name} is unresolved.`, + }, + ); + const callableIds = uniqueCallableIds( + candidates.flatMap((candidate) => candidate.callableIds), + ); + const resolution = { + callableIds, + complete: candidates.every(({ complete }) => complete), + reason: + callableIds.length === 1 && candidates.every(({ complete }) => complete) + ? "Unique local callable target." + : `Call target ${binding.name} has ambiguous or unsupported assignments.`, + }; + if (resolution.complete) + context.bindingCache.set(binding.bindingId, resolution); + return resolution; +}; + +const missingBindingInitializer = ( + binding: JavaScriptSemanticBindingState, +): LocalCallableResolution => ({ + callableIds: [], + complete: false, + reason: + binding.kind === "import" + ? `Call target ${binding.name} is imported from outside this source.` + : `Call target ${binding.name} has no local initializer.`, +}); + +/** Find all bindings introduced by one positional parameter pattern. */ +export const parameterBindings = ( + callable: JavaScriptSemanticCallable, + index: number, + state: JavaScriptSemanticAnalysisState, +): JavaScriptSemanticBindingState[] => { + if (callable.bodyScopeId === null) return []; + const scope = state.scopesById.get(callable.bodyScopeId); + if (scope === undefined) return []; + return [...scope.bindings.values()] + .filter( + (binding) => + binding.kind === "parameter" && + binding.definitions.some( + ({ kind, location }) => + kind === "parameter" && + parameterIndex(callable, location, state) === index, + ), + ) + .sort((left, right) => compareCodePoints(left.bindingId, right.bindingId)); +}; + +const parameterIndex = ( + callable: JavaScriptSemanticCallable, + location: JavaScriptSemanticArgumentFlow["parameterLocation"], + state: JavaScriptSemanticAnalysisState, +): number => { + const node = state.callableNodesById.get(callable.callableId); + if (!t.isFunction(node)) return -1; + return node.params.findIndex((parameter) => + rangeContains(range(parameter), location), + ); +}; + +const rangeContains = ( + outer: JavaScriptSemanticArgumentFlow["parameterLocation"], + inner: JavaScriptSemanticArgumentFlow["parameterLocation"], +): boolean => + (outer.start.line < inner.start.line || + (outer.start.line === inner.start.line && + outer.start.column <= inner.start.column)) && + (outer.end.line > inner.end.line || + (outer.end.line === inner.end.line && + outer.end.column >= inner.end.column)); + +const nestedContext = ( + context: CallResolutionContext, +): CallResolutionContext => ({ ...context, depth: context.depth + 1 }); + +const uniqueCallableIds = (values: readonly string[]): string[] => + [...new Set(values)].sort(compareCodePoints); diff --git a/src/domain/javascriptSemanticCalls.ts b/src/domain/javascriptSemanticCalls.ts new file mode 100644 index 000000000..31c2c8aa2 --- /dev/null +++ b/src/domain/javascriptSemanticCalls.ts @@ -0,0 +1,386 @@ +import * as t from "@babel/types"; + +import type { + JavaScriptSemanticArgumentFlow, + JavaScriptSemanticCallable, + JavaScriptSemanticCallReturnFlow, + JavaScriptSemanticCallSite, + JavaScriptSemanticClosureCapture, + JavaScriptSemanticFrontier, +} from "./javascriptSemanticIr.js"; +import { + reachSemanticLimit, + semanticCallableIdForNode, + semanticReferenceRole, + semanticStaticPropertyName, +} from "./javascriptSemanticProjection.js"; +import { + resolveSemanticBindingState, + type JavaScriptSemanticAnalysisState, + type JavaScriptSemanticBindingState, +} from "./javascriptSemanticState.js"; +import { + type LocalCallableResolution, + parameterBindings, + resolveLocalCallables, +} from "./javascriptSemanticCallResolution.js"; +import { traverseJavaScriptAst } from "./javascriptSemanticTraversal.js"; +import { range } from "./javascriptStaticAnalysisHelpers.js"; + +/** Local call, flow, capture, and unsupported-frontier facts. */ +export interface JavaScriptSemanticCallAnalysis { + readonly callSites: readonly JavaScriptSemanticCallSite[]; + readonly argumentFlows: readonly JavaScriptSemanticArgumentFlow[]; + readonly callReturnFlows: readonly JavaScriptSemanticCallReturnFlow[]; + readonly closureCaptures: readonly JavaScriptSemanticClosureCapture[]; + readonly frontiers: readonly JavaScriptSemanticFrontier[]; +} + +interface MutableCallAnalysis { + readonly callSites: JavaScriptSemanticCallSite[]; + readonly argumentFlows: JavaScriptSemanticArgumentFlow[]; + readonly callReturnFlows: JavaScriptSemanticCallReturnFlow[]; + readonly closureCaptures: JavaScriptSemanticClosureCapture[]; + readonly frontiers: JavaScriptSemanticFrontier[]; + retainedArguments: number; +} + +interface CallCollectionContext { + readonly state: JavaScriptSemanticAnalysisState; + readonly callableById: ReadonlyMap; + readonly output: MutableCallAnalysis; + readonly bindingResolutionCache: Map; + readonly parameterBindingCache: Map< + string, + readonly JavaScriptSemanticBindingState[] + >; +} + +/** Recover bounded direct-call and lexical-capture candidates from inert syntax. */ +export const collectJavaScriptSemanticCalls = ( + program: t.Program, + state: JavaScriptSemanticAnalysisState, + callables: readonly JavaScriptSemanticCallable[], +): JavaScriptSemanticCallAnalysis => { + const output: MutableCallAnalysis = { + callSites: [], + argumentFlows: [], + callReturnFlows: [], + closureCaptures: [], + frontiers: [], + retainedArguments: 0, + }; + const callableById = new Map( + callables.map((callable) => [callable.callableId, callable]), + ); + const context: CallCollectionContext = { + state, + callableById, + output, + bindingResolutionCache: new Map(), + parameterBindingCache: new Map(), + }; + const callableStack: JavaScriptSemanticCallable[] = []; + traverseJavaScriptAst(program, { + enter: (node, parent) => { + const callableId = semanticCallableIdForNode(node); + const callable = + callableId === null ? undefined : callableById.get(callableId); + if (callable !== undefined) callableStack.push(callable); + const owner = enclosingFunction(callableStack); + collectCapture(node, parent, owner, context); + collectDynamicProperty(node, owner, state, output); + if ( + t.isCallExpression(node) || + t.isOptionalCallExpression(node) || + t.isNewExpression(node) + ) + collectCallSite(node, owner, context); + }, + exit: (node) => { + const callableId = semanticCallableIdForNode(node); + if ( + callableId !== null && + callableStack.at(-1)?.callableId === callableId + ) + callableStack.pop(); + }, + }); + return { + callSites: output.callSites, + argumentFlows: output.argumentFlows, + callReturnFlows: output.callReturnFlows, + closureCaptures: output.closureCaptures, + frontiers: output.frontiers, + }; +}; + +const collectCallSite = ( + node: t.CallExpression | t.OptionalCallExpression | t.NewExpression, + owner: JavaScriptSemanticCallable | undefined, + context: CallCollectionContext, +): void => { + const { state, callableById, output } = context; + if (output.callSites.length >= state.limits.maxCallSites) { + reachSemanticLimit(state, "maxCallSites"); + return; + } + const resolution = resolveLocalCallables({ + node: node.callee, + state, + callableById, + seenBindings: new Set(), + depth: 0, + bindingCache: context.bindingResolutionCache, + }); + const callSiteId = semanticCallSiteId(node); + const argumentsValue = retainedArguments(node.arguments, state, output); + const site: JavaScriptSemanticCallSite = { + callSiteId, + kind: t.isNewExpression(node) ? "construct" : "call", + callerCallableId: owner?.callableId ?? null, + location: range(node), + calleeLocation: range(node.callee), + resolution: + resolution.callableIds.length === 1 && resolution.complete + ? "exact" + : resolution.callableIds.length > 0 + ? "ambiguous" + : "unresolved", + calleeCallableIds: resolution.callableIds, + arguments: argumentsValue, + }; + output.callSites.push(site); + if (site.resolution !== "exact") + addFrontier( + { + kind: "dynamic-call", + callableId: owner?.callableId ?? null, + location: range(node.callee), + reason: resolution.reason, + }, + state, + output, + ); + collectArgumentFlows(site, node, context); + if (site.kind === "call") + collectReturnFlows(site, callableById, state, output); +}; + +const retainedArguments = ( + nodes: readonly ( + | t.Expression + | t.SpreadElement + | t.JSXNamespacedName + | t.ArgumentPlaceholder + )[], + state: JavaScriptSemanticAnalysisState, + output: MutableCallAnalysis, +): JavaScriptSemanticCallSite["arguments"] => { + const retained: JavaScriptSemanticCallSite["arguments"][number][] = []; + for (const [index, node] of nodes.entries()) { + if (output.retainedArguments >= state.limits.maxCallArguments) { + reachSemanticLimit(state, "maxCallArguments"); + break; + } + output.retainedArguments += 1; + retained.push({ + index, + location: range(node), + spread: t.isSpreadElement(node), + }); + } + return retained; +}; + +const collectArgumentFlows = ( + site: JavaScriptSemanticCallSite, + node: t.CallExpression | t.OptionalCallExpression | t.NewExpression, + context: CallCollectionContext, +): void => { + const { state, callableById, output } = context; + const retainedIndexes = new Set(site.arguments.map(({ index }) => index)); + let positionIsExact = true; + for (const [index, argument] of node.arguments.entries()) { + if (t.isSpreadElement(argument)) { + positionIsExact = false; + continue; + } + if ( + !positionIsExact || + !retainedIndexes.has(index) || + !t.isExpression(argument) + ) + continue; + for (const callableId of site.calleeCallableIds) { + const callable = callableById.get(callableId); + if (callable === undefined) continue; + for (const parameter of cachedParameterBindings( + callable, + index, + context, + )) { + if (output.argumentFlows.length >= state.limits.maxArgumentFlows) { + reachSemanticLimit(state, "maxArgumentFlows"); + return; + } + const definition = parameter.definitions.find( + ({ kind }) => kind === "parameter", + ); + if (definition === undefined) continue; + output.argumentFlows.push({ + callSiteId: site.callSiteId, + argumentIndex: index, + argumentLocation: range(argument), + callableId, + parameterBindingId: parameter.bindingId, + parameterLocation: definition.location, + }); + } + } + } +}; + +const cachedParameterBindings = ( + callable: JavaScriptSemanticCallable, + index: number, + context: CallCollectionContext, +): readonly JavaScriptSemanticBindingState[] => { + const key = `${callable.callableId}:${String(index)}`; + const cached = context.parameterBindingCache.get(key); + if (cached !== undefined) return cached; + const bindings = parameterBindings(callable, index, context.state); + context.parameterBindingCache.set(key, bindings); + return bindings; +}; + +const collectReturnFlows = ( + site: JavaScriptSemanticCallSite, + callableById: ReadonlyMap, + state: JavaScriptSemanticAnalysisState, + output: MutableCallAnalysis, +): void => { + for (const callableId of site.calleeCallableIds) { + const callable = callableById.get(callableId); + if (callable === undefined) continue; + for (const returnSite of callable.returnSites) { + if (output.callReturnFlows.length >= state.limits.maxCallReturnFlows) { + reachSemanticLimit(state, "maxCallReturnFlows"); + return; + } + output.callReturnFlows.push({ + callSiteId: site.callSiteId, + callableId, + returnSiteId: returnSite.returnSiteId, + returnLocation: returnSite.location, + }); + } + } +}; + +const collectCapture = ( + node: t.Node, + parent: t.Node | null, + owner: JavaScriptSemanticCallable | undefined, + context: CallCollectionContext, +): void => { + const { state, output } = context; + if ( + !t.isIdentifier(node) || + parent === null || + owner === undefined || + owner.bodyScopeId === null + ) + return; + if (semanticReferenceRole(node, parent) === null) return; + const binding = resolveSemanticBindingState(state, node, node.name); + if ( + binding === undefined || + bindingIsWithinCallable(binding, owner.bodyScopeId, state) + ) + return; + if (output.closureCaptures.length >= state.limits.maxClosureCaptures) { + reachSemanticLimit(state, "maxClosureCaptures"); + return; + } + output.closureCaptures.push({ + callableId: owner.callableId, + bindingId: binding.bindingId, + referenceLocation: range(node), + }); +}; + +const collectDynamicProperty = ( + node: t.Node, + owner: JavaScriptSemanticCallable | undefined, + state: JavaScriptSemanticAnalysisState, + output: MutableCallAnalysis, +): void => { + if ( + (t.isMemberExpression(node) || t.isOptionalMemberExpression(node)) && + semanticStaticPropertyName(node.property, node.computed) === "" + ) + addFrontier( + { + kind: "dynamic-property", + callableId: owner?.callableId ?? null, + location: range(node.property), + reason: "Computed member property is not a static string or number.", + }, + state, + output, + ); + else if ( + (t.isObjectProperty(node) || t.isObjectMethod(node)) && + node.computed && + semanticStaticPropertyName(node.key, true) === "" + ) + addFrontier( + { + kind: "dynamic-property", + callableId: owner?.callableId ?? null, + location: range(node.key), + reason: "Computed object property is not a static string or number.", + }, + state, + output, + ); +}; + +const addFrontier = ( + frontier: JavaScriptSemanticFrontier, + state: JavaScriptSemanticAnalysisState, + output: MutableCallAnalysis, +): void => { + if (output.frontiers.length >= state.limits.maxFrontiers) { + reachSemanticLimit(state, "maxFrontiers"); + return; + } + output.frontiers.push(frontier); +}; + +const bindingIsWithinCallable = ( + binding: JavaScriptSemanticBindingState, + bodyScopeId: string, + state: JavaScriptSemanticAnalysisState, +): boolean => { + let scope = state.scopesById.get(binding.scopeId); + while (scope !== undefined) { + if (scope.scopeId === bodyScopeId) return true; + scope = + scope.parentScopeId === null + ? undefined + : state.scopesById.get(scope.parentScopeId); + } + return false; +}; + +const enclosingFunction = ( + stack: readonly JavaScriptSemanticCallable[], +): JavaScriptSemanticCallable | undefined => + stack.findLast(({ kind }) => kind !== "class"); + +const semanticCallSiteId = ( + node: t.CallExpression | t.OptionalCallExpression | t.NewExpression, +): string => + `call:${t.isNewExpression(node) ? "construct" : "call"}:${String(node.start ?? -1)}:${String(node.end ?? -1)}`; diff --git a/src/domain/javascriptSemanticIr.ts b/src/domain/javascriptSemanticIr.ts index ed89b2cf1..7cb47e217 100644 --- a/src/domain/javascriptSemanticIr.ts +++ b/src/domain/javascriptSemanticIr.ts @@ -8,6 +8,12 @@ export interface JavaScriptSemanticLimits { readonly maxReferences: number; readonly maxModuleLinks: number; readonly maxReturnSites: number; + readonly maxCallSites: number; + readonly maxCallArguments: number; + readonly maxArgumentFlows: number; + readonly maxCallReturnFlows: number; + readonly maxClosureCaptures: number; + readonly maxFrontiers: number; readonly maxValueDepth: number; readonly maxUnionValues: number; readonly maxObjectProperties: number; @@ -21,6 +27,12 @@ export const DEFAULT_JAVASCRIPT_SEMANTIC_LIMITS: JavaScriptSemanticLimits = { maxReferences: 100_000, maxModuleLinks: 20_000, maxReturnSites: 20_000, + maxCallSites: 20_000, + maxCallArguments: 100_000, + maxArgumentFlows: 100_000, + maxCallReturnFlows: 100_000, + maxClosureCaptures: 100_000, + maxFrontiers: 20_000, maxValueDepth: 16, maxUnionValues: 32, maxObjectProperties: 256, @@ -139,10 +151,63 @@ export interface JavaScriptSemanticCallable { /** One direct return expression belonging to exactly one callable. */ export interface JavaScriptSemanticReturnSite { + readonly returnSiteId: string; readonly location: JavaScriptSourceRange; readonly value: JavaScriptSemanticValue; } +/** One statically visible call or construction expression. */ +export interface JavaScriptSemanticCallSite { + readonly callSiteId: string; + readonly kind: "call" | "construct"; + readonly callerCallableId: string | null; + readonly location: JavaScriptSourceRange; + readonly calleeLocation: JavaScriptSourceRange; + readonly resolution: "exact" | "ambiguous" | "unresolved"; + readonly calleeCallableIds: readonly string[]; + readonly arguments: readonly JavaScriptSemanticCallArgument[]; +} + +/** One argument occurrence retained without evaluating application code. */ +export interface JavaScriptSemanticCallArgument { + readonly index: number; + readonly location: JavaScriptSourceRange; + readonly spread: boolean; +} + +/** One positional argument that may initialize one local parameter binding. */ +export interface JavaScriptSemanticArgumentFlow { + readonly callSiteId: string; + readonly argumentIndex: number; + readonly argumentLocation: JavaScriptSourceRange; + readonly callableId: string; + readonly parameterBindingId: string; + readonly parameterLocation: JavaScriptSourceRange; +} + +/** One direct local return that may supply a call expression's value. */ +export interface JavaScriptSemanticCallReturnFlow { + readonly callSiteId: string; + readonly callableId: string; + readonly returnSiteId: string; + readonly returnLocation: JavaScriptSourceRange; +} + +/** One resolved outer lexical binding referenced by a nested callable. */ +export interface JavaScriptSemanticClosureCapture { + readonly callableId: string; + readonly bindingId: string; + readonly referenceLocation: JavaScriptSourceRange; +} + +/** One syntax location where exact local semantic continuation is unavailable. */ +export interface JavaScriptSemanticFrontier { + readonly kind: "dynamic-call" | "dynamic-property"; + readonly callableId: string | null; + readonly location: JavaScriptSourceRange; + readonly reason: string; +} + /** Exact retention state for one callable's direct returns. */ interface JavaScriptSemanticReturnCoverage { readonly status: "complete" | "partial" | "truncated"; @@ -174,15 +239,20 @@ interface JavaScriptSemanticCoverage { readonly limitsReached: readonly (keyof JavaScriptSemanticLimits)[]; } -/** Provider-neutral JavaScript semantic IR v2. */ +/** Provider-neutral JavaScript semantic IR v3. */ export interface JavaScriptSemanticIr { readonly schema: "JavaScriptSemanticIR"; - readonly schemaVersion: 2; + readonly schemaVersion: 3; readonly scopes: readonly JavaScriptSemanticScope[]; readonly bindings: readonly JavaScriptSemanticBinding[]; readonly callables: readonly JavaScriptSemanticCallable[]; readonly references: readonly JavaScriptSemanticReference[]; readonly moduleLinks: readonly JavaScriptSemanticModuleLink[]; + readonly callSites: readonly JavaScriptSemanticCallSite[]; + readonly argumentFlows: readonly JavaScriptSemanticArgumentFlow[]; + readonly callReturnFlows: readonly JavaScriptSemanticCallReturnFlow[]; + readonly closureCaptures: readonly JavaScriptSemanticClosureCapture[]; + readonly frontiers: readonly JavaScriptSemanticFrontier[]; readonly coverage: JavaScriptSemanticCoverage; readonly limitations: readonly string[]; } @@ -208,12 +278,17 @@ export const semanticBinding = ( /** Fail-closed result when Babel cannot produce an inert syntax tree. */ export const failedJavaScriptSemanticIr = (): JavaScriptSemanticIr => ({ schema: "JavaScriptSemanticIR", - schemaVersion: 2, + schemaVersion: 3, scopes: [], bindings: [], callables: [], references: [], moduleLinks: [], + callSites: [], + argumentFlows: [], + callReturnFlows: [], + closureCaptures: [], + frontiers: [], coverage: { status: "failed", omittedCount: null, limitsReached: [] }, limitations: [ "JavaScript parsing failed; no semantic absence claim is available.", diff --git a/src/domain/javascriptSemanticProjection.ts b/src/domain/javascriptSemanticProjection.ts index e8ed83cec..c13bfe954 100644 --- a/src/domain/javascriptSemanticProjection.ts +++ b/src/domain/javascriptSemanticProjection.ts @@ -42,8 +42,10 @@ export const collectSemanticCallable = (input: CollectCallableInput): void => { reachSemanticLimit(state, "maxCallables"); return; } + const callableId = + semanticCallableIdForNode(node) ?? "callable:unknown:-1:-1"; state.callables.push({ - callableId: semanticCallableIdForNode(node) ?? "callable:unknown:-1:-1", + callableId, kind, name: callableName(node, parent), containerScopeId: containerScope.scopeId, @@ -60,6 +62,7 @@ export const collectSemanticCallable = (input: CollectCallableInput): void => { limitsReached: [], }, }); + state.callableNodesById.set(callableId, node); }; /** Collect one static import/export relationship for later composition. */ @@ -102,7 +105,7 @@ export const collectSemanticReferences = ( traverseJavaScriptAst(program, { enter: (node, parent) => { if (!t.isIdentifier(node) || parent === null) return; - const role = referenceRole(node, parent); + const role = semanticReferenceRole(node, parent); if (role === null) return; const key = `${String(node.start)}:${role}:${node.name}`; if (seen.has(key)) return; @@ -403,7 +406,8 @@ const declarationCallableId = ( : semanticCallableIdForNode(declarator.init); }; -const referenceRole = ( +/** Classify one identifier occurrence consistently with semantic references. */ +export const semanticReferenceRole = ( node: t.Identifier, parent: t.Node, ): JavaScriptSemanticReference["role"] | null => { diff --git a/src/domain/javascriptSemanticReturns.ts b/src/domain/javascriptSemanticReturns.ts index c078f4a02..017a032cd 100644 --- a/src/domain/javascriptSemanticReturns.ts +++ b/src/domain/javascriptSemanticReturns.ts @@ -55,6 +55,10 @@ export const collectSemanticReturns = ( } retained += 1; sites.push({ + returnSiteId: semanticReturnSiteId( + callable.callableId, + expression.location, + ), location: expression.location, value: expression.node === null @@ -76,6 +80,12 @@ export const collectSemanticReturns = ( }); }; +const semanticReturnSiteId = ( + callableId: string, + location: JavaScriptSemanticReturnSite["location"], +): string => + `${callableId}:return:${String(location.start.line)}:${String(location.start.column)}:${String(location.end.line)}:${String(location.end.column)}`; + /** Link exports to callables only when lexical resolution is unique and exact. */ export const resolveSemanticModuleCallables = ( state: JavaScriptSemanticAnalysisState, diff --git a/src/domain/javascriptSemanticState.ts b/src/domain/javascriptSemanticState.ts index 69ad149cf..ee0a73e55 100644 --- a/src/domain/javascriptSemanticState.ts +++ b/src/domain/javascriptSemanticState.ts @@ -45,6 +45,7 @@ export interface JavaScriptSemanticAnalysisState { readonly scopeByNode: WeakMap; readonly bindingsById: Map; readonly callables: JavaScriptSemanticCallable[]; + readonly callableNodesById: Map; readonly moduleLinks: JavaScriptSemanticModuleLink[]; readonly limitsReached: Set; omittedCount: number; diff --git a/tests/javascriptSemanticAnalysis.test.ts b/tests/javascriptSemanticAnalysis.test.ts index d846b480c..e182e4d86 100644 --- a/tests/javascriptSemanticAnalysis.test.ts +++ b/tests/javascriptSemanticAnalysis.test.ts @@ -200,6 +200,153 @@ describe("JavaScript semantic analysis", () => { expect(bindingsNamed(ir, "method")).toEqual([]); }); + it("links exact local calls to parameters, returns, and closure captures", () => { + const ir = analyzeJavaScriptSemantics(` + const prefix = "rea"; + function render(value, { mode }) { + return prefix + value + mode; + } + const alias = render; + const result = alias(input, { mode: "fast" }); + `); + + expect(ir.schemaVersion).toBe(3); + const render = onlyCallable(ir, "render"); + const call = ir.callSites.find( + ({ calleeCallableIds }) => calleeCallableIds[0] === render.callableId, + ); + expect(call).toMatchObject({ + kind: "call", + callerCallableId: null, + resolution: "exact", + calleeCallableIds: [render.callableId], + arguments: [ + { index: 0, spread: false }, + { index: 1, spread: false }, + ], + }); + if (call === undefined) throw new Error("Missing render call"); + expect( + ir.argumentFlows + .filter(({ callSiteId }) => callSiteId === call.callSiteId) + .map(({ argumentIndex, parameterBindingId }) => ({ + argumentIndex, + parameter: semanticBinding(ir, parameterBindingId)?.name, + })), + ).toEqual([ + { argumentIndex: 0, parameter: "value" }, + { argumentIndex: 1, parameter: "mode" }, + ]); + expect(ir.callReturnFlows).toEqual([ + expect.objectContaining({ + callSiteId: call.callSiteId, + callableId: render.callableId, + returnSiteId: render.returnSites[0]?.returnSiteId, + }), + ]); + expect(ir.closureCaptures).toEqual([ + expect.objectContaining({ + callableId: render.callableId, + bindingId: topLevelBinding(ir, "prefix").bindingId, + }), + ]); + expect(ir.frontiers).toEqual([]); + }); + + it("retains ambiguous local callees and explicit dynamic frontiers", () => { + const ir = analyzeJavaScriptSemantics(` + function left(value) { return value; } + function right(value) { return value; } + const selected = chooseLeft ? left : right; + selected(input); + receiver[key](input); + external(input); + `); + + const selected = ir.callSites.find( + ({ resolution }) => resolution === "ambiguous", + ); + expect(selected?.calleeCallableIds).toEqual( + [ + onlyCallable(ir, "left").callableId, + onlyCallable(ir, "right").callableId, + ].sort(), + ); + expect( + ir.argumentFlows.filter( + ({ callSiteId }) => callSiteId === selected?.callSiteId, + ), + ).toHaveLength(2); + expect(ir.frontiers).toEqual( + expect.arrayContaining([ + expect.objectContaining({ + kind: "dynamic-call", + reason: expect.stringMatching(/ambiguous/iu), + }), + expect.objectContaining({ + kind: "dynamic-property", + reason: expect.stringMatching(/computed member/iu), + }), + expect.objectContaining({ + kind: "dynamic-call", + reason: "Unresolved call target external.", + }), + ]), + ); + }); + + it("does not project positional argument flow after a spread", () => { + const ir = analyzeJavaScriptSemantics(` + function target(first, second, third) { return third; } + target(one, ...rest, three); + `); + + expect(ir.argumentFlows.map(({ argumentIndex }) => argumentIndex)).toEqual([ + 0, + ]); + }); + + it("applies independent hard bounds to every new semantic fact family", () => { + const ir = analyzeJavaScriptSemantics( + ` + const captured = 1; + function target(value, other) { return value + other + captured; } + target(1, 2); + target(3, 4); + object[first]; + object[second]; + `, + { + maxCallSites: 1, + maxCallArguments: 1, + maxArgumentFlows: 0, + maxCallReturnFlows: 0, + maxClosureCaptures: 0, + maxFrontiers: 1, + }, + ); + + expect(ir.callSites).toHaveLength(1); + expect(ir.callSites[0]?.arguments).toEqual([ + expect.objectContaining({ index: 0, spread: false }), + ]); + expect(ir.argumentFlows).toEqual([]); + expect(ir.callReturnFlows).toEqual([]); + expect(ir.closureCaptures).toEqual([]); + expect(ir.frontiers).toHaveLength(1); + expect(ir.coverage).toMatchObject({ + status: "truncated", + limitsReached: expect.arrayContaining([ + "maxCallSites", + "maxCallArguments", + "maxArgumentFlows", + "maxCallReturnFlows", + "maxClosureCaptures", + "maxFrontiers", + ]), + }); + }); + it("recovers only direct return sites with literal and unknown object fields", () => { const ir = analyzeJavaScriptSemantics(` export default function parseMarkdown(value) { From b3939ea9ffc79a6f0998af76b0f6c9f166d2963b Mon Sep 17 00:00:00 2001 From: morluto <76467478+morluto@users.noreply.github.com> Date: Wed, 22 Jul 2026 21:42:35 +0000 Subject: [PATCH 2/4] feat(javascript): add bounded semantic relation graph --- src/domain/javascriptSemanticGraph.ts | 546 +++++++++++++++++ src/domain/javascriptSemanticGraphSchemas.ts | 393 ++++++++++++ src/domain/javascriptSemanticQuery.ts | 568 ++++++++++++++++++ .../javascriptSemanticQueryAssessment.ts | 110 ++++ src/domain/javascriptSemanticQuerySchemas.ts | 227 +++++++ tests/javascriptSemanticGraphQuery.test.ts | 477 +++++++++++++++ 6 files changed, 2321 insertions(+) create mode 100644 src/domain/javascriptSemanticGraph.ts create mode 100644 src/domain/javascriptSemanticGraphSchemas.ts create mode 100644 src/domain/javascriptSemanticQuery.ts create mode 100644 src/domain/javascriptSemanticQueryAssessment.ts create mode 100644 src/domain/javascriptSemanticQuerySchemas.ts create mode 100644 tests/javascriptSemanticGraphQuery.test.ts diff --git a/src/domain/javascriptSemanticGraph.ts b/src/domain/javascriptSemanticGraph.ts new file mode 100644 index 000000000..257e3adf3 --- /dev/null +++ b/src/domain/javascriptSemanticGraph.ts @@ -0,0 +1,546 @@ +import { createHash } from "node:crypto"; + +import canonicalize from "canonicalize"; +import { z } from "zod"; + +import { compareCodePoints } from "./javascriptApplicationGraph.js"; +import type { ApplicationGraphEvidence } from "./javascriptApplicationEvidenceSchemas.js"; +import { + JAVASCRIPT_SEMANTIC_RELATION_FAMILIES, + JAVASCRIPT_SEMANTIC_RELATION_FAMILY, + javaScriptSemanticFingerprintInputSchema, + javaScriptSemanticFingerprintSchema, + javaScriptSemanticGraphInputSchema, + javaScriptSemanticGraphRecordSchema, + javaScriptSemanticNodeInputSchema, + javaScriptSemanticNodeSchema, + javaScriptSemanticRelationInputSchema, + javaScriptSemanticRelationSchema, + javaScriptSemanticUnknownInputSchema, + javaScriptSemanticUnknownSchema, + type JavaScriptSemanticGraphInput, + type JavaScriptSemanticGraphNode, + type JavaScriptSemanticGraphRelation, +} from "./javascriptSemanticGraphSchemas.js"; + +const canonicalJson = (value: unknown): string => { + const encoded = canonicalize(value); + if (encoded === undefined) + throw new TypeError( + "JavaScript semantic graph could not canonicalize data", + ); + return encoded; +}; + +const digestCanonical = (value: unknown): string => + createHash("sha256").update(canonicalJson(value)).digest("hex"); + +const uniqueSorted = ( + values: readonly Value[], +): Value[] => [...new Set(values)].sort(compareCodePoints); + +const normalizeEvidence = ( + evidence: ApplicationGraphEvidence, +): ApplicationGraphEvidence => ({ + ...evidence, + coverage: { + ...evidence.coverage, + limits: [...evidence.coverage.limits].sort((left, right) => + compareCodePoints(canonicalJson(left), canonicalJson(right)), + ), + }, + limitations: uniqueSorted(evidence.limitations), + evidence_ids: uniqueSorted(evidence.evidence_ids), +}); + +const nodeIdentity = ( + node: Pick, +) => ({ kind: node.kind, identity: node.identity }); + +const nodeId = ( + node: Pick, +): string => `jsrg_node_${digestCanonical(nodeIdentity(node))}`; + +/** Normalize one semantic entity and derive its artifact-version identifier. */ +export const createJavaScriptSemanticGraphNode = ( + input: unknown, +): JavaScriptSemanticGraphNode => { + const parsed = javaScriptSemanticNodeInputSchema.parse(input); + const semantic = { + ...parsed, + application_node_ids: uniqueSorted(parsed.application_node_ids), + evidence: normalizeEvidence(parsed.evidence), + identifier_strategy: { + strategy: "semantic-content-sha256" as const, + stability: "artifact-version" as const, + }, + }; + return javaScriptSemanticNodeSchema.parse({ + ...semantic, + node_id: nodeId(semantic), + }); +}; + +/** Normalize one semantic relationship and derive its exact identifier. */ +export const createJavaScriptSemanticGraphRelation = ( + input: unknown, +): JavaScriptSemanticGraphRelation => { + const parsed = javaScriptSemanticRelationInputSchema.parse(input); + const semantic = { + ...parsed, + evidence: normalizeEvidence(parsed.evidence), + identifier_strategy: { + strategy: "semantic-content-sha256" as const, + stability: "relationship-exact" as const, + }, + }; + return javaScriptSemanticRelationSchema.parse({ + ...semantic, + relation_id: `jsrg_relation_${digestCanonical(semantic)}`, + }); +}; + +/** Normalize one unresolved semantic frontier and derive its identifier. */ +export const createJavaScriptSemanticGraphUnknown = (input: unknown) => { + const parsed = javaScriptSemanticUnknownInputSchema.parse(input); + const semantic = { + ...parsed, + relation_kinds: uniqueSorted(parsed.relation_kinds), + candidate_node_ids: uniqueSorted(parsed.candidate_node_ids), + evidence: normalizeEvidence(parsed.evidence), + }; + return javaScriptSemanticUnknownSchema.parse({ + ...semantic, + unknown_id: `jsrg_unknown_${digestCanonical(semantic)}`, + }); +}; + +/** Normalize one function fingerprint and derive its component commitment. */ +export const createJavaScriptSemanticFingerprint = (input: unknown) => { + const parsed = javaScriptSemanticFingerprintInputSchema.parse(input); + const semantic = { + ...parsed, + components: { + ...parsed.components, + effects: uniqueSorted(parsed.components.effects), + }, + limitations: uniqueSorted(parsed.limitations), + evidence: normalizeEvidence(parsed.evidence), + }; + const fingerprintSha256 = digestCanonical(semantic.components); + return javaScriptSemanticFingerprintSchema.parse({ + ...semantic, + fingerprint_sha256: fingerprintSha256, + fingerprint_id: `jsrg_fingerprint_${digestCanonical({ + function_node_id: semantic.function_node_id, + algorithm: semantic.algorithm, + fingerprint_sha256: fingerprintSha256, + })}`, + }); +}; + +type GraphRecord = z.infer; + +const sortedUniqueIssue = ( + values: readonly string[], + path: PropertyKey[], + label: string, + context: z.RefinementCtx, +): void => { + for (let index = 1; index < values.length; index += 1) { + if (compareCodePoints(values[index - 1] ?? "", values[index] ?? "") < 0) + continue; + context.addIssue({ + code: "custom", + path: [...path, index], + message: `${label} must be sorted and unique`, + }); + return; + } +}; + +const checkCoverage = (graph: GraphRecord, context: z.RefinementCtx): void => { + const families = graph.coverage.families.map(({ family }) => family); + if ( + canonicalJson(families) !== + canonicalJson(JAVASCRIPT_SEMANTIC_RELATION_FAMILIES) + ) + context.addIssue({ + code: "custom", + path: ["coverage", "families"], + message: + "Coverage must name every semantic relation family once in canonical order", + }); + if ( + graph.coverage.status === "complete" && + (graph.coverage.truncated || + graph.coverage.omitted_nodes !== 0 || + graph.coverage.omitted_relations !== 0 || + graph.coverage.families.some(({ status }) => status !== "complete")) + ) + context.addIssue({ + code: "custom", + path: ["coverage"], + message: "Complete graph coverage cannot omit or truncate semantic facts", + }); + if (graph.coverage.truncated && graph.coverage.limits.length === 0) + context.addIssue({ + code: "custom", + path: ["coverage", "limits"], + message: "Truncated graph coverage must identify an effective limit", + }); + const relationsByFamily = new Map( + JAVASCRIPT_SEMANTIC_RELATION_FAMILIES.map((family) => [family, 0]), + ); + for (const relation of graph.relations) { + const family = JAVASCRIPT_SEMANTIC_RELATION_FAMILY[relation.relation]; + relationsByFamily.set(family, (relationsByFamily.get(family) ?? 0) + 1); + } + const unknownsById = new Map( + graph.unknowns.map((unknown) => [unknown.unknown_id, unknown]), + ); + for (const [index, family] of graph.coverage.families.entries()) { + if (family.retained_relations !== relationsByFamily.get(family.family)) + context.addIssue({ + code: "custom", + path: ["coverage", "families", index, "retained_relations"], + message: "Family retained relation count does not match graph content", + }); + if ( + family.unknown_ids.some( + (identifier) => unknownsById.get(identifier)?.family !== family.family, + ) + ) + context.addIssue({ + code: "custom", + path: ["coverage", "families", index, "unknown_ids"], + message: "Family coverage references an unknown from another family", + }); + if ( + family.status === "complete" && + (family.omitted_relations !== 0 || family.unknown_ids.length > 0) + ) + context.addIssue({ + code: "custom", + path: ["coverage", "families", index], + message: "Complete family coverage cannot omit or retain unknown facts", + }); + } +}; + +const checkCanonicalOrder = ( + graph: GraphRecord, + context: z.RefinementCtx, +): void => { + sortedUniqueIssue( + graph.root_node_ids, + ["root_node_ids"], + "Root nodes", + context, + ); + sortedUniqueIssue( + graph.nodes.map(({ node_id }) => node_id), + ["nodes"], + "Nodes", + context, + ); + sortedUniqueIssue( + graph.relations.map(({ relation_id }) => relation_id), + ["relations"], + "Relations", + context, + ); + sortedUniqueIssue( + graph.fingerprints.map(({ fingerprint_id }) => fingerprint_id), + ["fingerprints"], + "Fingerprints", + context, + ); + sortedUniqueIssue( + graph.unknowns.map(({ unknown_id }) => unknown_id), + ["unknowns"], + "Unknown frontiers", + context, + ); + sortedUniqueIssue(graph.limitations, ["limitations"], "Limitations", context); +}; + +const checkNodes = ( + graph: GraphRecord, + nodes: ReadonlyMap, + context: z.RefinementCtx, +): void => { + for (const [index, node] of graph.nodes.entries()) { + if (node.node_id !== nodeId(node)) + context.addIssue({ + code: "custom", + path: ["nodes", index, "node_id"], + message: "Node identifier is stale", + }); + if ( + node.function_node_id !== null && + nodes.get(node.function_node_id)?.kind !== "function" + ) + context.addIssue({ + code: "custom", + path: ["nodes", index, "function_node_id"], + message: "Function owner must name a function node", + }); + } +}; + +const checkRoots = ( + graph: GraphRecord, + nodes: ReadonlyMap, + context: z.RefinementCtx, +): void => { + for (const root of graph.root_node_ids) + if (!nodes.has(root)) + context.addIssue({ + code: "custom", + path: ["root_node_ids"], + message: "Root identifier must name a semantic node", + }); +}; + +const checkRelations = ( + graph: GraphRecord, + nodes: ReadonlyMap, + context: z.RefinementCtx, +): void => { + for (const [index, relation] of graph.relations.entries()) { + const { relation_id: identifier, ...semantic } = relation; + if (identifier !== "jsrg_relation_" + digestCanonical(semantic)) + context.addIssue({ + code: "custom", + path: ["relations", index, "relation_id"], + message: "Relation identifier is stale", + }); + if ( + !nodes.has(relation.source_node_id) || + !nodes.has(relation.target_node_id) + ) + context.addIssue({ + code: "custom", + path: ["relations", index], + message: "Relation endpoints must name semantic nodes", + }); + if (relation.source_node_id === relation.target_node_id) + context.addIssue({ + code: "custom", + path: ["relations", index], + message: "Semantic relations cannot be self-referential", + }); + if ( + relation.resolution === "candidate" && + relation.evidence.state === "observed" + ) + context.addIssue({ + code: "custom", + path: ["relations", index, "evidence", "state"], + message: "Candidate relations cannot claim observed resolution", + }); + } +}; + +const checkUnknowns = ( + graph: GraphRecord, + nodes: ReadonlyMap, + context: z.RefinementCtx, +): void => { + for (const [index, unknown] of graph.unknowns.entries()) { + const { unknown_id: identifier, ...semantic } = unknown; + if (identifier !== "jsrg_unknown_" + digestCanonical(semantic)) + context.addIssue({ + code: "custom", + path: ["unknowns", index, "unknown_id"], + message: "Unknown frontier identifier is stale", + }); + if (unknown.node_id !== null && !nodes.has(unknown.node_id)) + context.addIssue({ + code: "custom", + path: ["unknowns", index, "node_id"], + message: "Unknown frontier node is absent", + }); + if ( + unknown.relation_kinds.some( + (kind) => JAVASCRIPT_SEMANTIC_RELATION_FAMILY[kind] !== unknown.family, + ) + ) + context.addIssue({ + code: "custom", + path: ["unknowns", index, "relation_kinds"], + message: "Unknown relation kinds must belong to their declared family", + }); + if (!["unknown", "unavailable"].includes(unknown.evidence.state)) + context.addIssue({ + code: "custom", + path: ["unknowns", index, "evidence", "state"], + message: "Unknown frontiers require unknown or unavailable evidence", + }); + } +}; + +const checkUnknownReferences = ( + graph: GraphRecord, + context: z.RefinementCtx, +): void => { + const unknownIds = new Set( + graph.unknowns.map(({ unknown_id }) => unknown_id), + ); + for (const [index, family] of graph.coverage.families.entries()) + if (family.unknown_ids.some((identifier) => !unknownIds.has(identifier))) + context.addIssue({ + code: "custom", + path: ["coverage", "families", index, "unknown_ids"], + message: "Family coverage references an absent unknown frontier", + }); +}; + +const checkFingerprints = ( + graph: GraphRecord, + nodes: ReadonlyMap, + context: z.RefinementCtx, +): void => { + for (const [index, fingerprint] of graph.fingerprints.entries()) { + if (nodes.get(fingerprint.function_node_id)?.kind !== "function") + context.addIssue({ + code: "custom", + path: ["fingerprints", index, "function_node_id"], + message: "Fingerprint must name a function node", + }); + if ( + fingerprint.fingerprint_sha256 !== digestCanonical(fingerprint.components) + ) + context.addIssue({ + code: "custom", + path: ["fingerprints", index, "fingerprint_sha256"], + message: "Fingerprint component commitment is stale", + }); + const expectedIdentifier = + "jsrg_fingerprint_" + + digestCanonical({ + function_node_id: fingerprint.function_node_id, + algorithm: fingerprint.algorithm, + fingerprint_sha256: fingerprint.fingerprint_sha256, + }); + if (fingerprint.fingerprint_id !== expectedIdentifier) + context.addIssue({ + code: "custom", + path: ["fingerprints", index, "fingerprint_id"], + message: "Fingerprint identifier is stale", + }); + if ( + fingerprint.status !== "complete" && + fingerprint.limitations.length === 0 + ) + context.addIssue({ + code: "custom", + path: ["fingerprints", index, "limitations"], + message: "Incomplete fingerprints require a limitation", + }); + } +}; + +const checkGraph = (graph: GraphRecord, context: z.RefinementCtx): void => { + checkCanonicalOrder(graph, context); + const nodes = new Map(graph.nodes.map((node) => [node.node_id, node])); + checkNodes(graph, nodes, context); + checkRoots(graph, nodes, context); + checkRelations(graph, nodes, context); + checkUnknowns(graph, nodes, context); + checkUnknownReferences(graph, context); + checkFingerprints(graph, nodes, context); + checkCoverage(graph, context); + if (graph.coverage.status !== "complete" && graph.limitations.length === 0) + context.addIssue({ + code: "custom", + path: ["limitations"], + message: "Non-complete graph coverage requires a limitation", + }); + const { graph_id: identifier, ...semantic } = graph; + if (identifier !== "jsrg_" + digestCanonical(semantic)) + context.addIssue({ + code: "custom", + path: ["graph_id"], + message: "Graph identifier is stale", + }); +}; + +/** Strict semantic graph schema with verified canonical commitments. */ +export const javaScriptSemanticGraphSchema = + javaScriptSemanticGraphRecordSchema.superRefine(checkGraph); + +/** Fully validated JavaScript Semantic Relation Graph v1. */ +export type JavaScriptSemanticGraph = z.infer< + typeof javaScriptSemanticGraphSchema +>; + +/** Normalize a complete companion graph and derive its graph ID. */ +export const createJavaScriptSemanticGraph = ( + input: unknown, +): JavaScriptSemanticGraph => { + const parsed = javaScriptSemanticGraphInputSchema.parse(input); + const semantic: JavaScriptSemanticGraphInput = { + ...parsed, + root_node_ids: uniqueSorted(parsed.root_node_ids), + nodes: [...parsed.nodes].sort((left, right) => + compareCodePoints(left.node_id, right.node_id), + ), + relations: [...parsed.relations].sort((left, right) => + compareCodePoints(left.relation_id, right.relation_id), + ), + fingerprints: [...parsed.fingerprints].sort((left, right) => + compareCodePoints(left.fingerprint_id, right.fingerprint_id), + ), + unknowns: [...parsed.unknowns].sort((left, right) => + compareCodePoints(left.unknown_id, right.unknown_id), + ), + coverage: { + ...parsed.coverage, + limits: [...parsed.coverage.limits].sort((left, right) => + compareCodePoints(canonicalJson(left), canonicalJson(right)), + ), + families: [...parsed.coverage.families] + .map((family) => ({ + ...family, + unknown_ids: uniqueSorted(family.unknown_ids), + })) + .sort((left, right) => compareCodePoints(left.family, right.family)), + }, + limitations: uniqueSorted(parsed.limitations), + }; + return javaScriptSemanticGraphSchema.parse({ + ...semantic, + graph_id: `jsrg_${digestCanonical(semantic)}`, + }); +}; + +/** Parse a stored semantic graph and reject unsupported versions or stale IDs. */ +export const parseJavaScriptSemanticGraph = ( + input: unknown, +): JavaScriptSemanticGraph => { + const envelope = z + .object({ schema: z.string(), schema_version: z.number() }) + .passthrough() + .safeParse(input); + if ( + envelope.success && + envelope.data.schema === "JavaScriptSemanticRelationGraph" && + envelope.data.schema_version !== 1 + ) + throw new TypeError( + `Unsupported JavaScript Semantic Relation Graph schema version: ${String(envelope.data.schema_version)}`, + ); + return javaScriptSemanticGraphSchema.parse(input); +}; + +/** Serialize a verified semantic graph as canonical JSON. */ +export const serializeJavaScriptSemanticGraph = (input: unknown): string => + canonicalJson(parseJavaScriptSemanticGraph(input)); + +/** Compute the canonical SHA-256 commitment of a verified semantic graph. */ +export const computeJavaScriptSemanticGraphSha256 = (input: unknown): string => + digestCanonical(parseJavaScriptSemanticGraph(input)); + +export type { JavaScriptSemanticGraphNode, JavaScriptSemanticGraphRelation }; diff --git a/src/domain/javascriptSemanticGraphSchemas.ts b/src/domain/javascriptSemanticGraphSchemas.ts new file mode 100644 index 000000000..ad7246147 --- /dev/null +++ b/src/domain/javascriptSemanticGraphSchemas.ts @@ -0,0 +1,393 @@ +import { z } from "zod"; + +import { applicationGraphEvidenceSchema } from "./javascriptApplicationEvidenceSchemas.js"; +import { isJsonWithinLimits } from "./jsonLimits.js"; +import { jsonValueSchema } from "./jsonValue.js"; + +const digestSchema = z.string().regex(/^[a-f0-9]{64}$/u); +const boundedTextSchema = z.string().min(1).max(4_096); +const semanticNodeIdSchema = z.string().regex(/^jsrg_node_[a-f0-9]{64}$/u); +const semanticRelationIdSchema = z + .string() + .regex(/^jsrg_relation_[a-f0-9]{64}$/u); +const semanticUnknownIdSchema = z + .string() + .regex(/^jsrg_unknown_[a-f0-9]{64}$/u); + +/** Semantic entity kinds admitted by JavaScript Semantic Relation Graph v1. */ +export const JAVASCRIPT_SEMANTIC_NODE_KINDS = [ + "module", + "function", + "parameter", + "binding", + "expression", + "literal", + "property-slot", + "call-site", + "return-site", + "closure-capture", + "promise", + "task", + "event", + "listener", + "timer", + "child-process", + "stdio", + "signal", + "config-source", + "request", + "response", + "boundary", + "resource", + "unknown-frontier", +] as const; + +/** Required semantic-analysis coverage families. */ +export const JAVASCRIPT_SEMANTIC_RELATION_FAMILIES = [ + "boundary", + "call-flow", + "child-process", + "closure", + "configuration", + "data-flow", + "event", + "object-flow", + "promise-ownership", + "request", + "resource-lifecycle", + "timer", +] as const; + +/** Directed semantic relations admitted by graph v1. */ +export const JAVASCRIPT_SEMANTIC_RELATIONS = [ + "acquires", + "aggregates", + "aliases", + "argument-to-parameter", + "awaits", + "calls", + "cancels-timer", + "captures", + "chains", + "coerces", + "connects-stdio", + "consumed-by", + "constructs-request", + "creates-promise", + "defaults", + "defines", + "destructures", + "detaches-task", + "dispatches-candidate", + "forwards-signal", + "listens-error", + "listens-exit", + "owns", + "overrides", + "parses", + "reads", + "reads-argv", + "reads-config", + "reads-environment", + "reads-property", + "registers-listener", + "releases", + "removes-listener", + "returns-task", + "returns-to-call", + "schedules-timer", + "spawns", + "spreads", + "supplies-argv", + "supplies-env", + "supplies-request-field", + "validates", + "writes", + "writes-property", +] as const; + +/** Map each semantic relation to its coverage family. */ +export const JAVASCRIPT_SEMANTIC_RELATION_FAMILY = { + acquires: "resource-lifecycle", + aggregates: "promise-ownership", + aliases: "data-flow", + "argument-to-parameter": "call-flow", + awaits: "promise-ownership", + calls: "call-flow", + "cancels-timer": "timer", + captures: "closure", + chains: "promise-ownership", + coerces: "boundary", + "connects-stdio": "child-process", + "consumed-by": "request", + "constructs-request": "request", + "creates-promise": "promise-ownership", + defaults: "configuration", + defines: "data-flow", + destructures: "object-flow", + "detaches-task": "promise-ownership", + "dispatches-candidate": "event", + "forwards-signal": "child-process", + "listens-error": "child-process", + "listens-exit": "child-process", + owns: "promise-ownership", + overrides: "configuration", + parses: "boundary", + reads: "data-flow", + "reads-argv": "configuration", + "reads-config": "configuration", + "reads-environment": "configuration", + "reads-property": "object-flow", + "registers-listener": "event", + releases: "resource-lifecycle", + "removes-listener": "event", + "returns-task": "promise-ownership", + "returns-to-call": "call-flow", + "schedules-timer": "timer", + spawns: "child-process", + spreads: "object-flow", + "supplies-argv": "child-process", + "supplies-env": "child-process", + "supplies-request-field": "request", + validates: "boundary", + writes: "data-flow", + "writes-property": "object-flow", +} as const satisfies Readonly< + Record< + (typeof JAVASCRIPT_SEMANTIC_RELATIONS)[number], + (typeof JAVASCRIPT_SEMANTIC_RELATION_FAMILIES)[number] + > +>; + +const relativePathSchema = z + .string() + .min(1) + .max(4_096) + .refine( + (path) => + !path.startsWith("/") && + !path.includes("\\") && + path.split("/").every((part) => !["", ".", ".."].includes(part)), + "Expected a normalized relative POSIX path without traversal", + ); + +const sourcePointSchema = z.strictObject({ + line: z.number().int().min(1), + column: z.number().int().min(0), +}); + +const sourceRangeSchema = z + .strictObject({ start: sourcePointSchema, end: sourcePointSchema }) + .superRefine((range, context) => { + if ( + range.end.line < range.start.line || + (range.end.line === range.start.line && + range.end.column < range.start.column) + ) + context.addIssue({ + code: "custom", + path: ["end"], + message: "Source range end must not precede its start", + }); + }); + +const semanticPropertiesSchema = z + .record(z.string().min(1).max(128), jsonValueSchema) + .superRefine((properties, context) => { + if (Object.keys(properties).length > 64) + context.addIssue({ + code: "custom", + message: "Semantic properties exceed 64 keys", + }); + if ( + !isJsonWithinLimits(properties, { + maxDepth: 6, + maxNodes: 512, + maxStringLength: 4_096, + }) + ) + context.addIssue({ + code: "custom", + message: "Semantic properties exceed structural limits", + }); + }); + +const semanticNodeIdentitySchema = z.strictObject({ + artifact_sha256: digestSchema, + module_path: relativePathSchema, + source_range: sourceRangeSchema.nullable(), + role_key: z.string().min(1).max(1_024), +}); + +/** One semantic entity before its canonical identifier is derived. */ +export const javaScriptSemanticNodeInputSchema = z.strictObject({ + kind: z.enum(JAVASCRIPT_SEMANTIC_NODE_KINDS), + identity: semanticNodeIdentitySchema, + function_node_id: semanticNodeIdSchema.nullable(), + application_node_ids: z + .array(z.string().regex(/^jag_node_[a-f0-9]{64}$/u)) + .max(64), + label: z.string().min(1).max(1_024).nullable(), + properties: semanticPropertiesSchema, + evidence: applicationGraphEvidenceSchema, +}); + +/** One exact, artifact-version semantic entity. */ +export const javaScriptSemanticNodeSchema = + javaScriptSemanticNodeInputSchema.extend({ + node_id: semanticNodeIdSchema, + identifier_strategy: z.strictObject({ + strategy: z.literal("semantic-content-sha256"), + stability: z.literal("artifact-version"), + }), + }); + +/** One directed semantic relationship before its ID is derived. */ +export const javaScriptSemanticRelationInputSchema = z.strictObject({ + source_node_id: semanticNodeIdSchema, + target_node_id: semanticNodeIdSchema, + relation: z.enum(JAVASCRIPT_SEMANTIC_RELATIONS), + resolution: z.enum(["resolved", "candidate"]), + properties: semanticPropertiesSchema, + evidence: applicationGraphEvidenceSchema, +}); + +/** One canonical directed semantic relationship. */ +export const javaScriptSemanticRelationSchema = + javaScriptSemanticRelationInputSchema.extend({ + relation_id: semanticRelationIdSchema, + identifier_strategy: z.strictObject({ + strategy: z.literal("semantic-content-sha256"), + stability: z.literal("relationship-exact"), + }), + }); + +/** Reasons semantic analysis can leave an explicit frontier. */ +export const JAVASCRIPT_SEMANTIC_UNKNOWN_REASONS = [ + "ambiguous-target", + "bound-reached", + "dynamic-call", + "dynamic-property", + "eval-or-generated-code", + "incomplete-module", + "missing-source", + "unsupported-syntax", +] as const; + +/** One unresolved semantic frontier before its ID is derived. */ +export const javaScriptSemanticUnknownInputSchema = z.strictObject({ + node_id: semanticNodeIdSchema.nullable(), + family: z.enum(JAVASCRIPT_SEMANTIC_RELATION_FAMILIES), + relation_kinds: z.array(z.enum(JAVASCRIPT_SEMANTIC_RELATIONS)).min(1).max(45), + reason: z.enum(JAVASCRIPT_SEMANTIC_UNKNOWN_REASONS), + detail: boundedTextSchema, + candidate_node_ids: z.array(semanticNodeIdSchema).max(1_000), + evidence: applicationGraphEvidenceSchema, +}); + +/** One canonical unresolved semantic frontier. */ +export const javaScriptSemanticUnknownSchema = + javaScriptSemanticUnknownInputSchema.extend({ + unknown_id: semanticUnknownIdSchema, + }); + +const fingerprintComponentsSchema = z.strictObject({ + parameter_arity: z.number().int().min(0).max(10_000), + normalized_ast_sha256: digestSchema, + control_flow_sha256: digestSchema, + relation_shape_sha256: digestSchema, + literal_set_sha256: digestSchema, + effects: z + .array( + z.enum([ + "async", + "child-process", + "event", + "network", + "promise", + "resource", + "timer", + ]), + ) + .max(7), +}); + +/** One bounded rename-resistant function fingerprint. */ +export const javaScriptSemanticFingerprintInputSchema = z.strictObject({ + function_node_id: semanticNodeIdSchema, + algorithm: z.literal("rea.javascript-semantic-function/v1"), + status: z.enum(["complete", "partial", "unavailable"]), + components: fingerprintComponentsSchema, + limitations: z.array(boundedTextSchema).max(100), + evidence: applicationGraphEvidenceSchema, +}); + +/** One canonical function fingerprint record. */ +export const javaScriptSemanticFingerprintSchema = + javaScriptSemanticFingerprintInputSchema.extend({ + fingerprint_id: z.string().regex(/^jsrg_fingerprint_[a-f0-9]{64}$/u), + fingerprint_sha256: digestSchema, + }); + +const familyCoverageSchema = z.strictObject({ + family: z.enum(JAVASCRIPT_SEMANTIC_RELATION_FAMILIES), + status: z.enum(["complete", "partial", "unknown", "unsupported"]), + retained_relations: z.number().int().min(0), + omitted_relations: z.number().int().min(0).nullable(), + unknown_ids: z.array(semanticUnknownIdSchema).max(100_000), +}); + +const graphCoverageSchema = z.strictObject({ + status: z.enum(["complete", "partial", "unknown", "unavailable"]), + truncated: z.boolean(), + omitted_nodes: z.number().int().min(0).nullable(), + omitted_relations: z.number().int().min(0).nullable(), + limits: z + .array( + z.strictObject({ + name: z.string().min(1).max(128), + value: z.number().int().min(0), + unit: z.enum(["items", "bytes", "milliseconds", "depth", "other"]), + }), + ) + .max(32), + families: z.array(familyCoverageSchema).length(12), +}); + +/** Graph content before its top-level commitment is derived. */ +export const javaScriptSemanticGraphInputSchema = z.strictObject({ + schema: z.literal("JavaScriptSemanticRelationGraph"), + schema_version: z.literal(1), + root_artifact_sha256: digestSchema, + application_graph_id: z.string().regex(/^jag_[a-f0-9]{64}$/u), + root_node_ids: z.array(semanticNodeIdSchema).min(1).max(10_000), + nodes: z.array(javaScriptSemanticNodeSchema).min(1).max(100_000), + relations: z.array(javaScriptSemanticRelationSchema).max(200_000), + fingerprints: z.array(javaScriptSemanticFingerprintSchema).max(20_000), + unknowns: z.array(javaScriptSemanticUnknownSchema).max(100_000), + coverage: graphCoverageSchema, + limitations: z.array(boundedTextSchema).max(1_000), +}); + +/** Stored JavaScript Semantic Relation Graph v1. */ +export const javaScriptSemanticGraphRecordSchema = + javaScriptSemanticGraphInputSchema.extend({ + graph_id: z.string().regex(/^jsrg_[a-f0-9]{64}$/u), + }); + +/** Canonical semantic node. */ +export type JavaScriptSemanticGraphNode = z.infer< + typeof javaScriptSemanticNodeSchema +>; +/** Canonical semantic relation. */ +export type JavaScriptSemanticGraphRelation = z.infer< + typeof javaScriptSemanticRelationSchema +>; +/** Canonical unresolved semantic frontier. */ +export type JavaScriptSemanticGraphUnknown = z.infer< + typeof javaScriptSemanticUnknownSchema +>; +/** Canonical graph content before graph ID derivation. */ +export type JavaScriptSemanticGraphInput = z.infer< + typeof javaScriptSemanticGraphInputSchema +>; diff --git a/src/domain/javascriptSemanticQuery.ts b/src/domain/javascriptSemanticQuery.ts new file mode 100644 index 000000000..7dfe45094 --- /dev/null +++ b/src/domain/javascriptSemanticQuery.ts @@ -0,0 +1,568 @@ +import { createHash } from "node:crypto"; + +import canonicalize from "canonicalize"; + +import { compareCodePoints } from "./javascriptApplicationGraph.js"; +import type { + JavaScriptSemanticGraph, + JavaScriptSemanticGraphNode, + JavaScriptSemanticGraphRelation, +} from "./javascriptSemanticGraph.js"; +import { + JAVASCRIPT_SEMANTIC_RELATION_FAMILY, + type JavaScriptSemanticGraphUnknown, +} from "./javascriptSemanticGraphSchemas.js"; +import { + javaScriptSemanticQueryInputSchema, + javaScriptSemanticQueryResultSchema, + type JavaScriptSemanticQueryInput, + type JavaScriptSemanticQueryResult, +} from "./javascriptSemanticQuerySchemas.js"; +import { + assessJavaScriptSemanticQuery, + type JavaScriptSemanticQueryFrontier, +} from "./javascriptSemanticQueryAssessment.js"; + +interface TraversalEntry { + readonly relation: JavaScriptSemanticGraphRelation; + readonly nextNodeId: string; +} + +type QueryFrontier = JavaScriptSemanticQueryFrontier; + +interface Traversal { + readonly nodeIds: Set; + readonly relationIds: Set; + readonly functionIds: Set; + readonly modules: Set; + readonly frontier: QueryFrontier[]; +} + +interface SeedAdmission { + readonly nodeIds: string[]; + readonly frontier: QueryFrontier[]; +} + +const canonicalJson = (value: unknown): string => { + const encoded = canonicalize(value); + if (encoded === undefined) + throw new TypeError( + "JavaScript semantic query could not canonicalize data", + ); + return encoded; +}; + +const digest = (value: unknown): string => + createHash("sha256").update(canonicalJson(value)).digest("hex"); + +const uniqueSorted = (values: readonly string[]): string[] => + [...new Set(values)].sort(compareCodePoints); + +/** Run one deterministic bounded traversal over a verified semantic graph. */ +export const queryJavaScriptSemanticGraph = ( + graph: JavaScriptSemanticGraph, + rawInput: unknown, +): JavaScriptSemanticQueryResult => { + const input = javaScriptSemanticQueryInputSchema.parse(rawInput); + const queryId = queryIdentifier(graph, input); + const offset = parseCursor(input.cursor, queryId); + const seeds = resolveSeeds(graph, input); + const admission = admitSeeds(graph, seeds, input); + const retainedSeeds = admission.nodeIds; + const adjacency = buildAdjacency(graph.relations, input); + const traversal = traverse(graph, retainedSeeds, adjacency, input); + const frontier = uniqueFrontier([ + ...traversal.frontier, + ...admission.frontier, + ]); + const retainedNodes = graph.nodes.filter(({ node_id }) => + traversal.nodeIds.has(node_id), + ); + const retainedRelations = graph.relations.filter(({ relation_id }) => + traversal.relationIds.has(relation_id), + ); + if (offset > retainedRelations.length) + throw new TypeError("Semantic query cursor is beyond the retained result"); + const pageRelations = retainedRelations.slice( + offset, + offset + input.limits.page_size, + ); + const pageNodeIds = new Set([ + ...retainedSeeds, + ...pageRelations.flatMap(({ source_node_id, target_node_id }) => [ + source_node_id, + target_node_id, + ]), + ]); + const pageNodes = graph.nodes.filter(({ node_id }) => + pageNodeIds.has(node_id), + ); + const relevantUnknowns = relevantUnknownFrontiers( + graph, + traversal.nodeIds, + input, + ); + const expectedMatches = expectedMatchesFor( + retainedNodes, + retainedRelations, + input, + ); + const assessment = assessJavaScriptSemanticQuery({ + graph, + totalSeeds: seeds.length, + retainedSeeds: retainedSeeds.length, + expectedMatches: expectedMatches.length, + hasExpectation: input.expected !== null, + frontier, + unknowns: relevantUnknowns, + }); + const nextOffset = offset + pageRelations.length; + const nextCursor = + nextOffset < retainedRelations.length + ? createCursor(queryId, nextOffset) + : null; + return javaScriptSemanticQueryResultSchema.parse({ + schema_version: 1, + query_id: queryId, + source_graph_id: graph.graph_id, + seed: input.seed, + direction: input.direction, + status: assessment.status, + seed_node_ids: retainedSeeds, + nodes: pageNodes, + relations: pageRelations, + unknowns: relevantUnknowns, + expected_match_node_ids: expectedMatches.map(({ node_id }) => node_id), + summary: { + total_seed_matches: seeds.length, + retained_seed_matches: retainedSeeds.length, + traversed_nodes: traversal.nodeIds.size, + traversed_relations: traversal.relationIds.size, + traversed_functions: traversal.functionIds.size, + traversed_modules: traversal.modules.size, + relevant_unknowns: relevantUnknowns.length, + }, + coverage: assessment.coverage, + page: { + offset, + size: pageRelations.length, + next_cursor: nextCursor, + }, + applied_limits: input.limits, + accepted_limit_ranges: assessment.acceptedLimitRanges, + limitations: assessment.limitations, + }); +}; + +const queryIdentifier = ( + graph: JavaScriptSemanticGraph, + input: JavaScriptSemanticQueryInput, +): string => { + const { cursor: _cursor, ...semantic } = input; + return `jsrq_${digest({ + source_graph_id: graph.graph_id, + ...semantic, + allowed_relations: + semantic.allowed_relations === undefined + ? undefined + : uniqueSorted(semantic.allowed_relations), + expected: + semantic.expected === null + ? null + : { + ...semantic.expected, + classes: uniqueSorted(semantic.expected.classes), + }, + })}`; +}; + +const createCursor = (queryId: string, offset: number): string => + `jsrqc_${String(offset)}_${digest({ query_id: queryId, offset })}`; + +const parseCursor = (cursor: string | null, queryId: string): number => { + if (cursor === null) return 0; + const match = /^jsrqc_([0-9]+)_([a-f0-9]{64})$/u.exec(cursor); + const offsetText = match?.[1]; + const commitment = match?.[2]; + if (offsetText === undefined || commitment === undefined) + throw new TypeError("Semantic query cursor is malformed"); + const offset = Number(offsetText); + if ( + !Number.isSafeInteger(offset) || + offset < 0 || + commitment !== digest({ query_id: queryId, offset }) + ) + throw new TypeError("Semantic query cursor does not match this query"); + return offset; +}; + +const resolveSeeds = ( + graph: JavaScriptSemanticGraph, + input: JavaScriptSemanticQueryInput, +): string[] => { + const fingerprintFunctions = new Set( + input.seed.kind === "function" + ? graph.fingerprints + .filter(({ fingerprint_sha256 }) => { + const seed = input.seed; + return ( + seed.kind === "function" && + fingerprint_sha256 === seed.fingerprint_sha256 + ); + }) + .map(({ function_node_id }) => function_node_id) + : [], + ); + return graph.nodes + .filter((node) => seedMatches(node, input, fingerprintFunctions)) + .map(({ node_id }) => node_id) + .sort(compareCodePoints); +}; + +const seedMatches = ( + node: JavaScriptSemanticGraphNode, + input: JavaScriptSemanticQueryInput, + fingerprintFunctions: ReadonlySet, +): boolean => { + const seed = input.seed; + if (seed.kind === "semantic-node") return node.node_id === seed.node_id; + if (seed.kind === "application-node") + return node.application_node_ids.includes(seed.node_id); + if (seed.kind === "function") return fingerprintFunctions.has(node.node_id); + if (seed.kind === "literal") + return ( + node.kind === "literal" && + canonicalJson(node.properties.value) === canonicalJson(seed.value) + ); + if (seed.kind === "property") + return node.kind === "property-slot" && node.properties.name === seed.name; + if (seed.kind === "endpoint") + return node.kind === "request" && node.properties.endpoint === seed.value; + if (seed.kind === "event") + return ( + ["event", "listener"].includes(node.kind) && + node.properties.event_name === seed.name + ); + return node.kind === "boundary" && node.properties.field === seed.field; +}; + +const admitSeeds = ( + graph: JavaScriptSemanticGraph, + seeds: readonly string[], + input: JavaScriptSemanticQueryInput, +): SeedAdmission => { + const nodes = new Map(graph.nodes.map((node) => [node.node_id, node])); + const retained: string[] = []; + const functions = new Set(); + const modules = new Set(); + const frontier: QueryFrontier[] = []; + for (const nodeId of seeds) { + const node = nodes.get(nodeId); + if (node === undefined) continue; + const reason = seedBlockedReason(node, { + retained, + functions, + modules, + input, + }); + if (reason !== null) { + frontier.push({ node_id: nodeId, depth: 0, reason }); + continue; + } + retained.push(nodeId); + retainOwners(node, functions, modules); + } + return { nodeIds: retained, frontier: uniqueFrontier(frontier) }; +}; + +interface SeedBounds { + readonly retained: readonly string[]; + readonly functions: ReadonlySet; + readonly modules: ReadonlySet; + readonly input: JavaScriptSemanticQueryInput; +} + +interface TraversalBounds { + readonly nodes: ReadonlyMap; + readonly nodeIds: ReadonlySet; + readonly relationIds: ReadonlySet; + readonly functionIds: ReadonlySet; + readonly modules: ReadonlySet; + readonly input: JavaScriptSemanticQueryInput; +} + +const seedBlockedReason = ( + node: JavaScriptSemanticGraphNode, + bounds: SeedBounds, +): QueryFrontier["reason"] | null => { + if (bounds.retained.length >= bounds.input.limits.max_seed_matches) + return "max-seed-matches"; + if (bounds.retained.length >= bounds.input.limits.max_nodes) + return "max-nodes"; + const functionId = functionOwner(node); + if ( + functionId !== null && + !bounds.functions.has(functionId) && + bounds.functions.size >= bounds.input.limits.max_functions + ) + return "max-functions"; + if ( + !bounds.modules.has(node.identity.module_path) && + bounds.modules.size >= bounds.input.limits.max_modules + ) + return "max-modules"; + return null; +}; + +const buildAdjacency = ( + relations: readonly JavaScriptSemanticGraphRelation[], + input: JavaScriptSemanticQueryInput, +): ReadonlyMap => { + const adjacency = new Map(); + const allowed = + input.allowed_relations === undefined + ? null + : new Set(input.allowed_relations); + for (const relation of relations) { + if (allowed !== null && !allowed.has(relation.relation)) continue; + if ( + relation.resolution === "candidate" && + !input.include_ambiguous_dynamic_edges + ) + continue; + addDirectedEntries(adjacency, relation, input.direction); + } + for (const entries of adjacency.values()) + entries.sort((left, right) => + compareCodePoints( + `${left.relation.relation_id}\0${left.nextNodeId}`, + `${right.relation.relation_id}\0${right.nextNodeId}`, + ), + ); + return adjacency; +}; + +const addDirectedEntries = ( + adjacency: Map, + relation: JavaScriptSemanticGraphRelation, + direction: JavaScriptSemanticQueryInput["direction"], +): void => { + if (direction === "callers") { + if (relation.relation === "calls") + addEntry( + adjacency, + relation.target_node_id, + relation.source_node_id, + relation, + ); + return; + } + if (direction === "ownership") { + if (!ownershipRelation(relation.relation)) return; + addEntry( + adjacency, + relation.source_node_id, + relation.target_node_id, + relation, + ); + addEntry( + adjacency, + relation.target_node_id, + relation.source_node_id, + relation, + ); + return; + } + if (direction === "backward-provenance") + addEntry( + adjacency, + relation.target_node_id, + relation.source_node_id, + relation, + ); + else + addEntry( + adjacency, + relation.source_node_id, + relation.target_node_id, + relation, + ); +}; + +const ownershipRelation = ( + relation: JavaScriptSemanticGraphRelation["relation"], +): boolean => + [ + "acquires", + "aggregates", + "awaits", + "chains", + "connects-stdio", + "creates-promise", + "detaches-task", + "owns", + "releases", + "returns-task", + "spawns", + ].includes(relation); + +const addEntry = ( + adjacency: Map, + from: string, + nextNodeId: string, + relation: JavaScriptSemanticGraphRelation, +): void => { + const entries = adjacency.get(from) ?? []; + entries.push({ relation, nextNodeId }); + adjacency.set(from, entries); +}; + +const traverse = ( + graph: JavaScriptSemanticGraph, + seeds: readonly string[], + adjacency: ReadonlyMap, + input: JavaScriptSemanticQueryInput, +): Traversal => { + const nodes = new Map(graph.nodes.map((node) => [node.node_id, node])); + const nodeIds = new Set(seeds); + const relationIds = new Set(); + const functionIds = new Set(); + const modules = new Set(); + const frontier: QueryFrontier[] = []; + for (const nodeId of nodeIds) + retainOwners(nodes.get(nodeId), functionIds, modules); + const depths = new Map([...nodeIds].map((nodeId) => [nodeId, 0])); + const queue = [...nodeIds]; + const bounds: TraversalBounds = { + nodes, + nodeIds, + relationIds, + functionIds, + modules, + input, + }; + for (let offset = 0; offset < queue.length; offset += 1) { + const current = queue[offset]; + if (current === undefined) continue; + const depth = depths.get(current) ?? 0; + for (const entry of adjacency.get(current) ?? []) { + const reason = blockedReason(entry, depth, bounds); + if (reason !== null) { + frontier.push({ node_id: current, depth, reason }); + continue; + } + relationIds.add(entry.relation.relation_id); + if (nodeIds.has(entry.nextNodeId)) continue; + const node = nodes.get(entry.nextNodeId); + if (node === undefined) continue; + nodeIds.add(entry.nextNodeId); + retainOwners(node, functionIds, modules); + depths.set(entry.nextNodeId, depth + 1); + queue.push(entry.nextNodeId); + } + } + return { + nodeIds, + relationIds, + functionIds, + modules, + frontier: uniqueFrontier(frontier), + }; +}; + +const blockedReason = ( + entry: TraversalEntry, + depth: number, + bounds: TraversalBounds, +): QueryFrontier["reason"] | null => { + if (depth >= bounds.input.limits.max_depth) return "max-depth"; + if ( + !bounds.relationIds.has(entry.relation.relation_id) && + bounds.relationIds.size >= bounds.input.limits.max_edges + ) + return "max-edges"; + if (bounds.nodeIds.has(entry.nextNodeId)) return null; + if (bounds.nodeIds.size >= bounds.input.limits.max_nodes) return "max-nodes"; + const node = bounds.nodes.get(entry.nextNodeId); + if (node === undefined) return null; + const functionId = functionOwner(node); + if ( + functionId !== null && + !bounds.functionIds.has(functionId) && + bounds.functionIds.size >= bounds.input.limits.max_functions + ) + return "max-functions"; + if ( + !bounds.modules.has(node.identity.module_path) && + bounds.modules.size >= bounds.input.limits.max_modules + ) + return "max-modules"; + return null; +}; + +const retainOwners = ( + node: JavaScriptSemanticGraphNode | undefined, + functions: Set, + modules: Set, +): void => { + if (node === undefined) return; + const functionId = functionOwner(node); + if (functionId !== null) functions.add(functionId); + modules.add(node.identity.module_path); +}; + +const functionOwner = (node: JavaScriptSemanticGraphNode): string | null => + node.kind === "function" ? node.node_id : node.function_node_id; + +const uniqueFrontier = (frontier: readonly QueryFrontier[]): QueryFrontier[] => + [ + ...new Map( + frontier.map((item) => [ + `${item.node_id}\0${String(item.depth)}\0${item.reason}`, + item, + ]), + ).values(), + ].sort((left, right) => + compareCodePoints(canonicalJson(left), canonicalJson(right)), + ); + +const relevantUnknownFrontiers = ( + graph: JavaScriptSemanticGraph, + nodeIds: ReadonlySet, + input: JavaScriptSemanticQueryInput, +): JavaScriptSemanticGraphUnknown[] => { + const allowed = + input.allowed_relations === undefined + ? null + : new Set(input.allowed_relations); + return graph.unknowns.filter( + (unknown) => + (unknown.node_id === null || nodeIds.has(unknown.node_id)) && + (allowed === null || + unknown.relation_kinds.some((relation) => allowed.has(relation))), + ); +}; + +const expectedMatchesFor = ( + nodes: readonly JavaScriptSemanticGraphNode[], + relations: readonly JavaScriptSemanticGraphRelation[], + input: JavaScriptSemanticQueryInput, +): JavaScriptSemanticGraphNode[] => { + if (input.expected === null) return []; + const classes = new Set(input.expected.classes); + const connected = new Set( + relations.map((relation) => + input.expected?.role === "source" + ? relation.target_node_id + : relation.source_node_id, + ), + ); + return nodes.filter( + ({ kind, node_id }) => classes.has(kind) && !connected.has(node_id), + ); +}; + +/** Return a relation's declared semantic coverage family. */ +export const javaScriptSemanticRelationFamily = ( + relation: JavaScriptSemanticGraphRelation["relation"], +) => JAVASCRIPT_SEMANTIC_RELATION_FAMILY[relation]; diff --git a/src/domain/javascriptSemanticQueryAssessment.ts b/src/domain/javascriptSemanticQueryAssessment.ts new file mode 100644 index 000000000..b73590354 --- /dev/null +++ b/src/domain/javascriptSemanticQueryAssessment.ts @@ -0,0 +1,110 @@ +import { compareCodePoints } from "./javascriptApplicationGraph.js"; +import type { JavaScriptSemanticGraph } from "./javascriptSemanticGraph.js"; +import type { JavaScriptSemanticGraphUnknown } from "./javascriptSemanticGraphSchemas.js"; +import type { JavaScriptSemanticQueryResult } from "./javascriptSemanticQuerySchemas.js"; + +/** One caller-limit frontier retained by deterministic query traversal. */ +export interface JavaScriptSemanticQueryFrontier { + readonly node_id: string; + readonly depth: number; + readonly reason: + | "max-depth" + | "max-edges" + | "max-functions" + | "max-modules" + | "max-nodes" + | "max-seed-matches"; +} + +/** Inputs needed to classify one completed semantic traversal. */ +export interface JavaScriptSemanticQueryAssessmentInput { + readonly graph: JavaScriptSemanticGraph; + readonly totalSeeds: number; + readonly retainedSeeds: number; + readonly expectedMatches: number; + readonly hasExpectation: boolean; + readonly frontier: readonly JavaScriptSemanticQueryFrontier[]; + readonly unknowns: readonly JavaScriptSemanticGraphUnknown[]; +} + +/** Status, coverage, limits, and limitations derived from one traversal. */ +export interface JavaScriptSemanticQueryAssessment { + readonly status: JavaScriptSemanticQueryResult["status"]; + readonly coverage: JavaScriptSemanticQueryResult["coverage"]; + readonly limitations: string[]; + readonly acceptedLimitRanges: JavaScriptSemanticQueryResult["accepted_limit_ranges"]; +} + +/** Classify a bounded semantic traversal without promoting unknowns to absence. */ +export const assessJavaScriptSemanticQuery = ( + input: JavaScriptSemanticQueryAssessmentInput, +): JavaScriptSemanticQueryAssessment => ({ + status: queryStatus(input), + coverage: queryCoverage(input), + limitations: queryLimitations(input), + acceptedLimitRanges: { + max_seed_matches: { minimum: 1, maximum: 1_000 }, + max_nodes: { minimum: 1, maximum: 50_000 }, + max_edges: { minimum: 1, maximum: 100_000 }, + max_depth: { minimum: 0, maximum: 64 }, + max_functions: { minimum: 1, maximum: 10_000 }, + max_modules: { minimum: 1, maximum: 10_000 }, + page_size: { minimum: 1, maximum: 1_000 }, + }, +}); + +const queryCoverage = ( + input: JavaScriptSemanticQueryAssessmentInput, +): JavaScriptSemanticQueryResult["coverage"] => ({ + status: + input.graph.coverage.status === "unavailable" + ? "unavailable" + : input.frontier.length > 0 + ? "truncated" + : input.unknowns.length > 0 || + input.graph.coverage.status !== "complete" + ? "partial" + : "complete", + frontier: [...input.frontier], +}); + +const queryStatus = ( + input: JavaScriptSemanticQueryAssessmentInput, +): JavaScriptSemanticQueryResult["status"] => { + if (input.graph.coverage.status === "unavailable") return "unsupported"; + if (input.frontier.length > 0) return "truncated"; + if (input.unknowns.length > 0 || input.graph.coverage.status !== "complete") + return "partial"; + if ( + input.totalSeeds === 0 || + (input.hasExpectation && input.expectedMatches === 0) + ) + return "no-match"; + return input.totalSeeds > 1 ? "ambiguous" : "found"; +}; + +const queryLimitations = ( + input: JavaScriptSemanticQueryAssessmentInput, +): string[] => + uniqueSorted([ + "Semantic graph reachability is a static inference and does not prove runtime execution.", + ...input.graph.limitations, + ...(input.totalSeeds === input.retainedSeeds + ? [] + : [ + "Seed matches exceeded caller bounds; omitted starts remain unexplored.", + ]), + ...(input.frontier.length === 0 + ? [] + : [ + "Traversal stopped at explicit caller limits; frontier facts remain unknown.", + ]), + ...(input.unknowns.length === 0 + ? [] + : [ + "Relevant dynamic, unsupported, incomplete, or ambiguous semantics remain unknown.", + ]), + ]); + +const uniqueSorted = (values: readonly string[]): string[] => + [...new Set(values)].sort(compareCodePoints); diff --git a/src/domain/javascriptSemanticQuerySchemas.ts b/src/domain/javascriptSemanticQuerySchemas.ts new file mode 100644 index 000000000..00d3e8ca3 --- /dev/null +++ b/src/domain/javascriptSemanticQuerySchemas.ts @@ -0,0 +1,227 @@ +import { z } from "zod"; + +import { + JAVASCRIPT_SEMANTIC_NODE_KINDS, + JAVASCRIPT_SEMANTIC_RELATIONS, + javaScriptSemanticNodeSchema, + javaScriptSemanticRelationSchema, + javaScriptSemanticUnknownSchema, +} from "./javascriptSemanticGraphSchemas.js"; +import { jsonValueSchema } from "./jsonValue.js"; + +const digestSchema = z.string().regex(/^[a-f0-9]{64}$/u); +const semanticNodeIdSchema = z.string().regex(/^jsrg_node_[a-f0-9]{64}$/u); +const evidenceIdSchema = z.string().regex(/^ev_[a-f0-9]{64}$/u); + +/** Exact accepted ranges and defaults for one semantic traversal. */ +export const JAVASCRIPT_SEMANTIC_QUERY_LIMIT_RANGES = { + max_seed_matches: { minimum: 1, maximum: 1_000, default: 25 }, + max_nodes: { minimum: 1, maximum: 50_000, default: 2_000 }, + max_edges: { minimum: 1, maximum: 100_000, default: 4_000 }, + max_depth: { minimum: 0, maximum: 64, default: 12 }, + max_functions: { minimum: 1, maximum: 10_000, default: 500 }, + max_modules: { minimum: 1, maximum: 10_000, default: 500 }, + page_size: { minimum: 1, maximum: 1_000, default: 200 }, +} as const; + +const literalSeedSchema = z.strictObject({ + kind: z.literal("literal"), + value: jsonValueSchema.refine( + (value) => + value === null || + typeof value === "string" || + typeof value === "number" || + typeof value === "boolean", + "Literal seeds accept only JSON primitive values", + ), +}); + +/** Authenticated starting points accepted by semantic tracing. */ +export const javaScriptSemanticQuerySeedSchema = z.discriminatedUnion("kind", [ + z.strictObject({ + kind: z.literal("semantic-node"), + node_id: semanticNodeIdSchema, + }), + z.strictObject({ + kind: z.literal("application-node"), + node_id: z.string().regex(/^jag_node_[a-f0-9]{64}$/u), + }), + literalSeedSchema, + z.strictObject({ + kind: z.literal("function"), + fingerprint_sha256: digestSchema, + }), + z.strictObject({ + kind: z.literal("property"), + name: z.string().min(1).max(4_096), + }), + z.strictObject({ + kind: z.literal("endpoint"), + value: z.string().min(1).max(16_384), + }), + z.strictObject({ + kind: z.literal("event"), + name: z.string().min(1).max(4_096), + }), + z.strictObject({ + kind: z.literal("boundary-field"), + field: z.string().min(1).max(4_096), + }), +]); + +/** Strict caller limits for deterministic bounded traversal. */ +export const javaScriptSemanticQueryLimitsSchema = z.strictObject({ + max_seed_matches: z.number().int().min(1).max(1_000).default(25), + max_nodes: z.number().int().min(1).max(50_000).default(2_000), + max_edges: z.number().int().min(1).max(100_000).default(4_000), + max_depth: z.number().int().min(0).max(64).default(12), + max_functions: z.number().int().min(1).max(10_000).default(500), + max_modules: z.number().int().min(1).max(10_000).default(500), + page_size: z.number().int().min(1).max(1_000).default(200), +}); + +const sourceMapAuthoritySchema = z.discriminatedUnion("authority", [ + z.strictObject({ authority: z.literal("none") }), + z.strictObject({ + authority: z.literal("authenticated"), + evidence_id: evidenceIdSchema, + source_map_sha256: digestSchema, + }), +]); + +/** Parsed pure-domain query over one authenticated companion graph. */ +export const javaScriptSemanticQueryInputSchema = z.strictObject({ + seed: javaScriptSemanticQuerySeedSchema, + direction: z.enum([ + "backward-provenance", + "forward-influence", + "callers", + "consumers", + "ownership", + ]), + allowed_relations: z + .array(z.enum(JAVASCRIPT_SEMANTIC_RELATIONS)) + .min(1) + .max(JAVASCRIPT_SEMANTIC_RELATIONS.length) + .optional(), + include_ambiguous_dynamic_edges: z.boolean().default(false), + expected: z + .strictObject({ + role: z.enum(["source", "sink"]), + classes: z.array(z.enum(JAVASCRIPT_SEMANTIC_NODE_KINDS)).min(1).max(24), + }) + .nullable() + .default(null), + source_map_authority: sourceMapAuthoritySchema.default({ authority: "none" }), + limits: javaScriptSemanticQueryLimitsSchema.default({ + max_seed_matches: 25, + max_nodes: 2_000, + max_edges: 4_000, + max_depth: 12, + max_functions: 500, + max_modules: 500, + page_size: 200, + }), + cursor: z + .string() + .regex(/^jsrqc_[0-9]+_[a-f0-9]{64}$/u) + .nullable() + .default(null), +}); + +const queryFrontierSchema = z.strictObject({ + node_id: semanticNodeIdSchema, + depth: z.number().int().min(0).max(65), + reason: z.enum([ + "max-depth", + "max-edges", + "max-functions", + "max-modules", + "max-nodes", + "max-seed-matches", + ]), +}); + +/** Deterministic, pageable semantic trace result. */ +export const javaScriptSemanticQueryResultSchema = z.strictObject({ + schema_version: z.literal(1), + query_id: z.string().regex(/^jsrq_[a-f0-9]{64}$/u), + source_graph_id: z.string().regex(/^jsrg_[a-f0-9]{64}$/u), + seed: javaScriptSemanticQuerySeedSchema, + direction: javaScriptSemanticQueryInputSchema.shape.direction, + status: z.enum([ + "found", + "no-match", + "ambiguous", + "partial", + "truncated", + "unsupported", + ]), + seed_node_ids: z.array(semanticNodeIdSchema).max(1_000), + nodes: z.array(javaScriptSemanticNodeSchema).max(3_000), + relations: z.array(javaScriptSemanticRelationSchema).max(1_000), + unknowns: z.array(javaScriptSemanticUnknownSchema).max(100_000), + expected_match_node_ids: z.array(semanticNodeIdSchema).max(50_000), + summary: z.strictObject({ + total_seed_matches: z.number().int().min(0), + retained_seed_matches: z.number().int().min(0), + traversed_nodes: z.number().int().min(0), + traversed_relations: z.number().int().min(0), + traversed_functions: z.number().int().min(0), + traversed_modules: z.number().int().min(0), + relevant_unknowns: z.number().int().min(0), + }), + coverage: z.strictObject({ + status: z.enum(["complete", "partial", "truncated", "unavailable"]), + frontier: z.array(queryFrontierSchema).max(100_000), + }), + page: z.strictObject({ + offset: z.number().int().min(0), + size: z.number().int().min(0).max(1_000), + next_cursor: z + .string() + .regex(/^jsrqc_[0-9]+_[a-f0-9]{64}$/u) + .nullable(), + }), + applied_limits: javaScriptSemanticQueryLimitsSchema, + accepted_limit_ranges: z.strictObject({ + max_seed_matches: z.strictObject({ + minimum: z.literal(1), + maximum: z.literal(1_000), + }), + max_nodes: z.strictObject({ + minimum: z.literal(1), + maximum: z.literal(50_000), + }), + max_edges: z.strictObject({ + minimum: z.literal(1), + maximum: z.literal(100_000), + }), + max_depth: z.strictObject({ + minimum: z.literal(0), + maximum: z.literal(64), + }), + max_functions: z.strictObject({ + minimum: z.literal(1), + maximum: z.literal(10_000), + }), + max_modules: z.strictObject({ + minimum: z.literal(1), + maximum: z.literal(10_000), + }), + page_size: z.strictObject({ + minimum: z.literal(1), + maximum: z.literal(1_000), + }), + }), + limitations: z.array(z.string().min(1).max(4_096)).max(1_000), +}); + +/** Validated semantic query input. */ +export type JavaScriptSemanticQueryInput = z.infer< + typeof javaScriptSemanticQueryInputSchema +>; +/** Validated semantic query result. */ +export type JavaScriptSemanticQueryResult = z.infer< + typeof javaScriptSemanticQueryResultSchema +>; diff --git a/tests/javascriptSemanticGraphQuery.test.ts b/tests/javascriptSemanticGraphQuery.test.ts new file mode 100644 index 000000000..771e92ac4 --- /dev/null +++ b/tests/javascriptSemanticGraphQuery.test.ts @@ -0,0 +1,477 @@ +import { describe, expect, it } from "vitest"; + +import type { ApplicationGraphEvidence } from "../src/domain/javascriptApplicationEvidenceSchemas.js"; +import { + JAVASCRIPT_SEMANTIC_NODE_KINDS, + JAVASCRIPT_SEMANTIC_RELATION_FAMILIES, + JAVASCRIPT_SEMANTIC_RELATIONS, +} from "../src/domain/javascriptSemanticGraphSchemas.js"; +import { + createJavaScriptSemanticFingerprint, + createJavaScriptSemanticGraph, + createJavaScriptSemanticGraphNode, + createJavaScriptSemanticGraphRelation, + createJavaScriptSemanticGraphUnknown, + parseJavaScriptSemanticGraph, + serializeJavaScriptSemanticGraph, + type JavaScriptSemanticGraph, + type JavaScriptSemanticGraphNode, +} from "../src/domain/javascriptSemanticGraph.js"; +import { queryJavaScriptSemanticGraph } from "../src/domain/javascriptSemanticQuery.js"; +import { javaScriptSemanticQueryInputSchema } from "../src/domain/javascriptSemanticQuerySchemas.js"; + +const SHA = "a".repeat(64); +const JAG_ID = `jag_${"b".repeat(64)}`; +const completeCoverage = { + status: "complete" as const, + truncated: false, + omitted_count: 0, + limits: [], +}; + +const evidence = ( + state: "observed" | "inferred" = "observed", +): ApplicationGraphEvidence => ({ + authority: + state === "observed" + ? "ast-static-analysis" + : "static-relationship-inference", + state, + confidence: state === "observed" ? "exact" : "high", + artifact: { available: true, artifact_id: `art_${SHA}`, sha256: SHA }, + location: { + available: true, + value: { + kind: "source-range", + source: "bundle.js", + start: { line: 1, column: 0 }, + end: { line: 1, column: 10 }, + }, + }, + extractor: { + name: "test", + version: "1", + operation: "recover-semantic-relation", + executable_sha256: null, + }, + coverage: completeCoverage, + limitations: + state === "observed" + ? [] + : ["Static reachability does not prove runtime execution."], + evidence_ids: [], +}); + +const unknownEvidence = (): ApplicationGraphEvidence => ({ + authority: "unknown", + state: "unknown", + confidence: "unknown", + artifact: { + available: false, + reason: "unknown", + detail: "Unknown artifact.", + }, + location: { + available: false, + reason: "unresolved", + detail: "Dynamic call target.", + }, + extractor: { + name: "test", + version: "1", + operation: "retain-dynamic-call", + executable_sha256: null, + }, + coverage: { + status: "unknown", + truncated: false, + omitted_count: null, + limits: [], + }, + limitations: ["Dynamic call target remains unknown."], + evidence_ids: [], +}); + +const node = ( + kind: (typeof JAVASCRIPT_SEMANTIC_NODE_KINDS)[number], + role: string, + properties: Record = {}, + functionNodeId: string | null = null, +): JavaScriptSemanticGraphNode => + createJavaScriptSemanticGraphNode({ + kind, + identity: { + artifact_sha256: SHA, + module_path: "bundle.js", + source_range: { + start: { line: 1, column: role.length }, + end: { line: 1, column: role.length + 1 }, + }, + role_key: role, + }, + function_node_id: functionNodeId, + application_node_ids: [], + label: role, + properties, + evidence: evidence(), + }); + +const fixtureGraph = (withUnknown = false): JavaScriptSemanticGraph => { + const module = node("module", "module"); + const literal = node("literal", "literal", { value: "TOKEN" }); + const binding = node("binding", "binding"); + const callable = node("function", "function"); + const request = node("request", "request", { + endpoint: "https://example.invalid/v1", + }); + const relations = [ + createJavaScriptSemanticGraphRelation({ + source_node_id: literal.node_id, + target_node_id: binding.node_id, + relation: "defines", + resolution: "resolved", + properties: {}, + evidence: evidence("inferred"), + }), + createJavaScriptSemanticGraphRelation({ + source_node_id: binding.node_id, + target_node_id: callable.node_id, + relation: "captures", + resolution: "resolved", + properties: {}, + evidence: evidence("inferred"), + }), + createJavaScriptSemanticGraphRelation({ + source_node_id: callable.node_id, + target_node_id: request.node_id, + relation: "constructs-request", + resolution: "resolved", + properties: {}, + evidence: evidence("inferred"), + }), + ]; + const dynamic = createJavaScriptSemanticGraphUnknown({ + node_id: callable.node_id, + family: "call-flow", + relation_kinds: ["calls"], + reason: "dynamic-call", + detail: "Computed callee is unresolved.", + candidate_node_ids: [], + evidence: unknownEvidence(), + }); + const unknowns = withUnknown ? [dynamic] : []; + const coverageFamilies = JAVASCRIPT_SEMANTIC_RELATION_FAMILIES.map( + (family) => ({ + family, + status: + withUnknown && family === "call-flow" + ? ("unknown" as const) + : ("complete" as const), + retained_relations: relations.filter(({ relation }) => + relation === "defines" + ? family === "data-flow" + : relation === "captures" + ? family === "closure" + : family === "request", + ).length, + omitted_relations: withUnknown && family === "call-flow" ? null : 0, + unknown_ids: + withUnknown && family === "call-flow" ? [dynamic.unknown_id] : [], + }), + ); + const fingerprint = createJavaScriptSemanticFingerprint({ + function_node_id: callable.node_id, + algorithm: "rea.javascript-semantic-function/v1", + status: "complete", + components: { + parameter_arity: 0, + normalized_ast_sha256: "1".repeat(64), + control_flow_sha256: "2".repeat(64), + relation_shape_sha256: "3".repeat(64), + literal_set_sha256: "4".repeat(64), + effects: ["network"], + }, + limitations: [], + evidence: evidence("inferred"), + }); + return createJavaScriptSemanticGraph({ + schema: "JavaScriptSemanticRelationGraph", + schema_version: 1, + root_artifact_sha256: SHA, + application_graph_id: JAG_ID, + root_node_ids: [module.node_id], + nodes: [request, callable, binding, literal, module], + relations, + fingerprints: [fingerprint], + unknowns, + coverage: { + status: withUnknown ? "partial" : "complete", + truncated: false, + omitted_nodes: withUnknown ? null : 0, + omitted_relations: withUnknown ? null : 0, + limits: [], + families: coverageFamilies, + }, + limitations: withUnknown ? ["Dynamic call target remains unknown."] : [], + }); +}; + +const graphWithCandidateEdge = (): JavaScriptSemanticGraph => { + const graph = fixtureGraph(); + const literal = graph.nodes.find(({ kind }) => kind === "literal"); + const request = graph.nodes.find(({ kind }) => kind === "request"); + if (literal === undefined || request === undefined) + throw new TypeError("Semantic fixture nodes are missing"); + const candidate = createJavaScriptSemanticGraphRelation({ + source_node_id: literal.node_id, + target_node_id: request.node_id, + relation: "supplies-request-field", + resolution: "candidate", + properties: { field: "authorization" }, + evidence: evidence("inferred"), + }); + const { graph_id: _graphId, ...input } = graph; + return createJavaScriptSemanticGraph({ + ...input, + relations: [...graph.relations, candidate], + coverage: { + ...graph.coverage, + families: graph.coverage.families.map((family) => + family.family === "request" + ? { + ...family, + retained_relations: family.retained_relations + 1, + } + : family, + ), + }, + }); +}; + +const graphWithDuplicateLiteral = (): JavaScriptSemanticGraph => { + const graph = fixtureGraph(); + const duplicate = node("literal", "literal-copy", { value: "TOKEN" }); + const { graph_id: _graphId, ...input } = graph; + return createJavaScriptSemanticGraph({ + ...input, + nodes: [...graph.nodes, duplicate], + }); +}; + +describe("JavaScript semantic relation graph", () => { + it("defines every required v1 relation family and canonicalizes records", () => { + expect(JAVASCRIPT_SEMANTIC_RELATIONS).toContain("argument-to-parameter"); + expect(JAVASCRIPT_SEMANTIC_RELATIONS).toContain("detaches-task"); + expect(JAVASCRIPT_SEMANTIC_RELATIONS).toContain("forwards-signal"); + expect(JAVASCRIPT_SEMANTIC_RELATIONS).toContain("validates"); + const graph = fixtureGraph(); + expect( + parseJavaScriptSemanticGraph( + JSON.parse(serializeJavaScriptSemanticGraph(graph)), + ), + ).toEqual(graph); + expect(graph.nodes.map(({ node_id }) => node_id)).toEqual( + graph.nodes.map(({ node_id }) => node_id).toSorted(), + ); + }); + + it("rejects stale identities, dangling endpoints, and incomplete coverage claims", () => { + const graph = fixtureGraph(); + const { graph_id: _graphId, ...graphInput } = graph; + expect(() => + parseJavaScriptSemanticGraph({ + ...graph, + graph_id: `jsrg_${"0".repeat(64)}`, + }), + ).toThrow(); + expect(() => + createJavaScriptSemanticGraph({ + ...graphInput, + relations: [ + { + ...graph.relations[0], + target_node_id: `jsrg_node_${"f".repeat(64)}`, + }, + ], + }), + ).toThrow(); + expect(() => + createJavaScriptSemanticGraph({ + ...graphInput, + coverage: { ...graph.coverage, omitted_nodes: 1 }, + }), + ).toThrow(/Complete graph coverage/u); + }); +}); + +describe("JavaScript semantic query", () => { + it("traces deterministic forward influence and pages without changing identity", () => { + const graph = fixtureGraph(); + const input = { + seed: { kind: "literal" as const, value: "TOKEN" }, + direction: "forward-influence" as const, + expected: { role: "sink" as const, classes: ["request" as const] }, + limits: { + max_seed_matches: 25, + max_nodes: 20, + max_edges: 20, + max_depth: 12, + max_functions: 10, + max_modules: 10, + page_size: 2, + }, + }; + const first = queryJavaScriptSemanticGraph(graph, input); + expect(first).toMatchObject({ + status: "found", + summary: { traversed_nodes: 4, traversed_relations: 3 }, + page: { offset: 0, size: 2 }, + }); + expect(first.expected_match_node_ids).toHaveLength(1); + expect(first.page.next_cursor).not.toBeNull(); + const second = queryJavaScriptSemanticGraph(graph, { + ...input, + cursor: first.page.next_cursor, + }); + expect(second.query_id).toBe(first.query_id); + expect(second.page).toMatchObject({ + offset: 2, + size: 1, + next_cursor: null, + }); + expect(second.relations[0]?.relation_id).not.toBe( + first.relations[0]?.relation_id, + ); + }); + + it("keeps relevant dynamic frontiers unknown", () => { + const result = queryJavaScriptSemanticGraph(fixtureGraph(true), { + seed: { kind: "literal", value: "TOKEN" }, + direction: "forward-influence", + }); + expect(result.status).toBe("partial"); + expect(result.coverage.status).toBe("partial"); + expect(result.unknowns).toMatchObject([ + { reason: "dynamic-call", relation_kinds: ["calls"] }, + ]); + }); + + it("excludes candidate edges unless the caller explicitly opts in", () => { + const graph = graphWithCandidateEdge(); + const input = { + seed: { kind: "literal" as const, value: "TOKEN" }, + direction: "forward-influence" as const, + allowed_relations: ["supplies-request-field" as const], + }; + const conservative = queryJavaScriptSemanticGraph(graph, input); + const optedIn = queryJavaScriptSemanticGraph(graph, { + ...input, + include_ambiguous_dynamic_edges: true, + }); + expect(conservative.summary.traversed_relations).toBe(0); + expect(optedIn.summary.traversed_relations).toBe(1); + expect(optedIn.relations).toMatchObject([ + { relation: "supplies-request-field", resolution: "candidate" }, + ]); + }); + + it("bounds seed ambiguity before traversal", () => { + const result = queryJavaScriptSemanticGraph(graphWithDuplicateLiteral(), { + seed: { kind: "literal", value: "TOKEN" }, + direction: "forward-influence", + limits: { + max_seed_matches: 1, + max_nodes: 20, + max_edges: 20, + max_depth: 12, + max_functions: 10, + max_modules: 10, + page_size: 10, + }, + }); + expect(result.summary).toMatchObject({ + total_seed_matches: 2, + retained_seed_matches: 1, + }); + expect(result.status).toBe("truncated"); + expect(result.coverage.frontier).toContainEqual( + expect.objectContaining({ reason: "max-seed-matches", depth: 0 }), + ); + }); + + it("reports exact caller limits and a deterministic truncation frontier", () => { + const result = queryJavaScriptSemanticGraph(fixtureGraph(), { + seed: { kind: "literal", value: "TOKEN" }, + direction: "forward-influence", + limits: { + max_seed_matches: 1, + max_nodes: 2, + max_edges: 10, + max_depth: 12, + max_functions: 10, + max_modules: 10, + page_size: 10, + }, + }); + expect(result.status).toBe("truncated"); + expect(result.coverage.frontier).toMatchObject([{ reason: "max-nodes" }]); + expect(result.applied_limits.max_nodes).toBe(2); + expect(result.accepted_limit_ranges.max_nodes).toEqual({ + minimum: 1, + maximum: 50_000, + }); + }); + + it.each([ + { + limit: { max_edges: 1 }, + reason: "max-edges", + }, + { + limit: { max_depth: 0 }, + reason: "max-depth", + }, + ])("reports the $reason frontier", ({ limit, reason }) => { + const result = queryJavaScriptSemanticGraph(fixtureGraph(), { + seed: { kind: "literal", value: "TOKEN" }, + direction: "forward-influence", + limits: { + max_seed_matches: 10, + max_nodes: 20, + max_edges: limit.max_edges ?? 20, + max_depth: limit.max_depth ?? 12, + max_functions: 10, + max_modules: 10, + page_size: 10, + }, + }); + expect(result.status).toBe("truncated"); + expect(result.coverage.frontier).toContainEqual( + expect.objectContaining({ reason }), + ); + }); + + it("rejects out-of-range limits and cursors from another query", () => { + expect(() => + javaScriptSemanticQueryInputSchema.parse({ + seed: { kind: "literal", value: "TOKEN" }, + direction: "forward-influence", + limits: { max_nodes: 50_001 }, + }), + ).toThrow(); + const graph = fixtureGraph(); + const first = queryJavaScriptSemanticGraph(graph, { + seed: { kind: "literal", value: "TOKEN" }, + direction: "forward-influence", + limits: { page_size: 1 }, + }); + expect(() => + queryJavaScriptSemanticGraph(graph, { + seed: { kind: "endpoint", value: "https://example.invalid/v1" }, + direction: "backward-provenance", + limits: { page_size: 1 }, + cursor: first.page.next_cursor, + }), + ).toThrow(/cursor does not match/u); + }); +}); From 8678b57b5e03b0fb103803eb0a9e9ebda3124647 Mon Sep 17 00:00:00 2001 From: morluto <76467478+morluto@users.noreply.github.com> Date: Wed, 22 Jul 2026 21:44:24 +0000 Subject: [PATCH 3/4] fix(javascript): preserve candidate trace ambiguity --- src/domain/javascriptSemanticQuery.ts | 24 +++++++++++++++++++ .../javascriptSemanticQueryAssessment.ts | 8 +++++++ tests/javascriptSemanticGraphQuery.test.ts | 3 +++ 3 files changed, 35 insertions(+) diff --git a/src/domain/javascriptSemanticQuery.ts b/src/domain/javascriptSemanticQuery.ts index 7dfe45094..ca97d51f2 100644 --- a/src/domain/javascriptSemanticQuery.ts +++ b/src/domain/javascriptSemanticQuery.ts @@ -102,6 +102,11 @@ export const queryJavaScriptSemanticGraph = ( traversal.nodeIds, input, ); + const candidateRelations = relevantCandidateRelationCount( + graph, + traversal.nodeIds, + input, + ); const expectedMatches = expectedMatchesFor( retainedNodes, retainedRelations, @@ -115,6 +120,7 @@ export const queryJavaScriptSemanticGraph = ( hasExpectation: input.expected !== null, frontier, unknowns: relevantUnknowns, + candidateRelations, }); const nextOffset = offset + pageRelations.length; const nextCursor = @@ -543,6 +549,24 @@ const relevantUnknownFrontiers = ( ); }; +const relevantCandidateRelationCount = ( + graph: JavaScriptSemanticGraph, + nodeIds: ReadonlySet, + input: JavaScriptSemanticQueryInput, +): number => { + const allowed = + input.allowed_relations === undefined + ? null + : new Set(input.allowed_relations); + return graph.relations.filter( + (relation) => + relation.resolution === "candidate" && + (allowed === null || allowed.has(relation.relation)) && + (nodeIds.has(relation.source_node_id) || + nodeIds.has(relation.target_node_id)), + ).length; +}; + const expectedMatchesFor = ( nodes: readonly JavaScriptSemanticGraphNode[], relations: readonly JavaScriptSemanticGraphRelation[], diff --git a/src/domain/javascriptSemanticQueryAssessment.ts b/src/domain/javascriptSemanticQueryAssessment.ts index b73590354..89b4a3d44 100644 --- a/src/domain/javascriptSemanticQueryAssessment.ts +++ b/src/domain/javascriptSemanticQueryAssessment.ts @@ -25,6 +25,7 @@ export interface JavaScriptSemanticQueryAssessmentInput { readonly hasExpectation: boolean; readonly frontier: readonly JavaScriptSemanticQueryFrontier[]; readonly unknowns: readonly JavaScriptSemanticGraphUnknown[]; + readonly candidateRelations: number; } /** Status, coverage, limits, and limitations derived from one traversal. */ @@ -62,6 +63,7 @@ const queryCoverage = ( : input.frontier.length > 0 ? "truncated" : input.unknowns.length > 0 || + input.candidateRelations > 0 || input.graph.coverage.status !== "complete" ? "partial" : "complete", @@ -75,6 +77,7 @@ const queryStatus = ( if (input.frontier.length > 0) return "truncated"; if (input.unknowns.length > 0 || input.graph.coverage.status !== "complete") return "partial"; + if (input.candidateRelations > 0) return "ambiguous"; if ( input.totalSeeds === 0 || (input.hasExpectation && input.expectedMatches === 0) @@ -104,6 +107,11 @@ const queryLimitations = ( : [ "Relevant dynamic, unsupported, incomplete, or ambiguous semantics remain unknown.", ]), + ...(input.candidateRelations === 0 + ? [] + : [ + "Candidate relations remain ambiguous and do not establish a resolved semantic path.", + ]), ]); const uniqueSorted = (values: readonly string[]): string[] => diff --git a/tests/javascriptSemanticGraphQuery.test.ts b/tests/javascriptSemanticGraphQuery.test.ts index 771e92ac4..5468e306b 100644 --- a/tests/javascriptSemanticGraphQuery.test.ts +++ b/tests/javascriptSemanticGraphQuery.test.ts @@ -370,6 +370,9 @@ describe("JavaScript semantic query", () => { }); expect(conservative.summary.traversed_relations).toBe(0); expect(optedIn.summary.traversed_relations).toBe(1); + expect(conservative.status).toBe("ambiguous"); + expect(conservative.coverage.status).toBe("partial"); + expect(optedIn.status).toBe("ambiguous"); expect(optedIn.relations).toMatchObject([ { relation: "supplies-request-field", resolution: "candidate" }, ]); From 508a9ccde9c4beef2fa4dbd39093a1030287e967 Mon Sep 17 00:00:00 2001 From: morluto <76467478+morluto@users.noreply.github.com> Date: Wed, 22 Jul 2026 23:07:51 +0000 Subject: [PATCH 4/4] feat(javascript): expose bounded semantic tracing --- README.md | 2 +- README_ar.md | 2 +- README_ja.md | 2 +- README_ko.md | 2 +- README_zh.md | 2 +- docs/javascript-application-workflows.md | 45 +- docs/product-catalog.json | 35 +- .../managed-conformance-manifest.json | 4 +- scripts/lib/catalog-core.mjs | 3 + scripts/lib/catalog-schemas.mjs | 17 +- skills/reverse-engineer-anything/SKILL.md | 4 +- .../ApplicationWorkflowEvidenceResolver.ts | 42 ++ .../JavaScriptApplicationEvidence.ts | 2 +- .../JavaScriptApplicationEvidenceGraph.ts | 17 +- .../JavaScriptApplicationService.ts | 2 +- .../JavaScriptApplicationWorkflowEvidence.ts | 19 + .../JavaScriptArtifactReconstruction.ts | 9 + .../JavaScriptSemanticGraphBuilder.ts | 462 ++++++++++++++++++ .../JavaScriptSemanticGraphConstruction.ts | 256 ++++++++++ .../JavaScriptSemanticGraphEvidence.ts | 111 +++++ .../JavaScriptSemanticGraphFlowProjection.ts | 127 +++++ .../JavaScriptSemanticGraphProjection.ts | 74 +++ .../JavaScriptSemanticTraceService.ts | 72 +++ src/cliApplicationCommands.ts | 11 + src/cliCommandNames.ts | 1 + src/contracts/applicationToolContracts.ts | 27 + .../applicationWorkflowInputContracts.ts | 20 + src/contracts/electronToolContracts.ts | 101 ++-- src/contracts/toolEffects.ts | 1 + src/domain/javascriptApplicationAnalysis.ts | 60 ++- .../javascriptRuntimeReconciliationParsing.ts | 8 +- src/domain/javascriptSemanticCalls.ts | 47 +- src/domain/javascriptSemanticGraph.ts | 40 +- .../javascriptSemanticGraphSerialization.ts | 39 ++ src/domain/javascriptSemanticIr.ts | 10 + src/domain/javascriptSemanticQuery.ts | 252 ++++------ .../javascriptSemanticQueryAssessment.ts | 18 +- src/domain/javascriptSemanticQueryIdentity.ts | 137 ++++++ .../javascriptSemanticQueryRelations.ts | 22 + src/domain/javascriptSemanticQuerySchemas.ts | 31 +- src/domain/javascriptSemanticReturns.ts | 4 + src/domain/javascriptSemanticTraceSchemas.ts | 27 + src/server/javascriptApplicationResult.ts | 36 +- src/server/registerApplicationTools.ts | 2 + .../traceSemantics.ts | 49 ++ tests/applicationWorkflowCli.test.ts | 49 ++ tests/applicationWorkflowMcp.test.ts | 30 ++ tests/contractSnapshot.test.ts | 2 +- tests/electronApplicationAnalysis.test.ts | 4 +- tests/electronMcp.test.ts | 10 +- .../javascriptArtifactReconstruction.test.ts | 48 ++ tests/javascriptSemanticGraphBuilder.test.ts | 314 ++++++++++++ tests/javascriptSemanticGraphQuery.test.ts | 43 +- tests/productCatalog.test.ts | 1 + tests/serverIdentity.test.ts | 6 +- tests/toolContracts.test.ts | 1 + 56 files changed, 2455 insertions(+), 307 deletions(-) create mode 100644 src/application/JavaScriptSemanticGraphBuilder.ts create mode 100644 src/application/JavaScriptSemanticGraphConstruction.ts create mode 100644 src/application/JavaScriptSemanticGraphEvidence.ts create mode 100644 src/application/JavaScriptSemanticGraphFlowProjection.ts create mode 100644 src/application/JavaScriptSemanticGraphProjection.ts create mode 100644 src/application/JavaScriptSemanticTraceService.ts create mode 100644 src/domain/javascriptSemanticGraphSerialization.ts create mode 100644 src/domain/javascriptSemanticQueryIdentity.ts create mode 100644 src/domain/javascriptSemanticQueryRelations.ts create mode 100644 src/domain/javascriptSemanticTraceSchemas.ts create mode 100644 src/server/registerApplicationTools/traceSemantics.ts create mode 100644 tests/javascriptSemanticGraphBuilder.test.ts diff --git a/README.md b/README.md index 34a49df3f..e613f6085 100644 --- a/README.md +++ b/README.md @@ -380,7 +380,7 @@ REA handles the app analysis in steps 1–5. The agent performs step 6 with its | Managed PE/CLI | 8 | PE/CLI identity, metadata members, CIL hashes, P/Invoke/native-boundary declarations and verification, application-graph projection, decompiler reconstruction import, token remapping, runtime-correlation plans, and version comparison | | Browser observation | 8 | exact-origin CDP capture, bundle and source-map analysis, WebMCP discovery, session timelines, capture diff, and visual evidence | | Electron analysis | 4 | passive root-confined observation, bounded static application mapping, and evidence-backed static/runtime reconciliation | -| Application workflows | 8 | bounded cross-layer traces, unique-only version matching, static export return-shape comparison, approved Linux-isolated extracted-module replay, managed-runtime characterization, and reconstruction coverage closure | +| Application workflows | 9 | bounded cross-layer traces, unique-only version matching, static export return-shape comparison, approved Linux-isolated extracted-module replay, managed-runtime characterization, and reconstruction coverage closure | | Workspace and observation | 19 | target lifecycle, Evidence v2 bundles, direct finite replay-machine evaluation, process/artifact/function comparison, evidence-linked residual-unknown lifecycle | The public interface describes what the agent is trying to learn. Providers decide how to answer. macOS utilities handle common semantic inspection without launching Hopper; Hopper handles deeper native analysis; the process harness implements controlled behavioral capture. diff --git a/README_ar.md b/README_ar.md index fa76f7c52..e18861583 100644 --- a/README_ar.md +++ b/README_ar.md @@ -168,7 +168,7 @@ rea uninstall --purge-data | Managed PE/CLI | 8 | هوية PE/CLI، أعضاء البيانات الوصفية، بصمات CIL، تصريحات P/Invoke/الحدود الأصلية والتحقق منها، إسقاط رسم التطبيق، استيراد إعادة البناء من فك الترجمة، إعادة ربط الرموز، خطط ترابط وقت التشغيل ومقارنة الإصدارات | | مراقبة المتصفح | 8 | التقاط CDP محدود الأصل وتحليل الحزم وخرائط المصدر واكتشاف WebMCP والفروق والأدلة المرئية | | تحليل Electron | 4 | مراقبة سلبية ضمن جذور canonical، ورسم ثابت ومحدود للتطبيق، ومطابقة ثابتة/وقت تشغيل قائمة على Evidence | -| سير عمل التطبيقات | 8 | تتبع محدود ومقارنة إصدارات فريدة ومقارنة ثابتة لأشكال قيم إرجاع الصادرات وإعادة تشغيل وحدات معزولة ومعتمدة على Linux وتوصيف وقت التشغيل المُدار وإغلاق تغطية إعادة البناء | +| سير عمل التطبيقات | 9 | تتبع محدود ومقارنة إصدارات فريدة ومقارنة ثابتة لأشكال قيم إرجاع الصادرات وإعادة تشغيل وحدات معزولة ومعتمدة على Linux وتوصيف وقت التشغيل المُدار وإغلاق تغطية إعادة البناء | | جلسة الملف التنفيذي | 19 | فتح الأهداف وحزم الأدلة وتشغيل آلة إعادة محدودة ومقارنة العمليات والقطع والدوال وسجل المجهولات المتبقية | ## استخدام REA مع وكلاء برمجة آخرين diff --git a/README_ja.md b/README_ja.md index fa56cd499..5b694ad63 100644 --- a/README_ja.md +++ b/README_ja.md @@ -158,7 +158,7 @@ REA は手順 1〜5 のバイナリ解析を処理し、手順 6 はエージェ | Managed PE/CLI | 8 | PE/CLI 識別、メタデータメンバー、CIL ハッシュ、P/Invoke/ネイティブ境界宣言と検証、アプリケーショングラフ投影、逆コンパイル再構成インポート、構造的 token 再マッピング、ランタイム相関計画、バージョン比較 | | ブラウザ観察 | 8 | origin 限定 CDP 取得、bundle/source map 解析、WebMCP 検出、セッション、capture diff、視覚証拠 | | Electron 解析 | 4 | canonical ファイルルート内の受動観察、有界な静的アプリマッピング、Evidence に基づく静的/実行時の照合 | -| アプリケーション | 8 | 有界なクロスレイヤー追跡、一意限定のバージョン比較、静的なエクスポート戻り値形状比較、承認済み Linux 隔離 extracted-module replay、マネージドランタイム特性評価、再構築カバレッジ閉包 | +| アプリケーション | 9 | 有界なクロスレイヤー追跡、一意限定のバージョン比較、静的なエクスポート戻り値形状比較、承認済み Linux 隔離 extracted-module replay、マネージドランタイム特性評価、再構築カバレッジ閉包 | | バイナリセッション | 19 | `open_binary`、`binary_session`、証拠バンドル、有限リプレイ状態機械、プロセス／成果物／関数比較、残余未知レジストリ | ## 他のコーディングエージェントで使う diff --git a/README_ko.md b/README_ko.md index 73eeb5639..d5a1eeee0 100644 --- a/README_ko.md +++ b/README_ko.md @@ -158,7 +158,7 @@ REA는 1–5단계의 바이너리 분석을 처리합니다. 6단계는 에이 | Managed PE/CLI | 8 | PE/CLI ID, 메타데이터 멤버, CIL 해시, P/Invoke/네이티브 경계 선언 및 검증, 애플리케이션 그래프 투영, 디컴파일 재구성 가져오기, 구조적 token 재매핑, 런타임 상관 계획, 버전 비교 | | 브라우저 관찰 | 8 | origin 제한 CDP 캡처, bundle/source map 분석, WebMCP 검색, 세션 타임라인, capture diff 및 시각 증거 | | Electron 분석 | 4 | canonical 파일 루트 내 수동 관찰, 제한된 정적 앱 매핑, Evidence 기반 정적/런타임 조정 | -| 애플리케이션 | 8 | 제한된 교차 계층 추적, 고유 일치 버전 비교, 정적 export 반환 형태 비교, 승인된 Linux 격리 extracted-module replay, 관리형 런타임 특성화, 재구성 커버리지 폐쇄 | +| 애플리케이션 | 9 | 제한된 교차 계층 추적, 고유 일치 버전 비교, 정적 export 반환 형태 비교, 승인된 Linux 격리 extracted-module replay, 관리형 런타임 특성화, 재구성 커버리지 폐쇄 | | 바이너리 세션 | 19 | `open_binary`, `binary_session`, 증거 번들, 유한 replay 상태 머신, 프로세스·아티팩트·함수 비교, 잔여 미지수 레지스트리 | ## 다른 코딩 에이전트에서 사용하기 diff --git a/README_zh.md b/README_zh.md index 46f5e595a..c7eeca376 100644 --- a/README_zh.md +++ b/README_zh.md @@ -158,7 +158,7 @@ REA 负责第 1–5 步中的二进制分析。第 6 步由智能体使用其常 | Managed PE/CLI | 8 | PE/CLI 身份、元数据成员、CIL 哈希、P/Invoke/原生边界声明与验证、应用图谱投影、反编译重建导入、结构化 token 重映射、运行时关联计划与跨版本比较 | | 浏览器观察 | 8 | 精确来源 CDP 抓取、bundle/source map 分析、WebMCP 发现、会话时间线、capture diff 与视觉证据 | | Electron 分析 | 4 | 限定 canonical 文件根的被动观察、有界静态应用映射，以及基于 Evidence 的静态/运行时对账 | -| 应用工作流 | 8 | 有界跨层追踪、唯一匹配跨版本图比较、静态导出返回形状比较、经批准的 Linux 隔离 extracted-module replay、托管运行时表征和重建覆盖闭合 | +| 应用工作流 | 9 | 有界跨层追踪、唯一匹配跨版本图比较、静态导出返回形状比较、经批准的 Linux 隔离 extracted-module replay、托管运行时表征和重建覆盖闭合 | | 二进制会话 | 19 | `open_binary`、`binary_session`、证据包、有限重放状态机、进程、制品与函数比较、残余未知项注册表 | ## 与其他编程智能体一起使用 diff --git a/docs/javascript-application-workflows.md b/docs/javascript-application-workflows.md index 379ba0b43..f23e1e713 100644 --- a/docs/javascript-application-workflows.md +++ b/docs/javascript-application-workflows.md @@ -2,10 +2,9 @@ REA derives a bounded feature trace from one authenticated JavaScript Application Graph and compares two authenticated graph versions. The MCP tools -are `trace_application_feature`, `compare_application_versions`, and -`compare_javascript_export_shapes`; their CLI equivalents are -`rea trace-application-feature`, `rea compare-application-versions`, and -`rea compare-javascript-export-shapes`. +are `trace_application_feature`, `trace_javascript_semantics`, +`compare_application_versions`, and `compare_javascript_export_shapes`; their +CLI equivalents use the same names with hyphens. Both workflows consume Evidence v2 produced by `analyze_javascript_application` or `reconcile_javascript_runtime`. They do not @@ -14,6 +13,13 @@ native-analysis provider. Static artifact observations, passive runtime observations, relationship inferences, and unknown or unavailable facts retain their original graph authority. +Current `analyze_javascript_application` Evidence uses result schema v2. It +retains the structural JavaScript Application Graph and a separate semantic +relation graph bound to the same root artifact digest and structural graph ID. +Legacy v1 Evidence remains valid for structural workflows, but semantic tracing +returns an actionable request to reanalyze it rather than treating missing +semantic data as an empty graph. + ## Feature tracing Select one literal seed kind: node ID, route, string, API, IPC channel, module, @@ -29,6 +35,33 @@ matches exactly. Otherwise the result recommends provider-neutral follow-up tools and reports `requires-provider-analysis`; it never starts or switches a provider implicitly. +## Semantic relation tracing + +`trace_javascript_semantics` queries the authenticated companion graph without +rereading mutable files. A seed may identify a semantic or application node, +literal, function fingerprint, property, endpoint, event, or boundary field. +The query declares backward provenance, forward influence, callers, or +ownership and can restrict admitted relation kinds. Node, relation, depth, +function, module, seed-match, and page limits are validated against exact +published ranges. A committed cursor returns the next deterministic relation +page only for the same graph, query, and limits. + +The initial extractor covers lexical definitions and reads, closure captures, +and uniquely resolved local calls with argument-to-parameter and direct +return-to-call candidates. Dynamic calls/properties, parser recovery, and bound +frontiers stay explicit. Promise, event, timer, child-process, request, +configuration, boundary, resource, and fingerprint families are represented by +the versioned schema but report unsupported or unknown coverage until their +extractors supply facts. Missing relations in those families are never reported +as proven absence. + +Semantic relations use static-inference authority. `resolved` means the static +target identity was unique within admitted analysis; it does not mean the path +executed. Candidate edges are excluded by default, require explicit opt-in, and +keep the result ambiguous. Runtime Evidence can later corroborate an exact +mapped candidate, but structural reachability, semantic influence, runtime +observation, and causal proof remain separate claims. + ## Version comparison REA pairs entities only when a tier produces one unique candidate on each side. @@ -75,8 +108,8 @@ replay recommendation; it does not execute JavaScript. ## CLI and verification -All three CLI commands accept inline JSON or a JSON file up to 64 MiB. The input is -the same object used by the corresponding MCP tool. +All four CLI commands accept inline JSON or a JSON file up to 64 MiB. The input +is the same object used by the corresponding MCP tool. For two operator-provided directories or ASARs, run: diff --git a/docs/product-catalog.json b/docs/product-catalog.json index 9526ab20e..b598ad899 100644 --- a/docs/product-catalog.json +++ b/docs/product-catalog.json @@ -11,7 +11,7 @@ "skill_version": "23" }, "tools": { - "total": 97, + "total": 98, "families": [ { "id": "direct", @@ -132,7 +132,7 @@ { "id": "application", "surface": "application-workflow", - "count": 8, + "count": 9, "tools": [ "commit_reconstruction_coverage", "compare_application_versions", @@ -141,7 +141,8 @@ "prepare_node_characterization", "query_reconstruction_coverage", "run_controlled_replay", - "trace_application_feature" + "trace_application_feature", + "trace_javascript_semantics" ] }, { @@ -318,7 +319,8 @@ "prepare_node_characterization", "query_reconstruction_coverage", "run_controlled_replay", - "trace_application_feature" + "trace_application_feature", + "trace_javascript_semantics" ] }, { @@ -427,7 +429,7 @@ }, { "id": "javascript_application_analysis", - "version": 1 + "version": 2 }, { "id": "javascript_application_graph", @@ -441,6 +443,18 @@ "id": "javascript_feature_trace", "version": 1 }, + { + "id": "javascript_semantic_graph", + "version": 1 + }, + { + "id": "javascript_semantic_query", + "version": 1 + }, + { + "id": "javascript_semantic_trace", + "version": 1 + }, { "id": "managed_artifact_inspection", "version": 1 @@ -503,7 +517,7 @@ } ], "cli": { - "primary_count": 57, + "primary_count": 58, "commands": [ "analyze", "analyze-javascript-application", @@ -558,6 +572,7 @@ "setup", "trace", "trace-application-feature", + "trace-javascript-semantics", "uninstall", "upgrade", "verify-managed-native-boundaries", @@ -572,17 +587,17 @@ }, "runtime_catalog": { "counts": { - "cli_commands": 57, - "mcp_tools": 97, + "cli_commands": 58, + "mcp_tools": 98, "mcp_prompts": 6, "mcp_resources": 2, "mcp_resource_templates": 9 }, "digests": { - "tools_sha256": "073b2d48443034a05ba56b649057f9e5f0ab21afcb39be2c380e5a05bc0cecae", + "tools_sha256": "501d405efbf8fc3537d13791f87ac81f3b97f3f328b62710ca0885de7213a62f", "prompts_sha256": "f6da3ad38a1000c11364d1710f122ac4990551f04efd8c53ffb2e819b5cf5707", "resources_sha256": "94bebbab1a1b3cf47644c38d3de6469de568caa7bd7b842f529ead8ad320a912", - "combined_sha256": "a555f03ab0881d6e140b15d59ba8d75ba596a16a9690d26a8341d6b32395da5f" + "combined_sha256": "71de8a7174176823fd82c2666130233b5a1b4a1613583270d268774775b410bc" } } } diff --git a/docs/verification/managed-conformance-manifest.json b/docs/verification/managed-conformance-manifest.json index 2ea26ef25..7d01a964e 100644 --- a/docs/verification/managed-conformance-manifest.json +++ b/docs/verification/managed-conformance-manifest.json @@ -1,6 +1,6 @@ { "schema_version": 1, - "manifest_id": "ecm_a77e4a40338b12af5108a59c7323c22a0f2367ff2cd925c8c3fb4213b5eba480", + "manifest_id": "ecm_e04a25d07a540df3bec2fd5ba6d8a158e1f582dc94fc59225c7ba329cbabbbf6", "verifier": { "id": "rea-managed-conformance", "version": "1" @@ -14,7 +14,7 @@ "skill_digests": [ { "skill_id": "reverse-engineer-anything", - "sha256": "6308c7735c8b1f16f58f9d7020102fdf0a2492d3f78d979f4537d49e8e83fdc9" + "sha256": "a8ffb68f043fc9dfd0eb401caa6e548adce991d1e228b4539333cf90be17a755" } ], "claims": [ diff --git a/scripts/lib/catalog-core.mjs b/scripts/lib/catalog-core.mjs index d7f5df1fc..c55669e6a 100644 --- a/scripts/lib/catalog-core.mjs +++ b/scripts/lib/catalog-core.mjs @@ -60,6 +60,9 @@ const SOURCE_PATHS = { webScreenshot: "dist/domain/webScreenshot.js", javascriptApplicationGraph: "dist/domain/javascriptApplicationGraph.js", javascriptApplicationAnalysis: "dist/domain/javascriptApplicationAnalysis.js", + javascriptSemanticGraph: "dist/domain/javascriptSemanticGraph.js", + javascriptSemanticQuery: "dist/domain/javascriptSemanticQuerySchemas.js", + javascriptSemanticTrace: "dist/domain/javascriptSemanticTraceSchemas.js", javascriptFeatureTrace: "dist/domain/javascriptFeatureTraceSchemas.js", javascriptVersionComparison: "dist/domain/javascriptApplicationVersionComparisonSchemas.js", diff --git a/scripts/lib/catalog-schemas.mjs b/scripts/lib/catalog-schemas.mjs index ed86cf6ca..6535fb934 100644 --- a/scripts/lib/catalog-schemas.mjs +++ b/scripts/lib/catalog-schemas.mjs @@ -92,6 +92,21 @@ const durableSchemaDefinitions = (sources) => [ .applicationVersionComparisonResultSchema, ["schema_version"], ], + [ + "javascript_semantic_graph", + sources.javascriptSemanticGraph.javaScriptSemanticGraphSchema, + ["schema_version"], + ], + [ + "javascript_semantic_query", + sources.javascriptSemanticQuery.javaScriptSemanticQueryResultSchema, + ["schema_version"], + ], + [ + "javascript_semantic_trace", + sources.javascriptSemanticTrace.javaScriptSemanticTraceResultSchema, + ["schema_version"], + ], [ "javascript_feature_trace", sources.javascriptFeatureTrace.applicationFeatureTraceResultSchema, @@ -158,7 +173,7 @@ const observationSchemaDefinitions = (sources) => [ [ "javascript_application_analysis", sources.javascriptApplicationAnalysis - .javascriptApplicationAnalysisResultSchema, + .javaScriptApplicationAnalysisResultV2Schema, ["schema_version"], ], [ diff --git a/skills/reverse-engineer-anything/SKILL.md b/skills/reverse-engineer-anything/SKILL.md index 04ccbda24..0958d96de 100644 --- a/skills/reverse-engineer-anything/SKILL.md +++ b/skills/reverse-engineer-anything/SKILL.md @@ -3,8 +3,8 @@ name: reverse-engineer-anything description: Reverse engineer native, managed, Electron/JavaScript, packaged, and browser applications with REA. Use shipped-artifact or approved runtime evidence to explain features, compare versions, decompile code, or guide a reconstruction. Skip REA for ordinary source-repository architecture analysis. metadata: version: "23" - tool_count: 97 - catalog_digest: "a555f03ab0881d6e140b15d59ba8d75ba596a16a9690d26a8341d6b32395da5f" + tool_count: 98 + catalog_digest: "71de8a7174176823fd82c2666130233b5a1b4a1613583270d268774775b410bc" --- # REA diff --git a/src/application/ApplicationWorkflowEvidenceResolver.ts b/src/application/ApplicationWorkflowEvidenceResolver.ts index e448079ac..f0231fbd6 100644 --- a/src/application/ApplicationWorkflowEvidenceResolver.ts +++ b/src/application/ApplicationWorkflowEvidenceResolver.ts @@ -4,15 +4,18 @@ import { compareApplicationVersionsRequestSchema, compareJavaScriptExportShapesRequestSchema, traceApplicationFeatureRequestSchema, + traceJavaScriptSemanticsRequestSchema, type CompareApplicationVersionsRequest, type CompareJavaScriptExportShapesRequest, type TraceApplicationFeatureRequest, + type TraceJavaScriptSemanticsRequest, } from "../contracts/applicationWorkflowInputContracts.js"; import { AnalysisInputError, type AnalysisError } from "../domain/errors.js"; import { projectInputIssues } from "../domain/inputIssueProjection.js"; import { compareApplicationVersionsInputSchema } from "../domain/javascriptApplicationVersionComparisonSchemas.js"; import { compareJavaScriptExportShapesInputSchema } from "../domain/javascriptExportShapeComparisonSchemas.js"; import { traceApplicationFeatureInputSchema } from "../domain/javascriptFeatureTraceSchemas.js"; +import { traceJavaScriptSemanticsInputSchema } from "../domain/javascriptSemanticTraceSchemas.js"; import { err, ok, type Result } from "../domain/result.js"; import { resolveEvidenceReferences, @@ -25,6 +28,10 @@ const APPLICATION_GRAPH_IDENTITIES = [ operation: "analyze_javascript_application", predicate: "rea.javascript-application-analysis/v1", }, + { + operation: "analyze_javascript_application", + predicate: "rea.javascript-application-analysis/v2", + }, { operation: "reconcile_javascript_runtime", predicate: "rea.javascript-runtime-reconciliation/v1", @@ -36,6 +43,7 @@ const APPLICATION_GRAPH_IDENTITIES = [ ] as const satisfies readonly EvidenceSemanticIdentity[]; type TraceInput = z.output; +type SemanticTraceInput = z.output; type ComparisonInput = z.output; type ExportShapeComparisonInput = z.output< typeof compareJavaScriptExportShapesInputSchema @@ -86,6 +94,40 @@ export const resolveTraceApplicationFeatureRequestValidated = ( : invalid("trace_application_feature", parsed.error, raw); }; +/** Parse and resolve one semantic trace adapter request. */ +export const resolveTraceJavaScriptSemanticsRequest = ( + input: unknown, + lookup?: EvidenceLookup, +): Result => { + const parsed = traceJavaScriptSemanticsRequestSchema.safeParse(input); + return parsed.success + ? resolveTraceJavaScriptSemanticsRequestValidated(parsed.data, lookup) + : invalid("trace_javascript_semantics", parsed.error, input); +}; + +/** Resolve a semantic trace request already parsed by its public contract. */ +export const resolveTraceJavaScriptSemanticsRequestValidated = ( + input: TraceJavaScriptSemanticsRequest, + lookup?: EvidenceLookup, +): Result => { + const application = + input.application === undefined + ? resolveEvidenceReferences( + lookup, + input.application_evidence_id === undefined + ? [] + : [input.application_evidence_id], + APPLICATION_GRAPH_IDENTITIES, + ) + : ok([input.application]); + if (!application.ok) return application; + const raw = { application: application.value[0], query: input.query }; + const parsed = traceJavaScriptSemanticsInputSchema.safeParse(raw); + return parsed.success + ? ok(parsed.data) + : invalid("trace_javascript_semantics", parsed.error, raw); +}; + /** Parse and resolve one comparison adapter request into canonical input. */ export const resolveCompareApplicationVersionsRequest = ( input: unknown, diff --git a/src/application/JavaScriptApplicationEvidence.ts b/src/application/JavaScriptApplicationEvidence.ts index fa38b51d3..14b73f989 100644 --- a/src/application/JavaScriptApplicationEvidence.ts +++ b/src/application/JavaScriptApplicationEvidence.ts @@ -23,7 +23,7 @@ export const createJavaScriptApplicationEvidence = ( }, JAVASCRIPT_APPLICATION_PROVIDER, { - predicateType: "rea.javascript-application-analysis/v1", + predicateType: `rea.javascript-application-analysis/v${String(result.schema_version)}`, operation: "analyze_javascript_application", parameters: parameters(input), result: jsonValueSchema.parse(result), diff --git a/src/application/JavaScriptApplicationEvidenceGraph.ts b/src/application/JavaScriptApplicationEvidenceGraph.ts index cc3379fed..eb2d7ac11 100644 --- a/src/application/JavaScriptApplicationEvidenceGraph.ts +++ b/src/application/JavaScriptApplicationEvidenceGraph.ts @@ -10,6 +10,7 @@ import { type JavaScriptApplicationAnalysisResult, } from "../domain/javascriptApplicationAnalysis.js"; import type { JavaScriptApplicationGraph } from "../domain/javascriptApplicationGraph.js"; +import type { JavaScriptSemanticGraph } from "../domain/javascriptSemanticGraph.js"; import { javascriptRuntimeReconciliationResultSchema } from "../domain/javascriptRuntimeReconciliationSchemas.js"; import { managedApplicationGraphResultSchema } from "../domain/managedApplicationGraph.js"; @@ -22,6 +23,7 @@ export interface ApplicationGraphEvidenceSource { | "static-runtime-reconciliation" | "managed-application"; readonly rootArtifactSha256: string; + readonly semanticGraph: JavaScriptSemanticGraph | null; } /** Parse and authenticate one REA-produced JavaScript Application Graph Evidence. */ @@ -31,7 +33,10 @@ export const parseApplicationGraphEvidence = ( const evidence = parseEvidence(input); if ( evidence.operation === "analyze_javascript_application" && - evidence.predicate_type === "rea.javascript-application-analysis/v1" && + [ + "rea.javascript-application-analysis/v1", + "rea.javascript-application-analysis/v2", + ].includes(evidence.predicate_type) && providerMatches(evidence, JAVASCRIPT_APPLICATION_PROVIDER) ) return staticApplicationSource(evidence); @@ -65,6 +70,13 @@ const staticApplicationSource = ( const result = javascriptApplicationAnalysisResultSchema.parse( evidence.normalized_result, ); + if ( + evidence.predicate_type !== + `rea.javascript-application-analysis/v${String(result.schema_version)}` + ) + throw new TypeError( + "JavaScript application Evidence predicate does not match its result schema version", + ); analyzeJavaScriptApplicationInputSchema.parse({ input_path: result.input_path, ...evidence.parameters, @@ -75,6 +87,7 @@ const staticApplicationSource = ( graph: result.graph, kind: "static-application", rootArtifactSha256: result.root_artifact_sha256, + semanticGraph: result.schema_version === 2 ? result.semantic_graph : null, }; }; @@ -109,6 +122,7 @@ const staticRuntimeSource = ( graph: result.graph, kind: "static-runtime-reconciliation", rootArtifactSha256: applicationLayer.root_artifact_sha256, + semanticGraph: null, }; }; @@ -138,6 +152,7 @@ const managedApplicationSource = ( graph: result.graph, kind: "managed-application", rootArtifactSha256: result.root_artifact_sha256, + semanticGraph: null, }; }; diff --git a/src/application/JavaScriptApplicationService.ts b/src/application/JavaScriptApplicationService.ts index 0528b36ad..aebe2f64a 100644 --- a/src/application/JavaScriptApplicationService.ts +++ b/src/application/JavaScriptApplicationService.ts @@ -96,7 +96,7 @@ export const analyzeJavaScriptApplicationValidated = async ( ); const { electron_summary: summary, ...application } = reconstructed; const result = javascriptApplicationAnalysisResultSchema.parse({ - schema_version: 1, + schema_version: 2, ...application, summary, limitations: reconstructed.graph.limitations, diff --git a/src/application/JavaScriptApplicationWorkflowEvidence.ts b/src/application/JavaScriptApplicationWorkflowEvidence.ts index 410d3c3cc..1101b5b6a 100644 --- a/src/application/JavaScriptApplicationWorkflowEvidence.ts +++ b/src/application/JavaScriptApplicationWorkflowEvidence.ts @@ -6,6 +6,7 @@ import { import type { ApplicationVersionComparisonResult } from "../domain/javascriptApplicationVersionComparisonSchemas.js"; import type { JavaScriptExportShapeComparisonResult } from "../domain/javascriptExportShapeComparisonSchemas.js"; import type { ApplicationFeatureTraceResult } from "../domain/javascriptFeatureTraceSchemas.js"; +import type { JavaScriptSemanticTraceResult } from "../domain/javascriptSemanticTraceSchemas.js"; import { jsonValueSchema } from "../domain/jsonValue.js"; import { JAVASCRIPT_APPLICATION_WORKFLOW_PROVIDER } from "./InvestigationProviders.js"; @@ -27,6 +28,24 @@ export const createApplicationFeatureTraceEvidence = ( evidenceLinks: result.evidence_links, }); +/** Create derived Evidence for one bounded JavaScript semantic trace. */ +export const createJavaScriptSemanticTraceEvidence = ( + parameters: EvidenceObservation["parameters"], + result: JavaScriptSemanticTraceResult, +): Evidence => + createEvidence(undefined, JAVASCRIPT_APPLICATION_WORKFLOW_PROVIDER, { + predicateType: "rea.javascript-semantic-trace/v1", + operation: "trace_javascript_semantics", + parameters, + result: jsonValueSchema.parse(result), + rawResult: null, + confidence: "inferred", + authority: "analyst-inference", + environment: null, + limitations: result.limitations, + evidenceLinks: result.evidence_links, + }); + /** Create derived Evidence for one tiered application version comparison. */ export const createApplicationVersionComparisonEvidence = ( parameters: EvidenceObservation["parameters"], diff --git a/src/application/JavaScriptArtifactReconstruction.ts b/src/application/JavaScriptArtifactReconstruction.ts index d4e8bdccd..c305d4ef2 100644 --- a/src/application/JavaScriptArtifactReconstruction.ts +++ b/src/application/JavaScriptArtifactReconstruction.ts @@ -8,6 +8,7 @@ import { } from "../artifacts/ArtifactReader.js"; import { DirectoryArtifactReader } from "../artifacts/DirectoryArtifactReader.js"; import type { JavaScriptApplicationGraph } from "../domain/javascriptApplicationGraph.js"; +import type { JavaScriptSemanticGraph } from "../domain/javascriptSemanticGraph.js"; import type { ElectronBoundarySummary } from "../domain/javascriptApplicationAnalysis.js"; import { analyzeJavaScriptArtifactFiles } from "./JavaScriptArtifactAnalysis.js"; import { readJavaScriptArtifactFiles } from "./JavaScriptArtifactFiles.js"; @@ -19,6 +20,7 @@ import { } from "./JavaScriptArtifactReconstructionInput.js"; import { scanCanonicalArtifactInventory } from "./ArtifactInventory.js"; import { summarizeElectronBoundaries } from "./ElectronBoundaryAnalysis.js"; +import { buildJavaScriptSemanticGraph } from "./JavaScriptSemanticGraphBuilder.js"; /** Application-layer result retaining local diagnostics outside the canonical graph. */ export interface JavaScriptArtifactReconstructionResult { @@ -28,6 +30,7 @@ export interface JavaScriptArtifactReconstructionResult { readonly inventory_manifest_id: string; readonly inventory_graph_sha256: string; readonly graph: JavaScriptApplicationGraph; + readonly semantic_graph: JavaScriptSemanticGraph; readonly electron_summary: ElectronBoundarySummary; readonly statistics: { readonly relevant_files: number; @@ -85,6 +88,11 @@ export const reconstructJavaScriptArtifact = async ( analysis, input, ); + const semanticGraph = buildJavaScriptSemanticGraph({ + rootArtifactSha256: snapshot.manifest.root_sha256, + applicationGraph: graph, + analysis, + }); return { input_path: path, format, @@ -92,6 +100,7 @@ export const reconstructJavaScriptArtifact = async ( inventory_manifest_id: snapshot.manifest.manifest_id, inventory_graph_sha256: snapshot.manifest.graph_sha256, graph, + semantic_graph: semanticGraph, electron_summary: summarizeElectronBoundaries(analysis), statistics: { relevant_files: files.files.length, diff --git a/src/application/JavaScriptSemanticGraphBuilder.ts b/src/application/JavaScriptSemanticGraphBuilder.ts new file mode 100644 index 000000000..c04e4adc9 --- /dev/null +++ b/src/application/JavaScriptSemanticGraphBuilder.ts @@ -0,0 +1,462 @@ +import { + createJavaScriptSemanticGraph, + type JavaScriptSemanticGraph, + type JavaScriptSemanticGraphNode, +} from "../domain/javascriptSemanticGraph.js"; +import { + JAVASCRIPT_SEMANTIC_RELATION_FAMILIES, + JAVASCRIPT_SEMANTIC_RELATION_FAMILY, +} from "../domain/javascriptSemanticGraphSchemas.js"; +import type { + JavaScriptSemanticCallArgument, + JavaScriptSemanticIr, +} from "../domain/javascriptSemanticIr.js"; +import type { JavaScriptApplicationGraph } from "../domain/javascriptApplicationGraph.js"; +import type { JavaScriptArtifactAnalysis } from "./JavaScriptArtifactAnalysisTypes.js"; +import type { JavaScriptArtifactFile } from "./JavaScriptArtifactFiles.js"; +import { inferredSemanticEvidenceAt } from "./JavaScriptSemanticGraphEvidence.js"; +import { + projectSemanticClosureCaptures, + projectSemanticFrontiers, + projectSemanticReturnValues, + semanticRecoveryLimits, + type SemanticFlowProjectionContext, +} from "./JavaScriptSemanticGraphFlowProjection.js"; +import { + owningSemanticCallableNode, + semanticFamilyStatus, + semanticNodesWithinRange, + semanticRangesEqual as rangesEqual, +} from "./JavaScriptSemanticGraphProjection.js"; +import { + addSemanticFallbackRoot as addFallbackRoot, + addSemanticGraphNode as addNode, + addSemanticGraphRelation as addRelation, + constructSemanticGraphNode as semanticNode, + createSemanticGraphProjectionState as emptyState, + MAX_APPLICATION_NODE_IDS_PER_SEMANTIC_NODE, + MAX_SEMANTIC_GRAPH_NODES, + MAX_SEMANTIC_GRAPH_RELATIONS, + MAX_SEMANTIC_GRAPH_UNKNOWNS, + type SemanticGraphProjectionState as BuilderState, +} from "./JavaScriptSemanticGraphConstruction.js"; + +interface BuilderInput { + readonly rootArtifactSha256: string; + readonly applicationGraph: Pick< + JavaScriptApplicationGraph, + "graph_id" | "nodes" + >; + readonly analysis: JavaScriptArtifactAnalysis; +} + +interface FileContext extends SemanticFlowProjectionContext { + readonly file: JavaScriptArtifactFile; + readonly ir: JavaScriptSemanticIr; + readonly state: BuilderState; + readonly moduleNode: JavaScriptSemanticGraphNode; + readonly bindingNodes: ReadonlyMap; + readonly callableNodes: ReadonlyMap; + readonly callSiteNodes: ReadonlyMap; + readonly returnSiteNodes: ReadonlyMap; + readonly argumentNodes: Map; + readonly referenceNodes: JavaScriptSemanticGraphNode[]; + readonly callResolutions: ReadonlyMap; +} + +/** Project bounded per-file semantic IR into an artifact-bound companion graph. */ +export const buildJavaScriptSemanticGraph = ({ + rootArtifactSha256, + applicationGraph, + analysis, +}: BuilderInput): JavaScriptSemanticGraph => { + const state = emptyState(applicationGraph); + for (const analyzed of analysis.files) { + if (analyzed.semantic === null) continue; + projectFile(analyzed.file, analyzed.semantic.ir, state); + } + if (state.roots.size === 0) addFallbackRoot(rootArtifactSha256, state); + const semanticTruncated = analysis.files.some( + ({ semantic }) => semantic?.ir.coverage.status === "truncated", + ); + const truncated = state.limitsReached.size > 0 || semanticTruncated; + const unknowns = [...state.unknowns.values()]; + return createJavaScriptSemanticGraph({ + schema: "JavaScriptSemanticRelationGraph", + schema_version: 1, + root_artifact_sha256: rootArtifactSha256, + application_graph_id: applicationGraph.graph_id, + root_node_ids: [...state.roots], + nodes: [...state.nodes.values()], + relations: [...state.relations.values()], + fingerprints: [], + unknowns, + coverage: { + status: truncated ? "partial" : "unknown", + truncated, + omitted_nodes: truncated ? null : 0, + omitted_relations: truncated ? null : 0, + limits: [ + { + name: "max_nodes", + value: MAX_SEMANTIC_GRAPH_NODES, + unit: "items", + }, + { + name: "max_relations", + value: MAX_SEMANTIC_GRAPH_RELATIONS, + unit: "items", + }, + { + name: "max_unknowns", + value: MAX_SEMANTIC_GRAPH_UNKNOWNS, + unit: "items", + }, + { + name: "max_application_node_ids_per_semantic_node", + value: MAX_APPLICATION_NODE_IDS_PER_SEMANTIC_NODE, + unit: "items", + }, + ...semanticRecoveryLimits(analysis), + ], + families: JAVASCRIPT_SEMANTIC_RELATION_FAMILIES.map((family) => ({ + family, + status: semanticFamilyStatus(family, analysis, truncated), + retained_relations: [...state.relations.values()].filter( + (relation) => + JAVASCRIPT_SEMANTIC_RELATION_FAMILY[relation.relation] === family, + ).length, + omitted_relations: truncated ? null : 0, + unknown_ids: unknowns + .filter((unknown) => unknown.family === family) + .map(({ unknown_id: identifier }) => identifier), + })), + }, + limitations: [ + "The semantic graph contains static syntax observations and conservative relationship candidates; it does not claim runtime execution.", + "Local data flow does not claim control-flow-sensitive reaching definitions or arbitrary dynamic property resolution.", + "Function fingerprints and promise, event, process, request, boundary, timer, configuration, and resource extraction are not available in this graph version's current extractor coverage.", + ...(truncated + ? [ + "Semantic graph projection reached one or more explicit hard limits.", + ] + : []), + ], + }); +}; + +const projectFile = ( + file: JavaScriptArtifactFile, + ir: JavaScriptSemanticIr, + state: BuilderState, +): void => { + const moduleNode = addNode( + state, + semanticNode( + file, + { + kind: "module", + roleKey: "module", + location: null, + label: file.path, + functionNodeId: null, + }, + state, + ), + ); + if (moduleNode === null) return; + state.roots.add(moduleNode.node_id); + const callableNodes = new Map( + ir.callables.flatMap((callable) => { + const node = addNode( + state, + semanticNode( + file, + { + kind: "function", + roleKey: `callable:${callable.callableId}`, + location: callable.location, + label: callable.name, + functionNodeId: null, + }, + state, + ), + ); + return node === null ? [] : [[callable.callableId, node] as const]; + }), + ); + const bindingNodes = new Map( + ir.bindings.flatMap((binding) => { + const location = binding.definitions[0]?.location ?? null; + const kind = binding.kind === "parameter" ? "parameter" : "binding"; + const owner = owningSemanticCallableNode(location, ir, callableNodes); + const node = addNode( + state, + semanticNode( + file, + { + kind, + roleKey: `binding:${binding.bindingId}`, + location, + label: binding.name, + functionNodeId: owner?.node_id ?? null, + }, + state, + ), + ); + return node === null ? [] : [[binding.bindingId, node] as const]; + }), + ); + const returnSiteNodes = createReturnSiteNodes(file, ir, callableNodes, state); + const callSiteNodes = createCallSiteNodes(file, ir, callableNodes, state); + const context: FileContext = { + file, + ir, + state, + moduleNode, + bindingNodes, + callableNodes, + callSiteNodes, + returnSiteNodes, + argumentNodes: new Map(), + referenceNodes: [], + callResolutions: new Map( + ir.callSites.map((call) => [ + call.callSiteId, + call.resolution === "exact" ? "resolved" : "candidate", + ]), + ), + }; + projectDefinitionsAndReferences(context); + projectCalls(context); + projectSemanticClosureCaptures(context); + projectSemanticFrontiers(context); +}; + +const createReturnSiteNodes = ( + file: JavaScriptArtifactFile, + ir: JavaScriptSemanticIr, + callables: ReadonlyMap, + state: BuilderState, +): Map => { + const result = new Map(); + for (const callable of ir.callables) { + const owner = callables.get(callable.callableId); + if (owner === undefined) continue; + for (const site of callable.returnSites) { + const identifier = site.returnSiteId; + const node = addNode( + state, + semanticNode( + file, + { + kind: "return-site", + roleKey: identifier, + location: site.location, + label: "return", + functionNodeId: owner.node_id, + }, + state, + ), + ); + if (node !== null) result.set(identifier, node); + } + } + return result; +}; + +const createCallSiteNodes = ( + file: JavaScriptArtifactFile, + ir: JavaScriptSemanticIr, + callables: ReadonlyMap, + state: BuilderState, +): Map => { + const result = new Map(); + for (const call of ir.callSites) { + const owner = + call.callerCallableId === null + ? null + : (callables.get(call.callerCallableId)?.node_id ?? null); + const node = addNode( + state, + semanticNode( + file, + { + kind: "call-site", + roleKey: `call:${call.callSiteId}`, + location: call.location, + label: call.kind, + functionNodeId: owner, + }, + state, + ), + ); + if (node !== null) result.set(call.callSiteId, node); + } + return result; +}; + +const projectDefinitionsAndReferences = (context: FileContext): void => { + for (const binding of context.ir.bindings) { + const bindingNode = context.bindingNodes.get(binding.bindingId); + if (bindingNode === undefined) continue; + for (const [index, definition] of binding.definitions.entries()) { + const definitionNode = addNode( + context.state, + semanticNode( + context.file, + { + kind: "expression", + roleKey: `definition:${binding.bindingId}:${String(index)}`, + location: definition.location, + label: definition.kind, + functionNodeId: bindingNode.function_node_id, + }, + context.state, + ), + ); + addRelation(context.state, { + source: definitionNode, + target: bindingNode, + relation: "defines", + resolution: "resolved", + }); + } + } + for (const [index, reference] of context.ir.references.entries()) { + const expression = addNode( + context.state, + semanticNode( + context.file, + { + kind: "expression", + roleKey: `reference:${String(index)}:${reference.role}:${reference.name}`, + location: reference.location, + label: reference.name, + functionNodeId: + owningSemanticCallableNode( + reference.location, + context.ir, + context.callableNodes, + )?.node_id ?? null, + }, + context.state, + ), + ); + const binding = + reference.bindingId === null + ? undefined + : context.bindingNodes.get(reference.bindingId); + if (expression !== null) context.referenceNodes.push(expression); + if (binding === undefined || expression === null) continue; + if (reference.role === "read") + addRelation(context.state, { + source: binding, + target: expression, + relation: "reads", + resolution: "resolved", + evidence: inferredSemanticEvidenceAt(context.file, reference.location), + }); + else + addRelation(context.state, { + source: expression, + target: binding, + relation: "writes", + resolution: "resolved", + }); + } +}; + +const projectCalls = (context: FileContext): void => { + for (const call of context.ir.callSites) { + const callNode = context.callSiteNodes.get(call.callSiteId); + if (callNode === undefined) continue; + for (const callableId of call.calleeCallableIds) { + const callable = context.callableNodes.get(callableId); + addRelation(context.state, { + source: callNode, + target: callable, + relation: "calls", + resolution: call.resolution === "exact" ? "resolved" : "candidate", + }); + } + for (const argument of call.arguments) { + const argumentNode = createArgumentNode( + context, + callNode, + call.callSiteId, + argument, + ); + if (argumentNode !== null) + context.argumentNodes.set( + `${call.callSiteId}\u0000${String(argument.index)}`, + argumentNode, + ); + const references = semanticNodesWithinRange( + context.referenceNodes, + argument.location, + ); + for (const reference of references) + addRelation(context.state, { + source: reference, + target: argumentNode, + relation: "aliases", + resolution: + references.length === 1 && + rangesEqual(reference.identity.source_range, argument.location) + ? "resolved" + : "candidate", + }); + } + } + for (const flow of context.ir.argumentFlows) { + const argument = context.argumentNodes.get( + `${flow.callSiteId}\u0000${String(flow.argumentIndex)}`, + ); + const parameter = context.bindingNodes.get(flow.parameterBindingId); + addRelation(context.state, { + source: argument, + target: parameter, + relation: "argument-to-parameter", + resolution: context.callResolutions.get(flow.callSiteId) ?? "candidate", + }); + } + for (const flow of context.ir.callReturnFlows) + addRelation(context.state, { + source: context.returnSiteNodes.get(flow.returnSiteId), + target: context.callSiteNodes.get(flow.callSiteId), + relation: "returns-to-call", + resolution: context.callResolutions.get(flow.callSiteId) ?? "candidate", + }); + for (const flow of context.ir.callResultFlows) + addRelation(context.state, { + source: context.callSiteNodes.get(flow.callSiteId), + target: context.bindingNodes.get(flow.bindingId), + relation: "aliases", + resolution: context.callResolutions.get(flow.callSiteId) ?? "candidate", + }); + projectSemanticReturnValues(context); +}; + +const createArgumentNode = ( + context: FileContext, + callNode: JavaScriptSemanticGraphNode, + callSiteId: string, + argument: JavaScriptSemanticCallArgument, +): JavaScriptSemanticGraphNode | null => + addNode( + context.state, + semanticNode( + context.file, + { + kind: "expression", + roleKey: `argument:${callSiteId}:${String(argument.index)}`, + location: argument.location, + label: argument.spread + ? "spread argument" + : `argument ${String(argument.index)}`, + functionNodeId: callNode.function_node_id, + }, + context.state, + ), + ); diff --git a/src/application/JavaScriptSemanticGraphConstruction.ts b/src/application/JavaScriptSemanticGraphConstruction.ts new file mode 100644 index 000000000..b2614afd9 --- /dev/null +++ b/src/application/JavaScriptSemanticGraphConstruction.ts @@ -0,0 +1,256 @@ +import type { JavaScriptApplicationGraph } from "../domain/javascriptApplicationGraph.js"; +import { + createJavaScriptSemanticGraphNode, + createJavaScriptSemanticGraphRelation, + type JavaScriptSemanticGraphNode, +} from "../domain/javascriptSemanticGraph.js"; +import type { ApplicationGraphEvidence } from "../domain/javascriptApplicationEvidenceSchemas.js"; +import type { + JavaScriptSemanticGraphRelation, + JavaScriptSemanticGraphUnknown, +} from "../domain/javascriptSemanticGraphSchemas.js"; +import type { JavaScriptSourceRange } from "../domain/javascriptStaticAnalysisTypes.js"; +import type { JavaScriptArtifactFile } from "./JavaScriptArtifactFiles.js"; +import { + inferredSemanticEvidence, + observedSemanticEvidence, + unavailableSemanticRootEvidence, +} from "./JavaScriptSemanticGraphEvidence.js"; + +/** Hard semantic graph node bound enforced during artifact projection. */ +export const MAX_SEMANTIC_GRAPH_NODES = 100_000; +/** Hard semantic graph relation bound enforced during artifact projection. */ +export const MAX_SEMANTIC_GRAPH_RELATIONS = 200_000; +/** Hard unresolved-frontier bound enforced across all analyzed files. */ +export const MAX_SEMANTIC_GRAPH_UNKNOWNS = 100_000; +/** Maximum structural nodes linked to one exact semantic location. */ +export const MAX_APPLICATION_NODE_IDS_PER_SEMANTIC_NODE = 64; + +type SemanticGraphProjectionLimit = + | "max_application_node_ids_per_semantic_node" + | "max_nodes" + | "max_relations" + | "max_unknowns"; + +/** Mutable local projection state hidden from graph callers. */ +export interface SemanticGraphProjectionState { + readonly nodes: Map; + readonly relations: Map; + readonly unknowns: Map; + readonly roots: Set; + readonly limitsReached: Set; + readonly applicationNodeIdsByLocation: ReadonlyMap; +} + +/** Input for one exact artifact-version semantic node. */ +export interface SemanticNodeConstructionInput { + readonly kind: JavaScriptSemanticGraphNode["kind"]; + readonly roleKey: string; + readonly location: JavaScriptSourceRange | null; + readonly label: string | null; + readonly functionNodeId: string | null; +} + +/** Input for one directed static semantic relationship. */ +export interface SemanticRelationConstructionInput { + readonly source: JavaScriptSemanticGraphNode | undefined | null; + readonly target: JavaScriptSemanticGraphNode | undefined | null; + readonly relation: JavaScriptSemanticGraphRelation["relation"]; + readonly resolution?: JavaScriptSemanticGraphRelation["resolution"]; + readonly evidence?: ApplicationGraphEvidence; +} + +/** Create empty projection state with structural application-node mappings. */ +export const createSemanticGraphProjectionState = ( + applicationGraph: Pick, +): SemanticGraphProjectionState => { + const index = indexApplicationNodes(applicationGraph.nodes); + return { + nodes: new Map(), + relations: new Map(), + unknowns: new Map(), + roots: new Set(), + limitsReached: new Set( + index.truncated ? ["max_application_node_ids_per_semantic_node"] : [], + ), + applicationNodeIdsByLocation: index.identifiers, + }; +}; + +/** Construct one canonical semantic node backed by an exact artifact file. */ +export const constructSemanticGraphNode = ( + file: JavaScriptArtifactFile, + input: SemanticNodeConstructionInput, + state: SemanticGraphProjectionState, +): JavaScriptSemanticGraphNode => + createJavaScriptSemanticGraphNode({ + kind: input.kind, + identity: { + artifact_sha256: file.sha256, + module_path: file.path, + source_range: input.location, + role_key: input.roleKey, + }, + function_node_id: input.functionNodeId, + application_node_ids: matchingApplicationNodeIds(file, input, state), + label: input.label, + properties: {}, + evidence: observedSemanticEvidence(file, input.location), + }); + +/** Retain one canonical node unless the hard graph bound was reached. */ +export const addSemanticGraphNode = ( + state: SemanticGraphProjectionState, + node: JavaScriptSemanticGraphNode, +): JavaScriptSemanticGraphNode | null => { + const existing = state.nodes.get(node.node_id); + if (existing !== undefined) return existing; + if (state.nodes.size >= MAX_SEMANTIC_GRAPH_NODES) { + state.limitsReached.add("max_nodes"); + return null; + } + state.nodes.set(node.node_id, node); + return node; +}; + +/** Retain one non-self semantic relationship unless its hard bound was reached. */ +export const addSemanticGraphRelation = ( + state: SemanticGraphProjectionState, + input: SemanticRelationConstructionInput, +): void => { + if ( + input.source === undefined || + input.source === null || + input.target === undefined || + input.target === null || + input.source.node_id === input.target.node_id + ) + return; + const value = createJavaScriptSemanticGraphRelation({ + source_node_id: input.source.node_id, + target_node_id: input.target.node_id, + relation: input.relation, + resolution: input.resolution ?? "candidate", + properties: {}, + evidence: input.evidence ?? inferredSemanticEvidence(input.source), + }); + if (state.relations.has(value.relation_id)) return; + if (state.relations.size >= MAX_SEMANTIC_GRAPH_RELATIONS) { + state.limitsReached.add("max_relations"); + return; + } + state.relations.set(value.relation_id, value); +}; + +/** Retain one unresolved frontier unless the global graph bound was reached. */ +export const addSemanticGraphUnknown = ( + state: SemanticGraphProjectionState, + unknown: JavaScriptSemanticGraphUnknown, +): void => { + if (state.unknowns.has(unknown.unknown_id)) return; + if (state.unknowns.size >= MAX_SEMANTIC_GRAPH_UNKNOWNS) { + state.limitsReached.add("max_unknowns"); + return; + } + state.unknowns.set(unknown.unknown_id, unknown); +}; + +/** Add a truthful root when no source file could produce semantic IR. */ +export const addSemanticFallbackRoot = ( + rootArtifactSha256: string, + state: SemanticGraphProjectionState, +): void => { + const node = addSemanticGraphNode( + state, + createJavaScriptSemanticGraphNode({ + kind: "module", + identity: { + artifact_sha256: rootArtifactSha256, + module_path: "unknown-semantic-root", + source_range: null, + role_key: "artifact-root", + }, + function_node_id: null, + application_node_ids: [], + label: "unavailable semantic root", + properties: {}, + evidence: unavailableSemanticRootEvidence(rootArtifactSha256), + }), + ); + if (node !== null) state.roots.add(node.node_id); +}; + +const indexApplicationNodes = ( + nodes: JavaScriptApplicationGraph["nodes"], +): { + readonly identifiers: ReadonlyMap; + readonly truncated: boolean; +} => { + const identifiers = new Map>(); + for (const node of nodes) { + for (const observation of node.observations) { + const { artifact, location } = observation.evidence; + if (!artifact.available || !location.available) continue; + if (location.value.kind === "artifact-path") { + addApplicationNodeIdentifier( + identifiers, + applicationLocationKey(artifact.sha256, location.value.path, null), + node.node_id, + ); + } + if (location.value.kind === "source-range") { + addApplicationNodeIdentifier( + identifiers, + applicationLocationKey(artifact.sha256, location.value.source, { + start: location.value.start, + end: location.value.end, + }), + node.node_id, + ); + } + } + } + let truncated = false; + const indexed = new Map( + [...identifiers].map(([key, values]) => { + const sorted = [...values].sort(); + if (sorted.length > MAX_APPLICATION_NODE_IDS_PER_SEMANTIC_NODE) + truncated = true; + return [key, sorted.slice(0, MAX_APPLICATION_NODE_IDS_PER_SEMANTIC_NODE)]; + }), + ); + return { identifiers: indexed, truncated }; +}; + +const matchingApplicationNodeIds = ( + file: JavaScriptArtifactFile, + input: SemanticNodeConstructionInput, + state: SemanticGraphProjectionState, +): string[] => [ + ...(state.applicationNodeIdsByLocation.get( + applicationLocationKey(file.sha256, file.path, input.location), + ) ?? []), +]; + +const addApplicationNodeIdentifier = ( + identifiers: Map>, + key: string, + nodeId: string, +): void => { + const values = identifiers.get(key) ?? new Set(); + values.add(nodeId); + identifiers.set(key, values); +}; + +const applicationLocationKey = ( + artifactSha256: string, + path: string, + range: JavaScriptSourceRange | null, +): string => + [ + artifactSha256, + path, + range === null + ? "artifact-path" + : `${range.start.line}:${range.start.column}-${range.end.line}:${range.end.column}`, + ].join("\u0000"); diff --git a/src/application/JavaScriptSemanticGraphEvidence.ts b/src/application/JavaScriptSemanticGraphEvidence.ts new file mode 100644 index 000000000..18b778e61 --- /dev/null +++ b/src/application/JavaScriptSemanticGraphEvidence.ts @@ -0,0 +1,111 @@ +import type { ApplicationGraphEvidence } from "../domain/javascriptApplicationEvidenceSchemas.js"; +import type { JavaScriptSemanticGraphNode } from "../domain/javascriptSemanticGraph.js"; +import type { JavaScriptSourceRange } from "../domain/javascriptStaticAnalysisTypes.js"; +import type { JavaScriptArtifactFile } from "./JavaScriptArtifactFiles.js"; + +const INFERENCE_LIMITATION = + "This relationship is a bounded static candidate; it does not prove runtime execution or causal flow."; + +/** Exact artifact-backed syntax evidence for one semantic graph node. */ +export const observedSemanticEvidence = ( + file: JavaScriptArtifactFile, + location: JavaScriptSourceRange | null, +): ApplicationGraphEvidence => ({ + authority: "ast-static-analysis", + state: "observed", + confidence: "exact", + artifact: { + available: true, + artifact_id: `art_${file.sha256}`, + sha256: file.sha256, + }, + location: + location === null + ? { available: true, value: { kind: "artifact-path", path: file.path } } + : { + available: true, + value: { kind: "source-range", source: file.path, ...location }, + }, + extractor: { + name: "rea-javascript-semantic-analysis", + version: "1", + operation: "analyze_javascript_application", + executable_sha256: null, + }, + coverage: { + status: "complete", + truncated: false, + omitted_count: 0, + limits: [], + }, + limitations: [], + evidence_ids: [], +}); + +/** Conservative static relationship evidence derived from one syntax node. */ +export const inferredSemanticEvidence = ( + source: JavaScriptSemanticGraphNode, +): ApplicationGraphEvidence => ({ + ...source.evidence, + authority: "static-relationship-inference", + state: "inferred", + confidence: "high", + limitations: [INFERENCE_LIMITATION], +}); + +/** Static relationship evidence anchored at the syntax that proves the link. */ +export const inferredSemanticEvidenceAt = ( + file: JavaScriptArtifactFile, + location: JavaScriptSourceRange, +): ApplicationGraphEvidence => ({ + ...observedSemanticEvidence(file, location), + authority: "static-relationship-inference", + state: "inferred", + confidence: "high", + limitations: [INFERENCE_LIMITATION], +}); + +/** Explicit absence of semantic syntax for an artifact-level fallback root. */ +export const unavailableSemanticRootEvidence = ( + rootArtifactSha256: string, +): ApplicationGraphEvidence => ({ + authority: "unknown", + state: "unavailable", + confidence: "unknown", + artifact: { + available: true, + artifact_id: `art_${rootArtifactSha256}`, + sha256: rootArtifactSha256, + }, + location: { + available: false, + reason: "not-observed", + detail: "No admitted JavaScript source produced semantic IR.", + }, + extractor: { + name: "rea-javascript-semantic-analysis", + version: "1", + operation: "analyze_javascript_application", + executable_sha256: null, + }, + coverage: { + status: "unavailable", + truncated: false, + omitted_count: null, + limits: [], + }, + limitations: ["This synthetic graph root does not claim a source location."], + evidence_ids: [], +}); + +/** Explicit unknown evidence at one unresolved dynamic syntax location. */ +export const unknownSemanticEvidence = ( + file: JavaScriptArtifactFile, + location: JavaScriptSourceRange, +): ApplicationGraphEvidence => ({ + ...observedSemanticEvidence(file, location), + authority: "unknown", + state: "unknown", + confidence: "unknown", + limitations: ["Dynamic syntax prevents an exact static relationship."], +}); diff --git a/src/application/JavaScriptSemanticGraphFlowProjection.ts b/src/application/JavaScriptSemanticGraphFlowProjection.ts new file mode 100644 index 000000000..292deecd9 --- /dev/null +++ b/src/application/JavaScriptSemanticGraphFlowProjection.ts @@ -0,0 +1,127 @@ +import { createJavaScriptSemanticGraphUnknown } from "../domain/javascriptSemanticGraph.js"; +import type { JavaScriptSemanticGraphNode } from "../domain/javascriptSemanticGraph.js"; +import type { JavaScriptSemanticIr } from "../domain/javascriptSemanticIr.js"; +import type { JavaScriptArtifactAnalysis } from "./JavaScriptArtifactAnalysisTypes.js"; +import type { JavaScriptArtifactFile } from "./JavaScriptArtifactFiles.js"; +import { + addSemanticGraphRelation, + addSemanticGraphUnknown, + type SemanticGraphProjectionState, +} from "./JavaScriptSemanticGraphConstruction.js"; +import { + inferredSemanticEvidenceAt, + unknownSemanticEvidence, +} from "./JavaScriptSemanticGraphEvidence.js"; +import { semanticNodesWithinRange } from "./JavaScriptSemanticGraphProjection.js"; + +/** File-local graph state needed by return, capture, and frontier projection. */ +export interface SemanticFlowProjectionContext { + readonly file: JavaScriptArtifactFile; + readonly ir: JavaScriptSemanticIr; + readonly state: SemanticGraphProjectionState; + readonly moduleNode: JavaScriptSemanticGraphNode; + readonly bindingNodes: ReadonlyMap; + readonly callableNodes: ReadonlyMap; + readonly returnSiteNodes: ReadonlyMap; + readonly referenceNodes: readonly JavaScriptSemanticGraphNode[]; +} + +/** Link direct return expressions to their retained semantic references. */ +export const projectSemanticReturnValues = ( + context: SemanticFlowProjectionContext, +): void => { + for (const callable of context.ir.callables) + for (const site of callable.returnSites) { + const returnNode = context.returnSiteNodes.get(site.returnSiteId); + const references = semanticNodesWithinRange( + context.referenceNodes, + site.location, + ); + for (const reference of references) + addSemanticGraphRelation(context.state, { + source: reference, + target: returnNode, + relation: "aliases", + resolution: + site.identityReferenceLocation !== null && + rangesEqual( + reference.identity.source_range, + site.identityReferenceLocation, + ) + ? "resolved" + : "candidate", + }); + } +}; + +const rangesEqual = ( + left: JavaScriptSemanticGraphNode["identity"]["source_range"], + right: JavaScriptSemanticGraphNode["identity"]["source_range"], +): boolean => + left !== null && + right !== null && + left.start.line === right.start.line && + left.start.column === right.start.column && + left.end.line === right.end.line && + left.end.column === right.end.column; + +/** Link each captured binding to the callable at the exact reference site. */ +export const projectSemanticClosureCaptures = ( + context: SemanticFlowProjectionContext, +): void => { + for (const capture of context.ir.closureCaptures) + addSemanticGraphRelation(context.state, { + source: context.bindingNodes.get(capture.bindingId), + target: context.callableNodes.get(capture.callableId), + relation: "captures", + resolution: "resolved", + evidence: inferredSemanticEvidenceAt( + context.file, + capture.referenceLocation, + ), + }); +}; + +/** Retain bounded unresolved dynamic-call and dynamic-property frontiers. */ +export const projectSemanticFrontiers = ( + context: SemanticFlowProjectionContext, +): void => { + for (const frontier of context.ir.frontiers) { + const unknown = createJavaScriptSemanticGraphUnknown({ + node_id: + frontier.callableId === null + ? context.moduleNode.node_id + : (context.callableNodes.get(frontier.callableId)?.node_id ?? + context.moduleNode.node_id), + family: frontier.kind === "dynamic-call" ? "call-flow" : "object-flow", + relation_kinds: + frontier.kind === "dynamic-call" + ? ["calls"] + : ["reads-property", "writes-property"], + reason: frontier.kind, + detail: frontier.reason, + candidate_node_ids: [], + evidence: unknownSemanticEvidence(context.file, frontier.location), + }); + addSemanticGraphUnknown(context.state, unknown); + } +}; + +/** Publish exact semantic analyzer limits reached by any admitted source file. */ +export const semanticRecoveryLimits = ( + analysis: JavaScriptArtifactAnalysis, +): { + readonly name: string; + readonly value: number; + readonly unit: "items"; +}[] => { + const limits = new Map(); + for (const { semantic } of analysis.files) { + if (semantic === null) continue; + for (const name of semantic.ir.coverage.limitsReached) + limits.set(`semantic.${name}`, semantic.limits[name]); + } + return [...limits] + .sort(([left], [right]) => left.localeCompare(right)) + .map(([name, value]) => ({ name, value, unit: "items" })); +}; diff --git a/src/application/JavaScriptSemanticGraphProjection.ts b/src/application/JavaScriptSemanticGraphProjection.ts new file mode 100644 index 000000000..5de02a28d --- /dev/null +++ b/src/application/JavaScriptSemanticGraphProjection.ts @@ -0,0 +1,74 @@ +import { compareCodePoints } from "../domain/javascriptApplicationGraph.js"; +import type { JavaScriptSemanticGraphNode } from "../domain/javascriptSemanticGraph.js"; +import { JAVASCRIPT_SEMANTIC_RELATION_FAMILIES } from "../domain/javascriptSemanticGraphSchemas.js"; +import type { JavaScriptSemanticIr } from "../domain/javascriptSemanticIr.js"; +import type { JavaScriptSourceRange } from "../domain/javascriptStaticAnalysisTypes.js"; +import type { JavaScriptArtifactAnalysis } from "./JavaScriptArtifactAnalysisTypes.js"; + +/** Find the innermost callable that contains one exact source range. */ +export const owningSemanticCallableNode = ( + location: JavaScriptSourceRange | null, + ir: JavaScriptSemanticIr, + nodes: ReadonlyMap, +): JavaScriptSemanticGraphNode | undefined => { + if (location === null) return undefined; + return ir.callables + .filter((callable) => contains(callable.location, location)) + .sort((left, right) => + compareCodePoints( + positionKey(right.location), + positionKey(left.location), + ), + ) + .map(({ callableId }) => nodes.get(callableId)) + .find((node) => node !== undefined); +}; + +/** Find retained semantic nodes whose exact ranges are within one expression. */ +export const semanticNodesWithinRange = ( + nodes: readonly JavaScriptSemanticGraphNode[], + range: JavaScriptSourceRange, +): JavaScriptSemanticGraphNode[] => + nodes.filter( + (node) => + node.identity.source_range !== null && + contains(range, node.identity.source_range), + ); + +/** Report extractor support without treating missing families as absence. */ +export const semanticFamilyStatus = ( + family: (typeof JAVASCRIPT_SEMANTIC_RELATION_FAMILIES)[number], + analysis: JavaScriptArtifactAnalysis, + truncated: boolean, +): "complete" | "partial" | "unknown" | "unsupported" => { + if (!["call-flow", "closure", "data-flow", "object-flow"].includes(family)) + return "unsupported"; + if (truncated) return "unknown"; + return analysis.truncated_scopes === 0 ? "partial" : "unknown"; +}; + +/** Compare two exact semantic source ranges. */ +export const semanticRangesEqual = ( + left: JavaScriptSourceRange | null, + right: JavaScriptSourceRange, +): boolean => + left !== null && + left.start.line === right.start.line && + left.start.column === right.start.column && + left.end.line === right.end.line && + left.end.column === right.end.column; + +const contains = ( + outer: JavaScriptSourceRange, + inner: JavaScriptSourceRange, +): boolean => + pointCompare(outer.start, inner.start) <= 0 && + pointCompare(outer.end, inner.end) >= 0; + +const pointCompare = ( + left: JavaScriptSourceRange["start"], + right: JavaScriptSourceRange["start"], +): number => left.line - right.line || left.column - right.column; + +const positionKey = (range: JavaScriptSourceRange): string => + `${String(range.start.line).padStart(12, "0")}:${String(range.start.column).padStart(12, "0")}`; diff --git a/src/application/JavaScriptSemanticTraceService.ts b/src/application/JavaScriptSemanticTraceService.ts new file mode 100644 index 000000000..7c5c3d340 --- /dev/null +++ b/src/application/JavaScriptSemanticTraceService.ts @@ -0,0 +1,72 @@ +import { z } from "zod"; + +import { + AnalysisInputError, + AnalysisProtocolError, + type AnalysisError, +} from "../domain/errors.js"; +import type { Evidence } from "../domain/evidence.js"; +import { jsonValueSchema } from "../domain/jsonValue.js"; +import { queryJavaScriptSemanticGraph } from "../domain/javascriptSemanticQuery.js"; +import { JavaScriptSemanticQueryCursorError } from "../domain/javascriptSemanticQueryIdentity.js"; +import { + javaScriptSemanticTraceResultSchema, + traceJavaScriptSemanticsInputSchema, +} from "../domain/javascriptSemanticTraceSchemas.js"; +import { err, ok, type Result } from "../domain/result.js"; +import { parseApplicationGraphEvidence } from "./JavaScriptApplicationEvidenceGraph.js"; +import { createJavaScriptSemanticTraceEvidence } from "./JavaScriptApplicationWorkflowEvidence.js"; + +const OPERATION = "trace_javascript_semantics" as const; + +/** Trace semantic relations from input already parsed by a trusted adapter. */ +export const traceJavaScriptSemanticsEvidenceValidated = ( + input: z.output, +): Result => { + try { + const source = parseApplicationGraphEvidence(input.application); + if (source.semanticGraph === null) + return err( + new AnalysisInputError(OPERATION, undefined, [ + { + path: ["application"], + reason: "invalid_value", + expected: + "analyze_javascript_application v2 Evidence; reanalyze the artifact with the current REA version", + }, + ]), + ); + const query = queryJavaScriptSemanticGraph( + source.semanticGraph, + input.query, + ); + const result = javaScriptSemanticTraceResultSchema.parse({ + ...query, + source_evidence_id: source.evidence.evidence_id, + evidence_links: [source.evidence.evidence_id], + }); + return ok( + createJavaScriptSemanticTraceEvidence( + { + application_evidence_id: source.evidence.evidence_id, + query: jsonValueSchema.parse(input.query), + }, + result, + ), + ); + } catch (cause: unknown) { + if (cause instanceof JavaScriptSemanticQueryCursorError) + return err( + new AnalysisInputError(OPERATION, undefined, [ + { + path: ["query", "cursor"], + reason: "invalid_value", + expected: "a cursor returned by the same graph, query, and limits", + }, + ]), + ); + return err( + new AnalysisProtocolError("JavaScript semantic trace failed", { cause }), + ); + } +}; diff --git a/src/cliApplicationCommands.ts b/src/cliApplicationCommands.ts index d64f61b64..e94e1d3ed 100644 --- a/src/cliApplicationCommands.ts +++ b/src/cliApplicationCommands.ts @@ -5,10 +5,12 @@ import { compareJavaScriptExportShapesEvidenceValidated, traceApplicationFeatureEvidenceValidated, } from "./application/JavaScriptApplicationWorkflowService.js"; +import { traceJavaScriptSemanticsEvidenceValidated } from "./application/JavaScriptSemanticTraceService.js"; import { resolveCompareApplicationVersionsRequest, resolveCompareJavaScriptExportShapesRequest, resolveTraceApplicationFeatureRequest, + resolveTraceJavaScriptSemanticsRequest, } from "./application/ApplicationWorkflowEvidenceResolver.js"; import type { EvidenceLookup } from "./application/EvidenceReferenceResolver.js"; import { readEvidenceBundle } from "./application/EvidenceBundleFiles.js"; @@ -60,6 +62,15 @@ export const registerApplicationCommands = ( resolveInput: resolveTraceApplicationFeatureRequest, workflow: traceApplicationFeatureEvidenceValidated, }); + registerJsonCommand({ + cli, + logger, + name: CLI_COMMANDS.traceJavaScriptSemantics, + description: + "Trace bounded static semantic relations through authenticated application Evidence", + resolveInput: resolveTraceJavaScriptSemanticsRequest, + workflow: traceJavaScriptSemanticsEvidenceValidated, + }); registerJsonCommand({ cli, logger, diff --git a/src/cliCommandNames.ts b/src/cliCommandNames.ts index d06809b15..fca7d94bc 100644 --- a/src/cliCommandNames.ts +++ b/src/cliCommandNames.ts @@ -49,6 +49,7 @@ export const CLI_COMMANDS = Object.freeze({ analyzeJavaScriptApplication: "analyze-javascript-application", reconcileJavaScriptRuntime: "reconcile-javascript-runtime", traceApplicationFeature: "trace-application-feature", + traceJavaScriptSemantics: "trace-javascript-semantics", compareApplicationVersions: "compare-application-versions", compareJavaScriptExportShapes: "compare-javascript-export-shapes", runReplayMachine: "run-replay-machine", diff --git a/src/contracts/applicationToolContracts.ts b/src/contracts/applicationToolContracts.ts index 12cce5878..2db523427 100644 --- a/src/contracts/applicationToolContracts.ts +++ b/src/contracts/applicationToolContracts.ts @@ -1,10 +1,12 @@ import { applicationVersionComparisonResultSchema } from "../domain/javascriptApplicationVersionComparisonSchemas.js"; import { applicationFeatureTraceResultSchema } from "../domain/javascriptFeatureTraceSchemas.js"; +import { javaScriptSemanticTraceResultSchema } from "../domain/javascriptSemanticTraceSchemas.js"; import { javaScriptExportShapeComparisonResultSchema } from "../domain/javascriptExportShapeComparisonSchemas.js"; import { compareApplicationVersionsRequestSchema, compareJavaScriptExportShapesRequestSchema, traceApplicationFeatureRequestSchema, + traceJavaScriptSemanticsRequestSchema, } from "./applicationWorkflowInputContracts.js"; import { controlledReplayInputSchema, @@ -33,6 +35,9 @@ import { } from "./javascriptApplicationWorkflowExamples.js"; const traceOutputSchema = evidenceResultOf(applicationFeatureTraceResultSchema); +const semanticTraceOutputSchema = evidenceResultOf( + javaScriptSemanticTraceResultSchema, +); const comparisonOutputSchema = evidenceResultOf( applicationVersionComparisonResultSchema, ); @@ -120,6 +125,28 @@ export const APPLICATION_TOOL_CONTRACTS = [ }, ], }, + { + name: "trace_javascript_semantics", + ...toolContractMetadata("trace_javascript_semantics"), + description: + "Trace bounded static JavaScript data-flow, direct call/return, and closure relations from authenticated analyze_javascript_application v2 Evidence. Queries declare direction and exact node, relation, depth, function, module, and page limits. Dynamic or unsupported semantics remain explicit unknowns; static reachability never claims runtime execution.", + kind: "application", + inputSchema: traceJavaScriptSemanticsRequestSchema, + outputSchema: semanticTraceOutputSchema, + examples: [ + { + title: "Trace backward provenance from one semantic node", + input: { + application_evidence_id: `ev_${HASH}`, + query: { + seed: { kind: "semantic-node", node_id: `jsrg_node_${HASH}` }, + direction: "backward-provenance", + source_map_authority: { authority: "none" }, + }, + }, + }, + ], + }, { name: "compare_application_versions", ...toolContractMetadata("compare_application_versions"), diff --git a/src/contracts/applicationWorkflowInputContracts.ts b/src/contracts/applicationWorkflowInputContracts.ts index 8304365e9..00457de46 100644 --- a/src/contracts/applicationWorkflowInputContracts.ts +++ b/src/contracts/applicationWorkflowInputContracts.ts @@ -4,6 +4,7 @@ import { evidenceSchema } from "../domain/evidence.js"; import { compareApplicationVersionsInputSchema } from "../domain/javascriptApplicationVersionComparisonSchemas.js"; import { compareJavaScriptExportShapesInputSchema } from "../domain/javascriptExportShapeComparisonSchemas.js"; import { traceApplicationFeatureInputSchema } from "../domain/javascriptFeatureTraceSchemas.js"; +import { javaScriptSemanticQueryInputSchema } from "../domain/javascriptSemanticQuerySchemas.js"; const evidenceIdSchema = z .string() @@ -47,6 +48,22 @@ export const traceApplicationFeatureRequestSchema = z ]); }); +/** MCP/CLI semantic trace request accepting full Evidence or a ledger reference. */ +export const traceJavaScriptSemanticsRequestSchema = z + .strictObject({ + application: evidenceSchema.optional(), + application_evidence_id: evidenceIdSchema.optional(), + query: javaScriptSemanticQueryInputSchema, + }) + .superRefine((input, context) => { + requireExactlyOne(context, [ + "application", + input.application, + "application_evidence_id", + input.application_evidence_id, + ]); + }); + /** MCP/CLI comparison request accepting full Evidence or ledger references. */ export const compareApplicationVersionsRequestSchema = z .strictObject({ @@ -122,6 +139,9 @@ export const compareJavaScriptExportShapesRequestSchema = z export type TraceApplicationFeatureRequest = z.output< typeof traceApplicationFeatureRequestSchema >; +export type TraceJavaScriptSemanticsRequest = z.output< + typeof traceJavaScriptSemanticsRequestSchema +>; export type CompareApplicationVersionsRequest = z.output< typeof compareApplicationVersionsRequestSchema >; diff --git a/src/contracts/electronToolContracts.ts b/src/contracts/electronToolContracts.ts index 1efb36add..1791b461b 100644 --- a/src/contracts/electronToolContracts.ts +++ b/src/contracts/electronToolContracts.ts @@ -11,6 +11,7 @@ import { } from "../domain/electronObservation.js"; import { analyzeJavaScriptApplicationInputSchema, + javaScriptApplicationAnalysisResultV1Schema, javascriptApplicationAnalysisResultSchema, } from "../domain/javascriptApplicationAnalysis.js"; import { JAVASCRIPT_APPLICATION_NODE_KINDS } from "../domain/javascriptApplicationGraphSchemas.js"; @@ -35,44 +36,70 @@ export const analyzeJavaScriptApplicationToolInputSchema = }); /** Bounded default projection for agent-facing JavaScript application analysis. */ -export const javascriptApplicationAnalysisSummarySchema = - javascriptApplicationAnalysisResultSchema.omit({ graph: true }).extend({ - unknowns: z.array(z.string().min(1).max(4_096)).max(1_000), - graph: z.strictObject({ - graph_id: z.string().regex(/^jag_[a-f0-9]{64}$/u), - node_count: z.number().int().min(0), - edge_count: z.number().int().min(0), - roots: z.strictObject({ - items: z.array(z.string().regex(/^jag_node_[a-f0-9]{64}$/u)), - total: z.number().int().min(0), - truncated: z.boolean(), - }), - node_kinds: z.array( - z.strictObject({ - kind: z.enum(JAVASCRIPT_APPLICATION_NODE_KINDS), - count: z.number().int().min(1), - }), - ), - top_findings: z.array( - z.strictObject({ - node_id: z.string().regex(/^jag_node_[a-f0-9]{64}$/u), - kind: z.enum(JAVASCRIPT_APPLICATION_NODE_KINDS), - label: z.string().nullable(), - authority: z.string(), - state: z.string(), - confidence: z.string(), - location: z.json(), - }), - ), - coverage: applicationCoverageSchema, - limitations: z.array(z.string()), - pages: z.strictObject({ - nodes: z.string().startsWith("rea://evidence/"), - edges: z.string().startsWith("rea://evidence/"), - page_limit: z.number().int().min(1).max(500), - }), +const applicationGraphSummarySchema = z.strictObject({ + graph_id: z.string().regex(/^jag_[a-f0-9]{64}$/u), + node_count: z.number().int().min(0), + edge_count: z.number().int().min(0), + roots: z.strictObject({ + items: z.array(z.string().regex(/^jag_node_[a-f0-9]{64}$/u)), + total: z.number().int().min(0), + truncated: z.boolean(), + }), + node_kinds: z.array( + z.strictObject({ + kind: z.enum(JAVASCRIPT_APPLICATION_NODE_KINDS), + count: z.number().int().min(1), }), - }); + ), + top_findings: z.array( + z.strictObject({ + node_id: z.string().regex(/^jag_node_[a-f0-9]{64}$/u), + kind: z.enum(JAVASCRIPT_APPLICATION_NODE_KINDS), + label: z.string().nullable(), + authority: z.string(), + state: z.string(), + confidence: z.string(), + location: z.json(), + }), + ), + coverage: applicationCoverageSchema, + limitations: z.array(z.string()), + pages: z.strictObject({ + nodes: z.string().startsWith("rea://evidence/"), + edges: z.string().startsWith("rea://evidence/"), + page_limit: z.number().int().min(1).max(500), + }), +}); + +const semanticGraphSummarySchema = z.strictObject({ + graph_id: z.string().regex(/^jsrg_[a-f0-9]{64}$/u), + nodes: z.number().int().min(0), + relations: z.number().int().min(0), + unknown_frontiers: z.number().int().min(0), + fingerprints: z.number().int().min(0), + coverage: z.enum(["complete", "partial", "unknown", "unavailable"]), + query_tool: z.literal("trace_javascript_semantics"), +}); + +const summaryFields = { + unknowns: z.array(z.string().min(1).max(4_096)).max(1_000), + graph: applicationGraphSummarySchema, +}; + +/** Bounded default projection for agent-facing JavaScript application analysis. */ +export const javascriptApplicationAnalysisSummarySchema = z.discriminatedUnion( + "schema_version", + [ + javaScriptApplicationAnalysisResultV1Schema + .omit({ graph: true }) + .extend(summaryFields), + javaScriptApplicationAnalysisResultV1Schema.omit({ graph: true }).extend({ + schema_version: z.literal(2), + ...summaryFields, + semantic_graph: semanticGraphSummarySchema, + }), + ], +); const applicationOutputSchema = evidenceResultOf( z.union([ diff --git a/src/contracts/toolEffects.ts b/src/contracts/toolEffects.ts index 5aa624481..d9bca24be 100644 --- a/src/contracts/toolEffects.ts +++ b/src/contracts/toolEffects.ts @@ -115,6 +115,7 @@ export const TOOL_EFFECTS: Readonly> = { analyze_javascript_application: evidence, reconcile_javascript_runtime: evidence, trace_application_feature: evidence, + trace_javascript_semantics: evidence, compare_application_versions: evidence, compare_javascript_export_shapes: evidence, run_replay_machine: effects(), diff --git a/src/domain/javascriptApplicationAnalysis.ts b/src/domain/javascriptApplicationAnalysis.ts index 53912e597..8f1693112 100644 --- a/src/domain/javascriptApplicationAnalysis.ts +++ b/src/domain/javascriptApplicationAnalysis.ts @@ -3,6 +3,7 @@ import { isAbsolute } from "node:path"; import { z } from "zod"; import { javascriptApplicationGraphSchema } from "./javascriptApplicationGraph.js"; +import { javaScriptSemanticGraphSchema } from "./javascriptSemanticGraph.js"; const MAX_GRAPH_NODES = 100_000; const MAX_GRAPH_EDGES = 200_000; @@ -155,8 +156,8 @@ const reconstructionStatisticsSchema = z.strictObject({ truncated_scopes: countSchema, }); -/** Strict high-level result returned inside Evidence v2. */ -export const javascriptApplicationAnalysisResultSchema = z.strictObject({ +/** Legacy structural JavaScript application analysis result. */ +export const javaScriptApplicationAnalysisResultV1Schema = z.strictObject({ schema_version: z.literal(1), input_path: z.string().min(1).max(16_384), format: z.enum(["asar", "directory"]), @@ -169,6 +170,61 @@ export const javascriptApplicationAnalysisResultSchema = z.strictObject({ limitations: z.array(z.string().min(1).max(4_096)).max(1_000), }); +/** JavaScript application analysis with an authenticated semantic companion. */ +export const javaScriptApplicationAnalysisResultV2Schema = + javaScriptApplicationAnalysisResultV1Schema + .extend({ + schema_version: z.literal(2), + semantic_graph: javaScriptSemanticGraphSchema, + }) + .superRefine((result, context) => { + if (result.semantic_graph.application_graph_id !== result.graph.graph_id) + context.addIssue({ + code: "custom", + path: ["semantic_graph", "application_graph_id"], + message: + "Semantic graph must commit the containing application graph", + }); + if ( + result.semantic_graph.root_artifact_sha256 !== + result.root_artifact_sha256 + ) + context.addIssue({ + code: "custom", + path: ["semantic_graph", "root_artifact_sha256"], + message: "Semantic graph must commit the containing root artifact", + }); + const applicationNodeIds = new Set( + result.graph.nodes.map(({ node_id }) => node_id), + ); + for (const [nodeIndex, node] of result.semantic_graph.nodes.entries()) + for (const [ + identifierIndex, + identifier, + ] of node.application_node_ids.entries()) + if (!applicationNodeIds.has(identifier)) + context.addIssue({ + code: "custom", + path: [ + "semantic_graph", + "nodes", + nodeIndex, + "application_node_ids", + identifierIndex, + ], + message: "Semantic node references an absent application node", + }); + }); + +/** Strict versioned high-level result returned inside Evidence v2. */ +export const javascriptApplicationAnalysisResultSchema = z.discriminatedUnion( + "schema_version", + [ + javaScriptApplicationAnalysisResultV1Schema, + javaScriptApplicationAnalysisResultV2Schema, + ], +); + /** Parsed public JavaScript application analysis request. */ export type AnalyzeJavaScriptApplicationInput = z.infer< typeof analyzeJavaScriptApplicationInputSchema diff --git a/src/domain/javascriptRuntimeReconciliationParsing.ts b/src/domain/javascriptRuntimeReconciliationParsing.ts index 6862c1010..448afb2f0 100644 --- a/src/domain/javascriptRuntimeReconciliationParsing.ts +++ b/src/domain/javascriptRuntimeReconciliationParsing.ts @@ -82,18 +82,18 @@ export const digestCanonical = (value: unknown): string => { const parseStaticLayer = (layer: StaticLayerInput): ParsedStaticLayer => { const evidence = parseEvidence(layer.analysis); + const result = javascriptApplicationAnalysisResultSchema.parse( + evidence.normalized_result, + ); assertEvidenceIdentity(evidence, { operation: "analyze_javascript_application", - predicate: "rea.javascript-application-analysis/v1", + predicate: `rea.javascript-application-analysis/v${String(result.schema_version)}`, providerId: "rea-javascript-application", providerName: "REA JavaScript application analyzer", providerVersion: "1", authority: "shipped-artifact", confidence: "derived", }); - const result = javascriptApplicationAnalysisResultSchema.parse( - evidence.normalized_result, - ); if ( evidence.subject === null || evidence.subject.digest.sha256 !== result.root_artifact_sha256 || diff --git a/src/domain/javascriptSemanticCalls.ts b/src/domain/javascriptSemanticCalls.ts index 31c2c8aa2..5a11ecdf9 100644 --- a/src/domain/javascriptSemanticCalls.ts +++ b/src/domain/javascriptSemanticCalls.ts @@ -3,6 +3,7 @@ import * as t from "@babel/types"; import type { JavaScriptSemanticArgumentFlow, JavaScriptSemanticCallable, + JavaScriptSemanticCallResultFlow, JavaScriptSemanticCallReturnFlow, JavaScriptSemanticCallSite, JavaScriptSemanticClosureCapture, @@ -32,6 +33,7 @@ export interface JavaScriptSemanticCallAnalysis { readonly callSites: readonly JavaScriptSemanticCallSite[]; readonly argumentFlows: readonly JavaScriptSemanticArgumentFlow[]; readonly callReturnFlows: readonly JavaScriptSemanticCallReturnFlow[]; + readonly callResultFlows: readonly JavaScriptSemanticCallResultFlow[]; readonly closureCaptures: readonly JavaScriptSemanticClosureCapture[]; readonly frontiers: readonly JavaScriptSemanticFrontier[]; } @@ -40,6 +42,7 @@ interface MutableCallAnalysis { readonly callSites: JavaScriptSemanticCallSite[]; readonly argumentFlows: JavaScriptSemanticArgumentFlow[]; readonly callReturnFlows: JavaScriptSemanticCallReturnFlow[]; + readonly callResultFlows: JavaScriptSemanticCallResultFlow[]; readonly closureCaptures: JavaScriptSemanticClosureCapture[]; readonly frontiers: JavaScriptSemanticFrontier[]; retainedArguments: number; @@ -66,6 +69,7 @@ export const collectJavaScriptSemanticCalls = ( callSites: [], argumentFlows: [], callReturnFlows: [], + callResultFlows: [], closureCaptures: [], frontiers: [], retainedArguments: 0, @@ -95,7 +99,7 @@ export const collectJavaScriptSemanticCalls = ( t.isOptionalCallExpression(node) || t.isNewExpression(node) ) - collectCallSite(node, owner, context); + collectCallSite(node, parent, owner, context); }, exit: (node) => { const callableId = semanticCallableIdForNode(node); @@ -110,6 +114,7 @@ export const collectJavaScriptSemanticCalls = ( callSites: output.callSites, argumentFlows: output.argumentFlows, callReturnFlows: output.callReturnFlows, + callResultFlows: output.callResultFlows, closureCaptures: output.closureCaptures, frontiers: output.frontiers, }; @@ -117,6 +122,7 @@ export const collectJavaScriptSemanticCalls = ( const collectCallSite = ( node: t.CallExpression | t.OptionalCallExpression | t.NewExpression, + parent: t.Node | null, owner: JavaScriptSemanticCallable | undefined, context: CallCollectionContext, ): void => { @@ -151,6 +157,7 @@ const collectCallSite = ( arguments: argumentsValue, }; output.callSites.push(site); + collectCallResultFlow(site, node, parent, context); if (site.resolution !== "exact") addFrontier( { @@ -167,6 +174,44 @@ const collectCallSite = ( collectReturnFlows(site, callableById, state, output); }; +const collectCallResultFlow = ( + site: JavaScriptSemanticCallSite, + node: t.CallExpression | t.OptionalCallExpression | t.NewExpression, + parent: t.Node | null, + context: CallCollectionContext, +): void => { + const identifier = + t.isVariableDeclarator(parent) && + parent.init === node && + t.isIdentifier(parent.id) + ? parent.id + : t.isAssignmentExpression(parent) && + parent.right === node && + t.isIdentifier(parent.left) + ? parent.left + : null; + if (identifier === null) return; + const binding = resolveSemanticBindingState( + context.state, + identifier, + identifier.name, + ); + const identifierRange = range(identifier); + const definition = binding?.definitions.find( + ({ location }) => + location.start.line === identifierRange.start.line && + location.start.column === identifierRange.start.column && + location.end.line === identifierRange.end.line && + location.end.column === identifierRange.end.column, + ); + if (binding === undefined || definition === undefined) return; + context.output.callResultFlows.push({ + callSiteId: site.callSiteId, + bindingId: binding.bindingId, + definitionLocation: definition.location, + }); +}; + const retainedArguments = ( nodes: readonly ( | t.Expression diff --git a/src/domain/javascriptSemanticGraph.ts b/src/domain/javascriptSemanticGraph.ts index 257e3adf3..631525690 100644 --- a/src/domain/javascriptSemanticGraph.ts +++ b/src/domain/javascriptSemanticGraph.ts @@ -53,13 +53,10 @@ const normalizeEvidence = ( evidence_ids: uniqueSorted(evidence.evidence_ids), }); -const nodeIdentity = ( - node: Pick, -) => ({ kind: node.kind, identity: node.identity }); - const nodeId = ( node: Pick, -): string => `jsrg_node_${digestCanonical(nodeIdentity(node))}`; +): string => + `jsrg_node_${digestCanonical({ kind: node.kind, identity: node.identity })}`; /** Normalize one semantic entity and derive its artifact-version identifier. */ export const createJavaScriptSemanticGraphNode = ( @@ -362,6 +359,12 @@ const checkUnknowns = ( path: ["unknowns", index, "node_id"], message: "Unknown frontier node is absent", }); + if (unknown.candidate_node_ids.some((nodeId) => !nodes.has(nodeId))) + context.addIssue({ + code: "custom", + path: ["unknowns", index, "candidate_node_ids"], + message: "Unknown frontier candidate node is absent", + }); if ( unknown.relation_kinds.some( (kind) => JAVASCRIPT_SEMANTIC_RELATION_FAMILY[kind] !== unknown.family, @@ -516,31 +519,4 @@ export const createJavaScriptSemanticGraph = ( }); }; -/** Parse a stored semantic graph and reject unsupported versions or stale IDs. */ -export const parseJavaScriptSemanticGraph = ( - input: unknown, -): JavaScriptSemanticGraph => { - const envelope = z - .object({ schema: z.string(), schema_version: z.number() }) - .passthrough() - .safeParse(input); - if ( - envelope.success && - envelope.data.schema === "JavaScriptSemanticRelationGraph" && - envelope.data.schema_version !== 1 - ) - throw new TypeError( - `Unsupported JavaScript Semantic Relation Graph schema version: ${String(envelope.data.schema_version)}`, - ); - return javaScriptSemanticGraphSchema.parse(input); -}; - -/** Serialize a verified semantic graph as canonical JSON. */ -export const serializeJavaScriptSemanticGraph = (input: unknown): string => - canonicalJson(parseJavaScriptSemanticGraph(input)); - -/** Compute the canonical SHA-256 commitment of a verified semantic graph. */ -export const computeJavaScriptSemanticGraphSha256 = (input: unknown): string => - digestCanonical(parseJavaScriptSemanticGraph(input)); - export type { JavaScriptSemanticGraphNode, JavaScriptSemanticGraphRelation }; diff --git a/src/domain/javascriptSemanticGraphSerialization.ts b/src/domain/javascriptSemanticGraphSerialization.ts new file mode 100644 index 000000000..ec12fe6f1 --- /dev/null +++ b/src/domain/javascriptSemanticGraphSerialization.ts @@ -0,0 +1,39 @@ +import canonicalize from "canonicalize"; +import { z } from "zod"; + +import { + javaScriptSemanticGraphSchema, + type JavaScriptSemanticGraph, +} from "./javascriptSemanticGraph.js"; + +const canonicalJson = (value: unknown): string => { + const encoded = canonicalize(value); + if (encoded === undefined) + throw new TypeError( + "JavaScript semantic graph could not canonicalize data", + ); + return encoded; +}; + +/** Parse a stored semantic graph and reject unsupported versions or stale IDs. */ +export const parseJavaScriptSemanticGraph = ( + input: unknown, +): JavaScriptSemanticGraph => { + const envelope = z + .object({ schema: z.string(), schema_version: z.number() }) + .passthrough() + .safeParse(input); + if ( + envelope.success && + envelope.data.schema === "JavaScriptSemanticRelationGraph" && + envelope.data.schema_version !== 1 + ) + throw new TypeError( + `Unsupported JavaScript Semantic Relation Graph schema version: ${String(envelope.data.schema_version)}`, + ); + return javaScriptSemanticGraphSchema.parse(input); +}; + +/** Serialize a verified semantic graph as canonical JSON. */ +export const serializeJavaScriptSemanticGraph = (input: unknown): string => + canonicalJson(parseJavaScriptSemanticGraph(input)); diff --git a/src/domain/javascriptSemanticIr.ts b/src/domain/javascriptSemanticIr.ts index 7cb47e217..d098b51b8 100644 --- a/src/domain/javascriptSemanticIr.ts +++ b/src/domain/javascriptSemanticIr.ts @@ -153,6 +153,7 @@ export interface JavaScriptSemanticCallable { export interface JavaScriptSemanticReturnSite { readonly returnSiteId: string; readonly location: JavaScriptSourceRange; + readonly identityReferenceLocation: JavaScriptSourceRange | null; readonly value: JavaScriptSemanticValue; } @@ -193,6 +194,13 @@ export interface JavaScriptSemanticCallReturnFlow { readonly returnLocation: JavaScriptSourceRange; } +/** One local binding initialized or assigned from a retained call expression. */ +export interface JavaScriptSemanticCallResultFlow { + readonly callSiteId: string; + readonly bindingId: string; + readonly definitionLocation: JavaScriptSourceRange; +} + /** One resolved outer lexical binding referenced by a nested callable. */ export interface JavaScriptSemanticClosureCapture { readonly callableId: string; @@ -251,6 +259,7 @@ export interface JavaScriptSemanticIr { readonly callSites: readonly JavaScriptSemanticCallSite[]; readonly argumentFlows: readonly JavaScriptSemanticArgumentFlow[]; readonly callReturnFlows: readonly JavaScriptSemanticCallReturnFlow[]; + readonly callResultFlows: readonly JavaScriptSemanticCallResultFlow[]; readonly closureCaptures: readonly JavaScriptSemanticClosureCapture[]; readonly frontiers: readonly JavaScriptSemanticFrontier[]; readonly coverage: JavaScriptSemanticCoverage; @@ -287,6 +296,7 @@ export const failedJavaScriptSemanticIr = (): JavaScriptSemanticIr => ({ callSites: [], argumentFlows: [], callReturnFlows: [], + callResultFlows: [], closureCaptures: [], frontiers: [], coverage: { status: "failed", omittedCount: null, limitsReached: [] }, diff --git a/src/domain/javascriptSemanticQuery.ts b/src/domain/javascriptSemanticQuery.ts index ca97d51f2..42d2b8400 100644 --- a/src/domain/javascriptSemanticQuery.ts +++ b/src/domain/javascriptSemanticQuery.ts @@ -1,17 +1,11 @@ -import { createHash } from "node:crypto"; - -import canonicalize from "canonicalize"; - import { compareCodePoints } from "./javascriptApplicationGraph.js"; import type { JavaScriptSemanticGraph, JavaScriptSemanticGraphNode, JavaScriptSemanticGraphRelation, } from "./javascriptSemanticGraph.js"; -import { - JAVASCRIPT_SEMANTIC_RELATION_FAMILY, - type JavaScriptSemanticGraphUnknown, -} from "./javascriptSemanticGraphSchemas.js"; +import type { JavaScriptSemanticGraphUnknown } from "./javascriptSemanticGraphSchemas.js"; +import { isJavaScriptSemanticOwnershipRelation as ownershipRelation } from "./javascriptSemanticQueryRelations.js"; import { javaScriptSemanticQueryInputSchema, javaScriptSemanticQueryResultSchema, @@ -22,6 +16,14 @@ import { assessJavaScriptSemanticQuery, type JavaScriptSemanticQueryFrontier, } from "./javascriptSemanticQueryAssessment.js"; +import { + canonicalJavaScriptSemanticQueryJson as canonicalJson, + createJavaScriptSemanticQueryCursor as createCursor, + JavaScriptSemanticQueryCursorError, + javaScriptSemanticQueryIdentifier as queryIdentifier, + parseJavaScriptSemanticQueryCursor as parseCursor, + resolveJavaScriptSemanticQuerySeeds as resolveSeeds, +} from "./javascriptSemanticQueryIdentity.js"; interface TraversalEntry { readonly relation: JavaScriptSemanticGraphRelation; @@ -43,20 +45,22 @@ interface SeedAdmission { readonly frontier: QueryFrontier[]; } -const canonicalJson = (value: unknown): string => { - const encoded = canonicalize(value); - if (encoded === undefined) - throw new TypeError( - "JavaScript semantic query could not canonicalize data", - ); - return encoded; -}; +interface QueryPage { + readonly retainedNodes: JavaScriptSemanticGraphNode[]; + readonly retainedRelations: JavaScriptSemanticGraphRelation[]; + readonly pageRelations: JavaScriptSemanticGraphRelation[]; + readonly pageNodes: JavaScriptSemanticGraphNode[]; + readonly nextCursor: string | null; +} -const digest = (value: unknown): string => - createHash("sha256").update(canonicalJson(value)).digest("hex"); - -const uniqueSorted = (values: readonly string[]): string[] => - [...new Set(values)].sort(compareCodePoints); +interface QueryPageInput { + readonly graph: JavaScriptSemanticGraph; + readonly traversal: Traversal; + readonly query: JavaScriptSemanticQueryInput; + readonly queryId: string; + readonly offset: number; + readonly retainedSeeds: readonly string[]; +} /** Run one deterministic bounded traversal over a verified semantic graph. */ export const queryJavaScriptSemanticGraph = ( @@ -75,41 +79,31 @@ export const queryJavaScriptSemanticGraph = ( ...traversal.frontier, ...admission.frontier, ]); - const retainedNodes = graph.nodes.filter(({ node_id }) => - traversal.nodeIds.has(node_id), - ); - const retainedRelations = graph.relations.filter(({ relation_id }) => - traversal.relationIds.has(relation_id), - ); - if (offset > retainedRelations.length) - throw new TypeError("Semantic query cursor is beyond the retained result"); - const pageRelations = retainedRelations.slice( + const page = createQueryPage({ + graph, + traversal, + query: input, + queryId, offset, - offset + input.limits.page_size, - ); - const pageNodeIds = new Set([ - ...retainedSeeds, - ...pageRelations.flatMap(({ source_node_id, target_node_id }) => [ - source_node_id, - target_node_id, - ]), - ]); - const pageNodes = graph.nodes.filter(({ node_id }) => - pageNodeIds.has(node_id), - ); - const relevantUnknowns = relevantUnknownFrontiers( + retainedSeeds, + }); + const allRelevantUnknowns = relevantUnknownFrontiers( graph, traversal.nodeIds, input, ); + const relevantUnknowns = allRelevantUnknowns.slice( + 0, + input.limits.max_unknowns, + ); const candidateRelations = relevantCandidateRelationCount( graph, traversal.nodeIds, input, ); const expectedMatches = expectedMatchesFor( - retainedNodes, - retainedRelations, + page.retainedNodes, + page.retainedRelations, input, ); const assessment = assessJavaScriptSemanticQuery({ @@ -121,12 +115,8 @@ export const queryJavaScriptSemanticGraph = ( frontier, unknowns: relevantUnknowns, candidateRelations, + unknownsTruncated: allRelevantUnknowns.length > relevantUnknowns.length, }); - const nextOffset = offset + pageRelations.length; - const nextCursor = - nextOffset < retainedRelations.length - ? createCursor(queryId, nextOffset) - : null; return javaScriptSemanticQueryResultSchema.parse({ schema_version: 1, query_id: queryId, @@ -135,8 +125,8 @@ export const queryJavaScriptSemanticGraph = ( direction: input.direction, status: assessment.status, seed_node_ids: retainedSeeds, - nodes: pageNodes, - relations: pageRelations, + nodes: page.pageNodes, + relations: page.pageRelations, unknowns: relevantUnknowns, expected_match_node_ids: expectedMatches.map(({ node_id }) => node_id), summary: { @@ -146,13 +136,14 @@ export const queryJavaScriptSemanticGraph = ( traversed_relations: traversal.relationIds.size, traversed_functions: traversal.functionIds.size, traversed_modules: traversal.modules.size, - relevant_unknowns: relevantUnknowns.length, + relevant_unknowns: allRelevantUnknowns.length, + retained_unknowns: relevantUnknowns.length, }, coverage: assessment.coverage, page: { offset, - size: pageRelations.length, - next_cursor: nextCursor, + size: page.pageRelations.length, + next_cursor: page.nextCursor, }, applied_limits: input.limits, accepted_limit_ranges: assessment.acceptedLimitRanges, @@ -160,96 +151,49 @@ export const queryJavaScriptSemanticGraph = ( }); }; -const queryIdentifier = ( - graph: JavaScriptSemanticGraph, - input: JavaScriptSemanticQueryInput, -): string => { - const { cursor: _cursor, ...semantic } = input; - return `jsrq_${digest({ - source_graph_id: graph.graph_id, - ...semantic, - allowed_relations: - semantic.allowed_relations === undefined - ? undefined - : uniqueSorted(semantic.allowed_relations), - expected: - semantic.expected === null - ? null - : { - ...semantic.expected, - classes: uniqueSorted(semantic.expected.classes), - }, - })}`; -}; - -const createCursor = (queryId: string, offset: number): string => - `jsrqc_${String(offset)}_${digest({ query_id: queryId, offset })}`; - -const parseCursor = (cursor: string | null, queryId: string): number => { - if (cursor === null) return 0; - const match = /^jsrqc_([0-9]+)_([a-f0-9]{64})$/u.exec(cursor); - const offsetText = match?.[1]; - const commitment = match?.[2]; - if (offsetText === undefined || commitment === undefined) - throw new TypeError("Semantic query cursor is malformed"); - const offset = Number(offsetText); - if ( - !Number.isSafeInteger(offset) || - offset < 0 || - commitment !== digest({ query_id: queryId, offset }) - ) - throw new TypeError("Semantic query cursor does not match this query"); - return offset; -}; - -const resolveSeeds = ( - graph: JavaScriptSemanticGraph, - input: JavaScriptSemanticQueryInput, -): string[] => { - const fingerprintFunctions = new Set( - input.seed.kind === "function" - ? graph.fingerprints - .filter(({ fingerprint_sha256 }) => { - const seed = input.seed; - return ( - seed.kind === "function" && - fingerprint_sha256 === seed.fingerprint_sha256 - ); - }) - .map(({ function_node_id }) => function_node_id) - : [], +const createQueryPage = ({ + graph, + traversal, + query, + queryId, + offset, + retainedSeeds, +}: QueryPageInput): QueryPage => { + const retainedNodes = graph.nodes.filter(({ node_id }) => + traversal.nodeIds.has(node_id), ); - return graph.nodes - .filter((node) => seedMatches(node, input, fingerprintFunctions)) - .map(({ node_id }) => node_id) - .sort(compareCodePoints); -}; - -const seedMatches = ( - node: JavaScriptSemanticGraphNode, - input: JavaScriptSemanticQueryInput, - fingerprintFunctions: ReadonlySet, -): boolean => { - const seed = input.seed; - if (seed.kind === "semantic-node") return node.node_id === seed.node_id; - if (seed.kind === "application-node") - return node.application_node_ids.includes(seed.node_id); - if (seed.kind === "function") return fingerprintFunctions.has(node.node_id); - if (seed.kind === "literal") - return ( - node.kind === "literal" && - canonicalJson(node.properties.value) === canonicalJson(seed.value) + const retainedRelations = graph.relations.filter(({ relation_id }) => + traversal.relationIds.has(relation_id), + ); + if (offset > retainedRelations.length) + throw new JavaScriptSemanticQueryCursorError( + "Semantic query cursor is beyond the retained result", ); - if (seed.kind === "property") - return node.kind === "property-slot" && node.properties.name === seed.name; - if (seed.kind === "endpoint") - return node.kind === "request" && node.properties.endpoint === seed.value; - if (seed.kind === "event") - return ( - ["event", "listener"].includes(node.kind) && - node.properties.event_name === seed.name - ); - return node.kind === "boundary" && node.properties.field === seed.field; + const pageRelations = retainedRelations.slice( + offset, + offset + query.limits.page_size, + ); + const pageNodeIds = new Set([ + ...retainedSeeds, + ...pageRelations.flatMap(({ source_node_id, target_node_id }) => [ + source_node_id, + target_node_id, + ]), + ]); + const pageNodes = graph.nodes.filter(({ node_id }) => + pageNodeIds.has(node_id), + ); + const nextOffset = offset + pageRelations.length; + return { + retainedNodes, + retainedRelations, + pageRelations, + pageNodes, + nextCursor: + nextOffset < retainedRelations.length + ? createCursor(queryId, nextOffset) + : null, + }; }; const admitSeeds = ( @@ -395,23 +339,6 @@ const addDirectedEntries = ( ); }; -const ownershipRelation = ( - relation: JavaScriptSemanticGraphRelation["relation"], -): boolean => - [ - "acquires", - "aggregates", - "awaits", - "chains", - "connects-stdio", - "creates-promise", - "detaches-task", - "owns", - "releases", - "returns-task", - "spawns", - ].includes(relation); - const addEntry = ( adjacency: Map, from: string, @@ -585,8 +512,3 @@ const expectedMatchesFor = ( ({ kind, node_id }) => classes.has(kind) && !connected.has(node_id), ); }; - -/** Return a relation's declared semantic coverage family. */ -export const javaScriptSemanticRelationFamily = ( - relation: JavaScriptSemanticGraphRelation["relation"], -) => JAVASCRIPT_SEMANTIC_RELATION_FAMILY[relation]; diff --git a/src/domain/javascriptSemanticQueryAssessment.ts b/src/domain/javascriptSemanticQueryAssessment.ts index 89b4a3d44..44953094a 100644 --- a/src/domain/javascriptSemanticQueryAssessment.ts +++ b/src/domain/javascriptSemanticQueryAssessment.ts @@ -26,6 +26,7 @@ export interface JavaScriptSemanticQueryAssessmentInput { readonly frontier: readonly JavaScriptSemanticQueryFrontier[]; readonly unknowns: readonly JavaScriptSemanticGraphUnknown[]; readonly candidateRelations: number; + readonly unknownsTruncated: boolean; } /** Status, coverage, limits, and limitations derived from one traversal. */ @@ -50,6 +51,7 @@ export const assessJavaScriptSemanticQuery = ( max_depth: { minimum: 0, maximum: 64 }, max_functions: { minimum: 1, maximum: 10_000 }, max_modules: { minimum: 1, maximum: 10_000 }, + max_unknowns: { minimum: 0, maximum: 10_000 }, page_size: { minimum: 1, maximum: 1_000 }, }, }); @@ -60,7 +62,7 @@ const queryCoverage = ( status: input.graph.coverage.status === "unavailable" ? "unavailable" - : input.frontier.length > 0 + : input.frontier.length > 0 || input.unknownsTruncated ? "truncated" : input.unknowns.length > 0 || input.candidateRelations > 0 || @@ -74,15 +76,16 @@ const queryStatus = ( input: JavaScriptSemanticQueryAssessmentInput, ): JavaScriptSemanticQueryResult["status"] => { if (input.graph.coverage.status === "unavailable") return "unsupported"; - if (input.frontier.length > 0) return "truncated"; - if (input.unknowns.length > 0 || input.graph.coverage.status !== "complete") - return "partial"; + if (input.frontier.length > 0 || input.unknownsTruncated) return "truncated"; if (input.candidateRelations > 0) return "ambiguous"; if ( input.totalSeeds === 0 || (input.hasExpectation && input.expectedMatches === 0) ) - return "no-match"; + return input.unknowns.length > 0 || + input.graph.coverage.status !== "complete" + ? "partial" + : "no-match"; return input.totalSeeds > 1 ? "ambiguous" : "found"; }; @@ -107,6 +110,11 @@ const queryLimitations = ( : [ "Relevant dynamic, unsupported, incomplete, or ambiguous semantics remain unknown.", ]), + ...(input.unknownsTruncated + ? [ + "Relevant unknown frontiers exceeded the caller limit; omitted unknowns remain unresolved.", + ] + : []), ...(input.candidateRelations === 0 ? [] : [ diff --git a/src/domain/javascriptSemanticQueryIdentity.ts b/src/domain/javascriptSemanticQueryIdentity.ts new file mode 100644 index 000000000..b83d6c6ef --- /dev/null +++ b/src/domain/javascriptSemanticQueryIdentity.ts @@ -0,0 +1,137 @@ +import { createHash } from "node:crypto"; + +import canonicalize from "canonicalize"; + +import { compareCodePoints } from "./javascriptApplicationGraph.js"; +import type { + JavaScriptSemanticGraph, + JavaScriptSemanticGraphNode, +} from "./javascriptSemanticGraph.js"; +import type { JavaScriptSemanticQueryInput } from "./javascriptSemanticQuerySchemas.js"; + +/** Caller cursor that cannot identify a page of the current semantic query. */ +export class JavaScriptSemanticQueryCursorError extends TypeError {} + +/** Canonical JSON used by semantic query commitments and ordering. */ +export const canonicalJavaScriptSemanticQueryJson = ( + value: unknown, +): string => { + const encoded = canonicalize(value); + if (encoded === undefined) + throw new TypeError( + "JavaScript semantic query could not canonicalize data", + ); + return encoded; +}; + +const digest = (value: unknown): string => + createHash("sha256") + .update(canonicalJavaScriptSemanticQueryJson(value)) + .digest("hex"); + +const uniqueSorted = (values: readonly string[]): string[] => + [...new Set(values)].sort(compareCodePoints); + +/** Derive the stable identity for one graph-bound query without its cursor. */ +export const javaScriptSemanticQueryIdentifier = ( + graph: JavaScriptSemanticGraph, + input: JavaScriptSemanticQueryInput, +): string => { + const { cursor: _cursor, ...semantic } = input; + return `jsrq_${digest({ + source_graph_id: graph.graph_id, + ...semantic, + allowed_relations: + semantic.allowed_relations === undefined + ? undefined + : uniqueSorted(semantic.allowed_relations), + expected: + semantic.expected === null + ? null + : { + ...semantic.expected, + classes: uniqueSorted(semantic.expected.classes), + }, + })}`; +}; + +/** Commit one deterministic relation-page offset to a query identity. */ +export const createJavaScriptSemanticQueryCursor = ( + queryId: string, + offset: number, +): string => `jsrqc_${String(offset)}_${digest({ query_id: queryId, offset })}`; + +/** Parse a query-bound cursor and reject stale, malformed, or foreign values. */ +export const parseJavaScriptSemanticQueryCursor = ( + cursor: string | null, + queryId: string, +): number => { + if (cursor === null) return 0; + const match = /^jsrqc_([0-9]+)_([a-f0-9]{64})$/u.exec(cursor); + const offsetText = match?.[1]; + const commitment = match?.[2]; + if (offsetText === undefined || commitment === undefined) + throw new JavaScriptSemanticQueryCursorError( + "Semantic query cursor is malformed", + ); + const offset = Number(offsetText); + if ( + !Number.isSafeInteger(offset) || + offset < 0 || + commitment !== digest({ query_id: queryId, offset }) + ) + throw new JavaScriptSemanticQueryCursorError( + "Semantic query cursor does not match this query", + ); + return offset; +}; + +/** Resolve deterministic seed candidates without applying traversal limits. */ +export const resolveJavaScriptSemanticQuerySeeds = ( + graph: JavaScriptSemanticGraph, + input: JavaScriptSemanticQueryInput, +): string[] => { + const fingerprintFunctions = new Set( + input.seed.kind === "function" + ? graph.fingerprints + .filter( + ({ fingerprint_sha256 }) => + input.seed.kind === "function" && + fingerprint_sha256 === input.seed.fingerprint_sha256, + ) + .map(({ function_node_id }) => function_node_id) + : [], + ); + return graph.nodes + .filter((node) => seedMatches(node, input, fingerprintFunctions)) + .map(({ node_id }) => node_id) + .sort(compareCodePoints); +}; + +const seedMatches = ( + node: JavaScriptSemanticGraphNode, + input: JavaScriptSemanticQueryInput, + fingerprintFunctions: ReadonlySet, +): boolean => { + const seed = input.seed; + if (seed.kind === "semantic-node") return node.node_id === seed.node_id; + if (seed.kind === "application-node") + return node.application_node_ids.includes(seed.node_id); + if (seed.kind === "function") return fingerprintFunctions.has(node.node_id); + if (seed.kind === "literal") + return ( + node.kind === "literal" && + canonicalJavaScriptSemanticQueryJson(node.properties.value) === + canonicalJavaScriptSemanticQueryJson(seed.value) + ); + if (seed.kind === "property") + return node.kind === "property-slot" && node.properties.name === seed.name; + if (seed.kind === "endpoint") + return node.kind === "request" && node.properties.endpoint === seed.value; + if (seed.kind === "event") + return ( + ["event", "listener"].includes(node.kind) && + node.properties.event_name === seed.name + ); + return node.kind === "boundary" && node.properties.field === seed.field; +}; diff --git a/src/domain/javascriptSemanticQueryRelations.ts b/src/domain/javascriptSemanticQueryRelations.ts new file mode 100644 index 000000000..698ffa2a0 --- /dev/null +++ b/src/domain/javascriptSemanticQueryRelations.ts @@ -0,0 +1,22 @@ +import type { JavaScriptSemanticGraphRelation } from "./javascriptSemanticGraph.js"; + +const OWNERSHIP_RELATIONS = new Set< + JavaScriptSemanticGraphRelation["relation"] +>([ + "acquires", + "aggregates", + "awaits", + "chains", + "connects-stdio", + "creates-promise", + "detaches-task", + "owns", + "releases", + "returns-task", + "spawns", +]); + +/** Whether one relation participates in bidirectional ownership traversal. */ +export const isJavaScriptSemanticOwnershipRelation = ( + relation: JavaScriptSemanticGraphRelation["relation"], +): boolean => OWNERSHIP_RELATIONS.has(relation); diff --git a/src/domain/javascriptSemanticQuerySchemas.ts b/src/domain/javascriptSemanticQuerySchemas.ts index 00d3e8ca3..0b180cf90 100644 --- a/src/domain/javascriptSemanticQuerySchemas.ts +++ b/src/domain/javascriptSemanticQuerySchemas.ts @@ -11,18 +11,6 @@ import { jsonValueSchema } from "./jsonValue.js"; const digestSchema = z.string().regex(/^[a-f0-9]{64}$/u); const semanticNodeIdSchema = z.string().regex(/^jsrg_node_[a-f0-9]{64}$/u); -const evidenceIdSchema = z.string().regex(/^ev_[a-f0-9]{64}$/u); - -/** Exact accepted ranges and defaults for one semantic traversal. */ -export const JAVASCRIPT_SEMANTIC_QUERY_LIMIT_RANGES = { - max_seed_matches: { minimum: 1, maximum: 1_000, default: 25 }, - max_nodes: { minimum: 1, maximum: 50_000, default: 2_000 }, - max_edges: { minimum: 1, maximum: 100_000, default: 4_000 }, - max_depth: { minimum: 0, maximum: 64, default: 12 }, - max_functions: { minimum: 1, maximum: 10_000, default: 500 }, - max_modules: { minimum: 1, maximum: 10_000, default: 500 }, - page_size: { minimum: 1, maximum: 1_000, default: 200 }, -} as const; const literalSeedSchema = z.strictObject({ kind: z.literal("literal"), @@ -77,17 +65,13 @@ export const javaScriptSemanticQueryLimitsSchema = z.strictObject({ max_depth: z.number().int().min(0).max(64).default(12), max_functions: z.number().int().min(1).max(10_000).default(500), max_modules: z.number().int().min(1).max(10_000).default(500), + max_unknowns: z.number().int().min(0).max(10_000).default(1_000), page_size: z.number().int().min(1).max(1_000).default(200), }); -const sourceMapAuthoritySchema = z.discriminatedUnion("authority", [ - z.strictObject({ authority: z.literal("none") }), - z.strictObject({ - authority: z.literal("authenticated"), - evidence_id: evidenceIdSchema, - source_map_sha256: digestSchema, - }), -]); +const sourceMapAuthoritySchema = z.strictObject({ + authority: z.literal("none"), +}); /** Parsed pure-domain query over one authenticated companion graph. */ export const javaScriptSemanticQueryInputSchema = z.strictObject({ @@ -96,7 +80,6 @@ export const javaScriptSemanticQueryInputSchema = z.strictObject({ "backward-provenance", "forward-influence", "callers", - "consumers", "ownership", ]), allowed_relations: z @@ -120,6 +103,7 @@ export const javaScriptSemanticQueryInputSchema = z.strictObject({ max_depth: 12, max_functions: 500, max_modules: 500, + max_unknowns: 1_000, page_size: 200, }), cursor: z @@ -170,6 +154,7 @@ export const javaScriptSemanticQueryResultSchema = z.strictObject({ traversed_functions: z.number().int().min(0), traversed_modules: z.number().int().min(0), relevant_unknowns: z.number().int().min(0), + retained_unknowns: z.number().int().min(0).max(10_000), }), coverage: z.strictObject({ status: z.enum(["complete", "partial", "truncated", "unavailable"]), @@ -209,6 +194,10 @@ export const javaScriptSemanticQueryResultSchema = z.strictObject({ minimum: z.literal(1), maximum: z.literal(10_000), }), + max_unknowns: z.strictObject({ + minimum: z.literal(0), + maximum: z.literal(10_000), + }), page_size: z.strictObject({ minimum: z.literal(1), maximum: z.literal(1_000), diff --git a/src/domain/javascriptSemanticReturns.ts b/src/domain/javascriptSemanticReturns.ts index 017a032cd..2d43f5bc2 100644 --- a/src/domain/javascriptSemanticReturns.ts +++ b/src/domain/javascriptSemanticReturns.ts @@ -60,6 +60,10 @@ export const collectSemanticReturns = ( expression.location, ), location: expression.location, + identityReferenceLocation: + expression.node !== null && t.isIdentifier(expression.node) + ? range(expression.node) + : null, value: expression.node === null ? { status: "unknown", reason: "Return has no value." } diff --git a/src/domain/javascriptSemanticTraceSchemas.ts b/src/domain/javascriptSemanticTraceSchemas.ts new file mode 100644 index 000000000..20a94f935 --- /dev/null +++ b/src/domain/javascriptSemanticTraceSchemas.ts @@ -0,0 +1,27 @@ +import { z } from "zod"; + +import { evidenceSchema } from "./evidence.js"; +import { + javaScriptSemanticQueryInputSchema, + javaScriptSemanticQueryResultSchema, +} from "./javascriptSemanticQuerySchemas.js"; + +const evidenceIdSchema = z.string().regex(/^ev_[a-f0-9]{64}$/u); + +/** Authenticated application Evidence plus one bounded semantic query. */ +export const traceJavaScriptSemanticsInputSchema = z.strictObject({ + application: evidenceSchema, + query: javaScriptSemanticQueryInputSchema, +}); + +/** Evidence-linked semantic query result shared by CLI and MCP. */ +export const javaScriptSemanticTraceResultSchema = + javaScriptSemanticQueryResultSchema.extend({ + source_evidence_id: evidenceIdSchema, + evidence_links: z.array(evidenceIdSchema).length(1), + }); + +/** Validated Evidence-linked semantic trace result. */ +export type JavaScriptSemanticTraceResult = z.infer< + typeof javaScriptSemanticTraceResultSchema +>; diff --git a/src/server/javascriptApplicationResult.ts b/src/server/javascriptApplicationResult.ts index e38066531..2bc71044a 100644 --- a/src/server/javascriptApplicationResult.ts +++ b/src/server/javascriptApplicationResult.ts @@ -58,7 +58,8 @@ export const summarizeJavaScriptApplicationEvidence = ( location: observation.evidence.location, }; }); - const { graph: _graph, ...summary } = result; + const summary = + result.schema_version === 1 ? omitGraph(result) : omitGraphs(result); const unknowns = new Set([ ...result.limitations, ...result.graph.limitations, @@ -77,6 +78,19 @@ export const summarizeJavaScriptApplicationEvidence = ( ); return javascriptApplicationAnalysisSummarySchema.parse({ ...summary, + ...(result.schema_version === 1 + ? {} + : { + semantic_graph: { + graph_id: result.semantic_graph.graph_id, + nodes: result.semantic_graph.nodes.length, + relations: result.semantic_graph.relations.length, + unknown_frontiers: result.semantic_graph.unknowns.length, + fingerprints: result.semantic_graph.fingerprints.length, + coverage: result.semantic_graph.coverage.status, + query_tool: "trace_javascript_semantics", + }, + }), unknowns: [...unknowns], graph: { graph_id: result.graph.graph_id, @@ -101,3 +115,23 @@ export const summarizeJavaScriptApplicationEvidence = ( }, }); }; + +const omitGraph = ( + result: Extract< + ReturnType, + { readonly schema_version: 1 } + >, +) => { + const { graph: _graph, ...summary } = result; + return summary; +}; + +const omitGraphs = ( + result: Extract< + ReturnType, + { readonly schema_version: 2 } + >, +) => { + const { graph: _graph, semantic_graph: _semanticGraph, ...summary } = result; + return summary; +}; diff --git a/src/server/registerApplicationTools.ts b/src/server/registerApplicationTools.ts index 1c093b783..69f6b06f1 100644 --- a/src/server/registerApplicationTools.ts +++ b/src/server/registerApplicationTools.ts @@ -1,6 +1,7 @@ import type { McpServer } from "@modelcontextprotocol/server"; import { registerTraceFeatureTool } from "./registerApplicationTools/traceFeature.js"; +import { registerTraceJavaScriptSemanticsTool } from "./registerApplicationTools/traceSemantics.js"; import { registerCompareApplicationVersionsTool } from "./registerApplicationTools/compareVersions.js"; import { registerCompareJavaScriptExportShapesTool } from "./registerApplicationTools/compareExportShapes.js"; import { registerControlledReplayTool } from "./registerApplicationTools/controlledReplay.js"; @@ -16,6 +17,7 @@ export const registerApplicationTools = ( options: ApplicationToolRegistration, ): void => { registerTraceFeatureTool(server, options); + registerTraceJavaScriptSemanticsTool(server, options); registerCompareApplicationVersionsTool(server, options); registerCompareJavaScriptExportShapesTool(server, options); registerControlledReplayTool(server, options); diff --git a/src/server/registerApplicationTools/traceSemantics.ts b/src/server/registerApplicationTools/traceSemantics.ts new file mode 100644 index 000000000..19ef9c71a --- /dev/null +++ b/src/server/registerApplicationTools/traceSemantics.ts @@ -0,0 +1,49 @@ +import type { McpServer } from "@modelcontextprotocol/server"; + +import { resolveTraceJavaScriptSemanticsRequestValidated } from "../../application/ApplicationWorkflowEvidenceResolver.js"; +import { traceJavaScriptSemanticsEvidenceValidated } from "../../application/JavaScriptSemanticTraceService.js"; +import { applicationToolContract } from "../../contracts/applicationToolContracts.js"; +import { traceJavaScriptSemanticsRequestSchema } from "../../contracts/applicationWorkflowInputContracts.js"; +import { safeParseToolInput } from "../toolInputValidation.js"; +import { logToolExecution } from "../toolLogging.js"; +import { toolRegistrationOptions } from "../toolRegistrationOptions.js"; +import { toCallToolResult } from "../toolResult.js"; +import { recordResult, recordSources } from "./helpers.js"; +import type { ApplicationToolRegistration } from "./types.js"; + +const contract = applicationToolContract("trace_javascript_semantics"); + +/** Register the bounded JavaScript semantic relation trace tool. */ +export const registerTraceJavaScriptSemanticsTool = ( + server: McpServer, + options: ApplicationToolRegistration, +): void => { + server.registerTool( + contract.name, + toolRegistrationOptions(contract), + async (input) => { + const parsedInput = safeParseToolInput( + traceJavaScriptSemanticsRequestSchema, + input, + contract.name, + ); + if (!parsedInput.ok) return toCallToolResult(parsedInput, contract); + const resolved = resolveTraceJavaScriptSemanticsRequestValidated( + parsedInput.value, + options.evidenceLookup, + ); + if (!resolved.ok) return toCallToolResult(resolved, contract); + const result = await logToolExecution(options.logger, contract.name, () => + Promise.resolve( + traceJavaScriptSemanticsEvidenceValidated(resolved.value), + ), + ); + if (!result.ok) return toCallToolResult(result, contract); + const recorded = recordSources(options.recordEvidence, [ + resolved.value.application, + ]); + if (!recorded.ok) return toCallToolResult(recorded, contract); + return recordResult(options, contract, result.value); + }, + ); +}; diff --git a/tests/applicationWorkflowCli.test.ts b/tests/applicationWorkflowCli.test.ts index 5d17fd0e5..c6f1abd30 100644 --- a/tests/applicationWorkflowCli.test.ts +++ b/tests/applicationWorkflowCli.test.ts @@ -20,6 +20,7 @@ import { import { analyzeJavaScriptApplication } from "../src/application/JavaScriptApplicationService.js"; import { permissionAuthorityForRoot } from "./fixtures/permissionAuthority.js"; import { REPLAY_MACHINE_RUN_EXAMPLE } from "../src/contracts/replayMachineExample.js"; +import { javascriptApplicationAnalysisResultSchema } from "../src/domain/javascriptApplicationAnalysis.js"; const execute = promisify(execFile); @@ -66,6 +67,54 @@ describe("application workflow CLI parity", () => { }); }, 20_000); + it("traces the same authenticated semantic graph through the CLI", async () => { + const root = await createTestTempDirectory("rea-semantic-cli-"); + temporary.push(root); + await writeFile( + join(root, "app.js"), + "function add(value) { return value + 1; } add(2);", + ); + const authority = await permissionAuthorityForRoot( + root, + ["investigation_input"], + ["investigation_input"], + ); + const analyzed = await analyzeJavaScriptApplication(authority, { + input_path: root, + approved: true, + }); + if (!analyzed.ok) throw analyzed.error; + const result = javascriptApplicationAnalysisResultSchema.parse( + analyzed.value.normalized_result, + ); + if (result.schema_version !== 2) + throw new TypeError("Expected semantic application Evidence"); + const seed = result.semantic_graph.relations[0]?.source_node_id; + if (seed === undefined) + throw new TypeError("Expected at least one semantic relation"); + + const traced = await runCli([ + "trace-javascript-semantics", + JSON.stringify({ + application: analyzed.value, + query: { + seed: { kind: "semantic-node", node_id: seed }, + direction: "forward-influence", + include_ambiguous_dynamic_edges: true, + }, + }), + "--json", + ]); + expect(traced).toMatchObject({ + operation: "trace_javascript_semantics", + predicate_type: "rea.javascript-semantic-trace/v1", + normalized_result: { + source_evidence_id: analyzed.value.evidence_id, + source_graph_id: result.semantic_graph.graph_id, + }, + }); + }, 20_000); + it("resolves Evidence IDs from an explicitly authorized bundle", async () => { const root = await createTestTempDirectory("rea-application-bundle-cli-"); temporary.push(root); diff --git a/tests/applicationWorkflowMcp.test.ts b/tests/applicationWorkflowMcp.test.ts index 0c5980250..3c4a5d8d7 100644 --- a/tests/applicationWorkflowMcp.test.ts +++ b/tests/applicationWorkflowMcp.test.ts @@ -21,6 +21,7 @@ import { createPermissionPolicy } from "../src/domain/permissionPolicy.js"; import { controlledReplayOutputSchema } from "../src/domain/javascriptReplay.js"; import { nodeCharacterizationPreparationOutputSchema } from "../src/domain/nodeRuntimeCharacterization.js"; import { analyzeJavaScriptApplication } from "../src/application/JavaScriptApplicationService.js"; +import { javascriptApplicationAnalysisResultSchema } from "../src/domain/javascriptApplicationAnalysis.js"; import { permissionAuthorityForRoot } from "./fixtures/permissionAuthority.js"; describe("application workflow MCP parity", () => { @@ -114,6 +115,35 @@ describe("application workflow MCP parity", () => { changes: [expect.objectContaining({ path: "/depth" })], }, }); + + const analyzed = javascriptApplicationAnalysisResultSchema.parse( + left.value.normalized_result, + ); + if (analyzed.schema_version !== 2) + throw new TypeError("Expected semantic application Evidence"); + const seed = analyzed.semantic_graph.relations[0]?.source_node_id; + if (seed === undefined) + throw new TypeError("Expected at least one semantic relation"); + const semantic = await client.callTool({ + name: "trace_javascript_semantics", + arguments: { + application: left.value, + query: { + seed: { kind: "semantic-node", node_id: seed }, + direction: "forward-influence", + include_ambiguous_dynamic_edges: true, + }, + }, + }); + expect(semantic.isError).not.toBe(true); + expect(semantic.structuredContent).toMatchObject({ + result: { + schema_version: 1, + source_evidence_id: left.value.evidence_id, + source_graph_id: analyzed.semantic_graph.graph_id, + summary: { retained_seed_matches: 1 }, + }, + }); } finally { await client.close(); await server.close(); diff --git a/tests/contractSnapshot.test.ts b/tests/contractSnapshot.test.ts index edf57f1d0..555636aaa 100644 --- a/tests/contractSnapshot.test.ts +++ b/tests/contractSnapshot.test.ts @@ -97,7 +97,7 @@ describe("tool contract surface", () => { ...ELECTRON_TOOL_CONTRACTS, ...APPLICATION_TOOL_CONTRACTS, ]; - expect(contracts).toHaveLength(78); + expect(contracts).toHaveLength(79); for (const contract of contracts) { const inputSchema = contractJsonSchema(contract.inputSchema); const outputSchema = contractJsonSchema(contract.outputSchema); diff --git a/tests/electronApplicationAnalysis.test.ts b/tests/electronApplicationAnalysis.test.ts index b8d9b5da3..6b8b8b6f7 100644 --- a/tests/electronApplicationAnalysis.test.ts +++ b/tests/electronApplicationAnalysis.test.ts @@ -247,13 +247,13 @@ describe("static Electron application analysis", () => { const evidence = parseEvidence(result.value); expect(evidence).toMatchObject({ operation: "analyze_javascript_application", - predicate_type: "rea.javascript-application-analysis/v1", + predicate_type: "rea.javascript-application-analysis/v2", provider: { id: "rea-javascript-application" }, authority: "shipped-artifact", confidence: "derived", subject: { local_path: root, format: "directory" }, normalized_result: { - schema_version: 1, + schema_version: 2, input_path: root, summary: { browser_windows: 3 }, }, diff --git a/tests/electronMcp.test.ts b/tests/electronMcp.test.ts index c6dbe9049..2b16a29d7 100644 --- a/tests/electronMcp.test.ts +++ b/tests/electronMcp.test.ts @@ -229,7 +229,7 @@ describe("Electron MCP tools", () => { .object({ evidence_id: z.string(), result: z.object({ - schema_version: z.literal(1), + schema_version: z.literal(2), input_path: z.string(), unknowns: z.array(z.string()), summary: z.object({ @@ -244,6 +244,13 @@ describe("Electron MCP tools", () => { top_findings: z.array(z.unknown()), pages: z.object({ nodes: z.string(), edges: z.string() }), }), + semantic_graph: z.object({ + graph_id: z.string(), + nodes: z.number(), + relations: z.number(), + unknown_frontiers: z.number(), + query_tool: z.literal("trace_javascript_semantics"), + }), }), }) .parse(analyzed.structuredContent); @@ -256,6 +263,7 @@ describe("Electron MCP tools", () => { }, }); expect(projected.result.graph.node_count).toBeGreaterThan(0); + expect(projected.result.semantic_graph.nodes).toBeGreaterThan(0); expect(JSON.stringify(analyzed.structuredContent)).not.toContain( '"nodes":[', ); diff --git a/tests/javascriptArtifactReconstruction.test.ts b/tests/javascriptArtifactReconstruction.test.ts index 9a44a320a..777154702 100644 --- a/tests/javascriptArtifactReconstruction.test.ts +++ b/tests/javascriptArtifactReconstruction.test.ts @@ -19,6 +19,9 @@ import { type ArtifactReader, } from "../src/artifacts/ArtifactReader.js"; import { parseJavaScriptApplicationGraph } from "../src/domain/javascriptApplicationGraph.js"; +import { javaScriptApplicationAnalysisResultV2Schema } from "../src/domain/javascriptApplicationAnalysis.js"; +import { createJavaScriptSemanticGraph } from "../src/domain/javascriptSemanticGraph.js"; +import { parseJavaScriptSemanticGraph } from "../src/domain/javascriptSemanticGraphSerialization.js"; import { writeJavaScriptArtifactFixture } from "./fixtures/javascriptArtifactApplication.js"; describe("JavaScript artifact reconstruction", () => { @@ -31,6 +34,7 @@ describe("JavaScript artifact reconstruction", () => { source_map_read_approved: true, }); const graph = parseJavaScriptApplicationGraph(result.graph); + const semanticGraph = parseJavaScriptSemanticGraph(result.semantic_graph); expect(Reflect.get(globalThis, "__rea_bundle_executed")).toBeUndefined(); expect(result.input_path).toBe(root); @@ -106,6 +110,27 @@ describe("JavaScript artifact reconstruction", () => { "persists_to", ]), ); + expect(semanticGraph.application_graph_id).toBe(graph.graph_id); + const { graph_id: _semanticGraphId, ...semanticGraphInput } = semanticGraph; + const mismatchedSemanticGraph = createJavaScriptSemanticGraph({ + ...semanticGraphInput, + application_graph_id: `jag_${"f".repeat(64)}`, + }); + const { electron_summary: summary, ...analysisResult } = result; + expect(() => + javaScriptApplicationAnalysisResultV2Schema.parse({ + ...analysisResult, + schema_version: 2, + summary, + semantic_graph: mismatchedSemanticGraph, + }), + ).toThrow(/must commit the containing application graph/u); + expect( + semanticGraph.nodes.some( + ({ application_node_ids: identifiers }) => identifiers.length > 0, + ), + ).toBe(true); + expect(semanticGraph.relations.length).toBeGreaterThan(0); const mainRole = graph.nodes.find( (node) => node.kind === "electron-main" && @@ -134,6 +159,29 @@ describe("JavaScript artifact reconstruction", () => { resolution_status: "resolved", limitations: [], }); + const mainAsset = graph.nodes.find( + (node) => + node.kind === "javascript-asset" && + node.observations.some( + ({ evidence }) => + evidence.location.available && + evidence.location.value.kind === "artifact-path" && + evidence.location.value.path === "main.js", + ), + ); + if (mainAsset === undefined) + throw new Error("Expected main JavaScript asset"); + const mainSemanticNodes = semanticGraph.nodes.filter( + ({ identity }) => identity.module_path === "main.js", + ); + const linkedMainSemanticNodes = mainSemanticNodes.filter( + ({ application_node_ids: identifiers }) => + identifiers.includes(mainAsset.node_id), + ); + expect(linkedMainSemanticNodes.length).toBeGreaterThan(0); + expect(linkedMainSemanticNodes.length).toBeLessThan( + mainSemanticNodes.length, + ); for (const role of [mainRole, preloadRole]) { expect(role).toBeDefined(); expect( diff --git a/tests/javascriptSemanticGraphBuilder.test.ts b/tests/javascriptSemanticGraphBuilder.test.ts new file mode 100644 index 000000000..735cd67ca --- /dev/null +++ b/tests/javascriptSemanticGraphBuilder.test.ts @@ -0,0 +1,314 @@ +import { describe, expect, it } from "vitest"; + +import { buildJavaScriptSemanticGraph } from "../src/application/JavaScriptSemanticGraphBuilder.js"; +import type { JavaScriptArtifactAnalysis } from "../src/application/JavaScriptArtifactAnalysisTypes.js"; +import type { JavaScriptArtifactFile } from "../src/application/JavaScriptArtifactFiles.js"; +import { queryJavaScriptSemanticGraph } from "../src/domain/javascriptSemanticQuery.js"; +import { analyzeJavaScriptSemantics } from "../src/domain/javascriptSemanticAnalysis.js"; +import { + DEFAULT_JAVASCRIPT_SEMANTIC_LIMITS, + type JavaScriptSemanticLimits, +} from "../src/domain/javascriptSemanticIr.js"; + +const SHA256 = "a".repeat(64); +const GRAPH_ID = `jag_${"b".repeat(64)}`; + +describe("JavaScript semantic companion graph", () => { + it("projects closure and direct interprocedural flow without execution", () => { + const graph = graphFor(` + const outer = 2; + function add(value) { return value + outer; } + const input = 40; + const answer = add(input); + `); + + const relations = new Set(graph.relations.map(({ relation }) => relation)); + for (const expected of [ + "argument-to-parameter", + "calls", + "captures", + "defines", + "reads", + "returns-to-call", + ] as const) + expect(relations.has(expected)).toBe(true); + expect(graph.coverage.families).toHaveLength(12); + expect(graph.limitations).toContain( + "The semantic graph contains static syntax observations and conservative relationship candidates; it does not claim runtime execution.", + ); + const parameter = graph.nodes.find( + ({ kind, label }) => kind === "parameter" && label === "value", + ); + if (parameter === undefined) throw new Error("Expected parameter node"); + const provenance = queryJavaScriptSemanticGraph(graph, { + seed: { kind: "semantic-node", node_id: parameter.node_id }, + direction: "backward-provenance", + include_ambiguous_dynamic_edges: true, + }); + expect(provenance.relations.map(({ relation }) => relation)).toEqual( + expect.arrayContaining([ + "aliases", + "argument-to-parameter", + "defines", + "reads", + ]), + ); + }); + + it("keeps dynamic calls explicit and returns bounded deterministic pages", () => { + const graph = graphFor(` + const handlers = { ready() { return 1; } }; + const key = process.argv[2]; + handlers[key](); + `); + expect(graph.unknowns.some(({ reason }) => reason === "dynamic-call")).toBe( + true, + ); + const seedNodeId = graph.relations.find( + ({ relation }) => relation === "defines", + )?.source_node_id; + expect(seedNodeId).toBeDefined(); + if (seedNodeId === undefined) throw new Error("Expected a definition seed"); + + const first = queryJavaScriptSemanticGraph(graph, { + seed: { kind: "semantic-node", node_id: seedNodeId }, + direction: "forward-influence", + include_ambiguous_dynamic_edges: true, + limits: { + max_seed_matches: 25, + max_nodes: 2_000, + max_edges: 4_000, + max_depth: 12, + max_functions: 500, + max_modules: 500, + page_size: 1, + }, + }); + const repeated = queryJavaScriptSemanticGraph(graph, { + seed: { kind: "semantic-node", node_id: seedNodeId }, + direction: "forward-influence", + include_ambiguous_dynamic_edges: true, + limits: first.applied_limits, + }); + expect(repeated).toEqual(first); + }); + + it("keeps ambiguous interprocedural flow out of the default traversal", () => { + const graph = graphFor(` + function left(value) { return value; } + function right(value) { return value; } + const input = 1; + const selected = Math.random() ? left : right; + selected(input); + `); + const input = graph.nodes.find( + ({ kind, label }) => kind === "binding" && label === "input", + ); + if (input === undefined) throw new Error("Expected input binding"); + const query = { + seed: { kind: "semantic-node" as const, node_id: input.node_id }, + direction: "forward-influence" as const, + allowed_relations: [ + "reads" as const, + "aliases" as const, + "argument-to-parameter" as const, + ], + }; + const strict = queryJavaScriptSemanticGraph(graph, query); + expect( + strict.relations.some( + ({ relation }) => relation === "argument-to-parameter", + ), + ).toBe(false); + expect(strict.status).toBe("ambiguous"); + const admitted = queryJavaScriptSemanticGraph(graph, { + ...query, + include_ambiguous_dynamic_edges: true, + }); + expect( + admitted.relations.filter( + ({ relation }) => relation === "argument-to-parameter", + ), + ).toHaveLength(2); + expect(admitted.status).toBe("ambiguous"); + }); + + it("connects caller results, direct returns, parameters, and captures", () => { + const graph = graphFor(` + const outer = 2; + function identity(value) { return value; } + function readOuter() { return outer; } + const input = 40; + const output = identity(input); + `); + const binding = (label: string) => + graph.nodes.find( + ({ kind, label: nodeLabel }) => + ["binding", "parameter"].includes(kind) && nodeLabel === label, + ); + const output = binding("output"); + const input = binding("input"); + const outer = binding("outer"); + const readOuter = graph.nodes.find( + ({ kind, label }) => kind === "function" && label === "readOuter", + ); + if ( + output === undefined || + input === undefined || + outer === undefined || + readOuter === undefined + ) + throw new Error("Expected semantic flow nodes"); + const provenance = queryJavaScriptSemanticGraph(graph, { + seed: { kind: "semantic-node", node_id: output.node_id }, + direction: "backward-provenance", + }); + expect(provenance.nodes.map(({ node_id }) => node_id)).toContain( + input.node_id, + ); + const influence = queryJavaScriptSemanticGraph(graph, { + seed: { kind: "semantic-node", node_id: outer.node_id }, + direction: "forward-influence", + allowed_relations: ["captures"], + }); + expect(influence.nodes.map(({ node_id }) => node_id)).toContain( + readOuter.node_id, + ); + const capture = graph.relations.find( + ({ relation }) => relation === "captures", + ); + expect(capture?.evidence.location).toMatchObject({ + available: true, + value: { kind: "source-range", source: "app.js" }, + }); + expect(capture?.evidence.location).not.toEqual(readOuter.evidence.location); + }); + + it("preserves semantic analyzer truncation in graph coverage", () => { + const graph = graphFor( + "const first = 1; const second = first; const third = second;", + { maxReferences: 1 }, + ); + expect(graph.coverage).toMatchObject({ + status: "partial", + truncated: true, + omitted_nodes: null, + omitted_relations: null, + limits: expect.arrayContaining([ + { name: "semantic.maxReferences", value: 1, unit: "items" }, + ]), + }); + expect( + graph.coverage.families.find( + ({ family }) => family === "promise-ownership", + ), + ).toMatchObject({ status: "unsupported", retained_relations: 0 }); + }); + + it("does not promote transformed return expressions to resolved aliases", () => { + const graph = graphFor(` + function increment(value) { return value + 1; } + const input = 1; + const output = increment(input); + `); + const returnNode = graph.nodes.find(({ kind }) => kind === "return-site"); + if (returnNode === undefined) throw new Error("Expected return site"); + expect( + graph.relations.filter( + ({ relation, target_node_id }) => + relation === "aliases" && target_node_id === returnNode.node_id, + ), + ).toEqual([expect.objectContaining({ resolution: "candidate" })]); + const output = graph.nodes.find( + ({ kind, label }) => kind === "binding" && label === "output", + ); + const input = graph.nodes.find( + ({ kind, label }) => kind === "binding" && label === "input", + ); + if (output === undefined || input === undefined) + throw new Error("Expected caller bindings"); + const strict = queryJavaScriptSemanticGraph(graph, { + seed: { kind: "semantic-node", node_id: output.node_id }, + direction: "backward-provenance", + }); + expect(strict.nodes.map(({ node_id }) => node_id)).not.toContain( + input.node_id, + ); + expect(strict.status).toBe("ambiguous"); + }); + + it("uses unavailable evidence when no source produced semantic IR", () => { + const graph = buildJavaScriptSemanticGraph({ + rootArtifactSha256: SHA256, + applicationGraph: { graph_id: GRAPH_ID, nodes: [] }, + analysis: emptyAnalysis(), + }); + expect(graph.nodes).toHaveLength(1); + expect(graph.nodes[0]?.evidence).toMatchObject({ + authority: "unknown", + state: "unavailable", + location: { available: false, reason: "not-observed" }, + }); + }); +}); + +const emptyAnalysis = (): JavaScriptArtifactAnalysis => ({ + files: [], + packages: [], + json_modules: [], + html_scripts: [], + source_maps: [], + visited_ast_nodes: 0, + findings: 0, + modules: 0, + parse_failures: 0, + truncated_scopes: 0, + limitations: [], +}); + +const graphFor = ( + source: string, + inputLimits: Partial = {}, +) => { + const file: JavaScriptArtifactFile = { + path: "app.js", + container_sha256: SHA256, + sha256: SHA256, + bytes: Buffer.byteLength(source), + inventory_artifact_id: `art_${SHA256}`, + kind: "javascript", + unpacked: false, + text: { included: true, value: source }, + }; + const semanticLimits = { + ...DEFAULT_JAVASCRIPT_SEMANTIC_LIMITS, + ...inputLimits, + }; + const analysis: JavaScriptArtifactAnalysis = { + files: [ + { + file, + javascript: null, + semantic: { + ir: analyzeJavaScriptSemantics(source, inputLimits), + limits: semanticLimits, + }, + }, + ], + packages: [], + json_modules: [], + html_scripts: [], + source_maps: [], + visited_ast_nodes: 0, + findings: 0, + modules: 0, + parse_failures: 0, + truncated_scopes: 0, + limitations: [], + }; + return buildJavaScriptSemanticGraph({ + rootArtifactSha256: SHA256, + applicationGraph: { graph_id: GRAPH_ID, nodes: [] }, + analysis, + }); +}; diff --git a/tests/javascriptSemanticGraphQuery.test.ts b/tests/javascriptSemanticGraphQuery.test.ts index 5468e306b..8fbfbfd3f 100644 --- a/tests/javascriptSemanticGraphQuery.test.ts +++ b/tests/javascriptSemanticGraphQuery.test.ts @@ -12,11 +12,13 @@ import { createJavaScriptSemanticGraphNode, createJavaScriptSemanticGraphRelation, createJavaScriptSemanticGraphUnknown, - parseJavaScriptSemanticGraph, - serializeJavaScriptSemanticGraph, type JavaScriptSemanticGraph, type JavaScriptSemanticGraphNode, } from "../src/domain/javascriptSemanticGraph.js"; +import { + parseJavaScriptSemanticGraph, + serializeJavaScriptSemanticGraph, +} from "../src/domain/javascriptSemanticGraphSerialization.js"; import { queryJavaScriptSemanticGraph } from "../src/domain/javascriptSemanticQuery.js"; import { javaScriptSemanticQueryInputSchema } from "../src/domain/javascriptSemanticQuerySchemas.js"; @@ -301,10 +303,45 @@ describe("JavaScript semantic relation graph", () => { coverage: { ...graph.coverage, omitted_nodes: 1 }, }), ).toThrow(/Complete graph coverage/u); + const graphWithUnknown = fixtureGraph(true); + const { graph_id: _unknownGraphId, ...unknownGraphInput } = + graphWithUnknown; + const originalUnknown = graphWithUnknown.unknowns[0]; + if (originalUnknown === undefined) + throw new Error("Expected unknown frontier"); + const { unknown_id: _unknownId, ...unknownInput } = originalUnknown; + expect(() => + createJavaScriptSemanticGraph({ + ...unknownGraphInput, + unknowns: [ + createJavaScriptSemanticGraphUnknown({ + ...unknownInput, + candidate_node_ids: [`jsrg_node_${"f".repeat(64)}`], + }), + ], + }), + ).toThrow(/candidate node is absent/u); }); }); describe("JavaScript semantic query", () => { + it("bounds relevant unknown frontiers independently from relation pages", () => { + const graph = fixtureGraph(true); + const callable = graph.nodes.find(({ kind }) => kind === "function"); + if (callable === undefined) throw new Error("Expected callable node"); + const result = queryJavaScriptSemanticGraph(graph, { + seed: { kind: "semantic-node", node_id: callable.node_id }, + direction: "forward-influence", + limits: { max_unknowns: 0 }, + }); + expect(result).toMatchObject({ + status: "truncated", + unknowns: [], + summary: { relevant_unknowns: 1, retained_unknowns: 0 }, + coverage: { status: "truncated" }, + }); + }); + it("traces deterministic forward influence and pages without changing identity", () => { const graph = fixtureGraph(); const input = { @@ -349,7 +386,7 @@ describe("JavaScript semantic query", () => { seed: { kind: "literal", value: "TOKEN" }, direction: "forward-influence", }); - expect(result.status).toBe("partial"); + expect(result.status).toBe("found"); expect(result.coverage.status).toBe("partial"); expect(result.unknowns).toMatchObject([ { reason: "dynamic-call", relation_kinds: ["calls"] }, diff --git a/tests/productCatalog.test.ts b/tests/productCatalog.test.ts index 6119cf2cf..0674e1712 100644 --- a/tests/productCatalog.test.ts +++ b/tests/productCatalog.test.ts @@ -120,6 +120,7 @@ describe("canonical product catalog", () => { "query_reconstruction_coverage", "run_controlled_replay", "trace_application_feature", + "trace_javascript_semantics", ]); expect( catalog.providers.find(({ id }) => id === MANAGED_WORKFLOW_PROVIDER.id) diff --git a/tests/serverIdentity.test.ts b/tests/serverIdentity.test.ts index 9b788fecb..40808d595 100644 --- a/tests/serverIdentity.test.ts +++ b/tests/serverIdentity.test.ts @@ -26,10 +26,10 @@ describe("server and catalog identity", () => { }); expect(PRODUCT_IDENTITY.packageVersion).toBe(packageJson.version); expect(SDK_IDENTITY.server).toBe("2.0.0-beta.4"); - expect(CLI_COMMAND_NAMES).toHaveLength(57); - expect(new Set(CLI_COMMAND_NAMES).size).toBe(57); + expect(CLI_COMMAND_NAMES).toHaveLength(58); + expect(new Set(CLI_COMMAND_NAMES).size).toBe(58); expect(CATALOG_IDENTITY.counts).toEqual({ - cli_commands: 57, + cli_commands: 58, mcp_tools: TOOL_CONTRACTS.length, mcp_prompts: 6, mcp_resources: 2, diff --git a/tests/toolContracts.test.ts b/tests/toolContracts.test.ts index 0eb701f72..9ce8bb6b9 100644 --- a/tests/toolContracts.test.ts +++ b/tests/toolContracts.test.ts @@ -50,6 +50,7 @@ describe("tool contract inventory", () => { ]); expect(APPLICATION_TOOL_CONTRACTS.map(({ name }) => name)).toEqual([ "trace_application_feature", + "trace_javascript_semantics", "compare_application_versions", "compare_javascript_export_shapes", "run_controlled_replay",