diff --git a/src/application/ApplicationWorkflowEvidenceResolver.ts b/src/application/ApplicationWorkflowEvidenceResolver.ts index f0231fbd6..37f16fbd8 100644 --- a/src/application/ApplicationWorkflowEvidenceResolver.ts +++ b/src/application/ApplicationWorkflowEvidenceResolver.ts @@ -3,10 +3,12 @@ import { z } from "zod"; import { compareApplicationVersionsRequestSchema, compareJavaScriptExportShapesRequestSchema, + compareSourceToBundleRequestSchema, traceApplicationFeatureRequestSchema, traceJavaScriptSemanticsRequestSchema, type CompareApplicationVersionsRequest, type CompareJavaScriptExportShapesRequest, + type CompareSourceToBundleRequest, type TraceApplicationFeatureRequest, type TraceJavaScriptSemanticsRequest, } from "../contracts/applicationWorkflowInputContracts.js"; @@ -16,6 +18,7 @@ import { compareApplicationVersionsInputSchema } from "../domain/javascriptAppli import { compareJavaScriptExportShapesInputSchema } from "../domain/javascriptExportShapeComparisonSchemas.js"; import { traceApplicationFeatureInputSchema } from "../domain/javascriptFeatureTraceSchemas.js"; import { traceJavaScriptSemanticsInputSchema } from "../domain/javascriptSemanticTraceSchemas.js"; +import { compareSourceToBundleInputSchema } from "../domain/sourceToBundleComparisonSchemas.js"; import { err, ok, type Result } from "../domain/result.js"; import { resolveEvidenceReferences, @@ -48,6 +51,7 @@ type ComparisonInput = z.output; type ExportShapeComparisonInput = z.output< typeof compareJavaScriptExportShapesInputSchema >; +type SourceToBundleInput = z.output; /** Parse and resolve one trace adapter request into its canonical domain input. */ export const resolveTraceApplicationFeatureRequest = ( @@ -180,6 +184,42 @@ export const resolveCompareApplicationVersionsRequestValidated = ( : invalid("compare_application_versions", parsed.error, raw); }; +/** Parse and resolve one historical-source comparison adapter request. */ +export const resolveCompareSourceToBundleRequest = ( + input: unknown, + lookup?: EvidenceLookup, +): Result => { + const parsed = compareSourceToBundleRequestSchema.safeParse(input); + return parsed.success + ? resolveCompareSourceToBundleRequestValidated(parsed.data, lookup) + : invalid("compare_source_to_bundle", parsed.error, input); +}; + +/** Resolve a historical-source comparison request parsed by its public contract. */ +export const resolveCompareSourceToBundleRequestValidated = ( + input: CompareSourceToBundleRequest, + lookup?: EvidenceLookup, +): Result => { + const application = graphEvidence( + input.application, + input.application_evidence_id, + lookup, + ); + if (!application.ok) return application; + const raw = { + reference: input.reference, + application: application.value, + limits: input.limits, + ...(input.unknown_registry_approved === undefined + ? {} + : { unknown_registry_approved: input.unknown_registry_approved }), + }; + const parsed = compareSourceToBundleInputSchema.safeParse(raw); + return parsed.success + ? ok(parsed.data) + : invalid("compare_source_to_bundle", parsed.error, raw); +}; + /** Parse and resolve an export-shape adapter request into canonical input. */ export const resolveCompareJavaScriptExportShapesRequest = ( input: unknown, @@ -221,7 +261,8 @@ export const resolveCompareJavaScriptExportShapesRequestValidated = ( const graphEvidence = ( evidence: | CompareApplicationVersionsRequest["left"] - | CompareJavaScriptExportShapesRequest["left"], + | CompareJavaScriptExportShapesRequest["left"] + | CompareSourceToBundleRequest["application"], evidenceId: string | undefined, lookup: EvidenceLookup | undefined, ) => { diff --git a/src/application/JavaScriptApplicationWorkflowEvidence.ts b/src/application/JavaScriptApplicationWorkflowEvidence.ts index 1101b5b6a..a87b0a0f0 100644 --- a/src/application/JavaScriptApplicationWorkflowEvidence.ts +++ b/src/application/JavaScriptApplicationWorkflowEvidence.ts @@ -8,6 +8,7 @@ import type { JavaScriptExportShapeComparisonResult } from "../domain/javascript import type { ApplicationFeatureTraceResult } from "../domain/javascriptFeatureTraceSchemas.js"; import type { JavaScriptSemanticTraceResult } from "../domain/javascriptSemanticTraceSchemas.js"; import { jsonValueSchema } from "../domain/jsonValue.js"; +import type { SourceToBundleComparisonResult } from "../domain/sourceToBundleComparisonSchemas.js"; import { JAVASCRIPT_APPLICATION_WORKFLOW_PROVIDER } from "./InvestigationProviders.js"; /** Create derived Evidence for one bounded application-graph feature trace. */ @@ -64,6 +65,24 @@ export const createApplicationVersionComparisonEvidence = ( evidenceLinks: result.evidence_links, }); +/** Create derived Evidence for one historical-source to bundle comparison. */ +export const createSourceToBundleComparisonEvidence = ( + parameters: EvidenceObservation["parameters"], + result: SourceToBundleComparisonResult, +): Evidence => + createEvidence(undefined, JAVASCRIPT_APPLICATION_WORKFLOW_PROVIDER, { + predicateType: "rea.source-to-bundle-comparison/v1", + operation: "compare_source_to_bundle", + parameters, + result: jsonValueSchema.parse(result), + rawResult: null, + confidence: "inferred", + authority: "analyst-inference", + environment: null, + limitations: result.limitations, + evidenceLinks: result.evidence_links, + }); + /** Create derived Evidence for one bounded static export-shape comparison. */ export const createJavaScriptExportShapeComparisonEvidence = ( parameters: EvidenceObservation["parameters"], diff --git a/src/application/JavaScriptApplicationWorkflowService.ts b/src/application/JavaScriptApplicationWorkflowService.ts index dceb1f7e6..29f23173f 100644 --- a/src/application/JavaScriptApplicationWorkflowService.ts +++ b/src/application/JavaScriptApplicationWorkflowService.ts @@ -14,6 +14,8 @@ import { compareJavaScriptExportShapesInputSchema } from "../domain/javascriptEx import { traceApplicationFeature } from "../domain/javascriptFeatureTrace.js"; import { traceApplicationFeatureInputSchema } from "../domain/javascriptFeatureTraceSchemas.js"; import { err, ok, type Result } from "../domain/result.js"; +import { compareSourceToBundle } from "../domain/sourceToBundleComparison.js"; +import { compareSourceToBundleInputSchema } from "../domain/sourceToBundleComparisonSchemas.js"; import { parseApplicationGraphEvidence, parseNativeApplicationEvidence, @@ -22,6 +24,7 @@ import { createApplicationFeatureTraceEvidence, createApplicationVersionComparisonEvidence, createJavaScriptExportShapeComparisonEvidence, + createSourceToBundleComparisonEvidence, } from "./JavaScriptApplicationWorkflowEvidence.js"; /** Authenticate Evidence and derive one bounded cross-layer feature trace. */ @@ -141,6 +144,37 @@ export const compareApplicationVersionsEvidenceValidated = ( } }; +/** Compare source and bundle inputs already parsed by a trusted adapter. */ +export const compareSourceToBundleEvidenceValidated = ( + input: z.output, +): Result => { + const operation = "compare_source_to_bundle"; + try { + const application = parseApplicationGraphEvidence(input.application); + const result = compareSourceToBundle({ + reference: input.reference, + application: { + evidenceId: application.evidence.evidence_id, + rootArtifactSha256: application.rootArtifactSha256, + graph: application.graph, + }, + limits: input.limits, + }); + return ok( + createSourceToBundleComparisonEvidence( + { + reference_root_sha256: input.reference.root_sha256, + application_evidence_id: application.evidence.evidence_id, + limits: input.limits, + }, + result, + ), + ); + } catch (cause: unknown) { + return workflowFailure(operation, cause); + } +}; + /** * Authenticate both graphs and compare one exact export return shape. * @public diff --git a/src/contracts/applicationWorkflowInputContracts.ts b/src/contracts/applicationWorkflowInputContracts.ts index 00457de46..3523eeef1 100644 --- a/src/contracts/applicationWorkflowInputContracts.ts +++ b/src/contracts/applicationWorkflowInputContracts.ts @@ -5,6 +5,7 @@ import { compareApplicationVersionsInputSchema } from "../domain/javascriptAppli import { compareJavaScriptExportShapesInputSchema } from "../domain/javascriptExportShapeComparisonSchemas.js"; import { traceApplicationFeatureInputSchema } from "../domain/javascriptFeatureTraceSchemas.js"; import { javaScriptSemanticQueryInputSchema } from "../domain/javascriptSemanticQuerySchemas.js"; +import { compareSourceToBundleInputSchema } from "../domain/sourceToBundleComparisonSchemas.js"; const evidenceIdSchema = z .string() @@ -102,6 +103,25 @@ export const compareApplicationVersionsRequestSchema = z ]); }); +/** Historical-source comparison accepting full application Evidence or a ledger reference. */ +export const compareSourceToBundleRequestSchema = z + .strictObject({ + reference: compareSourceToBundleInputSchema.shape.reference, + application: evidenceSchema.optional(), + application_evidence_id: evidenceIdSchema.optional(), + limits: compareSourceToBundleInputSchema.shape.limits, + unknown_registry_approved: + compareSourceToBundleInputSchema.shape.unknown_registry_approved, + }) + .superRefine((input, context) => { + requireExactlyOne(context, [ + "application", + input.application, + "application_evidence_id", + input.application_evidence_id, + ]); + }); + /** MCP/CLI export-shape request accepting full Evidence or ledger references. */ export const compareJavaScriptExportShapesRequestSchema = z .strictObject({ @@ -145,6 +165,9 @@ export type TraceJavaScriptSemanticsRequest = z.output< export type CompareApplicationVersionsRequest = z.output< typeof compareApplicationVersionsRequestSchema >; +export type CompareSourceToBundleRequest = z.output< + typeof compareSourceToBundleRequestSchema +>; export type CompareJavaScriptExportShapesRequest = z.output< typeof compareJavaScriptExportShapesRequestSchema >; diff --git a/src/contracts/javascriptApplicationWorkflowExamples.ts b/src/contracts/javascriptApplicationWorkflowExamples.ts index 46872826f..bca29c1b1 100644 --- a/src/contracts/javascriptApplicationWorkflowExamples.ts +++ b/src/contracts/javascriptApplicationWorkflowExamples.ts @@ -4,6 +4,7 @@ import { JAVASCRIPT_APPLICATION_EVIDENCE_EXAMPLE, JAVASCRIPT_RUNTIME_RECONCILIATION_EXAMPLE, } from "./javascriptRuntimeReconciliationExample.js"; +import { createHistoricalSourceGraph } from "../domain/referenceSourceGraph.js"; const reconciliation = reconcileJavaScriptRuntime( JAVASCRIPT_RUNTIME_RECONCILIATION_EXAMPLE, @@ -66,6 +67,48 @@ export const JAVASCRIPT_APPLICATION_VERSION_COMPARISON_EXAMPLE = { right_evidence_id: SECOND_EXAMPLE_EVIDENCE_ID, }; +/** Historical source inventory compared with one producer-returned graph ID. */ +export const SOURCE_TO_BUNDLE_COMPARISON_EXAMPLE = { + reference: createHistoricalSourceGraph({ + schema: "HistoricalSourceGraph/v1", + authority: "historical-reference", + root_alias: "$REFERENCE_ROOT", + inventory_state: "complete", + entries: [ + { + path: "src", + kind: "directory", + classifications: ["source"], + tree_state: "enumerated", + limitations: [], + }, + { + path: "src/main.ts", + kind: "file", + sha256: "2".repeat(64), + size: 128, + language: "TypeScript", + classifications: ["source"], + content_state: "hashed", + limitations: [], + }, + ], + relationships: [], + parse_failures: [], + exclusions: [], + languages: ["TypeScript"], + manifests: [], + vcs: { kind: "none", head: null, dirty: null }, + provenance: { + importer: "rea", + importer_version: "1", + caller: "contract-example", + }, + limitations: [], + }), + application_evidence_id: EXAMPLE_EVIDENCE_ID, +}; + /** Exact static return-shape comparison using producer-returned Evidence IDs. */ export const JAVASCRIPT_EXPORT_SHAPE_COMPARISON_EXAMPLE = { left_evidence_id: EXAMPLE_EVIDENCE_ID, diff --git a/src/domain/javascriptStaticAnalysis.ts b/src/domain/javascriptStaticAnalysis.ts index 2cbc0b334..1112bfc20 100644 --- a/src/domain/javascriptStaticAnalysis.ts +++ b/src/domain/javascriptStaticAnalysis.ts @@ -16,7 +16,10 @@ import { inspectRouteProperty, inspectRoleProperty, } from "./javascriptStaticAnalysisCalls.js"; -import { inspectBundlerRegistration } from "./javascriptStaticAnalysisBundler.js"; +import { + inspectBundlerRegistration, + inspectEsbuildWrapper, +} from "./javascriptStaticAnalysisBundler.js"; import { finalizeLocatedFindings } from "./javascriptStaticAnalysisFindings.js"; import { createJavaScriptAnalysisAccumulator, @@ -167,8 +170,9 @@ const inspectNode = ( inspectElectronStaticNode(node, findings); if (t.isCallExpression(node)) { inspectBundlerRegistration(source, node, accumulator, limits); - inspectCall(node, findings); - } else if (t.isNewExpression(node)) inspectCall(node, findings); + inspectEsbuildWrapper(source, node, accumulator, limits); + inspectCall(source, node, findings); + } else if (t.isNewExpression(node)) inspectCall(source, node, findings); if ( (t.isImportDeclaration(node) || t.isExportAllDeclaration(node)) && node.source !== undefined diff --git a/src/domain/javascriptStaticAnalysisBundler.ts b/src/domain/javascriptStaticAnalysisBundler.ts index 6852b4a0c..1316e6756 100644 --- a/src/domain/javascriptStaticAnalysisBundler.ts +++ b/src/domain/javascriptStaticAnalysisBundler.ts @@ -22,6 +22,7 @@ import { import type { JavaScriptAnalysisAccumulator as AnalysisAccumulator } from "./javascriptStaticAnalysisState.js"; import type { JavaScriptBundlerModule, + JavaScriptBundlerRegistration, JavaScriptStaticAnalysisLimits, } from "./javascriptStaticAnalysisTypes.js"; @@ -105,6 +106,56 @@ export const inspectBundlerRegistration = ( ); }; +/** Inspect esbuild helper wrappers and recover their literal module tables. */ +export const inspectEsbuildWrapper = ( + source: string, + call: t.CallExpression, + accumulator: AnalysisAccumulator, + limits: JavaScriptStaticAnalysisLimits, +): void => { + const name = calleeName(call.callee); + const wrapperKind = + name === "__commonJS" || name.endsWith(".__commonJS") + ? "commonjs" + : name === "__esm" || name.endsWith(".__esm") + ? "esm" + : null; + const table = call.arguments[0]; + if (wrapperKind === null || !t.isObjectExpression(table)) return; + const recovered = recoverBundlerModules(source, table, accumulator, limits); + if (recovered.modules.length === 0) return; + if (recovered.omittedAsyncChunkKeys > 0) { + accumulator.droppedFindings += recovered.omittedAsyncChunkKeys; + accumulator.structuralTruncation = true; + } + accumulator.unknownFindings += recovered.unknownAsyncChunkKeys; + const registration: JavaScriptBundlerRegistration = { + bundler: "esbuild", + runtime: `esbuild-${wrapperKind}`, + chunk_keys: [`${wrapperKind}@${String(call.start ?? 0)}`], + omitted_chunk_keys: 0, + unknown_chunk_keys: 0, + runtime_require_name: null, + runtime_module_cache_status: "not-observed", + entry_module_keys: [], + omitted_entry_module_keys: 0, + unknown_entry_module_keys: 0, + async_chunk_keys: recovered.asyncChunkKeys, + omitted_async_chunk_keys: recovered.omittedAsyncChunkKeys, + unknown_async_chunk_keys: recovered.unknownAsyncChunkKeys, + modules: recovered.modules.sort((left, right) => + compareCodePoints(left.module_key, right.module_key), + ), + location: range(call), + }; + addBoundedFinding( + accumulator, + `registration\0${registrationKey(registration)}`, + limits.maxFindings, + () => accumulator.registrations.push(registration), + ); +}; + interface RecoveredBundlerModules { readonly modules: JavaScriptBundlerModule[]; readonly asyncChunkKeys: readonly string[]; diff --git a/src/domain/javascriptStaticAnalysisCalls.ts b/src/domain/javascriptStaticAnalysisCalls.ts index 6314eac2a..94843c727 100644 --- a/src/domain/javascriptStaticAnalysisCalls.ts +++ b/src/domain/javascriptStaticAnalysisCalls.ts @@ -30,6 +30,7 @@ import type { /** Inspect one call or new expression for references, endpoints, storage, and roles. */ export const inspectCall = ( + source: string, node: t.CallExpression | t.NewExpression, context: FindingContext, ): void => { @@ -52,6 +53,8 @@ export const inspectCall = ( kind: "dynamic-import", specifier: chunkLoad.specifier, }); + for (const specifier of vitePreloadDependencySpecifiers(source, node, name)) + addReference(context, { node, kind: "dynamic-import", specifier }); if (name === "Worker" || name.endsWith(".Worker")) addReference(context, { node, kind: "worker", specifier: first }); if (name.endsWith("serviceWorker.register")) @@ -118,6 +121,33 @@ const chunkLoadSpecifier = ( return { specifier: prefixedArgument("chunk:", node.arguments[0]) }; }; +const vitePreloadDependencySpecifiers = ( + source: string, + node: t.CallExpression | t.NewExpression, + name: string, +): readonly string[] => { + if (name !== "__vite__mapDeps" || !t.isArrayExpression(node.arguments[0])) + return []; + const indexes = node.arguments[0].elements.flatMap((element) => + t.isNumericLiteral(element) ? [element.value] : [], + ); + if (indexes.length !== node.arguments[0].elements.length) return []; + const table = vitePreloadDependencyTable(source); + return indexes.flatMap((index) => table[index] ?? []); +}; + +const vitePreloadDependencyTable = (source: string): readonly string[] => { + const match = + /\.\s*f\s*\|\|\s*\(\s*[^)]*?\.f\s*=\s*\[([\s\S]*?)\]\s*\)/u.exec(source); + const body = match?.[1]; + if (body === undefined) return []; + return [ + ...body.matchAll(/"([^"\\]*(?:\\.[^"\\]*)*)"|'([^'\\]*(?:\\.[^'\\]*)*)'/gu), + ] + .map((literal) => (literal[1] ?? literal[2] ?? "").slice(0, 4_096)) + .slice(0, 256); +}; + const inspectEndpointCall = ( node: t.CallExpression | t.NewExpression, name: string, diff --git a/src/domain/javascriptStaticAnalysisHelpers.ts b/src/domain/javascriptStaticAnalysisHelpers.ts index a3e75d459..35371e3d3 100644 --- a/src/domain/javascriptStaticAnalysisHelpers.ts +++ b/src/domain/javascriptStaticAnalysisHelpers.ts @@ -450,6 +450,8 @@ const vendorPatterns = [ { name: "webpack", patterns: ["__webpack_require__", "webpackChunk"] }, { name: "Rspack", patterns: ["__webpack_require__.f", "rspackChunk"] }, { name: "Vite", patterns: ["__vite__", "import.meta.hot"] }, + { name: "Rollup", patterns: ["ROLLUP_FILE_URL", "import.meta.ROLLUP"] }, + { name: "esbuild", patterns: ["__commonJS", "__esm"] }, { name: "React", patterns: ["React.createElement", "react/jsx-runtime"] }, { name: "Vue", patterns: ["__VUE__", "createApp("] }, { name: "Next.js", patterns: ["__NEXT_DATA__", "/_next/"] }, diff --git a/src/domain/javascriptStaticAnalysisTypes.ts b/src/domain/javascriptStaticAnalysisTypes.ts index 819bce044..4f3902ce3 100644 --- a/src/domain/javascriptStaticAnalysisTypes.ts +++ b/src/domain/javascriptStaticAnalysisTypes.ts @@ -67,9 +67,9 @@ export interface JavaScriptBundlerModule { readonly location: JavaScriptSourceRange; } -/** One statically recognized Webpack/Rspack chunk registration. */ +/** One statically recognized bundler chunk or module registration. */ export interface JavaScriptBundlerRegistration { - readonly bundler: "webpack" | "rspack"; + readonly bundler: "webpack" | "rspack" | "esbuild"; readonly runtime: string; readonly chunk_keys: readonly string[]; readonly omitted_chunk_keys: number; diff --git a/src/domain/sourceToBundleComparison.ts b/src/domain/sourceToBundleComparison.ts new file mode 100644 index 000000000..9af88f656 --- /dev/null +++ b/src/domain/sourceToBundleComparison.ts @@ -0,0 +1,482 @@ +import { createHash } from "node:crypto"; + +import canonicalize from "canonicalize"; + +import type { JavaScriptApplicationGraph } from "./javascriptApplicationGraph.js"; +import type { HistoricalSourceGraph } from "./referenceSourceGraph.js"; +import { + SOURCE_TO_BUNDLE_SIGNAL_WEIGHTS, + sourceToBundleComparisonResultSchema, + type CompareSourceToBundleInput, + type SourceToBundleCandidate, + type SourceToBundleComparisonItem, + type SourceToBundleComparisonResult, +} from "./sourceToBundleComparisonSchemas.js"; +import { + buildSourceToBundleCandidateIndex, + candidateIdsForSource, + historicalSourceFiles, + scoreSourceToBundleCandidate, + sourceBearingNodes, + type SourceToBundleCandidateIndex, +} from "./sourceToBundleSignals.js"; + +interface SourceToBundleComparisonInput { + readonly reference: HistoricalSourceGraph; + readonly application: { + readonly evidenceId: string; + readonly rootArtifactSha256: string; + readonly graph: JavaScriptApplicationGraph; + }; + readonly limits: CompareSourceToBundleInput["limits"]; +} + +interface ClassifiedMapping { + readonly status: SourceToBundleComparisonItem["status"]; + readonly confidence: SourceToBundleComparisonItem["confidence"]; + readonly currentNodeIds: string[]; + readonly limitations: string[]; +} + +interface ComparisonProjection { + readonly sources: ReturnType; + readonly currentNodes: ReturnType; + readonly index: SourceToBundleCandidateIndex; + readonly absenceComplete: boolean; + readonly factors: CoverageFactors; +} + +interface MappingProjection { + readonly items: SourceToBundleComparisonItem[]; + readonly candidateEvaluations: number; + readonly omittedCandidateEvaluations: number; + readonly omittedCandidateReferences: number; +} + +/** Compare historical source files to static source-bearing application nodes. */ +export const compareSourceToBundle = ( + input: SourceToBundleComparisonInput, +): SourceToBundleComparisonResult => { + const projection = projectComparison(input); + const mappings = compareMappings(input, projection); + const factors = { + ...projection.factors, + omittedCandidateEvaluations: mappings.omittedCandidateEvaluations, + omittedCandidateReferences: mappings.omittedCandidateReferences, + }; + const mappedCurrentIds = new Set( + mappings.items.flatMap(({ current_node_ids: nodeIds }) => nodeIds), + ); + const unmappedCurrentNodeIds = projection.currentNodes + .map(({ node_id: nodeId }) => nodeId) + .filter((nodeId) => !mappedCurrentIds.has(nodeId)); + const semanticResult = { + schema_version: 1 as const, + reference: { + root_sha256: input.reference.root_sha256, + inventory_state: input.reference.inventory_state, + }, + application: { + evidence_id: input.application.evidenceId, + graph_id: input.application.graph.graph_id, + root_artifact_sha256: input.application.rootArtifactSha256, + }, + scoring: scoringModel(), + summary: comparisonSummary(mappings.items), + items: mappings.items, + unmapped_current_node_ids: unmappedCurrentNodeIds, + coverage: comparisonCoverage(projection, mappings, factors), + evidence_links: [input.application.evidenceId], + limitations: comparisonLimitations(factors), + }; + return sourceToBundleComparisonResultSchema.parse({ + ...semanticResult, + comparison_id: `stbc_${digestCanonical(semanticResult)}`, + }); +}; + +const projectComparison = ( + input: SourceToBundleComparisonInput, +): ComparisonProjection => { + const allSources = historicalSourceFiles(input.reference); + const allCurrentNodes = sourceBearingNodes(input.application.graph.nodes); + const sources = allSources.slice(0, input.limits.max_source_files); + const currentNodes = allCurrentNodes.slice( + 0, + input.limits.max_application_nodes, + ); + const omittedSourceFiles = allSources.length - sources.length; + const omittedApplicationNodes = allCurrentNodes.length - currentNodes.length; + const index = buildSourceToBundleCandidateIndex(currentNodes); + return { + sources, + currentNodes, + index, + absenceComplete: + input.reference.inventory_state === "complete" && + input.application.graph.coverage.status === "complete" && + !input.application.graph.coverage.truncated && + omittedSourceFiles === 0 && + omittedApplicationNodes === 0 && + !index.pathIndexTruncated, + factors: { + input, + omittedSourceFiles, + omittedApplicationNodes, + omittedCandidateEvaluations: 0, + omittedCandidateReferences: 0, + pathIndexTruncated: index.pathIndexTruncated, + }, + }; +}; + +const compareMappings = ( + input: SourceToBundleComparisonInput, + projection: ComparisonProjection, +): MappingProjection => { + let evaluationsRemaining = input.limits.max_candidate_evaluations; + let candidateEvaluations = 0; + let omittedCandidateEvaluations = 0; + let omittedCandidateReferences = 0; + const provisionalItems: SourceToBundleComparisonItem[] = []; + for (const source of projection.sources) { + const candidateIds = candidateIdsForSource(source, projection.index); + const retainedIds = candidateIds.slice(0, evaluationsRemaining); + const omittedEvaluations = candidateIds.length - retainedIds.length; + evaluationsRemaining -= retainedIds.length; + candidateEvaluations += retainedIds.length; + omittedCandidateEvaluations += omittedEvaluations; + const scored = retainedIds + .map((nodeId) => + scoreSourceToBundleCandidate(source, nodeId, projection.index), + ) + .filter( + (candidate): candidate is SourceToBundleCandidate => candidate !== null, + ) + .sort(compareCandidates); + const retainedCandidates = scored.slice( + 0, + input.limits.max_candidate_nodes, + ); + const omittedCandidates = + omittedEvaluations + scored.length - retainedCandidates.length; + omittedCandidateReferences += omittedCandidates; + const classification = classifyMapping( + retainedCandidates, + omittedEvaluations === 0, + projection.absenceComplete, + ); + const semantic = { + source_path: source.path, + source_sha256: source.sha256, + source_language: source.language, + status: classification.status, + confidence: classification.confidence, + current_node_ids: classification.currentNodeIds, + candidates: retainedCandidates, + omitted_candidates: omittedCandidates, + limitations: uniqueSorted([ + ...source.limitations, + ...classification.limitations, + ...(omittedCandidates > 0 + ? [ + `${String(omittedCandidates)} candidate references were omitted by comparison limits.`, + ] + : []), + ]), + }; + provisionalItems.push({ + mapping_id: mappingId(input, semantic), + ...semantic, + }); + } + return { + items: markMergedMappings(input, provisionalItems), + candidateEvaluations, + omittedCandidateEvaluations, + omittedCandidateReferences, + }; +}; + +const comparisonCoverage = ( + projection: ComparisonProjection, + mappings: MappingProjection, + factors: CoverageFactors, +): SourceToBundleComparisonResult["coverage"] => ({ + status: coverageStatus(factors), + reference_inventory_state: factors.input.reference.inventory_state, + application_graph_status: factors.input.application.graph.coverage.status, + retained_source_files: projection.sources.length, + omitted_source_files: factors.omittedSourceFiles, + retained_application_nodes: projection.currentNodes.length, + omitted_application_nodes: factors.omittedApplicationNodes, + candidate_evaluations: mappings.candidateEvaluations, + omitted_candidate_evaluations: mappings.omittedCandidateEvaluations, + omitted_candidate_references: mappings.omittedCandidateReferences, + omitted_unmapped_current_nodes: 0, +}); + +const scoringModel = (): SourceToBundleComparisonResult["scoring"] => ({ + algorithm: "rea-source-to-bundle-signals/v1", + minimum_candidate_score: 20, + weights: Object.entries(SOURCE_TO_BUNDLE_SIGNAL_WEIGHTS).map( + ([signal, weight]) => ({ + signal: signal as keyof typeof SOURCE_TO_BUNDLE_SIGNAL_WEIGHTS, + weight, + }), + ), +}); + +const classifyMapping = ( + candidates: readonly SourceToBundleCandidate[], + evaluationComplete: boolean, + absenceComplete: boolean, +): ClassifiedMapping => { + if (!evaluationComplete) + return { + status: "unknown", + confidence: "unknown", + currentNodeIds: [], + limitations: [ + "Candidate evaluation stopped at the declared limit; a stronger match may be omitted.", + ], + }; + const exact = candidates.filter((candidate) => + hasSignal(candidate, "exact-source-digest"), + ); + if (exact.length === 1) + return { + status: "unchanged", + confidence: "exact", + currentNodeIds: [exact[0]?.current_node_id ?? ""].filter(Boolean), + limitations: [ + "Exact byte identity does not establish equivalent load context or runtime behavior.", + ], + }; + if (exact.length > 1) + return { + status: "duplicated", + confidence: "exact", + currentNodeIds: exact.map(({ current_node_id: nodeId }) => nodeId), + limitations: [ + "The same historical source digest occurs in multiple current graph nodes.", + ], + }; + const pathCandidates = candidates.filter((candidate) => + candidate.signals.some(({ kind }) => + [ + "source-map-original-path", + "current-path-exact", + "current-path-suffix", + ].includes(kind), + ), + ); + const topScore = pathCandidates[0]?.score; + const topPathCandidates = + topScore === undefined + ? [] + : pathCandidates.filter(({ score }) => score === topScore); + if (topPathCandidates.length === 1) { + const candidate = topPathCandidates[0]; + return { + status: "modified", + confidence: candidate?.confidence === "high" ? "high" : "medium", + currentNodeIds: + candidate === undefined ? [] : [candidate.current_node_id], + limitations: [ + "Location evidence links the source to the current node, but exact source bytes do not match or are unavailable.", + ], + }; + } + if (topPathCandidates.length > 1) + return { + status: "split", + confidence: "medium", + currentNodeIds: topPathCandidates.map( + ({ current_node_id: nodeId }) => nodeId, + ), + limitations: [ + "Multiple equally scored current nodes retain the historical source path; this is a static split inference, not runtime proof.", + ], + }; + if (candidates.length > 0) + return { + status: "unknown", + confidence: "unknown", + currentNodeIds: [], + limitations: [ + "Only weak basename or extension signals were available; no mapping was forced.", + ], + }; + return absenceComplete + ? { + status: "removed", + confidence: "high", + currentNodeIds: [], + limitations: [ + "No retained current node matched exact digest or path signals within complete static inventories.", + ], + } + : { + status: "unknown", + confidence: "unknown", + currentNodeIds: [], + limitations: [ + "No candidate was observed, but incomplete input or graph coverage prevents a removal claim.", + ], + }; +}; + +const markMergedMappings = ( + input: SourceToBundleComparisonInput, + items: readonly SourceToBundleComparisonItem[], +): SourceToBundleComparisonItem[] => { + const sourcesByNode = new Map(); + for (const item of items) { + if (item.status !== "unchanged" || item.current_node_ids.length !== 1) + continue; + const nodeId = item.current_node_ids[0]; + if (nodeId === undefined) continue; + sourcesByNode.set(nodeId, [...(sourcesByNode.get(nodeId) ?? []), item]); + } + const mergedSources = new Set( + [...sourcesByNode.values()] + .filter((group) => group.length > 1) + .flatMap((group) => group.map(({ source_path: path }) => path)), + ); + return items.map((item) => { + if (!mergedSources.has(item.source_path)) return item; + const { mapping_id: previousMappingId, ...base } = item; + void previousMappingId; + const semantic = { + ...base, + status: "merged" as const, + confidence: "exact" as const, + limitations: uniqueSorted([ + ...item.limitations, + "Multiple historical source paths have exact digest identity with the same current graph node.", + ]), + }; + return { ...semantic, mapping_id: mappingId(input, semantic) }; + }); +}; + +const mappingId = ( + input: SourceToBundleComparisonInput, + item: Omit, +): string => + `stbc_item_${digestCanonical({ + reference_root_sha256: input.reference.root_sha256, + application_graph_id: input.application.graph.graph_id, + ...item, + })}`; + +const comparisonSummary = ( + items: readonly SourceToBundleComparisonItem[], +): SourceToBundleComparisonResult["summary"] => ({ + unchanged: countStatus(items, "unchanged"), + modified: countStatus(items, "modified"), + removed: countStatus(items, "removed"), + split: countStatus(items, "split"), + merged: countStatus(items, "merged"), + duplicated: countStatus(items, "duplicated"), + unknown: countStatus(items, "unknown"), +}); + +const countStatus = ( + items: readonly SourceToBundleComparisonItem[], + status: SourceToBundleComparisonItem["status"], +): number => items.filter((item) => item.status === status).length; + +interface CoverageFactors { + readonly input: SourceToBundleComparisonInput; + readonly omittedSourceFiles: number; + readonly omittedApplicationNodes: number; + readonly omittedCandidateEvaluations: number; + readonly omittedCandidateReferences: number; + readonly pathIndexTruncated: boolean; +} + +const coverageStatus = ( + factors: CoverageFactors, +): SourceToBundleComparisonResult["coverage"]["status"] => { + if ( + factors.omittedSourceFiles > 0 || + factors.omittedApplicationNodes > 0 || + factors.omittedCandidateEvaluations > 0 || + factors.omittedCandidateReferences > 0 || + factors.pathIndexTruncated + ) + return "truncated"; + return factors.input.reference.inventory_state === "complete" && + factors.input.application.graph.coverage.status === "complete" + ? "complete-within-inputs" + : "partial"; +}; + +const comparisonLimitations = (factors: CoverageFactors): string[] => + uniqueSorted([ + "Static digest and location signals do not establish runtime behavior, semantic equivalence, or absence of dynamically generated modules.", + ...factors.input.reference.limitations, + ...factors.input.application.graph.limitations, + ...(factors.input.reference.inventory_state === "complete" + ? [] + : [ + `Historical source inventory is ${factors.input.reference.inventory_state}; unmatched files remain unknown.`, + ]), + ...(factors.input.application.graph.coverage.status === "complete" + ? [] + : [ + `Application graph coverage is ${factors.input.application.graph.coverage.status}; unmatched files remain unknown.`, + ]), + ...(factors.omittedSourceFiles > 0 + ? [ + `${String(factors.omittedSourceFiles)} historical source files were omitted by max_source_files.`, + ] + : []), + ...(factors.omittedApplicationNodes > 0 + ? [ + `${String(factors.omittedApplicationNodes)} source-bearing application nodes were omitted by max_application_nodes.`, + ] + : []), + ...(factors.omittedCandidateEvaluations > 0 + ? [ + `${String(factors.omittedCandidateEvaluations)} candidate evaluations were omitted by max_candidate_evaluations.`, + ] + : []), + ...(factors.omittedCandidateReferences > 0 + ? [ + `${String(factors.omittedCandidateReferences)} candidate references were omitted from item output.`, + ] + : []), + ...(factors.pathIndexTruncated + ? [ + "At least one current path exceeded the 128-segment suffix index bound.", + ] + : []), + ]); + +const hasSignal = ( + candidate: SourceToBundleCandidate, + kind: SourceToBundleCandidate["signals"][number]["kind"], +): boolean => candidate.signals.some((signal) => signal.kind === kind); + +const compareCandidates = ( + left: SourceToBundleCandidate, + right: SourceToBundleCandidate, +): number => + right.score - left.score || + compareText(left.current_node_id, right.current_node_id); + +const digestCanonical = (value: unknown): string => { + const encoded = canonicalize(value); + if (encoded === undefined) + throw new TypeError("Source-to-bundle comparison could not canonicalize"); + return createHash("sha256").update(encoded).digest("hex"); +}; + +const uniqueSorted = (values: readonly string[]): string[] => + [...new Set(values)].sort(compareText); + +const compareText = (left: string, right: string): number => + left < right ? -1 : left > right ? 1 : 0; diff --git a/src/domain/sourceToBundleComparisonSchemas.ts b/src/domain/sourceToBundleComparisonSchemas.ts new file mode 100644 index 000000000..a79e79032 --- /dev/null +++ b/src/domain/sourceToBundleComparisonSchemas.ts @@ -0,0 +1,157 @@ +import { z } from "zod"; + +import { evidenceSchema } from "./evidence.js"; +import { JAVASCRIPT_APPLICATION_NODE_KINDS } from "./javascriptApplicationGraphSchemas.js"; +import { historicalSourceGraphSchema } from "./referenceSourceGraph.js"; + +const digestSchema = z.string().regex(/^[a-f0-9]{64}$/u); +const evidenceIdSchema = z.string().regex(/^ev_[a-f0-9]{64}$/u); +const nodeIdSchema = z.string().regex(/^jag_node_[a-f0-9]{64}$/u); +const boundedTextSchema = z.string().min(1).max(4_096); + +export const SOURCE_TO_BUNDLE_SIGNAL_WEIGHTS = { + "exact-source-digest": 100, + "source-map-original-path": 70, + "current-path-exact": 60, + "current-path-suffix": 40, + "basename-match": 20, + "language-extension": 5, +} as const; + +const sourceToBundleSignalKindSchema = z.enum([ + "exact-source-digest", + "source-map-original-path", + "current-path-exact", + "current-path-suffix", + "basename-match", + "language-extension", +]); + +const sourceToBundleLimitsSchema = z.strictObject({ + max_source_files: z.number().int().min(1).max(100_000).default(10_000), + max_application_nodes: z.number().int().min(1).max(100_000).default(20_000), + max_candidate_nodes: z.number().int().min(1).max(1_000).default(100), + max_candidate_evaluations: z + .number() + .int() + .min(1) + .max(10_000_000) + .default(1_000_000), +}); + +/** Historical source inventory, authenticated application Evidence, and bounds. */ +export const compareSourceToBundleInputSchema = z.strictObject({ + reference: historicalSourceGraphSchema, + application: evidenceSchema, + limits: sourceToBundleLimitsSchema.default({ + max_source_files: 10_000, + max_application_nodes: 20_000, + max_candidate_nodes: 100, + max_candidate_evaluations: 1_000_000, + }), + unknown_registry_approved: z.literal(true).optional(), +}); + +const sourceToBundleSignalSchema = z.strictObject({ + kind: sourceToBundleSignalKindSchema, + weight: z.number().int().min(1).max(100), + source_value: boundedTextSchema, + current_values: z.array(boundedTextSchema).min(1).max(64), +}); + +const sourceToBundleCandidateSchema = z.strictObject({ + current_node_id: nodeIdSchema, + current_node_kind: z.enum(JAVASCRIPT_APPLICATION_NODE_KINDS), + score: z.number().int().min(1).max(300), + confidence: z.enum(["exact", "high", "medium", "low"]), + signals: z.array(sourceToBundleSignalSchema).min(1).max(6), +}); + +const sourceToBundleItemSchema = z.strictObject({ + mapping_id: z.string().regex(/^stbc_item_[a-f0-9]{64}$/u), + source_path: boundedTextSchema, + source_sha256: digestSchema.nullable(), + source_language: z.string().min(1).max(100).nullable(), + status: z.enum([ + "unchanged", + "modified", + "removed", + "split", + "merged", + "duplicated", + "unknown", + ]), + confidence: z.enum(["exact", "high", "medium", "unknown"]), + current_node_ids: z.array(nodeIdSchema).max(1_000), + candidates: z.array(sourceToBundleCandidateSchema).max(1_000), + omitted_candidates: z.number().int().min(0), + limitations: z.array(boundedTextSchema).max(100), +}); + +/** Deterministic, evidence-bearing historical-source to shipped-bundle comparison. */ +export const sourceToBundleComparisonResultSchema = z.strictObject({ + schema_version: z.literal(1), + comparison_id: z.string().regex(/^stbc_[a-f0-9]{64}$/u), + reference: z.strictObject({ + root_sha256: digestSchema, + inventory_state: z.enum(["complete", "partial", "unknown"]), + }), + application: z.strictObject({ + evidence_id: evidenceIdSchema, + graph_id: z.string().regex(/^jag_[a-f0-9]{64}$/u), + root_artifact_sha256: digestSchema, + }), + scoring: z.strictObject({ + algorithm: z.literal("rea-source-to-bundle-signals/v1"), + minimum_candidate_score: z.literal(20), + weights: z.array( + z.strictObject({ + signal: sourceToBundleSignalKindSchema, + weight: z.number().int().min(1).max(100), + }), + ), + }), + summary: z.strictObject({ + unchanged: z.number().int().min(0), + modified: z.number().int().min(0), + removed: z.number().int().min(0), + split: z.number().int().min(0), + merged: z.number().int().min(0), + duplicated: z.number().int().min(0), + unknown: z.number().int().min(0), + }), + items: z.array(sourceToBundleItemSchema).max(100_000), + unmapped_current_node_ids: z.array(nodeIdSchema).max(100_000), + coverage: z.strictObject({ + status: z.enum(["complete-within-inputs", "partial", "truncated"]), + reference_inventory_state: z.enum(["complete", "partial", "unknown"]), + application_graph_status: z.enum([ + "complete", + "partial", + "unknown", + "unavailable", + ]), + retained_source_files: z.number().int().min(0), + omitted_source_files: z.number().int().min(0), + retained_application_nodes: z.number().int().min(0), + omitted_application_nodes: z.number().int().min(0), + candidate_evaluations: z.number().int().min(0), + omitted_candidate_evaluations: z.number().int().min(0), + omitted_candidate_references: z.number().int().min(0), + omitted_unmapped_current_nodes: z.number().int().min(0), + }), + evidence_links: z.array(evidenceIdSchema).min(1).max(1), + limitations: z.array(boundedTextSchema).max(1_000), +}); + +export type CompareSourceToBundleInput = z.infer< + typeof compareSourceToBundleInputSchema +>; +export type SourceToBundleComparisonResult = z.infer< + typeof sourceToBundleComparisonResultSchema +>; +export type SourceToBundleComparisonItem = + SourceToBundleComparisonResult["items"][number]; +export type SourceToBundleCandidate = + SourceToBundleComparisonItem["candidates"][number]; +export type SourceToBundleSignal = SourceToBundleCandidate["signals"][number]; diff --git a/src/domain/sourceToBundleSignals.ts b/src/domain/sourceToBundleSignals.ts new file mode 100644 index 000000000..f3d67ac4f --- /dev/null +++ b/src/domain/sourceToBundleSignals.ts @@ -0,0 +1,314 @@ +import { posix } from "node:path"; + +import type { ApplicationNode } from "./javascriptApplicationGraphSchemas.js"; +import type { HistoricalSourceGraph } from "./referenceSourceGraph.js"; +import { + SOURCE_TO_BUNDLE_SIGNAL_WEIGHTS, + type SourceToBundleCandidate, + type SourceToBundleSignal, +} from "./sourceToBundleComparisonSchemas.js"; + +type SourceFile = Extract< + HistoricalSourceGraph["entries"][number], + { readonly kind: "file" } +>; + +type CurrentPathKind = + | "source-map-original" + | "canonical-path" + | "observation-path"; + +interface CurrentPath { + readonly kind: CurrentPathKind; + readonly value: string; +} + +interface CurrentProjection { + readonly node: ApplicationNode; + readonly digests: ReadonlySet; + readonly paths: readonly CurrentPath[]; +} + +/** Indexed current-node facts used by deterministic source matching. */ +export interface SourceToBundleCandidateIndex { + readonly nodes: ReadonlyMap; + readonly byDigest: ReadonlyMap>; + readonly byPathSuffix: ReadonlyMap>; + readonly byBasename: ReadonlyMap>; + readonly pathIndexTruncated: boolean; +} + +const RELEVANT_NODE_KINDS = new Set([ + "javascript-asset", + "javascript-module", + "source-module", +]); +const PATH_PROPERTIES = [ + "path", + "source", + "original_source", + "module_path", + "resolved_path", + "declared_path", +] as const; +const MAX_PATH_SUFFIXES = 128; + +/** Keep source-bearing application nodes in stable identifier order. */ +export const sourceBearingNodes = ( + nodes: readonly ApplicationNode[], +): ApplicationNode[] => + nodes + .filter(({ kind }) => RELEVANT_NODE_KINDS.has(kind)) + .sort((left, right) => compareText(left.node_id, right.node_id)); + +/** Keep historical source files in stable path order. */ +export const historicalSourceFiles = ( + graph: HistoricalSourceGraph, +): SourceFile[] => + graph.entries + .filter( + (entry): entry is SourceFile => + entry.kind === "file" && entry.classifications.includes("source"), + ) + .sort((left, right) => compareText(left.path, right.path)); + +/** Build bounded-key indices without assigning fuzzy matches. */ +export const buildSourceToBundleCandidateIndex = ( + nodes: readonly ApplicationNode[], +): SourceToBundleCandidateIndex => { + const projections = nodes.map(projectCurrentNode); + const byDigest = new Map>(); + const byPathSuffix = new Map>(); + const byBasename = new Map>(); + let pathIndexTruncated = false; + for (const projection of projections) { + for (const digest of projection.digests) + addIndexValue(byDigest, digest, projection.node.node_id); + for (const path of projection.paths) { + const suffixes = pathSuffixes(path.value); + pathIndexTruncated ||= suffixes.truncated; + for (const suffix of suffixes.values) + addIndexValue(byPathSuffix, suffix, projection.node.node_id); + addIndexValue( + byBasename, + posix.basename(path.value), + projection.node.node_id, + ); + } + } + return { + nodes: new Map( + projections.map((projection) => [projection.node.node_id, projection]), + ), + byDigest, + byPathSuffix, + byBasename, + pathIndexTruncated, + }; +}; + +/** Candidate identifiers ordered by strongest available signal, then ID. */ +export const candidateIdsForSource = ( + source: SourceFile, + index: SourceToBundleCandidateIndex, +): string[] => { + const groups = [ + source.sha256 === null ? undefined : index.byDigest.get(source.sha256), + index.byPathSuffix.get(source.path), + index.byBasename.get(posix.basename(source.path)), + ]; + const seen = new Set(); + const output: string[] = []; + for (const group of groups) { + if (group === undefined) continue; + for (const nodeId of [...group].sort(compareText)) { + if (seen.has(nodeId)) continue; + seen.add(nodeId); + output.push(nodeId); + } + } + return output; +}; + +/** Score one indexed candidate from explicit, caller-visible signals. */ +export const scoreSourceToBundleCandidate = ( + source: SourceFile, + nodeId: string, + index: SourceToBundleCandidateIndex, +): SourceToBundleCandidate | null => { + const projection = index.nodes.get(nodeId); + if (projection === undefined) return null; + const signals = candidateSignals(source, projection); + const score = signals.reduce((total, signal) => total + signal.weight, 0); + if (score < 20) return null; + return { + current_node_id: projection.node.node_id, + current_node_kind: projection.node.kind, + score, + confidence: candidateConfidence(signals), + signals, + }; +}; + +const projectCurrentNode = (node: ApplicationNode): CurrentProjection => { + const digests = new Set(); + const paths: CurrentPath[] = []; + if (node.identity.strategy === "content-digest") + digests.add(node.identity.sha256); + if (node.identity.strategy === "source-map-original") { + if (node.identity.source_sha256 !== null) + digests.add(node.identity.source_sha256); + addPath(paths, "source-map-original", node.identity.original_source); + } + if (node.identity.strategy === "canonical-path") + addPath(paths, "canonical-path", node.identity.path); + for (const observation of node.observations) { + const sourceDigest = observation.properties.source_sha256; + if (typeof sourceDigest === "string" && isDigest(sourceDigest)) + digests.add(sourceDigest); + for (const key of PATH_PROPERTIES) + addJsonPath(paths, observation.properties[key]); + } + return { + node, + digests, + paths: uniquePaths(paths), + }; +}; + +const candidateSignals = ( + source: SourceFile, + current: CurrentProjection, +): SourceToBundleSignal[] => { + const signals: SourceToBundleSignal[] = []; + if (source.sha256 !== null && current.digests.has(source.sha256)) + signals.push(signal("exact-source-digest", source.sha256, [source.sha256])); + const sourceMapPaths = matchingPaths( + source.path, + current.paths.filter(({ kind }) => kind === "source-map-original"), + ); + if (sourceMapPaths.length > 0) + signals.push( + signal("source-map-original-path", source.path, sourceMapPaths), + ); + const otherPaths = current.paths.filter( + ({ kind }) => kind !== "source-map-original", + ); + const exactPaths = otherPaths + .filter(({ value }) => value === source.path) + .map(({ value }) => value); + if (exactPaths.length > 0) + signals.push(signal("current-path-exact", source.path, exactPaths)); + const suffixPaths = otherPaths + .filter( + ({ value }) => value !== source.path && value.endsWith(`/${source.path}`), + ) + .map(({ value }) => value); + if (suffixPaths.length > 0) + signals.push(signal("current-path-suffix", source.path, suffixPaths)); + const basename = posix.basename(source.path); + const basenamePaths = current.paths + .filter(({ value }) => posix.basename(value) === basename) + .map(({ value }) => value); + if (basenamePaths.length > 0) + signals.push(signal("basename-match", basename, basenamePaths)); + const extension = posix.extname(source.path).toLowerCase(); + const extensionPaths = current.paths + .filter(({ value }) => posix.extname(value).toLowerCase() === extension) + .map(({ value }) => value); + if (extension !== "" && extensionPaths.length > 0) + signals.push(signal("language-extension", extension, extensionPaths)); + return signals; +}; + +const signal = ( + kind: SourceToBundleSignal["kind"], + sourceValue: string, + currentValues: readonly string[], +): SourceToBundleSignal => ({ + kind, + weight: SOURCE_TO_BUNDLE_SIGNAL_WEIGHTS[kind], + source_value: sourceValue, + current_values: [...new Set(currentValues)].sort(compareText).slice(0, 64), +}); + +const candidateConfidence = ( + signals: readonly SourceToBundleSignal[], +): SourceToBundleCandidate["confidence"] => { + const kinds = new Set(signals.map(({ kind }) => kind)); + if (kinds.has("exact-source-digest")) return "exact"; + if (kinds.has("source-map-original-path") || kinds.has("current-path-exact")) + return "high"; + return kinds.has("current-path-suffix") ? "medium" : "low"; +}; + +const matchingPaths = ( + sourcePath: string, + currentPaths: readonly CurrentPath[], +): string[] => + currentPaths + .filter( + ({ value }) => value === sourcePath || value.endsWith(`/${sourcePath}`), + ) + .map(({ value }) => value); + +const addJsonPath = (paths: CurrentPath[], value: unknown): void => { + if (typeof value === "string") addPath(paths, "observation-path", value); +}; + +const addPath = ( + paths: CurrentPath[], + kind: CurrentPathKind, + raw: string, +): void => { + const value = normalizeCurrentPath(raw); + if (value !== null) paths.push({ kind, value }); +}; + +const normalizeCurrentPath = (raw: string): string | null => { + const withoutQuery = raw.split(/[?#]/u, 1)[0] ?? ""; + const withoutScheme = withoutQuery.replace(/^[a-z][a-z0-9+.-]*:\/\/+/iu, ""); + const parts = withoutScheme + .replaceAll("\\", "/") + .split("/") + .filter((part) => part !== "" && part !== "."); + if (parts.length === 0 || parts.includes("..")) return null; + return parts.join("/"); +}; + +const uniquePaths = (paths: readonly CurrentPath[]): CurrentPath[] => + [ + ...new Map( + paths.map((path) => [`${path.kind}\0${path.value}`, path]), + ).values(), + ].sort((left, right) => + compareText(`${left.kind}\0${left.value}`, `${right.kind}\0${right.value}`), + ); + +const pathSuffixes = ( + path: string, +): { readonly values: readonly string[]; readonly truncated: boolean } => { + const parts = path.split("/"); + const start = Math.max(0, parts.length - MAX_PATH_SUFFIXES); + return { + values: parts + .slice(start) + .map((_, index) => parts.slice(start + index).join("/")), + truncated: start > 0, + }; +}; + +const addIndexValue = ( + index: Map>, + key: string, + nodeId: string, +): void => { + const values = index.get(key) ?? new Set(); + values.add(nodeId); + index.set(key, values); +}; + +const isDigest = (value: string): boolean => /^[a-f0-9]{64}$/u.test(value); + +const compareText = (left: string, right: string): number => + left < right ? -1 : left > right ? 1 : 0; diff --git a/src/server/registerApplicationTools/compareSourceToBundle.ts b/src/server/registerApplicationTools/compareSourceToBundle.ts new file mode 100644 index 000000000..67eb20932 --- /dev/null +++ b/src/server/registerApplicationTools/compareSourceToBundle.ts @@ -0,0 +1,59 @@ +import type { McpServer } from "@modelcontextprotocol/server"; + +import { compareSourceToBundleEvidenceValidated } from "../../application/JavaScriptApplicationWorkflowService.js"; +import { resolveCompareSourceToBundleRequestValidated } from "../../application/ApplicationWorkflowEvidenceResolver.js"; +import { applicationToolContract } from "../../contracts/applicationToolContracts.js"; +import { compareSourceToBundleRequestSchema } from "../../contracts/applicationWorkflowInputContracts.js"; +import { sourceToBundleComparisonResultSchema } from "../../domain/sourceToBundleComparisonSchemas.js"; +import { logToolExecution } from "../toolLogging.js"; +import { safeParseToolInput } from "../toolInputValidation.js"; +import { toolRegistrationOptions } from "../toolRegistrationOptions.js"; +import { toCallToolResult } from "../toolResult.js"; +import { recordResult, recordSources } from "./helpers.js"; +import type { ApplicationToolRegistration } from "./types.js"; + +const contract = applicationToolContract("compare_source_to_bundle"); + +/** Register conservative historical-source to bundle comparison. */ +export const registerCompareSourceToBundleTool = ( + server: McpServer, + options: ApplicationToolRegistration, +): void => { + server.registerTool( + contract.name, + toolRegistrationOptions(contract), + async (input) => { + const parsedInput = safeParseToolInput( + compareSourceToBundleRequestSchema, + input, + contract.name, + ); + if (!parsedInput.ok) return toCallToolResult(parsedInput, contract); + const resolved = resolveCompareSourceToBundleRequestValidated( + parsedInput.value, + options.evidenceLookup, + ); + if (!resolved.ok) return toCallToolResult(resolved, contract); + const result = await logToolExecution(options.logger, contract.name, () => + Promise.resolve(compareSourceToBundleEvidenceValidated(resolved.value)), + ); + if (!result.ok) return toCallToolResult(result, contract); + const recorded = recordSources(options.recordEvidence, [ + resolved.value.application, + ]); + if (!recorded.ok) return toCallToolResult(recorded, contract); + const comparison = sourceToBundleComparisonResultSchema.parse( + result.value.normalized_result, + ); + return recordResult( + options, + contract, + result.value, + resolved.value.unknown_registry_approved === true && + comparison.summary.unknown > 0 + ? "source-to-bundle-comparison" + : undefined, + ); + }, + ); +}; diff --git a/src/server/registerApplicationTools/helpers.ts b/src/server/registerApplicationTools/helpers.ts index 6549e7a56..ac63b038b 100644 --- a/src/server/registerApplicationTools/helpers.ts +++ b/src/server/registerApplicationTools/helpers.ts @@ -24,7 +24,10 @@ export const recordResult = ( options: ApplicationToolRegistration, contract: (typeof APPLICATION_TOOL_CONTRACTS)[number], evidence: Evidence, - unknownKind?: "application-version-comparison" | "javascript-export-shape", + unknownKind?: + | "application-version-comparison" + | "javascript-export-shape" + | "source-to-bundle-comparison", ) => { const recorded = unknownKind === undefined @@ -42,13 +45,18 @@ export const recordResult = ( const unknownRegistration = ( evidence: Evidence, - kind: "application-version-comparison" | "javascript-export-shape", + kind: + | "application-version-comparison" + | "javascript-export-shape" + | "source-to-bundle-comparison", ) => ({ approved: true as const, question: kind === "application-version-comparison" ? "Which application entities remain unmatched or ambiguous across these versions?" - : "Which selected JavaScript export return shapes remain dynamic, incomplete, or ambiguously paired?", + : kind === "source-to-bundle-comparison" + ? "Which historical source files remain unmatched or ambiguously mapped to the shipped bundle?" + : "Which selected JavaScript export return shapes remain dynamic, incomplete, or ambiguously paired?", severity: "medium" as const, domain: kind, supporting_evidence_ids: [evidence.evidence_id], @@ -70,13 +78,15 @@ const unknownRegistration = ( "Add approved passive runtime Evidence without promoting it to static fact.", }, ] - : [ - { - operation: "run_controlled_replay", - rationale: - "Validate exact approved module behavior separately when runtime semantics are required.", - }, - ]), + : kind === "source-to-bundle-comparison" + ? [] + : [ + { + operation: "run_controlled_replay", + rationale: + "Validate exact approved module behavior separately when runtime semantics are required.", + }, + ]), ], relationships: [], }); diff --git a/tests/applicationWorkflowCli.test.ts b/tests/applicationWorkflowCli.test.ts index c6f1abd30..d7bbb05e3 100644 --- a/tests/applicationWorkflowCli.test.ts +++ b/tests/applicationWorkflowCli.test.ts @@ -12,6 +12,7 @@ import { JAVASCRIPT_FEATURE_TRACE_FULL_EVIDENCE_EXAMPLE, JAVASCRIPT_FEATURE_TRACE_EXAMPLE, JAVASCRIPT_VERSION_COMPARISON_FULL_EVIDENCE_EXAMPLE, + SOURCE_TO_BUNDLE_COMPARISON_EXAMPLE, } from "../src/contracts/javascriptApplicationWorkflowExamples.js"; import { createEvidenceBundle, @@ -65,6 +66,18 @@ describe("application workflow CLI parity", () => { summary: { unknown: expect.any(Number) }, }, }); + const sourceCompared = await runCli([ + "compare-source-to-bundle", + JSON.stringify({ + reference: SOURCE_TO_BUNDLE_COMPARISON_EXAMPLE.reference, + application: JAVASCRIPT_FEATURE_TRACE_FULL_EVIDENCE_EXAMPLE.application, + }), + "--json", + ]); + expect(sourceCompared).toMatchObject({ + operation: "compare_source_to_bundle", + normalized_result: { schema_version: 1 }, + }); }, 20_000); it("traces the same authenticated semantic graph through the CLI", async () => { diff --git a/tests/applicationWorkflowMcp.test.ts b/tests/applicationWorkflowMcp.test.ts index 3c4a5d8d7..d99256c0c 100644 --- a/tests/applicationWorkflowMcp.test.ts +++ b/tests/applicationWorkflowMcp.test.ts @@ -12,6 +12,7 @@ import { JAVASCRIPT_FEATURE_TRACE_FULL_EVIDENCE_EXAMPLE, JAVASCRIPT_FEATURE_TRACE_EXAMPLE, JAVASCRIPT_VERSION_COMPARISON_FULL_EVIDENCE_EXAMPLE, + SOURCE_TO_BUNDLE_COMPARISON_EXAMPLE, } from "../src/contracts/javascriptApplicationWorkflowExamples.js"; import { createEvidence } from "../src/domain/evidence.js"; import { createServer } from "../src/server/createServer.js"; @@ -202,6 +203,25 @@ describe("application workflow MCP parity", () => { coverage: { status: expect.any(String) }, }, }); + const sourceCompared = await client.callTool({ + name: "compare_source_to_bundle", + arguments: { + reference: SOURCE_TO_BUNDLE_COMPARISON_EXAMPLE.reference, + application: + JAVASCRIPT_FEATURE_TRACE_FULL_EVIDENCE_EXAMPLE.application, + }, + }); + expect(sourceCompared.isError).not.toBe(true); + expect(sourceCompared.structuredContent).toMatchObject({ + result: { + schema_version: 1, + reference: { + root_sha256: + SOURCE_TO_BUNDLE_COMPARISON_EXAMPLE.reference.root_sha256, + }, + scoring: { algorithm: "rea-source-to-bundle-signals/v1" }, + }, + }); expect(session.exportEvidenceBundle().records.length).toBeGreaterThan(2); await new Promise((resolve) => setImmediate(resolve)); @@ -243,6 +263,16 @@ describe("application workflow MCP parity", () => { }, }, }); + const sourceComparedById = await client.callTool({ + name: "compare_source_to_bundle", + arguments: { + ...SOURCE_TO_BUNDLE_COMPARISON_EXAMPLE, + application_evidence_id: + JAVASCRIPT_FEATURE_TRACE_FULL_EVIDENCE_EXAMPLE.application + .evidence_id, + }, + }); + expect(sourceComparedById.isError).not.toBe(true); const missing = await client.callTool({ name: "trace_application_feature", diff --git a/tests/javascriptBundlerAdapters.test.ts b/tests/javascriptBundlerAdapters.test.ts index 93d8ac822..a941d8533 100644 --- a/tests/javascriptBundlerAdapters.test.ts +++ b/tests/javascriptBundlerAdapters.test.ts @@ -104,6 +104,40 @@ describe("Webpack and Rspack runtime adapters", () => { await rm(root, { recursive: true, force: true }); } }); + + it("recovers esbuild wrapper modules and Vite preload dependencies", async () => { + const root = await esmRuntimeFixture(); + try { + const { graph } = await reconstructJavaScriptArtifact({ + input_path: root, + }); + const commonJsModule = moduleNode(graph, "src/dep.js"); + const esmModule = moduleNode(graph, "src/core.js"); + + expect(commonJsModule?.observations[0]?.properties).toMatchObject({ + bundler: "esbuild", + runtime: "esbuild-commonjs", + module_key: "src/dep.js", + }); + expect(esmModule?.observations[0]?.properties).toMatchObject({ + bundler: "esbuild", + runtime: "esbuild-esm", + module_key: "src/core.js", + }); + expect( + graph.edges.filter( + ({ relation, properties }) => + relation === "imports" && + properties.kind === "dynamic-import" && + ["./feature.js", "./main.css"].includes( + String(properties.specifier), + ), + ), + ).toHaveLength(2); + } finally { + await rm(root, { recursive: true, force: true }); + } + }); }); const bundlerFixture = async (): Promise => { @@ -159,6 +193,33 @@ const bundlerFixture = async (): Promise => { return root; }; +const esmRuntimeFixture = async (): Promise => { + const root = await createTestTempDirectory("rea-esm-runtime-adapters-"); + await writeFile( + join(root, "main.js"), + ` + const __vite__mapDeps = (indexes, map = __vite__mapDeps, dependencies = + (map.f || (map.f = ["./feature.js", "./main.css"]))) => + indexes.map((index) => dependencies[index]); + const require_dep = __commonJS({ + "src/dep.js"(exports, module) { + module.exports = { value: 1 }; + } + }); + const init_core = __esm({ + "src/core.js"() { + require_dep(); + } + }); + __vite__mapDeps([0, 1]); + init_core(); + `, + ); + await writeFile(join(root, "feature.js"), "export const feature = true;"); + await writeFile(join(root, "main.css"), ".app { color: green; }"); + return root; +}; + const chunk = ( graph: JavaScriptApplicationGraph, runtime: string, diff --git a/tests/sourceToBundleComparison.test.ts b/tests/sourceToBundleComparison.test.ts new file mode 100644 index 000000000..5d74ad07b --- /dev/null +++ b/tests/sourceToBundleComparison.test.ts @@ -0,0 +1,303 @@ +import { describe, expect, it } from "vitest"; + +import { + createJavaScriptApplicationGraph, + createJavaScriptApplicationNode, +} from "../src/domain/javascriptApplicationGraph.js"; +import { createHistoricalSourceGraph } from "../src/domain/referenceSourceGraph.js"; +import { compareSourceToBundle } from "../src/domain/sourceToBundleComparison.js"; +import { sourceToBundleComparisonResultSchema } from "../src/domain/sourceToBundleComparisonSchemas.js"; +import { artifactEvidence } from "./fixtures/javascriptApplicationGraph.js"; + +const HASH = { + artifact: "1".repeat(64), + unchanged: "2".repeat(64), + modified: "3".repeat(64), + removed: "4".repeat(64), + split: "5".repeat(64), + merged: "6".repeat(64), + duplicated: "7".repeat(64), +} as const; +const EVIDENCE_ID = `ev_${"a".repeat(64)}`; + +describe("historical source to bundle comparison", () => { + it("classifies exact, changed, absent, split, and merged mappings stably", () => { + const reference = historicalGraph("complete"); + const graph = applicationGraph("complete"); + const input = { + reference, + application: { + evidenceId: EVIDENCE_ID, + rootArtifactSha256: HASH.artifact, + graph, + }, + limits: limits(), + }; + + const first = compareSourceToBundle(input); + const second = compareSourceToBundle(input); + + expect(first).toEqual(second); + expect(() => + sourceToBundleComparisonResultSchema.parse(first), + ).not.toThrow(); + expect(first.summary).toEqual({ + unchanged: 1, + modified: 1, + removed: 1, + split: 1, + merged: 2, + duplicated: 1, + unknown: 0, + }); + expect(item(first, "src/unchanged.ts")).toMatchObject({ + status: "unchanged", + confidence: "exact", + candidates: [ + expect.objectContaining({ + signals: expect.arrayContaining([ + expect.objectContaining({ + kind: "exact-source-digest", + weight: 100, + }), + ]), + }), + ], + }); + expect(item(first, "src/modified.ts")).toMatchObject({ + status: "modified", + confidence: "high", + candidates: [ + expect.objectContaining({ + signals: expect.arrayContaining([ + expect.objectContaining({ kind: "current-path-exact", weight: 60 }), + ]), + }), + ], + }); + expect(item(first, "src/removed.ts")).toMatchObject({ + status: "removed", + current_node_ids: [], + }); + expect(item(first, "src/split.ts")).toMatchObject({ + status: "split", + confidence: "medium", + current_node_ids: [expect.any(String), expect.any(String)], + }); + expect(item(first, "src/merged-a.ts").status).toBe("merged"); + expect(item(first, "src/merged-b.ts").status).toBe("merged"); + expect(item(first, "src/duplicated.ts")).toMatchObject({ + status: "duplicated", + confidence: "exact", + current_node_ids: [expect.any(String), expect.any(String)], + }); + expect(first.coverage.status).toBe("complete-within-inputs"); + }); + + it("keeps absence unknown when either inventory is incomplete", () => { + const reference = historicalGraph("partial", ["src/removed.ts"]); + const graph = applicationGraph("partial"); + const result = compareSourceToBundle({ + reference, + application: { + evidenceId: EVIDENCE_ID, + rootArtifactSha256: HASH.artifact, + graph, + }, + limits: limits(), + }); + + expect(result.summary).toMatchObject({ removed: 0, unknown: 1 }); + expect(result.items[0]).toMatchObject({ + source_path: "src/removed.ts", + status: "unknown", + confidence: "unknown", + }); + expect(result.coverage.status).toBe("partial"); + }); + + it("reports the candidate frontier instead of deciding past a limit", () => { + const result = compareSourceToBundle({ + reference: historicalGraph("complete", ["src/split.ts"]), + application: { + evidenceId: EVIDENCE_ID, + rootArtifactSha256: HASH.artifact, + graph: applicationGraph("complete"), + }, + limits: { ...limits(), max_candidate_evaluations: 1 }, + }); + + expect(result.items[0]).toMatchObject({ + status: "unknown", + confidence: "unknown", + omitted_candidates: 1, + }); + expect(result.coverage).toMatchObject({ + status: "truncated", + candidate_evaluations: 1, + omitted_candidate_evaluations: 1, + }); + }); +}); + +const historicalGraph = ( + inventoryState: "complete" | "partial", + paths = [ + "src/duplicated.ts", + "src/merged-a.ts", + "src/merged-b.ts", + "src/modified.ts", + "src/removed.ts", + "src/split.ts", + "src/unchanged.ts", + ], +) => + createHistoricalSourceGraph({ + schema: "HistoricalSourceGraph/v1", + authority: "historical-reference", + root_alias: "$REFERENCE_ROOT", + inventory_state: inventoryState, + entries: [ + { + path: "src", + kind: "directory", + classifications: ["source"], + tree_state: inventoryState === "complete" ? "enumerated" : "partial", + limitations: + inventoryState === "complete" + ? [] + : ["Fixture inventory is partial."], + }, + ...paths.map((path) => ({ + path, + kind: "file" as const, + sha256: sourceDigest(path), + size: 10, + language: "TypeScript", + classifications: ["source"] as const, + content_state: "hashed" as const, + limitations: [], + })), + ], + relationships: [], + parse_failures: [], + exclusions: [], + languages: ["TypeScript"], + manifests: [], + vcs: { kind: "none", head: null, dirty: null }, + provenance: { + importer: "rea-test", + importer_version: "1", + caller: "source-to-bundle-test", + }, + limitations: + inventoryState === "complete" ? [] : ["Fixture inventory is partial."], + }); + +const applicationGraph = (coverage: "complete" | "partial") => { + const nodes = [ + moduleNode("duplicated-a", { + path: "bundle-a.js", + source_sha256: HASH.duplicated, + }), + moduleNode("duplicated-b", { + path: "bundle-b.js", + source_sha256: HASH.duplicated, + }), + moduleNode("unchanged", { + path: "bundle.js", + source_sha256: HASH.unchanged, + }), + createJavaScriptApplicationNode({ + kind: "javascript-asset", + identity: { + strategy: "canonical-path", + stability: "artifact-version", + artifact_sha256: HASH.artifact, + path: "src/modified.ts", + }, + observations: [ + { + label: "src/modified.ts", + properties: { path: "src/modified.ts" }, + evidence: artifactEvidence(HASH.artifact, "src/modified.ts"), + }, + ], + }), + moduleNode("split-a", { path: "bundle-a/src/split.ts" }), + moduleNode("split-b", { path: "bundle-b/src/split.ts" }), + moduleNode("merged", { + path: "bundle.js", + source_sha256: HASH.merged, + }), + ]; + return createJavaScriptApplicationGraph({ + schema: "JavaScriptApplicationGraph", + schema_version: 1, + root_node_ids: nodes.map(({ node_id: nodeId }) => nodeId), + nodes, + edges: [], + coverage: + coverage === "complete" + ? { + status: "complete", + truncated: false, + omitted_count: 0, + limits: [], + } + : { + status: "partial", + truncated: true, + omitted_count: 1, + limits: [{ name: "fixture", value: 5, unit: "items" }], + }, + limitations: + coverage === "complete" ? [] : ["Fixture graph coverage is partial."], + }); +}; + +const moduleNode = (key: string, properties: Record) => + createJavaScriptApplicationNode({ + kind: "javascript-module", + identity: { + strategy: "artifact-local-key", + stability: "artifact-version", + artifact_sha256: HASH.artifact, + namespace: "test", + key, + }, + observations: [ + { + label: key, + properties, + evidence: artifactEvidence(HASH.artifact, "bundle.js"), + }, + ], + }); + +const sourceDigest = (path: string): string => { + if (path.includes("duplicated")) return HASH.duplicated; + if (path.includes("merged")) return HASH.merged; + if (path.includes("modified")) return HASH.modified; + if (path.includes("removed")) return HASH.removed; + if (path.includes("split")) return HASH.split; + return HASH.unchanged; +}; + +const limits = () => ({ + max_source_files: 100, + max_application_nodes: 100, + max_candidate_nodes: 100, + max_candidate_evaluations: 10_000, +}); + +const item = ( + result: ReturnType, + sourcePath: string, +) => { + const value = result.items.find( + ({ source_path: path }) => path === sourcePath, + ); + if (value === undefined) throw new TypeError(`Missing item: ${sourcePath}`); + return value; +};