import { isSecretLikePath } from "../domain/referenceSourceClassification.js"; import { createHistoricalSourceGraph, type HistoricalSourceGraph, type HistoricalSourceGraphInput, } from "../domain/referenceSourceGraph.js"; import { err, ok, type Result } from "../domain/result.js"; import { readReferenceSource } from "../reference/ReferenceSourceReader.js"; import { parseReferenceSourceEntries } from "./ReferenceSourceImportEntries.js"; import { readReferenceSourceVcs } from "./ReferenceSourceVcsAdapter.js"; import { type ReferenceSourceImportError, type ReferenceSourceImportOptions, } from "./ReferenceSourceImportTypes.js"; import { prepareReferenceSourceImport } from "./ReferenceSourceImportPolicy.js"; export type { ReferenceSourceImportError, ReferenceSourceImportOptions, } from "./ReferenceSourceImportTypes.js"; const compareCodePoints = (left: string, right: string): number => { const leftPoints = [...left].map((value) => value.codePointAt(0) ?? 0); const rightPoints = [...right].map((value) => value.codePointAt(0) ?? 0); const length = Math.min(leftPoints.length, rightPoints.length); for (let index = 0; index < length; index += 1) { const difference = (leftPoints[index] ?? 0) - (rightPoints[index] ?? 0); if (difference !== 0) return difference; } return leftPoints.length - rightPoints.length; }; const byCodePoint = (left: string, right: string): number => compareCodePoints(left, right); const failure = ( code: ReferenceSourceImportError["code"], message: string, ): ReferenceSourceImportError => ({ tag: "reference-source-import", code, message, }); const cancelled = (): ReferenceSourceImportError => failure("cancelled", "Reference source import cancelled"); const isAborted = (signal?: AbortSignal): boolean => signal?.aborted === true; const relationshipKey = ( relationship: HistoricalSourceGraphInput["relationships"][number], ): string => `${relationship.from_path}\u0000${relationship.to}\u0000${relationship.kind}\u0000${relationship.resolution}\u0000${relationship.parse_state}`; const deduplicateRelationships = ( relationships: HistoricalSourceGraphInput["relationships"], ): HistoricalSourceGraphInput["relationships"] => { const seen = new Set(); const result: HistoricalSourceGraphInput["relationships"] = []; for (const relationship of relationships) { const key = relationshipKey(relationship); if (seen.has(key)) continue; seen.add(key); result.push(relationship); } return result; }; const buildProvenance = ( options: ReferenceSourceImportOptions, ): HistoricalSourceGraphInput["provenance"] => ({ importer: options.importer ?? "rea-reference-source-import", importer_version: options.importerVersion ?? null, caller: options.caller, }); const deriveLanguages = ( entries: HistoricalSourceGraphInput["entries"], ): string[] => [ ...new Set( entries.flatMap((entry) => entry.kind === "file" && entry.language !== null ? [entry.language] : [], ), ), ].sort(byCodePoint); const deriveManifests = ( entries: HistoricalSourceGraphInput["entries"], ): string[] => entries .flatMap((entry) => entry.kind === "file" && entry.classifications.includes("manifest") ? [entry.path] : [], ) .sort(byCodePoint); const deriveInventoryState = ( input: Pick< HistoricalSourceGraphInput, | "entries" | "relationships" | "parse_failures" | "exclusions" | "limitations" >, ): "complete" | "partial" | "unknown" => { if (input.limitations.length > 0) return "partial"; if (input.exclusions.length > 0) return "partial"; if (input.parse_failures.length > 0) return "partial"; const partialEntry = input.entries.some( (entry) => entry.limitations.length > 0 || (entry.kind === "file" && entry.content_state !== "hashed") || (entry.kind === "directory" && entry.tree_state !== "enumerated") || (entry.kind === "symlink" && entry.target_state !== "internal"), ); if (partialEntry) return "partial"; const partialRelationship = input.relationships.some( ({ parse_state, resolution }) => parse_state !== "parsed" || ["unresolved", "unknown"].includes(resolution), ); if (partialRelationship) return "partial"; return "complete"; }; const sortAndLimit = ( exclusions: HistoricalSourceGraphInput["exclusions"], maxExclusions: number, ): HistoricalSourceGraphInput["exclusions"] => { const sorted = [...exclusions].sort((left, right) => { const byPath = byCodePoint(left.path, right.path); if (byPath !== 0) return byPath; return byCodePoint(left.reason, right.reason); }); if (sorted.length <= maxExclusions) return sorted; return sorted.slice(0, maxExclusions); }; /** * Import a reference source directory into a committed historical source graph. * * The import is deterministic, parallel-safe, and never executes source, hooks, * git subprocesses, or network requests. Secret-like paths are redacted before * they are committed to the graph. */ export const importReferenceSource = async ( options: ReferenceSourceImportOptions, ): Promise> => { if (isAborted(options.signal)) return err(cancelled()); const prepared = await prepareReferenceSourceImport(options); if (!prepared.ok) return prepared; const { ignored, limits, root, secrets } = prepared.value; if (isAborted(options.signal)) return err(cancelled()); const exclusions: HistoricalSourceGraphInput["exclusions"] = []; const shouldExclude = (path: string): boolean => { if (isSecretLikePath(path) || secrets.ignores(path)) { exclusions.push({ path, reason: "configured-secret" }); return true; } if (ignored.ignores(path)) { exclusions.push({ path, reason: "caller-excluded" }); return true; } return false; }; const [readResult, vcs] = await Promise.all([ readReferenceSource(root, limits, { ...(options.signal === undefined ? {} : { signal: options.signal }), shouldExclude, }), readReferenceSourceVcs(root, options.signal), ]); if (!readResult.ok) { const error = readResult.error; if (error.code === "cancelled") return err(cancelled()); return err( failure( error.code === "invalid-limits" ? "invalid-limits" : "io", error.message, ), ); } if (isAborted(options.signal)) return err(cancelled()); const read = readResult.value; const filePaths = new Set( read.entries .filter((entry) => entry.status === "read" && entry.kind === "file") .map((entry) => entry.path), ); const { entries, relationships, parseFailures, limitations } = parseReferenceSourceEntries(read, filePaths, options.signal); if (isAborted(options.signal)) return err(cancelled()); const uniqueRelationships = deduplicateRelationships(relationships); const uniqueFailures = [...parseFailures] .sort((left, right) => { const byPath = byCodePoint(left.path, right.path); if (byPath !== 0) return byPath; return byCodePoint(left.parser, right.parser); }) .filter((value, index, array) => { if (index === 0) return true; const previous = array[index - 1]; if (previous === undefined) return true; return value.path !== previous.path || value.parser !== previous.parser; }); const sortedEntries = [...entries].sort((left, right) => byCodePoint(left.path, right.path), ); const sortedExclusions = sortAndLimit(exclusions, limits.maxEntries); const sortedLimitations = [...limitations].sort(byCodePoint); const input: HistoricalSourceGraphInput = { schema: "HistoricalSourceGraph/v1", authority: "historical-reference", root_alias: "$REFERENCE_ROOT", inventory_state: deriveInventoryState({ entries: sortedEntries, relationships: uniqueRelationships, parse_failures: uniqueFailures, exclusions: sortedExclusions, limitations: sortedLimitations, }), entries: sortedEntries, relationships: uniqueRelationships.sort((left, right) => byCodePoint(relationshipKey(left), relationshipKey(right)), ), parse_failures: uniqueFailures, exclusions: sortedExclusions, languages: deriveLanguages(sortedEntries), manifests: deriveManifests(sortedEntries), vcs, provenance: buildProvenance(options), limitations: sortedLimitations, }; try { return ok(createHistoricalSourceGraph(input)); } catch (error) { return err( failure( "parse", error instanceof Error ? error.message : "Graph failed validation", ), ); } };