import { compareUnicodeCodePoints } from "../domain/unicodeCodePointOrder.js"; import { createHistoricalSourceGraph, historicalSourceLanguages, historicalSourceManifests, historicalSourceParseFailureKey, type HistoricalSourceGraph, type HistoricalSourceGraphInput, } from "../domain/referenceSourceGraph.js"; import { err, ok, type Result } from "../domain/result.js"; import type { ArtifactResourceScope } from "../artifacts/ArtifactResourceScope.js"; import type { AnalysisCleanupObservation } from "../domain/analysisErrorBase.js"; import { readReferenceSource } from "../reference/ReferenceSourceReader.js"; import type { ReferenceSourceExclusionKind, ReferenceSourceRead, ReferenceSourceReaderError, } from "../reference/ReferenceSourceReaderTypes.js"; import { parseReferenceSourceEntries } from "./ReferenceSourceImportEntries.js"; import { readReferenceSourceVcs } from "./ReferenceSourceVcsAdapter.js"; import type { ReferenceSourceImportError, ReferenceSourceImportOptions, } from "./ReferenceSourceImportTypes.js"; import { prepareReferenceSourceImport, type PreparedReferenceSourceImport, } from "./ReferenceSourceImportPolicy.js"; const failure = ( code: ReferenceSourceImportError["code"], message: string, ): ReferenceSourceImportError => ({ tag: "reference-source-import", code, message, }); const cancelled = ( partial?: ReferenceSourceRead, ): ReferenceSourceImportError => ({ ...failure("cancelled", "Reference source import cancelled"), ...(partial === undefined ? {} : { partial }), }); const isAborted = (signal?: AbortSignal): boolean => signal?.aborted === true; const relationshipKey = ( relationship: HistoricalSourceGraphInput["relationships"][number], ): string => `${relationship.from_path}\u0000${relationship.to}\u0000${relationship.kind}\u0000${relationship.resolution}\u0000${relationship.parse_state}`; const deduplicateRelationships = ( relationships: HistoricalSourceGraphInput["relationships"], ): HistoricalSourceGraphInput["relationships"] => { const seen = new Set(); const result: HistoricalSourceGraphInput["relationships"] = []; for (const relationship of relationships) { const key = relationshipKey(relationship); if (seen.has(key)) continue; seen.add(key); result.push(relationship); } return result; }; /** Sort parse failures deterministically while retaining distinct reasons. */ export const normalizeHistoricalSourceParseFailures = ( failures: HistoricalSourceGraphInput["parse_failures"], ): HistoricalSourceGraphInput["parse_failures"] => [...failures] .sort((left, right) => compareUnicodeCodePoints( historicalSourceParseFailureKey(left), historicalSourceParseFailureKey(right), ), ) .filter((value, index, array) => { if (index === 0) return true; const previous = array[index - 1]; return ( previous === undefined || historicalSourceParseFailureKey(value) !== historicalSourceParseFailureKey(previous) ); }); const buildProvenance = ( options: ReferenceSourceImportOptions, ): HistoricalSourceGraphInput["provenance"] => ({ importer: options.importer ?? "rea-reference-source-import", importer_version: options.importerVersion ?? null, caller: options.caller, }); const deriveInventoryState = ( input: Pick< HistoricalSourceGraphInput, | "entries" | "relationships" | "parse_failures" | "exclusions" | "limitations" >, ): "complete" | "partial" | "unknown" => { if (input.limitations.length > 0) return "partial"; if (input.exclusions.length > 0) return "partial"; if (input.parse_failures.length > 0) return "partial"; const partialEntry = input.entries.some( (entry) => entry.limitations.length > 0 || (entry.kind === "file" && entry.content_state !== "hashed") || (entry.kind === "directory" && entry.tree_state !== "enumerated") || (entry.kind === "symlink" && entry.target_state !== "internal") || entry.kind === "unknown", ); if (partialEntry) return "partial"; const partialRelationship = input.relationships.some( ({ parse_state, resolution }) => parse_state !== "parsed" || ["unresolved", "unknown"].includes(resolution), ); if (partialRelationship) return "partial"; return "complete"; }; const sortExclusions = ( exclusions: HistoricalSourceGraphInput["exclusions"], ): HistoricalSourceGraphInput["exclusions"] => [...exclusions].sort((left, right) => { const byPath = compareUnicodeCodePoints(left.path, right.path); if (byPath !== 0) return byPath; const byReason = compareUnicodeCodePoints(left.reason, right.reason); if (byReason !== 0) return byReason; return compareUnicodeCodePoints( "pattern" in left ? left.pattern : "", "pattern" in right ? right.pattern : "", ); }); const createShouldExclude = ( exclusions: HistoricalSourceGraphInput["exclusions"], secrets: PreparedReferenceSourceImport["secrets"], ignored: PreparedReferenceSourceImport["ignored"], ): ((path: string, kind: ReferenceSourceExclusionKind) => boolean) => (path, kind) => { // An entry of unknown kind may be a directory, so directory-only rules // must still exclude it; this fails closed for caller-marked secrets. const patternPaths = kind === "directory" ? [`${path}/`] : kind === "unknown" ? [path, `${path}/`] : [path]; const firstMatch = (matcher: typeof secrets) => { const results = patternPaths.map((patternPath) => matcher.test(patternPath), ); return results.find(({ ignored }) => ignored) ?? results[0]; }; const secretMatch = firstMatch(secrets); if (secretMatch?.ignored === true) { if (!secretMatch.rule) throw new Error(`Ignored secret path has no matching rule: ${path}`); exclusions.push({ path, entry_kind: kind, reason: "configured-secret", pattern: secretMatch.rule.pattern, }); return true; } const match = firstMatch(ignored); if (match?.ignored !== true) return false; if (!match.rule) throw new Error(`Ignored path has no matching rule: ${path}`); const reason = match.rule.mark; if ( reason !== "project-ignored" && reason !== "default-ignored" && reason !== "caller-excluded" ) throw new Error(`Ignored path has unknown rule origin: ${path}`); exclusions.push({ path, entry_kind: kind, reason, pattern: match.rule.pattern, }); return true; }; /** * Import a reference source directory into a committed historical source graph. * * The import is deterministic, parallel-safe, and never executes source, hooks, * git subprocesses, or network requests. Paths explicitly excluded by the * caller's reference-source policy are omitted from the graph. */ export const importReferenceSource = async ( options: ReferenceSourceImportOptions, resources: ArtifactResourceScope, ): Promise> => { if (isAborted(options.signal)) return err(cancelled()); const prepared = await prepareReferenceSourceImport(options, resources); if (!prepared.ok) return prepared; const { ignored, root, secrets } = prepared.value; if (isAborted(options.signal)) return err(cancelled()); const exclusions: HistoricalSourceGraphInput["exclusions"] = []; const shouldExclude = createShouldExclude(exclusions, secrets, ignored); const [readResult, vcsResult] = await Promise.all([ readReferenceSource(root, resources, { ...(options.signal === undefined ? {} : { signal: options.signal }), shouldExclude, }), readReferenceSourceVcs(root, resources, options.signal), ]); if (!readResult.ok) return err( combineReadFailures( readResult.error, vcsResult.ok ? undefined : vcsResult.error, readResult.error.partial, ), ); if (!vcsResult.ok) return err( combineReadFailures(vcsResult.error, undefined, readResult.value), ); const vcs = vcsResult.value; if (isAborted(options.signal)) return err(cancelled(readResult.value)); const read = readResult.value; const filePaths = new Set( read.entries .filter((entry) => entry.status === "read" && entry.kind === "file") .map((entry) => entry.path), ); const { entries, relationships, parseFailures, limitations } = parseReferenceSourceEntries(read, filePaths, options.signal); if (isAborted(options.signal)) return err(cancelled(read)); const uniqueRelationships = deduplicateRelationships(relationships); const uniqueFailures = normalizeHistoricalSourceParseFailures(parseFailures); const sortedEntries = [...entries].sort((left, right) => compareUnicodeCodePoints(left.path, right.path), ); const sortedExclusions = sortExclusions(exclusions); const sortedLimitations = [...limitations].sort(compareUnicodeCodePoints); const input: HistoricalSourceGraphInput = { schema: "HistoricalSourceGraph", authority: "historical-reference", root_alias: "$REFERENCE_ROOT", inventory_state: deriveInventoryState({ entries: sortedEntries, relationships: uniqueRelationships, parse_failures: uniqueFailures, exclusions: sortedExclusions, limitations: sortedLimitations, }), entries: sortedEntries, relationships: uniqueRelationships.sort((left, right) => compareUnicodeCodePoints(relationshipKey(left), relationshipKey(right)), ), parse_failures: uniqueFailures, exclusions: sortedExclusions, languages: historicalSourceLanguages(sortedEntries), manifests: historicalSourceManifests(sortedEntries), vcs, provenance: buildProvenance(options), limitations: sortedLimitations, }; try { return ok(createHistoricalSourceGraph(input)); } catch (cause: unknown) { return err( failure( "parse", cause instanceof Error ? cause.message : "Graph failed validation", ), ); } }; const combineReadFailures = ( primary: ReferenceSourceReaderError, other: ReferenceSourceReaderError | undefined, partial: ReferenceSourceRead | undefined, ): ReferenceSourceImportError => { const code = primary.code === "cancelled" || other?.code === "cancelled" ? "cancelled" : primary.code; const cleanup = mergeCleanup(primary.cleanup, other?.cleanup); const message = other === undefined ? primary.message : `${primary.message}; additional reference metadata failure: ${other.message}`; const cause = other === undefined ? primary.cause : new AggregateError( [primary, other], "Reference source inventory and metadata both failed", { cause: primary }, ); return { ...failure(code, message), ...(cleanup === undefined ? {} : { cleanup }), ...(partial === undefined ? {} : { partial }), ...(cause === undefined ? {} : { cause }), }; }; const mergeCleanup = ( first: AnalysisCleanupObservation | undefined, second: AnalysisCleanupObservation | undefined, ): AnalysisCleanupObservation | undefined => { if (first === undefined) return second; if (second === undefined) return first; return { reason: `${first.reason}; ${second.reason}`, resources: [...new Set([...first.resources, ...second.resources])], }; };