fix(reference): preserve distinct parse failures

This commit is contained in:
morluto 2026-07-14 22:24:03 +00:00
commit e02cb2f877
4 changed files with 78 additions and 14 deletions

View File

@ -1,6 +1,7 @@
import { isSecretLikePath } from "../domain/referenceSourceClassification.js";
import {
createHistoricalSourceGraph,
historicalSourceParseFailureKey,
type HistoricalSourceGraph,
type HistoricalSourceGraphInput,
} from "../domain/referenceSourceGraph.js";
@ -66,6 +67,27 @@ const deduplicateRelationships = (
return result;
};
/** Sort parse failures deterministically while retaining distinct reasons. */
export const normalizeHistoricalSourceParseFailures = (
failures: HistoricalSourceGraphInput["parse_failures"],
): HistoricalSourceGraphInput["parse_failures"] =>
[...failures]
.sort((left, right) =>
byCodePoint(
historicalSourceParseFailureKey(left),
historicalSourceParseFailureKey(right),
),
)
.filter((value, index, array) => {
if (index === 0) return true;
const previous = array[index - 1];
return (
previous === undefined ||
historicalSourceParseFailureKey(value) !==
historicalSourceParseFailureKey(previous)
);
});
const buildProvenance = (
options: ReferenceSourceImportOptions,
): HistoricalSourceGraphInput["provenance"] => ({
@ -204,18 +226,7 @@ export const importReferenceSource = async (
if (isAborted(options.signal)) return err(cancelled());
const uniqueRelationships = deduplicateRelationships(relationships);
const uniqueFailures = [...parseFailures]
.sort((left, right) => {
const byPath = byCodePoint(left.path, right.path);
if (byPath !== 0) return byPath;
return byCodePoint(left.parser, right.parser);
})
.filter((value, index, array) => {
if (index === 0) return true;
const previous = array[index - 1];
if (previous === undefined) return true;
return value.path !== previous.path || value.parser !== previous.parser;
});
const uniqueFailures = normalizeHistoricalSourceParseFailures(parseFailures);
const sortedEntries = [...entries].sort((left, right) =>
byCodePoint(left.path, right.path),

View File

@ -225,6 +225,11 @@ export type HistoricalSourceManifest = z.infer<
typeof historicalSourceManifestBaseSchema
>;
/** Stable identity used to sort and deduplicate source parse failures. */
export const historicalSourceParseFailureKey = (
failure: HistoricalSourceGraphInput["parse_failures"][number],
): string => `${failure.path}\u0000${failure.parser}\u0000${failure.reason}`;
const canonicalJson = (value: unknown): string => {
const serialized = canonicalize(value);
if (serialized === undefined)
@ -400,7 +405,7 @@ function checkGraphInvariants(
context,
);
checkSortedUnique(
graph.parse_failures.map(({ path, parser }) => `${path}\u0000${parser}`),
graph.parse_failures.map(historicalSourceParseFailureKey),
"parse_failures",
context,
);

View File

@ -5,6 +5,7 @@ import {
createHistoricalSourceGraph,
createHistoricalSourceManifest,
historicalSourceGraphSchema,
historicalSourceParseFailureKey,
parseHistoricalSourceGraph,
parseHistoricalSourceManifest,
type HistoricalSourceGraphInput,
@ -192,6 +193,29 @@ describe("historical source graph", () => {
).toThrow(/Complete inventory/u);
});
it("keeps distinct parse failure reasons for the same path and parser", () => {
const input = graphInput();
const malformed = {
path: "src/main.ts",
parser: "typescript",
reason: "Malformed input",
};
const unexpected = {
path: "src/main.ts",
parser: "typescript",
reason: "Unexpected token",
};
input.inventory_state = "partial";
input.parse_failures = [malformed, unexpected];
expect(createHistoricalSourceGraph(input).parse_failures).toEqual(
input.parse_failures,
);
expect(historicalSourceParseFailureKey(malformed)).not.toBe(
historicalSourceParseFailureKey(unexpected),
);
});
it("rejects strict unknown fields, invalid VCS, dangling internal edges, and false manifests", () => {
expect(() =>
createHistoricalSourceGraph({ ...graphInput(), surprise: true }),

View File

@ -6,7 +6,10 @@ import { join } from "node:path";
import { add, commit, init } from "isomorphic-git";
import { describe, expect, it } from "vitest";
import { importReferenceSource } from "../src/application/ReferenceSourceImport.js";
import {
importReferenceSource,
normalizeHistoricalSourceParseFailures,
} from "../src/application/ReferenceSourceImport.js";
import { projectReferenceSourceEntryFailure } from "../src/application/ReferenceSourceImportEntries.js";
import {
projectReferenceSourceImportError,
@ -52,6 +55,27 @@ const importTree = (
});
describe("reference source import", () => {
it("deduplicates exact parse failures without collapsing distinct reasons", () => {
const malformed = {
path: "src/main.ts",
parser: "babel",
reason: "Malformed input",
};
const unexpected = {
path: "src/main.ts",
parser: "babel",
reason: "Unexpected token",
};
expect(
normalizeHistoricalSourceParseFailures([
unexpected,
malformed,
unexpected,
]),
).toEqual([malformed, unexpected]);
});
it("projects entry failures without low-level reader diagnostics", () => {
for (const [kind, code] of [
["directory", "io"],