diff --git a/docs/cli.md b/docs/cli.md index 2d7567d06..dcd949e11 100644 --- a/docs/cli.md +++ b/docs/cli.md @@ -215,7 +215,9 @@ rea import-reference-source /absolute/path/to/source ``` The import records hashes and metadata for the supplied files, separately -from observations of the current app. It honors the source tree's `.gitignore` +from observations of the current app. For a directory inside a Git checkout, +it records the HEAD of the nearest enclosing repository; whether the working +tree matches that commit remains unknown. It honors the source tree's `.gitignore` and applies default exclusions for common generated and dependency paths such as `node_modules/`, `dist/`, and `*.log`. Pattern-based exclusions record the matched pattern and whether it came from project, default, caller, or sensitive diff --git a/src/application/ReferenceSourceImportEntries.ts b/src/application/ReferenceSourceImportEntries.ts index c7432c68d..3254d7cfb 100644 --- a/src/application/ReferenceSourceImportEntries.ts +++ b/src/application/ReferenceSourceImportEntries.ts @@ -109,12 +109,18 @@ const resolveInternalSpecifier = ( : { to: match, resolution: "internal" }; }; +type ReadFile = Extract; + const appendFile = ( - entry: Extract, + entry: ReadFile, lookupPath: ReferenceSourcePathLookup, output: ParsedReferenceSourceEntries, ): void => { const language = detectReferenceSourceLanguage(entry.path); + const parsed = + language === null || !PARSEABLE_REFERENCE_SOURCE_LANGUAGES.has(language) + ? undefined + : parseReferenceSourceImports(entry.path, entry.bytes, language); output.entries.push({ path: entry.path, kind: "file", @@ -123,11 +129,9 @@ const appendFile = ( language, classifications: classifyReferenceSourcePath(entry.path), content_state: "hashed", - limitations: [], + limitations: [...(parsed?.limitations ?? [])], }); - if (language === null || !PARSEABLE_REFERENCE_SOURCE_LANGUAGES.has(language)) - return; - const parsed = parseReferenceSourceImports(entry.path, entry.bytes, language); + if (parsed === undefined) return; for (const relationship of parsed.relationships) { if (relationship.resolution !== "internal") { output.relationships.push(relationship); diff --git a/src/application/ReferenceSourceVcsAdapter.ts b/src/application/ReferenceSourceVcsAdapter.ts index b2a21e2b5..346ae6ebe 100644 --- a/src/application/ReferenceSourceVcsAdapter.ts +++ b/src/application/ReferenceSourceVcsAdapter.ts @@ -1,6 +1,6 @@ import fs from "node:fs"; import { lstat } from "node:fs/promises"; -import { join, resolve } from "node:path"; +import { dirname, join, resolve } from "node:path"; import { err, ok, type Result } from "../domain/result.js"; import type { ArtifactResourceScope } from "../artifacts/ArtifactResourceScope.js"; import { ArtifactReaderFailure } from "../artifacts/ArtifactReader.js"; @@ -32,18 +32,24 @@ export const readReferenceSourceVcs = async ( ): Promise> => { try { return await resources.run(async () => { - if (isAborted(signal)) return ok(unknownVcs()); - try { - await lstat(join(root, ".git")); - } catch (cause: unknown) { - return ok( - errorCode(cause) === "ENOENT" - ? { kind: "none", head: null, dirty: null } - : unknownVcs(), - ); + // Like Git, a subdirectory belongs to its nearest enclosing repository; + // only a missing `.git` at every ancestor means no version control. + let repository = resolve(root); + for (;;) { + if (isAborted(signal)) return ok(unknownVcs()); + try { + await lstat(join(repository, ".git")); + break; + } catch (cause: unknown) { + if (errorCode(cause) !== "ENOENT") return ok(unknownVcs()); + } + const parent = dirname(repository); + if (parent === repository) + return ok({ kind: "none", head: null, dirty: null }); + repository = parent; } try { - const head = await resolveSourceHead(root, resources, signal); + const head = await resolveSourceHead(repository, resources, signal); if (isAborted(signal)) return ok(unknownVcs()); return ok({ kind: "git", head, dirty: null }); } catch (cause: unknown) { diff --git a/src/cli/utilityCommands.ts b/src/cli/utilityCommands.ts index 9c380eaff..33099ee28 100644 --- a/src/cli/utilityCommands.ts +++ b/src/cli/utilityCommands.ts @@ -255,7 +255,7 @@ const runReferenceSourceImport = async ( caller: "rea-cli", policy, importer: PRODUCT_IDENTITY.packageName, - importerVersion: null, + importerVersion: PRODUCT_IDENTITY.packageVersion, }, resources, ), diff --git a/src/domain/javascript/javascriptSourceParser.ts b/src/domain/javascript/javascriptSourceParser.ts index 6a8016887..ce5619ead 100644 --- a/src/domain/javascript/javascriptSourceParser.ts +++ b/src/domain/javascript/javascriptSourceParser.ts @@ -1,4 +1,4 @@ -import { parse, type ParserPlugin } from "@babel/parser"; +import { parse, type ParserOptions, type ParserPlugin } from "@babel/parser"; /** Babel AST produced by REA's inert JavaScript parser boundary. */ export type ParsedJavaScriptSource = ReturnType; @@ -37,6 +37,67 @@ export const parserPluginsForPath = (sourcePath?: string): ParserPlugin[] => { : ["decorators-legacy", typescript]; }; +/** `.mjs` and `.mts` are modules by extension; other paths leave the mode to syntax. */ +export const isExplicitModuleSourcePath = (path?: string): boolean => + /\.m[jt]s$/iu.test(path ?? ""); + +export interface RecoveredJavaScriptParse { + readonly file: ParsedJavaScriptSource; + /** Errors that apply only if a source of unknown mode is an ES module. */ + readonly moduleOnlyErrors: readonly string[]; +} + +/** + * Parse a module or script with recovery. Babel tries module mode first and + * keeps its strict-mode errors after falling back to a script, so a script + * with errors is parsed again as a script. Strict errors that only the module + * parse reports stay visible because the source may still be a module. + */ +export const parseRecoveredJavaScript = ( + source: string, + sourcePath: string | undefined, + options: Omit, +): RecoveredJavaScriptParse => { + // `.cjs` is CommonJS by extension, so module-only errors cannot apply. + if (/\.cjs$/iu.test(sourcePath ?? "")) + return { + file: parse(source, { + ...options, + sourceType: "script", + errorRecovery: true, + }), + moduleOnlyErrors: [], + }; + if (isExplicitModuleSourcePath(sourcePath)) + return { + file: parse(source, { + ...options, + sourceType: "module", + errorRecovery: true, + }), + moduleOnlyErrors: [], + }; + const file = parse(source, { + ...options, + sourceType: "unambiguous", + errorRecovery: true, + }); + if (file.program.sourceType !== "script" || file.errors.length === 0) + return { file, moduleOnlyErrors: [] }; + const script = parse(source, { + ...options, + sourceType: "script", + errorRecovery: true, + }); + const scriptErrors = new Set(script.errors.map((error) => error.message)); + return { + file: script, + moduleOnlyErrors: file.errors + .map((error) => error.message) + .filter((message) => !scriptErrors.has(message)), + }; +}; + /** Parse JavaScript or TypeScript once without attaching comments to AST nodes. */ export const parseJavaScriptSource = ( source: string, diff --git a/src/domain/referenceSourceClassification.ts b/src/domain/referenceSourceClassification.ts index c567cf06a..ea53a18a4 100644 --- a/src/domain/referenceSourceClassification.ts +++ b/src/domain/referenceSourceClassification.ts @@ -37,6 +37,8 @@ const CODE_EXTENSIONS = new Set([ "m", "mm", "sh", + "cs", + "php", ]); const LANGUAGE_BY_EXTENSION = new Map([ @@ -59,6 +61,8 @@ const LANGUAGE_BY_EXTENSION = new Map([ ["cc", "C++"], ["cxx", "C++"], ["swift", "Swift"], + ["cs", "C#"], + ["php", "PHP"], ["m", "Objective-C"], ["mm", "Objective-C++"], ["sh", "Shell"], diff --git a/src/domain/referenceSourceImportParsing.ts b/src/domain/referenceSourceImportParsing.ts index 62ad226dd..998e368c4 100644 --- a/src/domain/referenceSourceImportParsing.ts +++ b/src/domain/referenceSourceImportParsing.ts @@ -1,4 +1,3 @@ -import { parse } from "@babel/parser"; import type { CallExpression, File, @@ -7,7 +6,10 @@ import type { StringLiteral, } from "@babel/types"; import { traverseJavaScriptAst } from "./javascript/javascriptSemanticTraversal.js"; -import { parserPluginsForPath } from "./javascript/javascriptSourceParser.js"; +import { + parseRecoveredJavaScript, + parserPluginsForPath, +} from "./javascript/javascriptSourceParser.js"; import { unshadowedReferenceSourceRequireCalls } from "./referenceSourceRequireBindings.js"; import { isCallExpression, @@ -55,6 +57,7 @@ interface ReferenceSourceParseFailure { export interface ReferenceSourceImportParseResult { readonly relationships: readonly ReferenceSourceImportRelationship[]; readonly parse_failures: readonly ReferenceSourceParseFailure[]; + readonly limitations: readonly string[]; } const codeSourceLanguages = new Set(["JavaScript", "TypeScript", "JSX", "TSX"]); @@ -272,6 +275,7 @@ const parseWithBabel = ( ): { ast: File | undefined; reasons: readonly string[]; + moduleOnlyErrors: readonly string[]; } => { // Single parser-mode owner: path-driven plugins (dts/JSX/mts) shared with // the semantic pipeline. `language` only gates non-JS/TS callers upstream; @@ -280,19 +284,26 @@ const parseWithBabel = ( const plugins = parserPluginsForPath(path); try { - const ast = parse(source, { - sourceType: "unambiguous", - allowImportExportEverywhere: false, - allowReturnOutsideFunction: true, - allowUndeclaredExports: true, - errorRecovery: true, - plugins, - }); - return { ast, reasons: ast.errors.map((error) => error.message) }; + const { file: ast, moduleOnlyErrors } = parseRecoveredJavaScript( + source, + path, + { + allowImportExportEverywhere: false, + allowReturnOutsideFunction: true, + allowUndeclaredExports: true, + plugins, + }, + ); + return { + ast, + reasons: ast.errors.map((error) => error.message), + moduleOnlyErrors, + }; } catch (cause: unknown) { return { ast: undefined, reasons: [cause instanceof Error ? cause.message : "Parse failed"], + moduleOnlyErrors: [], }; } }; @@ -304,7 +315,7 @@ export const parseReferenceSourceImports = ( language: string | null, ): ReferenceSourceImportParseResult => { if (language !== null && !codeSourceLanguages.has(language)) - return { relationships: [], parse_failures: [] }; + return { relationships: [], parse_failures: [], limitations: [] }; let source: string; try { @@ -320,9 +331,20 @@ export const parseReferenceSourceImports = ( "Source bytes are not valid UTF-8; import targets were not parsed.", }, ], + limitations: [], }; } - const { ast, reasons } = parseWithBabel(path, source, language); + const { ast, reasons, moduleOnlyErrors } = parseWithBabel( + path, + source, + language, + ); + const limitations = + moduleOnlyErrors.length === 0 + ? [] + : [ + `Strict-mode errors apply only if this file is an ES module: ${moduleOnlyErrors.join("; ")}`, + ]; const parseFailures = reasons.map((reason) => ({ path, parser: "babel", @@ -333,6 +355,7 @@ export const parseReferenceSourceImports = ( return { relationships: [], parse_failures: parseFailures, + limitations, }; } @@ -354,5 +377,6 @@ export const parseReferenceSourceImports = ( : relationship.parse_state, })), parse_failures: parseFailures, + limitations, }; }; diff --git a/src/domain/referenceSourceRequireBindings.ts b/src/domain/referenceSourceRequireBindings.ts index c6df25180..f2f8d1ef5 100644 --- a/src/domain/referenceSourceRequireBindings.ts +++ b/src/domain/referenceSourceRequireBindings.ts @@ -7,7 +7,10 @@ import { } from "./javascript/javascriptSemanticState.js"; import { traverseJavaScriptAst } from "./javascript/javascriptSemanticTraversal.js"; import type { JavaScriptSemanticReference } from "./javascript/javascriptSemanticIr.js"; -import type { ParsedJavaScriptSource } from "./javascript/javascriptSourceParser.js"; +import { + isExplicitModuleSourcePath, + type ParsedJavaScriptSource, +} from "./javascript/javascriptSourceParser.js"; /** * Keep calls that resolve either to Node's CommonJS wrapper loader or to a @@ -29,7 +32,10 @@ export const unshadowedReferenceSourceRequireCalls = ( semanticScopeId("program", file.program), "require", ); - const allowsUnboundRequire = !isExplicitModuleSource(path); + // Syntax alone cannot establish Node's runtime mode: `.js` depends on its + // package scope, and `.ts` may be compiled with a CommonJS module target. + // Keep unmarked paths runtime-unknown and preserve candidate require edges. + const allowsUnboundRequire = !isExplicitModuleSourcePath(path); const hasTopLevelVarRequire = allowsUnboundRequire && hasUninitializedTopLevelVarRequire(file.program); const createRequireBinding = directCreateRequireBinding( @@ -82,16 +88,6 @@ const contains = (container: t.Node, node: t.Node): boolean => container.start <= node.start && node.end <= container.end; -const isExplicitModuleSource = (path: string): boolean => { - const lower = path.toLowerCase(); - if (lower.endsWith(".mjs") || lower.endsWith(".mts")) return true; - if (lower.endsWith(".cjs") || lower.endsWith(".cts")) return false; - // Syntax alone cannot establish Node's runtime mode: `.js` depends on its - // package scope, and `.ts` may be compiled with a CommonJS module target. - // Keep unmarked paths runtime-unknown and preserve candidate require edges. - return false; -}; - const hasUninitializedTopLevelVarRequire = (program: t.Program): boolean => { let found = false; let initialized = false; diff --git a/tests/boundary/cli/referenceSourceLegacyFacts.test.ts b/tests/boundary/cli/referenceSourceLegacyFacts.test.ts new file mode 100644 index 000000000..d254a04a0 --- /dev/null +++ b/tests/boundary/cli/referenceSourceLegacyFacts.test.ts @@ -0,0 +1,90 @@ +import { mkdir, writeFile } from "node:fs/promises"; +import { join } from "node:path"; + +import { expect } from "vitest"; + +import { historicalSourceGraphSchema } from "../../../src/domain/referenceSourceGraph.js"; +import { PRODUCT_IDENTITY } from "../../../src/identity.js"; +import { execFileOutput } from "../../../src/process/ExecFileOutput.js"; +import { createTestTempDirectory } from "../../fixtures/temporaryDirectory.js"; +import { cliTest } from "../../support/cli/cliFixture.js"; + +cliTest( + "imports a legacy subtree of a Git checkout with its own source facts", + async ({ cli }) => { + const repository = await createTestTempDirectory("rea-reference-legacy-"); + const source = join(repository, "source"); + await mkdir(source); + await Promise.all([ + // `.cjs` is CommonJS, so the strict-mode rule does not apply. + writeFile( + join(source, "legacy.cjs"), + "var messages = {};\ndelete messages;\n", + ), + // Deleting a plain binding is valid only in a sloppy-mode script. + writeFile( + join(source, "legacy.js"), + "var messages = {};\ndelete messages;\n", + ), + // An explicit module keeps the strict-mode error. + writeFile( + join(source, "module.mjs"), + "var messages = {};\ndelete messages;\n", + ), + writeFile(join(source, "index.php"), " + execFileOutput("git", args, { cwd: repository }); + await git("init", "--initial-branch=main"); + await git("add", "."); + await git( + "-c", + "user.name=REA fixture", + "-c", + "user.email=fixture@example.invalid", + "-c", + "commit.gpgsign=false", + "commit", + "-m", + "fixture", + ); + const head = (await git("rev-parse", "HEAD")).stdout.trim(); + + const result = await cli.run({ + arguments: ["import-reference-source", source, "--json"], + environment: { + HOME: repository, + XDG_CONFIG_HOME: repository, + XDG_CACHE_HOME: repository, + }, + }); + + expect(result.exitCode).toBe(0); + const graph = historicalSourceGraphSchema.parse(result.json); + expect(graph.parse_failures).toEqual([ + { + path: "module.mjs", + parser: "babel", + reason: "Deleting local variable in strict mode. (2:0)", + }, + ]); + // Outside `.mjs`/`.mts`/`.cjs` the mode stays unknown, not forced to script. + const limitations = (path: string) => + graph.entries.find((entry) => entry.path === path)?.limitations; + for (const path of ["legacy.js"]) + expect(limitations(path)).toEqual([ + "Strict-mode errors apply only if this file is an ES module: Deleting local variable in strict mode. (2:0)", + ]); + expect(limitations("legacy.cjs")).toEqual([]); + expect(graph.languages).toEqual(["C#", "JavaScript", "PHP"]); + for (const path of ["Program.cs", "index.php"]) + expect( + graph.entries.find((entry) => entry.path === path)?.classifications, + ).toContain("source"); + expect(graph.vcs).toEqual({ kind: "git", head, dirty: null }); + expect(graph.provenance.importer_version).toBe( + PRODUCT_IDENTITY.packageVersion, + ); + }, +);