fix(javascript): retain returned-reference mutation uncertainty (#1471)

docs/javascript-application-workflows.md

@@ -186,7 +186,10 @@ keys, and parser recovery remain partial or unknown.
 Objects and arrays passed to calls, constructors, or tagged templates, used
 as method receivers, or stored through property targets are not assumed
 unchanged afterward. Property stores conservatively retain reference escape
-uncertainty rather than pretending to execute later writes. Aliases
+uncertainty rather than pretending to execute later writes. References
+handed back by local functions, methods, constructors, async functions, and
+generators also retain escape uncertainty, including when the call is read-only;
+this does not establish that a mutation occurred. Aliases
 and shared children in spread and rest copies retain that uncertainty; copied
 primitive slots and unrelated containing properties remain known. Object rest
 excludes consumed keys, array rest excludes the consumed prefix for known

src/domain/javascript/javascriptSemanticCallableResults.ts

@@ -0,0 +1,53 @@
+import * as t from "@babel/types";
+
+import { childNodes } from "./javascriptSemanticTraversal.js";
+
+/** One direct return or yield, excluding every nested callable. */
+export interface SemanticCallableResultExpression {
+  readonly node: t.Node | null;
+  readonly site: t.Node;
+  readonly kind: "return" | "yield";
+  readonly delegated: boolean;
+}
+
+/** Collect direct result syntax without executing or evaluating a callable. */
+export const semanticCallableResultExpressions = (
+  callable: t.Node,
+): readonly SemanticCallableResultExpression[] => {
+  if (
+    t.isArrowFunctionExpression(callable) &&
+    !t.isBlockStatement(callable.body)
+  )
+    return [
+      {
+        node: callable.body,
+        site: callable.body,
+        kind: "return",
+        delegated: false,
+      },
+    ];
+  if (!t.isFunction(callable)) return [];
+  const output: SemanticCallableResultExpression[] = [];
+  const pending = childNodes(callable.body).reverse();
+  while (pending.length > 0) {
+    const node = pending.pop();
+    if (node === undefined || t.isFunction(node) || t.isClass(node)) continue;
+    if (t.isReturnStatement(node)) {
+      output.push({
+        node: node.argument ?? null,
+        site: node,
+        kind: "return",
+        delegated: false,
+      });
+    }
+    if (t.isYieldExpression(node))
+      output.push({
+        node: node.argument ?? null,
+        site: node,
+        kind: "yield",
+        delegated: node.delegate,
+      });
+    for (const child of childNodes(node).reverse()) pending.push(child);
+  }
+  return output;
+};

src/domain/javascript/javascriptSemanticMemberMutations.ts

@@ -15,6 +15,8 @@ import {
   semanticSlotAtPath,
 } from "./javascriptSemanticSlots.js";
 import { semanticIterationSources } from "./javascriptSemanticIterationSources.js";
+import { semanticCallableResultExpressions } from "./javascriptSemanticCallableResults.js";
+import { createSemanticReturnedReferences } from "./javascriptSemanticReturnedReferences.js";
 import { semanticMutationInitializers } from "./javascriptSemanticMutationInitializers.js";
 import {
   semanticArrayIndex,
@@ -44,6 +46,7 @@ export const collectSemanticMemberMutations = (
     },
   });
   const normalizePath = observableMutationPaths(state);
+  const returnedReferences = createSemanticReturnedReferences(state);
   const recordedEffects = new Set<string>();
   const selections = new Map<string, readonly string[]>();
   const coveringPaths = new Map<string, SemanticPropertyPathCoverage>();
@@ -340,6 +343,8 @@ export const collectSemanticMemberMutations = (
         t.isOptionalCallExpression(node) ||
         t.isNewExpression(node)
       ) {
+        for (const source of returnedReferences(node.callee))
+          markEscaped(source.node, node, source.projection);
         for (const argument of node.arguments)
           if (t.isSpreadElement(argument))
             deferIterable({
@@ -555,6 +560,11 @@ const referencedValues = (
   path: PropertyPath,
   effect: ValueEffect,
 ): readonly ReferencedValue[] => {
+  if (effect === "escape" && t.isFunction(node))
+    return semanticCallableResultExpressions(node).flatMap(
+      ({ node, delegated }) =>
+        node === null ? [] : [{ node, path: delegated ? [null] : [] }],
+    );
   // An initializer owns its slots; only deeper writes can affect shared children.
   if (t.isObjectExpression(node) || t.isArrayExpression(node)) {
     if (effect === "write" && path.length < 2) return [];
@@ -855,11 +865,11 @@ const objectReferencedValues = (
     }
     const name = semanticStaticPropertyKey(property.key, property.computed);
     if (name !== null && overwritten.has(name)) continue;
-    if (
-      t.isObjectProperty(property) &&
-      (name === null || semanticPropertyPathKeyMatches(key ?? null, name))
-    )
-      references.push({ node: property.value, path: remaining });
+    if (name === null || semanticPropertyPathKeyMatches(key ?? null, name))
+      references.push({
+        node: t.isObjectProperty(property) ? property.value : property,
+        path: remaining,
+      });
     // A prototype setter does not replace an own property from a spread.
     if (
       name !== null &&

src/domain/javascript/javascriptSemanticReturnedReferences.ts

@@ -0,0 +1,213 @@
+import * as t from "@babel/types";
+
+import {
+  semanticStaticPropertyKey,
+  unwrapJavaScriptExpression,
+} from "./javascriptAstValues.js";
+import {
+  resolveLocalCallables,
+  type LocalCallableResolution,
+} from "./javascriptSemanticCallResolution.js";
+import { semanticCallableResultExpressions } from "./javascriptSemanticCallableResults.js";
+import {
+  semanticArrayIndex,
+  type JavaScriptSemanticPropertyPath,
+} from "./javascriptSemanticPropertyPaths.js";
+import {
+  resolveSemanticBindingState,
+  type JavaScriptSemanticAnalysisState,
+} from "./javascriptSemanticState.js";
+
+/** A reference potentially handed to a caller or generator consumer. */
+export interface SemanticReturnedReference {
+  readonly node: t.Node;
+  readonly projection: JavaScriptSemanticPropertyPath;
+}
+
+interface Candidate {
+  readonly node: t.Node;
+  readonly path: readonly (string | number | null)[];
+  readonly bindings: ReadonlySet<string>;
+}
+
+/** Resolve local result references conservatively, without claiming call execution. */
+export const createSemanticReturnedReferences = (
+  state: JavaScriptSemanticAnalysisState,
+): ((callee: t.Node) => readonly SemanticReturnedReference[]) => {
+  const callableById = new Map(
+    state.callables.map((callable) => [callable.callableId, callable]),
+  );
+  const bindingCache = new Map<string, LocalCallableResolution>();
+  const results = new WeakMap<t.Node, readonly SemanticReturnedReference[]>();
+  return (callee) => {
+    const existing = results.get(callee);
+    if (existing !== undefined) return existing;
+    const functions = new Set<t.Node>();
+    const resolution = resolveLocalCallables({
+      node: callee,
+      state,
+      callableById,
+      bindingCache,
+      seenBindings: new Set(),
+    });
+    for (const id of resolution.callableIds) {
+      const node = state.callableNodesById.get(id);
+      if (node !== undefined) functions.add(node);
+    }
+    const pending: Candidate[] = [
+      { node: callee, path: [], bindings: new Set() },
+    ];
+    for (let cursor = 0; cursor < pending.length; cursor++) {
+      const item = pending[cursor];
+      if (item === undefined) break;
+      const node = unwrapJavaScriptExpression(item.node).node;
+      if (t.isIdentifier(node)) {
+        const binding = resolveSemanticBindingState(state, node, node.name);
+        if (binding === undefined || item.bindings.has(binding.bindingId))
+          continue;
+        const bindings = new Set([...item.bindings, binding.bindingId]);
+        for (const initializer of [
+          ...binding.initializers,
+          ...binding.referenceInitializers,
+        ])
+          pending.push({
+            node: initializer.node,
+            path: [...initializer.projection, ...item.path],
+            bindings,
+          });
+      } else if (
+        t.isMemberExpression(node) ||
+        t.isOptionalMemberExpression(node)
+      ) {
+        const path: (string | number | null)[] = [];
+        let receiver: t.Node = node;
+        while (
+          t.isMemberExpression(receiver) ||
+          t.isOptionalMemberExpression(receiver)
+        ) {
+          path.push(
+            semanticStaticPropertyKey(receiver.property, receiver.computed),
+          );
+          receiver = unwrapJavaScriptExpression(receiver.object).node;
+        }
+        pending.push({
+          ...item,
+          node: receiver,
+          path: [...path.reverse(), ...item.path],
+        });
+      } else if (
+        t.isConditionalExpression(node) ||
+        t.isLogicalExpression(node)
+      ) {
+        pending.push(
+          {
+            ...item,
+            node: t.isConditionalExpression(node) ? node.consequent : node.left,
+          },
+          {
+            ...item,
+            node: t.isConditionalExpression(node) ? node.alternate : node.right,
+          },
+        );
+      } else if (t.isSequenceExpression(node)) {
+        const last = node.expressions.at(-1);
+        if (last !== undefined) pending.push({ ...item, node: last });
+      } else if (t.isFunction(node) && item.path.length === 0)
+        functions.add(node);
+      else if (t.isObjectExpression(node))
+        selectObjectTargets(node, item, pending);
+      else if (t.isArrayExpression(node)) {
+        const [key, ...path] = item.path;
+        const index =
+          key === undefined || key === null ? null : semanticArrayIndex(key);
+        let uncertainOffset = false;
+        node.elements.forEach((element, offset) => {
+          if (element === null) return;
+          if (t.isSpreadElement(element)) {
+            uncertainOffset = true;
+            pending.push({
+              ...item,
+              node: element.argument,
+              path: [null, ...path],
+            });
+          } else if (index === null || uncertainOffset || index === offset)
+            pending.push({ ...item, node: element, path });
+        });
+      } else if (t.isNewExpression(node))
+        pending.push({ ...item, node: node.callee });
+      else if (t.isClass(node)) {
+        const [key, ...path] = item.path;
+        for (const member of node.body.body) {
+          if (t.isClassMethod(member)) {
+            if (
+              (item.path.length === 0 && member.kind === "constructor") ||
+              (key !== undefined &&
+                (key === null ||
+                  semanticStaticPropertyKey(member.key, member.computed) ===
+                    String(key)))
+            )
+              pending.push({ ...item, node: member, path });
+          } else if (
+            t.isClassProperty(member) &&
+            member.value !== null &&
+            member.value !== undefined &&
+            key !== undefined &&
+            (key === null ||
+              semanticStaticPropertyKey(member.key, member.computed) ===
+                String(key))
+          )
+            pending.push({ ...item, node: member.value, path });
+        }
+        if (node.superClass !== null && node.superClass !== undefined)
+          pending.push({ ...item, node: node.superClass });
+      }
+    }
+    const output = [...functions].flatMap((node) => {
+      const callables = t.isClass(node)
+        ? node.body.body.filter(
+            (member) =>
+              t.isClassMethod(member) && member.kind === "constructor",
+          )
+        : [node];
+      return callables.flatMap((callable) =>
+        semanticCallableResultExpressions(callable).flatMap(
+          ({ node, delegated }) =>
+            node === null
+              ? []
+              : [{ node, projection: delegated ? [null] : [] }],
+        ),
+      );
+    });
+    results.set(callee, output);
+    return output;
+  };
+};
+
+const selectObjectTargets = (
+  node: t.ObjectExpression,
+  item: Candidate,
+  pending: Candidate[],
+): void => {
+  const [key, ...path] = item.path;
+  let replaced = false;
+  for (const property of [...node.properties].reverse()) {
+    if (replaced) break;
+    if (t.isSpreadElement(property)) {
+      pending.push({ ...item, node: property.argument });
+      continue;
+    }
+    const name = semanticStaticPropertyKey(property.key, property.computed);
+    if (
+      key !== null &&
+      key !== undefined &&
+      name !== null &&
+      name !== String(key)
+    )
+      continue;
+    if (t.isObjectProperty(property))
+      pending.push({ ...item, node: property.value, path });
+    else pending.push({ ...item, node: property, path });
+    if (key !== null && key !== undefined && name === String(key))
+      replaced = true;
+  }
+};

src/domain/javascript/javascriptSemanticReturns.ts

@@ -26,10 +26,8 @@ import {
 import { evaluateSemanticExpression } from "./javascriptSemanticValues.js";
 import { range } from "./javascriptStaticAnalysisHelpers.js";
 import { semanticReturnCoverage } from "./javascriptSemanticCoverage.js";
-import {
-  childNodes,
-  traverseJavaScriptAst,
-} from "./javascriptSemanticTraversal.js";
+import { traverseJavaScriptAst } from "./javascriptSemanticTraversal.js";
+import { semanticCallableResultExpressions } from "./javascriptSemanticCallableResults.js";
 
 interface ReturnExpression {
   readonly node: t.Node | null;
@@ -123,27 +121,10 @@ const callableNodes = (program: t.Program): Map<string, t.Node> => {
   return output;
 };
 
-const directReturnExpressions = (callable: t.Node): ReturnExpression[] => {
-  if (
-    t.isArrowFunctionExpression(callable) &&
-    !t.isBlockStatement(callable.body)
-  )
-    return [{ node: callable.body, location: range(callable.body) }];
-  if (!t.isFunction(callable)) return [];
-  const output: ReturnExpression[] = [];
-  const pending = childNodes(callable.body).reverse();
-  while (pending.length > 0) {
-    const node = pending.pop();
-    if (node === undefined || t.isFunction(node) || t.isClass(node)) continue;
-    if (t.isReturnStatement(node)) {
-      output.push({ node: node.argument ?? null, location: range(node) });
-      continue;
-    }
-    // Reverse children so the explicit stack retains source-tree order.
-    for (const child of childNodes(node).reverse()) pending.push(child);
-  }
-  return output;
-};
+const directReturnExpressions = (callable: t.Node): ReturnExpression[] =>
+  semanticCallableResultExpressions(callable)
+    .filter(({ kind }) => kind === "return")
+    .map(({ node, site }) => ({ node, location: range(site) }));
 
 const callableIdsForBinding = (
   binding: JavaScriptSemanticBindingState,

tests/acceptance/applications/javascriptReturnedReferences.test.ts

@@ -0,0 +1,100 @@
+import { expect } from "vitest";
+
+import {
+  verifyJavaScriptReturnShapes,
+  type JavaScriptReturnFields,
+} from "../../fixtures/javascriptReturnShapes.js";
+import { cliTest } from "../../support/cli/cliFixture.js";
+
+const writes = {
+  direct: "const get = () => shared; get().x = 2;",
+  captured: "const get = () => shared; const t = get(); t.x = 2;",
+  declaration:
+    "function get() { return shared; } const o = {}; o.a = get(); o.a.x = 2;",
+  immediate: "const o = {}; o.a = (() => shared)(); o.a.x = 2;",
+  method:
+    "const box = {get() { return shared; }}; const o = {}; o.a = box.get(); o.a.x = 2;",
+  methodDirect: "const box = {get() { return shared; }}; box.get().x = 2;",
+  quotedMethod:
+    "const box = {['get/ref']() { return shared; }}; box['get/ref']().x = 2;",
+  alias: "const get = () => shared; const another = get; another().x = 2;",
+  alternative: "const get = true ? () => shared : () => ({x: 0}); get().x = 2;",
+  nestedReturn: "const get = () => ({child: shared}); get().child.x = 2;",
+  arrayReturn: "const get = () => [shared]; get()[0].x = 2;",
+  returnedProperty:
+    "const holder = {inner: shared}; const get = () => holder.inner; get().x = 2;",
+  transitive:
+    "const first = () => shared; const second = () => first(); second().x = 2;",
+  updated: "const get = () => shared; get().x++;",
+  asyncAwait: "const get = async () => shared; const t = await get(); t.x = 2;",
+  generator:
+    "function* get() { yield shared; } for (const t of get()) t.x = 2;",
+  returnedYield:
+    "function* get() { return yield shared; } for (const t of get()) t.x = 2;",
+  delegatedYield:
+    "function* get() { yield* [shared]; } for (const t of get()) t.x = 2;",
+  recursiveWrapper:
+    "let count = 0; const get = () => count++ === 0 ? {child: get()} : shared; get().child.x = 2;",
+  destructuredMethod:
+    "const box = {get() { return shared; }}; const {get} = box; get().x = 2;",
+  arrayCallable: "const methods = [() => shared]; methods[0]().x = 2;",
+  spreadArrayCallable:
+    "const methods = [...[], () => shared]; methods[0]().x = 2;",
+  classMethod:
+    "class Box { get() { return shared; } } const box = new Box(); box.get().x = 2;",
+  staticMethod:
+    "class Box { static get() { return shared; } } Box.get().x = 2;",
+  constructor:
+    "function Box() { return shared; } const t = new Box(); t.x = 2;",
+  callback: "const get = () => shared; [0].forEach(() => { get().x = 2; });",
+};
+const source = [
+  ...Object.entries(writes).map(
+    ([name, body]) =>
+      `export ${name === "asyncAwait" ? "async " : ""}function ${name}(){ const shared = {x: 1}; ${body} return shared.x; }`,
+  ),
+  "export function directSibling(){ const shared = {x: 1}; const parent = {shared, keep: 7}; const get = () => parent.shared; get().x = 2; return {x: shared.x, keep: parent.keep}; }",
+  "export function primitive(){ const n = 1; const get = () => n; const t = get(); return n; }",
+  "export function copy(){ const shared = {x: 1}; const get = () => ({...shared}); get().x = 2; return shared.x; }",
+  "export function methodCopy(){ const shared = {x: 1}; const box = {get(){ return {...shared}; }}; box.get().x = 2; return shared.x; }",
+  "export function arrayCopy(){ const shared = {x: 1}; const get = () => [shared.x]; get()[0] = 2; return shared.x; }",
+  "export function unused(){ const shared = {x: 1}; const get = () => shared; get(); return shared.x; }",
+  "export function nestedCallable(){ const shared = {x: 1}; const get = () => { const inner = () => shared; return {x: 0}; }; get().x = 2; return shared.x; }",
+  "export function unrelated(){ const shared = {x: 1}; const other = {x: 9}; const get = () => shared; get().x = 2; return other.x; }",
+].join("\n");
+
+const assertReturns = (fields: JavaScriptReturnFields): void => {
+  for (const name of Object.keys(writes)) {
+    const field = fields(name).find(({ path }) => path === "");
+    expect(field?.state, name).toMatch(/^(unknown|literal)$/);
+    if (field?.state === "literal") expect(field.value, name).toBe(2);
+  }
+  const changed = fields("directSibling").find(({ path }) => path === "/x");
+  expect(changed?.state).toMatch(/^(unknown|literal)$/);
+  if (changed?.state === "literal") expect(changed.value).toBe(2);
+  expect(fields("directSibling")).toContainEqual(
+    expect.objectContaining({ path: "/keep", state: "literal", value: 7 }),
+  );
+  for (const name of [
+    "primitive",
+    "copy",
+    "methodCopy",
+    "arrayCopy",
+    "nestedCallable",
+  ])
+    expect(fields(name), name).toContainEqual(
+      expect.objectContaining({ path: "", state: "literal", value: 1 }),
+    );
+  const unused = fields("unused").find(({ path }) => path === "");
+  expect(unused?.state).toMatch(/^(unknown|literal)$/);
+  if (unused?.state === "literal") expect(unused.value).toBe(1);
+  expect(fields("unrelated")).toContainEqual(
+    expect.objectContaining({ path: "", state: "literal", value: 9 }),
+  );
+};
+
+cliTest(
+  "preserves mutation uncertainty for objects returned by local calls",
+  ({ cli }) => verifyJavaScriptReturnShapes(cli, source, assertReturns),
+  120_000,
+);