artcraft: free image-to-3D thumbnail captures
captureModelThumbnail builds a throwaway renderer for every generated model's thumbnail. It never freed the model, never released the WebGL context (so each capture held one until garbage collection), and its 30s timeout was never cleared: it always fired, disposing a second time, and a load that finished after the timeout still rendered on the disposed renderer. Capture now tears down through a single guarded finish(): the timeout is cleared, the scene is freed with disposeObject3D, the renderer is disposed and its context released, exactly once whichever of load, error or timeout comes first. A model that loads after the timeout is freed instead of rendered. The function moves to its own file so it can be tested without the store's Tauri imports.
This commit is contained in:
Tanish Manku
parent
9c0f707536
commit
36bf18648d
|
|
@ -2,145 +2,7 @@ import { create } from "zustand";
|
|||
import { listen } from "@tauri-apps/api/event";
|
||||
import { MediaFilesApi, MediaUploadApi } from "@storyteller/api";
|
||||
import { v4 as uuidv4 } from "uuid";
|
||||
import * as THREE from "three";
|
||||
import { GLTFLoader } from "three/examples/jsm/loaders/GLTFLoader.js";
|
||||
|
||||
// Offscreen thumbnail capture function - produces a square image
|
||||
async function captureModelThumbnail(modelUrl: string): Promise<string | null> {
|
||||
return new Promise((resolve) => {
|
||||
try {
|
||||
// Square dimensions for thumbnail
|
||||
const size = 512;
|
||||
const fov = 50;
|
||||
|
||||
const scene = new THREE.Scene();
|
||||
scene.background = new THREE.Color(0x282828);
|
||||
|
||||
const camera = new THREE.PerspectiveCamera(fov, 1, 0.1, 1000); // Aspect ratio 1 for square
|
||||
|
||||
const renderer = new THREE.WebGLRenderer({
|
||||
antialias: true,
|
||||
alpha: false,
|
||||
preserveDrawingBuffer: true,
|
||||
});
|
||||
renderer.setSize(size, size);
|
||||
renderer.setPixelRatio(1);
|
||||
|
||||
// Add lights
|
||||
const ambientLight = new THREE.AmbientLight(0xffffff, 2);
|
||||
scene.add(ambientLight);
|
||||
|
||||
const hemisphereLight = new THREE.HemisphereLight(
|
||||
0xffffff,
|
||||
0x888888,
|
||||
1.2,
|
||||
);
|
||||
scene.add(hemisphereLight);
|
||||
|
||||
const keyLight = new THREE.DirectionalLight(0xffffff, 2);
|
||||
keyLight.position.set(2, 10, 8);
|
||||
scene.add(keyLight);
|
||||
|
||||
const fillLight = new THREE.DirectionalLight(0xffffff, 1.2);
|
||||
fillLight.position.set(-6, 6, -4);
|
||||
scene.add(fillLight);
|
||||
|
||||
const frontLight = new THREE.DirectionalLight(0xffffff, 1);
|
||||
frontLight.position.set(0, 4, 10);
|
||||
scene.add(frontLight);
|
||||
|
||||
const loader = new GLTFLoader();
|
||||
// Without anonymous CORS the GLTF + textures taint the WebGL canvas, so
|
||||
// toDataURL() below throws a SecurityError and the thumbnail is lost.
|
||||
loader.setCrossOrigin("anonymous");
|
||||
loader.load(
|
||||
modelUrl,
|
||||
(gltf) => {
|
||||
const model = gltf.scene;
|
||||
|
||||
const box = new THREE.Box3().setFromObject(model);
|
||||
const modelSize = box.getSize(new THREE.Vector3());
|
||||
|
||||
const maxDim = Math.max(modelSize.x, modelSize.y, modelSize.z);
|
||||
const scale = 2 / maxDim;
|
||||
model.scale.multiplyScalar(scale);
|
||||
|
||||
const scaledBox = new THREE.Box3().setFromObject(model);
|
||||
const scaledCenter = scaledBox.getCenter(new THREE.Vector3());
|
||||
const scaledSize = scaledBox.getSize(new THREE.Vector3());
|
||||
|
||||
model.position.x = -scaledCenter.x;
|
||||
model.position.z = -scaledCenter.z;
|
||||
model.position.y = -scaledBox.min.y;
|
||||
|
||||
scene.add(model);
|
||||
|
||||
// Calculate camera distance to fit model in view with padding
|
||||
const modelHeight = scaledSize.y;
|
||||
const maxModelDim = Math.max(
|
||||
scaledSize.x,
|
||||
scaledSize.y,
|
||||
scaledSize.z,
|
||||
);
|
||||
|
||||
// Calculate distance needed to fit the model based on FOV
|
||||
// For a 45-degree camera angle, we view from a diagonal
|
||||
const fovRad = (fov * Math.PI) / 180;
|
||||
const fitDistance = maxModelDim / 2 / Math.tan(fovRad / 2);
|
||||
|
||||
// Add some padding (1.3x) and account for diagonal viewing angle
|
||||
const cameraDistance = fitDistance * 1.3;
|
||||
|
||||
// Position camera at 45-degree angle
|
||||
const angle = Math.PI / 4; // 45 degrees
|
||||
camera.position.set(
|
||||
Math.sin(angle) * cameraDistance,
|
||||
modelHeight * 0.5 + cameraDistance * 0.4,
|
||||
Math.cos(angle) * cameraDistance,
|
||||
);
|
||||
camera.lookAt(0, modelHeight * 0.4, 0);
|
||||
|
||||
// Render and capture
|
||||
renderer.render(scene, camera);
|
||||
let dataUrl: string | null = null;
|
||||
try {
|
||||
dataUrl = renderer.domElement.toDataURL("image/png");
|
||||
} catch (err) {
|
||||
// A tainted canvas (CORS) throws SecurityError here. Log so the
|
||||
// failure is diagnosable instead of silently producing no thumbnail.
|
||||
console.warn(
|
||||
"[captureModelThumbnail] toDataURL failed (likely CORS taint):",
|
||||
err,
|
||||
);
|
||||
}
|
||||
|
||||
// Cleanup
|
||||
renderer.dispose();
|
||||
scene.clear();
|
||||
|
||||
resolve(dataUrl);
|
||||
},
|
||||
undefined,
|
||||
(error) => {
|
||||
console.error("[captureModelThumbnail] Error loading model:", error);
|
||||
renderer.dispose();
|
||||
scene.clear();
|
||||
resolve(null);
|
||||
},
|
||||
);
|
||||
|
||||
// Timeout after 30 seconds
|
||||
setTimeout(() => {
|
||||
renderer.dispose();
|
||||
scene.clear();
|
||||
resolve(null);
|
||||
}, 30000);
|
||||
} catch (error) {
|
||||
console.error("[captureModelThumbnail] Error:", error);
|
||||
resolve(null);
|
||||
}
|
||||
});
|
||||
}
|
||||
import { captureModelThumbnail } from "./captureModelThumbnail";
|
||||
|
||||
export interface PendingExternalImage {
|
||||
url: string;
|
||||
|
|
|
|||
|
|
@ -0,0 +1,106 @@
|
|||
import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
|
||||
import * as THREE from "three";
|
||||
import { captureModelThumbnail } from "./captureModelThumbnail";
|
||||
|
||||
const { renderers, pendingLoads } = vi.hoisted(() => ({
|
||||
renderers: [] as Array<{
|
||||
dispose: ReturnType<typeof vi.fn>;
|
||||
forceContextLoss: ReturnType<typeof vi.fn>;
|
||||
}>,
|
||||
pendingLoads: [] as Array<{
|
||||
onLoad: (gltf: { scene: THREE.Object3D }) => void;
|
||||
onError: (error: unknown) => void;
|
||||
}>,
|
||||
}));
|
||||
|
||||
// jsdom has no WebGL; only the teardown calls matter here.
|
||||
vi.mock("three", async (importOriginal) => {
|
||||
const actual = await importOriginal<typeof import("three")>();
|
||||
class WebGLRenderer {
|
||||
domElement = { toDataURL: () => "data:image/png;base64,AAAA" };
|
||||
dispose = vi.fn();
|
||||
forceContextLoss = vi.fn();
|
||||
constructor() {
|
||||
renderers.push(this);
|
||||
}
|
||||
setSize() {}
|
||||
setPixelRatio() {}
|
||||
render() {}
|
||||
}
|
||||
return { ...actual, WebGLRenderer };
|
||||
});
|
||||
|
||||
// Loads stay pending until a test settles them.
|
||||
vi.mock("three/examples/jsm/loaders/GLTFLoader.js", () => ({
|
||||
GLTFLoader: class {
|
||||
setCrossOrigin() {}
|
||||
load(
|
||||
_url: string,
|
||||
onLoad: (gltf: { scene: THREE.Object3D }) => void,
|
||||
_onProgress: unknown,
|
||||
onError: (error: unknown) => void,
|
||||
) {
|
||||
pendingLoads.push({ onLoad, onError });
|
||||
}
|
||||
},
|
||||
}));
|
||||
|
||||
describe("captureModelThumbnail", () => {
|
||||
beforeEach(() => {
|
||||
renderers.length = 0;
|
||||
pendingLoads.length = 0;
|
||||
vi.useFakeTimers();
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
vi.useRealTimers();
|
||||
});
|
||||
|
||||
it("returns the frame and frees the model, renderer and context once", async () => {
|
||||
const { model, onDispose } = trackedModel();
|
||||
const capture = captureModelThumbnail("model.glb");
|
||||
|
||||
pendingLoads[0].onLoad({ scene: model });
|
||||
const dataUrl = await capture;
|
||||
vi.runAllTimers();
|
||||
|
||||
expect(dataUrl).toMatch(/^data:image\/png/);
|
||||
expect(onDispose).toHaveBeenCalledTimes(1);
|
||||
expect(renderers[0].dispose).toHaveBeenCalledTimes(1);
|
||||
expect(renderers[0].forceContextLoss).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
|
||||
it("frees the renderer and context when the model fails to load", async () => {
|
||||
const capture = captureModelThumbnail("missing.glb");
|
||||
vi.spyOn(console, "error").mockImplementation(() => undefined);
|
||||
|
||||
pendingLoads[0].onError(new Error("404"));
|
||||
|
||||
expect(await capture).toBeNull();
|
||||
expect(renderers[0].dispose).toHaveBeenCalledTimes(1);
|
||||
expect(renderers[0].forceContextLoss).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
|
||||
it("gives up after the timeout and frees a model that loads later", async () => {
|
||||
const capture = captureModelThumbnail("slow.glb");
|
||||
|
||||
vi.advanceTimersByTime(30000);
|
||||
expect(await capture).toBeNull();
|
||||
expect(renderers[0].forceContextLoss).toHaveBeenCalledTimes(1);
|
||||
|
||||
const { model, onDispose } = trackedModel();
|
||||
pendingLoads[0].onLoad({ scene: model });
|
||||
|
||||
expect(onDispose).toHaveBeenCalledTimes(1);
|
||||
expect(renderers[0].dispose).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
});
|
||||
|
||||
function trackedModel() {
|
||||
const geometry = new THREE.BoxGeometry();
|
||||
const onDispose = vi.fn();
|
||||
geometry.addEventListener("dispose", onDispose);
|
||||
const model = new THREE.Group();
|
||||
model.add(new THREE.Mesh(geometry));
|
||||
return { model, onDispose };
|
||||
}
|
||||
|
|
@ -0,0 +1,160 @@
|
|||
import * as THREE from "three";
|
||||
import { GLTFLoader } from "three/examples/jsm/loaders/GLTFLoader.js";
|
||||
import { disposeObject3D } from "@storyteller/ui-viewer-3d";
|
||||
|
||||
const THUMBNAIL_SIZE = 512;
|
||||
const FOV = 50;
|
||||
const LOAD_TIMEOUT_MS = 30000;
|
||||
|
||||
// Offscreen thumbnail capture: loads the model into a throwaway renderer,
|
||||
// renders one square frame and returns it as a PNG data URL (null on
|
||||
// failure or timeout). The renderer, its WebGL context and the model are
|
||||
// freed exactly once, whichever of load, error or timeout comes first.
|
||||
export async function captureModelThumbnail(
|
||||
modelUrl: string,
|
||||
): Promise<string | null> {
|
||||
return new Promise((resolve) => {
|
||||
try {
|
||||
const scene = new THREE.Scene();
|
||||
scene.background = new THREE.Color(0x282828);
|
||||
|
||||
const camera = new THREE.PerspectiveCamera(FOV, 1, 0.1, 1000); // Aspect ratio 1 for square
|
||||
|
||||
const renderer = new THREE.WebGLRenderer({
|
||||
antialias: true,
|
||||
alpha: false,
|
||||
preserveDrawingBuffer: true,
|
||||
});
|
||||
renderer.setSize(THUMBNAIL_SIZE, THUMBNAIL_SIZE);
|
||||
renderer.setPixelRatio(1);
|
||||
|
||||
addLights(scene);
|
||||
|
||||
let done = false;
|
||||
const finish = (dataUrl: string | null) => {
|
||||
if (done) return;
|
||||
done = true;
|
||||
clearTimeout(timeout);
|
||||
disposeObject3D(scene);
|
||||
renderer.dispose();
|
||||
// The renderer's canvas is never attached, so nothing else can use
|
||||
// this context; release it now instead of at garbage collection.
|
||||
renderer.forceContextLoss();
|
||||
resolve(dataUrl);
|
||||
};
|
||||
|
||||
const timeout = setTimeout(() => finish(null), LOAD_TIMEOUT_MS);
|
||||
|
||||
const loader = new GLTFLoader();
|
||||
// Without anonymous CORS the GLTF + textures taint the WebGL canvas, so
|
||||
// toDataURL() below throws a SecurityError and the thumbnail is lost.
|
||||
loader.setCrossOrigin("anonymous");
|
||||
loader.load(
|
||||
modelUrl,
|
||||
(gltf) => {
|
||||
// Timed out already: the renderer is gone, just free the model.
|
||||
if (done) {
|
||||
disposeObject3D(gltf.scene);
|
||||
return;
|
||||
}
|
||||
const model = gltf.scene;
|
||||
const modelHeight = fitModel(model);
|
||||
scene.add(model);
|
||||
frameCamera(camera, model, modelHeight);
|
||||
|
||||
// Render and capture
|
||||
renderer.render(scene, camera);
|
||||
let dataUrl: string | null = null;
|
||||
try {
|
||||
dataUrl = renderer.domElement.toDataURL("image/png");
|
||||
} catch (err) {
|
||||
// A tainted canvas (CORS) throws SecurityError here. Log so the
|
||||
// failure is diagnosable instead of silently producing no thumbnail.
|
||||
console.warn(
|
||||
"[captureModelThumbnail] toDataURL failed (likely CORS taint):",
|
||||
err,
|
||||
);
|
||||
}
|
||||
finish(dataUrl);
|
||||
},
|
||||
undefined,
|
||||
(error) => {
|
||||
console.error("[captureModelThumbnail] Error loading model:", error);
|
||||
finish(null);
|
||||
},
|
||||
);
|
||||
} catch (error) {
|
||||
console.error("[captureModelThumbnail] Error:", error);
|
||||
resolve(null);
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
// Scale the model to a 2-unit max dimension and stand it centered on the
|
||||
// origin. Returns its scaled height.
|
||||
function fitModel(model: THREE.Object3D): number {
|
||||
const box = new THREE.Box3().setFromObject(model);
|
||||
const modelSize = box.getSize(new THREE.Vector3());
|
||||
|
||||
const maxDim = Math.max(modelSize.x, modelSize.y, modelSize.z);
|
||||
const scale = 2 / maxDim;
|
||||
model.scale.multiplyScalar(scale);
|
||||
|
||||
const scaledBox = new THREE.Box3().setFromObject(model);
|
||||
const scaledCenter = scaledBox.getCenter(new THREE.Vector3());
|
||||
|
||||
model.position.x = -scaledCenter.x;
|
||||
model.position.z = -scaledCenter.z;
|
||||
model.position.y = -scaledBox.min.y;
|
||||
|
||||
return scaledBox.getSize(new THREE.Vector3()).y;
|
||||
}
|
||||
|
||||
// Place the camera on a 45-degree diagonal, far enough to fit the model.
|
||||
function frameCamera(
|
||||
camera: THREE.PerspectiveCamera,
|
||||
model: THREE.Object3D,
|
||||
modelHeight: number,
|
||||
): void {
|
||||
const scaledSize = new THREE.Box3()
|
||||
.setFromObject(model)
|
||||
.getSize(new THREE.Vector3());
|
||||
const maxModelDim = Math.max(scaledSize.x, scaledSize.y, scaledSize.z);
|
||||
|
||||
// Calculate distance needed to fit the model based on FOV
|
||||
// For a 45-degree camera angle, we view from a diagonal
|
||||
const fovRad = (FOV * Math.PI) / 180;
|
||||
const fitDistance = maxModelDim / 2 / Math.tan(fovRad / 2);
|
||||
|
||||
// Add some padding (1.3x) and account for diagonal viewing angle
|
||||
const cameraDistance = fitDistance * 1.3;
|
||||
|
||||
// Position camera at 45-degree angle
|
||||
const angle = Math.PI / 4; // 45 degrees
|
||||
camera.position.set(
|
||||
Math.sin(angle) * cameraDistance,
|
||||
modelHeight * 0.5 + cameraDistance * 0.4,
|
||||
Math.cos(angle) * cameraDistance,
|
||||
);
|
||||
camera.lookAt(0, modelHeight * 0.4, 0);
|
||||
}
|
||||
|
||||
function addLights(scene: THREE.Scene): void {
|
||||
const ambientLight = new THREE.AmbientLight(0xffffff, 2);
|
||||
scene.add(ambientLight);
|
||||
|
||||
const hemisphereLight = new THREE.HemisphereLight(0xffffff, 0x888888, 1.2);
|
||||
scene.add(hemisphereLight);
|
||||
|
||||
const keyLight = new THREE.DirectionalLight(0xffffff, 2);
|
||||
keyLight.position.set(2, 10, 8);
|
||||
scene.add(keyLight);
|
||||
|
||||
const fillLight = new THREE.DirectionalLight(0xffffff, 1.2);
|
||||
fillLight.position.set(-6, 6, -4);
|
||||
scene.add(fillLight);
|
||||
|
||||
const frontLight = new THREE.DirectionalLight(0xffffff, 1);
|
||||
frontLight.position.set(0, 4, 10);
|
||||
scene.add(frontLight);
|
||||
}
|
||||
Loading…
Reference in New Issue