From b8b2091067aa4996f71e52f313cbd55ced015df3 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Mon, 28 Sep 2026 01:21:30 -0400 Subject: [PATCH] thumbnail speedup --- .../experiences/ImageTo3DExperience.tsx | 3 +- .../generation-feed/desktopMediaActions.ts | 9 +- .../useDesktopGenerationFeed.ts | 84 ++++++++ .../app/src/pages/PageAngles/Angles.tsx | 3 +- .../src/pages/PageDraw/BaseImageSelector.tsx | 2 +- .../ImageWatermarkRemover.tsx | 2 +- .../PageRemoveBackground/RemoveBackground.tsx | 2 +- .../adapters/export-sink-adapter.ts | 5 +- .../VideoFrameExtractor.tsx | 2 +- .../VideoWatermarkRemover.tsx | 2 +- frontend/libs/api/package.json | 6 +- frontend/libs/api/src/index.ts | 1 - .../src/lib/GalleryThumbnail.tsx | 137 +++--------- .../src/lib/pending-video-thumbnails.spec.tsx | 173 +++++++++++++++ .../src/lib/pending-video-thumbnails.ts | 200 ++++++++++++++++++ .../components/login-modal/tsconfig.lib.json | 5 +- .../promptbox/src/lib/CharactersModal.tsx | 2 +- .../promptbox/src/lib/ImagePromptRow.tsx | 2 +- .../promptbox/src/lib/deck/useDeckMedia.tsx | 3 +- .../lib/panes/DownloadsSettingsPane.spec.tsx | 6 +- .../src/lib/panes/DownloadsSettingsPane.tsx | 6 +- frontend/libs/tauri-api/package.json | 4 +- frontend/libs/tauri-api/src/index.ts | 1 + .../src/lib/download/LocalDownloads.ts} | 2 +- frontend/package-lock.json | 10 +- 25 files changed, 516 insertions(+), 156 deletions(-) create mode 100644 frontend/libs/components/generation-list/src/lib/pending-video-thumbnails.spec.tsx create mode 100644 frontend/libs/components/generation-list/src/lib/pending-video-thumbnails.ts rename frontend/libs/{api/src/lib/LocalApi.ts => tauri-api/src/lib/download/LocalDownloads.ts} (97%) diff --git a/frontend/apps/artcraft/app/src/components/experiences/ImageTo3DExperience.tsx b/frontend/apps/artcraft/app/src/components/experiences/ImageTo3DExperience.tsx index eb4f3d6420..80345e898d 100644 --- a/frontend/apps/artcraft/app/src/components/experiences/ImageTo3DExperience.tsx +++ b/frontend/apps/artcraft/app/src/components/experiences/ImageTo3DExperience.tsx @@ -8,13 +8,14 @@ import { BoxIcon, ImagesIcon, PlusIcon, UploadIcon, XIcon } from "lucide-react"; import { twMerge } from "tailwind-merge"; import { useImageTo3DStore } from "../../pages/PageImageTo3DObject/ImageTo3DStore"; import { useImageTo3DWorldStore } from "../../pages/PageImageTo3DWorld/ImageTo3DWorldStore"; -import { MediaUploadApi, downloadFileFromUrl } from "@storyteller/api"; +import { MediaUploadApi } from "@storyteller/api"; import { GalleryItem, GalleryModal } from "@storyteller/ui-gallery-modal"; import { GenerateMesh, EnqueueImageTo3dObjectModel, GenerateSplat, commandErrorMessage, + downloadFileFromUrl, } from "@storyteller/tauri-api"; import { toast } from "react-hot-toast"; import { v4 as uuidv4 } from "uuid"; diff --git a/frontend/apps/artcraft/app/src/components/generation-feed/desktopMediaActions.ts b/frontend/apps/artcraft/app/src/components/generation-feed/desktopMediaActions.ts index 1a658ea51c..165580074f 100644 --- a/frontend/apps/artcraft/app/src/components/generation-feed/desktopMediaActions.ts +++ b/frontend/apps/artcraft/app/src/components/generation-feed/desktopMediaActions.ts @@ -2,12 +2,15 @@ import toast from "react-hot-toast"; import { FilterMediaClasses, PromptsApi, +} from "@storyteller/api"; +import type { Prompts } from "@storyteller/api"; +import { + DownloadUrl, + useModelsStore, downloadUrlToPath, pickDownloadDirectory, promptDownloadLocationIfNeeded, -} from "@storyteller/api"; -import type { Prompts } from "@storyteller/api"; -import { DownloadUrl, useModelsStore } from "@storyteller/tauri-api"; +} from "@storyteller/tauri-api"; import { RefImage, RefVideo, diff --git a/frontend/apps/artcraft/app/src/components/generation-feed/useDesktopGenerationFeed.ts b/frontend/apps/artcraft/app/src/components/generation-feed/useDesktopGenerationFeed.ts index 557412531d..e0923cbf03 100644 --- a/frontend/apps/artcraft/app/src/components/generation-feed/useDesktopGenerationFeed.ts +++ b/frontend/apps/artcraft/app/src/components/generation-feed/useDesktopGenerationFeed.ts @@ -35,6 +35,11 @@ import { const POLL_INTERVAL_MS = 5000; +// Newly completed videos that arrived without a thumbnail URL are looked up +// again on this interval, for up to MISSING_THUMBNAIL_GIVE_UP_MS. +const MISSING_THUMBNAIL_POLL_MS = 15_000; +const MISSING_THUMBNAIL_GIVE_UP_MS = 10 * 60_000; + const FAILED_STATUSES = new Set([ "complete_failure", "attempt_failed", @@ -197,6 +202,71 @@ export function useDesktopGenerationFeed(options: { load(); }); + // Newly completed videos occasionally arrive without a thumbnail URL (the + // Rust media-file lookup fails open). The library copy would have it, but the + // newly completed item takes precedence in the feed, so look it up again + // until the URL shows up. GalleryThumbnail then re-checks the image itself. + const newlyCompletedRef = useRef(newlyCompleted); + newlyCompletedRef.current = newlyCompleted; + const missingThumbnailSinceRef = useRef>(new Map()); + + useEffect(() => { + if (mediaType !== "video") return; + let cancelled = false; + let running = false; + + const lookUpMissingThumbnails = async () => { + if (running) return; // A slow round is still going. + running = true; + try { + await lookUpMissingThumbnailsOnce(); + } finally { + running = false; + } + }; + + const lookUpMissingThumbnailsOnce = async () => { + const now = Date.now(); + const missing = newlyCompletedRef.current.filter((item) => { + if (item.thumbnail) return false; + const since = missingThumbnailSinceRef.current.get(item.id) ?? now; + missingThumbnailSinceRef.current.set(item.id, since); + return now - since < MISSING_THUMBNAIL_GIVE_UP_MS; + }); + for (const item of missing) { + const response = await mediaApiRef.current.GetMediaFileByToken({ + mediaFileToken: item.id, + }); + if (cancelled) return; + const links = response.success + ? (response.data?.media_links as VideoMediaLinks | undefined) + : undefined; + // Without a preview or template, getMediaThumbnail would fall back to + // the .mp4 itself, which an can't show. Try again next round. + if (!links?.maybe_video_previews?.animated && !links?.maybe_thumbnail_template) { + continue; + } + const thumbnail = getMediaThumbnail(links, "video", { + size: THUMBNAIL_SIZES.LARGE, + }); + const stillThumbnail = getMediaStillThumbnail(links, { + size: THUMBNAIL_SIZES.LARGE, + }); + setNewlyCompleted((prev) => + prev.map((i) => + i.id === item.id ? { ...i, thumbnail, stillThumbnail } : i, + ), + ); + } + }; + + const intervalId = setInterval(lookUpMissingThumbnails, MISSING_THUMBNAIL_POLL_MS); + return () => { + cancelled = true; + clearInterval(intervalId); + }; + }, [mediaType]); + // Poll + reload on Tauri generation events and local "task-queue-update" // dispatches (fired by the pages right after enqueueing). useEffect(() => { @@ -249,6 +319,20 @@ export function useDesktopGenerationFeed(options: { }; } +// ── Types ─────────────────────────────────────────────────────────────────── + +// The media-file API's link fields used for video thumbnails. (The shared +// MediaFile type predates these fields.) +interface VideoMediaLinks { + cdn_url?: string | null; + maybe_thumbnail_template?: string | null; + maybe_video_previews?: { + animated?: string | null; + still?: string | null; + still_thumbnail_template?: string | null; + } | null; +} + // ── Task mapping helpers ──────────────────────────────────────────────────── function getTaskMediaType(t: TaskQueueItem): "image" | "video" | "other" { diff --git a/frontend/apps/artcraft/app/src/pages/PageAngles/Angles.tsx b/frontend/apps/artcraft/app/src/pages/PageAngles/Angles.tsx index 0e6ca4c749..940cdf434b 100644 --- a/frontend/apps/artcraft/app/src/pages/PageAngles/Angles.tsx +++ b/frontend/apps/artcraft/app/src/pages/PageAngles/Angles.tsx @@ -5,7 +5,6 @@ import { GalleryItem, GalleryModal } from "@storyteller/ui-gallery-modal"; import { MediaUploadApi, MediaFilesApi, - downloadFileFromUrl, } from "@storyteller/api"; import toast from "react-hot-toast"; import { v4 as uuidv4 } from "uuid"; @@ -25,7 +24,7 @@ import { twMerge } from "tailwind-merge"; import { LoadingSpinner } from "@storyteller/ui-loading-spinner"; import { SliderV2 } from "@storyteller/ui-sliderv2"; // import { Switch } from "@headlessui/react"; -import { GenerateImage } from "@storyteller/tauri-api"; +import { GenerateImage, downloadFileFromUrl } from "@storyteller/tauri-api"; import { ClassyModelSelector, useAnglesPageModelList, diff --git a/frontend/apps/artcraft/app/src/pages/PageDraw/BaseImageSelector.tsx b/frontend/apps/artcraft/app/src/pages/PageDraw/BaseImageSelector.tsx index 5b3814df96..39a18ab615 100644 --- a/frontend/apps/artcraft/app/src/pages/PageDraw/BaseImageSelector.tsx +++ b/frontend/apps/artcraft/app/src/pages/PageDraw/BaseImageSelector.tsx @@ -1,5 +1,5 @@ import { ImagesIcon, MaximizeIcon, PencilIcon } from "lucide-react"; -import { downloadFileFromUrl } from "libs/api/src/lib/LocalApi"; +import { downloadFileFromUrl } from "@storyteller/tauri-api"; import { GalleryModal, GalleryItem } from "@storyteller/ui-gallery-modal"; import { useState } from "react"; import toast from "react-hot-toast"; diff --git a/frontend/apps/artcraft/app/src/pages/PageImageWatermarkRemover/ImageWatermarkRemover.tsx b/frontend/apps/artcraft/app/src/pages/PageImageWatermarkRemover/ImageWatermarkRemover.tsx index d9e96fdfa0..c3ec895138 100644 --- a/frontend/apps/artcraft/app/src/pages/PageImageWatermarkRemover/ImageWatermarkRemover.tsx +++ b/frontend/apps/artcraft/app/src/pages/PageImageWatermarkRemover/ImageWatermarkRemover.tsx @@ -2,7 +2,7 @@ import { useState, useRef, useEffect } from "react"; import { DropletIcon, ImagesIcon, RotateCwIcon } from "lucide-react"; import { Button } from "@storyteller/ui-button"; import { GalleryItem, GalleryModal } from "@storyteller/ui-gallery-modal"; -import { downloadFileFromUrl } from "@storyteller/api"; +import { downloadFileFromUrl } from "@storyteller/tauri-api"; import { UploadEntryCard } from "../../components/media/UploadEntryCard"; import toast from "react-hot-toast"; diff --git a/frontend/apps/artcraft/app/src/pages/PageRemoveBackground/RemoveBackground.tsx b/frontend/apps/artcraft/app/src/pages/PageRemoveBackground/RemoveBackground.tsx index f03a53829a..a37a78fe3a 100644 --- a/frontend/apps/artcraft/app/src/pages/PageRemoveBackground/RemoveBackground.tsx +++ b/frontend/apps/artcraft/app/src/pages/PageRemoveBackground/RemoveBackground.tsx @@ -2,7 +2,6 @@ import { useState, useRef, useEffect, useCallback, useMemo } from "react"; import { DownloadIcon, EyeIcon, ImagesIcon, PlusIcon, UploadIcon, WandSparklesIcon } from "lucide-react"; import { Button } from "@storyteller/ui-button"; import { GalleryItem, GalleryModal } from "@storyteller/ui-gallery-modal"; -import { downloadFileFromUrl } from "@storyteller/api"; import toast from "react-hot-toast"; import { v4 as uuidv4 } from "uuid"; import { UploadEntryCard } from "../../components/media/UploadEntryCard"; @@ -13,6 +12,7 @@ import { import { EnqueueImageBgRemoval, useCanvasBgRemovedEvent, + downloadFileFromUrl, } from "@storyteller/tauri-api"; import { PopoverMenu, PopoverItem } from "@storyteller/ui-popover"; import { twMerge } from "tailwind-merge"; diff --git a/frontend/apps/artcraft/app/src/pages/PageVideoEditor/adapters/export-sink-adapter.ts b/frontend/apps/artcraft/app/src/pages/PageVideoEditor/adapters/export-sink-adapter.ts index c9621749cb..cbb2f55cfb 100644 --- a/frontend/apps/artcraft/app/src/pages/PageVideoEditor/adapters/export-sink-adapter.ts +++ b/frontend/apps/artcraft/app/src/pages/PageVideoEditor/adapters/export-sink-adapter.ts @@ -3,10 +3,7 @@ import type { ExportSinkAdapter, } from "@storyteller/ui-video-editor"; import { kindFromMime } from "@storyteller/ui-video-editor"; -import { - promptDownloadLocationIfNeeded, - downloadUrlToPath, -} from "@storyteller/api"; +import { promptDownloadLocationIfNeeded, downloadUrlToPath } from "@storyteller/tauri-api"; import { downloadDir } from "@tauri-apps/api/path"; import { uploadByKind } from "./upload-by-kind"; diff --git a/frontend/apps/artcraft/app/src/pages/PageVideoFrameExtractor/VideoFrameExtractor.tsx b/frontend/apps/artcraft/app/src/pages/PageVideoFrameExtractor/VideoFrameExtractor.tsx index c69d8eae60..b61c4ffc7f 100644 --- a/frontend/apps/artcraft/app/src/pages/PageVideoFrameExtractor/VideoFrameExtractor.tsx +++ b/frontend/apps/artcraft/app/src/pages/PageVideoFrameExtractor/VideoFrameExtractor.tsx @@ -4,10 +4,10 @@ import { DynamicIcon } from "@storyteller/icons"; import { Button } from "@storyteller/ui-button"; import { GalleryItem, GalleryModal } from "@storyteller/ui-gallery-modal"; import { - downloadFileFromUrl, MediaUploadApi, EIntermediateFile, } from "@storyteller/api"; +import { downloadFileFromUrl } from "@storyteller/tauri-api"; import toast from "react-hot-toast"; import { v4 as uuidv4 } from "uuid"; import { usePromptVideoStore, RefImage } from "@storyteller/ui-promptbox"; diff --git a/frontend/apps/artcraft/app/src/pages/PageVideoWatermarkRemover/VideoWatermarkRemover.tsx b/frontend/apps/artcraft/app/src/pages/PageVideoWatermarkRemover/VideoWatermarkRemover.tsx index 67a9e58512..9a0ea41b43 100644 --- a/frontend/apps/artcraft/app/src/pages/PageVideoWatermarkRemover/VideoWatermarkRemover.tsx +++ b/frontend/apps/artcraft/app/src/pages/PageVideoWatermarkRemover/VideoWatermarkRemover.tsx @@ -3,7 +3,7 @@ import { DropletIcon, ImagesIcon, PauseIcon, PlayIcon, RotateCwIcon, SkipBackIco import { DynamicIcon } from "@storyteller/icons"; import { Button } from "@storyteller/ui-button"; import { GalleryItem, GalleryModal } from "@storyteller/ui-gallery-modal"; -import { downloadFileFromUrl } from "@storyteller/api"; +import { downloadFileFromUrl } from "@storyteller/tauri-api"; import { UploadEntryCard } from "../../components/media/UploadEntryCard"; import toast from "react-hot-toast"; diff --git a/frontend/libs/api/package.json b/frontend/libs/api/package.json index 8dd315e0ea..06903204fa 100644 --- a/frontend/libs/api/package.json +++ b/frontend/libs/api/package.json @@ -14,9 +14,5 @@ "import": "./dist/index.js", "default": "./dist/index.js" } - }, - "dependencies": { - "@tauri-apps/plugin-dialog": "^2.7.1", - "@tauri-apps/plugin-upload": "2.4.0" } -} \ No newline at end of file +} diff --git a/frontend/libs/api/src/index.ts b/frontend/libs/api/src/index.ts index 7286c75276..a2859cdda8 100644 --- a/frontend/libs/api/src/index.ts +++ b/frontend/libs/api/src/index.ts @@ -11,7 +11,6 @@ export * from "./lib/JobsApi.js"; export * from "./lib/MediaFilesApi.js"; export * from "./lib/MediaUploadApi.js"; export * from "./lib/MiscApi.js"; -export * from "./lib/LocalApi.js"; export * from "./lib/ProjectsApi.js"; export * from "./lib/PromptsApi.js"; export * from "./lib/TagsApi.js"; diff --git a/frontend/libs/components/generation-list/src/lib/GalleryThumbnail.tsx b/frontend/libs/components/generation-list/src/lib/GalleryThumbnail.tsx index 7edb70e509..be9b2557a4 100644 --- a/frontend/libs/components/generation-list/src/lib/GalleryThumbnail.tsx +++ b/frontend/libs/components/generation-list/src/lib/GalleryThumbnail.tsx @@ -1,41 +1,19 @@ -import { useCallback, useEffect, useRef, useState } from "react"; +import { useCallback } from "react"; import { LoaderCircleIcon } from "lucide-react"; import type { LucideIcon } from "lucide-react"; import { DynamicIcon } from "@storyteller/icons"; import { PLACEHOLDER_IMAGES } from "@storyteller/common"; import { useGalleryViewStore } from "./gallery-view-store"; +import { + markVideoThumbnailFailed, + useVideoThumbnailState, +} from "./pending-video-thumbnails"; // Media thumbnail shared by the masonry GalleryCard and the list GalleryRow. // Encapsulates the two awkward bits of thumbnail rendering: freshly generated -// video thumbnails 404 until the render job finishes (so we retry on a timer -// and on tab refocus), and broken image URLs fall back to a placeholder. - -// ── Shared visibility listener for video thumbnail retries ──────────────── - -const visibilityCallbacks = new Set<() => void>(); - -function onTabVisible(cb: () => void) { - visibilityCallbacks.add(cb); - if (visibilityCallbacks.size === 1) { - document.addEventListener("visibilitychange", fireVisibilityCallbacks); - } - return () => { - visibilityCallbacks.delete(cb); - if (visibilityCallbacks.size === 0) { - document.removeEventListener("visibilitychange", fireVisibilityCallbacks); - } - }; -} - -function fireVisibilityCallbacks() { - if (document.hidden) return; - visibilityCallbacks.forEach((cb) => cb()); -} - -// ── Retry constants ──────────────────────────────────────────────────────── - -const MAX_RETRIES = 20; -const RETRY_INTERVAL = 5000; +// video thumbnails 404 until the render job finishes (so failed ones are +// marked and re-checked in the background, see pending-video-thumbnails), and +// broken image URLs fall back to a placeholder. // ── Component ────────────────────────────────────────────────────────────── @@ -70,71 +48,17 @@ export function GalleryThumbnail({ }: GalleryThumbnailProps) { const autoplayVideos = useGalleryViewStore((s) => s.autoplayVideos); // With autoplay off, videos freeze on their still first frame. The still is - // also what the retry machinery below polls in that mode. + // also what gets re-checked in that mode. const thumbnail = isVideo && !autoplayVideos && stillThumbnail ? stillThumbnail : animatedThumbnail; - // "retrying" flips to true only after the first error, so videos with ready - // thumbnails render the normal path with zero overhead. While retrying, - // a hidden loads via ref (no re-renders) and the spinner stays stable. - const [retrying, setRetrying] = useState(false); - const retryImgRef = useRef(null); - const retryTimerRef = useRef>(undefined); - const retryCountRef = useRef(0); - - const kickRetry = useCallback(() => { - if (!retryImgRef.current || !thumbnail) return; - retryImgRef.current.src = `${thumbnail}?_r=${Date.now()}`; - }, [thumbnail]); - - const scheduleRetry = useCallback(() => { - if (retryCountRef.current >= MAX_RETRIES || !thumbnail) return; - if (document.hidden) return; - retryTimerRef.current = setTimeout(() => { - retryCountRef.current++; - kickRetry(); - }, RETRY_INTERVAL); - }, [thumbnail, kickRetry]); - - // Subscribe to the shared visibility listener while a video card is retrying. - useEffect(() => { - if (!isVideo || !thumbnail || !retrying) return; - const unsubscribe = onTabVisible(() => { - retryCountRef.current = 0; - kickRetry(); - }); - return () => { - unsubscribe(); - if (retryTimerRef.current) clearTimeout(retryTimerRef.current); - }; - }, [isVideo, thumbnail, retrying, kickRetry]); - - // Reset retry state when the thumbnail URL changes. - useEffect(() => { - if (!isVideo) return; - setRetrying(false); - retryCountRef.current = 0; - if (retryTimerRef.current) clearTimeout(retryTimerRef.current); - }, [isVideo, thumbnail]); - - const handleLoad = useCallback( - (e: React.SyntheticEvent) => { - if (isVideo && retrying) { - setRetrying(false); - retryCountRef.current = 0; - if (retryTimerRef.current) clearTimeout(retryTimerRef.current); - } - onLoad?.(e); - }, - [isVideo, retrying, onLoad], - ); + const videoState = useVideoThumbnailState(isVideo ? thumbnail : null); const handleError = useCallback( (e: React.SyntheticEvent) => { if (isVideo) { - setRetrying(true); - scheduleRetry(); + if (thumbnail) markVideoThumbnailFailed(thumbnail); } else { const target = e.currentTarget; if (target.dataset.fallback) return; @@ -143,43 +67,32 @@ export function GalleryThumbnail({ target.style.opacity = "0.3"; } }, - [isVideo, scheduleRetry], + [isVideo, thumbnail], ); - if (retrying) { + if (videoState?.status === "pending") { return ( - <> -
- - {showRetryLabel && ( - Loading thumbnail… - )} -
- {/* Hidden img retries in the background via ref — zero re-renders */} - - +
+ + {showRetryLabel && ( + Loading thumbnail… + )} +
); } - if (thumbnail) { + // Loaded after a re-check: render the (cache-busted) URL that worked. + const src = videoState?.status === "ready" ? videoState.src : thumbnail; + + if (src && videoState?.status !== "gave_up") { return ( {alt} ); diff --git a/frontend/libs/components/generation-list/src/lib/pending-video-thumbnails.spec.tsx b/frontend/libs/components/generation-list/src/lib/pending-video-thumbnails.spec.tsx new file mode 100644 index 0000000000..f12e4a2100 --- /dev/null +++ b/frontend/libs/components/generation-list/src/lib/pending-video-thumbnails.spec.tsx @@ -0,0 +1,173 @@ +import { act, cleanup, fireEvent, render, screen } from "@testing-library/react"; +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; +import { VideoIcon } from "lucide-react"; +import { + getVideoThumbnailState, + markVideoThumbnailFailed, + resetVideoThumbnailsForTests, +} from "./pending-video-thumbnails"; +import { GalleryThumbnail } from "./GalleryThumbnail"; + +vi.mock("@storyteller/icons", () => ({ + DynamicIcon: () => , +})); +vi.mock("@storyteller/common", () => ({ + PLACEHOLDER_IMAGES: { DEFAULT: "placeholder.png" }, +})); + +const THUMB = + "https://cdn-2.fakeyou.com/cdn-cgi/image/width=512,quality=95/media/a/b/storyteller_x.mp4-thumb.gif"; + +// Probes made by the checker; a test settles each one by hand. +let probes: FakeImage[] = []; + +class FakeImage { + onload: (() => void) | null = null; + onerror: (() => void) | null = null; + src = ""; + constructor() { + probes.push(this); + } +} + +beforeEach(() => { + vi.useFakeTimers(); + vi.stubGlobal("Image", FakeImage); + probes = []; + resetVideoThumbnailsForTests(); +}); + +afterEach(() => { + cleanup(); + vi.unstubAllGlobals(); + vi.useRealTimers(); +}); + +describe("checker", () => { + it("re-checks a failed thumbnail and hands back the URL that loaded", () => { + markVideoThumbnailFailed(THUMB); + expect(getVideoThumbnailState(THUMB)).toEqual({ status: "pending" }); + + vi.advanceTimersByTime(5_000); + expect(probes).toHaveLength(1); + expect(probes[0].src).toMatch(/^https:\/\/cdn-2\.fakeyou\.com\/.*\.mp4-thumb\.gif\?_r=\d+$/); + + probes[0].onload!(); + expect(getVideoThumbnailState(THUMB)).toEqual({ status: "ready", src: probes[0].src }); + }); + + it("keeps checking past the old 20-retry limit, backing off", () => { + markVideoThumbnailFailed(THUMB); + failProbesFor(2 * 60_000); + expect(probes).toHaveLength(24); // Every 5s for the first two minutes. + + failProbesFor(8 * 60_000); + expect(probes).toHaveLength(24 + 32); // Then every 15s. + expect(getVideoThumbnailState(THUMB)).toEqual({ status: "pending" }); + }); + + it("gives up after an hour", () => { + markVideoThumbnailFailed(THUMB); + failProbesFor(60 * 60_000 + 60_000); + expect(getVideoThumbnailState(THUMB)).toEqual({ status: "gave_up" }); + + const probeCount = probes.length; + vi.advanceTimersByTime(10 * 60_000); + expect(probes).toHaveLength(probeCount); + }); + + it("probes a URL once however many cards mark it", () => { + markVideoThumbnailFailed(THUMB); + markVideoThumbnailFailed(THUMB); + vi.advanceTimersByTime(5_000); + markVideoThumbnailFailed(THUMB); + vi.advanceTimersByTime(5_000); + expect(probes).toHaveLength(1); // The first probe is still in flight. + }); + + it("checks right away when the window regains focus", () => { + markVideoThumbnailFailed(THUMB); + failProbesFor(3 * 60_000); // Now on the 15s interval. + const probeCount = probes.length; + + vi.advanceTimersByTime(1_000); + window.dispatchEvent(new Event("focus")); + vi.advanceTimersByTime(0); + expect(probes).toHaveLength(probeCount + 1); + }); + + it("pauses while the window is hidden", () => { + const hidden = vi.spyOn(document, "hidden", "get").mockReturnValue(true); + markVideoThumbnailFailed(THUMB); + vi.advanceTimersByTime(60_000); + expect(probes).toHaveLength(0); + + hidden.mockReturnValue(false); + document.dispatchEvent(new Event("visibilitychange")); + vi.advanceTimersByTime(0); + expect(probes).toHaveLength(1); + }); +}); + +describe("GalleryThumbnail", () => { + it("shows a spinner for a failed video thumbnail, then swaps in the loaded one", () => { + renderVideoThumbnail(); + fireEvent.error(screen.getByRole("img")); + expect(screen.getByText("Loading thumbnail…")).toBeTruthy(); + + act(() => { + vi.advanceTimersByTime(5_000); + probes[0].onerror!(); + vi.advanceTimersByTime(5_000); + probes[1].onload!(); + }); + expect(screen.getByRole("img").getAttribute("src")).toBe(probes[1].src); + }); + + it("renders an already-recovered thumbnail straight away when remounted", () => { + markVideoThumbnailFailed(THUMB); + vi.advanceTimersByTime(5_000); + probes[0].onload!(); + + renderVideoThumbnail(); + expect(screen.getByRole("img").getAttribute("src")).toBe(probes[0].src); + }); + + it("falls back to the video icon once the thumbnail is given up on", () => { + markVideoThumbnailFailed(THUMB); + failProbesFor(60 * 60_000 + 60_000); + + renderVideoThumbnail(); + expect(screen.getByTestId("fallback-icon")).toBeTruthy(); + }); + + it("leaves image thumbnails on the placeholder fallback", () => { + render( + , + ); + fireEvent.error(screen.getByRole("img")); + expect(screen.getByRole("img").getAttribute("src")).toBe("placeholder.png"); + expect(getVideoThumbnailState(THUMB)).toBeUndefined(); + }); +}); + +function renderVideoThumbnail() { + render( + , + ); +} + +// Advance time, failing every probe the checker starts along the way. +function failProbesFor(ms: number) { + const end = Date.now() + ms; + while (Date.now() < end) { + vi.advanceTimersByTime(1_000); + for (const probe of probes) { + if (probe.onerror) { + const onerror = probe.onerror; + probe.onerror = null; + onerror(); + } + } + } +} diff --git a/frontend/libs/components/generation-list/src/lib/pending-video-thumbnails.ts b/frontend/libs/components/generation-list/src/lib/pending-video-thumbnails.ts new file mode 100644 index 0000000000..3357b15f37 --- /dev/null +++ b/frontend/libs/components/generation-list/src/lib/pending-video-thumbnails.ts @@ -0,0 +1,200 @@ +import { useSyncExternalStore } from "react"; + +// Freshly generated video thumbnails are rendered asynchronously after the +// video lands, so the first load usually 404s. A card whose video +// thumbnail fails marks it here; one shared checker then probes every marked +// URL on a backoff until it loads, and subscribed cards swap it in. +// +// Probes add a cache-busting query param (the CDN may have cached the 404 for +// the plain URL), and the URL that loaded is the one handed back to render. + +// ── Check schedule ──────────────────────────────────────────────────────── + +// How often to re-check a marked thumbnail, by time since it was marked. +const CHECK_SCHEDULE: { untilMs: number; everyMs: number }[] = [ + { untilMs: 2 * 60_000, everyMs: 5_000 }, + { untilMs: 10 * 60_000, everyMs: 15_000 }, + { untilMs: 60 * 60_000, everyMs: 60_000 }, +]; + +// Past the last window the thumbnail is treated as never coming. +const GIVE_UP_AFTER_MS = CHECK_SCHEDULE[CHECK_SCHEDULE.length - 1].untilMs; + +// ── Types ───────────────────────────────────────────────────────────────── + +export type VideoThumbnailState = + // Render this src (the original URL, or the cache-busted URL that loaded). + | { status: "ready"; src: string } + // Failed to load; being re-checked in the background. + | { status: "pending" } + // Never loaded within GIVE_UP_AFTER_MS. + | { status: "gave_up" }; + +interface PendingEntry { + markedAt: number; + nextCheckAt: number; + probing: boolean; +} + +// ── Registry state ──────────────────────────────────────────────────────── + +const PENDING_STATE: VideoThumbnailState = { status: "pending" }; +const GAVE_UP_STATE: VideoThumbnailState = { status: "gave_up" }; + +const pending = new Map(); +// Settled states, kept for the session so remounted cards skip the 404. +const settled = new Map(); +const listeners = new Map void>>(); + +let timer: ReturnType | undefined; +let wakeListenersAttached = false; + +// ── Public API ──────────────────────────────────────────────────────────── + +/** Mark a video thumbnail URL as failed so it gets re-checked. */ +export function markVideoThumbnailFailed(url: string) { + if (pending.has(url) || settled.get(url) === GAVE_UP_STATE) return; + const now = Date.now(); + // Also covers a previously loaded src failing again: start over. + settled.delete(url); + pending.set(url, { + markedAt: now, + nextCheckAt: now + CHECK_SCHEDULE[0].everyMs, + probing: false, + }); + attachWakeListeners(); + notify(url); + scheduleNextTick(); +} + +export function getVideoThumbnailState( + url: string, +): VideoThumbnailState | undefined { + if (pending.has(url)) return PENDING_STATE; + return settled.get(url); +} + +/** + * The render state for a video thumbnail URL. Untracked URLs render as-is. + * Pass null to opt out (non-video media, or no thumbnail). + */ +export function useVideoThumbnailState( + url: string | null, +): VideoThumbnailState | undefined { + return useSyncExternalStore( + (onChange) => (url ? subscribe(url, onChange) : () => {}), + () => (url ? getVideoThumbnailState(url) : undefined), + ); +} + +/** Test-only: forget all tracked thumbnails and stop the checker. */ +export function resetVideoThumbnailsForTests() { + pending.clear(); + settled.clear(); + listeners.clear(); + if (timer) clearTimeout(timer); + timer = undefined; +} + +// ── Checker ─────────────────────────────────────────────────────────────── + +function scheduleNextTick() { + if (timer) clearTimeout(timer); + timer = undefined; + if (pending.size === 0) return; + let earliest = Infinity; + for (const entry of pending.values()) { + if (!entry.probing) earliest = Math.min(earliest, entry.nextCheckAt); + } + if (earliest === Infinity) return; // Every entry is mid-probe. + timer = setTimeout(tick, Math.max(0, earliest - Date.now())); +} + +function tick() { + timer = undefined; + // Paused while the window is hidden; the wake listeners resume it. + if (typeof document !== "undefined" && document.hidden) return; + const now = Date.now(); + for (const [url, entry] of pending) { + if (entry.probing || entry.nextCheckAt > now) continue; + if (now - entry.markedAt >= GIVE_UP_AFTER_MS) { + pending.delete(url); + settled.set(url, GAVE_UP_STATE); + notify(url); + continue; + } + probe(url, entry); + } + scheduleNextTick(); +} + +function probe(url: string, entry: PendingEntry) { + entry.probing = true; + const src = withCacheBust(url); + const img = new Image(); + img.onload = () => { + if (pending.get(url) !== entry) return; // Reset while in flight. + pending.delete(url); + settled.set(url, { status: "ready", src }); + notify(url); + scheduleNextTick(); + }; + img.onerror = () => { + if (pending.get(url) !== entry) return; + entry.probing = false; + entry.nextCheckAt = Date.now() + checkInterval(Date.now() - entry.markedAt); + scheduleNextTick(); + }; + img.src = src; +} + +function checkInterval(ageMs: number): number { + const slot = CHECK_SCHEDULE.find((w) => ageMs < w.untilMs); + return (slot ?? CHECK_SCHEDULE[CHECK_SCHEDULE.length - 1]).everyMs; +} + +// Re-check everything immediately when the app comes back into view or focus +// (the desktop window rarely fires visibilitychange, but does fire focus). +function attachWakeListeners() { + if (wakeListenersAttached || typeof window === "undefined") return; + wakeListenersAttached = true; + const wake = () => { + if (document.hidden || pending.size === 0) return; + const now = Date.now(); + for (const entry of pending.values()) { + entry.nextCheckAt = Math.min(entry.nextCheckAt, now); + } + scheduleNextTick(); + }; + document.addEventListener("visibilitychange", wake); + window.addEventListener("focus", wake); +} + +// ── Subscriptions ───────────────────────────────────────────────────────── + +function subscribe(url: string, onChange: () => void) { + let set = listeners.get(url); + if (!set) { + set = new Set(); + listeners.set(url, set); + } + set.add(onChange); + return () => { + set.delete(onChange); + if (set.size === 0) listeners.delete(url); + }; +} + +function notify(url: string) { + listeners.get(url)?.forEach((cb) => cb()); +} + +function withCacheBust(url: string): string { + try { + const parsed = new URL(url); + parsed.searchParams.set("_r", Date.now().toString()); + return parsed.toString(); + } catch { + return `${url}${url.includes("?") ? "&" : "?"}_r=${Date.now()}`; + } +} diff --git a/frontend/libs/components/login-modal/tsconfig.lib.json b/frontend/libs/components/login-modal/tsconfig.lib.json index 45cf3e4614..d7517fe035 100644 --- a/frontend/libs/components/login-modal/tsconfig.lib.json +++ b/frontend/libs/components/login-modal/tsconfig.lib.json @@ -43,9 +43,6 @@ "src/**/*.tsx" ], "references": [ - { - "path": "../../api/tsconfig.lib.json" - }, { "path": "../../icons/tsconfig.lib.json" }, @@ -56,7 +53,7 @@ "path": "../input/tsconfig.lib.json" }, { - "path": "../../tauri-api/tsconfig.lib.json" + "path": "../../api/tsconfig.lib.json" } ] } diff --git a/frontend/libs/components/promptbox/src/lib/CharactersModal.tsx b/frontend/libs/components/promptbox/src/lib/CharactersModal.tsx index 7d0fcf4b3a..e4031ac98f 100644 --- a/frontend/libs/components/promptbox/src/lib/CharactersModal.tsx +++ b/frontend/libs/components/promptbox/src/lib/CharactersModal.tsx @@ -6,8 +6,8 @@ import { CharactersApi, Character, MediaUploadApi, - downloadFileFromUrl, } from "@storyteller/api"; +import { downloadFileFromUrl } from "@storyteller/tauri-api"; import { toast } from "@storyteller/ui-toaster"; import { v4 as uuidv4 } from "uuid"; import { GalleryItem, GalleryModal } from "@storyteller/ui-gallery-modal"; diff --git a/frontend/libs/components/promptbox/src/lib/ImagePromptRow.tsx b/frontend/libs/components/promptbox/src/lib/ImagePromptRow.tsx index 16a2299307..f6e6a2574f 100644 --- a/frontend/libs/components/promptbox/src/lib/ImagePromptRow.tsx +++ b/frontend/libs/components/promptbox/src/lib/ImagePromptRow.tsx @@ -9,7 +9,7 @@ import { import { Button } from "@storyteller/ui-button"; import { Tooltip } from "@storyteller/ui-tooltip"; import { GalleryItem, GalleryModal } from "@storyteller/ui-gallery-modal"; -import { downloadFileFromUrl } from "@storyteller/api"; +import { downloadFileFromUrl } from "@storyteller/tauri-api"; import { ImageIcon, ImagesIcon, LoaderCircleIcon, MusicIcon, PlayIcon, PlusIcon, SquareIcon, Trash2Icon, VideoIcon, XIcon } from "lucide-react"; import { DynamicIcon } from "@storyteller/icons"; import { RefImage, RefVideo, RefAudio } from "./promptStore"; diff --git a/frontend/libs/components/promptbox/src/lib/deck/useDeckMedia.tsx b/frontend/libs/components/promptbox/src/lib/deck/useDeckMedia.tsx index 772b0f913c..f2945ecccc 100644 --- a/frontend/libs/components/promptbox/src/lib/deck/useDeckMedia.tsx +++ b/frontend/libs/components/promptbox/src/lib/deck/useDeckMedia.tsx @@ -1,6 +1,7 @@ import { useEffect, useMemo, useRef, useState, type ReactNode } from "react"; import { GalleryItem, GalleryModal } from "@storyteller/ui-gallery-modal"; -import { downloadFileFromUrl, type UploadMediaFn } from "@storyteller/api"; +import { type UploadMediaFn } from "@storyteller/api"; +import { downloadFileFromUrl } from "@storyteller/tauri-api"; import { toast } from "@storyteller/ui-toaster"; import { UploaderStates } from "@storyteller/common"; import { diff --git a/frontend/libs/components/settings-modal/src/lib/panes/DownloadsSettingsPane.spec.tsx b/frontend/libs/components/settings-modal/src/lib/panes/DownloadsSettingsPane.spec.tsx index c37ce47bcf..523129c109 100644 --- a/frontend/libs/components/settings-modal/src/lib/panes/DownloadsSettingsPane.spec.tsx +++ b/frontend/libs/components/settings-modal/src/lib/panes/DownloadsSettingsPane.spec.tsx @@ -9,6 +9,8 @@ vi.mock("@storyteller/tauri-api", () => ({ GetAppPreferences: api.get, UpdateAppPreferences: api.update, DownloadDirectoryReveal: vi.fn(), + getAskLocationBeforeDownload: () => false, + setAskLocationBeforeDownload: vi.fn(), PreferenceName: { AutoDownload: "auto_download", PreferredDownloadDirectory: "preferred_download_directory", @@ -16,10 +18,6 @@ vi.mock("@storyteller/tauri-api", () => ({ }, })); vi.mock("@tauri-apps/plugin-dialog", () => ({ open: vi.fn() })); -vi.mock("@storyteller/api", () => ({ - getAskLocationBeforeDownload: () => false, - setAskLocationBeforeDownload: vi.fn(), -})); beforeEach(() => { vi.stubGlobal("ResizeObserver", class { diff --git a/frontend/libs/components/settings-modal/src/lib/panes/DownloadsSettingsPane.tsx b/frontend/libs/components/settings-modal/src/lib/panes/DownloadsSettingsPane.tsx index fecc5b67b4..a2e7678993 100644 --- a/frontend/libs/components/settings-modal/src/lib/panes/DownloadsSettingsPane.tsx +++ b/frontend/libs/components/settings-modal/src/lib/panes/DownloadsSettingsPane.tsx @@ -9,16 +9,14 @@ import { PreferredDownloadFilename, SystemDirectory, UpdateAppPreferences, + getAskLocationBeforeDownload, + setAskLocationBeforeDownload, } from "@storyteller/tauri-api"; import { open } from "@tauri-apps/plugin-dialog"; import { Label } from "@storyteller/ui-label"; import { Switch } from "@storyteller/ui-switch"; import { Select } from "@storyteller/ui-select"; import { FolderIcon, RotateCcwIcon, SearchIcon } from "lucide-react"; -import { - getAskLocationBeforeDownload, - setAskLocationBeforeDownload, -} from "@storyteller/api"; const DEFAULT_CUSTOM_FORMAT = "{model}_{date}"; const FILENAME_OPTIONS = [ diff --git a/frontend/libs/tauri-api/package.json b/frontend/libs/tauri-api/package.json index b0dae5660c..1bd7356f6b 100644 --- a/frontend/libs/tauri-api/package.json +++ b/frontend/libs/tauri-api/package.json @@ -15,6 +15,8 @@ } }, "dependencies": { - "@tauri-apps/api": "^2.11.0" + "@tauri-apps/api": "^2.11.0", + "@tauri-apps/plugin-dialog": "^2.7.1", + "@tauri-apps/plugin-upload": "2.4.0" } } diff --git a/frontend/libs/tauri-api/src/index.ts b/frontend/libs/tauri-api/src/index.ts index 17c54adc4b..f36cc0c39d 100644 --- a/frontend/libs/tauri-api/src/index.ts +++ b/frontend/libs/tauri-api/src/index.ts @@ -6,6 +6,7 @@ export * from "./lib/artcraft/task_queue/TasksNukeAll.js"; export * from "./lib/common/CommandStatus.js"; export * from "./lib/download/DownloadDirectoryReveal.js"; export * from "./lib/download/DownloadUrl.js"; +export * from "./lib/download/LocalDownloads.js"; export * from "./lib/enqueue/EnqueueEditImage.js"; export * from "./lib/enqueue/EnqueueImageBgRemovalCommand.js"; export * from "./lib/enqueue/EnqueueImageInpaint.js"; diff --git a/frontend/libs/api/src/lib/LocalApi.ts b/frontend/libs/tauri-api/src/lib/download/LocalDownloads.ts similarity index 97% rename from frontend/libs/api/src/lib/LocalApi.ts rename to frontend/libs/tauri-api/src/lib/download/LocalDownloads.ts index f9e92c9780..364b65b688 100644 --- a/frontend/libs/api/src/lib/LocalApi.ts +++ b/frontend/libs/tauri-api/src/lib/download/LocalDownloads.ts @@ -1,7 +1,7 @@ import { download } from "@tauri-apps/plugin-upload"; import { open, save } from "@tauri-apps/plugin-dialog"; -import { DownloadUrl, GetDownloadPath } from "@storyteller/tauri-api"; +import { DownloadUrl, GetDownloadPath } from "./DownloadUrl"; const ASK_LOCATION_BEFORE_DOWNLOAD_KEY = "artcraft_ask_location_before_download"; diff --git a/frontend/package-lock.json b/frontend/package-lock.json index 18561a8b56..8125ddc863 100644 --- a/frontend/package-lock.json +++ b/frontend/package-lock.json @@ -443,11 +443,7 @@ }, "libs/api": { "name": "@storyteller/api", - "version": "0.0.1", - "dependencies": { - "@tauri-apps/plugin-dialog": "^2.7.1", - "@tauri-apps/plugin-upload": "2.4.0" - } + "version": "0.0.1" }, "libs/api-enums": { "name": "@storyteller/api-enums", @@ -681,7 +677,9 @@ "name": "@storyteller/tauri-api", "version": "0.0.1", "dependencies": { - "@tauri-apps/api": "^2.11.0" + "@tauri-apps/api": "^2.11.0", + "@tauri-apps/plugin-dialog": "^2.7.1", + "@tauri-apps/plugin-upload": "2.4.0" } }, "libs/tauri-events": {