lightcraft/crates/engine/src/media.rs

560 lines
24 KiB
Rust
Raw Blame History

This file contains ambiguous Unicode characters

This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.

//! Source proxies and render jobs.
//!
//! Sources are decoded (or generated) once per level and cached in memory LRUs: `Thumb` (≤ 512 px
//! long edge) for the grid and filmstrip, `Preview` (≤ 2560 px) for the loupe, `Full` (native
//! resolution, the last one only) for exports larger than a preview. A [`RenderJob`] is
//! self-contained and `Send`: frontends run it on a worker thread; if the source wasn't cached yet
//! the job loads it and hands it back in the [`RenderResult`] so the cache can keep it.
//!
//! Rendered thumbnails are cached too ([`lightcraft_preview::PreviewCache`]: memory LRU, plus a
//! disk cache in the library's `thumbs/` folder), keyed by the photo's content hash, the develop
//! settings hash, the size and [`RENDER_CACHE_VERSION`] — so reopening a library shows its grid
//! without decoding a single original, and an edit simply produces a new key.
//!
//! Opening a photo shows something at once ([`QuickJob`]): the loupe's last render for the current
//! settings (kept in the same cache under a size-independent "view" key), else the camera's
//! embedded JPEG for an unedited raw, else a thumbnail-level render — while the real render is
//! prepared. Grid thumbnails of unedited raws likewise start from the embedded preview.
use std::sync::Arc;
use lightcraft_catalog::{MediaKind, Photo, PhotoId, Source};
use lightcraft_develop::DevelopSettings;
use lightcraft_pipeline::{Quality, RenderRequest, Rendered, SourceInfo, StageCache};
use lightcraft_preview::{Hash128, Hasher128, Lru, PreviewCache};
use lightcraft_raster::{Histogram, Rgb32f, Rgba8};
use serde::{Deserialize, Serialize};
/// Bump when the pipeline's output changes, to invalidate cached thumbnails.
pub const RENDER_CACHE_VERSION: u64 = 3;
/// Thumbnails render at one of these long edges (so window/cell size changes reuse the cache).
pub const THUMB_SIZES: [usize; 4] = [128, 256, 384, 512];
/// Memory budgets.
const THUMB_SOURCE_BYTES: usize = 384 << 20;
const RENDERED_MEM_BYTES: usize = 128 << 20;
/// Disk budget for the thumbnail cache.
pub const DISK_CACHE_BYTES: u64 = 2 << 30;
#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, Serialize, Deserialize)]
#[serde(rename_all = "lowercase")]
pub enum SourceLevel {
Thumb,
Preview,
/// The original at its native resolution (exports larger than a preview).
Full,
}
impl SourceLevel {
pub fn max_edge(self) -> usize {
match self {
SourceLevel::Thumb => 512,
SourceLevel::Preview => 2560,
SourceLevel::Full => usize::MAX,
}
}
/// Smallest level that can serve an output of `long_edge` pixels.
pub fn for_size(long_edge: usize) -> SourceLevel {
match long_edge {
0..=520 => SourceLevel::Thumb,
521..=2560 => SourceLevel::Preview,
_ => SourceLevel::Full,
}
}
}
/// Decodes a file into a linear Rec.2020 image no larger than `max_edge` (set by the app).
pub type FileLoader = Arc<dyn Fn(&str, usize) -> Result<(Rgb32f, SourceInfo), String> + Send + Sync>;
/// A raw file's embedded (camera-rendered) preview as display sRGB, oriented, no larger than
/// `max_edge` (set by the app). `None`: no usable preview.
pub type PreviewLoader = Arc<dyn Fn(&str, usize) -> Option<Rgba8> + Send + Sync>;
#[derive(Clone)]
pub enum SourceRef {
Loaded(Arc<Rgb32f>),
Demo { scene: Box<lightcraft_scenes::Scene>, max_edge: usize },
File { path: String, max_edge: usize, loader: Option<FileLoader> },
}
impl SourceRef {
pub fn load(&self) -> Result<Arc<Rgb32f>, String> {
match self {
SourceRef::Loaded(a) => Ok(a.clone()),
SourceRef::Demo { scene, max_edge } => Ok(Arc::new(scene.render_fit(*max_edge))),
SourceRef::File { path, max_edge, loader } => match loader {
Some(l) => l(path, *max_edge).map(|(img, _)| Arc::new(img)),
None => Err(format!("no decoder available for {path}")),
},
}
}
}
pub struct MediaCache {
/// Decoded thumbnail-level sources (LRU by bytes).
thumbs: Lru<PhotoId, Arc<Rgb32f>>,
previews: Vec<(PhotoId, Arc<Rgb32f>)>,
/// The last full-resolution original (exports; one at a time: ~300 MB at 24 MP).
full: Option<(PhotoId, Arc<Rgb32f>)>,
/// How many previews to keep (LRU).
pub preview_capacity: usize,
pub file_loader: Option<FileLoader>,
pub file_probe: Option<FileProbe>,
pub preview_loader: Option<PreviewLoader>,
scenes: Vec<lightcraft_scenes::Scene>,
/// Rendered thumbnails (memory, plus disk once a library is attached).
pub rendered: Arc<PreviewCache>,
}
impl Default for MediaCache {
fn default() -> Self {
MediaCache {
thumbs: Lru::new(THUMB_SOURCE_BYTES),
previews: Vec::new(),
full: None,
preview_capacity: 0,
file_loader: None,
file_probe: None,
preview_loader: None,
scenes: Vec::new(),
rendered: Arc::new(PreviewCache::memory(RENDERED_MEM_BYTES)),
}
}
}
impl MediaCache {
/// Keep rendered thumbnails on disk in `dir` as well.
pub fn attach_disk_cache(&mut self, dir: &std::path::Path) {
self.rendered = Arc::new(PreviewCache::with_disk(RENDERED_MEM_BYTES, dir, DISK_CACHE_BYTES));
}
pub fn get(&mut self, id: PhotoId, level: SourceLevel) -> Option<Arc<Rgb32f>> {
match level {
SourceLevel::Thumb => self.thumbs.get(&id).cloned(),
SourceLevel::Preview => self.previews.iter().find(|(p, _)| *p == id).map(|(_, a)| a.clone()),
SourceLevel::Full => self.full.as_ref().filter(|(p, _)| *p == id).map(|(_, a)| a.clone()),
}
}
pub fn insert(&mut self, id: PhotoId, level: SourceLevel, img: Arc<Rgb32f>) {
match level {
SourceLevel::Thumb => {
let cost = img.width * img.height * 12 + 64;
self.thumbs.insert(id, img, cost);
}
SourceLevel::Preview => {
self.previews.retain(|(p, _)| *p != id);
self.previews.push((id, img));
let cap = if self.preview_capacity == 0 { 4 } else { self.preview_capacity };
while self.previews.len() > cap {
self.previews.remove(0);
}
}
SourceLevel::Full => self.full = Some((id, img)),
}
}
/// Forget a photo's decoded sources (e.g. after its file changed).
pub fn forget(&mut self, id: PhotoId) {
self.thumbs.remove(&id);
self.previews.retain(|(p, _)| *p != id);
if self.full.as_ref().is_some_and(|(p, _)| *p == id) {
self.full = None;
}
}
/// (decoded thumbnail sources, bytes).
pub fn source_usage(&self) -> (usize, usize) {
(self.thumbs.len(), self.thumbs.cost())
}
/// Decoded sources held: (thumbnail level, preview level, full size).
pub fn usage(&self) -> (crate::memory::Usage, crate::memory::Usage, crate::memory::Usage) {
use crate::memory::Usage;
let bytes = |i: &Rgb32f| i.data.len() * 12;
let previews = Usage::new(self.previews.len(), self.previews.iter().map(|(_, a)| bytes(a)).sum());
let full = self.full.as_ref().map(|(_, a)| Usage::new(1, bytes(a))).unwrap_or_default();
(Usage::new(self.thumbs.len(), self.thumbs.cost()), previews, full)
}
pub fn source_ref(&mut self, p: &Photo, level: SourceLevel) -> SourceRef {
if let Some(a) = self.get(p.id, level) {
return SourceRef::Loaded(a);
}
// Procedural scenes have a nominal size: "full" is that size, not unbounded.
let max_edge = level.max_edge().min(match (&p.source, level) {
(Source::Demo { .. }, SourceLevel::Full) => p.width.max(p.height).max(1) as usize,
_ => usize::MAX,
});
self.origin_ref(&p.source, max_edge)
}
/// How to load `origin` at most `max_edge` pixels long, ignoring decoded sources in memory (a
/// render worker in another wasm instance builds its sources from this).
pub fn origin_ref(&mut self, origin: &Source, max_edge: usize) -> SourceRef {
match origin {
Source::Demo { scene } => {
if self.scenes.is_empty() {
self.scenes = lightcraft_scenes::demo_library();
}
match self.scenes.iter().find(|s| s.id == *scene) {
Some(s) => SourceRef::Demo { scene: Box::new(s.clone()), max_edge },
None => SourceRef::File { path: format!("demo:{scene}"), max_edge, loader: None },
}
}
Source::File { path } => SourceRef::File { path: path.clone(), max_edge, loader: self.file_loader.clone() },
}
}
}
/// Everything needed to render one photo, detached from the session.
#[derive(Clone)]
pub struct RenderJob {
pub photo: PhotoId,
pub level: SourceLevel,
pub source: SourceRef,
/// Where the photo's pixels come from (for executors that can't share `source`, e.g. a web
/// worker).
pub origin: Source,
pub info: SourceInfo,
pub settings: Arc<DevelopSettings>,
pub request: RenderRequest,
/// Identifies the result for caching: hash of settings + request.
pub key: u64,
/// Rendered-thumbnail cache and this job's key in it.
pub cache: Option<(Arc<PreviewCache>, Hash128)>,
/// Intermediate results of this view's previous renders (set by the frontend for the loupe):
/// slider drags then only redo the stages the changed setting feeds.
pub stages: Option<Arc<StageCache>>,
/// Keep a full-quality result here as the photo's view preview ([`crate::Session::loupe_job`]).
pub view_cache: Option<(Arc<PreviewCache>, Hash128)>,
}
pub struct RenderResult {
pub photo: PhotoId,
pub level: SourceLevel,
pub key: u64,
pub rendered: Result<Rendered, String>,
/// A source that was loaded by this job (to be inserted into the cache).
pub loaded: Option<Arc<Rgb32f>>,
/// Set for a [`QuickJob`]'s stand-in: where the image came from.
pub quick: Option<QuickSource>,
}
impl RenderJob {
/// Render at draft quality (interactive drags). Gets its own result key.
pub fn draft(mut self) -> Self {
if self.request.quality != Quality::Draft {
self.request.quality = Quality::Draft;
self.key ^= 0x9e37_79b9_7f4a_7c15;
}
self
}
/// Reuse `stages` across this view's renders.
pub fn with_stages(mut self, stages: Arc<StageCache>) -> Self {
self.stages = Some(stages);
self
}
pub fn run(self) -> RenderResult {
if let Some((cache, key)) = &self.cache
&& let Some(img) = cache.get(*key)
{
let image = Arc::unwrap_or_clone(img);
let histogram = Histogram::of_srgb8(&image);
return RenderResult {
photo: self.photo,
level: self.level,
key: self.key,
rendered: Ok(Rendered { image, histogram }),
loaded: None,
quick: None,
};
}
let was_loaded = matches!(self.source, SourceRef::Loaded(_));
match self.source.load() {
Ok(src) => {
// Thumbnails (many small jobs side by side) stay on the CPU; views and exports use
// the GPU when there is one.
let gpu = self.cache.is_none();
let rendered = develop(&src, &self.info, &self.settings, &self.request, self.stages.as_deref(), gpu);
if let Some((cache, key)) = &self.cache {
cache.put(*key, Arc::new(rendered.image.clone()));
}
if let Some((cache, key)) = &self.view_cache
&& self.request.quality == Quality::Full
{
cache.put_deferred(*key, Arc::new(rendered.image.clone()));
}
RenderResult {
photo: self.photo,
level: self.level,
key: self.key,
rendered: Ok(rendered),
loaded: (!was_loaded).then_some(src),
quick: None,
}
}
Err(e) => RenderResult { photo: self.photo, level: self.level, key: self.key, rendered: Err(e), loaded: None, quick: None },
}
}
}
/// Render `src`: on the GPU when `gpu` is set and a GPU is available (`lightcraft_gpu`), else on
/// the CPU, reusing `stages` either way. Both produce the same image within 1–3 LSB (see
/// `docs/gpu-pipeline.md`); `LIGHTCRAFT_GPU=0` or [`lightcraft_gpu::set_enabled`] forces the CPU.
pub fn develop(src: &Arc<Rgb32f>, info: &SourceInfo, s: &DevelopSettings, req: &RenderRequest, stages: Option<&StageCache>, gpu: bool) -> Rendered {
if gpu && let Some(r) = lightcraft_gpu::render(src, info, s, req, stages) {
return r;
}
match stages {
Some(st) => lightcraft_pipeline::render_cached(src, info, s, req, st),
None => lightcraft_pipeline::render(src, info, s, req),
}
}
/// Where a [`QuickJob`]'s stand-in image came from.
#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)]
#[serde(rename_all = "camelCase")]
pub enum QuickSource {
/// The photo's own develop render for its current settings (cached view or thumbnail render).
Cached,
/// The camera's embedded JPEG (unedited raws).
Embedded,
/// A thumbnail-level develop render made for the occasion.
Small,
}
/// Something to show *now* for a photo while its real render is prepared. Tried in order: cached
/// renders, the embedded preview of an unedited raw, a thumbnail-level render.
#[derive(Clone)]
pub struct QuickJob {
pub photo: PhotoId,
pub key: u64,
/// Cached renders to try first, in order: (cache, key).
pub cached: Vec<(Arc<PreviewCache>, Hash128)>,
/// Embedded preview of an unedited raw: (path, loader, max edge).
pub embedded: Option<(String, PreviewLoader, usize)>,
/// Last resort: run this thumbnail job.
pub small: Option<Box<RenderJob>>,
}
impl QuickJob {
pub fn run(self) -> RenderResult {
let (photo, key) = (self.photo, self.key);
let done = |rendered: Result<Rendered, String>, quick| RenderResult { photo, level: SourceLevel::Thumb, key, rendered, loaded: None, quick };
let rendered = |image: Rgba8| {
let histogram = Histogram::of_srgb8(&image);
Ok(Rendered { image, histogram })
};
for (cache, k) in &self.cached {
if let Some(img) = cache.get(*k) {
return done(rendered(Arc::unwrap_or_clone(img)), Some(QuickSource::Cached));
}
}
if let Some((path, loader, edge)) = &self.embedded
&& let Some(image) = loader(path, *edge)
{
return done(rendered(image), Some(QuickSource::Embedded));
}
if let Some(job) = self.small {
let r = job.run();
return RenderResult { key, quick: Some(QuickSource::Small), ..r };
}
done(Err("no quick preview".into()), None)
}
}
/// What identifies a photo's pixels for caching: its content hash, else its source.
pub fn content_key(p: &Photo) -> String {
match (&p.content_hash, &p.source) {
(Some(h), _) => h.clone(),
(None, Source::Demo { scene }) => format!("demo:{scene}"),
(None, Source::File { path }) => format!("file:{path}:{}", p.file_size),
}
}
pub fn source_info(p: &Photo) -> SourceInfo {
// Procedural demo scenes are scene-referred HDR (like raw files): use the filmic tone map.
if matches!(p.source, Source::Demo { .. }) {
return SourceInfo { raw: true, as_shot_temp: 6500.0, as_shot_tint: 0.0, lens: None };
}
if p.kind == MediaKind::Raw {
SourceInfo { raw: true, as_shot_temp: 5500.0, as_shot_tint: 0.0, lens: p.embedded_lens }
} else {
SourceInfo::default()
}
}
impl crate::Session {
/// Build a render job for `id` fitting `max_w × max_h`. `before` renders the unedited look.
pub fn render_job(&mut self, id: PhotoId, max_w: usize, max_h: usize, before: bool, apply_crop: bool) -> Option<RenderJob> {
self.build_job(id, max_w, max_h, before, apply_crop, None)
}
/// A grid/filmstrip thumbnail job: the long edge is rounded up to one of [`THUMB_SIZES`]
/// (≥ `long_edge`) and the result comes from / goes to the thumbnail cache (memory + disk).
pub fn thumb_job(&mut self, id: PhotoId, long_edge: usize) -> Option<RenderJob> {
let b = THUMB_SIZES.iter().copied().find(|s| *s >= long_edge).unwrap_or(THUMB_SIZES[THUMB_SIZES.len() - 1]);
self.build_job(id, b, b, false, true, Some(b))
}
fn build_job(
&mut self,
id: PhotoId,
max_w: usize,
max_h: usize,
before: bool,
apply_crop: bool,
thumb_bucket: Option<usize>,
) -> Option<RenderJob> {
let p = self.catalog.photo(id)?.clone();
let level = SourceLevel::for_size(max_w.max(max_h));
let source = self.media.source_ref(&p, level);
let settings = if before { Arc::new(lightcraft_pipeline::before_settings(&p.develop)) } else { p.develop.clone() };
let request = RenderRequest { max_w, max_h, quality: Quality::Full, apply_crop };
let key = settings.hash64() ^ ((max_w as u64) << 40) ^ ((max_h as u64) << 20) ^ (apply_crop as u64) ^ (level as u64) << 60;
let cache = thumb_bucket.map(|b| {
let k = Hasher128::new().str(&content_key(&p)).u64(settings.hash64()).u64(b as u64).u64(RENDER_CACHE_VERSION).finish();
(self.media.rendered.clone(), k)
});
Some(RenderJob {
photo: id,
level,
source,
origin: p.source.clone(),
info: source_info(&p),
settings,
request,
key,
cache,
stages: None,
view_cache: None,
})
}
/// Size-independent cache key of a photo's view render (loupe) for its current settings.
fn view_key(p: &Photo, apply_crop: bool) -> Hash128 {
Hasher128::new().str(&content_key(p)).str("view").u64(p.develop.hash64()).u64(apply_crop as u64).u64(RENDER_CACHE_VERSION).finish()
}
/// The loupe's render job: like [`Self::render_job`], and a full-quality result is kept as the
/// photo's view preview (memory, and disk with a library) for [`Self::quick_view_job`].
pub fn loupe_job(&mut self, id: PhotoId, max_w: usize, max_h: usize, apply_crop: bool) -> Option<RenderJob> {
let mut job = self.render_job(id, max_w, max_h, false, apply_crop)?;
let p = self.catalog.photo(id)?;
job.view_cache = Some((self.media.rendered.clone(), Self::view_key(p, apply_crop)));
Some(job)
}
/// The embedded preview of an unedited raw (path, loader), when the app installed a loader.
fn embedded_of(&self, p: &Photo) -> Option<(String, PreviewLoader)> {
match (&p.source, &self.media.preview_loader) {
(Source::File { path }, Some(l)) if p.kind == MediaKind::Raw && crate::import::has_import_look(p) => Some((path.clone(), l.clone())),
_ => None,
}
}
/// Something to show in the loupe right away for `id` (see [`QuickJob`]): its cached view
/// render, else the embedded preview of an unedited raw, else a cached or fresh thumbnail.
pub fn quick_view_job(&mut self, id: PhotoId, max_edge: usize, apply_crop: bool) -> Option<QuickJob> {
let p = self.catalog.photo(id)?.clone();
let small = self.thumb_job(id, THUMB_SIZES[THUMB_SIZES.len() - 1])?;
// (the thumbnail job itself starts with the cached thumbnail, after the sharper embedded preview)
let cached = vec![(self.media.rendered.clone(), Self::view_key(&p, apply_crop))];
let h = Hasher128::new().str(&content_key(&p)).str("quick").u64(p.develop.hash64()).u64(apply_crop as u64).finish();
let key = h.0 as u64;
let embedded = self.embedded_of(&p).map(|(path, l)| (path, l, max_edge.clamp(1, SourceLevel::Preview.max_edge())));
Some(QuickJob { photo: id, key, cached, embedded, small: apply_crop.then(|| Box::new(small)) })
}
/// A first grid thumbnail for an unedited raw: the cached render for `job` (the real
/// thumbnail job, from [`Self::thumb_job`]) if there is one — then the result carries
/// [`QuickSource::Cached`] and `job.key` and is final — else its embedded preview. `None` for
/// other photos (render them directly).
pub fn quick_thumb_job(&mut self, job: &RenderJob) -> Option<QuickJob> {
let p = self.catalog.photo(job.photo)?.clone();
let (path, l) = self.embedded_of(&p)?;
let edge = job.request.max_w.max(job.request.max_h);
Some(QuickJob {
photo: job.photo,
key: job.key,
cached: job.cache.clone().into_iter().collect(),
embedded: Some((path, l, edge)),
small: None,
})
}
/// Accept a finished job's loaded source into the cache.
pub fn accept(&mut self, r: &RenderResult) {
if let Some(src) = &r.loaded {
self.media.insert(r.photo, r.level, src.clone());
}
}
/// Synchronous render (CLI, MCP, tests).
pub fn render_now(&mut self, id: PhotoId, max_w: usize, max_h: usize) -> Result<Rendered, String> {
let job = self.render_job(id, max_w, max_h, false, true).ok_or("no such photo")?;
let r = job.run();
self.accept(&r);
r.rendered
}
/// The source proxy for pixel-statistics commands (auto tone/WB), loading synchronously.
pub fn source_now(&mut self, id: PhotoId, level: SourceLevel) -> Result<Arc<Rgb32f>, String> {
let p = self.catalog.photo(id).ok_or("no such photo")?.clone();
let r = self.media.source_ref(&p, level).load()?;
self.media.insert(id, level, r.clone());
Ok(r)
}
}
/// What an import learns from a file header (set by the app from `lightcraft-codecs`/`-raw`).
#[derive(Clone, Debug, Default)]
pub struct ProbeInfo {
pub width: u32,
pub height: u32,
pub format: String,
pub kind: MediaKind,
pub file_size: u64,
pub captured: Option<String>,
pub meta: lightcraft_catalog::Meta,
pub as_shot_wb: Option<(f64, f64)>,
/// Hash of the file's bytes (hex), for duplicate detection.
pub content_hash: Option<String>,
/// Lens corrections embedded in the file (DNG opcodes).
pub embedded_lens: Option<lightcraft_develop::EmbeddedLens>,
/// The file's embedded XMP packet (raw/DNG files), for develop settings stored inside the file.
pub xmp: Option<String>,
}
pub type FileProbe = Arc<dyn Fn(&str) -> Result<ProbeInfo, String> + Send + Sync>;
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn levels_cover_sizes_and_large_exports_use_the_original() {
assert_eq!(SourceLevel::for_size(256), SourceLevel::Thumb);
assert_eq!(SourceLevel::for_size(2560), SourceLevel::Preview);
assert_eq!(SourceLevel::for_size(2561), SourceLevel::Full);
let mut s = crate::Session::with_demo();
let p = s.catalog.photos().next().unwrap().clone();
let long = p.width.max(p.height) as usize;
assert!(long > 2560, "demo photos are camera-sized");
// a full-size export reads the source at its native size, not an upscaled preview
let job = s.render_job(p.id, long, long, false, true).unwrap();
assert_eq!(job.level, SourceLevel::Full);
match &job.source {
SourceRef::Demo { max_edge, .. } => assert_eq!(*max_edge, long),
_ => panic!("demo source expected"),
}
let loupe = s.render_job(p.id, 1600, 1600, false, true).unwrap();
assert_eq!(loupe.level, SourceLevel::Preview);
assert_ne!(loupe.key, s.render_job(p.id, 1600, 1600, false, true).unwrap().draft().key);
}
}