diff --git a/Cargo.lock b/Cargo.lock index 3fc4e721..397094ed 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -209,7 +209,7 @@ version = "0.6.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "dc890384c8602f339876ded803c97ad529f3842aba97f6392b3dba0dd171769b" dependencies = [ - "equator", + "equator 0.4.2", ] [[package]] @@ -967,6 +967,12 @@ dependencies = [ "unicode-width", ] +[[package]] +name = "coe-rs" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7e8f1e641542c07631228b1e0dc04b69ae3c1d58ef65d5691a439711d805c698" + [[package]] name = "color" version = "0.3.3" @@ -1256,6 +1262,16 @@ version = "0.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d8b14ccef22fc6f5a8f4d7d768562a182c04ce9a3b3157b91390b52ddfdf1a76" +[[package]] +name = "dyn-stack" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "56e53799688f5632f364f8fb387488dd05db9fe45db7011be066fc20e7027f8b" +dependencies = [ + "bytemuck", + "reborrow", +] + [[package]] name = "dyn-stack" version = "0.13.2" @@ -1537,13 +1553,33 @@ version = "0.36.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "773fa9c96dd0dbef887e39d0ed6177f141cce4d68e6041df77570aa3702dfa13" +[[package]] +name = "equator" +version = "0.1.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a3b0a88aa91d0ad2b9684e4479aed31a17d3f9051bdbbc634bd2c01bc5a5eee8" +dependencies = [ + "equator-macro 0.1.9", +] + [[package]] name = "equator" version = "0.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4711b213838dfee0117e3be6ac926007d7f433d7bbe33595975d4190cb07e6fc" dependencies = [ - "equator-macro", + "equator-macro 0.4.2", +] + +[[package]] +name = "equator-macro" +version = "0.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "60d08acb9849f7fb4401564f251be5a526829183a3645a90197dea8e786cf3ae" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", ] [[package]] @@ -1608,6 +1644,42 @@ dependencies = [ "pin-project-lite", ] +[[package]] +name = "faer-core" +version = "0.17.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32da0f6262445261106407bdfb548df4f8f1c89bcc3cff1bb6d08e7f525b575a" +dependencies = [ + "bytemuck", + "coe-rs", + "dyn-stack 0.10.0", + "equator 0.1.10", + "faer-entity", + "gemm 0.17.1", + "matrixcompare-core", + "num-complex", + "num-traits", + "paste", + "pulp 0.18.22", + "reborrow", + "seq-macro", +] + +[[package]] +name = "faer-entity" +version = "0.17.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "93fb12f36da3162b3fefc0c7343d905d5bd2c2bf444a7f63ceeaa2bc003311d3" +dependencies = [ + "bytemuck", + "coe-rs", + "libm", + "num-complex", + "num-traits", + "pulp 0.18.22", + "reborrow", +] + [[package]] name = "fastrand" version = "2.5.0" @@ -1783,13 +1855,33 @@ dependencies = [ "slab", ] +[[package]] +name = "gemm" +version = "0.17.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ab24cc62135b40090e31a76a9b2766a501979f3070fa27f689c27ec04377d32" +dependencies = [ + "dyn-stack 0.10.0", + "gemm-c32 0.17.1", + "gemm-c64 0.17.1", + "gemm-common 0.17.1", + "gemm-f16 0.17.1", + "gemm-f32 0.17.1", + "gemm-f64 0.17.1", + "num-complex", + "num-traits", + "paste", + "raw-cpuid 10.7.0", + "seq-macro", +] + [[package]] name = "gemm" version = "0.18.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ab96b703d31950f1aeddded248bc95543c9efc7ac9c4a21fda8703a83ee35451" dependencies = [ - "dyn-stack", + "dyn-stack 0.13.2", "gemm-c32 0.18.2", "gemm-c64 0.18.2", "gemm-common 0.18.2", @@ -1799,7 +1891,7 @@ dependencies = [ "num-complex", "num-traits", "paste", - "raw-cpuid", + "raw-cpuid 11.6.0", "seq-macro", ] @@ -1809,7 +1901,7 @@ version = "0.19.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "aa0673db364b12263d103b68337a68fbecc541d6f6b61ba72fe438654709eacb" dependencies = [ - "dyn-stack", + "dyn-stack 0.13.2", "gemm-c32 0.19.0", "gemm-c64 0.19.0", "gemm-common 0.19.0", @@ -1819,7 +1911,22 @@ dependencies = [ "num-complex", "num-traits", "paste", - "raw-cpuid", + "raw-cpuid 11.6.0", + "seq-macro", +] + +[[package]] +name = "gemm-c32" +version = "0.17.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b9c030d0b983d1e34a546b86e08f600c11696fde16199f971cd46c12e67512c0" +dependencies = [ + "dyn-stack 0.10.0", + "gemm-common 0.17.1", + "num-complex", + "num-traits", + "paste", + "raw-cpuid 10.7.0", "seq-macro", ] @@ -1829,12 +1936,12 @@ version = "0.18.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f6db9fd9f40421d00eea9dd0770045a5603b8d684654816637732463f4073847" dependencies = [ - "dyn-stack", + "dyn-stack 0.13.2", "gemm-common 0.18.2", "num-complex", "num-traits", "paste", - "raw-cpuid", + "raw-cpuid 11.6.0", "seq-macro", ] @@ -1844,12 +1951,27 @@ version = "0.19.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "086936dbdcb99e37aad81d320f98f670e53c1e55a98bee70573e83f95beb128c" dependencies = [ - "dyn-stack", + "dyn-stack 0.13.2", "gemm-common 0.19.0", "num-complex", "num-traits", "paste", - "raw-cpuid", + "raw-cpuid 11.6.0", + "seq-macro", +] + +[[package]] +name = "gemm-c64" +version = "0.17.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fbb5f2e79fefb9693d18e1066a557b4546cd334b226beadc68b11a8f9431852a" +dependencies = [ + "dyn-stack 0.10.0", + "gemm-common 0.17.1", + "num-complex", + "num-traits", + "paste", + "raw-cpuid 10.7.0", "seq-macro", ] @@ -1859,12 +1981,12 @@ version = "0.18.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "dfcad8a3d35a43758330b635d02edad980c1e143dc2f21e6fd25f9e4eada8edf" dependencies = [ - "dyn-stack", + "dyn-stack 0.13.2", "gemm-common 0.18.2", "num-complex", "num-traits", "paste", - "raw-cpuid", + "raw-cpuid 11.6.0", "seq-macro", ] @@ -1874,15 +1996,34 @@ version = "0.19.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "20c8aeeeec425959bda4d9827664029ba1501a90a0d1e6228e48bef741db3a3f" dependencies = [ - "dyn-stack", + "dyn-stack 0.13.2", "gemm-common 0.19.0", "num-complex", "num-traits", "paste", - "raw-cpuid", + "raw-cpuid 11.6.0", "seq-macro", ] +[[package]] +name = "gemm-common" +version = "0.17.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a2e7ea062c987abcd8db95db917b4ffb4ecdfd0668471d8dc54734fdff2354e8" +dependencies = [ + "bytemuck", + "dyn-stack 0.10.0", + "half", + "num-complex", + "num-traits", + "once_cell", + "paste", + "pulp 0.18.22", + "raw-cpuid 10.7.0", + "seq-macro", + "sysctl 0.5.5", +] + [[package]] name = "gemm-common" version = "0.18.2" @@ -1890,7 +2031,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a352d4a69cbe938b9e2a9cb7a3a63b7e72f9349174a2752a558a8a563510d0f3" dependencies = [ "bytemuck", - "dyn-stack", + "dyn-stack 0.13.2", "half", "libm", "num-complex", @@ -1898,10 +2039,10 @@ dependencies = [ "once_cell", "paste", "pulp 0.21.5", - "raw-cpuid", + "raw-cpuid 11.6.0", "rayon", "seq-macro", - "sysctl", + "sysctl 0.6.0", ] [[package]] @@ -1911,7 +2052,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "88027625910cc9b1085aaaa1c4bc46bb3a36aad323452b33c25b5e4e7c8e2a3e" dependencies = [ "bytemuck", - "dyn-stack", + "dyn-stack 0.13.2", "half", "libm", "num-complex", @@ -1919,10 +2060,27 @@ dependencies = [ "once_cell", "paste", "pulp 0.22.3", - "raw-cpuid", + "raw-cpuid 11.6.0", "rayon", "seq-macro", - "sysctl", + "sysctl 0.6.0", +] + +[[package]] +name = "gemm-f16" +version = "0.17.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7ca4c06b9b11952071d317604acb332e924e817bd891bec8dfb494168c7cedd4" +dependencies = [ + "dyn-stack 0.10.0", + "gemm-common 0.17.1", + "gemm-f32 0.17.1", + "half", + "num-complex", + "num-traits", + "paste", + "raw-cpuid 10.7.0", + "seq-macro", ] [[package]] @@ -1931,14 +2089,14 @@ version = "0.18.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cff95ae3259432f3c3410eaa919033cd03791d81cebd18018393dc147952e109" dependencies = [ - "dyn-stack", + "dyn-stack 0.13.2", "gemm-common 0.18.2", "gemm-f32 0.18.2", "half", "num-complex", "num-traits", "paste", - "raw-cpuid", + "raw-cpuid 11.6.0", "rayon", "seq-macro", ] @@ -1949,30 +2107,45 @@ version = "0.19.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e3df7a55202e6cd6739d82ae3399c8e0c7e1402859b30e4cb780e61525d9486e" dependencies = [ - "dyn-stack", + "dyn-stack 0.13.2", "gemm-common 0.19.0", "gemm-f32 0.19.0", "half", "num-complex", "num-traits", "paste", - "raw-cpuid", + "raw-cpuid 11.6.0", "rayon", "seq-macro", ] +[[package]] +name = "gemm-f32" +version = "0.17.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e9a69f51aaefbd9cf12d18faf273d3e982d9d711f60775645ed5c8047b4ae113" +dependencies = [ + "dyn-stack 0.10.0", + "gemm-common 0.17.1", + "num-complex", + "num-traits", + "paste", + "raw-cpuid 10.7.0", + "seq-macro", +] + [[package]] name = "gemm-f32" version = "0.18.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bc8d3d4385393304f407392f754cd2dc4b315d05063f62cf09f47b58de276864" dependencies = [ - "dyn-stack", + "dyn-stack 0.13.2", "gemm-common 0.18.2", "num-complex", "num-traits", "paste", - "raw-cpuid", + "raw-cpuid 11.6.0", "seq-macro", ] @@ -1982,12 +2155,27 @@ version = "0.19.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "02e0b8c9da1fbec6e3e3ab2ce6bc259ef18eb5f6f0d3e4edf54b75f9fd41a81c" dependencies = [ - "dyn-stack", + "dyn-stack 0.13.2", "gemm-common 0.19.0", "num-complex", "num-traits", "paste", - "raw-cpuid", + "raw-cpuid 11.6.0", + "seq-macro", +] + +[[package]] +name = "gemm-f64" +version = "0.17.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aa397a48544fadf0b81ec8741e5c0fba0043008113f71f2034def1935645d2b0" +dependencies = [ + "dyn-stack 0.10.0", + "gemm-common 0.17.1", + "num-complex", + "num-traits", + "paste", + "raw-cpuid 10.7.0", "seq-macro", ] @@ -1997,12 +2185,12 @@ version = "0.18.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "35b2a4f76ce4b8b16eadc11ccf2e083252d8237c1b589558a49b0183545015bd" dependencies = [ - "dyn-stack", + "dyn-stack 0.13.2", "gemm-common 0.18.2", "num-complex", "num-traits", "paste", - "raw-cpuid", + "raw-cpuid 11.6.0", "seq-macro", ] @@ -2012,12 +2200,12 @@ version = "0.19.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "056131e8f2a521bfab322f804ccd652520c79700d81209e9d9275bbdecaadc6a" dependencies = [ - "dyn-stack", + "dyn-stack 0.13.2", "gemm-common 0.19.0", "num-complex", "num-traits", "paste", - "raw-cpuid", + "raw-cpuid 11.6.0", "seq-macro", ] @@ -2869,6 +3057,38 @@ dependencies = [ "serde", ] +[[package]] +name = "lightcraft-denoise" +version = "0.4.0" +dependencies = [ + "faer-core", + "lightcraft-denoise-core", + "lightcraft-raster", + "miniz_oxide 0.8.9", + "proptest", + "rayon", + "serde", + "serde_json", + "sha2", + "thiserror 2.0.21", + "web-time", +] + +[[package]] +name = "lightcraft-denoise-core" +version = "0.4.0" +dependencies = [ + "lightcraft-raster", + "miniz_oxide 0.8.9", + "proptest", + "rayon", + "serde", + "serde_json", + "sha2", + "thiserror 2.0.21", + "web-time", +] + [[package]] name = "lightcraft-develop" version = "0.4.0" @@ -2891,6 +3111,8 @@ dependencies = [ "lightcraft-catalog", "lightcraft-codecs", "lightcraft-color", + "lightcraft-denoise", + "lightcraft-denoise-core", "lightcraft-develop", "lightcraft-faces", "lightcraft-fetch", @@ -2954,6 +3176,7 @@ version = "0.4.0" dependencies = [ "bytemuck", "lightcraft-color", + "lightcraft-denoise", "lightcraft-develop", "lightcraft-geom", "lightcraft-pipeline", @@ -3118,6 +3341,7 @@ dependencies = [ "lightcraft-catalog", "lightcraft-codecs", "lightcraft-color", + "lightcraft-denoise", "lightcraft-develop", "lightcraft-engine", "lightcraft-faces", @@ -3211,6 +3435,12 @@ dependencies = [ "libc", ] +[[package]] +name = "matrixcompare-core" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b0bdabb30db18805d5290b3da7ceaccbddba795620b86c02145d688e04900a73" + [[package]] name = "matrixmultiply" version = "0.3.11" @@ -4353,6 +4583,18 @@ dependencies = [ "unarray", ] +[[package]] +name = "pulp" +version = "0.18.22" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a0a01a0dc67cf4558d279f0c25b0962bd08fc6dec0137699eae304103e882fe6" +dependencies = [ + "bytemuck", + "libm", + "num-complex", + "reborrow", +] + [[package]] name = "pulp" version = "0.21.5" @@ -4379,7 +4621,7 @@ dependencies = [ "num-complex", "paste", "pulp-wasm-simd-flag", - "raw-cpuid", + "raw-cpuid 11.6.0", "reborrow", "version_check", ] @@ -4580,6 +4822,15 @@ dependencies = [ "rgb", ] +[[package]] +name = "raw-cpuid" +version = "10.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6c297679cb867470fa8c9f67dbba74a78d78e3e98d7cf2b08d6d71540f797332" +dependencies = [ + "bitflags 1.3.2", +] + [[package]] name = "raw-cpuid" version = "11.6.0" @@ -5313,6 +5564,20 @@ dependencies = [ "syn 3.0.6", ] +[[package]] +name = "sysctl" +version = "0.5.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec7dddc5f0fee506baf8b9fdb989e242f17e4b11c61dfbb0635b705217199eea" +dependencies = [ + "bitflags 2.13.2", + "byteorder", + "enum-as-inner", + "libc", + "thiserror 1.0.69", + "walkdir", +] + [[package]] name = "sysctl" version = "0.6.0" diff --git a/Cargo.toml b/Cargo.toml index 058da8f9..5f7c85db 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -11,6 +11,8 @@ repository = "https://github.com/storytold/lightcraft" [workspace.dependencies] # internal +lightcraft-denoise-core = { path = "crates/denoise-core" } +lightcraft-denoise = { path = "crates/denoise", default-features = false } lightcraft-geom = { path = "crates/geom" } lightcraft-faces = { path = "crates/faces" } lightcraft-sysmem = { path = "crates/sysmem" } diff --git a/NOTICE b/NOTICE index f3f6499f..254e2628 100644 --- a/NOTICE +++ b/NOTICE @@ -27,6 +27,15 @@ the facebook/sam3 checkpoint only when they choose to (in the app, or by hand), License (https://github.com/facebookresearch/sam3/blob/main/LICENSE), a separate, non-OSI licence. See docs/ai-masks.md. +AI denoise weights are not part of LightCraft and are never distributed with it. +The default build offers user-supplied models only. The optional rawnind-model +feature offers a separately downloaded RawNIND UtNet2 Bayer checkpoint under +GPL-3.0 (https://www.gnu.org/licenses/gpl-3.0.html), after the user accepts its terms. +Commercial use is subject to those terms, including applicable source, licence and +notice obligations on redistribution. Enabling that offer in releases remains a +maintainer decision. The ONNX reader and CPU/GPU executor are our Rust implementation; +no darktable, RawSpeed, LibRaw or other GPL decoder/runtime code is included. + Builds made with craft-fonts (the optional CRAFT_FONTS_DIR build input; all official releases) also embed the fonts listed in https://github.com/storytold/craft-fonts/blob/main/ATTRIBUTION.md (the Japanese fonts BIZ UDPGothic, BIZ UDMincho and Shippori Mincho; the web build embeds only BIZ diff --git a/ROADMAP.md b/ROADMAP.md index f816f7d1..4a6b101a 100644 --- a/ROADMAP.md +++ b/ROADMAP.md @@ -25,7 +25,7 @@ duplicate Local entries #22, black GPU exports on an Intel iGPU #78). | **Feature checklist** | 79.7% | P0 core and P1 nearly complete: import (Add / Copy / Move, templates, devices), library, grid/loupe/compare/survey, every Edit slider, curves, colour grading, masking tools, crop/Upright, heal/clone, presets/profiles, versions/history, sync, export, menus, shortcuts | P0: full-resolution zoom detail; P1: lens-profile database, content-aware fill (patch synthesis), video playback/trim | | **RAW coverage** (formats people shoot) | ~55% | DNG (all kinds), CR2, CR3 lossless Bayer and version 0x100/0x200 C-RAW (M50/R100/R8 pixel-exact corpus), ARW, NEF (uncompressed + Huffman lossless/lossy), RAF (uncompressed, lossless/lossy compressed Bayer and X-Trans), uncompressed ORF, RW2 / RWL / Panasonic RAW (every raw format, checked on 178 files from 118 Panasonic and Leica bodies), PEF; every container's embedded preview (incl. CR3) | **CR3** (every Canon since ~2018), compressed ORF, Nikon lossy-after-split, Canon sRAW, HEIC/AVIF. NEFs labelled compressed but stored uncompressed (Z 6 packed 14-bit, D850 12-bit uncompressed). Per-model verification is thin (~55 corpus files vs >1,000 models) | | **Colour & image quality** | ~55–65% | Pipeline is complete and fast; GPU path CPU-exact within 1/255 | **No measured camera calibration database**: ARW, NEF, RW2, RAF, CR3, CR2 and PEF have guarded camera-JPEG colour estimates, with bundled ILCE-7M4, X-H2S and X-T4 profiles (docs/camera-preview-colour.md); other non-DNG raws and rejected estimates use a neutral matrix. Colour fidelity remains incomplete. No lens-profile database. No measured fidelity against Lightroom (tone, highlights, texture/clarity, NR, sharpening are tuned by eye) | -| **AI & computational** | ~15–20% | Assisted culling (focus, bursts), auto tone, HDR/panorama merge; subject/sky/background masks as classical heuristics, or the photo's own segmentation mattes when it ships them (DNG semantic masks, e.g. iPhone ProRAW's sky matte); **faces (MVP, [docs/faces.md](docs/faces.md))**: a pure-Rust detector (a 232 KB opt-in download), opt-in recognition models (one-click download or add a file; they run on the shared pure-Rust CPU interpreter; pinned SFace/AuraFace embeddings agree with an independent reference runtime on 40 public crops; one-thread medians 34–36 / 233–252 ms (AuraFace remains expensive)), a background scan that finds and embeds faces, name suggestions, a person's page with look-alike faces to confirm, an Unnamed faces section (look-alikes together, select a group and name it at once) | Real segmentation masks (subject, sky, people, objects, landscape, depth), AI denoise, super resolution, lens blur, generative remove, natural-language search; for faces: clusters with separators between groups, small faces in large photos (the detector looks at 640 px), thresholds calibrated on live faces, writing face regions back to XMP. **Blocked on a model strategy** (licensable weights or our own training; pure-Rust inference is feasible) | +| **AI & computational** | ~15–20% | Assisted culling (focus, bursts), auto tone, HDR/panorama merge; subject/sky/background masks as classical heuristics, or the photo's own segmentation mattes when it ships them (DNG semantic masks, e.g. iPhone ProRAW's sky matte); non-destructive Bayer RAW denoise with opt-in weights on pure-Rust CPU/GPU; **faces (MVP, [docs/faces.md](docs/faces.md))**: a pure-Rust detector (a 232 KB opt-in download), opt-in recognition models (one-click download or add a file; they run on the shared pure-Rust CPU interpreter; pinned SFace/AuraFace embeddings agree with an independent reference runtime on 40 public crops; one-thread medians 34–36 / 233–252 ms (AuraFace remains expensive)), a background scan that finds and embeds faces, name suggestions, a person's page with look-alike faces to confirm, an Unnamed faces section (look-alikes together, select a group and name it at once) | Real segmentation masks (subject, sky, people, objects, landscape, depth), denoise model policy / linear RGB / quality verification, super resolution, lens blur, generative remove, natural-language search; for faces: clusters with separators between groups, small faces in large photos (the detector looks at 640 px), thresholds calibrated on live faces, writing face regions back to XMP. **Blocked on a model strategy** (licensable weights or our own training; pure-Rust inference is feasible) | | **Workflow & library** | ~85% (single machine) | Robust catalog (journal + snapshots, background compaction, crash-tested), 85k-photo libraries stay responsive, Local browsing with automatic cleanup, XMP interop, keywords, smart albums, Move import | Opening an 85k library takes 1.7–4.7 s; no cloud sync (out of scope), no tablet companion (#74, roadmap), shared albums, publish services, tethering | | **Classic modules** | ~30% | Geotagging from GPX track logs, soft proofing (partial), slideshow (basic) | **Map view, Book, Print, Slideshow module, Web, publish services**: ~40 tracker rows ⬜ | | **HDR & video** | 0% | | HDR edit/display/export; video play/trim/edit/export | @@ -40,7 +40,7 @@ duplicate Local entries #22, black GPU exports on an Intel iGPU #78). | Fujifilm raw shooter | ~65% | Uncompressed and lossless/lossy compressed RAF decoded; camera colour calibration remains missing. Verified on 13 bodies, including 14/16-bit GFX; see `docs/raf-compression.md` | | Canon CR3 / Olympus compressed-raw shooter | ~45% | CR3 lossless Bayer and version 0x100/0x200 C-RAW develop from sensor data (M50, R100, R8 verified pixel for pixel); other CRX variants and compressed ORF open as embedded JPEG previews. Camera colour and broader model verification remain | | Lightroom Classic power user | ~45% | Print, Book, Map, publish, tethering | -| Relies on AI (masks, denoise) | ~25% | Object / Describe masks via optional SAM 3 (download not yet hosted); no AI denoise | +| Relies on AI (masks, denoise) | ~25% | Object / Describe masks via optional SAM 3 (download not yet hosted); Bayer AI denoise needs separately installed weights; X-Trans/linear RGB, model policy and quality parity remain open | ## Where we're going @@ -59,9 +59,9 @@ Priorities, in order. Each points at tracker rows in [`docs/parity.md`](docs/par (references stay in the local `plan/`), then tune against the numbers. 5. **Lens profiles of our own** (LR-EDIT-OPTICS-PROFILE, P1). 6. **AI model strategy** (maintainer decision): which permissively licensed models (or our own training) for - segmentation masks and denoise; then pure-Rust inference. Unblocks M12 and Enhance. Face detection and recognition run on our own checked Rust engine. Object / Describe masks now + segmentation masks and denoise. Bayer denoise inference is implemented in pure Rust with an opt-in GPL model; release policy and independently licensed weights remain open. Unblocks M12 and Enhance. Face detection and recognition run on our own checked Rust engine. Object / Describe masks now run Meta's SAM 3 in pure Rust (optional, user-consented download under the non-OSI SAM License; CDN mirrors still - to be set up, docs/ai-masks.md); Subject / Sky / People and Enhance are still open. + to be set up, docs/ai-masks.md); Subject / Sky / People, Raw Details and Super Resolution are still open. 7. **Then:** HDR (Q), the Classic output modules (Print first, then Map view, Book, Slideshow), video (R), localisation and accessibility. @@ -86,7 +86,7 @@ yet guarantee original-pixel detail; visible-region rendering remains a gap (LR- | M4 | Crop, geometry, optics | crop tool + overlays, straighten, Upright (auto/level/vertical/full/guided), manual transforms, CA, defringe, manual lens corrections | 6–10 | ✅ | | M5 | Performance | source pyramids, wgpu compute pipeline (CPU oracle), draft/full renders, prefetch, budgets (16 ms slider updates on 24 MP) | 10–15 | 🚧 (stage cache, source pyramid, wgpu pipeline, prefetch, memory budget ✅; colour NR at half resolution, GPU histogram ⬜) | | M6 | Masking | brush, linear/radial gradients, colour/luminance/depth range, add/subtract/intersect/invert, all local adjustments, masks panel | 8–12 | 🚧 (brush/linear/radial/colour/luminance range, add/subtract/intersect, masks panel ✅; depth range, AI masks ⬜) | -| M7 | Detail | sharpening + masking preview, luminance/colour NR, Denoise, Raw Details, Super Resolution | 6–10 | 🚧 (sharpening, luminance/colour NR ✅; AI Denoise, Raw Details, Super Resolution ⬜) | +| M7 | Detail | sharpening + masking preview, luminance/colour NR, Denoise, Raw Details, Super Resolution | 6–10 | 🚧 (sharpening, luminance/colour NR ✅; AI Denoise 🟡 (Bayer, optional weights; policy/fidelity open); Raw Details, Super Resolution ⬜) | | M8 | Heal / Remove | content-aware remove (PatchMatch), heal, clone, brush spots, visualize spots, red/pet eye | 6–10 | 🚧 (heal, clone, auto source, visualize spots, red/pet eye ✅; PatchMatch remove ⬜) | | M9 | Presets, profiles, versions, sync | preset browser + amount, create/import presets, profile browser, versions, history, copy/paste/sync settings | 5–8 | ✅ | | M10 | Export & share | export dialog (JPEG/PNG/TIFF/DNG/AVIF/JXL/original), sizing, sharpening, metadata, watermark, naming, batch jobs, XMP sidecars, HDR export | 6–10 | 🚧 (all formats incl. DNG/original, sizing, presets, background jobs ✅; JXL encode, HDR export ⬜) | diff --git a/apps/lightcraft-cli/Cargo.toml b/apps/lightcraft-cli/Cargo.toml index 4612b9a5..6471ab86 100644 --- a/apps/lightcraft-cli/Cargo.toml +++ b/apps/lightcraft-cli/Cargo.toml @@ -25,7 +25,9 @@ serde_json = { workspace = true } dhat = { workspace = true, optional = true } [features] -default = [] +default = ["denoise"] +denoise = ["lightcraft-engine/denoise"] +rawnind-model = ["denoise", "lightcraft-engine/rawnind-model"] # Heap profiling: `cargo run --release -p lightcraft-cli --features dhat-heap -- snapshot …` counts # every allocation (live/peak bytes in `library.memory` / `ui.inspect`) and writes dhat-heap.json # (open it in DHAT's viewer: allocation sites by bytes at the peak). Slows allocation-heavy work. diff --git a/apps/lightcraft-cli/src/main.rs b/apps/lightcraft-cli/src/main.rs index 15edac61..f8ddf550 100644 --- a/apps/lightcraft-cli/src/main.rs +++ b/apps/lightcraft-cli/src/main.rs @@ -391,7 +391,7 @@ fn merge(args: &[String]) -> Result<(), String> { } i += 1; } - let mut s = Session::new().with_fs(); + let mut s = Session::new().with_fs().with_default_denoise_models(); let paths = expand_paths(&files); let r = s.execute("library.import", &json!({"paths": paths})).map_err(|e| e.to_string())?; let mut ids: Vec = r["imported"].as_array().map(|a| a.iter().filter_map(Value::as_u64).collect()).unwrap_or_default(); @@ -652,7 +652,7 @@ fn render(args: &[String]) -> Result<(), String> { } let input = input.ok_or("render: missing input file")?; let output = output.ok_or("render: missing -o OUTPUT")?; - let mut s = Session::new().with_fs(); + let mut s = Session::new().with_fs().with_default_denoise_models(); let abs = expand_paths(std::slice::from_ref(&input)); let r = s.execute("library.import", &json!({"paths": abs})).map_err(|e| e.to_string())?; let id = r["imported"][0].as_u64().ok_or_else(|| format!("{input}: not a readable photo"))?; @@ -732,12 +732,12 @@ fn snapshot(args: &[String]) -> Result<(), String> { let t0 = Instant::now(); let mut session = match &library { Some(dir) => { - let mut s = Session::new().with_fs().with_default_face_models(); + let mut s = Session::new().with_fs().with_default_denoise_models().with_default_face_models(); s.open_library(dir, false).map_err(|e| library_error(dir, e))?; s } - None if files.is_empty() => Session::with_demo().with_fs().with_default_face_models(), - None => Session::new().with_fs().with_default_face_models(), + None if files.is_empty() => Session::with_demo().with_fs().with_default_denoise_models().with_default_face_models(), + None => Session::new().with_fs().with_default_denoise_models().with_default_face_models(), }; // Apply compute policy before imports or the first scripted query can discover an adapter. session.execute("app.gpu", &json!({"enabled": false})).map_err(|e| e.to_string())?; diff --git a/apps/lightcraft-web/src/web.rs b/apps/lightcraft-web/src/web.rs index a1c9d76d..f78ac861 100644 --- a/apps/lightcraft-web/src/web.rs +++ b/apps/lightcraft-web/src/web.rs @@ -205,6 +205,7 @@ fn services(originals: Originals, backend: Option, files: Files, frozen open_picker(originals.clone(), backend.clone(), ctx.clone()); Vec::new() // files arrive asynchronously and are imported on a later frame })), + pick_denoise_model: None, // Preset files: browser pickers are asynchronous; not wired on the web yet. pick_preset_files: None, pick_tracklog: None, diff --git a/apps/lightcraft/Cargo.toml b/apps/lightcraft/Cargo.toml index f9173c2a..e4d7f2b3 100644 --- a/apps/lightcraft/Cargo.toml +++ b/apps/lightcraft/Cargo.toml @@ -10,7 +10,9 @@ default-run = "lightcraft" workspace = true [features] -default = [] +default = ["denoise"] +denoise = ["lightcraft-engine/denoise"] +rawnind-model = ["denoise", "lightcraft-engine/rawnind-model"] # Open HEIC/HEIF photos (the optional lightcraft-heif crate). Not default so distributors choose # it (HEVC patents); build with `--features heif` to include it. heif = ["lightcraft-codecs/heif"] diff --git a/apps/lightcraft/src/main.rs b/apps/lightcraft/src/main.rs index be44cc33..a0394da9 100644 --- a/apps/lightcraft/src/main.rs +++ b/apps/lightcraft/src/main.rs @@ -125,7 +125,7 @@ fn gpu_crash_notice(what: &str) -> String { } fn config_dir() -> Option { - lightcraft_engine::camera_profiles::config_dir() + lightcraft_engine::config::config_dir() } /// `/ui.json`, the saved UI state and app settings — `None` when nothing is read or @@ -387,6 +387,13 @@ fn services(ctx: egui::Context, log_file: Option<&std::path::Path>) -> Services .map(|p| p.to_string_lossy().to_string()) .collect() })), + pick_denoise_model: Some(Box::new(|| { + rfd::FileDialog::new() + .add_filter_nocase(lightcraft_ui_egui::i18n::tr("ONNX model"), &["onnx"]) + .pick_file() + .map(|p| vec![p.to_string_lossy().into_owned()]) + .unwrap_or_default() + })), pick_preset_files: Some(Box::new(|| { rfd::FileDialog::new() .set_title(lightcraft_ui_egui::i18n::tr("Import Presets")) @@ -468,15 +475,15 @@ fn open_session(in_memory: bool, dir: Option, seed_demo: boo /// and offers Try Again / Choose Another Library… / Continue Without Saving / Quit (issue #100). fn open_library_session(in_memory: bool, dir: Option, seed_demo: bool) -> (Session, Option) { if in_memory { - return (if seed_demo { Session::with_demo() } else { Session::new() }.with_fs().with_system_clock(), None); + return (if seed_demo { Session::with_demo() } else { Session::new() }.with_fs().with_default_denoise_models().with_system_clock(), None); } - let unopened = || Session::new().with_fs().with_system_clock(); + let unopened = || Session::new().with_fs().with_default_denoise_models().with_system_clock(); let Some(dir) = dir else { log::warn!("no library location (set --library or LIGHTCRAFT_LIBRARY)"); return (unopened(), Some(LibraryProblem::new("", "There is no home folder to keep the library in. Choose a folder for it."))); }; let t0 = std::time::Instant::now(); - let mut s = Session::new().with_fs().with_system_clock(); + let mut s = Session::new().with_fs().with_default_denoise_models().with_system_clock(); match s.open_library(&dir, seed_demo) { Ok(r) => { let (replayed, torn) = (r.replayed, r.torn_bytes); diff --git a/crates/denoise-core/Cargo.toml b/crates/denoise-core/Cargo.toml new file mode 100644 index 00000000..b65f90ae --- /dev/null +++ b/crates/denoise-core/Cargo.toml @@ -0,0 +1,26 @@ +[package] +name = "lightcraft-denoise-core" +version.workspace = true +edition.workspace = true +license.workspace = true +description = "LightCraft denoise model metadata, Bayer tiling and cache formats (L1; no inference backend)" + +[lints] +workspace = true + +[features] +default = [] +rawnind-model = [] + +[dependencies] +lightcraft-raster = { workspace = true } +serde = { workspace = true } +serde_json = { workspace = true } +sha2 = { workspace = true } +thiserror = { workspace = true } +miniz_oxide = { workspace = true } +rayon = { workspace = true } +web-time = { workspace = true } + +[dev-dependencies] +proptest = { workspace = true } diff --git a/crates/denoise-core/README.md b/crates/denoise-core/README.md new file mode 100644 index 00000000..74d1e324 --- /dev/null +++ b/crates/denoise-core/README.md @@ -0,0 +1,7 @@ +# lightcraft-denoise-core + +L1 model metadata, bounded archive/cache formats and Bayer packing/tiling helpers. +No weights or inference backend. See [the user workflow](../../docs/denoise.md). +`lightcraft-denoise` re-exports these types for CPU/GPU users; the engine uses this +crate directly when inference is disabled. The default download catalog is empty. +Run `cargo test -p lightcraft-denoise-core`; optional catalog checks use `--features rawnind-model`. diff --git a/crates/denoise-core/src/archive.rs b/crates/denoise-core/src/archive.rs new file mode 100644 index 00000000..ccfea471 --- /dev/null +++ b/crates/denoise-core/src/archive.rs @@ -0,0 +1,279 @@ +//! Taking one file out of a `.zip` (a `.dtmodel` is one), the little of the format a model download needs: stored and +//! deflated entries, found by name, checked against their CRC-32 and a size limit, written next to where they are +//! going and moved into place. It is a stranger's file, so every length and offset in it is checked, a name is only +//! ever compared (never used as a path), and zip64, encrypted and multi-disk archives are refused. + +use std::io::{Read, Seek, SeekFrom, Write}; +use std::path::Path; + +/// How far from the end the end-of-central-directory record can start (a 22-byte record and a 64 KiB comment). +const EOCD_SEARCH: u64 = 22 + 65535; +/// Largest central directory we read. +const MAX_DIRECTORY: u64 = 1 << 20; + +#[derive(Debug, PartialEq, thiserror::Error)] +pub enum ArchiveError { + #[error("the archive could not be read: {0}")] + Io(String), + #[error("not a usable archive: {0}")] + Format(String), + #[error("the archive has no file called `{0}`")] + Missing(String), +} + +impl From for ArchiveError { + fn from(e: std::io::Error) -> Self { + ArchiveError::Io(e.to_string()) + } +} + +fn format(why: &str) -> Result { + Err(ArchiveError::Format(why.into())) +} + +fn u16_at(b: &[u8], at: usize) -> Option { + Some(u16::from_le_bytes(b.get(at..at.checked_add(2)?)?.try_into().ok()?)) +} + +fn u32_at(b: &[u8], at: usize) -> Option { + Some(u32::from_le_bytes(b.get(at..at.checked_add(4)?)?.try_into().ok()?)) +} + +/// CRC-32 (IEEE) of `bytes`. +fn crc32(bytes: &[u8]) -> u32 { + let mut table = [0u32; 256]; + for (i, t) in table.iter_mut().enumerate() { + let mut c = i as u32; + for _ in 0..8 { + c = if c & 1 != 0 { 0xedb8_8320 ^ (c >> 1) } else { c >> 1 }; + } + *t = c; + } + !bytes.iter().fold(!0u32, |c, &b| table[((c ^ u32::from(b)) & 0xff) as usize] ^ (c >> 8)) +} + +struct Entry { + method: u16, + crc: u32, + compressed: u64, + size: u64, + local_header: u64, +} + +/// Find the entry called `name` (exactly, or as the last part of a longer name: `folder/name`) in the central directory. +fn find(f: &mut (impl Read + Seek), name: &str) -> Result { + let len = f.seek(SeekFrom::End(0))?; + let tail_len = len.min(EOCD_SEARCH); + f.seek(SeekFrom::Start(len - tail_len))?; + let mut tail = vec![0u8; tail_len as usize]; + f.read_exact(&mut tail)?; + let eocd = (0..tail.len().saturating_sub(21)).rev().find(|&i| u32_at(&tail, i) == Some(0x0605_4b50)); + let Some(e) = eocd else { return format("this is not a zip file") }; + let (entries, size, offset) = (u16_at(&tail, e + 10), u32_at(&tail, e + 12), u32_at(&tail, e + 16)); + let (Some(entries), Some(size), Some(offset)) = (entries, size, offset) else { return format("the end of the archive is cut short") }; + if entries == 0xffff || size == u32::MAX || offset == u32::MAX { + return format("zip64 archives are not supported"); + } + if u64::from(size) > MAX_DIRECTORY || u64::from(offset) + u64::from(size) > len { + return format("the list of files is not where it says"); + } + f.seek(SeekFrom::Start(u64::from(offset)))?; + let mut dir = vec![0u8; size as usize]; + f.read_exact(&mut dir)?; + let mut at = 0usize; + for _ in 0..entries { + if u32_at(&dir, at) != Some(0x0201_4b50) { + return format("the list of files is damaged"); + } + let get = |o: usize| at.checked_add(o); + let (flags, method, crc) = (u16_at(&dir, at + 8), u16_at(&dir, at + 10), u32_at(&dir, at + 16)); + let (compressed, uncompressed) = (u32_at(&dir, at + 20), u32_at(&dir, at + 24)); + let (n, x, c) = (u16_at(&dir, at + 28), u16_at(&dir, at + 30), u16_at(&dir, at + 32)); + let local = u32_at(&dir, at + 42); + let (Some(flags), Some(method), Some(crc), Some(compressed), Some(uncompressed), Some(n), Some(x), Some(c), Some(local), Some(start)) = + (flags, method, crc, compressed, uncompressed, n, x, c, local, get(46)) + else { + return format("the list of files is cut short"); + }; + let Some(entry_name) = dir.get(start..start + usize::from(n)) else { return format("a file name is cut short") }; + let matches = entry_name == name.as_bytes() || entry_name.strip_suffix(name.as_bytes()).is_some_and(|p| p.ends_with(b"/")); + if matches { + if flags & 1 != 0 { + return format("encrypted files are not supported"); + } + if compressed == u32::MAX || uncompressed == u32::MAX { + return format("zip64 files are not supported"); + } + return Ok(Entry { method, crc, compressed: u64::from(compressed), size: u64::from(uncompressed), local_header: u64::from(local) }); + } + at = start + usize::from(n) + usize::from(x) + usize::from(c); + } + Err(ArchiveError::Missing(name.into())) +} + +/// Take the file called `name` out of the archive at `archive` and write it to `dest` (through a temporary file next +/// to it); `max_bytes` bounds its size. Returns the number of bytes written. +pub fn extract(archive: &Path, name: &str, dest: &Path, max_bytes: u64) -> Result { + let mut f = std::io::BufReader::new(std::fs::File::open(archive)?); + let e = find(&mut f, name)?; + if e.size > max_bytes || e.compressed > max_bytes { + return format("the file in the archive is larger than allowed"); + } + f.seek(SeekFrom::Start(e.local_header))?; + let mut head = [0u8; 30]; + f.read_exact(&mut head)?; + if u32_at(&head, 0) != Some(0x0403_4b50) { + return format("the file's header is not where it says"); + } + let skip = u64::from(u16_at(&head, 26).unwrap_or(0)) + u64::from(u16_at(&head, 28).unwrap_or(0)); + f.seek_relative(i64::try_from(skip).map_err(|_| ArchiveError::Format("a header is too long".into()))?)?; + let len = usize::try_from(e.compressed) + .ok() + .filter(|&n| n <= 256 * 1024 * 1024) + .ok_or_else(|| ArchiveError::Format("the compressed model is too large".into()))?; + if e.size > 256 * 1024 * 1024 { + return format("the extracted model is too large"); + } + let mut packed = Vec::new(); + packed.try_reserve_exact(len).map_err(|_| ArchiveError::Format("not enough memory for the model archive".into()))?; + packed.resize(len, 0u8); + f.read_exact(&mut packed)?; + let data = match e.method { + 0 => packed, + 8 => miniz_oxide::inflate::decompress_to_vec_with_limit(&packed, max_bytes.min(256 * 1024 * 1024) as usize) + .map_err(|err| ArchiveError::Format(format!("{err:?}")))?, + m => return format(&format!("compression method {m} is not supported")), + }; + if data.len() as u64 != e.size || crc32(&data) != e.crc { + return format("the file in the archive is damaged"); + } + let tmp = dest.with_extension("part"); + let written = (|| -> std::io::Result<()> { + let mut out = std::fs::File::create(&tmp)?; + out.write_all(&data)?; + out.flush()?; + drop(out); + std::fs::rename(&tmp, dest) + })(); + if let Err(err) = written { + let _ = std::fs::remove_file(&tmp); + return Err(err.into()); + } + Ok(data.len() as u64) +} + +#[cfg(test)] +mod tests { + use super::*; + + /// A zip with the given (name, bytes, deflate) entries, written by hand. + pub(crate) fn make_zip(files: &[(&str, &[u8], bool)]) -> Vec { + let mut out = Vec::new(); + let mut dir = Vec::new(); + for (name, data, deflate) in files { + let packed = if *deflate { miniz_oxide::deflate::compress_to_vec(data, 6) } else { data.to_vec() }; + let offset = out.len() as u32; + let method: u16 = if *deflate { 8 } else { 0 }; + out.extend_from_slice(&0x0403_4b50u32.to_le_bytes()); + out.extend_from_slice(&[20, 0, 0, 0]); + out.extend_from_slice(&method.to_le_bytes()); + out.extend_from_slice(&[0, 0, 0, 0]); + out.extend_from_slice(&crc32(data).to_le_bytes()); + out.extend_from_slice(&(packed.len() as u32).to_le_bytes()); + out.extend_from_slice(&(data.len() as u32).to_le_bytes()); + out.extend_from_slice(&(name.len() as u16).to_le_bytes()); + out.extend_from_slice(&[0, 0]); + out.extend_from_slice(name.as_bytes()); + out.extend_from_slice(&packed); + dir.extend_from_slice(&0x0201_4b50u32.to_le_bytes()); + dir.extend_from_slice(&[20, 0, 20, 0, 0, 0]); + dir.extend_from_slice(&method.to_le_bytes()); + dir.extend_from_slice(&[0, 0, 0, 0]); + dir.extend_from_slice(&crc32(data).to_le_bytes()); + dir.extend_from_slice(&(packed.len() as u32).to_le_bytes()); + dir.extend_from_slice(&(data.len() as u32).to_le_bytes()); + dir.extend_from_slice(&(name.len() as u16).to_le_bytes()); + dir.extend_from_slice(&[0; 12]); + dir.extend_from_slice(&offset.to_le_bytes()); + dir.extend_from_slice(name.as_bytes()); + } + let cd_offset = out.len() as u32; + out.extend_from_slice(&dir); + out.extend_from_slice(&0x0605_4b50u32.to_le_bytes()); + out.extend_from_slice(&[0, 0, 0, 0]); + out.extend_from_slice(&(files.len() as u16).to_le_bytes()); + out.extend_from_slice(&(files.len() as u16).to_le_bytes()); + out.extend_from_slice(&(dir.len() as u32).to_le_bytes()); + out.extend_from_slice(&cd_offset.to_le_bytes()); + out.extend_from_slice(&[0, 0]); + out + } + + fn temp(name: &str) -> std::path::PathBuf { + std::env::temp_dir().join(format!("lc-denoise-zip-{name}-{}", std::process::id())) + } + + #[test] + fn a_stored_and_a_deflated_file_come_out_whole() { + let big: Vec = (0..200_000u32).map(|i| (i * 7 % 251) as u8).collect(); + let zip = make_zip(&[("m/config.json", b"{}", false), ("m/model.onnx", &big, true), ("m/other.onnx", b"zz", true)]); + let (a, out) = (temp("a.zip"), temp("a.onnx")); + std::fs::write(&a, &zip).unwrap(); + assert_eq!(extract(&a, "model.onnx", &out, 1 << 20).unwrap(), big.len() as u64); + assert_eq!(std::fs::read(&out).unwrap(), big); + assert_eq!(extract(&a, "m/config.json", &out, 1 << 20).unwrap(), 2); + assert_eq!(std::fs::read(&out).unwrap(), b"{}"); + // a name must match whole path parts: `del.onnx` is not `model.onnx` + assert!(matches!(extract(&a, "del.onnx", &out, 1 << 20), Err(ArchiveError::Missing(_)))); + assert!(matches!(extract(&a, "nothing", &out, 1 << 20), Err(ArchiveError::Missing(_)))); + // the size limit applies + assert!(matches!(extract(&a, "model.onnx", &out, 1000), Err(ArchiveError::Format(_)))); + for p in [a, out] { + let _ = std::fs::remove_file(p); + } + } + + /// Opt-in: `LC_DENOISE_DTMODEL= cargo test -p lightcraft-denoise -- --ignored archive` + #[test] + #[ignore = "needs the real download: set LC_DENOISE_DTMODEL"] + fn the_real_download_gives_the_model_it_should() { + let Some(path) = std::env::var_os("LC_DENOISE_DTMODEL") else { return }; + let known = crate::known::find("rawnind-bayer").unwrap(); + let (d, entry) = (known.download.unwrap(), known.entry.unwrap()); + assert_eq!(crate::hash::sha256_file(Path::new(&path)).unwrap(), d.sha256); + let out = temp("real.onnx"); + let n = extract(Path::new(&path), entry, &out, crate::manifest::MAX_MODEL_BYTES).unwrap(); + assert_eq!(Some(n), known.manifest.size_bytes); + assert_eq!(Some(crate::hash::sha256_file(&out).unwrap()), known.manifest.sha256); + let _ = std::fs::remove_file(out); + } + + #[test] + fn damaged_and_hostile_archives_are_errors_not_panics() { + let zip = make_zip(&[("m/model.onnx", &vec![9u8; 5000], true)]); + let (a, out) = (temp("b.zip"), temp("b.onnx")); + // cut anywhere + for n in (0..zip.len()).step_by(3) { + std::fs::write(&a, &zip[..n]).unwrap(); + assert!(extract(&a, "model.onnx", &out, 1 << 20).is_err(), "cut at {n}"); + } + // one byte changed anywhere: an error, or (when it falls in an unused field) the right file, never a panic + for at in 0..zip.len() { + let mut bad = zip.clone(); + bad[at] ^= 0x5a; + std::fs::write(&a, &bad).unwrap(); + if extract(&a, "model.onnx", &out, 1 << 20).is_ok() { + assert_eq!(std::fs::read(&out).unwrap(), vec![9u8; 5000], "a changed byte at {at} gave another file"); + } + } + // not a zip at all, and an empty file + for junk in [Vec::new(), b"PK".to_vec(), vec![0u8; 100_000]] { + std::fs::write(&a, junk).unwrap(); + assert!(extract(&a, "model.onnx", &out, 1 << 20).is_err()); + } + assert!(extract(&temp("missing.zip"), "model.onnx", &out, 1 << 20).is_err()); + for p in [a, out] { + let _ = std::fs::remove_file(p); + } + } +} diff --git a/crates/denoise-core/src/bayer.rs b/crates/denoise-core/src/bayer.rs new file mode 100644 index 00000000..a4a00d94 --- /dev/null +++ b/crates/denoise-core/src/bayer.rs @@ -0,0 +1,191 @@ +//! Bayer geometry: which way a mosaic is laid out, and how a patch of it is packed into the four planes +//! (red, green on the red row, green on the blue row, blue) a denoise model takes. +//! +//! Every Bayer layout is the same 2 × 2 cell shifted by one pixel at most, so a layout is just where its +//! first red sample is. The cell grid starts one pixel before the picture when that red sample is in +//! column or row 1, and the picture is read with its edge mirrored (without repeating the edge sample, so +//! the colour of a mirrored sample is the colour that belongs at that place). + +/// Where the first red sample of a Bayer mosaic sits, `(dx, dy)` in `{0, 1}²`: RGGB is `(0, 0)`, GRBG `(1, 0)`, +/// GBRG `(0, 1)`, BGGR `(1, 1)`. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct Layout { + pub dx: usize, + pub dy: usize, +} + +impl Layout { + /// The layout of a 2 × 2 colour pattern, row-major, colours 0 = red, 1 = green, 2 = blue; `None` when + /// it is not a Bayer cell (one red, one blue, the greens on the other diagonal). + pub fn from_cell(cell: [u8; 4]) -> Option { + let at = |c: u8| cell.iter().position(|&v| v == c); + let red = at(0)?; + let blue = at(2)?; + let greens = cell.iter().filter(|&&v| v == 1).count(); + // red and blue sit on one diagonal: indices 0+3 or 1+2 + if greens != 2 || red + blue != 3 { + return None; + } + Some(Layout { dx: red % 2, dy: red / 2 }) + } + + /// The cell grid's origin: `-1` along an axis whose first red sample is at 1, so that cell 0 starts on a red column or row. + fn origin(self) -> (isize, isize) { + (isize::try_from(self.dx).unwrap_or(isize::MAX).saturating_neg(), isize::try_from(self.dy).unwrap_or(isize::MAX).saturating_neg()) + } + + /// Cells across and down covering a `width × height` mosaic. + pub fn cells(self, width: usize, height: usize) -> (usize, usize) { + let cells = |n: usize, phase: usize| n / 2 + (n % 2 + phase.min(1)).div_ceil(2); + (cells(width, self.dx), cells(height, self.dy)) + } + + /// The mosaic position of cell `(cx, cy)`'s top-left sample (may lie outside the picture). + pub fn position(self, cx: isize, cy: isize) -> (isize, isize) { + let (ox, oy) = self.origin(); + (ox.saturating_add(cx.saturating_mul(2)), oy.saturating_add(cy.saturating_mul(2))) + } +} + +/// `i` mirrored into `0..n` (ping-pong, edge not repeated), so `-1` is `1` and `n` is `n - 2`. `n == 0` gives 0. +pub fn reflect(i: isize, n: usize) -> usize { + if n <= 1 { + return 0; + } + if let Some(period) = isize::try_from(n).ok().and_then(|n| (n - 1).checked_mul(2)) { + let m = i.rem_euclid(period); + return (if m < n as isize { m } else { period - m }) as usize; + } + let period = 2 * (n as i128 - 1); + let m = (i as i128).rem_euclid(period); + // The reflected value is in 0..n, so this conversion always fits usize. + (if m < n as i128 { m } else { period - m }) as usize +} + +/// Pack the `tile × tile` cells whose top-left cell is `(cx0, cy0)` into `out` as four planes `[R, G1, G2, B]` +/// (`4 · tile²` values, NCHW). G1 is the green beside the red sample, G2 the green under it. Samples outside +/// the picture are mirrored. Returns `false` (and writes nothing) when the sizes do not fit. +pub fn pack_tile(mosaic: &[f32], width: usize, height: usize, layout: Layout, cx0: isize, cy0: isize, tile: usize, out: &mut [f32]) -> bool { + let Some(plane) = tile.checked_mul(tile) else { return false }; + let Some(values) = plane.checked_mul(4) else { return false }; + if width == 0 + || height == 0 + || tile == 0 + || layout.dx > 1 + || layout.dy > 1 + || width.checked_mul(height) != Some(mosaic.len()) + || out.len() != values + { + return false; + } + let Ok(last) = isize::try_from(tile - 1) else { return false }; + let (ox, oy) = layout.origin(); + for (start, origin) in [(cx0, ox), (cy0, oy)] { + if start.checked_mul(2).and_then(|v| v.checked_add(origin)).is_none() + || start.checked_add(last).and_then(|v| v.checked_mul(2)).and_then(|v| v.checked_add(origin)).and_then(|v| v.checked_add(1)).is_none() + { + return false; + } + } + // the four samples of a cell, as (dx, dy) from its top-left sample + const SUB: [(isize, isize); 4] = [(0, 0), (1, 0), (0, 1), (1, 1)]; + for (p, &(sx, sy)) in SUB.iter().enumerate() { + let Some(dst) = out.get_mut(p * plane..(p + 1) * plane) else { return false }; + for (j, row) in dst.chunks_exact_mut(tile).enumerate() { + let (_, y) = layout.position(cx0, cy0 + j as isize); + let ry = reflect(y + sy, height); + let Some(src) = mosaic.get(ry * width..(ry + 1) * width) else { return false }; + for (i, v) in row.iter_mut().enumerate() { + let (x, _) = layout.position(cx0 + i as isize, cy0); + *v = src.get(reflect(x + sx, width)).copied().unwrap_or(0.0); + } + } + } + true +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn hostile_sizes_and_positions_never_panic_or_write_output() { + let layout = Layout::from_cell([0, 1, 1, 2]).unwrap(); + let mut out = [7.0; 4]; + for (w, h, tile, x) in [(usize::MAX, 2, 1, 0), (1, 1, usize::MAX, 0), (1, 1, 0, 0), (1, 1, 1, isize::MAX), (1, 1, 1, isize::MIN)] { + assert!(!pack_tile(&[0.5], w, h, layout, x, 0, tile, &mut out)); + assert_eq!(out, [7.0; 4]); + } + for n in [1, 2, isize::MAX as usize, usize::MAX] { + for i in [isize::MIN, -1, 0, isize::MAX] { + assert!(reflect(i, n) < n); + } + } + assert_eq!(layout.cells(usize::MAX, usize::MAX), (usize::MAX / 2 + 1, usize::MAX / 2 + 1)); + } + + #[test] + fn layouts_from_cells() { + assert_eq!(Layout::from_cell([0, 1, 1, 2]), Some(Layout { dx: 0, dy: 0 })); + assert_eq!(Layout::from_cell([1, 0, 2, 1]), Some(Layout { dx: 1, dy: 0 })); + assert_eq!(Layout::from_cell([1, 2, 0, 1]), Some(Layout { dx: 0, dy: 1 })); + assert_eq!(Layout::from_cell([2, 1, 1, 0]), Some(Layout { dx: 1, dy: 1 })); + // not Bayer: two reds, a missing blue, greens on the wrong diagonal + assert_eq!(Layout::from_cell([0, 0, 1, 2]), None); + assert_eq!(Layout::from_cell([0, 1, 2, 1]), None); + assert_eq!(Layout::from_cell([1, 1, 1, 1]), None); + } + + #[test] + fn reflection_keeps_the_colour_of_a_site() { + assert_eq!(reflect(-1, 10), 1); + assert_eq!(reflect(-2, 10), 2); + assert_eq!(reflect(10, 10), 8); + assert_eq!(reflect(11, 10), 7); + assert_eq!(reflect(25, 10), 7); + assert_eq!(reflect(5, 0), 0); + assert_eq!(reflect(5, 1), 0); + for i in -40..60 { + assert_eq!(reflect(i, 11) % 2, (i.rem_euclid(2)) as usize, "parity at {i}"); + } + } + + #[test] + fn packing_puts_each_colour_in_its_plane_for_every_layout() { + // a mosaic whose sample value says which colour it is: R = 10, G = 20, B = 30 + for cell in [[0u8, 1, 1, 2], [1, 0, 2, 1], [1, 2, 0, 1], [2, 1, 1, 0]] { + let layout = Layout::from_cell(cell).unwrap(); + let (w, h) = (10usize, 8usize); + let mosaic: Vec = (0..w * h).map(|i| [10.0, 20.0, 30.0][cell[(i / w % 2) * 2 + i % w % 2] as usize]).collect(); + let (nx, ny) = layout.cells(w, h); + let tile = nx.max(ny); + let mut out = vec![0.0; 4 * tile * tile]; + assert!(pack_tile(&mosaic, w, h, layout, 0, 0, tile, &mut out)); + let plane = tile * tile; + for (p, want) in [10.0, 20.0, 20.0, 30.0].into_iter().enumerate() { + assert!(out[p * plane..(p + 1) * plane].iter().all(|&v| v == want), "layout {cell:?} plane {p}"); + } + } + } + + #[test] + fn g1_is_beside_the_red_sample_and_g2_under_it() { + // RGGB with a distinct value at every site + let (w, h) = (4, 4); + let mosaic: Vec = (0..16).map(|i| i as f32).collect(); + let mut out = vec![0.0; 4 * 4]; + assert!(pack_tile(&mosaic, w, h, Layout { dx: 0, dy: 0 }, 0, 0, 2, &mut out)); + // cell (0,0): R = site (0,0)=0, G1 = site (1,0)=1, G2 = site (0,1)=4, B = site (1,1)=5 + assert_eq!([out[0], out[4], out[8], out[12]], [0.0, 1.0, 4.0, 5.0]); + // cell (1,0): sites (2,0)=2, (3,0)=3, (2,1)=6, (3,1)=7 + assert_eq!([out[1], out[5], out[9], out[13]], [2.0, 3.0, 6.0, 7.0]); + } + + #[test] + fn packing_rejects_wrong_sizes() { + let mut out = vec![0.0; 16]; + assert!(!pack_tile(&[0.0; 15], 4, 4, Layout { dx: 0, dy: 0 }, 0, 0, 2, &mut out)); + assert!(!pack_tile(&[0.0; 16], 4, 4, Layout { dx: 0, dy: 0 }, 0, 0, 3, &mut out)); + assert!(!pack_tile(&[], 0, 0, Layout { dx: 0, dy: 0 }, 0, 0, 2, &mut out)); + } +} diff --git a/crates/denoise-core/src/hash.rs b/crates/denoise-core/src/hash.rs new file mode 100644 index 00000000..50f83be3 --- /dev/null +++ b/crates/denoise-core/src/hash.rs @@ -0,0 +1,46 @@ +//! SHA-256 of a model file, streamed (a model can be hundreds of megabytes). + +use std::io::{self, Read}; +use std::path::Path; + +use sha2::{Digest, Sha256}; + +/// Lowercase hex SHA-256 of everything `r` yields. +pub fn sha256_reader(mut r: R) -> io::Result { + let mut h = Sha256::new(); + let mut buf = vec![0u8; 1 << 20]; + loop { + let n = r.read(&mut buf)?; + if n == 0 { + break; + } + h.update(buf.get(..n).unwrap_or(&[])); + } + Ok(hex(&h.finalize())) +} + +/// Lowercase hex SHA-256 of a file. +pub fn sha256_file(path: &Path) -> io::Result { + sha256_reader(io::BufReader::new(std::fs::File::open(path)?)) +} + +/// Lowercase hex SHA-256 of `bytes`. +pub fn sha256_hex(bytes: &[u8]) -> String { + hex(&Sha256::digest(bytes)) +} + +fn hex(bytes: &[u8]) -> String { + bytes.iter().map(|b| format!("{b:02x}")).collect() +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn known_digests() { + assert_eq!(sha256_hex(b""), "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"); + assert_eq!(sha256_hex(b"abc"), "ba7816bf8f01cfea414140de5dae2223b00361a396177a9cb410ff61f20015ad"); + assert_eq!(sha256_reader(&b"abc"[..]).unwrap(), sha256_hex(b"abc")); + } +} diff --git a/crates/denoise-core/src/known.rs b/crates/denoise-core/src/known.rs new file mode 100644 index 00000000..f9a51109 --- /dev/null +++ b/crates/denoise-core/src/known.rs @@ -0,0 +1,128 @@ +//! The denoise models LightCraft knows about: what each one is, who may use it, and where to fetch it. +//! +//! None is part of LightCraft. A model with an address here can be downloaded when the user asks and has accepted +//! its terms; anything else is brought by the user as an `.onnx` file with a `denoise-model.json` beside it. + +#[cfg(any(feature = "rawnind-model", test))] +use crate::licence::{Commercial, Licence}; + +use crate::manifest::DenoiserManifest; +#[cfg(any(feature = "rawnind-model", test))] +use crate::manifest::{Domain, Gain}; + +/// A known model, and how to get it. +pub struct Known { + pub manifest: DenoiserManifest, + /// What to fetch (an archive or the `.onnx` itself); `None` when the user must bring the file. + pub download: Option, + /// The `.onnx` inside the downloaded archive (`None`: the download is the `.onnx`). + pub entry: Option<&'static str>, +} + +/// RawNIND's UtNet2 for Bayer sensors, as published for darktable (`rawdenoise-nind.dtmodel`, a zip with a Bayer and a +/// linear model). Its weights are GPL-3.0; the training pictures are CC BY 4.0 and CC0. +#[cfg(any(feature = "rawnind-model", test))] +fn rawnind_bayer() -> Known { + let manifest = DenoiserManifest { + id: "rawnind-bayer".into(), + name: "RawNIND UtNet2 (Bayer)".into(), + version: "1.0".into(), + licence: Licence { + name: "GPL-3.0".into(), + commercial: Commercial::Yes, + url: Some("https://www.gnu.org/licenses/gpl-3.0.html".into()), + notice: "The weights are published under the GNU General Public License v3. LightCraft does not include them: they \ + are downloaded to your computer at your request and used by it only. If you pass the model file on, the \ + GPL applies to that copy. Commercial use is subject to the GPL terms, including applicable source, licence and notice obligations on redistribution." + .into(), + }, + source: Some("https://github.com/darktable-org/darktable-ai/tree/master/models/rawdenoise-nind".into()), + sha256: Some("da27509dab6a2915da67e988acd86cf71f9d5bbc8d1aa0ed32933578a887b901".into()), + size_bytes: Some(31_056_425), + provenance: "Trained on RawNIND, real photographs of the same scenes taken noisy and clean (images CC BY 4.0 and CC0, \ + published on Wikimedia Commons and by the University of Louvain). Paper: arXiv 2501.08924." + .into(), + domain: Domain::BayerToRgb, + tile: 512, + overlap: 64, + gain: Gain::MatchMean { nominal: 1.0e6, max_deviation: 0.05 }, + }; + let download = Download { + id: manifest.id.clone(), + url: "https://github.com/darktable-org/darktable-ai/releases/download/release-5.6.0/rawdenoise-nind.dtmodel".into(), + file_name: "rawdenoise-nind.dtmodel".into(), + size_bytes: 57_700_134, + sha256: "d71b5f1e727c85a359e6f74dca9e2016c9d8fc3e2f7ac3e9b347d80ceca969af".into(), + }; + Known { manifest, download: Some(download), entry: Some("model_bayer.onnx") } +} + +/// Every model LightCraft knows. +pub fn all() -> Vec { + #[cfg(feature = "rawnind-model")] + { + vec![rawnind_bayer()] + } + #[cfg(not(feature = "rawnind-model"))] + { + Vec::new() + } +} + +/// The known model with this id. +pub fn find(id: &str) -> Option { + all().into_iter().find(|k| k.manifest.id == id) +} + +/// The known model whose `.onnx` has this SHA-256. +pub fn by_sha256(sha256: &str) -> Option { + all().into_iter().map(|k| k.manifest).find(|m| m.sha256.as_deref() == Some(sha256)) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::manifest::validate; + + #[test] + fn known_models_are_valid_unique_and_downloaded_over_https() { + let all = all(); + assert_eq!(all.is_empty(), !cfg!(feature = "rawnind-model")); + validate(&rawnind_bayer().manifest).unwrap(); + for (i, k) in all.iter().enumerate() { + validate(&k.manifest).unwrap(); + assert!(all.iter().skip(i + 1).all(|o| o.manifest.id != k.manifest.id), "unique ids"); + assert!(!k.manifest.licence.name.is_empty() && !k.manifest.licence.notice.is_empty(), "terms are shown before use"); + if let Some(d) = &k.download { + assert_eq!(d.id, k.manifest.id); + assert!(d.url.starts_with("https://") && d.sha256.len() == 64 && d.size_bytes > 0); + // a name that cannot leave the staging folder + assert!(!d.file_name.contains(['/', '\\']) && !d.file_name.starts_with('.')); + } + // a model that may not be used commercially is never offered for download + if k.manifest.licence.commercial == Commercial::No { + assert!(k.download.is_none(), "{}", k.manifest.id); + } + } + assert_eq!(find("rawnind-bayer").is_some(), cfg!(feature = "rawnind-model")); + assert!(find("nothing").is_none()); + assert_eq!( + by_sha256("da27509dab6a2915da67e988acd86cf71f9d5bbc8d1aa0ed32933578a887b901").map(|m| m.id), + if cfg!(feature = "rawnind-model") { Some("rawnind-bayer".into()) } else { None } + ); + } +} + +#[derive(Clone, Debug, PartialEq)] +pub struct Download { + pub id: String, + pub url: String, + /// The name the file is kept under while it waits for the user to accept its terms. + pub file_name: String, + pub size_bytes: u64, + pub sha256: String, +} + +pub fn host(url: &str) -> &str { + url.strip_prefix("https://").and_then(|r| r.split('/').next()).unwrap_or("") +} diff --git a/crates/denoise-core/src/lib.rs b/crates/denoise-core/src/lib.rs new file mode 100644 index 00000000..568ed9f8 --- /dev/null +++ b/crates/denoise-core/src/lib.rs @@ -0,0 +1,16 @@ +//! Model metadata, bounded archive/cache formats and Bayer tiling, independent of inference. +//! Used by featureless engine and web builds; no neural-network or matrix backend is linked. +#![forbid(unsafe_code)] +#![deny(clippy::unwrap_used, clippy::expect_used, clippy::panic, clippy::unimplemented, clippy::todo, clippy::unreachable)] +pub mod archive; +pub mod bayer; +pub mod hash; +pub mod known; +pub mod licence; +pub mod manifest; +pub mod product; +pub mod run; +pub mod tiles; + +pub use manifest::{DenoiserManifest, Domain, Gain, ManifestError}; +pub use run::{Control, Error, Params, TileRunner, denoise_bayer}; diff --git a/crates/denoise-core/src/licence.rs b/crates/denoise-core/src/licence.rs new file mode 100644 index 00000000..1d89146b --- /dev/null +++ b/crates/denoise-core/src/licence.rs @@ -0,0 +1,34 @@ +use serde::{Deserialize, Serialize}; + +#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub enum Commercial { + Yes, + /// Research or personal use only. Never bundled or redistributed by LightCraft. + No, + /// The licence or the training data is unclear: the user is told so before installing. + Unknown, +} + +#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] +#[serde(default, rename_all = "camelCase")] +pub struct Licence { + /// SPDX id or a short name ("MIT", "Apache-2.0", "InsightFace non-commercial research"). + pub name: String, + pub commercial: Commercial, + pub url: Option, + /// Plain-language terms shown before the model is enabled. + pub notice: String, +} + +impl Default for Licence { + fn default() -> Self { + Self { name: String::new(), commercial: Commercial::Unknown, url: None, notice: String::new() } + } +} + +pub fn valid_id(id: &str) -> bool { + let mut bytes = id.bytes(); + let first_ok = bytes.next().is_some_and(|b| b.is_ascii_lowercase() || b.is_ascii_digit()); + first_ok && id.len() <= 64 && bytes.all(|b| b.is_ascii_lowercase() || b.is_ascii_digit() || matches!(b, b'.' | b'_' | b'-')) +} diff --git a/crates/denoise-core/src/manifest.rs b/crates/denoise-core/src/manifest.rs new file mode 100644 index 00000000..9fbc9f49 --- /dev/null +++ b/crates/denoise-core/src/manifest.rs @@ -0,0 +1,217 @@ +//! A denoise model's manifest: what the model takes and gives, how big its tiles are and who may use it, as +//! plain data. +//! +//! It comes from a `denoise-model.json` next to the `.onnx` file or from the user's `catalog.json`. Both are +//! untrusted: [`validate`] must pass before a manifest is used, and [`parse`] applies it after reading JSON of +//! bounded size. The licence, id and address rules are the face models' (`crate::licence`). + +use crate::licence::{Licence, valid_id}; +use serde::{Deserialize, Serialize}; + +/// Largest `denoise-model.json` we read. +pub const MAX_MANIFEST_BYTES: usize = 64 * 1024; +/// Largest model file we accept (bytes). +pub const MAX_MODEL_BYTES: u64 = 2 * 1024 * 1024 * 1024; + +/// What the model takes and what it gives. +#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub enum Domain { + /// The sensor's Bayer mosaic (black subtracted, white at 1, not white balanced) packed as four planes + /// `[R, G1, G2, B]` of its 2 × 2 cells, forced to RGGB; gives camera RGB at the mosaic's resolution, so + /// the picture is denoised and demosaiced in one step and must not be demosaiced again. + BayerToRgb, +} + +/// How the model's output scale is brought back to the input's. +#[derive(Clone, Copy, Debug, PartialEq, Serialize, Deserialize)] +#[serde(tag = "kind", rename_all = "camelCase")] +pub enum Gain { + /// The output already has the input's scale. + None, + /// The output has a scale of its own, `nominal` times the input's (a million, for one model) and a little + /// different from tile to tile: each tile is scaled so that its channel means equal the input's. A tile too + /// dark to measure, or whose measured scale is further than `maxDeviation` (a fraction, 0.05 = 5 %) from + /// `nominal`, is scaled as if it were within that limit. + MatchMean { nominal: f32, max_deviation: f32 }, +} + +#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct DenoiserManifest { + /// Lowercase letters, digits, `.` `_` `-`. + pub id: String, + pub name: String, + pub version: String, + #[serde(default)] + pub licence: Licence, + /// Where the weights come from (an `https://` page or file), for the user's information. + #[serde(default)] + pub source: Option, + /// Lowercase hex SHA-256 of the `.onnx` file, when known. + #[serde(default)] + pub sha256: Option, + #[serde(default)] + pub size_bytes: Option, + /// What the model was trained on, in plain words; says "undisclosed" when it is. + #[serde(default)] + pub provenance: String, + pub domain: Domain, + /// Side of the square the model runs on, in packed cells (the file's fixed input size). + pub tile: u32, + /// Cells neighbouring tiles share; the picture is blended across them. + pub overlap: u32, + pub gain: Gain, +} + +#[derive(Debug, PartialEq, thiserror::Error)] +pub enum ManifestError { + #[error("`{0}` is not valid: {1}")] + Field(&'static str, String), + #[error("not a denoise-model manifest: {0}")] + Parse(String), + #[error("a manifest may be at most {MAX_MANIFEST_BYTES} bytes")] + TooLarge, +} + +fn bad(field: &'static str, why: impl Into) -> Result { + Err(ManifestError::Field(field, why.into())) +} + +fn text(field: &'static str, s: &str, min: usize, max: usize) -> Result<(), ManifestError> { + let n = s.chars().count(); + if n < min || n > max { + return bad(field, format!("must be {min} to {max} characters")); + } + if s.chars().any(|c| c.is_control() && c != '\n') { + return bad(field, "must not contain control characters"); + } + Ok(()) +} + +fn url(field: &'static str, u: &Option) -> Result<(), ManifestError> { + match u { + None => Ok(()), + Some(u) if u.len() <= 500 && u.starts_with("https://") && !u.chars().any(|c| c.is_control() || c.is_whitespace()) => Ok(()), + Some(_) => bad(field, "must be an https:// address of at most 500 characters"), + } +} + +/// Check a manifest from outside before trusting it. +pub fn validate(m: &DenoiserManifest) -> Result<(), ManifestError> { + if !valid_id(&m.id) { + return bad("id", "1 to 64 lowercase letters, digits, `.`, `_` or `-`, starting with a letter or digit"); + } + text("name", &m.name, 1, 80)?; + text("version", &m.version, 1, 40)?; + text("provenance", &m.provenance, 0, 1000)?; + text("licence.name", &m.licence.name, 0, 80)?; + text("licence.notice", &m.licence.notice, 0, 2000)?; + url("licence.url", &m.licence.url)?; + url("source", &m.source)?; + if let Some(h) = &m.sha256 + && !(h.len() == 64 && h.bytes().all(|b| b.is_ascii_digit() || (b'a'..=b'f').contains(&b))) + { + return bad("sha256", "must be 64 lowercase hex digits"); + } + if let Some(n) = m.size_bytes + && (n == 0 || n > MAX_MODEL_BYTES) + { + return bad("sizeBytes", format!("must be between 1 and {MAX_MODEL_BYTES}")); + } + // a U-Net with four poolings wants a multiple of 16; nothing smaller than 64 or larger than 2048 is a tile + if !(64..=2048).contains(&m.tile) || !m.tile.is_multiple_of(16) { + return bad("tile", "must be a multiple of 16 from 64 to 2048"); + } + // tiles must overlap by less than half, or one tile's blend ramps would run into its other neighbour's + if !m.overlap.is_multiple_of(2) || m.overlap < 16 || m.overlap.saturating_mul(2) >= m.tile { + return bad("overlap", "must be even, at least 16 and less than half the tile"); + } + if let Gain::MatchMean { nominal, max_deviation } = m.gain { + if !(nominal.is_finite() && (1e-6..=1e12).contains(&nominal)) { + return bad("gain.nominal", "must be between 0.000001 and 1000000000000"); + } + if !(max_deviation.is_finite() && (0.0..=0.5).contains(&max_deviation)) { + return bad("gain.maxDeviation", "must be between 0 and 0.5"); + } + } + Ok(()) +} + +/// Read and validate a `denoise-model.json`. +pub fn parse(bytes: &[u8]) -> Result { + if bytes.len() > MAX_MANIFEST_BYTES { + return Err(ManifestError::TooLarge); + } + let bytes = bytes.strip_prefix(&[0xEF, 0xBB, 0xBF]).unwrap_or(bytes); + let m: DenoiserManifest = serde_json::from_slice(bytes).map_err(|e| ManifestError::Parse(e.to_string()))?; + validate(&m)?; + Ok(m) +} + +#[cfg(test)] +mod tests { + use super::*; + + pub(crate) fn sample() -> DenoiserManifest { + DenoiserManifest { + id: "nind-bayer".into(), + name: "Bayer denoiser".into(), + version: "1".into(), + licence: Licence::default(), + source: Some("https://example.org/model.onnx".into()), + sha256: None, + size_bytes: Some(31_000_000), + provenance: "trained on real raw noise pairs".into(), + domain: Domain::BayerToRgb, + tile: 512, + overlap: 64, + gain: Gain::MatchMean { nominal: 1.0e6, max_deviation: 0.05 }, + } + } + + #[test] + fn a_good_manifest_round_trips() { + let m = sample(); + let json = serde_json::to_vec(&m).unwrap(); + assert_eq!(parse(&json).unwrap(), m); + } + + #[test] + fn bad_fields_are_named() { + let cases: Vec<(&str, Box)> = vec![ + ("id", Box::new(|m| m.id = "../evil".into())), + ("id", Box::new(|m| m.id = String::new())), + ("name", Box::new(|m| m.name = "x\u{7}".into())), + ("source", Box::new(|m| m.source = Some("http://insecure".into()))), + ("sha256", Box::new(|m| m.sha256 = Some("XYZ".into()))), + ("sizeBytes", Box::new(|m| m.size_bytes = Some(0))), + ("tile", Box::new(|m| m.tile = 500)), + ("tile", Box::new(|m| m.tile = 4096)), + ("overlap", Box::new(|m| m.overlap = 300)), + ("overlap", Box::new(|m| m.overlap = 63)), + ("gain.maxDeviation", Box::new(|m| m.gain = Gain::MatchMean { nominal: 1.0, max_deviation: f32::NAN })), + ("gain.maxDeviation", Box::new(|m| m.gain = Gain::MatchMean { nominal: 1.0, max_deviation: 2.0 })), + ("gain.nominal", Box::new(|m| m.gain = Gain::MatchMean { nominal: 0.0, max_deviation: 0.1 })), + ("gain.nominal", Box::new(|m| m.gain = Gain::MatchMean { nominal: f32::INFINITY, max_deviation: 0.1 })), + ]; + for (field, edit) in cases { + let mut m = sample(); + edit(&mut m); + match validate(&m) { + Err(ManifestError::Field(f, _)) => assert_eq!(f, field), + other => panic!("{field}: {other:?}"), + } + } + } + + #[test] + fn parse_is_bounded_and_tolerant() { + assert_eq!(parse(&vec![b' '; MAX_MANIFEST_BYTES + 1]), Err(ManifestError::TooLarge)); + assert!(matches!(parse(b"{ nope"), Err(ManifestError::Parse(_)))); + assert!(matches!(parse(b""), Err(ManifestError::Parse(_)))); + let mut with_bom = vec![0xEF, 0xBB, 0xBF]; + with_bom.extend(serde_json::to_vec(&sample()).unwrap()); + assert!(parse(&with_bom).is_ok()); + } +} diff --git a/crates/denoise-core/src/product.rs b/crates/denoise-core/src/product.rs new file mode 100644 index 00000000..5a607c8f --- /dev/null +++ b/crates/denoise-core/src/product.rs @@ -0,0 +1,616 @@ +//! The denoised picture as cached data, and the blend the Amount slider controls. +//! +//! A product is camera RGB at the sensor's resolution (what the model gave, before the raw's own opcodes and +//! crop), kept as half floats in strips of rows, each strip byte-shuffled and deflated with its own checksum. +//! It is derived data: any problem reading it (missing, truncated, corrupt, made for another photo or model) is +//! just a cache miss, never a panic, and the picture can be made again. +//! +//! File: `LCDN`, version, header length, header JSON (`width`, `height`, `strip`, `key`), then for each strip its +//! compressed length and CRC-32, then the strips' bytes. `key` says what the product was made from (the photo's +//! content, the model, this code's version, the settings it depends on); a file with another key is stale. + +use std::io::{Read, Write}; +use std::path::Path; + +use lightcraft_raster::Rgb32f; +use rayon::prelude::*; +use serde::{Deserialize, Serialize}; + +const MAGIC: &[u8; 4] = b"LCDN"; +const VERSION: u8 = 1; +const MAX_HEADER: usize = 4096; +const MAX_SIDE: usize = 1 << 17; +/// Most pixels a product may have (the denoiser's own limit, [`crate::run::MAX_PIXELS`]). +const MAX_PIXELS: usize = crate::run::MAX_PIXELS; +/// Rows per strip: about a megapixel of three planes at 24 MP widths. +const STRIP_ROWS: usize = 128; +/// Bumped when the meaning of a product changes (included in keys by the caller). +pub const ALGORITHM: u32 = 1; + +#[derive(Debug, thiserror::Error)] +pub enum ProductError { + #[error("the denoised picture could not be read or written: {0}")] + Io(String), + #[error("not a usable denoised picture: {0}")] + Format(String), + /// A product made from something else (another photo state, model or version). + #[error("the denoised picture is out of date")] + Stale, +} + +impl From for ProductError { + fn from(e: std::io::Error) -> Self { + ProductError::Io(e.to_string()) + } +} + +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +pub struct Header { + pub width: usize, + pub height: usize, + pub strip: usize, + pub key: String, +} + +fn allocate(n: usize, value: T) -> Result, ProductError> { + let mut v = Vec::new(); + v.try_reserve_exact(n).map_err(|_| ProductError::Format("not enough memory for the denoised picture".into()))?; + v.resize(n, value); + Ok(v) +} + +/// `f32` as IEEE half-float bits, rounded to nearest even; beyond the range it saturates, NaN becomes 0. +pub fn to_f16(v: f32) -> u16 { + if !v.is_finite() { + return if v.is_nan() { + 0 + } else if v > 0.0 { + 0x7bff + } else { + 0xfbff + }; + } + let b = v.to_bits(); + let sign = ((b >> 16) & 0x8000) as u16; + let exp = ((b >> 23) & 0xff) as i32 - 127 + 15; + let man = b & 0x7f_ffff; + if exp >= 31 { + return sign | 0x7bff; + } + if exp <= 0 { + if exp < -10 { + return sign; + } + let man = man | 0x80_0000; + let shift = (14 - exp) as u32; + let (half, rem, halfway) = (man >> shift, man & ((1u32 << shift) - 1), 1u32 << (shift - 1)); + let mut h = half as u16; + if rem > halfway || (rem == halfway && h & 1 == 1) { + h += 1; + } + return sign | h; + } + let mut h = (((exp as u32) << 10) | (man >> 13)) as u16; + let rem = man & 0x1fff; + if rem > 0x1000 || (rem == 0x1000 && h & 1 == 1) { + h += 1; + } + if h >= 0x7c00 { + h = 0x7bff; + } + sign | h +} + +/// Half-float bits as `f32`. +pub fn from_f16(h: u16) -> f32 { + let sign = if h & 0x8000 != 0 { -1.0f32 } else { 1.0 }; + let exp = i32::from((h >> 10) & 0x1f); + let man = f32::from(h & 0x3ff); + match exp { + 0 => sign * man * 2f32.powi(-24), + 31 => sign * 65504.0, + _ => sign * (1.0 + man / 1024.0) * 2f32.powi(exp - 15), + } +} + +fn crc(bytes: &[u8]) -> u32 { + // CRC-32 (IEEE), bitwise: strips are small and this runs once per strip + let mut table = [0u32; 256]; + for (i, t) in table.iter_mut().enumerate() { + let mut c = i as u32; + for _ in 0..8 { + c = if c & 1 != 0 { 0xedb8_8320 ^ (c >> 1) } else { c >> 1 }; + } + *t = c; + } + !bytes.iter().fold(!0u32, |c, &b| table[((c ^ u32::from(b)) & 0xff) as usize] ^ (c >> 8)) +} + +/// One strip as bytes: for each plane (R, G, B) the low bytes of its half floats, then the high bytes. +fn encode_strip(img: &Rgb32f, y0: usize, y1: usize) -> Result, ProductError> { + let n = (y1 - y0) * img.width; + let mut raw = allocate(6 * n, 0u8)?; + let rows = img.data.get(y0 * img.width..y1 * img.width).unwrap_or(&[]); + for (i, px) in rows.iter().enumerate() { + for (c, &v) in px.iter().enumerate() { + let h = to_f16(v); + if let Some(lo) = raw.get_mut(2 * c * n + i) { + *lo = h as u8; + } + if let Some(hi) = raw.get_mut((2 * c + 1) * n + i) { + *hi = (h >> 8) as u8; + } + } + } + Ok(miniz_oxide::deflate::compress_to_vec(&raw, 3)) +} + +/// Write `img` to `path` (through a temporary file next to it, so a reader never sees half a product). +pub fn write(path: &Path, img: &Rgb32f, key: &str) -> Result<(), ProductError> { + let pixels = img.width.checked_mul(img.height).filter(|&n| n <= MAX_PIXELS); + if img.width == 0 || img.height == 0 || img.width > MAX_SIDE || img.height > MAX_SIDE || pixels != Some(img.data.len()) { + return Err(ProductError::Format("the picture has no usable size".into())); + } + let strips: Vec<(usize, usize)> = (0..img.height).step_by(STRIP_ROWS).map(|y| (y, (y + STRIP_ROWS).min(img.height))).collect(); + let packed: Vec> = strips.par_iter().map(|&(a, b)| encode_strip(img, a, b)).collect::>()?; + let header = serde_json::to_vec(&Header { width: img.width, height: img.height, strip: STRIP_ROWS, key: key.to_string() }) + .map_err(|e| ProductError::Format(e.to_string()))?; + let tmp = path.with_extension("tmp"); + let result = (|| -> Result<(), ProductError> { + if let Some(dir) = path.parent() { + std::fs::create_dir_all(dir)?; + } + let mut f = std::io::BufWriter::new(std::fs::File::create(&tmp)?); + f.write_all(MAGIC)?; + f.write_all(&[VERSION, 0, 0, 0])?; + f.write_all(&(header.len() as u32).to_le_bytes())?; + f.write_all(&header)?; + for p in &packed { + f.write_all(&(p.len() as u32).to_le_bytes())?; + f.write_all(&crc(p).to_le_bytes())?; + } + for p in &packed { + f.write_all(p)?; + } + f.flush()?; + drop(f); + std::fs::rename(&tmp, path)?; + Ok(()) + })(); + if result.is_err() { + let _ = std::fs::remove_file(&tmp); + } + result +} + +/// The parsed start of a product file: header, each strip's (compressed length, CRC-32), and the file length the +/// start says the whole file has. +struct Start { + header: Header, + table: Vec<(usize, u32)>, + expected_len: u64, +} + +fn read_start(r: &mut impl Read) -> Result { + let mut fixed = [0u8; 12]; + r.read_exact(&mut fixed)?; + if fixed.get(0..4) != Some(&MAGIC[..]) || fixed.get(4) != Some(&VERSION) { + return Err(ProductError::Format("not a denoised picture of this version".into())); + } + let len = fixed.get(8..12).and_then(|b| <[u8; 4]>::try_from(b).ok()).map(u32::from_le_bytes).unwrap_or(u32::MAX) as usize; + if len > MAX_HEADER { + return Err(ProductError::Format("the header is too large".into())); + } + let mut hb = allocate(len, 0u8)?; + r.read_exact(&mut hb)?; + let h: Header = serde_json::from_slice(&hb).map_err(|e| ProductError::Format(e.to_string()))?; + let pixels = h.width.saturating_mul(h.height); + if h.width == 0 || h.height == 0 || h.width > MAX_SIDE || h.height > MAX_SIDE || pixels > MAX_PIXELS || h.strip != STRIP_ROWS { + return Err(ProductError::Format("the size is not usable".into())); + } + let n = h.height.div_ceil(h.strip); + let mut table = Vec::with_capacity(n); + let mut total = 12 + len as u64 + 8 * n as u64; + for _ in 0..n { + let mut e = [0u8; 8]; + r.read_exact(&mut e)?; + let l = u32::from_le_bytes([e[0], e[1], e[2], e[3]]) as usize; + let c = u32::from_le_bytes([e[4], e[5], e[6], e[7]]); + // a strip never compresses to much more than its raw size + if l > 12 * h.strip * h.width + 4096 { + return Err(ProductError::Format("a strip is larger than it can be".into())); + } + total += l as u64; + table.push((l, c)); + } + Ok(Start { header: h, table, expected_len: total }) +} + +/// The header of the product at `path`, when it is one. +pub fn read_header_at(path: &Path) -> Result { + let mut f = std::io::BufReader::new(std::fs::File::open(path)?); + Ok(read_start(&mut f)?.header) +} + +/// Whether `path` holds a complete-looking product made from `key` (the header and the file's length are checked; +/// strips are checked when read). +pub fn is_current(path: &Path, key: &str) -> bool { + let check = || -> Result { + let mut f = std::io::BufReader::new(std::fs::File::open(path)?); + let s = read_start(&mut f)?; + Ok(s.header.key == key && std::fs::metadata(path)?.len() == s.expected_len) + }; + check().unwrap_or(false) +} + +/// A rectangle of a product, in the product's pixels. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct Window { + pub x: usize, + pub y: usize, + pub width: usize, + pub height: usize, +} + +/// Half floats as `f32`, for all 65536 bit patterns (a table is much faster than computing each). +fn f16_table() -> &'static [f32] { + static TABLE: std::sync::OnceLock> = std::sync::OnceLock::new(); + TABLE.get_or_init(|| (0..=u16::MAX).map(from_f16).collect()) +} + +/// Strips decoded together (they are inflated in parallel, then folded in order). +const GROUP: usize = 8; + +/// One strip, checked and inflated: for each plane the low bytes of its half floats, then the high bytes. +fn decode_strip(index: usize, packed: &[u8], expect_crc: u32, n: usize) -> Result, ProductError> { + if crc(packed) != expect_crc { + return Err(ProductError::Format(format!("strip {index} is damaged"))); + } + let raw = miniz_oxide::inflate::decompress_to_vec_with_limit(packed, 6 * n).map_err(|e| ProductError::Format(format!("strip {index}: {e:?}")))?; + if raw.len() != 6 * n { + return Err(ProductError::Format(format!("strip {index} has the wrong size"))); + } + Ok(raw) +} + +/// Read the product at `path`, made from `key`, with every `factor × factor` block of pixels averaged into one +/// (`factor` 1 = full size). The result is camera RGB. +/// +/// With a `window`, only that rectangle is read and the blocks start at its corner; only whole blocks are made, so +/// the result is `window.width / factor × window.height / factor` (the same blocks a binned development of the same +/// crop makes). Without one the whole product is read and blocks at the right and bottom edges may be smaller. +/// +/// With `clip` (white is 1), a block in which any sample of a colour is at or above it takes that colour's +/// maximum instead of the mean, so a clipped highlight stays clipped for highlight reconstruction. +pub fn read_window(path: &Path, key: &str, window: Option, factor: usize, clip: Option) -> Result { + let factor = factor.clamp(1, 1 << 12); + let mut f = std::io::BufReader::new(std::fs::File::open(path)?); + let Start { header: h, table, expected_len } = read_start(&mut f)?; + if h.key != key { + return Err(ProductError::Stale); + } + if std::fs::metadata(path)?.len() != expected_len { + return Err(ProductError::Format("the file is not the length its header says".into())); + } + let w = window.unwrap_or(Window { x: 0, y: 0, width: h.width, height: h.height }); + let fits = |a: usize, b: usize, limit: usize| a.checked_add(b).is_some_and(|e| e <= limit); + if w.width == 0 || w.height == 0 || !fits(w.x, w.width, h.width) || !fits(w.y, w.height, h.height) { + return Err(ProductError::Format("the window does not lie inside the picture".into())); + } + // whole blocks for a window; partial ones at the edges when the whole picture is read + let whole = window.is_none(); + let (ow, oh) = if whole { (w.width.div_ceil(factor), w.height.div_ceil(factor)) } else { (w.width / factor, w.height / factor) }; + if ow == 0 || oh == 0 { + return Err(ProductError::Format("the window is smaller than one block".into())); + } + // the picture rows and columns the blocks cover + let (x_end, y_end) = if whole { (w.x + w.width, w.y + w.height) } else { (w.x + ow * factor, w.y + oh * factor) }; + let mut out = Rgb32f { width: ow, height: oh, data: allocate(ow * oh, [0.0; 3])? }; + let lut = f16_table(); + let value = |raw: &[u8], c: usize, n: usize, i: usize| -> f32 { + let (lo, hi) = (raw.get(2 * c * n + i).copied().unwrap_or(0), raw.get((2 * c + 1) * n + i).copied().unwrap_or(0)); + lut.get(usize::from(lo) | usize::from(hi) << 8).copied().unwrap_or(0.0) + }; + // which strips are needed, and where each starts in the file (after the header and the strip table) + let wanted = |s: usize| s * h.strip < y_end && (s + 1) * h.strip > w.y; + let (mut sum, mut peak) = (allocate(ow, [0f32; 3])?, allocate(ow, [f32::MIN; 3])?); + let (mut rows_in, mut oy) = (0usize, 0usize); + let strips: Vec = (0..table.len()).filter(|&s| wanted(s)).collect(); + // skip the strips before the first one wanted + let skipped: usize = table.iter().take(strips.first().copied().unwrap_or(0)).map(|e| e.0).sum(); + f.seek_relative(i64::try_from(skipped).map_err(|_| ProductError::Format("the file is too large".into()))?)?; + for group in strips.chunks(GROUP) { + let mut packed = Vec::with_capacity(group.len()); + for &s in group { + let Some(&(len, _)) = table.get(s) else { return Err(ProductError::Format("a strip is missing".into())) }; + let mut p = allocate(len, 0u8)?; + f.read_exact(&mut p)?; + packed.push(p); + } + let decoded: Vec, ProductError>> = group + .par_iter() + .zip(packed.par_iter()) + .map(|(&s, p)| { + let rows = h.height.saturating_sub(s * h.strip).min(h.strip); + decode_strip(s, p, table.get(s).map_or(0, |e| e.1), rows * h.width) + }) + .collect(); + for (&s, raw) in group.iter().zip(decoded) { + let raw = raw?; + let y0 = s * h.strip; + let rows = h.height.saturating_sub(y0).min(h.strip); + let n = rows * h.width; + for r in 0..rows { + let y = y0 + r; + if y < w.y || y >= y_end { + continue; + } + let base = r * h.width; + for (ox, (sm, pk)) in sum.iter_mut().zip(peak.iter_mut()).enumerate() { + let x0 = w.x + ox * factor; + let x1 = (x0 + factor).min(x_end); + for x in x0..x1 { + for c in 0..3 { + let v = value(&raw, c, n, base + x); + if let (Some(s_), Some(p_)) = (sm.get_mut(c), pk.get_mut(c)) { + *s_ += v; + *p_ = p_.max(v); + } + } + } + } + rows_in += 1; + if rows_in == factor || y + 1 == y_end { + for (ox, (sm, pk)) in sum.iter_mut().zip(peak.iter_mut()).enumerate() { + // the block's size at the right and bottom edges may be smaller + let bw = (x_end - (w.x + ox * factor)).min(factor); + let k = 1.0 / (rows_in * bw).max(1) as f32; + if let Some(o) = out.data.get_mut(oy * ow + ox) { + for c in 0..3 { + let (s_, p_) = (sm.get(c).copied().unwrap_or(0.0), pk.get(c).copied().unwrap_or(0.0)); + if let Some(slot) = o.get_mut(c) { + *slot = if factor > 1 && clip.is_some_and(|t| p_ >= t) { p_ } else { s_ * k }; + } + } + } + *sm = [0.0; 3]; + *pk = [f32::MIN; 3]; + } + rows_in = 0; + oy += 1; + } + } + } + } + Ok(out) +} + +/// [`read_window`] of the whole product: every `factor × factor` block averaged into one. +pub fn read(path: &Path, key: &str, factor: usize) -> Result { + read_window(path, key, None, factor, None) +} + +/// `base + (denoised − base) · amount` for every pixel (`amount` 0 to 1), in place in `base`. The two pictures must +/// be the same size; if they are not, `base` is left as it was and `false` is returned. +pub fn blend(base: &mut Rgb32f, denoised: &Rgb32f, amount: f32) -> bool { + if base.width != denoised.width || base.height != denoised.height { + return false; + } + let a = if amount.is_finite() { amount.clamp(0.0, 1.0) } else { 0.0 }; + base.data.par_iter_mut().zip(denoised.data.par_iter()).for_each(|(b, d)| { + for c in 0..3 { + b[c] += (d[c] - b[c]) * a; + } + }); + true +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn oversized_product_header_is_rejected_before_allocating_pixels() { + let p = temp("oversized-header"); + let header = serde_json::to_vec(&Header { width: 10001, height: 10000, strip: STRIP_ROWS, key: "k".into() }).unwrap(); + let mut bytes = Vec::from(MAGIC.as_slice()); + bytes.extend_from_slice(&[VERSION, 0, 0, 0]); + bytes.extend_from_slice(&(header.len() as u32).to_le_bytes()); + bytes.extend_from_slice(&header); + std::fs::write(&p, bytes).unwrap(); + assert!(read_header_at(&p).is_err()); + let _ = std::fs::remove_file(p); + } + + fn temp(name: &str) -> std::path::PathBuf { + std::env::temp_dir().join(format!("lc-denoise-{name}-{}.lcdn", std::process::id())) + } + + fn picture(w: usize, h: usize) -> Rgb32f { + Rgb32f { + width: w, + height: h, + data: (0..w * h).map(|i| [(i % w) as f32 / w as f32, (i / w) as f32 / h as f32, 0.001 + 0.5 * ((i % 7) as f32 / 7.0)]).collect(), + } + } + + #[test] + fn half_floats_round_trip_closely() { + for v in [0.0f32, 1.0, -1.0, 0.5, 0.333_333, 1e-4, 6.1e-5, 3e-6, 123.456, 65000.0, -0.002] { + let back = from_f16(to_f16(v)); + assert!((back - v).abs() <= v.abs() * 0.001 + 1e-7, "{v} -> {back}"); + } + assert_eq!(from_f16(to_f16(1e10)), 65504.0); + assert_eq!(to_f16(f32::NAN), 0); + assert_eq!(from_f16(to_f16(f32::NEG_INFINITY)), -65504.0); + } + + #[test] + fn a_product_survives_a_round_trip_at_full_and_reduced_size() { + let (w, h) = (301, 517); + let img = picture(w, h); + let p = temp("round"); + write(&p, &img, "k1").unwrap(); + assert!(is_current(&p, "k1") && !is_current(&p, "k2")); + assert_eq!(read_header_at(&p).unwrap().width, w); + let full = read(&p, "k1", 1).unwrap(); + assert_eq!((full.width, full.height), (w, h)); + for (a, b) in full.data.iter().zip(&img.data) { + for c in 0..3 { + assert!((a[c] - b[c]).abs() < 0.002, "{a:?} vs {b:?}"); + } + } + // averaged into blocks of 4: the sizes round up, and the mean of the picture is kept + let small = read(&p, "k1", 4).unwrap(); + assert_eq!((small.width, small.height), (w.div_ceil(4), h.div_ceil(4))); + let mean = |i: &Rgb32f| i.data.iter().map(|p| f64::from(p[1])).sum::() / i.data.len() as f64; + assert!((mean(&small) - mean(&img)).abs() < 0.01); + // a factor that does not divide the strip height is the same average + let odd = read(&p, "k1", 5).unwrap(); + assert_eq!((odd.width, odd.height), (w.div_ceil(5), h.div_ceil(5))); + assert!((mean(&odd) - mean(&img)).abs() < 0.01); + assert!(matches!(read(&p, "other", 1), Err(ProductError::Stale))); + std::fs::remove_file(&p).unwrap(); + } + + /// What binning a window of `img` by `k` means, written the slow obvious way. + fn blocks(img: &Rgb32f, win: Window, k: usize, clip: Option) -> Rgb32f { + let (ow, oh) = (win.width / k, win.height / k); + let mut out = Rgb32f { width: ow, height: oh, data: vec![[0.0; 3]; ow * oh] }; + for by in 0..oh { + for bx in 0..ow { + for c in 0..3 { + let vals: Vec = (0..k * k).map(|i| img.data[(win.y + by * k + i / k) * img.width + win.x + bx * k + i % k][c]).collect(); + let max = vals.iter().copied().fold(f32::MIN, f32::max); + out.data[by * ow + bx][c] = + if k > 1 && clip.is_some_and(|t| max >= t) { max } else { vals.iter().sum::() / vals.len() as f32 }; + } + } + } + out + } + + #[test] + fn a_window_is_read_as_the_blocks_of_that_crop() { + let (w, h) = (301, 517); + let mut img = picture(w, h); + // a clipped sample in the middle of a block + img.data[(130 + 2) * w + 61 + 1] = [1.0, 0.4, 0.4]; + let p = temp("window"); + write(&p, &img, "k1").unwrap(); + let near = |a: &Rgb32f, b: &Rgb32f| { + assert_eq!((a.width, a.height), (b.width, b.height)); + for (x, y) in a.data.iter().zip(&b.data) { + for c in 0..3 { + assert!((x[c] - y[c]).abs() < 0.003, "{x:?} vs {y:?}"); + } + } + }; + // windows at odd places, across strip boundaries (128 rows), with factors that do and do not divide them + for (win, k) in [ + (Window { x: 3, y: 5, width: 200, height: 300 }, 1), + (Window { x: 3, y: 5, width: 200, height: 300 }, 4), + (Window { x: 61, y: 130, width: 121, height: 250 }, 5), + (Window { x: 0, y: 0, width: w, height: h }, 2), + (Window { x: 1, y: 380, width: 90, height: 137 }, 3), + ] { + let got = read_window(&p, "k1", Some(win), k, None).unwrap(); + near(&got, &blocks(&img, win, k, None)); + } + // with a clip level the block with the clipped sample keeps its highest value, the others stay means + let win = Window { x: 61, y: 130, width: 40, height: 20 }; + let clipped = read_window(&p, "k1", Some(win), 4, Some(0.99)).unwrap(); + near(&clipped, &blocks(&img, win, 4, Some(0.99))); + assert!(clipped.data[0][0] > 0.99, "the clipped block stays clipped: {:?}", clipped.data[0]); + let plain = read_window(&p, "k1", Some(win), 4, None).unwrap(); + assert!(plain.data[0][0] < 0.99, "an average falls below the clip level"); + // windows that do not fit, or are smaller than a block, are errors + for bad in [ + Window { x: 0, y: 0, width: 0, height: 5 }, + Window { x: 290, y: 0, width: 20, height: 5 }, + Window { x: 0, y: 510, width: 5, height: 10 }, + Window { x: usize::MAX, y: 0, width: 2, height: 2 }, + Window { x: 0, y: 0, width: 3, height: 3 }, + ] { + assert!(read_window(&p, "k1", Some(bad), 4, None).is_err(), "{bad:?}"); + } + std::fs::remove_file(&p).unwrap(); + } + + #[test] + fn a_smooth_picture_compresses_well() { + let img = Rgb32f { + width: 512, + height: 512, + data: (0..512 * 512) + .map(|i| { + let v = 0.2 + 0.0001 * (i % 512) as f32; + [v, v * 0.8, v * 0.6] + }) + .collect(), + }; + let p = temp("size"); + write(&p, &img, "k").unwrap(); + let bytes = std::fs::metadata(&p).unwrap().len() as usize; + assert!(bytes < 512 * 512 * 6 / 2, "{bytes} bytes for a smooth picture"); + std::fs::remove_file(&p).unwrap(); + } + + #[test] + fn damaged_files_are_errors_not_panics() { + let img = picture(100, 300); + let p = temp("damage"); + write(&p, &img, "k").unwrap(); + let good = std::fs::read(&p).unwrap(); + // truncated at every length + for cut in (0..good.len()).step_by(37) { + std::fs::write(&p, &good[..cut]).unwrap(); + assert!(read(&p, "k", 1).is_err(), "cut at {cut}"); + assert!(!is_current(&p, "k")); + } + // a flipped byte anywhere is caught by a checksum, the header parse, or the key + for at in (0..good.len()).step_by(53) { + let mut bad = good.clone(); + bad[at] ^= 0xff; + std::fs::write(&p, &bad).unwrap(); + let _ = read(&p, "k", 1); // must not panic; most are errors + } + // a header that claims an absurd size + let mut huge = b"LCDN\x01\0\0\0".to_vec(); + let hdr = br#"{"width":4000000000,"height":4000000000,"strip":128,"key":"k"}"#; + huge.extend((hdr.len() as u32).to_le_bytes()); + huge.extend(hdr); + std::fs::write(&p, &huge).unwrap(); + assert!(matches!(read(&p, "k", 1), Err(ProductError::Format(_)))); + std::fs::write(&p, b"").unwrap(); + assert!(read(&p, "k", 1).is_err()); + assert!(read(&temp("missing"), "k", 1).is_err()); + std::fs::remove_file(&p).unwrap(); + } + + #[test] + fn writing_rejects_unusable_pictures() { + let p = temp("bad"); + assert!(write(&p, &Rgb32f { width: 0, height: 0, data: vec![] }, "k").is_err()); + assert!(write(&p, &Rgb32f { width: 3, height: 3, data: vec![[0.0; 3]; 8] }, "k").is_err()); + assert!(!p.exists()); + } + + #[test] + fn blend_mixes_by_amount_and_rejects_other_sizes() { + let mut base = Rgb32f { width: 2, height: 1, data: vec![[0.0; 3], [1.0; 3]] }; + let dn = Rgb32f { width: 2, height: 1, data: vec![[1.0; 3], [0.0; 3]] }; + assert!(blend(&mut base, &dn, 0.25)); + assert_eq!(base.data, vec![[0.25; 3], [0.75; 3]]); + assert!(blend(&mut base, &dn, f32::NAN)); + assert_eq!(base.data, vec![[0.25; 3], [0.75; 3]]); + assert!(!blend(&mut base, &Rgb32f { width: 1, height: 1, data: vec![[0.0; 3]] }, 1.0)); + assert!(blend(&mut base, &dn, 1.0)); + assert_eq!(base.data, dn.data); + } + + #[test] + fn crc_matches_the_standard_check_value() { + assert_eq!(crc(b"123456789"), 0xcbf4_3926); + } +} diff --git a/crates/denoise-core/src/run.rs b/crates/denoise-core/src/run.rs new file mode 100644 index 00000000..1df96a9e --- /dev/null +++ b/crates/denoise-core/src/run.rs @@ -0,0 +1,589 @@ +//! Denoising a whole Bayer mosaic with a tile model: cut it into overlapping tiles, run the model on each, bring +//! each tile's scale back to the input's, keep clipped highlights as they were, and blend the tiles together. +//! +//! The model itself is a [`TileRunner`], so everything around it (packing, tiling, scale matching, highlight +//! protection, blending, cancelling, progress) is tested without one. + +use std::sync::atomic::{AtomicBool, Ordering}; + +use lightcraft_raster::Rgb32f; +use rayon::prelude::*; + +use crate::bayer::{Layout, pack_tile}; +use crate::manifest::Gain; +use crate::tiles::{starts, weight}; + +/// Most pixels one picture may have (guards the allocations that follow from its size). +pub const MAX_PIXELS: usize = 100_000_000; +/// A tile whose mean signal is below this (white = 1) is too dark to measure its scale from. +const MIN_SIGNAL: f32 = 0.003; + +#[derive(Debug, thiserror::Error)] +pub enum Error { + #[error("{0}")] + Input(String), + #[error("the model gave {0}")] + Model(String), + #[error("the model could not run: {0}")] + Runtime(String), + #[error("the model could not be loaded: {0}")] + Load(String), + #[error("cancelled")] + Cancelled, +} + +fn allocate(n: usize, value: T) -> Result, Error> { + let mut v = Vec::new(); + v.try_reserve_exact(n).map_err(|_| Error::Input("not enough memory for denoise".into()))?; + v.resize(n, value); + Ok(v) +} + +/// Something that runs the model on one tile. +pub trait TileRunner: Sync { + /// `input` is the tile as four planes `[R, G1, G2, B]` of `tile × tile` cells (NCHW, batch 1); the answer is + /// camera RGB, three planes of `2·tile × 2·tile` pixels (CHW), in the model's own scale. + fn run(&self, input: &[f32]) -> Result, Error>; +} + +pub struct Params { + /// Side of the model's square, in cells. + pub tile: usize, + /// Cells neighbouring tiles share. + pub overlap: usize, + pub gain: Gain, + /// White is 1; samples at or above this (minus a little) are clipped and keep their value. + pub clip: Option, + /// Tiles run at once (each model call may itself be single-threaded). + pub parallel: usize, +} + +#[derive(Default)] +pub struct Control<'a> { + pub cancel: Option<&'a AtomicBool>, + /// `(tiles done, tiles in all)`. + pub progress: Option<&'a (dyn Fn(usize, usize) + Sync)>, +} + +/// The tile grid of a picture: `(starts across, starts down)`. +pub fn grid(width: usize, height: usize, layout: Layout, tile: usize, overlap: usize) -> (Vec, Vec) { + let (nx, ny) = layout.cells(width, height); + (starts(nx, tile, overlap), starts(ny, tile, overlap)) +} + +/// How many model calls [`denoise_bayer`] makes for a picture. +pub fn tile_count(width: usize, height: usize, layout: Layout, tile: usize, overlap: usize) -> usize { + let (sx, sy) = grid(width, height, layout, tile, overlap); + sx.len().saturating_mul(sy.len()) +} + +/// Denoise (and demosaic) the normalised mosaic `mosaic` of `width × height` samples, laid out as `layout`. +/// Gives camera RGB at the mosaic's resolution. +pub fn denoise_bayer( + mosaic: &[f32], + width: usize, + height: usize, + layout: Layout, + runner: &dyn TileRunner, + p: &Params, + ctl: &Control<'_>, +) -> Result { + let pixels = width + .checked_mul(height) + .filter(|&n| n > 0 && n <= MAX_PIXELS) + .ok_or_else(|| Error::Input("the picture has no size, or is too large".into()))?; + if mosaic.len() != pixels { + return Err(Error::Input("the mosaic does not have width × height samples".into())); + } + if !(16..=2048).contains(&p.tile) || p.overlap >= p.tile.div_ceil(2) || layout.dx > 1 || layout.dy > 1 { + return Err(Error::Input("the tile size and overlap do not fit".into())); + } + if let Gain::MatchMean { nominal, max_deviation } = p.gain + && !(nominal.is_finite() && (1e-6..=1e12).contains(&nominal) && max_deviation.is_finite() && (0.0..=0.5).contains(&max_deviation)) + { + return Err(Error::Input("the model gain is not finite or within its supported range".into())); + } + if p.clip.is_some_and(|v| !v.is_finite() || v <= 0.0) { + return Err(Error::Input("the highlight clip must be finite and positive".into())); + } + let (nx, ny) = layout.cells(width, height); + let (sx, sy) = (starts(nx, p.tile, p.overlap), starts(ny, p.tile, p.overlap)); + let (cols, rows) = (sx.len(), sy.len()); + if cols == 0 || rows == 0 { + return Err(Error::Input("the picture needs too many tiles".into())); + } + let tiles: Vec<(usize, usize, bool, bool, bool, bool)> = sy + .iter() + .enumerate() + .flat_map(|(j, &y0)| sx.iter().enumerate().map(move |(i, &x0)| (x0, y0, i > 0, i + 1 < cols, j > 0, j + 1 < rows))) + .collect(); + let total = tiles.len(); + + let mut acc = allocate(pixels, [0f32; 3])?; + let mut wsum = allocate(nx * ny, 0f32)?; + let group = p.parallel.clamp(1, 4); + let mut done = 0; + // where the time goes, printed under `LIGHTCRAFT_PROFILE` + let (mut t_wait, mut t_total) = (std::time::Duration::ZERO, std::time::Duration::ZERO); + let began = web_time::Instant::now(); + + // One group of tiles at a time: pack, run the model, check and bring each to the input's scale, in parallel. + // Blending (sequential: the tiles add into one picture, in tile order so the result does not depend on timing) of + // the previous group runs while the next group is on the model, so neither the card nor the cores wait for it. + type Tile = (usize, usize, bool, bool, bool, bool); + let compute = |chunk: &[Tile]| -> Vec, Error>> { + chunk + .par_iter() + .map(|&(x0, y0, bx, ax, by, ay)| { + let mut input = allocate(4 * p.tile * p.tile, 0f32)?; + if !pack_tile(mosaic, width, height, layout, x0 as isize, y0 as isize, p.tile, &mut input) { + return Err(Error::Input("a tile could not be packed".into())); + } + let mut out = runner.run(&input)?; + let side = 2 * p.tile; + if out.len() != 3 * side * side { + return Err(Error::Model(format!("{} numbers for a tile, not the {} expected", out.len(), 3 * side * side))); + } + if out.iter().any(|v| !v.is_finite()) { + return Err(Error::Model("numbers that are not finite".into())); + } + finish_tile(&input, &mut out, p, (bx, ax, by, ay)); + Ok(out) + }) + .collect() + }; + let mut blend = |chunk: &[Tile], outs: Vec, Error>>| -> Result<(), Error> { + for (&(x0, y0, bx, ax, by, ay), r) in chunk.iter().zip(outs) { + blend_tile(&r?, &mut acc, &mut wsum, (width, height), (nx, ny), layout, (x0, y0), p, (bx, ax, by, ay)); + done += 1; + if let Some(f) = ctl.progress { + f(done, total); + } + } + Ok(()) + }; + let mut previous: Option<(&[Tile], Vec, Error>>)> = None; + for chunk in tiles.chunks(group) { + if ctl.cancel.is_some_and(|c| c.load(Ordering::Relaxed)) { + return Err(Error::Cancelled); + } + let (outs, blended) = rayon::join( + || compute(chunk), + || match previous.take() { + Some((c, o)) => { + let started = web_time::Instant::now(); + let r = blend(c, o); + (r, started.elapsed()) + } + None => (Ok(()), std::time::Duration::ZERO), + }, + ); + blended.0?; + t_wait += blended.1; + previous = Some((chunk, outs)); + } + if let Some((c, o)) = previous.take() { + let started = web_time::Instant::now(); + blend(c, o)?; + t_wait += started.elapsed(); + } + t_total += began.elapsed(); + let started = web_time::Instant::now(); + // normalise by the weights + let (ox, oy) = (layout.position(0, 0).0, layout.position(0, 0).1); + acc.par_chunks_mut(width).enumerate().for_each(|(y, row)| { + let cy = ((y as isize - oy) / 2) as usize; + for (x, px) in row.iter_mut().enumerate() { + let cx = ((x as isize - ox) / 2) as usize; + let w = wsum.get(cy * nx + cx).copied().unwrap_or(0.0); + if w > 0.0 { + *px = px.map(|v| v / w); + } + } + }); + if std::env::var_os("LIGHTCRAFT_PROFILE").is_some() { + eprintln!( + "[denoise] {total} tiles, {} at once: tiles {:.0} ms (blending, {:.0} ms of it, overlaps the model), normalise {:.0} ms", + group, + t_total.as_secs_f64() * 1e3, + t_wait.as_secs_f64() * 1e3, + started.elapsed().as_secs_f64() * 1e3 + ); + } + Ok(Rgb32f { width, height, data: acc }) +} + +/// Bring a tile's output to the input's scale and keep clipped highlights as they were (in place). +fn finish_tile(input: &[f32], out: &mut [f32], p: &Params, edges: (bool, bool, bool, bool)) { + let t = p.tile; + let side = 2 * t; + if let Gain::MatchMean { nominal, max_deviation } = p.gain { + // the part of the tile whose scale is measured: away from the edges that blend into a neighbour + let m = p.overlap / 2; + let (i0, i1) = (if edges.0 { m } else { 0 }, t - if edges.1 { m } else { 0 }); + let (j0, j1) = (if edges.2 { m } else { 0 }, t - if edges.3 { m } else { 0 }); + let cells = ((i1 - i0) * (j1 - j0)) as f64; + let plane_sum = |pl: usize| -> f64 { + (j0..j1) + .map(|j| input.get(pl * t * t + j * t + i0..pl * t * t + j * t + i1).map_or(0.0, |r| r.iter().map(|&v| f64::from(v)).sum::())) + .sum() + }; + let in_mean = [plane_sum(0) / cells, (plane_sum(1) + plane_sum(2)) / (2.0 * cells), plane_sum(3) / cells]; + for (c, &mean_in) in in_mean.iter().enumerate() { + let sum_out: f64 = (2 * j0..2 * j1) + .map(|y| { + out.get(c * side * side + y * side + 2 * i0..c * side * side + y * side + 2 * i1) + .map_or(0.0, |r| r.iter().map(|&v| f64::from(v)).sum::()) + }) + .sum(); + let mean_out = sum_out / (4.0 * cells); + let measured = if mean_in as f32 >= MIN_SIGNAL && mean_out > 0.0 { (mean_out / mean_in) as f32 } else { nominal }; + let g = measured.clamp(nominal * (1.0 - max_deviation), nominal * (1.0 + max_deviation)); + if g.is_finite() && g > 0.0 { + let inv = 1.0 / g; + if let Some(plane) = out.get_mut(c * side * side..(c + 1) * side * side) { + plane.iter_mut().for_each(|v| *v *= inv); + } + } + } + } + if let Some(clip) = p.clip { + protect_clipped(input, out, t, clip); + } +} + +/// Where the mosaic is at (or within a cell of) white, the output becomes the plain cell values again, so a clipped +/// area stays exactly as clipped as the camera recorded it (highlight recovery later depends on that). +fn protect_clipped(input: &[f32], out: &mut [f32], t: usize, clip: f32) { + let side = 2 * t; + let plane = t * t; + let (Some(r), Some(g1), Some(g2), Some(b)) = + (input.get(0..plane), input.get(plane..2 * plane), input.get(2 * plane..3 * plane), input.get(3 * plane..4 * plane)) + else { + return; + }; + let mut m: Vec = (0..plane).map(|i| r[i].max(g1[i]).max(g2[i]).max(b[i])).collect(); + if !m.iter().any(|&v| v >= clip) { + return; + } + // 3×3 maximum over cells (separable) + let dilate = |src: &[f32], horizontal: bool| -> Vec { + (0..plane) + .map(|i| { + let (x, y) = (i % t, i / t); + let mut best = src[i]; + for d in [-1isize, 1] { + let (nx, ny) = if horizontal { (x as isize + d, y as isize) } else { (x as isize, y as isize + d) }; + if nx >= 0 && ny >= 0 && (nx as usize) < t && (ny as usize) < t { + best = best.max(src[ny as usize * t + nx as usize]); + } + } + best + }) + .collect() + }; + m = dilate(&dilate(&m, true), false); + let lo = clip - 0.03; + for j in 0..t { + for i in 0..t { + let k = j * t + i; + let s = ((m[k] - lo) / (clip - lo)).clamp(0.0, 1.0); + if s <= 0.0 { + continue; + } + let s = s * s * (3.0 - 2.0 * s); + let plain = [r[k], 0.5 * (g1[k] + g2[k]), b[k]]; + for (c, &pv) in plain.iter().enumerate() { + for (sy, sx) in [(0, 0), (0, 1), (1, 0), (1, 1)] { + if let Some(v) = out.get_mut(c * side * side + (2 * j + sy) * side + 2 * i + sx) { + *v += (pv - *v) * s; + } + } + } + } + } +} + +/// Add a finished tile to the picture with its blend weights. +#[allow(clippy::too_many_arguments)] +fn blend_tile( + out: &[f32], + acc: &mut [[f32; 3]], + wsum: &mut [f32], + size: (usize, usize), + cells: (usize, usize), + layout: Layout, + at: (usize, usize), + p: &Params, + edges: (bool, bool, bool, bool), +) { + let (width, height) = size; + let (nx, ny) = cells; + let (t, side) = (p.tile, 2 * p.tile); + let fade = p.overlap; + let wx: Vec = (0..t).map(|i| weight(i, t, fade, edges.0, edges.1)).collect(); + let wy: Vec = (0..t).map(|j| weight(j, t, fade, edges.2, edges.3)).collect(); + for j in 0..t { + let cy = at.1 + j; + for i in 0..t { + let cx = at.0 + i; + // cells past the picture's edge (a picture smaller than one tile is padded) belong to no pixel + if cx >= nx || cy >= ny { + continue; + } + let (Some(&a), Some(&b)) = (wx.get(i), wy.get(j)) else { continue }; + let w = a * b; + if let Some(s) = wsum.get_mut(cy * nx + cx) { + *s += w; + } + let (px0, py0) = layout.position(cx as isize, cy as isize); + for (sy, sx) in [(0isize, 0isize), (0, 1), (1, 0), (1, 1)] { + let (x, y) = (px0 + sx, py0 + sy); + if x < 0 || y < 0 || x as usize >= width || y as usize >= height { + continue; + } + let o = (2 * j + sy as usize) * side + 2 * i + sx as usize; + let (Some(&r), Some(&g), Some(&bl)) = (out.get(o), out.get(side * side + o), out.get(2 * side * side + o)) else { continue }; + if let Some(px) = acc.get_mut(y as usize * width + x as usize) { + px[0] += w * r; + px[1] += w * g; + px[2] += w * bl; + } + } + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + /// A stand-in model: the cell's samples as an RGB picture at twice the resolution (a plain 2 × 2 "demosaic"), + /// times `scale`, so the output has a scale of its own like the real model's. + struct Mock { + tile: usize, + scale: f32, + } + impl TileRunner for Mock { + fn run(&self, input: &[f32]) -> Result, Error> { + let (t, side) = (self.tile, 2 * self.tile); + let plane = t * t; + let mut out = vec![0.0; 3 * side * side]; + for j in 0..t { + for i in 0..t { + let k = j * t + i; + let rgb = [input[k], 0.5 * (input[plane + k] + input[2 * plane + k]), input[3 * plane + k]]; + for (c, v) in rgb.iter().enumerate() { + for (sy, sx) in [(0, 0), (0, 1), (1, 0), (1, 1)] { + out[c * side * side + (2 * j + sy) * side + 2 * i + sx] = v * self.scale; + } + } + } + } + Ok(out) + } + } + + fn params(tile: usize, overlap: usize, gain: Gain) -> Params { + Params { tile, overlap, gain, clip: None, parallel: 2 } + } + + #[test] + fn overflowing_tiles_and_nan_gain_are_errors_before_the_model_runs() { + let layout = Layout::from_cell([0, 1, 1, 2]).unwrap(); + let model = Mock { tile: 16, scale: 1.0 }; + for (tile, overlap, gain) in [ + (usize::MAX, 0, Gain::None), + (16, usize::MAX, Gain::None), + (16, 0, Gain::MatchMean { nominal: 1.0, max_deviation: f32::NAN }), + (16, 0, Gain::MatchMean { nominal: f32::INFINITY, max_deviation: 0.1 }), + ] { + assert!(matches!(denoise_bayer(&[0.5], 1, 1, layout, &model, ¶ms(tile, overlap, gain), &Control::default()), Err(Error::Input(_)))); + } + } + + /// A smooth picture whose colour at a sample is `base[colour] · (1 + shading)`. + fn mosaic(w: usize, h: usize, cell: [u8; 4]) -> Vec { + (0..w * h) + .map(|i| { + let (x, y) = (i % w, i / w); + let c = cell[(y % 2) * 2 + x % 2] as usize; + [0.5, 0.3, 0.2][c] * (0.6 + 0.4 * ((x as f32 / 23.0).sin() * (y as f32 / 31.0).cos())) + }) + .collect() + } + + fn reference(m: &[f32], w: usize, h: usize, layout: Layout) -> Rgb32f { + // what the mock gives for the whole picture in one tile + let (nx, ny) = layout.cells(w, h); + let t = nx.max(ny).div_ceil(16) * 16; + denoise_bayer(m, w, h, layout, &Mock { tile: t, scale: 1.0 }, ¶ms(t, 16, Gain::None), &Control::default()).unwrap() + } + + #[test] + fn tiles_blend_to_the_same_picture_as_one_tile_for_every_layout() { + for cell in [[0u8, 1, 1, 2], [1, 0, 2, 1], [1, 2, 0, 1], [2, 1, 1, 0]] { + let layout = Layout::from_cell(cell).unwrap(); + let (w, h) = (150usize, 110usize); + let m = mosaic(w, h, cell); + let want = reference(&m, w, h, layout); + let got = denoise_bayer(&m, w, h, layout, &Mock { tile: 32, scale: 1.0 }, ¶ms(32, 8, Gain::None), &Control::default()).unwrap(); + assert_eq!((got.width, got.height), (w, h)); + for (a, b) in got.data.iter().zip(&want.data) { + for c in 0..3 { + assert!((a[c] - b[c]).abs() < 1e-5, "layout {cell:?}: {a:?} vs {b:?}"); + } + } + } + } + + #[test] + fn the_picture_does_not_depend_on_how_many_tiles_run_at_once_or_how_long_each_takes() { + // tiles that finish in a different order every time: blending still adds them in tile order + struct Jittery(Mock); + impl TileRunner for Jittery { + fn run(&self, input: &[f32]) -> Result, Error> { + let wait = (input.iter().take(64).sum::() * 1000.0) as u64 % 7; + std::thread::sleep(std::time::Duration::from_millis(wait)); + self.0.run(input) + } + } + let layout = Layout { dx: 0, dy: 1 }; + let (w, h) = (150usize, 110usize); + let m = mosaic(w, h, [1, 0, 2, 1]); + let run = |parallel: usize| { + let mut p = params(32, 8, Gain::MatchMean { nominal: 1.0e6, max_deviation: 0.05 }); + p.parallel = parallel; + denoise_bayer(&m, w, h, layout, &Jittery(Mock { tile: 32, scale: 1.01e6 }), &p, &Control::default()).unwrap() + }; + let one = run(1); + for parallel in [2, 3, 8] { + assert_eq!(run(parallel).data, one.data, "{parallel} at once"); + } + } + + #[test] + fn a_failing_tile_or_a_cancel_ends_the_picture_without_a_panic() { + struct Fails(Mock, std::sync::atomic::AtomicUsize); + impl TileRunner for Fails { + fn run(&self, input: &[f32]) -> Result, Error> { + if self.1.fetch_add(1, Ordering::Relaxed) == 4 { + return Err(Error::Runtime("boom".into())); + } + self.0.run(input) + } + } + let layout = Layout { dx: 0, dy: 0 }; + let (w, h) = (150usize, 110usize); + let m = mosaic(w, h, [0, 1, 1, 2]); + let r = denoise_bayer( + &m, + w, + h, + layout, + &Fails(Mock { tile: 32, scale: 1.0 }, Default::default()), + ¶ms(32, 8, Gain::None), + &Control::default(), + ); + assert!(matches!(r, Err(Error::Runtime(_))), "{r:?}"); + let stop = AtomicBool::new(true); + let ctl = Control { cancel: Some(&stop), progress: None }; + let r = denoise_bayer(&m, w, h, layout, &Mock { tile: 32, scale: 1.0 }, ¶ms(32, 8, Gain::None), &ctl); + assert!(matches!(r, Err(Error::Cancelled))); + } + + #[test] + fn a_scale_of_its_own_is_taken_out_tile_by_tile() { + let layout = Layout { dx: 0, dy: 0 }; + let (w, h) = (150usize, 110usize); + let m = mosaic(w, h, [0, 1, 1, 2]); + let want = reference(&m, w, h, layout); + let gain = Gain::MatchMean { nominal: 1.0e6, max_deviation: 0.05 }; + // the scale differs a little from nominal; each tile's own scale is measured and removed + let got = denoise_bayer(&m, w, h, layout, &Mock { tile: 32, scale: 1.02e6 }, ¶ms(32, 8, gain), &Control::default()).unwrap(); + let mean = |img: &Rgb32f, c: usize| img.data.iter().map(|p| f64::from(p[c])).sum::() / img.data.len() as f64; + for c in 0..3 { + let (a, b) = (mean(&got, c), mean(&want, c)); + assert!((a - b).abs() / b < 0.01, "channel {c}: {a} vs {b}"); + } + // a scale far from nominal is clamped rather than trusted + let wild = denoise_bayer(&m, w, h, layout, &Mock { tile: 32, scale: 3.0e6 }, ¶ms(32, 8, gain), &Control::default()).unwrap(); + assert!(mean(&wild, 1) > 2.0 * mean(&want, 1)); + } + + #[test] + fn a_picture_smaller_than_a_tile_is_padded_by_mirroring() { + let layout = Layout { dx: 1, dy: 1 }; + let (w, h) = (20usize, 14usize); + let m = mosaic(w, h, [2, 1, 1, 0]); + let got = denoise_bayer(&m, w, h, layout, &Mock { tile: 64, scale: 1.0 }, ¶ms(64, 16, Gain::None), &Control::default()).unwrap(); + assert_eq!((got.width, got.height), (w, h)); + assert!(got.data.iter().all(|p| p.iter().all(|v| v.is_finite() && *v > 0.0))); + } + + #[test] + fn clipped_highlights_stay_clipped() { + // a model that darkens everything by 10 % would pull a clipped plateau below white + struct Dim(Mock); + impl TileRunner for Dim { + fn run(&self, input: &[f32]) -> Result, Error> { + Ok(self.0.run(input)?.into_iter().map(|v| v * 0.9).collect()) + } + } + let layout = Layout { dx: 0, dy: 0 }; + let (w, h) = (96usize, 96usize); + let m: Vec = (0..w * h).map(|i| if i % w > 40 && i % w < 70 { 1.0 } else { 0.4 }).collect(); + let mut p = params(32, 8, Gain::None); + p.clip = Some(0.99); + let got = denoise_bayer(&m, w, h, layout, &Dim(Mock { tile: 32, scale: 1.0 }), &p, &Control::default()).unwrap(); + let mid = got.data[48 * w + 55]; + assert!(mid.iter().all(|&v| v > 0.995), "plateau {mid:?}"); + let dark = got.data[48 * w + 10]; + assert!(dark.iter().all(|&v| (v - 0.36).abs() < 0.01), "unclipped area still goes through the model: {dark:?}"); + } + + #[test] + fn cancelling_and_progress_work() { + let layout = Layout { dx: 0, dy: 0 }; + let (w, h) = (150usize, 110usize); + let m = mosaic(w, h, [0, 1, 1, 2]); + let cancel = AtomicBool::new(true); + let r = denoise_bayer( + &m, + w, + h, + layout, + &Mock { tile: 32, scale: 1.0 }, + ¶ms(32, 8, Gain::None), + &Control { cancel: Some(&cancel), progress: None }, + ); + assert!(matches!(r, Err(Error::Cancelled))); + let seen = std::sync::Mutex::new(Vec::new()); + let f = |d: usize, n: usize| seen.lock().unwrap().push((d, n)); + denoise_bayer(&m, w, h, layout, &Mock { tile: 32, scale: 1.0 }, ¶ms(32, 8, Gain::None), &Control { cancel: None, progress: Some(&f) }) + .unwrap(); + let seen = seen.into_inner().unwrap(); + let total = tile_count(w, h, layout, 32, 8); + assert!(total > 1); + assert_eq!(seen.last(), Some(&(total, total))); + assert!(seen.windows(2).all(|p| p[1].0 == p[0].0 + 1)); + } + + #[test] + fn hostile_inputs_are_errors() { + let layout = Layout { dx: 0, dy: 0 }; + let mock = Mock { tile: 32, scale: 1.0 }; + let p = params(32, 8, Gain::None); + assert!(matches!(denoise_bayer(&[], 0, 0, layout, &mock, &p, &Control::default()), Err(Error::Input(_)))); + assert!(matches!(denoise_bayer(&[0.0; 10], 4, 4, layout, &mock, &p, &Control::default()), Err(Error::Input(_)))); + assert!(matches!(denoise_bayer(&[0.0; 16], 4, 4, layout, &mock, ¶ms(32, 16, Gain::None), &Control::default()), Err(Error::Input(_)))); + struct Bad(Vec); + impl TileRunner for Bad { + fn run(&self, _: &[f32]) -> Result, Error> { + Ok(self.0.clone()) + } + } + assert!(matches!(denoise_bayer(&[0.5; 16], 4, 4, layout, &Bad(vec![0.0; 5]), &p, &Control::default()), Err(Error::Model(_)))); + assert!(matches!(denoise_bayer(&[0.5; 16], 4, 4, layout, &Bad(vec![f32::NAN; 3 * 64 * 64]), &p, &Control::default()), Err(Error::Model(_)))); + } +} diff --git a/crates/denoise-core/src/tiles.rs b/crates/denoise-core/src/tiles.rs new file mode 100644 index 00000000..59898c5a --- /dev/null +++ b/crates/denoise-core/src/tiles.rs @@ -0,0 +1,100 @@ +//! Cutting a picture into overlapping tiles and putting the results back together. +//! +//! A model runs on a fixed square, so a picture is covered by tiles spaced evenly along each axis, each sharing +//! at least `overlap` cells with its neighbours. Each tile's influence fades towards its own edges (except at the +//! picture's edge), and the result is the sum of tile × weight divided by the sum of weights, so any set of +//! overlapping tiles blends consistently and a seam between two tiles of slightly different scale is a smooth ramp. + +/// Where the tiles start along one axis of `n` cells: `[0]` when the axis fits one tile (the tile is then padded by +/// mirroring), else evenly spaced starts, the first at 0 and the last at `n − tile`, each tile sharing at least +/// `overlap` cells with the next. +pub fn starts(n: usize, tile: usize, overlap: usize) -> Vec { + if n <= tile || tile == 0 { + return vec![0]; + } + let step = tile.saturating_sub(overlap).max(1); + // tiles needed so that start k·spacing with spacing ≤ step reaches n − tile + let Some(k) = (n - tile).div_ceil(step).checked_add(1).filter(|&k| k <= 1_000_000) else { return Vec::new() }; + let last = n - tile; + (0..k).map(|i| if k == 1 { 0 } else { ((i as u128 * last as u128 + (k - 1) as u128 / 2) / (k - 1) as u128) as usize }).collect() +} + +/// Weight of the cell at offset `p` (0-based, inside a tile of `tile` cells) along one axis: 1 in the middle, easing +/// down to a sliver over `fade` cells at an edge that has a neighbour, 1 right up to an edge that is the picture's. +pub fn weight(p: usize, tile: usize, fade: usize, has_before: bool, has_after: bool) -> f32 { + let fade = fade.max(1) as f32; + let ramp = |d: usize| { + let t = ((d as f32 + 0.5) / fade).min(1.0); + // smoothstep, never exactly 0 so that a cell is always covered + (t * t * (3.0 - 2.0 * t)).max(1e-3) + }; + let before = if has_before { ramp(p) } else { 1.0 }; + let after = if has_after { ramp(tile.saturating_sub(1).saturating_sub(p)) } else { 1.0 }; + before.min(after) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn hostile_axes_have_bounded_tile_counts_and_checked_spacing() { + assert!(starts(usize::MAX, 1, 0).is_empty()); + assert!(starts(usize::MAX, 2, usize::MAX).is_empty()); + let s = starts(usize::MAX, usize::MAX / 2, 0); + assert_eq!(s.first(), Some(&0)); + assert_eq!(s.last(), Some(&(usize::MAX - usize::MAX / 2))); + assert!(s.windows(2).all(|s| s[0] < s[1])); + } + + #[test] + fn one_tile_when_it_fits() { + assert_eq!(starts(100, 512, 64), vec![0]); + assert_eq!(starts(512, 512, 64), vec![0]); + assert_eq!(starts(0, 512, 64), vec![0]); + assert_eq!(starts(10, 0, 0), vec![0]); + } + + #[test] + fn tiles_cover_the_axis_and_overlap_enough() { + for n in [513usize, 600, 900, 1000, 1024, 1500, 3024, 3600, 5000, 12345] { + let s = starts(n, 512, 64); + assert_eq!(s[0], 0, "n={n}"); + assert_eq!(*s.last().unwrap(), n - 512, "n={n}"); + for w in s.windows(2) { + assert!(w[1] > w[0], "starts increase, n={n}"); + assert!(w[0] + 512 >= w[1] + 64, "at least 64 shared cells between {} and {}, n={n}", w[0], w[1]); + } + } + } + + #[test] + fn weights_fade_only_towards_neighbours() { + let (t, f) = (512, 64); + // a lone tile: full weight everywhere + assert!((0..t).all(|p| weight(p, t, f, false, false) == 1.0)); + // the first tile fades on its far side only + assert_eq!(weight(0, t, f, false, true), 1.0); + assert!(weight(t - 1, t, f, false, true) < 0.01); + // the last tile fades on its near side only + assert!(weight(0, t, f, true, false) < 0.01); + assert_eq!(weight(t - 1, t, f, true, false), 1.0); + // a middle tile is 1 in the middle and everywhere positive + assert_eq!(weight(t / 2, t, f, true, true), 1.0); + assert!((0..t).all(|p| weight(p, t, f, true, true) > 0.0)); + } + + #[test] + fn two_overlapping_tiles_blend_to_a_partition_of_unity() { + // at every cell of the shared region the normalised weights are in [0, 1] and sum to 1 + let (t, f) = (512usize, 64usize); + let s = starts(900, t, 64); + assert_eq!(s.len(), 2); + for cell in s[1]..s[0] + t { + let a = weight(cell - s[0], t, f, false, true); + let b = weight(cell - s[1], t, f, true, false); + let (na, nb) = (a / (a + b), b / (a + b)); + assert!((na + nb - 1.0).abs() < 1e-6 && na >= 0.0 && nb >= 0.0); + } + } +} diff --git a/crates/denoise/Cargo.toml b/crates/denoise/Cargo.toml new file mode 100644 index 00000000..621fd3d6 --- /dev/null +++ b/crates/denoise/Cargo.toml @@ -0,0 +1,32 @@ +[package] +name = "lightcraft-denoise" +version.workspace = true +edition.workspace = true +license.workspace = true +description = "LightCraft AI denoise (L1): denoise-model manifests, Bayer packing, tiling and blending, the cache file, and pure-Rust CPU inference" + +[lints] +workspace = true + +[features] +default = [] +runtime = ["dep:faer"] +# Product-policy switch: absent by default; users may install their own model. +rawnind-model = ["lightcraft-denoise-core/rawnind-model"] + +[dependencies] +lightcraft-denoise-core = { workspace = true } +# Safe matrix views over the pure-Rust GEMM backend; no BLAS, cc/build scripts +# or inner thread pool. Its dependencies include architecture-specific assembly. +faer = { package = "faer-core", version = "=0.17.1", default-features = false, features = ["std"], optional = true } +lightcraft-raster = { workspace = true } +serde = { workspace = true } +serde_json = { workspace = true } +sha2 = { workspace = true } +thiserror = { workspace = true } +miniz_oxide = { workspace = true } +rayon = { workspace = true } +web-time = { workspace = true } + +[dev-dependencies] +proptest = { workspace = true } diff --git a/crates/denoise/README.md b/crates/denoise/README.md new file mode 100644 index 00000000..0397193f --- /dev/null +++ b/crates/denoise/README.md @@ -0,0 +1,18 @@ +# lightcraft-denoise + +L1 pure-Rust AI RAW denoise primitives and CPU inference. No weights are bundled. +See [user workflow and limitations](../../docs/denoise.md). + +`lightcraft-denoise-core` provides metadata, Bayer helpers and cache formats, +re-exported here. `manifest` and `known` validate opt-in models; `onnx` reads a bounded operator subset +into `net`; `cpu` executes it and `reference` supplies scalar checks. `bayer`, `tiles` +and `run` pack sensor data and blend overlapping tiles while preserving clipped +highlights. `product` reads/writes the disposable `.lcdn` cache. `runtime` supplies +the installation self-test. Synthetic networks are generated in Rust for tests. + +Run `cargo test -p lightcraft-denoise --features runtime`. The CPU backend is optional; +featureless and web builds keep the model metadata and cache format without faer/GEMM. +The `rawnind-model` feature enables the otherwise empty download catalog and requires +an explicit release-policy decision. Real-model tests are ignored by default and +require an explicitly supplied local model. The GPU twin is `lightcraft-gpu::nn`; +model management, background jobs and export integration live in `lightcraft-engine`. diff --git a/crates/denoise/src/cpu.rs b/crates/denoise/src/cpu.rs new file mode 100644 index 00000000..34e5269e --- /dev/null +++ b/crates/denoise/src/cpu.rs @@ -0,0 +1,386 @@ +//! Single-thread CPU execution of the checked denoise network. +//! +//! Convolutions lower at most 256 spatial positions at a time, then use faer's safe views over pure-Rust GEMM. +//! No inner thread pool: `run::denoise_bayer` already runs tiles on the application's Rayon pool. Tensor lifetimes +//! are planned once; buffers and im2col storage are recycled between calls, independently for concurrent callers. + +use crate::net::{Net, NetError, Op}; +use crate::run::{Error, TileRunner}; +use faer::{Parallelism, mat}; +use std::sync::{Arc, Mutex}; + +const BLOCK: usize = 256; +/// Cap all tensor buffers and convolution scratch for one tile together (512 MiB). +const MAX_VALUES: usize = 128 * 1024 * 1024; +const MAX_TILE: usize = 2048; +const MAX_IDLE: usize = 1; + +type Result = std::result::Result; +fn bad(why: &str) -> NetError { + NetError::Invalid(why.into()) +} +fn count(c: usize, side: usize) -> Result { + c.checked_mul(side) + .and_then(|n| n.checked_mul(side)) + .filter(|&n| n > 0 && n <= MAX_VALUES) + .ok_or_else(|| bad("a tensor has no size or is too large")) +} +fn zeros(n: usize) -> Result> { + let mut v = Vec::new(); + v.try_reserve_exact(n).map_err(|_| bad("not enough memory for a denoise tile"))?; + v.resize(n, 0.0); + Ok(v) +} +fn sources(op: &Op) -> (usize, Option) { + match op { + Op::Conv(c) => (c.src, c.src2), + Op::ConvT2(c) => (c.src, None), + Op::MaxPool2 { src } | Op::DepthToSpace2 { src } => (*src, None), + } +} +#[derive(Clone, Copy)] +struct Tensor { + slot: usize, + side: usize, + channels: usize, +} +struct Plan { + net: Net, + tensors: Vec, + sizes: Vec, + lower: usize, + product: usize, + up_weights: Vec>>, +} +struct Workspace { + buffers: Vec>, + lower: Vec, + product: Vec, +} + +/// A checked Net on square CHW tiles. Clones share immutable weights and the idle workspace cache. +#[derive(Clone)] +pub struct NetRunner { + plan: Arc, + idle: Arc>>, +} + +impl NetRunner { + pub fn new(net: &Net, tile: usize) -> Result { + if tile == 0 + || tile > MAX_TILE + || !net.fits_tile(tile) + || net.in_channels != 4 + || net.out_channels() != 3 + || net.shape(0).is_none_or(|s| s.channels != net.in_channels) + { + return Err(bad("the tile or Bayer-to-RGB contract does not fit the network")); + } + let mut last = vec![0; net.ops().len() + 1]; + for (i, op) in net.ops().iter().enumerate() { + let (a, b) = sources(op); + for id in std::iter::once(a).chain(b) { + *last.get_mut(id).ok_or_else(|| bad("missing tensor"))? = i + 1; + } + } + let mut tensors = vec![Tensor { slot: 0, side: tile, channels: net.in_channels }]; + let mut sizes = vec![count(net.in_channels, tile)?]; + let mut busy = vec![true]; + let (mut lower, mut product) = (0, 0); + let mut up_weights = Vec::new(); + for (i, op) in net.ops().iter().enumerate() { + let (src, src2) = sources(op); + let a = *tensors.get(src).ok_or_else(|| bad("missing tensor"))?; + let (channels, side) = match op { + Op::Conv(c) => { + if let Some(s) = src2 + && tensors.get(s).is_none_or(|b| b.side != a.side) + { + return Err(bad("a join has different sizes")); + } + (c.cout, a.side) + } + Op::ConvT2(c) => (c.cout, a.side.checked_mul(2).ok_or_else(|| bad("size overflow"))?), + Op::MaxPool2 { .. } => { + if a.side < 2 || !a.side.is_multiple_of(2) { + return Err(bad("pooling does not divide the tile")); + } + (a.channels, a.side / 2) + } + Op::DepthToSpace2 { .. } => (a.channels / 4, a.side.checked_mul(2).ok_or_else(|| bad("size overflow"))?), + }; + let n = count(channels, side)?; + let slot = busy + .iter() + .zip(&sizes) + .enumerate() + .filter(|(_, (used, _))| !**used) + .min_by_key(|(_, (_, size))| (n.saturating_sub(**size), **size)) + .map_or(busy.len(), |(i, _)| i); + if slot == sizes.len() { + sizes.push(n); + busy.push(true); + } else { + let sz = sizes.get_mut(slot).ok_or_else(|| bad("missing buffer"))?; + *sz = (*sz).max(n); + *busy.get_mut(slot).ok_or_else(|| bad("missing buffer"))? = true; + } + tensors.push(Tensor { slot, side, channels }); + // Release sources only after the output has its own buffer (also handles reading the same tensor twice). + for id in std::iter::once(src).chain(src2) { + if last.get(id) == Some(&(i + 1)) { + let t = tensors.get(id).ok_or_else(|| bad("missing tensor"))?; + *busy.get_mut(t.slot).ok_or_else(|| bad("missing buffer"))? = false; + } + } + let (k, m, positions) = match op { + Op::Conv(c) => (c.cin * c.k * c.k, c.cout, a.side * a.side), + Op::ConvT2(c) => (c.cin, c.cout * 4, a.side * a.side), + _ => (0, 0, 0), + }; + lower = lower.max(k.saturating_mul(BLOCK.min(positions))); + product = product.max(m.saturating_mul(BLOCK.min(positions))); + let packed = if let Op::ConvT2(c) = op { + let mut w = zeros(c.weight.len())?; + // [cin][cout][dy][dx] -> [cout*4][cin], so an up-convolution is one GEMM and a shuffle. + for (ci, row) in c.weight.chunks_exact(c.cout * 4).enumerate() { + for (co, &v) in row.iter().enumerate() { + *w.get_mut(co * c.cin + ci).ok_or_else(|| bad("invalid transposed weights"))? = v; + } + } + Some(w) + } else { + None + }; + up_weights.push(packed); + } + let total = sizes + .iter() + .try_fold(lower.saturating_add(product), |n, &s| n.checked_add(s)) + .filter(|&n| n <= MAX_VALUES) + .ok_or_else(|| bad("the tile needs too much working memory"))?; + let _ = total; + Ok(Self { plan: Arc::new(Plan { net: net.clone(), tensors, sizes, lower, product, up_weights }), idle: Arc::new(Mutex::new(Vec::new())) }) + } + /// Immutable data for the GPU runner, without re-reading the model. + pub fn net(&self) -> &Net { + &self.plan.net + } + pub fn bytes_per_tile(&self) -> usize { + (self.plan.sizes.iter().sum::() + self.plan.lower + self.plan.product) * 4 + } + fn workspace(&self) -> Result { + if let Some(w) = self.idle.lock().unwrap_or_else(std::sync::PoisonError::into_inner).pop() { + return Ok(w); + } + Ok(Workspace { + buffers: self.plan.sizes.iter().map(|&n| zeros(n)).collect::>()?, + lower: zeros(self.plan.lower)?, + product: zeros(self.plan.product)?, + }) + } + fn execute(&self, input: &[f32], w: &mut Workspace) -> Result> { + let first = self.plan.tensors.first().ok_or_else(|| bad("no input"))?; + let want = count(first.channels, first.side)?; + w.buffers.first_mut().and_then(|b| b.get_mut(..want)).ok_or_else(|| bad("missing input buffer"))?.copy_from_slice(input); + let profile = std::env::var_os("LIGHTCRAFT_PROFILE").is_some(); + for (i, op) in self.plan.net.ops().iter().enumerate() { + let started = web_time::Instant::now(); + let out = *self.plan.tensors.get(i + 1).ok_or_else(|| bad("missing output tensor"))?; + let (before, rest) = w.buffers.split_at_mut_checked(out.slot).ok_or_else(|| bad("missing output buffer"))?; + let (dest, after) = rest.split_first_mut().ok_or_else(|| bad("missing output buffer"))?; + let get = |id: usize| -> Result<(&[f32], Tensor)> { + let t = *self.plan.tensors.get(id).ok_or_else(|| bad("missing source tensor"))?; + let data = if t.slot < out.slot { + before.get(t.slot) + } else { + after.get(t.slot.checked_sub(out.slot + 1).ok_or_else(|| bad("aliased buffers"))?) + } + .and_then(|b| b.get(..count(t.channels, t.side).ok()?)) + .ok_or_else(|| bad("missing source buffer"))?; + Ok((data, t)) + }; + let (src, src2) = sources(op); + let (a, shape) = get(src)?; + let b = src2.map(get).transpose()?.map(|v| v.0); + let dest = dest.get_mut(..count(out.channels, out.side)?).ok_or_else(|| bad("short output buffer"))?; + match op { + Op::Conv(c) => conv(c, a, b, shape.side, dest, &mut w.lower, &mut w.product)?, + Op::ConvT2(c) => up( + c, + a, + shape.side, + dest, + self.plan.up_weights.get(i).and_then(Option::as_deref).ok_or_else(|| bad("missing up weights"))?, + &mut w.lower, + &mut w.product, + )?, + Op::MaxPool2 { .. } => pool(a, shape.side, dest)?, + Op::DepthToSpace2 { .. } => shuffle(a, shape.side, dest)?, + } + if dest.iter().any(|v| !v.is_finite()) { + return Err(bad("the model produced non-finite values")); + } + if profile { + eprintln!("[denoise-cpu] layer {i}: {}x{}x{} {:.2} ms", out.channels, out.side, out.side, started.elapsed().as_secs_f64() * 1e3); + } + } + let t = self.plan.tensors.last().ok_or_else(|| bad("no output"))?; + let data = w.buffers.get(t.slot).and_then(|b| b.get(..count(t.channels, t.side).ok()?)).ok_or_else(|| bad("missing result"))?; + let mut out = zeros(data.len())?; + out.copy_from_slice(data); + Ok(out) + } +} +impl TileRunner for NetRunner { + fn run(&self, input: &[f32]) -> std::result::Result, Error> { + let first = self.plan.tensors.first().ok_or_else(|| Error::Runtime("missing input shape".into()))?; + let want = count(first.channels, first.side).map_err(|e| Error::Runtime(e.to_string()))?; + if input.len() != want || input.iter().any(|v| !v.is_finite()) { + return Err(Error::Input("the tile has the wrong size or non-finite values".into())); + } + let mut w = self.workspace().map_err(|e| Error::Runtime(e.to_string()))?; + let result = self.execute(input, &mut w).map_err(|e| Error::Runtime(e.to_string())); + let mut idle = self.idle.lock().unwrap_or_else(std::sync::PoisonError::into_inner); + if idle.len() < MAX_IDLE { + idle.push(w); + } + result + } +} +fn activate(x: f32, bias: f32, alpha: Option) -> f32 { + let x = x + bias; + if x < 0.0 { x * alpha.unwrap_or(1.0) } else { x } +} + +/// All views are over exact, checked slices. The matrix library's assertions therefore never see file-derived sizes. +fn multiply(w: &[f32], x: &[f32], y: &mut [f32], m: usize, k: usize, n: usize) -> Result<()> { + if m.checked_mul(k) != Some(w.len()) || k.checked_mul(n) != Some(x.len()) || m.checked_mul(n) != Some(y.len()) || m == 0 || k == 0 || n == 0 { + return Err(bad("invalid matrix dimensions")); + } + faer::mul::matmul( + mat::from_row_major_slice_mut::(y, m, n), + mat::from_row_major_slice::(w, m, k), + mat::from_row_major_slice::(x, k, n), + None, + 1.0, + Parallelism::None, + ); + Ok(()) +} +fn lower(a: &[f32], b: Option<&[f32]>, side: usize, k: usize, start: usize, n: usize, dest: &mut [f32]) -> Result<()> { + let pad = k / 2; + let pixels = side * side; + // Whole row segments are copied, so bounds checks and edge branches stay out of the multiply loop. + for (ci, plane) in a.chunks_exact(pixels).chain(b.into_iter().flat_map(|b| b.chunks_exact(pixels))).enumerate() { + for ky in 0..k { + for kx in 0..k { + let row = (ci * k * k + ky * k + kx) * n; + let out = dest.get_mut(row..row + n).ok_or_else(|| bad("short im2col buffer"))?; + let mut at = 0; + while at < n { + let p = start + at; + let (y, x) = (p / side, p % side); + let len = (side - x).min(n - at); + if let Some(yy) = y.checked_add(ky).and_then(|v| v.checked_sub(pad)).filter(|&v| v < side) { + let lo = x.max(pad.saturating_sub(kx)); + let hi = (x + len).min((side + pad).saturating_sub(kx)); + if lo < hi { + let from = yy * side + lo + kx - pad; + let to = at + lo - x; + out.get_mut(at..to).ok_or_else(|| bad("short im2col padding"))?.fill(0.0); + out.get_mut(to..to + hi - lo) + .ok_or_else(|| bad("short im2col row"))? + .copy_from_slice(plane.get(from..from + hi - lo).ok_or_else(|| bad("short source row"))?); + out.get_mut(to + hi - lo..at + len).ok_or_else(|| bad("short im2col padding"))?.fill(0.0); + } else { + out.get_mut(at..at + len).ok_or_else(|| bad("short im2col row"))?.fill(0.0); + } + } else { + out.get_mut(at..at + len).ok_or_else(|| bad("short im2col row"))?.fill(0.0); + } + at += len; + } + } + } + } + Ok(()) +} +fn conv(c: &crate::net::Conv, a: &[f32], b: Option<&[f32]>, side: usize, dest: &mut [f32], scratch: &mut [f32], product: &mut [f32]) -> Result<()> { + let pixels = side * side; + let k = c.cin * c.k * c.k; + for start in (0..pixels).step_by(BLOCK) { + let n = BLOCK.min(pixels - start); + let x = scratch.get_mut(..k * n).ok_or_else(|| bad("short im2col storage"))?; + lower(a, b, side, c.k, start, n, x)?; + let y = product.get_mut(..c.cout * n).ok_or_else(|| bad("short product storage"))?; + multiply(&c.weight, x, y, c.cout, k, n)?; + for ((row, &bias), out) in y.chunks_exact(n).zip(&c.bias).zip(dest.chunks_exact_mut(pixels)) { + let to = out.get_mut(start..start + n).ok_or_else(|| bad("short convolution output"))?; + for (d, &v) in to.iter_mut().zip(row) { + *d = activate(v, bias, c.leaky); + } + } + } + Ok(()) +} +fn up(c: &crate::net::ConvT2, a: &[f32], side: usize, dest: &mut [f32], weights: &[f32], scratch: &mut [f32], product: &mut [f32]) -> Result<()> { + let pixels = side * side; + let out_side = 2 * side; + for start in (0..pixels).step_by(BLOCK) { + let n = BLOCK.min(pixels - start); + let x = scratch.get_mut(..c.cin * n).ok_or_else(|| bad("short up input storage"))?; + for (to, from) in x.chunks_exact_mut(n).zip(a.chunks_exact(pixels)) { + to.copy_from_slice(from.get(start..start + n).ok_or_else(|| bad("short up input"))?); + } + let y = product.get_mut(..c.cout * 4 * n).ok_or_else(|| bad("short up product storage"))?; + multiply(weights, x, y, c.cout * 4, c.cin, n)?; + for (co, (&bias, out)) in c.bias.iter().zip(dest.chunks_exact_mut(4 * pixels)).enumerate() { + for d in 0..4 { + let row = y.get((4 * co + d) * n..(4 * co + d + 1) * n).ok_or_else(|| bad("short up product"))?; + for (j, &v) in row.iter().enumerate() { + let p = start + j; + let at = (2 * (p / side) + d / 2) * out_side + 2 * (p % side) + d % 2; + *out.get_mut(at).ok_or_else(|| bad("short up output"))? = activate(v, bias, c.leaky); + } + } + } + } + Ok(()) +} +fn pool(a: &[f32], side: usize, dest: &mut [f32]) -> Result<()> { + let s = side / 2; + for (input, out) in a.chunks_exact(side * side).zip(dest.chunks_exact_mut(s * s)) { + for (y, row) in out.chunks_exact_mut(s).enumerate() { + let lo = input.get(2 * y * side..(2 * y + 1) * side).ok_or_else(|| bad("short pool input"))?; + let hi = input.get((2 * y + 1) * side..(2 * y + 2) * side).ok_or_else(|| bad("short pool input"))?; + for ((d, lo), hi) in row.iter_mut().zip(lo.as_chunks::<2>().0).zip(hi.as_chunks::<2>().0) { + let [a, b] = *lo; + let [c, e] = *hi; + *d = a.max(b).max(c).max(e); + } + } + } + Ok(()) +} +fn shuffle(a: &[f32], side: usize, dest: &mut [f32]) -> Result<()> { + let pixels = side * side; + for (input, out) in a.chunks_exact(4 * pixels).zip(dest.chunks_exact_mut(4 * pixels)) { + for d in 0..4 { + let plane = input.get(d * pixels..(d + 1) * pixels).ok_or_else(|| bad("short shuffle input"))?; + for (y, row) in plane.chunks_exact(side).enumerate() { + let at = (2 * y + d / 2) * 2 * side; + let to = out.get_mut(at..at + 2 * side).ok_or_else(|| bad("short shuffle output"))?; + for (to, &v) in to.as_chunks_mut::<2>().0.iter_mut().zip(row) { + if let Some(t) = to.get_mut(d % 2) { + *t = v; + } + } + } + } + } + Ok(()) +} + +#[cfg(test)] +mod tests; diff --git a/crates/denoise/src/cpu/tests.rs b/crates/denoise/src/cpu/tests.rs new file mode 100644 index 00000000..16a496b5 --- /dev/null +++ b/crates/denoise/src/cpu/tests.rs @@ -0,0 +1,151 @@ +use super::*; +use crate::{ + net::{Conv, ConvT2}, + reference, synthetic, +}; + +fn random(n: usize, seed: u32) -> Vec { + let mut s = seed | 1; + (0..n) + .map(|_| { + s ^= s << 13; + s ^= s >> 17; + s ^= s << 5; + (s >> 8) as f32 / (1u32 << 23) as f32 - 1.0 + }) + .collect() +} +fn net(tile: usize, ch: u64, depth: usize, seed: u64) -> Net { + let p = std::env::temp_dir().join(format!("lc-purecpu-{}-{tile}-{ch}-{depth}-{seed}.onnx", std::process::id())); + std::fs::write(&p, synthetic::unet_onnx(tile as u64, ch, depth, seed)).unwrap(); + let n = crate::onnx::read(&p).unwrap(); + std::fs::remove_file(p).unwrap(); + n +} +fn agree(a: &[f32], b: &[f32]) { + assert_eq!(a.len(), b.len()); + assert!(a.iter().chain(b).all(|v| v.is_finite())); + let scale = b.iter().fold(1e-6f32, |m, v| m.max(v.abs())); + let error = a.iter().zip(b).fold(0f32, |m, (a, b)| m.max((a - b).abs())) / scale; + assert!(error <= 1e-3, "relative maximum error {error:e}"); +} +#[test] +fn all_operators_match_scalar_on_random_nets_and_odd_and_tiny_shapes() { + for (t, ch, d) in [(1, 3, 0), (3, 5, 0), (5, 7, 0), (6, 3, 1), (20, 5, 2), (36, 9, 2), (64, 8, 3)] { + for seed in [1, 73] { + let n = net(t, ch, d, seed); + let r = NetRunner::new(&n, t).unwrap(); + let x = random(4 * t * t, seed as u32); + let want = reference::run(&n, t, &x).unwrap(); + agree(&r.run(&x).unwrap(), &want); + assert_eq!(r.run(&x).unwrap(), r.run(&x).unwrap()); + } + } +} + +#[test] +fn malformed_values_fail_before_execution_and_workspace_sizes_are_bounded() { + let mut n = net(4, 3, 1, 11); + for t in [0, 1, 3, usize::MAX, 4096] { + assert!(NetRunner::new(&n, t).is_err()); + } + for ch in [0, 3, usize::MAX] { + n.in_channels = ch; + assert!(NetRunner::new(&n, 4).is_err()); + } + let n = Net::new( + 4, + vec![ + Op::Conv(Conv { src: 0, src2: None, cin: 4, cout: 4096, k: 1, weight: vec![0.1; 4 * 4096], bias: vec![0.0; 4096], leaky: None }), + Op::Conv(Conv { src: 1, src2: None, cin: 4096, cout: 12, k: 1, weight: vec![0.1; 4096 * 12], bias: vec![0.0; 12], leaky: None }), + Op::DepthToSpace2 { src: 2 }, + ], + ) + .unwrap(); + assert!(NetRunner::new(&n, 2048).is_err()); + let ops = (0..100).map(|i| Op::MaxPool2 { src: i }).collect(); + let n = Net::new(4, ops).unwrap(); + assert!(NetRunner::new(&n, 16).is_err()); + let c = Conv { src: 0, src2: None, cin: 4, cout: 12, k: 1, weight: vec![0.25; 48], bias: vec![0.0; 12], leaky: None }; + for mode in 0..6 { + let mut bad = c.clone(); + match mode { + 0 => bad.src = usize::MAX, + 1 => bad.weight.pop().map(|_| ()).unwrap(), + 2 => bad.bias[0] = f32::NAN, + 3 => bad.leaky = Some(f32::INFINITY), + 4 => bad.cout = usize::MAX, + _ => bad.k = usize::MAX, + } + assert!(Net::new(4, vec![Op::Conv(bad), Op::DepthToSpace2 { src: 1 }]).is_err()); + } +} +#[test] +fn bad_tiles_and_arithmetic_overflow_are_errors_and_do_not_poison_reuse() { + let c = Conv { src: 0, src2: None, cin: 4, cout: 12, k: 1, weight: vec![f32::MAX; 48], bias: vec![0.0; 12], leaky: None }; + let n = Net::new(4, vec![Op::Conv(c), Op::DepthToSpace2 { src: 1 }]).unwrap(); + let r = NetRunner::new(&n, 1).unwrap(); + assert!(r.run(&[]).is_err()); + assert!(r.run(&[f32::NAN; 4]).is_err()); + assert!(r.run(&[f32::INFINITY; 4]).is_err()); + assert_eq!(r.idle.lock().unwrap().len(), 0, "invalid input never allocates a workspace"); + assert!(r.run(&[2.0; 4]).is_err()); + assert_eq!(r.run(&[0.0; 4]).unwrap(), vec![0.0; 12]); + // Matrix shape checks are the boundary in front of the dependency's asserting constructors. + assert!(multiply(&[], &[], &mut [], usize::MAX, 2, 2).is_err()); +} +#[test] +fn shared_runner_serves_concurrent_tiles_without_cross_contamination() { + let n = net(12, 5, 1, 97); + let r = NetRunner::new(&n, 12).unwrap(); + let inputs: Vec<_> = (0..8).map(|s| random(4 * 12 * 12, s)).collect(); + let answers: Vec<_> = inputs.iter().map(|x| reference::run(&n, 12, x).unwrap()).collect(); + std::thread::scope(|scope| { + let mut handles = Vec::new(); + for (x, want) in inputs.iter().zip(&answers) { + let r = &r; + handles.push(scope.spawn(move || { + for _ in 0..3 { + agree(&r.run(x).unwrap(), want); + } + })); + } + for h in handles { + h.join().unwrap(); + } + }); +} +#[test] +fn join_may_read_the_same_tensor_twice_and_leaky_up_convolution_is_supported() { + let c = ConvT2 { src: 0, cin: 4, cout: 3, weight: random(48, 2), bias: vec![0.1, -0.1, 0.0], leaky: Some(0.13) }; + let n = Net::new( + 4, + vec![ + Op::ConvT2(c), + Op::Conv(Conv { src: 1, src2: Some(1), cin: 6, cout: 3, k: 3, weight: random(162, 3), bias: vec![0.0; 3], leaky: Some(0.0) }), + ], + ) + .unwrap(); + let x = random(4 * 3 * 3, 7); + agree(&NetRunner::new(&n, 3).unwrap().run(&x).unwrap(), &reference::run(&n, 3, &x).unwrap()); +} + +#[test] +fn reused_im2col_clears_all_edge_padding_between_images() { + for t in [1, 3, 5, 36] { + let n = net(t, 5, 0, 121); + let r = NetRunner::new(&n, t).unwrap(); + for x in [vec![1.0; 4 * t * t], vec![0.0; 4 * t * t], random(4 * t * t, 9)] { + agree(&r.run(&x).unwrap(), &reference::run(&n, t, &x).unwrap()); + } + } +} +#[test] +#[ignore = "needs LC_DENOISE_MODEL; one tile with LIGHTCRAFT_PROFILE=1"] +fn profile_one_real_tile() { + let model = std::env::var_os("LC_DENOISE_MODEL").expect("set LC_DENOISE_MODEL"); + let m = crate::known::find("rawnind-bayer").unwrap().manifest; + let r = crate::runtime::CpuRunner::load(std::path::Path::new(&model), &m).unwrap(); + println!("workspace {:.1} MB", r.bytes_per_tile() as f64 / 1e6); + r.run(&crate::runtime::check_tile(512)).unwrap(); +} diff --git a/crates/denoise/src/lib.rs b/crates/denoise/src/lib.rs new file mode 100644 index 00000000..e72860ba --- /dev/null +++ b/crates/denoise/src/lib.rs @@ -0,0 +1,37 @@ +//! AI denoise for LightCraft, as a step between the sensor and the demosaic that the rest of the develop +//! pipeline never sees: the raw file stays untouched, the denoised picture is cached data that can always be +//! made again, and the amount is an ordinary develop setting. +//! +//! - [`manifest`]: what a denoise model takes and gives, and who may use it (untrusted JSON, validated). +//! - [`known`] and [`archive`]: the models LightCraft can fetch, and taking the model file out of the `.zip` it comes in. +//! - [`bayer`]: Bayer layouts and the packing of a mosaic into the four planes a model takes. +//! - [`tiles`] and [`run`]: cutting a picture into overlapping tiles for a fixed-size model, bringing each tile's +//! output scale back to the input's, keeping clipped highlights as they were, and blending the tiles together. +//! The model itself is a [`run::TileRunner`], so all of this is tested without one. +//! - [`product`]: the cache file the denoised picture is kept in (half floats, compressed in strips) and the +//! blend with the plain demosaic that the Amount slider controls. +//! - [`cpu`] and [`runtime`]: a single-thread pure-Rust CPU runner and its installation self-test. +//! - [`net`] and [`onnx`]: checked plain network data and bounded ONNX decoding, shared by CPU and GPU. +//! +//! Everything here treats model files, manifests and cache files as hostile input: sizes are capped, numbers must +//! be finite, and a malformed file is an error, never a panic. +#![forbid(unsafe_code)] +#![deny(clippy::unwrap_used, clippy::expect_used, clippy::panic, clippy::unimplemented, clippy::todo, clippy::unreachable)] + +#[cfg(feature = "runtime")] +pub mod cpu; + +pub mod net; +pub mod onnx; +mod onnx_proto; + +pub mod reference; + +#[cfg(feature = "runtime")] +pub mod runtime; +#[doc(hidden)] +pub mod synthetic; + +pub use lightcraft_denoise_core::*; + +pub mod synthetic_proto; diff --git a/crates/denoise/src/net.rs b/crates/denoise/src/net.rs new file mode 100644 index 00000000..42241833 --- /dev/null +++ b/crates/denoise/src/net.rs @@ -0,0 +1,345 @@ +//! A denoise network as plain data: the handful of layer kinds a U-Net denoiser is made of, with their weights. +//! +//! [`onnx`](crate::onnx) reads the file into this description, shared by the pure-Rust CPU and GPU runners. +//! Unsupported operators, groups, dilation or activations are reported as errors when loading the model. +//! +//! The description is checked ([`Net::new`]) because it comes from a stranger's file: channel counts must chain, +//! weights must have the sizes the shapes need and be finite, and the depth must stay small. + +/// Most layers a network may have. +pub const MAX_OPS: usize = 512; +/// Most channels a layer may have. +pub const MAX_CHANNELS: usize = 4096; +/// Most weights (numbers) a network may have in all. +pub const MAX_WEIGHTS: usize = 128 * 1024 * 1024; + +#[derive(Debug, PartialEq, thiserror::Error)] +pub enum NetError { + /// The model uses something this denoise network description cannot express. + #[error("not supported by the denoise runners: {0}")] + Unsupported(String), + /// The model is not a well-formed network. + #[error("not a valid network: {0}")] + Invalid(String), +} + +fn invalid(why: impl Into) -> Result { + Err(NetError::Invalid(why.into())) +} + +/// A tensor is named by its number: 0 is the network's input, and layer `i` makes tensor `i + 1`. +pub type TensorId = usize; + +/// A convolution with stride 1 and "same" padding (`k` is 1 or 3), over one tensor or two joined along the channels +/// (`src` first), then a bias and optionally a leaky ReLU. +#[derive(Clone, Debug, PartialEq)] +pub struct Conv { + pub src: TensorId, + /// A second source, whose channels follow `src`'s (a skip connection's concatenation). + pub src2: Option, + pub cin: usize, + pub cout: usize, + pub k: usize, + /// `[cout][cin][k][k]`, as ONNX has it. + pub weight: Vec, + pub bias: Vec, + /// `Some(alpha)`: a leaky ReLU with this slope follows. + pub leaky: Option, +} + +/// A transposed convolution with a 2 × 2 kernel and stride 2: every input pixel becomes a 2 × 2 block. +#[derive(Clone, Debug, PartialEq)] +pub struct ConvT2 { + pub src: TensorId, + pub cin: usize, + pub cout: usize, + /// `[cin][cout][2][2]`, as ONNX has it. + pub weight: Vec, + pub bias: Vec, + pub leaky: Option, +} + +#[derive(Clone, Debug, PartialEq)] +pub enum Op { + Conv(Conv), + ConvT2(ConvT2), + /// 2 × 2 maximum, stride 2. + MaxPool2 { + src: TensorId, + }, + /// Depth-to-space with block size 2 in "CRD" order: output channel `c` at `(2y + dy, 2x + dx)` is input channel + /// `4c + 2dy + dx` at `(y, x)`. + DepthToSpace2 { + src: TensorId, + }, +} + +/// What a tensor looks like: its channels, and its resolution as a power of two below the input's (a negative +/// level is above it). +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct Shape { + pub channels: usize, + pub level: i32, +} + +#[derive(Clone, Debug, PartialEq)] +pub struct Net { + pub in_channels: usize, + ops: Vec, + shapes: Vec, +} + +impl Net { + /// Check `ops` (layer `i` makes tensor `i + 1`) over an input of `in_channels` and describe every tensor. + pub fn new(in_channels: usize, ops: Vec) -> Result { + if in_channels == 0 || in_channels > MAX_CHANNELS { + return invalid("the input has no channels or too many"); + } + if ops.is_empty() || ops.len() > MAX_OPS { + return invalid("the network has no layers or too many"); + } + let mut shapes = vec![Shape { channels: in_channels, level: 0 }]; + let mut weights = 0usize; + for (i, op) in ops.iter().enumerate() { + let shape_of = |t: TensorId| { + shapes + .get(t) + .copied() + .filter(|_| t <= i) + .ok_or_else(|| NetError::Invalid(format!("layer {i} reads tensor {t}, which does not exist yet"))) + }; + let out = match op { + Op::Conv(c) => { + let a = shape_of(c.src)?; + let (cin, level) = match c.src2 { + Some(s2) => { + let b = shape_of(s2)?; + if b.level != a.level { + return invalid(format!("layer {i} joins tensors of different resolutions")); + } + (a.channels + b.channels, a.level) + } + None => (a.channels, a.level), + }; + if c.cin != cin || c.cout == 0 || c.cout > MAX_CHANNELS || !(c.k == 1 || c.k == 3) { + return invalid(format!( + "layer {i}: a convolution of {} to {} channels with a {}-wide kernel does not fit", + c.cin, c.cout, c.k + )); + } + check_weights( + i, + &c.weight, + c.cout.checked_mul(cin).and_then(|n| n.checked_mul(c.k * c.k)), + &c.bias, + c.cout, + c.leaky, + &mut weights, + )?; + Shape { channels: c.cout, level } + } + Op::ConvT2(c) => { + let a = shape_of(c.src)?; + if c.cin != a.channels || c.cout == 0 || c.cout > MAX_CHANNELS { + return invalid(format!("layer {i}: a transposed convolution of {} to {} channels does not fit", c.cin, c.cout)); + } + check_weights(i, &c.weight, c.cin.checked_mul(c.cout).and_then(|n| n.checked_mul(4)), &c.bias, c.cout, c.leaky, &mut weights)?; + Shape { channels: c.cout, level: a.level - 1 } + } + Op::MaxPool2 { src } => { + let a = shape_of(*src)?; + Shape { channels: a.channels, level: a.level + 1 } + } + Op::DepthToSpace2 { src } => { + let a = shape_of(*src)?; + if !a.channels.is_multiple_of(4) { + return invalid(format!("layer {i}: depth-to-space needs a multiple of 4 channels")); + } + Shape { channels: a.channels / 4, level: a.level - 1 } + } + }; + shapes.push(out); + } + Ok(Net { in_channels, ops, shapes }) + } + + /// The layers: layer `i` makes tensor `i + 1`. + pub fn ops(&self) -> &[Op] { + &self.ops + } + + /// The shape of tensor `t` (0 is the input). + pub fn shape(&self, t: TensorId) -> Option { + self.shapes.get(t).copied() + } + + /// The tensor the network gives (the last layer's). + pub fn output(&self) -> TensorId { + self.ops.len() + } + + pub fn out_channels(&self) -> usize { + self.shapes.last().map_or(0, |s| s.channels) + } + + /// The deepest resolution level: a tile must be a multiple of `2^this` cells wide. + pub fn depth(&self) -> u32 { + self.shapes.iter().map(|s| s.level.max(0) as u32).max().unwrap_or(0) + } + + /// Can the network run on a `tile × tile` input and give `[out_channels, 2·tile, 2·tile]`? (Pooling must divide + /// the tile, and the output must be one level above the input.) + pub fn fits_tile(&self, tile: usize) -> bool { + let Some(multiple) = 1usize.checked_shl(self.depth()) else { return false }; + tile > 0 && tile.is_multiple_of(multiple) && self.shapes.last().is_some_and(|s| s.level == -1) + } + + /// Multiply-adds for one tile, for estimating time. + pub fn macs(&self, tile: usize) -> u64 { + let pixels = |level: i32| -> u64 { + let side = if level >= 0 { + tile.checked_shr(level as u32).unwrap_or(0) + } else { + tile.saturating_mul(1usize.checked_shl(level.unsigned_abs()).unwrap_or(usize::MAX)) + }; + (side as u64).saturating_mul(side as u64) + }; + self.ops + .iter() + .enumerate() + .map(|(i, op)| { + let out = self.shapes.get(i + 1).map_or(0, |s| pixels(s.level)); + match op { + Op::Conv(c) => out.saturating_mul((c.cin * c.cout * c.k * c.k) as u64), + // every input pixel makes a 2 × 2 block: as many multiply-adds as the 2 × 2-wide kernel has weights + Op::ConvT2(c) => pixels(self.shapes.get(c.src).map_or(0, |s| s.level)).saturating_mul((c.cin * c.cout * 4) as u64), + _ => 0, + } + }) + .fold(0u64, u64::saturating_add) + } +} + +fn check_weights( + layer: usize, + weight: &[f32], + want: Option, + bias: &[f32], + cout: usize, + leaky: Option, + total: &mut usize, +) -> Result<(), NetError> { + if want != Some(weight.len()) || bias.len() != cout { + return invalid(format!("layer {layer}: the weights are not the size its shape needs")); + } + *total = total.saturating_add(weight.len()).saturating_add(bias.len()); + if *total > MAX_WEIGHTS { + return invalid("the network has too many weights"); + } + if weight.iter().chain(bias).any(|v| !v.is_finite()) || leaky.is_some_and(|a| !a.is_finite()) { + return invalid(format!("layer {layer}: a weight is not a finite number")); + } + Ok(()) +} + +#[cfg(test)] +mod tests { + use super::*; + + fn conv(src: TensorId, src2: Option, cin: usize, cout: usize, k: usize) -> Op { + Op::Conv(Conv { src, src2, cin, cout, k, weight: vec![0.1; cout * cin * k * k], bias: vec![0.0; cout], leaky: Some(0.2) }) + } + + fn convt(src: TensorId, cin: usize, cout: usize) -> Op { + Op::ConvT2(ConvT2 { src, cin, cout, weight: vec![0.1; cin * cout * 4], bias: vec![0.0; cout], leaky: None }) + } + + /// A two-level network of the shape the denoisers have: 4 → 8 channels, pool, 16 channels, up, join, 12, and the + /// 1 × 1 convolution to the colours. + fn unet() -> Vec { + vec![ + conv(0, None, 4, 8, 3), // 1: 8 ch, level 0 + Op::MaxPool2 { src: 1 }, // 2: level 1 + conv(2, None, 8, 16, 3), // 3: 16 ch, level 1 + convt(3, 16, 8), // 4: 8 ch, level 0 + conv(4, Some(1), 16, 8, 3), // 5: join [up, skip] + conv(5, None, 8, 12, 1), // 6: 12 ch + Op::DepthToSpace2 { src: 6 }, // 7: 3 ch, level -1 + ] + } + + #[test] + fn deep_networks_and_huge_cost_queries_never_overflow_or_truncate_depth() { + let n = Net::new(4, unet()).unwrap(); + assert_eq!(n.macs(usize::MAX), u64::MAX); + assert_eq!(n.macs(0), 0); + let mut ops = Vec::new(); + for i in 0..21 { + ops.push(Op::MaxPool2 { src: i }); + } + for i in 21..42 { + ops.push(convt(i, 4, 4)); + } + ops.push(conv(42, None, 4, 12, 1)); + ops.push(Op::DepthToSpace2 { src: 43 }); + let n = Net::new(4, ops).unwrap(); + assert!(!n.fits_tile(1 << 20)); + assert!(n.fits_tile(1 << 21)); + let _ = n.macs(16); + } + + #[test] + fn a_u_net_is_described_with_its_shapes_and_costs() { + let net = Net::new(4, unet()).unwrap(); + assert_eq!(net.out_channels(), 3); + assert_eq!(net.shape(3), Some(Shape { channels: 16, level: 1 })); + assert_eq!(net.shape(7), Some(Shape { channels: 3, level: -1 })); + assert_eq!(net.depth(), 1); + assert!(net.fits_tile(64) && !net.fits_tile(63) && !net.fits_tile(0)); + // 64 × 64 cells: layer 1 = 4096·4·8·9, and so on; just check it is the sum of what the layers cost + let t = 64u64; + let want = t * t * 4 * 8 * 9 + (t / 2) * (t / 2) * 8 * 16 * 9 + (t / 2) * (t / 2) * 16 * 8 * 4 + t * t * 16 * 8 * 9 + t * t * 8 * 12; + assert_eq!(net.macs(64), want); + } + + #[test] + fn a_network_that_does_not_chain_is_refused() { + // wrong input channel count for the first convolution + assert!(Net::new(3, unet()).is_err()); + // a layer reading a tensor that does not exist yet + let mut ops = unet(); + ops[2] = conv(9, None, 8, 16, 3); + assert!(Net::new(4, ops).is_err()); + // joining tensors of different resolutions + let mut ops = unet(); + ops[4] = conv(3, Some(1), 24, 8, 3); + assert!(Net::new(4, ops).is_err()); + // weights of the wrong size, and not finite + let mut ops = unet(); + if let Op::Conv(c) = &mut ops[0] { + c.weight.pop(); + } + assert!(Net::new(4, ops).is_err()); + let mut ops = unet(); + if let Op::Conv(c) = &mut ops[0] { + c.bias[0] = f32::NAN; + } + assert!(Net::new(4, ops).is_err()); + // a kernel that is neither 1 nor 3 wide, no layers, depth-to-space on a channel count it cannot split + let mut ops = unet(); + if let Op::Conv(c) = &mut ops[0] { + c.k = 5; + } + assert!(Net::new(4, ops).is_err()); + assert!(Net::new(4, vec![]).is_err()); + assert!(Net::new(4, vec![Op::DepthToSpace2 { src: 0 }]).is_ok()); + assert!(Net::new(4, vec![Op::DepthToSpace2 { src: 0 }, Op::DepthToSpace2 { src: 1 }]).is_err()); + assert!(Net::new(2, vec![Op::DepthToSpace2 { src: 0 }]).is_err()); + } + + #[test] + fn a_network_that_does_not_end_one_level_up_does_not_fit_a_tile() { + let net = Net::new(4, unet()[..6].to_vec()).unwrap(); + assert!(!net.fits_tile(64), "no depth-to-space: the output is not at twice the resolution"); + } +} diff --git a/crates/denoise/src/onnx.rs b/crates/denoise/src/onnx.rs new file mode 100644 index 00000000..fc84a36c --- /dev/null +++ b/crates/denoise/src/onnx.rs @@ -0,0 +1,369 @@ +//! Reading an `.onnx` file into a [`Net`] with bounded, pure-Rust protobuf decoding. +//! +//! Only what the denoise U-Nets are made of is understood: `Conv` (3 × 3 "same" or 1 × 1, stride 1, no groups, no +//! dilation) optionally followed by `LeakyRelu`, `ConvTranspose` (2 × 2, stride 2) likewise, `MaxPool` (2 × 2, stride 2), +//! a two-input `Concat` on the channels that feeds a convolution, and a final `DepthToSpace` (2, `CRD`). Any other +//! operator or attribute is [`NetError::Unsupported`], which is reported to the user by both runners. + +use std::collections::HashMap; +use std::path::Path; + +use crate::onnx_proto::{self, AttributeProto, NodeProto, TensorProto, ValueInfoProto}; +use std::io::Read; + +use crate::net::{Conv, ConvT2, MAX_WEIGHTS, Net, NetError, Op, TensorId}; + +fn unsupported(why: impl Into) -> Result { + Err(NetError::Unsupported(why.into())) +} + +fn invalid(why: impl Into) -> Result { + Err(NetError::Invalid(why.into())) +} + +/// Read the model at `path`. +pub fn read(path: &Path) -> Result { + let file = std::fs::File::open(path).map_err(|e| NetError::Invalid(e.to_string()))?; + let size = usize::try_from(file.metadata().map_err(|e| NetError::Invalid(e.to_string()))?.len()) + .ok() + .filter(|&n| n <= onnx_proto::MAX_BYTES) + .ok_or_else(|| NetError::Invalid("model file too large".into()))?; + let mut bytes = Vec::new(); + bytes.try_reserve_exact(size).map_err(|_| NetError::Invalid("not enough memory to load the model".into()))?; + file.take(onnx_proto::MAX_BYTES as u64 + 1).read_to_end(&mut bytes).map_err(|e| NetError::Invalid(e.to_string()))?; + read_bytes(&bytes) +} + +fn read_bytes(bytes: &[u8]) -> Result { + let graph = onnx_proto::decode(bytes)?; + + let initialisers: HashMap<&str, &TensorProto> = graph.initializer.iter().map(|t| (t.name.as_str(), t)).collect(); + if initialisers.len() != graph.initializer.len() { + return invalid("duplicate initializer names"); + } + let inputs: Vec<_> = graph.input.iter().filter(|i| !initialisers.contains_key(i.name.as_str())).collect(); + let ([input], [output]) = (inputs.as_slice(), graph.output.as_slice()) else { return unsupported("not one input and one output") }; + let in_channels = match channel_dim(input) { + Some(c) => c, + None => return unsupported("the input's channel count is not written in the file"), + }; + + // how many nodes read each name (a graph output counts as a reader) + let mut readers: HashMap<&str, usize> = HashMap::new(); + for n in &graph.node { + for i in &n.input { + *readers.entry(i.as_str()).or_default() += 1; + } + } + *readers.entry(output.name.as_str()).or_default() += 1; + + let mut ids: HashMap = HashMap::new(); + ids.insert(input.name.clone(), 0); + // a Concat is not a layer of its own: the convolution that reads it takes both sources + let mut joins: HashMap = HashMap::new(); + let mut ops: Vec = Vec::new(); + let mut weights_total = 0usize; + + let nodes = &graph.node; + let mut i = 0; + while i < nodes.len() { + let Some(node) = nodes.get(i) else { break }; + i += 1; + if node.output.len() != 1 || node.output.first().is_none_or(|n| n.is_empty() || ids.contains_key(n) || joins.contains_key(n)) { + return invalid("a layer has missing, repeated or multiple outputs"); + } + let allowed: &[&str] = match node.op_type.as_str() { + "Conv" => &["kernel_shape", "pads", "strides", "dilations", "group", "auto_pad"], + "ConvTranspose" => &["kernel_shape", "pads", "strides", "dilations", "group", "output_padding", "auto_pad"], + "MaxPool" => &["kernel_shape", "pads", "strides", "dilations", "ceil_mode", "storage_order", "auto_pad"], + "Concat" => &["axis"], + "DepthToSpace" => &["blocksize", "mode"], + _ => &[], + }; + if node.attribute.iter().any(|a| !allowed.contains(&a.name.as_str())) { + return unsupported("an unknown operator attribute"); + } + let this = ops.len() + 1; + let lookup = |name: &str| -> Result { + ids.get(name).copied().ok_or_else(|| NetError::Invalid(format!("`{}` reads `{name}`, which nothing makes", node.op_type))) + }; + // a LeakyRelu straight after this layer, which nothing else reads, is part of it + let mut fuse_leaky = |out: &str| -> Result<(Option, String), NetError> { + if let Some(next) = nodes.get(i) + && next.op_type == "LeakyRelu" + && next.input.first().map(String::as_str) == Some(out) + && readers.get(out).copied() == Some(1) + { + if next.input.len() != 1 || next.output.len() != 1 || next.attribute.iter().any(|a| a.name != "alpha") { + return invalid("malformed LeakyRelu"); + } + let alpha = attr(next, "alpha").map_or(0.01, |a| a.f); + i += 1; + let name = next.output.first().cloned().ok_or_else(|| NetError::Invalid("LeakyRelu without an output".into()))?; + return Ok((Some(alpha), name)); + } + Ok((None, out.to_string())) + }; + match node.op_type.as_str() { + "Concat" => { + if attr(node, "axis").map(|a| a.i) != Some(1) { + return unsupported("Concat on another axis than the channels"); + } + let [a, b] = node.input.as_slice() else { return unsupported("Concat of other than two tensors") }; + let out = node.output.first().ok_or_else(|| NetError::Invalid("Concat without an output".into()))?; + if readers.get(out.as_str()).copied() != Some(1) { + return unsupported("a Concat that is read by more than one layer"); + } + joins.insert(out.clone(), (lookup(a)?, lookup(b)?)); + } + "Conv" => { + let (w, b) = weights_of(node, &initialisers, &mut weights_total)?; + let (cout, cin, kh, kw) = four(&w.0)?; + let b = if b.is_empty() { vec![0.0; cout] } else { b }; + if kh != kw || !(kh == 1 || kh == 3) { + return unsupported(format!("a {kh} × {kw} convolution")); + } + if ints_or(node, "strides", &[1, 1]) != [1, 1] + || ints_or(node, "dilations", &[1, 1]) != [1, 1] + || attr(node, "group").is_some_and(|a| a.i != 1) + { + return unsupported("a convolution with strides, dilation or groups"); + } + if attr(node, "kernel_shape").is_some_and(|a| a.ints != [kh as i64, kw as i64]) { + return invalid("kernel shape does not match the weights"); + } + let pad = (kh as i64 - 1) / 2; + if ints_or(node, "pads", &[0; 4]) != [pad; 4] || attr(node, "auto_pad").is_some_and(|a| !a.s.is_empty() && a.s != b"NOTSET") { + return unsupported("a convolution that is not padded to keep its size"); + } + let src_name = node.input.first().ok_or_else(|| NetError::Invalid("Conv without an input".into()))?; + let (src, src2) = match joins.remove(src_name) { + Some((a, b)) => (a, Some(b)), + None => (lookup(src_name)?, None), + }; + let out = node.output.first().ok_or_else(|| NetError::Invalid("Conv without an output".into()))?; + let (leaky, out_name) = fuse_leaky(out)?; + ids.insert(out_name, this); + ops.push(Op::Conv(Conv { src, src2, cin, cout, k: kh, weight: w.1, bias: b, leaky })); + } + "ConvTranspose" => { + let (w, b) = weights_of(node, &initialisers, &mut weights_total)?; + let (cin, cout, kh, kw) = four(&w.0)?; + let b = if b.is_empty() { vec![0.0; cout] } else { b }; + if (kh, kw) != (2, 2) + || ints_or(node, "strides", &[1, 1]) != [2, 2] + || ints_or(node, "pads", &[0; 4]) != [0; 4] + || ints_or(node, "dilations", &[1, 1]) != [1, 1] + || attr(node, "group").is_some_and(|a| a.i != 1) + || attr(node, "output_padding").is_some_and(|a| a.ints.iter().any(|&v| v != 0)) + { + return unsupported("a transposed convolution other than 2 × 2 with stride 2"); + } + if attr(node, "auto_pad").is_some_and(|a| !a.s.is_empty() && a.s != b"NOTSET") { + return unsupported("transposed convolution auto padding"); + } + if attr(node, "kernel_shape").is_some_and(|a| a.ints != [2, 2]) { + return invalid("transposed kernel shape does not match the weights"); + } + let src = lookup(node.input.first().map_or("", String::as_str))?; + let out = node.output.first().ok_or_else(|| NetError::Invalid("ConvTranspose without an output".into()))?; + let (leaky, out_name) = fuse_leaky(out)?; + ids.insert(out_name, this); + ops.push(Op::ConvT2(ConvT2 { src, cin, cout, weight: w.1, bias: b, leaky })); + } + "MaxPool" => { + if ints_or(node, "kernel_shape", &[]) != [2, 2] + || ints_or(node, "strides", &[1, 1]) != [2, 2] + || ints_or(node, "pads", &[0; 4]) != [0; 4] + || ints_or(node, "dilations", &[1, 1]) != [1, 1] + || attr(node, "ceil_mode").is_some_and(|a| a.i != 0) + || attr(node, "storage_order").is_some_and(|a| a.i != 0) + || attr(node, "auto_pad").is_some_and(|a| !a.s.is_empty() && a.s != b"NOTSET") + { + return unsupported("a max-pool other than 2 × 2 with stride 2"); + } + let src = lookup(node.input.first().map_or("", String::as_str))?; + let out = node.output.first().ok_or_else(|| NetError::Invalid("MaxPool without an output".into()))?; + ids.insert(out.clone(), this); + ops.push(Op::MaxPool2 { src }); + } + "DepthToSpace" => { + if attr(node, "blocksize").map(|a| a.i) != Some(2) || attr(node, "mode").map(|a| a.s.as_slice()) != Some(&b"CRD"[..]) { + return unsupported("depth-to-space other than block size 2 in CRD order"); + } + let src = lookup(node.input.first().map_or("", String::as_str))?; + let out = node.output.first().ok_or_else(|| NetError::Invalid("DepthToSpace without an output".into()))?; + ids.insert(out.clone(), this); + ops.push(Op::DepthToSpace2 { src }); + } + other => return unsupported(format!("the `{other}` operator")), + } + } + if !joins.is_empty() { + return unsupported("a Concat whose result no convolution reads"); + } + if ids.get(output.name.as_str()) != Some(&ops.len()) { + return unsupported("the output is not the last layer"); + } + Net::new(in_channels, ops) +} + +fn attr<'a>(n: &'a NodeProto, name: &str) -> Option<&'a AttributeProto> { + n.attribute.iter().find(|a| a.name == name) +} + +/// The integer list attribute `name`, or `default` when the node does not have it. +fn ints_or(n: &NodeProto, name: &str, default: &[i64]) -> Vec { + attr(n, name).map_or_else(|| default.to_vec(), |a| a.ints.clone()) +} + +fn four(dims: &[i64]) -> Result<(usize, usize, usize, usize), NetError> { + match dims { + [a, b, c, d] if dims.iter().all(|&v| v > 0 && v <= 1 << 20) => Ok((*a as usize, *b as usize, *c as usize, *d as usize)), + _ => invalid("a weight tensor that is not four-dimensional"), + } +} + +type Weights = ((Vec, Vec), Vec); + +/// The weight and bias initialisers of a convolution node. +fn weights_of(node: &NodeProto, initialisers: &HashMap<&str, &TensorProto>, total: &mut usize) -> Result { + let mut get = |idx: usize| -> Result<(Vec, Vec), NetError> { + let name = node.input.get(idx).ok_or_else(|| NetError::Invalid(format!("`{}` without its weights", node.op_type)))?; + let t = initialisers.get(name.as_str()).ok_or_else(|| NetError::Unsupported(format!("weights that are computed (`{name}`)")))?; + let values = floats(t)?; + *total = total.saturating_add(values.len()); + if *total > MAX_WEIGHTS { + return invalid("the network has too many weights"); + } + Ok((t.dims.clone(), values)) + }; + let w = get(1)?; + let b = if node.input.len() > 2 { + let (dims, values) = get(2)?; + if dims.len() != 1 || values.is_empty() { + return invalid("a bias tensor must be a nonempty vector"); + } + values + } else { + Vec::new() + }; + Ok((w, b)) +} + +fn floats(t: &TensorProto) -> Result, NetError> { + if t.data_type != 1 { + return unsupported("weights that are not 32-bit floats"); + } + let want: usize = t + .dims + .iter() + .try_fold(1usize, |n, &d| usize::try_from(d).ok().and_then(|d| n.checked_mul(d))) + .ok_or_else(|| NetError::Invalid("a tensor with a negative or huge size".into()))?; + if want > MAX_WEIGHTS { + return invalid("a tensor with too many numbers"); + } + if !t.raw_data.is_empty() && (!t.float_data.is_empty() || !t.raw_data.len().is_multiple_of(4)) { + return invalid("ambiguous or truncated tensor data"); + } + let actual = if t.raw_data.is_empty() { t.float_data.len() } else { t.raw_data.len() / 4 }; + if actual != want { + return invalid("tensor data does not match its dimensions"); + } + let mut values = Vec::new(); + values.try_reserve_exact(want).map_err(|_| NetError::Invalid("not enough memory for weights".into()))?; + if !t.raw_data.is_empty() { + values.extend(t.raw_data.as_chunks::<4>().0.iter().map(|b| f32::from_le_bytes(*b))); + } else { + values.extend_from_slice(&t.float_data); + } + if values.len() != want { + return invalid(format!("tensor `{}` has {} numbers, not the {want} its shape says", t.name, values.len())); + } + Ok(values) +} + +/// The channel count in an input's declared shape `[batch, channels, …]`. +fn channel_dim(v: &ValueInfoProto) -> Option { + v.channels +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::synthetic::unet_onnx; + + fn temp_model(name: &str, bytes: &[u8]) -> std::path::PathBuf { + let p = std::env::temp_dir().join(format!("lc-denoise-onnx-{name}-{}.onnx", std::process::id())); + std::fs::write(&p, bytes).unwrap(); + p + } + + #[test] + fn a_u_net_is_read_with_its_layers_fused() { + let (tile, depth) = (32usize, 2usize); + let p = temp_model("unet", &unet_onnx(tile as u64, 8, depth, 7)); + let net = read(&p).unwrap(); + // per level two convolutions; a pool per level but the last; going up a transposed convolution and two convolutions + // per level (the join is part of the first); then the 1 × 1 convolution and depth-to-space + assert_eq!(net.ops().len(), 2 * (depth + 1) + depth + depth * 3 + 2); + assert!(matches!(net.ops().last(), Some(crate::net::Op::DepthToSpace2 { .. }))); + assert!(net.ops().iter().any(|o| matches!(o, crate::net::Op::Conv(c) if c.src2.is_some())), "the joins became two-source convolutions"); + assert!( + net.ops().iter().all(|o| !matches!(o, crate::net::Op::Conv(c) if c.leaky.is_none() && c.k == 3)), + "every 3 × 3 convolution has its activation" + ); + assert_eq!((net.in_channels, net.out_channels(), net.depth()), (4, 3, 2)); + assert!(net.fits_tile(tile) && !net.fits_tile(30)); + + let _ = std::fs::remove_file(p); + } + + #[test] + fn an_explicit_empty_or_nonvector_bias_is_not_treated_as_missing() { + let w = TensorProto { name: "w".into(), dims: vec![12, 4, 1, 1], data_type: 1, float_data: vec![0.1; 48], ..Default::default() }; + let n = NodeProto { input: vec!["data".into(), "w".into(), "b".into()], ..Default::default() }; + for dims in [vec![0], vec![3, 4]] { + let b = TensorProto { + name: "b".into(), + float_data: vec![0.0; dims.iter().product::() as usize], + dims, + data_type: 1, + ..Default::default() + }; + let inits = HashMap::from([("w", &w), ("b", &b)]); + assert!(weights_of(&n, &inits, &mut 0).is_err()); + } + } + + #[test] + fn truncated_and_mutated_protobuf_is_bounded_and_never_panics() { + let good = unet_onnx(4, 3, 1, 7); + for n in (0..good.len()).step_by(31) { + let _ = read_bytes(&good[..n]); + } + for at in (0..good.len()).step_by(37) { + let mut bad = good.clone(); + bad[at] ^= 0xff; + let _ = read_bytes(&bad); + } + assert!(read_bytes(&[0x3a, 0xff, 0xff, 0xff, 0xff, 0x7f]).is_err()); + assert!(read_bytes(&[0x3a, 0, 0x3a, 0]).is_err()); + } + proptest::proptest! { + #[test] + fn arbitrary_model_bytes_never_panic(bytes in proptest::collection::vec(proptest::prelude::any::(), 0..4096)) { + let _ = read_bytes(&bytes); + } + } + + #[test] + fn what_the_description_cannot_say_is_unsupported_and_garbage_is_an_error() { + // not a model + let p = temp_model("junk", b"this is not an onnx file"); + assert!(matches!(read(&p), Err(NetError::Invalid(_)) | Err(NetError::Unsupported(_)))); + let _ = std::fs::remove_file(p); + // a missing file + assert!(read(Path::new("/does/not/exist.onnx")).is_err()); + } +} diff --git a/crates/denoise/src/onnx_proto.rs b/crates/denoise/src/onnx_proto.rs new file mode 100644 index 00000000..d8bbba41 --- /dev/null +++ b/crates/denoise/src/onnx_proto.rs @@ -0,0 +1,295 @@ +//! Bounded, borrowed decoding of the ONNX fields used by denoise, from the public Apache-2.0 schema: +//! https://github.com/onnx/onnx/blob/main/onnx/onnx.proto3 +//! No generated code, protoc, recursive graph decoding, or external tensor files. + +use crate::net::{MAX_OPS, MAX_WEIGHTS, NetError}; + +const MAX_ITEMS: usize = MAX_OPS * 4; +const MAX_STRING: usize = 4096; +pub(super) const MAX_BYTES: usize = MAX_WEIGHTS * 4 + 16 * 1024 * 1024; + +type Result = std::result::Result; +fn bad() -> NetError { + NetError::Invalid("malformed or oversized ONNX protobuf".into()) +} + +struct Fields<'a> { + rest: &'a [u8], + count: usize, +} +enum Wire<'a> { + Int(u64), + Bytes(&'a [u8]), + F32(f32), + Other, +} + +fn varint(rest: &mut &[u8]) -> Result { + let mut value = 0u64; + for shift in (0..70).step_by(7) { + let (b, tail) = rest.split_first().ok_or_else(bad)?; + *rest = tail; + if shift == 63 && *b > 1 { + return Err(bad()); + } + value |= u64::from(*b & 127) << shift; + if b & 128 == 0 { + return Ok(value); + } + } + Err(bad()) +} + +impl<'a> Fields<'a> { + fn new(rest: &'a [u8]) -> Self { + Self { rest, count: 0 } + } + fn take(&mut self, n: usize) -> Result<&'a [u8]> { + let out = self.rest.get(..n).ok_or_else(bad)?; + self.rest = self.rest.get(n..).ok_or_else(bad)?; + Ok(out) + } + fn next(&mut self) -> Result)>> { + if self.rest.is_empty() { + return Ok(None); + } + self.count += 1; + if self.count > MAX_WEIGHTS { + return Err(bad()); + } + let tag = varint(&mut self.rest)?; + let number = u32::try_from(tag >> 3).map_err(|_| bad())?; + if number == 0 || number >= 1 << 29 { + return Err(bad()); + } + let wire = match tag & 7 { + 0 => Wire::Int(varint(&mut self.rest)?), + 1 => { + self.take(8)?; + Wire::Other + } + 2 => { + let n = usize::try_from(varint(&mut self.rest)?).map_err(|_| bad())?; + Wire::Bytes(self.take(n)?) + } + 5 => { + let b: [u8; 4] = self.take(4)?.try_into().map_err(|_| bad())?; + Wire::F32(f32::from_le_bytes(b)) + } + _ => return Err(bad()), + }; + Ok(Some((number, wire))) + } +} + +fn string(b: &[u8]) -> Result { + if b.len() > MAX_STRING { + return Err(bad()); + } + std::str::from_utf8(b).map(str::to_owned).map_err(|_| bad()) +} +fn push(v: &mut Vec, value: T, cap: usize) -> Result<()> { + if v.len() >= cap { + return Err(bad()); + } + v.try_reserve(1).map_err(|_| bad())?; + v.push(value); + Ok(()) +} +fn ints(w: Wire<'_>, out: &mut Vec, cap: usize) -> Result<()> { + match w { + Wire::Int(v) => push(out, v as i64, cap), + Wire::Bytes(mut b) => { + while !b.is_empty() { + let v = varint(&mut b)?; + push(out, v as i64, cap)?; + } + Ok(()) + } + _ => Err(bad()), + } +} + +#[derive(Default)] +pub(super) struct AttributeProto { + pub name: String, + pub f: f32, + pub i: i64, + pub s: Vec, + pub ints: Vec, +} +#[derive(Default)] +pub(super) struct NodeProto { + pub input: Vec, + pub output: Vec, + pub op_type: String, + pub attribute: Vec, +} +#[derive(Default)] +pub(super) struct TensorProto<'a> { + pub name: String, + pub dims: Vec, + pub data_type: i32, + pub raw_data: &'a [u8], + pub float_data: Vec, +} +pub(super) struct ValueInfoProto { + pub name: String, + pub channels: Option, +} +#[derive(Default)] +pub(super) struct Graph<'a> { + pub node: Vec, + pub initializer: Vec>, + pub input: Vec, + pub output: Vec, +} + +fn attribute(b: &[u8]) -> Result { + let mut a = AttributeProto::default(); + let mut fields = Fields::new(b); + while let Some((n, w)) = fields.next()? { + match (n, w) { + (1, Wire::Bytes(b)) => a.name = string(b)?, + (2, Wire::F32(v)) => a.f = v, + (3, Wire::Int(v)) => a.i = v as i64, + (4, Wire::Bytes(b)) => { + if b.len() > MAX_STRING { + return Err(bad()); + } + a.s = b.to_vec(); + } + (8, w) => ints(w, &mut a.ints, 16)?, + (20, Wire::Int(1 | 2 | 3 | 7)) => {} + _ => return Err(NetError::Unsupported("unknown attribute value".into())), + } + } + Ok(a) +} +fn node(b: &[u8]) -> Result { + let mut a = NodeProto::default(); + let mut fields = Fields::new(b); + while let Some((n, w)) = fields.next()? { + match (n, w) { + (1, Wire::Bytes(b)) => push(&mut a.input, string(b)?, 4)?, + (2, Wire::Bytes(b)) => push(&mut a.output, string(b)?, 2)?, + (4, Wire::Bytes(b)) => a.op_type = string(b)?, + (5, Wire::Bytes(b)) => push(&mut a.attribute, attribute(b)?, 16)?, + (7, Wire::Bytes(b)) if !b.is_empty() => return Err(NetError::Unsupported("a custom operator domain".into())), + _ => {} + } + } + Ok(a) +} +fn tensor(b: &[u8]) -> Result> { + let mut a = TensorProto::default(); + let mut fields = Fields::new(b); + while let Some((n, w)) = fields.next()? { + match (n, w) { + (1, w) => ints(w, &mut a.dims, 8)?, + (2, Wire::Int(v)) => a.data_type = i32::try_from(v).map_err(|_| bad())?, + (4, Wire::F32(v)) => push(&mut a.float_data, v, MAX_WEIGHTS)?, + (4, Wire::Bytes(b)) => { + if !b.len().is_multiple_of(4) { + return Err(bad()); + } + for chunk in b.as_chunks::<4>().0 { + push(&mut a.float_data, f32::from_le_bytes(*chunk), MAX_WEIGHTS)?; + } + } + (8, Wire::Bytes(b)) => a.name = string(b)?, + (9, Wire::Bytes(b)) => { + if b.len() > MAX_WEIGHTS * 4 { + return Err(bad()); + } + a.raw_data = b; + } + (13, _) | (14, Wire::Int(1..)) => return Err(NetError::Unsupported("external tensor data".into())), + _ => {} + } + } + Ok(a) +} +fn channel_shape(b: &[u8]) -> Result> { + let mut fields = Fields::new(b); + let mut dim = 0; + let mut channels = None; + while let Some((n, w)) = fields.next()? { + if let (1, Wire::Bytes(b)) = (n, w) { + if dim >= 8 { + return Err(bad()); + } + let mut df = Fields::new(b); + while let Some((n, w)) = df.next()? { + if let (1, Wire::Int(v)) = (n, w) + && dim == 1 + { + channels = usize::try_from(v).ok().filter(|&v| v > 0); + } + } + dim += 1; + } + } + Ok(channels) +} +fn channel_type(b: &[u8]) -> Result> { + let mut fields = Fields::new(b); + while let Some((n, w)) = fields.next()? { + if let (1, Wire::Bytes(b)) = (n, w) { + let mut tf = Fields::new(b); + let (mut elem, mut channels) = (0, None); + while let Some((n, w)) = tf.next()? { + match (n, w) { + (1, Wire::Int(v)) => elem = v, + (2, Wire::Bytes(b)) => channels = channel_shape(b)?, + _ => {} + } + } + return Ok(if elem == 1 { channels } else { None }); + } + } + Ok(None) +} +fn value_info(b: &[u8]) -> Result { + let mut out = ValueInfoProto { name: String::new(), channels: None }; + let mut fields = Fields::new(b); + while let Some((n, w)) = fields.next()? { + match (n, w) { + (1, Wire::Bytes(b)) => out.name = string(b)?, + (2, Wire::Bytes(b)) => out.channels = channel_type(b)?, + _ => {} + } + } + Ok(out) +} +fn graph(b: &[u8]) -> Result> { + let mut out = Graph::default(); + let mut fields = Fields::new(b); + while let Some((n, w)) = fields.next()? { + match (n, w) { + (1, Wire::Bytes(b)) => push(&mut out.node, node(b)?, MAX_ITEMS)?, + (5, Wire::Bytes(b)) => push(&mut out.initializer, tensor(b)?, MAX_ITEMS)?, + (11, Wire::Bytes(b)) => push(&mut out.input, value_info(b)?, MAX_ITEMS)?, + (12, Wire::Bytes(b)) => push(&mut out.output, value_info(b)?, 1)?, + (15, _) => return Err(NetError::Unsupported("sparse initializers".into())), + _ => {} + } + } + Ok(out) +} +pub(super) fn decode(b: &[u8]) -> Result> { + if b.len() > MAX_BYTES { + return Err(bad()); + } + let mut fields = Fields::new(b); + let mut out = None; + while let Some((n, w)) = fields.next()? { + if let (7, Wire::Bytes(b)) = (n, w) { + if out.is_some() { + return Err(bad()); + } + out = Some(graph(b)?); + } + } + out.ok_or_else(bad) +} diff --git a/crates/denoise/src/reference.rs b/crates/denoise/src/reference.rs new file mode 100644 index 00000000..ef58b5b9 --- /dev/null +++ b/crates/denoise/src/reference.rs @@ -0,0 +1,133 @@ +//! A plain-loop interpreter of a [`Net`], slow and obviously right: the reference the parser and the GPU runner are +//! tested against (tract's answer on the real model is the other). It is not used on photos. + +use crate::net::{Net, NetError, Op, TensorId}; +use crate::run::{Error, TileRunner}; + +/// A tensor in `[channels][side][side]` order. +struct T { + channels: usize, + side: usize, + data: Vec, +} + +/// Run `net` on `input` (`in_channels` planes of `tile × tile`). +pub fn run(net: &Net, tile: usize, input: &[f32]) -> Result, NetError> { + if !net.fits_tile(tile) || input.len() != net.in_channels * tile * tile { + return Err(NetError::Invalid("the tile does not fit the network".into())); + } + let mut tensors: Vec = vec![T { channels: net.in_channels, side: tile, data: input.to_vec() }]; + let get = |tensors: &[T], id: TensorId| -> Result { + if id < tensors.len() { Ok(id) } else { Err(NetError::Invalid("a layer reads a tensor that does not exist yet".into())) } + }; + for op in net.ops() { + let out = match op { + Op::Conv(c) => { + let a = get(&tensors, c.src)?; + let b = c.src2.map(|s| get(&tensors, s)).transpose()?; + let side = tensors[a].side; + let pad = (c.k - 1) / 2; + let mut data = vec![0f32; c.cout * side * side]; + // the input channels in order: the first source's, then the second's + let planes: Vec<&[f32]> = + std::iter::once(&tensors[a]).chain(b.map(|b| &tensors[b])).flat_map(|t| t.data.chunks(side * side)).collect(); + for co in 0..c.cout { + for y in 0..side { + for x in 0..side { + let mut sum = c.bias[co]; + for (ci, plane) in planes.iter().enumerate() { + for ky in 0..c.k { + for kx in 0..c.k { + let (yy, xx) = ((y + ky) as isize - pad as isize, (x + kx) as isize - pad as isize); + if yy < 0 || xx < 0 || yy >= side as isize || xx >= side as isize { + continue; + } + sum += plane[yy as usize * side + xx as usize] * c.weight[((co * c.cin + ci) * c.k + ky) * c.k + kx]; + } + } + } + data[(co * side + y) * side + x] = leaky(sum, c.leaky); + } + } + } + T { channels: c.cout, side, data } + } + Op::ConvT2(c) => { + let a = get(&tensors, c.src)?; + let side = tensors[a].side; + let out_side = 2 * side; + let mut data = vec![0f32; c.cout * out_side * out_side]; + for co in 0..c.cout { + for y in 0..side { + for x in 0..side { + for dy in 0..2 { + for dx in 0..2 { + let mut sum = c.bias[co]; + for ci in 0..c.cin { + sum += tensors[a].data[(ci * side + y) * side + x] * c.weight[((ci * c.cout + co) * 2 + dy) * 2 + dx]; + } + data[(co * out_side + 2 * y + dy) * out_side + 2 * x + dx] = leaky(sum, c.leaky); + } + } + } + } + } + T { channels: c.cout, side: out_side, data } + } + Op::MaxPool2 { src } => { + let a = get(&tensors, *src)?; + let (channels, side) = (tensors[a].channels, tensors[a].side / 2); + let mut data = vec![0f32; channels * side * side]; + for ch in 0..channels { + for y in 0..side { + for x in 0..side { + let at = |dy: usize, dx: usize| tensors[a].data[(ch * 2 * side + 2 * y + dy) * 2 * side + 2 * x + dx]; + data[(ch * side + y) * side + x] = at(0, 0).max(at(0, 1)).max(at(1, 0)).max(at(1, 1)); + } + } + } + T { channels, side, data } + } + Op::DepthToSpace2 { src } => { + let a = get(&tensors, *src)?; + let (channels, side) = (tensors[a].channels / 4, tensors[a].side); + let out_side = 2 * side; + let mut data = vec![0f32; channels * out_side * out_side]; + for ch in 0..channels { + for y in 0..side { + for x in 0..side { + for dy in 0..2 { + for dx in 0..2 { + data[(ch * out_side + 2 * y + dy) * out_side + 2 * x + dx] = + tensors[a].data[((4 * ch + 2 * dy + dx) * side + y) * side + x]; + } + } + } + } + } + T { channels, side: out_side, data } + } + }; + tensors.push(out); + } + tensors.pop().map(|t| t.data).ok_or_else(|| NetError::Invalid("no layers".into())) +} + +fn leaky(v: f32, alpha: Option) -> f32 { + match alpha { + Some(a) if v < 0.0 => v * a, + _ => v, + } +} + +/// The reference as a [`TileRunner`] (for tests of the code around the model). +pub struct ReferenceRunner { + pub net: Net, + pub tile: usize, +} + +impl TileRunner for ReferenceRunner { + fn run(&self, input: &[f32]) -> Result, Error> { + run(&self.net, self.tile, input).map_err(|e| Error::Runtime(e.to_string())) + } +} diff --git a/crates/denoise/src/runtime.rs b/crates/denoise/src/runtime.rs new file mode 100644 index 00000000..117f6775 --- /dev/null +++ b/crates/denoise/src/runtime.rs @@ -0,0 +1,433 @@ +//! Model loading, a shared installation self-test, and deterministic test tiles. +//! `CpuRunner` is the pure-Rust executor; scalar operator tests live in `reference`. + +use std::path::Path; + +use web_time::Instant; + +use serde::Serialize; + +use crate::manifest::{DenoiserManifest, Gain}; +use crate::run::{Error, TileRunner}; + +/// What `self_test` found. +#[derive(Clone, Debug, PartialEq, Serialize)] +#[serde(rename_all = "camelCase")] +pub struct SelfTest { + pub ok: bool, + /// Milliseconds to load and optimise the model. + pub load_ms: f64, + /// Median milliseconds to run one tile. + pub tile_ms: f64, + /// Megapixels of picture one tile covers, for estimating a whole photo's time. + pub tile_megapixels: f64, + /// The model's output scale relative to its input on the test picture (what `Gain::MatchMean` takes out). + pub scale: f64, + /// Each check: what was checked and whether it held. + pub checks: Vec<(String, bool)>, +} + +/// The default pure-Rust CPU model, with the same manifest and installation self-test as the reference. +#[derive(Clone)] +pub struct CpuRunner { + runner: crate::cpu::NetRunner, + manifest: DenoiserManifest, + load_ms: f64, +} +impl CpuRunner { + pub fn load(path: &Path, manifest: &DenoiserManifest) -> Result { + crate::manifest::validate(manifest).map_err(|e| Error::Load(e.to_string()))?; + let started = Instant::now(); + let net = crate::onnx::read(path).map_err(|e| Error::Load(e.to_string()))?; + let runner = crate::cpu::NetRunner::new(&net, manifest.tile as usize).map_err(|e| Error::Load(e.to_string()))?; + Ok(Self { runner, manifest: manifest.clone(), load_ms: started.elapsed().as_secs_f64() * 1e3 }) + } + pub fn manifest(&self) -> &DenoiserManifest { + &self.manifest + } + pub fn net(&self) -> &crate::net::Net { + self.runner.net() + } + pub fn bytes_per_tile(&self) -> usize { + self.runner.bytes_per_tile() + } + pub fn self_test(&self) -> SelfTest { + self_test(self, &self.manifest, self.load_ms) + } +} +impl TileRunner for CpuRunner { + fn run(&self, input: &[f32]) -> Result, Error> { + self.runner.run(input) + } +} +fn self_test(runner: &dyn TileRunner, manifest: &DenoiserManifest, load_ms: f64) -> SelfTest { + let t = manifest.tile as usize; + let mut checks: Vec<(String, bool)> = Vec::new(); + let (input, base) = test_tile(t); + let (mut times, mut scale) = (Vec::new(), 0.0f64); + let mut run = || { + let started = Instant::now(); + let r = runner.run(&input); + times.push(started.elapsed().as_secs_f64() * 1000.0); + r + }; + let (a, b) = (run(), run()); + match (&a, &b) { + (Ok(a), Ok(b)) => { + checks.push((format!("gives {} finite numbers per tile", a.len()), true)); + checks.push(( + "gives the same answer for the same tile".into(), + a.iter().zip(b).all(|(x, y)| (x - y).abs() <= 1e-4 * x.abs().max(y.abs()).max(1e-9)), + )); + let side = 2 * t; + let mean_in = base.iter().map(|&v| f64::from(v)).sum::() / base.len().max(1) as f64; + let mean_out = a.iter().map(|&v| f64::from(v)).sum::() / a.len().max(1) as f64; + scale = if mean_in > 0.0 { mean_out / mean_in } else { 0.0 }; + if let Gain::MatchMean { nominal, max_deviation } = manifest.gain { + let ratio = scale / f64::from(nominal); + let tolerance = f64::from(max_deviation).max(0.05) * 4.0; + checks.push((format!("its scale is the {nominal:.3e} the description says (found {scale:.3e})"), (ratio - 1.0).abs() <= tolerance)); + } + // noise left after bringing the answer back to the input's scale, against the noise that went in + let g = if scale > 0.0 { 1.0 / scale } else { 1.0 }; + let (mut noise_in, mut noise_out, mut n_in, mut n_out) = (0.0f64, 0.0f64, 0usize, 0usize); + for (i, (&v, &s)) in input.iter().zip(base.iter().cycle()).enumerate() { + // centre of each plane only: edges are the model's guess + let (x, y) = ((i % (t * t)) % t, (i % (t * t)) / t); + if x >= t / 4 && x < 3 * t / 4 && y >= t / 4 && y < 3 * t / 4 { + noise_in += f64::from(v - s).powi(2); + n_in += 1; + } + } + for c in 0..3 { + for y in side / 4..3 * side / 4 { + for x in side / 4..3 * side / 4 { + if let (Some(&o), Some(&s)) = (a.get(c * side * side + y * side + x), base.get((y / 2) * t + x / 2)) { + noise_out += (f64::from(o) * g - f64::from(s)).powi(2); + n_out += 1; + } + } + } + } + let (rms_in, rms_out) = ((noise_in / n_in.max(1) as f64).sqrt(), (noise_out / n_out.max(1) as f64).sqrt()); + checks.push((format!("takes noise down (from {rms_in:.4} to {rms_out:.4})"), rms_out < 0.8 * rms_in)); + } + (Err(e), _) | (_, Err(e)) => checks.push((format!("runs: {e}"), false)), + } + times.sort_by(|x, y| x.total_cmp(y)); + SelfTest { + ok: !checks.is_empty() && checks.iter().all(|(_, ok)| *ok), + load_ms, + tile_ms: times.get(times.len() / 2).copied().unwrap_or(0.0), + tile_megapixels: (2 * t * 2 * t) as f64 / 1e6, + scale, + checks, + } +} + +/// A deterministic test tile: four planes of a smooth gradient with noise on top, and the gradient alone for one +/// plane (`tile × tile`). +pub fn test_tile(t: usize) -> (Vec, Vec) { + if t == 0 || t > 2048 { + return (Vec::new(), Vec::new()); + } + let mut seed = 0x2545_f491u32; + let mut next = move || { + seed ^= seed << 13; + seed ^= seed >> 17; + seed ^= seed << 5; + (seed >> 8) as f32 / (1u32 << 24) as f32 + }; + let base: Vec = (0..t * t).map(|i| 0.12 + 0.3 * (((i % t) + (i / t)) as f32 / (2 * t) as f32)).collect(); + let input = (0..4 * t * t) + .map(|i| { + // sum of four uniforms: close to Gaussian, deviation 0.0577 × 2 before scaling + let n = next() + next() + next() + next() - 2.0; + base.get(i % (t * t)).copied().unwrap_or(0.0) + 0.05 * n + }) + .collect(); + (input, base) +} + +/// A tile for checking that two runners of one model agree: the whole range a raw photo has rather than the test tile's +/// mid-greys — a noisy ramp from black to white across the tile, a flat black corner and a clipped white one (four +/// planes of `tile × tile`). +pub fn check_tile(t: usize) -> Vec { + if t == 0 || t > 2048 { + return Vec::new(); + } + let mut seed = 0x9e37_79b9u32; + let mut next = move || { + seed ^= seed << 13; + seed ^= seed >> 17; + seed ^= seed << 5; + (seed >> 8) as f32 / (1u32 << 24) as f32 + }; + let (edge, span) = ((t / 8).max(1), (2 * t).saturating_sub(2).max(1) as f32); + (0..4 * t * t) + .map(|i| { + let (x, y) = ((i % (t * t)) % t, (i % (t * t)) / t); + let n = next() + next() + next() + next() - 2.0; + if x < edge && y < edge { + 0.0 + } else if x + edge >= t && y + edge >= t { + 1.0 + } else { + ((x + y) as f32 / span + 0.05 * n).clamp(0.0, 1.0) + } + }) + .collect() +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::licence::Licence; + use crate::manifest::Domain; + use crate::synthetic_proto::{len_field, value_info_bytes}; + + fn varint(mut v: u64, out: &mut Vec) { + loop { + let b = (v & 0x7f) as u8; + v >>= 7; + if v == 0 { + out.push(b); + return; + } + out.push(b | 0x80); + } + } + + fn varint_field(n: u64, v: u64, out: &mut Vec) { + varint(n << 3, out); + varint(v, out); + } + + fn node(op: &str, inputs: &[&str], output: &str, ints: &[(&str, i64)]) -> Vec { + let mut n = Vec::new(); + for i in inputs { + len_field(1, i.as_bytes(), &mut n); + } + len_field(2, output.as_bytes(), &mut n); + len_field(4, op.as_bytes(), &mut n); + for (name, value) in ints { + let mut a = Vec::new(); + len_field(1, name.as_bytes(), &mut a); + varint_field(3, *value as u64, &mut a); + varint_field(20, 2, &mut a); + len_field(5, &a, &mut n); + } + n + } + + fn tensor(name: &str, dims: &[u64], values: &[f32]) -> Vec { + let mut t = Vec::new(); + for d in dims { + varint_field(1, *d, &mut t); + } + varint_field(2, 1, &mut t); + len_field(8, name.as_bytes(), &mut t); + let raw: Vec = values.iter().flat_map(|v| v.to_le_bytes()).collect(); + len_field(9, &raw, &mut t); + t + } + + /// A tiny working denoiser-shaped network: `[1, 4, t, t]` → a 1 × 1 convolution to 12 channels (each output + /// channel is `scale` times one input plane, so the scale is known) → depth-to-space → `[1, 3, 2t, 2t]`. + fn tiny_model(t: u64, scale: f32) -> Vec { + tiny_model_with(t, scale, 12) + } + + /// The same with another number of convolution channels; only 12 can be arranged into the three colours. + fn tiny_model_with(t: u64, scale: f32, channels: u64) -> Vec { + let mut graph = Vec::new(); + len_field(1, &node("Conv", &["data", "w"], if channels == 12 { "c" } else { "rgb" }, &[("group", 1)]), &mut graph); + if channels == 12 { + let mut depth_to_space = node("DepthToSpace", &["c"], "rgb", &[("blocksize", 2)]); + let mut mode = Vec::new(); + len_field(1, b"mode", &mut mode); + len_field(4, b"CRD", &mut mode); + varint_field(20, 3, &mut mode); + len_field(5, &mode, &mut depth_to_space); + len_field(1, &depth_to_space, &mut graph); + } + // CRD output channel colour·4 + dy·2 + dx reads input plane [R, G1, B] of the tile. + let mut w = vec![0f32; channels as usize * 4]; + for oc in 0..channels as usize { + let colour = (oc / 4).min(2); + let plane = [0, 1, 3][colour]; + w[oc * 4 + plane] = scale; + } + len_field(5, &tensor("w", &[channels, 4, 1, 1], &w), &mut graph); + len_field(11, &value_info_bytes("data", 1, &[Ok(1), Ok(4), Ok(t), Ok(t)]), &mut graph); + len_field(12, &value_info_bytes("rgb", 1, &[Ok(1), Ok(3), Ok(2 * t), Ok(2 * t)]), &mut graph); + let mut ops = Vec::new(); + len_field(1, b"", &mut ops); + varint_field(2, 13, &mut ops); + let mut model = Vec::new(); + varint_field(1, 8, &mut model); + len_field(7, &graph, &mut model); + len_field(8, &ops, &mut model); + model + } + + fn manifest(tile: u32, nominal: f32) -> DenoiserManifest { + DenoiserManifest { + id: "tiny".into(), + name: "Tiny".into(), + version: "1".into(), + licence: Licence::default(), + source: None, + sha256: None, + size_bytes: None, + provenance: String::new(), + domain: Domain::BayerToRgb, + tile, + overlap: 16, + gain: Gain::MatchMean { nominal, max_deviation: 0.05 }, + } + } + + fn temp_model(name: &str, bytes: &[u8]) -> std::path::PathBuf { + let p = std::env::temp_dir().join(format!("lc-denoise-runtime-{name}-{}.onnx", std::process::id())); + std::fs::write(&p, bytes).unwrap(); + p + } + + #[test] + fn a_working_model_loads_runs_and_has_the_scale_it_says() { + let p = temp_model("ok", &tiny_model(64, 1000.0)); + let r = CpuRunner::load(&p, &manifest(64, 1000.0)).unwrap(); + let input: Vec = (0..4 * 64 * 64).map(|i| (i % 97) as f32 / 97.0).collect(); + let out = r.run(&input).unwrap(); + assert_eq!(out.len(), 3 * 128 * 128); + // output (colour c, pixel (2y+dy, 2x+dx)) = scale × input plane [R, G1, B][c] at (y, x) + for (c, plane) in [0usize, 1, 3].into_iter().enumerate() { + let (y, x) = (5usize, 9usize); + assert!((out[c * 128 * 128 + (2 * y + 1) * 128 + 2 * x] - 1000.0 * input[plane * 64 * 64 + y * 64 + x]).abs() < 1e-2); + } + let _ = std::fs::remove_file(p); + } + + #[test] + fn the_self_test_checks_scale_and_noise() { + // a model that only multiplies cannot take noise down: it must fail that check but pass the others + let p = temp_model("selftest", &tiny_model(64, 1000.0)); + let t = CpuRunner::load(&p, &manifest(64, 1000.0)).unwrap().self_test(); + assert!(!t.ok, "{t:?}"); + assert!(t.checks.iter().any(|(c, ok)| c.contains("same answer") && *ok), "{t:?}"); + assert!(t.checks.iter().any(|(c, ok)| c.contains("scale") && *ok), "{t:?}"); + assert!(t.checks.iter().any(|(c, ok)| c.contains("noise") && !*ok), "{t:?}"); + // and one that claims the wrong scale fails that check + let q = CpuRunner::load(&p, &manifest(64, 5.0)).unwrap().self_test(); + assert!(q.checks.iter().any(|(c, ok)| c.contains("scale") && !*ok), "{q:?}"); + let _ = std::fs::remove_file(p); + } + + #[test] + fn wrong_sizes_and_broken_files_are_errors() { + let p = temp_model("sizes", &tiny_model(64, 1.0)); + let r = CpuRunner::load(&p, &manifest(64, 1.0)).unwrap(); + assert!(matches!(r.run(&[0.0; 10]), Err(Error::Input(_)))); + // a fully convolutional model works at whatever tile size the manifest asks for + let wide = CpuRunner::load(&p, &manifest(128, 1.0)).unwrap(); + assert_eq!(wide.run(&vec![0.5; 4 * 128 * 128]).unwrap().len(), 3 * 256 * 256); + // a model that gives another number of values than the manifest promises is an error + let q = temp_model("channels", &tiny_model_with(64, 1.0, 8)); + assert!(CpuRunner::load(&q, &manifest(64, 1.0)).is_err(), "shape is refused before execution"); + let _ = std::fs::remove_file(q); + for (name, bytes) in [("junk", b"nonsense".to_vec()), ("empty", Vec::new())] { + let q = temp_model(name, &bytes); + assert!(CpuRunner::load(&q, &manifest(64, 1.0)).is_err(), "{name}"); + let _ = std::fs::remove_file(q); + } + assert!(CpuRunner::load(&std::env::temp_dir().join("does-not-exist.onnx"), &manifest(64, 1.0)).is_err()); + let _ = std::fs::remove_file(p); + } + + #[test] + fn the_check_tile_covers_black_to_white() { + for t in [1, 2, 7, 64] { + let c = check_tile(t); + assert_eq!(c.len(), 4 * t * t); + assert!(c.iter().all(|v| (0.0..=1.0).contains(v)), "{t}"); + } + let c = check_tile(64); + let near = |lo: f32, hi: f32| c.iter().filter(|v| (lo..=hi).contains(*v)).count(); + assert!(near(0.0, 0.0) >= 4 * 8 * 8 && near(1.0, 1.0) >= 4 * 8 * 8, "a flat black and a clipped white corner"); + for band in 0..10 { + let lo = band as f32 / 10.0; + assert!(near(lo, lo + 0.1) > 100, "values around {lo}"); + } + assert_eq!(check_tile(64), c, "the same tile every time"); + } + + #[test] + fn truncated_and_corrupted_models_never_panic() { + let good = tiny_model(64, 1.0); + for n in (0..good.len()).step_by(5) { + let q = temp_model("trunc", &good[..n]); + let _ = CpuRunner::load(&q, &manifest(64, 1.0)); + let _ = std::fs::remove_file(q); + } + for at in (0..good.len()).step_by(7) { + let mut bad = good.clone(); + if let Some(b) = bad.get_mut(at) { + *b = b.wrapping_add(0x55); + } + let q = temp_model("flip", &bad); + if let Ok(r) = CpuRunner::load(&q, &manifest(64, 1.0)) { + let _ = r.run(&vec![0.5; 4 * 64 * 64]); + } + let _ = std::fs::remove_file(q); + } + } + + /// Opt-in: `LC_DENOISE_MODEL= cargo test -p lightcraft-denoise --release + /// -- --ignored --nocapture` self-tests a real model (RawNIND's Bayer model: tile 512, scale about 1e6). + #[test] + #[ignore = "needs a real model file: set LC_DENOISE_MODEL"] + fn a_real_model_passes_the_self_test() { + let Some(path) = std::env::var_os("LC_DENOISE_MODEL") else { return }; + let m = manifest(512, 1.0e6); + let r = CpuRunner::load(Path::new(&path), &m).unwrap(); + let t = r.self_test(); + println!("load {:.0} ms | {:.0} ms per tile | scale {:.4e} | {:?}", t.load_ms, t.tile_ms, t.scale, t.checks); + assert!(t.ok); + } +} + +#[cfg(test)] +mod cpu_tests { + use super::*; + #[test] + fn manifest_and_test_tile_sizes_are_checked_without_tract() { + for t in [0, 2049, usize::MAX] { + assert!(test_tile(t).0.is_empty()); + assert!(check_tile(t).is_empty()); + } + let path = std::env::temp_dir().join(format!("lc-cpu-load-{}.onnx", std::process::id())); + std::fs::write(&path, crate::synthetic::unet_onnx(64, 5, 1, 7)).unwrap(); + let mut m = DenoiserManifest { + id: "synthetic".into(), + name: "Synthetic".into(), + version: "1".into(), + licence: Default::default(), + source: None, + sha256: None, + size_bytes: None, + provenance: "synthetic test".into(), + domain: crate::manifest::Domain::BayerToRgb, + tile: 64, + overlap: 16, + gain: Gain::None, + }; + let r = CpuRunner::load(&path, &m).unwrap(); + let x = check_tile(64); + assert_eq!(r.run(&x).unwrap(), r.run(&x).unwrap()); + assert!(r.self_test().checks.iter().any(|(text, ok)| text.contains("same answer") && *ok)); + m.tile = u32::MAX; + assert!(CpuRunner::load(&path, &m).is_err()); + std::fs::remove_file(path).unwrap(); + } +} diff --git a/crates/denoise/src/synthetic.rs b/crates/denoise/src/synthetic.rs new file mode 100644 index 00000000..8ceafca7 --- /dev/null +++ b/crates/denoise/src/synthetic.rs @@ -0,0 +1,234 @@ +//! A small ONNX U-Net built in memory, for tests of the layers that read and run denoise networks (here, in the GPU +//! runner and in the engine): the same layer kinds and wiring as the real denoisers, with deterministic random weights. +//! Nothing here is a model anyone would use. + +use crate::synthetic_proto::{len_field, value_info_bytes}; + +fn varint(mut v: u64, out: &mut Vec) { + loop { + let b = (v & 0x7f) as u8; + v >>= 7; + if v == 0 { + out.push(b); + return; + } + out.push(b | 0x80); + } +} + +fn varint_field(n: u64, v: u64, out: &mut Vec) { + varint(n << 3, out); + varint(v, out); +} + +enum A<'a> { + Int(i64), + Ints(&'a [i64]), + Float(f32), + Str(&'a str), +} + +fn node(op: &str, inputs: &[&str], output: &str, attrs: &[(&str, A)]) -> Vec { + let mut n = Vec::new(); + for i in inputs { + len_field(1, i.as_bytes(), &mut n); + } + len_field(2, output.as_bytes(), &mut n); + len_field(4, op.as_bytes(), &mut n); + for (name, value) in attrs { + let mut a = Vec::new(); + len_field(1, name.as_bytes(), &mut a); + match value { + A::Int(v) => { + varint_field(3, *v as u64, &mut a); + varint_field(20, 2, &mut a); + } + A::Ints(vs) => { + for v in *vs { + varint_field(8, *v as u64, &mut a); + } + varint_field(20, 7, &mut a); + } + A::Float(v) => { + varint((2 << 3) | 5, &mut a); + a.extend_from_slice(&v.to_le_bytes()); + varint_field(20, 1, &mut a); + } + A::Str(s) => { + len_field(4, s.as_bytes(), &mut a); + varint_field(20, 3, &mut a); + } + } + len_field(5, &a, &mut n); + } + n +} + +fn tensor(name: &str, dims: &[u64], values: &[f32]) -> Vec { + let mut t = Vec::new(); + for d in dims { + varint_field(1, *d, &mut t); + } + varint_field(2, 1, &mut t); + len_field(8, name.as_bytes(), &mut t); + let raw: Vec = values.iter().flat_map(|v| v.to_le_bytes()).collect(); + len_field(9, &raw, &mut t); + t +} + +struct Rng(u64); + +impl Rng { + fn next(&mut self) -> f32 { + self.0 ^= self.0 >> 12; + self.0 ^= self.0 << 25; + self.0 ^= self.0 >> 27; + ((self.0.wrapping_mul(0x2545_f491_4f6c_dd1d) >> 40) as f32 / (1u64 << 24) as f32) * 2.0 - 1.0 + } +} + +struct Builder { + nodes: Vec>, + inits: Vec>, + rng: Rng, + n: usize, +} + +impl Builder { + fn name(&mut self, kind: &str) -> String { + self.n += 1; + format!("{kind}{}", self.n) + } + + /// Random weights scaled so activations keep a sane size through the layers. + fn weights(&mut self, name: &str, dims: &[u64], fan_in: usize) -> String { + let count: u64 = dims.iter().product(); + let scale = (3.0 / fan_in as f32).sqrt(); + let values: Vec = (0..count).map(|_| self.rng.next() * scale).collect(); + self.inits.push(tensor(name, dims, &values)); + name.to_string() + } + + fn bias(&mut self, name: &str, n: u64) -> String { + let values: Vec = (0..n).map(|_| self.rng.next() * 0.1).collect(); + self.inits.push(tensor(name, &[n], &values)); + name.to_string() + } + + /// `Conv` (+ `LeakyRelu` when `leaky`); gives the name of the tensor that results. + fn conv(&mut self, src: &str, cin: u64, cout: u64, k: u64, leaky: bool) -> String { + let (w, b, out) = (self.name("w"), self.name("b"), self.name("conv")); + let w = self.weights(&w, &[cout, cin, k, k], (cin * k * k) as usize); + let b = self.bias(&b, cout); + let pad = (k as i64 - 1) / 2; + self.nodes.push(node( + "Conv", + &[src, &w, &b], + &out, + &[("kernel_shape", A::Ints(&[k as i64, k as i64])), ("pads", A::Ints(&[pad; 4])), ("strides", A::Ints(&[1, 1])), ("group", A::Int(1))], + )); + if !leaky { + return out; + } + let act = self.name("act"); + self.nodes.push(node("LeakyRelu", &[&out], &act, &[("alpha", A::Float(0.2))])); + act + } + + fn pool(&mut self, src: &str) -> String { + let out = self.name("pool"); + self.nodes.push(node( + "MaxPool", + &[src], + &out, + &[("kernel_shape", A::Ints(&[2, 2])), ("strides", A::Ints(&[2, 2])), ("pads", A::Ints(&[0; 4]))], + )); + out + } + + fn up(&mut self, src: &str, cin: u64, cout: u64) -> String { + let (w, b, out) = (self.name("uw"), self.name("ub"), self.name("up")); + let w = self.weights(&w, &[cin, cout, 2, 2], (cin * 4) as usize); + let b = self.bias(&b, cout); + self.nodes.push(node( + "ConvTranspose", + &[src, &w, &b], + &out, + &[("kernel_shape", A::Ints(&[2, 2])), ("strides", A::Ints(&[2, 2])), ("pads", A::Ints(&[0; 4]))], + )); + out + } + + fn concat(&mut self, a: &str, b: &str) -> String { + let out = self.name("cat"); + self.nodes.push(node("Concat", &[a, b], &out, &[("axis", A::Int(1))])); + out + } +} + +/// A denoiser-shaped U-Net as ONNX bytes: `[1, 4, tile, tile]` → `[1, 3, 2·tile, 2·tile]`, with `depth` poolings +/// (so `tile` must be a multiple of `2^depth`) and `ch` channels on the first level, doubling on each (every count a +/// multiple of 4). Each level is two 3 × 3 convolutions with a leaky ReLU, as in the real networks; the output is a +/// 1 × 1 convolution to 12 channels and depth-to-space. +pub fn unet_onnx(tile: u64, ch: u64, depth: usize, seed: u64) -> Vec { + let mut b = Builder { nodes: Vec::new(), inits: Vec::new(), rng: Rng(seed | 1), n: 0 }; + let mut skips: Vec<(String, u64)> = Vec::new(); + let (mut cur, mut c_in, mut c) = ("data".to_string(), 4u64, ch); + for level in 0..=depth { + let x = b.conv(&cur, c_in, c, 3, true); + let x = b.conv(&x, c, c, 3, true); + if level < depth { + skips.push((x.clone(), c)); + cur = b.pool(&x); + c_in = c; + c *= 2; + } else { + cur = x; + } + } + for (skip, sc) in skips.into_iter().rev() { + let up = b.up(&cur, c, sc); + let joined = b.concat(&up, &skip); + let x = b.conv(&joined, 2 * sc, sc, 3, true); + cur = b.conv(&x, sc, sc, 3, true); + c = sc; + } + let rgb = b.conv(&cur, c, 12, 1, false); + b.nodes.push(node("DepthToSpace", &[&rgb], "rgb", &[("blocksize", A::Int(2)), ("mode", A::Str("CRD"))])); + + let mut graph = Vec::new(); + for n in &b.nodes { + len_field(1, n, &mut graph); + } + for t in &b.inits { + len_field(5, t, &mut graph); + } + len_field(11, &value_info_bytes("data", 1, &[Ok(1), Ok(4), Ok(tile), Ok(tile)]), &mut graph); + len_field(12, &value_info_bytes("rgb", 1, &[Ok(1), Ok(3), Ok(2 * tile), Ok(2 * tile)]), &mut graph); + let mut ops = Vec::new(); + len_field(1, b"", &mut ops); + varint_field(2, 13, &mut ops); + let mut model = Vec::new(); + varint_field(1, 8, &mut model); + len_field(7, &graph, &mut model); + len_field(8, &ops, &mut model); + model +} + +/// A tiny smoothing network for installing a real CPU model in setup-flow tests. +/// It averages a 3 × 3 neighborhood of the four Bayer planes and returns three identical channels. +pub fn smoothing_onnx() -> Vec { + let mut graph = Vec::new(); + len_field(1, &node("Conv", &["data", "weights"], "samples", &[("kernel_shape", A::Ints(&[3, 3])), ("pads", A::Ints(&[1, 1, 1, 1]))]), &mut graph); + len_field(1, &node("DepthToSpace", &["samples"], "rgb", &[("blocksize", A::Int(2)), ("mode", A::Str("CRD"))]), &mut graph); + len_field(5, &tensor("weights", &[12, 4, 3, 3], &[1.0 / 36.0; 12 * 4 * 3 * 3]), &mut graph); + len_field(11, &value_info_bytes("data", 1, &[Ok(1), Ok(4), Ok(64), Ok(64)]), &mut graph); + len_field(12, &value_info_bytes("rgb", 1, &[Ok(1), Ok(3), Ok(128), Ok(128)]), &mut graph); + let mut model = Vec::new(); + varint_field(1, 8, &mut model); + len_field(7, &graph, &mut model); + let mut ops = Vec::new(); + varint_field(2, 13, &mut ops); + len_field(8, &ops, &mut model); + model +} diff --git a/crates/denoise/src/synthetic_proto.rs b/crates/denoise/src/synthetic_proto.rs new file mode 100644 index 00000000..a78d586b --- /dev/null +++ b/crates/denoise/src/synthetic_proto.rs @@ -0,0 +1,44 @@ +pub(crate) fn varint(mut v: u64, out: &mut Vec) { + loop { + let b = (v & 0x7f) as u8; + v >>= 7; + if v == 0 { + out.push(b); + return; + } + out.push(b | 0x80); + } +} + +pub fn len_field(n: u64, body: &[u8], out: &mut Vec) { + varint(n << 3 | 2, out); + varint(body.len() as u64, out); + out.extend_from_slice(body); +} + +pub(crate) fn varint_field(n: u64, v: u64, out: &mut Vec) { + varint(n << 3, out); + varint(v, out); +} + +/// A graph input or output: `Ok(n)` is a fixed dimension, `Err(name)` a named (dynamic) one. +pub fn value_info_bytes(name: &str, elem: u64, dims: &[std::result::Result]) -> Vec { + let mut shape = Vec::new(); + for d in dims { + let mut dim = Vec::new(); + match d { + Ok(v) => varint_field(1, *v, &mut dim), + Err(p) => len_field(2, p.as_bytes(), &mut dim), + } + len_field(1, &dim, &mut shape); + } + let mut tensor = Vec::new(); + varint_field(1, elem, &mut tensor); + len_field(2, &shape, &mut tensor); + let mut ty = Vec::new(); + len_field(1, &tensor, &mut ty); + let mut vi = Vec::new(); + len_field(1, name.as_bytes(), &mut vi); + len_field(2, &ty, &mut vi); + vi +} diff --git a/crates/denoise/tests/real_model.rs b/crates/denoise/tests/real_model.rs new file mode 100644 index 00000000..203e9937 --- /dev/null +++ b/crates/denoise/tests/real_model.rs @@ -0,0 +1,250 @@ +//! Opt-in end-to-end check of the real RawNIND Bayer model on real mosaics with noise added: +//! +//! ```text +//! LC_DENOISE_MODEL= LC_DENOISE_MOSAICS= LC_DENOISE_OUT= \ +//! cargo test --release -p lightcraft-denoise --features runtime --test real_model -- --ignored --nocapture +//! ``` +//! +//! `LC_DENOISE_MOSAICS` holds `.mosaic.f32` (little-endian f32, normalised: black 0, white 1, not white +//! balanced), `.rgb.f32` (a clean demosaic of it, camera RGB, interleaved) and `.txt` (`width`, +//! `height`, `cfa`, `wb` lines). Noise is Poisson-Gaussian, made here. Optional: `LC_DENOISE_PARALLEL` (tiles at +//! once, default 8), `LC_DENOISE_ONLY` (one name), `LC_DENOISE_ISO` (noise strength, default 1.0; 0 adds none), +//! `LC_DENOISE_FULL` (the whole frame instead of a 2048 × 1536 crop). + +#![cfg(feature = "runtime")] + +use std::path::{Path, PathBuf}; +use std::time::Instant; + +use lightcraft_denoise::bayer::Layout; +use lightcraft_denoise::manifest::{DenoiserManifest, Domain, Gain}; +use lightcraft_denoise::run::{Control, Params, denoise_bayer}; +use lightcraft_denoise::runtime::CpuRunner; + +struct Rng(u64); +impl Rng { + fn next(&mut self) -> f64 { + self.0 ^= self.0 >> 12; + self.0 ^= self.0 << 25; + self.0 ^= self.0 >> 27; + ((self.0.wrapping_mul(0x2545_f491_4f6c_dd1d) >> 11) as f64 + 0.5) / (1u64 << 53) as f64 + } + fn gauss(&mut self) -> f64 { + (-2.0 * self.next().ln()).sqrt() * (std::f64::consts::TAU * self.next()).cos() + } +} + +fn read_f32(path: &Path) -> Vec { + std::fs::read(path).unwrap().as_chunks::<4>().0.iter().map(|b| f32::from_le_bytes(*b)).collect() +} + +struct Dump { + width: usize, + height: usize, + layout: Layout, + wb: [f32; 3], + mosaic: Vec, + rgb: Vec, +} + +fn load(dir: &Path, name: &str) -> Dump { + let text = std::fs::read_to_string(dir.join(format!("{name}.txt"))).unwrap(); + let field = |k: &str| text.lines().find_map(|l| l.strip_prefix(k).map(|v| v.trim().to_string())).unwrap(); + let cell = match field("cfa").as_str() { + "RGGB" => [0, 1, 1, 2], + "BGGR" => [2, 1, 1, 0], + "GRBG" => [1, 0, 2, 1], + "GBRG" => [1, 2, 0, 1], + other => panic!("cfa {other}"), + }; + let wb: Vec = field("wb").split_whitespace().map(|v| v.parse().unwrap()).collect(); + Dump { + width: field("width").parse().unwrap(), + height: field("height").parse().unwrap(), + layout: Layout::from_cell(cell).unwrap(), + wb: [wb[0], wb[1], wb[2]], + mosaic: read_f32(&dir.join(format!("{name}.mosaic.f32"))), + rgb: read_f32(&dir.join(format!("{name}.rgb.f32"))), + } +} + +/// Crop with an even origin so the Bayer phase is kept. +fn crop(d: &Dump, x0: usize, y0: usize, w: usize, h: usize) -> (Vec, Vec) { + let mut m = Vec::with_capacity(w * h); + let mut c = Vec::with_capacity(w * h * 3); + for y in y0..y0 + h { + m.extend_from_slice(&d.mosaic[y * d.width + x0..y * d.width + x0 + w]); + c.extend_from_slice(&d.rgb[(y * d.width + x0) * 3..(y * d.width + x0 + w) * 3]); + } + (m, c) +} + +/// 0 = red, 1 = green, 2 = blue at a mosaic position. +fn site(layout: Layout, x: usize, y: usize) -> usize { + match ((x + layout.dx) % 2, (y + layout.dy) % 2) { + (0, 0) => 0, + (1, 1) => 2, + _ => 1, + } +} + +/// The simplest demosaic (each missing colour is the mean of the same-coloured samples in the 3 × 3 around), +/// as the baseline a user would get from the noisy picture without AI. +fn bilinear(m: &[f32], w: usize, h: usize, layout: Layout) -> Vec { + let mut out = vec![0f32; w * h * 3]; + for y in 0..h { + for x in 0..w { + let mut sum = [0f32; 3]; + let mut n = [0f32; 3]; + for yy in y.saturating_sub(1)..(y + 2).min(h) { + for xx in x.saturating_sub(1)..(x + 2).min(w) { + let c = site(layout, xx, yy); + sum[c] += m[yy * w + xx]; + n[c] += 1.0; + } + } + for c in 0..3 { + out[(y * w + x) * 3 + c] = if n[c] > 0.0 { sum[c] / n[c] } else { 0.0 }; + } + } + } + out +} + +/// Signal-to-error ratio in dB of `got` against `want` (both interleaved RGB), ignoring a border. +fn snr(got: &[f32], want: &[f32], w: usize, h: usize, border: usize) -> f64 { + let (mut sig, mut err) = (0f64, 0f64); + for y in border..h - border { + for x in border..w - border { + for c in 0..3 { + let (g, t) = (f64::from(got[(y * w + x) * 3 + c]), f64::from(want[(y * w + x) * 3 + c])); + sig += t * t; + err += (g - t).powi(2); + } + } + } + 10.0 * (sig / err.max(1e-30)).log10() +} + +fn crc32(data: &[u8]) -> u32 { + let mut c = !0u32; + for &b in data { + c ^= u32::from(b); + for _ in 0..8 { + c = if c & 1 != 0 { 0xedb8_8320 ^ (c >> 1) } else { c >> 1 }; + } + } + !c +} + +/// A viewable PNG of interleaved camera-RGB panels side by side: white balanced, one exposure for all, gamma. +fn write_png(path: &Path, panels: &[&[f32]], w: usize, h: usize, wb: [f32; 3], exposure: f32) { + let total_w = w * panels.len(); + let mut raw = Vec::with_capacity((total_w * 3 + 1) * h); + for y in 0..h { + raw.push(0u8); + for p in panels { + for x in 0..w { + for c in 0..3 { + let v = (p[(y * w + x) * 3 + c] * wb[c] * exposure).clamp(0.0, 1.0); + raw.push((v.powf(1.0 / 2.2) * 255.0 + 0.5) as u8); + } + } + } + } + let z = miniz_oxide::deflate::compress_to_vec_zlib(&raw, 6); + let mut png = b"\x89PNG\r\n\x1a\n".to_vec(); + let mut chunk = |kind: &[u8; 4], body: &[u8]| { + png.extend_from_slice(&(body.len() as u32).to_be_bytes()); + let mut k = kind.to_vec(); + k.extend_from_slice(body); + png.extend_from_slice(&k); + png.extend_from_slice(&crc32(&k).to_be_bytes()); + }; + let mut ihdr = Vec::new(); + ihdr.extend_from_slice(&(total_w as u32).to_be_bytes()); + ihdr.extend_from_slice(&(h as u32).to_be_bytes()); + ihdr.extend_from_slice(&[8, 2, 0, 0, 0]); + chunk(b"IHDR", &ihdr); + chunk(b"IDAT", &z); + chunk(b"IEND", &[]); + std::fs::write(path, png).unwrap(); +} + +#[test] +#[ignore = "needs a real model and mosaics: see the module docs"] +fn the_real_model_denoises_real_mosaics() { + let (Some(model), Some(dir)) = (std::env::var_os("LC_DENOISE_MODEL"), std::env::var_os("LC_DENOISE_MOSAICS")) else { return }; + let dir = PathBuf::from(dir); + let out_dir = std::env::var_os("LC_DENOISE_OUT").map(PathBuf::from); + let parallel: usize = std::env::var("LC_DENOISE_PARALLEL").ok().and_then(|v| v.parse().ok()).unwrap_or(8); + let iso: f64 = std::env::var("LC_DENOISE_ISO").ok().and_then(|v| v.parse().ok()).unwrap_or(1.0); + let names: Vec = match std::env::var("LC_DENOISE_ONLY") { + Ok(n) => vec![n], + Err(_) => ["arw-sony-a7m3-compressed", "cr2-canon-6d", "nef-nikon-d5100-lossless", "dng-google-pixel2xl", "pef-pentax-k3"] + .map(String::from) + .to_vec(), + }; + let manifest = DenoiserManifest { + id: "rawnind-bayer".into(), + name: "RawNIND Bayer".into(), + version: "1".into(), + licence: Default::default(), + source: None, + sha256: None, + size_bytes: None, + provenance: String::new(), + domain: Domain::BayerToRgb, + tile: 512, + overlap: 64, + gain: Gain::MatchMean { nominal: 1.0e6, max_deviation: 0.05 }, + }; + let runner = CpuRunner::load(Path::new(&model), &manifest).unwrap(); + let params = Params { tile: 512, overlap: 64, gain: manifest.gain, clip: Some(1.0), parallel }; + for name in names { + let d = load(&dir, &name); + // a crop of about 2 × 1.5 tiles, away from the edges, even origin + let full = std::env::var_os("LC_DENOISE_FULL").is_some(); + let (w, h) = if full { (d.width & !1, d.height & !1) } else { (2048usize, 1536usize) }; + let (x0, y0) = (((d.width - w) / 3) & !1, ((d.height - h) / 2) & !1); + let (clean, reference) = crop(&d, x0, y0, w, h); + // Poisson-Gaussian noise: variance a·signal + b, as a sensor at high ISO + let (a, b) = (1.5e-3 * iso, (4.0e-3 * iso.sqrt()).powi(2)); + let mut rng = Rng(0x9e37_79b9_7f4a_7c15 ^ name.len() as u64); + let noisy: Vec = clean + .iter() + .map(|&v| { + let v = f64::from(v); + let sigma = (a * v.max(0.0) + b).sqrt(); + ((v + sigma * rng.gauss()) as f32).min(1.0) + }) + .collect(); + let started = Instant::now(); + let done = std::sync::atomic::AtomicUsize::new(0); + let progress = |n: usize, total: usize| done.store(n * 1000 / total, std::sync::atomic::Ordering::Relaxed); + let denoised = denoise_bayer(&noisy, w, h, d.layout, &runner, ¶ms, &Control { cancel: None, progress: Some(&progress) }).unwrap(); + let secs = started.elapsed().as_secs_f64(); + let baseline = bilinear(&noisy, w, h, d.layout); + let clean_bilinear = bilinear(&clean, w, h, d.layout); + let den: Vec = denoised.data.iter().flat_map(|p| p.iter().copied()).collect(); + let (s_noisy, s_den) = (snr(&baseline, &reference, w, h, 64), snr(&den, &reference, w, h, 64)); + let s_floor = snr(&clean_bilinear, &reference, w, h, 64); + println!( + "{name}: {w}×{h} crop, {} tiles, parallel {parallel}, {secs:.1} s | SNR vs clean demosaic: noisy bilinear {s_noisy:.2} dB, denoised {s_den:.2} dB (clean bilinear {s_floor:.2} dB)", + lightcraft_denoise::run::tile_count(w, h, d.layout, 512, 64) + ); + if let Some(out) = &out_dir { + std::fs::create_dir_all(out).unwrap(); + let mean = reference.iter().map(|&v| f64::from(v)).sum::() / reference.len() as f64; + let exposure = (0.18 / mean.max(1e-6)) as f32; + // three panels, a 512 × 384 window each, at 1:1 + let (pw, ph, px, py) = (512usize, 384usize, 700usize, 500usize); + let window = + |buf: &[f32]| -> Vec { (0..ph).flat_map(|y| buf[((py + y) * w + px) * 3..((py + y) * w + px + pw) * 3].to_vec()).collect() }; + write_png(&out.join(format!("{name}.png")), &[&window(&baseline), &window(&den), &window(&reference)], pw, ph, d.wb, exposure); + } + if iso >= 4.0 { + assert!(s_den > s_noisy + 2.0, "{name}: denoised {s_den:.2} dB is not clearly better than noisy {s_noisy:.2} dB"); + } + } +} diff --git a/crates/develop/src/controls.rs b/crates/develop/src/controls.rs index 5696d670..c8a001b7 100644 --- a/crates/develop/src/controls.rs +++ b/crates/develop/src/controls.rs @@ -207,6 +207,7 @@ controls! { "grain.amount" => grain.amount, "Grain", Grain, 0, 100, 0, 1, 0, Plain; "grain.size" => grain.size, "Size", Grain, 0, 100, 25, 1, 0, Plain; "grain.roughness" => grain.roughness, "Roughness", Grain, 0, 100, 50, 1, 0, Plain; + "enhance.denoise" => enhance.denoise, "Amount", Detail, 0, 100, 0, 1, 0, Plain; "detail.sharpenAmount" => detail.sharpen_amount, "Sharpening", Detail, 0, 150, 0, 1, 0, Plain; "detail.sharpenRadius" => detail.sharpen_radius, "Radius", Detail, 0.5, 3, 1, 0.1, 1, Plain; "detail.sharpenDetail" => detail.sharpen_detail, "Detail", Detail, 0, 100, 25, 1, 0, Plain; diff --git a/crates/develop/src/lib.rs b/crates/develop/src/lib.rs index bfce1893..b7c95251 100644 --- a/crates/develop/src/lib.rs +++ b/crates/develop/src/lib.rs @@ -94,6 +94,13 @@ impl DevelopSettings { std::borrow::Cow::Owned(d) } + /// How much of the AI-denoised picture to mix in, 0 (none) to 1 (all of it): the Denoise amount, unless the Detail + /// section is switched off or the value is not a number. + pub fn denoise_amount(&self) -> f32 { + let a = self.enhance.denoise; + if a.is_finite() && self.enhance.denoise_enabled() && self.section_enabled("detail") { (a / 100.0).clamp(0.0, 1.0) as f32 } else { 0.0 } + } + pub fn set_section_enabled(&mut self, section: &str, on: bool) { self.disabled_sections.retain(|s| s != section); if !on { @@ -112,6 +119,7 @@ impl DevelopSettings { self.curve = d; } Section::Color => self.wb.mode = WbMode::AsShot, + Section::Detail => self.enhance.denoise_on = None, _ => {} } } @@ -193,4 +201,40 @@ mod tests { s.set_section_enabled("effects", true); assert!(s.section_enabled("effects")); } + + #[test] + fn the_denoise_amount_is_a_slider_and_follows_the_detail_section() { + let mut s = DevelopSettings::default(); + assert_eq!(s.denoise_amount(), 0.0); + assert!(controls::set(&mut s, "enhance.denoise", 60.0)); + assert_eq!(controls::get(&s, "enhance.denoise"), Some(60.0)); + assert!((s.denoise_amount() - 0.6).abs() < 1e-6); + // the slider's range is 0 to 100; values outside it are brought in + controls::set(&mut s, "enhance.denoise", 500.0); + assert_eq!(s.denoise_amount(), 1.0); + controls::set(&mut s, "enhance.denoise", -5.0); + assert_eq!(s.denoise_amount(), 0.0); + // switching the Detail section off switches the denoise off with it; a settings file with a bad number does nothing + controls::set(&mut s, "enhance.denoise", 80.0); + s.set_section_enabled("detail", false); + assert_eq!(s.denoise_amount(), 0.0); + s.set_section_enabled("detail", true); + s.enhance.denoise = f64::NAN; + assert_eq!(s.denoise_amount(), 0.0); + // it is part of Detail: a reset of the section clears it + controls::set(&mut s, "enhance.denoise", 40.0); + s.reset_section(Section::Detail); + assert_eq!(s.enhance.denoise, 0.0); + // The later independent switch preserves Amount when off, and can be on at zero. + controls::set(&mut s, "enhance.denoise", 75.0); + s.enhance.denoise_on = Some(false); + assert_eq!(s.denoise_amount(), 0.0); + assert_eq!(s.enhance.denoise, 75.0); + s.enhance.denoise_on = Some(true); + assert_eq!(s.denoise_amount(), 0.75); + controls::set(&mut s, "enhance.denoise", 0.0); + assert!(s.enhance.denoise_enabled()); + s.reset_section(Section::Detail); + assert_eq!(s.enhance.denoise_on, None); + } } diff --git a/crates/develop/src/settings.rs b/crates/develop/src/settings.rs index 910cd07d..5f0db8ab 100644 --- a/crates/develop/src/settings.rs +++ b/crates/develop/src/settings.rs @@ -915,6 +915,14 @@ pub struct LensBlur { #[serde(default)] pub struct Enhance { pub denoise: f64, + #[serde(skip_serializing_if = "Option::is_none")] + pub denoise_on: Option, pub raw_details: bool, pub super_resolution: bool, } + +impl Enhance { + pub fn denoise_enabled(&self) -> bool { + self.denoise_on.unwrap_or(self.denoise > 0.0) + } +} diff --git a/crates/engine/Cargo.toml b/crates/engine/Cargo.toml index 7a6ab439..9bd9845f 100644 --- a/crates/engine/Cargo.toml +++ b/crates/engine/Cargo.toml @@ -21,6 +21,8 @@ web-time = { workspace = true } lightcraft-preview = { workspace = true } lightcraft-codecs = { workspace = true } lightcraft-raw = { workspace = true } +lightcraft-denoise-core = { workspace = true } +lightcraft-denoise = { workspace = true, optional = true } lightcraft-meta = { workspace = true } lightcraft-faces = { workspace = true } miniz_oxide = { workspace = true } @@ -43,6 +45,8 @@ lightcraft-fetch = { workspace = true } [features] # AI masks with SAM 3 (Object / Describe). Off for the web and CLI builds. sam = ["dep:lightcraft-segment"] +denoise = ["dep:lightcraft-denoise", "lightcraft-denoise/runtime", "lightcraft-gpu/denoise"] +rawnind-model = ["denoise", "lightcraft-denoise/rawnind-model", "lightcraft-denoise-core/rawnind-model"] [dev-dependencies] libc = { workspace = true } diff --git a/crates/engine/src/camera_profiles.rs b/crates/engine/src/camera_profiles.rs index 97fb8ef2..80192cf6 100644 --- a/crates/engine/src/camera_profiles.rs +++ b/crates/engine/src/camera_profiles.rs @@ -65,19 +65,8 @@ impl CameraProfile { } } -/// LightCraft's configuration folder (settings, GPU marker, camera profiles). -pub fn config_dir() -> Option { - if cfg!(target_os = "macos") { - std::env::var_os("HOME").map(|h| PathBuf::from(h).join("Library/Application Support/LightCraft")) - } else if cfg!(windows) { - std::env::var_os("APPDATA").map(|a| PathBuf::from(a).join("LightCraft")) - } else { - std::env::var_os("XDG_CONFIG_HOME") - .map(PathBuf::from) - .or_else(|| std::env::var_os("HOME").map(|h| PathBuf::from(h).join(".config"))) - .map(|c| c.join("lightcraft")) - } -} +/// LightCraft's configuration folder (settings, GPU marker, camera profiles); one helper for the app. +pub use crate::config::config_dir; /// Where camera profiles are read from and written to. pub fn dir() -> Option { diff --git a/crates/engine/src/cmd/denoise.rs b/crates/engine/src/cmd/denoise.rs new file mode 100644 index 00000000..e1cc0900 --- /dev/null +++ b/crates/engine/src/cmd/denoise.rs @@ -0,0 +1,659 @@ +//! Denoise commands (`denoise.*`): the models (list, install, download, test, remove, choose), the settings, and the +//! work (queue, cancel, status, pump, clear). The machinery they drive is [`crate::denoise`]. +//! +//! Models live in the folder the host provides (`Session::denoise.models_dir`): one subfolder per model with +//! `model.onnx`, `denoise-model.json` (its manifest) and `installed.json` (what the user accepted and the file's hash), +//! plus `settings.json` beside them. Model files, archives and manifests are hostile input: sizes are capped, ids are +//! validated before they become folder names, and nothing outside a model's own folder is ever touched. + +use std::path::{Path, PathBuf}; +use std::sync::{Arc, PoisonError, mpsc}; + +use lightcraft_catalog::PhotoId; +use lightcraft_denoise::archive; +use lightcraft_denoise::hash::sha256_file; +use lightcraft_denoise::known::{self, Known}; +use lightcraft_denoise::manifest::{self, DenoiserManifest, MAX_MANIFEST_BYTES, MAX_MODEL_BYTES}; +use serde_json::{Value, json}; + +use super::{CommandSpec, always, bad, bool_or, cmd, str_param}; +use crate::denoise::{Pace, RunOn, Settings, forget_gpu_set_ups, installed_models, read_capped, read_settings, write_atomic, write_settings}; +use crate::model_download::State; +use crate::{EngineError, Result, Session}; + +fn fail(what: &str, e: impl std::fmt::Display) -> EngineError { + EngineError::Other(format!("{what}: {e}")) +} + +fn models_dir(s: &Session, cmd: &str) -> Result { + s.denoise.models_dir.clone().ok_or_else(|| bad(cmd, "this build has nowhere to keep denoise models (the desktop app does)")) +} + +/// What a command says about a model. +fn row(m: &DenoiserManifest, download_host: Option, installed: bool, selected: bool, accepted: &Value) -> Value { + json!({ + "id": m.id, + "name": m.name, + "version": m.version, + "licence": m.licence, + "provenance": m.provenance, + "source": m.source, + "sizeBytes": m.size_bytes, + "sha256": m.sha256, + "tile": m.tile, + "known": known::find(&m.id).is_some(), + // a pinned address LightCraft can fetch it from (the user presses Download), and which site that is + "downloadHost": download_host, + "installed": installed, + "selected": selected, + "accepted": accepted, + }) +} + +fn host_of(k: &Known) -> Option { + k.download.as_ref().map(|d| lightcraft_denoise::known::host(&d.url).to_string()) +} + +fn list(s: &mut Session, _: &Value) -> Result { + s.denoise_refresh_active(true); + let dir = s.denoise.models_dir.clone(); + let st = dir.as_deref().map(read_settings).unwrap_or_default(); + let installed = dir.as_deref().map(installed_models).unwrap_or_default(); + let on_disk = |id: &str| installed.iter().find(|i| i.manifest.id == id); + let chosen = |id: &str| st.model.as_deref() == Some(id); + let mut models: Vec = Vec::new(); + for k in known::all() { + let found = on_disk(&k.manifest.id); + models.push(row(&k.manifest, host_of(&k), found.is_some(), chosen(&k.manifest.id), found.map_or(&Value::Null, |f| &f.accepted))); + } + for i in installed.iter().filter(|i| known::find(&i.manifest.id).is_none()) { + models.push(row(&i.manifest, None, true, chosen(&i.manifest.id), &i.accepted)); + } + Ok(json!({ + "dir": dir.as_ref().map(|d| d.display().to_string()), + "productsDir": s.denoise_products_dir().map(|d| d.display().to_string()), + "model": st.model, + // whether this build can run denoise models (the `denoise` feature: our pure-Rust CPU/GPU runners) + "runtime": cfg!(feature = "denoise"), + "auto": st.auto(), + "cacheGb": st.cache_bytes() >> 30, + "threads": st.threads, + "runOn": st.run_on().name(), + "models": models, + })) +} + +/// What a file the user pointed at is, as far as installing it goes. +struct Source { + manifest: DenoiserManifest, + /// The `.onnx` inside the archive; `None` when the file is the `.onnx`. + entry: Option<&'static str>, + file_name: String, +} + +fn identify(path: &Path, c: &str, sha_known: Option<&str>) -> Result { + let meta = std::fs::metadata(path).map_err(|e| bad(c, format!("cannot read `{}`: {e}", path.display())))?; + if !meta.is_file() { + return Err(bad(c, format!("`{}` is not a file", path.display()))); + } + if meta.len() == 0 || meta.len() > MAX_MODEL_BYTES { + return Err(bad(c, "a model file must be between 1 byte and 2 GiB")); + } + let sha256 = match sha_known { + Some(h) => h.to_string(), + None => sha256_file(path).map_err(|e| fail(&format!("could not read {}", path.display()), e))?, + }; + let file_name = path.file_name().map(|n| n.to_string_lossy().into_owned()).unwrap_or_default(); + // an archive LightCraft knows (the file darktable publishes the model in) + if let Some(k) = known::all().into_iter().find(|k| k.download.as_ref().is_some_and(|d| d.sha256 == sha256)) { + return Ok(Source { manifest: k.manifest, entry: k.entry, file_name }); + } + // the `.onnx` of a known model + if let Some(m) = known::by_sha256(&sha256) { + return Ok(Source { manifest: m, entry: None, file_name }); + } + // an `.onnx` with its manifest beside it + let sibling = path.with_file_name("denoise-model.json"); + if path.extension().is_some_and(|e| e.eq_ignore_ascii_case("onnx")) + && let Some(bytes) = read_capped(&sibling, MAX_MANIFEST_BYTES) + { + let m = manifest::parse(&bytes).map_err(|e| bad(c, format!("{}: {e}", sibling.display())))?; + return Ok(Source { manifest: m, entry: None, file_name }); + } + Err(bad( + c, + "LightCraft does not know this file. A denoise model is an .onnx file with a denoise-model.json beside it that says how to use it \ + (see docs/denoise.md), or an archive LightCraft can fetch itself (`denoise.models.download`)", + )) +} + +/// Install the model in `src`. A model that passes its self-test becomes the one in use (`activate`, the default): installing +/// a model is how someone says they want it. `verified` is for a download whose hash was already checked. +fn install_file( + dir: &Path, + loader: &crate::denoise::Loader, + run_on: RunOn, + accepted_at: String, + c: &str, + src: &Path, + acknowledged: bool, + verified: bool, +) -> Result<(Value, Arc)> { + let ident = identify(src, c, None)?; + let m = ident.manifest.clone(); + manifest::validate(&m).map_err(|e| bad(c, e.to_string()))?; + if !acknowledged { + return Err(bad( + c, + "the licence has not been accepted: show the user its terms (see `denoise.models.list`) and pass `acknowledged: true` once they agree", + )); + } + let home = dir.join(&m.id); + if home.join("model.onnx").exists() { + return Err(bad(c, "that model is already installed; remove it before installing a replacement")); + } + std::fs::create_dir_all(&home).map_err(|e| fail("could not create the model's folder", e))?; + let mut cleanup = InstallCleanup { home: home.clone(), armed: true }; + let (part, final_path) = (home.join("model.onnx.part"), home.join("model.onnx")); + let _ = std::fs::remove_file(&part); + let placed: Result<()> = match ident.entry { + Some(entry) => { + archive::extract(src, entry, &part, MAX_MODEL_BYTES).map(|_| ()).map_err(|e| fail("could not take the model out of the archive", e)) + } + None if verified && std::fs::rename(src, &part).is_ok() => Ok(()), + None => std::fs::copy(src, &part).map(|_| ()).map_err(|e| fail("could not copy the model (is the disk full?)", e)), + }; + let sha256 = placed.and_then(|_| sha256_file(&part).map_err(|e| fail("could not check the model", e))).and_then(|h| match &m.sha256 { + Some(want) if *want != h => Err(EngineError::Other("the model file is not the one its description names (its SHA-256 differs)".into())), + _ => Ok(h), + }); + let sha256 = match sha256 { + Ok(h) => h, + Err(e) => { + let _ = std::fs::remove_file(&part); + return Err(e); + } + }; + // Test the staged file first. A failed replacement preserves the installed model. + let model = loader(&part, &m).map_err(|e| bad(c, e))?; + let test = match model.self_test(run_on) { + Ok(t) => t, + Err(e) => { + let _ = std::fs::remove_file(&part); + return Err(bad(c, e)); + } + }; + // An installed id is immutable: installing again must never destroy a usable model. + if final_path.exists() { + let _ = std::fs::remove_file(&part); + return Err(bad(c, "that model is already installed; remove it before installing a replacement")); + } + std::fs::rename(&part, &final_path).map_err(|e| fail("could not finish the copy", e))?; + write_atomic(&home.join("denoise-model.json"), &serde_json::to_vec_pretty(&m).map_err(|e| fail("manifest", e))?).map_err(EngineError::Other)?; + let accepted = json!({ + "acceptedAt": accepted_at, + "licence": m.licence.name, + "commercial": m.licence.commercial, + "fileName": ident.file_name, + "sha256": sha256, + "selfTest": test, + }); + write_atomic(&home.join("installed.json"), &serde_json::to_vec_pretty(&accepted).map_err(|e| fail("record", e))?).map_err(EngineError::Other)?; + cleanup.armed = false; + Ok((json!({"installed": row(&m, known::find(&m.id).and_then(|k| host_of(&k)), true, false, &accepted), "model": m.id}), model)) +} + +/// Removes only files created for this new installation, including on a worker panic. +struct InstallCleanup { + home: PathBuf, + armed: bool, +} +impl Drop for InstallCleanup { + fn drop(&mut self) { + if self.armed { + for name in ["model.onnx.part", "model.onnx", "denoise-model.json", "installed.json"] { + let _ = std::fs::remove_file(self.home.join(name)); + } + let _ = std::fs::remove_dir(&self.home); + } + } +} + +pub(crate) struct InstallJob { + id: String, + activate: bool, + result: mpsc::Receiver), String>>, +} + +fn activate_install(s: &mut Session, result: &Value, model: Arc, activate: bool) -> Result { + let dir = models_dir(s, "denoise.models.install")?; + let id = result["installed"]["id"].as_str().ok_or_else(|| bad("denoise.models.install", "missing installed id"))?; + let mut st = read_settings(&dir); + if activate { + st.model = Some(id.to_string()); + write_settings(&dir, &st).map_err(EngineError::Other)?; + } + s.denoise.touch(); + s.denoise_refresh_active(true); + if let Some(a) = &s.denoise.active + && a.id == id + { + *a.model.lock().unwrap_or_else(PoisonError::into_inner) = Some(model); + } + let mut v = result.clone(); + v["installed"]["selected"] = json!(st.model.as_deref() == Some(id)); + v["model"] = json!(st.model); + Ok(v) +} + +fn start_install(s: &mut Session, src: PathBuf, id: String, activate: bool, verified: bool) -> Result { + const C: &str = "denoise.models.install"; + if s.denoise.install.is_some() { + return Err(bad(C, "another model is being installed")); + } + let dir = models_dir(s, C)?; + let loader = s.denoise.loader.clone(); + let run_on = s.denoise.settings.run_on(); + let at = (s.clock)(); + let (tx, rx) = mpsc::channel(); + std::thread::Builder::new() + .name("denoise-model-install".into()) + .stack_size(16 * 1024 * 1024) + .spawn(move || { + let outcome = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| { + let _held = crate::memory::work_gate().acquire(512 * 1024 * 1024); + install_file(&dir, &loader, run_on, at, C, &src, true, verified).map_err(|e| plain(&e)) + })) + .unwrap_or_else(|_| Err("the model installation stopped unexpectedly".into())); + if verified { + let _ = std::fs::remove_file(&src); + } + let _ = tx.send(outcome); + }) + .map_err(|e| bad(C, format!("could not start installation: {e}")))?; + s.denoise.install = Some(InstallJob { id: id.clone(), activate, result: rx }); + s.denoise.downloads.set_outcome(&id, State::Installing); + Ok(json!({"started": id})) +} + +fn install(s: &mut Session, p: &Value) -> Result { + const C: &str = "denoise.models.install"; + let path = str_param(p, "path").ok_or_else(|| bad(C, "missing `path`"))?; + let acknowledged = bool_or(p, "acknowledged", false); + let activate = bool_or(p, "activate", true); + if !acknowledged { + return Err(bad(C, "the licence has not been accepted: show its terms first")); + } + if bool_or(p, "background", false) { + return start_install(s, PathBuf::from(path), "local-install".into(), activate, false); + } + if s.denoise.install.is_some() { + return Err(bad(C, "another model is being installed")); + } + let (result, model) = + install_file(&models_dir(s, C)?, &s.denoise.loader, s.denoise.settings.run_on(), (s.clock)(), C, Path::new(path), acknowledged, false)?; + activate_install(s, &result, model, activate) +} + +/// Read only the small adjacent manifest; hashing and execution happen in the install worker. +fn inspect(s: &mut Session, p: &Value) -> Result { + const C: &str = "denoise.models.inspect"; + let path = str_param(p, "path").ok_or_else(|| bad(C, "missing `path`"))?; + let src = Path::new(path); + if !src.extension().is_some_and(|e| e.eq_ignore_ascii_case("onnx")) { + return Err(bad(C, "choose an .onnx file with denoise-model.json beside it")); + } + let bytes = read_capped(&src.with_file_name("denoise-model.json"), MAX_MANIFEST_BYTES) + .ok_or_else(|| bad(C, "denoise-model.json is missing or too large"))?; + let m = manifest::parse(&bytes).map_err(|e| bad(C, e.to_string()))?; + let dir = models_dir(s, C)?; + Ok(json!({"kind": "model", "domain": "denoise", "path": path, "fileName": src.file_name().map(|n| n.to_string_lossy()), "model": m, + "sizeBytes": std::fs::metadata(src).ok().map(|m| m.len()), "alreadyInstalled": dir.join(&m.id).join("model.onnx").is_file()})) +} + +/// What an error says, without the command it came from (for a line shown to the user). +fn plain(e: &EngineError) -> String { + match e { + EngineError::BadParams { msg, .. } => msg.clone(), + other => other.to_string(), + } +} + +/// Install every download that has arrived: each was accepted when it was started, so it is installed and chosen without +/// another question. Called by `denoise.models.downloads` and by every frame's `denoise.pump`. +pub(crate) fn finish_downloads(s: &mut Session) { + if let Some(job) = &s.denoise.install { + let outcome = match job.result.try_recv() { + Ok(v) => Some(v), + Err(mpsc::TryRecvError::Empty) => None, + Err(mpsc::TryRecvError::Disconnected) => Some(Err("the installation worker stopped".into())), + }; + if let Some(outcome) = outcome + && let Some(job) = s.denoise.install.take() + { + let state = match outcome { + Ok((v, model)) => match activate_install(s, &v, model, job.activate) { + Ok(_) => State::Installed, + Err(e) => State::Failed(plain(&e)), + }, + Err(e) => State::Failed(e), + }; + s.denoise.downloads.set_outcome(&job.id, state); + } + } + if s.denoise.install.is_none() + && let Some((id, path, _sha)) = s.denoise.downloads.finished().into_iter().next() + && let Err(e) = start_install(s, path, id.clone(), true, true) + { + s.denoise.downloads.set_outcome(&id, State::Failed(plain(&e))); + } +} + +fn download(s: &mut Session, p: &Value) -> Result { + const C: &str = "denoise.models.download"; + let id = str_param(p, "id").ok_or_else(|| bad(C, "missing `id`"))?; + let dir = models_dir(s, C)?; + let k = known::find(id).ok_or_else(|| bad(C, "LightCraft has no download for that model: get the file from its page, then add it"))?; + let spec = k.download.clone().ok_or_else(|| bad(C, "LightCraft has no download for that model: get the file from its page, then add it"))?; + if !cfg!(feature = "denoise") { + return Err(bad(C, "this build cannot run denoise models, so there is nothing to download for it")); + } + if !bool_or(p, "acknowledged", false) { + return Err(bad( + C, + "the licence has not been accepted: show the user the model's terms (see `denoise.models.list`) and pass `acknowledged: true` once they agree", + )); + } + if installed_models(&dir).iter().any(|i| i.manifest.id == spec.id) { + return Err(bad(C, "that model is already installed")); + } + let host = lightcraft_denoise::known::host(&spec.url).to_string(); + s.denoise.downloads.start(spec, &dir).map_err(|e| bad(C, e))?; + Ok(json!({"started": id, "from": host})) +} + +fn download_row(id: &str, state: &State) -> Value { + let from = known::find(id).and_then(|k| host_of(&k)); + match state { + State::Running { bytes, total } => json!({"id": id, "state": "running", "bytes": bytes, "total": total, "from": from}), + State::Done { path, .. } => json!({"id": id, "state": "done", "path": path.display().to_string(), "from": from}), + State::Installing => json!({"id": id, "state": "installing", "from": from}), + State::Installed => json!({"id": id, "state": "installed", "from": from}), + State::Failed(why) => json!({"id": id, "state": "failed", "error": why, "from": from}), + State::Cancelled => json!({"id": id, "state": "cancelled", "from": from}), + } +} + +fn downloads(s: &mut Session, _: &Value) -> Result { + finish_downloads(s); + let all: Vec = s.denoise.downloads.snapshot().iter().map(|(id, st)| download_row(id, st)).collect(); + Ok(json!({"running": s.denoise.downloads.running(), "downloads": all})) +} + +fn download_cancel(s: &mut Session, p: &Value) -> Result { + const C: &str = "denoise.models.downloadCancel"; + let id = str_param(p, "id").ok_or_else(|| bad(C, "missing `id`"))?; + Ok(json!({"discarded": s.denoise.downloads.discard(id)})) +} + +fn test(s: &mut Session, p: &Value) -> Result { + const C: &str = "denoise.models.test"; + let id = str_param(p, "id").ok_or_else(|| bad(C, "missing `id`"))?; + let dir = models_dir(s, C)?; + let found = installed_models(&dir).into_iter().find(|i| i.manifest.id == id).ok_or_else(|| bad(C, "that model is not installed"))?; + let run_on = s.denoise.settings.run_on(); + let result = (s.denoise.loader)(&found.onnx, &found.manifest).and_then(|m| m.self_test(run_on)); + let (ok, detail) = match &result { + Ok(v) => (true, v.clone()), + Err(e) => (false, json!(e)), + }; + // keep the latest outcome with the model's record + let mut record = found.accepted; + if let Some(o) = record.as_object_mut() { + o.insert("selfTest".into(), if ok { detail.clone() } else { json!({"ok": false, "error": detail}) }); + write_atomic(&found.onnx.with_file_name("installed.json"), &serde_json::to_vec_pretty(&record).map_err(|e| fail("record", e))?) + .map_err(EngineError::Other)?; + } + Ok(json!({"id": id, "ok": ok, "result": detail})) +} + +fn remove(s: &mut Session, p: &Value) -> Result { + const C: &str = "denoise.models.remove"; + let id = str_param(p, "id").ok_or_else(|| bad(C, "missing `id`"))?; + let dir = models_dir(s, C)?; + if !lightcraft_denoise::licence::valid_id(id) { + return Err(bad(C, "not a model id")); + } + let home = dir.join(id); + // only a folder that is a model (it has a manifest) is ever deleted + if !home.join("denoise-model.json").is_file() { + return Err(bad(C, "that model is not installed")); + } + std::fs::remove_dir_all(&home).map_err(|e| fail("could not remove the model", e))?; + let mut st = read_settings(&dir); + if st.model.as_deref() == Some(id) { + st.model = installed_models(&dir).into_iter().map(|i| i.manifest.id).find(|m| m != id); + write_settings(&dir, &st).map_err(EngineError::Other)?; + } + s.denoise.touch(); + s.denoise_refresh_active(true); + Ok(json!({"removed": id, "model": st.model})) +} + +fn select(s: &mut Session, p: &Value) -> Result { + const C: &str = "denoise.models.select"; + let dir = models_dir(s, C)?; + let id = match p.get("id") { + None | Some(Value::Null) => None, + Some(v) => Some(v.as_str().ok_or_else(|| bad(C, "`id` must be a model id or null"))?.to_string()), + }; + if let Some(id) = &id + && !installed_models(&dir).iter().any(|i| &i.manifest.id == id) + { + return Err(bad(C, "that denoise model is not installed")); + } + let mut st = read_settings(&dir); + st.model = id; + write_settings(&dir, &st).map_err(EngineError::Other)?; + s.denoise.touch(); + s.denoise_refresh_active(true); + Ok(json!({"model": st.model})) +} + +fn settings(s: &mut Session, p: &Value) -> Result { + const C: &str = "denoise.settings"; + let dir = models_dir(s, C)?; + let mut st: Settings = read_settings(&dir); + if let Some(v) = p.get("auto") { + st.auto = Some(v.as_bool().ok_or_else(|| bad(C, "`auto` must be true or false"))?); + } + if let Some(v) = p.get("cacheGb") { + let n = + v.as_u64().filter(|n| (1..=10_000).contains(n)).ok_or_else(|| bad(C, "`cacheGb` must be a whole number of gigabytes from 1 to 10000"))?; + st.cache_gb = Some(n as u32); + } + if let Some(v) = p.get("threads") { + st.threads = match v { + Value::Null => None, + v => Some(v.as_u64().filter(|n| (1..=64).contains(n)).ok_or_else(|| bad(C, "`threads` must be from 1 to 64, or null"))? as u32), + }; + } + let run_on = match p.get("runOn") { + None => None, + Some(v) => Some(v.as_str().and_then(RunOn::parse).ok_or_else(|| bad(C, "`runOn` must be auto, gpu or cpu"))?), + }; + if let Some(r) = run_on { + st.run_on = Some(r.name().to_string()); + st.gpu = None; + } + write_settings(&dir, &st).map_err(EngineError::Other)?; + if run_on.is_some() { + // a choice of where denoise runs is also a go-ahead to try the card again: a set-up that crashed or gave up + // before is forgotten, and the loaded model sets the card up afresh for the next photo + forget_gpu_set_ups(&dir); + if let Some(a) = &s.denoise.active { + *a.model.lock().unwrap_or_else(PoisonError::into_inner) = None; + } + } + s.denoise.touch(); + s.denoise_refresh_active(true); + // a smaller limit takes effect now + s.denoise_evict(None); + Ok(json!({"auto": st.auto(), "cacheGb": st.cache_bytes() >> 30, "threads": st.threads, "runOn": st.run_on().name()})) +} + +fn ids_param(p: &Value) -> Option> { + p.get("ids").and_then(Value::as_array).map(|a| a.iter().filter_map(Value::as_u64).map(PhotoId).collect()) +} + +fn status_json(s: &Session, photo: Option) -> Value { + let (files, bytes) = s.denoise_cache_usage(); + let st = &s.denoise.settings; + let model = s.denoise.active.as_ref().map(|a| json!({"id": a.id, "name": a.manifest.name})); + let mut v = json!({ + "enabled": s.denoise.active.is_some(), + "model": model, + "runtime": cfg!(feature = "denoise"), + "productsDir": s.denoise_products_dir().map(|d| d.display().to_string()), + "queued": s.denoise.queued(), + "running": s.denoise_running_json(), + "ready": s.media.denoise.len(), + "made": s.denoise_made(), + "auto": st.auto(), + "runOn": st.run_on().name(), + "device": s.denoise_device(), + "cache": {"files": files, "bytes": bytes, "limitBytes": st.cache_bytes()}, + "generation": s.denoise.generation, + }); + if let (Some(id), Some(o)) = (photo, v.as_object_mut()) { + o.insert("photo".into(), serde_json::to_value(s.denoise_photo_state(id)).unwrap_or(Value::Null)); + } + v +} + +/// A per-photo switch, independent of the selection and Auto Sync. Turning on starts at 50%. +fn toggle(s: &mut Session, p: &Value) -> Result { + const C: &str = "denoise.toggle"; + let id = match p.get("id") { + Some(v) => PhotoId(v.as_u64().ok_or_else(|| bad(C, "`id` must be a photo id"))?), + None => s.active().ok_or_else(|| bad(C, "select a raw photo"))?, + }; + let photo = s.catalog.photo(id).ok_or_else(|| bad(C, "no such photo"))?; + if !crate::denoise::eligible(photo) { + return Err(bad(C, "AI Denoise applies to raw photos developed from their sensor data")); + } + let mut d = (*s.develop_of(id).ok_or_else(|| bad(C, "no such photo"))?).clone(); + let on = match p.get("enabled") { + Some(v) => v.as_bool().ok_or_else(|| bad(C, "`enabled` must be true or false"))?, + None => !d.enhance.denoise_enabled(), + }; + if on { + if !d.enhance.denoise.is_finite() { + return Err(bad(C, "the photo has an invalid Denoise amount")); + } + s.denoise_refresh_active(true); + if s.denoise.active.is_none() { + return Err(bad(C, "AI Denoise needs a model: install one in Settings > AI Denoise")); + } + } + // The switch is its own setting: the Amount stays where it is (0 included) when it is flipped, so flipping compares + // with and without. A photo that never had an Amount starts at 50 the first time. + let amount = if on && d.enhance.denoise_on.is_none() && d.enhance.denoise <= 0.0 { 50.0 } else { d.enhance.denoise }; + if d.enhance.denoise != amount || d.enhance.denoise_on != Some(on) { + s.end_interaction()?; + d.enhance.denoise = amount; + d.enhance.denoise_on = Some(on); + let op = s.develop_op(id, d, "AI Denoise").ok_or_else(|| bad(C, "no such photo"))?; + s.commit("AI Denoise", op)?; + } + if on { + s.denoise_enqueue(&[id], true, false); + } else { + s.denoise_cancel(Some(id), false); + } + Ok(json!({"id": id.0, "enabled": on, "amount": amount})) +} + +fn queue(s: &mut Session, p: &Value) -> Result { + const C: &str = "denoise.queue"; + s.denoise_refresh_active(true); + if s.denoise.active.is_none() { + return Err(bad(C, "no denoise model is chosen: install one first (`denoise.models.download` or `denoise.models.install`)")); + } + if s.denoise_products_dir().is_none() { + return Err(bad(C, "denoise keeps its pictures in the library's folder: open a library on disk first")); + } + let ids = match ids_param(p) { + Some(ids) => ids, + None => match str_param(p, "scope").unwrap_or("selected") { + "selected" => s.selection.active.iter().chain(s.selection.ids.iter()).copied().collect(), + "visible" => s.visible_cloned().to_vec(), + "withAmount" => s.catalog.photos().filter(|ph| ph.develop.denoise_amount() > 0.0).map(|ph| ph.id).collect(), + other => return Err(bad(C, format!("unknown scope `{other}`: use selected, visible or withAmount"))), + }, + }; + let queued = s.denoise_enqueue(&ids, false, bool_or(p, "retry", false)); + let mut v = status_json(s, None); + if let Some(o) = v.as_object_mut() { + o.insert("added".into(), json!(queued)); + } + Ok(v) +} + +fn cancel(s: &mut Session, p: &Value) -> Result { + let photo = p.get("id").and_then(Value::as_u64).map(PhotoId); + let n = s.denoise_cancel(photo, bool_or(p, "all", photo.is_none())); + Ok(json!({"cancelled": n})) +} + +fn status(s: &mut Session, p: &Value) -> Result { + s.denoise_refresh_active(false); + Ok(status_json(s, p.get("id").and_then(Value::as_u64).map(PhotoId))) +} + +/// `denoise.pump {pace?}`: cheap to call every frame. Takes in what the background job has finished and keeps one going. +fn pump(s: &mut Session, p: &Value) -> Result { + s.denoise_pump(Pace::parse(p.get("pace").and_then(Value::as_str))); + Ok(json!({ + "active": s.denoise.active.is_some(), + "running": s.denoise_running_json(), + "queued": s.denoise.queued(), + "ready": s.media.denoise.len(), + "generation": s.denoise.generation, + "downloading": s.denoise.downloads.running(), + })) +} + +fn clear(s: &mut Session, _: &Value) -> Result { + let (files, bytes) = s.denoise_clear(); + Ok(json!({"deleted": files, "bytes": bytes})) +} + +pub fn specs() -> Vec { + vec![ + cmd!(query "denoise.models.list", "Denoise Models", [], None, "{} → {dir, productsDir, model, runtime, auto, cacheGb, threads, runOn, models: [{id, name, version, licence{name, commercial, url, notice}, provenance, source, sizeBytes, sha256, tile, known, downloadHost, installed, selected, accepted}]}", always, list), + cmd!(query "denoise.models.inspect", "Inspect Denoise Model", [], None, "{path} → model terms from an adjacent manifest; never executes the file", always, inspect), + cmd!(query "denoise.models.install", "Install Denoise Model", [], None, "{path, acknowledged: true, activate?: true, background?: false} → {installed, model} or {started} — install a denoise model from a file: an .onnx with a denoise-model.json beside it, or an archive LightCraft knows (the darktable `.dtmodel`). `acknowledged` must be true: the user has been shown the model's licence and accepted it. Unless `activate` is false it becomes the model in use", always, install), + cmd!(query "denoise.models.download", "Download Denoise Model", [], None, "{id, acknowledged: true} → {started, from} — fetch a model LightCraft has a pinned address for (see `downloadHost` in the list) in the background over HTTPS with lightcraft-fetch. `acknowledged` must be true: the user has been shown the model's terms and accepted them. It is checked against its size and SHA-256, installed and chosen by itself; `denoise.models.downloads` shows how far it is", always, download), + cmd!(query "denoise.models.downloads", "Denoise Model Downloads", [], None, "{} → {running, downloads: [{id, state: running | done | installed | failed | cancelled, bytes, total, error, from}]} — also installs any download that has arrived", always, downloads), + cmd!(query "denoise.models.downloadCancel", "Cancel Denoise Model Download", [], None, "{id} → {discarded} — stop a download, or delete a finished one that was not installed", always, download_cancel), + cmd!(query "denoise.models.test", "Test Denoise Model", [], None, "{id} → {ok, result} — load an installed model and check it gives sensible pictures; needs the denoise runtime", always, test), + cmd!(query "denoise.models.remove", "Remove Denoise Model", [], None, "{id} → {removed, model} — delete an installed model (its cached pictures stay until they are evicted)", always, remove), + cmd!(query "denoise.models.select", "Choose Denoise Model", [], None, "{id: installed model | null} → {model} — null switches denoise off", always, select), + cmd!(query "denoise.settings", "Denoise Settings", [], None, "{auto?: bool, cacheGb?: 1..10000, threads?: 1..64 | null, runOn?: auto | gpu | cpu} → the settings — `auto`: make the picture of the photos being looked at that have a Denoise amount; `cacheGb`: how much the cached pictures may take; `threads`: most tiles run at once; `runOn`: where the model runs — `auto` (the default) the graphics card where it is faster than the processor (both timed on this computer when the card is set up), `gpu` the card whenever it can run the model, `cpu` the processor. Setting `runOn` also lets the card be tried again after a set-up that failed or closed LightCraft", always, settings), + cmd!( + "denoise.toggle", + "AI Denoise", + [], + None, + "{id?: photoId, enabled?: bool} → {id, enabled, amount} — turn AI Denoise on (50% initially) or off for one raw photo, independent of selection and Auto Sync; needs an installed model to turn on", + always, + toggle + ), + cmd!(query "denoise.queue", "Denoise Photos", [], None, "{ids?: [photoId], scope?: selected | visible | withAmount, retry?: false} → {added, …status} — make the denoised pictures of raw photos that have none, in the background. `retry` tries photos that failed before", always, queue), + cmd!(query "denoise.cancel", "Cancel Denoise", [], None, "{id?, all?} → {cancelled} — stop the photo being made and/or forget the ones waiting (all of them when `id` is omitted)", always, cancel), + cmd!(query "denoise.status", "Denoise Status", [], None, "{id?} → {enabled, model, runtime, productsDir, queued, running: {photo, done, total, seconds} | null, ready, made, auto, runOn, device: {kind: gpu | cpu | pending | none, adapter?, reason?, cardMs?, cpuMs?, fellBack?}, cache: {files, bytes, limitBytes}, generation, photo?: {state: notApplicable | noModel | ready | queued | running | failed | idle, …}}", always, status), + cmd!(query "denoise.pump", "Denoise Pump", [], None, "{pace?: pause | light | normal | full} → {active, running, queued, ready, generation, downloading} — called every frame by the app: takes in finished work and keeps one photo going", always, pump), + cmd!(query "denoise.clear", "Clear Denoise Cache", [], None, "{} → {deleted, bytes} — delete every cached denoised picture of this library (they are made again when wanted)", always, clear), + ] +} diff --git a/crates/engine/src/cmd/mod.rs b/crates/engine/src/cmd/mod.rs index 303a92d4..ef838fe0 100644 --- a/crates/engine/src/cmd/mod.rs +++ b/crates/engine/src/cmd/mod.rs @@ -10,6 +10,7 @@ mod color; pub(crate) mod convert; mod cull; pub mod curves; +pub(crate) mod denoise; mod develop; mod edit; mod export; @@ -143,6 +144,7 @@ pub fn command_specs() -> &'static [CommandSpec] { v.extend(missing::specs()); v.extend(metadata::specs()); v.extend(filters::specs()); + v.extend(denoise::specs()); v.extend(face_models::specs()); v.extend(face_detect::specs()); v.extend(face_recognize::specs()); diff --git a/crates/engine/src/config.rs b/crates/engine/src/config.rs index 933dc22d..7ab9b653 100644 --- a/crates/engine/src/config.rs +++ b/crates/engine/src/config.rs @@ -1,9 +1,10 @@ -//! Where LightCraft keeps per-user data on this machine: the config folder (app settings, face models). +//! Where LightCraft keeps per-user data on this machine: the config folder (app settings, camera +//! profiles, face and denoise models). //! //! Hosts that have a file system (the desktop app, the CLI, the MCP server) share these defaults so a model //! installed from one is there for the others. Nothing here is applied automatically: a [`Session`] has no -//! face-models folder until a host asks for one ([`Session::with_default_face_models`]), so tests stay -//! hermetic. +//! face- or denoise-models folder until a host asks for one ([`Session::with_default_face_models`], +//! [`Session::with_default_denoise_models`]), so tests stay hermetic. use std::path::PathBuf; @@ -28,10 +29,30 @@ pub fn default_face_models_dir() -> Option { std::env::var_os("LIGHTCRAFT_FACE_MODELS").filter(|v| !v.is_empty()).map(PathBuf::from).or_else(|| config_dir().map(|d| d.join("models"))) } +/// Where opt-in denoise models are kept: `$LIGHTCRAFT_DENOISE_MODELS` if set, else `/denoise-models`. +pub fn default_denoise_models_dir() -> Option { + std::env::var_os("LIGHTCRAFT_DENOISE_MODELS") + .filter(|v| !v.is_empty()) + .map(PathBuf::from) + .or_else(|| config_dir().map(|d| d.join("denoise-models"))) +} + impl Session { /// Keep face models in the shared default folder ([`default_face_models_dir`]). pub fn with_default_face_models(mut self) -> Self { self.face_models_dir = default_face_models_dir(); self } + + /// Keep denoise models in the shared default folder ([`default_denoise_models_dir`]). + pub fn with_default_denoise_models(mut self) -> Self { + self.set_denoise_models_dir(default_denoise_models_dir()); + self + } + + /// Point AI denoise at `dir` (or at no folder), dropping what was learned about the old one. + pub fn set_denoise_models_dir(&mut self, dir: Option) { + self.denoise.models_dir = dir; + self.denoise.touch(); + } } diff --git a/crates/engine/src/demo.rs b/crates/engine/src/demo.rs index f9d8b7c9..48da21e2 100644 --- a/crates/engine/src/demo.rs +++ b/crates/engine/src/demo.rs @@ -96,3 +96,143 @@ pub fn load(s: &mut Session) { s.selection = crate::Selection::single(*first); } } + +/// Procedural scenes are deterministic but costly to evaluate: every pixel runs the scene's noise +/// functions, so a 24 MP demo original takes minutes of CPU time on a small machine (a 4-core CI +/// VM). Each scene at each size is therefore generated once per process and shared: +/// +/// - concurrent requests wait for one generation (Before and After at 1:1 both load the +/// original; sessions side by side, e.g. parallel tests, open the same photo); +/// - the most recently used results are kept, up to [`scene_cache_limit`] bytes. They are the same +/// allocations the sessions' source caches hold, so they cost extra memory only once no session +/// holds them any more. +/// +/// Only demo photos use this; real files are decoded by their loader as before. +pub(crate) fn scene_pixels(scene: &lightcraft_scenes::Scene, max_edge: usize) -> Arc { + let key = SceneKey { kind: scene.kind, seed: scene.seed, width: scene.width, height: scene.height, max_edge }; + let slot = { + let mut c = scene_cache(); + c.tick += 1; + let tick = c.tick; + match c.entries.iter_mut().find(|e| e.key == key) { + Some(e) => { + e.used = tick; + e.slot.clone() + } + None => { + let slot = Arc::new(std::sync::OnceLock::new()); + c.entries.push(SceneEntry { key, slot: slot.clone(), used: tick }); + slot + } + } + }; + // (no lock held while generating: other scenes and sizes are served meanwhile) + let pixels = slot.get_or_init(|| Arc::new(scene.render_fit(max_edge))).clone(); + trim_scene_cache(&mut scene_cache(), scene_cache_limit()); + pixels +} + +/// Bytes of generated scenes kept for reuse: half the memory budget (768 MB by default; enough +/// for one 24 MP original and the previews around it). +fn scene_cache_limit() -> usize { + crate::memory::budget() / 2 +} + +#[derive(Clone, Copy, PartialEq, Eq, Debug)] +struct SceneKey { + kind: lightcraft_scenes::Kind, + seed: u32, + width: u32, + height: u32, + max_edge: usize, +} + +type SceneSlot = Arc>>; + +struct SceneEntry { + key: SceneKey, + slot: SceneSlot, + used: u64, +} + +#[derive(Default)] +struct SceneCache { + entries: Vec, + tick: u64, +} + +fn scene_cache() -> std::sync::MutexGuard<'static, SceneCache> { + static CACHE: std::sync::OnceLock> = std::sync::OnceLock::new(); + CACHE.get_or_init(Default::default).lock().unwrap_or_else(std::sync::PoisonError::into_inner) +} + +fn scene_bytes(slot: &SceneSlot) -> usize { + slot.get().map_or(0, |img| img.data.len().saturating_mul(std::mem::size_of::<[f32; 3]>())) +} + +/// Drop the least recently used generated scenes until the rest fit in `limit` bytes. Scenes still +/// being generated are kept (their waiters hold the slot anyway). +fn trim_scene_cache(c: &mut SceneCache, limit: usize) { + loop { + let total: usize = c.entries.iter().map(|e| scene_bytes(&e.slot)).fold(0, usize::saturating_add); + if total <= limit { + return; + } + let Some(i) = c.entries.iter().enumerate().filter(|(_, e)| e.slot.get().is_some()).min_by_key(|(_, e)| e.used).map(|(i, _)| i) else { + return; + }; + c.entries.swap_remove(i); + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn small_scene() -> lightcraft_scenes::Scene { + lightcraft_scenes::demo_library().swap_remove(0) + } + + #[test] + fn a_scene_is_generated_once_per_size_and_shared() { + let s = small_scene(); + let a = scene_pixels(&s, 48); + let b = scene_pixels(&s, 48); + assert!(Arc::ptr_eq(&a, &b), "the second request reuses the first generation"); + assert_eq!(a.width.max(a.height), 48); + assert_eq!(a.data, s.render_fit(48).data, "the same pixels as generating it directly"); + let c = scene_pixels(&s, 40); + assert_eq!(c.width.max(c.height), 40, "another size is its own generation"); + } + + #[test] + fn concurrent_requests_wait_for_one_generation() { + let s = small_scene(); + let edge = 37; // a size no other test asks for + let got: Vec<_> = std::thread::scope(|sc| { + let hs: Vec<_> = (0..4).map(|_| sc.spawn(|| scene_pixels(&s, edge))).collect(); + hs.into_iter().map(|h| h.join().unwrap()).collect() + }); + assert!(got.windows(2).all(|w| Arc::ptr_eq(&w[0], &w[1])), "every request got the same pixels"); + } + + #[test] + fn the_cache_keeps_the_most_recent_scenes_within_its_limit() { + let key = |max_edge| SceneKey { kind: lightcraft_scenes::Kind::Dunes, seed: 1, width: 3, height: 2, max_edge }; + let entry = |max_edge, used, px: Option| { + let slot: SceneSlot = Arc::new(std::sync::OnceLock::new()); + if let Some(n) = px { + let _ = slot.set(Arc::new(lightcraft_raster::Rgb32f::new(n, 1))); + } + SceneEntry { key: key(max_edge), slot, used } + }; + let px = std::mem::size_of::<[f32; 3]>(); + let mut c = SceneCache { entries: vec![entry(1, 3, Some(10)), entry(2, 1, Some(10)), entry(3, 0, None), entry(4, 2, Some(10))], tick: 3 }; + trim_scene_cache(&mut c, 25 * px); + let mut kept: Vec = c.entries.iter().map(|e| e.key.max_edge).collect(); + kept.sort_unstable(); + assert_eq!(kept, vec![1, 3, 4], "the least recently used finished scene went; the one in progress stays"); + trim_scene_cache(&mut c, 0); + assert_eq!(c.entries.iter().map(|e| e.key.max_edge).collect::>(), vec![3], "only the scene in progress is left"); + } +} diff --git a/crates/engine/src/denoise.rs b/crates/engine/src/denoise.rs new file mode 100644 index 00000000..01bcdc78 --- /dev/null +++ b/crates/engine/src/denoise.rs @@ -0,0 +1,1644 @@ +//! AI denoise, the session's side (`docs/denoise.md`): which photos have a denoised picture, making the missing ones +//! in the background, and keeping the cache within its size. +//! +//! The denoised picture of a raw photo is *cached data*, never a file in the library: it is made from the untouched +//! original by the model the user chose, kept in `/denoise/.lcdn`, and thrown away whenever space is +//! needed (it is made again when it is next wanted). Its name is a hash of the model, the algorithm and the photo's +//! content, so a changed file or another model simply does not find it. The Denoise *amount* is an ordinary develop +//! setting (`enhance.denoise`): a render mixes the plain and the denoised picture by it ([`crate::media`]). +//! +//! What runs where: the session thread only decides (which photos, which model) and keeps an index of what is ready +//! ([`crate::media::DenoiseIndex`], memory only). One background thread at a time decodes a photo, runs the model on +//! its mosaic tile by tile and writes the product ([`make_product`]); an export that finds its photo's product missing +//! makes it itself, on its own thread, so it never comes out without the denoise it was asked for. +//! +//! Failure is a message, never a panic: a file that is not a Bayer raw, a model that does not load, a full disk, a +//! panic inside the runtime and a cancelled job all end as an [`Outcome`] the pump records. + +use std::collections::{HashMap, HashSet, VecDeque}; +use std::path::{Path, PathBuf}; +use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering}; +use std::sync::{Arc, Condvar, Mutex, OnceLock, PoisonError}; +use std::time::{Duration, SystemTime}; + +use lightcraft_catalog::{Photo, PhotoId, Source}; +use lightcraft_denoise::bayer::Layout; +use lightcraft_denoise::manifest::DenoiserManifest; +use lightcraft_denoise::run::{Control, Params, TileRunner, denoise_bayer}; +use lightcraft_denoise::{Error as RunError, product}; +use lightcraft_develop::DevelopSettings; +use lightcraft_preview::Hasher128; +use serde::{Deserialize, Serialize}; +use serde_json::{Value, json}; + +use crate::media::{DenoiseSpec, MakeProduct, PairLoader}; +use crate::merge::ByteReader; +use crate::model_download::Downloads; +use crate::{Selection, Session}; + +/// Extension of a product file. +pub const EXT: &str = "lcdn"; +/// Where the products of a library are kept, inside the library's folder (safe to delete, like `thumbs/`). +pub const DIR: &str = "denoise"; +/// What the cache may take up unless the user says otherwise (GB). A product is about as large as the raw file's +/// decoded samples, compressed: ~100 MB for 24 megapixels. +pub const DEFAULT_CACHE_GB: u32 = 20; +/// Samples at or above this (white = 1) are clipped: the model leaves them as the camera recorded them. +const CLIP: f32 = 0.99; +/// Most photos waiting in the queue. +const MAX_QUEUE: usize = 5000; +/// Most selected photos the automatic queue looks at in one pump. +const AUTO_LOOK: usize = 64; +/// How long a look at the model settings is trusted. +const SETTINGS_TTL: Duration = Duration::from_secs(2); +/// Stack of the threads that run the model (the runtime recurses through its graph). +const STACK: usize = 16 << 20; + +// --------------------------------------------------------------------------------------------------------------------- +// the model + +/// Where the model runs (Settings ▸ AI Denoise ▸ Speed; `denoise.settings {runOn}`). +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] +pub enum RunOn { + /// The graphics card where it is faster than the processor at the pace the work goes at (both are timed on this + /// computer when the card is set up), else the processor. + #[default] + Auto, + /// The graphics card whenever it can run the model, even where the processor would be faster. + Gpu, + /// Always the processor. + Cpu, +} + +impl RunOn { + /// `auto`, `gpu` or `cpu`. + pub fn parse(name: &str) -> Option { + match name { + "auto" => Some(RunOn::Auto), + "gpu" => Some(RunOn::Gpu), + "cpu" => Some(RunOn::Cpu), + _ => None, + } + } + + pub fn name(self) -> &'static str { + match self { + RunOn::Auto => "auto", + RunOn::Gpu => "gpu", + RunOn::Cpu => "cpu", + } + } +} + +/// A loaded denoise model: what runs tiles, and a self-check. +pub(crate) trait Model: Send + Sync { + /// What runs a photo's tiles, and how many to run at once, when `threads` may be used and the setting is `run_on` + /// (a graphics card is kept busy by a few). The first time the card may be used it is set up and checked here. + fn runner(&self, run_on: RunOn, threads: usize) -> (&dyn TileRunner, usize); + /// Check it does something sensible: the test's result, or why not. Unless `run_on` is the processor the graphics + /// card is set up and checked too, and the result says where the work will run. + fn self_test(&self, run_on: RunOn) -> Result; + /// Conservative scratch bound for an unknown runner. + fn bytes_per_tile(&self) -> usize { + 512 * 1024 * 1024 + } + /// Where the tiles run: `{kind: "gpu", adapter}`, `{kind: "cpu", reason?}` or `{kind: "pending"}` (the card is not + /// set up yet), with the card's and the processor's time for the check tile (`cardMs`, `cpuMs`) once both were + /// timed. Never starts anything. + fn device(&self, _run_on: RunOn) -> Value { + json!({"kind": "cpu"}) + } +} + +/// Loads the model at a path as its manifest describes it. +pub(crate) type Loader = Arc Result, String> + Send + Sync>; + +/// Tiles run at once on a graphics card: enough to keep it fed while others are packed and blended. +#[cfg(feature = "denoise")] +const GPU_PARALLEL: usize = 4; +/// How far the GPU's answer on the check tile may be from the CPU's (of the answer's largest value) before it is not used. +#[cfg(feature = "denoise")] +const GPU_AGREE: f32 = 1e-3; +/// Longest the card may take to set up (build its kernels, run the check tile, time it) before the processor does the +/// work instead. +#[cfg(feature = "denoise")] +const GPU_SETUP_LIMIT: Duration = Duration::from_secs(30); +/// Tiles the card may get wrong (an error, numbers that are not finite) before it stops being used for the model. +#[cfg(feature = "denoise")] +const GPU_MAX_FALLBACKS: usize = 3; +/// The file beside a model while the graphics card is set up for it (see [`set_up_card`]). +pub(crate) const GPU_SETUP_MARKER: &str = "gpu-setup.marker"; +/// Why the card is not tried when a set-up marker is found. +#[cfg(feature = "denoise")] +const GPU_CRASHED: &str = "LightCraft closed while it was setting up the graphics card for AI Denoise last time"; +/// Why the card is not used when its set-up ran out of time (the start of it). +#[cfg(feature = "denoise")] +const GPU_SLOW: &str = "setting up the graphics card took longer than"; +/// Why the card is not used while a set-up that ran out of time is still going. +#[cfg(feature = "denoise")] +const GPU_STUCK: &str = "the graphics card is still setting up from before (it took too long)"; + +/// A set-up that may work another time (a crash, the time limit), unlike one the card or the model cannot do. +#[cfg(feature = "denoise")] +fn worth_retrying(why: &str) -> bool { + why == GPU_CRASHED || why == GPU_STUCK || why.starts_with(GPU_SLOW) +} + +/// The model on our pure-Rust CPU runner, and on the graphics card when there is one that can run it: the card is set up for the +/// first photo that may use it, checked against the CPU's answer on a check tile and timed against it, and used while +/// it keeps working; a tile it gets wrong is run on the CPU, so a photo is always finished. +#[cfg(feature = "denoise")] +struct Network { + cpu: lightcraft_denoise::runtime::CpuRunner, + path: PathBuf, + tile: usize, + gpu: OnceLock>, + /// Tiles the card got wrong (an error, numbers that are not finite) and the CPU ran. + fallbacks: AtomicUsize, + /// Tiles at once of the photo started last (0: none yet), to say where the work runs. + last_threads: AtomicUsize, +} + +#[cfg(feature = "denoise")] +struct GpuSide { + runner: Box, + adapter: String, + /// The check tile on the card, the fastest of a few runs (ms). + card_ms: f64, + /// The check tile on the CPU, on one thread (ms). + cpu_ms: f64, +} + +/// Whether the card (`card_ms` a tile) is faster than the CPU running `threads` tiles at once, from the CPU's time for +/// one tile alone (`cpu_ms`). Tiles at once share the memory and the cores' second threads, so each one more counts as +/// half a tile's speed: on a 16-core laptop one tile alone took 1.4 s and sixteen at once finished one every 0.2 s. +#[cfg(feature = "denoise")] +fn card_is_faster(card_ms: f64, cpu_ms: f64, threads: usize) -> bool { + let at_once = 1.0 + 0.5 * threads.saturating_sub(1) as f64; + card_ms.is_finite() && card_ms <= cpu_ms / at_once +} + +/// Card set-ups of this process that have not finished, by marker (their markers are not a crash's). +#[cfg(feature = "denoise")] +static SETTING_UP: Mutex> = Mutex::new(Vec::new()); + +/// Removes a set-up's marker when the set-up ends, however it ends — but not when the process dies inside the driver. +#[cfg(feature = "denoise")] +struct SetUpEnded(PathBuf); + +#[cfg(feature = "denoise")] +impl Drop for SetUpEnded { + fn drop(&mut self) { + let _ = std::fs::remove_file(&self.0); + SETTING_UP.lock().unwrap_or_else(PoisonError::into_inner).retain(|m| m != &self.0); + } +} + +/// Set the graphics card up with `card` on a thread of its own while `meanwhile` runs here, and give up on the card when +/// it takes longer than `limit` (a driver building kernels for minutes, or stuck). The `marker` file exists while `card` +/// runs: one found before it starts was left by a LightCraft that closed during a set-up (a driver crash takes the app +/// with it), so the card is not tried again until the user chooses where denoise runs (`denoise.settings {runOn}`). +#[cfg(feature = "denoise")] +fn set_up_card( + marker: &Path, + limit: Duration, + card: impl FnOnce() -> Result + Send + 'static, + meanwhile: impl FnOnce() -> R, +) -> Result<(T, R), String> { + let ended = { + let mut running = SETTING_UP.lock().unwrap_or_else(PoisonError::into_inner); + if running.iter().any(|m| m == marker) { + return Err(GPU_STUCK.into()); + } + if marker.exists() { + return Err(GPU_CRASHED.into()); + } + let secs = SystemTime::now().duration_since(SystemTime::UNIX_EPOCH).map_or(0, |d| d.as_secs()); + // a marker that cannot be written only loses the crash guard + let _ = std::fs::write(marker, format!("setting up the graphics card for AI Denoise (unix time {secs})\n")); + running.push(marker.to_path_buf()); + SetUpEnded(marker.to_path_buf()) + }; + let started = web_time::Instant::now(); + let (tx, rx) = std::sync::mpsc::channel(); + std::thread::Builder::new() + .name("denoise-gpu-setup".into()) + .stack_size(STACK) + .spawn(move || { + let r = std::panic::catch_unwind(std::panic::AssertUnwindSafe(card)).unwrap_or_else(|_| Err("the GPU runner gave up".into())); + drop(ended); + let _ = tx.send(r); + }) + .map_err(|e| format!("could not start setting up the graphics card: {e}"))?; + let here = meanwhile(); + match rx.recv_timeout(limit.saturating_sub(started.elapsed())) { + Ok(r) => r.map(|t| (t, here)), + Err(std::sync::mpsc::RecvTimeoutError::Timeout) => Err(format!("{GPU_SLOW} {} s, so it is not used", limit.as_secs_f64())), + Err(std::sync::mpsc::RecvTimeoutError::Disconnected) => Err("the GPU runner gave up".into()), + } +} + +/// The card's time for one tile once it is warm: the fastest of up to three runs, stopping after about two seconds (a +/// slow card is slow enough to tell at once). +#[cfg(feature = "denoise")] +fn card_tile_ms(runner: &dyn TileRunner, input: &[f32]) -> f64 { + if runner.run(input).is_err() { + return f64::INFINITY; + } + let began = web_time::Instant::now(); + let mut best = f64::INFINITY; + for _ in 0..3 { + let one = web_time::Instant::now(); + if runner.run(input).is_err() { + break; + } + best = best.min(one.elapsed().as_secs_f64() * 1000.0); + if began.elapsed() > Duration::from_secs(2) { + break; + } + } + best +} + +/// Set the card up for the model at `path`: kernels built, the check tile run on both and compared, both timed. +#[cfg(feature = "denoise")] +fn make_gpu(cpu: &lightcraft_denoise::runtime::CpuRunner, path: &Path, tile: usize) -> Result { + let input = Arc::new(lightcraft_denoise::runtime::check_tile(tile)); + let (on_card, net) = (input.clone(), cpu.net().clone()); + let card = move || -> Result<_, String> { + let runner = lightcraft_gpu::nn::runner(&net, tile)?; + let got = runner.run(&on_card).map_err(|e| format!("the GPU could not run the check tile: {e}"))?; + let card_ms = card_tile_ms(&runner, &on_card); + Ok((runner, got, card_ms)) + }; + let on_cpu = || { + let started = web_time::Instant::now(); + (cpu.run(&input), started.elapsed().as_secs_f64() * 1000.0) + }; + let ((runner, got, card_ms), (want, cpu_ms)) = set_up_card(&path.with_file_name(GPU_SETUP_MARKER), GPU_SETUP_LIMIT, card, on_cpu)?; + let want = want.map_err(|e| format!("the CPU could not check the GPU: {e}"))?; + let scale = want.iter().fold(0f32, |m, v| m.max(v.abs())).max(1e-12); + let worst = want.iter().zip(&got).fold(0f32, |m, (a, b)| m.max((a - b).abs())) / scale; + if got.len() != want.len() || !got.iter().all(|v| v.is_finite()) || worst > GPU_AGREE { + return Err(format!("its answer on the check tile is not the CPU's (off by {worst:.1e} of the largest value): not used")); + } + Ok(GpuSide { adapter: runner.adapter(), runner: Box::new(runner), card_ms, cpu_ms }) +} + +#[cfg(feature = "denoise")] +impl Network { + /// The card when it can be used now, else why not. `start` sets it up when that has not been tried yet; without it + /// nothing is started. + fn card(&self, start: bool) -> Result<&GpuSide, String> { + let side = if start { + Some(self.gpu.get_or_init(|| { + let made = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| make_gpu(&self.cpu, &self.path, self.tile))) + .unwrap_or_else(|_| Err("the GPU runner gave up".into())); + if let Err(why) = &made { + log::info!("denoise: running on the CPU ({why})"); + } + made + })) + } else { + self.gpu.get() + }; + let g = match side { + None => return Err("the graphics card is not set up yet".into()), + Some(Err(why)) => return Err(why.clone()), + Some(Ok(g)) => g, + }; + if let Some(why) = lightcraft_gpu::nn::broken_reason() { + return Err(format!("the graphics card stopped working ({why})")); + } + if self.fallbacks.load(Ordering::Relaxed) >= GPU_MAX_FALLBACKS { + return Err(format!("{} got {GPU_MAX_FALLBACKS} tiles wrong", g.adapter)); + } + Ok(g) + } + + /// The card, when a photo made `threads` tiles at a time under `run_on` runs on it; else why the CPU does. + fn choose(&self, run_on: RunOn, threads: usize, start: bool) -> Result<&GpuSide, String> { + if run_on == RunOn::Cpu { + return Err("the processor is chosen in Settings".into()); + } + let g = self.card(start)?; + if run_on == RunOn::Auto && !card_is_faster(g.card_ms, g.cpu_ms, threads) { + return Err(format!("it is faster than {} on this computer", g.adapter)); + } + Ok(g) + } + + fn fell_back(&self, why: &str) { + let n = self.fallbacks.fetch_add(1, Ordering::Relaxed) + 1; + if n == 1 { + log::warn!("denoise: a tile came back wrong from the GPU ({why}); the CPU runs it"); + } else if n == GPU_MAX_FALLBACKS { + log::warn!("denoise: the GPU got {n} tiles wrong; the CPU does the rest"); + } + } +} + +/// The card, with the CPU running any tile it gets wrong. Handed out only for photos that run on the card. +#[cfg(feature = "denoise")] +impl TileRunner for Network { + fn run(&self, input: &[f32]) -> Result, RunError> { + if let Ok(g) = self.card(false) { + match g.runner.run(input) { + Ok(out) if out.iter().all(|v| v.is_finite()) => return Ok(out), + Ok(_) => self.fell_back("numbers that are not finite"), + Err(e) => self.fell_back(&e.to_string()), + } + } + self.cpu.run(input) + } +} + +#[cfg(feature = "denoise")] +impl Model for Network { + fn bytes_per_tile(&self) -> usize { + self.cpu.bytes_per_tile() + } + fn runner(&self, run_on: RunOn, threads: usize) -> (&dyn TileRunner, usize) { + let threads = threads.max(1); + self.last_threads.store(threads, Ordering::Relaxed); + match self.choose(run_on, threads, true) { + Ok(_) => (self, threads.min(GPU_PARALLEL)), + Err(_) => (&self.cpu, threads), + } + } + + fn self_test(&self, run_on: RunOn) -> Result { + let t = self.cpu.self_test(); + if !t.ok { + let failed: Vec<&str> = t.checks.iter().filter(|(_, ok)| !*ok).map(|(c, _)| c.as_str()).collect(); + return Err(format!("the model failed its self-test: it does not {}", failed.join(", and does not "))); + } + let mut v = serde_json::to_value(&t).map_err(|e| e.to_string())?; + // the card is set up and checked here, so the test says where the work will run + if run_on != RunOn::Cpu { + let _ = self.card(true); + } + if let Some(o) = v.as_object_mut() { + o.insert("device".into(), self.device(run_on)); + } + Ok(v) + } + + fn device(&self, run_on: RunOn) -> Value { + if run_on != RunOn::Cpu && self.gpu.get().is_none() { + return json!({"kind": "pending"}); + } + let threads = match self.last_threads.load(Ordering::Relaxed) { + 0 => Pace::Full.parallel(std::thread::available_parallelism().map_or(1, |n| n.get()), None), + n => n, + }; + let mut v = match self.choose(run_on, threads, false) { + Ok(g) => json!({"kind": "gpu", "adapter": g.adapter}), + // a set-up that crashed or ran out of time can be tried again (`denoise.settings {runOn}`) + Err(why) => json!({"kind": "cpu", "retry": worth_retrying(&why), "reason": why}), + }; + if let (Some(Ok(g)), Some(o)) = (self.gpu.get(), v.as_object_mut()) { + // measured on this computer, on the check tile: the card's best run, and one tile alone on the CPU + o.insert("cardMs".into(), json!(g.card_ms)); + o.insert("cpuMs".into(), json!(g.cpu_ms)); + o.insert("fellBack".into(), json!(self.fallbacks.load(Ordering::Relaxed))); + } + v + } +} + +/// The loader of this build: our pure-Rust executor with the `denoise` feature, else one that says why nothing can run. +pub(crate) fn default_loader() -> Loader { + #[cfg(feature = "denoise")] + { + Arc::new(|path, manifest| { + let cpu = lightcraft_denoise::runtime::CpuRunner::load(path, manifest).map_err(|e| e.to_string())?; + Ok(Arc::new(Network { + cpu, + path: path.to_path_buf(), + tile: manifest.tile as usize, + gpu: OnceLock::new(), + fallbacks: AtomicUsize::new(0), + last_threads: AtomicUsize::new(0), + }) as Arc) + }) + } + #[cfg(not(feature = "denoise"))] + { + Arc::new(|_, _| Err("this build cannot run denoise models".to_string())) + } +} + +/// A model that is loaded on first use and kept, shared by the background job and the exports. +pub(crate) type ModelSlot = Arc>>>; + +// --------------------------------------------------------------------------------------------------------------------- +// settings and installed models + +/// `settings.json` in the models folder. +#[derive(Clone, Default, Serialize, Deserialize)] +#[serde(default, rename_all = "camelCase")] +pub(crate) struct Settings { + /// The installed model in use; `None`: denoise is off. + pub model: Option, + /// Photos with a Denoise amount above 0 that are being looked at (active, selected) get their picture made without + /// being asked (`None`: yes). + pub auto: Option, + /// Cache size limit (GB; `None`: [`DEFAULT_CACHE_GB`]). + pub cache_gb: Option, + /// Most tiles run at once (`None`: by how hard the pump says to work). + pub threads: Option, + /// Where the model runs: `auto`, `gpu` or `cpu` (`None` or anything else: `auto`; see [`RunOn`]). + pub run_on: Option, + /// What came before `runOn`: `false` was the processor only. + pub gpu: Option, +} + +impl Settings { + pub fn auto(&self) -> bool { + self.auto.unwrap_or(true) + } + + pub fn run_on(&self) -> RunOn { + match self.run_on.as_deref().and_then(RunOn::parse) { + Some(r) => r, + None if self.gpu == Some(false) => RunOn::Cpu, + None => RunOn::Auto, + } + } + + pub fn cache_bytes(&self) -> u64 { + u64::from(self.cache_gb.unwrap_or(DEFAULT_CACHE_GB).clamp(1, 10_000)) << 30 + } +} + +pub(crate) fn read_capped(path: &Path, max: usize) -> Option> { + use std::io::Read; + let mut buf = Vec::new(); + std::fs::File::open(path).ok()?.take(max as u64 + 1).read_to_end(&mut buf).ok()?; + (buf.len() <= max).then_some(buf) +} + +pub(crate) fn read_settings(dir: &Path) -> Settings { + read_capped(&dir.join("settings.json"), 4096).and_then(|b| serde_json::from_slice(&b).ok()).unwrap_or_default() +} + +/// Write a file next to its final name and rename it into place. +pub(crate) fn write_atomic(path: &Path, bytes: &[u8]) -> Result<(), String> { + let part = path.with_extension("part"); + std::fs::write(&part, bytes).map_err(|e| format!("could not write {}: {e}", path.display()))?; + std::fs::rename(&part, path).map_err(|e| { + let _ = std::fs::remove_file(&part); + format!("could not write {}: {e}", path.display()) + }) +} + +pub(crate) fn write_settings(dir: &Path, st: &Settings) -> Result<(), String> { + std::fs::create_dir_all(dir).map_err(|e| format!("could not create the denoise models folder: {e}"))?; + write_atomic(&dir.join("settings.json"), &serde_json::to_vec_pretty(st).map_err(|e| e.to_string())?) +} + +/// An installed model folder: `/model.onnx`, `denoise-model.json`, `installed.json`. +pub(crate) struct Installed { + pub manifest: DenoiserManifest, + pub onnx: PathBuf, + /// What was recorded when it was installed (terms accepted, hash of the file, self-test). + pub accepted: Value, +} + +impl Installed { + /// Names the model file the cache is made with: a hash of the file as installed, so another file with the same id + /// never finds this one's products. + pub fn fingerprint(&self) -> String { + let sha = self.accepted.get("sha256").and_then(Value::as_str).or(self.manifest.sha256.as_deref()).unwrap_or(""); + let len = std::fs::metadata(&self.onnx).map(|m| m.len()).unwrap_or(0); + format!("{}@{}:{len}", self.manifest.id, sha) + } +} + +/// The models in the folder. A folder that is not a valid model (wrong name, no model file, a manifest that does not +/// validate or does not match its folder) is ignored, never trusted. +pub(crate) fn installed_models(dir: &Path) -> Vec { + let Ok(entries) = std::fs::read_dir(dir) else { return Vec::new() }; + let mut out = Vec::new(); + for e in entries.flatten() { + let p = e.path(); + let Some(name) = p.file_name().and_then(|n| n.to_str()).map(str::to_string) else { continue }; + let onnx = p.join("model.onnx"); + if !lightcraft_denoise::licence::valid_id(&name) || !p.is_dir() || !onnx.is_file() { + continue; + } + let Some(m) = read_capped(&p.join("denoise-model.json"), lightcraft_denoise::manifest::MAX_MANIFEST_BYTES) + .and_then(|b| lightcraft_denoise::manifest::parse(&b).ok()) + else { + continue; + }; + if m.id != name { + continue; + } + let accepted = read_capped(&p.join("installed.json"), 8192).and_then(|b| serde_json::from_slice(&b).ok()).unwrap_or(Value::Null); + out.push(Installed { manifest: m, onnx, accepted }); + } + out.sort_by(|a, b| a.manifest.id.cmp(&b.manifest.id)); + out +} + +/// Forget that setting the graphics card up for a model closed LightCraft before (the markers [`set_up_card`] leaves): +/// the user chose where denoise runs, so the card may be tried again. +pub(crate) fn forget_gpu_set_ups(dir: &Path) { + for i in installed_models(dir) { + let _ = std::fs::remove_file(i.onnx.with_file_name(GPU_SETUP_MARKER)); + } +} + +/// The model in use. +pub(crate) struct Active { + pub id: String, + pub manifest: DenoiserManifest, + pub onnx: PathBuf, + pub fingerprint: String, + pub model: ModelSlot, +} + +// --------------------------------------------------------------------------------------------------------------------- +// products + +/// The photos whose files can be denoised at all: raw files developed from their sensor data and read from disk. +pub(crate) fn eligible(p: &Photo) -> bool { + p.develops_raw() && matches!(p.source, Source::File { .. }) && !p.deleted +} + +/// A product's name: what it was made from and with. +pub(crate) fn product_key(fingerprint: &str, p: &Photo) -> String { + let mut h = Hasher128::new(); + h.str("lcdn").u64(u64::from(product::ALGORITHM)).str(fingerprint).str(&crate::media::content_key(p)); + h.finish().to_string() +} + +pub(crate) fn product_path(dir: &Path, key: &str) -> PathBuf { + dir.join(format!("{key}.{EXT}")) +} + +/// The `.lcdn` files in `dir`: path, size, last change. +fn products_in(dir: &Path) -> Vec<(PathBuf, u64, SystemTime)> { + let Ok(entries) = std::fs::read_dir(dir) else { return Vec::new() }; + entries + .flatten() + .filter(|e| e.path().extension().is_some_and(|x| x == EXT)) + .filter_map(|e| { + let m = e.metadata().ok()?; + m.is_file().then(|| (e.path(), m.len(), m.modified().unwrap_or(SystemTime::UNIX_EPOCH))) + }) + .collect() +} + +/// Make sure only one thread writes a product at a time: a second one waits, then finds it made. +fn making() -> &'static (Mutex>, Condvar) { + static MAKING: OnceLock<(Mutex>, Condvar)> = OnceLock::new(); + MAKING.get_or_init(|| (Mutex::new(HashSet::new()), Condvar::new())) +} + +struct Slot(PathBuf); + +impl Slot { + fn enter(path: &Path) -> Slot { + let (set, cv) = making(); + let mut g = set.lock().unwrap_or_else(PoisonError::into_inner); + while g.contains(path) { + g = cv.wait(g).unwrap_or_else(PoisonError::into_inner); + } + g.insert(path.to_path_buf()); + Slot(path.to_path_buf()) + } +} + +impl Drop for Slot { + fn drop(&mut self) { + let (set, cv) = making(); + set.lock().unwrap_or_else(PoisonError::into_inner).remove(&self.0); + cv.notify_all(); + } +} + +// --------------------------------------------------------------------------------------------------------------------- +// making a product + +/// Everything a thread needs to make one product, detached from the session. +#[derive(Clone)] +pub(crate) struct JobSpec { + /// The original file. + pub path: String, + pub read: Option, + pub product: PathBuf, + pub key: String, + pub manifest: DenoiserManifest, + pub onnx: PathBuf, + pub model: ModelSlot, + pub loader: Loader, + /// Tiles run at once. + pub parallel: usize, + /// Where the model runs (the setting). + pub run_on: RunOn, +} + +/// Why a product was not made. +#[derive(Debug, PartialEq)] +pub(crate) enum MakeError { + /// Denoise does not apply to this file (not a Bayer raw): nothing is wrong. + Unsupported(String), + Failed(String), + Cancelled, +} + +impl From for MakeError { + fn from(e: RunError) -> Self { + match e { + RunError::Cancelled => MakeError::Cancelled, + other => MakeError::Failed(other.to_string()), + } + } +} + +/// Progress and cancel of a running job. +#[derive(Default)] +pub(crate) struct Progress { + done: AtomicUsize, + total: AtomicUsize, + cancel: AtomicBool, +} + +impl Progress { + /// Ask the job to stop at the next tile. + pub(crate) fn cancel_now(&self) { + self.cancel.store(true, Ordering::Relaxed); + } +} + +/// The loaded model, loading it first when it is not (kept for the next photo). +fn loaded(spec: &JobSpec) -> Result, MakeError> { + if let Some(m) = spec.model.lock().unwrap_or_else(PoisonError::into_inner).clone() { + return Ok(m); + } + let m = (spec.loader)(&spec.onnx, &spec.manifest).map_err(|e| MakeError::Failed(format!("the denoise model could not be loaded: {e}")))?; + *spec.model.lock().unwrap_or_else(PoisonError::into_inner) = Some(m.clone()); + Ok(m) +} + +/// The cell `[(0,0), (1,0), (0,1), (1,1)]` colours of a Bayer mosaic and so its layout. +fn layout_of(cfa: &lightcraft_raw::Cfa) -> Option { + Layout::from_cell([cfa.color_at(0, 0), cfa.color_at(1, 0), cfa.color_at(0, 1), cfa.color_at(1, 1)]) +} + +/// Make the product of `spec` unless it is there: decode the photo, run the model over its mosaic, write the result. +/// `Ok(true)` when it was made now, `Ok(false)` when it already existed. +pub(crate) fn make_product(spec: &JobSpec, progress: Option<&Progress>) -> Result { + let _slot = Slot::enter(&spec.product); + if product::is_current(&spec.product, &spec.key) { + return Ok(false); + } + let began = web_time::Instant::now(); + let cancelled = || progress.is_some_and(|p| p.cancel.load(Ordering::Relaxed)); + if cancelled() { + return Err(MakeError::Cancelled); + } + let bytes = match &spec.read { + Some(r) => r(&spec.path).map_err(MakeError::Failed)?, + None => std::fs::read(&spec.path).map_err(|e| MakeError::Failed(format!("{}: {e}", spec.path)))?, + }; + // the decode and the normalised mosaic wait their turn at the process-wide memory gate, as every decode does + let (mosaic, layout) = { + let _held = crate::memory::work_gate().acquire(bytes.len().saturating_mul(4)); + if lightcraft_raw::probe(&bytes).is_none() { + return Err(MakeError::Unsupported("not a raw file".into())); + } + let raw = match lightcraft_raw::decode(&bytes) { + Ok(r) => r, + Err(lightcraft_raw::RawError::Unsupported(why)) => return Err(MakeError::Unsupported(why)), + Err(e) => return Err(MakeError::Failed(e.to_string())), + }; + drop(bytes); + if raw.cpp != 1 || !raw.cfa.as_ref().is_some_and(lightcraft_raw::Cfa::is_bayer) { + return Err(MakeError::Unsupported( + "only Bayer raw files can be denoised (this one has another sensor layout or is already demosaiced)".into(), + )); + } + let n = raw.normalized().map_err(|e| MakeError::Failed(e.to_string()))?; + drop(raw); + let layout = + n.cfa.as_ref().and_then(layout_of).ok_or_else(|| MakeError::Unsupported("the colour filter layout is not a Bayer pattern".into()))?; + (n, layout) + }; + if cancelled() { + return Err(MakeError::Cancelled); + } + let decoded = began.elapsed(); + let model = loaded(spec)?; + // Packed input, returned RGB and the preceding group's RGB overlap while tiles run. + let scratch = + model.bytes_per_tile().saturating_add(128usize.saturating_mul(spec.manifest.tile as usize).saturating_mul(spec.manifest.tile as usize)); + let (_, budget) = crate::memory::work_gate().usage(); + let parallel = spec.parallel.clamp(1, 4).min((budget / scratch.max(1)).max(1)); + let _held = crate::memory::work_gate().acquire(mosaic.data.len().saturating_mul(20).saturating_add(scratch.saturating_mul(parallel))); + let (runner, parallel) = model.runner(spec.run_on, parallel); + let params = Params { + tile: spec.manifest.tile as usize, + overlap: spec.manifest.overlap as usize, + gain: spec.manifest.gain, + clip: Some(CLIP), + parallel: parallel.max(1), + }; + let report = |done: usize, total: usize| { + if let Some(p) = progress { + p.total.store(total, Ordering::Relaxed); + p.done.store(done, Ordering::Relaxed); + } + }; + let ctl = Control { cancel: progress.map(|p| &p.cancel), progress: Some(&report) }; + // the tiles run on threads of their own, so a busy window's renders keep theirs + let pool = rayon::ThreadPoolBuilder::new() + .num_threads(params.parallel) + .stack_size(STACK) + .thread_name(|i| format!("denoise-{i}")) + .build() + .map_err(|e| MakeError::Failed(format!("could not start the worker threads: {e}")))?; + let rgb = pool.install(|| denoise_bayer(&mosaic.data, mosaic.width, mosaic.height, layout, runner, ¶ms, &ctl))?; + drop(mosaic); + if cancelled() { + return Err(MakeError::Cancelled); + } + let ran = began.elapsed(); + product::write(&spec.product, &rgb, &spec.key).map_err(|e| MakeError::Failed(format!("could not save the denoised picture: {e}")))?; + if std::env::var_os("LIGHTCRAFT_PROFILE").is_some() { + eprintln!( + "[denoise] read + decode {:.0} ms, model (set-up, tiles, blend) {:.0} ms, write {:.0} ms", + decoded.as_secs_f64() * 1e3, + (ran - decoded).as_secs_f64() * 1e3, + (began.elapsed() - ran).as_secs_f64() * 1e3 + ); + } + Ok(true) +} + +// --------------------------------------------------------------------------------------------------------------------- +// the session's state + +/// How hard the background work may go (the app picks it from what the user is doing, like the face scan). +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum Pace { + /// Start nothing new. + Pause, + /// Two tiles at once. + Light, + /// Half the machine. + Normal, + /// Most of the machine: the user is watching the progress. + Full, +} + +impl Pace { + /// `pause`, `light`, `normal` or `full`; anything else is `normal`. + pub fn parse(name: Option<&str>) -> Pace { + match name { + Some("pause") => Pace::Pause, + Some("light") => Pace::Light, + Some("full") => Pace::Full, + _ => Pace::Normal, + } + } + + /// Tiles to run at once on a machine with `cores` threads (`cap`: the user's own limit). + pub fn parallel(self, cores: usize, cap: Option) -> usize { + let n = match self { + Pace::Pause | Pace::Light => 2, + Pace::Normal => cores / 2, + Pace::Full => cores * 4 / 5, + } + .clamp(1, 16); + cap.map_or(n, |c| n.min(c.max(1) as usize)) + } +} + +/// How a photo stands, for the interface. +#[derive(Clone, Debug, PartialEq, Serialize)] +#[serde(tag = "state", rename_all = "camelCase")] +pub enum PhotoState { + /// Not a photo denoise applies to (not a raw, offline, a demo). + NotApplicable, + /// No model is chosen: nothing can be made. + NoModel, + /// Its picture is made: the Amount slider works. + Ready, + Queued { + /// Photos ahead of it (0 = next). + ahead: usize, + }, + Running { + done: usize, + total: usize, + }, + /// Making it failed (`why`), or denoise does not apply to this file (`unsupported`). + Failed { + why: String, + unsupported: bool, + }, + /// Nothing has asked for it yet. + Idle, +} + +#[derive(Clone)] +struct Failure { + key: String, + why: String, + unsupported: bool, +} + +enum Outcome { + Done, + Failed(String, bool), + Cancelled, +} + +struct Running { + photo: PhotoId, + key: String, + product: PathBuf, + progress: Arc, + outcome: Arc>>, + started: web_time::Instant, +} + +pub(crate) struct State { + /// Where the models are kept (`None` where there is no file system). + pub models_dir: Option, + /// Models being downloaded at the user's request. + pub downloads: Downloads, + pub(crate) install: Option, + /// How a model is loaded (tests put another in). + pub loader: Loader, + pub(crate) active: Option, + active_seen: Option, + /// `settings.json` as last read (with the model choice). + pub(crate) settings: Settings, + /// The index must be made again (the library, the model or the cache changed). + dirty: bool, + /// The catalog revision and time of the last index, to look again when photos changed. + reindexed: Option<(u64, web_time::Instant)>, + queue: VecDeque, + running: Option, + failed: HashMap, + /// Counts every change of what is ready, so the interface knows to draw again. + pub generation: u64, + /// Photos made since the session started. + made: u64, +} + +impl Default for State { + fn default() -> Self { + State { + models_dir: None, + downloads: Downloads::default(), + install: None, + loader: default_loader(), + active: None, + active_seen: None, + settings: Settings::default(), + dirty: true, + reindexed: None, + queue: VecDeque::new(), + running: None, + failed: HashMap::new(), + generation: 0, + made: 0, + } + } +} + +impl Drop for State { + /// Quitting stops the job instead of leaving it to finish a photo nobody will see. + fn drop(&mut self) { + if let Some(r) = &self.running { + r.progress.cancel_now(); + } + } +} + +impl State { + /// The library was replaced: nothing carries over (photo ids are per library). + pub fn library_changed(&mut self) { + if let Some(r) = self.running.take() { + r.progress.cancel_now(); + } + self.queue.clear(); + self.failed.clear(); + self.dirty = true; + self.generation += 1; + } + + /// The model choice or the cache changed on disk: look again at the next pump. + pub fn touch(&mut self) { + self.active_seen = None; + self.dirty = true; + } + + pub fn queued(&self) -> usize { + self.queue.len() + } +} + +impl Session { + /// Where the active model's tiles run (see [`Model::device`]); `{kind: "none"}` while no model is loaded. + pub(crate) fn denoise_device(&self) -> Value { + let loaded = self.denoise.active.as_ref().and_then(|a| a.model.lock().unwrap_or_else(PoisonError::into_inner).clone()); + match loaded { + Some(m) => m.device(self.denoise.settings.run_on()), + None => json!({"kind": "none"}), + } + } + + /// The folder the products of the open library are kept in; `None` without a library on disk. + pub(crate) fn denoise_products_dir(&self) -> Option { + self.library.as_ref().filter(|l| l.on_disk).map(|l| l.dir.join(DIR)) + } + + /// Read which model is in use (at most every couple of seconds): the manifest and file of the model the settings + /// name, if it is installed. + pub(crate) fn denoise_refresh_active(&mut self, force: bool) { + let fresh = self.denoise.active_seen.is_some_and(|t| t.elapsed() < SETTINGS_TTL); + if fresh && !force { + return; + } + self.denoise.active_seen = Some(web_time::Instant::now()); + self.denoise.settings = self.denoise.models_dir.as_deref().map(read_settings).unwrap_or_default(); + let chosen = self.denoise.models_dir.as_deref().and_then(|dir| { + let id = self.denoise.settings.model.clone()?; + installed_models(dir).into_iter().find(|i| i.manifest.id == id) + }); + let found = chosen.map(|i| (i.fingerprint(), i)); + let same = match (&self.denoise.active, &found) { + (None, None) => true, + (Some(a), Some((fp, _))) => a.fingerprint == *fp, + _ => false, + }; + if same { + return; + } + // another model: what was made with the old one is not this one's, and what failed may work now + self.denoise.active = found.map(|(fingerprint, i)| Active { + id: i.manifest.id.clone(), + manifest: i.manifest, + onnx: i.onnx, + fingerprint, + model: Arc::new(Mutex::new(None)), + }); + if let Some(r) = self.denoise.running.take() { + r.progress.cancel_now(); + } + self.denoise.queue.clear(); + self.denoise.failed.clear(); + self.denoise.dirty = true; + } + + fn denoise_spec(&self, p: &Photo) -> Option<(DenoiseSpec, String)> { + let (dir, active) = (self.denoise_products_dir()?, self.denoise.active.as_ref()?); + let key = product_key(&active.fingerprint, p); + Some((DenoiseSpec { product: product_path(&dir, &key), key: key.clone(), make: None }, key)) + } + + /// Make the index of ready photos from the products on disk (a directory listing and a hash per photo: no file is + /// opened). Pictures that appeared or vanished drop the decoded sources they affect. + pub(crate) fn denoise_reindex(&mut self) { + self.denoise.dirty = false; + self.denoise.reindexed = Some((self.catalog.revision, web_time::Instant::now())); + let mut now: HashMap = HashMap::new(); + if let (Some(dir), Some(active)) = (self.denoise_products_dir(), self.denoise.active.as_ref()) { + let names: HashSet = + products_in(&dir).into_iter().filter_map(|(p, _, _)| p.file_stem().and_then(|s| s.to_str()).map(str::to_string)).collect(); + if !names.is_empty() { + for p in self.catalog.photos().filter(|p| eligible(p)) { + let key = product_key(&active.fingerprint, p); + if names.contains(&key) { + now.insert(p.id, DenoiseSpec { product: product_path(&dir, &key), key, make: None }); + } + } + } + } + let mut changed = false; + for id in self.media.denoise.ids() { + if !now.contains_key(&id) { + self.media.denoise.remove(id); + self.media.forget(id); + changed = true; + } + } + for (id, spec) in now { + if self.media.denoise.spec(id).is_none_or(|old| old.key != spec.key) { + self.media.denoise.set(id, spec); + self.media.forget(id); + changed = true; + } + } + if changed { + self.denoise.generation += 1; + } + } + + /// A photo whose file changed (its content key is not the one its index entry was made for) has no picture. + fn denoise_check_photo(&mut self, id: PhotoId) { + let Some(p) = self.catalog.photo(id).cloned() else { return }; + let Some((spec, _)) = self.denoise_spec(&p) else { return }; + match self.media.denoise.spec(id) { + Some(have) if have.key != spec.key => { + self.media.denoise.remove(id); + self.media.forget(id); + self.denoise.generation += 1; + } + None if spec.product.is_file() && product::is_current(&spec.product, &spec.key) => { + // made by an export, or by another session + self.media.denoise.set(id, spec); + self.media.forget(id); + self.denoise.generation += 1; + } + _ => {} + } + } + + /// How photo `id` stands. + pub fn denoise_photo_state(&self, id: PhotoId) -> PhotoState { + let Some(p) = self.catalog.photo(id) else { return PhotoState::NotApplicable }; + if !eligible(p) { + return PhotoState::NotApplicable; + } + if self.denoise.active.is_none() { + // the model folder has not been looked at yet (just started): not "no model" until it has been + return if self.denoise.active_seen.is_none() { PhotoState::Idle } else { PhotoState::NoModel }; + } + if self.denoise_products_dir().is_none() { + return PhotoState::Failed { + why: "AI Denoise keeps its pictures in the library folder, and this session has none.".into(), + unsupported: true, + }; + } + if self.media.denoise.spec(id).is_some() { + return PhotoState::Ready; + } + if let Some(r) = self.denoise.running.as_ref().filter(|r| r.photo == id) { + return PhotoState::Running { done: r.progress.done.load(Ordering::Relaxed), total: r.progress.total.load(Ordering::Relaxed) }; + } + if let Some(ahead) = self.denoise.queue.iter().position(|q| *q == id) { + return PhotoState::Queued { ahead: ahead + usize::from(self.denoise.running.is_some()) }; + } + let key = self.denoise_spec(p).map(|s| s.1); + match self.denoise.failed.get(&id) { + Some(f) if Some(&f.key) == key.as_ref() => PhotoState::Failed { why: f.why.clone(), unsupported: f.unsupported }, + _ => PhotoState::Idle, + } + } + + /// Ask for photos' pictures to be made (the ones that are not made, queued or known to be impossible). With `retry` + /// a photo that failed before is tried again. A photo that is not raw is skipped. Returns how many were queued. + pub(crate) fn denoise_enqueue(&mut self, ids: &[PhotoId], front: bool, retry: bool) -> usize { + self.denoise_refresh_active(false); + if self.denoise.active.is_none() || self.denoise_products_dir().is_none() { + return 0; + } + let mut added = 0; + for id in ids { + let Some(p) = self.catalog.photo(*id).cloned() else { continue }; + if !eligible(&p) || self.media.denoise.spec(*id).is_some() || self.denoise.queue.contains(id) { + continue; + } + if self.denoise.running.as_ref().is_some_and(|r| r.photo == *id) || self.denoise.queue.len() >= MAX_QUEUE { + continue; + } + let Some((_, key)) = self.denoise_spec(&p) else { continue }; + if retry { + self.denoise.failed.remove(id); + } else if self.denoise.failed.get(id).is_some_and(|f| f.key == key) { + continue; + } + if front { + self.denoise.queue.push_front(*id); + } else { + self.denoise.queue.push_back(*id); + } + added += 1; + } + added + } + + /// Stop the job that is running, and with `all` forget everything that waits. + pub(crate) fn denoise_cancel(&mut self, photo: Option, all: bool) -> usize { + let mut n = 0; + if let Some(r) = &self.denoise.running + && photo.is_none_or(|p| p == r.photo) + { + r.progress.cancel_now(); + n += 1; + } + let before = self.denoise.queue.len(); + match photo { + Some(p) => self.denoise.queue.retain(|q| *q != p), + None if all => self.denoise.queue.clear(), + None => {} + } + n + before - self.denoise.queue.len() + } + + /// The photos the automatic queue looks at: the one open and the ones selected, those with a Denoise amount. + fn denoise_wanted(&self) -> Vec { + let Selection { active, ids, .. } = &self.selection; + active + .iter() + .chain(ids.iter().take(AUTO_LOOK)) + .copied() + .filter(|id| self.catalog.photo(*id).is_some_and(|p| p.develop.denoise_amount() > 0.0)) + .collect() + } + + fn denoise_job_spec(&self, p: &Photo, parallel: usize) -> Option { + let Source::File { path } = &p.source else { return None }; + let (spec, _) = self.denoise_spec(p)?; + let active = self.denoise.active.as_ref()?; + Some(JobSpec { + path: path.clone(), + read: self.media.file_bytes.clone(), + product: spec.product, + key: spec.key, + manifest: active.manifest.clone(), + onnx: active.onnx.clone(), + model: active.model.clone(), + loader: self.denoise.loader.clone(), + parallel, + run_on: self.denoise.settings.run_on(), + }) + } + + /// Start the next queued photo on its own thread. + fn denoise_start_next(&mut self, parallel: usize) { + while let Some(id) = self.denoise.queue.pop_front() { + let Some(p) = self.catalog.photo(id).cloned() else { continue }; + if !eligible(&p) || self.media.denoise.spec(id).is_some() { + continue; + } + let Some(spec) = self.denoise_job_spec(&p, parallel) else { continue }; + // another session or an export may have made it since it was queued + if product::is_current(&spec.product, &spec.key) { + self.denoise_check_photo(id); + continue; + } + let progress = Arc::new(Progress::default()); + let outcome: Arc>> = Arc::new(Mutex::new(None)); + let (key, product) = (spec.key.clone(), spec.product.clone()); + let (worker_progress, worker_outcome) = (progress.clone(), outcome.clone()); + let started = std::thread::Builder::new().name("denoise".into()).stack_size(STACK).spawn(move || { + let result = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| make_product(&spec, Some(&worker_progress)))); + let o = match result { + Ok(Ok(_)) => Outcome::Done, + Ok(Err(MakeError::Cancelled)) => Outcome::Cancelled, + Ok(Err(MakeError::Unsupported(why))) => Outcome::Failed(why, true), + Ok(Err(MakeError::Failed(why))) => Outcome::Failed(why, false), + Err(_) => Outcome::Failed("the denoiser stopped unexpectedly".into(), false), + }; + *worker_outcome.lock().unwrap_or_else(PoisonError::into_inner) = Some(o); + }); + match started { + Ok(_) => { + self.denoise.running = Some(Running { photo: id, key, product, progress, outcome, started: web_time::Instant::now() }); + } + Err(e) => { + self.denoise.failed.insert(id, Failure { key, why: format!("could not start the denoiser: {e}"), unsupported: false }); + continue; + } + } + return; + } + } + + /// Take in the running job's result. + fn denoise_poll(&mut self) { + let Some(r) = &self.denoise.running else { return }; + let Some(outcome) = r.outcome.lock().unwrap_or_else(PoisonError::into_inner).take() else { return }; + let Some(r) = self.denoise.running.take() else { return }; + match outcome { + Outcome::Done => { + self.denoise.made += 1; + self.denoise_check_photo(r.photo); + self.denoise_evict(Some(&r.product)); + } + Outcome::Failed(why, unsupported) => { + self.denoise.failed.insert(r.photo, Failure { key: r.key, why, unsupported }); + } + Outcome::Cancelled => {} + } + // the picture's state changed: the interface draws it again + self.denoise.generation += 1; + } + + /// Keep the cache within its limit: the pictures nobody asks for go first, the oldest first; `keep` (the one just + /// made) never goes. The index forgets what was deleted. + pub(crate) fn denoise_evict(&mut self, keep: Option<&Path>) { + let Some(models) = self.denoise.models_dir.clone() else { return }; + self.denoise_evict_to(read_settings(&models).cache_bytes(), keep); + } + + /// [`Self::denoise_evict`] for a limit of `limit` bytes. + pub(crate) fn denoise_evict_to(&mut self, limit: u64, keep: Option<&Path>) { + let Some(dir) = self.denoise_products_dir() else { return }; + let mut files = products_in(&dir); + let mut total: u64 = files.iter().map(|f| f.1).sum(); + if total <= limit { + return; + } + // the pictures photos use now (a Denoise amount above 0) are the last to go + let wanted: HashSet = self + .media + .denoise + .ids() + .into_iter() + .filter(|id| self.catalog.photo(*id).is_some_and(|p| p.develop.denoise_amount() > 0.0)) + .filter_map(|id| self.media.denoise.spec(id).map(|s| s.product.clone())) + .collect(); + files.sort_by_key(|(path, _, modified)| (wanted.contains(path), *modified)); + let mut gone: HashSet = HashSet::new(); + for (path, size, _) in files { + if total <= limit { + break; + } + if keep == Some(path.as_path()) { + continue; + } + if std::fs::remove_file(&path).is_ok() { + total = total.saturating_sub(size); + gone.insert(path); + } + } + if gone.is_empty() { + return; + } + for id in self.media.denoise.ids() { + if self.media.denoise.spec(id).is_some_and(|s| gone.contains(&s.product)) { + self.media.denoise.remove(id); + self.media.forget(id); + } + } + self.denoise.generation += 1; + } + + /// Delete every product of this library (they are made again when wanted). + pub(crate) fn denoise_clear(&mut self) -> (usize, u64) { + self.denoise_cancel(None, true); + let (mut files, mut bytes) = (0, 0); + if let Some(dir) = self.denoise_products_dir() { + for (path, size, _) in products_in(&dir) { + if std::fs::remove_file(&path).is_ok() { + files += 1; + bytes += size; + } + } + } + self.denoise.failed.clear(); + self.denoise.touch(); + (files, bytes) + } + + /// Per frame: take in what finished, look at what the user is looking at, and keep one job going. Cheap when + /// nothing is happening (no file is opened unless a photo with a Denoise amount lacks its picture). + pub(crate) fn denoise_pump(&mut self, pace: Pace) { + crate::cmd::denoise::finish_downloads(self); + self.denoise_refresh_active(false); + // photos that changed (a replaced file has another key) must not keep the picture of what they were + let stale = !self.media.denoise.is_empty() + && self.denoise.reindexed.is_none_or(|(rev, at)| rev != self.catalog.revision && at.elapsed() > SETTINGS_TTL); + if self.denoise.dirty || stale { + self.denoise_reindex(); + } + self.denoise_poll(); + if self.denoise.active.is_none() || self.denoise_products_dir().is_none() { + return; + } + let (cap, auto) = (self.denoise.settings.threads, self.denoise.settings.auto()); + let wanted = self.denoise_wanted(); + for id in &wanted { + self.denoise_check_photo(*id); + } + if auto { + // the photo being edited goes first + let (open, others): (Vec, Vec) = wanted.into_iter().partition(|id| self.selection.active == Some(*id)); + self.denoise_enqueue(&others, false, false); + self.denoise_enqueue(&open, true, false); + } + if self.denoise.running.is_none() && pace != Pace::Pause { + let cores = std::thread::available_parallelism().map_or(1, |n| n.get()); + self.denoise_start_next(pace.parallel(cores, cap)); + } + } + + /// The statistics `denoise.status` reports besides the queue (reads the cache folder). + pub(crate) fn denoise_cache_usage(&self) -> (usize, u64) { + match self.denoise_products_dir() { + Some(dir) => { + let files = products_in(&dir); + (files.len(), files.iter().map(|f| f.1).sum()) + } + None => (0, 0), + } + } + + /// Are pictures made without being asked for photos being looked at (Settings ▸ AI Denoise)? + pub fn denoise_auto(&self) -> bool { + self.denoise.settings.auto() + } + + /// Is a picture being made, or waiting for its turn with a model to make it? (Headless runs wait for this.) + pub fn denoise_busy(&self) -> bool { + self.denoise.running.is_some() || (!self.denoise.queue.is_empty() && self.denoise.active.is_some()) + } + + /// Seconds the running job has been going, and its progress. + pub(crate) fn denoise_running_json(&self) -> Value { + match &self.denoise.running { + Some(r) => json!({ + "photo": r.photo.0, + "done": r.progress.done.load(Ordering::Relaxed), + "total": r.progress.total.load(Ordering::Relaxed), + "seconds": r.started.elapsed().as_secs_f64(), + }), + None => Value::Null, + } + } + + pub(crate) fn denoise_made(&self) -> u64 { + self.denoise.made + } + + /// What an export of photo `id` with `settings` needs for its Denoise amount: its denoised picture, made now when the + /// queue has not got to it. `Ok(None)` when denoise does not apply (amount 0, not a Bayer raw), `Err` when it should + /// have and cannot (no model chosen, nowhere to keep the picture, the model failing). + pub(crate) fn denoise_for_export(&mut self, id: PhotoId, settings: &DevelopSettings) -> Result, String> { + if settings.denoise_amount() <= 0.0 { + return Ok(None); + } + let Some(p) = self.catalog.photo(id).cloned() else { return Ok(None) }; + let Some(loader) = self.media.denoise.loader.clone() else { return Ok(None) }; + if !eligible(&p) { + return Ok(None); + } + self.denoise_refresh_active(false); + let name = &p.file_name; + if self.denoise.active.is_none() { + return Err(format!("{name}: Denoise is on for this photo but no denoise model is chosen; choose one in Settings, or set Denoise to 0")); + } + if self.denoise_products_dir().is_none() { + return Err(format!("{name}: Denoise needs a library on disk to keep its pictures in")); + } + self.denoise_check_photo(id); + let Some(job) = self.denoise_job_spec(&p, Pace::Full.parallel(std::thread::available_parallelism().map_or(1, |n| n.get()), None)) else { + return Ok(None); + }; + let label = name.clone(); + let spec = job.clone(); + let make: MakeProduct = Arc::new(move || match make_product(&spec, None) { + Ok(_) => Ok(true), + Err(MakeError::Unsupported(_)) => Ok(false), + Err(MakeError::Failed(why)) => Err(format!("{label}: could not denoise: {why}")), + Err(MakeError::Cancelled) => Err(format!("{label}: denoise was cancelled")), + }); + Ok(Some((DenoiseSpec { product: job.product, key: job.key, make: Some(make) }, loader))) + } +} + +#[cfg(all(test, feature = "denoise"))] +mod gpu_tests { + use super::*; + use lightcraft_denoise::manifest::{Domain, Gain}; + + /// A folder of the test's own: the card's set-up leaves its marker beside the model. + fn folder(name: &str) -> PathBuf { + let dir = std::env::temp_dir().join(format!("lc-engine-gpu-{}-{name}", std::process::id())); + let _ = std::fs::remove_dir_all(&dir); + std::fs::create_dir_all(&dir).unwrap(); + dir + } + + /// The model file is read again when the card is set up (for the first photo), so it stays until the test ends. + fn model(name: &str, tile: u64) -> (Arc, lightcraft_denoise::runtime::CpuRunner, PathBuf) { + let path = folder(name).join("model.onnx"); + std::fs::write(&path, lightcraft_denoise::synthetic::unet_onnx(tile, 8, 2, 7)).unwrap(); + let manifest = DenoiserManifest { + id: "synthetic".into(), + name: "Synthetic".into(), + version: "1".into(), + licence: Default::default(), + source: None, + sha256: None, + size_bytes: None, + provenance: String::new(), + domain: Domain::BayerToRgb, + tile: tile as u32, + overlap: 16, + gain: Gain::MatchMean { nominal: 1.0, max_deviation: 0.05 }, + }; + let loaded = default_loader()(&path, &manifest).unwrap(); + let cpu = lightcraft_denoise::runtime::CpuRunner::load(&path, &manifest).unwrap(); + (loaded, cpu, path) + } + + fn worst(want: &[f32], got: &[f32]) -> f32 { + assert_eq!(want.len(), got.len()); + let scale = want.iter().fold(1e-9f32, |m, v| m.max(v.abs())); + want.iter().zip(got).fold(0f32, |m, (a, b)| m.max((a - b).abs())) / scale + } + + #[test] + fn a_tile_on_the_card_is_the_cpus_tile() { + let (model, cpu, path) = model("same", 64); + let (input, _) = lightcraft_denoise::runtime::test_tile(64); + let want = cpu.run(&input).unwrap(); + let (runner, at_once) = model.runner(RunOn::Gpu, 16); + assert!(worst(&want, &runner.run(&input).unwrap()) < 1e-3); + let device = model.device(RunOn::Gpu); + eprintln!("device: {device}"); + match device["kind"].as_str() { + Some("gpu") => { + assert!(device["adapter"].as_str().is_some_and(|a| !a.is_empty())); + assert!(at_once <= GPU_PARALLEL, "a card is fed by a few tiles at once"); + assert!(device["cardMs"].as_f64().is_some_and(|ms| ms > 0.0) && device["cpuMs"].as_f64().is_some_and(|ms| ms > 0.0), "{device}"); + } + Some("cpu") => { + assert!(device["reason"].as_str().is_some_and(|r| !r.is_empty()), "the CPU is only used for a reason: {device}"); + assert_eq!(at_once, 16); + } + other => panic!("the first photo decides where the work runs, not {other:?}"), + } + assert!(!path.with_file_name(GPU_SETUP_MARKER).exists(), "a set-up that ended leaves no marker"); + let _ = std::fs::remove_dir_all(path.parent().unwrap()); + } + + #[test] + fn with_the_processor_chosen_everything_runs_on_the_cpu() { + let (model, cpu, path) = model("cpu", 64); + let (input, _) = lightcraft_denoise::runtime::test_tile(64); + let (runner, at_once) = model.runner(RunOn::Cpu, 16); + assert_eq!(runner.run(&input).unwrap(), cpu.run(&input).unwrap(), "the CPU path is the CPU runner, bit for bit"); + assert_eq!(at_once, 16, "the threads asked for are the threads used"); + assert_eq!(model.device(RunOn::Cpu)["kind"], "cpu"); + assert_eq!(model.device(RunOn::Auto)["kind"], "pending", "the card was not even set up"); + let _ = std::fs::remove_dir_all(path.parent().unwrap()); + } + + #[test] + fn the_card_is_used_where_it_is_faster_than_the_processor() { + // measured: a tile alone on one CPU thread 1.4 s; an RTX 4090 Laptop 13 ms; a Radeon 610M 490 ms + for threads in [1, 2, 8, 16, 64] { + assert!(card_is_faster(13.0, 1400.0, threads), "a big card wins at {threads}"); + } + assert!(card_is_faster(490.0, 1400.0, 2), "a small card beats the two threads background work gets"); + assert!(!card_is_faster(490.0, 1400.0, 16), "and loses to sixteen"); + assert!(!card_is_faster(f64::INFINITY, 1400.0, 1) && !card_is_faster(f64::NAN, 1400.0, 1), "a card that never ran is never faster"); + assert!(!card_is_faster(10.0, 0.0, 1)); + } + + /// A card that gives numbers that are not finite. + struct Garbage(AtomicUsize); + + impl TileRunner for Garbage { + fn run(&self, input: &[f32]) -> Result, RunError> { + self.0.fetch_add(1, Ordering::Relaxed); + let n = input.len() / 4 * 12; + Ok(vec![f32::NAN; n]) + } + } + + #[test] + fn a_tile_the_card_gets_wrong_is_run_on_the_processor_and_three_stop_the_card() { + let (_, cpu, path) = model("garbage", 64); + let card = GpuSide { runner: Box::new(Garbage(AtomicUsize::new(0))), adapter: "Test card".into(), card_ms: 1.0, cpu_ms: 1000.0 }; + let model = Network { + cpu: cpu.clone(), + path: path.clone(), + tile: 32, + gpu: OnceLock::from(Ok(card)), + fallbacks: AtomicUsize::new(0), + last_threads: AtomicUsize::new(0), + }; + let (input, _) = lightcraft_denoise::runtime::test_tile(64); + let want = cpu.run(&input).unwrap(); + let (runner, at_once) = model.runner(RunOn::Auto, 8); + assert_eq!(at_once, GPU_PARALLEL, "the (fast) card is chosen"); + for _ in 0..GPU_MAX_FALLBACKS { + assert_eq!(runner.run(&input).unwrap(), want, "a wrong tile is the CPU's tile"); + } + let device = model.device(RunOn::Auto); + assert_eq!(device["kind"], "cpu", "{device}"); + assert!(device["reason"].as_str().unwrap().contains("tiles wrong"), "{device}"); + assert_eq!(device["fellBack"], GPU_MAX_FALLBACKS); + let (_, at_once) = model.runner(RunOn::Gpu, 8); + assert_eq!(at_once, 8, "the next photo runs on the CPU, even with the card chosen"); + let _ = std::fs::remove_dir_all(path.parent().unwrap()); + } + + #[test] + fn a_slow_card_is_left_out_unless_it_is_chosen() { + let (_, cpu, path) = model("slow", 64); + let slow = GpuSide { runner: Box::new(cpu.clone()), adapter: "Slow card".into(), card_ms: 490.0, cpu_ms: 1400.0 }; + let model = Network { + cpu, + path: path.clone(), + tile: 32, + gpu: OnceLock::from(Ok(slow)), + fallbacks: AtomicUsize::new(0), + last_threads: AtomicUsize::new(0), + }; + assert_eq!(model.runner(RunOn::Auto, 16).1, 16, "sixteen threads beat it"); + let device = model.device(RunOn::Auto); + assert_eq!((device["kind"].as_str(), device["reason"].as_str()), (Some("cpu"), Some("it is faster than Slow card on this computer"))); + assert_eq!(device["cardMs"], 490.0, "the times measured are reported"); + assert_eq!(device["retry"], false, "trying again would not make it faster"); + assert_eq!(model.runner(RunOn::Auto, 2).1, GPU_PARALLEL.min(2), "two threads do not"); + assert_eq!(model.device(RunOn::Auto)["kind"], "gpu"); + assert_eq!(model.runner(RunOn::Gpu, 16).1, GPU_PARALLEL, "chosen, it is used anyway"); + let _ = std::fs::remove_dir_all(path.parent().unwrap()); + } + + #[test] + fn a_set_up_marker_left_behind_keeps_the_card_off() { + let (model, _, path) = model("crashed", 64); + let marker = path.with_file_name(GPU_SETUP_MARKER); + std::fs::write(&marker, "setting up the graphics card for AI Denoise (unix time 0)").unwrap(); + let (_, at_once) = model.runner(RunOn::Gpu, 4); + assert_eq!(at_once, 4, "the CPU does the work"); + let device = model.device(RunOn::Gpu); + assert_eq!((device["kind"].as_str(), device["reason"].as_str()), (Some("cpu"), Some(GPU_CRASHED)), "{device}"); + assert_eq!(device["retry"], true, "the interface offers to try again"); + assert!(marker.exists(), "it stays until the user chooses where denoise runs"); + let _ = std::fs::remove_dir_all(path.parent().unwrap()); + } + + #[test] + fn a_set_up_runs_beside_the_cpu_and_removes_its_marker() { + let dir = folder("setup"); + let marker = dir.join(GPU_SETUP_MARKER); + let seen = Arc::new(AtomicBool::new(false)); + let (m, s) = (marker.clone(), seen.clone()); + let r = set_up_card( + &marker, + Duration::from_secs(10), + move || { + s.store(m.exists(), Ordering::Relaxed); + Ok(7) + }, + || "cpu", + ); + assert_eq!(r, Ok((7, "cpu"))); + assert!(seen.load(Ordering::Relaxed), "the marker is there while the card is set up"); + assert!(!marker.exists(), "and gone after"); + // a set-up that fails or panics removes it too + assert_eq!(set_up_card(&marker, Duration::from_secs(10), || Err::<(), _>("no card".into()), || ()), Err("no card".into())); + assert!(!marker.exists()); + let r = set_up_card(&marker, Duration::from_secs(10), || -> Result<(), String> { panic!("driver") }, || ()); + assert!(r.is_err() && !marker.exists(), "{r:?}"); + // a marker found before the set-up starts means the last one took LightCraft down: the card is not touched + std::fs::write(&marker, "left behind").unwrap(); + let touched = Arc::new(AtomicBool::new(false)); + let t = touched.clone(); + let r = set_up_card( + &marker, + Duration::from_secs(10), + move || { + t.store(true, Ordering::Relaxed); + Ok(()) + }, + || (), + ); + assert_eq!(r, Err(GPU_CRASHED.to_string())); + assert!(!touched.load(Ordering::Relaxed) && marker.exists()); + let _ = std::fs::remove_dir_all(&dir); + } + + #[test] + fn a_set_up_that_takes_too_long_is_given_up_on() { + let dir = folder("slowsetup"); + let marker = dir.join(GPU_SETUP_MARKER); + let started = web_time::Instant::now(); + let slow = || -> Result<(), String> { + std::thread::sleep(Duration::from_millis(600)); + Ok(()) + }; + let r = set_up_card(&marker, Duration::from_millis(50), slow, || ()); + assert!(r.as_ref().is_err_and(|e| e.contains("took longer") && worth_retrying(e)), "{r:?}"); + assert!(started.elapsed() < Duration::from_millis(500), "the caller does not wait for the card"); + // while the first is still stuck, another set-up of the same model does not start a second one + let again = set_up_card(&marker, Duration::from_secs(5), || Ok(()), || ()); + assert!(again.is_err_and(|e| e.contains("still setting up") && worth_retrying(&e))); + // once it ends its marker goes, and the card can be set up again + while marker.exists() { + assert!(started.elapsed() < Duration::from_secs(10), "the stuck set-up ended"); + std::thread::sleep(Duration::from_millis(20)); + } + while SETTING_UP.lock().unwrap().iter().any(|m| m == &marker) { + std::thread::sleep(Duration::from_millis(5)); + } + assert_eq!(set_up_card(&marker, Duration::from_secs(5), || Ok(1), || 2), Ok((1, 2))); + let _ = std::fs::remove_dir_all(&dir); + } +} diff --git a/crates/engine/src/files.rs b/crates/engine/src/files.rs index 01de8a7a..95481a4f 100644 --- a/crates/engine/src/files.rs +++ b/crates/engine/src/files.rs @@ -11,7 +11,7 @@ use lightcraft_pipeline::SourceInfo; use lightcraft_raster::Rgb32f; use lightcraft_raster::resample::{Filter, fit}; -use crate::media::{FileLoader, FileProbe, PreviewLoader, ProbeInfo}; +use crate::media::{DenoiseSpec, FileLoader, FileProbe, PairLoader, PreviewLoader, ProbeInfo}; fn meta_of(m: &lightcraft_meta::Metadata) -> (Meta, Option) { let shutter = m.exposure_time.map(|t| if t >= 1.0 { format!("{t:.0}") } else { format!("1/{:.0}", 1.0 / t) }).unwrap_or_default(); @@ -220,22 +220,56 @@ pub fn bin_factor(raw: &lightcraft_raw::RawImage, max_edge: usize) -> Option Result<(Rgb32f, SourceInfo), String> { - rayon::scope(|_| load_bytes_now(std::borrow::Cow::Borrowed(bytes), max_edge)) + rayon::scope(|_| load_bytes_now(std::borrow::Cow::Borrowed(bytes), max_edge, None).map(|(img, _, info)| (img, info))) } /// [`load_bytes`] taking the file's bytes: a raw file's bytes are freed as soon as it is decoded /// (less memory held while it is developed). pub fn load_vec(bytes: Vec, max_edge: usize) -> Result<(Rgb32f, SourceInfo), String> { - rayon::scope(move |_| load_bytes_now(std::borrow::Cow::Owned(bytes), max_edge)) + rayon::scope(move |_| load_bytes_now(std::borrow::Cow::Owned(bytes), max_edge, None).map(|(img, _, info)| (img, info))) } -fn load_bytes_now(bytes: std::borrow::Cow<'_, [u8]>, max_edge: usize) -> Result<(Rgb32f, SourceInfo), String> { +/// [`load_vec`] that also develops the photo's cached denoised picture (`spec`) from the same decode: the plain +/// picture, the denoised one (`None` when it cannot be read or does not fit this raw: the plain one is then all +/// there is) and the source facts. Both pictures are the same size. +pub fn load_vec_pair(bytes: Vec, max_edge: usize, spec: &DenoiseSpec) -> Result<(Rgb32f, Option, SourceInfo), String> { + rayon::scope(move |_| load_bytes_now(std::borrow::Cow::Owned(bytes), max_edge, Some(spec))) +} + +/// The denoised camera RGB of `raw` from its cached product, as the plain development would have it: the default +/// crop, and `factor` × `factor` blocks averaged into one pixel when the plain one was binned. `None` when the product +/// is missing, damaged, made from something else or not the size of this raw's active area. +fn denoised_camera_rgb(raw: &lightcraft_raw::RawImage, spec: &DenoiseSpec, factor: usize) -> Option { + use lightcraft_denoise::product::{self, Window}; + let a = raw.active_area; + let header = product::read_header_at(&spec.product).ok()?; + if header.key != spec.key || (header.width, header.height) != (a.width, a.height) { + return None; + } + if raw.opcodes.list3.is_empty() { + let c = raw.develop_crop(a.width, a.height); + let window = Window { x: c.x, y: c.y, width: c.width, height: c.height }; + product::read_window(&spec.product, &spec.key, Some(window), factor, Some(HIGHLIGHT_CLIP)).ok() + } else { + // (the plain development is not binned either when the file has opcodes of this kind: `factor` is 1) + let whole = product::read(&spec.product, &spec.key, 1).ok()?; + Some(raw.finish_demosaiced(whole)) + } +} + +fn load_bytes_now( + bytes: std::borrow::Cow<'_, [u8]>, + max_edge: usize, + denoise: Option<&DenoiseSpec>, +) -> Result<(Rgb32f, Option, SourceInfo), String> { if lightcraft_raw::probe(&bytes).is_some() { let mut raw = match lightcraft_raw::decode(&bytes).map_err(|e| preview_reason(&bytes, e)) { Ok(r) => r, Err(Ok(why)) => { // show the camera's embedded JPEG (rendered, not raw) until the variant is supported - return load_embedded_preview(&bytes, max_edge).ok_or(format!("unsupported raw ({why}) without an embedded preview")); + return load_embedded_preview(&bytes, max_edge) + .map(|(img, info)| (img, None, info)) + .ok_or(format!("unsupported raw ({why}) without an embedded preview")); } Err(Err(e)) => return Err(e), }; @@ -251,48 +285,57 @@ fn load_bytes_now(bytes: std::borrow::Cow<'_, [u8]>, max_edge: usize) -> Result< // Previews and thumbnails bin the mosaic straight to (about) the size they need; only // larger levels (exports, 1:1) demosaic the whole sensor. let t0 = web_time::Instant::now(); - let binned = match bin_factor(&raw, max_edge) { + let bin = bin_factor(&raw, max_edge); + let binned = match bin { Some(k) => raw.develop_binned(k, HIGHLIGHT_CLIP).map_err(|e| e.to_string())?, None => None, }; - let mut img = match binned { + let factor = if binned.is_some() { bin.unwrap_or(1) } else { 1 }; + let img = match binned { Some(img) => img, None => { let method = if max_edge <= 600 { lightcraft_raw::Method::Bilinear } else { lightcraft_raw::Method::Ahd }; raw.develop(method).map_err(|e| e.to_string())? } }; + // the same development from the denoised mosaic, when the photo has been denoised + let twin = denoise.and_then(|spec| denoised_camera_rgb(&raw, spec, factor)).filter(|d| d.width == img.width && d.height == img.height); // the samples aren't needed any more (the colour model below reads only the tags) raw.data = lightcraft_raw::RawData::U16(Vec::new()); - let mut stages = vec![("develop", t0.elapsed())]; - stages.push(("transform", t0.elapsed())); - lightcraft_raw::highlight::reconstruct(&mut img, t.wb, HIGHLIGHT_CLIP); - stages.push(("highlights", t0.elapsed())); + let wb = t.wb; let m = camera_look.as_ref().map(|p| p.matrix.mul(&t.matrix)).unwrap_or(t.matrix).to_f32(); let hue_sat = camera_look.as_ref().and_then(|p| p.hue_sat.as_ref()).and_then(crate::camera_preview::HueSat::new); let gain = 2f32.powf(t.baseline_exposure as f32); - let wb = t.wb; // A DNG's own profile look (hue/saturation map, look table), DNG spec chapter 6. let tables = lightcraft_raw::profile::ProfileTables::new(&raw.color.profile, lightcraft_raw::color::illuminant_weight(&raw.color, xy)); - img.map_in_place(|p| { - let c = [p[0] * wb[0], p[1] * wb[1], p[2] * wb[2]]; - let rgb = [ - m[0][0] * c[0] + m[0][1] * c[1] + m[0][2] * c[2], - m[1][0] * c[0] + m[1][1] * c[1] + m[1][2] * c[2], - m[2][0] * c[0] + m[2][1] * c[1] + m[2][2] * c[2], - ]; - let rgb = match &tables { - Some(tables) => tables.apply(rgb, gain), - None => rgb.map(|v| v * gain), - }; - // after the baseline exposure, as when it was fitted - hue_sat.as_ref().map_or(rgb, |h| h.apply(rgb)).map(|v| v.max(0.0)) - }); - stages.push(("colour", t0.elapsed())); - let img = fit(&img, max_edge, max_edge, Filter::Box); - stages.push(("fit", t0.elapsed())); - let img = img.into_oriented(raw.orientation); - stages.push(("orient", t0.elapsed())); + // camera RGB → what the pipeline takes: clipped highlights rebuilt, white balance and the colour model applied, + // fitted to `max_edge` and upright (the same for the plain and the denoised picture) + let finish = |mut img: Rgb32f| { + let mut stages = vec![("develop", t0.elapsed())]; + stages.push(("transform", t0.elapsed())); + lightcraft_raw::highlight::reconstruct(&mut img, wb, HIGHLIGHT_CLIP); + stages.push(("highlights", t0.elapsed())); + img.map_in_place(|p| { + let c = [p[0] * wb[0], p[1] * wb[1], p[2] * wb[2]]; + let rgb = [ + m[0][0] * c[0] + m[0][1] * c[1] + m[0][2] * c[2], + m[1][0] * c[0] + m[1][1] * c[1] + m[1][2] * c[2], + m[2][0] * c[0] + m[2][1] * c[1] + m[2][2] * c[2], + ]; + let rgb = match &tables { + Some(tables) => tables.apply(rgb, gain), + None => rgb.map(|v| v * gain), + }; + hue_sat.as_ref().map_or(rgb, |h| h.apply(rgb)).map(|v| v.max(0.0)) + }); + stages.push(("colour", t0.elapsed())); + let img = fit(&img, max_edge, max_edge, Filter::Box); + stages.push(("fit", t0.elapsed())); + let img = img.into_oriented(raw.orientation); + stages.push(("orient", t0.elapsed())); + (img, stages) + }; + let (img, stages) = finish(img); if lightcraft_pipeline::profiling() { let mut prev = std::time::Duration::ZERO; let parts: Vec = stages @@ -305,6 +348,7 @@ fn load_bytes_now(bytes: std::borrow::Cow<'_, [u8]>, max_edge: usize) -> Result< .collect(); eprintln!("[profile] raw source {}×{} (max {max_edge}, ms after decode): {}", img.width, img.height, parts.join(", ")); } + let twin = twin.map(|picture| finish(picture).0); let (temp, tint) = xy_to_temp_tint(xy); let relative = crate::camera_preview::file_local_look(raw.format) && t.matrix_is_fallback; // White balance re-evaluates the file's own colour model (when it has one and no @@ -317,6 +361,7 @@ fn load_bytes_now(bytes: std::borrow::Cow<'_, [u8]>, max_edge: usize) -> Result< let (temp, tint) = if relative { (6500.0, 0.0) } else { (temp.round(), tint.round()) }; return Ok(( img, + twin, SourceInfo { raw: true, as_shot_temp: temp, as_shot_tint: tint, lens, relative_wb: relative, camera_color, camera_tone, mattes }, )); } @@ -324,7 +369,7 @@ fn load_bytes_now(bytes: std::borrow::Cow<'_, [u8]>, max_edge: usize) -> Result< drop(bytes); let img = d.to_working(); let img = if img.width.max(img.height) > max_edge { fit(&img, max_edge, max_edge, Filter::Mitchell) } else { img }; - Ok((img.into_oriented(Orientation::from_exif(d.orientation)), SourceInfo::default())) + Ok((img.into_oriented(Orientation::from_exif(d.orientation)), None, SourceInfo::default())) } /// The semantic masks of a DNG that AI masks understand, over the developed image (default crop, @@ -462,6 +507,21 @@ pub fn fs_preview_loader() -> PreviewLoader { }) } +/// The filesystem-backed [`PairLoader`]: the file is read and decoded once, and developed as usual and from the photo's +/// denoised picture. It holds two pictures at once, so it counts for more at the memory gate than a plain load. +pub fn fs_pair_loader() -> PairLoader { + Arc::new(|path: &str, max_edge: usize, spec: &DenoiseSpec| { + let len = std::fs::metadata(path).map(|m| m.len() as usize).unwrap_or(0); + let weight = len * if max_edge <= crate::media::SourceLevel::Thumb.max_edge() { 4 } else { 10 }; + let gate = crate::memory::work_gate(); + let _permit = if crate::memory::is_background() { gate.acquire(weight) } else { gate.acquire_urgent(weight) }; + let bytes = std::fs::read(path).map_err(|e| format!("{path}: {e}"))?; + let r = load_vec_pair(bytes, max_edge, spec); + crate::memory::release(); + r + }) +} + /// Filesystem-backed hooks (native). On the web the host installs bytes-based hooks instead. /// /// Their working memory is bounded by [`crate::memory::work_gate`]: background loads (grid @@ -497,6 +557,7 @@ impl crate::Session { self.media.file_loader = Some(l); self.media.file_probe = Some(p); self.media.preview_loader = Some(fs_preview_loader()); + self.media.denoise.loader = Some(fs_pair_loader()); self } } @@ -537,6 +598,37 @@ mod tests { assert!(probe_bytes("x.jpg", b"not an image").is_err()); } + #[test] + fn cached_denoise_uses_the_same_white_balance_profile_and_orientation_as_plain_raw() { + use lightcraft_denoise::product; + use lightcraft_raw::{DngWriteOptions, Method, Orientation, profile::HsvTable, write_dng}; + let dir = crate::tests_xmp::temp_dir("denoise-color-finish"); + let fixture = crate::tests_xmp::synthetic_dng_with(None, Default::default()); + let mut raw = lightcraft_raw::decode(&fixture).unwrap(); + raw.orientation = Orientation::Rotate90; + raw.color.baseline_exposure = 0.5; + raw.color.profile.look_table = + Some(HsvTable { hue_divisions: 1, sat_divisions: 2, val_divisions: 1, data: vec![[23.0, 0.65, 0.8]; 2], srgb_value: false }); + let bytes = write_dng(&raw, &DngWriteOptions::default()).unwrap(); + let decoded = lightcraft_raw::decode(&bytes).unwrap(); + assert!(decoded.color.profile.look_table.is_some()); + // A cache containing the ordinary demosaic must undergo the identical downstream finish. + let camera_rgb = decoded.develop(Method::Ahd).unwrap(); + let key = "synthetic-color-finish"; + let path = dir.join("same.lcdn"); + product::write(&path, &camera_rgb, key).unwrap(); + let spec = DenoiseSpec { product: path, key: key.into(), make: None }; + let (ordinary, _) = load_bytes(&bytes, 2048).unwrap(); + let (plain, cached, _) = load_vec_pair(bytes, 2048, &spec).unwrap(); + assert_eq!(plain.data, ordinary.data, "requesting a twin must not change the ordinary path"); + let cached = cached.unwrap(); + assert_eq!((plain.width, plain.height), (24, 32)); + assert_eq!((cached.width, cached.height), (plain.width, plain.height)); + let worst = plain.data.iter().zip(&cached.data).flat_map(|(a, b)| a.iter().zip(b).map(|(a, b)| (a - b).abs())).fold(0.0f32, f32::max); + assert!(worst < 0.002, "color finish differs beyond cache half-float rounding: {worst}"); + let _ = std::fs::remove_dir_all(dir); + } + /// Luminance (from linear RGB) minus its 9 × 9 local mean: tone differences between a render and a camera JPEG /// mostly cancel, edges remain. fn detail(w: usize, h: usize, lum: impl Fn(usize, usize) -> f32) -> Vec { diff --git a/crates/engine/src/lib.rs b/crates/engine/src/lib.rs index b2f57d24..a6c5f642 100644 --- a/crates/engine/src/lib.rs +++ b/crates/engine/src/lib.rs @@ -10,6 +10,10 @@ //! run them off the UI thread. #![forbid(unsafe_code)] +// Model/cache types remain available without linking the optional inference crate. +#[cfg(not(feature = "denoise"))] +extern crate lightcraft_denoise_core as lightcraft_denoise; + pub mod availability; mod camera_preview; pub mod camera_profiles; @@ -18,6 +22,7 @@ pub mod config; pub mod crs; pub mod crs_masks; pub mod demo; +pub mod denoise; pub mod devices; pub mod export; pub mod face_download; @@ -39,6 +44,7 @@ pub mod logging; pub mod media; pub mod memory; pub mod merge; +mod model_download; pub mod originals; pub mod preset_import; pub mod preset_luminar; @@ -138,8 +144,10 @@ pub struct Interaction { /// Source of [`Session::visible_shared`] generations (process-wide, so two sessions never share one). static VISIBLE_GEN: std::sync::atomic::AtomicU64 = std::sync::atomic::AtomicU64::new(1); +static LIBRARY_GEN: std::sync::atomic::AtomicU64 = std::sync::atomic::AtomicU64::new(1); pub struct Session { + library_generation: u64, /// Auto Sync: edits to the active photo also change the other selected photos (the settings /// that changed, nothing else). pub auto_sync: bool, @@ -163,6 +171,8 @@ pub struct Session { /// Set by a command whose change must not rewrite the photo's XMP sidecar even with auto-write on /// (a catalog-only edit of data the sidecar writer does not emit); consumed when the command ends. pub(crate) skip_auto_write: bool, + /// AI denoise: the model in use, the photos that have their picture and the work in progress. + pub(crate) denoise: denoise::State, /// Where the host keeps face models (one folder each); `None` where there is no file system (the web). pub face_models_dir: Option, /// Face model downloads started this session (the staged files wait in `/.downloads`). @@ -262,8 +272,13 @@ impl Default for Session { } impl Session { + pub fn library_generation(&self) -> u64 { + self.library_generation + } + pub fn new() -> Session { Session { + library_generation: LIBRARY_GEN.fetch_add(1, std::sync::atomic::Ordering::Relaxed), auto_sync: false, catalog: Catalog::new(), source: LibrarySource::All, @@ -278,6 +293,7 @@ impl Session { redo: Vec::new(), interaction: None, skip_auto_write: false, + denoise: Default::default(), face_models_dir: None, face_downloads: Default::default(), face_catalog: Default::default(), @@ -826,6 +842,8 @@ mod tests_album_order; #[cfg(test)] mod tests_color; #[cfg(test)] +mod tests_denoise; +#[cfg(test)] mod tests_export; #[cfg(test)] mod tests_face_models; diff --git a/crates/engine/src/library.rs b/crates/engine/src/library.rs index d01c1b72..a83dc4a8 100644 --- a/crates/engine/src/library.rs +++ b/crates/engine/src/library.rs @@ -297,6 +297,9 @@ impl Session { { log::error!("library: {e}"); } + // Photo ids belong to one library; denoise work must not carry into the next. + self.denoise.library_changed(); + self.media.denoise.clear(); // what was learned about the old library's faces is saved, and none of it carries over (photo ids are per library) #[cfg(not(target_arch = "wasm32"))] self.faces.library_changed(); @@ -308,6 +311,7 @@ impl Session { static LOADS: std::sync::atomic::AtomicU64 = std::sync::atomic::AtomicU64::new(0); catalog.revision = LOADS.fetch_add(1, std::sync::atomic::Ordering::Relaxed).wrapping_add(1) << 32; self.catalog = catalog; + self.library_generation = crate::LIBRARY_GEN.fetch_add(1, std::sync::atomic::Ordering::Relaxed); self.undo.clear(); self.redo.clear(); self.interaction = None; diff --git a/crates/engine/src/media.rs b/crates/engine/src/media.rs index e3ed5815..b0439390 100644 --- a/crates/engine/src/media.rs +++ b/crates/engine/src/media.rs @@ -125,6 +125,43 @@ pub type FileLoader = Arc Result<(Rgb32f, SourceInfo), St /// `max_edge` (set by the app). `None`: no usable preview. pub type PreviewLoader = Arc Option + Send + Sync>; +/// Makes a photo's denoised picture when it is not cached yet (an export does not wait for the background queue): +/// `Ok(true)` once it is there, `Ok(false)` when denoise does not apply to this photo (render it plain), `Err` when it +/// should have worked and did not. +pub type MakeProduct = Arc Result + Send + Sync>; + +/// A raw photo's cached AI-denoised picture (see [`crate::denoise`]): where the file is and the key it must have been +/// made with. A file that is missing, damaged or made for something else is a cache miss, never an error. +#[derive(Clone)] +pub struct DenoiseSpec { + pub product: std::path::PathBuf, + pub key: String, + /// Run before the picture is read, by a job that must not render without it (an export). + pub make: Option, +} + +/// Decodes a raw file once and develops it twice: as usual, and from the denoised picture in `spec` when that can be +/// read (`None` when it cannot). Both are at most `max_edge` long and the same size. +pub type PairLoader = Arc Result<(Rgb32f, Option, SourceInfo), String> + Send + Sync>; + +/// A photo's denoised picture next to its plain one, and the last mix of the two that was asked for. +pub struct Twin { + pub image: Arc, + /// The Amount (in thousandths) and the picture mixed for it, so dragging other sliders keeps one source picture. + mix: std::sync::Mutex)>>, +} + +impl Twin { + pub fn new(image: Arc) -> Twin { + Twin { image, mix: std::sync::Mutex::new(None) } + } + + /// Bytes reserved for the twin and its lazy mix, before either cache cost can grow. + fn bytes(&self) -> usize { + self.image.data.len().saturating_mul(24) + } +} + /// Pixels and the source interpretation learned while decoding; they must be cached together. #[derive(Clone)] pub struct DecodedSource { @@ -133,11 +170,46 @@ pub struct DecodedSource { /// A smart preview's stored camera tone curve: the one decoder fact its pixels need that the /// catalog's header facts lack (used when `info` is `None`). pub camera_tone: Option, + /// The same picture developed from the AI-denoised mosaic, when the photo has one cached. + pub denoised: Option>, } impl DecodedSource { pub fn new(image: Arc, info: Option) -> Self { - DecodedSource { image, info, camera_tone: None } + DecodedSource { image, info, camera_tone: None, denoised: None } + } + + /// The picture to develop for a Denoise `amount` (0 to 1, see [`DevelopSettings::denoise_amount`]): the plain one, + /// the denoised one, or a mix of the two. A mix is made once per amount and kept, so the stages after it are + /// reused until the amount changes. + pub fn image_for(&self, amount: f32) -> Arc { + let Some(twin) = &self.denoised else { return self.image.clone() }; + if amount.is_nan() || amount <= 0.0 || twin.image.width != self.image.width || twin.image.height != self.image.height { + return self.image.clone(); + } + if amount >= 1.0 { + return twin.image.clone(); + } + let thousandths = (amount * 1000.0).round().clamp(1.0, 999.0) as u32; + let mut memo = twin.mix.lock().unwrap_or_else(std::sync::PoisonError::into_inner); + if let Some((t, img)) = memo.as_ref() + && *t == thousandths + { + return img.clone(); + } + let mut mixed = (*self.image).clone(); + if !lightcraft_denoise::product::blend(&mut mixed, &twin.image, thousandths as f32 / 1000.0) { + return self.image.clone(); + } + let mixed = Arc::new(mixed); + *memo = Some((thousandths, mixed.clone())); + mixed + } + + /// Cache cost: the picture, its segmentation mattes, its denoised twin and room for the lazy mix. + pub fn bytes(&self) -> usize { + let mattes = self.info.as_ref().and_then(|i| i.mattes.as_ref()).map_or(0, |m| m.bytes()); + self.image.data.len() * 12 + 64 + std::mem::size_of::() + mattes + self.denoised.as_ref().map_or(0, |t| t.bytes()) } /// What to render these pixels against: the decoder's facts, else `header` (the catalog's) @@ -161,6 +233,8 @@ pub enum SourceRef { /// The photo's smart preview, used when the original can't be read (offline drive). The /// UI thread never checks whether the original is there; the render worker finds out. fallback: Option, + /// The photo has a cached denoised picture: load it with the plain one (see [`PairLoader`]). + denoise: Option<(DenoiseSpec, PairLoader)>, }, /// A smart preview standing in for a missing original. Smart { @@ -176,16 +250,46 @@ impl SourceRef { pub fn load_source(&self) -> Result { let image = match self { SourceRef::Loaded(a) => return Ok((**a).clone()), - SourceRef::Demo { scene, max_edge } => Arc::new(scene.render_fit(*max_edge)), - SourceRef::File { path, max_edge, loader, fallback } => { - let r = match loader { + SourceRef::Demo { scene, max_edge } => crate::demo::scene_pixels(scene, *max_edge), + SourceRef::File { path, max_edge, loader, fallback, denoise } => { + let plain = || match loader { Some(l) => l(path, *max_edge).map(|(image, info)| DecodedSource::new(Arc::new(image), Some(info))), None => Err(format!("no decoder available for {path}")), }; + // the denoised picture comes with the plain one from a single decode; when it cannot be read the + // pair loader gives the plain picture alone (the cached file is never a reason to fail a render) + let r = match denoise { + Some((spec, pair)) => { + // a job that needs the denoised picture has it made first; if that fails the job fails (no smart + // preview stands in: an export must not quietly come out without the denoise it was asked for) + let made = match &spec.make { + Some(make) => make()?, + None => true, + }; + if made { + pair(path, *max_edge, spec).and_then(|(image, twin, info)| { + if spec.make.is_some() && twin.is_none() { + return Err("the cached AI Denoise picture could not be read; clear the denoise cache and retry".into()); + } + Ok(DecodedSource { + image: Arc::new(image), + info: Some(info), + camera_tone: None, + denoised: twin.map(|t| Arc::new(Twin::new(Arc::new(t)))), + }) + }) + } else { + plain() + } + } + None => plain(), + }; // An offline original renders from its smart preview (no decoder facts: header ones). return match (r, fallback) { #[cfg(not(target_arch = "wasm32"))] - (Err(e), Some(sp)) if crate::smart::is_valid(sp) => crate::smart::load(sp).map_err(|e2| format!("{e}; smart preview: {e2}")), + (Err(e), Some(sp)) if denoise.as_ref().is_none_or(|(spec, _)| spec.make.is_none()) && crate::smart::is_valid(sp) => { + crate::smart::load(sp).map_err(|e2| format!("{e}; smart preview: {e2}")) + } (r, _) => r, }; } @@ -224,6 +328,54 @@ pub struct MediaCache { scenes: Vec, /// Rendered thumbnails (memory, plus disk once a library is attached). pub rendered: Arc, + /// Which photos have a cached denoised picture, and how to read it. + pub denoise: DenoiseIndex, +} + +/// The photos that have a cached denoised picture for the denoise model in use. Only memory: nothing here touches the +/// file system, so it can be asked on the UI thread. +#[derive(Default)] +pub struct DenoiseIndex { + ready: std::collections::HashMap, + /// Reads the pictures (`None`: this build cannot, and nothing is ever ready). + pub loader: Option, +} + +impl DenoiseIndex { + /// What to load for `id` besides its plain picture, and how. + pub fn get(&self, id: PhotoId) -> Option<(&DenoiseSpec, &PairLoader)> { + self.ready.get(&id).zip(self.loader.as_ref()) + } + + pub fn spec(&self, id: PhotoId) -> Option<&DenoiseSpec> { + self.ready.get(&id) + } + + pub fn set(&mut self, id: PhotoId, spec: DenoiseSpec) { + self.ready.insert(id, spec); + } + + pub fn remove(&mut self, id: PhotoId) -> bool { + self.ready.remove(&id).is_some() + } + + pub fn clear(&mut self) { + self.ready.clear(); + } + + pub fn len(&self) -> usize { + self.ready.len() + } + + pub fn is_empty(&self) -> bool { + self.ready.is_empty() + } + + pub fn ids(&self) -> Vec { + let mut v: Vec = self.ready.keys().copied().collect(); + v.sort(); + v + } } impl Default for MediaCache { @@ -244,6 +396,7 @@ impl Default for MediaCache { availability: Default::default(), scenes: Vec::new(), rendered: Arc::new(PreviewCache::memory(rendered_budget(budget))), + denoise: DenoiseIndex::default(), } } } @@ -253,11 +406,6 @@ fn rendered_budget(share: usize) -> usize { RENDERED_MEM_BYTES.min(share / 4) } -fn source_bytes(s: &DecodedSource) -> usize { - let mattes = s.info.as_ref().and_then(|i| i.mattes.as_ref()).map_or(0, |m| m.bytes()); - s.image.data.len() * 12 + 64 + std::mem::size_of::() + mattes -} - impl MediaCache { /// Keep rendered thumbnails on disk in `dir` as well. The cache this replaces is retired /// (its files stay): render jobs still holding it can't write through it any more, even @@ -320,8 +468,8 @@ impl MediaCache { let Some((tick, which)) = candidates.into_iter().flatten().min() else { break }; let freed = match which { 0 => self.thumbs.pop_oldest(), - 1 => self.previews.iter().position(|e| e.2 == tick).map(|i| source_bytes(&self.previews.remove(i).1)), - 2 => self.full.take().map(|e| source_bytes(&e.1)), + 1 => self.previews.iter().position(|e| e.2 == tick).map(|i| self.previews.remove(i).1.bytes()), + 2 => self.full.take().map(|e| e.1.bytes()), _ => self.rendered.evict_oldest(), }; match freed { @@ -373,7 +521,7 @@ impl MediaCache { let tick = lightcraft_preview::next_tick(); match level { SourceLevel::Thumb => { - let cost = source_bytes(&img); + let cost = img.bytes(); self.thumbs.insert(id, img, cost); } SourceLevel::Preview => { @@ -416,13 +564,42 @@ impl MediaCache { /// Decoded sources held: (thumbnail level, preview level, full size). pub fn usage(&self) -> (crate::memory::Usage, crate::memory::Usage, crate::memory::Usage) { use crate::memory::Usage; - let previews = Usage::new(self.previews.len(), self.previews.iter().map(|e| source_bytes(&e.1)).sum()); - let full = self.full.as_ref().map(|e| Usage::new(1, source_bytes(&e.1))).unwrap_or_default(); + let previews = Usage::new(self.previews.len(), self.previews.iter().map(|e| e.1.bytes()).sum()); + let full = self.full.as_ref().map(|e| Usage::new(1, e.1.bytes())).unwrap_or_default(); (Usage::new(self.thumbs.len(), self.thumbs.cost()), previews, full) } + /// What a render of photo `id` with settings `s` adds to its identity for the denoised picture it will use: 0 unless + /// it uses one (the Denoise amount is above 0 and the photo has a cached denoised picture). + pub fn denoise_salt(&self, id: PhotoId, s: &DevelopSettings) -> u64 { + match self.denoise.spec(id) { + Some(spec) if s.denoise_amount() > 0.0 => Hasher128::new().str(&spec.key).finish().0 as u64 | 1, + _ => 0, + } + } + + /// How to load photo `p` at `level`: from the cache when its source is there, else from its file (with its denoised + /// picture beside it when it has one), its smart preview or the demo scene. pub fn source_ref(&mut self, p: &Photo, level: SourceLevel) -> SourceRef { - if let Some(a) = self.get_source(p.id, level) { + self.source_ref_with(p, level, true) + } + + /// [`Self::source_ref`] that never takes a decoded source from memory: for a job that needs something the cached + /// one lacks (an export, whose denoised picture is made when it runs). + pub fn source_ref_uncached(&mut self, p: &Photo, level: SourceLevel) -> SourceRef { + self.source_ref_with(p, level, false) + } + + fn source_ref_with(&mut self, p: &Photo, level: SourceLevel, cached: bool) -> SourceRef { + let mut r = self.source_ref_plain(p, level, cached); + if let (SourceRef::File { denoise, .. }, Some((spec, loader))) = (&mut r, self.denoise.get(p.id)) { + *denoise = Some((spec.clone(), loader.clone())); + } + r + } + + fn source_ref_plain(&mut self, p: &Photo, level: SourceLevel, cached: bool) -> SourceRef { + if cached && let Some(a) = self.get_source(p.id, level) { return SourceRef::Loaded(Box::new(a)); } // Procedural scenes have a nominal size: "full" is that size, not unbounded. @@ -460,10 +637,12 @@ impl MediaCache { } match self.scenes.iter().find(|s| s.id == *scene) { Some(s) => SourceRef::Demo { scene: Box::new(s.clone()), max_edge }, - None => SourceRef::File { path: format!("demo:{scene}"), max_edge, loader: None, fallback: None }, + None => SourceRef::File { path: format!("demo:{scene}"), max_edge, loader: None, fallback: None, denoise: None }, } } - Source::File { path } => SourceRef::File { path: path.clone(), max_edge, loader: self.file_loader.clone(), fallback: None }, + Source::File { path } => { + SourceRef::File { path: path.clone(), max_edge, loader: self.file_loader.clone(), fallback: None, denoise: None } + } } } } @@ -574,7 +753,8 @@ impl RenderJob { let was_loaded = matches!(self.source, SourceRef::Loaded(_)); match self.source.load_source() { Ok(source) => { - let src = &source.image; + // the plain picture, or (Denoise amount above 0 and a denoised picture cached) the mix the amount asks for + let src = &source.image_for(self.settings.denoise_amount()); let info = source.info_or(self.info); // Thumbnails (many small jobs side by side) stay on the CPU; views and exports use // the GPU when there is one. @@ -758,22 +938,23 @@ impl crate::Session { let content = source_key.0 as u64; // Temporary "before" settings must not replace the photo's cached develop hash. let settings_hash = if before { settings.hash64() } else { self.media.settings_hashes.get(id, &settings) }; + let salt = self.media.denoise_salt(id, &settings); let key = settings_hash ^ ((max_w as u64) << 40) ^ ((max_h as u64) << 20) ^ (apply_crop as u64) ^ (level as u64) << 60 ^ id.0.wrapping_mul(0x9e37_79b9_7f4a_7c15) - ^ content.rotate_left(17); + ^ content.rotate_left(17) + ^ salt; let cache = thumb_bucket.map(|b| { - let k = Hasher128::new() - .str(&content_key) - .u64(settings_hash) - .u64(b as u64) - .u64(RENDER_CACHE_VERSION) - .u64(crate::camera_profiles::cache_key()) - .finish(); - (self.media.rendered.clone(), k) + let mut k = Hasher128::new(); + k.str(&content_key).u64(settings_hash).u64(b as u64).u64(RENDER_CACHE_VERSION).u64(crate::camera_profiles::cache_key()); + // (only a render that uses a denoised picture has it in its key, so every other thumbnail stays valid) + if salt != 0 { + k.u64(salt); + } + (self.media.rendered.clone(), k.finish()) }); Some(RenderJob { request_id: 0, @@ -1064,6 +1245,15 @@ impl crate::Session { depth: lightcraft_pipeline::OutputDepth, ) -> Result { let mut job = self.render_job(id, max_w, max_h, false, true).ok_or("no such photo")?; + // An export is made from the denoised picture its Denoise amount asks for, made when the job runs if the + // background queue has not got to it. (A source kept in memory may lack the picture, so the job reads the file.) + if let Some(wanted) = self.denoise_for_export(id, &job.settings)? { + let p = self.catalog.photo(id).ok_or("no such photo")?.clone(); + job.source = self.media.source_ref_uncached(&p, job.level); + if let SourceRef::File { denoise, .. } = &mut job.source { + *denoise = Some(wanted); + } + } job.request.space = space; job.request.depth = depth; job.key ^= (space as u64 + 1).wrapping_mul(0xa076_1d64_78bd_642f) ^ (depth as u64 + 1).wrapping_mul(0xe703_7ed1_a0b4_28db); @@ -1211,6 +1401,20 @@ mod thumbnail_hash_tests { mod tests { use super::*; + #[test] + fn denoise_mix_is_reserved_before_the_source_enters_the_cache() { + let mut plain = Rgb32f::new(8, 8); + plain.data.fill([1.0; 3]); + let mut source = DecodedSource::new(Arc::new(plain), None); + source.denoised = Some(Arc::new(Twin::new(Arc::new(Rgb32f::new(8, 8))))); + let reserved = source.bytes(); + assert!(reserved >= 3 * 8 * 8 * 12); + assert_eq!(source.image_for(0.5).data[0], [0.5; 3]); + assert_eq!(source.bytes(), reserved, "creating a mix cannot evade the LRU's recorded cost"); + let _ = source.image_for(0.25); + assert_eq!(source.bytes(), reserved, "changing Amount replaces the reserved mix"); + } + #[test] fn decoder_info_survives_render_jobs_cache_and_eviction() { let tone = lightcraft_pipeline::tone::CameraTone::new(std::array::from_fn(|i| { @@ -1232,6 +1436,7 @@ mod tests { Ok((image, loaded_info.clone())) })), fallback: None, + denoise: None, }; job.info = SourceInfo::default(); // Header facts cannot override decoder facts. job.settings = Arc::new(DevelopSettings::default()); diff --git a/crates/engine/src/model_download.rs b/crates/engine/src/model_download.rs new file mode 100644 index 00000000..98707f78 --- /dev/null +++ b/crates/engine/src/model_download.rs @@ -0,0 +1,359 @@ +//! Downloading a denoise model explicitly requested after showing its terms. +//! +//! The transfer is [`lightcraft_fetch`] (pure-Rust HTTPS with resume, size and SHA-256 checks), on a background thread, +//! so the UI never waits for the network. Only a model listed in [`lightcraft_denoise::known::Download`] can be fetched: +//! the address and hash come from our pinned catalog or the user's model catalog, and no token is sent with it. What arrives +//! matches the model's recorded size and SHA-256 or it is thrown away; it then waits in a staging folder until the model +//! is installed (`denoise.models.install`, or by itself once the user has accepted the model's terms in the download). +//! +//! Failure is a message, never a panic: no network, a stalled or truncated transfer, a full disk, a file that is not the +//! one expected, and a panic inside the thread itself all end as [`State::Failed`]. + +use std::collections::HashMap; +use std::path::{Path, PathBuf}; +use std::sync::atomic::{AtomicBool, Ordering}; +use std::sync::{Arc, Mutex, PoisonError}; +use std::time::Duration; + +use lightcraft_denoise::known::Download; + +/// The folder inside the models folder where a downloaded file waits to be installed. Its name starts with a dot, so it +/// can never be a model's own folder (ids start with a letter or digit). +pub const STAGING: &str = ".downloads"; + +/// Where a download stands. +#[derive(Clone, Debug, PartialEq)] +pub enum State { + /// Fetching (`bytes` of `total` so far); the file is checked as the last step. + Running { + bytes: u64, + total: u64, + }, + /// Fetched and verified (its SHA-256 is `sha256`), waiting in the staging folder at `path` to be installed. + Done { + path: PathBuf, + sha256: String, + }, + /// Installed and in use: nothing more to do but tell the user. + Installing, + Installed, + Failed(String), + Cancelled, +} + +struct Shared { + state: Mutex, + cancel: AtomicBool, +} + +impl Shared { + fn new(total: u64) -> Self { + Shared { state: Mutex::new(State::Running { bytes: 0, total }), cancel: AtomicBool::new(false) } + } + + fn set(&self, s: State) { + *self.state.lock().unwrap_or_else(PoisonError::into_inner) = s; + } + + fn get(&self) -> State { + self.state.lock().unwrap_or_else(PoisonError::into_inner).clone() + } +} + +/// The downloads of one session, by model id. +#[derive(Default)] +pub struct Downloads { + jobs: HashMap>, +} + +impl Downloads { + /// Start fetching `spec` into `/.downloads/`. Different models can download side by side; the same + /// one is started once. + pub fn start(&mut self, spec: Download, models_dir: &Path) -> Result<(), String> { + if self.jobs.get(&spec.id).is_some_and(|j| matches!(j.get(), State::Running { .. })) { + return Err("that model is already downloading".into()); + } + let staging = models_dir.join(STAGING); + std::fs::create_dir_all(&staging).map_err(|e| format!("could not create the download folder: {e}"))?; + let shared = Arc::new(Shared::new(spec.size_bytes)); + let worker = shared.clone(); + let id = spec.id.clone(); + std::thread::Builder::new() + .name("denoise-model-download".into()) + .spawn(move || { + let outcome = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| run(&spec, &spec.url, &staging, &worker))); + if outcome.is_err() { + worker.set(State::Failed("the download stopped unexpectedly".into())); + } + }) + .map_err(|e| format!("could not start the download: {e}"))?; + self.jobs.insert(id, shared); + Ok(()) + } + + /// Every download this session has started and not discarded, by model id. + pub fn snapshot(&self) -> Vec<(String, State)> { + let mut all: Vec<_> = self.jobs.iter().map(|(id, j)| (id.clone(), j.get())).filter(|(_, s)| *s != State::Cancelled).collect(); + all.sort_by(|a, b| a.0.cmp(&b.0)); + all + } + + pub fn running(&self) -> bool { + self.jobs.values().any(|j| matches!(j.get(), State::Running { .. })) + } + + /// Stop a running download, or discard a finished one (its staged file is deleted). `true` if there was one. + pub fn discard(&mut self, id: &str) -> bool { + let Some(job) = self.jobs.get(id) else { return false }; + job.cancel.store(true, Ordering::Relaxed); + match job.get() { + // the thread notices, deletes the partial file and marks itself cancelled + State::Running { .. } | State::Installing => {} + State::Done { path, .. } => { + let _ = std::fs::remove_file(path); + self.jobs.remove(id); + } + State::Installed | State::Failed(_) | State::Cancelled => { + self.jobs.remove(id); + } + } + true + } + + /// Downloads that have arrived and been checked, ready to install: (model id, staged file, its SHA-256). + pub fn finished(&self) -> Vec<(String, PathBuf, String)> { + let mut out: Vec<_> = self + .jobs + .iter() + .filter_map(|(id, j)| match j.get() { + State::Done { path, sha256 } => Some((id.clone(), path, sha256)), + _ => None, + }) + .collect(); + out.sort_by(|a, b| a.0.cmp(&b.0)); + out + } + + /// Record how installing a finished download went (`Installed` or `Failed`). + pub fn set_outcome(&mut self, id: &str, state: State) { + match self.jobs.get(id) { + Some(j) => j.set(state), + None => { + let j = Shared::new(0); + j.set(state); + self.jobs.insert(id.to_string(), Arc::new(j)); + } + } + } +} + +impl Drop for Downloads { + /// Quitting while a download runs stops it instead of leaving it behind. + fn drop(&mut self) { + for job in self.jobs.values() { + job.cancel.store(true, Ordering::Relaxed); + } + let started = web_time::Instant::now(); + while self.running() && started.elapsed() < Duration::from_secs(1) { + std::thread::sleep(Duration::from_millis(10)); + } + } +} + +/// What went wrong, in words the user can act on. +#[cfg(not(target_arch = "wasm32"))] +fn explain(e: &lightcraft_fetch::DownloadError) -> String { + use lightcraft_fetch::DownloadError; + match e { + DownloadError::AllMirrorsFailed { errors, .. } => { + let detail: String = errors.join("; ").chars().take(300).collect(); + format!("could not download the model ({detail}): check the internet connection, or get the file from its page and add it yourself") + } + other => other.to_string(), + } +} + +/// Fetch `spec` from its exact URL into `staging`, check it and leave it there. +#[cfg(not(target_arch = "wasm32"))] +fn fetch(spec: &Download, mirror: &str, staging: &Path, shared: &Shared) -> Result<(), lightcraft_fetch::DownloadError> { + let file = lightcraft_fetch::FileSpec { name: &spec.file_name, size: Some(spec.size_bytes), sha256: Some(&spec.sha256), max: spec.size_bytes }; + lightcraft_fetch::download_url(&file, mirror, staging, &lightcraft_fetch::Options::default(), &shared.cancel, &mut |p| { + shared.set(State::Running { bytes: p.done.min(spec.size_bytes), total: spec.size_bytes }); + }) +} + +#[cfg(target_arch = "wasm32")] +fn fetch(_: &Download, _: &str, _: &Path, _: &Shared) -> Result<(), String> { + Err("the web build downloads no models".into()) +} + +/// The whole job: reuse a verified earlier copy, else fetch, check and stage. Always ends in a final [`State`]. +fn run(spec: &Download, mirror: &str, staging: &Path, shared: &Shared) { + let target = staging.join(&spec.file_name); + let part = staging.join(format!("{}.part", spec.file_name)); + match fetch(spec, mirror, staging, shared) { + Ok(()) => shared.set(State::Done { path: target, sha256: spec.sha256.to_string() }), + #[cfg(not(target_arch = "wasm32"))] + Err(lightcraft_fetch::DownloadError::Cancelled) => { + let _ = std::fs::remove_file(&part); + shared.set(State::Cancelled); + } + #[cfg(not(target_arch = "wasm32"))] + Err(e) => { + shared.set(State::Failed(explain(&e))); + } + #[cfg(target_arch = "wasm32")] + Err(e) => { + let _ = std::fs::remove_file(&part); + shared.set(State::Failed(e)); + } + } +} + +#[cfg(all(test, not(target_arch = "wasm32")))] +mod tests { + use super::*; + use lightcraft_denoise::hash::sha256_hex; + use std::io::{BufRead, BufReader, Write}; + use std::net::TcpListener; + + fn scratch(name: &str) -> PathBuf { + let d = std::env::temp_dir().join(format!("lc-face-dl-{name}-{}", std::process::id())); + let _ = std::fs::remove_dir_all(&d); + std::fs::create_dir_all(&d).unwrap(); + d + } + + fn spec_for(bytes: &[u8]) -> Download { + Download { + id: "test-model".into(), + url: "https://example.invalid/m/test-model.onnx".into(), + file_name: "test-model.onnx".into(), + size_bytes: bytes.len() as u64, + sha256: sha256_hex(bytes), + } + } + + /// A local server that answers every request with `status` and `body` (the whole file, no ranges). + fn serve(status: &'static str, body: Vec) -> String { + let l = TcpListener::bind("127.0.0.1:0").unwrap(); + let base = format!("http://{}/m", l.local_addr().unwrap()); + std::thread::spawn(move || { + for mut s in l.incoming().flatten() { + let mut r = BufReader::new(s.try_clone().unwrap()); + loop { + let mut line = String::new(); + if r.read_line(&mut line).unwrap_or(0) == 0 || line.trim().is_empty() { + break; + } + } + let _ = s.write_all(format!("HTTP/1.1 {status}\r\nContent-Length: {}\r\nConnection: close\r\n\r\n", body.len()).as_bytes()); + let _ = s.write_all(&body); + } + }); + base + } + + #[test] + fn fetches_checks_and_stages_a_file() { + let dir = scratch("ok"); + let bytes: Vec = (0..200_000u32).map(|i| (i % 251) as u8).collect(); + let (staging, spec) = (dir.join(STAGING), spec_for(&bytes)); + std::fs::create_dir_all(&staging).unwrap(); + let mirror = serve("200 OK", bytes.clone()); + let shared = Shared::new(spec.size_bytes); + run(&spec, &mirror, &staging, &shared); + let State::Done { path, sha256 } = shared.get() else { panic!("{:?}", shared.get()) }; + assert_eq!((std::fs::read(&path).unwrap(), sha256), (bytes, spec.sha256.to_string())); + assert!(!staging.join("test-model.onnx.part").exists()); + // asked again with the copy still staged: no fetch needed (nothing listens there) + let again = Shared::new(spec.size_bytes); + run(&spec, "http://127.0.0.1:1/m", &staging, &again); + assert!(matches!(again.get(), State::Done { .. }), "{:?}", again.get()); + let _ = std::fs::remove_dir_all(&dir); + } + + #[test] + fn a_file_that_does_not_match_is_thrown_away() { + let dir = scratch("bad"); + let staging = dir.join(STAGING); + std::fs::create_dir_all(&staging).unwrap(); + // the right size and the wrong content + let spec = spec_for(&[7u8; 5000]); + let shared = Shared::new(spec.size_bytes); + run(&spec, &serve("200 OK", vec![8u8; 5000]), &staging, &shared); + assert!(matches!(shared.get(), State::Failed(_)), "{:?}", shared.get()); + // nothing is left behind: not the file, not a partial one + assert_eq!(std::fs::read_dir(&staging).unwrap().count(), 0); + let _ = std::fs::remove_dir_all(&dir); + } + + #[test] + fn an_unreachable_or_refusing_server_is_a_message() { + let dir = scratch("down"); + let staging = dir.join(STAGING); + std::fs::create_dir_all(&staging).unwrap(); + let spec = spec_for(b"abc"); + for mirror in ["http://127.0.0.1:1/m".to_string(), serve("404 Not Found", b"not found".to_vec())] { + let shared = Shared::new(3); + run(&spec, &mirror, &staging, &shared); + let State::Failed(why) = shared.get() else { panic!("{:?}", shared.get()) }; + assert!(why.contains("internet connection"), "{why}"); + } + assert_eq!(std::fs::read_dir(&staging).unwrap().count(), 0); + let _ = std::fs::remove_dir_all(&dir); + } + + #[test] + fn a_cancelled_download_leaves_nothing() { + let dir = scratch("cancel"); + let staging = dir.join(STAGING); + std::fs::create_dir_all(&staging).unwrap(); + let spec = spec_for(b"abc"); + let shared = Shared::new(3); + shared.cancel.store(true, Ordering::Relaxed); + run(&spec, &serve("200 OK", b"abc".to_vec()), &staging, &shared); + assert_eq!(shared.get(), State::Cancelled); + assert_eq!(std::fs::read_dir(&staging).unwrap().count(), 0); + let _ = std::fs::remove_dir_all(&dir); + } + + #[test] + fn dropping_the_session_stops_a_running_download() { + let shared = Arc::new(Shared::new(10)); + // a stand-in for the download thread: notices the cancel flag the way the downloader does + let worker = shared.clone(); + let handle = std::thread::spawn(move || { + while !worker.cancel.load(Ordering::Relaxed) { + std::thread::sleep(Duration::from_millis(5)); + } + worker.set(State::Cancelled); + }); + let mut d = Downloads::default(); + d.jobs.insert("m".into(), shared.clone()); + assert!(d.running()); + let started = web_time::Instant::now(); + drop(d); + assert!(started.elapsed() < Duration::from_millis(900), "quitting must not wait for a transfer to finish"); + assert_eq!(shared.get(), State::Cancelled); + handle.join().unwrap(); + } + + #[test] + fn discarding_a_finished_download_deletes_its_file() { + let dir = scratch("discard"); + let file = dir.join("staged.onnx"); + std::fs::write(&file, b"x").unwrap(); + let mut d = Downloads::default(); + let shared = Arc::new(Shared::new(1)); + shared.set(State::Done { path: file.clone(), sha256: String::new() }); + d.jobs.insert("m".into(), shared); + assert_eq!(d.snapshot().len(), 1); + assert!(!d.running()); + assert!(d.discard("m")); + assert!(!file.exists()); + assert!(d.snapshot().is_empty()); + assert!(!d.discard("m"), "nothing left to discard"); + let _ = std::fs::remove_dir_all(&dir); + } +} diff --git a/crates/engine/src/smart.rs b/crates/engine/src/smart.rs index 8deff7bd..a48fcd04 100644 --- a/crates/engine/src/smart.rs +++ b/crates/engine/src/smart.rs @@ -190,7 +190,7 @@ pub fn is_valid(path: &Path) -> bool { /// Load the proxy at `path`. pub fn load(path: &Path) -> Result { let b = std::fs::read(path).map_err(|e| format!("{}: {e}", path.display()))?; - decode(&b).map(|(image, camera_tone)| crate::media::DecodedSource { image: Arc::new(image), info: None, camera_tone }) + decode(&b).map(|(image, camera_tone)| crate::media::DecodedSource { image: Arc::new(image), info: None, camera_tone, denoised: None }) } #[cfg(test)] diff --git a/crates/engine/src/tests_denoise.rs b/crates/engine/src/tests_denoise.rs new file mode 100644 index 00000000..ef87747f --- /dev/null +++ b/crates/engine/src/tests_denoise.rs @@ -0,0 +1,759 @@ +//! AI denoise through the engine: installing a model, making a photo's denoised picture in the background and for an +//! export, the Amount slider mixing it in, and the cache. A tiny stand-in model (it halves every sample) takes the +//! place of a real one, which is enough to see what the machinery does with its answer; how well real models denoise is +//! measured separately (`crates/denoise/tests/real_model.rs`). + +use std::path::{Path, PathBuf}; +use std::sync::Arc; +use std::sync::atomic::{AtomicUsize, Ordering}; + +use lightcraft_denoise::manifest::{DenoiserManifest, Domain, Gain}; +use lightcraft_denoise::run::TileRunner; +use serde_json::{Value, json}; + +use crate::Session; +use crate::denoise::{Loader, Model, PhotoState, RunOn}; +use crate::export::{ExportOptions, export_photo}; +use crate::tests_xmp::{synthetic_dng_of, temp_dir}; + +/// A model that makes every output pixel half of its cell's colour: a picture that is plainly not the plain one. +struct Dim { + runs: Arc, +} + +impl TileRunner for Dim { + fn run(&self, input: &[f32]) -> Result, lightcraft_denoise::Error> { + self.runs.fetch_add(1, Ordering::Relaxed); + let t = ((input.len() / 4) as f64).sqrt() as usize; + let side = 2 * t; + let mut out = vec![0f32; 3 * side * side]; + for y in 0..side { + for x in 0..side { + let i = (y / 2) * t + x / 2; + let (r, g, b) = (input[i], (input[t * t + i] + input[2 * t * t + i]) / 2.0, input[3 * t * t + i]); + out[y * side + x] = r * 0.5; + out[side * side + y * side + x] = g * 0.5; + out[2 * side * side + y * side + x] = b * 0.5; + } + } + Ok(out) + } +} + +impl Model for Dim { + fn runner(&self, _: RunOn, threads: usize) -> (&dyn TileRunner, usize) { + (self, threads) + } + + fn self_test(&self, _: RunOn) -> Result { + Ok(json!({"ok": true})) + } +} + +fn dim_loader(runs: &Arc) -> Loader { + let runs = runs.clone(); + Arc::new(move |_, _| Ok(Arc::new(Dim { runs: runs.clone() }) as Arc)) +} + +fn manifest(id: &str) -> DenoiserManifest { + DenoiserManifest { + id: id.into(), + name: "Stand-in".into(), + version: "1".into(), + licence: Default::default(), + source: None, + sha256: None, + size_bytes: None, + provenance: "made up for a test".into(), + domain: Domain::BayerToRgb, + tile: 64, + overlap: 16, + gain: Gain::None, + } +} + +/// The model's `.onnx` (not a real one: the stand-in loader never reads it) with its manifest beside it. +fn model_files(dir: &Path, id: &str) -> PathBuf { + let src = dir.join(format!("src-{id}")); + std::fs::create_dir_all(&src).unwrap(); + std::fs::write(src.join("denoise-model.json"), serde_json::to_vec(&manifest(id)).unwrap()).unwrap(); + std::fs::write(src.join("model.onnx"), format!("not an onnx file, only a stand-in for {id}")).unwrap(); + src.join("model.onnx") +} + +#[test] +fn model_installation_keeps_the_frame_pump_responsive_and_reuses_the_loaded_model() { + use std::sync::{Mutex, mpsc}; + use std::time::Duration; + let mut x = setup("background-install", false); + let path = model_files(&x.dir, "stand-in"); + let (entered_tx, entered_rx) = mpsc::channel(); + let (release_tx, release_rx) = mpsc::channel(); + let release_rx = Mutex::new(release_rx); + let runs = x.runs.clone(); + let loads = Arc::new(AtomicUsize::new(0)); + let worker_loads = loads.clone(); + x.s.denoise.loader = Arc::new(move |_, _| { + worker_loads.fetch_add(1, Ordering::Relaxed); + entered_tx.send(()).map_err(|e| e.to_string())?; + release_rx.lock().unwrap().recv_timeout(Duration::from_secs(3)).map_err(|e| e.to_string())?; + Ok(Arc::new(Dim { runs: runs.clone() })) + }); + let info = x.s.execute("denoise.models.inspect", &json!({"path": path})).unwrap(); + assert_eq!(info["model"]["id"], "stand-in"); + assert_eq!(loads.load(Ordering::Relaxed), 0, "showing terms never executes the model"); + let r = x.s.execute("denoise.models.install", &json!({"path": path, "acknowledged": true, "background": true})).unwrap(); + assert_eq!(r["started"], "local-install"); + entered_rx.recv_timeout(Duration::from_secs(3)).unwrap(); + // The loader is blocked: the frame pump must still return, reporting installation in progress. + let state = x.s.execute("denoise.models.downloads", &json!({})).unwrap(); + assert_eq!(state["downloads"][0]["state"], "installing"); + assert_eq!(x.s.execute("denoise.pump", &json!({"pace": "pause"})).unwrap()["active"], false); + assert!(x.s.execute("denoise.models.install", &json!({"path": path, "acknowledged": true, "background": true})).is_err()); + release_tx.send(()).unwrap(); + for _ in 0..600 { + let v = x.s.execute("denoise.models.downloads", &json!({})).unwrap(); + if v["downloads"][0]["state"] == "installed" { + break; + } + std::thread::sleep(Duration::from_millis(5)); + } + assert_eq!(x.s.execute("denoise.models.downloads", &json!({})).unwrap()["downloads"][0]["state"], "installed"); + let a = x.s.denoise.active.as_ref().unwrap(); + assert!(a.model.lock().unwrap().is_some(), "the worker's tested model is reused"); + assert_eq!(loads.load(Ordering::Relaxed), 1); + drop(x.s); + let _ = std::fs::remove_dir_all(x.dir); +} + +#[test] +fn failed_model_replacement_preserves_the_installed_model() { + let mut x = setup("replacement", true); + let installed = x.dir.join("denoise-models/stand-in/model.onnx"); + let original = std::fs::read(&installed).unwrap(); + let source = model_files(&x.dir, "stand-in"); + std::fs::write(&source, "a different broken model").unwrap(); + assert!(x.s.execute("denoise.models.install", &json!({"path": source, "acknowledged": true})).is_err()); + assert_eq!(std::fs::read(installed).unwrap(), original); + assert_eq!(x.s.execute("denoise.models.list", &json!({})).unwrap()["model"], "stand-in"); + drop(x.s); + let _ = std::fs::remove_dir_all(x.dir); +} + +/// A Bayer DNG that is a smooth ramp, big enough for a 64-cell tile and a second one. +fn ramp_dng(w: usize, h: usize, cfa: &str) -> Vec { + let data: Vec = (0..w * h).map(|i| 600 + ((i % w) * 30 + (i / w) * 35) as u16).collect(); + synthetic_dng_of(w, h, cfa, data, None, Default::default()) +} + +struct Setup { + dir: PathBuf, + s: Session, + runs: Arc, +} + +/// A session with a library on disk holding one raw photo, and the stand-in model installed and chosen. +fn setup(tag: &str, with_model: bool) -> Setup { + let dir = temp_dir(tag); + let originals = dir.join("originals"); + std::fs::create_dir_all(&originals).unwrap(); + std::fs::write(originals.join("ramp.dng"), ramp_dng(192, 160, "RGGB")).unwrap(); + let runs = Arc::new(AtomicUsize::new(0)); + let mut s = Session::new().with_fs(); + s.open_library(dir.join("library"), false).unwrap(); + s.execute("library.import", &json!({"paths": [originals.join("ramp.dng").to_string_lossy()]})).unwrap(); + let id = s.catalog.photos().next().unwrap().id; + s.execute("library.select", &json!({"ids": [id.0]})).unwrap(); + s.set_denoise_models_dir(Some(dir.join("denoise-models"))); + s.denoise.loader = dim_loader(&runs); + if with_model { + let onnx = model_files(&dir, "stand-in"); + s.execute("denoise.models.install", &json!({"path": onnx.to_string_lossy(), "acknowledged": true})).unwrap(); + } + Setup { dir, s, runs } +} + +fn photo(s: &Session) -> lightcraft_catalog::PhotoId { + s.catalog.photos().next().unwrap().id +} + +/// Pump until `n` photos are ready (the job runs on its own thread). +fn pump_until_ready(s: &mut Session, n: u64) -> Value { + for _ in 0..600 { + let r = s.execute("denoise.pump", &json!({"pace": "full"})).unwrap(); + if r["ready"].as_u64() >= Some(n) { + return r; + } + std::thread::sleep(std::time::Duration::from_millis(50)); + } + panic!("not ready: {}", s.execute("denoise.status", &json!({})).unwrap()); +} + +fn mean(img: &lightcraft_raster::Rgba8) -> f64 { + let sum: u64 = img.data.iter().map(|p| u64::from(p[0]) + u64::from(p[1]) + u64::from(p[2])).sum(); + sum as f64 / (img.data.len() * 3) as f64 +} + +fn rendered_mean(s: &mut Session, id: lightcraft_catalog::PhotoId) -> f64 { + mean(&s.render_now(id, 160, 160).unwrap().image) +} + +fn set_amount(s: &mut Session, amount: f64) { + s.execute("develop.set", &json!({"control": "enhance.denoise", "value": amount})).unwrap(); +} + +#[test] +fn a_model_is_installed_chosen_and_removed_with_the_terms_accepted() { + let mut x = setup("model", false); + let onnx = model_files(&x.dir, "stand-in"); + // the terms come first + let e = x.s.execute("denoise.models.install", &json!({"path": onnx.to_string_lossy()})).unwrap_err().to_string(); + assert!(e.contains("licence has not been accepted"), "{e}"); + let r = x.s.execute("denoise.models.list", &json!({})).unwrap(); + assert_eq!(r["model"], Value::Null); + assert_eq!(r["runtime"], cfg!(feature = "denoise")); + // a file with no manifest is not guessed at + let stray = x.dir.join("stray.onnx"); + std::fs::write(&stray, b"who knows").unwrap(); + let e = x.s.execute("denoise.models.install", &json!({"path": stray.to_string_lossy(), "acknowledged": true})).unwrap_err().to_string(); + assert!(e.contains("does not know this file"), "{e}"); + // installed, it is the model in use + let r = x.s.execute("denoise.models.install", &json!({"path": onnx.to_string_lossy(), "acknowledged": true})).unwrap(); + assert_eq!((r["installed"]["id"].clone(), r["model"].clone()), (json!("stand-in"), json!("stand-in")), "{r}"); + assert!(r["installed"]["accepted"]["sha256"].as_str().is_some_and(|h| h.len() == 64), "the file's hash is recorded: {r}"); + assert!(x.s.denoise.active.is_some()); + assert_eq!(x.s.execute("denoise.status", &json!({})).unwrap()["model"]["id"], "stand-in"); + // it can be checked again, switched off and on, and removed + assert_eq!(x.s.execute("denoise.models.test", &json!({"id": "stand-in"})).unwrap()["ok"], true); + assert_eq!(x.s.execute("denoise.models.select", &json!({"id": null})).unwrap()["model"], Value::Null); + assert!(x.s.denoise.active.is_none()); + assert!(x.s.execute("denoise.models.select", &json!({"id": "nothing"})).is_err()); + x.s.execute("denoise.models.select", &json!({"id": "stand-in"})).unwrap(); + assert!(x.s.denoise.active.is_some()); + let r = x.s.execute("denoise.models.remove", &json!({"id": "stand-in"})).unwrap(); + assert_eq!(r["model"], Value::Null); + assert!(x.s.denoise.active.is_none()); + assert!(!x.dir.join("denoise-models").join("stand-in").exists()); + // ids that are not folder names, and folders that are not models, are never touched + for bad in ["../library", "", "a/b", "settings.json"] { + assert!(x.s.execute("denoise.models.remove", &json!({"id": bad})).is_err(), "{bad}"); + } + assert!(x.dir.join("library").is_dir()); + // a model that fails its self-test is not left installed + x.s.denoise.loader = Arc::new(|_, _| Err("it does not load".into())); + let e = x.s.execute("denoise.models.install", &json!({"path": onnx.to_string_lossy(), "acknowledged": true})).unwrap_err().to_string(); + assert!(e.contains("it does not load"), "{e}"); + assert!(!x.dir.join("denoise-models").join("stand-in").exists()); + let _ = std::fs::remove_dir_all(&x.dir); +} + +#[test] +fn a_manifest_that_does_not_validate_or_name_the_file_is_refused() { + let mut x = setup("manifest", false); + let onnx = model_files(&x.dir, "stand-in"); + let json_path = onnx.with_file_name("denoise-model.json"); + // a tile the model cannot run on + let mut m = manifest("stand-in"); + m.tile = 50; + std::fs::write(&json_path, serde_json::to_vec(&m).unwrap()).unwrap(); + let e = x.s.execute("denoise.models.install", &json!({"path": onnx.to_string_lossy(), "acknowledged": true})).unwrap_err().to_string(); + assert!(e.contains("tile"), "{e}"); + // a hash that is not the file's + let mut m = manifest("stand-in"); + m.sha256 = Some("0".repeat(64)); + std::fs::write(&json_path, serde_json::to_vec(&m).unwrap()).unwrap(); + let e = x.s.execute("denoise.models.install", &json!({"path": onnx.to_string_lossy(), "acknowledged": true})).unwrap_err().to_string(); + assert!(e.contains("SHA-256"), "{e}"); + assert!(!x.dir.join("denoise-models").join("stand-in").join("model.onnx").exists(), "nothing half-installed is left"); + // an id that would leave the folder + let mut m = manifest("../escape"); + m.id = "../escape".into(); + std::fs::write(&json_path, serde_json::to_vec(&m).unwrap()).unwrap(); + assert!(x.s.execute("denoise.models.install", &json!({"path": onnx.to_string_lossy(), "acknowledged": true})).is_err()); + assert!(!x.dir.join("escape").exists()); + // garbage instead of a manifest + std::fs::write(&json_path, b"{ not json").unwrap(); + assert!(x.s.execute("denoise.models.install", &json!({"path": onnx.to_string_lossy(), "acknowledged": true})).is_err()); + let _ = std::fs::remove_dir_all(&x.dir); +} + +#[test] +fn a_photo_is_denoised_in_the_background_and_the_amount_mixes_it_in() { + let mut x = setup("background", true); + let id = photo(&x.s); + // nothing is made until a photo has an amount + let r = x.s.execute("denoise.pump", &json!({"pace": "full"})).unwrap(); + assert_eq!((r["queued"].clone(), r["running"].clone(), r["ready"].clone()), (json!(0), Value::Null, json!(0)), "{r}"); + assert_eq!(x.s.denoise_photo_state(id), PhotoState::Idle); + let plain = rendered_mean(&mut x.s, id); + let key_before = x.s.render_job(id, 160, 160, false, true).unwrap().key; + + set_amount(&mut x.s, 100.0); + // the photo being edited is picked up by itself, and made on another thread + let r = pump_until_ready(&mut x.s, 1); + assert_eq!(r["queued"], 0, "{r}"); + assert_eq!(x.s.denoise_photo_state(id), PhotoState::Ready); + assert!(x.runs.load(Ordering::Relaxed) >= 4, "a 192 × 160 mosaic is four 64-cell tiles: {}", x.runs.load(Ordering::Relaxed)); + let products: Vec<_> = std::fs::read_dir(x.dir.join("library").join("denoise")).unwrap().flatten().collect(); + assert_eq!(products.len(), 1, "one cached picture, no DNG or anything else in the library"); + assert!(products[0].path().extension().is_some_and(|e| e == "lcdn")); + let originals: Vec<_> = std::fs::read_dir(x.dir.join("originals")).unwrap().flatten().collect(); + assert_eq!(originals.len(), 1, "the original is untouched and alone"); + + // the render now comes from the denoised picture: darker at 100, in between at 50, the plain one at 0 + let full = rendered_mean(&mut x.s, id); + assert!(full < plain - 3.0, "denoised {full} vs plain {plain}"); + set_amount(&mut x.s, 50.0); + let half = rendered_mean(&mut x.s, id); + assert!(half < plain - 1.0 && half > full + 1.0, "{full} < {half} < {plain}"); + set_amount(&mut x.s, 0.0); + assert!((rendered_mean(&mut x.s, id) - plain).abs() < 0.01, "amount 0 is the plain picture again"); + // …and a render that uses it has another identity than one that does not + assert_ne!(key_before, { + set_amount(&mut x.s, 100.0); + x.s.render_job(id, 160, 160, false, true).unwrap().key + }); + // switching the Detail section off switches the denoise off with it + set_amount(&mut x.s, 100.0); + x.s.execute("develop.sectionEnabled", &json!({"section": "detail", "enabled": false})).unwrap(); + assert!((rendered_mean(&mut x.s, id) - plain).abs() < 0.01); + assert_eq!(x.s.media.denoise_salt(id, &x.s.catalog.photo(id).unwrap().develop), 0); + let _ = std::fs::remove_dir_all(&x.dir); +} + +#[test] +fn what_was_made_is_found_again_by_a_new_session_and_a_new_model_starts_over() { + let mut x = setup("again", true); + let id = photo(&x.s); + set_amount(&mut x.s, 100.0); + pump_until_ready(&mut x.s, 1); + let made = x.runs.load(Ordering::Relaxed); + x.s.close_library().unwrap(); + drop(x.s); + + // reopened: ready without running the model again + let mut s2 = Session::new().with_fs(); + s2.set_denoise_models_dir(Some(x.dir.join("denoise-models"))); + s2.denoise.loader = dim_loader(&x.runs); + s2.open_library(x.dir.join("library"), false).unwrap(); + s2.denoise_pump(crate::denoise::Pace::Full); + assert_eq!(s2.denoise_photo_state(id), PhotoState::Ready); + assert_eq!(x.runs.load(Ordering::Relaxed), made, "nothing was made again"); + + // another model's pictures are different pictures + let onnx = model_files(&x.dir, "other"); + s2.execute("denoise.models.install", &json!({"path": onnx.to_string_lossy(), "acknowledged": true})).unwrap(); + s2.denoise_pump(crate::denoise::Pace::Pause); + assert!( + matches!(s2.denoise_photo_state(id), PhotoState::Queued { .. }), + "the first model's picture is not this model's: it is to be made: {:?}", + s2.denoise_photo_state(id) + ); + // and choosing the first again finds its picture where it was + s2.execute("denoise.models.select", &json!({"id": "stand-in"})).unwrap(); + s2.denoise_pump(crate::denoise::Pace::Pause); + assert_eq!(s2.denoise_photo_state(id), PhotoState::Ready); + let _ = std::fs::remove_dir_all(&x.dir); +} + +#[test] +fn an_export_makes_its_picture_when_the_queue_has_not_and_never_comes_out_without_it() { + let mut x = setup("export", true); + let id = photo(&x.s); + let o = ExportOptions::from_json(&json!({"format": "png", "width": 160, "height": 160})); + let decode = |bytes: &[u8]| { + let d = lightcraft_codecs::decode(bytes, lightcraft_codecs::DecodeOptions::fit(160, 160)).unwrap(); + let w = d.to_working(); + w.data.iter().map(|p| f64::from(p[0] + p[1] + p[2])).sum::() / (w.data.len() * 3) as f64 + }; + let plain = decode(&export_photo(&mut x.s, id, &o, 1).unwrap().bytes); + assert_eq!(x.runs.load(Ordering::Relaxed), 0, "no amount, no denoise"); + set_amount(&mut x.s, 100.0); + // no pump: the export makes the picture itself + let denoised = decode(&export_photo(&mut x.s, id, &o, 1).unwrap().bytes); + assert!(x.runs.load(Ordering::Relaxed) >= 4); + assert!(denoised < plain * 0.9, "denoised {denoised} vs plain {plain}"); + let products = std::fs::read_dir(x.dir.join("library").join("denoise")).unwrap().flatten().count(); + assert_eq!(products, 1); + // a second export finds it + let runs = x.runs.load(Ordering::Relaxed); + let again = decode(&export_photo(&mut x.s, id, &o, 1).unwrap().bytes); + assert_eq!(x.runs.load(Ordering::Relaxed), runs); + assert!((again - denoised).abs() < 1e-6); + // the pump takes in what the export made + x.s.denoise_pump(crate::denoise::Pace::Pause); + assert_eq!(x.s.denoise_photo_state(id), PhotoState::Ready); + + // no model: the export says so instead of leaving the denoise out + x.s.execute("denoise.models.select", &json!({"id": null})).unwrap(); + let e = export_photo(&mut x.s, id, &o, 1).err().unwrap(); + assert!(e.contains("no denoise model is chosen") && e.contains("ramp.dng"), "{e}"); + set_amount(&mut x.s, 0.0); + assert!(export_photo(&mut x.s, id, &o, 1).is_ok(), "without an amount nothing is asked of the model"); + let _ = std::fs::remove_dir_all(&x.dir); +} + +#[test] +fn a_model_that_fails_fails_the_export_and_is_not_retried_by_the_queue() { + let mut x = setup("fails", true); + let id = photo(&x.s); + x.s.denoise.loader = Arc::new(|_, _| Err("the model went away".into())); + // Simulate loss of the in-memory model; successful installation now retains its tested instance. + *x.s.denoise.active.as_ref().unwrap().model.lock().unwrap() = None; + set_amount(&mut x.s, 80.0); + let o = ExportOptions::from_json(&json!({"format": "png", "width": 100, "height": 100})); + let e = export_photo(&mut x.s, id, &o, 1).err().unwrap(); + assert!(e.contains("ramp.dng") && e.contains("the model went away"), "{e}"); + // the background job fails too, says why once and does not try again every frame + for _ in 0..200 { + x.s.execute("denoise.pump", &json!({"pace": "full"})).unwrap(); + if matches!(x.s.denoise_photo_state(id), PhotoState::Failed { .. }) { + break; + } + std::thread::sleep(std::time::Duration::from_millis(20)); + } + match x.s.denoise_photo_state(id) { + PhotoState::Failed { why, unsupported } => assert!(why.contains("the model went away") && !unsupported, "{why}"), + other => panic!("{other:?}"), + } + let r = x.s.execute("denoise.pump", &json!({"pace": "full"})).unwrap(); + assert_eq!((r["queued"].clone(), r["running"].clone()), (json!(0), Value::Null), "{r}"); + // asking again by hand does retry + x.s.denoise.loader = dim_loader(&x.runs); + let r = x.s.execute("denoise.queue", &json!({"retry": true})).unwrap(); + assert_eq!(r["added"], 1, "{r}"); + pump_until_ready(&mut x.s, 1); + let _ = std::fs::remove_dir_all(&x.dir); +} + +#[test] +fn a_corrupt_cache_with_a_current_header_never_silently_exports_the_plain_photo() { + let mut x = setup("corrupt-export", true); + let id = photo(&x.s); + set_amount(&mut x.s, 100.0); + let options = ExportOptions::from_json(&json!({"format": "png", "width": 160, "height": 160})); + export_photo(&mut x.s, id, &options, 1).unwrap(); + x.s.denoise_pump(crate::denoise::Pace::Pause); + let spec = x.s.media.denoise.spec(id).unwrap().clone(); + let mut bytes = std::fs::read(&spec.product).unwrap(); + for byte in bytes.iter_mut().rev().take(64) { + *byte ^= 0xff; + } + std::fs::write(&spec.product, bytes).unwrap(); + assert!(lightcraft_denoise::product::is_current(&spec.product, &spec.key), "header/length still look current"); + x.s.media.forget(id); + let error = export_photo(&mut x.s, id, &options, 1).err().unwrap(); + assert!(error.contains("AI Denoise picture could not be read"), "{error}"); + let _ = std::fs::remove_dir_all(x.dir); +} + +#[test] +fn only_raw_photos_on_disk_are_denoised_and_nothing_is_queued_without_a_model() { + let dir = temp_dir("eligible"); + let mut demo = Session::with_demo(); + demo.set_denoise_models_dir(Some(dir.join("models"))); + let id = demo.active().unwrap(); + assert_eq!(demo.denoise_photo_state(id), PhotoState::NotApplicable, "a generated photo has no sensor data"); + let e = demo.execute("denoise.queue", &json!({})).unwrap_err().to_string(); + assert!(e.contains("no denoise model is chosen"), "{e}"); + // no models folder at all (the web build): the commands say so, nothing panics + let mut web = Session::with_demo(); + for cmd in ["denoise.models.install", "denoise.models.select", "denoise.models.remove", "denoise.settings"] { + assert!(web.execute(cmd, &json!({"path": "x", "acknowledged": true, "id": "x"})).is_err(), "{cmd}"); + } + assert_eq!(web.execute("denoise.pump", &json!({})).unwrap()["active"], false); + assert_eq!(web.execute("denoise.status", &json!({})).unwrap()["enabled"], false); + // no library open: nowhere to keep pictures, and the queue says so + let x = setup("eligible2", true); + let mut bare = Session::new().with_fs(); + bare.set_denoise_models_dir(Some(x.dir.join("denoise-models"))); + assert!(bare.denoise_products_dir().is_none()); + let e = bare.execute("denoise.queue", &json!({})).unwrap_err().to_string(); + assert!(e.contains("open a library on disk first"), "{e}"); + assert_eq!(bare.execute("denoise.pump", &json!({})).unwrap()["queued"], 0); + let _ = std::fs::remove_dir_all(&dir); + let _ = std::fs::remove_dir_all(&x.dir); +} + +#[test] +fn files_that_cannot_be_denoised_fail_with_a_reason_and_never_a_panic() { + let dir = temp_dir("makefails"); + let spec = |path: &Path| crate::denoise::JobSpec { + path: path.to_string_lossy().into_owned(), + read: None, + product: dir.join("p.lcdn"), + key: "k".into(), + manifest: manifest("stand-in"), + onnx: dir.join("model.onnx"), + model: Default::default(), + loader: dim_loader(&Arc::new(AtomicUsize::new(0))), + parallel: 2, + run_on: RunOn::Auto, + }; + use crate::denoise::{MakeError, make_product}; + // not a file + assert!(matches!(make_product(&spec(&dir.join("missing.dng")), None), Err(MakeError::Failed(_)))); + // not a raw + let junk = dir.join("junk.dng"); + std::fs::write(&junk, b"this is not a raw file at all").unwrap(); + assert!(matches!(make_product(&spec(&junk), None), Err(MakeError::Unsupported(_)))); + // a raw cut short + let whole = ramp_dng(192, 160, "RGGB"); + let cut = dir.join("cut.dng"); + std::fs::write(&cut, &whole[..whole.len() / 3]).unwrap(); + assert!(make_product(&spec(&cut), None).is_err()); + // a model that gives the wrong number of values is an error from the model, not a crash + struct Wrong; + impl TileRunner for Wrong { + fn run(&self, _: &[f32]) -> Result, lightcraft_denoise::Error> { + Ok(vec![0.0; 7]) + } + } + impl Model for Wrong { + fn runner(&self, _: RunOn, threads: usize) -> (&dyn TileRunner, usize) { + (self, threads) + } + fn self_test(&self, _: RunOn) -> Result { + Err("no".into()) + } + } + let good = dir.join("good.dng"); + std::fs::write(&good, &whole).unwrap(); + let mut s = spec(&good); + s.loader = Arc::new(|_, _| Ok(Arc::new(Wrong) as Arc)); + match make_product(&s, None) { + Err(MakeError::Failed(why)) => assert!(why.contains("7 numbers"), "{why}"), + other => panic!("{other:?}"), + } + assert!(!dir.join("p.lcdn").exists(), "a failed job leaves no product"); + // a job told to stop stops before doing anything + let progress = crate::denoise::Progress::default(); + progress.cancel_now(); + assert_eq!(make_product(&spec(&good), Some(&progress)), Err(MakeError::Cancelled)); + // a Bayer layout of the other kinds is denoised too, and a product that exists is not made again + for cfa in ["BGGR", "GRBG", "GBRG"] { + let f = dir.join(format!("{cfa}.dng")); + std::fs::write(&f, ramp_dng(192, 160, cfa)).unwrap(); + let mut s = spec(&f); + s.product = dir.join(format!("{cfa}.lcdn")); + assert_eq!(make_product(&s, None), Ok(true), "{cfa}"); + assert_eq!(make_product(&s, None), Ok(false), "{cfa} is already there"); + } + let _ = std::fs::remove_dir_all(&dir); +} + +#[test] +fn two_threads_making_the_same_picture_make_it_once() { + let dir = temp_dir("twice"); + let f = dir.join("a.dng"); + std::fs::write(&f, ramp_dng(192, 160, "RGGB")).unwrap(); + let runs = Arc::new(AtomicUsize::new(0)); + let spec = crate::denoise::JobSpec { + path: f.to_string_lossy().into_owned(), + read: None, + product: dir.join("p.lcdn"), + key: "k".into(), + manifest: manifest("stand-in"), + onnx: dir.join("model.onnx"), + model: Default::default(), + loader: dim_loader(&runs), + parallel: 2, + run_on: RunOn::Auto, + }; + let made: Vec = std::thread::scope(|sc| { + let hs: Vec<_> = (0..4).map(|_| sc.spawn(|| crate::denoise::make_product(&spec, None).unwrap())).collect(); + hs.into_iter().map(|h| h.join().unwrap()).collect() + }); + assert_eq!(made.iter().filter(|m| **m).count(), 1, "{made:?}"); + assert_eq!(runs.load(Ordering::Relaxed), 4, "one photo's four tiles, once"); + assert!(lightcraft_denoise::product::is_current(&spec.product, "k")); + let _ = std::fs::remove_dir_all(&dir); +} + +#[test] +fn the_cache_is_kept_within_its_limit_and_what_photos_use_goes_last() { + let mut x = setup("evict", true); + let id = photo(&x.s); + set_amount(&mut x.s, 100.0); + pump_until_ready(&mut x.s, 1); + let used = x.dir.join("library").join("denoise"); + // two older pictures nobody uses + for (i, name) in ["aaaa", "bbbb"].iter().enumerate() { + let f = used.join(format!("{name}.lcdn")); + std::fs::write(&f, vec![7u8; 1000]).unwrap(); + let t = std::time::SystemTime::now() - std::time::Duration::from_secs(1000 - i as u64); + std::fs::File::options().write(true).open(&f).unwrap().set_modified(t).unwrap(); + } + let size_of = |p: &Path| std::fs::metadata(p).unwrap().len(); + let mine: PathBuf = std::fs::read_dir(&used).unwrap().flatten().map(|e| e.path()).find(|p| p.file_stem().is_some_and(|s| s.len() == 32)).unwrap(); + // room for the photo's own picture and one stray: the oldest stray goes, then the other + x.s.denoise_evict_to(size_of(&mine) + 1500, None); + assert!(!used.join("aaaa.lcdn").exists() && used.join("bbbb.lcdn").exists() && mine.exists()); + x.s.denoise_evict_to(size_of(&mine) + 10, None); + assert!(!used.join("bbbb.lcdn").exists() && mine.exists()); + assert_eq!(x.s.denoise_photo_state(id), PhotoState::Ready); + // too small for even that: the photo's own picture is the last thing to go, and the photo knows + x.s.denoise_evict_to(10, None); + assert!(!mine.exists()); + assert_ne!(x.s.denoise_photo_state(id), PhotoState::Ready); + // …and it is made again, because the photo still has its amount + pump_until_ready(&mut x.s, 1); + assert!(mine.exists()); + // clearing deletes it all + let r = x.s.execute("denoise.clear", &json!({})).unwrap(); + assert_eq!(r["deleted"], 1, "{r}"); + assert_eq!(std::fs::read_dir(&used).unwrap().flatten().count(), 0); + let _ = std::fs::remove_dir_all(&x.dir); +} + +#[test] +fn settings_are_checked_saved_and_a_paused_pump_starts_nothing() { + let mut x = setup("settings", true); + let r = x.s.execute("denoise.settings", &json!({"auto": false, "cacheGb": 5, "threads": 3})).unwrap(); + assert_eq!((r["auto"].clone(), r["cacheGb"].clone(), r["threads"].clone()), (json!(false), json!(5), json!(3)), "{r}"); + for bad in [ + json!({"cacheGb": 0}), + json!({"cacheGb": "big"}), + json!({"threads": 0}), + json!({"threads": 1000}), + json!({"auto": "yes"}), + json!({"runOn": "fastest"}), + json!({"runOn": true}), + ] { + assert!(x.s.execute("denoise.settings", &bad).is_err(), "{bad}"); + } + assert_eq!(x.s.execute("denoise.models.list", &json!({})).unwrap()["cacheGb"], 5, "a refused change changes nothing"); + // the faster of the card and the processor unless the user says otherwise, and the interface can read the choice back + assert_eq!(x.s.execute("denoise.models.list", &json!({})).unwrap()["runOn"], "auto"); + assert_eq!(x.s.execute("denoise.settings", &json!({"runOn": "cpu"})).unwrap()["runOn"], "cpu"); + assert_eq!(x.s.execute("denoise.models.list", &json!({})).unwrap()["runOn"], "cpu"); + let status = x.s.execute("denoise.status", &json!({})).unwrap(); + assert_eq!((status["runOn"].clone(), status["device"]["kind"].clone()), (json!("cpu"), json!("none")), "no model is loaded yet: {status}"); + // the setting before `runOn` still reads as what it meant + let models = x.s.denoise.models_dir.clone().unwrap(); + let mut st = crate::denoise::read_settings(&models); + (st.run_on, st.gpu) = (None, Some(false)); + crate::denoise::write_settings(&models, &st).unwrap(); + assert_eq!(x.s.execute("denoise.models.list", &json!({})).unwrap()["runOn"], "cpu"); + // choosing where it runs lets the card be tried again after a set-up that closed LightCraft + let onnx = crate::denoise::installed_models(&models).first().unwrap().onnx.clone(); + let marker = onnx.with_file_name(crate::denoise::GPU_SETUP_MARKER); + std::fs::write(&marker, "setting up").unwrap(); + assert_eq!(x.s.execute("denoise.settings", &json!({"runOn": "auto"})).unwrap()["runOn"], "auto"); + assert!(!marker.exists(), "the crash is forgotten"); + assert_eq!(crate::denoise::read_settings(&models).gpu, None, "the old setting is not kept beside the new one"); + // with auto off nothing is started by itself, and the photo is made when asked + let id = photo(&x.s); + set_amount(&mut x.s, 100.0); + for _ in 0..3 { + x.s.execute("denoise.pump", &json!({"pace": "full"})).unwrap(); + } + assert_eq!(x.s.denoise_photo_state(id), PhotoState::Idle); + x.s.execute("denoise.settings", &json!({"auto": true})).unwrap(); + // paused: it is queued but nothing starts + for _ in 0..3 { + let r = x.s.execute("denoise.pump", &json!({"pace": "pause"})).unwrap(); + assert_eq!(r["running"], Value::Null); + } + assert!(matches!(x.s.denoise_photo_state(id), PhotoState::Queued { .. }), "{:?}", x.s.denoise_photo_state(id)); + // …and cancelling forgets it + assert_eq!(x.s.execute("denoise.cancel", &json!({})).unwrap()["cancelled"], 1); + assert_eq!(x.s.denoise_photo_state(id), PhotoState::Idle); + // when the pace allows, it is made + x.s.execute("denoise.queue", &json!({"scope": "withAmount"})).unwrap(); + pump_until_ready(&mut x.s, 1); + let _ = std::fs::remove_dir_all(&x.dir); +} + +#[test] +fn a_photo_whose_file_changed_has_no_picture_until_it_is_made_again() { + let mut x = setup("changed", true); + let id = photo(&x.s); + set_amount(&mut x.s, 100.0); + pump_until_ready(&mut x.s, 1); + // the photo's content is something else now (a replaced file): its old picture is not its picture + let p = x.s.catalog.photo(id).unwrap().clone(); + x.s.commit( + "test", + lightcraft_catalog::Op::SetContent { + id, + width: p.width, + height: p.height, + file_size: p.file_size + 1, + content_hash: Some("ffffffffffffffffffffffffffffffff".into()), + preview_only: None, + }, + ) + .unwrap(); + x.s.denoise_pump(crate::denoise::Pace::Pause); + assert_ne!(x.s.denoise_photo_state(id), PhotoState::Ready); + // the old picture is not used by a render either + assert_eq!(x.s.media.denoise_salt(id, &x.s.catalog.photo(id).unwrap().develop), 0); + let _ = std::fs::remove_dir_all(&x.dir); +} + +#[test] +fn a_photo_waits_for_its_turn_unless_pictures_are_not_made_on_their_own_or_have_nowhere_to_go() { + let mut x = setup("waits", true); + let id = photo(&x.s); + set_amount(&mut x.s, 80.0); + // automatic off: the open photo is left alone until it is asked for + x.s.execute("denoise.settings", &json!({"auto": false})).unwrap(); + assert!(!x.s.denoise_auto()); + x.s.execute("denoise.pump", &json!({"pace": "full"})).unwrap(); + assert_eq!(x.s.denoise_photo_state(id), PhotoState::Idle); + assert!(!x.s.denoise_busy()); + x.s.execute("denoise.queue", &json!({"ids": [id.0]})).unwrap(); + assert!(x.s.denoise_busy(), "queued with a model: the headless runs wait for it"); + pump_until_ready(&mut x.s, 1); + assert_eq!(x.s.denoise_photo_state(id), PhotoState::Ready); + assert!(!x.s.denoise_busy()); + // a session whose library is not on disk has nowhere to keep it: the photo says so instead of waiting for ever + x.s.denoise.settings.auto = Some(true); + if let Some(l) = x.s.library.as_mut() { + l.on_disk = false; + } + match x.s.denoise_photo_state(id) { + PhotoState::Failed { why, unsupported } => assert!(unsupported && why.contains("library folder"), "{why}"), + other => panic!("{other:?}"), + } + let _ = std::fs::remove_dir_all(&x.dir); +} + +#[test] +fn the_denoise_switch_validates_arguments_and_accepts_explicit_targets_without_selection() { + let mut x = setup("toggle", false); + let id = photo(&x.s); + for params in [json!({"id": "bad"}), json!({"id": u64::MAX}), json!({"id": id.0, "enabled": "yes"})] { + assert!(x.s.execute("denoise.toggle", ¶ms).is_err()); + } + assert!(x.s.execute("denoise.toggle", &json!({"id": id.0, "enabled": true})).is_err()); + assert_eq!(x.s.develop_of(id).unwrap().enhance.denoise, 0.0); + let model = model_files(&x.dir, "toggle-model"); + x.s.execute("denoise.models.install", &json!({"path": model.to_string_lossy(), "acknowledged": true})).unwrap(); + x.s.selection = Default::default(); + let undo = x.s.undo.len(); + let r = x.s.execute("denoise.toggle", &json!({"id": id.0, "enabled": true})).unwrap(); + assert_eq!(r["amount"], 50.0); + assert_eq!(x.s.undo.len(), undo + 1); + x.s.execute("denoise.toggle", &json!({"id": id.0, "enabled": false})).unwrap(); + assert_eq!(x.s.develop_of(id).unwrap().denoise_amount(), 0.0); + // the switch is its own setting: flipping it keeps the Amount (so it compares with and without), and an Amount of 0 + // does not switch it off + x.s.execute("library.select", &json!({"ids": [id.0]})).unwrap(); + x.s.execute("denoise.toggle", &json!({"id": id.0, "enabled": true})).unwrap(); + x.s.execute("develop.set", &json!({"control": "enhance.denoise", "value": 80.0})).unwrap(); + let off = x.s.execute("denoise.toggle", &json!({"id": id.0, "enabled": false})).unwrap(); + assert_eq!(off["amount"], 80.0); + let d = x.s.develop_of(id).unwrap(); + assert!(!d.enhance.denoise_enabled() && d.denoise_amount() == 0.0); + assert_eq!(x.s.execute("denoise.toggle", &json!({"id": id.0, "enabled": true})).unwrap()["amount"], 80.0); + assert!((x.s.develop_of(id).unwrap().denoise_amount() - 0.8).abs() < 1e-6); + x.s.execute("develop.set", &json!({"control": "enhance.denoise", "value": 0.0})).unwrap(); + assert!(x.s.develop_of(id).unwrap().enhance.denoise_enabled(), "sliding to 0 leaves it on"); + assert_eq!(x.s.execute("denoise.toggle", &json!({"id": id.0, "enabled": true})).unwrap()["amount"], 0.0); + // with no `enabled`, the toggle flips what the switch says + assert_eq!(x.s.execute("denoise.toggle", &json!({"id": id.0})).unwrap()["enabled"], false); + drop(x.s); + let _ = std::fs::remove_dir_all(x.dir); +} diff --git a/crates/engine/src/tests_xmp.rs b/crates/engine/src/tests_xmp.rs index fede1fec..ab801e23 100644 --- a/crates/engine/src/tests_xmp.rs +++ b/crates/engine/src/tests_xmp.rs @@ -335,10 +335,15 @@ fn synthetic_dng(xmp: &str) -> Vec { /// A tiny Bayer DNG written by our own DNG writer (optional embedded XMP, camera metadata). pub(crate) fn synthetic_dng_with(xmp: Option<&str>, metadata: lightcraft_meta::Metadata) -> Vec { - use lightcraft_raw::*; let (w, h) = (32usize, 24usize); - let cfa = Cfa::bayer("RGGB").unwrap(); let data: Vec = (0..w * h).map(|i| 256 + ((i % w) * 300 + (i / w) * 200) as u16).collect(); + synthetic_dng_of(w, h, "RGGB", data, xmp, metadata) +} + +/// A synthetic Bayer DNG for denoise and cache tests. +pub(crate) fn synthetic_dng_of(w: usize, h: usize, cfa: &str, data: Vec, xmp: Option<&str>, metadata: lightcraft_meta::Metadata) -> Vec { + use lightcraft_raw::*; + let cfa = Cfa::bayer(cfa).unwrap(); let raw = RawImage { format: RawFormat::Dng, width: w, diff --git a/crates/engine/tests/denoise_cpu.rs b/crates/engine/tests/denoise_cpu.rs new file mode 100644 index 00000000..d01b7c5a --- /dev/null +++ b/crates/engine/tests/denoise_cpu.rs @@ -0,0 +1,101 @@ +//! Opt-in same-machine denoise model-stage benchmark on one public Bayer raw. +//! LC_DENOISE_MODEL= LC_DENOISE_RAW= cargo test --release +//! -p lightcraft-engine --features denoise --test denoise_cpu -- --ignored --nocapture +//! LC_DENOISE_ONLY selects a file (default arw-sony-a7m3-compressed.arw). +//! The whole model stage includes packing, scale/clip protection, blending and normalization; +//! file decode, model loading and GPU setup are outside the timer. It uses the application's tile pool convention. +#![cfg(feature = "denoise")] + +use lightcraft_denoise::{ + TileRunner, + bayer::Layout, + run::{Control, Params, denoise_bayer}, + runtime::{CpuRunner, check_tile}, +}; +use lightcraft_raster::Rgb32f; +use std::{path::Path, time::Instant}; + +fn agree(a: &[f32], b: &[f32]) -> f32 { + assert_eq!(a.len(), b.len()); + assert!(a.iter().chain(b).all(|v| v.is_finite())); + let scale = b.iter().fold(1e-6f32, |m, v| m.max(v.abs())); + let error = a.iter().zip(b).fold(0f32, |m, (a, b)| m.max((a - b).abs())) / scale; + assert!(error <= 1e-3, "maximum relative error {error:e}"); + error +} +fn tile_time(name: &str, r: &dyn TileRunner, x: &[f32]) { + r.run(x).unwrap(); + let mut times: Vec<_> = (0..5) + .map(|_| { + let t = Instant::now(); + r.run(x).unwrap(); + t.elapsed().as_secs_f64() * 1000.0 + }) + .collect(); + times.sort_by(f64::total_cmp); + println!("{name}, one 512-cell tile, one thread: {:.1} ms median ({:.1}..{:.1})", times[2], times[0], times[4]); +} +fn photo_time(name: &str, r: &dyn TileRunner, raw: &lightcraft_raw::Normalized, layout: Layout, p: &Params) -> Rgb32f { + let pool = rayon::ThreadPoolBuilder::new().num_threads(p.parallel).build().unwrap(); + let run = || pool.install(|| denoise_bayer(&raw.data, raw.width, raw.height, layout, r, p, &Control::default())).unwrap(); + drop(run()); // warm the workers and their workspace cache + let mut times = Vec::new(); + let mut answer = Rgb32f { width: 0, height: 0, data: Vec::new() }; + for i in 0..3 { + let t = Instant::now(); + answer = run(); + let secs = t.elapsed().as_secs_f64(); + times.push(secs); + println!("{name}, whole model stage, {} workers, run {i}: {secs:.3} s", p.parallel); + } + times.sort_by(f64::total_cmp); + println!("{name}, whole model stage median {:.3} s", times[1]); + answer +} +#[test] +#[ignore = "needs LC_DENOISE_MODEL and LC_DENOISE_RAW; release timing and real-model accuracy"] +fn denoise_stage_compares_cpu_and_gpu() { + let model = std::env::var_os("LC_DENOISE_MODEL").expect("set LC_DENOISE_MODEL"); + let folder = std::env::var_os("LC_DENOISE_RAW").expect("set LC_DENOISE_RAW to public CC0 corpus/raw"); + let name = std::env::var("LC_DENOISE_ONLY").unwrap_or_else(|_| "arw-sony-a7m3-compressed.arw".into()); + let bytes = std::fs::read(Path::new(&folder).join(&name)).unwrap(); + let raw = lightcraft_raw::decode(&bytes).unwrap().normalized().unwrap(); + let cfa = raw.cfa.as_ref().unwrap(); + assert!(cfa.is_bayer()); + let layout = Layout::from_cell([cfa.color_at(0, 0), cfa.color_at(1, 0), cfa.color_at(0, 1), cfa.color_at(1, 1)]).unwrap(); + let m = lightcraft_denoise::known::find("rawnind-bayer").unwrap().manifest; + let cpu = CpuRunner::load(Path::new(&model), &m).unwrap(); + let threads = std::env::var("LC_DENOISE_PARALLEL") + .ok() + .and_then(|v| v.parse::().ok()) + .unwrap_or_else(|| std::thread::available_parallelism().map_or(1, |n| n.get()).clamp(1, 16)); + let p = Params { tile: 512, overlap: 64, gain: m.gain, clip: Some(1.0), parallel: threads }; + println!( + "{name}: {}x{}, {:.2} MP, {} tiles, {} CPU workers, {:.1} MB/workspace, {:.1} GMAC/tile", + raw.width, + raw.height, + raw.width as f64 * raw.height as f64 / 1e6, + lightcraft_denoise::run::tile_count(raw.width, raw.height, layout, 512, 64), + threads, + cpu.bytes_per_tile() as f64 / 1e6, + cpu.net().macs(512) as f64 / 1e9 + ); + let x = check_tile(512); + let want = cpu.run(&x).unwrap(); + tile_time("pure Rust", &cpu, &x); + let done = photo_time("pure Rust", &cpu, &raw, layout, &p); + + match lightcraft_gpu::nn::runner(cpu.net(), 512) { + Ok(gpu) => { + println!("GPU: {}", gpu.adapter()); + println!("CPU/check tile vs GPU: {:e}", agree(&want, &gpu.run(&x).unwrap())); + tile_time("GPU", &gpu, &x); + let gp = Params { parallel: 4, ..p }; + let answer = photo_time("GPU", &gpu, &raw, layout, &gp); + let a: Vec<_> = done.data.iter().flatten().copied().collect(); + let b: Vec<_> = answer.data.iter().flatten().copied().collect(); + println!("CPU/whole photo vs GPU: {:e}", agree(&a, &b)); + } + Err(e) => println!("GPU unavailable: {e}"), + } +} diff --git a/crates/engine/tests/denoise_real.rs b/crates/engine/tests/denoise_real.rs new file mode 100644 index 00000000..8d858175 --- /dev/null +++ b/crates/engine/tests/denoise_real.rs @@ -0,0 +1,117 @@ +//! Opt-in end-to-end check of AI denoise with the real RawNIND Bayer model, through the engine: install it, make a real +//! raw photo's picture in the background, render and export it with the Amount slider, and see what a file that is not +//! Bayer says. +//! +//! ```text +//! LC_DENOISE_MODEL= LC_DENOISE_RAW= \ +//! cargo test --release -p lightcraft-engine --features denoise --test denoise_real -- --ignored --nocapture +//! ``` +//! +//! `LC_DENOISE_RAW` is a folder of CC0 raw files (`cargo xtask corpus --download` → `corpus/raw`); every file in it is +//! tried. Optional: `LC_DENOISE_ONLY` (a file name to try alone), `LC_DENOISE_RUN_ON=cpu|gpu|auto` (where the model runs; `cpu` to compare with the card). +#![cfg(feature = "denoise")] + +use std::path::{Path, PathBuf}; +use std::time::{Duration, Instant}; + +use lightcraft_engine::Session; +use serde_json::{Value, json}; + +fn temp(tag: &str) -> PathBuf { + let d = std::env::temp_dir().join(format!("lc-denoise-real-{tag}-{}", std::process::id())); + let _ = std::fs::remove_dir_all(&d); + std::fs::create_dir_all(&d).unwrap(); + d +} + +/// Mean absolute second difference of the luma: the fine detail and the noise of a picture. +fn roughness(img: &lightcraft_raster::Rgba8) -> f64 { + let (w, h) = (img.width, img.height); + let luma = |x: usize, y: usize| { + let p = img.data[y * w + x]; + 0.299 * f64::from(p[0]) + 0.587 * f64::from(p[1]) + 0.114 * f64::from(p[2]) + }; + let (mut sum, mut n) = (0.0, 0usize); + for y in 1..h - 1 { + for x in 1..w - 1 { + sum += (4.0 * luma(x, y) - luma(x - 1, y) - luma(x + 1, y) - luma(x, y - 1) - luma(x, y + 1)).abs(); + n += 1; + } + } + sum / n.max(1) as f64 +} + +fn files(dir: &Path) -> Vec { + let only = std::env::var("LC_DENOISE_ONLY").ok(); + let mut v: Vec = std::fs::read_dir(dir) + .unwrap() + .flatten() + .map(|e| e.path()) + .filter(|p| p.is_file() && only.as_deref().is_none_or(|o| p.file_name().is_some_and(|n| n.to_string_lossy() == o))) + .collect(); + v.sort(); + v +} + +#[test] +#[ignore = "needs the real model and raw files (see the file's header)"] +fn the_real_model_denoises_real_raws_through_the_engine() { + let (Some(model), Some(raws)) = (std::env::var_os("LC_DENOISE_MODEL"), std::env::var_os("LC_DENOISE_RAW")) else { + panic!("set LC_DENOISE_MODEL and LC_DENOISE_RAW"); + }; + let dir = temp("e2e"); + let mut s = Session::new().with_fs(); + s.open_library(dir.join("library"), false).unwrap(); + s.set_denoise_models_dir(Some(dir.join("models"))); + let t = Instant::now(); + let r = s.execute("denoise.models.install", &json!({"path": model.to_string_lossy(), "acknowledged": true})).unwrap(); + println!("installed {} in {:.1} s: self-test {}", r["installed"]["id"], t.elapsed().as_secs_f64(), r["installed"]["accepted"]["selfTest"]); + assert_eq!(r["installed"]["id"], "rawnind-bayer"); + if let Ok(run_on) = std::env::var("LC_DENOISE_RUN_ON") { + s.execute("denoise.settings", &json!({"runOn": run_on})).unwrap(); + } + + for file in files(Path::new(&raws)) { + let name = file.file_name().unwrap().to_string_lossy().into_owned(); + let imported = s.execute("library.import", &json!({"paths": [file.to_string_lossy()]})).unwrap(); + let Some(id) = imported["imported"].as_array().and_then(|a| a.first()).and_then(Value::as_u64) else { + println!("{name}: not imported ({imported})"); + continue; + }; + s.execute("library.select", &json!({"ids": [id]})).unwrap(); + s.execute("develop.set", &json!({"control": "enhance.denoise", "value": 100})).unwrap(); + let photo = lightcraft_engine::catalog::PhotoId(id); + let started = Instant::now(); + let mut state = Value::Null; + while started.elapsed() < Duration::from_secs(600) { + s.execute("denoise.pump", &json!({"pace": "full"})).unwrap(); + state = s.execute("denoise.status", &json!({"id": id})).unwrap()["photo"].clone(); + if matches!(state["state"].as_str(), Some("ready" | "failed")) { + break; + } + std::thread::sleep(Duration::from_millis(250)); + } + println!("{name}: {} in {:.1} s", state, started.elapsed().as_secs_f64()); + if state["state"] == "failed" { + // a file that is not a Bayer raw says why, and its export is the plain one + assert_eq!(state["unsupported"], true, "{name}: {state}"); + continue; + } + assert_eq!(state["state"], "ready", "{name}: {state}"); + let denoised = s.render_now(photo, 1600, 1600).unwrap().image; + s.execute("develop.set", &json!({"control": "enhance.denoise", "value": 0})).unwrap(); + let plain = s.render_now(photo, 1600, 1600).unwrap().image; + let (a, b) = (roughness(&plain), roughness(&denoised)); + println!("{name}: roughness at 1600 px: plain {a:.3}, denoised {b:.3} ({:+.1} %)", 100.0 * (b - a) / a); + assert!(b < a, "{name}: the denoised picture is not smoother"); + // the export is made from the same picture, without asking the queue + s.execute("develop.set", &json!({"control": "enhance.denoise", "value": 100})).unwrap(); + let o = lightcraft_engine::export::ExportOptions::from_json(&json!({"format": "jpeg", "width": 1200, "height": 1200})); + let e = lightcraft_engine::export::export_photo(&mut s, photo, &o, 1).unwrap(); + assert!(e.bytes.len() > 1000); + } + let status = s.execute("denoise.status", &json!({})).unwrap(); + println!("device: {}", status["device"]); + println!("cache: {}", status["cache"]); + let _ = std::fs::remove_dir_all(&dir); +} diff --git a/crates/gpu/Cargo.toml b/crates/gpu/Cargo.toml index 765e4c6d..b19acf54 100644 --- a/crates/gpu/Cargo.toml +++ b/crates/gpu/Cargo.toml @@ -9,6 +9,7 @@ description = "LightCraft GPU develop pipeline: wgpu compute (WGSL) twin of the workspace = true [dependencies] +lightcraft-denoise = { workspace = true, optional = true } lightcraft-geom = { workspace = true } lightcraft-color = { workspace = true } lightcraft-raster = { workspace = true } @@ -23,5 +24,9 @@ wgpu = { workspace = true } pollster = { workspace = true } bytemuck = { workspace = true } +[features] +default = [] +denoise = ["dep:lightcraft-denoise", "lightcraft-denoise/runtime"] + [dev-dependencies] lightcraft-scenes = { workspace = true } diff --git a/crates/gpu/src/lib.rs b/crates/gpu/src/lib.rs index 312d4263..9c234964 100644 --- a/crates/gpu/src/lib.rs +++ b/crates/gpu/src/lib.rs @@ -29,6 +29,8 @@ use lightcraft_raster::Rgb32f; pub mod backend; #[cfg(not(target_arch = "wasm32"))] mod ctx; +#[cfg(all(feature = "denoise", not(target_arch = "wasm32")))] +pub mod nn; #[cfg(not(target_arch = "wasm32"))] mod params; #[cfg(not(target_arch = "wasm32"))] @@ -145,6 +147,16 @@ pub fn enabled() -> bool { ENABLED.load(Ordering::Relaxed) && !BROKEN.load(Ordering::Relaxed) && !env_disabled() } +/// Why GPU compute that has a device of its own (denoise) is switched off by the user's settings, or `None`: the +/// environment switches and the rendering preference, but not a render-device failure (the devices are separate). +#[cfg(all(feature = "denoise", not(target_arch = "wasm32")))] +pub(crate) fn switched_off() -> Option { + if env_disabled() { + return Some(if backend::env_off() { "disabled by LIGHTCRAFT_GPU_BACKEND=off" } else { "disabled by LIGHTCRAFT_GPU=0" }.into()); + } + (!ENABLED.load(Ordering::Relaxed)).then(|| "disabled by the GPU rendering preference (app.gpu)".into()) +} + /// `LIGHTCRAFT_GPU=0` (or `LIGHTCRAFT_GPU_BACKEND=off`): no GPU for the whole process. fn env_disabled() -> bool { static OFF: std::sync::OnceLock = std::sync::OnceLock::new(); diff --git a/crates/gpu/src/nn.rs b/crates/gpu/src/nn.rs new file mode 100644 index 00000000..4231dae8 --- /dev/null +++ b/crates/gpu/src/nn.rs @@ -0,0 +1,1048 @@ +//! Denoise networks on the GPU: a [`NetRunner`] executes a [`Net`] (the plain description of a U-Net's layers) with +//! WGSL compute kernels, one tile at a time, behind the same [`TileRunner`] interface the CPU runner has. +//! +//! The runner has a device of its own, so a denoise job and the interactive renders are separate contexts that the +//! driver time-slices, and a failure of one never takes the other down. It respects the same switches as GPU rendering +//! (`LIGHTCRAFT_GPU`, `LIGHTCRAFT_GPU_BACKEND`, the GPU preference) and the same crash sentinel around device creation. +//! `LIGHTCRAFT_GPU_ADAPTER` picks an adapter by (part of) its name, e.g. to try another card. +//! +//! Every failure is an `Err` (never a panic), and a device that errs or is lost stops being used: callers then run the +//! CPU runner. The kernels (`wgsl/nn_conv.wgsl`, `wgsl/nn_pool.wgsl`) are tested against the plain-loop reference +//! interpreter in `lightcraft_denoise::reference` and, with the real model, against the pure-Rust CPU runner. + +use std::collections::HashMap; +use std::sync::atomic::{AtomicBool, Ordering}; +use std::sync::{Condvar, Mutex, OnceLock}; +use std::time::Duration; + +use bytemuck::{Pod, Zeroable}; +use lightcraft_denoise::net::{Net, Op}; +use lightcraft_denoise::run::{Error, TileRunner}; +use wgpu::util::DeviceExt; + +/// Longest wait for one tile's GPU work before the device is given up on. +const TIMEOUT: Duration = Duration::from_secs(60); +/// The largest tile side (in input cells) the dispatch sizes and 32-bit indexing are checked for. +const MAX_TILE: usize = 1024; +/// The largest single buffer used: indices in the kernels are 32-bit element offsets. +const MAX_BUFFER: u64 = 2 << 30; +/// Tiles that can be on the GPU at once (each has buffers of its own). +const WORKSPACES: usize = 2; + +// --------------------------------------------------------------------------------------------------------------------- +// the device + +struct Dev { + device: wgpu::Device, + queue: wgpu::Queue, + info: wgpu::AdapterInfo, + limits: wgpu::Limits, +} + +static DEV: OnceLock> = OnceLock::new(); +static BROKEN: AtomicBool = AtomicBool::new(false); +static BROKEN_WHY: Mutex> = Mutex::new(None); + +fn mark_broken(why: &str) { + let mut w = BROKEN_WHY.lock().unwrap_or_else(|e| e.into_inner()); + if w.is_none() { + *w = Some(why.to_string()); + } + BROKEN.store(true, Ordering::Relaxed); +} + +/// Why the GPU stopped being used for denoise (a device error), if it did. +pub fn broken_reason() -> Option { + BROKEN.load(Ordering::Relaxed).then(|| BROKEN_WHY.lock().unwrap_or_else(|e| e.into_inner()).clone().unwrap_or_else(|| "device error".into())) +} + +fn dev() -> Result<&'static Dev, String> { + if let Some(why) = broken_reason() { + return Err(format!("stopped after a GPU failure: {why}")); + } + if let Some(why) = crate::switched_off() { + return Err(why); + } + DEV.get_or_init(create_device).as_ref().map_err(|e| e.clone()) +} + +fn create_device() -> Result { + let Some(backends) = crate::backend::compute_backends() else { return Err("disabled by LIGHTCRAFT_GPU_BACKEND=off".into()) }; + crate::backend::with_init_marker(backends, || { + std::panic::catch_unwind(|| make_device(backends)).unwrap_or_else(|_| Err("device creation panicked".into())) + }) +} + +fn make_device(backends: wgpu::Backends) -> Result { + let mut desc = wgpu::InstanceDescriptor::new_without_display_handle(); + desc.backends = backends; + let instance = wgpu::Instance::new(desc); + let wanted = std::env::var("LIGHTCRAFT_GPU_ADAPTER").ok().map(|s| s.trim().to_lowercase()).filter(|s| !s.is_empty()); + let adapter = match &wanted { + Some(w) => { + let all = pollster::block_on(instance.enumerate_adapters(backends)); + let names: Vec = all.iter().map(|a| a.get_info().name).collect(); + all.into_iter() + .find(|a| { + let i = a.get_info(); + i.device_type != wgpu::DeviceType::Cpu && i.name.to_lowercase().contains(w.as_str()) + }) + .ok_or_else(|| format!("no GPU adapter named like `{w}` (found: {})", names.join(", ")))? + } + None => pollster::block_on( + instance.request_adapter(&wgpu::RequestAdapterOptions { power_preference: wgpu::PowerPreference::HighPerformance, ..Default::default() }), + ) + .map_err(|e| format!("no GPU adapter among {backends:?} ({e})"))?, + }; + let info = adapter.get_info(); + if info.device_type == wgpu::DeviceType::Cpu { + return Err(format!("software adapter ({}) skipped: the CPU is faster", info.name)); + } + let limits = adapter.limits(); + if limits.max_storage_buffers_per_shader_stage < 6 + || limits.max_compute_invocations_per_workgroup < 256 + || limits.max_compute_workgroup_size_x < 256 + { + return Err(format!("{} is too limited for the denoise kernels", info.name)); + } + if limits.max_compute_workgroup_storage_size < 16 * 1024 { + return Err(format!("{} has too little workgroup memory for the denoise kernels", info.name)); + } + let (device, queue) = pollster::block_on(adapter.request_device(&wgpu::DeviceDescriptor { + label: Some("lightcraft-denoise"), + required_limits: limits.clone(), + ..Default::default() + })) + .map_err(|e| format!("{}: device creation failed ({e})", info.name))?; + device.on_uncaptured_error(std::sync::Arc::new(|e| { + log::error!("gpu (denoise): {e}"); + mark_broken(&format!("device error: {e}")); + })); + device.set_device_lost_callback(|reason, msg| { + log::error!("gpu (denoise): device lost ({reason:?}): {msg}"); + mark_broken(&format!("device lost ({reason:?}): {msg}")); + }); + log::info!("gpu (denoise): {} ({:?})", info.name, info.backend); + Ok(Dev { device, queue, info, limits }) +} + +/// The adapter denoise would use ("NVIDIA GeForce RTX 4090 Laptop GPU (Dx12)"), or why it cannot use one. Creates the +/// device on first use. +pub fn adapter() -> Result { + dev().map(|d| format!("{} ({:?})", d.info.name, d.info.backend)) +} + +// --------------------------------------------------------------------------------------------------------------------- +// the plan: what to run, in what buffers + +#[repr(C)] +#[derive(Clone, Copy, Pod, Zeroable)] +struct ConvParams { + h: u32, + w: u32, + c0: u32, + c1: u32, + n: u32, + npad: u32, + k: u32, + mode: u32, + cout: u32, + leaky: u32, + alpha: f32, + pad: u32, +} + +#[repr(C)] +#[derive(Clone, Copy, Pod, Zeroable)] +struct PoolParams { + h: u32, + w: u32, + c4: u32, + total: u32, +} + +#[derive(Clone, Copy, PartialEq, Eq, Hash, Debug)] +enum Kernel { + Conv { bm: u32, bn: u32, bk: u32 }, + Pool, +} + +/// Where a step's tensor lives. +#[derive(Clone, Copy, Debug)] +enum Place { + Input, + Slot(usize), + Output, +} + +struct Step { + kernel: Kernel, + groups: [u32; 3], + src: Place, + src2: Option, + dst: Place, + /// Index into the per-layer GPU data (weights, bias), for the convolutions. + layer: Option, + params: Vec, +} + +struct Layout { + steps: Vec, + /// Bytes of each activation buffer. + slots: Vec, + in_floats: usize, + out_floats: usize, +} + +/// Weights arranged for the kernel: `[k][npad]` rows with `k = tap · cin + ci` (columns are output channels, zero past +/// `n`), and the bias padded the same. +struct Layer { + weight: Vec, + bias: Vec, +} + +fn unsupported(why: impl Into) -> Result { + Err(why.into()) +} + +/// The kernel shape for `n` output columns: 64 × 64 blocks when `n` is a multiple of 64, else 128 × 32. +fn blocks(n: usize, cin: usize) -> (u32, u32, u32) { + let bn = if n.is_multiple_of(64) { 64 } else { 32 }; + let bk = if cin.is_multiple_of(16) { + 16 + } else if cin.is_multiple_of(8) { + 8 + } else { + 4 + }; + (4096 / bn, bn, bk) +} + +fn round_up(v: usize, to: usize) -> usize { + v.div_ceil(to) * to +} + +/// Work out the steps, the buffers and the arranged weights for `net` on `tile × tile` input cells. +fn plan(net: &Net, tile: usize, max_binding: u64) -> Result<(Layout, Vec), String> { + if !net.fits_tile(tile) { + return unsupported("the tile does not fit the network"); + } + if tile > MAX_TILE { + return unsupported("the tile is larger than the kernels' dispatches reach"); + } + let side = |t: usize| -> usize { + let level = net.shape(t).map_or(0, |s| s.level); + if level >= 0 { tile >> level } else { tile << (-level) } + }; + let chan = |t: usize| net.shape(t).map_or(0, |s| s.channels); + if !net.in_channels.is_multiple_of(4) { + return unsupported("the input channel count is not a multiple of 4"); + } + let ops = net.ops(); + // the network must end in a 1 × 1 convolution and a depth-to-space (they run as one kernel) + let n_ops = ops.len(); + let ends_right = matches!( + (n_ops.checked_sub(2).and_then(|i| ops.get(i)), ops.last()), + (Some(Op::Conv(c)), Some(Op::DepthToSpace2 { src })) if c.k == 1 && c.src2.is_none() && *src == n_ops - 1 && c.cout == 12 && c.leaky.is_none() + ); + if !ends_right { + return unsupported("the network does not end in a 1 × 1 convolution to 12 channels and depth-to-space"); + } + // the last layer that reads each tensor + let mut last_use = vec![0usize; n_ops + 1]; + for (i, op) in ops.iter().enumerate() { + let reads: Vec = match op { + Op::Conv(c) => std::iter::once(c.src).chain(c.src2).collect(), + Op::ConvT2(c) => vec![c.src], + Op::MaxPool2 { src } | Op::DepthToSpace2 { src } => vec![*src], + }; + for t in reads { + if let Some(l) = last_use.get_mut(t) { + *l = i; + } + } + } + + use Place::{Input, Output, Slot}; + let mut slots: Vec = Vec::new(); + let mut free: Vec = Vec::new(); + let mut place: Vec> = vec![None; n_ops + 1]; + if let Some(p) = place.first_mut() { + *p = Some(Input); + } + let mut steps: Vec = Vec::new(); + let mut layers: Vec = Vec::new(); + /// A buffer of at least `bytes`: the smallest free one that is big enough, else the biggest free one grown, else new. + fn alloc(bytes: u64, slots: &mut Vec, free: &mut Vec) -> usize { + let size = |s: usize| slots.get(s).copied().unwrap_or(0); + let fits = free.iter().enumerate().filter(|&(_, &s)| size(s) >= bytes).min_by_key(|&(_, &s)| size(s)).map(|(i, _)| i); + let pick = fits.or_else(|| free.iter().enumerate().max_by_key(|&(_, &s)| size(s)).map(|(i, _)| i)); + match pick { + Some(i) => { + let s = free.swap_remove(i); + if let Some(b) = slots.get_mut(s) { + *b = (*b).max(bytes); + } + s + } + None => { + slots.push(bytes); + slots.len() - 1 + } + } + } + let get = |place: &[Option], t: usize| place.get(t).copied().flatten().ok_or_else(|| format!("tensor {t} has no buffer")); + + for (i, op) in ops.iter().enumerate() { + let out_t = i + 1; + match op { + Op::Conv(c) => { + if !c.cin.is_multiple_of(4) || !c.cout.is_multiple_of(4) { + return unsupported("a convolution whose channel counts are not multiples of 4"); + } + let s = side(c.src); + let last = i + 2 == n_ops; + let (n, mode) = (c.cout, if last { 2u32 } else { 0u32 }); + let (bm, bn, bk) = blocks(n, c.cin); + let npad = round_up(n, bn as usize); + // arranged weights: row (tap · cin + ci), column co + let taps = c.k * c.k; + let mut weight = vec![0f32; taps * c.cin * npad]; + for co in 0..c.cout { + for ci in 0..c.cin { + for ky in 0..c.k { + for kx in 0..c.k { + let v = c.weight.get(((co * c.cin + ci) * c.k + ky) * c.k + kx).copied().unwrap_or(0.0); + if let Some(w) = weight.get_mut(((ky * c.k + kx) * c.cin + ci) * npad + co) { + *w = v; + } + } + } + } + } + let mut bias = vec![0f32; npad]; + if let Some(b) = bias.get_mut(..c.cout) { + b.copy_from_slice(&c.bias); + } + let m = s * s; + let params = ConvParams { + h: s as u32, + w: s as u32, + c0: chan(c.src) as u32, + c1: c.src2.map_or(0, chan) as u32, + n: n as u32, + npad: npad as u32, + k: c.k as u32, + mode, + cout: c.cout as u32, + leaky: u32::from(c.leaky.is_some()), + alpha: c.leaky.unwrap_or(0.0), + pad: 0, + }; + let src = get(&place, c.src)?; + let src2 = c.src2.map(|t| get(&place, t)).transpose()?; + let dst = if last { + Output + } else { + let slot = alloc((m * c.cout * 4) as u64, &mut slots, &mut free); + Slot(slot) + }; + if let Some(p) = place.get_mut(out_t) { + *p = Some(dst); + } + layers.push(Layer { weight, bias }); + steps.push(Step { + kernel: Kernel::Conv { bm, bn, bk }, + groups: [m.div_ceil(bm as usize) as u32, (npad as u32) / bn, 1], + src, + src2, + dst, + layer: Some(layers.len() - 1), + params: bytemuck::bytes_of(¶ms).to_vec(), + }); + } + Op::ConvT2(c) => { + if !c.cin.is_multiple_of(4) || !c.cout.is_multiple_of(4) { + return unsupported("a transposed convolution whose channel counts are not multiples of 4"); + } + let s = side(c.src); + let n = 4 * c.cout; + let (bm, bn, bk) = blocks(n, c.cin); + let npad = round_up(n, bn as usize); + // arranged weights: row ci, column (ky · 2 + kx) · cout + co + let mut weight = vec![0f32; c.cin * npad]; + for ci in 0..c.cin { + for co in 0..c.cout { + for ky in 0..2 { + for kx in 0..2 { + let v = c.weight.get(((ci * c.cout + co) * 2 + ky) * 2 + kx).copied().unwrap_or(0.0); + if let Some(w) = weight.get_mut(ci * npad + (ky * 2 + kx) * c.cout + co) { + *w = v; + } + } + } + } + } + let mut bias = vec![0f32; npad]; + for q in 0..4 { + for co in 0..c.cout { + if let (Some(b), Some(v)) = (bias.get_mut(q * c.cout + co), c.bias.get(co)) { + *b = *v; + } + } + } + let m = s * s; + let params = ConvParams { + h: s as u32, + w: s as u32, + c0: c.cin as u32, + c1: 0, + n: n as u32, + npad: npad as u32, + k: 1, + mode: 1, + cout: c.cout as u32, + leaky: u32::from(c.leaky.is_some()), + alpha: c.leaky.unwrap_or(0.0), + pad: 0, + }; + let src = get(&place, c.src)?; + let slot = alloc((4 * m * c.cout * 4) as u64, &mut slots, &mut free); + if let Some(p) = place.get_mut(out_t) { + *p = Some(Slot(slot)); + } + layers.push(Layer { weight, bias }); + steps.push(Step { + kernel: Kernel::Conv { bm, bn, bk }, + groups: [m.div_ceil(bm as usize) as u32, (npad as u32) / bn, 1], + src, + src2: None, + dst: Slot(slot), + layer: Some(layers.len() - 1), + params: bytemuck::bytes_of(¶ms).to_vec(), + }); + } + Op::MaxPool2 { src } => { + let (s, c) = (side(*src), chan(*src)); + if !c.is_multiple_of(4) { + return unsupported("a max-pool over a channel count that is not a multiple of 4"); + } + let total = (s / 2) * (s / 2) * (c / 4); + let params = PoolParams { h: s as u32, w: s as u32, c4: (c / 4) as u32, total: total as u32 }; + let from = get(&place, *src)?; + let slot = alloc((total * 16) as u64, &mut slots, &mut free); + if let Some(p) = place.get_mut(out_t) { + *p = Some(Slot(slot)); + } + let groups = total.div_ceil(64); + steps.push(Step { + kernel: Kernel::Pool, + groups: [groups.min(8192) as u32, groups.div_ceil(8192) as u32, 1], + src: from, + src2: None, + dst: Slot(slot), + layer: None, + params: bytemuck::bytes_of(¶ms).to_vec(), + }); + } + // run as part of the 1 × 1 convolution before it + Op::DepthToSpace2 { .. } => {} + } + // buffers whose last reader was this layer are free for the next layer's result + for t in 0..=i { + if last_use.get(t).copied() == Some(i) + && let Some(Some(Slot(s))) = place.get(t) + && !free.contains(s) + { + free.push(*s); + } + } + } + if slots.iter().any(|&b| b > max_binding) { + return unsupported("an activation buffer is larger than this GPU allows"); + } + if layers.iter().any(|l| (l.weight.len() as u64).saturating_mul(4) > max_binding) { + return unsupported("a weight buffer is larger than this GPU allows"); + } + Ok((Layout { steps, slots, in_floats: net.in_channels * tile * tile, out_floats: 3 * (2 * tile) * (2 * tile) }, layers)) +} + +// --------------------------------------------------------------------------------------------------------------------- +// the GPU objects + +struct Pipelines { + conv: HashMap<(u32, u32, u32), wgpu::ComputePipeline>, + conv_layout: wgpu::BindGroupLayout, + pool: wgpu::ComputePipeline, + pool_layout: wgpu::BindGroupLayout, + /// How long each kernel took to build (the driver compiles them), for the log and the tests. + built: Vec<(String, f64)>, +} + +fn entry(binding: u32, ty: wgpu::BufferBindingType) -> wgpu::BindGroupLayoutEntry { + wgpu::BindGroupLayoutEntry { + binding, + visibility: wgpu::ShaderStages::COMPUTE, + ty: wgpu::BindingType::Buffer { ty, has_dynamic_offset: false, min_binding_size: None }, + count: None, + } +} + +fn pipeline(d: &Dev, label: &str, src: &str, layout: &wgpu::BindGroupLayout) -> wgpu::ComputePipeline { + let module = d.device.create_shader_module(wgpu::ShaderModuleDescriptor { label: Some(label), source: wgpu::ShaderSource::Wgsl(src.into()) }); + let pl = d.device.create_pipeline_layout(&wgpu::PipelineLayoutDescriptor { label: None, bind_group_layouts: &[Some(layout)], immediate_size: 0 }); + d.device.create_compute_pipeline(&wgpu::ComputePipelineDescriptor { + label: Some(label), + layout: Some(&pl), + module: &module, + entry_point: Some("main"), + // The kernels write all of their workgroup memory before reading any of it, so the zeroing wgpu adds by default + // is not needed; on DX12 it is written out as one assignment per element, which makes the shader compiler take + // a minute over a 2k-element array. + compilation_options: wgpu::PipelineCompilationOptions { zero_initialize_workgroup_memory: false, ..Default::default() }, + cache: None, + }) +} + +fn conv_source(bm: u32, bn: u32, bk: u32) -> String { + include_str!("wgsl/nn_conv.wgsl") + .replace("{{BM}}", &bm.to_string()) + .replace("{{BN}}", &bn.to_string()) + .replace("{{BK}}", &bk.to_string()) + .replace("{{A_VECS}}", &(bk * bm / 4).to_string()) + .replace("{{B_VECS}}", &(bk * bn / 4).to_string()) +} + +fn conv_bgl(d: &Dev) -> wgpu::BindGroupLayout { + use wgpu::BufferBindingType::{Storage, Uniform}; + d.device.create_bind_group_layout(&wgpu::BindGroupLayoutDescriptor { + label: None, + entries: &[ + entry(0, Uniform), + entry(1, Storage { read_only: true }), + entry(2, Storage { read_only: true }), + entry(3, Storage { read_only: true }), + entry(4, Storage { read_only: true }), + entry(5, Storage { read_only: false }), + ], + }) +} + +fn pool_bgl(d: &Dev) -> wgpu::BindGroupLayout { + use wgpu::BufferBindingType::{Storage, Uniform}; + d.device.create_bind_group_layout(&wgpu::BindGroupLayoutDescriptor { + label: None, + entries: &[entry(0, Uniform), entry(1, Storage { read_only: true }), entry(2, Storage { read_only: false })], + }) +} + +fn build_pipelines(d: &Dev, layout: &Layout) -> Pipelines { + let (conv_layout, pool_layout) = (conv_bgl(d), pool_bgl(d)); + let mut conv = HashMap::new(); + let mut built = Vec::new(); + for s in &layout.steps { + if let Kernel::Conv { bm, bn, bk } = s.kernel { + conv.entry((bm, bn, bk)).or_insert_with(|| { + let started = std::time::Instant::now(); + let p = pipeline(d, "nn_conv", &conv_source(bm, bn, bk), &conv_layout); + built.push((format!("conv {bm}x{bn}x{bk}"), started.elapsed().as_secs_f64() * 1000.0)); + p + }); + } + } + let started = std::time::Instant::now(); + let pool = pipeline(d, "nn_pool", include_str!("wgsl/nn_pool.wgsl"), &pool_layout); + built.push(("pool".into(), started.elapsed().as_secs_f64() * 1000.0)); + Pipelines { conv, conv_layout, pool, pool_layout, built } +} + +/// One tile's buffers and the bound steps. +struct Workspace { + input: wgpu::Buffer, + output: wgpu::Buffer, + staging: wgpu::Buffer, + bound: Vec<(Kernel, [u32; 3], wgpu::BindGroup)>, + _slots: Vec, +} + +struct Shared { + dev: &'static Dev, + layout: Layout, + pipes: Pipelines, + weights: Vec<(wgpu::Buffer, wgpu::Buffer)>, + uniforms: Vec, + dummy: wgpu::Buffer, +} + +fn storage(d: &Dev, label: &str, bytes: u64, extra: wgpu::BufferUsages) -> wgpu::Buffer { + d.device.create_buffer(&wgpu::BufferDescriptor { + label: Some(label), + size: bytes.max(16), + usage: wgpu::BufferUsages::STORAGE | extra, + mapped_at_creation: false, + }) +} + +impl Shared { + fn workspace(&self) -> Workspace { + let d = self.dev; + let slots: Vec = self.layout.slots.iter().map(|&b| storage(d, "activations", b, wgpu::BufferUsages::empty())).collect(); + let input = storage(d, "input", (self.layout.in_floats * 4) as u64, wgpu::BufferUsages::COPY_DST); + let output = storage(d, "output", (self.layout.out_floats * 4) as u64, wgpu::BufferUsages::COPY_SRC); + let staging = d.device.create_buffer(&wgpu::BufferDescriptor { + label: Some("readback"), + size: (self.layout.out_floats * 4) as u64, + usage: wgpu::BufferUsages::MAP_READ | wgpu::BufferUsages::COPY_DST, + mapped_at_creation: false, + }); + let buf = |p: Place| -> &wgpu::Buffer { + match p { + Place::Input => &input, + Place::Output => &output, + Place::Slot(i) => slots.get(i).unwrap_or(&self.dummy), + } + }; + let mut bound = Vec::new(); + for (i, s) in self.layout.steps.iter().enumerate() { + let uniform = self.uniforms.get(i).unwrap_or(&self.dummy); + let bg = match s.kernel { + Kernel::Conv { .. } => { + let (w, b) = s.layer.and_then(|l| self.weights.get(l)).map_or((&self.dummy, &self.dummy), |(w, b)| (w, b)); + d.device.create_bind_group(&wgpu::BindGroupDescriptor { + label: None, + layout: &self.pipes.conv_layout, + entries: &[ + wgpu::BindGroupEntry { binding: 0, resource: uniform.as_entire_binding() }, + wgpu::BindGroupEntry { binding: 1, resource: buf(s.src).as_entire_binding() }, + wgpu::BindGroupEntry { binding: 2, resource: s.src2.map_or(&self.dummy, buf).as_entire_binding() }, + wgpu::BindGroupEntry { binding: 3, resource: w.as_entire_binding() }, + wgpu::BindGroupEntry { binding: 4, resource: b.as_entire_binding() }, + wgpu::BindGroupEntry { binding: 5, resource: buf(s.dst).as_entire_binding() }, + ], + }) + } + Kernel::Pool => d.device.create_bind_group(&wgpu::BindGroupDescriptor { + label: None, + layout: &self.pipes.pool_layout, + entries: &[ + wgpu::BindGroupEntry { binding: 0, resource: uniform.as_entire_binding() }, + wgpu::BindGroupEntry { binding: 1, resource: buf(s.src).as_entire_binding() }, + wgpu::BindGroupEntry { binding: 2, resource: buf(s.dst).as_entire_binding() }, + ], + }), + }; + bound.push((s.kernel, s.groups, bg)); + } + Workspace { input, output, staging, bound, _slots: slots } + } +} + +/// A network on the GPU for one tile size. Cheap to share between threads: each call takes a free workspace. +pub struct NetRunner { + shared: Shared, + tile: usize, + in_channels: usize, + free: Mutex>, + /// Workspaces made so far (they are made when a call finds none free, up to [`WORKSPACES`]). + made: Mutex, + wake: Condvar, + bytes: u64, +} + +/// Put `net` on the GPU for `tile × tile` input cells, or say why it cannot be (no GPU, a layer or a size the kernels +/// do not do, the device refusing the buffers). Creates the device on first use and compiles the kernels. +pub fn runner(net: &Net, tile: usize) -> Result { + let d = dev()?; + let max_binding = d.limits.max_storage_buffer_binding_size.min(d.limits.max_buffer_size).min(MAX_BUFFER); + let (layout, layers) = plan(net, tile, max_binding)?; + let scope = [ + d.device.push_error_scope(wgpu::ErrorFilter::OutOfMemory), + d.device.push_error_scope(wgpu::ErrorFilter::Validation), + d.device.push_error_scope(wgpu::ErrorFilter::Internal), + ]; + let pipes = build_pipelines(d, &layout); + let weights: Vec<(wgpu::Buffer, wgpu::Buffer)> = layers + .iter() + .map(|l| { + let make = |v: &[f32]| { + d.device.create_buffer_init(&wgpu::util::BufferInitDescriptor { + label: Some("weights"), + contents: bytemuck::cast_slice(v), + usage: wgpu::BufferUsages::STORAGE, + }) + }; + (make(&l.weight), make(&l.bias)) + }) + .collect(); + let uniforms: Vec = layout + .steps + .iter() + .map(|s| { + d.device.create_buffer_init(&wgpu::util::BufferInitDescriptor { + label: Some("params"), + contents: &s.params, + usage: wgpu::BufferUsages::UNIFORM, + }) + }) + .collect(); + let dummy = storage(d, "dummy", 16, wgpu::BufferUsages::empty()); + let bytes: u64 = layout.slots.iter().sum::() + ((layout.in_floats + layout.out_floats) * 8) as u64; + let shared = Shared { dev: d, layout, pipes, weights, uniforms, dummy }; + let first = shared.workspace(); + let [oom, validation, internal] = scope; + let errors = [pollster::block_on(internal.pop()), pollster::block_on(validation.pop()), pollster::block_on(oom.pop())]; + if let Some(e) = errors.into_iter().flatten().next() { + return Err(format!("{}: the denoise kernels could not be set up ({e})", d.info.name)); + } + Ok(NetRunner { shared, tile, in_channels: net.in_channels, free: Mutex::new(vec![first]), made: Mutex::new(1), wake: Condvar::new(), bytes }) +} + +impl NetRunner { + /// The adapter this runner uses. + pub fn adapter(&self) -> String { + format!("{} ({:?})", self.shared.dev.info.name, self.shared.dev.info.backend) + } + + /// How long each kernel took to build, e.g. `conv 64x64x16: 812 ms`. + pub fn build_report(&self) -> String { + self.shared.pipes.built.iter().map(|(k, ms)| format!("{k}: {ms:.0} ms")).collect::>().join(", ") + } + + /// Device memory one tile in flight uses (bytes), without the weights. + pub fn bytes_per_tile(&self) -> u64 { + self.bytes + } + + /// A free workspace, a new one when fewer than [`WORKSPACES`] exist, else wait for one to come back. + fn take(&self) -> Result { + let mut free = self.free.lock().unwrap_or_else(|e| e.into_inner()); + loop { + if let Some(w) = free.pop() { + return Ok(w); + } + { + let mut made = self.made.lock().unwrap_or_else(|e| e.into_inner()); + if *made < WORKSPACES { + *made += 1; + drop(made); + drop(free); + let d = self.shared.dev; + let scopes = + [d.device.push_error_scope(wgpu::ErrorFilter::OutOfMemory), d.device.push_error_scope(wgpu::ErrorFilter::Validation)]; + let w = self.shared.workspace(); + let [oom, validation] = scopes; + let (validation, oom) = (pollster::block_on(validation.pop()), pollster::block_on(oom.pop())); + if let Some(e) = oom.or(validation) { + self.forget(); + return Err(Error::Runtime(format!("the GPU could not make room for another tile: {e}"))); + } + return Ok(w); + } + } + free = self.wake.wait(free).unwrap_or_else(|e| e.into_inner()); + } + } + + fn give(&self, w: Workspace) { + self.free.lock().unwrap_or_else(|e| e.into_inner()).push(w); + self.wake.notify_one(); + } + + /// A workspace is gone (dropped after a failure): room for a new one, and a waiting caller may make it. + fn forget(&self) { + let mut made = self.made.lock().unwrap_or_else(|e| e.into_inner()); + *made = made.saturating_sub(1); + drop(made); + self.wake.notify_one(); + } + + fn execute(&self, w: &Workspace, input: &[f32]) -> Result, Error> { + let d = self.shared.dev; + let (t, c) = (self.tile, self.in_channels); + // planar → channels innermost + let mut nhwc = vec![0f32; input.len()]; + for (ch, plane) in input.chunks_exact(t * t).enumerate() { + for (i, v) in plane.iter().enumerate() { + if let Some(o) = nhwc.get_mut(i * c + ch) { + *o = *v; + } + } + } + let mut calls = Vec::with_capacity(w.bound.len()); + for (kernel, groups, bg) in &w.bound { + let pipe = match kernel { + Kernel::Conv { bm, bn, bk } => self.shared.pipes.conv.get(&(*bm, *bn, *bk)), + Kernel::Pool => Some(&self.shared.pipes.pool), + }; + let Some(pipe) = pipe else { return Err(Error::Runtime("a kernel is missing".into())) }; + calls.push((pipe, groups, bg)); + } + let scopes = [ + d.device.push_error_scope(wgpu::ErrorFilter::OutOfMemory), + d.device.push_error_scope(wgpu::ErrorFilter::Validation), + d.device.push_error_scope(wgpu::ErrorFilter::Internal), + ]; + d.queue.write_buffer(&w.input, 0, bytemuck::cast_slice(&nhwc)); + let mut enc = d.device.create_command_encoder(&wgpu::CommandEncoderDescriptor { label: None }); + { + let mut pass = enc.begin_compute_pass(&wgpu::ComputePassDescriptor { label: Some("denoise tile"), timestamp_writes: None }); + for (pipe, groups, bg) in calls { + pass.set_pipeline(pipe); + pass.set_bind_group(0, bg, &[]); + pass.dispatch_workgroups(groups[0], groups[1], groups[2]); + } + } + let bytes = (self.shared.layout.out_floats * 4) as u64; + enc.copy_buffer_to_buffer(&w.output, 0, &w.staging, 0, bytes); + d.queue.submit([enc.finish()]); + let slice = w.staging.slice(..bytes); + let (tx, rx) = std::sync::mpsc::channel(); + slice.map_async(wgpu::MapMode::Read, move |r| { + let _ = tx.send(r); + }); + let waited = d.device.poll(wgpu::PollType::Wait { submission_index: None, timeout: Some(TIMEOUT) }); + let [oom, validation, internal] = scopes; + let errors = [pollster::block_on(internal.pop()), pollster::block_on(validation.pop()), pollster::block_on(oom.pop())]; + let [internal, validation, oom] = errors; + if let Some(e) = oom { + // not a broken device: this tile runs on the CPU, and the next may fit + return Err(Error::Runtime(format!("the GPU ran out of memory: {e}"))); + } + if let Some(e) = internal.or(validation) { + mark_broken(&format!("device error: {e}")); + return Err(Error::Runtime(format!("the GPU reported an error: {e}"))); + } + if let Err(e) = waited { + mark_broken(&format!("the GPU did not finish a tile: {e}")); + return Err(Error::Runtime(format!("the GPU did not finish: {e}"))); + } + match rx.recv_timeout(Duration::from_secs(10)) { + Ok(Ok(())) => {} + Ok(Err(e)) => return Err(Error::Runtime(format!("reading the result back failed: {e}"))), + Err(e) => return Err(Error::Runtime(format!("reading the result back failed: {e}"))), + } + let out = { + let data = slice.get_mapped_range().map_err(|e| Error::Runtime(format!("reading the result back failed: {e:?}")))?; + data.as_chunks::<4>().0.iter().map(|b| f32::from_le_bytes(*b)).collect::>() + }; + w.staging.unmap(); + Ok(out) + } +} + +impl TileRunner for NetRunner { + fn run(&self, input: &[f32]) -> Result, Error> { + if let Some(why) = broken_reason() { + return Err(Error::Runtime(format!("the GPU is not used any more: {why}"))); + } + if input.len() != self.shared.layout.in_floats { + return Err(Error::Input("the tile is not the size the model wants".into())); + } + let w = self.take()?; + let r = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| self.execute(&w, input))); + match r { + Ok(Ok(out)) => { + self.give(w); + Ok(out) + } + Ok(Err(e)) => { + // its readback buffer may be left half-mapped: a fresh workspace is made when one is next needed + drop(w); + self.forget(); + Err(e) + } + Err(_) => { + // a panic inside wgpu: this workspace is not given back, and the GPU is not used again + mark_broken("a denoise tile panicked"); + drop(w); + self.forget(); + Err(Error::Runtime("the GPU runner gave up".into())) + } + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + use lightcraft_denoise::{onnx, reference, synthetic}; + + fn net(tile: u64, ch: u64, depth: usize, seed: u64) -> Net { + let path = std::env::temp_dir().join(format!("lc-nn-{}-{tile}-{ch}-{depth}-{seed}.onnx", std::process::id())); + std::fs::write(&path, synthetic::unet_onnx(tile, ch, depth, seed)).unwrap(); + let n = onnx::read(&path).unwrap(); + let _ = std::fs::remove_file(&path); + n + } + + fn picture(len: usize, seed: u64) -> Vec { + let mut s = seed | 1; + (0..len) + .map(|_| { + s ^= s >> 12; + s ^= s << 25; + s ^= s >> 27; + (s.wrapping_mul(0x2545_f491_4f6c_dd1d) >> 40) as f32 / (1u64 << 24) as f32 + }) + .collect() + } + + /// The runner, or `None` (with a note) on a machine with no usable GPU. + fn gpu(net: &Net, tile: usize) -> Option { + match runner(net, tile) { + Ok(r) => Some(r), + Err(e) => { + eprintln!("skipped: no GPU runner here ({e})"); + None + } + } + } + + fn worst(a: &[f32], b: &[f32]) -> f32 { + assert_eq!(a.len(), b.len()); + let scale = b.iter().fold(1e-3f32, |m, v| m.max(v.abs())); + a.iter().zip(b).fold(0f32, |m, (x, y)| m.max((x - y).abs())) / scale + } + + #[test] + fn the_gpu_gives_what_the_reference_gives() { + // (tile, channels, depth): a tile that is not a multiple of the block size, odd channel counts, a deeper net + for (tile, ch, depth, seed) in + [(1u64, 4u64, 0usize, 1u64), (3, 4, 0, 8), (6, 4, 1, 9), (20, 12, 2, 2), (32, 8, 2, 3), (48, 12, 2, 4), (64, 16, 3, 5)] + { + let n = net(tile, ch, depth, seed); + let Some(g) = gpu(&n, tile as usize) else { return }; + let x = picture(4 * (tile * tile) as usize, seed); + let want = reference::run(&n, tile as usize, &x).unwrap(); + let got = g.run(&x).unwrap(); + let cpu = lightcraft_denoise::cpu::NetRunner::new(&n, tile as usize).unwrap().run(&x).unwrap(); + assert!(worst(&got, &cpu) < 1e-3, "GPU and pure-Rust CPU differ"); + let e = worst(&got, &want); + eprintln!("{}: tile {tile}, {ch} channels, depth {depth}: off by {e:e} of the output's size", g.adapter()); + assert!(e < 2e-4, "tile {tile}, {ch} channels, depth {depth}: the GPU is off by {e:e} of the output's size"); + } + } + + #[test] + fn a_runner_serves_many_tiles_from_many_threads() { + let n = net(32, 8, 2, 7); + let Some(g) = gpu(&n, 32) else { return }; + let inputs: Vec> = (0..12).map(|i| picture(4 * 32 * 32, 100 + i)).collect(); + let wants: Vec> = inputs.iter().map(|x| reference::run(&n, 32, x).unwrap()).collect(); + std::thread::scope(|s| { + for chunk in inputs.chunks(3).zip(wants.chunks(3)) { + let g = &g; + s.spawn(move || { + for (x, want) in chunk.0.iter().zip(chunk.1) { + let e = worst(&g.run(x).unwrap(), want); + assert!(e < 2e-4, "off by {e:e}"); + } + }); + } + }); + } + + /// The real model on the GPU against the pure-Rust CPU runner, with timings (the numbers in docs/denoise.md come from here): + /// `LC_DENOISE_MODEL= cargo test --release -p lightcraft-gpu --lib real_model -- --ignored --nocapture` + #[test] + #[ignore = "needs the real model: see the doc comment"] + fn real_model_on_the_gpu_matches_cpu() { + use lightcraft_denoise::manifest::{DenoiserManifest, Domain, Gain}; + use lightcraft_denoise::runtime::CpuRunner; + use std::time::Instant; + let Some(model) = std::env::var_os("LC_DENOISE_MODEL") else { return }; + let net = onnx::read(std::path::Path::new(&model)).unwrap(); + let tile = 512usize; + let started = Instant::now(); + let g = runner(&net, tile).unwrap(); + println!("{}: set up in {:.2} s, {:.0} MB per tile in flight", g.adapter(), started.elapsed().as_secs_f64(), g.bytes_per_tile() as f64 / 1e6); + println!("kernels: {}", g.build_report()); + let manifest = DenoiserManifest { + id: "rawnind-bayer".into(), + name: "RawNIND Bayer".into(), + version: "1".into(), + licence: Default::default(), + source: None, + sha256: None, + size_bytes: None, + provenance: String::new(), + domain: Domain::BayerToRgb, + tile: 512, + overlap: 64, + gain: Gain::MatchMean { nominal: 1.0e6, max_deviation: 0.05 }, + }; + let cpu = CpuRunner::load(std::path::Path::new(&model), &manifest).unwrap(); + // a gradient with noise, in the model's input range + let noise = picture(4 * tile * tile, 42); + let input: Vec = + (0..4 * tile * tile).map(|i| (0.2 + 0.3 * ((i % tile) as f32 / tile as f32) + 0.1 * (noise[i] - 0.5)).clamp(0.0, 1.0)).collect(); + let t = Instant::now(); + let want = cpu.run(&input).unwrap(); + println!("CPU: {:.0} ms for one tile (one call runs on one core)", t.elapsed().as_secs_f64() * 1000.0); + let t = Instant::now(); + let got = g.run(&input).unwrap(); + println!("gpu, first tile: {:.0} ms", t.elapsed().as_secs_f64() * 1000.0); + let e = worst(&got, &want); + println!("worst difference {e:e} of the output's size"); + assert!(e < 1e-3, "the GPU does not give the CPU answer ({e:e})"); + let mut times: Vec = (0..8) + .map(|_| { + let t = Instant::now(); + g.run(&input).unwrap(); + t.elapsed().as_secs_f64() * 1000.0 + }) + .collect(); + times.sort_by(|a, b| a.total_cmp(b)); + println!("gpu, one tile at a time: median {:.0} ms (best {:.0}, worst {:.0})", times[times.len() / 2], times[0], times[times.len() - 1]); + // two threads, as the denoiser feeds it + let t = Instant::now(); + std::thread::scope(|s| { + for _ in 0..2 { + s.spawn(|| { + for _ in 0..8 { + g.run(&input).unwrap(); + } + }); + } + }); + println!("gpu, 16 tiles from 2 threads: {:.0} ms per tile", t.elapsed().as_secs_f64() * 1000.0 / 16.0); + } + + /// How long the driver takes to build one conv kernel variant: `LC_SHADER= LC_BLOCKS=128,32,16`. + #[test] + #[ignore = "a probe for shader compile times"] + fn time_a_shader_build() { + let Some(path) = std::env::var_os("LC_SHADER") else { return }; + let blocks: Vec = std::env::var("LC_BLOCKS").unwrap_or_else(|_| "128,32,16".into()).split(',').filter_map(|v| v.parse().ok()).collect(); + let (bm, bn, bk) = (blocks[0], blocks[1], blocks[2]); + let src = std::fs::read_to_string(path) + .unwrap() + .replace("{{BM}}", &bm.to_string()) + .replace("{{BN}}", &bn.to_string()) + .replace("{{BK}}", &bk.to_string()) + .replace("{{A_VECS}}", &(bk * bm / 4).to_string()) + .replace("{{B_VECS}}", &(bk * bn / 4).to_string()); + let d = dev().unwrap(); + let layout = conv_bgl(d); + let started = std::time::Instant::now(); + let _ = pipeline(d, "probe", &src, &layout); + println!("{} {bm}x{bn}x{bk}: built in {:.0} ms", d.info.backend.to_str(), started.elapsed().as_secs_f64() * 1000.0); + } + + #[test] + fn a_tile_of_the_wrong_size_is_an_error_not_a_crash() { + let n = net(16, 4, 1, 9); + let Some(g) = gpu(&n, 16) else { return }; + assert!(g.run(&[0.0; 10]).is_err()); + assert!(g.run(&[]).is_err()); + } + + #[test] + fn networks_the_kernels_cannot_run_are_refused() { + let n = net(16, 4, 1, 11); + // a tile the network does not fit + assert!(plan(&n, 15, u64::MAX).is_err()); + // a buffer limit smaller than the activations + assert!(plan(&n, 16, 64).is_err()); + } +} diff --git a/crates/gpu/src/wgsl/nn_conv.wgsl b/crates/gpu/src/wgsl/nn_conv.wgsl new file mode 100644 index 00000000..d991e9e7 --- /dev/null +++ b/crates/gpu/src/wgsl/nn_conv.wgsl @@ -0,0 +1,162 @@ +// Convolution for the denoise networks, as an implicit matrix product: every output pixel is a row, every input +// channel of every kernel tap is a column of K, and the output channels are the columns of N. +// +// out[m][n] = bias[n] + sum_k A[m][k] * W[k][n] A[m][k] = input pixel (m + tap offset), channel ci +// +// Tensors are NHWC, channels innermost, read four at a time (every channel count is a multiple of 4). A workgroup of +// 256 threads makes a BM x BN block of the output, 4 x 4 values per thread, and walks K in steps of BK through +// workgroup memory. The source can be two tensors whose channels follow each other (a skip connection's join, so it +// is never made), and a leaky ReLU is applied on the way out. Specialised by the host: BM, BN, BK (BM * BN = 4096). +// +// Output modes: +// 0 NHWC at the input's size. +// 1 a 2x2 stride-2 transposed convolution: the columns are (dy, dx, co) and each lands in the output pixel +// (2y + dy, 2x + dx) of an output twice the size. +// 2 the last 1x1 convolution followed by depth-to-space (CRD): columns 4c + 2dy + dx land in plane c at +// (2y + dy, 2x + dx) of a planar (CHW) output twice the size. + +struct P { + h: u32, + w: u32, + c0: u32, + c1: u32, + n: u32, + npad: u32, + k: u32, + mode: u32, + cout: u32, + leaky: u32, + alpha: f32, + pad: u32, +} + +@group(0) @binding(0) var p: P; +@group(0) @binding(1) var src0: array>; +@group(0) @binding(2) var src1: array>; +@group(0) @binding(3) var wgt: array>; +@group(0) @binding(4) var bias: array>; +@group(0) @binding(5) var dst: array; + +const BM: u32 = {{BM}}u; +const BN: u32 = {{BN}}u; +const BK: u32 = {{BK}}u; +const TX: u32 = BN / 4u; +const A_ITEMS: u32 = BM * (BK / 4u); +const B_ITEMS: u32 = BK * (BN / 4u); + +// A[k][m] four pixels to a vec4, B[k][n] four channels to a vec4 +var As: array, {{A_VECS}}>; +var Bs: array, {{B_VECS}}>; + +@compute @workgroup_size(256) +fn main(@builtin(workgroup_id) wg: vec3, @builtin(local_invocation_index) tid: u32) { + let m0 = wg.x * BM; + let n0 = wg.y * BN; + let tx = tid % TX; + let ty = tid / TX; + let hw = p.h * p.w; + let cin = p.c0 + p.c1; + let taps = p.k * p.k; + let pad = (p.k - 1u) / 2u; + let chunks = cin / BK; + + var acc0 = vec4(0.0); + var acc1 = vec4(0.0); + var acc2 = vec4(0.0); + var acc3 = vec4(0.0); + + for (var tap = 0u; tap < taps; tap++) { + let ty_off = i32(tap / p.k) - i32(pad); + let tx_off = i32(tap % p.k) - i32(pad); + for (var chunk = 0u; chunk < chunks; chunk++) { + let ci0 = chunk * BK; + for (var item = tid; item < A_ITEMS; item += 256u) { + let ml = item / (BK / 4u); + let q = item % (BK / 4u); + let m = m0 + ml; + var v = vec4(0.0); + if (m < hw) { + let y = i32(m / p.w) + ty_off; + let x = i32(m % p.w) + tx_off; + if (y >= 0 && y < i32(p.h) && x >= 0 && x < i32(p.w)) { + let pix = u32(y) * p.w + u32(x); + let ci = ci0 + q * 4u; + if (ci < p.c0) { + v = src0[pix * (p.c0 / 4u) + ci / 4u]; + } else { + v = src1[pix * (p.c1 / 4u) + (ci - p.c0) / 4u]; + } + } + } + let base = (q * 4u) * (BM / 4u) + ml / 4u; + let lane = ml % 4u; + As[base][lane] = v.x; + As[base + BM / 4u][lane] = v.y; + As[base + 2u * (BM / 4u)][lane] = v.z; + As[base + 3u * (BM / 4u)][lane] = v.w; + } + for (var item = tid; item < B_ITEMS; item += 256u) { + let r = item / (BN / 4u); + let c4 = item % (BN / 4u); + let krow = tap * cin + ci0 + r; + Bs[item] = wgt[krow * (p.npad / 4u) + n0 / 4u + c4]; + } + workgroupBarrier(); + for (var kk = 0u; kk < BK; kk++) { + let a = As[kk * (BM / 4u) + ty]; + let b = Bs[kk * TX + tx]; + acc0 += a.x * b; + acc1 += a.y * b; + acc2 += a.z * b; + acc3 += a.w * b; + } + workgroupBarrier(); + } + } + + let n = n0 + tx * 4u; + if (n >= p.n) { + return; + } + let bv = bias[n / 4u]; + for (var i = 0u; i < 4u; i++) { + let m = m0 + ty * 4u + i; + if (m >= hw) { + continue; + } + var v = bv; + if (i == 0u) { v += acc0; } else if (i == 1u) { v += acc1; } else if (i == 2u) { v += acc2; } else { v += acc3; } + if (p.leaky != 0u) { + v = select(v * p.alpha, v, v >= vec4(0.0)); + } + if (p.mode == 0u) { + let o = m * p.n + n; + dst[o] = v.x; + dst[o + 1u] = v.y; + dst[o + 2u] = v.z; + dst[o + 3u] = v.w; + } else if (p.mode == 1u) { + let q = n / p.cout; + let co = n % p.cout; + let y = m / p.w; + let x = m % p.w; + let opix = (2u * y + q / 2u) * (2u * p.w) + 2u * x + q % 2u; + let o = opix * p.cout + co; + dst[o] = v.x; + dst[o + 1u] = v.y; + dst[o + 2u] = v.z; + dst[o + 3u] = v.w; + } else { + let c = n / 4u; + let y = m / p.w; + let x = m % p.w; + let ow = 2u * p.w; + let plane = c * (2u * p.h) * ow; + let o = plane + (2u * y) * ow + 2u * x; + dst[o] = v.x; + dst[o + 1u] = v.y; + dst[o + ow] = v.z; + dst[o + ow + 1u] = v.w; + } + } +} diff --git a/crates/gpu/src/wgsl/nn_pool.wgsl b/crates/gpu/src/wgsl/nn_pool.wgsl new file mode 100644 index 00000000..6117eb06 --- /dev/null +++ b/crates/gpu/src/wgsl/nn_pool.wgsl @@ -0,0 +1,28 @@ +// 2x2 maximum with stride 2 over an NHWC tensor read and written four channels at a time. + +struct P { + h: u32, + w: u32, + c4: u32, + total: u32, +} + +@group(0) @binding(0) var p: P; +@group(0) @binding(1) var src: array>; +@group(0) @binding(2) var dst: array>; + +@compute @workgroup_size(64) +fn main(@builtin(global_invocation_id) gid: vec3, @builtin(num_workgroups) nwg: vec3) { + let o = gid.x + gid.y * nwg.x * 64u; + if (o >= p.total) { + return; + } + let c = o % p.c4; + let pix = o / p.c4; + let ow = p.w / 2u; + let oy = pix / ow; + let ox = pix % ow; + let i00 = ((2u * oy) * p.w + 2u * ox) * p.c4 + c; + let i10 = i00 + p.w * p.c4; + dst[o] = max(max(src[i00], src[i00 + p.c4]), max(src[i10], src[i10 + p.c4])); +} diff --git a/crates/mcp/src/headless.rs b/crates/mcp/src/headless.rs index 7347d33b..837302d6 100644 --- a/crates/mcp/src/headless.rs +++ b/crates/mcp/src/headless.rs @@ -31,7 +31,7 @@ impl Drop for Headless { impl Default for Headless { fn default() -> Self { - Self::new(Session::new().with_fs().with_default_face_models()) + Self::new(Session::new().with_fs().with_default_denoise_models().with_default_face_models()) } } @@ -42,7 +42,7 @@ impl Headless { /// A headless session with the procedurally generated demo library. pub fn demo() -> Self { - Self::new(Session::with_demo().with_fs().with_default_face_models()) + Self::new(Session::with_demo().with_fs().with_default_denoise_models().with_default_face_models()) } fn photo_or_active(&self, p: &Value) -> Result { diff --git a/crates/raw/src/lib.rs b/crates/raw/src/lib.rs index 6732ad92..d38b2fc6 100644 --- a/crates/raw/src/lib.rs +++ b/crates/raw/src/lib.rs @@ -689,14 +689,27 @@ impl RawImage { /// The result is camera RGB (not white balanced), white level = 1.0, not oriented. pub fn develop(&self, method: Method) -> Result { let n = self.normalized()?; - let mut rgb = demosaic(&n, method); + let rgb = demosaic(&n, method); drop(n); + Ok(self.finish_demosaiced(rgb)) + } + + /// The last steps of [`Self::develop`] for camera RGB made some other way (a denoiser that demosaics too) from + /// [`Self::normalized`]'s mosaic: apply `OpcodeList3`, then the default crop. + pub fn finish_demosaiced(&self, mut rgb: Rgb32f) -> Rgb32f { opcodes::apply_list3(&self.opcodes.list3, &mut rgb); - let c = self.crop.clipped(rgb.width, rgb.height); - if c.width == 0 || c.height == 0 || (c.x == 0 && c.y == 0 && c.width == rgb.width && c.height == rgb.height) { - return Ok(rgb); + let c = self.develop_crop(rgb.width, rgb.height); + if c.x == 0 && c.y == 0 && c.width == rgb.width && c.height == rgb.height { + return rgb; } - Ok(rgb.into_crop(c.x, c.y, c.width, c.height)) + rgb.into_crop(c.x, c.y, c.width, c.height) + } + + /// The part of a `width × height` demosaiced picture (the active area) that [`Self::develop`] keeps: the default + /// crop, or all of it when there is none. + pub fn develop_crop(&self, width: usize, height: usize) -> Rect { + let c = self.crop.clipped(width, height); + if c.width == 0 || c.height == 0 { Rect::new(0, 0, width, height) } else { c } } } @@ -729,6 +742,43 @@ mod tests { } } + #[test] + fn a_picture_made_elsewhere_is_finished_like_develop_does() { + let (w, h) = (12usize, 10usize); + let raw = RawImage { + format: RawFormat::Dng, + width: w, + height: h, + cpp: 1, + data: RawData::U16((0..w * h).map(|i| (100 + (i * 37) % 900) as u16).collect()), + cfa: Cfa::bayer("RGGB"), + bits: 12, + black: BlackLevel::uniform(64.0), + white: vec![1023.0], + active_area: Rect::new(2, 2, 8, 6), + crop: Rect::new(1, 1, 5, 4), + orientation: Orientation::Normal, + color: ColorData::default(), + wb_multipliers: None, + linearized: false, + opcodes: OpcodeLists::default(), + metadata: Metadata::default(), + }; + let direct = raw.develop(Method::Bilinear).unwrap(); + let by_hand = raw.finish_demosaiced(demosaic(&raw.normalized().unwrap(), Method::Bilinear)); + assert_eq!((direct.width, direct.height), (5, 4)); + assert_eq!(direct, by_hand); + // the crop is what develop keeps; no usable crop keeps everything + assert_eq!(raw.develop_crop(8, 6), Rect::new(1, 1, 5, 4)); + let mut whole = raw.clone(); + whole.crop = Rect::default(); + assert_eq!(whole.develop_crop(8, 6), Rect::new(0, 0, 8, 6)); + assert_eq!(whole.develop(Method::Bilinear).unwrap().width, 8); + // a crop that pokes out is clipped to the picture + whole.crop = Rect::new(6, 4, 10, 10); + assert_eq!(whole.develop_crop(8, 6), Rect::new(6, 4, 2, 2)); + } + #[test] fn black_levels() { let b = BlackLevel { repeat_rows: 2, repeat_cols: 2, values: vec![1.0, 2.0, 3.0, 4.0], delta_h: vec![0.5, 0.0], delta_v: vec![0.0, 10.0] }; diff --git a/crates/segment/src/bin/lightcraft-sam3-worker.rs b/crates/segment/src/bin/lightcraft-sam3-worker.rs index 62c18334..f8e52541 100644 --- a/crates/segment/src/bin/lightcraft-sam3-worker.rs +++ b/crates/segment/src/bin/lightcraft-sam3-worker.rs @@ -3,6 +3,7 @@ #![forbid(unsafe_code)] #![deny(clippy::unwrap_used, clippy::expect_used, clippy::panic, clippy::unimplemented, clippy::todo, clippy::unreachable)] +#[cfg(not(target_arch = "wasm32"))] fn main() -> Result<(), Box> { let mut args = std::env::args().skip(1); let dir = args.next().ok_or("usage: lightcraft-sam3-worker MODEL_DIR [127.0.0.1:8793]")?; @@ -11,3 +12,7 @@ fn main() -> Result<(), Box> { lightcraft_segment::remote::serve(address, std::path::Path::new(&dir), device)?; Ok(()) } + +/// The worker serves a local GPU over a socket; there is nothing to serve in a browser. +#[cfg(target_arch = "wasm32")] +fn main() {} diff --git a/crates/ui-egui/Cargo.toml b/crates/ui-egui/Cargo.toml index 96779e31..d67494b7 100644 --- a/crates/ui-egui/Cargo.toml +++ b/crates/ui-egui/Cargo.toml @@ -25,6 +25,7 @@ serde_json = { workspace = true } log = { workspace = true } [dev-dependencies] +lightcraft-denoise = { workspace = true, features = ["runtime"] } lightcraft-codecs = { workspace = true } lightcraft-faces = { workspace = true } diff --git a/crates/ui-egui/locales/de-formats.json b/crates/ui-egui/locales/de-formats.json index af10202f..12672394 100644 --- a/crates/ui-egui/locales/de-formats.json +++ b/crates/ui-egui/locales/de-formats.json @@ -187,5 +187,24 @@ "Moved; {} photo(s) relinked": "Verschoben; {} Fotos neu verknüpft", "{} label (click more labels to show any of them)": "Markierung {} (weitere Markierungen auswählen, um alle passenden Fotos zu zeigen)", ", {} star{}": ", Sterne: {}{:.0}", - ", {} label": ", Markierung: {}" + ", {} label": ", Markierung: {}", + "Downloads from {host}. Once it has arrived and checked out, LightCraft installs it, {after}.": "Downloads from {host}. Once it has arrived and checked out, LightCraft installs it, {after}.", + "{:.1} MB": "{:.1} MB", + "Running on the processor: {}.": "Running on the processor: {}.", + "{} · {}. Kept in the library's “denoise” folder; the oldest ones that no photo uses go first when the limit is reached.": "{} · {}. Kept in the library's “denoise” folder; the oldest ones that no photo uses go first when the limit is reached.", + "Denoising {name}: {done} of {total} tiles{}": "Denoising {name}: {done} of {total} tiles{}", + "{size} GB": "{size} GB", + "{} of {} from {host}": "{} of {} from {host}", + "{} KB": "{} KB", + "{} MB": "{} MB", + "Making the denoised picture… {}%": "Making the denoised picture… {}%", + "{name} · {terms}": "{name} · {terms}", + "{n} {noun}": "{n} {noun}", + "Trained on: {provenance}": "Trained on: {provenance}", + " · {queued} waiting": " · {queued} waiting", + "Shows the model's terms, then downloads {} from {host}, installs it and starts using it. Nothing else is sent.": "Shows the model's terms, then downloads {} from {host}, installs it and starts using it. Nothing else is sent.", + "In line for AI Denoise ({ahead} ahead)…": "In line for AI Denoise ({ahead} ahead)…", + "Running on {}.": "Running on {}.", + "{file} · {size}": "{file} · {size}", + "• {assumption}": "• {assumption}" } diff --git a/crates/ui-egui/locales/de.json b/crates/ui-egui/locales/de.json index 520c73f9..64a485ed 100644 --- a/crates/ui-egui/locales/de.json +++ b/crates/ui-egui/locales/de.json @@ -1798,5 +1798,94 @@ "Install the SAM 3 Model": "SAM-3-Modell installieren", "This build can't download the model yet. To install it by hand, put model.safetensors, vocab.json and merges.txt from Meta's facebook/sam3 in the folder above: Object and Describe work as soon as they are there.": "Diese Version kann das Modell noch nicht herunterladen. Zum Installieren von Hand lege model.safetensors, vocab.json und merges.txt aus Metas facebook/sam3 in den obigen Ordner: Objekt und Beschreiben funktionieren, sobald sie dort liegen.", "Open Log Folder": "Protokollordner öffnen", - "Back to this person": "Back to this person" + "Back to this person": "Back to this person", + "AI Denoise": "AI Denoise", + "AI Denoise Model": "AI Denoise Model", + "AI Denoise is off: press Use on a model.": "AI Denoise is off: press Use on a model.", + "AI Denoise needs a model. Install one in Settings.": "AI Denoise needs a model. Install one in Settings.", + "Accept & Download": "Accept & Download", + "Already installed. Remove it before installing a replacement.": "Already installed. Remove it before installing a replacement.", + "Automatic uses the graphics card where it is faster than the processor: both are timed on this computer the first time a photo is made.": "Automatic uses the graphics card where it is faster than the processor: both are timed on this computer the first time a photo is made.", + "Cache": "Cache", + "Checking and installing the model…": "Checking and installing the model…", + "Checking…": "Checking…", + "Clear cache": "Clear cache", + "Denoise Models": "Denoise Models", + "Denoise Settings": "Denoise Settings", + "Denoise all photos with an Amount": "Denoise all photos with an Amount", + "Downloaded and checked. Installing…": "Downloaded and checked. Installing…", + "Failed its last test": "Failed its last test", + "Graphics card": "Graphics card", + "I have read these terms and accept them for my own use": "I have read these terms and accept them for my own use", + "Inspect Denoise Model": "Inspect Denoise Model", + "Install Denoise Model": "Install Denoise Model", + "Install from file…": "Install from file…", + "Install your own ONNX model with denoise-model.json beside it. This build offers no model downloads.": "Install your own ONNX model with denoise-model.json beside it. This build offers no model downloads.", + "Its denoised picture is not made yet, and pictures are not made on their own (Settings, AI Denoise).": "Its denoised picture is not made yet, and pictures are not made on their own (Settings, AI Denoise).", + "LightCraft does not know this model, so it assumed:": "LightCraft does not know this model, so it assumed:", + "Limit": "Limit", + "Make it now": "Make it now", + "Make the denoised picture of the photos I am looking at": "Make the denoised picture of the photos I am looking at", + "Next up for AI Denoise…": "Next up for AI Denoise…", + "Nothing is fetched until you accept. The model is kept on this computer only; LightCraft never uploads or shares it.": "Nothing is fetched until you accept. The model is kept on this computer only; LightCraft never uploads or shares it.", + "Only photos whose AI Denoise amount is above 0 are made. Exports always make theirs if it is missing.": "Only photos whose AI Denoise amount is above 0 are made. Exports always make theirs if it is missing.", + "Open page": "Open page", + "Open the model's page": "Open the model's page", + "Opens the model's own page in your browser, to read about it or get the file yourself.": "Opens the model's own page in your browser, to read about it or get the file yourself.", + "Processor": "Processor", + "Set up AI Denoise…": "Set up AI Denoise…", + "Speed": "Speed", + "Starting…": "Starting…", + "Tested and works on this computer": "Tested and works on this computer", + "The denoise model is installed and in use: the AI Denoise Amount slider under Detail now works": "The denoise model is installed and in use: the AI Denoise Amount slider under Detail now works", + "The graphics card is set up when the next photo is made.": "The graphics card is set up when the next photo is made.", + "The model is kept on this computer only. LightCraft never uploads or shares it.": "The model is kept on this computer only. LightCraft never uploads or shares it.", + "The processor does the work.": "The processor does the work.", + "There is no model file to install": "There is no model file to install", + "This build cannot run denoise models.": "This build cannot run denoise models.", + "This build has nowhere to keep denoise models, so AI Denoise is not available. (The desktop app has.)": "This build has nowhere to keep denoise models, so AI Denoise is not available. (The desktop app has.)", + "This file cannot be used yet": "This file cannot be used yet", + "Tick the box to accept the model's terms first": "Tick the box to accept the model's terms first", + "Try again": "Try again", + "Try the graphics card again": "Try the graphics card again", + "Turn off": "Turn off", + "Unknown licence": "Unknown licence", + "Use": "Use", + "Waiting to start…": "Waiting to start…", + "Works on raw files from cameras with a standard (Bayer) sensor. Other files keep their normal noise reduction.": "Works on raw files from cameras with a standard (Bayer) sensor. Other files keep their normal noise reduction.", + "cached picture": "cached picture", + "cached pictures": "cached pictures", + "commercial use allowed": "commercial use allowed", + "denoiseModel.accept": "denoiseModel.accept", + "in use": "in use", + "installed": "installed", + "makes AI Denoise available": "makes AI Denoise available", + "non-commercial use only": "non-commercial use only", + "photo has its denoised picture.": "photo has its denoised picture.", + "photo waiting": "photo waiting", + "photos have their denoised picture.": "photos have their denoised picture.", + "photos waiting": "photos waiting", + "starts using it for AI Denoise": "starts using it for AI Denoise", + "terms unclear": "terms unclear", + "the graphics card": "the graphics card", + "the graphics card is not used": "the graphics card is not used", + "Cancel Denoise": "Cancel Denoise", + "Cancel Denoise Model Download": "Cancel Denoise Model Download", + "Choose Denoise Model": "Choose Denoise Model", + "Clear Denoise Cache": "Clear Denoise Cache", + "Denoise Model Downloads": "Denoise Model Downloads", + "Denoise Photos": "Denoise Photos", + "Denoise Pump": "Denoise Pump", + "Denoise Status": "Denoise Status", + "Download Denoise Model": "Download Denoise Model", + "Remove Denoise Model": "Remove Denoise Model", + "Test Denoise Model": "Test Denoise Model", + "AI Denoise is enabled on the requesting photo": "AI Denoise is enabled on the requesting photo", + "After installation, AI Denoise will turn on for the requesting photo.": "After installation, AI Denoise will turn on for the requesting photo.", + "Cancel pending action": "Cancel pending action", + "Install a denoise model to enable AI Denoise on this photo": "Install a denoise model to enable AI Denoise on this photo", + "Pending photo action cancelled": "Pending photo action cancelled", + "Pending photo action cancelled because its library or photo changed": "Pending photo action cancelled because its library or photo changed", + "ONNX model": "ONNX model", + "commercial use subject to GPL conditions": "commercial use subject to GPL conditions" } diff --git a/crates/ui-egui/locales/es-formats.json b/crates/ui-egui/locales/es-formats.json index b67d6720..54418c76 100644 --- a/crates/ui-egui/locales/es-formats.json +++ b/crates/ui-egui/locales/es-formats.json @@ -187,5 +187,24 @@ "Moved; {} photo(s) relinked": "Movida; {} foto(s) vinculada(s) de nuevo", "{} label (click more labels to show any of them)": "Etiqueta {} (hacer clic en más etiquetas para mostrar cualquiera de ellas)", ", {} star{}": ", {} estrella{}", - ", {} label": ", etiqueta {}" + ", {} label": ", etiqueta {}", + "Downloads from {host}. Once it has arrived and checked out, LightCraft installs it, {after}.": "Downloads from {host}. Once it has arrived and checked out, LightCraft installs it, {after}.", + "{:.1} MB": "{:.1} MB", + "Running on the processor: {}.": "Running on the processor: {}.", + "{} · {}. Kept in the library's “denoise” folder; the oldest ones that no photo uses go first when the limit is reached.": "{} · {}. Kept in the library's “denoise” folder; the oldest ones that no photo uses go first when the limit is reached.", + "Denoising {name}: {done} of {total} tiles{}": "Denoising {name}: {done} of {total} tiles{}", + "{size} GB": "{size} GB", + "{} of {} from {host}": "{} of {} from {host}", + "{} KB": "{} KB", + "{} MB": "{} MB", + "Making the denoised picture… {}%": "Making the denoised picture… {}%", + "{name} · {terms}": "{name} · {terms}", + "{n} {noun}": "{n} {noun}", + "Trained on: {provenance}": "Trained on: {provenance}", + " · {queued} waiting": " · {queued} waiting", + "Shows the model's terms, then downloads {} from {host}, installs it and starts using it. Nothing else is sent.": "Shows the model's terms, then downloads {} from {host}, installs it and starts using it. Nothing else is sent.", + "In line for AI Denoise ({ahead} ahead)…": "In line for AI Denoise ({ahead} ahead)…", + "Running on {}.": "Running on {}.", + "{file} · {size}": "{file} · {size}", + "• {assumption}": "• {assumption}" } diff --git a/crates/ui-egui/locales/es.json b/crates/ui-egui/locales/es.json index 1c8b0013..461fa2f6 100644 --- a/crates/ui-egui/locales/es.json +++ b/crates/ui-egui/locales/es.json @@ -1643,5 +1643,94 @@ "Captured until": "Capturadas hasta", "Only": "Solo", "Open Log Folder": "Abrir carpeta de registros", - "Back to this person": "Back to this person" + "Back to this person": "Back to this person", + "AI Denoise": "AI Denoise", + "AI Denoise Model": "AI Denoise Model", + "AI Denoise is off: press Use on a model.": "AI Denoise is off: press Use on a model.", + "AI Denoise needs a model. Install one in Settings.": "AI Denoise needs a model. Install one in Settings.", + "Accept & Download": "Accept & Download", + "Already installed. Remove it before installing a replacement.": "Already installed. Remove it before installing a replacement.", + "Automatic uses the graphics card where it is faster than the processor: both are timed on this computer the first time a photo is made.": "Automatic uses the graphics card where it is faster than the processor: both are timed on this computer the first time a photo is made.", + "Cache": "Cache", + "Checking and installing the model…": "Checking and installing the model…", + "Checking…": "Checking…", + "Clear cache": "Clear cache", + "Denoise Models": "Denoise Models", + "Denoise Settings": "Denoise Settings", + "Denoise all photos with an Amount": "Denoise all photos with an Amount", + "Downloaded and checked. Installing…": "Downloaded and checked. Installing…", + "Failed its last test": "Failed its last test", + "Graphics card": "Graphics card", + "I have read these terms and accept them for my own use": "I have read these terms and accept them for my own use", + "Inspect Denoise Model": "Inspect Denoise Model", + "Install Denoise Model": "Install Denoise Model", + "Install from file…": "Install from file…", + "Install your own ONNX model with denoise-model.json beside it. This build offers no model downloads.": "Install your own ONNX model with denoise-model.json beside it. This build offers no model downloads.", + "Its denoised picture is not made yet, and pictures are not made on their own (Settings, AI Denoise).": "Its denoised picture is not made yet, and pictures are not made on their own (Settings, AI Denoise).", + "LightCraft does not know this model, so it assumed:": "LightCraft does not know this model, so it assumed:", + "Limit": "Limit", + "Make it now": "Make it now", + "Make the denoised picture of the photos I am looking at": "Make the denoised picture of the photos I am looking at", + "Next up for AI Denoise…": "Next up for AI Denoise…", + "Nothing is fetched until you accept. The model is kept on this computer only; LightCraft never uploads or shares it.": "Nothing is fetched until you accept. The model is kept on this computer only; LightCraft never uploads or shares it.", + "Only photos whose AI Denoise amount is above 0 are made. Exports always make theirs if it is missing.": "Only photos whose AI Denoise amount is above 0 are made. Exports always make theirs if it is missing.", + "Open page": "Open page", + "Open the model's page": "Open the model's page", + "Opens the model's own page in your browser, to read about it or get the file yourself.": "Opens the model's own page in your browser, to read about it or get the file yourself.", + "Processor": "Processor", + "Set up AI Denoise…": "Set up AI Denoise…", + "Speed": "Speed", + "Starting…": "Starting…", + "Tested and works on this computer": "Tested and works on this computer", + "The denoise model is installed and in use: the AI Denoise Amount slider under Detail now works": "The denoise model is installed and in use: the AI Denoise Amount slider under Detail now works", + "The graphics card is set up when the next photo is made.": "The graphics card is set up when the next photo is made.", + "The model is kept on this computer only. LightCraft never uploads or shares it.": "The model is kept on this computer only. LightCraft never uploads or shares it.", + "The processor does the work.": "The processor does the work.", + "There is no model file to install": "There is no model file to install", + "This build cannot run denoise models.": "This build cannot run denoise models.", + "This build has nowhere to keep denoise models, so AI Denoise is not available. (The desktop app has.)": "This build has nowhere to keep denoise models, so AI Denoise is not available. (The desktop app has.)", + "This file cannot be used yet": "This file cannot be used yet", + "Tick the box to accept the model's terms first": "Tick the box to accept the model's terms first", + "Try again": "Try again", + "Try the graphics card again": "Try the graphics card again", + "Turn off": "Turn off", + "Unknown licence": "Unknown licence", + "Use": "Use", + "Waiting to start…": "Waiting to start…", + "Works on raw files from cameras with a standard (Bayer) sensor. Other files keep their normal noise reduction.": "Works on raw files from cameras with a standard (Bayer) sensor. Other files keep their normal noise reduction.", + "cached picture": "cached picture", + "cached pictures": "cached pictures", + "commercial use allowed": "commercial use allowed", + "denoiseModel.accept": "denoiseModel.accept", + "in use": "in use", + "installed": "installed", + "makes AI Denoise available": "makes AI Denoise available", + "non-commercial use only": "non-commercial use only", + "photo has its denoised picture.": "photo has its denoised picture.", + "photo waiting": "photo waiting", + "photos have their denoised picture.": "photos have their denoised picture.", + "photos waiting": "photos waiting", + "starts using it for AI Denoise": "starts using it for AI Denoise", + "terms unclear": "terms unclear", + "the graphics card": "the graphics card", + "the graphics card is not used": "the graphics card is not used", + "Cancel Denoise": "Cancel Denoise", + "Cancel Denoise Model Download": "Cancel Denoise Model Download", + "Choose Denoise Model": "Choose Denoise Model", + "Clear Denoise Cache": "Clear Denoise Cache", + "Denoise Model Downloads": "Denoise Model Downloads", + "Denoise Photos": "Denoise Photos", + "Denoise Pump": "Denoise Pump", + "Denoise Status": "Denoise Status", + "Download Denoise Model": "Download Denoise Model", + "Remove Denoise Model": "Remove Denoise Model", + "Test Denoise Model": "Test Denoise Model", + "AI Denoise is enabled on the requesting photo": "AI Denoise is enabled on the requesting photo", + "After installation, AI Denoise will turn on for the requesting photo.": "After installation, AI Denoise will turn on for the requesting photo.", + "Cancel pending action": "Cancel pending action", + "Install a denoise model to enable AI Denoise on this photo": "Install a denoise model to enable AI Denoise on this photo", + "Pending photo action cancelled": "Pending photo action cancelled", + "Pending photo action cancelled because its library or photo changed": "Pending photo action cancelled because its library or photo changed", + "ONNX model": "ONNX model", + "commercial use subject to GPL conditions": "commercial use subject to GPL conditions" } diff --git a/crates/ui-egui/locales/ja-formats.json b/crates/ui-egui/locales/ja-formats.json index e462871c..b0e0f2f9 100644 --- a/crates/ui-egui/locales/ja-formats.json +++ b/crates/ui-egui/locales/ja-formats.json @@ -187,5 +187,24 @@ "Moved; {} photo(s) relinked": "移動しました。{}枚の写真の参照先を更新しました", "{} label (click more labels to show any of them)": "{}ラベル（ほかのラベルもクリックすると、いずれかに一致する写真を表示）", ", {} star{}": "、星{}つ{:.0}", - ", {} label": "、{}ラベル" + ", {} label": "、{}ラベル", + "Downloads from {host}. Once it has arrived and checked out, LightCraft installs it, {after}.": "Downloads from {host}. Once it has arrived and checked out, LightCraft installs it, {after}.", + "{:.1} MB": "{:.1} MB", + "Running on the processor: {}.": "Running on the processor: {}.", + "{} · {}. Kept in the library's “denoise” folder; the oldest ones that no photo uses go first when the limit is reached.": "{} · {}. Kept in the library's “denoise” folder; the oldest ones that no photo uses go first when the limit is reached.", + "Denoising {name}: {done} of {total} tiles{}": "Denoising {name}: {done} of {total} tiles{}", + "{size} GB": "{size} GB", + "{} of {} from {host}": "{} of {} from {host}", + "{} KB": "{} KB", + "{} MB": "{} MB", + "Making the denoised picture… {}%": "Making the denoised picture… {}%", + "{name} · {terms}": "{name} · {terms}", + "{n} {noun}": "{n} {noun}", + "Trained on: {provenance}": "Trained on: {provenance}", + " · {queued} waiting": " · {queued} waiting", + "Shows the model's terms, then downloads {} from {host}, installs it and starts using it. Nothing else is sent.": "Shows the model's terms, then downloads {} from {host}, installs it and starts using it. Nothing else is sent.", + "In line for AI Denoise ({ahead} ahead)…": "In line for AI Denoise ({ahead} ahead)…", + "Running on {}.": "Running on {}.", + "{file} · {size}": "{file} · {size}", + "• {assumption}": "• {assumption}" } diff --git a/crates/ui-egui/locales/ja.json b/crates/ui-egui/locales/ja.json index afc2fe44..49f2ef80 100644 --- a/crates/ui-egui/locales/ja.json +++ b/crates/ui-egui/locales/ja.json @@ -1643,5 +1643,94 @@ "Install the SAM 3 Model": "SAM 3モデルのインストール", "This build can't download the model yet. To install it by hand, put model.safetensors, vocab.json and merges.txt from Meta's facebook/sam3 in the folder above: Object and Describe work as soon as they are there.": "このビルドではまだモデルをダウンロードできません。手動でインストールするには、Meta の facebook/sam3 の model.safetensors、vocab.json、merges.txt を上のフォルダに置いてください。置いた時点で「オブジェクト」と「説明で選択」が使えるようになります。", "Open Log Folder": "ログフォルダーを開く", - "Back to this person": "Back to this person" + "Back to this person": "Back to this person", + "AI Denoise": "AI Denoise", + "AI Denoise Model": "AI Denoise Model", + "AI Denoise is off: press Use on a model.": "AI Denoise is off: press Use on a model.", + "AI Denoise needs a model. Install one in Settings.": "AI Denoise needs a model. Install one in Settings.", + "Accept & Download": "Accept & Download", + "Already installed. Remove it before installing a replacement.": "Already installed. Remove it before installing a replacement.", + "Automatic uses the graphics card where it is faster than the processor: both are timed on this computer the first time a photo is made.": "Automatic uses the graphics card where it is faster than the processor: both are timed on this computer the first time a photo is made.", + "Cache": "Cache", + "Checking and installing the model…": "Checking and installing the model…", + "Checking…": "Checking…", + "Clear cache": "Clear cache", + "Denoise Models": "Denoise Models", + "Denoise Settings": "Denoise Settings", + "Denoise all photos with an Amount": "Denoise all photos with an Amount", + "Downloaded and checked. Installing…": "Downloaded and checked. Installing…", + "Failed its last test": "Failed its last test", + "Graphics card": "Graphics card", + "I have read these terms and accept them for my own use": "I have read these terms and accept them for my own use", + "Inspect Denoise Model": "Inspect Denoise Model", + "Install Denoise Model": "Install Denoise Model", + "Install from file…": "Install from file…", + "Install your own ONNX model with denoise-model.json beside it. This build offers no model downloads.": "Install your own ONNX model with denoise-model.json beside it. This build offers no model downloads.", + "Its denoised picture is not made yet, and pictures are not made on their own (Settings, AI Denoise).": "Its denoised picture is not made yet, and pictures are not made on their own (Settings, AI Denoise).", + "LightCraft does not know this model, so it assumed:": "LightCraft does not know this model, so it assumed:", + "Limit": "Limit", + "Make it now": "Make it now", + "Make the denoised picture of the photos I am looking at": "Make the denoised picture of the photos I am looking at", + "Next up for AI Denoise…": "Next up for AI Denoise…", + "Nothing is fetched until you accept. The model is kept on this computer only; LightCraft never uploads or shares it.": "Nothing is fetched until you accept. The model is kept on this computer only; LightCraft never uploads or shares it.", + "Only photos whose AI Denoise amount is above 0 are made. Exports always make theirs if it is missing.": "Only photos whose AI Denoise amount is above 0 are made. Exports always make theirs if it is missing.", + "Open page": "Open page", + "Open the model's page": "Open the model's page", + "Opens the model's own page in your browser, to read about it or get the file yourself.": "Opens the model's own page in your browser, to read about it or get the file yourself.", + "Processor": "Processor", + "Set up AI Denoise…": "Set up AI Denoise…", + "Speed": "Speed", + "Starting…": "Starting…", + "Tested and works on this computer": "Tested and works on this computer", + "The denoise model is installed and in use: the AI Denoise Amount slider under Detail now works": "The denoise model is installed and in use: the AI Denoise Amount slider under Detail now works", + "The graphics card is set up when the next photo is made.": "The graphics card is set up when the next photo is made.", + "The model is kept on this computer only. LightCraft never uploads or shares it.": "The model is kept on this computer only. LightCraft never uploads or shares it.", + "The processor does the work.": "The processor does the work.", + "There is no model file to install": "There is no model file to install", + "This build cannot run denoise models.": "This build cannot run denoise models.", + "This build has nowhere to keep denoise models, so AI Denoise is not available. (The desktop app has.)": "This build has nowhere to keep denoise models, so AI Denoise is not available. (The desktop app has.)", + "This file cannot be used yet": "This file cannot be used yet", + "Tick the box to accept the model's terms first": "Tick the box to accept the model's terms first", + "Try again": "Try again", + "Try the graphics card again": "Try the graphics card again", + "Turn off": "Turn off", + "Unknown licence": "Unknown licence", + "Use": "Use", + "Waiting to start…": "Waiting to start…", + "Works on raw files from cameras with a standard (Bayer) sensor. Other files keep their normal noise reduction.": "Works on raw files from cameras with a standard (Bayer) sensor. Other files keep their normal noise reduction.", + "cached picture": "cached picture", + "cached pictures": "cached pictures", + "commercial use allowed": "commercial use allowed", + "denoiseModel.accept": "denoiseModel.accept", + "in use": "in use", + "installed": "installed", + "makes AI Denoise available": "makes AI Denoise available", + "non-commercial use only": "non-commercial use only", + "photo has its denoised picture.": "photo has its denoised picture.", + "photo waiting": "photo waiting", + "photos have their denoised picture.": "photos have their denoised picture.", + "photos waiting": "photos waiting", + "starts using it for AI Denoise": "starts using it for AI Denoise", + "terms unclear": "terms unclear", + "the graphics card": "the graphics card", + "the graphics card is not used": "the graphics card is not used", + "Cancel Denoise": "Cancel Denoise", + "Cancel Denoise Model Download": "Cancel Denoise Model Download", + "Choose Denoise Model": "Choose Denoise Model", + "Clear Denoise Cache": "Clear Denoise Cache", + "Denoise Model Downloads": "Denoise Model Downloads", + "Denoise Photos": "Denoise Photos", + "Denoise Pump": "Denoise Pump", + "Denoise Status": "Denoise Status", + "Download Denoise Model": "Download Denoise Model", + "Remove Denoise Model": "Remove Denoise Model", + "Test Denoise Model": "Test Denoise Model", + "AI Denoise is enabled on the requesting photo": "AI Denoise is enabled on the requesting photo", + "After installation, AI Denoise will turn on for the requesting photo.": "After installation, AI Denoise will turn on for the requesting photo.", + "Cancel pending action": "Cancel pending action", + "Install a denoise model to enable AI Denoise on this photo": "Install a denoise model to enable AI Denoise on this photo", + "Pending photo action cancelled": "Pending photo action cancelled", + "Pending photo action cancelled because its library or photo changed": "Pending photo action cancelled because its library or photo changed", + "ONNX model": "ONNX model", + "commercial use subject to GPL conditions": "commercial use subject to GPL conditions" } diff --git a/crates/ui-egui/locales/pt-br-formats.json b/crates/ui-egui/locales/pt-br-formats.json index 59bec59d..ab64e082 100644 --- a/crates/ui-egui/locales/pt-br-formats.json +++ b/crates/ui-egui/locales/pt-br-formats.json @@ -187,5 +187,24 @@ "Moved; {} photo(s) relinked": "Movida; {} foto(s) vinculada(s) novamente", "{} label (click more labels to show any of them)": "Rótulo {} (clique em mais rótulos para mostrar qualquer um deles)", ", {} star{}": ", {} estrela{}", - ", {} label": ", rótulo {}" + ", {} label": ", rótulo {}", + "Downloads from {host}. Once it has arrived and checked out, LightCraft installs it, {after}.": "Downloads from {host}. Once it has arrived and checked out, LightCraft installs it, {after}.", + "{:.1} MB": "{:.1} MB", + "Running on the processor: {}.": "Running on the processor: {}.", + "{} · {}. Kept in the library's “denoise” folder; the oldest ones that no photo uses go first when the limit is reached.": "{} · {}. Kept in the library's “denoise” folder; the oldest ones that no photo uses go first when the limit is reached.", + "Denoising {name}: {done} of {total} tiles{}": "Denoising {name}: {done} of {total} tiles{}", + "{size} GB": "{size} GB", + "{} of {} from {host}": "{} of {} from {host}", + "{} KB": "{} KB", + "{} MB": "{} MB", + "Making the denoised picture… {}%": "Making the denoised picture… {}%", + "{name} · {terms}": "{name} · {terms}", + "{n} {noun}": "{n} {noun}", + "Trained on: {provenance}": "Trained on: {provenance}", + " · {queued} waiting": " · {queued} waiting", + "Shows the model's terms, then downloads {} from {host}, installs it and starts using it. Nothing else is sent.": "Shows the model's terms, then downloads {} from {host}, installs it and starts using it. Nothing else is sent.", + "In line for AI Denoise ({ahead} ahead)…": "In line for AI Denoise ({ahead} ahead)…", + "Running on {}.": "Running on {}.", + "{file} · {size}": "{file} · {size}", + "• {assumption}": "• {assumption}" } diff --git a/crates/ui-egui/locales/pt-br.json b/crates/ui-egui/locales/pt-br.json index be7d22ba..81e331c5 100644 --- a/crates/ui-egui/locales/pt-br.json +++ b/crates/ui-egui/locales/pt-br.json @@ -1643,5 +1643,94 @@ "≥ at least": "≥ no mínimo", "日本語": "日本語", "繁體中文（台灣）": "繁體中文（台灣）", - "Back to this person": "Back to this person" + "Back to this person": "Back to this person", + "AI Denoise": "AI Denoise", + "AI Denoise Model": "AI Denoise Model", + "AI Denoise is off: press Use on a model.": "AI Denoise is off: press Use on a model.", + "AI Denoise needs a model. Install one in Settings.": "AI Denoise needs a model. Install one in Settings.", + "Accept & Download": "Accept & Download", + "Already installed. Remove it before installing a replacement.": "Already installed. Remove it before installing a replacement.", + "Automatic uses the graphics card where it is faster than the processor: both are timed on this computer the first time a photo is made.": "Automatic uses the graphics card where it is faster than the processor: both are timed on this computer the first time a photo is made.", + "Cache": "Cache", + "Checking and installing the model…": "Checking and installing the model…", + "Checking…": "Checking…", + "Clear cache": "Clear cache", + "Denoise Models": "Denoise Models", + "Denoise Settings": "Denoise Settings", + "Denoise all photos with an Amount": "Denoise all photos with an Amount", + "Downloaded and checked. Installing…": "Downloaded and checked. Installing…", + "Failed its last test": "Failed its last test", + "Graphics card": "Graphics card", + "I have read these terms and accept them for my own use": "I have read these terms and accept them for my own use", + "Inspect Denoise Model": "Inspect Denoise Model", + "Install Denoise Model": "Install Denoise Model", + "Install from file…": "Install from file…", + "Install your own ONNX model with denoise-model.json beside it. This build offers no model downloads.": "Install your own ONNX model with denoise-model.json beside it. This build offers no model downloads.", + "Its denoised picture is not made yet, and pictures are not made on their own (Settings, AI Denoise).": "Its denoised picture is not made yet, and pictures are not made on their own (Settings, AI Denoise).", + "LightCraft does not know this model, so it assumed:": "LightCraft does not know this model, so it assumed:", + "Limit": "Limit", + "Make it now": "Make it now", + "Make the denoised picture of the photos I am looking at": "Make the denoised picture of the photos I am looking at", + "Next up for AI Denoise…": "Next up for AI Denoise…", + "Nothing is fetched until you accept. The model is kept on this computer only; LightCraft never uploads or shares it.": "Nothing is fetched until you accept. The model is kept on this computer only; LightCraft never uploads or shares it.", + "Only photos whose AI Denoise amount is above 0 are made. Exports always make theirs if it is missing.": "Only photos whose AI Denoise amount is above 0 are made. Exports always make theirs if it is missing.", + "Open page": "Open page", + "Open the model's page": "Open the model's page", + "Opens the model's own page in your browser, to read about it or get the file yourself.": "Opens the model's own page in your browser, to read about it or get the file yourself.", + "Processor": "Processor", + "Set up AI Denoise…": "Set up AI Denoise…", + "Speed": "Speed", + "Starting…": "Starting…", + "Tested and works on this computer": "Tested and works on this computer", + "The denoise model is installed and in use: the AI Denoise Amount slider under Detail now works": "The denoise model is installed and in use: the AI Denoise Amount slider under Detail now works", + "The graphics card is set up when the next photo is made.": "The graphics card is set up when the next photo is made.", + "The model is kept on this computer only. LightCraft never uploads or shares it.": "The model is kept on this computer only. LightCraft never uploads or shares it.", + "The processor does the work.": "The processor does the work.", + "There is no model file to install": "There is no model file to install", + "This build cannot run denoise models.": "This build cannot run denoise models.", + "This build has nowhere to keep denoise models, so AI Denoise is not available. (The desktop app has.)": "This build has nowhere to keep denoise models, so AI Denoise is not available. (The desktop app has.)", + "This file cannot be used yet": "This file cannot be used yet", + "Tick the box to accept the model's terms first": "Tick the box to accept the model's terms first", + "Try again": "Try again", + "Try the graphics card again": "Try the graphics card again", + "Turn off": "Turn off", + "Unknown licence": "Unknown licence", + "Use": "Use", + "Waiting to start…": "Waiting to start…", + "Works on raw files from cameras with a standard (Bayer) sensor. Other files keep their normal noise reduction.": "Works on raw files from cameras with a standard (Bayer) sensor. Other files keep their normal noise reduction.", + "cached picture": "cached picture", + "cached pictures": "cached pictures", + "commercial use allowed": "commercial use allowed", + "denoiseModel.accept": "denoiseModel.accept", + "in use": "in use", + "installed": "installed", + "makes AI Denoise available": "makes AI Denoise available", + "non-commercial use only": "non-commercial use only", + "photo has its denoised picture.": "photo has its denoised picture.", + "photo waiting": "photo waiting", + "photos have their denoised picture.": "photos have their denoised picture.", + "photos waiting": "photos waiting", + "starts using it for AI Denoise": "starts using it for AI Denoise", + "terms unclear": "terms unclear", + "the graphics card": "the graphics card", + "the graphics card is not used": "the graphics card is not used", + "Cancel Denoise": "Cancel Denoise", + "Cancel Denoise Model Download": "Cancel Denoise Model Download", + "Choose Denoise Model": "Choose Denoise Model", + "Clear Denoise Cache": "Clear Denoise Cache", + "Denoise Model Downloads": "Denoise Model Downloads", + "Denoise Photos": "Denoise Photos", + "Denoise Pump": "Denoise Pump", + "Denoise Status": "Denoise Status", + "Download Denoise Model": "Download Denoise Model", + "Remove Denoise Model": "Remove Denoise Model", + "Test Denoise Model": "Test Denoise Model", + "AI Denoise is enabled on the requesting photo": "AI Denoise is enabled on the requesting photo", + "After installation, AI Denoise will turn on for the requesting photo.": "After installation, AI Denoise will turn on for the requesting photo.", + "Cancel pending action": "Cancel pending action", + "Install a denoise model to enable AI Denoise on this photo": "Install a denoise model to enable AI Denoise on this photo", + "Pending photo action cancelled": "Pending photo action cancelled", + "Pending photo action cancelled because its library or photo changed": "Pending photo action cancelled because its library or photo changed", + "ONNX model": "ONNX model", + "commercial use subject to GPL conditions": "commercial use subject to GPL conditions" } diff --git a/crates/ui-egui/locales/ru-formats.json b/crates/ui-egui/locales/ru-formats.json index 2beb4a5b..f3a0f647 100644 --- a/crates/ui-egui/locales/ru-formats.json +++ b/crates/ui-egui/locales/ru-formats.json @@ -187,5 +187,24 @@ "{} — show their photos": "{} — показать фото этого человека", "· {dups} already in the library": "· уже в библиотеке: {dups}", "· {n} photo{}": "· фото: {n}{:.0}", - "… and {more} more": "… и ещё {more}" + "… and {more} more": "… и ещё {more}", + "Downloads from {host}. Once it has arrived and checked out, LightCraft installs it, {after}.": "Downloads from {host}. Once it has arrived and checked out, LightCraft installs it, {after}.", + "{:.1} MB": "{:.1} MB", + "Running on the processor: {}.": "Running on the processor: {}.", + "{} · {}. Kept in the library's “denoise” folder; the oldest ones that no photo uses go first when the limit is reached.": "{} · {}. Kept in the library's “denoise” folder; the oldest ones that no photo uses go first when the limit is reached.", + "Denoising {name}: {done} of {total} tiles{}": "Denoising {name}: {done} of {total} tiles{}", + "{size} GB": "{size} GB", + "{} of {} from {host}": "{} of {} from {host}", + "{} KB": "{} KB", + "{} MB": "{} MB", + "Making the denoised picture… {}%": "Making the denoised picture… {}%", + "{name} · {terms}": "{name} · {terms}", + "{n} {noun}": "{n} {noun}", + "Trained on: {provenance}": "Trained on: {provenance}", + " · {queued} waiting": " · {queued} waiting", + "Shows the model's terms, then downloads {} from {host}, installs it and starts using it. Nothing else is sent.": "Shows the model's terms, then downloads {} from {host}, installs it and starts using it. Nothing else is sent.", + "In line for AI Denoise ({ahead} ahead)…": "In line for AI Denoise ({ahead} ahead)…", + "Running on {}.": "Running on {}.", + "{file} · {size}": "{file} · {size}", + "• {assumption}": "• {assumption}" } diff --git a/crates/ui-egui/locales/ru.json b/crates/ui-egui/locales/ru.json index f1d33491..382bfc05 100644 --- a/crates/ui-egui/locales/ru.json +++ b/crates/ui-egui/locales/ru.json @@ -1313,5 +1313,95 @@ "saved in": "сохранено в", "this photo": "это фото", "− harder border · + softer border": "− жёстче край · + мягче край", - "Back to this person": "Back to this person" + "Back to this person": "Back to this person", + "AI Denoise": "AI Denoise", + "AI Denoise Model": "AI Denoise Model", + "AI Denoise is off: press Use on a model.": "AI Denoise is off: press Use on a model.", + "AI Denoise needs a model. Install one in Settings.": "AI Denoise needs a model. Install one in Settings.", + "Accept & Download": "Accept & Download", + "Already installed. Remove it before installing a replacement.": "Already installed. Remove it before installing a replacement.", + "Automatic uses the graphics card where it is faster than the processor: both are timed on this computer the first time a photo is made.": "Automatic uses the graphics card where it is faster than the processor: both are timed on this computer the first time a photo is made.", + "Cache": "Cache", + "Checking and installing the model…": "Checking and installing the model…", + "Checking…": "Checking…", + "Clear cache": "Clear cache", + "Denoise Models": "Denoise Models", + "Denoise Settings": "Denoise Settings", + "Denoise all photos with an Amount": "Denoise all photos with an Amount", + "Downloaded and checked. Installing…": "Downloaded and checked. Installing…", + "Failed its last test": "Failed its last test", + "Graphics card": "Graphics card", + "I have read these terms and accept them for my own use": "I have read these terms and accept them for my own use", + "Inspect Denoise Model": "Inspect Denoise Model", + "Install Denoise Model": "Install Denoise Model", + "Install from file…": "Install from file…", + "Install your own ONNX model with denoise-model.json beside it. This build offers no model downloads.": "Install your own ONNX model with denoise-model.json beside it. This build offers no model downloads.", + "Its denoised picture is not made yet, and pictures are not made on their own (Settings, AI Denoise).": "Its denoised picture is not made yet, and pictures are not made on their own (Settings, AI Denoise).", + "LightCraft does not know this model, so it assumed:": "LightCraft does not know this model, so it assumed:", + "Limit": "Limit", + "Make it now": "Make it now", + "Make the denoised picture of the photos I am looking at": "Make the denoised picture of the photos I am looking at", + "Models": "Models", + "Next up for AI Denoise…": "Next up for AI Denoise…", + "Nothing is fetched until you accept. The model is kept on this computer only; LightCraft never uploads or shares it.": "Nothing is fetched until you accept. The model is kept on this computer only; LightCraft never uploads or shares it.", + "Only photos whose AI Denoise amount is above 0 are made. Exports always make theirs if it is missing.": "Only photos whose AI Denoise amount is above 0 are made. Exports always make theirs if it is missing.", + "Open page": "Open page", + "Open the model's page": "Open the model's page", + "Opens the model's own page in your browser, to read about it or get the file yourself.": "Opens the model's own page in your browser, to read about it or get the file yourself.", + "Processor": "Processor", + "Set up AI Denoise…": "Set up AI Denoise…", + "Speed": "Speed", + "Starting…": "Starting…", + "Tested and works on this computer": "Tested and works on this computer", + "The denoise model is installed and in use: the AI Denoise Amount slider under Detail now works": "The denoise model is installed and in use: the AI Denoise Amount slider under Detail now works", + "The graphics card is set up when the next photo is made.": "The graphics card is set up when the next photo is made.", + "The model is kept on this computer only. LightCraft never uploads or shares it.": "The model is kept on this computer only. LightCraft never uploads or shares it.", + "The processor does the work.": "The processor does the work.", + "There is no model file to install": "There is no model file to install", + "This build cannot run denoise models.": "This build cannot run denoise models.", + "This build has nowhere to keep denoise models, so AI Denoise is not available. (The desktop app has.)": "This build has nowhere to keep denoise models, so AI Denoise is not available. (The desktop app has.)", + "This file cannot be used yet": "This file cannot be used yet", + "Tick the box to accept the model's terms first": "Tick the box to accept the model's terms first", + "Try again": "Try again", + "Try the graphics card again": "Try the graphics card again", + "Turn off": "Turn off", + "Unknown licence": "Unknown licence", + "Use": "Use", + "Waiting to start…": "Waiting to start…", + "Works on raw files from cameras with a standard (Bayer) sensor. Other files keep their normal noise reduction.": "Works on raw files from cameras with a standard (Bayer) sensor. Other files keep their normal noise reduction.", + "cached picture": "cached picture", + "cached pictures": "cached pictures", + "commercial use allowed": "commercial use allowed", + "denoiseModel.accept": "denoiseModel.accept", + "in use": "in use", + "installed": "installed", + "makes AI Denoise available": "makes AI Denoise available", + "non-commercial use only": "non-commercial use only", + "photo has its denoised picture.": "photo has its denoised picture.", + "photo waiting": "photo waiting", + "photos have their denoised picture.": "photos have their denoised picture.", + "photos waiting": "photos waiting", + "starts using it for AI Denoise": "starts using it for AI Denoise", + "terms unclear": "terms unclear", + "the graphics card": "the graphics card", + "the graphics card is not used": "the graphics card is not used", + "Cancel Denoise": "Cancel Denoise", + "Cancel Denoise Model Download": "Cancel Denoise Model Download", + "Choose Denoise Model": "Choose Denoise Model", + "Clear Denoise Cache": "Clear Denoise Cache", + "Denoise Model Downloads": "Denoise Model Downloads", + "Denoise Photos": "Denoise Photos", + "Denoise Pump": "Denoise Pump", + "Denoise Status": "Denoise Status", + "Download Denoise Model": "Download Denoise Model", + "Remove Denoise Model": "Remove Denoise Model", + "Test Denoise Model": "Test Denoise Model", + "AI Denoise is enabled on the requesting photo": "AI Denoise is enabled on the requesting photo", + "After installation, AI Denoise will turn on for the requesting photo.": "After installation, AI Denoise will turn on for the requesting photo.", + "Cancel pending action": "Cancel pending action", + "Install a denoise model to enable AI Denoise on this photo": "Install a denoise model to enable AI Denoise on this photo", + "Pending photo action cancelled": "Pending photo action cancelled", + "Pending photo action cancelled because its library or photo changed": "Pending photo action cancelled because its library or photo changed", + "ONNX model": "ONNX model", + "commercial use subject to GPL conditions": "commercial use subject to GPL conditions" } diff --git a/crates/ui-egui/locales/zh-hans-formats.json b/crates/ui-egui/locales/zh-hans-formats.json index eca40f8e..e32c7f66 100644 --- a/crates/ui-egui/locales/zh-hans-formats.json +++ b/crates/ui-egui/locales/zh-hans-formats.json @@ -187,5 +187,24 @@ "Moved; {} photo(s) relinked": "已移动；已重新链接 {0} 张照片", "{} label (click more labels to show any of them)": "{0}标签（单击更多标签可显示符合其中任一标签的照片）", ", {} star{}": "，{0} 星{1:.0}", - ", {} label": "，{0}标签" + ", {} label": "，{0}标签", + "Downloads from {host}. Once it has arrived and checked out, LightCraft installs it, {after}.": "Downloads from {host}. Once it has arrived and checked out, LightCraft installs it, {after}.", + "{:.1} MB": "{:.1} MB", + "Running on the processor: {}.": "Running on the processor: {}.", + "{} · {}. Kept in the library's “denoise” folder; the oldest ones that no photo uses go first when the limit is reached.": "{} · {}. Kept in the library's “denoise” folder; the oldest ones that no photo uses go first when the limit is reached.", + "Denoising {name}: {done} of {total} tiles{}": "Denoising {name}: {done} of {total} tiles{}", + "{size} GB": "{size} GB", + "{} of {} from {host}": "{} of {} from {host}", + "{} KB": "{} KB", + "{} MB": "{} MB", + "Making the denoised picture… {}%": "Making the denoised picture… {}%", + "{name} · {terms}": "{name} · {terms}", + "{n} {noun}": "{n} {noun}", + "Trained on: {provenance}": "Trained on: {provenance}", + " · {queued} waiting": " · {queued} waiting", + "Shows the model's terms, then downloads {} from {host}, installs it and starts using it. Nothing else is sent.": "Shows the model's terms, then downloads {} from {host}, installs it and starts using it. Nothing else is sent.", + "In line for AI Denoise ({ahead} ahead)…": "In line for AI Denoise ({ahead} ahead)…", + "Running on {}.": "Running on {}.", + "{file} · {size}": "{file} · {size}", + "• {assumption}": "• {assumption}" } diff --git a/crates/ui-egui/locales/zh-hans.json b/crates/ui-egui/locales/zh-hans.json index 3d88dcc2..4074dfbd 100644 --- a/crates/ui-egui/locales/zh-hans.json +++ b/crates/ui-egui/locales/zh-hans.json @@ -1643,5 +1643,94 @@ "Install the SAM 3 Model": "安装 SAM 3 模型", "This build can't download the model yet. To install it by hand, put model.safetensors, vocab.json and merges.txt from Meta's facebook/sam3 in the folder above: Object and Describe work as soon as they are there.": "此版本尚无法下载模型。如需手动安装，请将 Meta 的 facebook/sam3 中的 model.safetensors、vocab.json 和 merges.txt 放入上方的文件夹：放入后即可使用“对象”和“描述”。", "Open Log Folder": "打开日志文件夹", - "Back to this person": "Back to this person" + "Back to this person": "Back to this person", + "AI Denoise": "AI Denoise", + "AI Denoise Model": "AI Denoise Model", + "AI Denoise is off: press Use on a model.": "AI Denoise is off: press Use on a model.", + "AI Denoise needs a model. Install one in Settings.": "AI Denoise needs a model. Install one in Settings.", + "Accept & Download": "Accept & Download", + "Already installed. Remove it before installing a replacement.": "Already installed. Remove it before installing a replacement.", + "Automatic uses the graphics card where it is faster than the processor: both are timed on this computer the first time a photo is made.": "Automatic uses the graphics card where it is faster than the processor: both are timed on this computer the first time a photo is made.", + "Cache": "Cache", + "Checking and installing the model…": "Checking and installing the model…", + "Checking…": "Checking…", + "Clear cache": "Clear cache", + "Denoise Models": "Denoise Models", + "Denoise Settings": "Denoise Settings", + "Denoise all photos with an Amount": "Denoise all photos with an Amount", + "Downloaded and checked. Installing…": "Downloaded and checked. Installing…", + "Failed its last test": "Failed its last test", + "Graphics card": "Graphics card", + "I have read these terms and accept them for my own use": "I have read these terms and accept them for my own use", + "Inspect Denoise Model": "Inspect Denoise Model", + "Install Denoise Model": "Install Denoise Model", + "Install from file…": "Install from file…", + "Install your own ONNX model with denoise-model.json beside it. This build offers no model downloads.": "Install your own ONNX model with denoise-model.json beside it. This build offers no model downloads.", + "Its denoised picture is not made yet, and pictures are not made on their own (Settings, AI Denoise).": "Its denoised picture is not made yet, and pictures are not made on their own (Settings, AI Denoise).", + "LightCraft does not know this model, so it assumed:": "LightCraft does not know this model, so it assumed:", + "Limit": "Limit", + "Make it now": "Make it now", + "Make the denoised picture of the photos I am looking at": "Make the denoised picture of the photos I am looking at", + "Next up for AI Denoise…": "Next up for AI Denoise…", + "Nothing is fetched until you accept. The model is kept on this computer only; LightCraft never uploads or shares it.": "Nothing is fetched until you accept. The model is kept on this computer only; LightCraft never uploads or shares it.", + "Only photos whose AI Denoise amount is above 0 are made. Exports always make theirs if it is missing.": "Only photos whose AI Denoise amount is above 0 are made. Exports always make theirs if it is missing.", + "Open page": "Open page", + "Open the model's page": "Open the model's page", + "Opens the model's own page in your browser, to read about it or get the file yourself.": "Opens the model's own page in your browser, to read about it or get the file yourself.", + "Processor": "Processor", + "Set up AI Denoise…": "Set up AI Denoise…", + "Speed": "Speed", + "Starting…": "Starting…", + "Tested and works on this computer": "Tested and works on this computer", + "The denoise model is installed and in use: the AI Denoise Amount slider under Detail now works": "The denoise model is installed and in use: the AI Denoise Amount slider under Detail now works", + "The graphics card is set up when the next photo is made.": "The graphics card is set up when the next photo is made.", + "The model is kept on this computer only. LightCraft never uploads or shares it.": "The model is kept on this computer only. LightCraft never uploads or shares it.", + "The processor does the work.": "The processor does the work.", + "There is no model file to install": "There is no model file to install", + "This build cannot run denoise models.": "This build cannot run denoise models.", + "This build has nowhere to keep denoise models, so AI Denoise is not available. (The desktop app has.)": "This build has nowhere to keep denoise models, so AI Denoise is not available. (The desktop app has.)", + "This file cannot be used yet": "This file cannot be used yet", + "Tick the box to accept the model's terms first": "Tick the box to accept the model's terms first", + "Try again": "Try again", + "Try the graphics card again": "Try the graphics card again", + "Turn off": "Turn off", + "Unknown licence": "Unknown licence", + "Use": "Use", + "Waiting to start…": "Waiting to start…", + "Works on raw files from cameras with a standard (Bayer) sensor. Other files keep their normal noise reduction.": "Works on raw files from cameras with a standard (Bayer) sensor. Other files keep their normal noise reduction.", + "cached picture": "cached picture", + "cached pictures": "cached pictures", + "commercial use allowed": "commercial use allowed", + "denoiseModel.accept": "denoiseModel.accept", + "in use": "in use", + "installed": "installed", + "makes AI Denoise available": "makes AI Denoise available", + "non-commercial use only": "non-commercial use only", + "photo has its denoised picture.": "photo has its denoised picture.", + "photo waiting": "photo waiting", + "photos have their denoised picture.": "photos have their denoised picture.", + "photos waiting": "photos waiting", + "starts using it for AI Denoise": "starts using it for AI Denoise", + "terms unclear": "terms unclear", + "the graphics card": "the graphics card", + "the graphics card is not used": "the graphics card is not used", + "Cancel Denoise": "Cancel Denoise", + "Cancel Denoise Model Download": "Cancel Denoise Model Download", + "Choose Denoise Model": "Choose Denoise Model", + "Clear Denoise Cache": "Clear Denoise Cache", + "Denoise Model Downloads": "Denoise Model Downloads", + "Denoise Photos": "Denoise Photos", + "Denoise Pump": "Denoise Pump", + "Denoise Status": "Denoise Status", + "Download Denoise Model": "Download Denoise Model", + "Remove Denoise Model": "Remove Denoise Model", + "Test Denoise Model": "Test Denoise Model", + "AI Denoise is enabled on the requesting photo": "AI Denoise is enabled on the requesting photo", + "After installation, AI Denoise will turn on for the requesting photo.": "After installation, AI Denoise will turn on for the requesting photo.", + "Cancel pending action": "Cancel pending action", + "Install a denoise model to enable AI Denoise on this photo": "Install a denoise model to enable AI Denoise on this photo", + "Pending photo action cancelled": "Pending photo action cancelled", + "Pending photo action cancelled because its library or photo changed": "Pending photo action cancelled because its library or photo changed", + "ONNX model": "ONNX model", + "commercial use subject to GPL conditions": "commercial use subject to GPL conditions" } diff --git a/crates/ui-egui/locales/zh-hant-formats.json b/crates/ui-egui/locales/zh-hant-formats.json index 640e3eb7..182a7dc6 100644 --- a/crates/ui-egui/locales/zh-hant-formats.json +++ b/crates/ui-egui/locales/zh-hant-formats.json @@ -187,5 +187,24 @@ "Moved; {} photo(s) relinked": "已移動；已重新連結 {} 張照片", "{} label (click more labels to show any of them)": "{}標籤（點擊更多標籤可顯示符合其中任一標籤的照片）", ", {} star{}": "，{} 顆星{:.0}", - ", {} label": "，{}標籤" + ", {} label": "，{}標籤", + "Downloads from {host}. Once it has arrived and checked out, LightCraft installs it, {after}.": "Downloads from {host}. Once it has arrived and checked out, LightCraft installs it, {after}.", + "{:.1} MB": "{:.1} MB", + "Running on the processor: {}.": "Running on the processor: {}.", + "{} · {}. Kept in the library's “denoise” folder; the oldest ones that no photo uses go first when the limit is reached.": "{} · {}. Kept in the library's “denoise” folder; the oldest ones that no photo uses go first when the limit is reached.", + "Denoising {name}: {done} of {total} tiles{}": "Denoising {name}: {done} of {total} tiles{}", + "{size} GB": "{size} GB", + "{} of {} from {host}": "{} of {} from {host}", + "{} KB": "{} KB", + "{} MB": "{} MB", + "Making the denoised picture… {}%": "Making the denoised picture… {}%", + "{name} · {terms}": "{name} · {terms}", + "{n} {noun}": "{n} {noun}", + "Trained on: {provenance}": "Trained on: {provenance}", + " · {queued} waiting": " · {queued} waiting", + "Shows the model's terms, then downloads {} from {host}, installs it and starts using it. Nothing else is sent.": "Shows the model's terms, then downloads {} from {host}, installs it and starts using it. Nothing else is sent.", + "In line for AI Denoise ({ahead} ahead)…": "In line for AI Denoise ({ahead} ahead)…", + "Running on {}.": "Running on {}.", + "{file} · {size}": "{file} · {size}", + "• {assumption}": "• {assumption}" } diff --git a/crates/ui-egui/locales/zh-hant.json b/crates/ui-egui/locales/zh-hant.json index 085ccadc..96fcbb18 100644 --- a/crates/ui-egui/locales/zh-hant.json +++ b/crates/ui-egui/locales/zh-hant.json @@ -1643,5 +1643,94 @@ "Install the SAM 3 Model": "安裝 SAM 3 模型", "This build can't download the model yet. To install it by hand, put model.safetensors, vocab.json and merges.txt from Meta's facebook/sam3 in the folder above: Object and Describe work as soon as they are there.": "此版本尚無法下載模型。如需手動安裝，請將 Meta 的 facebook/sam3 中的 model.safetensors、vocab.json 和 merges.txt 放入上方的資料夾：放入後即可使用「物件」與「描述」。", "Open Log Folder": "開啟記錄資料夾", - "Back to this person": "Back to this person" + "Back to this person": "Back to this person", + "AI Denoise": "AI Denoise", + "AI Denoise Model": "AI Denoise Model", + "AI Denoise is off: press Use on a model.": "AI Denoise is off: press Use on a model.", + "AI Denoise needs a model. Install one in Settings.": "AI Denoise needs a model. Install one in Settings.", + "Accept & Download": "Accept & Download", + "Already installed. Remove it before installing a replacement.": "Already installed. Remove it before installing a replacement.", + "Automatic uses the graphics card where it is faster than the processor: both are timed on this computer the first time a photo is made.": "Automatic uses the graphics card where it is faster than the processor: both are timed on this computer the first time a photo is made.", + "Cache": "Cache", + "Checking and installing the model…": "Checking and installing the model…", + "Checking…": "Checking…", + "Clear cache": "Clear cache", + "Denoise Models": "Denoise Models", + "Denoise Settings": "Denoise Settings", + "Denoise all photos with an Amount": "Denoise all photos with an Amount", + "Downloaded and checked. Installing…": "Downloaded and checked. Installing…", + "Failed its last test": "Failed its last test", + "Graphics card": "Graphics card", + "I have read these terms and accept them for my own use": "I have read these terms and accept them for my own use", + "Inspect Denoise Model": "Inspect Denoise Model", + "Install Denoise Model": "Install Denoise Model", + "Install from file…": "Install from file…", + "Install your own ONNX model with denoise-model.json beside it. This build offers no model downloads.": "Install your own ONNX model with denoise-model.json beside it. This build offers no model downloads.", + "Its denoised picture is not made yet, and pictures are not made on their own (Settings, AI Denoise).": "Its denoised picture is not made yet, and pictures are not made on their own (Settings, AI Denoise).", + "LightCraft does not know this model, so it assumed:": "LightCraft does not know this model, so it assumed:", + "Limit": "Limit", + "Make it now": "Make it now", + "Make the denoised picture of the photos I am looking at": "Make the denoised picture of the photos I am looking at", + "Next up for AI Denoise…": "Next up for AI Denoise…", + "Nothing is fetched until you accept. The model is kept on this computer only; LightCraft never uploads or shares it.": "Nothing is fetched until you accept. The model is kept on this computer only; LightCraft never uploads or shares it.", + "Only photos whose AI Denoise amount is above 0 are made. Exports always make theirs if it is missing.": "Only photos whose AI Denoise amount is above 0 are made. Exports always make theirs if it is missing.", + "Open page": "Open page", + "Open the model's page": "Open the model's page", + "Opens the model's own page in your browser, to read about it or get the file yourself.": "Opens the model's own page in your browser, to read about it or get the file yourself.", + "Processor": "Processor", + "Set up AI Denoise…": "Set up AI Denoise…", + "Speed": "Speed", + "Starting…": "Starting…", + "Tested and works on this computer": "Tested and works on this computer", + "The denoise model is installed and in use: the AI Denoise Amount slider under Detail now works": "The denoise model is installed and in use: the AI Denoise Amount slider under Detail now works", + "The graphics card is set up when the next photo is made.": "The graphics card is set up when the next photo is made.", + "The model is kept on this computer only. LightCraft never uploads or shares it.": "The model is kept on this computer only. LightCraft never uploads or shares it.", + "The processor does the work.": "The processor does the work.", + "There is no model file to install": "There is no model file to install", + "This build cannot run denoise models.": "This build cannot run denoise models.", + "This build has nowhere to keep denoise models, so AI Denoise is not available. (The desktop app has.)": "This build has nowhere to keep denoise models, so AI Denoise is not available. (The desktop app has.)", + "This file cannot be used yet": "This file cannot be used yet", + "Tick the box to accept the model's terms first": "Tick the box to accept the model's terms first", + "Try again": "Try again", + "Try the graphics card again": "Try the graphics card again", + "Turn off": "Turn off", + "Unknown licence": "Unknown licence", + "Use": "Use", + "Waiting to start…": "Waiting to start…", + "Works on raw files from cameras with a standard (Bayer) sensor. Other files keep their normal noise reduction.": "Works on raw files from cameras with a standard (Bayer) sensor. Other files keep their normal noise reduction.", + "cached picture": "cached picture", + "cached pictures": "cached pictures", + "commercial use allowed": "commercial use allowed", + "denoiseModel.accept": "denoiseModel.accept", + "in use": "in use", + "installed": "installed", + "makes AI Denoise available": "makes AI Denoise available", + "non-commercial use only": "non-commercial use only", + "photo has its denoised picture.": "photo has its denoised picture.", + "photo waiting": "photo waiting", + "photos have their denoised picture.": "photos have their denoised picture.", + "photos waiting": "photos waiting", + "starts using it for AI Denoise": "starts using it for AI Denoise", + "terms unclear": "terms unclear", + "the graphics card": "the graphics card", + "the graphics card is not used": "the graphics card is not used", + "Cancel Denoise": "Cancel Denoise", + "Cancel Denoise Model Download": "Cancel Denoise Model Download", + "Choose Denoise Model": "Choose Denoise Model", + "Clear Denoise Cache": "Clear Denoise Cache", + "Denoise Model Downloads": "Denoise Model Downloads", + "Denoise Photos": "Denoise Photos", + "Denoise Pump": "Denoise Pump", + "Denoise Status": "Denoise Status", + "Download Denoise Model": "Download Denoise Model", + "Remove Denoise Model": "Remove Denoise Model", + "Test Denoise Model": "Test Denoise Model", + "AI Denoise is enabled on the requesting photo": "AI Denoise is enabled on the requesting photo", + "After installation, AI Denoise will turn on for the requesting photo.": "After installation, AI Denoise will turn on for the requesting photo.", + "Cancel pending action": "Cancel pending action", + "Install a denoise model to enable AI Denoise on this photo": "Install a denoise model to enable AI Denoise on this photo", + "Pending photo action cancelled": "Pending photo action cancelled", + "Pending photo action cancelled because its library or photo changed": "Pending photo action cancelled because its library or photo changed", + "ONNX model": "ONNX model", + "commercial use subject to GPL conditions": "commercial use subject to GPL conditions" } diff --git a/crates/ui-egui/src/headless.rs b/crates/ui-egui/src/headless.rs index 77aa77b3..d1fae1e1 100644 --- a/crates/ui-egui/src/headless.rs +++ b/crates/ui-egui/src/headless.rs @@ -268,6 +268,7 @@ impl Headless { || self.app.scan.is_some() || self.app.import.is_some() || self.app.export.is_some() + || self.app.session.denoise_busy() || !self.app.tasks.is_empty() || !self.app.synthetic.is_empty() || !self.events.is_empty() @@ -365,6 +366,62 @@ impl Headless { #[cfg(test)] mod tests { + #[test] + fn detail_offers_ai_denoise_for_raw_photos_and_leads_to_its_settings() { + use crate::state::Dialog; + use lightcraft_catalog::{MediaKind, Op, Photo, PhotoId, Source}; + let mut h = demo([1400.0, 900.0]); + let t = Duration::from_secs(10); + let dir = std::env::temp_dir().join(format!("lc-ui-denoise-{}", std::process::id())); + let _ = std::fs::remove_dir_all(&dir); + std::fs::create_dir_all(&dir).unwrap(); + h.app.session.set_denoise_models_dir(Some(dir.join("denoise-models"))); + // a raw photo (its file need not exist: nothing here reads it) + let id = PhotoId(h.app.session.catalog.photos().map(|p| p.id.0).max().unwrap_or(0) + 1); + let mut raw = Photo::new( + id, + Source::File { path: dir.join("IMG_1.dng").to_string_lossy().into_owned() }, + "IMG_1.dng", + "DNG", + 6000, + 4000, + "2026-10-01T00:00:00", + ); + raw.kind = MediaKind::Raw; + h.app.session.commit("setup", Op::AddPhoto { photo: Box::new(raw) }).unwrap(); + let widgets = |h: &mut Headless| h.request("ui.widgets", json!({}), t).to_string(); + h.request("engine.execute", json!({"command": "library.select", "params": {"ids": [id.0]}}), t); + h.request("ui.set", json!({"view": "detail"}), t); + h.app.ui.open_sections = vec!["detail".to_string()]; + h.settle(SETTLE); + h.step(); + h.step(); + let w = widgets(&mut h); + assert!(w.contains("\"denoise:setup\""), "a raw photo with no model is offered the setup: {w}"); + // One click lands in Settings, with local installation available even without a download offer. + let r = h.request("ui.clickWidget", json!({"id": "denoise:setup"}), t); + assert_eq!(r["ok"], true, "{r}"); + h.step(); + h.step(); + assert_eq!(h.app.ui.dialog, Some(Dialog::Settings { tab: "denoise".into() })); + h.settle(SETTLE); + let listed = h.app.session.execute("denoise.models.list", &json!({})).unwrap(); + assert_eq!(listed["models"].as_array().unwrap().len(), lightcraft_denoise::known::all().len(), "{listed}"); + assert!(widgets(&mut h).contains("\"denoise:installFile\"")); + h.app.ui.dialog = None; + // a photo that is not raw: no offer + h.request( + "engine.execute", + json!({"command": "library.select", "params": {"ids": [h.app.session.catalog.photos().find(|p| p.id != id).unwrap().id.0]}}), + t, + ); + h.settle(SETTLE); + h.step(); + h.step(); + assert!(!widgets(&mut h).contains("\"denoise:setup\""), "nothing is offered for a photo denoise does not apply to"); + let _ = std::fs::remove_dir_all(&dir); + } + use super::*; #[test] diff --git a/crates/ui-egui/src/lib.rs b/crates/ui-egui/src/lib.rs index 91cf64f3..7684f00d 100644 --- a/crates/ui-egui/src/lib.rs +++ b/crates/ui-egui/src/lib.rs @@ -19,6 +19,7 @@ pub mod menu_level; pub mod menubar; pub mod menus; pub mod merge; +mod model_setup; pub mod panels; pub mod pick; pub mod region; @@ -106,6 +107,8 @@ pub struct Services { pub picker: Option, /// Show an open dialog for photos; returns paths. pub pick_files: Option, + /// Local ONNX model; a manifest must accompany it. + pub pick_denoise_model: Option, /// Open dialog for a face model (`.onnx`; Settings ▸ Faces ▸ Add a model file…). pub pick_model_file: Option, /// Open dialog for preset files (`.lcpreset`, `.xmp`, `.lrtemplate`, `.zip`, `.dng`, Luminar `.lmp` / `.mplumpack`). @@ -159,6 +162,7 @@ pub struct Perf { } pub struct LightcraftApp { + pub(crate) model_setup: model_setup::Pending, /// Per-catalog-revision caches of library-wide results the panels show every frame /// (expensive on big libraries). pub caches: Caches, @@ -314,6 +318,7 @@ impl LightcraftApp { gpu_applied: None, memory_applied: None, library_problem: None, + model_setup: Default::default(), } } @@ -326,6 +331,9 @@ impl LightcraftApp { /// Run a UI or engine command by id. The single entry point for every frontend path. pub fn run(&mut self, id: &str, params: Value) -> Result { + if let Some(result) = model_setup::intercept(self, id, ¶ms) { + return result; + } if let Some(r) = menus::run_ui_command(self, id, ¶ms) { return r; } @@ -765,6 +773,8 @@ impl LightcraftApp { ctx.request_repaint_after(std::time::Duration::from_secs(3)); } self.session.persist_if_dirty(); + panels::denoise::pump(self, ctx); + model_setup::pump(self, ctx); panels::faces::pump(self, ctx); self.collect_screenshots(ctx); self.issue_screenshots(ctx); @@ -1052,6 +1062,8 @@ pub struct Caches { album_counts: Option<(u64, std::sync::Arc>)>, /// How often the album counts were recomputed (tests check that unchanged frames don't). pub album_count_scans: usize, + /// AI denoise: what the pump last saw, the model list and the downloads being watched. + pub denoise: panels::denoise::Ui, /// Bumped when a face model is installed, removed or chosen, so Settings re-reads the list at once. pub faces_epoch: u64, /// Face model downloads the user started, followed until installed (see `panels::faces::pump`). @@ -1313,3 +1325,6 @@ mod cache_tests { assert_eq!(c.album_count_scans, 1); } } + +#[cfg(test)] +mod tests_model_setup; diff --git a/crates/ui-egui/src/menus.rs b/crates/ui-egui/src/menus.rs index c024a44d..5b79548b 100644 --- a/crates/ui-egui/src/menus.rs +++ b/crates/ui-egui/src/menus.rs @@ -47,6 +47,7 @@ pub fn language_from_command(id: &str) -> Option { } pub const UI_COMMANDS: &[UiCommand] = &[ + ("modelSetup.cancel", "Cancel Pending Photo Action", None, ""), ("view.photoGrid", "Photo Grid", None, "View"), ("view.squareGrid", "Square Grid", None, "View"), // G: Photo Grid ↔ Square Grid (from other views: the photo grid) @@ -464,7 +465,8 @@ pub fn run_ui_command(app: &mut LightcraftApp, id: &str, p: &Value) -> Option { let tab = p.get("tab").and_then(Value::as_str).unwrap_or("general"); if !crate::panels::settings::TABS.iter().any(|(id, _)| *id == tab) { - return Some(Err(format!("unknown settings tab `{tab}` (general|import|performance|interface|faces)"))); + let tabs: Vec<&str> = crate::panels::settings::TABS.iter().map(|(id, _)| *id).collect(); + return Some(Err(format!("unknown settings tab `{tab}` ({})", tabs.join("|")))); } app.ui.dialog = Some(Dialog::Settings { tab: tab.into() }); Ok(Value::Null) diff --git a/crates/ui-egui/src/model_setup.rs b/crates/ui-egui/src/model_setup.rs new file mode 100644 index 00000000..e7094954 --- /dev/null +++ b/crates/ui-egui/src/model_setup.rs @@ -0,0 +1,134 @@ +//! Resume a denoise request after model setup, bound to its original photo and library. + +use crate::i18n::tr; +use crate::{LightcraftApp, state::Dialog}; +use lightcraft_catalog::{PhotoId, Source}; +use lightcraft_engine::denoise::PhotoState; +use serde_json::{Value, json}; + +struct Request { + library: u64, + photo: PhotoId, + source: Source, +} + +impl Request { + fn capture(app: &LightcraftApp, id: PhotoId) -> Result { + let photo = app.session.catalog.photo(id).filter(|p| !p.deleted).ok_or("the requested photo is no longer available")?; + Ok(Self { library: app.session.library_generation(), photo: id, source: photo.source.clone() }) + } + + fn valid(&self, app: &LightcraftApp) -> bool { + self.library == app.session.library_generation() + && app.session.catalog.photo(self.photo).is_some_and(|p| !p.deleted && p.source == self.source) + } +} + +#[derive(Default)] +pub(crate) struct Pending { + denoise: Option, + next_check: f64, +} + +pub(crate) fn denoise_requested(app: &LightcraftApp, id: PhotoId) -> bool { + app.model_setup.denoise.as_ref().is_some_and(|r| r.photo == id && r.valid(app)) +} + +pub(crate) fn intercept(app: &mut LightcraftApp, command: &str, params: &Value) -> Option> { + if matches!(command, "app.openLibrary" | "file.restoreLibrary") { + app.model_setup = Pending::default(); + return None; + } + if command == "modelSetup.cancel" { + if params.get("kind").and_then(Value::as_str) != Some("denoise") { + return Some(Err("modelSetup.cancel: use kind denoise".into())); + } + app.model_setup.denoise = None; + app.ui.status = tr("Pending photo action cancelled").into(); + return Some(Ok(json!({"cancelled": true}))); + } + if command == "denoise.models.downloadCancel" { + app.model_setup.denoise = None; + } + if command != "denoise.toggle" { + return None; + } + let id = match params.get("id") { + Some(v) => match v.as_u64() { + Some(id) => PhotoId(id), + None => return Some(Err("denoise.toggle: id must be a photo id".into())), + }, + None => match app.session.active() { + Some(id) => id, + None => return Some(Err("select a raw photo".into())), + }, + }; + let enabled = app.session.develop_of(id).is_some_and(|d| d.enhance.denoise_enabled()); + let on = match params.get("enabled") { + Some(v) => match v.as_bool() { + Some(on) => on, + None => return Some(Err("denoise.toggle: enabled must be true or false".into())), + }, + None => !enabled && !denoise_requested(app, id), + }; + if !on { + app.model_setup.denoise = None; + return Some(app.session.execute("denoise.toggle", &json!({"id": id.0, "enabled": false})).map_err(|e| e.to_string())); + } + let _ = app.session.execute("denoise.status", &json!({"id": id.0})); + if !matches!(app.session.denoise_photo_state(id), PhotoState::NoModel) { + return None; + } + if app.session.execute("denoise.models.list", &json!({})).ok().is_none_or(|v| v["dir"].is_null()) { + return None; + } + Some(Request::capture(app, id).map(|request| { + app.model_setup.denoise = Some(request); + app.model_setup.next_check = 0.0; + app.ui.dialog = Some(Dialog::Settings { tab: "denoise".into() }); + app.ui.status = tr("Install a denoise model to enable AI Denoise on this photo").into(); + json!({"setupRequired": true, "photo": id.0}) + })) +} + +pub(crate) fn notice(app: &mut LightcraftApp, ui: &mut egui::Ui, kind: &str) { + if kind != "denoise" || !app.model_setup.denoise.as_ref().is_some_and(|r| r.valid(app)) { + return; + } + ui.add(egui::Label::new(tr("After installation, AI Denoise will turn on for the requesting photo.")).wrap()); + let r = ui.small_button(tr("Cancel pending action")); + crate::widgets::register(ui.ctx(), "modelSetup:cancel:denoise", r.rect); + if r.clicked() { + let _ = app.run("modelSetup.cancel", json!({"kind": "denoise"})); + } + ui.add_space(6.0); +} + +pub(crate) fn pump(app: &mut LightcraftApp, ctx: &egui::Context) { + if app.model_setup.denoise.is_none() { + return; + } + if app.session.interaction.is_some() { + ctx.request_repaint_after(std::time::Duration::from_millis(250)); + return; + } + let now = ctx.input(|i| i.time); + if now < app.model_setup.next_check { + return; + } + app.model_setup.next_check = now + 0.25; + let Some(request) = app.model_setup.denoise.take() else { return }; + if !request.valid(app) { + app.ui.status = tr("Pending photo action cancelled because its library or photo changed").into(); + return; + } + if matches!(app.session.denoise_photo_state(request.photo), PhotoState::NoModel | PhotoState::NotApplicable) { + app.model_setup.denoise = Some(request); + ctx.request_repaint_after(std::time::Duration::from_millis(250)); + return; + } + match app.run("denoise.toggle", json!({"id": request.photo.0, "enabled": true})) { + Ok(_) => app.ui.status = tr("AI Denoise is enabled on the requesting photo").into(), + Err(error) => app.ui.status = error, + } +} diff --git a/crates/ui-egui/src/panels/denoise.rs b/crates/ui-egui/src/panels/denoise.rs new file mode 100644 index 00000000..bf89acd2 --- /dev/null +++ b/crates/ui-egui/src/panels/denoise.rs @@ -0,0 +1,688 @@ +//! AI denoise in the interface: the line under the Detail panel's Denoise slider (what the picture is doing), the +//! Settings ▸ Denoise tab (the model and the cache) and the per-frame pump that keeps the engine's background work going. +//! +//! The Amount slider is an ordinary develop control (`enhance.denoise`); everything here is about the cleaned picture the +//! slider mixes in. It is cached data made in the background for the photos being looked at, and never a file in the +//! library. A model is fetched only when the user presses Download and accepts its terms. + +use crate::i18n::{tr, tr_format}; +use egui::RichText; +use lightcraft_catalog::PhotoId; +use lightcraft_engine::denoise::PhotoState; +use serde_json::{Value, json}; + +use super::settings::{check, choices, heading, hint}; +use crate::LightcraftApp; +use crate::theme::Tokens; +use crate::widgets::register; + +/// How long a read of the model list is reused (it is a few small files, but not for every frame). +const REFRESH_SECS: f64 = 1.5; +/// Cache limits offered (GB). +const CACHE_LIMITS: [u32; 6] = [5, 10, 20, 50, 100, 250]; + +/// What the interface keeps between frames. +#[derive(Default)] +pub struct Ui { + next_pump: f64, + generation: u64, + active: bool, + /// (tiles done, tiles in all) of the photo being made. + running: Option<(u64, u64)>, + queued: u64, + /// Bumped when a model is installed, removed or chosen, so Settings re-reads the list at once. + pub epoch: u64, + /// Models the user pressed Download for that have not been installed yet. + dl_watch: Vec, + list: Option<(u64, f64, Value)>, +} + +/// Called every frame: keeps the engine's denoise work going and asks for a redraw when a photo's picture appears (the +/// loupe and the thumbnails then render from it) or while one is being made (the progress). +pub fn pump(app: &mut LightcraftApp, ctx: &egui::Context) { + watch_downloads(app, ctx); + let now = ctx.input(|i| i.time); + let (working, moving) = ctx.input(|i| { + let keys = i.events.iter().any(|e| matches!(e, egui::Event::Key { .. } | egui::Event::Text(_) | egui::Event::MouseWheel { .. })); + (i.pointer.any_down() || keys, i.pointer.is_moving()) + }); + if working { + app.caches.last_input = now; + } + if moving || working { + app.caches.last_move = now; + } + if now < app.caches.denoise.next_pump { + return; + } + let watching = matches!(&app.ui.dialog, Some(crate::state::Dialog::Settings { tab }) if tab == "denoise"); + let pace = window_pace(app, ctx, now, watching); + let Ok(v) = app.session.execute("denoise.pump", &json!({"pace": pace})) else { return }; + let c = &mut app.caches.denoise; + let generation = v["generation"].as_u64().unwrap_or(0); + if generation != c.generation { + c.generation = generation; + ctx.request_repaint(); + } + c.active = v["active"] == true; + c.queued = v["queued"].as_u64().unwrap_or(0); + c.running = v["running"].is_object().then(|| (v["running"]["done"].as_u64().unwrap_or(0), v["running"]["total"].as_u64().unwrap_or(0))); + let busy = c.running.is_some() || c.queued > 0; + c.next_pump = now + + if busy { + 0.1 + } else if c.active { + 0.5 + } else { + 1.0 + }; + // frames are drawn only when something asks for one: while a photo is being made, ask again for its progress + if busy { + ctx.request_repaint_after(std::time::Duration::from_millis(100)); + } +} + +/// The engine's model list, re-read at most every [`REFRESH_SECS`], or at once after an action changed it. +fn models(app: &mut LightcraftApp, ctx: &egui::Context) -> Value { + let now = ctx.input(|i| i.time); + let epoch = app.caches.denoise.epoch; + if let Some((e, at, v)) = &app.caches.denoise.list + && *e == epoch + && now - at < REFRESH_SECS + { + return v.clone(); + } + let v = app.run("denoise.models.list", json!({})).unwrap_or(Value::Null); + app.caches.denoise.list = Some((epoch, now, v.clone())); + v +} + +/// Follows the downloads the user started: keeps redrawing while one runs (the progress bar), and says so when a +/// model has been installed and is in use (the engine does all of that by itself). +fn watch_downloads(app: &mut LightcraftApp, ctx: &egui::Context) { + if app.caches.denoise.dl_watch.is_empty() { + return; + } + let Ok(v) = app.session.execute("denoise.models.downloads", &json!({})) else { + app.caches.denoise.dl_watch.clear(); + return; + }; + let rows: Vec = v["downloads"].as_array().cloned().unwrap_or_default(); + let (mut keep, mut installed) = (Vec::new(), Vec::new()); + for id in std::mem::take(&mut app.caches.denoise.dl_watch) { + match rows.iter().find(|r| r["id"] == id.as_str()).and_then(|r| r["state"].as_str()) { + Some("running" | "done" | "installing") => keep.push(id), + Some("installed") => installed.push(id), + _ => {} + } + } + app.caches.denoise.dl_watch = keep; + for id in installed { + let _ = app.session.execute("denoise.models.downloadCancel", &json!({"id": id})); + app.caches.denoise.epoch += 1; + let text = tr("The denoise model is installed and in use: the AI Denoise Amount slider under Detail now works"); + app.ui.status = text.into(); + app.toast(ctx, text); + } + ctx.request_repaint_after(std::time::Duration::from_millis(120)); +} + +/// Per-photo AI Denoise switch, including a request waiting for model installation. +pub fn detail_toggle(app: &mut LightcraftApp, ui: &mut egui::Ui, id: PhotoId, on: bool, applicable: bool) { + let mut enabled = on || crate::model_setup::denoise_requested(app, id); + egui::Frame::NONE.inner_margin(egui::Margin { left: 24, right: 22, top: 6, bottom: 2 }).show(ui, |ui| { + let r = ui.add_enabled(applicable, egui::Checkbox::new(&mut enabled, crate::i18n::tr("AI Denoise"))); + register(ui.ctx(), "denoise:enabled", r.rect); + if r.changed() { + let _ = app.run("denoise.toggle", json!({"id": id.0, "enabled": enabled})); + } + }); +} + +/// The line under the Denoise slider for the photo `id`, whose Amount is `amount`. +pub fn detail_status(app: &mut LightcraftApp, ui: &mut egui::Ui, id: PhotoId, amount: f64) { + let t = Tokens::get(ui.ctx()); + let state = app.session.denoise_photo_state(id); + let pad = egui::Margin { left: 24, right: 22, top: 0, bottom: 6 }; + let note = |ui: &mut egui::Ui, text: &str, color: egui::Color32| { + ui.add(egui::Label::new(RichText::new(crate::i18n::tr(text)).size(11.0).color(color)).wrap()); + }; + match state { + PhotoState::NotApplicable | PhotoState::Ready => {} + PhotoState::NoModel => { + egui::Frame::NONE.inner_margin(pad).show(ui, |ui| { + note(ui, "AI Denoise needs a model. Install one in Settings.", t.text_dim); + ui.add_space(2.0); + let r = ui.small_button(crate::i18n::tr("Set up AI Denoise…")); + register(ui.ctx(), "denoise:setup", r.rect); + if r.clicked() { + let _ = app.run("denoise.toggle", json!({"id": id.0, "enabled": true})); + } + }); + } + PhotoState::Idle => { + if amount > 0.0 && app.session.denoise_auto() { + // the pump has not met this photo yet: let it come round soon instead of waiting out its pause + let now = ui.ctx().input(|i| i.time); + app.caches.denoise.next_pump = app.caches.denoise.next_pump.min(now + 0.05); + ui.ctx().request_repaint_after(std::time::Duration::from_millis(60)); + egui::Frame::NONE.inner_margin(pad).show(ui, |ui| note(ui, "Waiting to start…", t.text_dim)); + } else if amount > 0.0 { + egui::Frame::NONE.inner_margin(pad).show(ui, |ui| { + note(ui, "Its denoised picture is not made yet, and pictures are not made on their own (Settings, AI Denoise).", t.text_dim); + ui.add_space(2.0); + let r = ui.small_button(crate::i18n::tr("Make it now")); + register(ui.ctx(), "denoise:make", r.rect); + if r.clicked() { + let _ = app.run("denoise.queue", json!({"ids": [id.0]})); + } + }); + } + } + PhotoState::Queued { ahead } => { + egui::Frame::NONE.inner_margin(pad).show(ui, |ui| { + let text = if ahead == 0 { + tr("Next up for AI Denoise…").to_string() + } else { + tr_format!("In line for AI Denoise ({ahead} ahead)…", ahead = ahead) + }; + note(ui, &text, t.text_dim); + }); + } + PhotoState::Running { done, total } => { + egui::Frame::NONE.inner_margin(pad).show(ui, |ui| { + let frac = if total > 0 { (done as f32 / total as f32).clamp(0.0, 1.0) } else { 0.0 }; + let text = if total > 0 { + tr_format!("Making the denoised picture… {}%", (frac * 100.0).round() as u32) + } else { + tr("Starting…").to_string() + }; + let bar = ui.add(egui::ProgressBar::new(frac).desired_width(ui.available_width().min(240.0)).text(RichText::new(text).size(11.0))); + register(ui.ctx(), "denoise:progress", bar.rect); + }); + } + PhotoState::Failed { why, unsupported } => { + egui::Frame::NONE.inner_margin(pad).show(ui, |ui| { + let color = if unsupported { t.text_dim } else { t.caution }; + note(ui, &sentence(&why), color); + if !unsupported { + let r = ui.small_button(crate::i18n::tr("Try again")); + register(ui.ctx(), "denoise:retry", r.rect); + if r.clicked() { + let _ = app.run("denoise.queue", json!({"ids": [id.0], "retry": true})); + } + } + }); + } + } +} + +/// The Settings ▸ Denoise tab. +pub fn settings_tab(app: &mut LightcraftApp, ui: &mut egui::Ui, t: &Tokens) { + let list = models(app, ui.ctx()); + heading(ui, t, "AI Denoise"); + crate::model_setup::notice(app, ui, "denoise"); + if list["dir"].is_null() { + hint(ui, t, "This build has nowhere to keep denoise models, so AI Denoise is not available. (The desktop app has.)"); + return; + } + hint( + ui, + t, + "Cleans noise out of raw photos with an AI model, before the picture is built: the AI Denoise Amount slider under Detail sets how much of it you see. \ + Your files are never changed and no extra files are made in your library: the cleaned picture is cached data that is made again when needed.", + ); + let runtime = list["runtime"].as_bool() == Some(true); + if !runtime { + hint(ui, t, "This build cannot run denoise models."); + } + let all: Vec = list["models"].as_array().cloned().unwrap_or_default(); + let downloads = app + .session + .execute("denoise.models.downloads", &json!({})) + .map(|v| v["downloads"].as_array().cloned().unwrap_or_default()) + .unwrap_or_default(); + heading(ui, t, "Models"); + let r = ui.add_enabled(runtime && app.services.pick_denoise_model.is_some(), egui::Button::new(tr("Install from file…"))); + register(ui.ctx(), "denoise:installFile", r.rect); + if r.clicked() + && let Some(path) = app.services.pick_denoise_model.as_mut().and_then(|f| f().into_iter().next()) + && let Ok(info) = app.run("denoise.models.inspect", json!({"path": path})) + { + app.ui.dialog = Some(crate::state::Dialog::DenoiseModel { info, accepted: false }); + } + if all.is_empty() { + hint(ui, t, "Install your own ONNX model with denoise-model.json beside it. This build offers no model downloads."); + } + if let Some(job) = downloads.iter().find(|d| d["id"] == "local-install") { + match job["state"].as_str() { + Some("installing") => hint(ui, t, "Checking and installing the model…"), + Some("failed") => hint(ui, t, job["error"].as_str().unwrap_or("Model installation failed")), + _ => {} + } + } + for m in &all { + let dl = downloads.iter().find(|d| d["id"] == m["id"]); + model_row(app, ui, t, m, dl, runtime); + } + if all.iter().all(|m| m["selected"] != true) && all.iter().any(|m| m["installed"] == true) { + hint(ui, t, "AI Denoise is off: press Use on a model."); + } + hint(ui, t, "Works on raw files from cameras with a standard (Bayer) sensor. Other files keep their normal noise reduction."); + work_section(app, ui, t, &list); +} + +/// What the cache holds, its limit, and what is being made. +fn work_section(app: &mut LightcraftApp, ui: &mut egui::Ui, t: &Tokens, list: &Value) { + let status = app.session.execute("denoise.status", &json!({})).unwrap_or(Value::Null); + if status["enabled"] != true { + return; + } + heading(ui, t, "Speed"); + let current = list["runOn"].as_str().unwrap_or("auto").to_string(); + let mut run_on = current.as_str(); + let options = [("auto", "Automatic"), ("gpu", "Graphics card"), ("cpu", "Processor")]; + if ui.horizontal(|ui| choices(ui, "denoiseRunOn", &options, &mut run_on)).inner { + let _ = app.run("denoise.settings", json!({"runOn": run_on})); + app.caches.denoise.epoch += 1; + } + hint( + ui, + t, + "Automatic uses the graphics card where it is faster than the processor: both are timed on this computer the first time a photo is made.", + ); + hint(ui, t, &device_line(&status["device"], run_on)); + if run_on != "cpu" && status["device"]["retry"] == true { + let r = ui.small_button(crate::i18n::tr("Try the graphics card again")); + register(ui.ctx(), "denoise:retryGpu", r.rect); + if r.clicked() { + // choosing where it runs, even the same again, forgets the failed set-up + let _ = app.run("denoise.settings", json!({"runOn": run_on})); + app.caches.denoise.epoch += 1; + } + } + heading(ui, t, "Photos"); + let mut auto = list["auto"].as_bool().unwrap_or(true); + if check(ui, "settings.denoiseAuto", &mut auto, "Make the denoised picture of the photos I am looking at") { + let _ = app.run("denoise.settings", json!({"auto": auto})); + app.caches.denoise.epoch += 1; + } + hint(ui, t, "Only photos whose AI Denoise amount is above 0 are made. Exports always make theirs if it is missing."); + let (running, queued) = (app.caches.denoise.running, app.caches.denoise.queued); + match running { + Some((done, total)) => { + let frac = if total > 0 { (done as f32 / total as f32).clamp(0.0, 1.0) } else { 0.0 }; + let name = status["running"]["photo"] + .as_u64() + .and_then(|p| app.session.catalog.photo(PhotoId(p)).map(|p| p.file_name.clone())) + .unwrap_or_default(); + let text = tr_format!( + "Denoising {name}: {done} of {total} tiles{}", + if queued > 0 { tr_format!(" · {queued} waiting", queued = queued) } else { String::new() }, + name = name, + done = done, + total = total + ); + let bar = ui.add(egui::ProgressBar::new(frac).desired_width(380.0).text(RichText::new(text).size(11.5))); + register(ui.ctx(), "denoise:settingsProgress", bar.rect); + } + None if queued > 0 => hint(ui, t, &count(queued, "photo waiting", "photos waiting")), + None => { + let ready = status["ready"].as_u64().unwrap_or(0); + hint(ui, t, &count(ready, "photo has its denoised picture.", "photos have their denoised picture.")) + } + } + ui.horizontal(|ui| { + let r = ui.button(tr("Denoise all photos with an Amount")); + register(ui.ctx(), "denoise:queueAll", r.rect); + if r.clicked() { + let _ = app.run("denoise.queue", json!({"scope": "withAmount"})); + } + if running.is_some() || queued > 0 { + let r = ui.button(tr("Stop")); + register(ui.ctx(), "denoise:stop", r.rect); + if r.clicked() { + let _ = app.run("denoise.cancel", json!({})); + } + } + }); + heading(ui, t, "Cache"); + let (files, bytes) = (status["cache"]["files"].as_u64().unwrap_or(0), status["cache"]["bytes"].as_u64().unwrap_or(0)); + hint( + ui, + t, + &tr_format!( + "{} · {}. Kept in the library's “denoise” folder; the oldest ones that no photo uses go first when the limit is reached.", + count(files, "cached picture", "cached pictures"), + mb(Some(bytes)) + ), + ); + ui.horizontal(|ui| { + ui.label(RichText::new(crate::i18n::tr("Limit")).color(t.text_label)); + let current = list["cacheGb"].as_u64().unwrap_or(20) as u32; + let mut choice = current; + egui::ComboBox::from_id_salt("denoise-cache-limit").selected_text(tr_format!("{size} GB", size = current)).show_ui(ui, |ui| { + for gb in CACHE_LIMITS { + ui.selectable_value(&mut choice, gb, tr_format!("{size} GB", size = gb)); + } + }); + if choice != current { + let _ = app.run("denoise.settings", json!({"cacheGb": choice})); + app.caches.denoise.epoch += 1; + } + let r = ui.button(tr("Clear cache")); + register(ui.ctx(), "denoise:clear", r.rect); + if r.clicked() { + let _ = app.run("denoise.clear", json!({})); + } + }); +} + +/// Where the work runs, in a sentence (`denoise.status` → `device`), for the setting `run_on`. No timings: they are this +/// computer's, and only the comparison matters. +fn device_line(d: &Value, run_on: &str) -> String { + match d["kind"].as_str() { + _ if run_on == "cpu" => tr("The processor does the work.").into(), + Some("gpu") => tr_format!("Running on {}.", d["adapter"].as_str().unwrap_or(tr("the graphics card"))), + Some("cpu") => { + tr_format!("Running on the processor: {}.", tr(d["reason"].as_str().unwrap_or("the graphics card is not used")).trim_end_matches('.')) + } + _ => tr("The graphics card is set up when the next photo is made.").into(), + } +} + +fn model_row(app: &mut LightcraftApp, ui: &mut egui::Ui, t: &Tokens, m: &Value, dl: Option<&Value>, can_run: bool) { + let id = m["id"].as_str().unwrap_or("").to_string(); + let (installed, selected) = (m["installed"].as_bool() == Some(true), m["selected"].as_bool() == Some(true)); + let dl_state = if installed { None } else { dl.and_then(|d| d["state"].as_str()) }; + let host = if can_run { m["downloadHost"].as_str().unwrap_or("").to_string() } else { String::new() }; + egui::Frame::NONE.inner_margin(egui::Margin::symmetric(0, 3)).show(ui, |ui| { + ui.horizontal(|ui| { + ui.vertical(|ui| { + ui.horizontal(|ui| { + ui.label(RichText::new(m["name"].as_str().unwrap_or("")).color(t.text)); + let size = mb(m["sizeBytes"].as_u64()); + if !size.is_empty() { + ui.label(RichText::new(size).color(t.text_dim)); + } + if selected { + ui.label(RichText::new(tr("in use")).color(t.accent)); + } else if installed { + ui.label(RichText::new(tr("installed")).color(t.text_dim)); + } + }); + ui.label(RichText::new(licence_line(m)).font(t.font(11.5)).color(if m["licence"]["commercial"] == "yes" { + t.text_dim + } else { + t.caution + })); + match m["accepted"]["selfTest"].as_object() { + Some(test) if test.get("ok").and_then(Value::as_bool) == Some(true) => { + ui.label(RichText::new(tr("Tested and works on this computer")).font(t.font(11.5)).color(t.text_dim)); + } + Some(_) => { + ui.label(RichText::new(tr("Failed its last test")).font(t.font(11.5)).color(t.caution)); + } + None => {} + } + match (dl_state, dl) { + (Some("running"), Some(d)) => { + let (got, total) = (d["bytes"].as_u64().unwrap_or(0), d["total"].as_u64().unwrap_or(0)); + let frac = if total > 0 { (got as f32 / total as f32).clamp(0.0, 1.0) } else { 0.0 }; + let text = if total > 0 && got >= total { + tr("Checking…").to_string() + } else { + tr_format!("{} of {} from {host}", mb(Some(got)), mb(Some(total)), host = host) + }; + let bar = ui.add(egui::ProgressBar::new(frac).desired_width(260.0).text(RichText::new(text).font(t.font(11.5)))); + register(ui.ctx(), format!("denoise:progress:{id}"), bar.rect); + } + (Some("done" | "installing"), _) => { + ui.label(RichText::new(tr("Downloaded and checked. Installing…")).font(t.font(11.5)).color(t.text_dim)); + } + (Some("failed"), Some(d)) => { + let why = sentence(d["error"].as_str().unwrap_or("The download failed")); + ui.add(egui::Label::new(RichText::new(why).font(t.font(11.5)).color(t.caution)).wrap()); + } + _ => {} + } + }); + ui.with_layout(egui::Layout::right_to_left(egui::Align::Center), |ui| { + if installed { + let r = ui.button(tr("Remove")); + register(ui.ctx(), format!("denoise:remove:{id}"), r.rect); + if r.clicked() { + let _ = app.run("denoise.models.remove", json!({"id": id})); + app.caches.denoise.epoch += 1; + } + if selected { + let r = ui.button(tr("Turn off")); + register(ui.ctx(), format!("denoise:off:{id}"), r.rect); + if r.clicked() { + let _ = app.run("denoise.models.select", json!({"id": null})); + app.caches.denoise.epoch += 1; + } + } else { + let r = ui.button(tr("Use")); + register(ui.ctx(), format!("denoise:use:{id}"), r.rect); + if r.clicked() { + let _ = app.run("denoise.models.select", json!({"id": id})); + app.caches.denoise.epoch += 1; + } + } + } else { + match dl_state { + Some("running") => { + let r = ui.button(tr("Cancel")); + register(ui.ctx(), format!("denoise:cancelDownload:{id}"), r.rect); + if r.clicked() { + let _ = app.run("denoise.models.downloadCancel", json!({"id": id})); + app.caches.denoise.dl_watch.retain(|w| w != &id); + } + } + // the engine installs it within a moment; nothing to press + Some("done" | "installing") => {} + _ => { + if let Some(url) = m["source"].as_str() { + let r = ui + .button(tr("Open page")) + .on_hover_text(tr("Opens the model's own page in your browser, to read about it or get the file yourself.")); + register(ui.ctx(), format!("denoise:get:{id}"), r.rect); + if r.clicked() { + open_page(app, url); + } + } + if !host.is_empty() { + let label = if dl_state == Some("failed") { "Try again" } else { "Download" }; + let tip = tr_format!( + "Shows the model's terms, then downloads {} from {host}, installs it and starts using it. Nothing else is sent.", + mb(m["sizeBytes"].as_u64()), + host = host + ); + let r = ui.button(tr(label)).on_hover_text(tip); + register(ui.ctx(), format!("denoise:download:{id}"), r.rect); + if r.clicked() { + open_download_dialog(app, m); + } + } + } + } + } + }); + }); + }); +} + +/// Pressing Download: the model's terms first, in the licence dialog. Accepting them starts the download; +/// nothing is fetched before. +fn open_download_dialog(app: &mut LightcraftApp, m: &Value) { + let info = json!({ + "kind": "known", + "domain": "denoise", + "fileName": m["id"], + "sizeBytes": m["sizeBytes"], + "model": m, + "alreadyInstalled": false, + "download": m["id"], + "host": m["downloadHost"], + "assumptions": [], + "reason": null, + }); + app.ui.dialog = Some(crate::state::Dialog::DenoiseModel { info, accepted: false }); +} + +/// The dialog's OK for a denoise model: start the download and watch it. +pub fn install(app: &mut LightcraftApp, info: &Value, accepted: bool) -> Result { + if !accepted { + return Err(tr("Tick the box to accept the model's terms first").into()); + } + app.caches.denoise.epoch += 1; + let (id, r) = if let Some(id) = info["download"].as_str() { + (id.to_string(), app.run("denoise.models.download", json!({"id": id, "acknowledged": true}))) + } else { + let path = info["path"].as_str().ok_or_else(|| tr("There is no model file to install").to_string())?; + ("local-install".into(), app.run("denoise.models.install", json!({"path": path, "acknowledged": true, "background": true}))) + }; + if r.is_ok() { + app.caches.denoise.dl_watch.push(id); + app.ui.dialog = Some(crate::state::Dialog::Settings { tab: "denoise".into() }); + } + r +} + +/// "1 photo" / "3 photos": the count with the right noun. +fn count(n: u64, one: &str, many: &str) -> String { + tr_format!("{n} {noun}", n = n, noun = tr(if n == 1 { one } else { many })) +} + +pub(super) fn window_pace(app: &LightcraftApp, ctx: &egui::Context, now: f64, watching: bool) -> &'static str { + let (focused, minimized) = ctx.input(|i| (i.focused, i.raw.viewports.get(&i.raw.viewport_id).and_then(|v| v.minimized).unwrap_or(false))); + pace_for(focused, minimized, now - app.caches.last_input, now - app.caches.last_move, watching) +} + +fn pace_for(focused: bool, minimized: bool, worked: f64, moved: f64, watching: bool) -> &'static str { + if minimized { + "light" + } else if !focused { + // still on screen, but the user is working in another app: no input here to get in the way of + "normal" + } else if worked < 0.4 { + "pause" + } else if moved < 3.0 { + "light" + } else if watching { + "full" + } else { + "normal" + } +} + +pub(super) fn mb(bytes: Option) -> String { + match bytes { + Some(b) if b >= 10_000_000 => tr_format!("{} MB", (b as f64 / 1e6).round() as u64), + Some(b) if b >= 1_000_000 => tr_format!("{:.1} MB", b as f64 / 1e6), + Some(b) => tr_format!("{} KB", (b as f64 / 1e3).round().max(1.0) as u64), + None => String::new(), + } +} + +pub(super) fn sentence(s: &str) -> String { + let s = s.trim(); + let mut c = s.chars(); + let mut out: String = c.next().map(|f| f.to_uppercase().collect()).unwrap_or_default(); + out.push_str(c.as_str()); + if !out.is_empty() && !out.ends_with(['.', '!', '?']) { + out.push('.'); + } + out +} + +pub(super) fn licence_line(m: &Value) -> String { + let name = m["licence"]["name"].as_str().filter(|s| !s.is_empty()).unwrap_or("Unknown licence"); + let terms = match m["licence"]["commercial"].as_str() { + Some("yes") if name.starts_with("GPL") => "commercial use subject to GPL conditions", + Some("yes") => "commercial use allowed", + Some("no") => "non-commercial use only", + _ => "terms unclear", + }; + tr_format!("{name} · {terms}", name = tr(name), terms = tr(terms)) +} + +pub(super) fn open_page(app: &mut LightcraftApp, url: &str) { + if let Some(f) = app.services.open_url.as_mut() { + let _ = f(url); + } +} + +pub fn model_dialog(app: &mut LightcraftApp, ui: &mut egui::Ui, t: &Tokens, info: &Value, accepted: &mut bool) { + let kind = info["kind"].as_str().unwrap_or("unsupported"); + let file = info["fileName"].as_str().unwrap_or(""); + if kind == "unsupported" { + ui.label(RichText::new(tr("This file cannot be used yet")).font(t.semibold(13.5)).color(t.caution)); + ui.add( + egui::Label::new( + RichText::new(sentence(tr(info["reason"].as_str().unwrap_or("It is not a denoise model LightCraft understands")))) + .color(t.text_label), + ) + .wrap(), + ); + ui.label(RichText::new(tr_format!("{file} · {size}", file = file, size = mb(info["sizeBytes"].as_u64()))).color(t.text_dim)); + return; + } + let m = &info["model"]; + ui.horizontal(|ui| { + ui.label(RichText::new(m["name"].as_str().unwrap_or(file)).font(t.semibold(14.0)).color(t.text)); + ui.label(RichText::new(mb(info["sizeBytes"].as_u64())).color(t.text_dim)); + }); + if info["alreadyInstalled"] == true { + ui.label(RichText::new(tr("Already installed. Remove it before installing a replacement.")).color(t.text_dim)); + } + let download = info["download"].is_string(); + if download { + let host = info["host"].as_str().unwrap_or("its own repository"); + let after = if info["domain"] == "denoise" { "starts using it for AI Denoise" } else { "makes AI Denoise available" }; + ui.add( + egui::Label::new( + RichText::new(tr_format!( + "Downloads from {host}. Once it has arrived and checked out, LightCraft installs it, {after}.", + host = host, + after = tr(after) + )) + .color(t.text_label), + ) + .wrap(), + ); + } + let commercial = m["licence"]["commercial"].as_str().unwrap_or("unknown"); + ui.label(RichText::new(licence_line(m)).font(t.semibold(12.5)).color(if commercial == "yes" { t.text } else { t.caution })); + let notice = m["licence"]["notice"].as_str().unwrap_or(""); + if !notice.is_empty() { + ui.add(egui::Label::new(RichText::new(notice).color(t.text_label)).wrap()); + } + let provenance = m["provenance"].as_str().unwrap_or(""); + if !provenance.is_empty() { + ui.add(egui::Label::new(RichText::new(tr_format!("Trained on: {provenance}", provenance = provenance)).color(t.text_dim)).wrap()); + } + if kind == "draft" { + ui.add_space(2.0); + ui.label(RichText::new(tr("LightCraft does not know this model, so it assumed:")).color(t.text_label)); + for a in info["assumptions"].as_array().into_iter().flatten().filter_map(Value::as_str) { + ui.add(egui::Label::new(RichText::new(tr_format!("• {assumption}", assumption = tr(a))).font(t.font(12.0)).color(t.text_dim)).wrap()); + } + } + if let Some(url) = m["source"].as_str().or(m["licence"]["url"].as_str()) { + let r = ui.link(tr("Open the model's page")); + register(ui.ctx(), "denoiseModel:page", r.rect); + if r.clicked() { + open_page(app, url); + } + } + ui.add_space(4.0); + check(ui, "denoiseModel.accept", accepted, "I have read these terms and accept them for my own use"); + let closing = if download { + "Nothing is fetched until you accept. The model is kept on this computer only; LightCraft never uploads or shares it." + } else { + "The model is kept on this computer only. LightCraft never uploads or shares it." + }; + ui.label(RichText::new(tr(closing)).font(t.font(11.5)).color(t.text_dim)); +} diff --git a/crates/ui-egui/src/panels/dialogs.rs b/crates/ui-egui/src/panels/dialogs.rs index 37a60e74..f2bdd668 100644 --- a/crates/ui-egui/src/panels/dialogs.rs +++ b/crates/ui-egui/src/panels/dialogs.rs @@ -86,6 +86,7 @@ pub fn show(app: &mut LightcraftApp, ctx: &egui::Context) { Dialog::NewSmartAlbum { .. } => "Create Smart Album", Dialog::AllMetadata { .. } => "All Metadata", Dialog::SystemInfo { .. } => "System Info", + Dialog::DenoiseModel { .. } => "AI Denoise Model", Dialog::FaceModel { info, .. } if info["download"].is_string() => "Download Face Model", Dialog::FaceModel { .. } => "Add Face Model", Dialog::WhatsNew => "What's New", @@ -178,6 +179,7 @@ pub fn show(app: &mut LightcraftApp, ctx: &egui::Context) { } }); } + Dialog::DenoiseModel { info, accepted } => crate::panels::denoise::model_dialog(app, ui, &t, info, accepted), Dialog::FaceModel { info, accepted, .. } => crate::panels::faces::model_dialog(app, ui, &t, info, accepted), Dialog::SystemInfo { rows } => { egui::Grid::new("sysinfo").num_columns(2).spacing([16.0, 4.0]).striped(true).show(ui, |ui| { @@ -790,7 +792,7 @@ pub fn show(app: &mut LightcraftApp, ctx: &egui::Context) { ui.add_space(4.0); ui.horizontal(|ui| { // a model file LightCraft cannot use has nothing to confirm - let unusable_model = matches!(&dlg, Dialog::FaceModel { info, .. } if info["kind"] == "unsupported"); + let unusable_model = matches!(&dlg, Dialog::DenoiseModel { info, .. } | Dialog::FaceModel { info, .. } if info["kind"] == "unsupported"); let informational = unusable_model || matches!(dlg, Dialog::About | Dialog::Shortcuts | Dialog::Settings { .. }); let sam = &app.session.segmenter; let (sam_installed, sam_running, sam_failed) = (sam.installed(), sam.download_status().running, sam.download_status().error.is_some()); @@ -817,6 +819,8 @@ pub fn show(app: &mut LightcraftApp, ctx: &egui::Context) { add_label.as_str() } Dialog::Merge { .. } => "Merge", + Dialog::DenoiseModel { info, .. } if info["download"].is_string() => "Accept & Download", + Dialog::DenoiseModel { .. } => "Install", Dialog::FaceModel { info, .. } if !informational && info["download"].is_string() => "Download", Dialog::FaceModel { .. } if !informational => "Install", Dialog::ConfirmDelete { .. } => "Delete", @@ -829,7 +833,7 @@ pub fn show(app: &mut LightcraftApp, ctx: &egui::Context) { _ => "OK", }; // installing waits for the licence to be accepted - let can_confirm = informational || !matches!(&dlg, Dialog::FaceModel { accepted: false, .. }); + let can_confirm = informational || !matches!(&dlg, Dialog::DenoiseModel { accepted: false, .. } | Dialog::FaceModel { accepted: false, .. }); let r = (!ok.is_empty()).then(|| ui.add_enabled(can_confirm, egui::Button::new(crate::i18n::tr(ok)))); if let Some(r) = &r { crate::widgets::register(ui.ctx(), "button:dialogOk", r.rect); @@ -1060,6 +1064,7 @@ pub fn confirm_dialog(app: &mut LightcraftApp, dlg: &Dialog) -> Result app.run("keyword.rename", json!({"from": from, "to": to})), Dialog::MergeKeywords { from, into } => app.run("keyword.merge", json!({"from": from, "into": into})), Dialog::AutoStack { gap } => app.run("stack.auto", json!({"gap": gap})), + Dialog::DenoiseModel { info, accepted } => crate::panels::denoise::install(app, info, *accepted), Dialog::FaceModel { path, info, accepted } => crate::panels::faces::install(app, path, info, *accepted), Dialog::AllMetadata { .. } | Dialog::SystemInfo { .. } | Dialog::WhatsNew => Ok(serde_json::Value::Null), Dialog::Cull { reject_below, pick_best } => { diff --git a/crates/ui-egui/src/panels/edit.rs b/crates/ui-egui/src/panels/edit.rs index 2188afa6..fb8d121e 100644 --- a/crates/ui-egui/src/panels/edit.rs +++ b/crates/ui-egui/src/panels/edit.rs @@ -272,6 +272,14 @@ pub fn show(app: &mut LightcraftApp, ui: &mut egui::Ui, id: PhotoId) { for c in ["detail.sharpenAmount", "detail.sharpenRadius", "detail.sharpenDetail", "detail.sharpenMasking"] { control(app, ui, d, c, c == "detail.sharpenAmount" || d.detail.sharpen_amount > 0.0); } + // AI Denoise is its own tool, apart from the manual noise reduction sliders below it + sub_title(ui, crate::i18n::tr("AI Denoise")); + // AI Denoise: how much of the photo's cleaned picture to mix in (raw files only), and what that picture is doing + let applicable = !matches!(app.session.denoise_photo_state(id), lightcraft_engine::denoise::PhotoState::NotApplicable); + let on = d.enhance.denoise_enabled(); + super::denoise::detail_toggle(app, ui, id, on, applicable); + control(app, ui, d, "enhance.denoise", applicable && on); + super::denoise::detail_status(app, ui, id, if on { d.enhance.denoise } else { 0.0 }); sub_title(ui, crate::i18n::tr("Noise Reduction")); for c in ["detail.nrLuminance", "detail.nrDetail", "detail.nrContrast", "detail.nrColor", "detail.nrColorDetail", "detail.nrColorSmoothness"] { diff --git a/crates/ui-egui/src/panels/mod.rs b/crates/ui-egui/src/panels/mod.rs index e95f8c7b..abb01551 100644 --- a/crates/ui-egui/src/panels/mod.rs +++ b/crates/ui-egui/src/panels/mod.rs @@ -4,6 +4,7 @@ pub mod bottombar; pub mod chips; pub mod compare; pub mod crop_overlay; +pub mod denoise; pub mod detail; pub mod dialogs; pub mod edit; diff --git a/crates/ui-egui/src/panels/settings.rs b/crates/ui-egui/src/panels/settings.rs index 822d2127..2774a962 100644 --- a/crates/ui-egui/src/panels/settings.rs +++ b/crates/ui-egui/src/panels/settings.rs @@ -1,4 +1,4 @@ -//! The Settings dialog (⌘,): General, Import, Performance, Interface, Faces. +//! The Settings dialog (⌘,): General, Import, Performance, Interface, Faces, AI Denoise. //! //! Changes apply immediately (no OK/Cancel). Where they are stored: //! - **app settings** ([`crate::state::AppSettings`]: startup view, delete confirmation, GPU, @@ -17,8 +17,14 @@ use crate::theme::Tokens; use crate::widgets::register; /// (id, label) of the tabs, in order. -pub const TABS: &[(&str, &str)] = - &[("general", "General"), ("import", "Import"), ("performance", "Performance"), ("interface", "Interface"), ("faces", "Faces")]; +pub const TABS: &[(&str, &str)] = &[ + ("general", "General"), + ("import", "Import"), + ("performance", "Performance"), + ("interface", "Interface"), + ("faces", "Faces"), + ("denoise", "AI Denoise"), +]; /// Thumbnail cache sizes offered (MB). const CACHE_SIZES: [u32; 5] = [512, 1024, 2048, 4096, 8192]; @@ -45,6 +51,7 @@ pub fn body(app: &mut LightcraftApp, ui: &mut egui::Ui, tab: &mut String) { "performance" => performance_tab(app, ui, &t), "interface" => interface_tab(app, ui, &t), "faces" => super::faces::settings_tab(app, ui, &t), + "denoise" => super::denoise::settings_tab(app, ui, &t), _ => general_tab(app, ui, &t), } } @@ -77,7 +84,7 @@ pub(super) fn check(ui: &mut egui::Ui, id: &str, value: &mut bool, label: &str) } /// Mutually exclusive buttons (`button:{id}-{index}`). -fn choices(ui: &mut egui::Ui, id: &str, options: &[(V, &str)], value: &mut V) -> bool { +pub(super) fn choices(ui: &mut egui::Ui, id: &str, options: &[(V, &str)], value: &mut V) -> bool { let mut changed = false; ui.spacing_mut().item_spacing.x = 4.0; for (i, (v, l)) in options.iter().enumerate() { diff --git a/crates/ui-egui/src/state.rs b/crates/ui-egui/src/state.rs index f8e6c330..83b3df29 100644 --- a/crates/ui-egui/src/state.rs +++ b/crates/ui-egui/src/state.rs @@ -581,6 +581,10 @@ pub enum Dialog { pick_best: bool, }, /// Help ▸ System Info: (label, value) rows. + DenoiseModel { + info: serde_json::Value, + accepted: bool, + }, /// A face model file the user chose: what it is, its licence terms, and the "I accept" box. FaceModel { path: String, diff --git a/crates/ui-egui/src/tests_model_setup.rs b/crates/ui-egui/src/tests_model_setup.rs new file mode 100644 index 00000000..85a7ca7f --- /dev/null +++ b/crates/ui-egui/src/tests_model_setup.rs @@ -0,0 +1,202 @@ +//! Synthetic setup flows: model installation must resume the requested action on its original targets. +use std::path::{Path, PathBuf}; + +use lightcraft_catalog::{MediaKind, Op, Photo, PhotoId, Source}; +use lightcraft_denoise::manifest::{DenoiserManifest, Domain, Gain}; +use lightcraft_engine::Session; +use serde_json::json; + +use crate::{LightcraftApp, Services, model_setup, state::Dialog}; + +fn setup(name: &str) -> (LightcraftApp, PathBuf, PhotoId) { + let dir = std::env::temp_dir().join(format!("lc-model-flow-{name}-{}", std::process::id())); + let _ = std::fs::remove_dir_all(&dir); + let mut s = Session::new().with_fs(); + s.open_library(dir.join("library"), true).unwrap(); + s.set_denoise_models_dir(Some(dir.join("denoise"))); + s.execute("denoise.settings", &json!({"runOn": "cpu"})).unwrap(); + let id = PhotoId(s.catalog.photos().map(|p| p.id.0).max().unwrap() + 1); + let mut raw = Photo::new( + id, + Source::File { path: dir.join("synthetic.dng").to_string_lossy().into_owned() }, + "synthetic.dng", + "DNG", + 128, + 128, + "2026-10-07", + ); + raw.kind = MediaKind::Raw; + s.commit("fixture", Op::AddPhoto { photo: Box::new(raw) }).unwrap(); + (LightcraftApp::new(s, Services::default()), dir, id) +} + +fn denoise_file(dir: &Path) -> PathBuf { + let source = dir.join("source"); + std::fs::create_dir_all(&source).unwrap(); + let bytes = lightcraft_denoise::synthetic::smoothing_onnx(); + std::fs::write(source.join("model.onnx"), &bytes).unwrap(); + let m = DenoiserManifest { + id: "test-smoothing".into(), + name: "Synthetic smoothing".into(), + version: "1".into(), + licence: Default::default(), + source: None, + sha256: None, + size_bytes: Some(bytes.len() as u64), + provenance: "Synthetic tests".into(), + domain: Domain::BayerToRgb, + tile: 64, + overlap: 16, + gain: Gain::None, + }; + std::fs::write(source.join("denoise-model.json"), serde_json::to_vec(&m).unwrap()).unwrap(); + source.join("model.onnx") +} + +fn install_denoise(app: &mut LightcraftApp, dir: &Path) { + app.run("denoise.models.install", json!({"path": denoise_file(dir), "acknowledged": true})).unwrap(); +} + +#[test] +fn accepting_a_local_model_installs_in_background_and_resumes_the_pending_action() { + let (mut app, dir, raw) = setup("local-install"); + let path = denoise_file(&dir); + app.run("denoise.toggle", json!({"id": raw.0, "enabled": true})).unwrap(); + let info = app.run("denoise.models.inspect", json!({"path": path})).unwrap(); + assert!(crate::panels::denoise::install(&mut app, &info, false).is_err()); + assert!(!dir.join("denoise/test-smoothing/model.onnx").exists()); + let result = crate::panels::denoise::install(&mut app, &info, true).unwrap(); + assert_eq!(result["started"], "local-install"); + assert_eq!(app.ui.dialog, Some(Dialog::Settings { tab: "denoise".into() })); + let ctx = egui::Context::default(); + for _ in 0..600 { + crate::panels::denoise::pump(&mut app, &ctx); + if app.session.denoise_photo_state(raw) != lightcraft_engine::denoise::PhotoState::NoModel { + break; + } + std::thread::sleep(std::time::Duration::from_millis(5)); + } + resume(&mut app); + assert_eq!(amount(&app, raw), 50.0); + assert!(dir.join("denoise/test-smoothing/model.onnx").is_file()); + assert!(path.is_file(), "a user-supplied original is preserved"); + drop(app); + let _ = std::fs::remove_dir_all(dir); +} + +fn amount(app: &LightcraftApp, id: PhotoId) -> f64 { + app.session.develop_of(id).unwrap().enhance.denoise +} +fn resume(app: &mut LightcraftApp) { + model_setup::pump(app, &egui::Context::default()); +} + +#[test] +fn denoise_installation_enables_only_the_requesting_photo_once_and_is_undoable() { + let (mut app, dir, raw) = setup("denoise"); + let other = app.session.active().unwrap(); + app.session.auto_sync = true; + let r = app.run("denoise.toggle", json!({"id": raw.0, "enabled": true})).unwrap(); + assert_eq!(r["setupRequired"], true); + assert_eq!(app.ui.dialog, Some(Dialog::Settings { tab: "denoise".into() })); + assert_eq!(amount(&app, raw), 0.0, "setup alone never changes the photo"); + app.run("library.select", json!({"ids": [other.0]})).unwrap(); + let undo = app.session.undo.len(); + install_denoise(&mut app, &dir); + resume(&mut app); + assert_eq!(amount(&app, raw), 50.0); + assert_eq!(amount(&app, other), 0.0); + assert_eq!(app.session.active(), Some(other)); + assert_eq!(app.session.undo.len(), undo + 1); + resume(&mut app); + assert_eq!(app.session.undo.len(), undo + 1, "a later frame cannot apply it again"); + app.run("edit.undo", json!({})).unwrap(); + assert_eq!(amount(&app, raw), 0.0); + app.run("denoise.toggle", json!({"id": raw.0, "enabled": true})).unwrap(); + app.run("develop.set", json!({"ids": [raw.0], "control": "enhance.denoise", "value": 75})).unwrap(); + app.run("denoise.toggle", json!({"id": raw.0, "enabled": true})).unwrap(); + assert_eq!(amount(&app, raw), 75.0, "enabling an already chosen amount keeps it"); + app.run("denoise.toggle", json!({"id": raw.0, "enabled": false})).unwrap(); + assert_eq!(amount(&app, raw), 75.0, "switching off keeps the Amount"); + assert_eq!(app.session.develop_of(raw).unwrap().denoise_amount(), 0.0, "but mixes nothing in"); + drop(app); + let _ = std::fs::remove_dir_all(dir); +} + +#[test] +fn cancelled_denoise_requests_do_not_apply_after_installation() { + for (name, command, params) in [ + ("cancel-action", "modelSetup.cancel", json!({"kind": "denoise"})), + ("cancel-toggle", "denoise.toggle", json!({"enabled": false})), + ("cancel-download", "denoise.models.downloadCancel", json!({"id": "rawnind-bayer"})), + ] { + let (mut app, dir, raw) = setup(name); + app.run("library.select", json!({"ids": [raw.0]})).unwrap(); + app.run("denoise.toggle", json!({"enabled": true})).unwrap(); + app.run(command, params).unwrap(); + install_denoise(&mut app, &dir); + resume(&mut app); + assert_eq!(amount(&app, raw), 0.0, "{name}"); + drop(app); + let _ = std::fs::remove_dir_all(dir); + } +} + +#[test] +fn pending_actions_do_not_cross_libraries_or_relinked_sources() { + for changed in ["library", "source"] { + let (mut app, dir, raw) = setup(changed); + app.run("denoise.toggle", json!({"id": raw.0, "enabled": true})).unwrap(); + if changed == "library" { + app.session.open_library(dir.join("other-library"), true).unwrap(); + } else { + app.session + .commit( + "relink", + Op::Relink { + id: raw, + file_name: "another.dng".into(), + source: Source::File { path: dir.join("another.dng").to_string_lossy().into_owned() }, + format: Some("DNG".into()), + }, + ) + .unwrap(); + } + install_denoise(&mut app, &dir); + resume(&mut app); + assert!(!model_setup::denoise_requested(&app, raw)); + assert!(app.session.catalog.photos().all(|p| p.develop.enhance.denoise == 0.0)); + assert!(app.ui.status.contains("cancelled")); + drop(app); + let _ = std::fs::remove_dir_all(dir); + } +} + +#[test] +fn hostile_or_inapplicable_toggle_arguments_are_errors_without_changes() { + let (mut app, dir, raw) = setup("hostile"); + let jpeg = app.session.active().unwrap(); + for params in [json!({"id": "wrong"}), json!({"id": u64::MAX}), json!({"id": raw.0, "enabled": 1}), json!({"id": jpeg.0, "enabled": true})] { + assert!(app.run("denoise.toggle", params).is_err()); + assert_eq!(amount(&app, raw), 0.0); + } + assert!(app.run("modelSetup.cancel", json!({"kind": "nonsense"})).is_err()); + drop(app); + let _ = std::fs::remove_dir_all(dir); +} + +#[test] +fn model_installation_waits_for_an_active_slider_gesture_to_finish() { + let (mut app, dir, raw) = setup("interaction"); + app.run("denoise.toggle", json!({"id": raw.0, "enabled": true})).unwrap(); + app.session.begin_interaction("Exposure").unwrap(); + install_denoise(&mut app, &dir); + resume(&mut app); + assert_eq!(amount(&app, raw), 0.0); + assert!(app.session.interaction.is_some()); + app.session.end_interaction().unwrap(); + resume(&mut app); + assert_eq!(amount(&app, raw), 50.0); + drop(app); + let _ = std::fs::remove_dir_all(dir); +} diff --git a/docs/denoise.md b/docs/denoise.md new file mode 100644 index 00000000..f1196e9b --- /dev/null +++ b/docs/denoise.md @@ -0,0 +1,131 @@ +# AI Denoise + +Edit ▸ Detail ▸ **AI Denoise** is a per-photo switch above **Amount** (0–100). +Turning it on starts at 50, or preserves an already positive Amount. Turning it off +preserves Amount and mixes none of the cleaned picture. It affects only the requesting +photo, independently of Auto Sync, and is undoable. Manual luminance and colour noise +reduction remain separate controls. + +If no model is selected, turning it on opens **Settings ▸ AI Denoise** without changing +the photo. After installation, the pending request enables the original photo even if +the selection changed. Cancel the pending action in Settings. Cancelled downloads, +deleted or relinked photos, switching libraries and quitting cancel the request. +An active slider gesture finishes before the pending action resumes. + +## Models and terms + +**No model weights are bundled.** Models live in the per-user `denoise-models` folder +(override with `LIGHTCRAFT_DENOISE_MODELS`). Settings provides **Install from file…** +for a user-supplied ONNX file with a `denoise-model.json` beside it. The default build +offers no model downloads. The separate `rawnind-model` Cargo feature enables the +pinned RawNIND offer only when a release's model policy permits it. +Terms must be acknowledged before installation or +download. Downloads use the shared pure-Rust `lightcraft-fetch` client, with resume, +progress, cancellation, size limits and SHA-256 verification. + +RawNIND's optional ONNX weights are **GPL-3.0**, as recorded in the pinned model catalog. +They are downloaded separately after consent and read by our own Rust executor. +Whether releases should offer this download remains a maintainer policy decision; +permissively licensed or independently trained weights remain a gap. This is not a +claim that optional downloading settles licensing questions. No GPL decoder or neural +runtime code is included. Model metadata is in `crates/denoise-core/src/known.rs`. + +Installation checks the manifest, model hash and size, then runs a synthetic self-test: +finite output of the promised shape, repeatability, the declared scale and reduced +noise. Unsupported operators, malformed files and a failed self-test return errors. +The UI performs copying, extraction, hashing, parsing and CPU/GPU self-tests on one +background installation worker. `denoise.models.downloads` and `denoise.pump` report +`installing`, `installed` or `failed`; an existing model is preserved on a failed replacement. +Command clients can request the same asynchronous path with `background: true`. +The synchronous command remains available for scripts that require the result immediately. + +The current contract is `bayerToRgb`: normalized, black-subtracted, non-white-balanced +RGGB planes `[1, 4, tile, tile]` produce camera RGB `[1, 3, 2*tile, 2*tile]`. +RawNIND uses a 512-cell tile, 64-cell overlap and a mean-matched nominal gain of 1e6. +See `DenoiserManifest` for the validated JSON schema. + +## Photos, previews and exports + +Original RAW files stay untouched. A cleaned picture is cached under +`/denoise/.lcdn`, keyed by source content, model and algorithm version. +The compressed half-float cache has checked headers, bounded windows and checksums. +It can be cleared or recreated; it is not a new catalog photo or a duplicate DNG. + +Plain and cleaned camera RGB go through the same highlight reconstruction, white +balance, colour matrix, DNG profile tables, camera hue/saturation look, crop and +orientation. Amount blends the two before the remaining develop pipeline, so edits +and exports use the same data. At zero Amount or with Detail disabled, the ordinary +render path remains in use. + +The queue prioritizes the open and selected photos, pauses after input, and adjusts +its worker count while the user works. Progress, cancellation and retry are available +in Settings and through commands. The default cache limit is 20 GB; unused products +are evicted oldest first. A failed file is not continuously retried. + +An export with a nonzero effective Amount creates a missing product synchronously. +It returns a clear error if it cannot produce the denoised result; it never silently +exports the plain image instead. Unsupported sensor layouts retain manual noise +reduction and report their limitation. + +## Pure-Rust CPU and GPU execution + +The CPU executor reads a bounded subset of ONNX into checked network data. Convolutions +use bounded im2col blocks and `faer-core 0.17.1` with its pure-Rust GEMM backend, +without BLAS, C dependencies or an inner thread pool. That backend includes +architecture-specific assembly in its dependencies. Tiles share the outer worker +pool. A scalar implementation checks the optimized operators on synthetic networks. +Tract is not a workspace dependency or a product feature. +CPU inference and the GPU model module are feature-gated; ordinary engine/web builds +do not compile the faer/GEMM backend. The engine depends on the inference crate only through `denoise = ["dep:lightcraft-denoise", …]`; +lightweight `lightcraft-denoise-core` metadata and cache formats remain available without inference. Pictures are capped at 100 megapixels; at most +four tile calls run together, chosen within the process-wide working-memory budget. +One idle CPU workspace is retained, and cache costs reserve room for the blended picture. + +The GPU executor uses our WGSL kernels through wgpu. It checks a real model's output +against the CPU and falls back on errors, unsupported networks or a slower device. +Setup has a time limit and a crash marker. Automatic selection estimates throughput +from a check tile; this estimate can choose imperfectly on unmeasured hardware. +Settings offers Automatic, Graphics card and Processor. GPU tests cover supported +synthetic networks against the scalar executor; a real-model test compares CPU/GPU. + +### Historical performance, measured 2026-10-07 + +These are the contributor's measurements **before the current upstream integration**, +on a Ryzen 9 7945HX and RTX 4090 Laptop GPU (DX12), release build, model SHA-256 +`da27509dab6a2915da67e988acd86cf71f9d5bbc8d1aa0ed32933578a887b901`. +The 24.34 MP Sony a7 III photo is 35 tiles. Numbers exclude model loading, source +decode, GPU setup, cache encoding and export, but include packing and blending. + +| Work | Pure-Rust CPU | Prior Tract reference | GPU | +|---|---:|---:|---:| +| One warm 512-cell tile, one caller | 1148 ms | 1463 ms | 15.1 ms | +| Whole photo, 16 CPU / 4 GPU feeding workers | 5.185 s | 9.534 s | 0.582 s | + +The largest relative differences were 1.68e-6 CPU/reference and 1.57e-6 CPU/GPU for +the check tile, below the required 1e-3 tolerance. Reference comparisons were made +on the older contributor branch; that reference runtime is intentionally excluded +from the current workspace. These numbers are not a fresh benchmark of this rebase. + +Fresh checks can be run using an already installed model and CC0 raw samples: + +```text +LC_DENOISE_MODEL= cargo test --release -p lightcraft-gpu --lib real_model_on_the_gpu_matches_cpu -- --ignored --nocapture +LC_DENOISE_MODEL= LC_DENOISE_RAW= cargo test --release -p lightcraft-engine --features rawnind-model --test denoise_cpu -- --ignored --nocapture +LC_DENOISE_MODEL= LC_DENOISE_RAW= cargo test --release -p lightcraft-engine --features rawnind-model --test denoise_real -- --ignored --nocapture +``` + +## Commands and limits + +UI, CLI, control channel and MCP dispatch `denoise.toggle`, `denoise.settings`, +`denoise.queue`, `.cancel`, `.status`, `.pump`, `.clear`, and `denoise.models.*` +(`list`, `install`, `download`, `downloads`, `downloadCancel`, `test`, `remove`, +`select`). The develop control is `enhance.denoise`. + +Native apps enable inference through their `denoise` feature. A build without it and +the web build report inference unavailable. Manifest/cache primitives still compile +for WASM. Settings and models remain opt-in; nothing is downloaded on launch. + +This implementation supports **Bayer RAW only**. X-Trans, Foveon, demosaiced DNG and +non-RAW images remain unsupported by this model contract. No measured quality parity +with Lightroom is claimed. Apple/Metal and wider hardware verification, model policy, +a linear-RGB contract and a side-by-side fidelity suite remain future work. diff --git a/docs/parity.md b/docs/parity.md index 066bd45a..c06dad1b 100644 --- a/docs/parity.md +++ b/docs/parity.md @@ -26,7 +26,7 @@ and prints the summary; `cargo xtask parity --write` refreshes the summary table | C. Views & navigation (VIEW) | 15 | 1 | 1 | 0 | 8/9 (89%) | 4/4 (100%) | | D. Search & filter (FILT) | 11 | 1 | 1 | 0 | 4/4 (100%) | 4/4 (100%) | | E. Metadata (META) | 5 | 1 | 0 | 0 | 2/2 (100%) | 2/2 (100%) | -| F. Edit panel — global adjustments (EDIT) | 42 | 1 | 5 | 1 | 28/28 (100%) | 13/14 (93%) | +| F. Edit panel — global adjustments (EDIT) | 42 | 2 | 4 | 1 | 28/28 (100%) | 13/14 (93%) | | G. Profiles (PROF) | 7 | 1 | 3 | 0 | 3/4 (75%) | 3/3 (100%) | | H. Crop & rotate (CROP) | 9 | 0 | 0 | 1 | 6/6 (100%) | 3/3 (100%) | | I. Remove / healing (REM) | 7 | 1 | 2 | 2 | 4/4 (100%) | 2/3 (67%) | @@ -48,9 +48,9 @@ and prints the summary; `cargo xtask parity --write` refreshes the summary table | Y. Menus | 82 | 0 | 4 | 8 | 47/47 (100%) | 23/23 (100%) | | Z. Keyboard shortcuts (desktop) | 75 | 3 | 3 | 1 | 49/52 (94%) | 23/24 (96%) | | Lightroom Classic extras | 32 | 11 | 39 | 9 | — | 22/24 (92%) | -| **Total** | 394 | 37 | 85 | 36 | 193/200 (97%) | 142/152 (93%) | +| **Total** | 394 | 38 | 84 | 36 | 193/200 (97%) | 142/152 (93%) | -Weighted completion (✅ = 1, 🟡 = ½, 🚫 left out): **79.9%** of 516 in-scope rows — P0 98.2% of 200 · P1 95.7% of 152 · P2 42.6% of 163. +Weighted completion (✅ = 1, 🟡 = ½, 🚫 left out): **80.0%** of 516 in-scope rows — P0 98.2% of 200 · P1 95.7% of 152 · P2 42.9% of 163. ## Top gaps @@ -267,7 +267,7 @@ under the UI command that wraps it (a no-op error); the UI command now wins. | LR-EDIT-DETAIL-SHARPEN | Sharpening | P0 | ✅ | `ctl:detail.sharpenAmount`, `ctl:detail.sharpenRadius`, `ctl:detail.sharpenDetail`, `ctl:detail.sharpenMasking` | no Alt-drag mask preview | | LR-EDIT-DETAIL-NR | Luminance noise reduction | P0 | ✅ | `ctl:detail.nrLuminance`, `ctl:detail.nrDetail`, `ctl:detail.nrContrast` | | | LR-EDIT-DETAIL-CNR | Colour noise reduction | P0 | ✅ | `ctl:detail.nrColor`, `ctl:detail.nrColorDetail`, `ctl:detail.nrColorSmoothness` | | -| LR-EDIT-DETAIL-DENOISE | AI denoise | P2 | ⬜ | | settings field reserved, not rendered | +| LR-EDIT-DETAIL-DENOISE | AI denoise | P2 | 🟡 | cmd:denoise.toggle, cmd:denoise.models.install, cmd:denoise.queue, ctl:enhance.denoise, `crates/denoise/`, `docs/denoise.md` | non-destructive Bayer RAW, opt-in weights, pure-Rust CPU/GPU; X-Trans/linear RGB, model policy and Lightroom fidelity remain | | LR-EDIT-DETAIL-RAWDETAILS | Improved demosaic toggle | P2 | ⬜ | | | | LR-EDIT-DETAIL-SUPERRES | Super resolution | P2 | ⬜ | | | | LR-EDIT-DETAIL-AISHARPEN | AI sharpen | OOS | 🚫 | | | diff --git a/docs/whats-new.md b/docs/whats-new.md index 133770dc..ccb0be80 100644 --- a/docs/whats-new.md +++ b/docs/whats-new.md @@ -15,6 +15,10 @@ - The catalog format is now version 3 (`Album.order`, `Op::SetAlbumOrder`). Once this version has opened a library, older versions refuse it, as with every format change. +### AI RAW denoise +- Detail has a per-photo AI Denoise switch and Amount. Bayer RAW photos use a disposable cache, with matching previews and exports and pure-Rust CPU/GPU inference. +- Models are installed separately after accepting their terms; no weights are bundled. See [setup and limitations](denoise.md). + ### Library keyboard culling - Colour labels tint thumbnail surrounds in Square Grid and the Detail filmstrip, and a translucent footer along the bottom of the photo in Photo Grid. Label confirmations use a matching pale colour. - Setting or clearing a colour label shows a brief bottom confirmation, like rating a photo; custom label names appear in the message too. diff --git a/xtask/src/layers.rs b/xtask/src/layers.rs index b6eab0f0..051b3aed 100644 --- a/xtask/src/layers.rs +++ b/xtask/src/layers.rs @@ -39,6 +39,8 @@ pub const TABLE: &[(&str, Class)] = &[ ("tiff", Class::Layer(0)), ("sysmem", Class::Layer(0)), ("fetch", Class::Layer(0)), + ("denoise-core", Class::Layer(1)), + ("denoise", Class::Layer(1)), ("heif", Class::Layer(0)), ("raw", Class::Layer(1)), ("codecs", Class::Layer(1)), @@ -70,6 +72,7 @@ pub const TABLE: &[(&str, Class)] = &[ pub const INTRA_LAYER_ORDER: &[&[&str]] = &[ &["geom", "color", "raster"], &["tiff", "raster"], + &["denoise-core", "denoise"], &["meta", "raw"], &["meta", "codecs"], &["meta", "develop"],