shifathosn
/
lightcraft
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge origin/main into #447 (render cache version 22, as main is at 21; parity row keeps main's CR3 AspectInfo note) 

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Brandon Thomas 2026-10-09 09:39:22 -04:00
parent 82f904495f 65ad290e89
commit b7e2f4eff4
134 changed files with 16228 additions and 421 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

331
Cargo.lock generated
View File

@ -209,7 +209,7 @@ version = "0.6.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "dc890384c8602f339876ded803c97ad529f3842aba97f6392b3dba0dd171769b"
dependencies = [
"equator",
"equator 0.4.2",
]
[[package]]
@ -967,6 +967,12 @@ dependencies = [
"unicode-width",
]
[[package]]
name = "coe-rs"
version = "0.1.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7e8f1e641542c07631228b1e0dc04b69ae3c1d58ef65d5691a439711d805c698"
[[package]]
name = "color"
version = "0.3.3"
@ -1256,6 +1262,16 @@ version = "0.1.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d8b14ccef22fc6f5a8f4d7d768562a182c04ce9a3b3157b91390b52ddfdf1a76"
[[package]]
name = "dyn-stack"
version = "0.10.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "56e53799688f5632f364f8fb387488dd05db9fe45db7011be066fc20e7027f8b"
dependencies = [
"bytemuck",
"reborrow",
]
[[package]]
name = "dyn-stack"
version = "0.13.2"
@ -1537,13 +1553,33 @@ version = "0.36.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "773fa9c96dd0dbef887e39d0ed6177f141cce4d68e6041df77570aa3702dfa13"
[[package]]
name = "equator"
version = "0.1.10"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a3b0a88aa91d0ad2b9684e4479aed31a17d3f9051bdbbc634bd2c01bc5a5eee8"
dependencies = [
"equator-macro 0.1.9",
]
[[package]]
name = "equator"
version = "0.4.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4711b213838dfee0117e3be6ac926007d7f433d7bbe33595975d4190cb07e6fc"
dependencies = [
"equator-macro",
"equator-macro 0.4.2",
]
[[package]]
name = "equator-macro"
version = "0.1.9"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "60d08acb9849f7fb4401564f251be5a526829183a3645a90197dea8e786cf3ae"
dependencies = [
"proc-macro2",
"quote",
"syn 2.0.119",
]
[[package]]
@ -1608,6 +1644,42 @@ dependencies = [
"pin-project-lite",
]
[[package]]
name = "faer-core"
version = "0.17.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "32da0f6262445261106407bdfb548df4f8f1c89bcc3cff1bb6d08e7f525b575a"
dependencies = [
"bytemuck",
"coe-rs",
"dyn-stack 0.10.0",
"equator 0.1.10",
"faer-entity",
"gemm 0.17.1",
"matrixcompare-core",
"num-complex",
"num-traits",
"paste",
"pulp 0.18.22",
"reborrow",
"seq-macro",
]
[[package]]
name = "faer-entity"
version = "0.17.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "93fb12f36da3162b3fefc0c7343d905d5bd2c2bf444a7f63ceeaa2bc003311d3"
dependencies = [
"bytemuck",
"coe-rs",
"libm",
"num-complex",
"num-traits",
"pulp 0.18.22",
"reborrow",
]
[[package]]
name = "fastrand"
version = "2.5.0"
@ -1783,13 +1855,33 @@ dependencies = [
"slab",
]
[[package]]
name = "gemm"
version = "0.17.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6ab24cc62135b40090e31a76a9b2766a501979f3070fa27f689c27ec04377d32"
dependencies = [
"dyn-stack 0.10.0",
"gemm-c32 0.17.1",
"gemm-c64 0.17.1",
"gemm-common 0.17.1",
"gemm-f16 0.17.1",
"gemm-f32 0.17.1",
"gemm-f64 0.17.1",
"num-complex",
"num-traits",
"paste",
"raw-cpuid 10.7.0",
"seq-macro",
]
[[package]]
name = "gemm"
version = "0.18.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ab96b703d31950f1aeddded248bc95543c9efc7ac9c4a21fda8703a83ee35451"
dependencies = [
"dyn-stack",
"dyn-stack 0.13.2",
"gemm-c32 0.18.2",
"gemm-c64 0.18.2",
"gemm-common 0.18.2",
@ -1799,7 +1891,7 @@ dependencies = [
"num-complex",
"num-traits",
"paste",
"raw-cpuid",
"raw-cpuid 11.6.0",
"seq-macro",
]
@ -1809,7 +1901,7 @@ version = "0.19.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "aa0673db364b12263d103b68337a68fbecc541d6f6b61ba72fe438654709eacb"
dependencies = [
"dyn-stack",
"dyn-stack 0.13.2",
"gemm-c32 0.19.0",
"gemm-c64 0.19.0",
"gemm-common 0.19.0",
@ -1819,7 +1911,22 @@ dependencies = [
"num-complex",
"num-traits",
"paste",
"raw-cpuid",
"raw-cpuid 11.6.0",
"seq-macro",
]
[[package]]
name = "gemm-c32"
version = "0.17.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b9c030d0b983d1e34a546b86e08f600c11696fde16199f971cd46c12e67512c0"
dependencies = [
"dyn-stack 0.10.0",
"gemm-common 0.17.1",
"num-complex",
"num-traits",
"paste",
"raw-cpuid 10.7.0",
"seq-macro",
]
@ -1829,12 +1936,12 @@ version = "0.18.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f6db9fd9f40421d00eea9dd0770045a5603b8d684654816637732463f4073847"
dependencies = [
"dyn-stack",
"dyn-stack 0.13.2",
"gemm-common 0.18.2",
"num-complex",
"num-traits",
"paste",
"raw-cpuid",
"raw-cpuid 11.6.0",
"seq-macro",
]
@ -1844,12 +1951,27 @@ version = "0.19.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "086936dbdcb99e37aad81d320f98f670e53c1e55a98bee70573e83f95beb128c"
dependencies = [
"dyn-stack",
"dyn-stack 0.13.2",
"gemm-common 0.19.0",
"num-complex",
"num-traits",
"paste",
"raw-cpuid",
"raw-cpuid 11.6.0",
"seq-macro",
]
[[package]]
name = "gemm-c64"
version = "0.17.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "fbb5f2e79fefb9693d18e1066a557b4546cd334b226beadc68b11a8f9431852a"
dependencies = [
"dyn-stack 0.10.0",
"gemm-common 0.17.1",
"num-complex",
"num-traits",
"paste",
"raw-cpuid 10.7.0",
"seq-macro",
]
@ -1859,12 +1981,12 @@ version = "0.18.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "dfcad8a3d35a43758330b635d02edad980c1e143dc2f21e6fd25f9e4eada8edf"
dependencies = [
"dyn-stack",
"dyn-stack 0.13.2",
"gemm-common 0.18.2",
"num-complex",
"num-traits",
"paste",
"raw-cpuid",
"raw-cpuid 11.6.0",
"seq-macro",
]
@ -1874,15 +1996,34 @@ version = "0.19.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "20c8aeeeec425959bda4d9827664029ba1501a90a0d1e6228e48bef741db3a3f"
dependencies = [
"dyn-stack",
"dyn-stack 0.13.2",
"gemm-common 0.19.0",
"num-complex",
"num-traits",
"paste",
"raw-cpuid",
"raw-cpuid 11.6.0",
"seq-macro",
]
[[package]]
name = "gemm-common"
version = "0.17.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a2e7ea062c987abcd8db95db917b4ffb4ecdfd0668471d8dc54734fdff2354e8"
dependencies = [
"bytemuck",
"dyn-stack 0.10.0",
"half",
"num-complex",
"num-traits",
"once_cell",
"paste",
"pulp 0.18.22",
"raw-cpuid 10.7.0",
"seq-macro",
"sysctl 0.5.5",
]
[[package]]
name = "gemm-common"
version = "0.18.2"
@ -1890,7 +2031,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a352d4a69cbe938b9e2a9cb7a3a63b7e72f9349174a2752a558a8a563510d0f3"
dependencies = [
"bytemuck",
"dyn-stack",
"dyn-stack 0.13.2",
"half",
"libm",
"num-complex",
@ -1898,10 +2039,10 @@ dependencies = [
"once_cell",
"paste",
"pulp 0.21.5",
"raw-cpuid",
"raw-cpuid 11.6.0",
"rayon",
"seq-macro",
"sysctl",
"sysctl 0.6.0",
]
[[package]]
@ -1911,7 +2052,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "88027625910cc9b1085aaaa1c4bc46bb3a36aad323452b33c25b5e4e7c8e2a3e"
dependencies = [
"bytemuck",
"dyn-stack",
"dyn-stack 0.13.2",
"half",
"libm",
"num-complex",
@ -1919,10 +2060,27 @@ dependencies = [
"once_cell",
"paste",
"pulp 0.22.3",
"raw-cpuid",
"raw-cpuid 11.6.0",
"rayon",
"seq-macro",
"sysctl",
"sysctl 0.6.0",
]
[[package]]
name = "gemm-f16"
version = "0.17.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7ca4c06b9b11952071d317604acb332e924e817bd891bec8dfb494168c7cedd4"
dependencies = [
"dyn-stack 0.10.0",
"gemm-common 0.17.1",
"gemm-f32 0.17.1",
"half",
"num-complex",
"num-traits",
"paste",
"raw-cpuid 10.7.0",
"seq-macro",
]
[[package]]
@ -1931,14 +2089,14 @@ version = "0.18.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "cff95ae3259432f3c3410eaa919033cd03791d81cebd18018393dc147952e109"
dependencies = [
"dyn-stack",
"dyn-stack 0.13.2",
"gemm-common 0.18.2",
"gemm-f32 0.18.2",
"half",
"num-complex",
"num-traits",
"paste",
"raw-cpuid",
"raw-cpuid 11.6.0",
"rayon",
"seq-macro",
]
@ -1949,30 +2107,45 @@ version = "0.19.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e3df7a55202e6cd6739d82ae3399c8e0c7e1402859b30e4cb780e61525d9486e"
dependencies = [
"dyn-stack",
"dyn-stack 0.13.2",
"gemm-common 0.19.0",
"gemm-f32 0.19.0",
"half",
"num-complex",
"num-traits",
"paste",
"raw-cpuid",
"raw-cpuid 11.6.0",
"rayon",
"seq-macro",
]
[[package]]
name = "gemm-f32"
version = "0.17.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e9a69f51aaefbd9cf12d18faf273d3e982d9d711f60775645ed5c8047b4ae113"
dependencies = [
"dyn-stack 0.10.0",
"gemm-common 0.17.1",
"num-complex",
"num-traits",
"paste",
"raw-cpuid 10.7.0",
"seq-macro",
]
[[package]]
name = "gemm-f32"
version = "0.18.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "bc8d3d4385393304f407392f754cd2dc4b315d05063f62cf09f47b58de276864"
dependencies = [
"dyn-stack",
"dyn-stack 0.13.2",
"gemm-common 0.18.2",
"num-complex",
"num-traits",
"paste",
"raw-cpuid",
"raw-cpuid 11.6.0",
"seq-macro",
]
@ -1982,12 +2155,27 @@ version = "0.19.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "02e0b8c9da1fbec6e3e3ab2ce6bc259ef18eb5f6f0d3e4edf54b75f9fd41a81c"
dependencies = [
"dyn-stack",
"dyn-stack 0.13.2",
"gemm-common 0.19.0",
"num-complex",
"num-traits",
"paste",
"raw-cpuid",
"raw-cpuid 11.6.0",
"seq-macro",
]
[[package]]
name = "gemm-f64"
version = "0.17.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "aa397a48544fadf0b81ec8741e5c0fba0043008113f71f2034def1935645d2b0"
dependencies = [
"dyn-stack 0.10.0",
"gemm-common 0.17.1",
"num-complex",
"num-traits",
"paste",
"raw-cpuid 10.7.0",
"seq-macro",
]
@ -1997,12 +2185,12 @@ version = "0.18.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "35b2a4f76ce4b8b16eadc11ccf2e083252d8237c1b589558a49b0183545015bd"
dependencies = [
"dyn-stack",
"dyn-stack 0.13.2",
"gemm-common 0.18.2",
"num-complex",
"num-traits",
"paste",
"raw-cpuid",
"raw-cpuid 11.6.0",
"seq-macro",
]
@ -2012,12 +2200,12 @@ version = "0.19.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "056131e8f2a521bfab322f804ccd652520c79700d81209e9d9275bbdecaadc6a"
dependencies = [
"dyn-stack",
"dyn-stack 0.13.2",
"gemm-common 0.19.0",
"num-complex",
"num-traits",
"paste",
"raw-cpuid",
"raw-cpuid 11.6.0",
"seq-macro",
]
@ -2869,6 +3057,38 @@ dependencies = [
"serde",
]
[[package]]
name = "lightcraft-denoise"
version = "0.4.0"
dependencies = [
"faer-core",
"lightcraft-denoise-core",
"lightcraft-raster",
"miniz_oxide 0.8.9",
"proptest",
"rayon",
"serde",
"serde_json",
"sha2",
"thiserror 2.0.21",
"web-time",
]
[[package]]
name = "lightcraft-denoise-core"
version = "0.4.0"
dependencies = [
"lightcraft-raster",
"miniz_oxide 0.8.9",
"proptest",
"rayon",
"serde",
"serde_json",
"sha2",
"thiserror 2.0.21",
"web-time",
]
[[package]]
name = "lightcraft-develop"
version = "0.4.0"
@ -2891,6 +3111,8 @@ dependencies = [
"lightcraft-catalog",
"lightcraft-codecs",
"lightcraft-color",
"lightcraft-denoise",
"lightcraft-denoise-core",
"lightcraft-develop",
"lightcraft-faces",
"lightcraft-fetch",
@ -2954,6 +3176,7 @@ version = "0.4.0"
dependencies = [
"bytemuck",
"lightcraft-color",
"lightcraft-denoise",
"lightcraft-develop",
"lightcraft-geom",
"lightcraft-pipeline",
@ -3118,6 +3341,7 @@ dependencies = [
"lightcraft-catalog",
"lightcraft-codecs",
"lightcraft-color",
"lightcraft-denoise",
"lightcraft-develop",
"lightcraft-engine",
"lightcraft-faces",
@ -3211,6 +3435,12 @@ dependencies = [
"libc",
]
[[package]]
name = "matrixcompare-core"
version = "0.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b0bdabb30db18805d5290b3da7ceaccbddba795620b86c02145d688e04900a73"
[[package]]
name = "matrixmultiply"
version = "0.3.11"
@ -4353,6 +4583,18 @@ dependencies = [
"unarray",
]
[[package]]
name = "pulp"
version = "0.18.22"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a0a01a0dc67cf4558d279f0c25b0962bd08fc6dec0137699eae304103e882fe6"
dependencies = [
"bytemuck",
"libm",
"num-complex",
"reborrow",
]
[[package]]
name = "pulp"
version = "0.21.5"
@ -4379,7 +4621,7 @@ dependencies = [
"num-complex",
"paste",
"pulp-wasm-simd-flag",
"raw-cpuid",
"raw-cpuid 11.6.0",
"reborrow",
"version_check",
]
@ -4580,6 +4822,15 @@ dependencies = [
"rgb",
]
[[package]]
name = "raw-cpuid"
version = "10.7.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6c297679cb867470fa8c9f67dbba74a78d78e3e98d7cf2b08d6d71540f797332"
dependencies = [
"bitflags 1.3.2",
]
[[package]]
name = "raw-cpuid"
version = "11.6.0"
@ -5313,6 +5564,20 @@ dependencies = [
"syn 3.0.6",
]
[[package]]
name = "sysctl"
version = "0.5.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ec7dddc5f0fee506baf8b9fdb989e242f17e4b11c61dfbb0635b705217199eea"
dependencies = [
"bitflags 2.13.2",
"byteorder",
"enum-as-inner",
"libc",
"thiserror 1.0.69",
"walkdir",
]
[[package]]
name = "sysctl"
version = "0.6.0"

2
Cargo.toml
View File

@ -11,6 +11,8 @@ repository = "https://github.com/storytold/lightcraft"
[workspace.dependencies]
# internal
lightcraft-denoise-core = { path = "crates/denoise-core" }
lightcraft-denoise = { path = "crates/denoise", default-features = false }
lightcraft-geom = { path = "crates/geom" }
lightcraft-faces = { path = "crates/faces" }
lightcraft-sysmem = { path = "crates/sysmem" }

9
NOTICE
View File

@ -27,6 +27,15 @@ the facebook/sam3 checkpoint only when they choose to (in the app, or by hand),
License (https://github.com/facebookresearch/sam3/blob/main/LICENSE), a separate, non-OSI
licence. See docs/ai-masks.md.
AI denoise weights are not part of LightCraft and are never distributed with it.
The default build offers user-supplied models only. The optional rawnind-model
feature offers a separately downloaded RawNIND UtNet2 Bayer checkpoint under
GPL-3.0 (https://www.gnu.org/licenses/gpl-3.0.html), after the user accepts its terms.
Commercial use is subject to those terms, including applicable source, licence and
notice obligations on redistribution. Enabling that offer in releases remains a
maintainer decision. The ONNX reader and CPU/GPU executor are our Rust implementation;
no darktable, RawSpeed, LibRaw or other GPL decoder/runtime code is included.
Builds made with craft-fonts (the optional CRAFT_FONTS_DIR build input; all official releases)
also embed the fonts listed in https://github.com/storytold/craft-fonts/blob/main/ATTRIBUTION.md
(the Japanese fonts BIZ UDPGothic, BIZ UDMincho and Shippori Mincho; the web build embeds only BIZ

10
ROADMAP.md
View File

@ -25,7 +25,7 @@ duplicate Local entries #22, black GPU exports on an Intel iGPU #78).
| **Feature checklist** | 79.7% | P0 core and P1 nearly complete: import (Add / Copy / Move, templates, devices), library, grid/loupe/compare/survey, every Edit slider, curves, colour grading, masking tools, crop/Upright, heal/clone, presets/profiles, versions/history, sync, export, menus, shortcuts | P0: full-resolution zoom detail; P1: lens-profile database, content-aware fill (patch synthesis), video playback/trim |
| **RAW coverage** (formats people shoot) | ~55% | DNG (all kinds), CR2, CR3 lossless Bayer and version 0x100/0x200 C-RAW (M50/R100/R8 pixel-exact corpus), ARW, NEF (uncompressed + Huffman lossless/lossy), RAF (uncompressed, lossless/lossy compressed Bayer and X-Trans), uncompressed ORF, RW2 / RWL / Panasonic RAW (every raw format, checked on 178 files from 118 Panasonic and Leica bodies), PEF; every container's embedded preview (incl. CR3) | **CR3** (every Canon since ~2018), compressed ORF, Nikon lossy-after-split, Canon sRAW, HEIC/AVIF. NEFs labelled compressed but stored uncompressed (Z 6 packed 14-bit, D850 12-bit uncompressed). Per-model verification is thin (~55 corpus files vs >1,000 models) |
| **Colour & image quality** | ~55–65% | Pipeline is complete and fast; GPU path CPU-exact within 1/255 | **No measured camera calibration database**: ARW, NEF, RW2, RAF, CR3, CR2 and PEF have guarded camera-JPEG colour estimates, with bundled ILCE-7M4, X-H2S and X-T4 profiles (docs/camera-preview-colour.md); other non-DNG raws and rejected estimates use a neutral matrix. Colour fidelity remains incomplete. No lens-profile database. No measured fidelity against Lightroom (tone, highlights, texture/clarity, NR, sharpening are tuned by eye) |
| **AI & computational** | ~15–20% | Assisted culling (focus, bursts), auto tone, HDR/panorama merge; subject/sky/background masks as classical heuristics, or the photo's own segmentation mattes when it ships them (DNG semantic masks, e.g. iPhone ProRAW's sky matte); **faces (MVP, [docs/faces.md](docs/faces.md))**: a pure-Rust detector (a 232 KB opt-in download), opt-in recognition models (one-click download or add a file; they run on the shared pure-Rust CPU interpreter; pinned SFace/AuraFace embeddings agree with an independent reference runtime on 40 public crops; one-thread medians 34–36 / 233–252 ms (AuraFace remains expensive)), a background scan that finds and embeds faces, name suggestions, a person's page with look-alike faces to confirm, an Unnamed faces section (look-alikes together, select a group and name it at once) | Real segmentation masks (subject, sky, people, objects, landscape, depth), AI denoise, super resolution, lens blur, generative remove, natural-language search; for faces: clusters with separators between groups, small faces in large photos (the detector looks at 640 px), thresholds calibrated on live faces, writing face regions back to XMP. **Blocked on a model strategy** (licensable weights or our own training; pure-Rust inference is feasible) |
| **AI & computational** | ~15–20% | Assisted culling (focus, bursts), auto tone, HDR/panorama merge; subject/sky/background masks as classical heuristics, or the photo's own segmentation mattes when it ships them (DNG semantic masks, e.g. iPhone ProRAW's sky matte); non-destructive Bayer RAW denoise with opt-in weights on pure-Rust CPU/GPU; **faces (MVP, [docs/faces.md](docs/faces.md))**: a pure-Rust detector (a 232 KB opt-in download), opt-in recognition models (one-click download or add a file; they run on the shared pure-Rust CPU interpreter; pinned SFace/AuraFace embeddings agree with an independent reference runtime on 40 public crops; one-thread medians 34–36 / 233–252 ms (AuraFace remains expensive)), a background scan that finds and embeds faces, name suggestions, a person's page with look-alike faces to confirm, an Unnamed faces section (look-alikes together, select a group and name it at once) | Real segmentation masks (subject, sky, people, objects, landscape, depth), denoise model policy / linear RGB / quality verification, super resolution, lens blur, generative remove, natural-language search; for faces: clusters with separators between groups, small faces in large photos (the detector looks at 640 px), thresholds calibrated on live faces, writing face regions back to XMP. **Blocked on a model strategy** (licensable weights or our own training; pure-Rust inference is feasible) |
| **Workflow & library** | ~85% (single machine) | Robust catalog (journal + snapshots, background compaction, crash-tested), 85k-photo libraries stay responsive, Local browsing with automatic cleanup, XMP interop, keywords, smart albums, Move import | Opening an 85k library takes 1.7–4.7 s; no cloud sync (out of scope), no tablet companion (#74, roadmap), shared albums, publish services, tethering |
| **Classic modules** | ~30% | Geotagging from GPX track logs, soft proofing (partial), slideshow (basic) | **Map view, Book, Print, Slideshow module, Web, publish services**: ~40 tracker rows ⬜ |
| **HDR & video** | 0% | | HDR edit/display/export; video play/trim/edit/export |
@ -40,7 +40,7 @@ duplicate Local entries #22, black GPU exports on an Intel iGPU #78).
| Fujifilm raw shooter | ~65% | Uncompressed and lossless/lossy compressed RAF decoded; camera colour calibration remains missing. Verified on 13 bodies, including 14/16-bit GFX; see `docs/raf-compression.md` |
| Canon CR3 / Olympus compressed-raw shooter | ~45% | CR3 lossless Bayer and version 0x100/0x200 C-RAW develop from sensor data (M50, R100, R8 verified pixel for pixel); other CRX variants and compressed ORF open as embedded JPEG previews. Camera colour and broader model verification remain |
| Lightroom Classic power user | ~45% | Print, Book, Map, publish, tethering |
| Relies on AI (masks, denoise) | ~25% | Object / Describe masks via optional SAM 3 (download not yet hosted); no AI denoise |
| Relies on AI (masks, denoise) | ~25% | Object / Describe masks via optional SAM 3 (download not yet hosted); Bayer AI denoise needs separately installed weights; X-Trans/linear RGB, model policy and quality parity remain open |
## Where we're going
@ -59,9 +59,9 @@ Priorities, in order. Each points at tracker rows in [`docs/parity.md`](docs/par
(references stay in the local `plan/`), then tune against the numbers.
5. **Lens profiles of our own** (LR-EDIT-OPTICS-PROFILE, P1).
6. **AI model strategy** (maintainer decision): which permissively licensed models (or our own training) for
segmentation masks and denoise; then pure-Rust inference. Unblocks M12 and Enhance. Face detection and recognition run on our own checked Rust engine. Object / Describe masks now
segmentation masks and denoise. Bayer denoise inference is implemented in pure Rust with an opt-in GPL model; release policy and independently licensed weights remain open. Unblocks M12 and Enhance. Face detection and recognition run on our own checked Rust engine. Object / Describe masks now
run Meta's SAM 3 in pure Rust (optional, user-consented download under the non-OSI SAM License; CDN mirrors still
to be set up, docs/ai-masks.md); Subject / Sky / People and Enhance are still open.
to be set up, docs/ai-masks.md); Subject / Sky / People, Raw Details and Super Resolution are still open.
7. **Then:** HDR (Q), the Classic output modules (Print first, then Map view, Book, Slideshow), video (R), localisation
and accessibility.
@ -86,7 +86,7 @@ yet guarantee original-pixel detail; visible-region rendering remains a gap (LR-
| M4 | Crop, geometry, optics | crop tool + overlays, straighten, Upright (auto/level/vertical/full/guided), manual transforms, CA, defringe, manual lens corrections | 6–10 | ✅ |
| M5 | Performance | source pyramids, wgpu compute pipeline (CPU oracle), draft/full renders, prefetch, budgets (16 ms slider updates on 24 MP) | 10–15 | 🚧 (stage cache, source pyramid, wgpu pipeline, prefetch, memory budget ✅; colour NR at half resolution, GPU histogram ⬜) |
| M6 | Masking | brush, linear/radial gradients, colour/luminance/depth range, add/subtract/intersect/invert, all local adjustments, masks panel | 8–12 | 🚧 (brush/linear/radial/colour/luminance range, add/subtract/intersect, masks panel ✅; depth range, AI masks ⬜) |
| M7 | Detail | sharpening + masking preview, luminance/colour NR, Denoise, Raw Details, Super Resolution | 6–10 | 🚧 (sharpening, luminance/colour NR ✅; AI Denoise, Raw Details, Super Resolution ⬜) |
| M7 | Detail | sharpening + masking preview, luminance/colour NR, Denoise, Raw Details, Super Resolution | 6–10 | 🚧 (sharpening, luminance/colour NR ✅; AI Denoise 🟡 (Bayer, optional weights; policy/fidelity open); Raw Details, Super Resolution ⬜) |
| M8 | Heal / Remove | content-aware remove (PatchMatch), heal, clone, brush spots, visualize spots, red/pet eye | 6–10 | 🚧 (heal, clone, auto source, visualize spots, red/pet eye ✅; PatchMatch remove ⬜) |
| M9 | Presets, profiles, versions, sync | preset browser + amount, create/import presets, profile browser, versions, history, copy/paste/sync settings | 5–8 | ✅ |
| M10 | Export & share | export dialog (JPEG/PNG/TIFF/DNG/AVIF/JXL/original), sizing, sharpening, metadata, watermark, naming, batch jobs, XMP sidecars, HDR export | 6–10 | 🚧 (all formats incl. DNG/original, sizing, presets, background jobs ✅; JXL encode, HDR export ⬜) |

4
apps/lightcraft-cli/Cargo.toml
View File

@ -25,7 +25,9 @@ serde_json = { workspace = true }
dhat = { workspace = true, optional = true }
[features]
default = []
default = ["denoise"]
denoise = ["lightcraft-engine/denoise"]
rawnind-model = ["denoise", "lightcraft-engine/rawnind-model"]
# Heap profiling: `cargo run --release -p lightcraft-cli --features dhat-heap -- snapshot …` counts
# every allocation (live/peak bytes in `library.memory` / `ui.inspect`) and writes dhat-heap.json
# (open it in DHAT's viewer: allocation sites by bytes at the peak). Slows allocation-heavy work.

10
apps/lightcraft-cli/src/main.rs
View File

@ -391,7 +391,7 @@ fn merge(args: &[String]) -> Result<(), String> {
}
i += 1;
}
let mut s = Session::new().with_fs();
let mut s = Session::new().with_fs().with_default_denoise_models();
let paths = expand_paths(&files);
let r = s.execute("library.import", &json!({"paths": paths})).map_err(|e| e.to_string())?;
let mut ids: Vec<u64> = r["imported"].as_array().map(|a| a.iter().filter_map(Value::as_u64).collect()).unwrap_or_default();
@ -652,7 +652,7 @@ fn render(args: &[String]) -> Result<(), String> {
}
let input = input.ok_or("render: missing input file")?;
let output = output.ok_or("render: missing -o OUTPUT")?;
let mut s = Session::new().with_fs();
let mut s = Session::new().with_fs().with_default_denoise_models();
let abs = expand_paths(std::slice::from_ref(&input));
let r = s.execute("library.import", &json!({"paths": abs})).map_err(|e| e.to_string())?;
let id = r["imported"][0].as_u64().ok_or_else(|| format!("{input}: not a readable photo"))?;
@ -732,12 +732,12 @@ fn snapshot(args: &[String]) -> Result<(), String> {
let t0 = Instant::now();
let mut session = match &library {
Some(dir) => {
let mut s = Session::new().with_fs().with_default_face_models();
let mut s = Session::new().with_fs().with_default_denoise_models().with_default_face_models();
s.open_library(dir, false).map_err(|e| library_error(dir, e))?;
s
}
None if files.is_empty() => Session::with_demo().with_fs().with_default_face_models(),
None => Session::new().with_fs().with_default_face_models(),
None if files.is_empty() => Session::with_demo().with_fs().with_default_denoise_models().with_default_face_models(),
None => Session::new().with_fs().with_default_denoise_models().with_default_face_models(),
};
// Apply compute policy before imports or the first scripted query can discover an adapter.
session.execute("app.gpu", &json!({"enabled": false})).map_err(|e| e.to_string())?;

26
apps/lightcraft-web/src/safety.rs
View File

@ -93,25 +93,41 @@ pub fn install_panic_hook() {
const TAB_LOCK: &str = "lightcraft-library";
/// Hold the library lock for as long as this page lives (Web Locks API). `Some(true)`: ours;
/// `Some(false)`: another tab of this site has it; `None`: the browser can't tell (no Web Locks).
/// `Some(false)`: another tab of this site has it; `None`: the browser can't tell (no Web Locks, or
/// it refused the request).
pub async fn acquire_tab_lock() -> Option<bool> {
let nav = Reflect::get(&js_sys::global(), &"navigator".into()).ok()?;
let locks = Reflect::get(&nav, &"locks".into()).ok().filter(|l| !l.is_undefined() && !l.is_null())?;
let request: js_sys::Function = Reflect::get(&locks, &"request".into()).ok()?.dyn_into().ok()?;
let opts = js_sys::Object::new();
Reflect::set(&opts, &"ifAvailable".into(), &true.into()).ok()?;
let granted = js_sys::Promise::new(&mut |resolve, _reject| {
let granted = js_sys::Promise::new(&mut |resolve, reject| {
let cb = Closure::once_into_js(move |lock: JsValue| -> JsValue {
let ours = !lock.is_null() && !lock.is_undefined();
let _ = resolve.call1(&JsValue::NULL, &ours.into());
// keep the lock until the page goes away: a promise that never settles
if ours { js_sys::Promise::new(&mut |_, _| {}).into() } else { JsValue::NULL }
});
if let Err(e) = request.call3(&locks, &TAB_LOCK.into(), &opts, &cb) {
log::warn!("Web Locks request failed: {e:?}");
// a refused request never calls `cb`: settle with its error instead of waiting forever
match request.call3(&locks, &TAB_LOCK.into(), &opts, &cb) {
Ok(pending) => {
let catch = Reflect::get(&pending, &"catch".into()).ok().and_then(|c| c.dyn_into::<js_sys::Function>().ok());
if let Some(catch) = catch {
let _ = catch.call1(&pending, &reject);
}
}
Err(e) => {
let _ = reject.call1(&JsValue::NULL, &e);
}
}
});
JsFuture::from(granted).await.ok().map(|v| v.is_truthy())
match JsFuture::from(granted).await {
Ok(v) => Some(v.is_truthy()),
Err(e) => {
log::warn!("Web Locks request failed, starting without the one-tab guard: {e:?}");
None
}
}
}
/// Offer a blob as a download named `name`.

1
apps/lightcraft-web/src/web.rs
View File

@ -205,6 +205,7 @@ fn services(originals: Originals, backend: Option<Backend>, files: Files, frozen
open_picker(originals.clone(), backend.clone(), ctx.clone());
Vec::new() // files arrive asynchronously and are imported on a later frame
})),
pick_denoise_model: None,
// Preset files: browser pickers are asynchronous; not wired on the web yet.
pick_preset_files: None,
pick_tracklog: None,

4
apps/lightcraft/Cargo.toml
View File

@ -10,7 +10,9 @@ default-run = "lightcraft"
workspace = true
[features]
default = []
default = ["denoise"]
denoise = ["lightcraft-engine/denoise"]
rawnind-model = ["denoise", "lightcraft-engine/rawnind-model"]
# Open HEIC/HEIF photos (the optional lightcraft-heif crate). Not default so distributors choose
# it (HEVC patents); build with `--features heif` to include it.
heif = ["lightcraft-codecs/heif"]

40
apps/lightcraft/src/main.rs
View File

@ -52,8 +52,15 @@ struct App(LightcraftApp, PrefsWriter, #[cfg(target_os = "macos")] Option<native
impl eframe::App for App {
fn logic(&mut self, ctx: &egui::Context, _frame: &mut eframe::Frame) {
#[cfg(target_os = "macos")]
if let Some(m) = self.2.as_mut() {
m.update(&mut self.0, ctx);
if let Some(m) = self.2.as_mut()
&& m.update(&mut self.0, ctx)
&& lightcraft_ui_egui::panels::notices::may_close(&mut self.0)
{
// Quit from the menu bar: save exactly what closing the window saves (settings and
// library, after the unsaved-changes check; otherwise its prompt shows), then end the
// process instead of closing the viewport while AppKit is terminating (PR #444).
eframe::App::on_exit(self);
std::process::exit(0);
}
self.0.logic(ctx);
self.1.tick(&mut self.0, ctx);
@ -76,11 +83,13 @@ impl eframe::App for App {
/// The window's renderer (egui-wgpu): eframe's defaults, with LightCraft's backend choice — DX12
/// alone on Windows unless `LIGHTCRAFT_GPU_BACKEND` / `WGPU_BACKEND` say otherwise (issue #136:
/// with Vulkan in the set, wgpu loads the Vulkan driver even when it then picks DX12).
/// with Vulkan in the set, wgpu loads the Vulkan driver even when it then picks DX12) — and DX12
/// shaders compiled with FXC, never a stray `dxcompiler.dll` (issue #471).
fn window_wgpu_options() -> eframe::egui_wgpu::WgpuConfiguration {
let mut c = eframe::egui_wgpu::WgpuConfiguration::default();
if let eframe::egui_wgpu::WgpuSetup::CreateNew(n) = &mut c.wgpu_setup {
n.instance_descriptor.backends = lightcraft_engine::gpu::backend::window_backends();
n.instance_descriptor.backend_options = lightcraft_engine::gpu::backend::backend_options();
}
c
}
@ -125,7 +134,7 @@ fn gpu_crash_notice(what: &str) -> String {
}
fn config_dir() -> Option<std::path::PathBuf> {
lightcraft_engine::camera_profiles::config_dir()
lightcraft_engine::config::config_dir()
}
/// `<config>/ui.json`, the saved UI state and app settings — `None` when nothing is read or
@ -387,6 +396,13 @@ fn services(ctx: egui::Context, log_file: Option<&std::path::Path>) -> Services
.map(|p| p.to_string_lossy().to_string())
.collect()
})),
pick_denoise_model: Some(Box::new(|| {
rfd::FileDialog::new()
.add_filter_nocase(lightcraft_ui_egui::i18n::tr("ONNX model"), &["onnx"])
.pick_file()
.map(|p| vec![p.to_string_lossy().into_owned()])
.unwrap_or_default()
})),
pick_preset_files: Some(Box::new(|| {
rfd::FileDialog::new()
.set_title(lightcraft_ui_egui::i18n::tr("Import Presets"))
@ -468,15 +484,15 @@ fn open_session(in_memory: bool, dir: Option<std::path::PathBuf>, seed_demo: boo
/// and offers Try Again / Choose Another Library… / Continue Without Saving / Quit (issue #100).
fn open_library_session(in_memory: bool, dir: Option<std::path::PathBuf>, seed_demo: bool) -> (Session, Option<LibraryProblem>) {
if in_memory {
return (if seed_demo { Session::with_demo() } else { Session::new() }.with_fs().with_system_clock(), None);
return (if seed_demo { Session::with_demo() } else { Session::new() }.with_fs().with_default_denoise_models().with_system_clock(), None);
}
let unopened = || Session::new().with_fs().with_system_clock();
let unopened = || Session::new().with_fs().with_default_denoise_models().with_system_clock();
let Some(dir) = dir else {
log::warn!("no library location (set --library or LIGHTCRAFT_LIBRARY)");
return (unopened(), Some(LibraryProblem::new("", "There is no home folder to keep the library in. Choose a folder for it.")));
};
let t0 = std::time::Instant::now();
let mut s = Session::new().with_fs().with_system_clock();
let mut s = Session::new().with_fs().with_default_denoise_models().with_system_clock();
match s.open_library(&dir, seed_demo) {
Ok(r) => {
let (replayed, torn) = (r.replayed, r.torn_bytes);
@ -891,6 +907,16 @@ mod tests {
assert!(!b.is_empty());
}
/// Issue #471: the window's DX12 shaders compile with FXC — wgpu's default loaded any
/// `dxcompiler.dll` on the search path, and one without `dxil.dll` kept the window from opening.
#[test]
fn window_compiles_dx12_shaders_with_fxc() {
let eframe::egui_wgpu::WgpuSetup::CreateNew(n) = window_wgpu_options().wgpu_setup else { panic!("expected CreateNew") };
if std::env::var_os("WGPU_DX12_COMPILER").is_none() {
assert!(matches!(n.instance_descriptor.backend_options.dx12.shader_compiler, eframe::wgpu::Dx12Compiler::Fxc));
}
}
/// Issue #234: Windows opens links with the URL protocol handler (the default browser), not
/// File Explorer, and passes the whole map link, `?`, `&` and `#` included, as one argument.
#[test]

12
apps/lightcraft/src/native_menu.rs
View File

@ -355,12 +355,15 @@ impl NativeMenu {
}
/// Per frame: run chosen items, then sync labels / enabled / checked and the text-focus
/// accelerators with the app state.
pub fn update(&mut self, app: &mut LightcraftApp, ctx: &egui::Context) {
/// accelerators with the app state. Returns whether Quit was chosen: the caller saves
/// everything and exits the process itself (closing the viewport while AppKit terminates the
/// app can crash in winit's teardown, PR #444).
pub fn update(&mut self, app: &mut LightcraftApp, ctx: &egui::Context) -> bool {
lightcraft_ui_egui::i18n::set_language(app.ui.language);
let mut quit = false;
while let Ok(key) = self.rx.try_recv() {
if key == QUIT {
ctx.send_viewport_cmd(egui::ViewportCommand::Close);
quit = true;
continue;
}
if key == "app.about" || key == SETTINGS {
@ -375,7 +378,7 @@ impl NativeMenu {
let bar = menu_bar(app);
if structure_of(&bar) != self.structure {
self.rebuild(app);
return;
return quit;
}
fn walk(nodes: &[MenuNode], items: &mut HashMap<String, Item>) {
for n in nodes {
@ -429,6 +432,7 @@ impl NativeMenu {
}
self.publish_shortcuts(app);
}
quit
}
}

3
contributors/people.toml
View File

@ -22,5 +22,8 @@ real_name = "Brandon Thomas"
display_name = "Brandon Thomas"
sync_github_name = false
[people.rock-meister]
sync_github_name = true
[people.sanyamsingh001]
display_name = "sanyamsingh001"

3
crates/catalog/src/journal.rs
View File

@ -45,6 +45,7 @@
//! | (none) | early builds: a log without a snapshot | |
//! | 1 | up to v0.2.0 | |
//! | 2 | after v0.2.0 | `Op::SetBrowsed`, `Catalog.browsed`, `Photo.local_baseline` |
//! | 3 | albums by hand | `Op::SetAlbumOrder`, `Album.order` |
//!
//! Rules:
//! - **Bump [`VERSION`]** (and add a row above) in the change that adds an [`Op`] variant or a
@ -69,7 +70,7 @@ pub const LOG: &str = "catalog.log";
const FORMAT: &str = "lightcraft-catalog";
/// The catalog format this build writes (and the newest it reads). See the module docs →
/// *Format versions*; bump it whenever an [`Op`] variant or a serialized field is added.
pub const VERSION: u32 = 2;
pub const VERSION: u32 = 3;
/// When [`Journal::wants_snapshot`] says it's time to compact the log.
#[derive(Clone, Copy, Debug)]

50
crates/catalog/src/lib.rs
View File

@ -136,6 +136,11 @@ pub enum Op {
id: AlbumId,
parent: Option<AlbumId>,
},
/// Where the album stands among its siblings (`None`: by name). Format version 3.
SetAlbumOrder {
id: AlbumId,
order: Option<u32>,
},
SetAlbumPhotos {
id: AlbumId,
photos: Vec<PhotoId>,
@ -282,6 +287,28 @@ impl Catalog {
pub fn albums(&self) -> impl Iterator<Item = &Album> {
self.albums.values()
}
/// The albums and folders directly inside `parent` (`None`: the top level), in the order the
/// sidebar lists them: folders first, then by their place if the user ordered them (those
/// without one after, by name), else by name.
pub fn album_children(&self, parent: Option<AlbumId>) -> Vec<&Album> {
let mut kids: Vec<&Album> = self.albums.values().filter(|a| a.parent == parent).collect();
kids.sort_by_cached_key(|a| album_order_key(a));
kids
}
/// [`Self::album_children`] of every folder (and of the top level, under `None`) in one pass:
/// what a tree view that draws all of them wants each frame.
pub fn album_children_by_parent(&self) -> std::collections::HashMap<Option<AlbumId>, Vec<&Album>> {
let mut map: std::collections::HashMap<Option<AlbumId>, Vec<&Album>> = std::collections::HashMap::new();
for a in self.albums.values() {
map.entry(a.parent).or_default().push(a);
}
map.values_mut().for_each(|kids| kids.sort_by_cached_key(|a| album_order_key(a)));
map
}
/// Whether anything inside `parent` has a place of its own (the folder is ordered by hand).
pub fn album_children_are_ordered(&self, parent: Option<AlbumId>) -> bool {
self.albums.values().any(|a| a.parent == parent && a.order.is_some())
}
/// The Quick Collection, once something was added to it.
pub fn quick_collection(&self) -> Option<AlbumId> {
self.albums.values().find(|a| a.quick).map(|a| a.id)
@ -479,7 +506,20 @@ impl Catalog {
}
}
let a = self.album_mut(id)?;
Op::MoveAlbum { id, parent: std::mem::replace(&mut a.parent, parent) }
if a.parent == parent {
return Ok(Op::MoveAlbum { id, parent });
}
// its place belonged to the old neighbours: the undo gives it back with the folder
let old_order = a.order.take();
let old_parent = std::mem::replace(&mut a.parent, parent);
match old_order {
Some(_) => Op::Batch { ops: vec![Op::MoveAlbum { id, parent: old_parent }, Op::SetAlbumOrder { id, order: old_order }] },
None => Op::MoveAlbum { id, parent: old_parent },
}
}
Op::SetAlbumOrder { id, order } => {
let a = self.album_mut(id)?;
Op::SetAlbumOrder { id, order: std::mem::replace(&mut a.order, order) }
}
Op::SetAlbumPhotos { id, photos } => {
let a = self.album_mut(id)?;
@ -668,9 +708,17 @@ impl Catalog {
}
}
/// How siblings are listed: folders first, then the ones with a place of their own by it, then by
/// name; the id settles any tie.
fn album_order_key(a: &Album) -> (bool, bool, u32, String, AlbumId) {
(!a.folder, a.order.is_none(), a.order.unwrap_or(0), a.name.to_lowercase(), a.id)
}
#[cfg(test)]
mod tests;
#[cfg(test)]
mod tests_album_order;
#[cfg(test)]
mod tests_background;
#[cfg(test)]
mod tests_folders;

1
crates/catalog/src/local.rs
View File

@ -295,6 +295,7 @@ impl Op {
Op::RemoveAlbum { .. }
| Op::RenameAlbum { .. }
| Op::MoveAlbum { .. }
| Op::SetAlbumOrder { .. }
| Op::SetAlbumRules { .. }
| Op::RemoveStack { .. }
| Op::SetLabelName { .. }

6
crates/catalog/src/model.rs
View File

@ -405,12 +405,16 @@ pub struct Album {
/// The Quick Collection: a regular album that B adds to (one per library).
#[serde(default, skip_serializing_if = "std::ops::Not::not")]
pub quick: bool,
/// Its place among the albums of the same kind in its folder, once the user ordered them by
/// hand (see [`crate::Catalog::album_children`]); `None`: listed by name.
#[serde(default, skip_serializing_if = "Option::is_none")]
pub order: Option<u32>,
}
impl Album {
/// A regular (manual) album.
pub fn new(id: AlbumId, name: impl Into<String>) -> Album {
Album { id, name: name.into(), parent: None, folder: false, photos: Vec::new(), cover: None, smart: None, quick: false }
Album { id, name: name.into(), parent: None, folder: false, photos: Vec::new(), cover: None, smart: None, quick: false, order: None }
}
pub fn is_smart(&self) -> bool {
self.smart.is_some()

14
crates/catalog/src/tests.rs
View File

@ -58,12 +58,22 @@ fn albums_and_folders() {
let a = photo(&mut c, "a.jpg", "2026-04-01");
let f = c.alloc_album_id();
c.apply(Op::AddAlbum {
album: Album { id: f, name: "Trips".into(), parent: None, folder: true, photos: vec![], cover: None, smart: None, quick: false },
album: Album { id: f, name: "Trips".into(), parent: None, folder: true, photos: vec![], cover: None, smart: None, quick: false, order: None },
})
.unwrap();
let al = c.alloc_album_id();
c.apply(Op::AddAlbum {
album: Album { id: al, name: "Alps".into(), parent: Some(f), folder: false, photos: vec![a], cover: None, smart: None, quick: false },
album: Album {
id: al,
name: "Alps".into(),
parent: Some(f),
folder: false,
photos: vec![a],
cover: None,
smart: None,
quick: false,
order: None,
},
})
.unwrap();
assert_eq!(c.albums_of(a), vec![al]);

122
crates/catalog/src/tests_album_order.rs Normal file
View File

@ -0,0 +1,122 @@
//! Album order: the children of a folder (or of the top level) are listed folders first, then
//! alphabetically, until some of them are given a place of their own (`Album.order`); moving an
//! album to another folder drops its place, and undoing the move brings it back.
use crate::*;
fn add(c: &mut Catalog, name: &str, parent: Option<AlbumId>, folder: bool) -> AlbumId {
let id = c.alloc_album_id();
c.apply(Op::AddAlbum { album: Album { parent, folder, ..Album::new(id, name) } }).unwrap();
id
}
fn names(c: &Catalog, parent: Option<AlbumId>) -> Vec<String> {
c.album_children(parent).iter().map(|a| a.name.clone()).collect()
}
fn place(c: &mut Catalog, id: AlbumId, order: Option<u32>) {
c.apply(Op::SetAlbumOrder { id, order }).unwrap();
}
/// Given albums that were never ordered, when they are listed, then folders come first and each
/// group is alphabetical regardless of case.
#[test]
fn children_are_folders_first_then_alphabetical() {
let mut c = Catalog::new();
add(&mut c, "beta", None, false);
add(&mut c, "Alpha", None, false);
add(&mut c, "zoo", None, true);
add(&mut c, "Archive", None, true);
assert_eq!(names(&c, None), ["Archive", "zoo", "Alpha", "beta"]);
assert!(!c.album_children_are_ordered(None));
}
/// Given some children with a place of their own, then they follow it; the others come after, by
/// name; folders stay ahead of albums.
#[test]
fn ordered_children_follow_their_order() {
let mut c = Catalog::new();
let a = add(&mut c, "A", None, false);
let b = add(&mut c, "B", None, false);
let cc = add(&mut c, "C", None, false);
add(&mut c, "D", None, false);
let f = add(&mut c, "F", None, true);
place(&mut c, cc, Some(0));
place(&mut c, a, Some(1));
assert_eq!(names(&c, None), ["F", "C", "A", "B", "D"]);
assert!(c.album_children_are_ordered(None));
// equal places fall back to the name, so a damaged catalog still lists deterministically
place(&mut c, b, Some(1));
assert_eq!(names(&c, None), ["F", "C", "A", "B", "D"]);
// another container is unaffected
let inner = add(&mut c, "Z", Some(f), false);
add(&mut c, "Y", Some(f), false);
assert_eq!(names(&c, Some(f)), ["Y", "Z"]);
assert!(!c.album_children_are_ordered(Some(f)));
place(&mut c, inner, Some(0));
assert_eq!(names(&c, Some(f)), ["Z", "Y"]);
}
/// The one-pass map lists every folder's children exactly as `album_children` does.
#[test]
fn the_children_map_matches_the_children_of_each_folder() {
let mut c = Catalog::new();
let f = add(&mut c, "F", None, true);
let g = add(&mut c, "G", Some(f), true);
for (name, parent) in [("b", None), ("a", None), ("y", Some(f)), ("x", Some(f)), ("q", Some(g))] {
add(&mut c, name, parent, false);
}
let x = c.albums().find(|a| a.name == "x").unwrap().id;
place(&mut c, x, Some(7));
let map = c.album_children_by_parent();
for parent in [None, Some(f), Some(g)] {
let from_map: Vec<AlbumId> = map.get(&parent).map(|v| v.iter().map(|a| a.id).collect()).unwrap_or_default();
let direct: Vec<AlbumId> = c.album_children(parent).iter().map(|a| a.id).collect();
assert_eq!(from_map, direct, "{parent:?}");
}
assert!(!map.contains_key(&Some(AlbumId(999))));
}
/// A place is one undo step, and `None` puts the album back among the alphabetical ones.
#[test]
fn a_place_can_be_set_cleared_and_undone() {
let mut c = Catalog::new();
let a = add(&mut c, "A", None, false);
add(&mut c, "B", None, false);
let inv = c.apply(Op::SetAlbumOrder { id: a, order: Some(5) }).unwrap();
assert_eq!(c.album(a).unwrap().order, Some(5));
c.apply(inv).unwrap();
assert_eq!(c.album(a).unwrap().order, None);
assert!(c.apply(Op::SetAlbumOrder { id: AlbumId(999), order: Some(1) }).is_err(), "no such album");
}
/// Moving to another folder drops the album's place there (it belonged to the old neighbours),
/// and undoing the move gives it back.
#[test]
fn moving_drops_the_place_and_undo_restores_it() {
let mut c = Catalog::new();
let from = add(&mut c, "From", None, true);
let to = add(&mut c, "To", None, true);
let a = add(&mut c, "A", Some(from), false);
place(&mut c, a, Some(3));
let inv = c.apply(Op::MoveAlbum { id: a, parent: Some(to) }).unwrap();
assert_eq!((c.album(a).unwrap().parent, c.album(a).unwrap().order), (Some(to), None));
c.apply(inv).unwrap();
assert_eq!((c.album(a).unwrap().parent, c.album(a).unwrap().order), (Some(from), Some(3)));
// "moving" to where it already is changes nothing
c.apply(Op::MoveAlbum { id: a, parent: Some(from) }).unwrap();
assert_eq!(c.album(a).unwrap().order, Some(3));
}
/// Catalogs written before albums had an order load as they were; an unset order is not written.
#[test]
fn albums_without_an_order_load_and_save_unchanged() {
let old = r#"{"id":4,"name":"Old","parent":null,"folder":false,"photos":[],"cover":null}"#;
let a: Album = serde_json::from_str(old).unwrap();
assert_eq!(a.order, None);
assert!(!serde_json::to_string(&a).unwrap().contains("order"));
let mut a = a;
a.order = Some(2);
let back: Album = serde_json::from_str(&serde_json::to_string(&a).unwrap()).unwrap();
assert_eq!(back.order, Some(2));
}

16
crates/catalog/src/tests_format_version.rs
View File

@ -47,6 +47,19 @@ fn v1_library_loads_and_is_upgraded() {
assert_eq!((r.replayed, r.upgraded_from), (1, None));
}
/// A library saved before albums could be ordered by hand opens as it was and is rewritten as v3.
#[test]
fn v2_library_loads_and_is_upgraded() {
let (base, log, full) = legacy_parts();
let m = MemStore::new();
m.set(SNAPSHOT, format!("{{\"format\":\"lightcraft-catalog\",\"version\":2,\"seq\":2,\"catalog\":{}}}\n", base.to_snapshot()).into_bytes());
m.set(LOG, log.into_bytes());
let (_, c, r) = Journal::open(Box::new(m.clone())).unwrap();
assert_eq!(c.to_snapshot(), full.to_snapshot());
assert_eq!((r.replayed, r.upgraded_from), (1, Some(2)));
assert_eq!(snapshot_version(&m), 3);
}
#[test]
fn versionless_log_only_library_is_upgraded() {
let mut c = Catalog::new();
@ -179,8 +192,9 @@ fn op_variants_are_versioned() {
| Op::SetLabelName { .. }
| Op::Batch { .. } => 1,
Op::SetBrowsed { .. } => 2,
Op::SetAlbumOrder { .. } => 3,
}
}
let newest = since(&Op::SetBrowsed { folder: String::new(), at: None });
let newest = since(&Op::SetAlbumOrder { id: AlbumId(0), order: None });
assert_eq!(newest, VERSION, "the newest op's version must be the current format version");
}

2
crates/catalog/src/tests_journal.rs
View File

@ -59,6 +59,7 @@ fn op_for(c: &mut Catalog, kind: u8, a: u8, b: u8) -> Op {
cover: None,
smart: None,
quick: false,
order: None,
},
}
}
@ -77,6 +78,7 @@ fn op_for(c: &mut Catalog, kind: u8, a: u8, b: u8) -> Op {
13 => Op::Batch { ops: c.remove_from_stacks_ops(&[pid]) },
14 => c.set_top_ops(pid).unwrap_or(Op::Batch { ops: vec![] }),
9 if b.is_multiple_of(5) => Op::SetLabelName { label: ColorLabel::ALL[a as usize % 5], name: (a % 3 != 1).then(|| format!("Label {b}")) },
9 if b % 7 == 3 => Op::SetAlbumOrder { id: aid, order: (!b.is_multiple_of(3)).then_some(u32::from(b % 5)) },
9 if b.is_multiple_of(2) => {
Op::SetFile { id: pid, file_name: format!("Renamed_{b}.jpg"), source: Source::File { path: format!("/p/Renamed_{b}.jpg") } }
}

26
crates/denoise-core/Cargo.toml Normal file
View File

@ -0,0 +1,26 @@
[package]
name = "lightcraft-denoise-core"
version.workspace = true
edition.workspace = true
license.workspace = true
description = "LightCraft denoise model metadata, Bayer tiling and cache formats (L1; no inference backend)"
[lints]
workspace = true
[features]
default = []
rawnind-model = []
[dependencies]
lightcraft-raster = { workspace = true }
serde = { workspace = true }
serde_json = { workspace = true }
sha2 = { workspace = true }
thiserror = { workspace = true }
miniz_oxide = { workspace = true }
rayon = { workspace = true }
web-time = { workspace = true }
[dev-dependencies]
proptest = { workspace = true }

7
crates/denoise-core/README.md Normal file
View File

@ -0,0 +1,7 @@
# lightcraft-denoise-core
L1 model metadata, bounded archive/cache formats and Bayer packing/tiling helpers.
No weights or inference backend. See [the user workflow](../../docs/denoise.md).
`lightcraft-denoise` re-exports these types for CPU/GPU users; the engine uses this
crate directly when inference is disabled. The default download catalog is empty.
Run `cargo test -p lightcraft-denoise-core`; optional catalog checks use `--features rawnind-model`.

279
crates/denoise-core/src/archive.rs Normal file
View File

@ -0,0 +1,279 @@
//! Taking one file out of a `.zip` (a `.dtmodel` is one), the little of the format a model download needs: stored and
//! deflated entries, found by name, checked against their CRC-32 and a size limit, written next to where they are
//! going and moved into place. It is a stranger's file, so every length and offset in it is checked, a name is only
//! ever compared (never used as a path), and zip64, encrypted and multi-disk archives are refused.
use std::io::{Read, Seek, SeekFrom, Write};
use std::path::Path;
/// How far from the end the end-of-central-directory record can start (a 22-byte record and a 64 KiB comment).
const EOCD_SEARCH: u64 = 22 + 65535;
/// Largest central directory we read.
const MAX_DIRECTORY: u64 = 1 << 20;
#[derive(Debug, PartialEq, thiserror::Error)]
pub enum ArchiveError {
#[error("the archive could not be read: {0}")]
Io(String),
#[error("not a usable archive: {0}")]
Format(String),
#[error("the archive has no file called `{0}`")]
Missing(String),
}
impl From<std::io::Error> for ArchiveError {
fn from(e: std::io::Error) -> Self {
ArchiveError::Io(e.to_string())
}
}
fn format<T>(why: &str) -> Result<T, ArchiveError> {
Err(ArchiveError::Format(why.into()))
}
fn u16_at(b: &[u8], at: usize) -> Option<u16> {
Some(u16::from_le_bytes(b.get(at..at.checked_add(2)?)?.try_into().ok()?))
}
fn u32_at(b: &[u8], at: usize) -> Option<u32> {
Some(u32::from_le_bytes(b.get(at..at.checked_add(4)?)?.try_into().ok()?))
}
/// CRC-32 (IEEE) of `bytes`.
fn crc32(bytes: &[u8]) -> u32 {
let mut table = [0u32; 256];
for (i, t) in table.iter_mut().enumerate() {
let mut c = i as u32;
for _ in 0..8 {
c = if c & 1 != 0 { 0xedb8_8320 ^ (c >> 1) } else { c >> 1 };
}
*t = c;
}
!bytes.iter().fold(!0u32, |c, &b| table[((c ^ u32::from(b)) & 0xff) as usize] ^ (c >> 8))
}
struct Entry {
method: u16,
crc: u32,
compressed: u64,
size: u64,
local_header: u64,
}
/// Find the entry called `name` (exactly, or as the last part of a longer name: `folder/name`) in the central directory.
fn find(f: &mut (impl Read + Seek), name: &str) -> Result<Entry, ArchiveError> {
let len = f.seek(SeekFrom::End(0))?;
let tail_len = len.min(EOCD_SEARCH);
f.seek(SeekFrom::Start(len - tail_len))?;
let mut tail = vec![0u8; tail_len as usize];
f.read_exact(&mut tail)?;
let eocd = (0..tail.len().saturating_sub(21)).rev().find(|&i| u32_at(&tail, i) == Some(0x0605_4b50));
let Some(e) = eocd else { return format("this is not a zip file") };
let (entries, size, offset) = (u16_at(&tail, e + 10), u32_at(&tail, e + 12), u32_at(&tail, e + 16));
let (Some(entries), Some(size), Some(offset)) = (entries, size, offset) else { return format("the end of the archive is cut short") };
if entries == 0xffff || size == u32::MAX || offset == u32::MAX {
return format("zip64 archives are not supported");
}
if u64::from(size) > MAX_DIRECTORY || u64::from(offset) + u64::from(size) > len {
return format("the list of files is not where it says");
}
f.seek(SeekFrom::Start(u64::from(offset)))?;
let mut dir = vec![0u8; size as usize];
f.read_exact(&mut dir)?;
let mut at = 0usize;
for _ in 0..entries {
if u32_at(&dir, at) != Some(0x0201_4b50) {
return format("the list of files is damaged");
}
let get = |o: usize| at.checked_add(o);
let (flags, method, crc) = (u16_at(&dir, at + 8), u16_at(&dir, at + 10), u32_at(&dir, at + 16));
let (compressed, uncompressed) = (u32_at(&dir, at + 20), u32_at(&dir, at + 24));
let (n, x, c) = (u16_at(&dir, at + 28), u16_at(&dir, at + 30), u16_at(&dir, at + 32));
let local = u32_at(&dir, at + 42);
let (Some(flags), Some(method), Some(crc), Some(compressed), Some(uncompressed), Some(n), Some(x), Some(c), Some(local), Some(start)) =
(flags, method, crc, compressed, uncompressed, n, x, c, local, get(46))
else {
return format("the list of files is cut short");
};
let Some(entry_name) = dir.get(start..start + usize::from(n)) else { return format("a file name is cut short") };
let matches = entry_name == name.as_bytes() || entry_name.strip_suffix(name.as_bytes()).is_some_and(|p| p.ends_with(b"/"));
if matches {
if flags & 1 != 0 {
return format("encrypted files are not supported");
}
if compressed == u32::MAX || uncompressed == u32::MAX {
return format("zip64 files are not supported");
}
return Ok(Entry { method, crc, compressed: u64::from(compressed), size: u64::from(uncompressed), local_header: u64::from(local) });
}
at = start + usize::from(n) + usize::from(x) + usize::from(c);
}
Err(ArchiveError::Missing(name.into()))
}
/// Take the file called `name` out of the archive at `archive` and write it to `dest` (through a temporary file next
/// to it); `max_bytes` bounds its size. Returns the number of bytes written.
pub fn extract(archive: &Path, name: &str, dest: &Path, max_bytes: u64) -> Result<u64, ArchiveError> {
let mut f = std::io::BufReader::new(std::fs::File::open(archive)?);
let e = find(&mut f, name)?;
if e.size > max_bytes || e.compressed > max_bytes {
return format("the file in the archive is larger than allowed");
}
f.seek(SeekFrom::Start(e.local_header))?;
let mut head = [0u8; 30];
f.read_exact(&mut head)?;
if u32_at(&head, 0) != Some(0x0403_4b50) {
return format("the file's header is not where it says");
}
let skip = u64::from(u16_at(&head, 26).unwrap_or(0)) + u64::from(u16_at(&head, 28).unwrap_or(0));
f.seek_relative(i64::try_from(skip).map_err(|_| ArchiveError::Format("a header is too long".into()))?)?;
let len = usize::try_from(e.compressed)
.ok()
.filter(|&n| n <= 256 * 1024 * 1024)
.ok_or_else(|| ArchiveError::Format("the compressed model is too large".into()))?;
if e.size > 256 * 1024 * 1024 {
return format("the extracted model is too large");
}
let mut packed = Vec::new();
packed.try_reserve_exact(len).map_err(|_| ArchiveError::Format("not enough memory for the model archive".into()))?;
packed.resize(len, 0u8);
f.read_exact(&mut packed)?;
let data = match e.method {
0 => packed,
8 => miniz_oxide::inflate::decompress_to_vec_with_limit(&packed, max_bytes.min(256 * 1024 * 1024) as usize)
.map_err(|err| ArchiveError::Format(format!("{err:?}")))?,
m => return format(&format!("compression method {m} is not supported")),
};
if data.len() as u64 != e.size || crc32(&data) != e.crc {
return format("the file in the archive is damaged");
}
let tmp = dest.with_extension("part");
let written = (|| -> std::io::Result<()> {
let mut out = std::fs::File::create(&tmp)?;
out.write_all(&data)?;
out.flush()?;
drop(out);
std::fs::rename(&tmp, dest)
})();
if let Err(err) = written {
let _ = std::fs::remove_file(&tmp);
return Err(err.into());
}
Ok(data.len() as u64)
}
#[cfg(test)]
mod tests {
use super::*;
/// A zip with the given (name, bytes, deflate) entries, written by hand.
pub(crate) fn make_zip(files: &[(&str, &[u8], bool)]) -> Vec<u8> {
let mut out = Vec::new();
let mut dir = Vec::new();
for (name, data, deflate) in files {
let packed = if *deflate { miniz_oxide::deflate::compress_to_vec(data, 6) } else { data.to_vec() };
let offset = out.len() as u32;
let method: u16 = if *deflate { 8 } else { 0 };
out.extend_from_slice(&0x0403_4b50u32.to_le_bytes());
out.extend_from_slice(&[20, 0, 0, 0]);
out.extend_from_slice(&method.to_le_bytes());
out.extend_from_slice(&[0, 0, 0, 0]);
out.extend_from_slice(&crc32(data).to_le_bytes());
out.extend_from_slice(&(packed.len() as u32).to_le_bytes());
out.extend_from_slice(&(data.len() as u32).to_le_bytes());
out.extend_from_slice(&(name.len() as u16).to_le_bytes());
out.extend_from_slice(&[0, 0]);
out.extend_from_slice(name.as_bytes());
out.extend_from_slice(&packed);
dir.extend_from_slice(&0x0201_4b50u32.to_le_bytes());
dir.extend_from_slice(&[20, 0, 20, 0, 0, 0]);
dir.extend_from_slice(&method.to_le_bytes());
dir.extend_from_slice(&[0, 0, 0, 0]);
dir.extend_from_slice(&crc32(data).to_le_bytes());
dir.extend_from_slice(&(packed.len() as u32).to_le_bytes());
dir.extend_from_slice(&(data.len() as u32).to_le_bytes());
dir.extend_from_slice(&(name.len() as u16).to_le_bytes());
dir.extend_from_slice(&[0; 12]);
dir.extend_from_slice(&offset.to_le_bytes());
dir.extend_from_slice(name.as_bytes());
}
let cd_offset = out.len() as u32;
out.extend_from_slice(&dir);
out.extend_from_slice(&0x0605_4b50u32.to_le_bytes());
out.extend_from_slice(&[0, 0, 0, 0]);
out.extend_from_slice(&(files.len() as u16).to_le_bytes());
out.extend_from_slice(&(files.len() as u16).to_le_bytes());
out.extend_from_slice(&(dir.len() as u32).to_le_bytes());
out.extend_from_slice(&cd_offset.to_le_bytes());
out.extend_from_slice(&[0, 0]);
out
}
fn temp(name: &str) -> std::path::PathBuf {
std::env::temp_dir().join(format!("lc-denoise-zip-{name}-{}", std::process::id()))
}
#[test]
fn a_stored_and_a_deflated_file_come_out_whole() {
let big: Vec<u8> = (0..200_000u32).map(|i| (i * 7 % 251) as u8).collect();
let zip = make_zip(&[("m/config.json", b"{}", false), ("m/model.onnx", &big, true), ("m/other.onnx", b"zz", true)]);
let (a, out) = (temp("a.zip"), temp("a.onnx"));
std::fs::write(&a, &zip).unwrap();
assert_eq!(extract(&a, "model.onnx", &out, 1 << 20).unwrap(), big.len() as u64);
assert_eq!(std::fs::read(&out).unwrap(), big);
assert_eq!(extract(&a, "m/config.json", &out, 1 << 20).unwrap(), 2);
assert_eq!(std::fs::read(&out).unwrap(), b"{}");
// a name must match whole path parts: `del.onnx` is not `model.onnx`
assert!(matches!(extract(&a, "del.onnx", &out, 1 << 20), Err(ArchiveError::Missing(_))));
assert!(matches!(extract(&a, "nothing", &out, 1 << 20), Err(ArchiveError::Missing(_))));
// the size limit applies
assert!(matches!(extract(&a, "model.onnx", &out, 1000), Err(ArchiveError::Format(_))));
for p in [a, out] {
let _ = std::fs::remove_file(p);
}
}
/// Opt-in: `LC_DENOISE_DTMODEL=<rawdenoise-nind.dtmodel> cargo test -p lightcraft-denoise -- --ignored archive`
#[test]
#[ignore = "needs the real download: set LC_DENOISE_DTMODEL"]
fn the_real_download_gives_the_model_it_should() {
let Some(path) = std::env::var_os("LC_DENOISE_DTMODEL") else { return };
let known = crate::known::find("rawnind-bayer").unwrap();
let (d, entry) = (known.download.unwrap(), known.entry.unwrap());
assert_eq!(crate::hash::sha256_file(Path::new(&path)).unwrap(), d.sha256);
let out = temp("real.onnx");
let n = extract(Path::new(&path), entry, &out, crate::manifest::MAX_MODEL_BYTES).unwrap();
assert_eq!(Some(n), known.manifest.size_bytes);
assert_eq!(Some(crate::hash::sha256_file(&out).unwrap()), known.manifest.sha256);
let _ = std::fs::remove_file(out);
}
#[test]
fn damaged_and_hostile_archives_are_errors_not_panics() {
let zip = make_zip(&[("m/model.onnx", &vec![9u8; 5000], true)]);
let (a, out) = (temp("b.zip"), temp("b.onnx"));
// cut anywhere
for n in (0..zip.len()).step_by(3) {
std::fs::write(&a, &zip[..n]).unwrap();
assert!(extract(&a, "model.onnx", &out, 1 << 20).is_err(), "cut at {n}");
}
// one byte changed anywhere: an error, or (when it falls in an unused field) the right file, never a panic
for at in 0..zip.len() {
let mut bad = zip.clone();
bad[at] ^= 0x5a;
std::fs::write(&a, &bad).unwrap();
if extract(&a, "model.onnx", &out, 1 << 20).is_ok() {
assert_eq!(std::fs::read(&out).unwrap(), vec![9u8; 5000], "a changed byte at {at} gave another file");
}
}
// not a zip at all, and an empty file
for junk in [Vec::new(), b"PK".to_vec(), vec![0u8; 100_000]] {
std::fs::write(&a, junk).unwrap();
assert!(extract(&a, "model.onnx", &out, 1 << 20).is_err());
}
assert!(extract(&temp("missing.zip"), "model.onnx", &out, 1 << 20).is_err());
for p in [a, out] {
let _ = std::fs::remove_file(p);
}
}
}

191
crates/denoise-core/src/bayer.rs Normal file
View File

@ -0,0 +1,191 @@
//! Bayer geometry: which way a mosaic is laid out, and how a patch of it is packed into the four planes
//! (red, green on the red row, green on the blue row, blue) a denoise model takes.
//!
//! Every Bayer layout is the same 2 × 2 cell shifted by one pixel at most, so a layout is just where its
//! first red sample is. The cell grid starts one pixel before the picture when that red sample is in
//! column or row 1, and the picture is read with its edge mirrored (without repeating the edge sample, so
//! the colour of a mirrored sample is the colour that belongs at that place).
/// Where the first red sample of a Bayer mosaic sits, `(dx, dy)` in `{0, 1}²`: RGGB is `(0, 0)`, GRBG `(1, 0)`,
/// GBRG `(0, 1)`, BGGR `(1, 1)`.
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub struct Layout {
pub dx: usize,
pub dy: usize,
}
impl Layout {
/// The layout of a 2 × 2 colour pattern, row-major, colours 0 = red, 1 = green, 2 = blue; `None` when
/// it is not a Bayer cell (one red, one blue, the greens on the other diagonal).
pub fn from_cell(cell: [u8; 4]) -> Option<Layout> {
let at = |c: u8| cell.iter().position(|&v| v == c);
let red = at(0)?;
let blue = at(2)?;
let greens = cell.iter().filter(|&&v| v == 1).count();
// red and blue sit on one diagonal: indices 0+3 or 1+2
if greens != 2 || red + blue != 3 {
return None;
}
Some(Layout { dx: red % 2, dy: red / 2 })
}
/// The cell grid's origin: `-1` along an axis whose first red sample is at 1, so that cell 0 starts on a red column or row.
fn origin(self) -> (isize, isize) {
(isize::try_from(self.dx).unwrap_or(isize::MAX).saturating_neg(), isize::try_from(self.dy).unwrap_or(isize::MAX).saturating_neg())
}
/// Cells across and down covering a `width × height` mosaic.
pub fn cells(self, width: usize, height: usize) -> (usize, usize) {
let cells = |n: usize, phase: usize| n / 2 + (n % 2 + phase.min(1)).div_ceil(2);
(cells(width, self.dx), cells(height, self.dy))
}
/// The mosaic position of cell `(cx, cy)`'s top-left sample (may lie outside the picture).
pub fn position(self, cx: isize, cy: isize) -> (isize, isize) {
let (ox, oy) = self.origin();
(ox.saturating_add(cx.saturating_mul(2)), oy.saturating_add(cy.saturating_mul(2)))
}
}
/// `i` mirrored into `0..n` (ping-pong, edge not repeated), so `-1` is `1` and `n` is `n - 2`. `n == 0` gives 0.
pub fn reflect(i: isize, n: usize) -> usize {
if n <= 1 {
return 0;
}
if let Some(period) = isize::try_from(n).ok().and_then(|n| (n - 1).checked_mul(2)) {
let m = i.rem_euclid(period);
return (if m < n as isize { m } else { period - m }) as usize;
}
let period = 2 * (n as i128 - 1);
let m = (i as i128).rem_euclid(period);
// The reflected value is in 0..n, so this conversion always fits usize.
(if m < n as i128 { m } else { period - m }) as usize
}
/// Pack the `tile × tile` cells whose top-left cell is `(cx0, cy0)` into `out` as four planes `[R, G1, G2, B]`
/// (`4 · tile²` values, NCHW). G1 is the green beside the red sample, G2 the green under it. Samples outside
/// the picture are mirrored. Returns `false` (and writes nothing) when the sizes do not fit.
pub fn pack_tile(mosaic: &[f32], width: usize, height: usize, layout: Layout, cx0: isize, cy0: isize, tile: usize, out: &mut [f32]) -> bool {
let Some(plane) = tile.checked_mul(tile) else { return false };
let Some(values) = plane.checked_mul(4) else { return false };
if width == 0
|| height == 0
|| tile == 0
|| layout.dx > 1
|| layout.dy > 1
|| width.checked_mul(height) != Some(mosaic.len())
|| out.len() != values
{
return false;
}
let Ok(last) = isize::try_from(tile - 1) else { return false };
let (ox, oy) = layout.origin();
for (start, origin) in [(cx0, ox), (cy0, oy)] {
if start.checked_mul(2).and_then(|v| v.checked_add(origin)).is_none()
|| start.checked_add(last).and_then(|v| v.checked_mul(2)).and_then(|v| v.checked_add(origin)).and_then(|v| v.checked_add(1)).is_none()
{
return false;
}
}
// the four samples of a cell, as (dx, dy) from its top-left sample
const SUB: [(isize, isize); 4] = [(0, 0), (1, 0), (0, 1), (1, 1)];
for (p, &(sx, sy)) in SUB.iter().enumerate() {
let Some(dst) = out.get_mut(p * plane..(p + 1) * plane) else { return false };
for (j, row) in dst.chunks_exact_mut(tile).enumerate() {
let (_, y) = layout.position(cx0, cy0 + j as isize);
let ry = reflect(y + sy, height);
let Some(src) = mosaic.get(ry * width..(ry + 1) * width) else { return false };
for (i, v) in row.iter_mut().enumerate() {
let (x, _) = layout.position(cx0 + i as isize, cy0);
*v = src.get(reflect(x + sx, width)).copied().unwrap_or(0.0);
}
}
}
true
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn hostile_sizes_and_positions_never_panic_or_write_output() {
let layout = Layout::from_cell([0, 1, 1, 2]).unwrap();
let mut out = [7.0; 4];
for (w, h, tile, x) in [(usize::MAX, 2, 1, 0), (1, 1, usize::MAX, 0), (1, 1, 0, 0), (1, 1, 1, isize::MAX), (1, 1, 1, isize::MIN)] {
assert!(!pack_tile(&[0.5], w, h, layout, x, 0, tile, &mut out));
assert_eq!(out, [7.0; 4]);
}
for n in [1, 2, isize::MAX as usize, usize::MAX] {
for i in [isize::MIN, -1, 0, isize::MAX] {
assert!(reflect(i, n) < n);
}
}
assert_eq!(layout.cells(usize::MAX, usize::MAX), (usize::MAX / 2 + 1, usize::MAX / 2 + 1));
}
#[test]
fn layouts_from_cells() {
assert_eq!(Layout::from_cell([0, 1, 1, 2]), Some(Layout { dx: 0, dy: 0 }));
assert_eq!(Layout::from_cell([1, 0, 2, 1]), Some(Layout { dx: 1, dy: 0 }));
assert_eq!(Layout::from_cell([1, 2, 0, 1]), Some(Layout { dx: 0, dy: 1 }));
assert_eq!(Layout::from_cell([2, 1, 1, 0]), Some(Layout { dx: 1, dy: 1 }));
// not Bayer: two reds, a missing blue, greens on the wrong diagonal
assert_eq!(Layout::from_cell([0, 0, 1, 2]), None);
assert_eq!(Layout::from_cell([0, 1, 2, 1]), None);
assert_eq!(Layout::from_cell([1, 1, 1, 1]), None);
}
#[test]
fn reflection_keeps_the_colour_of_a_site() {
assert_eq!(reflect(-1, 10), 1);
assert_eq!(reflect(-2, 10), 2);
assert_eq!(reflect(10, 10), 8);
assert_eq!(reflect(11, 10), 7);
assert_eq!(reflect(25, 10), 7);
assert_eq!(reflect(5, 0), 0);
assert_eq!(reflect(5, 1), 0);
for i in -40..60 {
assert_eq!(reflect(i, 11) % 2, (i.rem_euclid(2)) as usize, "parity at {i}");
}
}
#[test]
fn packing_puts_each_colour_in_its_plane_for_every_layout() {
// a mosaic whose sample value says which colour it is: R = 10, G = 20, B = 30
for cell in [[0u8, 1, 1, 2], [1, 0, 2, 1], [1, 2, 0, 1], [2, 1, 1, 0]] {
let layout = Layout::from_cell(cell).unwrap();
let (w, h) = (10usize, 8usize);
let mosaic: Vec<f32> = (0..w * h).map(|i| [10.0, 20.0, 30.0][cell[(i / w % 2) * 2 + i % w % 2] as usize]).collect();
let (nx, ny) = layout.cells(w, h);
let tile = nx.max(ny);
let mut out = vec![0.0; 4 * tile * tile];
assert!(pack_tile(&mosaic, w, h, layout, 0, 0, tile, &mut out));
let plane = tile * tile;
for (p, want) in [10.0, 20.0, 20.0, 30.0].into_iter().enumerate() {
assert!(out[p * plane..(p + 1) * plane].iter().all(|&v| v == want), "layout {cell:?} plane {p}");
}
}
}
#[test]
fn g1_is_beside_the_red_sample_and_g2_under_it() {
// RGGB with a distinct value at every site
let (w, h) = (4, 4);
let mosaic: Vec<f32> = (0..16).map(|i| i as f32).collect();
let mut out = vec![0.0; 4 * 4];
assert!(pack_tile(&mosaic, w, h, Layout { dx: 0, dy: 0 }, 0, 0, 2, &mut out));
// cell (0,0): R = site (0,0)=0, G1 = site (1,0)=1, G2 = site (0,1)=4, B = site (1,1)=5
assert_eq!([out[0], out[4], out[8], out[12]], [0.0, 1.0, 4.0, 5.0]);
// cell (1,0): sites (2,0)=2, (3,0)=3, (2,1)=6, (3,1)=7
assert_eq!([out[1], out[5], out[9], out[13]], [2.0, 3.0, 6.0, 7.0]);
}
#[test]
fn packing_rejects_wrong_sizes() {
let mut out = vec![0.0; 16];
assert!(!pack_tile(&[0.0; 15], 4, 4, Layout { dx: 0, dy: 0 }, 0, 0, 2, &mut out));
assert!(!pack_tile(&[0.0; 16], 4, 4, Layout { dx: 0, dy: 0 }, 0, 0, 3, &mut out));
assert!(!pack_tile(&[], 0, 0, Layout { dx: 0, dy: 0 }, 0, 0, 2, &mut out));
}
}

46
crates/denoise-core/src/hash.rs Normal file
View File

@ -0,0 +1,46 @@
//! SHA-256 of a model file, streamed (a model can be hundreds of megabytes).
use std::io::{self, Read};
use std::path::Path;
use sha2::{Digest, Sha256};
/// Lowercase hex SHA-256 of everything `r` yields.
pub fn sha256_reader<R: Read>(mut r: R) -> io::Result<String> {
let mut h = Sha256::new();
let mut buf = vec![0u8; 1 << 20];
loop {
let n = r.read(&mut buf)?;
if n == 0 {
break;
}
h.update(buf.get(..n).unwrap_or(&[]));
}
Ok(hex(&h.finalize()))
}
/// Lowercase hex SHA-256 of a file.
pub fn sha256_file(path: &Path) -> io::Result<String> {
sha256_reader(io::BufReader::new(std::fs::File::open(path)?))
}
/// Lowercase hex SHA-256 of `bytes`.
pub fn sha256_hex(bytes: &[u8]) -> String {
hex(&Sha256::digest(bytes))
}
fn hex(bytes: &[u8]) -> String {
bytes.iter().map(|b| format!("{b:02x}")).collect()
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn known_digests() {
assert_eq!(sha256_hex(b""), "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855");
assert_eq!(sha256_hex(b"abc"), "ba7816bf8f01cfea414140de5dae2223b00361a396177a9cb410ff61f20015ad");
assert_eq!(sha256_reader(&b"abc"[..]).unwrap(), sha256_hex(b"abc"));
}
}

128
crates/denoise-core/src/known.rs Normal file
View File

@ -0,0 +1,128 @@
//! The denoise models LightCraft knows about: what each one is, who may use it, and where to fetch it.
//!
//! None is part of LightCraft. A model with an address here can be downloaded when the user asks and has accepted
//! its terms; anything else is brought by the user as an `.onnx` file with a `denoise-model.json` beside it.
#[cfg(any(feature = "rawnind-model", test))]
use crate::licence::{Commercial, Licence};
use crate::manifest::DenoiserManifest;
#[cfg(any(feature = "rawnind-model", test))]
use crate::manifest::{Domain, Gain};
/// A known model, and how to get it.
pub struct Known {
pub manifest: DenoiserManifest,
/// What to fetch (an archive or the `.onnx` itself); `None` when the user must bring the file.
pub download: Option<Download>,
/// The `.onnx` inside the downloaded archive (`None`: the download is the `.onnx`).
pub entry: Option<&'static str>,
}
/// RawNIND's UtNet2 for Bayer sensors, as published for darktable (`rawdenoise-nind.dtmodel`, a zip with a Bayer and a
/// linear model). Its weights are GPL-3.0; the training pictures are CC BY 4.0 and CC0.
#[cfg(any(feature = "rawnind-model", test))]
fn rawnind_bayer() -> Known {
let manifest = DenoiserManifest {
id: "rawnind-bayer".into(),
name: "RawNIND UtNet2 (Bayer)".into(),
version: "1.0".into(),
licence: Licence {
name: "GPL-3.0".into(),
commercial: Commercial::Yes,
url: Some("https://www.gnu.org/licenses/gpl-3.0.html".into()),
notice: "The weights are published under the GNU General Public License v3. LightCraft does not include them: they \
are downloaded to your computer at your request and used by it only. If you pass the model file on, the \
GPL applies to that copy. Commercial use is subject to the GPL terms, including applicable source, licence and notice obligations on redistribution."
.into(),
},
source: Some("https://github.com/darktable-org/darktable-ai/tree/master/models/rawdenoise-nind".into()),
sha256: Some("da27509dab6a2915da67e988acd86cf71f9d5bbc8d1aa0ed32933578a887b901".into()),
size_bytes: Some(31_056_425),
provenance: "Trained on RawNIND, real photographs of the same scenes taken noisy and clean (images CC BY 4.0 and CC0, \
published on Wikimedia Commons and by the University of Louvain). Paper: arXiv 2501.08924."
.into(),
domain: Domain::BayerToRgb,
tile: 512,
overlap: 64,
gain: Gain::MatchMean { nominal: 1.0e6, max_deviation: 0.05 },
};
let download = Download {
id: manifest.id.clone(),
url: "https://github.com/darktable-org/darktable-ai/releases/download/release-5.6.0/rawdenoise-nind.dtmodel".into(),
file_name: "rawdenoise-nind.dtmodel".into(),
size_bytes: 57_700_134,
sha256: "d71b5f1e727c85a359e6f74dca9e2016c9d8fc3e2f7ac3e9b347d80ceca969af".into(),
};
Known { manifest, download: Some(download), entry: Some("model_bayer.onnx") }
}
/// Every model LightCraft knows.
pub fn all() -> Vec<Known> {
#[cfg(feature = "rawnind-model")]
{
vec![rawnind_bayer()]
}
#[cfg(not(feature = "rawnind-model"))]
{
Vec::new()
}
}
/// The known model with this id.
pub fn find(id: &str) -> Option<Known> {
all().into_iter().find(|k| k.manifest.id == id)
}
/// The known model whose `.onnx` has this SHA-256.
pub fn by_sha256(sha256: &str) -> Option<DenoiserManifest> {
all().into_iter().map(|k| k.manifest).find(|m| m.sha256.as_deref() == Some(sha256))
}
#[cfg(test)]
mod tests {
use super::*;
use crate::manifest::validate;
#[test]
fn known_models_are_valid_unique_and_downloaded_over_https() {
let all = all();
assert_eq!(all.is_empty(), !cfg!(feature = "rawnind-model"));
validate(&rawnind_bayer().manifest).unwrap();
for (i, k) in all.iter().enumerate() {
validate(&k.manifest).unwrap();
assert!(all.iter().skip(i + 1).all(|o| o.manifest.id != k.manifest.id), "unique ids");
assert!(!k.manifest.licence.name.is_empty() && !k.manifest.licence.notice.is_empty(), "terms are shown before use");
if let Some(d) = &k.download {
assert_eq!(d.id, k.manifest.id);
assert!(d.url.starts_with("https://") && d.sha256.len() == 64 && d.size_bytes > 0);
// a name that cannot leave the staging folder
assert!(!d.file_name.contains(['/', '\\']) && !d.file_name.starts_with('.'));
}
// a model that may not be used commercially is never offered for download
if k.manifest.licence.commercial == Commercial::No {
assert!(k.download.is_none(), "{}", k.manifest.id);
}
}
assert_eq!(find("rawnind-bayer").is_some(), cfg!(feature = "rawnind-model"));
assert!(find("nothing").is_none());
assert_eq!(
by_sha256("da27509dab6a2915da67e988acd86cf71f9d5bbc8d1aa0ed32933578a887b901").map(|m| m.id),
if cfg!(feature = "rawnind-model") { Some("rawnind-bayer".into()) } else { None }
);
}
}
#[derive(Clone, Debug, PartialEq)]
pub struct Download {
pub id: String,
pub url: String,
/// The name the file is kept under while it waits for the user to accept its terms.
pub file_name: String,
pub size_bytes: u64,
pub sha256: String,
}
pub fn host(url: &str) -> &str {
url.strip_prefix("https://").and_then(|r| r.split('/').next()).unwrap_or("")
}

16
crates/denoise-core/src/lib.rs Normal file
View File

@ -0,0 +1,16 @@
//! Model metadata, bounded archive/cache formats and Bayer tiling, independent of inference.
//! Used by featureless engine and web builds; no neural-network or matrix backend is linked.
#![forbid(unsafe_code)]
#![deny(clippy::unwrap_used, clippy::expect_used, clippy::panic, clippy::unimplemented, clippy::todo, clippy::unreachable)]
pub mod archive;
pub mod bayer;
pub mod hash;
pub mod known;
pub mod licence;
pub mod manifest;
pub mod product;
pub mod run;
pub mod tiles;
pub use manifest::{DenoiserManifest, Domain, Gain, ManifestError};
pub use run::{Control, Error, Params, TileRunner, denoise_bayer};

34
crates/denoise-core/src/licence.rs Normal file
View File

@ -0,0 +1,34 @@
use serde::{Deserialize, Serialize};
#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)]
#[serde(rename_all = "camelCase")]
pub enum Commercial {
Yes,
/// Research or personal use only. Never bundled or redistributed by LightCraft.
No,
/// The licence or the training data is unclear: the user is told so before installing.
Unknown,
}
#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
#[serde(default, rename_all = "camelCase")]
pub struct Licence {
/// SPDX id or a short name ("MIT", "Apache-2.0", "InsightFace non-commercial research").
pub name: String,
pub commercial: Commercial,
pub url: Option<String>,
/// Plain-language terms shown before the model is enabled.
pub notice: String,
}
impl Default for Licence {
fn default() -> Self {
Self { name: String::new(), commercial: Commercial::Unknown, url: None, notice: String::new() }
}
}
pub fn valid_id(id: &str) -> bool {
let mut bytes = id.bytes();
let first_ok = bytes.next().is_some_and(|b| b.is_ascii_lowercase() || b.is_ascii_digit());
first_ok && id.len() <= 64 && bytes.all(|b| b.is_ascii_lowercase() || b.is_ascii_digit() || matches!(b, b'.' | b'_' | b'-'))
}

217
crates/denoise-core/src/manifest.rs Normal file
View File

@ -0,0 +1,217 @@
//! A denoise model's manifest: what the model takes and gives, how big its tiles are and who may use it, as
//! plain data.
//!
//! It comes from a `denoise-model.json` next to the `.onnx` file or from the user's `catalog.json`. Both are
//! untrusted: [`validate`] must pass before a manifest is used, and [`parse`] applies it after reading JSON of
//! bounded size. The licence, id and address rules are the face models' (`crate::licence`).
use crate::licence::{Licence, valid_id};
use serde::{Deserialize, Serialize};
/// Largest `denoise-model.json` we read.
pub const MAX_MANIFEST_BYTES: usize = 64 * 1024;
/// Largest model file we accept (bytes).
pub const MAX_MODEL_BYTES: u64 = 2 * 1024 * 1024 * 1024;
/// What the model takes and what it gives.
#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)]
#[serde(rename_all = "camelCase")]
pub enum Domain {
/// The sensor's Bayer mosaic (black subtracted, white at 1, not white balanced) packed as four planes
/// `[R, G1, G2, B]` of its 2 × 2 cells, forced to RGGB; gives camera RGB at the mosaic's resolution, so
/// the picture is denoised and demosaiced in one step and must not be demosaiced again.
BayerToRgb,
}
/// How the model's output scale is brought back to the input's.
#[derive(Clone, Copy, Debug, PartialEq, Serialize, Deserialize)]
#[serde(tag = "kind", rename_all = "camelCase")]
pub enum Gain {
/// The output already has the input's scale.
None,
/// The output has a scale of its own, `nominal` times the input's (a million, for one model) and a little
/// different from tile to tile: each tile is scaled so that its channel means equal the input's. A tile too
/// dark to measure, or whose measured scale is further than `maxDeviation` (a fraction, 0.05 = 5 %) from
/// `nominal`, is scaled as if it were within that limit.
MatchMean { nominal: f32, max_deviation: f32 },
}
#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
#[serde(rename_all = "camelCase")]
pub struct DenoiserManifest {
/// Lowercase letters, digits, `.` `_` `-`.
pub id: String,
pub name: String,
pub version: String,
#[serde(default)]
pub licence: Licence,
/// Where the weights come from (an `https://` page or file), for the user's information.
#[serde(default)]
pub source: Option<String>,
/// Lowercase hex SHA-256 of the `.onnx` file, when known.
#[serde(default)]
pub sha256: Option<String>,
#[serde(default)]
pub size_bytes: Option<u64>,
/// What the model was trained on, in plain words; says "undisclosed" when it is.
#[serde(default)]
pub provenance: String,
pub domain: Domain,
/// Side of the square the model runs on, in packed cells (the file's fixed input size).
pub tile: u32,
/// Cells neighbouring tiles share; the picture is blended across them.
pub overlap: u32,
pub gain: Gain,
}
#[derive(Debug, PartialEq, thiserror::Error)]
pub enum ManifestError {
#[error("`{0}` is not valid: {1}")]
Field(&'static str, String),
#[error("not a denoise-model manifest: {0}")]
Parse(String),
#[error("a manifest may be at most {MAX_MANIFEST_BYTES} bytes")]
TooLarge,
}
fn bad<T>(field: &'static str, why: impl Into<String>) -> Result<T, ManifestError> {
Err(ManifestError::Field(field, why.into()))
}
fn text(field: &'static str, s: &str, min: usize, max: usize) -> Result<(), ManifestError> {
let n = s.chars().count();
if n < min || n > max {
return bad(field, format!("must be {min} to {max} characters"));
}
if s.chars().any(|c| c.is_control() && c != '\n') {
return bad(field, "must not contain control characters");
}
Ok(())
}
fn url(field: &'static str, u: &Option<String>) -> Result<(), ManifestError> {
match u {
None => Ok(()),
Some(u) if u.len() <= 500 && u.starts_with("https://") && !u.chars().any(|c| c.is_control() || c.is_whitespace()) => Ok(()),
Some(_) => bad(field, "must be an https:// address of at most 500 characters"),
}
}
/// Check a manifest from outside before trusting it.
pub fn validate(m: &DenoiserManifest) -> Result<(), ManifestError> {
if !valid_id(&m.id) {
return bad("id", "1 to 64 lowercase letters, digits, `.`, `_` or `-`, starting with a letter or digit");
}
text("name", &m.name, 1, 80)?;
text("version", &m.version, 1, 40)?;
text("provenance", &m.provenance, 0, 1000)?;
text("licence.name", &m.licence.name, 0, 80)?;
text("licence.notice", &m.licence.notice, 0, 2000)?;
url("licence.url", &m.licence.url)?;
url("source", &m.source)?;
if let Some(h) = &m.sha256
&& !(h.len() == 64 && h.bytes().all(|b| b.is_ascii_digit() || (b'a'..=b'f').contains(&b)))
{
return bad("sha256", "must be 64 lowercase hex digits");
}
if let Some(n) = m.size_bytes
&& (n == 0 || n > MAX_MODEL_BYTES)
{
return bad("sizeBytes", format!("must be between 1 and {MAX_MODEL_BYTES}"));
}
// a U-Net with four poolings wants a multiple of 16; nothing smaller than 64 or larger than 2048 is a tile
if !(64..=2048).contains(&m.tile) || !m.tile.is_multiple_of(16) {
return bad("tile", "must be a multiple of 16 from 64 to 2048");
}
// tiles must overlap by less than half, or one tile's blend ramps would run into its other neighbour's
if !m.overlap.is_multiple_of(2) || m.overlap < 16 || m.overlap.saturating_mul(2) >= m.tile {
return bad("overlap", "must be even, at least 16 and less than half the tile");
}
if let Gain::MatchMean { nominal, max_deviation } = m.gain {
if !(nominal.is_finite() && (1e-6..=1e12).contains(&nominal)) {
return bad("gain.nominal", "must be between 0.000001 and 1000000000000");
}
if !(max_deviation.is_finite() && (0.0..=0.5).contains(&max_deviation)) {
return bad("gain.maxDeviation", "must be between 0 and 0.5");
}
}
Ok(())
}
/// Read and validate a `denoise-model.json`.
pub fn parse(bytes: &[u8]) -> Result<DenoiserManifest, ManifestError> {
if bytes.len() > MAX_MANIFEST_BYTES {
return Err(ManifestError::TooLarge);
}
let bytes = bytes.strip_prefix(&[0xEF, 0xBB, 0xBF]).unwrap_or(bytes);
let m: DenoiserManifest = serde_json::from_slice(bytes).map_err(|e| ManifestError::Parse(e.to_string()))?;
validate(&m)?;
Ok(m)
}
#[cfg(test)]
mod tests {
use super::*;
pub(crate) fn sample() -> DenoiserManifest {
DenoiserManifest {
id: "nind-bayer".into(),
name: "Bayer denoiser".into(),
version: "1".into(),
licence: Licence::default(),
source: Some("https://example.org/model.onnx".into()),
sha256: None,
size_bytes: Some(31_000_000),
provenance: "trained on real raw noise pairs".into(),
domain: Domain::BayerToRgb,
tile: 512,
overlap: 64,
gain: Gain::MatchMean { nominal: 1.0e6, max_deviation: 0.05 },
}
}
#[test]
fn a_good_manifest_round_trips() {
let m = sample();
let json = serde_json::to_vec(&m).unwrap();
assert_eq!(parse(&json).unwrap(), m);
}
#[test]
fn bad_fields_are_named() {
let cases: Vec<(&str, Box<dyn Fn(&mut DenoiserManifest)>)> = vec![
("id", Box::new(|m| m.id = "../evil".into())),
("id", Box::new(|m| m.id = String::new())),
("name", Box::new(|m| m.name = "x\u{7}".into())),
("source", Box::new(|m| m.source = Some("http://insecure".into()))),
("sha256", Box::new(|m| m.sha256 = Some("XYZ".into()))),
("sizeBytes", Box::new(|m| m.size_bytes = Some(0))),
("tile", Box::new(|m| m.tile = 500)),
("tile", Box::new(|m| m.tile = 4096)),
("overlap", Box::new(|m| m.overlap = 300)),
("overlap", Box::new(|m| m.overlap = 63)),
("gain.maxDeviation", Box::new(|m| m.gain = Gain::MatchMean { nominal: 1.0, max_deviation: f32::NAN })),
("gain.maxDeviation", Box::new(|m| m.gain = Gain::MatchMean { nominal: 1.0, max_deviation: 2.0 })),
("gain.nominal", Box::new(|m| m.gain = Gain::MatchMean { nominal: 0.0, max_deviation: 0.1 })),
("gain.nominal", Box::new(|m| m.gain = Gain::MatchMean { nominal: f32::INFINITY, max_deviation: 0.1 })),
];
for (field, edit) in cases {
let mut m = sample();
edit(&mut m);
match validate(&m) {
Err(ManifestError::Field(f, _)) => assert_eq!(f, field),
other => panic!("{field}: {other:?}"),
}
}
}
#[test]
fn parse_is_bounded_and_tolerant() {
assert_eq!(parse(&vec![b' '; MAX_MANIFEST_BYTES + 1]), Err(ManifestError::TooLarge));
assert!(matches!(parse(b"{ nope"), Err(ManifestError::Parse(_))));
assert!(matches!(parse(b""), Err(ManifestError::Parse(_))));
let mut with_bom = vec![0xEF, 0xBB, 0xBF];
with_bom.extend(serde_json::to_vec(&sample()).unwrap());
assert!(parse(&with_bom).is_ok());
}
}

616
crates/denoise-core/src/product.rs Normal file
View File

@ -0,0 +1,616 @@
//! The denoised picture as cached data, and the blend the Amount slider controls.
//!
//! A product is camera RGB at the sensor's resolution (what the model gave, before the raw's own opcodes and
//! crop), kept as half floats in strips of rows, each strip byte-shuffled and deflated with its own checksum.
//! It is derived data: any problem reading it (missing, truncated, corrupt, made for another photo or model) is
//! just a cache miss, never a panic, and the picture can be made again.
//!
//! File: `LCDN`, version, header length, header JSON (`width`, `height`, `strip`, `key`), then for each strip its
//! compressed length and CRC-32, then the strips' bytes. `key` says what the product was made from (the photo's
//! content, the model, this code's version, the settings it depends on); a file with another key is stale.
use std::io::{Read, Write};
use std::path::Path;
use lightcraft_raster::Rgb32f;
use rayon::prelude::*;
use serde::{Deserialize, Serialize};
const MAGIC: &[u8; 4] = b"LCDN";
const VERSION: u8 = 1;
const MAX_HEADER: usize = 4096;
const MAX_SIDE: usize = 1 << 17;
/// Most pixels a product may have (the denoiser's own limit, [`crate::run::MAX_PIXELS`]).
const MAX_PIXELS: usize = crate::run::MAX_PIXELS;
/// Rows per strip: about a megapixel of three planes at 24 MP widths.
const STRIP_ROWS: usize = 128;
/// Bumped when the meaning of a product changes (included in keys by the caller).
pub const ALGORITHM: u32 = 1;
#[derive(Debug, thiserror::Error)]
pub enum ProductError {
#[error("the denoised picture could not be read or written: {0}")]
Io(String),
#[error("not a usable denoised picture: {0}")]
Format(String),
/// A product made from something else (another photo state, model or version).
#[error("the denoised picture is out of date")]
Stale,
}
impl From<std::io::Error> for ProductError {
fn from(e: std::io::Error) -> Self {
ProductError::Io(e.to_string())
}
}
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
pub struct Header {
pub width: usize,
pub height: usize,
pub strip: usize,
pub key: String,
}
fn allocate<T: Clone>(n: usize, value: T) -> Result<Vec<T>, ProductError> {
let mut v = Vec::new();
v.try_reserve_exact(n).map_err(|_| ProductError::Format("not enough memory for the denoised picture".into()))?;
v.resize(n, value);
Ok(v)
}
/// `f32` as IEEE half-float bits, rounded to nearest even; beyond the range it saturates, NaN becomes 0.
pub fn to_f16(v: f32) -> u16 {
if !v.is_finite() {
return if v.is_nan() {
0
} else if v > 0.0 {
0x7bff
} else {
0xfbff
};
}
let b = v.to_bits();
let sign = ((b >> 16) & 0x8000) as u16;
let exp = ((b >> 23) & 0xff) as i32 - 127 + 15;
let man = b & 0x7f_ffff;
if exp >= 31 {
return sign | 0x7bff;
}
if exp <= 0 {
if exp < -10 {
return sign;
}
let man = man | 0x80_0000;
let shift = (14 - exp) as u32;
let (half, rem, halfway) = (man >> shift, man & ((1u32 << shift) - 1), 1u32 << (shift - 1));
let mut h = half as u16;
if rem > halfway || (rem == halfway && h & 1 == 1) {
h += 1;
}
return sign | h;
}
let mut h = (((exp as u32) << 10) | (man >> 13)) as u16;
let rem = man & 0x1fff;
if rem > 0x1000 || (rem == 0x1000 && h & 1 == 1) {
h += 1;
}
if h >= 0x7c00 {
h = 0x7bff;
}
sign | h
}
/// Half-float bits as `f32`.
pub fn from_f16(h: u16) -> f32 {
let sign = if h & 0x8000 != 0 { -1.0f32 } else { 1.0 };
let exp = i32::from((h >> 10) & 0x1f);
let man = f32::from(h & 0x3ff);
match exp {
0 => sign * man * 2f32.powi(-24),
31 => sign * 65504.0,
_ => sign * (1.0 + man / 1024.0) * 2f32.powi(exp - 15),
}
}
fn crc(bytes: &[u8]) -> u32 {
// CRC-32 (IEEE), bitwise: strips are small and this runs once per strip
let mut table = [0u32; 256];
for (i, t) in table.iter_mut().enumerate() {
let mut c = i as u32;
for _ in 0..8 {
c = if c & 1 != 0 { 0xedb8_8320 ^ (c >> 1) } else { c >> 1 };
}
*t = c;
}
!bytes.iter().fold(!0u32, |c, &b| table[((c ^ u32::from(b)) & 0xff) as usize] ^ (c >> 8))
}
/// One strip as bytes: for each plane (R, G, B) the low bytes of its half floats, then the high bytes.
fn encode_strip(img: &Rgb32f, y0: usize, y1: usize) -> Result<Vec<u8>, ProductError> {
let n = (y1 - y0) * img.width;
let mut raw = allocate(6 * n, 0u8)?;
let rows = img.data.get(y0 * img.width..y1 * img.width).unwrap_or(&[]);
for (i, px) in rows.iter().enumerate() {
for (c, &v) in px.iter().enumerate() {
let h = to_f16(v);
if let Some(lo) = raw.get_mut(2 * c * n + i) {
*lo = h as u8;
}
if let Some(hi) = raw.get_mut((2 * c + 1) * n + i) {
*hi = (h >> 8) as u8;
}
}
}
Ok(miniz_oxide::deflate::compress_to_vec(&raw, 3))
}
/// Write `img` to `path` (through a temporary file next to it, so a reader never sees half a product).
pub fn write(path: &Path, img: &Rgb32f, key: &str) -> Result<(), ProductError> {
let pixels = img.width.checked_mul(img.height).filter(|&n| n <= MAX_PIXELS);
if img.width == 0 || img.height == 0 || img.width > MAX_SIDE || img.height > MAX_SIDE || pixels != Some(img.data.len()) {
return Err(ProductError::Format("the picture has no usable size".into()));
}
let strips: Vec<(usize, usize)> = (0..img.height).step_by(STRIP_ROWS).map(|y| (y, (y + STRIP_ROWS).min(img.height))).collect();
let packed: Vec<Vec<u8>> = strips.par_iter().map(|&(a, b)| encode_strip(img, a, b)).collect::<Result<_, _>>()?;
let header = serde_json::to_vec(&Header { width: img.width, height: img.height, strip: STRIP_ROWS, key: key.to_string() })
.map_err(|e| ProductError::Format(e.to_string()))?;
let tmp = path.with_extension("tmp");
let result = (|| -> Result<(), ProductError> {
if let Some(dir) = path.parent() {
std::fs::create_dir_all(dir)?;
}
let mut f = std::io::BufWriter::new(std::fs::File::create(&tmp)?);
f.write_all(MAGIC)?;
f.write_all(&[VERSION, 0, 0, 0])?;
f.write_all(&(header.len() as u32).to_le_bytes())?;
f.write_all(&header)?;
for p in &packed {
f.write_all(&(p.len() as u32).to_le_bytes())?;
f.write_all(&crc(p).to_le_bytes())?;
}
for p in &packed {
f.write_all(p)?;
}
f.flush()?;
drop(f);
std::fs::rename(&tmp, path)?;
Ok(())
})();
if result.is_err() {
let _ = std::fs::remove_file(&tmp);
}
result
}
/// The parsed start of a product file: header, each strip's (compressed length, CRC-32), and the file length the
/// start says the whole file has.
struct Start {
header: Header,
table: Vec<(usize, u32)>,
expected_len: u64,
}
fn read_start(r: &mut impl Read) -> Result<Start, ProductError> {
let mut fixed = [0u8; 12];
r.read_exact(&mut fixed)?;
if fixed.get(0..4) != Some(&MAGIC[..]) || fixed.get(4) != Some(&VERSION) {
return Err(ProductError::Format("not a denoised picture of this version".into()));
}
let len = fixed.get(8..12).and_then(|b| <[u8; 4]>::try_from(b).ok()).map(u32::from_le_bytes).unwrap_or(u32::MAX) as usize;
if len > MAX_HEADER {
return Err(ProductError::Format("the header is too large".into()));
}
let mut hb = allocate(len, 0u8)?;
r.read_exact(&mut hb)?;
let h: Header = serde_json::from_slice(&hb).map_err(|e| ProductError::Format(e.to_string()))?;
let pixels = h.width.saturating_mul(h.height);
if h.width == 0 || h.height == 0 || h.width > MAX_SIDE || h.height > MAX_SIDE || pixels > MAX_PIXELS || h.strip != STRIP_ROWS {
return Err(ProductError::Format("the size is not usable".into()));
}
let n = h.height.div_ceil(h.strip);
let mut table = Vec::with_capacity(n);
let mut total = 12 + len as u64 + 8 * n as u64;
for _ in 0..n {
let mut e = [0u8; 8];
r.read_exact(&mut e)?;
let l = u32::from_le_bytes([e[0], e[1], e[2], e[3]]) as usize;
let c = u32::from_le_bytes([e[4], e[5], e[6], e[7]]);
// a strip never compresses to much more than its raw size
if l > 12 * h.strip * h.width + 4096 {
return Err(ProductError::Format("a strip is larger than it can be".into()));
}
total += l as u64;
table.push((l, c));
}
Ok(Start { header: h, table, expected_len: total })
}
/// The header of the product at `path`, when it is one.
pub fn read_header_at(path: &Path) -> Result<Header, ProductError> {
let mut f = std::io::BufReader::new(std::fs::File::open(path)?);
Ok(read_start(&mut f)?.header)
}
/// Whether `path` holds a complete-looking product made from `key` (the header and the file's length are checked;
/// strips are checked when read).
pub fn is_current(path: &Path, key: &str) -> bool {
let check = || -> Result<bool, ProductError> {
let mut f = std::io::BufReader::new(std::fs::File::open(path)?);
let s = read_start(&mut f)?;
Ok(s.header.key == key && std::fs::metadata(path)?.len() == s.expected_len)
};
check().unwrap_or(false)
}
/// A rectangle of a product, in the product's pixels.
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub struct Window {
pub x: usize,
pub y: usize,
pub width: usize,
pub height: usize,
}
/// Half floats as `f32`, for all 65536 bit patterns (a table is much faster than computing each).
fn f16_table() -> &'static [f32] {
static TABLE: std::sync::OnceLock<Vec<f32>> = std::sync::OnceLock::new();
TABLE.get_or_init(|| (0..=u16::MAX).map(from_f16).collect())
}
/// Strips decoded together (they are inflated in parallel, then folded in order).
const GROUP: usize = 8;
/// One strip, checked and inflated: for each plane the low bytes of its half floats, then the high bytes.
fn decode_strip(index: usize, packed: &[u8], expect_crc: u32, n: usize) -> Result<Vec<u8>, ProductError> {
if crc(packed) != expect_crc {
return Err(ProductError::Format(format!("strip {index} is damaged")));
}
let raw = miniz_oxide::inflate::decompress_to_vec_with_limit(packed, 6 * n).map_err(|e| ProductError::Format(format!("strip {index}: {e:?}")))?;
if raw.len() != 6 * n {
return Err(ProductError::Format(format!("strip {index} has the wrong size")));
}
Ok(raw)
}
/// Read the product at `path`, made from `key`, with every `factor × factor` block of pixels averaged into one
/// (`factor` 1 = full size). The result is camera RGB.
///
/// With a `window`, only that rectangle is read and the blocks start at its corner; only whole blocks are made, so
/// the result is `window.width / factor × window.height / factor` (the same blocks a binned development of the same
/// crop makes). Without one the whole product is read and blocks at the right and bottom edges may be smaller.
///
/// With `clip` (white is 1), a block in which any sample of a colour is at or above it takes that colour's
/// maximum instead of the mean, so a clipped highlight stays clipped for highlight reconstruction.
pub fn read_window(path: &Path, key: &str, window: Option<Window>, factor: usize, clip: Option<f32>) -> Result<Rgb32f, ProductError> {
let factor = factor.clamp(1, 1 << 12);
let mut f = std::io::BufReader::new(std::fs::File::open(path)?);
let Start { header: h, table, expected_len } = read_start(&mut f)?;
if h.key != key {
return Err(ProductError::Stale);
}
if std::fs::metadata(path)?.len() != expected_len {
return Err(ProductError::Format("the file is not the length its header says".into()));
}
let w = window.unwrap_or(Window { x: 0, y: 0, width: h.width, height: h.height });
let fits = |a: usize, b: usize, limit: usize| a.checked_add(b).is_some_and(|e| e <= limit);
if w.width == 0 || w.height == 0 || !fits(w.x, w.width, h.width) || !fits(w.y, w.height, h.height) {
return Err(ProductError::Format("the window does not lie inside the picture".into()));
}
// whole blocks for a window; partial ones at the edges when the whole picture is read
let whole = window.is_none();
let (ow, oh) = if whole { (w.width.div_ceil(factor), w.height.div_ceil(factor)) } else { (w.width / factor, w.height / factor) };
if ow == 0 || oh == 0 {
return Err(ProductError::Format("the window is smaller than one block".into()));
}
// the picture rows and columns the blocks cover
let (x_end, y_end) = if whole { (w.x + w.width, w.y + w.height) } else { (w.x + ow * factor, w.y + oh * factor) };
let mut out = Rgb32f { width: ow, height: oh, data: allocate(ow * oh, [0.0; 3])? };
let lut = f16_table();
let value = |raw: &[u8], c: usize, n: usize, i: usize| -> f32 {
let (lo, hi) = (raw.get(2 * c * n + i).copied().unwrap_or(0), raw.get((2 * c + 1) * n + i).copied().unwrap_or(0));
lut.get(usize::from(lo) | usize::from(hi) << 8).copied().unwrap_or(0.0)
};
// which strips are needed, and where each starts in the file (after the header and the strip table)
let wanted = |s: usize| s * h.strip < y_end && (s + 1) * h.strip > w.y;
let (mut sum, mut peak) = (allocate(ow, [0f32; 3])?, allocate(ow, [f32::MIN; 3])?);
let (mut rows_in, mut oy) = (0usize, 0usize);
let strips: Vec<usize> = (0..table.len()).filter(|&s| wanted(s)).collect();
// skip the strips before the first one wanted
let skipped: usize = table.iter().take(strips.first().copied().unwrap_or(0)).map(|e| e.0).sum();
f.seek_relative(i64::try_from(skipped).map_err(|_| ProductError::Format("the file is too large".into()))?)?;
for group in strips.chunks(GROUP) {
let mut packed = Vec::with_capacity(group.len());
for &s in group {
let Some(&(len, _)) = table.get(s) else { return Err(ProductError::Format("a strip is missing".into())) };
let mut p = allocate(len, 0u8)?;
f.read_exact(&mut p)?;
packed.push(p);
}
let decoded: Vec<Result<Vec<u8>, ProductError>> = group
.par_iter()
.zip(packed.par_iter())
.map(|(&s, p)| {
let rows = h.height.saturating_sub(s * h.strip).min(h.strip);
decode_strip(s, p, table.get(s).map_or(0, |e| e.1), rows * h.width)
})
.collect();
for (&s, raw) in group.iter().zip(decoded) {
let raw = raw?;
let y0 = s * h.strip;
let rows = h.height.saturating_sub(y0).min(h.strip);
let n = rows * h.width;
for r in 0..rows {
let y = y0 + r;
if y < w.y || y >= y_end {
continue;
}
let base = r * h.width;
for (ox, (sm, pk)) in sum.iter_mut().zip(peak.iter_mut()).enumerate() {
let x0 = w.x + ox * factor;
let x1 = (x0 + factor).min(x_end);
for x in x0..x1 {
for c in 0..3 {
let v = value(&raw, c, n, base + x);
if let (Some(s_), Some(p_)) = (sm.get_mut(c), pk.get_mut(c)) {
*s_ += v;
*p_ = p_.max(v);
}
}
}
}
rows_in += 1;
if rows_in == factor || y + 1 == y_end {
for (ox, (sm, pk)) in sum.iter_mut().zip(peak.iter_mut()).enumerate() {
// the block's size at the right and bottom edges may be smaller
let bw = (x_end - (w.x + ox * factor)).min(factor);
let k = 1.0 / (rows_in * bw).max(1) as f32;
if let Some(o) = out.data.get_mut(oy * ow + ox) {
for c in 0..3 {
let (s_, p_) = (sm.get(c).copied().unwrap_or(0.0), pk.get(c).copied().unwrap_or(0.0));
if let Some(slot) = o.get_mut(c) {
*slot = if factor > 1 && clip.is_some_and(|t| p_ >= t) { p_ } else { s_ * k };
}
}
}
*sm = [0.0; 3];
*pk = [f32::MIN; 3];
}
rows_in = 0;
oy += 1;
}
}
}
}
Ok(out)
}
/// [`read_window`] of the whole product: every `factor × factor` block averaged into one.
pub fn read(path: &Path, key: &str, factor: usize) -> Result<Rgb32f, ProductError> {
read_window(path, key, None, factor, None)
}
/// `base + (denoised − base) · amount` for every pixel (`amount` 0 to 1), in place in `base`. The two pictures must
/// be the same size; if they are not, `base` is left as it was and `false` is returned.
pub fn blend(base: &mut Rgb32f, denoised: &Rgb32f, amount: f32) -> bool {
if base.width != denoised.width || base.height != denoised.height {
return false;
}
let a = if amount.is_finite() { amount.clamp(0.0, 1.0) } else { 0.0 };
base.data.par_iter_mut().zip(denoised.data.par_iter()).for_each(|(b, d)| {
for c in 0..3 {
b[c] += (d[c] - b[c]) * a;
}
});
true
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn oversized_product_header_is_rejected_before_allocating_pixels() {
let p = temp("oversized-header");
let header = serde_json::to_vec(&Header { width: 10001, height: 10000, strip: STRIP_ROWS, key: "k".into() }).unwrap();
let mut bytes = Vec::from(MAGIC.as_slice());
bytes.extend_from_slice(&[VERSION, 0, 0, 0]);
bytes.extend_from_slice(&(header.len() as u32).to_le_bytes());
bytes.extend_from_slice(&header);
std::fs::write(&p, bytes).unwrap();
assert!(read_header_at(&p).is_err());
let _ = std::fs::remove_file(p);
}
fn temp(name: &str) -> std::path::PathBuf {
std::env::temp_dir().join(format!("lc-denoise-{name}-{}.lcdn", std::process::id()))
}
fn picture(w: usize, h: usize) -> Rgb32f {
Rgb32f {
width: w,
height: h,
data: (0..w * h).map(|i| [(i % w) as f32 / w as f32, (i / w) as f32 / h as f32, 0.001 + 0.5 * ((i % 7) as f32 / 7.0)]).collect(),
}
}
#[test]
fn half_floats_round_trip_closely() {
for v in [0.0f32, 1.0, -1.0, 0.5, 0.333_333, 1e-4, 6.1e-5, 3e-6, 123.456, 65000.0, -0.002] {
let back = from_f16(to_f16(v));
assert!((back - v).abs() <= v.abs() * 0.001 + 1e-7, "{v} -> {back}");
}
assert_eq!(from_f16(to_f16(1e10)), 65504.0);
assert_eq!(to_f16(f32::NAN), 0);
assert_eq!(from_f16(to_f16(f32::NEG_INFINITY)), -65504.0);
}
#[test]
fn a_product_survives_a_round_trip_at_full_and_reduced_size() {
let (w, h) = (301, 517);
let img = picture(w, h);
let p = temp("round");
write(&p, &img, "k1").unwrap();
assert!(is_current(&p, "k1") && !is_current(&p, "k2"));
assert_eq!(read_header_at(&p).unwrap().width, w);
let full = read(&p, "k1", 1).unwrap();
assert_eq!((full.width, full.height), (w, h));
for (a, b) in full.data.iter().zip(&img.data) {
for c in 0..3 {
assert!((a[c] - b[c]).abs() < 0.002, "{a:?} vs {b:?}");
}
}
// averaged into blocks of 4: the sizes round up, and the mean of the picture is kept
let small = read(&p, "k1", 4).unwrap();
assert_eq!((small.width, small.height), (w.div_ceil(4), h.div_ceil(4)));
let mean = |i: &Rgb32f| i.data.iter().map(|p| f64::from(p[1])).sum::<f64>() / i.data.len() as f64;
assert!((mean(&small) - mean(&img)).abs() < 0.01);
// a factor that does not divide the strip height is the same average
let odd = read(&p, "k1", 5).unwrap();
assert_eq!((odd.width, odd.height), (w.div_ceil(5), h.div_ceil(5)));
assert!((mean(&odd) - mean(&img)).abs() < 0.01);
assert!(matches!(read(&p, "other", 1), Err(ProductError::Stale)));
std::fs::remove_file(&p).unwrap();
}
/// What binning a window of `img` by `k` means, written the slow obvious way.
fn blocks(img: &Rgb32f, win: Window, k: usize, clip: Option<f32>) -> Rgb32f {
let (ow, oh) = (win.width / k, win.height / k);
let mut out = Rgb32f { width: ow, height: oh, data: vec![[0.0; 3]; ow * oh] };
for by in 0..oh {
for bx in 0..ow {
for c in 0..3 {
let vals: Vec<f32> = (0..k * k).map(|i| img.data[(win.y + by * k + i / k) * img.width + win.x + bx * k + i % k][c]).collect();
let max = vals.iter().copied().fold(f32::MIN, f32::max);
out.data[by * ow + bx][c] =
if k > 1 && clip.is_some_and(|t| max >= t) { max } else { vals.iter().sum::<f32>() / vals.len() as f32 };
}
}
}
out
}
#[test]
fn a_window_is_read_as_the_blocks_of_that_crop() {
let (w, h) = (301, 517);
let mut img = picture(w, h);
// a clipped sample in the middle of a block
img.data[(130 + 2) * w + 61 + 1] = [1.0, 0.4, 0.4];
let p = temp("window");
write(&p, &img, "k1").unwrap();
let near = |a: &Rgb32f, b: &Rgb32f| {
assert_eq!((a.width, a.height), (b.width, b.height));
for (x, y) in a.data.iter().zip(&b.data) {
for c in 0..3 {
assert!((x[c] - y[c]).abs() < 0.003, "{x:?} vs {y:?}");
}
}
};
// windows at odd places, across strip boundaries (128 rows), with factors that do and do not divide them
for (win, k) in [
(Window { x: 3, y: 5, width: 200, height: 300 }, 1),
(Window { x: 3, y: 5, width: 200, height: 300 }, 4),
(Window { x: 61, y: 130, width: 121, height: 250 }, 5),
(Window { x: 0, y: 0, width: w, height: h }, 2),
(Window { x: 1, y: 380, width: 90, height: 137 }, 3),
] {
let got = read_window(&p, "k1", Some(win), k, None).unwrap();
near(&got, &blocks(&img, win, k, None));
}
// with a clip level the block with the clipped sample keeps its highest value, the others stay means
let win = Window { x: 61, y: 130, width: 40, height: 20 };
let clipped = read_window(&p, "k1", Some(win), 4, Some(0.99)).unwrap();
near(&clipped, &blocks(&img, win, 4, Some(0.99)));
assert!(clipped.data[0][0] > 0.99, "the clipped block stays clipped: {:?}", clipped.data[0]);
let plain = read_window(&p, "k1", Some(win), 4, None).unwrap();
assert!(plain.data[0][0] < 0.99, "an average falls below the clip level");
// windows that do not fit, or are smaller than a block, are errors
for bad in [
Window { x: 0, y: 0, width: 0, height: 5 },
Window { x: 290, y: 0, width: 20, height: 5 },
Window { x: 0, y: 510, width: 5, height: 10 },
Window { x: usize::MAX, y: 0, width: 2, height: 2 },
Window { x: 0, y: 0, width: 3, height: 3 },
] {
assert!(read_window(&p, "k1", Some(bad), 4, None).is_err(), "{bad:?}");
}
std::fs::remove_file(&p).unwrap();
}
#[test]
fn a_smooth_picture_compresses_well() {
let img = Rgb32f {
width: 512,
height: 512,
data: (0..512 * 512)
.map(|i| {
let v = 0.2 + 0.0001 * (i % 512) as f32;
[v, v * 0.8, v * 0.6]
})
.collect(),
};
let p = temp("size");
write(&p, &img, "k").unwrap();
let bytes = std::fs::metadata(&p).unwrap().len() as usize;
assert!(bytes < 512 * 512 * 6 / 2, "{bytes} bytes for a smooth picture");
std::fs::remove_file(&p).unwrap();
}
#[test]
fn damaged_files_are_errors_not_panics() {
let img = picture(100, 300);
let p = temp("damage");
write(&p, &img, "k").unwrap();
let good = std::fs::read(&p).unwrap();
// truncated at every length
for cut in (0..good.len()).step_by(37) {
std::fs::write(&p, &good[..cut]).unwrap();
assert!(read(&p, "k", 1).is_err(), "cut at {cut}");
assert!(!is_current(&p, "k"));
}
// a flipped byte anywhere is caught by a checksum, the header parse, or the key
for at in (0..good.len()).step_by(53) {
let mut bad = good.clone();
bad[at] ^= 0xff;
std::fs::write(&p, &bad).unwrap();
let _ = read(&p, "k", 1); // must not panic; most are errors
}
// a header that claims an absurd size
let mut huge = b"LCDN\x01\0\0\0".to_vec();
let hdr = br#"{"width":4000000000,"height":4000000000,"strip":128,"key":"k"}"#;
huge.extend((hdr.len() as u32).to_le_bytes());
huge.extend(hdr);
std::fs::write(&p, &huge).unwrap();
assert!(matches!(read(&p, "k", 1), Err(ProductError::Format(_))));
std::fs::write(&p, b"").unwrap();
assert!(read(&p, "k", 1).is_err());
assert!(read(&temp("missing"), "k", 1).is_err());
std::fs::remove_file(&p).unwrap();
}
#[test]
fn writing_rejects_unusable_pictures() {
let p = temp("bad");
assert!(write(&p, &Rgb32f { width: 0, height: 0, data: vec![] }, "k").is_err());
assert!(write(&p, &Rgb32f { width: 3, height: 3, data: vec![[0.0; 3]; 8] }, "k").is_err());
assert!(!p.exists());
}
#[test]
fn blend_mixes_by_amount_and_rejects_other_sizes() {
let mut base = Rgb32f { width: 2, height: 1, data: vec![[0.0; 3], [1.0; 3]] };
let dn = Rgb32f { width: 2, height: 1, data: vec![[1.0; 3], [0.0; 3]] };
assert!(blend(&mut base, &dn, 0.25));
assert_eq!(base.data, vec![[0.25; 3], [0.75; 3]]);
assert!(blend(&mut base, &dn, f32::NAN));
assert_eq!(base.data, vec![[0.25; 3], [0.75; 3]]);
assert!(!blend(&mut base, &Rgb32f { width: 1, height: 1, data: vec![[0.0; 3]] }, 1.0));
assert!(blend(&mut base, &dn, 1.0));
assert_eq!(base.data, dn.data);
}
#[test]
fn crc_matches_the_standard_check_value() {
assert_eq!(crc(b"123456789"), 0xcbf4_3926);
}
}

589
crates/denoise-core/src/run.rs Normal file
View File

@ -0,0 +1,589 @@
//! Denoising a whole Bayer mosaic with a tile model: cut it into overlapping tiles, run the model on each, bring
//! each tile's scale back to the input's, keep clipped highlights as they were, and blend the tiles together.
//!
//! The model itself is a [`TileRunner`], so everything around it (packing, tiling, scale matching, highlight
//! protection, blending, cancelling, progress) is tested without one.
use std::sync::atomic::{AtomicBool, Ordering};
use lightcraft_raster::Rgb32f;
use rayon::prelude::*;
use crate::bayer::{Layout, pack_tile};
use crate::manifest::Gain;
use crate::tiles::{starts, weight};
/// Most pixels one picture may have (guards the allocations that follow from its size).
pub const MAX_PIXELS: usize = 100_000_000;
/// A tile whose mean signal is below this (white = 1) is too dark to measure its scale from.
const MIN_SIGNAL: f32 = 0.003;
#[derive(Debug, thiserror::Error)]
pub enum Error {
#[error("{0}")]
Input(String),
#[error("the model gave {0}")]
Model(String),
#[error("the model could not run: {0}")]
Runtime(String),
#[error("the model could not be loaded: {0}")]
Load(String),
#[error("cancelled")]
Cancelled,
}
fn allocate<T: Clone>(n: usize, value: T) -> Result<Vec<T>, Error> {
let mut v = Vec::new();
v.try_reserve_exact(n).map_err(|_| Error::Input("not enough memory for denoise".into()))?;
v.resize(n, value);
Ok(v)
}
/// Something that runs the model on one tile.
pub trait TileRunner: Sync {
/// `input` is the tile as four planes `[R, G1, G2, B]` of `tile × tile` cells (NCHW, batch 1); the answer is
/// camera RGB, three planes of `2·tile × 2·tile` pixels (CHW), in the model's own scale.
fn run(&self, input: &[f32]) -> Result<Vec<f32>, Error>;
}
pub struct Params {
/// Side of the model's square, in cells.
pub tile: usize,
/// Cells neighbouring tiles share.
pub overlap: usize,
pub gain: Gain,
/// White is 1; samples at or above this (minus a little) are clipped and keep their value.
pub clip: Option<f32>,
/// Tiles run at once (each model call may itself be single-threaded).
pub parallel: usize,
}
#[derive(Default)]
pub struct Control<'a> {
pub cancel: Option<&'a AtomicBool>,
/// `(tiles done, tiles in all)`.
pub progress: Option<&'a (dyn Fn(usize, usize) + Sync)>,
}
/// The tile grid of a picture: `(starts across, starts down)`.
pub fn grid(width: usize, height: usize, layout: Layout, tile: usize, overlap: usize) -> (Vec<usize>, Vec<usize>) {
let (nx, ny) = layout.cells(width, height);
(starts(nx, tile, overlap), starts(ny, tile, overlap))
}
/// How many model calls [`denoise_bayer`] makes for a picture.
pub fn tile_count(width: usize, height: usize, layout: Layout, tile: usize, overlap: usize) -> usize {
let (sx, sy) = grid(width, height, layout, tile, overlap);
sx.len().saturating_mul(sy.len())
}
/// Denoise (and demosaic) the normalised mosaic `mosaic` of `width × height` samples, laid out as `layout`.
/// Gives camera RGB at the mosaic's resolution.
pub fn denoise_bayer(
mosaic: &[f32],
width: usize,
height: usize,
layout: Layout,
runner: &dyn TileRunner,
p: &Params,
ctl: &Control<'_>,
) -> Result<Rgb32f, Error> {
let pixels = width
.checked_mul(height)
.filter(|&n| n > 0 && n <= MAX_PIXELS)
.ok_or_else(|| Error::Input("the picture has no size, or is too large".into()))?;
if mosaic.len() != pixels {
return Err(Error::Input("the mosaic does not have width × height samples".into()));
}
if !(16..=2048).contains(&p.tile) || p.overlap >= p.tile.div_ceil(2) || layout.dx > 1 || layout.dy > 1 {
return Err(Error::Input("the tile size and overlap do not fit".into()));
}
if let Gain::MatchMean { nominal, max_deviation } = p.gain
&& !(nominal.is_finite() && (1e-6..=1e12).contains(&nominal) && max_deviation.is_finite() && (0.0..=0.5).contains(&max_deviation))
{
return Err(Error::Input("the model gain is not finite or within its supported range".into()));
}
if p.clip.is_some_and(|v| !v.is_finite() || v <= 0.0) {
return Err(Error::Input("the highlight clip must be finite and positive".into()));
}
let (nx, ny) = layout.cells(width, height);
let (sx, sy) = (starts(nx, p.tile, p.overlap), starts(ny, p.tile, p.overlap));
let (cols, rows) = (sx.len(), sy.len());
if cols == 0 || rows == 0 {
return Err(Error::Input("the picture needs too many tiles".into()));
}
let tiles: Vec<(usize, usize, bool, bool, bool, bool)> = sy
.iter()
.enumerate()
.flat_map(|(j, &y0)| sx.iter().enumerate().map(move |(i, &x0)| (x0, y0, i > 0, i + 1 < cols, j > 0, j + 1 < rows)))
.collect();
let total = tiles.len();
let mut acc = allocate(pixels, [0f32; 3])?;
let mut wsum = allocate(nx * ny, 0f32)?;
let group = p.parallel.clamp(1, 4);
let mut done = 0;
// where the time goes, printed under `LIGHTCRAFT_PROFILE`
let (mut t_wait, mut t_total) = (std::time::Duration::ZERO, std::time::Duration::ZERO);
let began = web_time::Instant::now();
// One group of tiles at a time: pack, run the model, check and bring each to the input's scale, in parallel.
// Blending (sequential: the tiles add into one picture, in tile order so the result does not depend on timing) of
// the previous group runs while the next group is on the model, so neither the card nor the cores wait for it.
type Tile = (usize, usize, bool, bool, bool, bool);
let compute = |chunk: &[Tile]| -> Vec<Result<Vec<f32>, Error>> {
chunk
.par_iter()
.map(|&(x0, y0, bx, ax, by, ay)| {
let mut input = allocate(4 * p.tile * p.tile, 0f32)?;
if !pack_tile(mosaic, width, height, layout, x0 as isize, y0 as isize, p.tile, &mut input) {
return Err(Error::Input("a tile could not be packed".into()));
}
let mut out = runner.run(&input)?;
let side = 2 * p.tile;
if out.len() != 3 * side * side {
return Err(Error::Model(format!("{} numbers for a tile, not the {} expected", out.len(), 3 * side * side)));
}
if out.iter().any(|v| !v.is_finite()) {
return Err(Error::Model("numbers that are not finite".into()));
}
finish_tile(&input, &mut out, p, (bx, ax, by, ay));
Ok(out)
})
.collect()
};
let mut blend = |chunk: &[Tile], outs: Vec<Result<Vec<f32>, Error>>| -> Result<(), Error> {
for (&(x0, y0, bx, ax, by, ay), r) in chunk.iter().zip(outs) {
blend_tile(&r?, &mut acc, &mut wsum, (width, height), (nx, ny), layout, (x0, y0), p, (bx, ax, by, ay));
done += 1;
if let Some(f) = ctl.progress {
f(done, total);
}
}
Ok(())
};
let mut previous: Option<(&[Tile], Vec<Result<Vec<f32>, Error>>)> = None;
for chunk in tiles.chunks(group) {
if ctl.cancel.is_some_and(|c| c.load(Ordering::Relaxed)) {
return Err(Error::Cancelled);
}
let (outs, blended) = rayon::join(
|| compute(chunk),
|| match previous.take() {
Some((c, o)) => {
let started = web_time::Instant::now();
let r = blend(c, o);
(r, started.elapsed())
}
None => (Ok(()), std::time::Duration::ZERO),
},
);
blended.0?;
t_wait += blended.1;
previous = Some((chunk, outs));
}
if let Some((c, o)) = previous.take() {
let started = web_time::Instant::now();
blend(c, o)?;
t_wait += started.elapsed();
}
t_total += began.elapsed();
let started = web_time::Instant::now();
// normalise by the weights
let (ox, oy) = (layout.position(0, 0).0, layout.position(0, 0).1);
acc.par_chunks_mut(width).enumerate().for_each(|(y, row)| {
let cy = ((y as isize - oy) / 2) as usize;
for (x, px) in row.iter_mut().enumerate() {
let cx = ((x as isize - ox) / 2) as usize;
let w = wsum.get(cy * nx + cx).copied().unwrap_or(0.0);
if w > 0.0 {
*px = px.map(|v| v / w);
}
}
});
if std::env::var_os("LIGHTCRAFT_PROFILE").is_some() {
eprintln!(
"[denoise] {total} tiles, {} at once: tiles {:.0} ms (blending, {:.0} ms of it, overlaps the model), normalise {:.0} ms",
group,
t_total.as_secs_f64() * 1e3,
t_wait.as_secs_f64() * 1e3,
started.elapsed().as_secs_f64() * 1e3
);
}
Ok(Rgb32f { width, height, data: acc })
}
/// Bring a tile's output to the input's scale and keep clipped highlights as they were (in place).
fn finish_tile(input: &[f32], out: &mut [f32], p: &Params, edges: (bool, bool, bool, bool)) {
let t = p.tile;
let side = 2 * t;
if let Gain::MatchMean { nominal, max_deviation } = p.gain {
// the part of the tile whose scale is measured: away from the edges that blend into a neighbour
let m = p.overlap / 2;
let (i0, i1) = (if edges.0 { m } else { 0 }, t - if edges.1 { m } else { 0 });
let (j0, j1) = (if edges.2 { m } else { 0 }, t - if edges.3 { m } else { 0 });
let cells = ((i1 - i0) * (j1 - j0)) as f64;
let plane_sum = |pl: usize| -> f64 {
(j0..j1)
.map(|j| input.get(pl * t * t + j * t + i0..pl * t * t + j * t + i1).map_or(0.0, |r| r.iter().map(|&v| f64::from(v)).sum::<f64>()))
.sum()
};
let in_mean = [plane_sum(0) / cells, (plane_sum(1) + plane_sum(2)) / (2.0 * cells), plane_sum(3) / cells];
for (c, &mean_in) in in_mean.iter().enumerate() {
let sum_out: f64 = (2 * j0..2 * j1)
.map(|y| {
out.get(c * side * side + y * side + 2 * i0..c * side * side + y * side + 2 * i1)
.map_or(0.0, |r| r.iter().map(|&v| f64::from(v)).sum::<f64>())
})
.sum();
let mean_out = sum_out / (4.0 * cells);
let measured = if mean_in as f32 >= MIN_SIGNAL && mean_out > 0.0 { (mean_out / mean_in) as f32 } else { nominal };
let g = measured.clamp(nominal * (1.0 - max_deviation), nominal * (1.0 + max_deviation));
if g.is_finite() && g > 0.0 {
let inv = 1.0 / g;
if let Some(plane) = out.get_mut(c * side * side..(c + 1) * side * side) {
plane.iter_mut().for_each(|v| *v *= inv);
}
}
}
}
if let Some(clip) = p.clip {
protect_clipped(input, out, t, clip);
}
}
/// Where the mosaic is at (or within a cell of) white, the output becomes the plain cell values again, so a clipped
/// area stays exactly as clipped as the camera recorded it (highlight recovery later depends on that).
fn protect_clipped(input: &[f32], out: &mut [f32], t: usize, clip: f32) {
let side = 2 * t;
let plane = t * t;
let (Some(r), Some(g1), Some(g2), Some(b)) =
(input.get(0..plane), input.get(plane..2 * plane), input.get(2 * plane..3 * plane), input.get(3 * plane..4 * plane))
else {
return;
};
let mut m: Vec<f32> = (0..plane).map(|i| r[i].max(g1[i]).max(g2[i]).max(b[i])).collect();
if !m.iter().any(|&v| v >= clip) {
return;
}
// 3×3 maximum over cells (separable)
let dilate = |src: &[f32], horizontal: bool| -> Vec<f32> {
(0..plane)
.map(|i| {
let (x, y) = (i % t, i / t);
let mut best = src[i];
for d in [-1isize, 1] {
let (nx, ny) = if horizontal { (x as isize + d, y as isize) } else { (x as isize, y as isize + d) };
if nx >= 0 && ny >= 0 && (nx as usize) < t && (ny as usize) < t {
best = best.max(src[ny as usize * t + nx as usize]);
}
}
best
})
.collect()
};
m = dilate(&dilate(&m, true), false);
let lo = clip - 0.03;
for j in 0..t {
for i in 0..t {
let k = j * t + i;
let s = ((m[k] - lo) / (clip - lo)).clamp(0.0, 1.0);
if s <= 0.0 {
continue;
}
let s = s * s * (3.0 - 2.0 * s);
let plain = [r[k], 0.5 * (g1[k] + g2[k]), b[k]];
for (c, &pv) in plain.iter().enumerate() {
for (sy, sx) in [(0, 0), (0, 1), (1, 0), (1, 1)] {
if let Some(v) = out.get_mut(c * side * side + (2 * j + sy) * side + 2 * i + sx) {
*v += (pv - *v) * s;
}
}
}
}
}
}
/// Add a finished tile to the picture with its blend weights.
#[allow(clippy::too_many_arguments)]
fn blend_tile(
out: &[f32],
acc: &mut [[f32; 3]],
wsum: &mut [f32],
size: (usize, usize),
cells: (usize, usize),
layout: Layout,
at: (usize, usize),
p: &Params,
edges: (bool, bool, bool, bool),
) {
let (width, height) = size;
let (nx, ny) = cells;
let (t, side) = (p.tile, 2 * p.tile);
let fade = p.overlap;
let wx: Vec<f32> = (0..t).map(|i| weight(i, t, fade, edges.0, edges.1)).collect();
let wy: Vec<f32> = (0..t).map(|j| weight(j, t, fade, edges.2, edges.3)).collect();
for j in 0..t {
let cy = at.1 + j;
for i in 0..t {
let cx = at.0 + i;
// cells past the picture's edge (a picture smaller than one tile is padded) belong to no pixel
if cx >= nx || cy >= ny {
continue;
}
let (Some(&a), Some(&b)) = (wx.get(i), wy.get(j)) else { continue };
let w = a * b;
if let Some(s) = wsum.get_mut(cy * nx + cx) {
*s += w;
}
let (px0, py0) = layout.position(cx as isize, cy as isize);
for (sy, sx) in [(0isize, 0isize), (0, 1), (1, 0), (1, 1)] {
let (x, y) = (px0 + sx, py0 + sy);
if x < 0 || y < 0 || x as usize >= width || y as usize >= height {
continue;
}
let o = (2 * j + sy as usize) * side + 2 * i + sx as usize;
let (Some(&r), Some(&g), Some(&bl)) = (out.get(o), out.get(side * side + o), out.get(2 * side * side + o)) else { continue };
if let Some(px) = acc.get_mut(y as usize * width + x as usize) {
px[0] += w * r;
px[1] += w * g;
px[2] += w * bl;
}
}
}
}
}
#[cfg(test)]
mod tests {
use super::*;
/// A stand-in model: the cell's samples as an RGB picture at twice the resolution (a plain 2 × 2 "demosaic"),
/// times `scale`, so the output has a scale of its own like the real model's.
struct Mock {
tile: usize,
scale: f32,
}
impl TileRunner for Mock {
fn run(&self, input: &[f32]) -> Result<Vec<f32>, Error> {
let (t, side) = (self.tile, 2 * self.tile);
let plane = t * t;
let mut out = vec![0.0; 3 * side * side];
for j in 0..t {
for i in 0..t {
let k = j * t + i;
let rgb = [input[k], 0.5 * (input[plane + k] + input[2 * plane + k]), input[3 * plane + k]];
for (c, v) in rgb.iter().enumerate() {
for (sy, sx) in [(0, 0), (0, 1), (1, 0), (1, 1)] {
out[c * side * side + (2 * j + sy) * side + 2 * i + sx] = v * self.scale;
}
}
}
}
Ok(out)
}
}
fn params(tile: usize, overlap: usize, gain: Gain) -> Params {
Params { tile, overlap, gain, clip: None, parallel: 2 }
}
#[test]
fn overflowing_tiles_and_nan_gain_are_errors_before_the_model_runs() {
let layout = Layout::from_cell([0, 1, 1, 2]).unwrap();
let model = Mock { tile: 16, scale: 1.0 };
for (tile, overlap, gain) in [
(usize::MAX, 0, Gain::None),
(16, usize::MAX, Gain::None),
(16, 0, Gain::MatchMean { nominal: 1.0, max_deviation: f32::NAN }),
(16, 0, Gain::MatchMean { nominal: f32::INFINITY, max_deviation: 0.1 }),
] {
assert!(matches!(denoise_bayer(&[0.5], 1, 1, layout, &model, &params(tile, overlap, gain), &Control::default()), Err(Error::Input(_))));
}
}
/// A smooth picture whose colour at a sample is `base[colour] · (1 + shading)`.
fn mosaic(w: usize, h: usize, cell: [u8; 4]) -> Vec<f32> {
(0..w * h)
.map(|i| {
let (x, y) = (i % w, i / w);
let c = cell[(y % 2) * 2 + x % 2] as usize;
[0.5, 0.3, 0.2][c] * (0.6 + 0.4 * ((x as f32 / 23.0).sin() * (y as f32 / 31.0).cos()))
})
.collect()
}
fn reference(m: &[f32], w: usize, h: usize, layout: Layout) -> Rgb32f {
// what the mock gives for the whole picture in one tile
let (nx, ny) = layout.cells(w, h);
let t = nx.max(ny).div_ceil(16) * 16;
denoise_bayer(m, w, h, layout, &Mock { tile: t, scale: 1.0 }, &params(t, 16, Gain::None), &Control::default()).unwrap()
}
#[test]
fn tiles_blend_to_the_same_picture_as_one_tile_for_every_layout() {
for cell in [[0u8, 1, 1, 2], [1, 0, 2, 1], [1, 2, 0, 1], [2, 1, 1, 0]] {
let layout = Layout::from_cell(cell).unwrap();
let (w, h) = (150usize, 110usize);
let m = mosaic(w, h, cell);
let want = reference(&m, w, h, layout);
let got = denoise_bayer(&m, w, h, layout, &Mock { tile: 32, scale: 1.0 }, &params(32, 8, Gain::None), &Control::default()).unwrap();
assert_eq!((got.width, got.height), (w, h));
for (a, b) in got.data.iter().zip(&want.data) {
for c in 0..3 {
assert!((a[c] - b[c]).abs() < 1e-5, "layout {cell:?}: {a:?} vs {b:?}");
}
}
}
}
#[test]
fn the_picture_does_not_depend_on_how_many_tiles_run_at_once_or_how_long_each_takes() {
// tiles that finish in a different order every time: blending still adds them in tile order
struct Jittery(Mock);
impl TileRunner for Jittery {
fn run(&self, input: &[f32]) -> Result<Vec<f32>, Error> {
let wait = (input.iter().take(64).sum::<f32>() * 1000.0) as u64 % 7;
std::thread::sleep(std::time::Duration::from_millis(wait));
self.0.run(input)
}
}
let layout = Layout { dx: 0, dy: 1 };
let (w, h) = (150usize, 110usize);
let m = mosaic(w, h, [1, 0, 2, 1]);
let run = |parallel: usize| {
let mut p = params(32, 8, Gain::MatchMean { nominal: 1.0e6, max_deviation: 0.05 });
p.parallel = parallel;
denoise_bayer(&m, w, h, layout, &Jittery(Mock { tile: 32, scale: 1.01e6 }), &p, &Control::default()).unwrap()
};
let one = run(1);
for parallel in [2, 3, 8] {
assert_eq!(run(parallel).data, one.data, "{parallel} at once");
}
}
#[test]
fn a_failing_tile_or_a_cancel_ends_the_picture_without_a_panic() {
struct Fails(Mock, std::sync::atomic::AtomicUsize);
impl TileRunner for Fails {
fn run(&self, input: &[f32]) -> Result<Vec<f32>, Error> {
if self.1.fetch_add(1, Ordering::Relaxed) == 4 {
return Err(Error::Runtime("boom".into()));
}
self.0.run(input)
}
}
let layout = Layout { dx: 0, dy: 0 };
let (w, h) = (150usize, 110usize);
let m = mosaic(w, h, [0, 1, 1, 2]);
let r = denoise_bayer(
&m,
w,
h,
layout,
&Fails(Mock { tile: 32, scale: 1.0 }, Default::default()),
&params(32, 8, Gain::None),
&Control::default(),
);
assert!(matches!(r, Err(Error::Runtime(_))), "{r:?}");
let stop = AtomicBool::new(true);
let ctl = Control { cancel: Some(&stop), progress: None };
let r = denoise_bayer(&m, w, h, layout, &Mock { tile: 32, scale: 1.0 }, &params(32, 8, Gain::None), &ctl);
assert!(matches!(r, Err(Error::Cancelled)));
}
#[test]
fn a_scale_of_its_own_is_taken_out_tile_by_tile() {
let layout = Layout { dx: 0, dy: 0 };
let (w, h) = (150usize, 110usize);
let m = mosaic(w, h, [0, 1, 1, 2]);
let want = reference(&m, w, h, layout);
let gain = Gain::MatchMean { nominal: 1.0e6, max_deviation: 0.05 };
// the scale differs a little from nominal; each tile's own scale is measured and removed
let got = denoise_bayer(&m, w, h, layout, &Mock { tile: 32, scale: 1.02e6 }, &params(32, 8, gain), &Control::default()).unwrap();
let mean = |img: &Rgb32f, c: usize| img.data.iter().map(|p| f64::from(p[c])).sum::<f64>() / img.data.len() as f64;
for c in 0..3 {
let (a, b) = (mean(&got, c), mean(&want, c));
assert!((a - b).abs() / b < 0.01, "channel {c}: {a} vs {b}");
}
// a scale far from nominal is clamped rather than trusted
let wild = denoise_bayer(&m, w, h, layout, &Mock { tile: 32, scale: 3.0e6 }, &params(32, 8, gain), &Control::default()).unwrap();
assert!(mean(&wild, 1) > 2.0 * mean(&want, 1));
}
#[test]
fn a_picture_smaller_than_a_tile_is_padded_by_mirroring() {
let layout = Layout { dx: 1, dy: 1 };
let (w, h) = (20usize, 14usize);
let m = mosaic(w, h, [2, 1, 1, 0]);
let got = denoise_bayer(&m, w, h, layout, &Mock { tile: 64, scale: 1.0 }, &params(64, 16, Gain::None), &Control::default()).unwrap();
assert_eq!((got.width, got.height), (w, h));
assert!(got.data.iter().all(|p| p.iter().all(|v| v.is_finite() && *v > 0.0)));
}
#[test]
fn clipped_highlights_stay_clipped() {
// a model that darkens everything by 10 % would pull a clipped plateau below white
struct Dim(Mock);
impl TileRunner for Dim {
fn run(&self, input: &[f32]) -> Result<Vec<f32>, Error> {
Ok(self.0.run(input)?.into_iter().map(|v| v * 0.9).collect())
}
}
let layout = Layout { dx: 0, dy: 0 };
let (w, h) = (96usize, 96usize);
let m: Vec<f32> = (0..w * h).map(|i| if i % w > 40 && i % w < 70 { 1.0 } else { 0.4 }).collect();
let mut p = params(32, 8, Gain::None);
p.clip = Some(0.99);
let got = denoise_bayer(&m, w, h, layout, &Dim(Mock { tile: 32, scale: 1.0 }), &p, &Control::default()).unwrap();
let mid = got.data[48 * w + 55];
assert!(mid.iter().all(|&v| v > 0.995), "plateau {mid:?}");
let dark = got.data[48 * w + 10];
assert!(dark.iter().all(|&v| (v - 0.36).abs() < 0.01), "unclipped area still goes through the model: {dark:?}");
}
#[test]
fn cancelling_and_progress_work() {
let layout = Layout { dx: 0, dy: 0 };
let (w, h) = (150usize, 110usize);
let m = mosaic(w, h, [0, 1, 1, 2]);
let cancel = AtomicBool::new(true);
let r = denoise_bayer(
&m,
w,
h,
layout,
&Mock { tile: 32, scale: 1.0 },
&params(32, 8, Gain::None),
&Control { cancel: Some(&cancel), progress: None },
);
assert!(matches!(r, Err(Error::Cancelled)));
let seen = std::sync::Mutex::new(Vec::new());
let f = |d: usize, n: usize| seen.lock().unwrap().push((d, n));
denoise_bayer(&m, w, h, layout, &Mock { tile: 32, scale: 1.0 }, &params(32, 8, Gain::None), &Control { cancel: None, progress: Some(&f) })
.unwrap();
let seen = seen.into_inner().unwrap();
let total = tile_count(w, h, layout, 32, 8);
assert!(total > 1);
assert_eq!(seen.last(), Some(&(total, total)));
assert!(seen.windows(2).all(|p| p[1].0 == p[0].0 + 1));
}
#[test]
fn hostile_inputs_are_errors() {
let layout = Layout { dx: 0, dy: 0 };
let mock = Mock { tile: 32, scale: 1.0 };
let p = params(32, 8, Gain::None);
assert!(matches!(denoise_bayer(&[], 0, 0, layout, &mock, &p, &Control::default()), Err(Error::Input(_))));
assert!(matches!(denoise_bayer(&[0.0; 10], 4, 4, layout, &mock, &p, &Control::default()), Err(Error::Input(_))));
assert!(matches!(denoise_bayer(&[0.0; 16], 4, 4, layout, &mock, &params(32, 16, Gain::None), &Control::default()), Err(Error::Input(_))));
struct Bad(Vec<f32>);
impl TileRunner for Bad {
fn run(&self, _: &[f32]) -> Result<Vec<f32>, Error> {
Ok(self.0.clone())
}
}
assert!(matches!(denoise_bayer(&[0.5; 16], 4, 4, layout, &Bad(vec![0.0; 5]), &p, &Control::default()), Err(Error::Model(_))));
assert!(matches!(denoise_bayer(&[0.5; 16], 4, 4, layout, &Bad(vec![f32::NAN; 3 * 64 * 64]), &p, &Control::default()), Err(Error::Model(_))));
}
}

100
crates/denoise-core/src/tiles.rs Normal file
View File

@ -0,0 +1,100 @@
//! Cutting a picture into overlapping tiles and putting the results back together.
//!
//! A model runs on a fixed square, so a picture is covered by tiles spaced evenly along each axis, each sharing
//! at least `overlap` cells with its neighbours. Each tile's influence fades towards its own edges (except at the
//! picture's edge), and the result is the sum of tile × weight divided by the sum of weights, so any set of
//! overlapping tiles blends consistently and a seam between two tiles of slightly different scale is a smooth ramp.
/// Where the tiles start along one axis of `n` cells: `[0]` when the axis fits one tile (the tile is then padded by
/// mirroring), else evenly spaced starts, the first at 0 and the last at `n − tile`, each tile sharing at least
/// `overlap` cells with the next.
pub fn starts(n: usize, tile: usize, overlap: usize) -> Vec<usize> {
if n <= tile || tile == 0 {
return vec![0];
}
let step = tile.saturating_sub(overlap).max(1);
// tiles needed so that start k·spacing with spacing ≤ step reaches n − tile
let Some(k) = (n - tile).div_ceil(step).checked_add(1).filter(|&k| k <= 1_000_000) else { return Vec::new() };
let last = n - tile;
(0..k).map(|i| if k == 1 { 0 } else { ((i as u128 * last as u128 + (k - 1) as u128 / 2) / (k - 1) as u128) as usize }).collect()
}
/// Weight of the cell at offset `p` (0-based, inside a tile of `tile` cells) along one axis: 1 in the middle, easing
/// down to a sliver over `fade` cells at an edge that has a neighbour, 1 right up to an edge that is the picture's.
pub fn weight(p: usize, tile: usize, fade: usize, has_before: bool, has_after: bool) -> f32 {
let fade = fade.max(1) as f32;
let ramp = |d: usize| {
let t = ((d as f32 + 0.5) / fade).min(1.0);
// smoothstep, never exactly 0 so that a cell is always covered
(t * t * (3.0 - 2.0 * t)).max(1e-3)
};
let before = if has_before { ramp(p) } else { 1.0 };
let after = if has_after { ramp(tile.saturating_sub(1).saturating_sub(p)) } else { 1.0 };
before.min(after)
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn hostile_axes_have_bounded_tile_counts_and_checked_spacing() {
assert!(starts(usize::MAX, 1, 0).is_empty());
assert!(starts(usize::MAX, 2, usize::MAX).is_empty());
let s = starts(usize::MAX, usize::MAX / 2, 0);
assert_eq!(s.first(), Some(&0));
assert_eq!(s.last(), Some(&(usize::MAX - usize::MAX / 2)));
assert!(s.windows(2).all(|s| s[0] < s[1]));
}
#[test]
fn one_tile_when_it_fits() {
assert_eq!(starts(100, 512, 64), vec![0]);
assert_eq!(starts(512, 512, 64), vec![0]);
assert_eq!(starts(0, 512, 64), vec![0]);
assert_eq!(starts(10, 0, 0), vec![0]);
}
#[test]
fn tiles_cover_the_axis_and_overlap_enough() {
for n in [513usize, 600, 900, 1000, 1024, 1500, 3024, 3600, 5000, 12345] {
let s = starts(n, 512, 64);
assert_eq!(s[0], 0, "n={n}");
assert_eq!(*s.last().unwrap(), n - 512, "n={n}");
for w in s.windows(2) {
assert!(w[1] > w[0], "starts increase, n={n}");
assert!(w[0] + 512 >= w[1] + 64, "at least 64 shared cells between {} and {}, n={n}", w[0], w[1]);
}
}
}
#[test]
fn weights_fade_only_towards_neighbours() {
let (t, f) = (512, 64);
// a lone tile: full weight everywhere
assert!((0..t).all(|p| weight(p, t, f, false, false) == 1.0));
// the first tile fades on its far side only
assert_eq!(weight(0, t, f, false, true), 1.0);
assert!(weight(t - 1, t, f, false, true) < 0.01);
// the last tile fades on its near side only
assert!(weight(0, t, f, true, false) < 0.01);
assert_eq!(weight(t - 1, t, f, true, false), 1.0);
// a middle tile is 1 in the middle and everywhere positive
assert_eq!(weight(t / 2, t, f, true, true), 1.0);
assert!((0..t).all(|p| weight(p, t, f, true, true) > 0.0));
}
#[test]
fn two_overlapping_tiles_blend_to_a_partition_of_unity() {
// at every cell of the shared region the normalised weights are in [0, 1] and sum to 1
let (t, f) = (512usize, 64usize);
let s = starts(900, t, 64);
assert_eq!(s.len(), 2);
for cell in s[1]..s[0] + t {
let a = weight(cell - s[0], t, f, false, true);
let b = weight(cell - s[1], t, f, true, false);
let (na, nb) = (a / (a + b), b / (a + b));
assert!((na + nb - 1.0).abs() < 1e-6 && na >= 0.0 && nb >= 0.0);
}
}
}

32
crates/denoise/Cargo.toml Normal file
View File

@ -0,0 +1,32 @@
[package]
name = "lightcraft-denoise"
version.workspace = true
edition.workspace = true
license.workspace = true
description = "LightCraft AI denoise (L1): denoise-model manifests, Bayer packing, tiling and blending, the cache file, and pure-Rust CPU inference"
[lints]
workspace = true
[features]
default = []
runtime = ["dep:faer"]
# Product-policy switch: absent by default; users may install their own model.
rawnind-model = ["lightcraft-denoise-core/rawnind-model"]
[dependencies]
lightcraft-denoise-core = { workspace = true }
# Safe matrix views over the pure-Rust GEMM backend; no BLAS, cc/build scripts
# or inner thread pool. Its dependencies include architecture-specific assembly.
faer = { package = "faer-core", version = "=0.17.1", default-features = false, features = ["std"], optional = true }
lightcraft-raster = { workspace = true }
serde = { workspace = true }
serde_json = { workspace = true }
sha2 = { workspace = true }
thiserror = { workspace = true }
miniz_oxide = { workspace = true }
rayon = { workspace = true }
web-time = { workspace = true }
[dev-dependencies]
proptest = { workspace = true }

18
crates/denoise/README.md Normal file
View File

@ -0,0 +1,18 @@
# lightcraft-denoise
L1 pure-Rust AI RAW denoise primitives and CPU inference. No weights are bundled.
See [user workflow and limitations](../../docs/denoise.md).
`lightcraft-denoise-core` provides metadata, Bayer helpers and cache formats,
re-exported here. `manifest` and `known` validate opt-in models; `onnx` reads a bounded operator subset
into `net`; `cpu` executes it and `reference` supplies scalar checks. `bayer`, `tiles`
and `run` pack sensor data and blend overlapping tiles while preserving clipped
highlights. `product` reads/writes the disposable `.lcdn` cache. `runtime` supplies
the installation self-test. Synthetic networks are generated in Rust for tests.
Run `cargo test -p lightcraft-denoise --features runtime`. The CPU backend is optional;
featureless and web builds keep the model metadata and cache format without faer/GEMM.
The `rawnind-model` feature enables the otherwise empty download catalog and requires
an explicit release-policy decision. Real-model tests are ignored by default and
require an explicitly supplied local model. The GPU twin is `lightcraft-gpu::nn`;
model management, background jobs and export integration live in `lightcraft-engine`.

386
crates/denoise/src/cpu.rs Normal file
View File

@ -0,0 +1,386 @@
//! Single-thread CPU execution of the checked denoise network.
//!
//! Convolutions lower at most 256 spatial positions at a time, then use faer's safe views over pure-Rust GEMM.
//! No inner thread pool: `run::denoise_bayer` already runs tiles on the application's Rayon pool. Tensor lifetimes
//! are planned once; buffers and im2col storage are recycled between calls, independently for concurrent callers.
use crate::net::{Net, NetError, Op};
use crate::run::{Error, TileRunner};
use faer::{Parallelism, mat};
use std::sync::{Arc, Mutex};
const BLOCK: usize = 256;
/// Cap all tensor buffers and convolution scratch for one tile together (512 MiB).
const MAX_VALUES: usize = 128 * 1024 * 1024;
const MAX_TILE: usize = 2048;
const MAX_IDLE: usize = 1;
type Result<T> = std::result::Result<T, NetError>;
fn bad(why: &str) -> NetError {
NetError::Invalid(why.into())
}
fn count(c: usize, side: usize) -> Result<usize> {
c.checked_mul(side)
.and_then(|n| n.checked_mul(side))
.filter(|&n| n > 0 && n <= MAX_VALUES)
.ok_or_else(|| bad("a tensor has no size or is too large"))
}
fn zeros(n: usize) -> Result<Vec<f32>> {
let mut v = Vec::new();
v.try_reserve_exact(n).map_err(|_| bad("not enough memory for a denoise tile"))?;
v.resize(n, 0.0);
Ok(v)
}
fn sources(op: &Op) -> (usize, Option<usize>) {
match op {
Op::Conv(c) => (c.src, c.src2),
Op::ConvT2(c) => (c.src, None),
Op::MaxPool2 { src } | Op::DepthToSpace2 { src } => (*src, None),
}
}
#[derive(Clone, Copy)]
struct Tensor {
slot: usize,
side: usize,
channels: usize,
}
struct Plan {
net: Net,
tensors: Vec<Tensor>,
sizes: Vec<usize>,
lower: usize,
product: usize,
up_weights: Vec<Option<Vec<f32>>>,
}
struct Workspace {
buffers: Vec<Vec<f32>>,
lower: Vec<f32>,
product: Vec<f32>,
}
/// A checked Net on square CHW tiles. Clones share immutable weights and the idle workspace cache.
#[derive(Clone)]
pub struct NetRunner {
plan: Arc<Plan>,
idle: Arc<Mutex<Vec<Workspace>>>,
}
impl NetRunner {
pub fn new(net: &Net, tile: usize) -> Result<Self> {
if tile == 0
|| tile > MAX_TILE
|| !net.fits_tile(tile)
|| net.in_channels != 4
|| net.out_channels() != 3
|| net.shape(0).is_none_or(|s| s.channels != net.in_channels)
{
return Err(bad("the tile or Bayer-to-RGB contract does not fit the network"));
}
let mut last = vec![0; net.ops().len() + 1];
for (i, op) in net.ops().iter().enumerate() {
let (a, b) = sources(op);
for id in std::iter::once(a).chain(b) {
*last.get_mut(id).ok_or_else(|| bad("missing tensor"))? = i + 1;
}
}
let mut tensors = vec![Tensor { slot: 0, side: tile, channels: net.in_channels }];
let mut sizes = vec![count(net.in_channels, tile)?];
let mut busy = vec![true];
let (mut lower, mut product) = (0, 0);
let mut up_weights = Vec::new();
for (i, op) in net.ops().iter().enumerate() {
let (src, src2) = sources(op);
let a = *tensors.get(src).ok_or_else(|| bad("missing tensor"))?;
let (channels, side) = match op {
Op::Conv(c) => {
if let Some(s) = src2
&& tensors.get(s).is_none_or(|b| b.side != a.side)
{
return Err(bad("a join has different sizes"));
}
(c.cout, a.side)
}
Op::ConvT2(c) => (c.cout, a.side.checked_mul(2).ok_or_else(|| bad("size overflow"))?),
Op::MaxPool2 { .. } => {
if a.side < 2 || !a.side.is_multiple_of(2) {
return Err(bad("pooling does not divide the tile"));
}
(a.channels, a.side / 2)
}
Op::DepthToSpace2 { .. } => (a.channels / 4, a.side.checked_mul(2).ok_or_else(|| bad("size overflow"))?),
};
let n = count(channels, side)?;
let slot = busy
.iter()
.zip(&sizes)
.enumerate()
.filter(|(_, (used, _))| !**used)
.min_by_key(|(_, (_, size))| (n.saturating_sub(**size), **size))
.map_or(busy.len(), |(i, _)| i);
if slot == sizes.len() {
sizes.push(n);
busy.push(true);
} else {
let sz = sizes.get_mut(slot).ok_or_else(|| bad("missing buffer"))?;
*sz = (*sz).max(n);
*busy.get_mut(slot).ok_or_else(|| bad("missing buffer"))? = true;
}
tensors.push(Tensor { slot, side, channels });
// Release sources only after the output has its own buffer (also handles reading the same tensor twice).
for id in std::iter::once(src).chain(src2) {
if last.get(id) == Some(&(i + 1)) {
let t = tensors.get(id).ok_or_else(|| bad("missing tensor"))?;
*busy.get_mut(t.slot).ok_or_else(|| bad("missing buffer"))? = false;
}
}
let (k, m, positions) = match op {
Op::Conv(c) => (c.cin * c.k * c.k, c.cout, a.side * a.side),
Op::ConvT2(c) => (c.cin, c.cout * 4, a.side * a.side),
_ => (0, 0, 0),
};
lower = lower.max(k.saturating_mul(BLOCK.min(positions)));
product = product.max(m.saturating_mul(BLOCK.min(positions)));
let packed = if let Op::ConvT2(c) = op {
let mut w = zeros(c.weight.len())?;
// [cin][cout][dy][dx] -> [cout*4][cin], so an up-convolution is one GEMM and a shuffle.
for (ci, row) in c.weight.chunks_exact(c.cout * 4).enumerate() {
for (co, &v) in row.iter().enumerate() {
*w.get_mut(co * c.cin + ci).ok_or_else(|| bad("invalid transposed weights"))? = v;
}
}
Some(w)
} else {
None
};
up_weights.push(packed);
}
let total = sizes
.iter()
.try_fold(lower.saturating_add(product), |n, &s| n.checked_add(s))
.filter(|&n| n <= MAX_VALUES)
.ok_or_else(|| bad("the tile needs too much working memory"))?;
let _ = total;
Ok(Self { plan: Arc::new(Plan { net: net.clone(), tensors, sizes, lower, product, up_weights }), idle: Arc::new(Mutex::new(Vec::new())) })
}
/// Immutable data for the GPU runner, without re-reading the model.
pub fn net(&self) -> &Net {
&self.plan.net
}
pub fn bytes_per_tile(&self) -> usize {
(self.plan.sizes.iter().sum::<usize>() + self.plan.lower + self.plan.product) * 4
}
fn workspace(&self) -> Result<Workspace> {
if let Some(w) = self.idle.lock().unwrap_or_else(std::sync::PoisonError::into_inner).pop() {
return Ok(w);
}
Ok(Workspace {
buffers: self.plan.sizes.iter().map(|&n| zeros(n)).collect::<Result<_>>()?,
lower: zeros(self.plan.lower)?,
product: zeros(self.plan.product)?,
})
}
fn execute(&self, input: &[f32], w: &mut Workspace) -> Result<Vec<f32>> {
let first = self.plan.tensors.first().ok_or_else(|| bad("no input"))?;
let want = count(first.channels, first.side)?;
w.buffers.first_mut().and_then(|b| b.get_mut(..want)).ok_or_else(|| bad("missing input buffer"))?.copy_from_slice(input);
let profile = std::env::var_os("LIGHTCRAFT_PROFILE").is_some();
for (i, op) in self.plan.net.ops().iter().enumerate() {
let started = web_time::Instant::now();
let out = *self.plan.tensors.get(i + 1).ok_or_else(|| bad("missing output tensor"))?;
let (before, rest) = w.buffers.split_at_mut_checked(out.slot).ok_or_else(|| bad("missing output buffer"))?;
let (dest, after) = rest.split_first_mut().ok_or_else(|| bad("missing output buffer"))?;
let get = |id: usize| -> Result<(&[f32], Tensor)> {
let t = *self.plan.tensors.get(id).ok_or_else(|| bad("missing source tensor"))?;
let data = if t.slot < out.slot {
before.get(t.slot)
} else {
after.get(t.slot.checked_sub(out.slot + 1).ok_or_else(|| bad("aliased buffers"))?)
}
.and_then(|b| b.get(..count(t.channels, t.side).ok()?))
.ok_or_else(|| bad("missing source buffer"))?;
Ok((data, t))
};
let (src, src2) = sources(op);
let (a, shape) = get(src)?;
let b = src2.map(get).transpose()?.map(|v| v.0);
let dest = dest.get_mut(..count(out.channels, out.side)?).ok_or_else(|| bad("short output buffer"))?;
match op {
Op::Conv(c) => conv(c, a, b, shape.side, dest, &mut w.lower, &mut w.product)?,
Op::ConvT2(c) => up(
c,
a,
shape.side,
dest,
self.plan.up_weights.get(i).and_then(Option::as_deref).ok_or_else(|| bad("missing up weights"))?,
&mut w.lower,
&mut w.product,
)?,
Op::MaxPool2 { .. } => pool(a, shape.side, dest)?,
Op::DepthToSpace2 { .. } => shuffle(a, shape.side, dest)?,
}
if dest.iter().any(|v| !v.is_finite()) {
return Err(bad("the model produced non-finite values"));
}
if profile {
eprintln!("[denoise-cpu] layer {i}: {}x{}x{} {:.2} ms", out.channels, out.side, out.side, started.elapsed().as_secs_f64() * 1e3);
}
}
let t = self.plan.tensors.last().ok_or_else(|| bad("no output"))?;
let data = w.buffers.get(t.slot).and_then(|b| b.get(..count(t.channels, t.side).ok()?)).ok_or_else(|| bad("missing result"))?;
let mut out = zeros(data.len())?;
out.copy_from_slice(data);
Ok(out)
}
}
impl TileRunner for NetRunner {
fn run(&self, input: &[f32]) -> std::result::Result<Vec<f32>, Error> {
let first = self.plan.tensors.first().ok_or_else(|| Error::Runtime("missing input shape".into()))?;
let want = count(first.channels, first.side).map_err(|e| Error::Runtime(e.to_string()))?;
if input.len() != want || input.iter().any(|v| !v.is_finite()) {
return Err(Error::Input("the tile has the wrong size or non-finite values".into()));
}
let mut w = self.workspace().map_err(|e| Error::Runtime(e.to_string()))?;
let result = self.execute(input, &mut w).map_err(|e| Error::Runtime(e.to_string()));
let mut idle = self.idle.lock().unwrap_or_else(std::sync::PoisonError::into_inner);
if idle.len() < MAX_IDLE {
idle.push(w);
}
result
}
}
fn activate(x: f32, bias: f32, alpha: Option<f32>) -> f32 {
let x = x + bias;
if x < 0.0 { x * alpha.unwrap_or(1.0) } else { x }
}
/// All views are over exact, checked slices. The matrix library's assertions therefore never see file-derived sizes.
fn multiply(w: &[f32], x: &[f32], y: &mut [f32], m: usize, k: usize, n: usize) -> Result<()> {
if m.checked_mul(k) != Some(w.len()) || k.checked_mul(n) != Some(x.len()) || m.checked_mul(n) != Some(y.len()) || m == 0 || k == 0 || n == 0 {
return Err(bad("invalid matrix dimensions"));
}
faer::mul::matmul(
mat::from_row_major_slice_mut::<f32>(y, m, n),
mat::from_row_major_slice::<f32>(w, m, k),
mat::from_row_major_slice::<f32>(x, k, n),
None,
1.0,
Parallelism::None,
);
Ok(())
}
fn lower(a: &[f32], b: Option<&[f32]>, side: usize, k: usize, start: usize, n: usize, dest: &mut [f32]) -> Result<()> {
let pad = k / 2;
let pixels = side * side;
// Whole row segments are copied, so bounds checks and edge branches stay out of the multiply loop.
for (ci, plane) in a.chunks_exact(pixels).chain(b.into_iter().flat_map(|b| b.chunks_exact(pixels))).enumerate() {
for ky in 0..k {
for kx in 0..k {
let row = (ci * k * k + ky * k + kx) * n;
let out = dest.get_mut(row..row + n).ok_or_else(|| bad("short im2col buffer"))?;
let mut at = 0;
while at < n {
let p = start + at;
let (y, x) = (p / side, p % side);
let len = (side - x).min(n - at);
if let Some(yy) = y.checked_add(ky).and_then(|v| v.checked_sub(pad)).filter(|&v| v < side) {
let lo = x.max(pad.saturating_sub(kx));
let hi = (x + len).min((side + pad).saturating_sub(kx));
if lo < hi {
let from = yy * side + lo + kx - pad;
let to = at + lo - x;
out.get_mut(at..to).ok_or_else(|| bad("short im2col padding"))?.fill(0.0);
out.get_mut(to..to + hi - lo)
.ok_or_else(|| bad("short im2col row"))?
.copy_from_slice(plane.get(from..from + hi - lo).ok_or_else(|| bad("short source row"))?);
out.get_mut(to + hi - lo..at + len).ok_or_else(|| bad("short im2col padding"))?.fill(0.0);
} else {
out.get_mut(at..at + len).ok_or_else(|| bad("short im2col row"))?.fill(0.0);
}
} else {
out.get_mut(at..at + len).ok_or_else(|| bad("short im2col row"))?.fill(0.0);
}
at += len;
}
}
}
}
Ok(())
}
fn conv(c: &crate::net::Conv, a: &[f32], b: Option<&[f32]>, side: usize, dest: &mut [f32], scratch: &mut [f32], product: &mut [f32]) -> Result<()> {
let pixels = side * side;
let k = c.cin * c.k * c.k;
for start in (0..pixels).step_by(BLOCK) {
let n = BLOCK.min(pixels - start);
let x = scratch.get_mut(..k * n).ok_or_else(|| bad("short im2col storage"))?;
lower(a, b, side, c.k, start, n, x)?;
let y = product.get_mut(..c.cout * n).ok_or_else(|| bad("short product storage"))?;
multiply(&c.weight, x, y, c.cout, k, n)?;
for ((row, &bias), out) in y.chunks_exact(n).zip(&c.bias).zip(dest.chunks_exact_mut(pixels)) {
let to = out.get_mut(start..start + n).ok_or_else(|| bad("short convolution output"))?;
for (d, &v) in to.iter_mut().zip(row) {
*d = activate(v, bias, c.leaky);
}
}
}
Ok(())
}
fn up(c: &crate::net::ConvT2, a: &[f32], side: usize, dest: &mut [f32], weights: &[f32], scratch: &mut [f32], product: &mut [f32]) -> Result<()> {
let pixels = side * side;
let out_side = 2 * side;
for start in (0..pixels).step_by(BLOCK) {
let n = BLOCK.min(pixels - start);
let x = scratch.get_mut(..c.cin * n).ok_or_else(|| bad("short up input storage"))?;
for (to, from) in x.chunks_exact_mut(n).zip(a.chunks_exact(pixels)) {
to.copy_from_slice(from.get(start..start + n).ok_or_else(|| bad("short up input"))?);
}
let y = product.get_mut(..c.cout * 4 * n).ok_or_else(|| bad("short up product storage"))?;
multiply(weights, x, y, c.cout * 4, c.cin, n)?;
for (co, (&bias, out)) in c.bias.iter().zip(dest.chunks_exact_mut(4 * pixels)).enumerate() {
for d in 0..4 {
let row = y.get((4 * co + d) * n..(4 * co + d + 1) * n).ok_or_else(|| bad("short up product"))?;
for (j, &v) in row.iter().enumerate() {
let p = start + j;
let at = (2 * (p / side) + d / 2) * out_side + 2 * (p % side) + d % 2;
*out.get_mut(at).ok_or_else(|| bad("short up output"))? = activate(v, bias, c.leaky);
}
}
}
}
Ok(())
}
fn pool(a: &[f32], side: usize, dest: &mut [f32]) -> Result<()> {
let s = side / 2;
for (input, out) in a.chunks_exact(side * side).zip(dest.chunks_exact_mut(s * s)) {
for (y, row) in out.chunks_exact_mut(s).enumerate() {
let lo = input.get(2 * y * side..(2 * y + 1) * side).ok_or_else(|| bad("short pool input"))?;
let hi = input.get((2 * y + 1) * side..(2 * y + 2) * side).ok_or_else(|| bad("short pool input"))?;
for ((d, lo), hi) in row.iter_mut().zip(lo.as_chunks::<2>().0).zip(hi.as_chunks::<2>().0) {
let [a, b] = *lo;
let [c, e] = *hi;
*d = a.max(b).max(c).max(e);
}
}
}
Ok(())
}
fn shuffle(a: &[f32], side: usize, dest: &mut [f32]) -> Result<()> {
let pixels = side * side;
for (input, out) in a.chunks_exact(4 * pixels).zip(dest.chunks_exact_mut(4 * pixels)) {
for d in 0..4 {
let plane = input.get(d * pixels..(d + 1) * pixels).ok_or_else(|| bad("short shuffle input"))?;
for (y, row) in plane.chunks_exact(side).enumerate() {
let at = (2 * y + d / 2) * 2 * side;
let to = out.get_mut(at..at + 2 * side).ok_or_else(|| bad("short shuffle output"))?;
for (to, &v) in to.as_chunks_mut::<2>().0.iter_mut().zip(row) {
if let Some(t) = to.get_mut(d % 2) {
*t = v;
}
}
}
}
}
Ok(())
}
#[cfg(test)]
mod tests;

151
crates/denoise/src/cpu/tests.rs Normal file
View File

@ -0,0 +1,151 @@
use super::*;
use crate::{
net::{Conv, ConvT2},
reference, synthetic,
};
fn random(n: usize, seed: u32) -> Vec<f32> {
let mut s = seed | 1;
(0..n)
.map(|_| {
s ^= s << 13;
s ^= s >> 17;
s ^= s << 5;
(s >> 8) as f32 / (1u32 << 23) as f32 - 1.0
})
.collect()
}
fn net(tile: usize, ch: u64, depth: usize, seed: u64) -> Net {
let p = std::env::temp_dir().join(format!("lc-purecpu-{}-{tile}-{ch}-{depth}-{seed}.onnx", std::process::id()));
std::fs::write(&p, synthetic::unet_onnx(tile as u64, ch, depth, seed)).unwrap();
let n = crate::onnx::read(&p).unwrap();
std::fs::remove_file(p).unwrap();
n
}
fn agree(a: &[f32], b: &[f32]) {
assert_eq!(a.len(), b.len());
assert!(a.iter().chain(b).all(|v| v.is_finite()));
let scale = b.iter().fold(1e-6f32, |m, v| m.max(v.abs()));
let error = a.iter().zip(b).fold(0f32, |m, (a, b)| m.max((a - b).abs())) / scale;
assert!(error <= 1e-3, "relative maximum error {error:e}");
}
#[test]
fn all_operators_match_scalar_on_random_nets_and_odd_and_tiny_shapes() {
for (t, ch, d) in [(1, 3, 0), (3, 5, 0), (5, 7, 0), (6, 3, 1), (20, 5, 2), (36, 9, 2), (64, 8, 3)] {
for seed in [1, 73] {
let n = net(t, ch, d, seed);
let r = NetRunner::new(&n, t).unwrap();
let x = random(4 * t * t, seed as u32);
let want = reference::run(&n, t, &x).unwrap();
agree(&r.run(&x).unwrap(), &want);
assert_eq!(r.run(&x).unwrap(), r.run(&x).unwrap());
}
}
}
#[test]
fn malformed_values_fail_before_execution_and_workspace_sizes_are_bounded() {
let mut n = net(4, 3, 1, 11);
for t in [0, 1, 3, usize::MAX, 4096] {
assert!(NetRunner::new(&n, t).is_err());
}
for ch in [0, 3, usize::MAX] {
n.in_channels = ch;
assert!(NetRunner::new(&n, 4).is_err());
}
let n = Net::new(
4,
vec![
Op::Conv(Conv { src: 0, src2: None, cin: 4, cout: 4096, k: 1, weight: vec![0.1; 4 * 4096], bias: vec![0.0; 4096], leaky: None }),
Op::Conv(Conv { src: 1, src2: None, cin: 4096, cout: 12, k: 1, weight: vec![0.1; 4096 * 12], bias: vec![0.0; 12], leaky: None }),
Op::DepthToSpace2 { src: 2 },
],
)
.unwrap();
assert!(NetRunner::new(&n, 2048).is_err());
let ops = (0..100).map(|i| Op::MaxPool2 { src: i }).collect();
let n = Net::new(4, ops).unwrap();
assert!(NetRunner::new(&n, 16).is_err());
let c = Conv { src: 0, src2: None, cin: 4, cout: 12, k: 1, weight: vec![0.25; 48], bias: vec![0.0; 12], leaky: None };
for mode in 0..6 {
let mut bad = c.clone();
match mode {
0 => bad.src = usize::MAX,
1 => bad.weight.pop().map(|_| ()).unwrap(),
2 => bad.bias[0] = f32::NAN,
3 => bad.leaky = Some(f32::INFINITY),
4 => bad.cout = usize::MAX,
_ => bad.k = usize::MAX,
}
assert!(Net::new(4, vec![Op::Conv(bad), Op::DepthToSpace2 { src: 1 }]).is_err());
}
}
#[test]
fn bad_tiles_and_arithmetic_overflow_are_errors_and_do_not_poison_reuse() {
let c = Conv { src: 0, src2: None, cin: 4, cout: 12, k: 1, weight: vec![f32::MAX; 48], bias: vec![0.0; 12], leaky: None };
let n = Net::new(4, vec![Op::Conv(c), Op::DepthToSpace2 { src: 1 }]).unwrap();
let r = NetRunner::new(&n, 1).unwrap();
assert!(r.run(&[]).is_err());
assert!(r.run(&[f32::NAN; 4]).is_err());
assert!(r.run(&[f32::INFINITY; 4]).is_err());
assert_eq!(r.idle.lock().unwrap().len(), 0, "invalid input never allocates a workspace");
assert!(r.run(&[2.0; 4]).is_err());
assert_eq!(r.run(&[0.0; 4]).unwrap(), vec![0.0; 12]);
// Matrix shape checks are the boundary in front of the dependency's asserting constructors.
assert!(multiply(&[], &[], &mut [], usize::MAX, 2, 2).is_err());
}
#[test]
fn shared_runner_serves_concurrent_tiles_without_cross_contamination() {
let n = net(12, 5, 1, 97);
let r = NetRunner::new(&n, 12).unwrap();
let inputs: Vec<_> = (0..8).map(|s| random(4 * 12 * 12, s)).collect();
let answers: Vec<_> = inputs.iter().map(|x| reference::run(&n, 12, x).unwrap()).collect();
std::thread::scope(|scope| {
let mut handles = Vec::new();
for (x, want) in inputs.iter().zip(&answers) {
let r = &r;
handles.push(scope.spawn(move || {
for _ in 0..3 {
agree(&r.run(x).unwrap(), want);
}
}));
}
for h in handles {
h.join().unwrap();
}
});
}
#[test]
fn join_may_read_the_same_tensor_twice_and_leaky_up_convolution_is_supported() {
let c = ConvT2 { src: 0, cin: 4, cout: 3, weight: random(48, 2), bias: vec![0.1, -0.1, 0.0], leaky: Some(0.13) };
let n = Net::new(
4,
vec![
Op::ConvT2(c),
Op::Conv(Conv { src: 1, src2: Some(1), cin: 6, cout: 3, k: 3, weight: random(162, 3), bias: vec![0.0; 3], leaky: Some(0.0) }),
],
)
.unwrap();
let x = random(4 * 3 * 3, 7);
agree(&NetRunner::new(&n, 3).unwrap().run(&x).unwrap(), &reference::run(&n, 3, &x).unwrap());
}
#[test]
fn reused_im2col_clears_all_edge_padding_between_images() {
for t in [1, 3, 5, 36] {
let n = net(t, 5, 0, 121);
let r = NetRunner::new(&n, t).unwrap();
for x in [vec![1.0; 4 * t * t], vec![0.0; 4 * t * t], random(4 * t * t, 9)] {
agree(&r.run(&x).unwrap(), &reference::run(&n, t, &x).unwrap());
}
}
}
#[test]
#[ignore = "needs LC_DENOISE_MODEL; one tile with LIGHTCRAFT_PROFILE=1"]
fn profile_one_real_tile() {
let model = std::env::var_os("LC_DENOISE_MODEL").expect("set LC_DENOISE_MODEL");
let m = crate::known::find("rawnind-bayer").unwrap().manifest;
let r = crate::runtime::CpuRunner::load(std::path::Path::new(&model), &m).unwrap();
println!("workspace {:.1} MB", r.bytes_per_tile() as f64 / 1e6);
r.run(&crate::runtime::check_tile(512)).unwrap();
}

37
crates/denoise/src/lib.rs Normal file
View File

@ -0,0 +1,37 @@
//! AI denoise for LightCraft, as a step between the sensor and the demosaic that the rest of the develop
//! pipeline never sees: the raw file stays untouched, the denoised picture is cached data that can always be
//! made again, and the amount is an ordinary develop setting.
//!
//! - [`manifest`]: what a denoise model takes and gives, and who may use it (untrusted JSON, validated).
//! - [`known`] and [`archive`]: the models LightCraft can fetch, and taking the model file out of the `.zip` it comes in.
//! - [`bayer`]: Bayer layouts and the packing of a mosaic into the four planes a model takes.
//! - [`tiles`] and [`run`]: cutting a picture into overlapping tiles for a fixed-size model, bringing each tile's
//! output scale back to the input's, keeping clipped highlights as they were, and blending the tiles together.
//! The model itself is a [`run::TileRunner`], so all of this is tested without one.
//! - [`product`]: the cache file the denoised picture is kept in (half floats, compressed in strips) and the
//! blend with the plain demosaic that the Amount slider controls.
//! - [`cpu`] and [`runtime`]: a single-thread pure-Rust CPU runner and its installation self-test.
//! - [`net`] and [`onnx`]: checked plain network data and bounded ONNX decoding, shared by CPU and GPU.
//!
//! Everything here treats model files, manifests and cache files as hostile input: sizes are capped, numbers must
//! be finite, and a malformed file is an error, never a panic.
#![forbid(unsafe_code)]
#![deny(clippy::unwrap_used, clippy::expect_used, clippy::panic, clippy::unimplemented, clippy::todo, clippy::unreachable)]
#[cfg(feature = "runtime")]
pub mod cpu;
pub mod net;
pub mod onnx;
mod onnx_proto;
pub mod reference;
#[cfg(feature = "runtime")]
pub mod runtime;
#[doc(hidden)]
pub mod synthetic;
pub use lightcraft_denoise_core::*;
pub mod synthetic_proto;

345
crates/denoise/src/net.rs Normal file
View File

@ -0,0 +1,345 @@
//! A denoise network as plain data: the handful of layer kinds a U-Net denoiser is made of, with their weights.
//!
//! [`onnx`](crate::onnx) reads the file into this description, shared by the pure-Rust CPU and GPU runners.
//! Unsupported operators, groups, dilation or activations are reported as errors when loading the model.
//!
//! The description is checked ([`Net::new`]) because it comes from a stranger's file: channel counts must chain,
//! weights must have the sizes the shapes need and be finite, and the depth must stay small.
/// Most layers a network may have.
pub const MAX_OPS: usize = 512;
/// Most channels a layer may have.
pub const MAX_CHANNELS: usize = 4096;
/// Most weights (numbers) a network may have in all.
pub const MAX_WEIGHTS: usize = 128 * 1024 * 1024;
#[derive(Debug, PartialEq, thiserror::Error)]
pub enum NetError {
/// The model uses something this denoise network description cannot express.
#[error("not supported by the denoise runners: {0}")]
Unsupported(String),
/// The model is not a well-formed network.
#[error("not a valid network: {0}")]
Invalid(String),
}
fn invalid<T>(why: impl Into<String>) -> Result<T, NetError> {
Err(NetError::Invalid(why.into()))
}
/// A tensor is named by its number: 0 is the network's input, and layer `i` makes tensor `i + 1`.
pub type TensorId = usize;
/// A convolution with stride 1 and "same" padding (`k` is 1 or 3), over one tensor or two joined along the channels
/// (`src` first), then a bias and optionally a leaky ReLU.
#[derive(Clone, Debug, PartialEq)]
pub struct Conv {
pub src: TensorId,
/// A second source, whose channels follow `src`'s (a skip connection's concatenation).
pub src2: Option<TensorId>,
pub cin: usize,
pub cout: usize,
pub k: usize,
/// `[cout][cin][k][k]`, as ONNX has it.
pub weight: Vec<f32>,
pub bias: Vec<f32>,
/// `Some(alpha)`: a leaky ReLU with this slope follows.
pub leaky: Option<f32>,
}
/// A transposed convolution with a 2 × 2 kernel and stride 2: every input pixel becomes a 2 × 2 block.
#[derive(Clone, Debug, PartialEq)]
pub struct ConvT2 {
pub src: TensorId,
pub cin: usize,
pub cout: usize,
/// `[cin][cout][2][2]`, as ONNX has it.
pub weight: Vec<f32>,
pub bias: Vec<f32>,
pub leaky: Option<f32>,
}
#[derive(Clone, Debug, PartialEq)]
pub enum Op {
Conv(Conv),
ConvT2(ConvT2),
/// 2 × 2 maximum, stride 2.
MaxPool2 {
src: TensorId,
},
/// Depth-to-space with block size 2 in "CRD" order: output channel `c` at `(2y + dy, 2x + dx)` is input channel
/// `4c + 2dy + dx` at `(y, x)`.
DepthToSpace2 {
src: TensorId,
},
}
/// What a tensor looks like: its channels, and its resolution as a power of two below the input's (a negative
/// level is above it).
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub struct Shape {
pub channels: usize,
pub level: i32,
}
#[derive(Clone, Debug, PartialEq)]
pub struct Net {
pub in_channels: usize,
ops: Vec<Op>,
shapes: Vec<Shape>,
}
impl Net {
/// Check `ops` (layer `i` makes tensor `i + 1`) over an input of `in_channels` and describe every tensor.
pub fn new(in_channels: usize, ops: Vec<Op>) -> Result<Net, NetError> {
if in_channels == 0 || in_channels > MAX_CHANNELS {
return invalid("the input has no channels or too many");
}
if ops.is_empty() || ops.len() > MAX_OPS {
return invalid("the network has no layers or too many");
}
let mut shapes = vec![Shape { channels: in_channels, level: 0 }];
let mut weights = 0usize;
for (i, op) in ops.iter().enumerate() {
let shape_of = |t: TensorId| {
shapes
.get(t)
.copied()
.filter(|_| t <= i)
.ok_or_else(|| NetError::Invalid(format!("layer {i} reads tensor {t}, which does not exist yet")))
};
let out = match op {
Op::Conv(c) => {
let a = shape_of(c.src)?;
let (cin, level) = match c.src2 {
Some(s2) => {
let b = shape_of(s2)?;
if b.level != a.level {
return invalid(format!("layer {i} joins tensors of different resolutions"));
}
(a.channels + b.channels, a.level)
}
None => (a.channels, a.level),
};
if c.cin != cin || c.cout == 0 || c.cout > MAX_CHANNELS || !(c.k == 1 || c.k == 3) {
return invalid(format!(
"layer {i}: a convolution of {} to {} channels with a {}-wide kernel does not fit",
c.cin, c.cout, c.k
));
}
check_weights(
i,
&c.weight,
c.cout.checked_mul(cin).and_then(|n| n.checked_mul(c.k * c.k)),
&c.bias,
c.cout,
c.leaky,
&mut weights,
)?;
Shape { channels: c.cout, level }
}
Op::ConvT2(c) => {
let a = shape_of(c.src)?;
if c.cin != a.channels || c.cout == 0 || c.cout > MAX_CHANNELS {
return invalid(format!("layer {i}: a transposed convolution of {} to {} channels does not fit", c.cin, c.cout));
}
check_weights(i, &c.weight, c.cin.checked_mul(c.cout).and_then(|n| n.checked_mul(4)), &c.bias, c.cout, c.leaky, &mut weights)?;
Shape { channels: c.cout, level: a.level - 1 }
}
Op::MaxPool2 { src } => {
let a = shape_of(*src)?;
Shape { channels: a.channels, level: a.level + 1 }
}
Op::DepthToSpace2 { src } => {
let a = shape_of(*src)?;
if !a.channels.is_multiple_of(4) {
return invalid(format!("layer {i}: depth-to-space needs a multiple of 4 channels"));
}
Shape { channels: a.channels / 4, level: a.level - 1 }
}
};
shapes.push(out);
}
Ok(Net { in_channels, ops, shapes })
}
/// The layers: layer `i` makes tensor `i + 1`.
pub fn ops(&self) -> &[Op] {
&self.ops
}
/// The shape of tensor `t` (0 is the input).
pub fn shape(&self, t: TensorId) -> Option<Shape> {
self.shapes.get(t).copied()
}
/// The tensor the network gives (the last layer's).
pub fn output(&self) -> TensorId {
self.ops.len()
}
pub fn out_channels(&self) -> usize {
self.shapes.last().map_or(0, |s| s.channels)
}
/// The deepest resolution level: a tile must be a multiple of `2^this` cells wide.
pub fn depth(&self) -> u32 {
self.shapes.iter().map(|s| s.level.max(0) as u32).max().unwrap_or(0)
}
/// Can the network run on a `tile × tile` input and give `[out_channels, 2·tile, 2·tile]`? (Pooling must divide
/// the tile, and the output must be one level above the input.)
pub fn fits_tile(&self, tile: usize) -> bool {
let Some(multiple) = 1usize.checked_shl(self.depth()) else { return false };
tile > 0 && tile.is_multiple_of(multiple) && self.shapes.last().is_some_and(|s| s.level == -1)
}
/// Multiply-adds for one tile, for estimating time.
pub fn macs(&self, tile: usize) -> u64 {
let pixels = |level: i32| -> u64 {
let side = if level >= 0 {
tile.checked_shr(level as u32).unwrap_or(0)
} else {
tile.saturating_mul(1usize.checked_shl(level.unsigned_abs()).unwrap_or(usize::MAX))
};
(side as u64).saturating_mul(side as u64)
};
self.ops
.iter()
.enumerate()
.map(|(i, op)| {
let out = self.shapes.get(i + 1).map_or(0, |s| pixels(s.level));
match op {
Op::Conv(c) => out.saturating_mul((c.cin * c.cout * c.k * c.k) as u64),
// every input pixel makes a 2 × 2 block: as many multiply-adds as the 2 × 2-wide kernel has weights
Op::ConvT2(c) => pixels(self.shapes.get(c.src).map_or(0, |s| s.level)).saturating_mul((c.cin * c.cout * 4) as u64),
_ => 0,
}
})
.fold(0u64, u64::saturating_add)
}
}
fn check_weights(
layer: usize,
weight: &[f32],
want: Option<usize>,
bias: &[f32],
cout: usize,
leaky: Option<f32>,
total: &mut usize,
) -> Result<(), NetError> {
if want != Some(weight.len()) || bias.len() != cout {
return invalid(format!("layer {layer}: the weights are not the size its shape needs"));
}
*total = total.saturating_add(weight.len()).saturating_add(bias.len());
if *total > MAX_WEIGHTS {
return invalid("the network has too many weights");
}
if weight.iter().chain(bias).any(|v| !v.is_finite()) || leaky.is_some_and(|a| !a.is_finite()) {
return invalid(format!("layer {layer}: a weight is not a finite number"));
}
Ok(())
}
#[cfg(test)]
mod tests {
use super::*;
fn conv(src: TensorId, src2: Option<TensorId>, cin: usize, cout: usize, k: usize) -> Op {
Op::Conv(Conv { src, src2, cin, cout, k, weight: vec![0.1; cout * cin * k * k], bias: vec![0.0; cout], leaky: Some(0.2) })
}
fn convt(src: TensorId, cin: usize, cout: usize) -> Op {
Op::ConvT2(ConvT2 { src, cin, cout, weight: vec![0.1; cin * cout * 4], bias: vec![0.0; cout], leaky: None })
}
/// A two-level network of the shape the denoisers have: 4 → 8 channels, pool, 16 channels, up, join, 12, and the
/// 1 × 1 convolution to the colours.
fn unet() -> Vec<Op> {
vec![
conv(0, None, 4, 8, 3), // 1: 8 ch, level 0
Op::MaxPool2 { src: 1 }, // 2: level 1
conv(2, None, 8, 16, 3), // 3: 16 ch, level 1
convt(3, 16, 8), // 4: 8 ch, level 0
conv(4, Some(1), 16, 8, 3), // 5: join [up, skip]
conv(5, None, 8, 12, 1), // 6: 12 ch
Op::DepthToSpace2 { src: 6 }, // 7: 3 ch, level -1
]
}
#[test]
fn deep_networks_and_huge_cost_queries_never_overflow_or_truncate_depth() {
let n = Net::new(4, unet()).unwrap();
assert_eq!(n.macs(usize::MAX), u64::MAX);
assert_eq!(n.macs(0), 0);
let mut ops = Vec::new();
for i in 0..21 {
ops.push(Op::MaxPool2 { src: i });
}
for i in 21..42 {
ops.push(convt(i, 4, 4));
}
ops.push(conv(42, None, 4, 12, 1));
ops.push(Op::DepthToSpace2 { src: 43 });
let n = Net::new(4, ops).unwrap();
assert!(!n.fits_tile(1 << 20));
assert!(n.fits_tile(1 << 21));
let _ = n.macs(16);
}
#[test]
fn a_u_net_is_described_with_its_shapes_and_costs() {
let net = Net::new(4, unet()).unwrap();
assert_eq!(net.out_channels(), 3);
assert_eq!(net.shape(3), Some(Shape { channels: 16, level: 1 }));
assert_eq!(net.shape(7), Some(Shape { channels: 3, level: -1 }));
assert_eq!(net.depth(), 1);
assert!(net.fits_tile(64) && !net.fits_tile(63) && !net.fits_tile(0));
// 64 × 64 cells: layer 1 = 4096·4·8·9, and so on; just check it is the sum of what the layers cost
let t = 64u64;
let want = t * t * 4 * 8 * 9 + (t / 2) * (t / 2) * 8 * 16 * 9 + (t / 2) * (t / 2) * 16 * 8 * 4 + t * t * 16 * 8 * 9 + t * t * 8 * 12;
assert_eq!(net.macs(64), want);
}
#[test]
fn a_network_that_does_not_chain_is_refused() {
// wrong input channel count for the first convolution
assert!(Net::new(3, unet()).is_err());
// a layer reading a tensor that does not exist yet
let mut ops = unet();
ops[2] = conv(9, None, 8, 16, 3);
assert!(Net::new(4, ops).is_err());
// joining tensors of different resolutions
let mut ops = unet();
ops[4] = conv(3, Some(1), 24, 8, 3);
assert!(Net::new(4, ops).is_err());
// weights of the wrong size, and not finite
let mut ops = unet();
if let Op::Conv(c) = &mut ops[0] {
c.weight.pop();
}
assert!(Net::new(4, ops).is_err());
let mut ops = unet();
if let Op::Conv(c) = &mut ops[0] {
c.bias[0] = f32::NAN;
}
assert!(Net::new(4, ops).is_err());
// a kernel that is neither 1 nor 3 wide, no layers, depth-to-space on a channel count it cannot split
let mut ops = unet();
if let Op::Conv(c) = &mut ops[0] {
c.k = 5;
}
assert!(Net::new(4, ops).is_err());
assert!(Net::new(4, vec![]).is_err());
assert!(Net::new(4, vec![Op::DepthToSpace2 { src: 0 }]).is_ok());
assert!(Net::new(4, vec![Op::DepthToSpace2 { src: 0 }, Op::DepthToSpace2 { src: 1 }]).is_err());
assert!(Net::new(2, vec![Op::DepthToSpace2 { src: 0 }]).is_err());
}
#[test]
fn a_network_that_does_not_end_one_level_up_does_not_fit_a_tile() {
let net = Net::new(4, unet()[..6].to_vec()).unwrap();
assert!(!net.fits_tile(64), "no depth-to-space: the output is not at twice the resolution");
}
}

369
crates/denoise/src/onnx.rs Normal file
View File

@ -0,0 +1,369 @@
//! Reading an `.onnx` file into a [`Net`] with bounded, pure-Rust protobuf decoding.
//!
//! Only what the denoise U-Nets are made of is understood: `Conv` (3 × 3 "same" or 1 × 1, stride 1, no groups, no
//! dilation) optionally followed by `LeakyRelu`, `ConvTranspose` (2 × 2, stride 2) likewise, `MaxPool` (2 × 2, stride 2),
//! a two-input `Concat` on the channels that feeds a convolution, and a final `DepthToSpace` (2, `CRD`). Any other
//! operator or attribute is [`NetError::Unsupported`], which is reported to the user by both runners.
use std::collections::HashMap;
use std::path::Path;
use crate::onnx_proto::{self, AttributeProto, NodeProto, TensorProto, ValueInfoProto};
use std::io::Read;
use crate::net::{Conv, ConvT2, MAX_WEIGHTS, Net, NetError, Op, TensorId};
fn unsupported<T>(why: impl Into<String>) -> Result<T, NetError> {
Err(NetError::Unsupported(why.into()))
}
fn invalid<T>(why: impl Into<String>) -> Result<T, NetError> {
Err(NetError::Invalid(why.into()))
}
/// Read the model at `path`.
pub fn read(path: &Path) -> Result<Net, NetError> {
let file = std::fs::File::open(path).map_err(|e| NetError::Invalid(e.to_string()))?;
let size = usize::try_from(file.metadata().map_err(|e| NetError::Invalid(e.to_string()))?.len())
.ok()
.filter(|&n| n <= onnx_proto::MAX_BYTES)
.ok_or_else(|| NetError::Invalid("model file too large".into()))?;
let mut bytes = Vec::new();
bytes.try_reserve_exact(size).map_err(|_| NetError::Invalid("not enough memory to load the model".into()))?;
file.take(onnx_proto::MAX_BYTES as u64 + 1).read_to_end(&mut bytes).map_err(|e| NetError::Invalid(e.to_string()))?;
read_bytes(&bytes)
}
fn read_bytes(bytes: &[u8]) -> Result<Net, NetError> {
let graph = onnx_proto::decode(bytes)?;
let initialisers: HashMap<&str, &TensorProto> = graph.initializer.iter().map(|t| (t.name.as_str(), t)).collect();
if initialisers.len() != graph.initializer.len() {
return invalid("duplicate initializer names");
}
let inputs: Vec<_> = graph.input.iter().filter(|i| !initialisers.contains_key(i.name.as_str())).collect();
let ([input], [output]) = (inputs.as_slice(), graph.output.as_slice()) else { return unsupported("not one input and one output") };
let in_channels = match channel_dim(input) {
Some(c) => c,
None => return unsupported("the input's channel count is not written in the file"),
};
// how many nodes read each name (a graph output counts as a reader)
let mut readers: HashMap<&str, usize> = HashMap::new();
for n in &graph.node {
for i in &n.input {
*readers.entry(i.as_str()).or_default() += 1;
}
}
*readers.entry(output.name.as_str()).or_default() += 1;
let mut ids: HashMap<String, TensorId> = HashMap::new();
ids.insert(input.name.clone(), 0);
// a Concat is not a layer of its own: the convolution that reads it takes both sources
let mut joins: HashMap<String, (TensorId, TensorId)> = HashMap::new();
let mut ops: Vec<Op> = Vec::new();
let mut weights_total = 0usize;
let nodes = &graph.node;
let mut i = 0;
while i < nodes.len() {
let Some(node) = nodes.get(i) else { break };
i += 1;
if node.output.len() != 1 || node.output.first().is_none_or(|n| n.is_empty() || ids.contains_key(n) || joins.contains_key(n)) {
return invalid("a layer has missing, repeated or multiple outputs");
}
let allowed: &[&str] = match node.op_type.as_str() {
"Conv" => &["kernel_shape", "pads", "strides", "dilations", "group", "auto_pad"],
"ConvTranspose" => &["kernel_shape", "pads", "strides", "dilations", "group", "output_padding", "auto_pad"],
"MaxPool" => &["kernel_shape", "pads", "strides", "dilations", "ceil_mode", "storage_order", "auto_pad"],
"Concat" => &["axis"],
"DepthToSpace" => &["blocksize", "mode"],
_ => &[],
};
if node.attribute.iter().any(|a| !allowed.contains(&a.name.as_str())) {
return unsupported("an unknown operator attribute");
}
let this = ops.len() + 1;
let lookup = |name: &str| -> Result<TensorId, NetError> {
ids.get(name).copied().ok_or_else(|| NetError::Invalid(format!("`{}` reads `{name}`, which nothing makes", node.op_type)))
};
// a LeakyRelu straight after this layer, which nothing else reads, is part of it
let mut fuse_leaky = |out: &str| -> Result<(Option<f32>, String), NetError> {
if let Some(next) = nodes.get(i)
&& next.op_type == "LeakyRelu"
&& next.input.first().map(String::as_str) == Some(out)
&& readers.get(out).copied() == Some(1)
{
if next.input.len() != 1 || next.output.len() != 1 || next.attribute.iter().any(|a| a.name != "alpha") {
return invalid("malformed LeakyRelu");
}
let alpha = attr(next, "alpha").map_or(0.01, |a| a.f);
i += 1;
let name = next.output.first().cloned().ok_or_else(|| NetError::Invalid("LeakyRelu without an output".into()))?;
return Ok((Some(alpha), name));
}
Ok((None, out.to_string()))
};
match node.op_type.as_str() {
"Concat" => {
if attr(node, "axis").map(|a| a.i) != Some(1) {
return unsupported("Concat on another axis than the channels");
}
let [a, b] = node.input.as_slice() else { return unsupported("Concat of other than two tensors") };
let out = node.output.first().ok_or_else(|| NetError::Invalid("Concat without an output".into()))?;
if readers.get(out.as_str()).copied() != Some(1) {
return unsupported("a Concat that is read by more than one layer");
}
joins.insert(out.clone(), (lookup(a)?, lookup(b)?));
}
"Conv" => {
let (w, b) = weights_of(node, &initialisers, &mut weights_total)?;
let (cout, cin, kh, kw) = four(&w.0)?;
let b = if b.is_empty() { vec![0.0; cout] } else { b };
if kh != kw || !(kh == 1 || kh == 3) {
return unsupported(format!("a {kh} × {kw} convolution"));
}
if ints_or(node, "strides", &[1, 1]) != [1, 1]
|| ints_or(node, "dilations", &[1, 1]) != [1, 1]
|| attr(node, "group").is_some_and(|a| a.i != 1)
{
return unsupported("a convolution with strides, dilation or groups");
}
if attr(node, "kernel_shape").is_some_and(|a| a.ints != [kh as i64, kw as i64]) {
return invalid("kernel shape does not match the weights");
}
let pad = (kh as i64 - 1) / 2;
if ints_or(node, "pads", &[0; 4]) != [pad; 4] || attr(node, "auto_pad").is_some_and(|a| !a.s.is_empty() && a.s != b"NOTSET") {
return unsupported("a convolution that is not padded to keep its size");
}
let src_name = node.input.first().ok_or_else(|| NetError::Invalid("Conv without an input".into()))?;
let (src, src2) = match joins.remove(src_name) {
Some((a, b)) => (a, Some(b)),
None => (lookup(src_name)?, None),
};
let out = node.output.first().ok_or_else(|| NetError::Invalid("Conv without an output".into()))?;
let (leaky, out_name) = fuse_leaky(out)?;
ids.insert(out_name, this);
ops.push(Op::Conv(Conv { src, src2, cin, cout, k: kh, weight: w.1, bias: b, leaky }));
}
"ConvTranspose" => {
let (w, b) = weights_of(node, &initialisers, &mut weights_total)?;
let (cin, cout, kh, kw) = four(&w.0)?;
let b = if b.is_empty() { vec![0.0; cout] } else { b };
if (kh, kw) != (2, 2)
|| ints_or(node, "strides", &[1, 1]) != [2, 2]
|| ints_or(node, "pads", &[0; 4]) != [0; 4]
|| ints_or(node, "dilations", &[1, 1]) != [1, 1]
|| attr(node, "group").is_some_and(|a| a.i != 1)
|| attr(node, "output_padding").is_some_and(|a| a.ints.iter().any(|&v| v != 0))
{
return unsupported("a transposed convolution other than 2 × 2 with stride 2");
}
if attr(node, "auto_pad").is_some_and(|a| !a.s.is_empty() && a.s != b"NOTSET") {
return unsupported("transposed convolution auto padding");
}
if attr(node, "kernel_shape").is_some_and(|a| a.ints != [2, 2]) {
return invalid("transposed kernel shape does not match the weights");
}
let src = lookup(node.input.first().map_or("", String::as_str))?;
let out = node.output.first().ok_or_else(|| NetError::Invalid("ConvTranspose without an output".into()))?;
let (leaky, out_name) = fuse_leaky(out)?;
ids.insert(out_name, this);
ops.push(Op::ConvT2(ConvT2 { src, cin, cout, weight: w.1, bias: b, leaky }));
}
"MaxPool" => {
if ints_or(node, "kernel_shape", &[]) != [2, 2]
|| ints_or(node, "strides", &[1, 1]) != [2, 2]
|| ints_or(node, "pads", &[0; 4]) != [0; 4]
|| ints_or(node, "dilations", &[1, 1]) != [1, 1]
|| attr(node, "ceil_mode").is_some_and(|a| a.i != 0)
|| attr(node, "storage_order").is_some_and(|a| a.i != 0)
|| attr(node, "auto_pad").is_some_and(|a| !a.s.is_empty() && a.s != b"NOTSET")
{
return unsupported("a max-pool other than 2 × 2 with stride 2");
}
let src = lookup(node.input.first().map_or("", String::as_str))?;
let out = node.output.first().ok_or_else(|| NetError::Invalid("MaxPool without an output".into()))?;
ids.insert(out.clone(), this);
ops.push(Op::MaxPool2 { src });
}
"DepthToSpace" => {
if attr(node, "blocksize").map(|a| a.i) != Some(2) || attr(node, "mode").map(|a| a.s.as_slice()) != Some(&b"CRD"[..]) {
return unsupported("depth-to-space other than block size 2 in CRD order");
}
let src = lookup(node.input.first().map_or("", String::as_str))?;
let out = node.output.first().ok_or_else(|| NetError::Invalid("DepthToSpace without an output".into()))?;
ids.insert(out.clone(), this);
ops.push(Op::DepthToSpace2 { src });
}
other => return unsupported(format!("the `{other}` operator")),
}
}
if !joins.is_empty() {
return unsupported("a Concat whose result no convolution reads");
}
if ids.get(output.name.as_str()) != Some(&ops.len()) {
return unsupported("the output is not the last layer");
}
Net::new(in_channels, ops)
}
fn attr<'a>(n: &'a NodeProto, name: &str) -> Option<&'a AttributeProto> {
n.attribute.iter().find(|a| a.name == name)
}
/// The integer list attribute `name`, or `default` when the node does not have it.
fn ints_or(n: &NodeProto, name: &str, default: &[i64]) -> Vec<i64> {
attr(n, name).map_or_else(|| default.to_vec(), |a| a.ints.clone())
}
fn four(dims: &[i64]) -> Result<(usize, usize, usize, usize), NetError> {
match dims {
[a, b, c, d] if dims.iter().all(|&v| v > 0 && v <= 1 << 20) => Ok((*a as usize, *b as usize, *c as usize, *d as usize)),
_ => invalid("a weight tensor that is not four-dimensional"),
}
}
type Weights = ((Vec<i64>, Vec<f32>), Vec<f32>);
/// The weight and bias initialisers of a convolution node.
fn weights_of(node: &NodeProto, initialisers: &HashMap<&str, &TensorProto>, total: &mut usize) -> Result<Weights, NetError> {
let mut get = |idx: usize| -> Result<(Vec<i64>, Vec<f32>), NetError> {
let name = node.input.get(idx).ok_or_else(|| NetError::Invalid(format!("`{}` without its weights", node.op_type)))?;
let t = initialisers.get(name.as_str()).ok_or_else(|| NetError::Unsupported(format!("weights that are computed (`{name}`)")))?;
let values = floats(t)?;
*total = total.saturating_add(values.len());
if *total > MAX_WEIGHTS {
return invalid("the network has too many weights");
}
Ok((t.dims.clone(), values))
};
let w = get(1)?;
let b = if node.input.len() > 2 {
let (dims, values) = get(2)?;
if dims.len() != 1 || values.is_empty() {
return invalid("a bias tensor must be a nonempty vector");
}
values
} else {
Vec::new()
};
Ok((w, b))
}
fn floats(t: &TensorProto) -> Result<Vec<f32>, NetError> {
if t.data_type != 1 {
return unsupported("weights that are not 32-bit floats");
}
let want: usize = t
.dims
.iter()
.try_fold(1usize, |n, &d| usize::try_from(d).ok().and_then(|d| n.checked_mul(d)))
.ok_or_else(|| NetError::Invalid("a tensor with a negative or huge size".into()))?;
if want > MAX_WEIGHTS {
return invalid("a tensor with too many numbers");
}
if !t.raw_data.is_empty() && (!t.float_data.is_empty() || !t.raw_data.len().is_multiple_of(4)) {
return invalid("ambiguous or truncated tensor data");
}
let actual = if t.raw_data.is_empty() { t.float_data.len() } else { t.raw_data.len() / 4 };
if actual != want {
return invalid("tensor data does not match its dimensions");
}
let mut values = Vec::new();
values.try_reserve_exact(want).map_err(|_| NetError::Invalid("not enough memory for weights".into()))?;
if !t.raw_data.is_empty() {
values.extend(t.raw_data.as_chunks::<4>().0.iter().map(|b| f32::from_le_bytes(*b)));
} else {
values.extend_from_slice(&t.float_data);
}
if values.len() != want {
return invalid(format!("tensor `{}` has {} numbers, not the {want} its shape says", t.name, values.len()));
}
Ok(values)
}
/// The channel count in an input's declared shape `[batch, channels, …]`.
fn channel_dim(v: &ValueInfoProto) -> Option<usize> {
v.channels
}
#[cfg(test)]
mod tests {
use super::*;
use crate::synthetic::unet_onnx;
fn temp_model(name: &str, bytes: &[u8]) -> std::path::PathBuf {
let p = std::env::temp_dir().join(format!("lc-denoise-onnx-{name}-{}.onnx", std::process::id()));
std::fs::write(&p, bytes).unwrap();
p
}
#[test]
fn a_u_net_is_read_with_its_layers_fused() {
let (tile, depth) = (32usize, 2usize);
let p = temp_model("unet", &unet_onnx(tile as u64, 8, depth, 7));
let net = read(&p).unwrap();
// per level two convolutions; a pool per level but the last; going up a transposed convolution and two convolutions
// per level (the join is part of the first); then the 1 × 1 convolution and depth-to-space
assert_eq!(net.ops().len(), 2 * (depth + 1) + depth + depth * 3 + 2);
assert!(matches!(net.ops().last(), Some(crate::net::Op::DepthToSpace2 { .. })));
assert!(net.ops().iter().any(|o| matches!(o, crate::net::Op::Conv(c) if c.src2.is_some())), "the joins became two-source convolutions");
assert!(
net.ops().iter().all(|o| !matches!(o, crate::net::Op::Conv(c) if c.leaky.is_none() && c.k == 3)),
"every 3 × 3 convolution has its activation"
);
assert_eq!((net.in_channels, net.out_channels(), net.depth()), (4, 3, 2));
assert!(net.fits_tile(tile) && !net.fits_tile(30));
let _ = std::fs::remove_file(p);
}
#[test]
fn an_explicit_empty_or_nonvector_bias_is_not_treated_as_missing() {
let w = TensorProto { name: "w".into(), dims: vec![12, 4, 1, 1], data_type: 1, float_data: vec![0.1; 48], ..Default::default() };
let n = NodeProto { input: vec!["data".into(), "w".into(), "b".into()], ..Default::default() };
for dims in [vec![0], vec![3, 4]] {
let b = TensorProto {
name: "b".into(),
float_data: vec![0.0; dims.iter().product::<i64>() as usize],
dims,
data_type: 1,
..Default::default()
};
let inits = HashMap::from([("w", &w), ("b", &b)]);
assert!(weights_of(&n, &inits, &mut 0).is_err());
}
}
#[test]
fn truncated_and_mutated_protobuf_is_bounded_and_never_panics() {
let good = unet_onnx(4, 3, 1, 7);
for n in (0..good.len()).step_by(31) {
let _ = read_bytes(&good[..n]);
}
for at in (0..good.len()).step_by(37) {
let mut bad = good.clone();
bad[at] ^= 0xff;
let _ = read_bytes(&bad);
}
assert!(read_bytes(&[0x3a, 0xff, 0xff, 0xff, 0xff, 0x7f]).is_err());
assert!(read_bytes(&[0x3a, 0, 0x3a, 0]).is_err());
}
proptest::proptest! {
#[test]
fn arbitrary_model_bytes_never_panic(bytes in proptest::collection::vec(proptest::prelude::any::<u8>(), 0..4096)) {
let _ = read_bytes(&bytes);
}
}
#[test]
fn what_the_description_cannot_say_is_unsupported_and_garbage_is_an_error() {
// not a model
let p = temp_model("junk", b"this is not an onnx file");
assert!(matches!(read(&p), Err(NetError::Invalid(_)) | Err(NetError::Unsupported(_))));
let _ = std::fs::remove_file(p);
// a missing file
assert!(read(Path::new("/does/not/exist.onnx")).is_err());
}
}

295
crates/denoise/src/onnx_proto.rs Normal file
View File

@ -0,0 +1,295 @@
//! Bounded, borrowed decoding of the ONNX fields used by denoise, from the public Apache-2.0 schema:
//! https://github.com/onnx/onnx/blob/main/onnx/onnx.proto3
//! No generated code, protoc, recursive graph decoding, or external tensor files.
use crate::net::{MAX_OPS, MAX_WEIGHTS, NetError};
const MAX_ITEMS: usize = MAX_OPS * 4;
const MAX_STRING: usize = 4096;
pub(super) const MAX_BYTES: usize = MAX_WEIGHTS * 4 + 16 * 1024 * 1024;
type Result<T> = std::result::Result<T, NetError>;
fn bad() -> NetError {
NetError::Invalid("malformed or oversized ONNX protobuf".into())
}
struct Fields<'a> {
rest: &'a [u8],
count: usize,
}
enum Wire<'a> {
Int(u64),
Bytes(&'a [u8]),
F32(f32),
Other,
}
fn varint(rest: &mut &[u8]) -> Result<u64> {
let mut value = 0u64;
for shift in (0..70).step_by(7) {
let (b, tail) = rest.split_first().ok_or_else(bad)?;
*rest = tail;
if shift == 63 && *b > 1 {
return Err(bad());
}
value |= u64::from(*b & 127) << shift;
if b & 128 == 0 {
return Ok(value);
}
}
Err(bad())
}
impl<'a> Fields<'a> {
fn new(rest: &'a [u8]) -> Self {
Self { rest, count: 0 }
}
fn take(&mut self, n: usize) -> Result<&'a [u8]> {
let out = self.rest.get(..n).ok_or_else(bad)?;
self.rest = self.rest.get(n..).ok_or_else(bad)?;
Ok(out)
}
fn next(&mut self) -> Result<Option<(u32, Wire<'a>)>> {
if self.rest.is_empty() {
return Ok(None);
}
self.count += 1;
if self.count > MAX_WEIGHTS {
return Err(bad());
}
let tag = varint(&mut self.rest)?;
let number = u32::try_from(tag >> 3).map_err(|_| bad())?;
if number == 0 || number >= 1 << 29 {
return Err(bad());
}
let wire = match tag & 7 {
0 => Wire::Int(varint(&mut self.rest)?),
1 => {
self.take(8)?;
Wire::Other
}
2 => {
let n = usize::try_from(varint(&mut self.rest)?).map_err(|_| bad())?;
Wire::Bytes(self.take(n)?)
}
5 => {
let b: [u8; 4] = self.take(4)?.try_into().map_err(|_| bad())?;
Wire::F32(f32::from_le_bytes(b))
}
_ => return Err(bad()),
};
Ok(Some((number, wire)))
}
}
fn string(b: &[u8]) -> Result<String> {
if b.len() > MAX_STRING {
return Err(bad());
}
std::str::from_utf8(b).map(str::to_owned).map_err(|_| bad())
}
fn push<T>(v: &mut Vec<T>, value: T, cap: usize) -> Result<()> {
if v.len() >= cap {
return Err(bad());
}
v.try_reserve(1).map_err(|_| bad())?;
v.push(value);
Ok(())
}
fn ints(w: Wire<'_>, out: &mut Vec<i64>, cap: usize) -> Result<()> {
match w {
Wire::Int(v) => push(out, v as i64, cap),
Wire::Bytes(mut b) => {
while !b.is_empty() {
let v = varint(&mut b)?;
push(out, v as i64, cap)?;
}
Ok(())
}
_ => Err(bad()),
}
}
#[derive(Default)]
pub(super) struct AttributeProto {
pub name: String,
pub f: f32,
pub i: i64,
pub s: Vec<u8>,
pub ints: Vec<i64>,
}
#[derive(Default)]
pub(super) struct NodeProto {
pub input: Vec<String>,
pub output: Vec<String>,
pub op_type: String,
pub attribute: Vec<AttributeProto>,
}
#[derive(Default)]
pub(super) struct TensorProto<'a> {
pub name: String,
pub dims: Vec<i64>,
pub data_type: i32,
pub raw_data: &'a [u8],
pub float_data: Vec<f32>,
}
pub(super) struct ValueInfoProto {
pub name: String,
pub channels: Option<usize>,
}
#[derive(Default)]
pub(super) struct Graph<'a> {
pub node: Vec<NodeProto>,
pub initializer: Vec<TensorProto<'a>>,
pub input: Vec<ValueInfoProto>,
pub output: Vec<ValueInfoProto>,
}
fn attribute(b: &[u8]) -> Result<AttributeProto> {
let mut a = AttributeProto::default();
let mut fields = Fields::new(b);
while let Some((n, w)) = fields.next()? {
match (n, w) {
(1, Wire::Bytes(b)) => a.name = string(b)?,
(2, Wire::F32(v)) => a.f = v,
(3, Wire::Int(v)) => a.i = v as i64,
(4, Wire::Bytes(b)) => {
if b.len() > MAX_STRING {
return Err(bad());
}
a.s = b.to_vec();
}
(8, w) => ints(w, &mut a.ints, 16)?,
(20, Wire::Int(1 | 2 | 3 | 7)) => {}
_ => return Err(NetError::Unsupported("unknown attribute value".into())),
}
}
Ok(a)
}
fn node(b: &[u8]) -> Result<NodeProto> {
let mut a = NodeProto::default();
let mut fields = Fields::new(b);
while let Some((n, w)) = fields.next()? {
match (n, w) {
(1, Wire::Bytes(b)) => push(&mut a.input, string(b)?, 4)?,
(2, Wire::Bytes(b)) => push(&mut a.output, string(b)?, 2)?,
(4, Wire::Bytes(b)) => a.op_type = string(b)?,
(5, Wire::Bytes(b)) => push(&mut a.attribute, attribute(b)?, 16)?,
(7, Wire::Bytes(b)) if !b.is_empty() => return Err(NetError::Unsupported("a custom operator domain".into())),
_ => {}
}
}
Ok(a)
}
fn tensor(b: &[u8]) -> Result<TensorProto<'_>> {
let mut a = TensorProto::default();
let mut fields = Fields::new(b);
while let Some((n, w)) = fields.next()? {
match (n, w) {
(1, w) => ints(w, &mut a.dims, 8)?,
(2, Wire::Int(v)) => a.data_type = i32::try_from(v).map_err(|_| bad())?,
(4, Wire::F32(v)) => push(&mut a.float_data, v, MAX_WEIGHTS)?,
(4, Wire::Bytes(b)) => {
if !b.len().is_multiple_of(4) {
return Err(bad());
}
for chunk in b.as_chunks::<4>().0 {
push(&mut a.float_data, f32::from_le_bytes(*chunk), MAX_WEIGHTS)?;
}
}
(8, Wire::Bytes(b)) => a.name = string(b)?,
(9, Wire::Bytes(b)) => {
if b.len() > MAX_WEIGHTS * 4 {
return Err(bad());
}
a.raw_data = b;
}
(13, _) | (14, Wire::Int(1..)) => return Err(NetError::Unsupported("external tensor data".into())),
_ => {}
}
}
Ok(a)
}
fn channel_shape(b: &[u8]) -> Result<Option<usize>> {
let mut fields = Fields::new(b);
let mut dim = 0;
let mut channels = None;
while let Some((n, w)) = fields.next()? {
if let (1, Wire::Bytes(b)) = (n, w) {
if dim >= 8 {
return Err(bad());
}
let mut df = Fields::new(b);
while let Some((n, w)) = df.next()? {
if let (1, Wire::Int(v)) = (n, w)
&& dim == 1
{
channels = usize::try_from(v).ok().filter(|&v| v > 0);
}
}
dim += 1;
}
}
Ok(channels)
}
fn channel_type(b: &[u8]) -> Result<Option<usize>> {
let mut fields = Fields::new(b);
while let Some((n, w)) = fields.next()? {
if let (1, Wire::Bytes(b)) = (n, w) {
let mut tf = Fields::new(b);
let (mut elem, mut channels) = (0, None);
while let Some((n, w)) = tf.next()? {
match (n, w) {
(1, Wire::Int(v)) => elem = v,
(2, Wire::Bytes(b)) => channels = channel_shape(b)?,
_ => {}
}
}
return Ok(if elem == 1 { channels } else { None });
}
}
Ok(None)
}
fn value_info(b: &[u8]) -> Result<ValueInfoProto> {
let mut out = ValueInfoProto { name: String::new(), channels: None };
let mut fields = Fields::new(b);
while let Some((n, w)) = fields.next()? {
match (n, w) {
(1, Wire::Bytes(b)) => out.name = string(b)?,
(2, Wire::Bytes(b)) => out.channels = channel_type(b)?,
_ => {}
}
}
Ok(out)
}
fn graph(b: &[u8]) -> Result<Graph<'_>> {
let mut out = Graph::default();
let mut fields = Fields::new(b);
while let Some((n, w)) = fields.next()? {
match (n, w) {
(1, Wire::Bytes(b)) => push(&mut out.node, node(b)?, MAX_ITEMS)?,
(5, Wire::Bytes(b)) => push(&mut out.initializer, tensor(b)?, MAX_ITEMS)?,
(11, Wire::Bytes(b)) => push(&mut out.input, value_info(b)?, MAX_ITEMS)?,
(12, Wire::Bytes(b)) => push(&mut out.output, value_info(b)?, 1)?,
(15, _) => return Err(NetError::Unsupported("sparse initializers".into())),
_ => {}
}
}
Ok(out)
}
pub(super) fn decode(b: &[u8]) -> Result<Graph<'_>> {
if b.len() > MAX_BYTES {
return Err(bad());
}
let mut fields = Fields::new(b);
let mut out = None;
while let Some((n, w)) = fields.next()? {
if let (7, Wire::Bytes(b)) = (n, w) {
if out.is_some() {
return Err(bad());
}
out = Some(graph(b)?);
}
}
out.ok_or_else(bad)
}

133
crates/denoise/src/reference.rs Normal file
View File

@ -0,0 +1,133 @@
//! A plain-loop interpreter of a [`Net`], slow and obviously right: the reference the parser and the GPU runner are
//! tested against (tract's answer on the real model is the other). It is not used on photos.
use crate::net::{Net, NetError, Op, TensorId};
use crate::run::{Error, TileRunner};
/// A tensor in `[channels][side][side]` order.
struct T {
channels: usize,
side: usize,
data: Vec<f32>,
}
/// Run `net` on `input` (`in_channels` planes of `tile × tile`).
pub fn run(net: &Net, tile: usize, input: &[f32]) -> Result<Vec<f32>, NetError> {
if !net.fits_tile(tile) || input.len() != net.in_channels * tile * tile {
return Err(NetError::Invalid("the tile does not fit the network".into()));
}
let mut tensors: Vec<T> = vec![T { channels: net.in_channels, side: tile, data: input.to_vec() }];
let get = |tensors: &[T], id: TensorId| -> Result<usize, NetError> {
if id < tensors.len() { Ok(id) } else { Err(NetError::Invalid("a layer reads a tensor that does not exist yet".into())) }
};
for op in net.ops() {
let out = match op {
Op::Conv(c) => {
let a = get(&tensors, c.src)?;
let b = c.src2.map(|s| get(&tensors, s)).transpose()?;
let side = tensors[a].side;
let pad = (c.k - 1) / 2;
let mut data = vec![0f32; c.cout * side * side];
// the input channels in order: the first source's, then the second's
let planes: Vec<&[f32]> =
std::iter::once(&tensors[a]).chain(b.map(|b| &tensors[b])).flat_map(|t| t.data.chunks(side * side)).collect();
for co in 0..c.cout {
for y in 0..side {
for x in 0..side {
let mut sum = c.bias[co];
for (ci, plane) in planes.iter().enumerate() {
for ky in 0..c.k {
for kx in 0..c.k {
let (yy, xx) = ((y + ky) as isize - pad as isize, (x + kx) as isize - pad as isize);
if yy < 0 || xx < 0 || yy >= side as isize || xx >= side as isize {
continue;
}
sum += plane[yy as usize * side + xx as usize] * c.weight[((co * c.cin + ci) * c.k + ky) * c.k + kx];
}
}
}
data[(co * side + y) * side + x] = leaky(sum, c.leaky);
}
}
}
T { channels: c.cout, side, data }
}
Op::ConvT2(c) => {
let a = get(&tensors, c.src)?;
let side = tensors[a].side;
let out_side = 2 * side;
let mut data = vec![0f32; c.cout * out_side * out_side];
for co in 0..c.cout {
for y in 0..side {
for x in 0..side {
for dy in 0..2 {
for dx in 0..2 {
let mut sum = c.bias[co];
for ci in 0..c.cin {
sum += tensors[a].data[(ci * side + y) * side + x] * c.weight[((ci * c.cout + co) * 2 + dy) * 2 + dx];
}
data[(co * out_side + 2 * y + dy) * out_side + 2 * x + dx] = leaky(sum, c.leaky);
}
}
}
}
}
T { channels: c.cout, side: out_side, data }
}
Op::MaxPool2 { src } => {
let a = get(&tensors, *src)?;
let (channels, side) = (tensors[a].channels, tensors[a].side / 2);
let mut data = vec![0f32; channels * side * side];
for ch in 0..channels {
for y in 0..side {
for x in 0..side {
let at = |dy: usize, dx: usize| tensors[a].data[(ch * 2 * side + 2 * y + dy) * 2 * side + 2 * x + dx];
data[(ch * side + y) * side + x] = at(0, 0).max(at(0, 1)).max(at(1, 0)).max(at(1, 1));
}
}
}
T { channels, side, data }
}
Op::DepthToSpace2 { src } => {
let a = get(&tensors, *src)?;
let (channels, side) = (tensors[a].channels / 4, tensors[a].side);
let out_side = 2 * side;
let mut data = vec![0f32; channels * out_side * out_side];
for ch in 0..channels {
for y in 0..side {
for x in 0..side {
for dy in 0..2 {
for dx in 0..2 {
data[(ch * out_side + 2 * y + dy) * out_side + 2 * x + dx] =
tensors[a].data[((4 * ch + 2 * dy + dx) * side + y) * side + x];
}
}
}
}
}
T { channels, side: out_side, data }
}
};
tensors.push(out);
}
tensors.pop().map(|t| t.data).ok_or_else(|| NetError::Invalid("no layers".into()))
}
fn leaky(v: f32, alpha: Option<f32>) -> f32 {
match alpha {
Some(a) if v < 0.0 => v * a,
_ => v,
}
}
/// The reference as a [`TileRunner`] (for tests of the code around the model).
pub struct ReferenceRunner {
pub net: Net,
pub tile: usize,
}
impl TileRunner for ReferenceRunner {
fn run(&self, input: &[f32]) -> Result<Vec<f32>, Error> {
run(&self.net, self.tile, input).map_err(|e| Error::Runtime(e.to_string()))
}
}

433
crates/denoise/src/runtime.rs Normal file
View File

@ -0,0 +1,433 @@
//! Model loading, a shared installation self-test, and deterministic test tiles.
//! `CpuRunner` is the pure-Rust executor; scalar operator tests live in `reference`.
use std::path::Path;
use web_time::Instant;
use serde::Serialize;
use crate::manifest::{DenoiserManifest, Gain};
use crate::run::{Error, TileRunner};
/// What `self_test` found.
#[derive(Clone, Debug, PartialEq, Serialize)]
#[serde(rename_all = "camelCase")]
pub struct SelfTest {
pub ok: bool,
/// Milliseconds to load and optimise the model.
pub load_ms: f64,
/// Median milliseconds to run one tile.
pub tile_ms: f64,
/// Megapixels of picture one tile covers, for estimating a whole photo's time.
pub tile_megapixels: f64,
/// The model's output scale relative to its input on the test picture (what `Gain::MatchMean` takes out).
pub scale: f64,
/// Each check: what was checked and whether it held.
pub checks: Vec<(String, bool)>,
}
/// The default pure-Rust CPU model, with the same manifest and installation self-test as the reference.
#[derive(Clone)]
pub struct CpuRunner {
runner: crate::cpu::NetRunner,
manifest: DenoiserManifest,
load_ms: f64,
}
impl CpuRunner {
pub fn load(path: &Path, manifest: &DenoiserManifest) -> Result<Self, Error> {
crate::manifest::validate(manifest).map_err(|e| Error::Load(e.to_string()))?;
let started = Instant::now();
let net = crate::onnx::read(path).map_err(|e| Error::Load(e.to_string()))?;
let runner = crate::cpu::NetRunner::new(&net, manifest.tile as usize).map_err(|e| Error::Load(e.to_string()))?;
Ok(Self { runner, manifest: manifest.clone(), load_ms: started.elapsed().as_secs_f64() * 1e3 })
}
pub fn manifest(&self) -> &DenoiserManifest {
&self.manifest
}
pub fn net(&self) -> &crate::net::Net {
self.runner.net()
}
pub fn bytes_per_tile(&self) -> usize {
self.runner.bytes_per_tile()
}
pub fn self_test(&self) -> SelfTest {
self_test(self, &self.manifest, self.load_ms)
}
}
impl TileRunner for CpuRunner {
fn run(&self, input: &[f32]) -> Result<Vec<f32>, Error> {
self.runner.run(input)
}
}
fn self_test(runner: &dyn TileRunner, manifest: &DenoiserManifest, load_ms: f64) -> SelfTest {
let t = manifest.tile as usize;
let mut checks: Vec<(String, bool)> = Vec::new();
let (input, base) = test_tile(t);
let (mut times, mut scale) = (Vec::new(), 0.0f64);
let mut run = || {
let started = Instant::now();
let r = runner.run(&input);
times.push(started.elapsed().as_secs_f64() * 1000.0);
r
};
let (a, b) = (run(), run());
match (&a, &b) {
(Ok(a), Ok(b)) => {
checks.push((format!("gives {} finite numbers per tile", a.len()), true));
checks.push((
"gives the same answer for the same tile".into(),
a.iter().zip(b).all(|(x, y)| (x - y).abs() <= 1e-4 * x.abs().max(y.abs()).max(1e-9)),
));
let side = 2 * t;
let mean_in = base.iter().map(|&v| f64::from(v)).sum::<f64>() / base.len().max(1) as f64;
let mean_out = a.iter().map(|&v| f64::from(v)).sum::<f64>() / a.len().max(1) as f64;
scale = if mean_in > 0.0 { mean_out / mean_in } else { 0.0 };
if let Gain::MatchMean { nominal, max_deviation } = manifest.gain {
let ratio = scale / f64::from(nominal);
let tolerance = f64::from(max_deviation).max(0.05) * 4.0;
checks.push((format!("its scale is the {nominal:.3e} the description says (found {scale:.3e})"), (ratio - 1.0).abs() <= tolerance));
}
// noise left after bringing the answer back to the input's scale, against the noise that went in
let g = if scale > 0.0 { 1.0 / scale } else { 1.0 };
let (mut noise_in, mut noise_out, mut n_in, mut n_out) = (0.0f64, 0.0f64, 0usize, 0usize);
for (i, (&v, &s)) in input.iter().zip(base.iter().cycle()).enumerate() {
// centre of each plane only: edges are the model's guess
let (x, y) = ((i % (t * t)) % t, (i % (t * t)) / t);
if x >= t / 4 && x < 3 * t / 4 && y >= t / 4 && y < 3 * t / 4 {
noise_in += f64::from(v - s).powi(2);
n_in += 1;
}
}
for c in 0..3 {
for y in side / 4..3 * side / 4 {
for x in side / 4..3 * side / 4 {
if let (Some(&o), Some(&s)) = (a.get(c * side * side + y * side + x), base.get((y / 2) * t + x / 2)) {
noise_out += (f64::from(o) * g - f64::from(s)).powi(2);
n_out += 1;
}
}
}
}
let (rms_in, rms_out) = ((noise_in / n_in.max(1) as f64).sqrt(), (noise_out / n_out.max(1) as f64).sqrt());
checks.push((format!("takes noise down (from {rms_in:.4} to {rms_out:.4})"), rms_out < 0.8 * rms_in));
}
(Err(e), _) | (_, Err(e)) => checks.push((format!("runs: {e}"), false)),
}
times.sort_by(|x, y| x.total_cmp(y));
SelfTest {
ok: !checks.is_empty() && checks.iter().all(|(_, ok)| *ok),
load_ms,
tile_ms: times.get(times.len() / 2).copied().unwrap_or(0.0),
tile_megapixels: (2 * t * 2 * t) as f64 / 1e6,
scale,
checks,
}
}
/// A deterministic test tile: four planes of a smooth gradient with noise on top, and the gradient alone for one
/// plane (`tile × tile`).
pub fn test_tile(t: usize) -> (Vec<f32>, Vec<f32>) {
if t == 0 || t > 2048 {
return (Vec::new(), Vec::new());
}
let mut seed = 0x2545_f491u32;
let mut next = move || {
seed ^= seed << 13;
seed ^= seed >> 17;
seed ^= seed << 5;
(seed >> 8) as f32 / (1u32 << 24) as f32
};
let base: Vec<f32> = (0..t * t).map(|i| 0.12 + 0.3 * (((i % t) + (i / t)) as f32 / (2 * t) as f32)).collect();
let input = (0..4 * t * t)
.map(|i| {
// sum of four uniforms: close to Gaussian, deviation 0.0577 × 2 before scaling
let n = next() + next() + next() + next() - 2.0;
base.get(i % (t * t)).copied().unwrap_or(0.0) + 0.05 * n
})
.collect();
(input, base)
}
/// A tile for checking that two runners of one model agree: the whole range a raw photo has rather than the test tile's
/// mid-greys — a noisy ramp from black to white across the tile, a flat black corner and a clipped white one (four
/// planes of `tile × tile`).
pub fn check_tile(t: usize) -> Vec<f32> {
if t == 0 || t > 2048 {
return Vec::new();
}
let mut seed = 0x9e37_79b9u32;
let mut next = move || {
seed ^= seed << 13;
seed ^= seed >> 17;
seed ^= seed << 5;
(seed >> 8) as f32 / (1u32 << 24) as f32
};
let (edge, span) = ((t / 8).max(1), (2 * t).saturating_sub(2).max(1) as f32);
(0..4 * t * t)
.map(|i| {
let (x, y) = ((i % (t * t)) % t, (i % (t * t)) / t);
let n = next() + next() + next() + next() - 2.0;
if x < edge && y < edge {
0.0
} else if x + edge >= t && y + edge >= t {
1.0
} else {
((x + y) as f32 / span + 0.05 * n).clamp(0.0, 1.0)
}
})
.collect()
}
#[cfg(test)]
mod tests {
use super::*;
use crate::licence::Licence;
use crate::manifest::Domain;
use crate::synthetic_proto::{len_field, value_info_bytes};
fn varint(mut v: u64, out: &mut Vec<u8>) {
loop {
let b = (v & 0x7f) as u8;
v >>= 7;
if v == 0 {
out.push(b);
return;
}
out.push(b | 0x80);
}
}
fn varint_field(n: u64, v: u64, out: &mut Vec<u8>) {
varint(n << 3, out);
varint(v, out);
}
fn node(op: &str, inputs: &[&str], output: &str, ints: &[(&str, i64)]) -> Vec<u8> {
let mut n = Vec::new();
for i in inputs {
len_field(1, i.as_bytes(), &mut n);
}
len_field(2, output.as_bytes(), &mut n);
len_field(4, op.as_bytes(), &mut n);
for (name, value) in ints {
let mut a = Vec::new();
len_field(1, name.as_bytes(), &mut a);
varint_field(3, *value as u64, &mut a);
varint_field(20, 2, &mut a);
len_field(5, &a, &mut n);
}
n
}
fn tensor(name: &str, dims: &[u64], values: &[f32]) -> Vec<u8> {
let mut t = Vec::new();
for d in dims {
varint_field(1, *d, &mut t);
}
varint_field(2, 1, &mut t);
len_field(8, name.as_bytes(), &mut t);
let raw: Vec<u8> = values.iter().flat_map(|v| v.to_le_bytes()).collect();
len_field(9, &raw, &mut t);
t
}
/// A tiny working denoiser-shaped network: `[1, 4, t, t]` → a 1 × 1 convolution to 12 channels (each output
/// channel is `scale` times one input plane, so the scale is known) → depth-to-space → `[1, 3, 2t, 2t]`.
fn tiny_model(t: u64, scale: f32) -> Vec<u8> {
tiny_model_with(t, scale, 12)
}
/// The same with another number of convolution channels; only 12 can be arranged into the three colours.
fn tiny_model_with(t: u64, scale: f32, channels: u64) -> Vec<u8> {
let mut graph = Vec::new();
len_field(1, &node("Conv", &["data", "w"], if channels == 12 { "c" } else { "rgb" }, &[("group", 1)]), &mut graph);
if channels == 12 {
let mut depth_to_space = node("DepthToSpace", &["c"], "rgb", &[("blocksize", 2)]);
let mut mode = Vec::new();
len_field(1, b"mode", &mut mode);
len_field(4, b"CRD", &mut mode);
varint_field(20, 3, &mut mode);
len_field(5, &mode, &mut depth_to_space);
len_field(1, &depth_to_space, &mut graph);
}
// CRD output channel colour·4 + dy·2 + dx reads input plane [R, G1, B] of the tile.
let mut w = vec![0f32; channels as usize * 4];
for oc in 0..channels as usize {
let colour = (oc / 4).min(2);
let plane = [0, 1, 3][colour];
w[oc * 4 + plane] = scale;
}
len_field(5, &tensor("w", &[channels, 4, 1, 1], &w), &mut graph);
len_field(11, &value_info_bytes("data", 1, &[Ok(1), Ok(4), Ok(t), Ok(t)]), &mut graph);
len_field(12, &value_info_bytes("rgb", 1, &[Ok(1), Ok(3), Ok(2 * t), Ok(2 * t)]), &mut graph);
let mut ops = Vec::new();
len_field(1, b"", &mut ops);
varint_field(2, 13, &mut ops);
let mut model = Vec::new();
varint_field(1, 8, &mut model);
len_field(7, &graph, &mut model);
len_field(8, &ops, &mut model);
model
}
fn manifest(tile: u32, nominal: f32) -> DenoiserManifest {
DenoiserManifest {
id: "tiny".into(),
name: "Tiny".into(),
version: "1".into(),
licence: Licence::default(),
source: None,
sha256: None,
size_bytes: None,
provenance: String::new(),
domain: Domain::BayerToRgb,
tile,
overlap: 16,
gain: Gain::MatchMean { nominal, max_deviation: 0.05 },
}
}
fn temp_model(name: &str, bytes: &[u8]) -> std::path::PathBuf {
let p = std::env::temp_dir().join(format!("lc-denoise-runtime-{name}-{}.onnx", std::process::id()));
std::fs::write(&p, bytes).unwrap();
p
}
#[test]
fn a_working_model_loads_runs_and_has_the_scale_it_says() {
let p = temp_model("ok", &tiny_model(64, 1000.0));
let r = CpuRunner::load(&p, &manifest(64, 1000.0)).unwrap();
let input: Vec<f32> = (0..4 * 64 * 64).map(|i| (i % 97) as f32 / 97.0).collect();
let out = r.run(&input).unwrap();
assert_eq!(out.len(), 3 * 128 * 128);
// output (colour c, pixel (2y+dy, 2x+dx)) = scale × input plane [R, G1, B][c] at (y, x)
for (c, plane) in [0usize, 1, 3].into_iter().enumerate() {
let (y, x) = (5usize, 9usize);
assert!((out[c * 128 * 128 + (2 * y + 1) * 128 + 2 * x] - 1000.0 * input[plane * 64 * 64 + y * 64 + x]).abs() < 1e-2);
}
let _ = std::fs::remove_file(p);
}
#[test]
fn the_self_test_checks_scale_and_noise() {
// a model that only multiplies cannot take noise down: it must fail that check but pass the others
let p = temp_model("selftest", &tiny_model(64, 1000.0));
let t = CpuRunner::load(&p, &manifest(64, 1000.0)).unwrap().self_test();
assert!(!t.ok, "{t:?}");
assert!(t.checks.iter().any(|(c, ok)| c.contains("same answer") && *ok), "{t:?}");
assert!(t.checks.iter().any(|(c, ok)| c.contains("scale") && *ok), "{t:?}");
assert!(t.checks.iter().any(|(c, ok)| c.contains("noise") && !*ok), "{t:?}");
// and one that claims the wrong scale fails that check
let q = CpuRunner::load(&p, &manifest(64, 5.0)).unwrap().self_test();
assert!(q.checks.iter().any(|(c, ok)| c.contains("scale") && !*ok), "{q:?}");
let _ = std::fs::remove_file(p);
}
#[test]
fn wrong_sizes_and_broken_files_are_errors() {
let p = temp_model("sizes", &tiny_model(64, 1.0));
let r = CpuRunner::load(&p, &manifest(64, 1.0)).unwrap();
assert!(matches!(r.run(&[0.0; 10]), Err(Error::Input(_))));
// a fully convolutional model works at whatever tile size the manifest asks for
let wide = CpuRunner::load(&p, &manifest(128, 1.0)).unwrap();
assert_eq!(wide.run(&vec![0.5; 4 * 128 * 128]).unwrap().len(), 3 * 256 * 256);
// a model that gives another number of values than the manifest promises is an error
let q = temp_model("channels", &tiny_model_with(64, 1.0, 8));
assert!(CpuRunner::load(&q, &manifest(64, 1.0)).is_err(), "shape is refused before execution");
let _ = std::fs::remove_file(q);
for (name, bytes) in [("junk", b"nonsense".to_vec()), ("empty", Vec::new())] {
let q = temp_model(name, &bytes);
assert!(CpuRunner::load(&q, &manifest(64, 1.0)).is_err(), "{name}");
let _ = std::fs::remove_file(q);
}
assert!(CpuRunner::load(&std::env::temp_dir().join("does-not-exist.onnx"), &manifest(64, 1.0)).is_err());
let _ = std::fs::remove_file(p);
}
#[test]
fn the_check_tile_covers_black_to_white() {
for t in [1, 2, 7, 64] {
let c = check_tile(t);
assert_eq!(c.len(), 4 * t * t);
assert!(c.iter().all(|v| (0.0..=1.0).contains(v)), "{t}");
}
let c = check_tile(64);
let near = |lo: f32, hi: f32| c.iter().filter(|v| (lo..=hi).contains(*v)).count();
assert!(near(0.0, 0.0) >= 4 * 8 * 8 && near(1.0, 1.0) >= 4 * 8 * 8, "a flat black and a clipped white corner");
for band in 0..10 {
let lo = band as f32 / 10.0;
assert!(near(lo, lo + 0.1) > 100, "values around {lo}");
}
assert_eq!(check_tile(64), c, "the same tile every time");
}
#[test]
fn truncated_and_corrupted_models_never_panic() {
let good = tiny_model(64, 1.0);
for n in (0..good.len()).step_by(5) {
let q = temp_model("trunc", &good[..n]);
let _ = CpuRunner::load(&q, &manifest(64, 1.0));
let _ = std::fs::remove_file(q);
}
for at in (0..good.len()).step_by(7) {
let mut bad = good.clone();
if let Some(b) = bad.get_mut(at) {
*b = b.wrapping_add(0x55);
}
let q = temp_model("flip", &bad);
if let Ok(r) = CpuRunner::load(&q, &manifest(64, 1.0)) {
let _ = r.run(&vec![0.5; 4 * 64 * 64]);
}
let _ = std::fs::remove_file(q);
}
}
/// Opt-in: `LC_DENOISE_MODEL=<model_bayer.onnx> cargo test -p lightcraft-denoise --release
/// -- --ignored --nocapture` self-tests a real model (RawNIND's Bayer model: tile 512, scale about 1e6).
#[test]
#[ignore = "needs a real model file: set LC_DENOISE_MODEL"]
fn a_real_model_passes_the_self_test() {
let Some(path) = std::env::var_os("LC_DENOISE_MODEL") else { return };
let m = manifest(512, 1.0e6);
let r = CpuRunner::load(Path::new(&path), &m).unwrap();
let t = r.self_test();
println!("load {:.0} ms | {:.0} ms per tile | scale {:.4e} | {:?}", t.load_ms, t.tile_ms, t.scale, t.checks);
assert!(t.ok);
}
}
#[cfg(test)]
mod cpu_tests {
use super::*;
#[test]
fn manifest_and_test_tile_sizes_are_checked_without_tract() {
for t in [0, 2049, usize::MAX] {
assert!(test_tile(t).0.is_empty());
assert!(check_tile(t).is_empty());
}
let path = std::env::temp_dir().join(format!("lc-cpu-load-{}.onnx", std::process::id()));
std::fs::write(&path, crate::synthetic::unet_onnx(64, 5, 1, 7)).unwrap();
let mut m = DenoiserManifest {
id: "synthetic".into(),
name: "Synthetic".into(),
version: "1".into(),
licence: Default::default(),
source: None,
sha256: None,
size_bytes: None,
provenance: "synthetic test".into(),
domain: crate::manifest::Domain::BayerToRgb,
tile: 64,
overlap: 16,
gain: Gain::None,
};
let r = CpuRunner::load(&path, &m).unwrap();
let x = check_tile(64);
assert_eq!(r.run(&x).unwrap(), r.run(&x).unwrap());
assert!(r.self_test().checks.iter().any(|(text, ok)| text.contains("same answer") && *ok));
m.tile = u32::MAX;
assert!(CpuRunner::load(&path, &m).is_err());
std::fs::remove_file(path).unwrap();
}
}

234
crates/denoise/src/synthetic.rs Normal file
View File

@ -0,0 +1,234 @@
//! A small ONNX U-Net built in memory, for tests of the layers that read and run denoise networks (here, in the GPU
//! runner and in the engine): the same layer kinds and wiring as the real denoisers, with deterministic random weights.
//! Nothing here is a model anyone would use.
use crate::synthetic_proto::{len_field, value_info_bytes};
fn varint(mut v: u64, out: &mut Vec<u8>) {
loop {
let b = (v & 0x7f) as u8;
v >>= 7;
if v == 0 {
out.push(b);
return;
}
out.push(b | 0x80);
}
}
fn varint_field(n: u64, v: u64, out: &mut Vec<u8>) {
varint(n << 3, out);
varint(v, out);
}
enum A<'a> {
Int(i64),
Ints(&'a [i64]),
Float(f32),
Str(&'a str),
}
fn node(op: &str, inputs: &[&str], output: &str, attrs: &[(&str, A)]) -> Vec<u8> {
let mut n = Vec::new();
for i in inputs {
len_field(1, i.as_bytes(), &mut n);
}
len_field(2, output.as_bytes(), &mut n);
len_field(4, op.as_bytes(), &mut n);
for (name, value) in attrs {
let mut a = Vec::new();
len_field(1, name.as_bytes(), &mut a);
match value {
A::Int(v) => {
varint_field(3, *v as u64, &mut a);
varint_field(20, 2, &mut a);
}
A::Ints(vs) => {
for v in *vs {
varint_field(8, *v as u64, &mut a);
}
varint_field(20, 7, &mut a);
}
A::Float(v) => {
varint((2 << 3) | 5, &mut a);
a.extend_from_slice(&v.to_le_bytes());
varint_field(20, 1, &mut a);
}
A::Str(s) => {
len_field(4, s.as_bytes(), &mut a);
varint_field(20, 3, &mut a);
}
}
len_field(5, &a, &mut n);
}
n
}
fn tensor(name: &str, dims: &[u64], values: &[f32]) -> Vec<u8> {
let mut t = Vec::new();
for d in dims {
varint_field(1, *d, &mut t);
}
varint_field(2, 1, &mut t);
len_field(8, name.as_bytes(), &mut t);
let raw: Vec<u8> = values.iter().flat_map(|v| v.to_le_bytes()).collect();
len_field(9, &raw, &mut t);
t
}
struct Rng(u64);
impl Rng {
fn next(&mut self) -> f32 {
self.0 ^= self.0 >> 12;
self.0 ^= self.0 << 25;
self.0 ^= self.0 >> 27;
((self.0.wrapping_mul(0x2545_f491_4f6c_dd1d) >> 40) as f32 / (1u64 << 24) as f32) * 2.0 - 1.0
}
}
struct Builder {
nodes: Vec<Vec<u8>>,
inits: Vec<Vec<u8>>,
rng: Rng,
n: usize,
}
impl Builder {
fn name(&mut self, kind: &str) -> String {
self.n += 1;
format!("{kind}{}", self.n)
}
/// Random weights scaled so activations keep a sane size through the layers.
fn weights(&mut self, name: &str, dims: &[u64], fan_in: usize) -> String {
let count: u64 = dims.iter().product();
let scale = (3.0 / fan_in as f32).sqrt();
let values: Vec<f32> = (0..count).map(|_| self.rng.next() * scale).collect();
self.inits.push(tensor(name, dims, &values));
name.to_string()
}
fn bias(&mut self, name: &str, n: u64) -> String {
let values: Vec<f32> = (0..n).map(|_| self.rng.next() * 0.1).collect();
self.inits.push(tensor(name, &[n], &values));
name.to_string()
}
/// `Conv` (+ `LeakyRelu` when `leaky`); gives the name of the tensor that results.
fn conv(&mut self, src: &str, cin: u64, cout: u64, k: u64, leaky: bool) -> String {
let (w, b, out) = (self.name("w"), self.name("b"), self.name("conv"));
let w = self.weights(&w, &[cout, cin, k, k], (cin * k * k) as usize);
let b = self.bias(&b, cout);
let pad = (k as i64 - 1) / 2;
self.nodes.push(node(
"Conv",
&[src, &w, &b],
&out,
&[("kernel_shape", A::Ints(&[k as i64, k as i64])), ("pads", A::Ints(&[pad; 4])), ("strides", A::Ints(&[1, 1])), ("group", A::Int(1))],
));
if !leaky {
return out;
}
let act = self.name("act");
self.nodes.push(node("LeakyRelu", &[&out], &act, &[("alpha", A::Float(0.2))]));
act
}
fn pool(&mut self, src: &str) -> String {
let out = self.name("pool");
self.nodes.push(node(
"MaxPool",
&[src],
&out,
&[("kernel_shape", A::Ints(&[2, 2])), ("strides", A::Ints(&[2, 2])), ("pads", A::Ints(&[0; 4]))],
));
out
}
fn up(&mut self, src: &str, cin: u64, cout: u64) -> String {
let (w, b, out) = (self.name("uw"), self.name("ub"), self.name("up"));
let w = self.weights(&w, &[cin, cout, 2, 2], (cin * 4) as usize);
let b = self.bias(&b, cout);
self.nodes.push(node(
"ConvTranspose",
&[src, &w, &b],
&out,
&[("kernel_shape", A::Ints(&[2, 2])), ("strides", A::Ints(&[2, 2])), ("pads", A::Ints(&[0; 4]))],
));
out
}
fn concat(&mut self, a: &str, b: &str) -> String {
let out = self.name("cat");
self.nodes.push(node("Concat", &[a, b], &out, &[("axis", A::Int(1))]));
out
}
}
/// A denoiser-shaped U-Net as ONNX bytes: `[1, 4, tile, tile]` → `[1, 3, 2·tile, 2·tile]`, with `depth` poolings
/// (so `tile` must be a multiple of `2^depth`) and `ch` channels on the first level, doubling on each (every count a
/// multiple of 4). Each level is two 3 × 3 convolutions with a leaky ReLU, as in the real networks; the output is a
/// 1 × 1 convolution to 12 channels and depth-to-space.
pub fn unet_onnx(tile: u64, ch: u64, depth: usize, seed: u64) -> Vec<u8> {
let mut b = Builder { nodes: Vec::new(), inits: Vec::new(), rng: Rng(seed | 1), n: 0 };
let mut skips: Vec<(String, u64)> = Vec::new();
let (mut cur, mut c_in, mut c) = ("data".to_string(), 4u64, ch);
for level in 0..=depth {
let x = b.conv(&cur, c_in, c, 3, true);
let x = b.conv(&x, c, c, 3, true);
if level < depth {
skips.push((x.clone(), c));
cur = b.pool(&x);
c_in = c;
c *= 2;
} else {
cur = x;
}
}
for (skip, sc) in skips.into_iter().rev() {
let up = b.up(&cur, c, sc);
let joined = b.concat(&up, &skip);
let x = b.conv(&joined, 2 * sc, sc, 3, true);
cur = b.conv(&x, sc, sc, 3, true);
c = sc;
}
let rgb = b.conv(&cur, c, 12, 1, false);
b.nodes.push(node("DepthToSpace", &[&rgb], "rgb", &[("blocksize", A::Int(2)), ("mode", A::Str("CRD"))]));
let mut graph = Vec::new();
for n in &b.nodes {
len_field(1, n, &mut graph);
}
for t in &b.inits {
len_field(5, t, &mut graph);
}
len_field(11, &value_info_bytes("data", 1, &[Ok(1), Ok(4), Ok(tile), Ok(tile)]), &mut graph);
len_field(12, &value_info_bytes("rgb", 1, &[Ok(1), Ok(3), Ok(2 * tile), Ok(2 * tile)]), &mut graph);
let mut ops = Vec::new();
len_field(1, b"", &mut ops);
varint_field(2, 13, &mut ops);
let mut model = Vec::new();
varint_field(1, 8, &mut model);
len_field(7, &graph, &mut model);
len_field(8, &ops, &mut model);
model
}
/// A tiny smoothing network for installing a real CPU model in setup-flow tests.
/// It averages a 3 × 3 neighborhood of the four Bayer planes and returns three identical channels.
pub fn smoothing_onnx() -> Vec<u8> {
let mut graph = Vec::new();
len_field(1, &node("Conv", &["data", "weights"], "samples", &[("kernel_shape", A::Ints(&[3, 3])), ("pads", A::Ints(&[1, 1, 1, 1]))]), &mut graph);
len_field(1, &node("DepthToSpace", &["samples"], "rgb", &[("blocksize", A::Int(2)), ("mode", A::Str("CRD"))]), &mut graph);
len_field(5, &tensor("weights", &[12, 4, 3, 3], &[1.0 / 36.0; 12 * 4 * 3 * 3]), &mut graph);
len_field(11, &value_info_bytes("data", 1, &[Ok(1), Ok(4), Ok(64), Ok(64)]), &mut graph);
len_field(12, &value_info_bytes("rgb", 1, &[Ok(1), Ok(3), Ok(128), Ok(128)]), &mut graph);
let mut model = Vec::new();
varint_field(1, 8, &mut model);
len_field(7, &graph, &mut model);
let mut ops = Vec::new();
varint_field(2, 13, &mut ops);
len_field(8, &ops, &mut model);
model
}

44
crates/denoise/src/synthetic_proto.rs Normal file
View File

@ -0,0 +1,44 @@
pub(crate) fn varint(mut v: u64, out: &mut Vec<u8>) {
loop {
let b = (v & 0x7f) as u8;
v >>= 7;
if v == 0 {
out.push(b);
return;
}
out.push(b | 0x80);
}
}
pub fn len_field(n: u64, body: &[u8], out: &mut Vec<u8>) {
varint(n << 3 | 2, out);
varint(body.len() as u64, out);
out.extend_from_slice(body);
}
pub(crate) fn varint_field(n: u64, v: u64, out: &mut Vec<u8>) {
varint(n << 3, out);
varint(v, out);
}
/// A graph input or output: `Ok(n)` is a fixed dimension, `Err(name)` a named (dynamic) one.
pub fn value_info_bytes(name: &str, elem: u64, dims: &[std::result::Result<u64, &str>]) -> Vec<u8> {
let mut shape = Vec::new();
for d in dims {
let mut dim = Vec::new();
match d {
Ok(v) => varint_field(1, *v, &mut dim),
Err(p) => len_field(2, p.as_bytes(), &mut dim),
}
len_field(1, &dim, &mut shape);
}
let mut tensor = Vec::new();
varint_field(1, elem, &mut tensor);
len_field(2, &shape, &mut tensor);
let mut ty = Vec::new();
len_field(1, &tensor, &mut ty);
let mut vi = Vec::new();
len_field(1, name.as_bytes(), &mut vi);
len_field(2, &ty, &mut vi);
vi
}

250
crates/denoise/tests/real_model.rs Normal file
View File

@ -0,0 +1,250 @@
//! Opt-in end-to-end check of the real RawNIND Bayer model on real mosaics with noise added:
//!
//! ```text
//! LC_DENOISE_MODEL=<model_bayer.onnx> LC_DENOISE_MOSAICS=<folder> LC_DENOISE_OUT=<folder> \
//! cargo test --release -p lightcraft-denoise --features runtime --test real_model -- --ignored --nocapture
//! ```
//!
//! `LC_DENOISE_MOSAICS` holds `<name>.mosaic.f32` (little-endian f32, normalised: black 0, white 1, not white
//! balanced), `<name>.rgb.f32` (a clean demosaic of it, camera RGB, interleaved) and `<name>.txt` (`width`,
//! `height`, `cfa`, `wb` lines). Noise is Poisson-Gaussian, made here. Optional: `LC_DENOISE_PARALLEL` (tiles at
//! once, default 8), `LC_DENOISE_ONLY` (one name), `LC_DENOISE_ISO` (noise strength, default 1.0; 0 adds none),
//! `LC_DENOISE_FULL` (the whole frame instead of a 2048 × 1536 crop).
#![cfg(feature = "runtime")]
use std::path::{Path, PathBuf};
use std::time::Instant;
use lightcraft_denoise::bayer::Layout;
use lightcraft_denoise::manifest::{DenoiserManifest, Domain, Gain};
use lightcraft_denoise::run::{Control, Params, denoise_bayer};
use lightcraft_denoise::runtime::CpuRunner;
struct Rng(u64);
impl Rng {
fn next(&mut self) -> f64 {
self.0 ^= self.0 >> 12;
self.0 ^= self.0 << 25;
self.0 ^= self.0 >> 27;
((self.0.wrapping_mul(0x2545_f491_4f6c_dd1d) >> 11) as f64 + 0.5) / (1u64 << 53) as f64
}
fn gauss(&mut self) -> f64 {
(-2.0 * self.next().ln()).sqrt() * (std::f64::consts::TAU * self.next()).cos()
}
}
fn read_f32(path: &Path) -> Vec<f32> {
std::fs::read(path).unwrap().as_chunks::<4>().0.iter().map(|b| f32::from_le_bytes(*b)).collect()
}
struct Dump {
width: usize,
height: usize,
layout: Layout,
wb: [f32; 3],
mosaic: Vec<f32>,
rgb: Vec<f32>,
}
fn load(dir: &Path, name: &str) -> Dump {
let text = std::fs::read_to_string(dir.join(format!("{name}.txt"))).unwrap();
let field = |k: &str| text.lines().find_map(|l| l.strip_prefix(k).map(|v| v.trim().to_string())).unwrap();
let cell = match field("cfa").as_str() {
"RGGB" => [0, 1, 1, 2],
"BGGR" => [2, 1, 1, 0],
"GRBG" => [1, 0, 2, 1],
"GBRG" => [1, 2, 0, 1],
other => panic!("cfa {other}"),
};
let wb: Vec<f32> = field("wb").split_whitespace().map(|v| v.parse().unwrap()).collect();
Dump {
width: field("width").parse().unwrap(),
height: field("height").parse().unwrap(),
layout: Layout::from_cell(cell).unwrap(),
wb: [wb[0], wb[1], wb[2]],
mosaic: read_f32(&dir.join(format!("{name}.mosaic.f32"))),
rgb: read_f32(&dir.join(format!("{name}.rgb.f32"))),
}
}
/// Crop with an even origin so the Bayer phase is kept.
fn crop(d: &Dump, x0: usize, y0: usize, w: usize, h: usize) -> (Vec<f32>, Vec<f32>) {
let mut m = Vec::with_capacity(w * h);
let mut c = Vec::with_capacity(w * h * 3);
for y in y0..y0 + h {
m.extend_from_slice(&d.mosaic[y * d.width + x0..y * d.width + x0 + w]);
c.extend_from_slice(&d.rgb[(y * d.width + x0) * 3..(y * d.width + x0 + w) * 3]);
}
(m, c)
}
/// 0 = red, 1 = green, 2 = blue at a mosaic position.
fn site(layout: Layout, x: usize, y: usize) -> usize {
match ((x + layout.dx) % 2, (y + layout.dy) % 2) {
(0, 0) => 0,
(1, 1) => 2,
_ => 1,
}
}
/// The simplest demosaic (each missing colour is the mean of the same-coloured samples in the 3 × 3 around),
/// as the baseline a user would get from the noisy picture without AI.
fn bilinear(m: &[f32], w: usize, h: usize, layout: Layout) -> Vec<f32> {
let mut out = vec![0f32; w * h * 3];
for y in 0..h {
for x in 0..w {
let mut sum = [0f32; 3];
let mut n = [0f32; 3];
for yy in y.saturating_sub(1)..(y + 2).min(h) {
for xx in x.saturating_sub(1)..(x + 2).min(w) {
let c = site(layout, xx, yy);
sum[c] += m[yy * w + xx];
n[c] += 1.0;
}
}
for c in 0..3 {
out[(y * w + x) * 3 + c] = if n[c] > 0.0 { sum[c] / n[c] } else { 0.0 };
}
}
}
out
}
/// Signal-to-error ratio in dB of `got` against `want` (both interleaved RGB), ignoring a border.
fn snr(got: &[f32], want: &[f32], w: usize, h: usize, border: usize) -> f64 {
let (mut sig, mut err) = (0f64, 0f64);
for y in border..h - border {
for x in border..w - border {
for c in 0..3 {
let (g, t) = (f64::from(got[(y * w + x) * 3 + c]), f64::from(want[(y * w + x) * 3 + c]));
sig += t * t;
err += (g - t).powi(2);
}
}
}
10.0 * (sig / err.max(1e-30)).log10()
}
fn crc32(data: &[u8]) -> u32 {
let mut c = !0u32;
for &b in data {
c ^= u32::from(b);
for _ in 0..8 {
c = if c & 1 != 0 { 0xedb8_8320 ^ (c >> 1) } else { c >> 1 };
}
}
!c
}
/// A viewable PNG of interleaved camera-RGB panels side by side: white balanced, one exposure for all, gamma.
fn write_png(path: &Path, panels: &[&[f32]], w: usize, h: usize, wb: [f32; 3], exposure: f32) {
let total_w = w * panels.len();
let mut raw = Vec::with_capacity((total_w * 3 + 1) * h);
for y in 0..h {
raw.push(0u8);
for p in panels {
for x in 0..w {
for c in 0..3 {
let v = (p[(y * w + x) * 3 + c] * wb[c] * exposure).clamp(0.0, 1.0);
raw.push((v.powf(1.0 / 2.2) * 255.0 + 0.5) as u8);
}
}
}
}
let z = miniz_oxide::deflate::compress_to_vec_zlib(&raw, 6);
let mut png = b"\x89PNG\r\n\x1a\n".to_vec();
let mut chunk = |kind: &[u8; 4], body: &[u8]| {
png.extend_from_slice(&(body.len() as u32).to_be_bytes());
let mut k = kind.to_vec();
k.extend_from_slice(body);
png.extend_from_slice(&k);
png.extend_from_slice(&crc32(&k).to_be_bytes());
};
let mut ihdr = Vec::new();
ihdr.extend_from_slice(&(total_w as u32).to_be_bytes());
ihdr.extend_from_slice(&(h as u32).to_be_bytes());
ihdr.extend_from_slice(&[8, 2, 0, 0, 0]);
chunk(b"IHDR", &ihdr);
chunk(b"IDAT", &z);
chunk(b"IEND", &[]);
std::fs::write(path, png).unwrap();
}
#[test]
#[ignore = "needs a real model and mosaics: see the module docs"]
fn the_real_model_denoises_real_mosaics() {
let (Some(model), Some(dir)) = (std::env::var_os("LC_DENOISE_MODEL"), std::env::var_os("LC_DENOISE_MOSAICS")) else { return };
let dir = PathBuf::from(dir);
let out_dir = std::env::var_os("LC_DENOISE_OUT").map(PathBuf::from);
let parallel: usize = std::env::var("LC_DENOISE_PARALLEL").ok().and_then(|v| v.parse().ok()).unwrap_or(8);
let iso: f64 = std::env::var("LC_DENOISE_ISO").ok().and_then(|v| v.parse().ok()).unwrap_or(1.0);
let names: Vec<String> = match std::env::var("LC_DENOISE_ONLY") {
Ok(n) => vec![n],
Err(_) => ["arw-sony-a7m3-compressed", "cr2-canon-6d", "nef-nikon-d5100-lossless", "dng-google-pixel2xl", "pef-pentax-k3"]
.map(String::from)
.to_vec(),
};
let manifest = DenoiserManifest {
id: "rawnind-bayer".into(),
name: "RawNIND Bayer".into(),
version: "1".into(),
licence: Default::default(),
source: None,
sha256: None,
size_bytes: None,
provenance: String::new(),
domain: Domain::BayerToRgb,
tile: 512,
overlap: 64,
gain: Gain::MatchMean { nominal: 1.0e6, max_deviation: 0.05 },
};
let runner = CpuRunner::load(Path::new(&model), &manifest).unwrap();
let params = Params { tile: 512, overlap: 64, gain: manifest.gain, clip: Some(1.0), parallel };
for name in names {
let d = load(&dir, &name);
// a crop of about 2 × 1.5 tiles, away from the edges, even origin
let full = std::env::var_os("LC_DENOISE_FULL").is_some();
let (w, h) = if full { (d.width & !1, d.height & !1) } else { (2048usize, 1536usize) };
let (x0, y0) = (((d.width - w) / 3) & !1, ((d.height - h) / 2) & !1);
let (clean, reference) = crop(&d, x0, y0, w, h);
// Poisson-Gaussian noise: variance a·signal + b, as a sensor at high ISO
let (a, b) = (1.5e-3 * iso, (4.0e-3 * iso.sqrt()).powi(2));
let mut rng = Rng(0x9e37_79b9_7f4a_7c15 ^ name.len() as u64);
let noisy: Vec<f32> = clean
.iter()
.map(|&v| {
let v = f64::from(v);
let sigma = (a * v.max(0.0) + b).sqrt();
((v + sigma * rng.gauss()) as f32).min(1.0)
})
.collect();
let started = Instant::now();
let done = std::sync::atomic::AtomicUsize::new(0);
let progress = |n: usize, total: usize| done.store(n * 1000 / total, std::sync::atomic::Ordering::Relaxed);
let denoised = denoise_bayer(&noisy, w, h, d.layout, &runner, &params, &Control { cancel: None, progress: Some(&progress) }).unwrap();
let secs = started.elapsed().as_secs_f64();
let baseline = bilinear(&noisy, w, h, d.layout);
let clean_bilinear = bilinear(&clean, w, h, d.layout);
let den: Vec<f32> = denoised.data.iter().flat_map(|p| p.iter().copied()).collect();
let (s_noisy, s_den) = (snr(&baseline, &reference, w, h, 64), snr(&den, &reference, w, h, 64));
let s_floor = snr(&clean_bilinear, &reference, w, h, 64);
println!(
"{name}: {w}×{h} crop, {} tiles, parallel {parallel}, {secs:.1} s | SNR vs clean demosaic: noisy bilinear {s_noisy:.2} dB, denoised {s_den:.2} dB (clean bilinear {s_floor:.2} dB)",
lightcraft_denoise::run::tile_count(w, h, d.layout, 512, 64)
);
if let Some(out) = &out_dir {
std::fs::create_dir_all(out).unwrap();
let mean = reference.iter().map(|&v| f64::from(v)).sum::<f64>() / reference.len() as f64;
let exposure = (0.18 / mean.max(1e-6)) as f32;
// three panels, a 512 × 384 window each, at 1:1
let (pw, ph, px, py) = (512usize, 384usize, 700usize, 500usize);
let window =
|buf: &[f32]| -> Vec<f32> { (0..ph).flat_map(|y| buf[((py + y) * w + px) * 3..((py + y) * w + px + pw) * 3].to_vec()).collect() };
write_png(&out.join(format!("{name}.png")), &[&window(&baseline), &window(&den), &window(&reference)], pw, ph, d.wb, exposure);
}
if iso >= 4.0 {
assert!(s_den > s_noisy + 2.0, "{name}: denoised {s_den:.2} dB is not clearly better than noisy {s_noisy:.2} dB");
}
}
}

1
crates/develop/src/controls.rs
View File

@ -207,6 +207,7 @@ controls! {
"grain.amount" => grain.amount, "Grain", Grain, 0, 100, 0, 1, 0, Plain;
"grain.size" => grain.size, "Size", Grain, 0, 100, 25, 1, 0, Plain;
"grain.roughness" => grain.roughness, "Roughness", Grain, 0, 100, 50, 1, 0, Plain;
"enhance.denoise" => enhance.denoise, "Amount", Detail, 0, 100, 0, 1, 0, Plain;
"detail.sharpenAmount" => detail.sharpen_amount, "Sharpening", Detail, 0, 150, 0, 1, 0, Plain;
"detail.sharpenRadius" => detail.sharpen_radius, "Radius", Detail, 0.5, 3, 1, 0.1, 1, Plain;
"detail.sharpenDetail" => detail.sharpen_detail, "Detail", Detail, 0, 100, 25, 1, 0, Plain;

44
crates/develop/src/lib.rs
View File

@ -94,6 +94,13 @@ impl DevelopSettings {
std::borrow::Cow::Owned(d)
}
/// How much of the AI-denoised picture to mix in, 0 (none) to 1 (all of it): the Denoise amount, unless the Detail
/// section is switched off or the value is not a number.
pub fn denoise_amount(&self) -> f32 {
let a = self.enhance.denoise;
if a.is_finite() && self.enhance.denoise_enabled() && self.section_enabled("detail") { (a / 100.0).clamp(0.0, 1.0) as f32 } else { 0.0 }
}
pub fn set_section_enabled(&mut self, section: &str, on: bool) {
self.disabled_sections.retain(|s| s != section);
if !on {
@ -112,6 +119,7 @@ impl DevelopSettings {
self.curve = d;
}
Section::Color => self.wb.mode = WbMode::AsShot,
Section::Detail => self.enhance.denoise_on = None,
_ => {}
}
}
@ -193,4 +201,40 @@ mod tests {
s.set_section_enabled("effects", true);
assert!(s.section_enabled("effects"));
}
#[test]
fn the_denoise_amount_is_a_slider_and_follows_the_detail_section() {
let mut s = DevelopSettings::default();
assert_eq!(s.denoise_amount(), 0.0);
assert!(controls::set(&mut s, "enhance.denoise", 60.0));
assert_eq!(controls::get(&s, "enhance.denoise"), Some(60.0));
assert!((s.denoise_amount() - 0.6).abs() < 1e-6);
// the slider's range is 0 to 100; values outside it are brought in
controls::set(&mut s, "enhance.denoise", 500.0);
assert_eq!(s.denoise_amount(), 1.0);
controls::set(&mut s, "enhance.denoise", -5.0);
assert_eq!(s.denoise_amount(), 0.0);
// switching the Detail section off switches the denoise off with it; a settings file with a bad number does nothing
controls::set(&mut s, "enhance.denoise", 80.0);
s.set_section_enabled("detail", false);
assert_eq!(s.denoise_amount(), 0.0);
s.set_section_enabled("detail", true);
s.enhance.denoise = f64::NAN;
assert_eq!(s.denoise_amount(), 0.0);
// it is part of Detail: a reset of the section clears it
controls::set(&mut s, "enhance.denoise", 40.0);
s.reset_section(Section::Detail);
assert_eq!(s.enhance.denoise, 0.0);
// The later independent switch preserves Amount when off, and can be on at zero.
controls::set(&mut s, "enhance.denoise", 75.0);
s.enhance.denoise_on = Some(false);
assert_eq!(s.denoise_amount(), 0.0);
assert_eq!(s.enhance.denoise, 75.0);
s.enhance.denoise_on = Some(true);
assert_eq!(s.denoise_amount(), 0.75);
controls::set(&mut s, "enhance.denoise", 0.0);
assert!(s.enhance.denoise_enabled());
s.reset_section(Section::Detail);
assert_eq!(s.enhance.denoise_on, None);
}
}

8
crates/develop/src/settings.rs
View File

@ -915,6 +915,14 @@ pub struct LensBlur {
#[serde(default)]
pub struct Enhance {
pub denoise: f64,
#[serde(skip_serializing_if = "Option::is_none")]
pub denoise_on: Option<bool>,
pub raw_details: bool,
pub super_resolution: bool,
}
impl Enhance {
pub fn denoise_enabled(&self) -> bool {
self.denoise_on.unwrap_or(self.denoise > 0.0)
}
}

4
crates/engine/Cargo.toml
View File

@ -21,6 +21,8 @@ web-time = { workspace = true }
lightcraft-preview = { workspace = true }
lightcraft-codecs = { workspace = true }
lightcraft-raw = { workspace = true }
lightcraft-denoise-core = { workspace = true }
lightcraft-denoise = { workspace = true, optional = true }
lightcraft-meta = { workspace = true }
lightcraft-faces = { workspace = true }
miniz_oxide = { workspace = true }
@ -43,6 +45,8 @@ lightcraft-fetch = { workspace = true }
[features]
# AI masks with SAM 3 (Object / Describe). Off for the web and CLI builds.
sam = ["dep:lightcraft-segment"]
denoise = ["dep:lightcraft-denoise", "lightcraft-denoise/runtime", "lightcraft-gpu/denoise"]
rawnind-model = ["denoise", "lightcraft-denoise/rawnind-model", "lightcraft-denoise-core/rawnind-model"]
[dev-dependencies]
libc = { workspace = true }

15
crates/engine/src/camera_profiles.rs
View File

@ -65,19 +65,8 @@ impl CameraProfile {
}
}
/// LightCraft's configuration folder (settings, GPU marker, camera profiles).
pub fn config_dir() -> Option<PathBuf> {
if cfg!(target_os = "macos") {
std::env::var_os("HOME").map(|h| PathBuf::from(h).join("Library/Application Support/LightCraft"))
} else if cfg!(windows) {
std::env::var_os("APPDATA").map(|a| PathBuf::from(a).join("LightCraft"))
} else {
std::env::var_os("XDG_CONFIG_HOME")
.map(PathBuf::from)
.or_else(|| std::env::var_os("HOME").map(|h| PathBuf::from(h).join(".config")))
.map(|c| c.join("lightcraft"))
}
}
/// LightCraft's configuration folder (settings, GPU marker, camera profiles); one helper for the app.
pub use crate::config::config_dir;
/// Where camera profiles are read from and written to.
pub fn dir() -> Option<PathBuf> {

659
crates/engine/src/cmd/denoise.rs Normal file
View File

@ -0,0 +1,659 @@
//! Denoise commands (`denoise.*`): the models (list, install, download, test, remove, choose), the settings, and the
//! work (queue, cancel, status, pump, clear). The machinery they drive is [`crate::denoise`].
//!
//! Models live in the folder the host provides (`Session::denoise.models_dir`): one subfolder per model with
//! `model.onnx`, `denoise-model.json` (its manifest) and `installed.json` (what the user accepted and the file's hash),
//! plus `settings.json` beside them. Model files, archives and manifests are hostile input: sizes are capped, ids are
//! validated before they become folder names, and nothing outside a model's own folder is ever touched.
use std::path::{Path, PathBuf};
use std::sync::{Arc, PoisonError, mpsc};
use lightcraft_catalog::PhotoId;
use lightcraft_denoise::archive;
use lightcraft_denoise::hash::sha256_file;
use lightcraft_denoise::known::{self, Known};
use lightcraft_denoise::manifest::{self, DenoiserManifest, MAX_MANIFEST_BYTES, MAX_MODEL_BYTES};
use serde_json::{Value, json};
use super::{CommandSpec, always, bad, bool_or, cmd, str_param};
use crate::denoise::{Pace, RunOn, Settings, forget_gpu_set_ups, installed_models, read_capped, read_settings, write_atomic, write_settings};
use crate::model_download::State;
use crate::{EngineError, Result, Session};
fn fail(what: &str, e: impl std::fmt::Display) -> EngineError {
EngineError::Other(format!("{what}: {e}"))
}
fn models_dir(s: &Session, cmd: &str) -> Result<PathBuf> {
s.denoise.models_dir.clone().ok_or_else(|| bad(cmd, "this build has nowhere to keep denoise models (the desktop app does)"))
}
/// What a command says about a model.
fn row(m: &DenoiserManifest, download_host: Option<String>, installed: bool, selected: bool, accepted: &Value) -> Value {
json!({
"id": m.id,
"name": m.name,
"version": m.version,
"licence": m.licence,
"provenance": m.provenance,
"source": m.source,
"sizeBytes": m.size_bytes,
"sha256": m.sha256,
"tile": m.tile,
"known": known::find(&m.id).is_some(),
// a pinned address LightCraft can fetch it from (the user presses Download), and which site that is
"downloadHost": download_host,
"installed": installed,
"selected": selected,
"accepted": accepted,
})
}
fn host_of(k: &Known) -> Option<String> {
k.download.as_ref().map(|d| lightcraft_denoise::known::host(&d.url).to_string())
}
fn list(s: &mut Session, _: &Value) -> Result<Value> {
s.denoise_refresh_active(true);
let dir = s.denoise.models_dir.clone();
let st = dir.as_deref().map(read_settings).unwrap_or_default();
let installed = dir.as_deref().map(installed_models).unwrap_or_default();
let on_disk = |id: &str| installed.iter().find(|i| i.manifest.id == id);
let chosen = |id: &str| st.model.as_deref() == Some(id);
let mut models: Vec<Value> = Vec::new();
for k in known::all() {
let found = on_disk(&k.manifest.id);
models.push(row(&k.manifest, host_of(&k), found.is_some(), chosen(&k.manifest.id), found.map_or(&Value::Null, |f| &f.accepted)));
}
for i in installed.iter().filter(|i| known::find(&i.manifest.id).is_none()) {
models.push(row(&i.manifest, None, true, chosen(&i.manifest.id), &i.accepted));
}
Ok(json!({
"dir": dir.as_ref().map(|d| d.display().to_string()),
"productsDir": s.denoise_products_dir().map(|d| d.display().to_string()),
"model": st.model,
// whether this build can run denoise models (the `denoise` feature: our pure-Rust CPU/GPU runners)
"runtime": cfg!(feature = "denoise"),
"auto": st.auto(),
"cacheGb": st.cache_bytes() >> 30,
"threads": st.threads,
"runOn": st.run_on().name(),
"models": models,
}))
}
/// What a file the user pointed at is, as far as installing it goes.
struct Source {
manifest: DenoiserManifest,
/// The `.onnx` inside the archive; `None` when the file is the `.onnx`.
entry: Option<&'static str>,
file_name: String,
}
fn identify(path: &Path, c: &str, sha_known: Option<&str>) -> Result<Source> {
let meta = std::fs::metadata(path).map_err(|e| bad(c, format!("cannot read `{}`: {e}", path.display())))?;
if !meta.is_file() {
return Err(bad(c, format!("`{}` is not a file", path.display())));
}
if meta.len() == 0 || meta.len() > MAX_MODEL_BYTES {
return Err(bad(c, "a model file must be between 1 byte and 2 GiB"));
}
let sha256 = match sha_known {
Some(h) => h.to_string(),
None => sha256_file(path).map_err(|e| fail(&format!("could not read {}", path.display()), e))?,
};
let file_name = path.file_name().map(|n| n.to_string_lossy().into_owned()).unwrap_or_default();
// an archive LightCraft knows (the file darktable publishes the model in)
if let Some(k) = known::all().into_iter().find(|k| k.download.as_ref().is_some_and(|d| d.sha256 == sha256)) {
return Ok(Source { manifest: k.manifest, entry: k.entry, file_name });
}
// the `.onnx` of a known model
if let Some(m) = known::by_sha256(&sha256) {
return Ok(Source { manifest: m, entry: None, file_name });
}
// an `.onnx` with its manifest beside it
let sibling = path.with_file_name("denoise-model.json");
if path.extension().is_some_and(|e| e.eq_ignore_ascii_case("onnx"))
&& let Some(bytes) = read_capped(&sibling, MAX_MANIFEST_BYTES)
{
let m = manifest::parse(&bytes).map_err(|e| bad(c, format!("{}: {e}", sibling.display())))?;
return Ok(Source { manifest: m, entry: None, file_name });
}
Err(bad(
c,
"LightCraft does not know this file. A denoise model is an .onnx file with a denoise-model.json beside it that says how to use it \
(see docs/denoise.md), or an archive LightCraft can fetch itself (`denoise.models.download`)",
))
}
/// Install the model in `src`. A model that passes its self-test becomes the one in use (`activate`, the default): installing
/// a model is how someone says they want it. `verified` is for a download whose hash was already checked.
fn install_file(
dir: &Path,
loader: &crate::denoise::Loader,
run_on: RunOn,
accepted_at: String,
c: &str,
src: &Path,
acknowledged: bool,
verified: bool,
) -> Result<(Value, Arc<dyn crate::denoise::Model>)> {
let ident = identify(src, c, None)?;
let m = ident.manifest.clone();
manifest::validate(&m).map_err(|e| bad(c, e.to_string()))?;
if !acknowledged {
return Err(bad(
c,
"the licence has not been accepted: show the user its terms (see `denoise.models.list`) and pass `acknowledged: true` once they agree",
));
}
let home = dir.join(&m.id);
if home.join("model.onnx").exists() {
return Err(bad(c, "that model is already installed; remove it before installing a replacement"));
}
std::fs::create_dir_all(&home).map_err(|e| fail("could not create the model's folder", e))?;
let mut cleanup = InstallCleanup { home: home.clone(), armed: true };
let (part, final_path) = (home.join("model.onnx.part"), home.join("model.onnx"));
let _ = std::fs::remove_file(&part);
let placed: Result<()> = match ident.entry {
Some(entry) => {
archive::extract(src, entry, &part, MAX_MODEL_BYTES).map(|_| ()).map_err(|e| fail("could not take the model out of the archive", e))
}
None if verified && std::fs::rename(src, &part).is_ok() => Ok(()),
None => std::fs::copy(src, &part).map(|_| ()).map_err(|e| fail("could not copy the model (is the disk full?)", e)),
};
let sha256 = placed.and_then(|_| sha256_file(&part).map_err(|e| fail("could not check the model", e))).and_then(|h| match &m.sha256 {
Some(want) if *want != h => Err(EngineError::Other("the model file is not the one its description names (its SHA-256 differs)".into())),
_ => Ok(h),
});
let sha256 = match sha256 {
Ok(h) => h,
Err(e) => {
let _ = std::fs::remove_file(&part);
return Err(e);
}
};
// Test the staged file first. A failed replacement preserves the installed model.
let model = loader(&part, &m).map_err(|e| bad(c, e))?;
let test = match model.self_test(run_on) {
Ok(t) => t,
Err(e) => {
let _ = std::fs::remove_file(&part);
return Err(bad(c, e));
}
};
// An installed id is immutable: installing again must never destroy a usable model.
if final_path.exists() {
let _ = std::fs::remove_file(&part);
return Err(bad(c, "that model is already installed; remove it before installing a replacement"));
}
std::fs::rename(&part, &final_path).map_err(|e| fail("could not finish the copy", e))?;
write_atomic(&home.join("denoise-model.json"), &serde_json::to_vec_pretty(&m).map_err(|e| fail("manifest", e))?).map_err(EngineError::Other)?;
let accepted = json!({
"acceptedAt": accepted_at,
"licence": m.licence.name,
"commercial": m.licence.commercial,
"fileName": ident.file_name,
"sha256": sha256,
"selfTest": test,
});
write_atomic(&home.join("installed.json"), &serde_json::to_vec_pretty(&accepted).map_err(|e| fail("record", e))?).map_err(EngineError::Other)?;
cleanup.armed = false;
Ok((json!({"installed": row(&m, known::find(&m.id).and_then(|k| host_of(&k)), true, false, &accepted), "model": m.id}), model))
}
/// Removes only files created for this new installation, including on a worker panic.
struct InstallCleanup {
home: PathBuf,
armed: bool,
}
impl Drop for InstallCleanup {
fn drop(&mut self) {
if self.armed {
for name in ["model.onnx.part", "model.onnx", "denoise-model.json", "installed.json"] {
let _ = std::fs::remove_file(self.home.join(name));
}
let _ = std::fs::remove_dir(&self.home);
}
}
}
pub(crate) struct InstallJob {
id: String,
activate: bool,
result: mpsc::Receiver<std::result::Result<(Value, Arc<dyn crate::denoise::Model>), String>>,
}
fn activate_install(s: &mut Session, result: &Value, model: Arc<dyn crate::denoise::Model>, activate: bool) -> Result<Value> {
let dir = models_dir(s, "denoise.models.install")?;
let id = result["installed"]["id"].as_str().ok_or_else(|| bad("denoise.models.install", "missing installed id"))?;
let mut st = read_settings(&dir);
if activate {
st.model = Some(id.to_string());
write_settings(&dir, &st).map_err(EngineError::Other)?;
}
s.denoise.touch();
s.denoise_refresh_active(true);
if let Some(a) = &s.denoise.active
&& a.id == id
{
*a.model.lock().unwrap_or_else(PoisonError::into_inner) = Some(model);
}
let mut v = result.clone();
v["installed"]["selected"] = json!(st.model.as_deref() == Some(id));
v["model"] = json!(st.model);
Ok(v)
}
fn start_install(s: &mut Session, src: PathBuf, id: String, activate: bool, verified: bool) -> Result<Value> {
const C: &str = "denoise.models.install";
if s.denoise.install.is_some() {
return Err(bad(C, "another model is being installed"));
}
let dir = models_dir(s, C)?;
let loader = s.denoise.loader.clone();
let run_on = s.denoise.settings.run_on();
let at = (s.clock)();
let (tx, rx) = mpsc::channel();
std::thread::Builder::new()
.name("denoise-model-install".into())
.stack_size(16 * 1024 * 1024)
.spawn(move || {
let outcome = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| {
let _held = crate::memory::work_gate().acquire(512 * 1024 * 1024);
install_file(&dir, &loader, run_on, at, C, &src, true, verified).map_err(|e| plain(&e))
}))
.unwrap_or_else(|_| Err("the model installation stopped unexpectedly".into()));
if verified {
let _ = std::fs::remove_file(&src);
}
let _ = tx.send(outcome);
})
.map_err(|e| bad(C, format!("could not start installation: {e}")))?;
s.denoise.install = Some(InstallJob { id: id.clone(), activate, result: rx });
s.denoise.downloads.set_outcome(&id, State::Installing);
Ok(json!({"started": id}))
}
fn install(s: &mut Session, p: &Value) -> Result<Value> {
const C: &str = "denoise.models.install";
let path = str_param(p, "path").ok_or_else(|| bad(C, "missing `path`"))?;
let acknowledged = bool_or(p, "acknowledged", false);
let activate = bool_or(p, "activate", true);
if !acknowledged {
return Err(bad(C, "the licence has not been accepted: show its terms first"));
}
if bool_or(p, "background", false) {
return start_install(s, PathBuf::from(path), "local-install".into(), activate, false);
}
if s.denoise.install.is_some() {
return Err(bad(C, "another model is being installed"));
}
let (result, model) =
install_file(&models_dir(s, C)?, &s.denoise.loader, s.denoise.settings.run_on(), (s.clock)(), C, Path::new(path), acknowledged, false)?;
activate_install(s, &result, model, activate)
}
/// Read only the small adjacent manifest; hashing and execution happen in the install worker.
fn inspect(s: &mut Session, p: &Value) -> Result<Value> {
const C: &str = "denoise.models.inspect";
let path = str_param(p, "path").ok_or_else(|| bad(C, "missing `path`"))?;
let src = Path::new(path);
if !src.extension().is_some_and(|e| e.eq_ignore_ascii_case("onnx")) {
return Err(bad(C, "choose an .onnx file with denoise-model.json beside it"));
}
let bytes = read_capped(&src.with_file_name("denoise-model.json"), MAX_MANIFEST_BYTES)
.ok_or_else(|| bad(C, "denoise-model.json is missing or too large"))?;
let m = manifest::parse(&bytes).map_err(|e| bad(C, e.to_string()))?;
let dir = models_dir(s, C)?;
Ok(json!({"kind": "model", "domain": "denoise", "path": path, "fileName": src.file_name().map(|n| n.to_string_lossy()), "model": m,
"sizeBytes": std::fs::metadata(src).ok().map(|m| m.len()), "alreadyInstalled": dir.join(&m.id).join("model.onnx").is_file()}))
}
/// What an error says, without the command it came from (for a line shown to the user).
fn plain(e: &EngineError) -> String {
match e {
EngineError::BadParams { msg, .. } => msg.clone(),
other => other.to_string(),
}
}
/// Install every download that has arrived: each was accepted when it was started, so it is installed and chosen without
/// another question. Called by `denoise.models.downloads` and by every frame's `denoise.pump`.
pub(crate) fn finish_downloads(s: &mut Session) {
if let Some(job) = &s.denoise.install {
let outcome = match job.result.try_recv() {
Ok(v) => Some(v),
Err(mpsc::TryRecvError::Empty) => None,
Err(mpsc::TryRecvError::Disconnected) => Some(Err("the installation worker stopped".into())),
};
if let Some(outcome) = outcome
&& let Some(job) = s.denoise.install.take()
{
let state = match outcome {
Ok((v, model)) => match activate_install(s, &v, model, job.activate) {
Ok(_) => State::Installed,
Err(e) => State::Failed(plain(&e)),
},
Err(e) => State::Failed(e),
};
s.denoise.downloads.set_outcome(&job.id, state);
}
}
if s.denoise.install.is_none()
&& let Some((id, path, _sha)) = s.denoise.downloads.finished().into_iter().next()
&& let Err(e) = start_install(s, path, id.clone(), true, true)
{
s.denoise.downloads.set_outcome(&id, State::Failed(plain(&e)));
}
}
fn download(s: &mut Session, p: &Value) -> Result<Value> {
const C: &str = "denoise.models.download";
let id = str_param(p, "id").ok_or_else(|| bad(C, "missing `id`"))?;
let dir = models_dir(s, C)?;
let k = known::find(id).ok_or_else(|| bad(C, "LightCraft has no download for that model: get the file from its page, then add it"))?;
let spec = k.download.clone().ok_or_else(|| bad(C, "LightCraft has no download for that model: get the file from its page, then add it"))?;
if !cfg!(feature = "denoise") {
return Err(bad(C, "this build cannot run denoise models, so there is nothing to download for it"));
}
if !bool_or(p, "acknowledged", false) {
return Err(bad(
C,
"the licence has not been accepted: show the user the model's terms (see `denoise.models.list`) and pass `acknowledged: true` once they agree",
));
}
if installed_models(&dir).iter().any(|i| i.manifest.id == spec.id) {
return Err(bad(C, "that model is already installed"));
}
let host = lightcraft_denoise::known::host(&spec.url).to_string();
s.denoise.downloads.start(spec, &dir).map_err(|e| bad(C, e))?;
Ok(json!({"started": id, "from": host}))
}
fn download_row(id: &str, state: &State) -> Value {
let from = known::find(id).and_then(|k| host_of(&k));
match state {
State::Running { bytes, total } => json!({"id": id, "state": "running", "bytes": bytes, "total": total, "from": from}),
State::Done { path, .. } => json!({"id": id, "state": "done", "path": path.display().to_string(), "from": from}),
State::Installing => json!({"id": id, "state": "installing", "from": from}),
State::Installed => json!({"id": id, "state": "installed", "from": from}),
State::Failed(why) => json!({"id": id, "state": "failed", "error": why, "from": from}),
State::Cancelled => json!({"id": id, "state": "cancelled", "from": from}),
}
}
fn downloads(s: &mut Session, _: &Value) -> Result<Value> {
finish_downloads(s);
let all: Vec<Value> = s.denoise.downloads.snapshot().iter().map(|(id, st)| download_row(id, st)).collect();
Ok(json!({"running": s.denoise.downloads.running(), "downloads": all}))
}
fn download_cancel(s: &mut Session, p: &Value) -> Result<Value> {
const C: &str = "denoise.models.downloadCancel";
let id = str_param(p, "id").ok_or_else(|| bad(C, "missing `id`"))?;
Ok(json!({"discarded": s.denoise.downloads.discard(id)}))
}
fn test(s: &mut Session, p: &Value) -> Result<Value> {
const C: &str = "denoise.models.test";
let id = str_param(p, "id").ok_or_else(|| bad(C, "missing `id`"))?;
let dir = models_dir(s, C)?;
let found = installed_models(&dir).into_iter().find(|i| i.manifest.id == id).ok_or_else(|| bad(C, "that model is not installed"))?;
let run_on = s.denoise.settings.run_on();
let result = (s.denoise.loader)(&found.onnx, &found.manifest).and_then(|m| m.self_test(run_on));
let (ok, detail) = match &result {
Ok(v) => (true, v.clone()),
Err(e) => (false, json!(e)),
};
// keep the latest outcome with the model's record
let mut record = found.accepted;
if let Some(o) = record.as_object_mut() {
o.insert("selfTest".into(), if ok { detail.clone() } else { json!({"ok": false, "error": detail}) });
write_atomic(&found.onnx.with_file_name("installed.json"), &serde_json::to_vec_pretty(&record).map_err(|e| fail("record", e))?)
.map_err(EngineError::Other)?;
}
Ok(json!({"id": id, "ok": ok, "result": detail}))
}
fn remove(s: &mut Session, p: &Value) -> Result<Value> {
const C: &str = "denoise.models.remove";
let id = str_param(p, "id").ok_or_else(|| bad(C, "missing `id`"))?;
let dir = models_dir(s, C)?;
if !lightcraft_denoise::licence::valid_id(id) {
return Err(bad(C, "not a model id"));
}
let home = dir.join(id);
// only a folder that is a model (it has a manifest) is ever deleted
if !home.join("denoise-model.json").is_file() {
return Err(bad(C, "that model is not installed"));
}
std::fs::remove_dir_all(&home).map_err(|e| fail("could not remove the model", e))?;
let mut st = read_settings(&dir);
if st.model.as_deref() == Some(id) {
st.model = installed_models(&dir).into_iter().map(|i| i.manifest.id).find(|m| m != id);
write_settings(&dir, &st).map_err(EngineError::Other)?;
}
s.denoise.touch();
s.denoise_refresh_active(true);
Ok(json!({"removed": id, "model": st.model}))
}
fn select(s: &mut Session, p: &Value) -> Result<Value> {
const C: &str = "denoise.models.select";
let dir = models_dir(s, C)?;
let id = match p.get("id") {
None | Some(Value::Null) => None,
Some(v) => Some(v.as_str().ok_or_else(|| bad(C, "`id` must be a model id or null"))?.to_string()),
};
if let Some(id) = &id
&& !installed_models(&dir).iter().any(|i| &i.manifest.id == id)
{
return Err(bad(C, "that denoise model is not installed"));
}
let mut st = read_settings(&dir);
st.model = id;
write_settings(&dir, &st).map_err(EngineError::Other)?;
s.denoise.touch();
s.denoise_refresh_active(true);
Ok(json!({"model": st.model}))
}
fn settings(s: &mut Session, p: &Value) -> Result<Value> {
const C: &str = "denoise.settings";
let dir = models_dir(s, C)?;
let mut st: Settings = read_settings(&dir);
if let Some(v) = p.get("auto") {
st.auto = Some(v.as_bool().ok_or_else(|| bad(C, "`auto` must be true or false"))?);
}
if let Some(v) = p.get("cacheGb") {
let n =
v.as_u64().filter(|n| (1..=10_000).contains(n)).ok_or_else(|| bad(C, "`cacheGb` must be a whole number of gigabytes from 1 to 10000"))?;
st.cache_gb = Some(n as u32);
}
if let Some(v) = p.get("threads") {
st.threads = match v {
Value::Null => None,
v => Some(v.as_u64().filter(|n| (1..=64).contains(n)).ok_or_else(|| bad(C, "`threads` must be from 1 to 64, or null"))? as u32),
};
}
let run_on = match p.get("runOn") {
None => None,
Some(v) => Some(v.as_str().and_then(RunOn::parse).ok_or_else(|| bad(C, "`runOn` must be auto, gpu or cpu"))?),
};
if let Some(r) = run_on {
st.run_on = Some(r.name().to_string());
st.gpu = None;
}
write_settings(&dir, &st).map_err(EngineError::Other)?;
if run_on.is_some() {
// a choice of where denoise runs is also a go-ahead to try the card again: a set-up that crashed or gave up
// before is forgotten, and the loaded model sets the card up afresh for the next photo
forget_gpu_set_ups(&dir);
if let Some(a) = &s.denoise.active {
*a.model.lock().unwrap_or_else(PoisonError::into_inner) = None;
}
}
s.denoise.touch();
s.denoise_refresh_active(true);
// a smaller limit takes effect now
s.denoise_evict(None);
Ok(json!({"auto": st.auto(), "cacheGb": st.cache_bytes() >> 30, "threads": st.threads, "runOn": st.run_on().name()}))
}
fn ids_param(p: &Value) -> Option<Vec<PhotoId>> {
p.get("ids").and_then(Value::as_array).map(|a| a.iter().filter_map(Value::as_u64).map(PhotoId).collect())
}
fn status_json(s: &Session, photo: Option<PhotoId>) -> Value {
let (files, bytes) = s.denoise_cache_usage();
let st = &s.denoise.settings;
let model = s.denoise.active.as_ref().map(|a| json!({"id": a.id, "name": a.manifest.name}));
let mut v = json!({
"enabled": s.denoise.active.is_some(),
"model": model,
"runtime": cfg!(feature = "denoise"),
"productsDir": s.denoise_products_dir().map(|d| d.display().to_string()),
"queued": s.denoise.queued(),
"running": s.denoise_running_json(),
"ready": s.media.denoise.len(),
"made": s.denoise_made(),
"auto": st.auto(),
"runOn": st.run_on().name(),
"device": s.denoise_device(),
"cache": {"files": files, "bytes": bytes, "limitBytes": st.cache_bytes()},
"generation": s.denoise.generation,
});
if let (Some(id), Some(o)) = (photo, v.as_object_mut()) {
o.insert("photo".into(), serde_json::to_value(s.denoise_photo_state(id)).unwrap_or(Value::Null));
}
v
}
/// A per-photo switch, independent of the selection and Auto Sync. Turning on starts at 50%.
fn toggle(s: &mut Session, p: &Value) -> Result<Value> {
const C: &str = "denoise.toggle";
let id = match p.get("id") {
Some(v) => PhotoId(v.as_u64().ok_or_else(|| bad(C, "`id` must be a photo id"))?),
None => s.active().ok_or_else(|| bad(C, "select a raw photo"))?,
};
let photo = s.catalog.photo(id).ok_or_else(|| bad(C, "no such photo"))?;
if !crate::denoise::eligible(photo) {
return Err(bad(C, "AI Denoise applies to raw photos developed from their sensor data"));
}
let mut d = (*s.develop_of(id).ok_or_else(|| bad(C, "no such photo"))?).clone();
let on = match p.get("enabled") {
Some(v) => v.as_bool().ok_or_else(|| bad(C, "`enabled` must be true or false"))?,
None => !d.enhance.denoise_enabled(),
};
if on {
if !d.enhance.denoise.is_finite() {
return Err(bad(C, "the photo has an invalid Denoise amount"));
}
s.denoise_refresh_active(true);
if s.denoise.active.is_none() {
return Err(bad(C, "AI Denoise needs a model: install one in Settings > AI Denoise"));
}
}
// The switch is its own setting: the Amount stays where it is (0 included) when it is flipped, so flipping compares
// with and without. A photo that never had an Amount starts at 50 the first time.
let amount = if on && d.enhance.denoise_on.is_none() && d.enhance.denoise <= 0.0 { 50.0 } else { d.enhance.denoise };
if d.enhance.denoise != amount || d.enhance.denoise_on != Some(on) {
s.end_interaction()?;
d.enhance.denoise = amount;
d.enhance.denoise_on = Some(on);
let op = s.develop_op(id, d, "AI Denoise").ok_or_else(|| bad(C, "no such photo"))?;
s.commit("AI Denoise", op)?;
}
if on {
s.denoise_enqueue(&[id], true, false);
} else {
s.denoise_cancel(Some(id), false);
}
Ok(json!({"id": id.0, "enabled": on, "amount": amount}))
}
fn queue(s: &mut Session, p: &Value) -> Result<Value> {
const C: &str = "denoise.queue";
s.denoise_refresh_active(true);
if s.denoise.active.is_none() {
return Err(bad(C, "no denoise model is chosen: install one first (`denoise.models.download` or `denoise.models.install`)"));
}
if s.denoise_products_dir().is_none() {
return Err(bad(C, "denoise keeps its pictures in the library's folder: open a library on disk first"));
}
let ids = match ids_param(p) {
Some(ids) => ids,
None => match str_param(p, "scope").unwrap_or("selected") {
"selected" => s.selection.active.iter().chain(s.selection.ids.iter()).copied().collect(),
"visible" => s.visible_cloned().to_vec(),
"withAmount" => s.catalog.photos().filter(|ph| ph.develop.denoise_amount() > 0.0).map(|ph| ph.id).collect(),
other => return Err(bad(C, format!("unknown scope `{other}`: use selected, visible or withAmount"))),
},
};
let queued = s.denoise_enqueue(&ids, false, bool_or(p, "retry", false));
let mut v = status_json(s, None);
if let Some(o) = v.as_object_mut() {
o.insert("added".into(), json!(queued));
}
Ok(v)
}
fn cancel(s: &mut Session, p: &Value) -> Result<Value> {
let photo = p.get("id").and_then(Value::as_u64).map(PhotoId);
let n = s.denoise_cancel(photo, bool_or(p, "all", photo.is_none()));
Ok(json!({"cancelled": n}))
}
fn status(s: &mut Session, p: &Value) -> Result<Value> {
s.denoise_refresh_active(false);
Ok(status_json(s, p.get("id").and_then(Value::as_u64).map(PhotoId)))
}
/// `denoise.pump {pace?}`: cheap to call every frame. Takes in what the background job has finished and keeps one going.
fn pump(s: &mut Session, p: &Value) -> Result<Value> {
s.denoise_pump(Pace::parse(p.get("pace").and_then(Value::as_str)));
Ok(json!({
"active": s.denoise.active.is_some(),
"running": s.denoise_running_json(),
"queued": s.denoise.queued(),
"ready": s.media.denoise.len(),
"generation": s.denoise.generation,
"downloading": s.denoise.downloads.running(),
}))
}
fn clear(s: &mut Session, _: &Value) -> Result<Value> {
let (files, bytes) = s.denoise_clear();
Ok(json!({"deleted": files, "bytes": bytes}))
}
pub fn specs() -> Vec<CommandSpec> {
vec![
cmd!(query "denoise.models.list", "Denoise Models", [], None, "{} → {dir, productsDir, model, runtime, auto, cacheGb, threads, runOn, models: [{id, name, version, licence{name, commercial, url, notice}, provenance, source, sizeBytes, sha256, tile, known, downloadHost, installed, selected, accepted}]}", always, list),
cmd!(query "denoise.models.inspect", "Inspect Denoise Model", [], None, "{path} → model terms from an adjacent manifest; never executes the file", always, inspect),
cmd!(query "denoise.models.install", "Install Denoise Model", [], None, "{path, acknowledged: true, activate?: true, background?: false} → {installed, model} or {started} — install a denoise model from a file: an .onnx with a denoise-model.json beside it, or an archive LightCraft knows (the darktable `.dtmodel`). `acknowledged` must be true: the user has been shown the model's licence and accepted it. Unless `activate` is false it becomes the model in use", always, install),
cmd!(query "denoise.models.download", "Download Denoise Model", [], None, "{id, acknowledged: true} → {started, from} — fetch a model LightCraft has a pinned address for (see `downloadHost` in the list) in the background over HTTPS with lightcraft-fetch. `acknowledged` must be true: the user has been shown the model's terms and accepted them. It is checked against its size and SHA-256, installed and chosen by itself; `denoise.models.downloads` shows how far it is", always, download),
cmd!(query "denoise.models.downloads", "Denoise Model Downloads", [], None, "{} → {running, downloads: [{id, state: running | done | installed | failed | cancelled, bytes, total, error, from}]} — also installs any download that has arrived", always, downloads),
cmd!(query "denoise.models.downloadCancel", "Cancel Denoise Model Download", [], None, "{id} → {discarded} — stop a download, or delete a finished one that was not installed", always, download_cancel),
cmd!(query "denoise.models.test", "Test Denoise Model", [], None, "{id} → {ok, result} — load an installed model and check it gives sensible pictures; needs the denoise runtime", always, test),
cmd!(query "denoise.models.remove", "Remove Denoise Model", [], None, "{id} → {removed, model} — delete an installed model (its cached pictures stay until they are evicted)", always, remove),
cmd!(query "denoise.models.select", "Choose Denoise Model", [], None, "{id: installed model | null} → {model} — null switches denoise off", always, select),
cmd!(query "denoise.settings", "Denoise Settings", [], None, "{auto?: bool, cacheGb?: 1..10000, threads?: 1..64 | null, runOn?: auto | gpu | cpu} → the settings — `auto`: make the picture of the photos being looked at that have a Denoise amount; `cacheGb`: how much the cached pictures may take; `threads`: most tiles run at once; `runOn`: where the model runs — `auto` (the default) the graphics card where it is faster than the processor (both timed on this computer when the card is set up), `gpu` the card whenever it can run the model, `cpu` the processor. Setting `runOn` also lets the card be tried again after a set-up that failed or closed LightCraft", always, settings),
cmd!(
"denoise.toggle",
"AI Denoise",
[],
None,
"{id?: photoId, enabled?: bool} → {id, enabled, amount} — turn AI Denoise on (50% initially) or off for one raw photo, independent of selection and Auto Sync; needs an installed model to turn on",
always,
toggle
),
cmd!(query "denoise.queue", "Denoise Photos", [], None, "{ids?: [photoId], scope?: selected | visible | withAmount, retry?: false} → {added, …status} — make the denoised pictures of raw photos that have none, in the background. `retry` tries photos that failed before", always, queue),
cmd!(query "denoise.cancel", "Cancel Denoise", [], None, "{id?, all?} → {cancelled} — stop the photo being made and/or forget the ones waiting (all of them when `id` is omitted)", always, cancel),
cmd!(query "denoise.status", "Denoise Status", [], None, "{id?} → {enabled, model, runtime, productsDir, queued, running: {photo, done, total, seconds} | null, ready, made, auto, runOn, device: {kind: gpu | cpu | pending | none, adapter?, reason?, cardMs?, cpuMs?, fellBack?}, cache: {files, bytes, limitBytes}, generation, photo?: {state: notApplicable | noModel | ready | queued | running | failed | idle, …}}", always, status),
cmd!(query "denoise.pump", "Denoise Pump", [], None, "{pace?: pause | light | normal | full} → {active, running, queued, ready, generation, downloading} — called every frame by the app: takes in finished work and keeps one photo going", always, pump),
cmd!(query "denoise.clear", "Clear Denoise Cache", [], None, "{} → {deleted, bytes} — delete every cached denoised picture of this library (they are made again when wanted)", always, clear),
]
}

84
crates/engine/src/cmd/library.rs
View File

@ -119,6 +119,14 @@ fn album_param(p: &Value, key: &str, c: &str) -> Result<AlbumId> {
p.get(key).and_then(Value::as_u64).map(AlbumId).ok_or_else(|| bad(c, format!("missing album `{key}`")))
}
/// An optional album id: absent or `null` is `None`, anything but a number is an error.
fn opt_album_param(p: &Value, key: &str, c: &str) -> Result<Option<AlbumId>> {
match p.get(key) {
None | Some(Value::Null) => Ok(None),
Some(v) => v.as_u64().map(|n| Some(AlbumId(n))).ok_or_else(|| bad(c, format!("`{key}` must be an album id or null"))),
}
}
fn strs(p: &Value, key: &str) -> Vec<String> {
p.get(key).and_then(Value::as_array).map(|a| a.iter().filter_map(Value::as_str).map(str::to_string).collect()).unwrap_or_default()
}
@ -784,7 +792,7 @@ pub fn specs() -> Vec<CommandSpec> {
let cover = photos.first().copied();
s.commit(
if folder { "New Folder" } else { "New Album" },
Op::AddAlbum { album: Album { id, name, parent, folder, photos, cover, smart: None, quick: false } },
Op::AddAlbum { album: Album { id, name, parent, folder, photos, cover, smart: None, quick: false, order: None } },
)?;
Ok(json!({"id": id.0}))
}),
@ -880,6 +888,80 @@ pub fn specs() -> Vec<CommandSpec> {
s.commit("Move Album", Op::MoveAlbum { id, parent })?;
ok()
}),
cmd!(
"album.reorder",
"Reorder Album",
[],
None,
"{id, parent?: folderId|null, before?: albumId|null} — places the album (or folder) in `parent` (default: where it is; null: the top level) before the sibling `before` of the same kind, or last of its kind; the folder is then ordered by hand. One undo step. → {changed}: how many edits it took (0: already there)",
always,
|s, p| {
let id = album_param(p, "id", "album.reorder")?;
let me = s.catalog.album(id).ok_or_else(|| bad("album.reorder", "no such album"))?;
let parent = if p.get("parent").is_some() { opt_album_param(p, "parent", "album.reorder")? } else { me.parent };
let before = opt_album_param(p, "before", "album.reorder")?;
if before == Some(id) {
// before itself: where it is
return Ok(json!({"changed": 0}));
}
let mut order: Vec<AlbumId> = s.catalog.album_children(parent).into_iter().filter(|a| a.id != id).map(|a| a.id).collect();
let at = match before {
Some(b) => {
let at = order.iter().position(|x| *x == b).ok_or_else(|| bad("album.reorder", "`before` is not inside that folder"))?;
if s.catalog.album(b).is_none_or(|x| x.folder != me.folder) {
return Err(bad("album.reorder", "folders are placed among folders and albums among albums"));
}
at
}
None if me.folder => order.iter().take_while(|x| s.catalog.album(**x).is_some_and(|a| a.folder)).count(),
None => order.len(),
};
order.insert(at.min(order.len()), id);
let moved = me.parent != parent;
let mut ops = Vec::new();
if moved {
ops.push(Op::MoveAlbum { id, parent });
}
for (i, aid) in order.iter().enumerate() {
let want = u32::try_from(i).ok();
// a moved album starts without a place in its new folder
let have = if moved && *aid == id { None } else { s.catalog.album(*aid).and_then(|a| a.order) };
if have != want {
ops.push(Op::SetAlbumOrder { id: *aid, order: want });
}
}
if ops.is_empty() {
return Ok(json!({"changed": 0}));
}
let changed = ops.len();
s.commit("Reorder Album", Op::Batch { ops })?;
Ok(json!({"changed": changed}))
}
),
cmd!(
"album.sort",
"Sort Albums by Name",
[],
None,
"{parent?: folderId|null} — drops the hand order of the albums inside the folder (default: the top level), so they are listed by name → {changed}: how many albums lost their place",
always,
|s, p| {
let parent = opt_album_param(p, "parent", "album.sort")?;
let ops: Vec<Op> = s
.catalog
.album_children(parent)
.into_iter()
.filter(|a| a.order.is_some())
.map(|a| Op::SetAlbumOrder { id: a.id, order: None })
.collect();
if ops.is_empty() {
return Ok(json!({"changed": 0}));
}
let changed = ops.len();
s.commit("Sort Albums by Name", Op::Batch { ops })?;
Ok(json!({"changed": changed}))
}
),
cmd!("album.addPhotos", "Add to Album", ["Photo"], None, "{id: albumId, ids?: [photoIds]} (default: selection)", has_selection, |s, p| {
let id = album_param(p, "id", "album.addPhotos")?;
let targets = ids_param(p).unwrap_or_else(|| s.targets(&Value::Null));

1
crates/engine/src/cmd/masks.rs
View File

@ -432,6 +432,7 @@ pub fn specs() -> Vec<CommandSpec> {
"available": crate::segment::Segmenter::AVAILABLE,
"installed": g.installed(),
"dir": g.dir.as_ref().map(|d| d.display().to_string()),
"remote": g.remote_endpoint(),
"loaded": g.loaded(),
"busy": g.busy(),
"analyzing": g.analyzing(),

2
crates/engine/src/cmd/mod.rs
View File

@ -10,6 +10,7 @@ mod color;
pub(crate) mod convert;
mod cull;
pub mod curves;
pub(crate) mod denoise;
mod develop;
mod edit;
mod export;
@ -143,6 +144,7 @@ pub fn command_specs() -> &'static [CommandSpec] {
v.extend(missing::specs());
v.extend(metadata::specs());
v.extend(filters::specs());
v.extend(denoise::specs());
v.extend(face_models::specs());
v.extend(face_detect::specs());
v.extend(face_recognize::specs());

27
crates/engine/src/config.rs
View File

@ -1,9 +1,10 @@
//! Where LightCraft keeps per-user data on this machine: the config folder (app settings, face models).
//! Where LightCraft keeps per-user data on this machine: the config folder (app settings, camera
//! profiles, face and denoise models).
//!
//! Hosts that have a file system (the desktop app, the CLI, the MCP server) share these defaults so a model
//! installed from one is there for the others. Nothing here is applied automatically: a [`Session`] has no
//! face-models folder until a host asks for one ([`Session::with_default_face_models`]), so tests stay
//! hermetic.
//! face- or denoise-models folder until a host asks for one ([`Session::with_default_face_models`],
//! [`Session::with_default_denoise_models`]), so tests stay hermetic.
use std::path::PathBuf;
@ -28,10 +29,30 @@ pub fn default_face_models_dir() -> Option<PathBuf> {
std::env::var_os("LIGHTCRAFT_FACE_MODELS").filter(|v| !v.is_empty()).map(PathBuf::from).or_else(|| config_dir().map(|d| d.join("models")))
}
/// Where opt-in denoise models are kept: `$LIGHTCRAFT_DENOISE_MODELS` if set, else `<config>/denoise-models`.
pub fn default_denoise_models_dir() -> Option<PathBuf> {
std::env::var_os("LIGHTCRAFT_DENOISE_MODELS")
.filter(|v| !v.is_empty())
.map(PathBuf::from)
.or_else(|| config_dir().map(|d| d.join("denoise-models")))
}
impl Session {
/// Keep face models in the shared default folder ([`default_face_models_dir`]).
pub fn with_default_face_models(mut self) -> Self {
self.face_models_dir = default_face_models_dir();
self
}
/// Keep denoise models in the shared default folder ([`default_denoise_models_dir`]).
pub fn with_default_denoise_models(mut self) -> Self {
self.set_denoise_models_dir(default_denoise_models_dir());
self
}
/// Point AI denoise at `dir` (or at no folder), dropping what was learned about the old one.
pub fn set_denoise_models_dir(&mut self, dir: Option<PathBuf>) {
self.denoise.models_dir = dir;
self.denoise.touch();
}
}

156
crates/engine/src/demo.rs
View File

@ -62,7 +62,17 @@ pub fn load(s: &mut Session) {
}
let folder = s.catalog.alloc_album_id();
ops.push(Op::AddAlbum {
album: Album { id: folder, name: "Travel 2026".into(), parent: None, folder: true, photos: vec![], cover: None, smart: None, quick: false },
album: Album {
id: folder,
name: "Travel 2026".into(),
parent: None,
folder: true,
photos: vec![],
cover: None,
smart: None,
quick: false,
order: None,
},
});
let by_kw = |kw: &str| -> Vec<_> { scenes.iter().zip(&ids).filter(|(sc, _)| sc.meta.keywords.contains(&kw)).map(|(_, id)| *id).collect() };
for (name, parent, photos) in [
@ -75,7 +85,9 @@ pub fn load(s: &mut Session) {
] {
let id = s.catalog.alloc_album_id();
let cover = photos.first().copied();
ops.push(Op::AddAlbum { album: Album { id, name: name.into(), parent, folder: false, photos, cover, smart: None, quick: false } });
ops.push(Op::AddAlbum {
album: Album { id, name: name.into(), parent, folder: false, photos, cover, smart: None, quick: false, order: None },
});
}
for op in ops {
let _ = s.catalog.apply(op);
@ -84,3 +96,143 @@ pub fn load(s: &mut Session) {
s.selection = crate::Selection::single(*first);
}
}
/// Procedural scenes are deterministic but costly to evaluate: every pixel runs the scene's noise
/// functions, so a 24 MP demo original takes minutes of CPU time on a small machine (a 4-core CI
/// VM). Each scene at each size is therefore generated once per process and shared:
///
/// - concurrent requests wait for one generation (Before and After at 1:1 both load the
/// original; sessions side by side, e.g. parallel tests, open the same photo);
/// - the most recently used results are kept, up to [`scene_cache_limit`] bytes. They are the same
/// allocations the sessions' source caches hold, so they cost extra memory only once no session
/// holds them any more.
///
/// Only demo photos use this; real files are decoded by their loader as before.
pub(crate) fn scene_pixels(scene: &lightcraft_scenes::Scene, max_edge: usize) -> Arc<lightcraft_raster::Rgb32f> {
let key = SceneKey { kind: scene.kind, seed: scene.seed, width: scene.width, height: scene.height, max_edge };
let slot = {
let mut c = scene_cache();
c.tick += 1;
let tick = c.tick;
match c.entries.iter_mut().find(|e| e.key == key) {
Some(e) => {
e.used = tick;
e.slot.clone()
}
None => {
let slot = Arc::new(std::sync::OnceLock::new());
c.entries.push(SceneEntry { key, slot: slot.clone(), used: tick });
slot
}
}
};
// (no lock held while generating: other scenes and sizes are served meanwhile)
let pixels = slot.get_or_init(|| Arc::new(scene.render_fit(max_edge))).clone();
trim_scene_cache(&mut scene_cache(), scene_cache_limit());
pixels
}
/// Bytes of generated scenes kept for reuse: half the memory budget (768 MB by default; enough
/// for one 24 MP original and the previews around it).
fn scene_cache_limit() -> usize {
crate::memory::budget() / 2
}
#[derive(Clone, Copy, PartialEq, Eq, Debug)]
struct SceneKey {
kind: lightcraft_scenes::Kind,
seed: u32,
width: u32,
height: u32,
max_edge: usize,
}
type SceneSlot = Arc<std::sync::OnceLock<Arc<lightcraft_raster::Rgb32f>>>;
struct SceneEntry {
key: SceneKey,
slot: SceneSlot,
used: u64,
}
#[derive(Default)]
struct SceneCache {
entries: Vec<SceneEntry>,
tick: u64,
}
fn scene_cache() -> std::sync::MutexGuard<'static, SceneCache> {
static CACHE: std::sync::OnceLock<std::sync::Mutex<SceneCache>> = std::sync::OnceLock::new();
CACHE.get_or_init(Default::default).lock().unwrap_or_else(std::sync::PoisonError::into_inner)
}
fn scene_bytes(slot: &SceneSlot) -> usize {
slot.get().map_or(0, |img| img.data.len().saturating_mul(std::mem::size_of::<[f32; 3]>()))
}
/// Drop the least recently used generated scenes until the rest fit in `limit` bytes. Scenes still
/// being generated are kept (their waiters hold the slot anyway).
fn trim_scene_cache(c: &mut SceneCache, limit: usize) {
loop {
let total: usize = c.entries.iter().map(|e| scene_bytes(&e.slot)).fold(0, usize::saturating_add);
if total <= limit {
return;
}
let Some(i) = c.entries.iter().enumerate().filter(|(_, e)| e.slot.get().is_some()).min_by_key(|(_, e)| e.used).map(|(i, _)| i) else {
return;
};
c.entries.swap_remove(i);
}
}
#[cfg(test)]
mod tests {
use super::*;
fn small_scene() -> lightcraft_scenes::Scene {
lightcraft_scenes::demo_library().swap_remove(0)
}
#[test]
fn a_scene_is_generated_once_per_size_and_shared() {
let s = small_scene();
let a = scene_pixels(&s, 48);
let b = scene_pixels(&s, 48);
assert!(Arc::ptr_eq(&a, &b), "the second request reuses the first generation");
assert_eq!(a.width.max(a.height), 48);
assert_eq!(a.data, s.render_fit(48).data, "the same pixels as generating it directly");
let c = scene_pixels(&s, 40);
assert_eq!(c.width.max(c.height), 40, "another size is its own generation");
}
#[test]
fn concurrent_requests_wait_for_one_generation() {
let s = small_scene();
let edge = 37; // a size no other test asks for
let got: Vec<_> = std::thread::scope(|sc| {
let hs: Vec<_> = (0..4).map(|_| sc.spawn(|| scene_pixels(&s, edge))).collect();
hs.into_iter().map(|h| h.join().unwrap()).collect()
});
assert!(got.windows(2).all(|w| Arc::ptr_eq(&w[0], &w[1])), "every request got the same pixels");
}
#[test]
fn the_cache_keeps_the_most_recent_scenes_within_its_limit() {
let key = |max_edge| SceneKey { kind: lightcraft_scenes::Kind::Dunes, seed: 1, width: 3, height: 2, max_edge };
let entry = |max_edge, used, px: Option<usize>| {
let slot: SceneSlot = Arc::new(std::sync::OnceLock::new());
if let Some(n) = px {
let _ = slot.set(Arc::new(lightcraft_raster::Rgb32f::new(n, 1)));
}
SceneEntry { key: key(max_edge), slot, used }
};
let px = std::mem::size_of::<[f32; 3]>();
let mut c = SceneCache { entries: vec![entry(1, 3, Some(10)), entry(2, 1, Some(10)), entry(3, 0, None), entry(4, 2, Some(10))], tick: 3 };
trim_scene_cache(&mut c, 25 * px);
let mut kept: Vec<usize> = c.entries.iter().map(|e| e.key.max_edge).collect();
kept.sort_unstable();
assert_eq!(kept, vec![1, 3, 4], "the least recently used finished scene went; the one in progress stays");
trim_scene_cache(&mut c, 0);
assert_eq!(c.entries.iter().map(|e| e.key.max_edge).collect::<Vec<_>>(), vec![3], "only the scene in progress is left");
}
}

1644
crates/engine/src/denoise.rs Normal file
View File

File diff suppressed because it is too large Load Diff

156
crates/engine/src/files.rs
View File

@ -11,7 +11,7 @@ use lightcraft_pipeline::SourceInfo;
use lightcraft_raster::Rgb32f;
use lightcraft_raster::resample::{Filter, fit};
use crate::media::{FileLoader, FileProbe, PreviewLoader, ProbeInfo};
use crate::media::{DenoiseSpec, FileLoader, FileProbe, PairLoader, PreviewLoader, ProbeInfo};
fn meta_of(m: &lightcraft_meta::Metadata) -> (Meta, Option<String>) {
let shutter = m.exposure_time.map(|t| if t >= 1.0 { format!("{t:.0}") } else { format!("1/{:.0}", 1.0 / t) }).unwrap_or_default();
@ -220,22 +220,56 @@ pub fn bin_factor(raw: &lightcraft_raw::RawImage, max_edge: usize) -> Option<usi
/// instead of each one waking the pool from outside and waiting for it (which costs more than the
/// loops themselves when the machine is busy).
pub fn load_bytes(bytes: &[u8], max_edge: usize) -> Result<(Rgb32f, SourceInfo), String> {
rayon::scope(|_| load_bytes_now(std::borrow::Cow::Borrowed(bytes), max_edge))
rayon::scope(|_| load_bytes_now(std::borrow::Cow::Borrowed(bytes), max_edge, None).map(|(img, _, info)| (img, info)))
}
/// [`load_bytes`] taking the file's bytes: a raw file's bytes are freed as soon as it is decoded
/// (less memory held while it is developed).
pub fn load_vec(bytes: Vec<u8>, max_edge: usize) -> Result<(Rgb32f, SourceInfo), String> {
rayon::scope(move |_| load_bytes_now(std::borrow::Cow::Owned(bytes), max_edge))
rayon::scope(move |_| load_bytes_now(std::borrow::Cow::Owned(bytes), max_edge, None).map(|(img, _, info)| (img, info)))
}
fn load_bytes_now(bytes: std::borrow::Cow<'_, [u8]>, max_edge: usize) -> Result<(Rgb32f, SourceInfo), String> {
/// [`load_vec`] that also develops the photo's cached denoised picture (`spec`) from the same decode: the plain
/// picture, the denoised one (`None` when it cannot be read or does not fit this raw: the plain one is then all
/// there is) and the source facts. Both pictures are the same size.
pub fn load_vec_pair(bytes: Vec<u8>, max_edge: usize, spec: &DenoiseSpec) -> Result<(Rgb32f, Option<Rgb32f>, SourceInfo), String> {
rayon::scope(move |_| load_bytes_now(std::borrow::Cow::Owned(bytes), max_edge, Some(spec)))
}
/// The denoised camera RGB of `raw` from its cached product, as the plain development would have it: the default
/// crop, and `factor` × `factor` blocks averaged into one pixel when the plain one was binned. `None` when the product
/// is missing, damaged, made from something else or not the size of this raw's active area.
fn denoised_camera_rgb(raw: &lightcraft_raw::RawImage, spec: &DenoiseSpec, factor: usize) -> Option<Rgb32f> {
use lightcraft_denoise::product::{self, Window};
let a = raw.active_area;
let header = product::read_header_at(&spec.product).ok()?;
if header.key != spec.key || (header.width, header.height) != (a.width, a.height) {
return None;
}
if raw.opcodes.list3.is_empty() {
let c = raw.develop_crop(a.width, a.height);
let window = Window { x: c.x, y: c.y, width: c.width, height: c.height };
product::read_window(&spec.product, &spec.key, Some(window), factor, Some(HIGHLIGHT_CLIP)).ok()
} else {
// (the plain development is not binned either when the file has opcodes of this kind: `factor` is 1)
let whole = product::read(&spec.product, &spec.key, 1).ok()?;
Some(raw.finish_demosaiced(whole))
}
}
fn load_bytes_now(
bytes: std::borrow::Cow<'_, [u8]>,
max_edge: usize,
denoise: Option<&DenoiseSpec>,
) -> Result<(Rgb32f, Option<Rgb32f>, SourceInfo), String> {
if lightcraft_raw::probe(&bytes).is_some() {
let mut raw = match lightcraft_raw::decode(&bytes).map_err(|e| preview_reason(&bytes, e)) {
Ok(r) => r,
Err(Ok(why)) => {
// show the camera's embedded JPEG (rendered, not raw) until the variant is supported
return load_embedded_preview(&bytes, max_edge).ok_or(format!("unsupported raw ({why}) without an embedded preview"));
return load_embedded_preview(&bytes, max_edge)
.map(|(img, info)| (img, None, info))
.ok_or(format!("unsupported raw ({why}) without an embedded preview"));
}
Err(Err(e)) => return Err(e),
};
@ -251,48 +285,57 @@ fn load_bytes_now(bytes: std::borrow::Cow<'_, [u8]>, max_edge: usize) -> Result<
// Previews and thumbnails bin the mosaic straight to (about) the size they need; only
// larger levels (exports, 1:1) demosaic the whole sensor.
let t0 = web_time::Instant::now();
let binned = match bin_factor(&raw, max_edge) {
let bin = bin_factor(&raw, max_edge);
let binned = match bin {
Some(k) => raw.develop_binned(k, HIGHLIGHT_CLIP).map_err(|e| e.to_string())?,
None => None,
};
let mut img = match binned {
let factor = if binned.is_some() { bin.unwrap_or(1) } else { 1 };
let img = match binned {
Some(img) => img,
None => {
let method = if max_edge <= 600 { lightcraft_raw::Method::Bilinear } else { lightcraft_raw::Method::Ahd };
raw.develop(method).map_err(|e| e.to_string())?
}
};
// the same development from the denoised mosaic, when the photo has been denoised
let twin = denoise.and_then(|spec| denoised_camera_rgb(&raw, spec, factor)).filter(|d| d.width == img.width && d.height == img.height);
// the samples aren't needed any more (the colour model below reads only the tags)
raw.data = lightcraft_raw::RawData::U16(Vec::new());
let mut stages = vec![("develop", t0.elapsed())];
stages.push(("transform", t0.elapsed()));
lightcraft_raw::highlight::reconstruct(&mut img, t.wb, HIGHLIGHT_CLIP);
stages.push(("highlights", t0.elapsed()));
let wb = t.wb;
let m = camera_look.as_ref().map(|p| p.matrix.mul(&t.matrix)).unwrap_or(t.matrix).to_f32();
let hue_sat = camera_look.as_ref().and_then(|p| p.hue_sat.as_ref()).and_then(crate::camera_preview::HueSat::new);
let gain = 2f32.powf(t.baseline_exposure as f32);
let wb = t.wb;
// A DNG's own profile look (hue/saturation map, look table), DNG spec chapter 6.
let tables = lightcraft_raw::profile::ProfileTables::new(&raw.color.profile, lightcraft_raw::color::illuminant_weight(&raw.color, xy));
img.map_in_place(|p| {
let c = [p[0] * wb[0], p[1] * wb[1], p[2] * wb[2]];
let rgb = [
m[0][0] * c[0] + m[0][1] * c[1] + m[0][2] * c[2],
m[1][0] * c[0] + m[1][1] * c[1] + m[1][2] * c[2],
m[2][0] * c[0] + m[2][1] * c[1] + m[2][2] * c[2],
];
let rgb = match &tables {
Some(tables) => tables.apply(rgb, gain),
None => rgb.map(|v| v * gain),
};
// after the baseline exposure, as when it was fitted
hue_sat.as_ref().map_or(rgb, |h| h.apply(rgb)).map(|v| v.max(0.0))
});
stages.push(("colour", t0.elapsed()));
let img = fit(&img, max_edge, max_edge, Filter::Box);
stages.push(("fit", t0.elapsed()));
let img = img.into_oriented(raw.orientation);
stages.push(("orient", t0.elapsed()));
// camera RGB → what the pipeline takes: clipped highlights rebuilt, white balance and the colour model applied,
// fitted to `max_edge` and upright (the same for the plain and the denoised picture)
let finish = |mut img: Rgb32f| {
let mut stages = vec![("develop", t0.elapsed())];
stages.push(("transform", t0.elapsed()));
lightcraft_raw::highlight::reconstruct(&mut img, wb, HIGHLIGHT_CLIP);
stages.push(("highlights", t0.elapsed()));
img.map_in_place(|p| {
let c = [p[0] * wb[0], p[1] * wb[1], p[2] * wb[2]];
let rgb = [
m[0][0] * c[0] + m[0][1] * c[1] + m[0][2] * c[2],
m[1][0] * c[0] + m[1][1] * c[1] + m[1][2] * c[2],
m[2][0] * c[0] + m[2][1] * c[1] + m[2][2] * c[2],
];
let rgb = match &tables {
Some(tables) => tables.apply(rgb, gain),
None => rgb.map(|v| v * gain),
};
hue_sat.as_ref().map_or(rgb, |h| h.apply(rgb)).map(|v| v.max(0.0))
});
stages.push(("colour", t0.elapsed()));
let img = fit(&img, max_edge, max_edge, Filter::Box);
stages.push(("fit", t0.elapsed()));
let img = img.into_oriented(raw.orientation);
stages.push(("orient", t0.elapsed()));
(img, stages)
};
let (img, stages) = finish(img);
if lightcraft_pipeline::profiling() {
let mut prev = std::time::Duration::ZERO;
let parts: Vec<String> = stages
@ -305,6 +348,7 @@ fn load_bytes_now(bytes: std::borrow::Cow<'_, [u8]>, max_edge: usize) -> Result<
.collect();
eprintln!("[profile] raw source {}×{} (max {max_edge}, ms after decode): {}", img.width, img.height, parts.join(", "));
}
let twin = twin.map(|picture| finish(picture).0);
let (temp, tint) = xy_to_temp_tint(xy);
let relative = crate::camera_preview::file_local_look(raw.format) && t.matrix_is_fallback;
// White balance re-evaluates the file's own colour model (when it has one and no
@ -317,6 +361,7 @@ fn load_bytes_now(bytes: std::borrow::Cow<'_, [u8]>, max_edge: usize) -> Result<
let (temp, tint) = if relative { (6500.0, 0.0) } else { (temp.round(), tint.round()) };
return Ok((
img,
twin,
SourceInfo { raw: true, as_shot_temp: temp, as_shot_tint: tint, lens, relative_wb: relative, camera_color, camera_tone, mattes },
));
}
@ -324,7 +369,7 @@ fn load_bytes_now(bytes: std::borrow::Cow<'_, [u8]>, max_edge: usize) -> Result<
drop(bytes);
let img = d.to_working();
let img = if img.width.max(img.height) > max_edge { fit(&img, max_edge, max_edge, Filter::Mitchell) } else { img };
Ok((img.into_oriented(Orientation::from_exif(d.orientation)), SourceInfo::default()))
Ok((img.into_oriented(Orientation::from_exif(d.orientation)), None, SourceInfo::default()))
}
/// The semantic masks of a DNG that AI masks understand, over the developed image (default crop,
@ -462,6 +507,21 @@ pub fn fs_preview_loader() -> PreviewLoader {
})
}
/// The filesystem-backed [`PairLoader`]: the file is read and decoded once, and developed as usual and from the photo's
/// denoised picture. It holds two pictures at once, so it counts for more at the memory gate than a plain load.
pub fn fs_pair_loader() -> PairLoader {
Arc::new(|path: &str, max_edge: usize, spec: &DenoiseSpec| {
let len = std::fs::metadata(path).map(|m| m.len() as usize).unwrap_or(0);
let weight = len * if max_edge <= crate::media::SourceLevel::Thumb.max_edge() { 4 } else { 10 };
let gate = crate::memory::work_gate();
let _permit = if crate::memory::is_background() { gate.acquire(weight) } else { gate.acquire_urgent(weight) };
let bytes = std::fs::read(path).map_err(|e| format!("{path}: {e}"))?;
let r = load_vec_pair(bytes, max_edge, spec);
crate::memory::release();
r
})
}
/// Filesystem-backed hooks (native). On the web the host installs bytes-based hooks instead.
///
/// Their working memory is bounded by [`crate::memory::work_gate`]: background loads (grid
@ -497,6 +557,7 @@ impl crate::Session {
self.media.file_loader = Some(l);
self.media.file_probe = Some(p);
self.media.preview_loader = Some(fs_preview_loader());
self.media.denoise.loader = Some(fs_pair_loader());
self
}
}
@ -537,6 +598,37 @@ mod tests {
assert!(probe_bytes("x.jpg", b"not an image").is_err());
}
#[test]
fn cached_denoise_uses_the_same_white_balance_profile_and_orientation_as_plain_raw() {
use lightcraft_denoise::product;
use lightcraft_raw::{DngWriteOptions, Method, Orientation, profile::HsvTable, write_dng};
let dir = crate::tests_xmp::temp_dir("denoise-color-finish");
let fixture = crate::tests_xmp::synthetic_dng_with(None, Default::default());
let mut raw = lightcraft_raw::decode(&fixture).unwrap();
raw.orientation = Orientation::Rotate90;
raw.color.baseline_exposure = 0.5;
raw.color.profile.look_table =
Some(HsvTable { hue_divisions: 1, sat_divisions: 2, val_divisions: 1, data: vec![[23.0, 0.65, 0.8]; 2], srgb_value: false });
let bytes = write_dng(&raw, &DngWriteOptions::default()).unwrap();
let decoded = lightcraft_raw::decode(&bytes).unwrap();
assert!(decoded.color.profile.look_table.is_some());
// A cache containing the ordinary demosaic must undergo the identical downstream finish.
let camera_rgb = decoded.develop(Method::Ahd).unwrap();
let key = "synthetic-color-finish";
let path = dir.join("same.lcdn");
product::write(&path, &camera_rgb, key).unwrap();
let spec = DenoiseSpec { product: path, key: key.into(), make: None };
let (ordinary, _) = load_bytes(&bytes, 2048).unwrap();
let (plain, cached, _) = load_vec_pair(bytes, 2048, &spec).unwrap();
assert_eq!(plain.data, ordinary.data, "requesting a twin must not change the ordinary path");
let cached = cached.unwrap();
assert_eq!((plain.width, plain.height), (24, 32));
assert_eq!((cached.width, cached.height), (plain.width, plain.height));
let worst = plain.data.iter().zip(&cached.data).flat_map(|(a, b)| a.iter().zip(b).map(|(a, b)| (a - b).abs())).fold(0.0f32, f32::max);
assert!(worst < 0.002, "color finish differs beyond cache half-float rounding: {worst}");
let _ = std::fs::remove_dir_all(dir);
}
/// Luminance (from linear RGB) minus its 9 × 9 local mean: tone differences between a render and a camera JPEG
/// mostly cancel, edges remain.
fn detail(w: usize, h: usize, lum: impl Fn(usize, usize) -> f32) -> Vec<f32> {

20
crates/engine/src/lib.rs
View File

@ -10,6 +10,10 @@
//! run them off the UI thread.
#![forbid(unsafe_code)]
// Model/cache types remain available without linking the optional inference crate.
#[cfg(not(feature = "denoise"))]
extern crate lightcraft_denoise_core as lightcraft_denoise;
pub mod availability;
mod camera_preview;
pub mod camera_profiles;
@ -18,6 +22,7 @@ pub mod config;
pub mod crs;
pub mod crs_masks;
pub mod demo;
pub mod denoise;
pub mod devices;
pub mod export;
pub mod face_download;
@ -39,6 +44,7 @@ pub mod logging;
pub mod media;
pub mod memory;
pub mod merge;
mod model_download;
pub mod originals;
pub mod preset_import;
pub mod preset_luminar;
@ -138,8 +144,10 @@ pub struct Interaction {
/// Source of [`Session::visible_shared`] generations (process-wide, so two sessions never share one).
static VISIBLE_GEN: std::sync::atomic::AtomicU64 = std::sync::atomic::AtomicU64::new(1);
static LIBRARY_GEN: std::sync::atomic::AtomicU64 = std::sync::atomic::AtomicU64::new(1);
pub struct Session {
library_generation: u64,
/// Auto Sync: edits to the active photo also change the other selected photos (the settings
/// that changed, nothing else).
pub auto_sync: bool,
@ -163,6 +171,8 @@ pub struct Session {
/// Set by a command whose change must not rewrite the photo's XMP sidecar even with auto-write on
/// (a catalog-only edit of data the sidecar writer does not emit); consumed when the command ends.
pub(crate) skip_auto_write: bool,
/// AI denoise: the model in use, the photos that have their picture and the work in progress.
pub(crate) denoise: denoise::State,
/// Where the host keeps face models (one folder each); `None` where there is no file system (the web).
pub face_models_dir: Option<std::path::PathBuf>,
/// Face model downloads started this session (the staged files wait in `<face_models_dir>/.downloads`).
@ -262,8 +272,13 @@ impl Default for Session {
}
impl Session {
pub fn library_generation(&self) -> u64 {
self.library_generation
}
pub fn new() -> Session {
Session {
library_generation: LIBRARY_GEN.fetch_add(1, std::sync::atomic::Ordering::Relaxed),
auto_sync: false,
catalog: Catalog::new(),
source: LibrarySource::All,
@ -278,6 +293,7 @@ impl Session {
redo: Vec::new(),
interaction: None,
skip_auto_write: false,
denoise: Default::default(),
face_models_dir: None,
face_downloads: Default::default(),
face_catalog: Default::default(),
@ -822,8 +838,12 @@ pub fn json_delta(old: &Value, new: &Value) -> Option<Value> {
#[cfg(test)]
mod tests;
#[cfg(test)]
mod tests_album_order;
#[cfg(test)]
mod tests_color;
#[cfg(test)]
mod tests_denoise;
#[cfg(test)]
mod tests_export;
#[cfg(test)]
mod tests_face_models;

4
crates/engine/src/library.rs
View File

@ -297,6 +297,9 @@ impl Session {
{
log::error!("library: {e}");
}
// Photo ids belong to one library; denoise work must not carry into the next.
self.denoise.library_changed();
self.media.denoise.clear();
// what was learned about the old library's faces is saved, and none of it carries over (photo ids are per library)
#[cfg(not(target_arch = "wasm32"))]
self.faces.library_changed();
@ -308,6 +311,7 @@ impl Session {
static LOADS: std::sync::atomic::AtomicU64 = std::sync::atomic::AtomicU64::new(0);
catalog.revision = LOADS.fetch_add(1, std::sync::atomic::Ordering::Relaxed).wrapping_add(1) << 32;
self.catalog = catalog;
self.library_generation = crate::LIBRARY_GEN.fetch_add(1, std::sync::atomic::Ordering::Relaxed);
self.undo.clear();
self.redo.clear();
self.interaction = None;

230
crates/engine/src/lightroom_catalog.rs
View File

@ -10,6 +10,7 @@ use std::sync::{
use crate::lightroom_sqlite::{Database, LiveTable, Value as SqlValue};
use lightcraft_catalog::{Album, Flag, Op, Photo, PhotoId, Source};
use lightcraft_geom::{Affine, Orientation, Point, Rect};
use serde::{Deserialize, Serialize};
use serde_json::{Value, json};
@ -477,17 +478,77 @@ fn ensure_read_active(cancel: &AtomicBool) -> Result<(), String> {
if cancel.load(Ordering::Relaxed) { Err("Lightroom import cancelled".into()) } else { Ok(()) }
}
fn mapped_settings(text: &str, raw: bool, aspect: f64) -> Result<(Value, Vec<String>), String> {
fn mapped_settings(text: &str, raw: bool, aspect: f64, orientation: Orientation, stored_aspect: f64) -> Result<(Value, Vec<String>), String> {
let mut root = crate::preset_import::parse_lua(text)?;
let deferred = strip_lua_sentinels(&mut root);
let (props, values) = crate::preset_import::lua_settings_props(&root)?;
let (partial, mut unknown) = crate::crs::to_partial_report(&props, Some(&values), Some(raw), aspect);
let (mut props, values) = crate::preset_import::lua_settings_props(&root)?;
// A catalog's develop settings carry the crop edges and angle but no `HasCrop` (XMP has it):
// any crop field means the photo is cropped.
const CROP: [&str; 5] = ["crs:CropLeft", "crs:CropTop", "crs:CropRight", "crs:CropBottom", "crs:CropAngle"];
if !props.contains_key("crs:HasCrop") && CROP.iter().any(|k| props.contains_key(*k)) {
props.insert("crs:HasCrop".into(), vec!["True".into()]);
}
let (mut partial, mut unknown) = crate::crs::to_partial_report(&props, Some(&values), Some(raw), aspect);
catalog_crop(&mut partial, orientation, stored_aspect);
if deferred > 0 {
unknown.push("Deferred Lightroom adjustments (-999999)".into());
}
Ok((partial, unknown))
}
/// Lightroom's `Adobe_images.orientation`: where the stored image's top-left and top-right corners
/// land in the displayed photo, naming the displayed corners A, B, C, D clockwise from the top left
/// (`AB` upright, `BC` a quarter turn clockwise, `DA` a quarter turn counter-clockwise, `BA` mirrored).
fn lightroom_orientation(code: &str) -> Option<Orientation> {
const ALL: [Orientation; 8] = [
Orientation::Normal,
Orientation::FlipH,
Orientation::Rotate180,
Orientation::FlipV,
Orientation::Transpose,
Orientation::Rotate90,
Orientation::Transverse,
Orientation::Rotate270,
];
let corner = |o: Orientation, x: f64, y: f64| match o.map(x, y, 1.0, 1.0) {
(u, v) if u < 0.5 && v < 0.5 => 'A',
(_, v) if v < 0.5 => 'B',
(u, _) if u >= 0.5 => 'C',
_ => 'D',
};
ALL.into_iter().find(|&o| code.chars().eq([corner(o, 0.0, 0.0), corner(o, 1.0, 0.0)]))
}
/// Lightroom's crop in LightCraft's frame. `crs` gives the crop's top-left (`CropLeft`, `CropTop`)
/// and bottom-right (`CropRight`, `CropBottom`) corners on the stored image (before orientation and
/// straightening), normalized to its size, plus a straighten angle that turns the other way to
/// LightCraft's. LightCraft's crop is a rectangle in the oriented image rotated by `angle` about its
/// centre, normalized to the oriented image's size, so the corners go through the orientation and
/// then the rotation; `stored_aspect` (width / height of the stored image) matters once the
/// rectangle turns. On a real catalog this reproduced the frame aspect of all 223 straightened crops
/// in Lightroom's exports within 1%.
fn catalog_crop(partial: &mut Value, orientation: Orientation, stored_aspect: f64) {
let Some(geometry) = partial.pointer_mut("/crop/geometry").and_then(Value::as_object_mut) else { return };
let edge = |k: &str| geometry.get("rect").and_then(|r| r.get(k)).and_then(Value::as_f64);
let (Some(x0), Some(y0), Some(x1), Some(y1)) = (edge("x0"), edge("y0"), edge("x1"), edge("y1")) else { return };
let lightroom_angle = geometry.get("angle").and_then(Value::as_f64).unwrap_or(0.0);
if orientation == Orientation::Normal && lightroom_angle == 0.0 {
return;
}
let w = if stored_aspect.is_finite() && stored_aspect > 0.0 { stored_aspect } else { 1.0 };
let (ow, oh) = if orientation.swaps_axes() { (1.0, w) } else { (w, 1.0) };
// a mirror turns the straighten the other way too
let angle = if orientation.to_parts().0 { lightroom_angle } else { -lightroom_angle };
let to_rotated = Affine::rotate_about(angle.to_radians(), Point::new(ow / 2.0, oh / 2.0));
let corner = |x: f64, y: f64| {
let (u, v) = orientation.map(x * w, y, w, 1.0);
to_rotated.apply(Point::new(u, v))
};
let r = Rect::from_points(corner(x0, y0), corner(x1, y1));
geometry.insert("rect".into(), json!({"x0": r.x0 / ow, "y0": r.y0 / oh, "x1": r.x1 / ow, "y1": r.y1 / oh}));
geometry.insert("angle".into(), json!(angle));
}
fn strip_lua_sentinels(value: &mut crate::preset_import::Lua) -> usize {
use crate::preset_import::Lua;
match value {
@ -680,7 +741,15 @@ pub(crate) fn apply_prepared(s: &mut crate::Session, data: CatalogImport, contex
p.meta.keywords = src.keywords.clone();
}
if !src.settings.is_empty() && !keep_develop {
match mapped_settings(&src.settings, p.kind == lightcraft_catalog::MediaKind::Raw, p.width.max(1) as f64 / p.height.max(1) as f64) {
let orientation = lightroom_orientation(text(&src.image, "orientation")).unwrap_or_default();
let stored_aspect = number(&src.image, "fileWidth") as f64 / number(&src.image, "fileHeight") as f64;
match mapped_settings(
&src.settings,
p.kind == lightcraft_catalog::MediaKind::Raw,
p.width.max(1) as f64 / p.height.max(1) as f64,
orientation,
stored_aspect,
) {
Ok((partial, unknown)) => {
p.develop = Arc::new(lightcraft_develop::apply_partial(&p.develop, &partial, 1.0));
p.edited = Some(now.clone());
@ -788,6 +857,7 @@ pub(crate) fn apply_prepared(s: &mut crate::Session, data: CatalogImport, contex
#[cfg(test)]
mod tests {
use super::*;
use lightcraft_geom::CropGeometry;
fn row(v: Value) -> Row {
serde_json::from_value(v).unwrap()
}
@ -866,7 +936,7 @@ mod tests {
cyclic.collections[0].insert("parent".into(), json!(10));
assert!(apply(&mut s, cyclic, false).is_err());
assert_eq!(s.catalog.to_snapshot(), before);
assert!(mapped_settings(&format!("{}0{}", "{".repeat(1000), "}".repeat(1000)), true, 1.0).is_err());
assert!(mapped_settings(&format!("{}0{}", "{".repeat(1000), "}".repeat(1000)), true, 1.0, Orientation::Normal, 1.5).is_err());
}
#[test]
fn compressed_xmp_is_bounded_and_settings_are_data_only() {
@ -876,10 +946,156 @@ mod tests {
assert_eq!(xmp(Some(&json!(encoded))).unwrap(), String::from_utf8_lossy(packet));
encoded[0] = 0xff;
assert!(xmp(Some(&json!(encoded))).is_err());
let (partial, _) = mapped_settings("s = { Exposure2012 = 0.75, ToneCurvePV2012 = {0,0,128,150,255,255} }", true, 1.5).unwrap();
let (partial, _) =
mapped_settings("s = { Exposure2012 = 0.75, ToneCurvePV2012 = {0,0,128,150,255,255} }", true, 1.5, Orientation::Normal, 1.5).unwrap();
assert_eq!(partial["light"]["exposure"], 0.75);
}
#[test]
fn catalog_crops_without_has_crop_are_imported() {
// Lightroom Classic catalogs store crop edges and angle but never `HasCrop`
let (partial, unknown) = mapped_settings(
"s = { CropConstrainAspectRatio = true, CropLeft = 0.042702, CropRight = 0.957298, CropTop = 0.1 }",
true,
1.5,
Orientation::Normal,
1.5,
)
.unwrap();
let crop = &partial["crop"]["geometry"];
assert_eq!(crop["rect"]["x0"], 0.042702);
assert_eq!(crop["rect"]["y0"], 0.1);
assert_eq!(crop["rect"]["x1"], 0.957298);
assert_eq!(crop["rect"]["y1"], 1.0);
assert_eq!(crop["angle"], 0.0);
assert!(!unknown.iter().any(|k| k.starts_with("CropLeft")), "{unknown:?}");
// an angle alone still means a crop (Lightroom's angle turns the other way to LightCraft's)
let (partial, unknown) = mapped_settings("s = { CropAngle = -1.5 }", true, 1.5, Orientation::Normal, 1.5).unwrap();
assert_eq!(partial["crop"]["geometry"]["angle"], 1.5);
assert!(!unknown.iter().any(|k| k.starts_with("CropAngle")), "{unknown:?}");
// no crop fields: the crop stays untouched
let (partial, _) = mapped_settings("s = { Exposure2012 = 0.5 }", true, 1.5, Orientation::Normal, 1.5).unwrap();
assert!(partial.get("crop").is_none());
}
#[test]
fn lightroom_orientation_codes() {
// the three seen in a real catalog, checked against each file's EXIF orientation (1, 6, 8)
assert_eq!(lightroom_orientation("AB"), Some(Orientation::Normal));
assert_eq!(lightroom_orientation("BC"), Some(Orientation::Rotate90));
assert_eq!(lightroom_orientation("DA"), Some(Orientation::Rotate270));
assert_eq!(lightroom_orientation("CD"), Some(Orientation::Rotate180));
assert_eq!(lightroom_orientation("BA"), Some(Orientation::FlipH));
assert_eq!(lightroom_orientation("DC"), Some(Orientation::FlipV));
assert_eq!(lightroom_orientation("AD"), Some(Orientation::Transpose));
assert_eq!(lightroom_orientation("CB"), Some(Orientation::Transverse));
for code in ["", "A", "ABC", "AC", "ab", "XY"] {
assert_eq!(lightroom_orientation(code), None, "{code:?}");
}
}
#[test]
fn catalog_crops_follow_the_photo_orientation() {
let crop = |o: Orientation| {
let settings = "s = { CropLeft = 0.1, CropRight = 0.6, CropTop = 0.2, CropBottom = 0.9 }";
let (partial, _) = mapped_settings(settings, true, 1.5, o, 1.5).unwrap();
let g = &partial["crop"]["geometry"];
let r = &g["rect"];
[r["x0"].as_f64(), r["y0"].as_f64(), r["x1"].as_f64(), r["y1"].as_f64(), g["angle"].as_f64()].map(|v| (v.unwrap() * 1e9).round() / 1e9)
};
// (x, y) of the stored image → (1 − y, x) a quarter turn clockwise, (y, 1 − x) counter-clockwise
assert_eq!(crop(Orientation::Normal), [0.1, 0.2, 0.6, 0.9, 0.0]);
assert_eq!(crop(Orientation::Rotate90), [0.1, 0.1, 0.8, 0.6, 0.0]);
assert_eq!(crop(Orientation::Rotate270), [0.2, 0.4, 0.9, 0.9, 0.0]);
assert_eq!(crop(Orientation::Rotate180), [0.4, 0.1, 0.9, 0.8, 0.0]);
assert_eq!(crop(Orientation::FlipH), [0.4, 0.2, 0.9, 0.9, 0.0]);
// a mirror turns the straighten the other way
let angle = |o: Orientation| {
let (partial, _) = mapped_settings("s = { CropAngle = 2 }", true, 1.5, o, 1.5).unwrap();
partial["crop"]["geometry"]["angle"].as_f64().unwrap()
};
assert_eq!(angle(Orientation::Rotate90), -2.0);
assert_eq!(angle(Orientation::FlipH), 2.0);
// no crop: nothing to orient
let (partial, _) = mapped_settings("s = { Exposure2012 = 0.5 }", true, 1.5, Orientation::Rotate90, 1.5).unwrap();
assert!(partial.get("crop").is_none());
}
/// The `crs` corners and angle Lightroom would store for LightCraft crop `g` on a stored image
/// of aspect `w` (height 1) shown with orientation `o`.
fn lightroom_crop(g: CropGeometry, o: Orientation, w: f64) -> String {
let (ow, oh) = if o.swaps_axes() { (1.0, w) } else { (w, 1.0) };
let to_source = Affine::rotate_about(-g.angle.to_radians(), Point::new(ow / 2.0, oh / 2.0));
let stored = |x: f64, y: f64| {
let p = to_source.apply(Point::new(x * ow, y * oh));
o.inverse().map(p.x, p.y, ow, oh)
};
// the rectangle's corners that land top-left and bottom-right on the stored image
let corners = [stored(g.rect.x0, g.rect.y0), stored(g.rect.x1, g.rect.y0), stored(g.rect.x1, g.rect.y1), stored(g.rect.x0, g.rect.y1)];
let (tl, br) = (0..4).map(|i| (corners[i], corners[(i + 2) % 4])).find(|(a, b)| a.0 < b.0 && a.1 < b.1).unwrap();
let angle = if o.to_parts().0 { g.angle } else { -g.angle };
format!("s = {{ CropLeft = {}, CropTop = {}, CropRight = {}, CropBottom = {}, CropAngle = {angle} }}", tl.0 / w, tl.1, br.0 / w, br.1)
}
#[test]
fn straightened_catalog_crops_round_trip() {
let g = CropGeometry { rect: Rect::new(0.2, 0.15, 0.7, 0.8), angle: 6.5 };
for o in [Orientation::Normal, Orientation::Rotate90, Orientation::Rotate270, Orientation::Rotate180, Orientation::FlipH] {
for angle in [6.5, -6.5] {
let g = CropGeometry { angle, ..g };
let (partial, _) = mapped_settings(&lightroom_crop(g, o, 1.5), true, 1.5, o, 1.5).unwrap();
let got: CropGeometry = serde_json::from_value(partial["crop"]["geometry"].clone()).unwrap();
let close = |a: f64, b: f64| (a - b).abs() < 1e-9;
assert!(
close(got.angle, g.angle)
&& [(got.rect.x0, g.rect.x0), (got.rect.y0, g.rect.y0), (got.rect.x1, g.rect.x1), (got.rect.y1, g.rect.y1)]
.iter()
.all(|&(a, b)| close(a, b)),
"{o:?} {angle}: {got:?}"
);
}
}
}
#[test]
fn straightened_catalog_crops_match_lightroom_exports() {
// Settings and pixel sizes from a real Lightroom Classic catalog (9504 × 6336 stored) and the
// JPEGs Lightroom exported from it at full size.
for (settings, o, export) in [
(
"s = { CropAngle = -9.34444, CropBottom = 0.802034, CropLeft = 0.05499, CropRight = 0.94501, CropTop = 0.197966 }",
Orientation::Normal,
(7725.0, 5150.0),
),
(
"s = { CropAngle = -1.48964, CropBottom = 0.962457, CropLeft = 0.010429, CropRight = 0.989571, CropTop = 0.037543 }",
Orientation::Rotate90,
(6100.0, 9150.0),
),
] {
let (partial, _) = mapped_settings(settings, true, 1.5, o, 9504.0 / 6336.0).unwrap();
let r: Rect = serde_json::from_value(partial["crop"]["geometry"]["rect"].clone()).unwrap();
let (ow, oh) = if o.swaps_axes() { (6336.0, 9504.0) } else { (9504.0, 6336.0) };
let (w, h) = (r.width() * ow, r.height() * oh);
assert!((w - export.0).abs() <= 3.0 && (h - export.1).abs() <= 3.0, "{o:?}: {w:.1} × {h:.1}, Lightroom exported {export:?}");
}
}
#[test]
fn imported_crops_use_the_catalog_orientation() {
let mut data = sample();
data.photos.truncate(1);
data.photos[0].image.remove("masterImage");
data.photos[0].image.insert("orientation".into(), json!("BC"));
data.photos[0].settings = "s = { CropLeft = 0.1, CropRight = 0.6, CropTop = 0.2, CropBottom = 0.9 }".into();
data.photos[0].xmp = String::new();
let mut s = crate::Session::new();
apply(&mut s, data, false).unwrap();
let g = s.catalog.photos().next().unwrap().develop.crop.geometry;
let r = [g.rect.x0, g.rect.y0, g.rect.x1, g.rect.y1].map(|v| (v * 1e9).round() / 1e9);
assert_eq!(r, [0.1, 0.1, 0.8, 0.6]);
}
#[test]
fn deferred_auto_tone_sentinels_never_become_real_slider_values() {
let mut data = sample();
@ -896,7 +1112,7 @@ mod tests {
assert_eq!(p.develop.light.shadows, 65.0);
assert_eq!(p.develop.light.blacks, -22.0);
assert!(report["unmapped"]["2"].as_array().unwrap().iter().any(|v| v.as_str().is_some_and(|v| v.contains("Deferred"))));
let (partial, _) = mapped_settings("s = { Exposure2012 = -5, Contrast2012 = -100 }", true, 1.0).unwrap();
let (partial, _) = mapped_settings("s = { Exposure2012 = -5, Contrast2012 = -100 }", true, 1.0, Orientation::Normal, 1.5).unwrap();
assert_eq!(partial["light"]["exposure"], -5.0);
assert_eq!(partial["light"]["contrast"], -100.0);
}

263
crates/engine/src/media.rs
View File

@ -62,7 +62,7 @@ impl SettingsHashes {
}
/// Bump when the pipeline's output changes, to invalidate cached thumbnails.
pub const RENDER_CACHE_VERSION: u64 = 17;
pub const RENDER_CACHE_VERSION: u64 = 22;
/// Thumbnails render at one of these long edges (so window/cell size changes reuse the cache).
pub const THUMB_SIZES: [usize; 4] = [128, 256, 384, 512];
@ -125,6 +125,43 @@ pub type FileLoader = Arc<dyn Fn(&str, usize) -> Result<(Rgb32f, SourceInfo), St
/// `max_edge` (set by the app). `None`: no usable preview.
pub type PreviewLoader = Arc<dyn Fn(&str, usize) -> Option<Rgba8> + Send + Sync>;
/// Makes a photo's denoised picture when it is not cached yet (an export does not wait for the background queue):
/// `Ok(true)` once it is there, `Ok(false)` when denoise does not apply to this photo (render it plain), `Err` when it
/// should have worked and did not.
pub type MakeProduct = Arc<dyn Fn() -> Result<bool, String> + Send + Sync>;
/// A raw photo's cached AI-denoised picture (see [`crate::denoise`]): where the file is and the key it must have been
/// made with. A file that is missing, damaged or made for something else is a cache miss, never an error.
#[derive(Clone)]
pub struct DenoiseSpec {
pub product: std::path::PathBuf,
pub key: String,
/// Run before the picture is read, by a job that must not render without it (an export).
pub make: Option<MakeProduct>,
}
/// Decodes a raw file once and develops it twice: as usual, and from the denoised picture in `spec` when that can be
/// read (`None` when it cannot). Both are at most `max_edge` long and the same size.
pub type PairLoader = Arc<dyn Fn(&str, usize, &DenoiseSpec) -> Result<(Rgb32f, Option<Rgb32f>, SourceInfo), String> + Send + Sync>;
/// A photo's denoised picture next to its plain one, and the last mix of the two that was asked for.
pub struct Twin {
pub image: Arc<Rgb32f>,
/// The Amount (in thousandths) and the picture mixed for it, so dragging other sliders keeps one source picture.
mix: std::sync::Mutex<Option<(u32, Arc<Rgb32f>)>>,
}
impl Twin {
pub fn new(image: Arc<Rgb32f>) -> Twin {
Twin { image, mix: std::sync::Mutex::new(None) }
}
/// Bytes reserved for the twin and its lazy mix, before either cache cost can grow.
fn bytes(&self) -> usize {
self.image.data.len().saturating_mul(24)
}
}
/// Pixels and the source interpretation learned while decoding; they must be cached together.
#[derive(Clone)]
pub struct DecodedSource {
@ -133,11 +170,46 @@ pub struct DecodedSource {
/// A smart preview's stored camera tone curve: the one decoder fact its pixels need that the
/// catalog's header facts lack (used when `info` is `None`).
pub camera_tone: Option<lightcraft_pipeline::tone::CameraTone>,
/// The same picture developed from the AI-denoised mosaic, when the photo has one cached.
pub denoised: Option<Arc<Twin>>,
}
impl DecodedSource {
pub fn new(image: Arc<Rgb32f>, info: Option<SourceInfo>) -> Self {
DecodedSource { image, info, camera_tone: None }
DecodedSource { image, info, camera_tone: None, denoised: None }
}
/// The picture to develop for a Denoise `amount` (0 to 1, see [`DevelopSettings::denoise_amount`]): the plain one,
/// the denoised one, or a mix of the two. A mix is made once per amount and kept, so the stages after it are
/// reused until the amount changes.
pub fn image_for(&self, amount: f32) -> Arc<Rgb32f> {
let Some(twin) = &self.denoised else { return self.image.clone() };
if amount.is_nan() || amount <= 0.0 || twin.image.width != self.image.width || twin.image.height != self.image.height {
return self.image.clone();
}
if amount >= 1.0 {
return twin.image.clone();
}
let thousandths = (amount * 1000.0).round().clamp(1.0, 999.0) as u32;
let mut memo = twin.mix.lock().unwrap_or_else(std::sync::PoisonError::into_inner);
if let Some((t, img)) = memo.as_ref()
&& *t == thousandths
{
return img.clone();
}
let mut mixed = (*self.image).clone();
if !lightcraft_denoise::product::blend(&mut mixed, &twin.image, thousandths as f32 / 1000.0) {
return self.image.clone();
}
let mixed = Arc::new(mixed);
*memo = Some((thousandths, mixed.clone()));
mixed
}
/// Cache cost: the picture, its segmentation mattes, its denoised twin and room for the lazy mix.
pub fn bytes(&self) -> usize {
let mattes = self.info.as_ref().and_then(|i| i.mattes.as_ref()).map_or(0, |m| m.bytes());
self.image.data.len() * 12 + 64 + std::mem::size_of::<SourceInfo>() + mattes + self.denoised.as_ref().map_or(0, |t| t.bytes())
}
/// What to render these pixels against: the decoder's facts, else `header` (the catalog's)
@ -161,6 +233,8 @@ pub enum SourceRef {
/// The photo's smart preview, used when the original can't be read (offline drive). The
/// UI thread never checks whether the original is there; the render worker finds out.
fallback: Option<std::path::PathBuf>,
/// The photo has a cached denoised picture: load it with the plain one (see [`PairLoader`]).
denoise: Option<(DenoiseSpec, PairLoader)>,
},
/// A smart preview standing in for a missing original.
Smart {
@ -176,16 +250,46 @@ impl SourceRef {
pub fn load_source(&self) -> Result<DecodedSource, String> {
let image = match self {
SourceRef::Loaded(a) => return Ok((**a).clone()),
SourceRef::Demo { scene, max_edge } => Arc::new(scene.render_fit(*max_edge)),
SourceRef::File { path, max_edge, loader, fallback } => {
let r = match loader {
SourceRef::Demo { scene, max_edge } => crate::demo::scene_pixels(scene, *max_edge),
SourceRef::File { path, max_edge, loader, fallback, denoise } => {
let plain = || match loader {
Some(l) => l(path, *max_edge).map(|(image, info)| DecodedSource::new(Arc::new(image), Some(info))),
None => Err(format!("no decoder available for {path}")),
};
// the denoised picture comes with the plain one from a single decode; when it cannot be read the
// pair loader gives the plain picture alone (the cached file is never a reason to fail a render)
let r = match denoise {
Some((spec, pair)) => {
// a job that needs the denoised picture has it made first; if that fails the job fails (no smart
// preview stands in: an export must not quietly come out without the denoise it was asked for)
let made = match &spec.make {
Some(make) => make()?,
None => true,
};
if made {
pair(path, *max_edge, spec).and_then(|(image, twin, info)| {
if spec.make.is_some() && twin.is_none() {
return Err("the cached AI Denoise picture could not be read; clear the denoise cache and retry".into());
}
Ok(DecodedSource {
image: Arc::new(image),
info: Some(info),
camera_tone: None,
denoised: twin.map(|t| Arc::new(Twin::new(Arc::new(t)))),
})
})
} else {
plain()
}
}
None => plain(),
};
// An offline original renders from its smart preview (no decoder facts: header ones).
return match (r, fallback) {
#[cfg(not(target_arch = "wasm32"))]
(Err(e), Some(sp)) if crate::smart::is_valid(sp) => crate::smart::load(sp).map_err(|e2| format!("{e}; smart preview: {e2}")),
(Err(e), Some(sp)) if denoise.as_ref().is_none_or(|(spec, _)| spec.make.is_none()) && crate::smart::is_valid(sp) => {
crate::smart::load(sp).map_err(|e2| format!("{e}; smart preview: {e2}"))
}
(r, _) => r,
};
}
@ -224,6 +328,54 @@ pub struct MediaCache {
scenes: Vec<lightcraft_scenes::Scene>,
/// Rendered thumbnails (memory, plus disk once a library is attached).
pub rendered: Arc<PreviewCache>,
/// Which photos have a cached denoised picture, and how to read it.
pub denoise: DenoiseIndex,
}
/// The photos that have a cached denoised picture for the denoise model in use. Only memory: nothing here touches the
/// file system, so it can be asked on the UI thread.
#[derive(Default)]
pub struct DenoiseIndex {
ready: std::collections::HashMap<PhotoId, DenoiseSpec>,
/// Reads the pictures (`None`: this build cannot, and nothing is ever ready).
pub loader: Option<PairLoader>,
}
impl DenoiseIndex {
/// What to load for `id` besides its plain picture, and how.
pub fn get(&self, id: PhotoId) -> Option<(&DenoiseSpec, &PairLoader)> {
self.ready.get(&id).zip(self.loader.as_ref())
}
pub fn spec(&self, id: PhotoId) -> Option<&DenoiseSpec> {
self.ready.get(&id)
}
pub fn set(&mut self, id: PhotoId, spec: DenoiseSpec) {
self.ready.insert(id, spec);
}
pub fn remove(&mut self, id: PhotoId) -> bool {
self.ready.remove(&id).is_some()
}
pub fn clear(&mut self) {
self.ready.clear();
}
pub fn len(&self) -> usize {
self.ready.len()
}
pub fn is_empty(&self) -> bool {
self.ready.is_empty()
}
pub fn ids(&self) -> Vec<PhotoId> {
let mut v: Vec<PhotoId> = self.ready.keys().copied().collect();
v.sort();
v
}
}
impl Default for MediaCache {
@ -244,6 +396,7 @@ impl Default for MediaCache {
availability: Default::default(),
scenes: Vec::new(),
rendered: Arc::new(PreviewCache::memory(rendered_budget(budget))),
denoise: DenoiseIndex::default(),
}
}
}
@ -253,11 +406,6 @@ fn rendered_budget(share: usize) -> usize {
RENDERED_MEM_BYTES.min(share / 4)
}
fn source_bytes(s: &DecodedSource) -> usize {
let mattes = s.info.as_ref().and_then(|i| i.mattes.as_ref()).map_or(0, |m| m.bytes());
s.image.data.len() * 12 + 64 + std::mem::size_of::<SourceInfo>() + mattes
}
impl MediaCache {
/// Keep rendered thumbnails on disk in `dir` as well. The cache this replaces is retired
/// (its files stay): render jobs still holding it can't write through it any more, even
@ -320,8 +468,8 @@ impl MediaCache {
let Some((tick, which)) = candidates.into_iter().flatten().min() else { break };
let freed = match which {
0 => self.thumbs.pop_oldest(),
1 => self.previews.iter().position(|e| e.2 == tick).map(|i| source_bytes(&self.previews.remove(i).1)),
2 => self.full.take().map(|e| source_bytes(&e.1)),
1 => self.previews.iter().position(|e| e.2 == tick).map(|i| self.previews.remove(i).1.bytes()),
2 => self.full.take().map(|e| e.1.bytes()),
_ => self.rendered.evict_oldest(),
};
match freed {
@ -373,7 +521,7 @@ impl MediaCache {
let tick = lightcraft_preview::next_tick();
match level {
SourceLevel::Thumb => {
let cost = source_bytes(&img);
let cost = img.bytes();
self.thumbs.insert(id, img, cost);
}
SourceLevel::Preview => {
@ -416,13 +564,42 @@ impl MediaCache {
/// Decoded sources held: (thumbnail level, preview level, full size).
pub fn usage(&self) -> (crate::memory::Usage, crate::memory::Usage, crate::memory::Usage) {
use crate::memory::Usage;
let previews = Usage::new(self.previews.len(), self.previews.iter().map(|e| source_bytes(&e.1)).sum());
let full = self.full.as_ref().map(|e| Usage::new(1, source_bytes(&e.1))).unwrap_or_default();
let previews = Usage::new(self.previews.len(), self.previews.iter().map(|e| e.1.bytes()).sum());
let full = self.full.as_ref().map(|e| Usage::new(1, e.1.bytes())).unwrap_or_default();
(Usage::new(self.thumbs.len(), self.thumbs.cost()), previews, full)
}
/// What a render of photo `id` with settings `s` adds to its identity for the denoised picture it will use: 0 unless
/// it uses one (the Denoise amount is above 0 and the photo has a cached denoised picture).
pub fn denoise_salt(&self, id: PhotoId, s: &DevelopSettings) -> u64 {
match self.denoise.spec(id) {
Some(spec) if s.denoise_amount() > 0.0 => Hasher128::new().str(&spec.key).finish().0 as u64 | 1,
_ => 0,
}
}
/// How to load photo `p` at `level`: from the cache when its source is there, else from its file (with its denoised
/// picture beside it when it has one), its smart preview or the demo scene.
pub fn source_ref(&mut self, p: &Photo, level: SourceLevel) -> SourceRef {
if let Some(a) = self.get_source(p.id, level) {
self.source_ref_with(p, level, true)
}
/// [`Self::source_ref`] that never takes a decoded source from memory: for a job that needs something the cached
/// one lacks (an export, whose denoised picture is made when it runs).
pub fn source_ref_uncached(&mut self, p: &Photo, level: SourceLevel) -> SourceRef {
self.source_ref_with(p, level, false)
}
fn source_ref_with(&mut self, p: &Photo, level: SourceLevel, cached: bool) -> SourceRef {
let mut r = self.source_ref_plain(p, level, cached);
if let (SourceRef::File { denoise, .. }, Some((spec, loader))) = (&mut r, self.denoise.get(p.id)) {
*denoise = Some((spec.clone(), loader.clone()));
}
r
}
fn source_ref_plain(&mut self, p: &Photo, level: SourceLevel, cached: bool) -> SourceRef {
if cached && let Some(a) = self.get_source(p.id, level) {
return SourceRef::Loaded(Box::new(a));
}
// Procedural scenes have a nominal size: "full" is that size, not unbounded.
@ -460,10 +637,12 @@ impl MediaCache {
}
match self.scenes.iter().find(|s| s.id == *scene) {
Some(s) => SourceRef::Demo { scene: Box::new(s.clone()), max_edge },
None => SourceRef::File { path: format!("demo:{scene}"), max_edge, loader: None, fallback: None },
None => SourceRef::File { path: format!("demo:{scene}"), max_edge, loader: None, fallback: None, denoise: None },
}
}
Source::File { path } => SourceRef::File { path: path.clone(), max_edge, loader: self.file_loader.clone(), fallback: None },
Source::File { path } => {
SourceRef::File { path: path.clone(), max_edge, loader: self.file_loader.clone(), fallback: None, denoise: None }
}
}
}
}
@ -574,7 +753,8 @@ impl RenderJob {
let was_loaded = matches!(self.source, SourceRef::Loaded(_));
match self.source.load_source() {
Ok(source) => {
let src = &source.image;
// the plain picture, or (Denoise amount above 0 and a denoised picture cached) the mix the amount asks for
let src = &source.image_for(self.settings.denoise_amount());
let info = source.info_or(self.info);
// Thumbnails (many small jobs side by side) stay on the CPU; views and exports use
// the GPU when there is one.
@ -758,22 +938,23 @@ impl crate::Session {
let content = source_key.0 as u64;
// Temporary "before" settings must not replace the photo's cached develop hash.
let settings_hash = if before { settings.hash64() } else { self.media.settings_hashes.get(id, &settings) };
let salt = self.media.denoise_salt(id, &settings);
let key = settings_hash
^ ((max_w as u64) << 40)
^ ((max_h as u64) << 20)
^ (apply_crop as u64)
^ (level as u64) << 60
^ id.0.wrapping_mul(0x9e37_79b9_7f4a_7c15)
^ content.rotate_left(17);
^ content.rotate_left(17)
^ salt;
let cache = thumb_bucket.map(|b| {
let k = Hasher128::new()
.str(&content_key)
.u64(settings_hash)
.u64(b as u64)
.u64(RENDER_CACHE_VERSION)
.u64(crate::camera_profiles::cache_key())
.finish();
(self.media.rendered.clone(), k)
let mut k = Hasher128::new();
k.str(&content_key).u64(settings_hash).u64(b as u64).u64(RENDER_CACHE_VERSION).u64(crate::camera_profiles::cache_key());
// (only a render that uses a denoised picture has it in its key, so every other thumbnail stays valid)
if salt != 0 {
k.u64(salt);
}
(self.media.rendered.clone(), k.finish())
});
Some(RenderJob {
request_id: 0,
@ -1064,6 +1245,15 @@ impl crate::Session {
depth: lightcraft_pipeline::OutputDepth,
) -> Result<RenderJob, String> {
let mut job = self.render_job(id, max_w, max_h, false, true).ok_or("no such photo")?;
// An export is made from the denoised picture its Denoise amount asks for, made when the job runs if the
// background queue has not got to it. (A source kept in memory may lack the picture, so the job reads the file.)
if let Some(wanted) = self.denoise_for_export(id, &job.settings)? {
let p = self.catalog.photo(id).ok_or("no such photo")?.clone();
job.source = self.media.source_ref_uncached(&p, job.level);
if let SourceRef::File { denoise, .. } = &mut job.source {
*denoise = Some(wanted);
}
}
job.request.space = space;
job.request.depth = depth;
job.key ^= (space as u64 + 1).wrapping_mul(0xa076_1d64_78bd_642f) ^ (depth as u64 + 1).wrapping_mul(0xe703_7ed1_a0b4_28db);
@ -1211,6 +1401,20 @@ mod thumbnail_hash_tests {
mod tests {
use super::*;
#[test]
fn denoise_mix_is_reserved_before_the_source_enters_the_cache() {
let mut plain = Rgb32f::new(8, 8);
plain.data.fill([1.0; 3]);
let mut source = DecodedSource::new(Arc::new(plain), None);
source.denoised = Some(Arc::new(Twin::new(Arc::new(Rgb32f::new(8, 8)))));
let reserved = source.bytes();
assert!(reserved >= 3 * 8 * 8 * 12);
assert_eq!(source.image_for(0.5).data[0], [0.5; 3]);
assert_eq!(source.bytes(), reserved, "creating a mix cannot evade the LRU's recorded cost");
let _ = source.image_for(0.25);
assert_eq!(source.bytes(), reserved, "changing Amount replaces the reserved mix");
}
#[test]
fn decoder_info_survives_render_jobs_cache_and_eviction() {
let tone = lightcraft_pipeline::tone::CameraTone::new(std::array::from_fn(|i| {
@ -1232,6 +1436,7 @@ mod tests {
Ok((image, loaded_info.clone()))
})),
fallback: None,
denoise: None,
};
job.info = SourceInfo::default(); // Header facts cannot override decoder facts.
job.settings = Arc::new(DevelopSettings::default());

359
crates/engine/src/model_download.rs Normal file
View File

@ -0,0 +1,359 @@
//! Downloading a denoise model explicitly requested after showing its terms.
//!
//! The transfer is [`lightcraft_fetch`] (pure-Rust HTTPS with resume, size and SHA-256 checks), on a background thread,
//! so the UI never waits for the network. Only a model listed in [`lightcraft_denoise::known::Download`] can be fetched:
//! the address and hash come from our pinned catalog or the user's model catalog, and no token is sent with it. What arrives
//! matches the model's recorded size and SHA-256 or it is thrown away; it then waits in a staging folder until the model
//! is installed (`denoise.models.install`, or by itself once the user has accepted the model's terms in the download).
//!
//! Failure is a message, never a panic: no network, a stalled or truncated transfer, a full disk, a file that is not the
//! one expected, and a panic inside the thread itself all end as [`State::Failed`].
use std::collections::HashMap;
use std::path::{Path, PathBuf};
use std::sync::atomic::{AtomicBool, Ordering};
use std::sync::{Arc, Mutex, PoisonError};
use std::time::Duration;
use lightcraft_denoise::known::Download;
/// The folder inside the models folder where a downloaded file waits to be installed. Its name starts with a dot, so it
/// can never be a model's own folder (ids start with a letter or digit).
pub const STAGING: &str = ".downloads";
/// Where a download stands.
#[derive(Clone, Debug, PartialEq)]
pub enum State {
/// Fetching (`bytes` of `total` so far); the file is checked as the last step.
Running {
bytes: u64,
total: u64,
},
/// Fetched and verified (its SHA-256 is `sha256`), waiting in the staging folder at `path` to be installed.
Done {
path: PathBuf,
sha256: String,
},
/// Installed and in use: nothing more to do but tell the user.
Installing,
Installed,
Failed(String),
Cancelled,
}
struct Shared {
state: Mutex<State>,
cancel: AtomicBool,
}
impl Shared {
fn new(total: u64) -> Self {
Shared { state: Mutex::new(State::Running { bytes: 0, total }), cancel: AtomicBool::new(false) }
}
fn set(&self, s: State) {
*self.state.lock().unwrap_or_else(PoisonError::into_inner) = s;
}
fn get(&self) -> State {
self.state.lock().unwrap_or_else(PoisonError::into_inner).clone()
}
}
/// The downloads of one session, by model id.
#[derive(Default)]
pub struct Downloads {
jobs: HashMap<String, Arc<Shared>>,
}
impl Downloads {
/// Start fetching `spec` into `<models dir>/.downloads/`. Different models can download side by side; the same
/// one is started once.
pub fn start(&mut self, spec: Download, models_dir: &Path) -> Result<(), String> {
if self.jobs.get(&spec.id).is_some_and(|j| matches!(j.get(), State::Running { .. })) {
return Err("that model is already downloading".into());
}
let staging = models_dir.join(STAGING);
std::fs::create_dir_all(&staging).map_err(|e| format!("could not create the download folder: {e}"))?;
let shared = Arc::new(Shared::new(spec.size_bytes));
let worker = shared.clone();
let id = spec.id.clone();
std::thread::Builder::new()
.name("denoise-model-download".into())
.spawn(move || {
let outcome = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| run(&spec, &spec.url, &staging, &worker)));
if outcome.is_err() {
worker.set(State::Failed("the download stopped unexpectedly".into()));
}
})
.map_err(|e| format!("could not start the download: {e}"))?;
self.jobs.insert(id, shared);
Ok(())
}
/// Every download this session has started and not discarded, by model id.
pub fn snapshot(&self) -> Vec<(String, State)> {
let mut all: Vec<_> = self.jobs.iter().map(|(id, j)| (id.clone(), j.get())).filter(|(_, s)| *s != State::Cancelled).collect();
all.sort_by(|a, b| a.0.cmp(&b.0));
all
}
pub fn running(&self) -> bool {
self.jobs.values().any(|j| matches!(j.get(), State::Running { .. }))
}
/// Stop a running download, or discard a finished one (its staged file is deleted). `true` if there was one.
pub fn discard(&mut self, id: &str) -> bool {
let Some(job) = self.jobs.get(id) else { return false };
job.cancel.store(true, Ordering::Relaxed);
match job.get() {
// the thread notices, deletes the partial file and marks itself cancelled
State::Running { .. } | State::Installing => {}
State::Done { path, .. } => {
let _ = std::fs::remove_file(path);
self.jobs.remove(id);
}
State::Installed | State::Failed(_) | State::Cancelled => {
self.jobs.remove(id);
}
}
true
}
/// Downloads that have arrived and been checked, ready to install: (model id, staged file, its SHA-256).
pub fn finished(&self) -> Vec<(String, PathBuf, String)> {
let mut out: Vec<_> = self
.jobs
.iter()
.filter_map(|(id, j)| match j.get() {
State::Done { path, sha256 } => Some((id.clone(), path, sha256)),
_ => None,
})
.collect();
out.sort_by(|a, b| a.0.cmp(&b.0));
out
}
/// Record how installing a finished download went (`Installed` or `Failed`).
pub fn set_outcome(&mut self, id: &str, state: State) {
match self.jobs.get(id) {
Some(j) => j.set(state),
None => {
let j = Shared::new(0);
j.set(state);
self.jobs.insert(id.to_string(), Arc::new(j));
}
}
}
}
impl Drop for Downloads {
/// Quitting while a download runs stops it instead of leaving it behind.
fn drop(&mut self) {
for job in self.jobs.values() {
job.cancel.store(true, Ordering::Relaxed);
}
let started = web_time::Instant::now();
while self.running() && started.elapsed() < Duration::from_secs(1) {
std::thread::sleep(Duration::from_millis(10));
}
}
}
/// What went wrong, in words the user can act on.
#[cfg(not(target_arch = "wasm32"))]
fn explain(e: &lightcraft_fetch::DownloadError) -> String {
use lightcraft_fetch::DownloadError;
match e {
DownloadError::AllMirrorsFailed { errors, .. } => {
let detail: String = errors.join("; ").chars().take(300).collect();
format!("could not download the model ({detail}): check the internet connection, or get the file from its page and add it yourself")
}
other => other.to_string(),
}
}
/// Fetch `spec` from its exact URL into `staging`, check it and leave it there.
#[cfg(not(target_arch = "wasm32"))]
fn fetch(spec: &Download, mirror: &str, staging: &Path, shared: &Shared) -> Result<(), lightcraft_fetch::DownloadError> {
let file = lightcraft_fetch::FileSpec { name: &spec.file_name, size: Some(spec.size_bytes), sha256: Some(&spec.sha256), max: spec.size_bytes };
lightcraft_fetch::download_url(&file, mirror, staging, &lightcraft_fetch::Options::default(), &shared.cancel, &mut |p| {
shared.set(State::Running { bytes: p.done.min(spec.size_bytes), total: spec.size_bytes });
})
}
#[cfg(target_arch = "wasm32")]
fn fetch(_: &Download, _: &str, _: &Path, _: &Shared) -> Result<(), String> {
Err("the web build downloads no models".into())
}
/// The whole job: reuse a verified earlier copy, else fetch, check and stage. Always ends in a final [`State`].
fn run(spec: &Download, mirror: &str, staging: &Path, shared: &Shared) {
let target = staging.join(&spec.file_name);
let part = staging.join(format!("{}.part", spec.file_name));
match fetch(spec, mirror, staging, shared) {
Ok(()) => shared.set(State::Done { path: target, sha256: spec.sha256.to_string() }),
#[cfg(not(target_arch = "wasm32"))]
Err(lightcraft_fetch::DownloadError::Cancelled) => {
let _ = std::fs::remove_file(&part);
shared.set(State::Cancelled);
}
#[cfg(not(target_arch = "wasm32"))]
Err(e) => {
shared.set(State::Failed(explain(&e)));
}
#[cfg(target_arch = "wasm32")]
Err(e) => {
let _ = std::fs::remove_file(&part);
shared.set(State::Failed(e));
}
}
}
#[cfg(all(test, not(target_arch = "wasm32")))]
mod tests {
use super::*;
use lightcraft_denoise::hash::sha256_hex;
use std::io::{BufRead, BufReader, Write};
use std::net::TcpListener;
fn scratch(name: &str) -> PathBuf {
let d = std::env::temp_dir().join(format!("lc-face-dl-{name}-{}", std::process::id()));
let _ = std::fs::remove_dir_all(&d);
std::fs::create_dir_all(&d).unwrap();
d
}
fn spec_for(bytes: &[u8]) -> Download {
Download {
id: "test-model".into(),
url: "https://example.invalid/m/test-model.onnx".into(),
file_name: "test-model.onnx".into(),
size_bytes: bytes.len() as u64,
sha256: sha256_hex(bytes),
}
}
/// A local server that answers every request with `status` and `body` (the whole file, no ranges).
fn serve(status: &'static str, body: Vec<u8>) -> String {
let l = TcpListener::bind("127.0.0.1:0").unwrap();
let base = format!("http://{}/m", l.local_addr().unwrap());
std::thread::spawn(move || {
for mut s in l.incoming().flatten() {
let mut r = BufReader::new(s.try_clone().unwrap());
loop {
let mut line = String::new();
if r.read_line(&mut line).unwrap_or(0) == 0 || line.trim().is_empty() {
break;
}
}
let _ = s.write_all(format!("HTTP/1.1 {status}\r\nContent-Length: {}\r\nConnection: close\r\n\r\n", body.len()).as_bytes());
let _ = s.write_all(&body);
}
});
base
}
#[test]
fn fetches_checks_and_stages_a_file() {
let dir = scratch("ok");
let bytes: Vec<u8> = (0..200_000u32).map(|i| (i % 251) as u8).collect();
let (staging, spec) = (dir.join(STAGING), spec_for(&bytes));
std::fs::create_dir_all(&staging).unwrap();
let mirror = serve("200 OK", bytes.clone());
let shared = Shared::new(spec.size_bytes);
run(&spec, &mirror, &staging, &shared);
let State::Done { path, sha256 } = shared.get() else { panic!("{:?}", shared.get()) };
assert_eq!((std::fs::read(&path).unwrap(), sha256), (bytes, spec.sha256.to_string()));
assert!(!staging.join("test-model.onnx.part").exists());
// asked again with the copy still staged: no fetch needed (nothing listens there)
let again = Shared::new(spec.size_bytes);
run(&spec, "http://127.0.0.1:1/m", &staging, &again);
assert!(matches!(again.get(), State::Done { .. }), "{:?}", again.get());
let _ = std::fs::remove_dir_all(&dir);
}
#[test]
fn a_file_that_does_not_match_is_thrown_away() {
let dir = scratch("bad");
let staging = dir.join(STAGING);
std::fs::create_dir_all(&staging).unwrap();
// the right size and the wrong content
let spec = spec_for(&[7u8; 5000]);
let shared = Shared::new(spec.size_bytes);
run(&spec, &serve("200 OK", vec![8u8; 5000]), &staging, &shared);
assert!(matches!(shared.get(), State::Failed(_)), "{:?}", shared.get());
// nothing is left behind: not the file, not a partial one
assert_eq!(std::fs::read_dir(&staging).unwrap().count(), 0);
let _ = std::fs::remove_dir_all(&dir);
}
#[test]
fn an_unreachable_or_refusing_server_is_a_message() {
let dir = scratch("down");
let staging = dir.join(STAGING);
std::fs::create_dir_all(&staging).unwrap();
let spec = spec_for(b"abc");
for mirror in ["http://127.0.0.1:1/m".to_string(), serve("404 Not Found", b"not found".to_vec())] {
let shared = Shared::new(3);
run(&spec, &mirror, &staging, &shared);
let State::Failed(why) = shared.get() else { panic!("{:?}", shared.get()) };
assert!(why.contains("internet connection"), "{why}");
}
assert_eq!(std::fs::read_dir(&staging).unwrap().count(), 0);
let _ = std::fs::remove_dir_all(&dir);
}
#[test]
fn a_cancelled_download_leaves_nothing() {
let dir = scratch("cancel");
let staging = dir.join(STAGING);
std::fs::create_dir_all(&staging).unwrap();
let spec = spec_for(b"abc");
let shared = Shared::new(3);
shared.cancel.store(true, Ordering::Relaxed);
run(&spec, &serve("200 OK", b"abc".to_vec()), &staging, &shared);
assert_eq!(shared.get(), State::Cancelled);
assert_eq!(std::fs::read_dir(&staging).unwrap().count(), 0);
let _ = std::fs::remove_dir_all(&dir);
}
#[test]
fn dropping_the_session_stops_a_running_download() {
let shared = Arc::new(Shared::new(10));
// a stand-in for the download thread: notices the cancel flag the way the downloader does
let worker = shared.clone();
let handle = std::thread::spawn(move || {
while !worker.cancel.load(Ordering::Relaxed) {
std::thread::sleep(Duration::from_millis(5));
}
worker.set(State::Cancelled);
});
let mut d = Downloads::default();
d.jobs.insert("m".into(), shared.clone());
assert!(d.running());
let started = web_time::Instant::now();
drop(d);
assert!(started.elapsed() < Duration::from_millis(900), "quitting must not wait for a transfer to finish");
assert_eq!(shared.get(), State::Cancelled);
handle.join().unwrap();
}
#[test]
fn discarding_a_finished_download_deletes_its_file() {
let dir = scratch("discard");
let file = dir.join("staged.onnx");
std::fs::write(&file, b"x").unwrap();
let mut d = Downloads::default();
let shared = Arc::new(Shared::new(1));
shared.set(State::Done { path: file.clone(), sha256: String::new() });
d.jobs.insert("m".into(), shared);
assert_eq!(d.snapshot().len(), 1);
assert!(!d.running());
assert!(d.discard("m"));
assert!(!file.exists());
assert!(d.snapshot().is_empty());
assert!(!d.discard("m"), "nothing left to discard");
let _ = std::fs::remove_dir_all(&dir);
}
}

13
crates/engine/src/segment/mod.rs
View File

@ -157,7 +157,14 @@ impl Segmenter {
/// Whether this build can compute AI masks at all.
pub const AVAILABLE: bool = cfg!(feature = "sam");
/// Whether the model's files are in place.
pub fn remote_endpoint(&self) -> Option<String> {
#[cfg(feature = "sam")]
return lightcraft_segment::remote::configured();
#[cfg(not(feature = "sam"))]
None
}
/// Whether the local model files are in place.
pub fn installed(&self) -> bool {
#[cfg(feature = "sam")]
if let Some(dir) = &self.dir {
@ -230,6 +237,10 @@ impl Segmenter {
if !Self::AVAILABLE {
return Err("AI masks are not available in this build".into());
}
#[cfg(feature = "sam")]
if lightcraft_segment::remote::configured().is_some() {
return Ok(PathBuf::new());
}
let dir = self.dir.clone().ok_or("no folder is set for the SAM 3 model")?;
if self.installed() {
return Ok(dir);

75
crates/engine/src/segment/worker.rs
View File

@ -17,7 +17,10 @@ use std::sync::mpsc::{self, RecvTimeoutError};
use std::time::Duration;
use lightcraft_develop::SegMask;
use lightcraft_segment::{Encoded, Sam3};
use lightcraft_segment::{
Encoded, Sam3,
remote::{self, Operation, Request, text_logits},
};
use super::{Input, Prompt, Tag};
use crate::media::RenderJob;
@ -175,6 +178,7 @@ impl Worker {
struct State {
model: Option<(PathBuf, Sam3)>,
cache: Option<(u64, Encoded)>,
remote: Option<remote::Client>,
}
/// Whether `later` makes `job` pointless (it asks for the same selection again).
@ -195,7 +199,7 @@ fn run(rx: &mpsc::Receiver<Job>, shared: &Shared, idle: Duration) {
match rx.recv_timeout(idle) {
Ok(j) => queue.push_back(j),
Err(RecvTimeoutError::Timeout) => {
if state.model.is_some() {
if state.model.is_some() || state.remote.is_some() {
state = State::default();
shared.loaded.store(false, Ordering::SeqCst);
log::info!("SAM 3 unloaded after {} minutes without use", idle.as_secs() / 60);
@ -231,7 +235,7 @@ fn run(rx: &mpsc::Receiver<Job>, shared: &Shared, idle: Duration) {
if result.is_err() && !is_detail {
state.cache = None;
}
shared.loaded.store(state.model.is_some(), Ordering::SeqCst);
shared.loaded.store(state.model.is_some() || state.remote.is_some(), Ordering::SeqCst);
drop(done);
let _ = job.reply.send(Outcome { tag: job.tag, result, superseded: false });
}
@ -284,29 +288,68 @@ fn encoded<'a>(state: &'a mut State, shared: &Shared, job: &mut Job) -> Result<(
log::info!("SAM 3 encoded {}×{} in {:?}", input.w, input.h, t.elapsed());
state.cache = Some((job.key, enc));
}
let State { model, cache } = state;
let State { model, cache, .. } = state;
match (model.as_mut(), cache.as_mut()) {
(Some((_, m)), Some((_, e))) => Ok((m, e)),
_ => Err("the model is not ready".into()),
}
}
/// Every phrase of `text` ("car, road"), merged (the per-pixel maximum); `None` when nothing
/// matches.
fn text_logits(model: &mut Sam3, enc: &mut Encoded, text: &str) -> Result<Option<Vec<f32>>, String> {
let mut merged: Option<Vec<f32>> = None;
for phrase in text.split([',', ';']).map(str::trim).filter(|p| !p.is_empty()) {
if let Some(p) = model.segment_text(enc, phrase, 0.5).map_err(|e| e.to_string())? {
merged = Some(match merged {
None => p.logits,
Some(m) => m.iter().zip(&p.logits).map(|(a, b)| a.max(*b)).collect(),
});
}
fn remote_job(state: &mut State, shared: &Shared, job: &mut Job, endpoint: &str) -> Result<Option<SegMask>, String> {
let _busy = Flag::raise(&shared.encoding);
if state.remote.is_none() {
state.remote = Some(remote::Client::connect(endpoint)?);
}
Ok(merged)
let client = state.remote.as_mut().ok_or("remote SAM unavailable")?;
let mut region = None;
let operation = match std::mem::replace(&mut job.kind, Kind::Prepare) {
Kind::Prepare => Operation::Prepare,
Kind::Text(text) => Operation::Text(text),
Kind::Clicks(clicks) => Operation::Clicks(clicks.iter().map(|c| [c.x, c.y, if c.positive { 1.0 } else { 0.0 }]).collect()),
Kind::Detail { prompt, region: r } => {
region = Some(r);
let (rw, rh) = (r[2] - r[0], r[3] - r[1]);
if !(rw > 0.0 && rh > 0.0) {
return Ok(None);
}
match prompt {
Prompt::Text(text) => Operation::Text(text),
Prompt::Clicks(clicks) => {
let inside: Vec<_> = clicks
.iter()
.map(|c| [((c.at.x - r[0]) / rw) as f32, ((c.at.y - r[1]) / rh) as f32, if c.include { 1.0 } else { 0.0 }])
.filter(|p| (0.0..=1.0).contains(&p[0]) && (0.0..=1.0).contains(&p[1]))
.collect();
if !inside.iter().any(|p| p[2] == 1.0) {
return Ok(None);
}
Operation::Clicks(inside)
}
}
}
#[cfg(test)]
Kind::Panic => return Err("test panic request".into()),
};
let input = if region.is_some() || client.key != Some(job.key) { render(job)? } else { Input { rgb: Vec::new(), w: 0, h: 0 } };
let request = Request { width: input.w, height: input.h, temporary: region.is_some(), operation };
let logits = client.request(&request, &input.rgb)?;
if region.is_none() {
client.key = Some(job.key);
}
Ok(logits.map(|l| match region {
Some(r) => SegMask::from_logits_in(lightcraft_segment::MASK_SIDE, &l, r),
None => SegMask::from_logits(lightcraft_segment::MASK_SIDE, &l),
}))
}
fn run_job(state: &mut State, shared: &Shared, job: &mut Job) -> Result<Option<SegMask>, String> {
if let Some(endpoint) = remote::configured() {
let result = remote_job(state, shared, job, &endpoint);
if result.is_err() {
state.remote = None;
}
return result.map_err(|e| format!("Remote SAM 3 failed: {e}. Check the worker and SSH tunnel, then retry."));
}
let side = lightcraft_segment::MASK_SIDE;
match std::mem::replace(&mut job.kind, Kind::Prepare) {
Kind::Prepare => {

2
crates/engine/src/smart.rs
View File

@ -190,7 +190,7 @@ pub fn is_valid(path: &Path) -> bool {
/// Load the proxy at `path`.
pub fn load(path: &Path) -> Result<crate::media::DecodedSource, String> {
let b = std::fs::read(path).map_err(|e| format!("{}: {e}", path.display()))?;
decode(&b).map(|(image, camera_tone)| crate::media::DecodedSource { image: Arc::new(image), info: None, camera_tone })
decode(&b).map(|(image, camera_tone)| crate::media::DecodedSource { image: Arc::new(image), info: None, camera_tone, denoised: None })
}
#[cfg(test)]

23
crates/engine/src/tests.rs
View File

@ -118,6 +118,29 @@ fn albums_crud() {
s.execute("album.delete", &json!({"id": f})).unwrap();
}
/// Given an album and a folder, when something is created inside each, then only the folder
/// accepts it: the sidebar only lists children of folders, so a child of a plain album would
/// be invisible.
#[test]
fn albums_and_smart_albums_are_created_only_inside_folders() {
let mut s = demo();
let folder = s.execute("album.create", &json!({"name": "Trips", "folder": true})).unwrap()["id"].as_u64().unwrap();
let album = s.execute("album.create", &json!({"name": "Best", "parent": folder})).unwrap()["id"].as_u64().unwrap();
for (cmd, params) in [
("album.create", json!({"name": "Nested", "parent": album})),
("album.create", json!({"name": "Nested", "folder": true, "parent": album})),
("album.createSmart", json!({"name": "Nested", "parent": album})),
("album.create", json!({"name": "Nested", "parent": 987_654})),
] {
assert!(s.execute(cmd, &params).is_err(), "{cmd} {params} must be refused");
}
let n = s.catalog.albums().count();
let sub = s.execute("album.create", &json!({"name": "Sub", "folder": true, "parent": folder})).unwrap()["id"].as_u64().unwrap();
let smart = s.execute("album.createSmart", &json!({"name": "Rated", "parent": sub})).unwrap()["id"].as_u64().unwrap();
assert_eq!(s.catalog.albums().count(), n + 2);
assert_eq!(s.catalog.album(lightcraft_catalog::AlbumId(smart)).unwrap().parent, Some(lightcraft_catalog::AlbumId(sub)));
}
#[test]
fn filters_and_delete_restore() {
let mut s = demo();

159
crates/engine/src/tests_album_order.rs Normal file
View File

@ -0,0 +1,159 @@
//! Ordering albums by hand: `album.reorder` puts an album (or folder) before a sibling, or last of
//! its kind, in one undo step, also when it changes folder; `album.sort` goes back to by name.
use lightcraft_catalog::AlbumId;
use serde_json::{Value, json};
use crate::Session;
fn make(s: &mut Session, name: &str, parent: Option<u64>, folder: bool) -> u64 {
let mut p = json!({"name": name, "folder": folder});
if let Some(parent) = parent {
p["parent"] = json!(parent);
}
s.execute("album.create", &p).unwrap()["id"].as_u64().unwrap()
}
fn names(s: &Session, parent: Option<u64>) -> Vec<String> {
s.catalog.album_children(parent.map(AlbumId)).iter().map(|a| a.name.clone()).collect()
}
fn reorder(s: &mut Session, id: u64, params: Value) -> Result<Value, String> {
let mut p = params;
p["id"] = json!(id);
s.execute("album.reorder", &p).map_err(|e| e.to_string())
}
/// Given three albums listed by name, when one is placed before another, then the list follows;
/// "no sibling" puts it last (among albums).
#[test]
fn an_album_is_placed_before_a_sibling_or_last() {
let mut s = Session::new();
let (a, b, c) = (make(&mut s, "A", None, false), make(&mut s, "B", None, false), make(&mut s, "C", None, false));
reorder(&mut s, c, json!({"before": a})).unwrap();
assert_eq!(names(&s, None), ["C", "A", "B"]);
assert!(s.catalog.album_children_are_ordered(None));
reorder(&mut s, c, json!({"before": null})).unwrap();
assert_eq!(names(&s, None), ["A", "B", "C"]);
reorder(&mut s, a, json!({})).unwrap();
assert_eq!(names(&s, None), ["B", "C", "A"], "no `before`: last");
assert_eq!(reorder(&mut s, b, json!({"before": b})).unwrap()["changed"], 0, "before itself is where it is");
assert_eq!(names(&s, None), ["B", "C", "A"]);
}
/// Folders stay ahead of albums: a folder is placed among folders, an album among albums.
#[test]
fn folders_and_albums_keep_to_their_own_group() {
let mut s = Session::new();
let (f1, f2) = (make(&mut s, "F1", None, true), make(&mut s, "F2", None, true));
let (a1, a2) = (make(&mut s, "A1", None, false), make(&mut s, "A2", None, false));
reorder(&mut s, f2, json!({"before": f1})).unwrap();
reorder(&mut s, a2, json!({"before": a1})).unwrap();
assert_eq!(names(&s, None), ["F2", "F1", "A2", "A1"]);
reorder(&mut s, f1, json!({"before": a1})).expect_err("a folder is not placed among albums");
reorder(&mut s, a1, json!({"before": f1})).expect_err("an album is not placed among folders");
reorder(&mut s, f2, json!({"before": null})).unwrap();
assert_eq!(names(&s, None), ["F1", "F2", "A2", "A1"], "last of the folders");
}
/// Placing an album in another folder moves it there, and one undo undoes both.
#[test]
fn placing_across_folders_moves_it_and_undoes_in_one_step() {
let mut s = Session::new();
let folder = make(&mut s, "Trips", None, true);
let (x, y) = (make(&mut s, "X", Some(folder), false), make(&mut s, "Y", Some(folder), false));
let loose = make(&mut s, "Loose", None, false);
reorder(&mut s, y, json!({"before": x})).unwrap();
assert_eq!(names(&s, Some(folder)), ["Y", "X"]);
reorder(&mut s, loose, json!({"parent": folder, "before": x})).unwrap();
assert_eq!(names(&s, Some(folder)), ["Y", "Loose", "X"]);
assert_eq!(s.catalog.album(AlbumId(loose)).unwrap().parent, Some(AlbumId(folder)));
s.execute("edit.undo", &json!({})).unwrap();
assert_eq!(names(&s, Some(folder)), ["Y", "X"]);
assert_eq!(s.catalog.album(AlbumId(loose)).unwrap().parent, None);
assert_eq!(s.catalog.album(AlbumId(loose)).unwrap().order, None);
}
/// Undo and redo of placing across folders, of sorting, and of moving a folder that has children.
#[test]
fn placing_and_sorting_undo_and_redo() {
let mut s = Session::new();
let top = make(&mut s, "Top", None, true);
let child = make(&mut s, "Child", Some(top), false);
let (a, b) = (make(&mut s, "A", None, false), make(&mut s, "B", None, false));
reorder(&mut s, b, json!({"before": a})).unwrap();
// a folder with children, placed inside another folder
let other = make(&mut s, "Other", None, true);
reorder(&mut s, top, json!({"parent": other})).unwrap();
assert_eq!(names(&s, Some(other)), ["Top"]);
assert_eq!(names(&s, Some(top)), ["Child"], "its children came along");
s.execute("edit.undo", &json!({})).unwrap();
assert_eq!(s.catalog.album(AlbumId(top)).unwrap().parent, None);
s.execute("edit.redo", &json!({})).unwrap();
assert_eq!(s.catalog.album(AlbumId(top)).unwrap().parent, Some(AlbumId(other)));
// sort, undo, redo
s.execute("album.sort", &json!({})).unwrap();
assert_eq!(names(&s, None), ["Other", "A", "B"]);
s.execute("edit.undo", &json!({})).unwrap();
assert_eq!(names(&s, None), ["Other", "B", "A"], "the hand order is back");
s.execute("edit.redo", &json!({})).unwrap();
assert_eq!(names(&s, None), ["Other", "A", "B"]);
assert_eq!(s.catalog.album(AlbumId(child)).unwrap().parent, Some(AlbumId(top)));
}
/// Bad requests change nothing: a `before` that is not in the target folder, a target that is not
/// a folder or lies inside the album, an unknown album.
#[test]
fn bad_placements_are_refused_and_change_nothing() {
let mut s = Session::new();
let f = make(&mut s, "F", None, true);
let inner = make(&mut s, "Inner", Some(f), true);
let (a, b) = (make(&mut s, "A", None, false), make(&mut s, "B", Some(f), false));
let before = s.catalog.to_snapshot();
reorder(&mut s, a, json!({"before": b})).expect_err("B is in F, not at the top level");
reorder(&mut s, a, json!({"parent": b})).expect_err("a plain album is no folder");
reorder(&mut s, f, json!({"parent": inner})).expect_err("a folder into itself");
reorder(&mut s, 987_654, json!({})).expect_err("no such album");
reorder(&mut s, a, json!({"before": 987_654})).expect_err("no such sibling");
assert_eq!(s.catalog.to_snapshot(), before, "nothing changed");
}
/// `album.sort` drops the hand order of one folder (and only that one): by name again.
#[test]
fn sorting_goes_back_to_by_name_for_one_folder() {
let mut s = Session::new();
let f = make(&mut s, "F", None, true);
let (x, y) = (make(&mut s, "X", Some(f), false), make(&mut s, "Y", Some(f), false));
let (a, b) = (make(&mut s, "A", None, false), make(&mut s, "B", None, false));
reorder(&mut s, y, json!({"before": x})).unwrap();
reorder(&mut s, b, json!({"before": a})).unwrap();
assert_eq!(names(&s, Some(f)), ["Y", "X"]);
assert_eq!(names(&s, None), ["F", "B", "A"]);
let r = s.execute("album.sort", &json!({"parent": f})).unwrap();
assert_eq!(r["changed"], 2);
assert_eq!(names(&s, Some(f)), ["X", "Y"]);
assert_eq!(names(&s, None), ["F", "B", "A"], "the top level keeps its order");
assert!(!s.catalog.album_children_are_ordered(Some(AlbumId(f))));
// nothing to do: no undo step is made
let r = s.execute("album.sort", &json!({"parent": f})).unwrap();
assert_eq!(r["changed"], 0);
s.execute("edit.undo", &json!({})).unwrap();
assert_eq!(names(&s, Some(f)), ["Y", "X"], "the one undo step was the sort");
let r = s.execute("album.sort", &json!({})).unwrap();
assert_eq!(r["changed"], 3, "every child of the top level had a place: F, B and A");
assert_eq!(names(&s, None), ["F", "A", "B"]);
}
/// Moving an album with `album.move` out of an ordered folder leaves the old neighbours' order.
#[test]
fn moving_out_of_an_ordered_folder_keeps_the_others_in_place() {
let mut s = Session::new();
let f = make(&mut s, "F", None, true);
let (x, y, z) = (make(&mut s, "X", Some(f), false), make(&mut s, "Y", Some(f), false), make(&mut s, "Z", Some(f), false));
reorder(&mut s, z, json!({"before": x})).unwrap();
assert_eq!(names(&s, Some(f)), ["Z", "X", "Y"]);
s.execute("album.move", &json!({"id": x, "parent": null})).unwrap();
assert_eq!(names(&s, Some(f)), ["Z", "Y"]);
assert_eq!(names(&s, None), ["F", "X"]);
let _ = y;
}

759
crates/engine/src/tests_denoise.rs Normal file
View File

@ -0,0 +1,759 @@
//! AI denoise through the engine: installing a model, making a photo's denoised picture in the background and for an
//! export, the Amount slider mixing it in, and the cache. A tiny stand-in model (it halves every sample) takes the
//! place of a real one, which is enough to see what the machinery does with its answer; how well real models denoise is
//! measured separately (`crates/denoise/tests/real_model.rs`).
use std::path::{Path, PathBuf};
use std::sync::Arc;
use std::sync::atomic::{AtomicUsize, Ordering};
use lightcraft_denoise::manifest::{DenoiserManifest, Domain, Gain};
use lightcraft_denoise::run::TileRunner;
use serde_json::{Value, json};
use crate::Session;
use crate::denoise::{Loader, Model, PhotoState, RunOn};
use crate::export::{ExportOptions, export_photo};
use crate::tests_xmp::{synthetic_dng_of, temp_dir};
/// A model that makes every output pixel half of its cell's colour: a picture that is plainly not the plain one.
struct Dim {
runs: Arc<AtomicUsize>,
}
impl TileRunner for Dim {
fn run(&self, input: &[f32]) -> Result<Vec<f32>, lightcraft_denoise::Error> {
self.runs.fetch_add(1, Ordering::Relaxed);
let t = ((input.len() / 4) as f64).sqrt() as usize;
let side = 2 * t;
let mut out = vec![0f32; 3 * side * side];
for y in 0..side {
for x in 0..side {
let i = (y / 2) * t + x / 2;
let (r, g, b) = (input[i], (input[t * t + i] + input[2 * t * t + i]) / 2.0, input[3 * t * t + i]);
out[y * side + x] = r * 0.5;
out[side * side + y * side + x] = g * 0.5;
out[2 * side * side + y * side + x] = b * 0.5;
}
}
Ok(out)
}
}
impl Model for Dim {
fn runner(&self, _: RunOn, threads: usize) -> (&dyn TileRunner, usize) {
(self, threads)
}
fn self_test(&self, _: RunOn) -> Result<Value, String> {
Ok(json!({"ok": true}))
}
}
fn dim_loader(runs: &Arc<AtomicUsize>) -> Loader {
let runs = runs.clone();
Arc::new(move |_, _| Ok(Arc::new(Dim { runs: runs.clone() }) as Arc<dyn Model>))
}
fn manifest(id: &str) -> DenoiserManifest {
DenoiserManifest {
id: id.into(),
name: "Stand-in".into(),
version: "1".into(),
licence: Default::default(),
source: None,
sha256: None,
size_bytes: None,
provenance: "made up for a test".into(),
domain: Domain::BayerToRgb,
tile: 64,
overlap: 16,
gain: Gain::None,
}
}
/// The model's `.onnx` (not a real one: the stand-in loader never reads it) with its manifest beside it.
fn model_files(dir: &Path, id: &str) -> PathBuf {
let src = dir.join(format!("src-{id}"));
std::fs::create_dir_all(&src).unwrap();
std::fs::write(src.join("denoise-model.json"), serde_json::to_vec(&manifest(id)).unwrap()).unwrap();
std::fs::write(src.join("model.onnx"), format!("not an onnx file, only a stand-in for {id}")).unwrap();
src.join("model.onnx")
}
#[test]
fn model_installation_keeps_the_frame_pump_responsive_and_reuses_the_loaded_model() {
use std::sync::{Mutex, mpsc};
use std::time::Duration;
let mut x = setup("background-install", false);
let path = model_files(&x.dir, "stand-in");
let (entered_tx, entered_rx) = mpsc::channel();
let (release_tx, release_rx) = mpsc::channel();
let release_rx = Mutex::new(release_rx);
let runs = x.runs.clone();
let loads = Arc::new(AtomicUsize::new(0));
let worker_loads = loads.clone();
x.s.denoise.loader = Arc::new(move |_, _| {
worker_loads.fetch_add(1, Ordering::Relaxed);
entered_tx.send(()).map_err(|e| e.to_string())?;
release_rx.lock().unwrap().recv_timeout(Duration::from_secs(3)).map_err(|e| e.to_string())?;
Ok(Arc::new(Dim { runs: runs.clone() }))
});
let info = x.s.execute("denoise.models.inspect", &json!({"path": path})).unwrap();
assert_eq!(info["model"]["id"], "stand-in");
assert_eq!(loads.load(Ordering::Relaxed), 0, "showing terms never executes the model");
let r = x.s.execute("denoise.models.install", &json!({"path": path, "acknowledged": true, "background": true})).unwrap();
assert_eq!(r["started"], "local-install");
entered_rx.recv_timeout(Duration::from_secs(3)).unwrap();
// The loader is blocked: the frame pump must still return, reporting installation in progress.
let state = x.s.execute("denoise.models.downloads", &json!({})).unwrap();
assert_eq!(state["downloads"][0]["state"], "installing");
assert_eq!(x.s.execute("denoise.pump", &json!({"pace": "pause"})).unwrap()["active"], false);
assert!(x.s.execute("denoise.models.install", &json!({"path": path, "acknowledged": true, "background": true})).is_err());
release_tx.send(()).unwrap();
for _ in 0..600 {
let v = x.s.execute("denoise.models.downloads", &json!({})).unwrap();
if v["downloads"][0]["state"] == "installed" {
break;
}
std::thread::sleep(Duration::from_millis(5));
}
assert_eq!(x.s.execute("denoise.models.downloads", &json!({})).unwrap()["downloads"][0]["state"], "installed");
let a = x.s.denoise.active.as_ref().unwrap();
assert!(a.model.lock().unwrap().is_some(), "the worker's tested model is reused");
assert_eq!(loads.load(Ordering::Relaxed), 1);
drop(x.s);
let _ = std::fs::remove_dir_all(x.dir);
}
#[test]
fn failed_model_replacement_preserves_the_installed_model() {
let mut x = setup("replacement", true);
let installed = x.dir.join("denoise-models/stand-in/model.onnx");
let original = std::fs::read(&installed).unwrap();
let source = model_files(&x.dir, "stand-in");
std::fs::write(&source, "a different broken model").unwrap();
assert!(x.s.execute("denoise.models.install", &json!({"path": source, "acknowledged": true})).is_err());
assert_eq!(std::fs::read(installed).unwrap(), original);
assert_eq!(x.s.execute("denoise.models.list", &json!({})).unwrap()["model"], "stand-in");
drop(x.s);
let _ = std::fs::remove_dir_all(x.dir);
}
/// A Bayer DNG that is a smooth ramp, big enough for a 64-cell tile and a second one.
fn ramp_dng(w: usize, h: usize, cfa: &str) -> Vec<u8> {
let data: Vec<u16> = (0..w * h).map(|i| 600 + ((i % w) * 30 + (i / w) * 35) as u16).collect();
synthetic_dng_of(w, h, cfa, data, None, Default::default())
}
struct Setup {
dir: PathBuf,
s: Session,
runs: Arc<AtomicUsize>,
}
/// A session with a library on disk holding one raw photo, and the stand-in model installed and chosen.
fn setup(tag: &str, with_model: bool) -> Setup {
let dir = temp_dir(tag);
let originals = dir.join("originals");
std::fs::create_dir_all(&originals).unwrap();
std::fs::write(originals.join("ramp.dng"), ramp_dng(192, 160, "RGGB")).unwrap();
let runs = Arc::new(AtomicUsize::new(0));
let mut s = Session::new().with_fs();
s.open_library(dir.join("library"), false).unwrap();
s.execute("library.import", &json!({"paths": [originals.join("ramp.dng").to_string_lossy()]})).unwrap();
let id = s.catalog.photos().next().unwrap().id;
s.execute("library.select", &json!({"ids": [id.0]})).unwrap();
s.set_denoise_models_dir(Some(dir.join("denoise-models")));
s.denoise.loader = dim_loader(&runs);
if with_model {
let onnx = model_files(&dir, "stand-in");
s.execute("denoise.models.install", &json!({"path": onnx.to_string_lossy(), "acknowledged": true})).unwrap();
}
Setup { dir, s, runs }
}
fn photo(s: &Session) -> lightcraft_catalog::PhotoId {
s.catalog.photos().next().unwrap().id
}
/// Pump until `n` photos are ready (the job runs on its own thread).
fn pump_until_ready(s: &mut Session, n: u64) -> Value {
for _ in 0..600 {
let r = s.execute("denoise.pump", &json!({"pace": "full"})).unwrap();
if r["ready"].as_u64() >= Some(n) {
return r;
}
std::thread::sleep(std::time::Duration::from_millis(50));
}
panic!("not ready: {}", s.execute("denoise.status", &json!({})).unwrap());
}
fn mean(img: &lightcraft_raster::Rgba8) -> f64 {
let sum: u64 = img.data.iter().map(|p| u64::from(p[0]) + u64::from(p[1]) + u64::from(p[2])).sum();
sum as f64 / (img.data.len() * 3) as f64
}
fn rendered_mean(s: &mut Session, id: lightcraft_catalog::PhotoId) -> f64 {
mean(&s.render_now(id, 160, 160).unwrap().image)
}
fn set_amount(s: &mut Session, amount: f64) {
s.execute("develop.set", &json!({"control": "enhance.denoise", "value": amount})).unwrap();
}
#[test]
fn a_model_is_installed_chosen_and_removed_with_the_terms_accepted() {
let mut x = setup("model", false);
let onnx = model_files(&x.dir, "stand-in");
// the terms come first
let e = x.s.execute("denoise.models.install", &json!({"path": onnx.to_string_lossy()})).unwrap_err().to_string();
assert!(e.contains("licence has not been accepted"), "{e}");
let r = x.s.execute("denoise.models.list", &json!({})).unwrap();
assert_eq!(r["model"], Value::Null);
assert_eq!(r["runtime"], cfg!(feature = "denoise"));
// a file with no manifest is not guessed at
let stray = x.dir.join("stray.onnx");
std::fs::write(&stray, b"who knows").unwrap();
let e = x.s.execute("denoise.models.install", &json!({"path": stray.to_string_lossy(), "acknowledged": true})).unwrap_err().to_string();
assert!(e.contains("does not know this file"), "{e}");
// installed, it is the model in use
let r = x.s.execute("denoise.models.install", &json!({"path": onnx.to_string_lossy(), "acknowledged": true})).unwrap();
assert_eq!((r["installed"]["id"].clone(), r["model"].clone()), (json!("stand-in"), json!("stand-in")), "{r}");
assert!(r["installed"]["accepted"]["sha256"].as_str().is_some_and(|h| h.len() == 64), "the file's hash is recorded: {r}");
assert!(x.s.denoise.active.is_some());
assert_eq!(x.s.execute("denoise.status", &json!({})).unwrap()["model"]["id"], "stand-in");
// it can be checked again, switched off and on, and removed
assert_eq!(x.s.execute("denoise.models.test", &json!({"id": "stand-in"})).unwrap()["ok"], true);
assert_eq!(x.s.execute("denoise.models.select", &json!({"id": null})).unwrap()["model"], Value::Null);
assert!(x.s.denoise.active.is_none());
assert!(x.s.execute("denoise.models.select", &json!({"id": "nothing"})).is_err());
x.s.execute("denoise.models.select", &json!({"id": "stand-in"})).unwrap();
assert!(x.s.denoise.active.is_some());
let r = x.s.execute("denoise.models.remove", &json!({"id": "stand-in"})).unwrap();
assert_eq!(r["model"], Value::Null);
assert!(x.s.denoise.active.is_none());
assert!(!x.dir.join("denoise-models").join("stand-in").exists());
// ids that are not folder names, and folders that are not models, are never touched
for bad in ["../library", "", "a/b", "settings.json"] {
assert!(x.s.execute("denoise.models.remove", &json!({"id": bad})).is_err(), "{bad}");
}
assert!(x.dir.join("library").is_dir());
// a model that fails its self-test is not left installed
x.s.denoise.loader = Arc::new(|_, _| Err("it does not load".into()));
let e = x.s.execute("denoise.models.install", &json!({"path": onnx.to_string_lossy(), "acknowledged": true})).unwrap_err().to_string();
assert!(e.contains("it does not load"), "{e}");
assert!(!x.dir.join("denoise-models").join("stand-in").exists());
let _ = std::fs::remove_dir_all(&x.dir);
}
#[test]
fn a_manifest_that_does_not_validate_or_name_the_file_is_refused() {
let mut x = setup("manifest", false);
let onnx = model_files(&x.dir, "stand-in");
let json_path = onnx.with_file_name("denoise-model.json");
// a tile the model cannot run on
let mut m = manifest("stand-in");
m.tile = 50;
std::fs::write(&json_path, serde_json::to_vec(&m).unwrap()).unwrap();
let e = x.s.execute("denoise.models.install", &json!({"path": onnx.to_string_lossy(), "acknowledged": true})).unwrap_err().to_string();
assert!(e.contains("tile"), "{e}");
// a hash that is not the file's
let mut m = manifest("stand-in");
m.sha256 = Some("0".repeat(64));
std::fs::write(&json_path, serde_json::to_vec(&m).unwrap()).unwrap();
let e = x.s.execute("denoise.models.install", &json!({"path": onnx.to_string_lossy(), "acknowledged": true})).unwrap_err().to_string();
assert!(e.contains("SHA-256"), "{e}");
assert!(!x.dir.join("denoise-models").join("stand-in").join("model.onnx").exists(), "nothing half-installed is left");
// an id that would leave the folder
let mut m = manifest("../escape");
m.id = "../escape".into();
std::fs::write(&json_path, serde_json::to_vec(&m).unwrap()).unwrap();
assert!(x.s.execute("denoise.models.install", &json!({"path": onnx.to_string_lossy(), "acknowledged": true})).is_err());
assert!(!x.dir.join("escape").exists());
// garbage instead of a manifest
std::fs::write(&json_path, b"{ not json").unwrap();
assert!(x.s.execute("denoise.models.install", &json!({"path": onnx.to_string_lossy(), "acknowledged": true})).is_err());
let _ = std::fs::remove_dir_all(&x.dir);
}
#[test]
fn a_photo_is_denoised_in_the_background_and_the_amount_mixes_it_in() {
let mut x = setup("background", true);
let id = photo(&x.s);
// nothing is made until a photo has an amount
let r = x.s.execute("denoise.pump", &json!({"pace": "full"})).unwrap();
assert_eq!((r["queued"].clone(), r["running"].clone(), r["ready"].clone()), (json!(0), Value::Null, json!(0)), "{r}");
assert_eq!(x.s.denoise_photo_state(id), PhotoState::Idle);
let plain = rendered_mean(&mut x.s, id);
let key_before = x.s.render_job(id, 160, 160, false, true).unwrap().key;
set_amount(&mut x.s, 100.0);
// the photo being edited is picked up by itself, and made on another thread
let r = pump_until_ready(&mut x.s, 1);
assert_eq!(r["queued"], 0, "{r}");
assert_eq!(x.s.denoise_photo_state(id), PhotoState::Ready);
assert!(x.runs.load(Ordering::Relaxed) >= 4, "a 192 × 160 mosaic is four 64-cell tiles: {}", x.runs.load(Ordering::Relaxed));
let products: Vec<_> = std::fs::read_dir(x.dir.join("library").join("denoise")).unwrap().flatten().collect();
assert_eq!(products.len(), 1, "one cached picture, no DNG or anything else in the library");
assert!(products[0].path().extension().is_some_and(|e| e == "lcdn"));
let originals: Vec<_> = std::fs::read_dir(x.dir.join("originals")).unwrap().flatten().collect();
assert_eq!(originals.len(), 1, "the original is untouched and alone");
// the render now comes from the denoised picture: darker at 100, in between at 50, the plain one at 0
let full = rendered_mean(&mut x.s, id);
assert!(full < plain - 3.0, "denoised {full} vs plain {plain}");
set_amount(&mut x.s, 50.0);
let half = rendered_mean(&mut x.s, id);
assert!(half < plain - 1.0 && half > full + 1.0, "{full} < {half} < {plain}");
set_amount(&mut x.s, 0.0);
assert!((rendered_mean(&mut x.s, id) - plain).abs() < 0.01, "amount 0 is the plain picture again");
// …and a render that uses it has another identity than one that does not
assert_ne!(key_before, {
set_amount(&mut x.s, 100.0);
x.s.render_job(id, 160, 160, false, true).unwrap().key
});
// switching the Detail section off switches the denoise off with it
set_amount(&mut x.s, 100.0);
x.s.execute("develop.sectionEnabled", &json!({"section": "detail", "enabled": false})).unwrap();
assert!((rendered_mean(&mut x.s, id) - plain).abs() < 0.01);
assert_eq!(x.s.media.denoise_salt(id, &x.s.catalog.photo(id).unwrap().develop), 0);
let _ = std::fs::remove_dir_all(&x.dir);
}
#[test]
fn what_was_made_is_found_again_by_a_new_session_and_a_new_model_starts_over() {
let mut x = setup("again", true);
let id = photo(&x.s);
set_amount(&mut x.s, 100.0);
pump_until_ready(&mut x.s, 1);
let made = x.runs.load(Ordering::Relaxed);
x.s.close_library().unwrap();
drop(x.s);
// reopened: ready without running the model again
let mut s2 = Session::new().with_fs();
s2.set_denoise_models_dir(Some(x.dir.join("denoise-models")));
s2.denoise.loader = dim_loader(&x.runs);
s2.open_library(x.dir.join("library"), false).unwrap();
s2.denoise_pump(crate::denoise::Pace::Full);
assert_eq!(s2.denoise_photo_state(id), PhotoState::Ready);
assert_eq!(x.runs.load(Ordering::Relaxed), made, "nothing was made again");
// another model's pictures are different pictures
let onnx = model_files(&x.dir, "other");
s2.execute("denoise.models.install", &json!({"path": onnx.to_string_lossy(), "acknowledged": true})).unwrap();
s2.denoise_pump(crate::denoise::Pace::Pause);
assert!(
matches!(s2.denoise_photo_state(id), PhotoState::Queued { .. }),
"the first model's picture is not this model's: it is to be made: {:?}",
s2.denoise_photo_state(id)
);
// and choosing the first again finds its picture where it was
s2.execute("denoise.models.select", &json!({"id": "stand-in"})).unwrap();
s2.denoise_pump(crate::denoise::Pace::Pause);
assert_eq!(s2.denoise_photo_state(id), PhotoState::Ready);
let _ = std::fs::remove_dir_all(&x.dir);
}
#[test]
fn an_export_makes_its_picture_when_the_queue_has_not_and_never_comes_out_without_it() {
let mut x = setup("export", true);
let id = photo(&x.s);
let o = ExportOptions::from_json(&json!({"format": "png", "width": 160, "height": 160}));
let decode = |bytes: &[u8]| {
let d = lightcraft_codecs::decode(bytes, lightcraft_codecs::DecodeOptions::fit(160, 160)).unwrap();
let w = d.to_working();
w.data.iter().map(|p| f64::from(p[0] + p[1] + p[2])).sum::<f64>() / (w.data.len() * 3) as f64
};
let plain = decode(&export_photo(&mut x.s, id, &o, 1).unwrap().bytes);
assert_eq!(x.runs.load(Ordering::Relaxed), 0, "no amount, no denoise");
set_amount(&mut x.s, 100.0);
// no pump: the export makes the picture itself
let denoised = decode(&export_photo(&mut x.s, id, &o, 1).unwrap().bytes);
assert!(x.runs.load(Ordering::Relaxed) >= 4);
assert!(denoised < plain * 0.9, "denoised {denoised} vs plain {plain}");
let products = std::fs::read_dir(x.dir.join("library").join("denoise")).unwrap().flatten().count();
assert_eq!(products, 1);
// a second export finds it
let runs = x.runs.load(Ordering::Relaxed);
let again = decode(&export_photo(&mut x.s, id, &o, 1).unwrap().bytes);
assert_eq!(x.runs.load(Ordering::Relaxed), runs);
assert!((again - denoised).abs() < 1e-6);
// the pump takes in what the export made
x.s.denoise_pump(crate::denoise::Pace::Pause);
assert_eq!(x.s.denoise_photo_state(id), PhotoState::Ready);
// no model: the export says so instead of leaving the denoise out
x.s.execute("denoise.models.select", &json!({"id": null})).unwrap();
let e = export_photo(&mut x.s, id, &o, 1).err().unwrap();
assert!(e.contains("no denoise model is chosen") && e.contains("ramp.dng"), "{e}");
set_amount(&mut x.s, 0.0);
assert!(export_photo(&mut x.s, id, &o, 1).is_ok(), "without an amount nothing is asked of the model");
let _ = std::fs::remove_dir_all(&x.dir);
}
#[test]
fn a_model_that_fails_fails_the_export_and_is_not_retried_by_the_queue() {
let mut x = setup("fails", true);
let id = photo(&x.s);
x.s.denoise.loader = Arc::new(|_, _| Err("the model went away".into()));
// Simulate loss of the in-memory model; successful installation now retains its tested instance.
*x.s.denoise.active.as_ref().unwrap().model.lock().unwrap() = None;
set_amount(&mut x.s, 80.0);
let o = ExportOptions::from_json(&json!({"format": "png", "width": 100, "height": 100}));
let e = export_photo(&mut x.s, id, &o, 1).err().unwrap();
assert!(e.contains("ramp.dng") && e.contains("the model went away"), "{e}");
// the background job fails too, says why once and does not try again every frame
for _ in 0..200 {
x.s.execute("denoise.pump", &json!({"pace": "full"})).unwrap();
if matches!(x.s.denoise_photo_state(id), PhotoState::Failed { .. }) {
break;
}
std::thread::sleep(std::time::Duration::from_millis(20));
}
match x.s.denoise_photo_state(id) {
PhotoState::Failed { why, unsupported } => assert!(why.contains("the model went away") && !unsupported, "{why}"),
other => panic!("{other:?}"),
}
let r = x.s.execute("denoise.pump", &json!({"pace": "full"})).unwrap();
assert_eq!((r["queued"].clone(), r["running"].clone()), (json!(0), Value::Null), "{r}");
// asking again by hand does retry
x.s.denoise.loader = dim_loader(&x.runs);
let r = x.s.execute("denoise.queue", &json!({"retry": true})).unwrap();
assert_eq!(r["added"], 1, "{r}");
pump_until_ready(&mut x.s, 1);
let _ = std::fs::remove_dir_all(&x.dir);
}
#[test]
fn a_corrupt_cache_with_a_current_header_never_silently_exports_the_plain_photo() {
let mut x = setup("corrupt-export", true);
let id = photo(&x.s);
set_amount(&mut x.s, 100.0);
let options = ExportOptions::from_json(&json!({"format": "png", "width": 160, "height": 160}));
export_photo(&mut x.s, id, &options, 1).unwrap();
x.s.denoise_pump(crate::denoise::Pace::Pause);
let spec = x.s.media.denoise.spec(id).unwrap().clone();
let mut bytes = std::fs::read(&spec.product).unwrap();
for byte in bytes.iter_mut().rev().take(64) {
*byte ^= 0xff;
}
std::fs::write(&spec.product, bytes).unwrap();
assert!(lightcraft_denoise::product::is_current(&spec.product, &spec.key), "header/length still look current");
x.s.media.forget(id);
let error = export_photo(&mut x.s, id, &options, 1).err().unwrap();
assert!(error.contains("AI Denoise picture could not be read"), "{error}");
let _ = std::fs::remove_dir_all(x.dir);
}
#[test]
fn only_raw_photos_on_disk_are_denoised_and_nothing_is_queued_without_a_model() {
let dir = temp_dir("eligible");
let mut demo = Session::with_demo();
demo.set_denoise_models_dir(Some(dir.join("models")));
let id = demo.active().unwrap();
assert_eq!(demo.denoise_photo_state(id), PhotoState::NotApplicable, "a generated photo has no sensor data");
let e = demo.execute("denoise.queue", &json!({})).unwrap_err().to_string();
assert!(e.contains("no denoise model is chosen"), "{e}");
// no models folder at all (the web build): the commands say so, nothing panics
let mut web = Session::with_demo();
for cmd in ["denoise.models.install", "denoise.models.select", "denoise.models.remove", "denoise.settings"] {
assert!(web.execute(cmd, &json!({"path": "x", "acknowledged": true, "id": "x"})).is_err(), "{cmd}");
}
assert_eq!(web.execute("denoise.pump", &json!({})).unwrap()["active"], false);
assert_eq!(web.execute("denoise.status", &json!({})).unwrap()["enabled"], false);
// no library open: nowhere to keep pictures, and the queue says so
let x = setup("eligible2", true);
let mut bare = Session::new().with_fs();
bare.set_denoise_models_dir(Some(x.dir.join("denoise-models")));
assert!(bare.denoise_products_dir().is_none());
let e = bare.execute("denoise.queue", &json!({})).unwrap_err().to_string();
assert!(e.contains("open a library on disk first"), "{e}");
assert_eq!(bare.execute("denoise.pump", &json!({})).unwrap()["queued"], 0);
let _ = std::fs::remove_dir_all(&dir);
let _ = std::fs::remove_dir_all(&x.dir);
}
#[test]
fn files_that_cannot_be_denoised_fail_with_a_reason_and_never_a_panic() {
let dir = temp_dir("makefails");
let spec = |path: &Path| crate::denoise::JobSpec {
path: path.to_string_lossy().into_owned(),
read: None,
product: dir.join("p.lcdn"),
key: "k".into(),
manifest: manifest("stand-in"),
onnx: dir.join("model.onnx"),
model: Default::default(),
loader: dim_loader(&Arc::new(AtomicUsize::new(0))),
parallel: 2,
run_on: RunOn::Auto,
};
use crate::denoise::{MakeError, make_product};
// not a file
assert!(matches!(make_product(&spec(&dir.join("missing.dng")), None), Err(MakeError::Failed(_))));
// not a raw
let junk = dir.join("junk.dng");
std::fs::write(&junk, b"this is not a raw file at all").unwrap();
assert!(matches!(make_product(&spec(&junk), None), Err(MakeError::Unsupported(_))));
// a raw cut short
let whole = ramp_dng(192, 160, "RGGB");
let cut = dir.join("cut.dng");
std::fs::write(&cut, &whole[..whole.len() / 3]).unwrap();
assert!(make_product(&spec(&cut), None).is_err());
// a model that gives the wrong number of values is an error from the model, not a crash
struct Wrong;
impl TileRunner for Wrong {
fn run(&self, _: &[f32]) -> Result<Vec<f32>, lightcraft_denoise::Error> {
Ok(vec![0.0; 7])
}
}
impl Model for Wrong {
fn runner(&self, _: RunOn, threads: usize) -> (&dyn TileRunner, usize) {
(self, threads)
}
fn self_test(&self, _: RunOn) -> Result<Value, String> {
Err("no".into())
}
}
let good = dir.join("good.dng");
std::fs::write(&good, &whole).unwrap();
let mut s = spec(&good);
s.loader = Arc::new(|_, _| Ok(Arc::new(Wrong) as Arc<dyn Model>));
match make_product(&s, None) {
Err(MakeError::Failed(why)) => assert!(why.contains("7 numbers"), "{why}"),
other => panic!("{other:?}"),
}
assert!(!dir.join("p.lcdn").exists(), "a failed job leaves no product");
// a job told to stop stops before doing anything
let progress = crate::denoise::Progress::default();
progress.cancel_now();
assert_eq!(make_product(&spec(&good), Some(&progress)), Err(MakeError::Cancelled));
// a Bayer layout of the other kinds is denoised too, and a product that exists is not made again
for cfa in ["BGGR", "GRBG", "GBRG"] {
let f = dir.join(format!("{cfa}.dng"));
std::fs::write(&f, ramp_dng(192, 160, cfa)).unwrap();
let mut s = spec(&f);
s.product = dir.join(format!("{cfa}.lcdn"));
assert_eq!(make_product(&s, None), Ok(true), "{cfa}");
assert_eq!(make_product(&s, None), Ok(false), "{cfa} is already there");
}
let _ = std::fs::remove_dir_all(&dir);
}
#[test]
fn two_threads_making_the_same_picture_make_it_once() {
let dir = temp_dir("twice");
let f = dir.join("a.dng");
std::fs::write(&f, ramp_dng(192, 160, "RGGB")).unwrap();
let runs = Arc::new(AtomicUsize::new(0));
let spec = crate::denoise::JobSpec {
path: f.to_string_lossy().into_owned(),
read: None,
product: dir.join("p.lcdn"),
key: "k".into(),
manifest: manifest("stand-in"),
onnx: dir.join("model.onnx"),
model: Default::default(),
loader: dim_loader(&runs),
parallel: 2,
run_on: RunOn::Auto,
};
let made: Vec<bool> = std::thread::scope(|sc| {
let hs: Vec<_> = (0..4).map(|_| sc.spawn(|| crate::denoise::make_product(&spec, None).unwrap())).collect();
hs.into_iter().map(|h| h.join().unwrap()).collect()
});
assert_eq!(made.iter().filter(|m| **m).count(), 1, "{made:?}");
assert_eq!(runs.load(Ordering::Relaxed), 4, "one photo's four tiles, once");
assert!(lightcraft_denoise::product::is_current(&spec.product, "k"));
let _ = std::fs::remove_dir_all(&dir);
}
#[test]
fn the_cache_is_kept_within_its_limit_and_what_photos_use_goes_last() {
let mut x = setup("evict", true);
let id = photo(&x.s);
set_amount(&mut x.s, 100.0);
pump_until_ready(&mut x.s, 1);
let used = x.dir.join("library").join("denoise");
// two older pictures nobody uses
for (i, name) in ["aaaa", "bbbb"].iter().enumerate() {
let f = used.join(format!("{name}.lcdn"));
std::fs::write(&f, vec![7u8; 1000]).unwrap();
let t = std::time::SystemTime::now() - std::time::Duration::from_secs(1000 - i as u64);
std::fs::File::options().write(true).open(&f).unwrap().set_modified(t).unwrap();
}
let size_of = |p: &Path| std::fs::metadata(p).unwrap().len();
let mine: PathBuf = std::fs::read_dir(&used).unwrap().flatten().map(|e| e.path()).find(|p| p.file_stem().is_some_and(|s| s.len() == 32)).unwrap();
// room for the photo's own picture and one stray: the oldest stray goes, then the other
x.s.denoise_evict_to(size_of(&mine) + 1500, None);
assert!(!used.join("aaaa.lcdn").exists() && used.join("bbbb.lcdn").exists() && mine.exists());
x.s.denoise_evict_to(size_of(&mine) + 10, None);
assert!(!used.join("bbbb.lcdn").exists() && mine.exists());
assert_eq!(x.s.denoise_photo_state(id), PhotoState::Ready);
// too small for even that: the photo's own picture is the last thing to go, and the photo knows
x.s.denoise_evict_to(10, None);
assert!(!mine.exists());
assert_ne!(x.s.denoise_photo_state(id), PhotoState::Ready);
// …and it is made again, because the photo still has its amount
pump_until_ready(&mut x.s, 1);
assert!(mine.exists());
// clearing deletes it all
let r = x.s.execute("denoise.clear", &json!({})).unwrap();
assert_eq!(r["deleted"], 1, "{r}");
assert_eq!(std::fs::read_dir(&used).unwrap().flatten().count(), 0);
let _ = std::fs::remove_dir_all(&x.dir);
}
#[test]
fn settings_are_checked_saved_and_a_paused_pump_starts_nothing() {
let mut x = setup("settings", true);
let r = x.s.execute("denoise.settings", &json!({"auto": false, "cacheGb": 5, "threads": 3})).unwrap();
assert_eq!((r["auto"].clone(), r["cacheGb"].clone(), r["threads"].clone()), (json!(false), json!(5), json!(3)), "{r}");
for bad in [
json!({"cacheGb": 0}),
json!({"cacheGb": "big"}),
json!({"threads": 0}),
json!({"threads": 1000}),
json!({"auto": "yes"}),
json!({"runOn": "fastest"}),
json!({"runOn": true}),
] {
assert!(x.s.execute("denoise.settings", &bad).is_err(), "{bad}");
}
assert_eq!(x.s.execute("denoise.models.list", &json!({})).unwrap()["cacheGb"], 5, "a refused change changes nothing");
// the faster of the card and the processor unless the user says otherwise, and the interface can read the choice back
assert_eq!(x.s.execute("denoise.models.list", &json!({})).unwrap()["runOn"], "auto");
assert_eq!(x.s.execute("denoise.settings", &json!({"runOn": "cpu"})).unwrap()["runOn"], "cpu");
assert_eq!(x.s.execute("denoise.models.list", &json!({})).unwrap()["runOn"], "cpu");
let status = x.s.execute("denoise.status", &json!({})).unwrap();
assert_eq!((status["runOn"].clone(), status["device"]["kind"].clone()), (json!("cpu"), json!("none")), "no model is loaded yet: {status}");
// the setting before `runOn` still reads as what it meant
let models = x.s.denoise.models_dir.clone().unwrap();
let mut st = crate::denoise::read_settings(&models);
(st.run_on, st.gpu) = (None, Some(false));
crate::denoise::write_settings(&models, &st).unwrap();
assert_eq!(x.s.execute("denoise.models.list", &json!({})).unwrap()["runOn"], "cpu");
// choosing where it runs lets the card be tried again after a set-up that closed LightCraft
let onnx = crate::denoise::installed_models(&models).first().unwrap().onnx.clone();
let marker = onnx.with_file_name(crate::denoise::GPU_SETUP_MARKER);
std::fs::write(&marker, "setting up").unwrap();
assert_eq!(x.s.execute("denoise.settings", &json!({"runOn": "auto"})).unwrap()["runOn"], "auto");
assert!(!marker.exists(), "the crash is forgotten");
assert_eq!(crate::denoise::read_settings(&models).gpu, None, "the old setting is not kept beside the new one");
// with auto off nothing is started by itself, and the photo is made when asked
let id = photo(&x.s);
set_amount(&mut x.s, 100.0);
for _ in 0..3 {
x.s.execute("denoise.pump", &json!({"pace": "full"})).unwrap();
}
assert_eq!(x.s.denoise_photo_state(id), PhotoState::Idle);
x.s.execute("denoise.settings", &json!({"auto": true})).unwrap();
// paused: it is queued but nothing starts
for _ in 0..3 {
let r = x.s.execute("denoise.pump", &json!({"pace": "pause"})).unwrap();
assert_eq!(r["running"], Value::Null);
}
assert!(matches!(x.s.denoise_photo_state(id), PhotoState::Queued { .. }), "{:?}", x.s.denoise_photo_state(id));
// …and cancelling forgets it
assert_eq!(x.s.execute("denoise.cancel", &json!({})).unwrap()["cancelled"], 1);
assert_eq!(x.s.denoise_photo_state(id), PhotoState::Idle);
// when the pace allows, it is made
x.s.execute("denoise.queue", &json!({"scope": "withAmount"})).unwrap();
pump_until_ready(&mut x.s, 1);
let _ = std::fs::remove_dir_all(&x.dir);
}
#[test]
fn a_photo_whose_file_changed_has_no_picture_until_it_is_made_again() {
let mut x = setup("changed", true);
let id = photo(&x.s);
set_amount(&mut x.s, 100.0);
pump_until_ready(&mut x.s, 1);
// the photo's content is something else now (a replaced file): its old picture is not its picture
let p = x.s.catalog.photo(id).unwrap().clone();
x.s.commit(
"test",
lightcraft_catalog::Op::SetContent {
id,
width: p.width,
height: p.height,
file_size: p.file_size + 1,
content_hash: Some("ffffffffffffffffffffffffffffffff".into()),
preview_only: None,
},
)
.unwrap();
x.s.denoise_pump(crate::denoise::Pace::Pause);
assert_ne!(x.s.denoise_photo_state(id), PhotoState::Ready);
// the old picture is not used by a render either
assert_eq!(x.s.media.denoise_salt(id, &x.s.catalog.photo(id).unwrap().develop), 0);
let _ = std::fs::remove_dir_all(&x.dir);
}
#[test]
fn a_photo_waits_for_its_turn_unless_pictures_are_not_made_on_their_own_or_have_nowhere_to_go() {
let mut x = setup("waits", true);
let id = photo(&x.s);
set_amount(&mut x.s, 80.0);
// automatic off: the open photo is left alone until it is asked for
x.s.execute("denoise.settings", &json!({"auto": false})).unwrap();
assert!(!x.s.denoise_auto());
x.s.execute("denoise.pump", &json!({"pace": "full"})).unwrap();
assert_eq!(x.s.denoise_photo_state(id), PhotoState::Idle);
assert!(!x.s.denoise_busy());
x.s.execute("denoise.queue", &json!({"ids": [id.0]})).unwrap();
assert!(x.s.denoise_busy(), "queued with a model: the headless runs wait for it");
pump_until_ready(&mut x.s, 1);
assert_eq!(x.s.denoise_photo_state(id), PhotoState::Ready);
assert!(!x.s.denoise_busy());
// a session whose library is not on disk has nowhere to keep it: the photo says so instead of waiting for ever
x.s.denoise.settings.auto = Some(true);
if let Some(l) = x.s.library.as_mut() {
l.on_disk = false;
}
match x.s.denoise_photo_state(id) {
PhotoState::Failed { why, unsupported } => assert!(unsupported && why.contains("library folder"), "{why}"),
other => panic!("{other:?}"),
}
let _ = std::fs::remove_dir_all(&x.dir);
}
#[test]
fn the_denoise_switch_validates_arguments_and_accepts_explicit_targets_without_selection() {
let mut x = setup("toggle", false);
let id = photo(&x.s);
for params in [json!({"id": "bad"}), json!({"id": u64::MAX}), json!({"id": id.0, "enabled": "yes"})] {
assert!(x.s.execute("denoise.toggle", &params).is_err());
}
assert!(x.s.execute("denoise.toggle", &json!({"id": id.0, "enabled": true})).is_err());
assert_eq!(x.s.develop_of(id).unwrap().enhance.denoise, 0.0);
let model = model_files(&x.dir, "toggle-model");
x.s.execute("denoise.models.install", &json!({"path": model.to_string_lossy(), "acknowledged": true})).unwrap();
x.s.selection = Default::default();
let undo = x.s.undo.len();
let r = x.s.execute("denoise.toggle", &json!({"id": id.0, "enabled": true})).unwrap();
assert_eq!(r["amount"], 50.0);
assert_eq!(x.s.undo.len(), undo + 1);
x.s.execute("denoise.toggle", &json!({"id": id.0, "enabled": false})).unwrap();
assert_eq!(x.s.develop_of(id).unwrap().denoise_amount(), 0.0);
// the switch is its own setting: flipping it keeps the Amount (so it compares with and without), and an Amount of 0
// does not switch it off
x.s.execute("library.select", &json!({"ids": [id.0]})).unwrap();
x.s.execute("denoise.toggle", &json!({"id": id.0, "enabled": true})).unwrap();
x.s.execute("develop.set", &json!({"control": "enhance.denoise", "value": 80.0})).unwrap();
let off = x.s.execute("denoise.toggle", &json!({"id": id.0, "enabled": false})).unwrap();
assert_eq!(off["amount"], 80.0);
let d = x.s.develop_of(id).unwrap();
assert!(!d.enhance.denoise_enabled() && d.denoise_amount() == 0.0);
assert_eq!(x.s.execute("denoise.toggle", &json!({"id": id.0, "enabled": true})).unwrap()["amount"], 80.0);
assert!((x.s.develop_of(id).unwrap().denoise_amount() - 0.8).abs() < 1e-6);
x.s.execute("develop.set", &json!({"control": "enhance.denoise", "value": 0.0})).unwrap();
assert!(x.s.develop_of(id).unwrap().enhance.denoise_enabled(), "sliding to 0 leaves it on");
assert_eq!(x.s.execute("denoise.toggle", &json!({"id": id.0, "enabled": true})).unwrap()["amount"], 0.0);
// with no `enabled`, the toggle flips what the switch says
assert_eq!(x.s.execute("denoise.toggle", &json!({"id": id.0})).unwrap()["enabled"], false);
drop(x.s);
let _ = std::fs::remove_dir_all(x.dir);
}

9
crates/engine/src/tests_xmp.rs
View File

@ -335,10 +335,15 @@ fn synthetic_dng(xmp: &str) -> Vec<u8> {
/// A tiny Bayer DNG written by our own DNG writer (optional embedded XMP, camera metadata).
pub(crate) fn synthetic_dng_with(xmp: Option<&str>, metadata: lightcraft_meta::Metadata) -> Vec<u8> {
use lightcraft_raw::*;
let (w, h) = (32usize, 24usize);
let cfa = Cfa::bayer("RGGB").unwrap();
let data: Vec<u16> = (0..w * h).map(|i| 256 + ((i % w) * 300 + (i / w) * 200) as u16).collect();
synthetic_dng_of(w, h, "RGGB", data, xmp, metadata)
}
/// A synthetic Bayer DNG for denoise and cache tests.
pub(crate) fn synthetic_dng_of(w: usize, h: usize, cfa: &str, data: Vec<u16>, xmp: Option<&str>, metadata: lightcraft_meta::Metadata) -> Vec<u8> {
use lightcraft_raw::*;
let cfa = Cfa::bayer(cfa).unwrap();
let raw = RawImage {
format: RawFormat::Dng,
width: w,

101
crates/engine/tests/denoise_cpu.rs Normal file
View File

@ -0,0 +1,101 @@
//! Opt-in same-machine denoise model-stage benchmark on one public Bayer raw.
//! LC_DENOISE_MODEL=<onnx> LC_DENOISE_RAW=<CC0 corpus/raw> cargo test --release
//! -p lightcraft-engine --features denoise --test denoise_cpu -- --ignored --nocapture
//! LC_DENOISE_ONLY selects a file (default arw-sony-a7m3-compressed.arw).
//! The whole model stage includes packing, scale/clip protection, blending and normalization;
//! file decode, model loading and GPU setup are outside the timer. It uses the application's tile pool convention.
#![cfg(feature = "denoise")]
use lightcraft_denoise::{
TileRunner,
bayer::Layout,
run::{Control, Params, denoise_bayer},
runtime::{CpuRunner, check_tile},
};
use lightcraft_raster::Rgb32f;
use std::{path::Path, time::Instant};
fn agree(a: &[f32], b: &[f32]) -> f32 {
assert_eq!(a.len(), b.len());
assert!(a.iter().chain(b).all(|v| v.is_finite()));
let scale = b.iter().fold(1e-6f32, |m, v| m.max(v.abs()));
let error = a.iter().zip(b).fold(0f32, |m, (a, b)| m.max((a - b).abs())) / scale;
assert!(error <= 1e-3, "maximum relative error {error:e}");
error
}
fn tile_time(name: &str, r: &dyn TileRunner, x: &[f32]) {
r.run(x).unwrap();
let mut times: Vec<_> = (0..5)
.map(|_| {
let t = Instant::now();
r.run(x).unwrap();
t.elapsed().as_secs_f64() * 1000.0
})
.collect();
times.sort_by(f64::total_cmp);
println!("{name}, one 512-cell tile, one thread: {:.1} ms median ({:.1}..{:.1})", times[2], times[0], times[4]);
}
fn photo_time(name: &str, r: &dyn TileRunner, raw: &lightcraft_raw::Normalized, layout: Layout, p: &Params) -> Rgb32f {
let pool = rayon::ThreadPoolBuilder::new().num_threads(p.parallel).build().unwrap();
let run = || pool.install(|| denoise_bayer(&raw.data, raw.width, raw.height, layout, r, p, &Control::default())).unwrap();
drop(run()); // warm the workers and their workspace cache
let mut times = Vec::new();
let mut answer = Rgb32f { width: 0, height: 0, data: Vec::new() };
for i in 0..3 {
let t = Instant::now();
answer = run();
let secs = t.elapsed().as_secs_f64();
times.push(secs);
println!("{name}, whole model stage, {} workers, run {i}: {secs:.3} s", p.parallel);
}
times.sort_by(f64::total_cmp);
println!("{name}, whole model stage median {:.3} s", times[1]);
answer
}
#[test]
#[ignore = "needs LC_DENOISE_MODEL and LC_DENOISE_RAW; release timing and real-model accuracy"]
fn denoise_stage_compares_cpu_and_gpu() {
let model = std::env::var_os("LC_DENOISE_MODEL").expect("set LC_DENOISE_MODEL");
let folder = std::env::var_os("LC_DENOISE_RAW").expect("set LC_DENOISE_RAW to public CC0 corpus/raw");
let name = std::env::var("LC_DENOISE_ONLY").unwrap_or_else(|_| "arw-sony-a7m3-compressed.arw".into());
let bytes = std::fs::read(Path::new(&folder).join(&name)).unwrap();
let raw = lightcraft_raw::decode(&bytes).unwrap().normalized().unwrap();
let cfa = raw.cfa.as_ref().unwrap();
assert!(cfa.is_bayer());
let layout = Layout::from_cell([cfa.color_at(0, 0), cfa.color_at(1, 0), cfa.color_at(0, 1), cfa.color_at(1, 1)]).unwrap();
let m = lightcraft_denoise::known::find("rawnind-bayer").unwrap().manifest;
let cpu = CpuRunner::load(Path::new(&model), &m).unwrap();
let threads = std::env::var("LC_DENOISE_PARALLEL")
.ok()
.and_then(|v| v.parse::<usize>().ok())
.unwrap_or_else(|| std::thread::available_parallelism().map_or(1, |n| n.get()).clamp(1, 16));
let p = Params { tile: 512, overlap: 64, gain: m.gain, clip: Some(1.0), parallel: threads };
println!(
"{name}: {}x{}, {:.2} MP, {} tiles, {} CPU workers, {:.1} MB/workspace, {:.1} GMAC/tile",
raw.width,
raw.height,
raw.width as f64 * raw.height as f64 / 1e6,
lightcraft_denoise::run::tile_count(raw.width, raw.height, layout, 512, 64),
threads,
cpu.bytes_per_tile() as f64 / 1e6,
cpu.net().macs(512) as f64 / 1e9
);
let x = check_tile(512);
let want = cpu.run(&x).unwrap();
tile_time("pure Rust", &cpu, &x);
let done = photo_time("pure Rust", &cpu, &raw, layout, &p);
match lightcraft_gpu::nn::runner(cpu.net(), 512) {
Ok(gpu) => {
println!("GPU: {}", gpu.adapter());
println!("CPU/check tile vs GPU: {:e}", agree(&want, &gpu.run(&x).unwrap()));
tile_time("GPU", &gpu, &x);
let gp = Params { parallel: 4, ..p };
let answer = photo_time("GPU", &gpu, &raw, layout, &gp);
let a: Vec<_> = done.data.iter().flatten().copied().collect();
let b: Vec<_> = answer.data.iter().flatten().copied().collect();
println!("CPU/whole photo vs GPU: {:e}", agree(&a, &b));
}
Err(e) => println!("GPU unavailable: {e}"),
}
}

117
crates/engine/tests/denoise_real.rs Normal file
View File

@ -0,0 +1,117 @@
//! Opt-in end-to-end check of AI denoise with the real RawNIND Bayer model, through the engine: install it, make a real
//! raw photo's picture in the background, render and export it with the Amount slider, and see what a file that is not
//! Bayer says.
//!
//! ```text
//! LC_DENOISE_MODEL=<model_bayer.onnx> LC_DENOISE_RAW=<folder with raw files> \
//! cargo test --release -p lightcraft-engine --features denoise --test denoise_real -- --ignored --nocapture
//! ```
//!
//! `LC_DENOISE_RAW` is a folder of CC0 raw files (`cargo xtask corpus --download` → `corpus/raw`); every file in it is
//! tried. Optional: `LC_DENOISE_ONLY` (a file name to try alone), `LC_DENOISE_RUN_ON=cpu|gpu|auto` (where the model runs; `cpu` to compare with the card).
#![cfg(feature = "denoise")]
use std::path::{Path, PathBuf};
use std::time::{Duration, Instant};
use lightcraft_engine::Session;
use serde_json::{Value, json};
fn temp(tag: &str) -> PathBuf {
let d = std::env::temp_dir().join(format!("lc-denoise-real-{tag}-{}", std::process::id()));
let _ = std::fs::remove_dir_all(&d);
std::fs::create_dir_all(&d).unwrap();
d
}
/// Mean absolute second difference of the luma: the fine detail and the noise of a picture.
fn roughness(img: &lightcraft_raster::Rgba8) -> f64 {
let (w, h) = (img.width, img.height);
let luma = |x: usize, y: usize| {
let p = img.data[y * w + x];
0.299 * f64::from(p[0]) + 0.587 * f64::from(p[1]) + 0.114 * f64::from(p[2])
};
let (mut sum, mut n) = (0.0, 0usize);
for y in 1..h - 1 {
for x in 1..w - 1 {
sum += (4.0 * luma(x, y) - luma(x - 1, y) - luma(x + 1, y) - luma(x, y - 1) - luma(x, y + 1)).abs();
n += 1;
}
}
sum / n.max(1) as f64
}
fn files(dir: &Path) -> Vec<PathBuf> {
let only = std::env::var("LC_DENOISE_ONLY").ok();
let mut v: Vec<PathBuf> = std::fs::read_dir(dir)
.unwrap()
.flatten()
.map(|e| e.path())
.filter(|p| p.is_file() && only.as_deref().is_none_or(|o| p.file_name().is_some_and(|n| n.to_string_lossy() == o)))
.collect();
v.sort();
v
}
#[test]
#[ignore = "needs the real model and raw files (see the file's header)"]
fn the_real_model_denoises_real_raws_through_the_engine() {
let (Some(model), Some(raws)) = (std::env::var_os("LC_DENOISE_MODEL"), std::env::var_os("LC_DENOISE_RAW")) else {
panic!("set LC_DENOISE_MODEL and LC_DENOISE_RAW");
};
let dir = temp("e2e");
let mut s = Session::new().with_fs();
s.open_library(dir.join("library"), false).unwrap();
s.set_denoise_models_dir(Some(dir.join("models")));
let t = Instant::now();
let r = s.execute("denoise.models.install", &json!({"path": model.to_string_lossy(), "acknowledged": true})).unwrap();
println!("installed {} in {:.1} s: self-test {}", r["installed"]["id"], t.elapsed().as_secs_f64(), r["installed"]["accepted"]["selfTest"]);
assert_eq!(r["installed"]["id"], "rawnind-bayer");
if let Ok(run_on) = std::env::var("LC_DENOISE_RUN_ON") {
s.execute("denoise.settings", &json!({"runOn": run_on})).unwrap();
}
for file in files(Path::new(&raws)) {
let name = file.file_name().unwrap().to_string_lossy().into_owned();
let imported = s.execute("library.import", &json!({"paths": [file.to_string_lossy()]})).unwrap();
let Some(id) = imported["imported"].as_array().and_then(|a| a.first()).and_then(Value::as_u64) else {
println!("{name}: not imported ({imported})");
continue;
};
s.execute("library.select", &json!({"ids": [id]})).unwrap();
s.execute("develop.set", &json!({"control": "enhance.denoise", "value": 100})).unwrap();
let photo = lightcraft_engine::catalog::PhotoId(id);
let started = Instant::now();
let mut state = Value::Null;
while started.elapsed() < Duration::from_secs(600) {
s.execute("denoise.pump", &json!({"pace": "full"})).unwrap();
state = s.execute("denoise.status", &json!({"id": id})).unwrap()["photo"].clone();
if matches!(state["state"].as_str(), Some("ready" | "failed")) {
break;
}
std::thread::sleep(Duration::from_millis(250));
}
println!("{name}: {} in {:.1} s", state, started.elapsed().as_secs_f64());
if state["state"] == "failed" {
// a file that is not a Bayer raw says why, and its export is the plain one
assert_eq!(state["unsupported"], true, "{name}: {state}");
continue;
}
assert_eq!(state["state"], "ready", "{name}: {state}");
let denoised = s.render_now(photo, 1600, 1600).unwrap().image;
s.execute("develop.set", &json!({"control": "enhance.denoise", "value": 0})).unwrap();
let plain = s.render_now(photo, 1600, 1600).unwrap().image;
let (a, b) = (roughness(&plain), roughness(&denoised));
println!("{name}: roughness at 1600 px: plain {a:.3}, denoised {b:.3} ({:+.1} %)", 100.0 * (b - a) / a);
assert!(b < a, "{name}: the denoised picture is not smoother");
// the export is made from the same picture, without asking the queue
s.execute("develop.set", &json!({"control": "enhance.denoise", "value": 100})).unwrap();
let o = lightcraft_engine::export::ExportOptions::from_json(&json!({"format": "jpeg", "width": 1200, "height": 1200}));
let e = lightcraft_engine::export::export_photo(&mut s, photo, &o, 1).unwrap();
assert!(e.bytes.len() > 1000);
}
let status = s.execute("denoise.status", &json!({})).unwrap();
println!("device: {}", status["device"]);
println!("cache: {}", status["cache"]);
let _ = std::fs::remove_dir_all(&dir);
}

5
crates/gpu/Cargo.toml
View File

@ -9,6 +9,7 @@ description = "LightCraft GPU develop pipeline: wgpu compute (WGSL) twin of the
workspace = true
[dependencies]
lightcraft-denoise = { workspace = true, optional = true }
lightcraft-geom = { workspace = true }
lightcraft-color = { workspace = true }
lightcraft-raster = { workspace = true }
@ -23,5 +24,9 @@ wgpu = { workspace = true }
pollster = { workspace = true }
bytemuck = { workspace = true }
[features]
default = []
denoise = ["dep:lightcraft-denoise", "lightcraft-denoise/runtime"]
[dev-dependencies]
lightcraft-scenes = { workspace = true }

38
crates/gpu/src/backend.rs
View File

@ -121,6 +121,30 @@ pub fn window_backends() -> Backends {
resolve(choice, default, wgpu::Instance::enabled_backend_features()).unwrap_or(default)
}
/// The DX12 shader compiler: FXC (`d3dcompiler_47.dll`, part of Windows) unless
/// `WGPU_DX12_COMPILER` (`wgpu`, wgpu's own variable: `fxc`, `dxc`, `auto`…) names another.
///
/// Issue #471: wgpu's default (`Auto`) loads whichever `dxcompiler.dll` the DLL search path finds
/// first — LightCraft ships none, so it is some other program's (an SDK's, a folder on `PATH`). A
/// copy without its `dxil.dll` beside it warns that the DXIL is unsigned; wgpu takes the warning for
/// a compile error, its own validation pipelines fail, the device is lost and the window never
/// opens. FXC is always there and compiles everything LightCraft's shaders need.
pub fn dx12_compiler(env: Option<&str>) -> wgpu::Dx12Compiler {
let Some(v) = env else { return wgpu::Dx12Compiler::Fxc };
v.parse().unwrap_or_else(|e| {
log::warn!("gpu: WGPU_DX12_COMPILER={v} ignored: {e}");
wgpu::Dx12Compiler::Fxc
})
}
/// wgpu's backend options for every instance LightCraft creates — the window's and the compute
/// devices': wgpu's environment variables, with the DX12 compiler from [`dx12_compiler`].
pub fn backend_options() -> wgpu::BackendOptions {
let mut o = wgpu::BackendOptions::from_env_or_default();
o.dx12.shader_compiler = dx12_compiler(std::env::var("WGPU_DX12_COMPILER").ok().as_deref());
o
}
/// `LIGHTCRAFT_GPU_BACKEND=off`.
pub(crate) fn env_off() -> bool {
env_choice() == BackendChoice::Off
@ -211,6 +235,20 @@ mod tests {
assert!(default_window_backends("linux").contains(Backends::VULKAN | Backends::GL));
}
/// Issue #471: DX12 shaders compile with FXC, never a `dxcompiler.dll` found on the search path,
/// unless `WGPU_DX12_COMPILER` asks for it.
#[test]
fn dx12_compiles_with_fxc_unless_asked() {
assert!(matches!(dx12_compiler(None), wgpu::Dx12Compiler::Fxc));
assert!(matches!(dx12_compiler(Some("bogus")), wgpu::Dx12Compiler::Fxc));
assert!(matches!(dx12_compiler(Some("FXC")), wgpu::Dx12Compiler::Fxc));
assert!(matches!(dx12_compiler(Some("dxc")), wgpu::Dx12Compiler::DynamicDxc { .. }));
assert!(matches!(dx12_compiler(Some("auto")), wgpu::Dx12Compiler::Auto));
if std::env::var_os("WGPU_DX12_COMPILER").is_none() {
assert!(matches!(backend_options().dx12.shader_compiler, wgpu::Dx12Compiler::Fxc));
}
}
#[test]
fn resolves_against_the_build() {
let compiled = Backends::DX12 | Backends::VULKAN;

2
crates/gpu/src/ctx.rs
View File

@ -282,6 +282,8 @@ impl Gpu {
let mut desc = wgpu::InstanceDescriptor::new_without_display_handle();
// only these drivers are loaded (`crate::backend`: never Vulkan on Windows unless asked)
desc.backends = backends;
// DX12 shaders compile with FXC (issue #471)
desc.backend_options = crate::backend::backend_options();
let instance = wgpu::Instance::new(desc);
let adapter = pollster::block_on(
instance.request_adapter(&wgpu::RequestAdapterOptions { power_preference: wgpu::PowerPreference::HighPerformance, ..Default::default() }),

12
crates/gpu/src/lib.rs
View File

@ -29,6 +29,8 @@ use lightcraft_raster::Rgb32f;
pub mod backend;
#[cfg(not(target_arch = "wasm32"))]
mod ctx;
#[cfg(all(feature = "denoise", not(target_arch = "wasm32")))]
pub mod nn;
#[cfg(not(target_arch = "wasm32"))]
mod params;
#[cfg(not(target_arch = "wasm32"))]
@ -145,6 +147,16 @@ pub fn enabled() -> bool {
ENABLED.load(Ordering::Relaxed) && !BROKEN.load(Ordering::Relaxed) && !env_disabled()
}
/// Why GPU compute that has a device of its own (denoise) is switched off by the user's settings, or `None`: the
/// environment switches and the rendering preference, but not a render-device failure (the devices are separate).
#[cfg(all(feature = "denoise", not(target_arch = "wasm32")))]
pub(crate) fn switched_off() -> Option<String> {
if env_disabled() {
return Some(if backend::env_off() { "disabled by LIGHTCRAFT_GPU_BACKEND=off" } else { "disabled by LIGHTCRAFT_GPU=0" }.into());
}
(!ENABLED.load(Ordering::Relaxed)).then(|| "disabled by the GPU rendering preference (app.gpu)".into())
}
/// `LIGHTCRAFT_GPU=0` (or `LIGHTCRAFT_GPU_BACKEND=off`): no GPU for the whole process.
fn env_disabled() -> bool {
static OFF: std::sync::OnceLock<bool> = std::sync::OnceLock::new();

1050
crates/gpu/src/nn.rs Normal file
View File

File diff suppressed because it is too large Load Diff

162
crates/gpu/src/wgsl/nn_conv.wgsl Normal file
View File

@ -0,0 +1,162 @@
// Convolution for the denoise networks, as an implicit matrix product: every output pixel is a row, every input
// channel of every kernel tap is a column of K, and the output channels are the columns of N.
//
// out[m][n] = bias[n] + sum_k A[m][k] * W[k][n] A[m][k] = input pixel (m + tap offset), channel ci
//
// Tensors are NHWC, channels innermost, read four at a time (every channel count is a multiple of 4). A workgroup of
// 256 threads makes a BM x BN block of the output, 4 x 4 values per thread, and walks K in steps of BK through
// workgroup memory. The source can be two tensors whose channels follow each other (a skip connection's join, so it
// is never made), and a leaky ReLU is applied on the way out. Specialised by the host: BM, BN, BK (BM * BN = 4096).
//
// Output modes:
// 0 NHWC at the input's size.
// 1 a 2x2 stride-2 transposed convolution: the columns are (dy, dx, co) and each lands in the output pixel
// (2y + dy, 2x + dx) of an output twice the size.
// 2 the last 1x1 convolution followed by depth-to-space (CRD): columns 4c + 2dy + dx land in plane c at
// (2y + dy, 2x + dx) of a planar (CHW) output twice the size.
struct P {
h: u32,
w: u32,
c0: u32,
c1: u32,
n: u32,
npad: u32,
k: u32,
mode: u32,
cout: u32,
leaky: u32,
alpha: f32,
pad: u32,
}
@group(0) @binding(0) var<uniform> p: P;
@group(0) @binding(1) var<storage, read> src0: array<vec4<f32>>;
@group(0) @binding(2) var<storage, read> src1: array<vec4<f32>>;
@group(0) @binding(3) var<storage, read> wgt: array<vec4<f32>>;
@group(0) @binding(4) var<storage, read> bias: array<vec4<f32>>;
@group(0) @binding(5) var<storage, read_write> dst: array<f32>;
const BM: u32 = {{BM}}u;
const BN: u32 = {{BN}}u;
const BK: u32 = {{BK}}u;
const TX: u32 = BN / 4u;
const A_ITEMS: u32 = BM * (BK / 4u);
const B_ITEMS: u32 = BK * (BN / 4u);
// A[k][m] four pixels to a vec4, B[k][n] four channels to a vec4
var<workgroup> As: array<vec4<f32>, {{A_VECS}}>;
var<workgroup> Bs: array<vec4<f32>, {{B_VECS}}>;
@compute @workgroup_size(256)
fn main(@builtin(workgroup_id) wg: vec3<u32>, @builtin(local_invocation_index) tid: u32) {
let m0 = wg.x * BM;
let n0 = wg.y * BN;
let tx = tid % TX;
let ty = tid / TX;
let hw = p.h * p.w;
let cin = p.c0 + p.c1;
let taps = p.k * p.k;
let pad = (p.k - 1u) / 2u;
let chunks = cin / BK;
var acc0 = vec4<f32>(0.0);
var acc1 = vec4<f32>(0.0);
var acc2 = vec4<f32>(0.0);
var acc3 = vec4<f32>(0.0);
for (var tap = 0u; tap < taps; tap++) {
let ty_off = i32(tap / p.k) - i32(pad);
let tx_off = i32(tap % p.k) - i32(pad);
for (var chunk = 0u; chunk < chunks; chunk++) {
let ci0 = chunk * BK;
for (var item = tid; item < A_ITEMS; item += 256u) {
let ml = item / (BK / 4u);
let q = item % (BK / 4u);
let m = m0 + ml;
var v = vec4<f32>(0.0);
if (m < hw) {
let y = i32(m / p.w) + ty_off;
let x = i32(m % p.w) + tx_off;
if (y >= 0 && y < i32(p.h) && x >= 0 && x < i32(p.w)) {
let pix = u32(y) * p.w + u32(x);
let ci = ci0 + q * 4u;
if (ci < p.c0) {
v = src0[pix * (p.c0 / 4u) + ci / 4u];
} else {
v = src1[pix * (p.c1 / 4u) + (ci - p.c0) / 4u];
}
}
}
let base = (q * 4u) * (BM / 4u) + ml / 4u;
let lane = ml % 4u;
As[base][lane] = v.x;
As[base + BM / 4u][lane] = v.y;
As[base + 2u * (BM / 4u)][lane] = v.z;
As[base + 3u * (BM / 4u)][lane] = v.w;
}
for (var item = tid; item < B_ITEMS; item += 256u) {
let r = item / (BN / 4u);
let c4 = item % (BN / 4u);
let krow = tap * cin + ci0 + r;
Bs[item] = wgt[krow * (p.npad / 4u) + n0 / 4u + c4];
}
workgroupBarrier();
for (var kk = 0u; kk < BK; kk++) {
let a = As[kk * (BM / 4u) + ty];
let b = Bs[kk * TX + tx];
acc0 += a.x * b;
acc1 += a.y * b;
acc2 += a.z * b;
acc3 += a.w * b;
}
workgroupBarrier();
}
}
let n = n0 + tx * 4u;
if (n >= p.n) {
return;
}
let bv = bias[n / 4u];
for (var i = 0u; i < 4u; i++) {
let m = m0 + ty * 4u + i;
if (m >= hw) {
continue;
}
var v = bv;
if (i == 0u) { v += acc0; } else if (i == 1u) { v += acc1; } else if (i == 2u) { v += acc2; } else { v += acc3; }
if (p.leaky != 0u) {
v = select(v * p.alpha, v, v >= vec4<f32>(0.0));
}
if (p.mode == 0u) {
let o = m * p.n + n;
dst[o] = v.x;
dst[o + 1u] = v.y;
dst[o + 2u] = v.z;
dst[o + 3u] = v.w;
} else if (p.mode == 1u) {
let q = n / p.cout;
let co = n % p.cout;
let y = m / p.w;
let x = m % p.w;
let opix = (2u * y + q / 2u) * (2u * p.w) + 2u * x + q % 2u;
let o = opix * p.cout + co;
dst[o] = v.x;
dst[o + 1u] = v.y;
dst[o + 2u] = v.z;
dst[o + 3u] = v.w;
} else {
let c = n / 4u;
let y = m / p.w;
let x = m % p.w;
let ow = 2u * p.w;
let plane = c * (2u * p.h) * ow;
let o = plane + (2u * y) * ow + 2u * x;
dst[o] = v.x;
dst[o + 1u] = v.y;
dst[o + ow] = v.z;
dst[o + ow + 1u] = v.w;
}
}
}

28
crates/gpu/src/wgsl/nn_pool.wgsl Normal file
View File

@ -0,0 +1,28 @@
// 2x2 maximum with stride 2 over an NHWC tensor read and written four channels at a time.
struct P {
h: u32,
w: u32,
c4: u32,
total: u32,
}
@group(0) @binding(0) var<uniform> p: P;
@group(0) @binding(1) var<storage, read> src: array<vec4<f32>>;
@group(0) @binding(2) var<storage, read_write> dst: array<vec4<f32>>;
@compute @workgroup_size(64)
fn main(@builtin(global_invocation_id) gid: vec3<u32>, @builtin(num_workgroups) nwg: vec3<u32>) {
let o = gid.x + gid.y * nwg.x * 64u;
if (o >= p.total) {
return;
}
let c = o % p.c4;
let pix = o / p.c4;
let ow = p.w / 2u;
let oy = pix / ow;
let ox = pix % ow;
let i00 = ((2u * oy) * p.w + 2u * ox) * p.c4 + c;
let i10 = i00 + p.w * p.c4;
dst[o] = max(max(src[i00], src[i00 + p.c4]), max(src[i10], src[i10 + p.c4]));
}

4
crates/mcp/src/headless.rs
View File

@ -31,7 +31,7 @@ impl Drop for Headless {
impl Default for Headless {
fn default() -> Self {
Self::new(Session::new().with_fs().with_default_face_models())
Self::new(Session::new().with_fs().with_default_denoise_models().with_default_face_models())
}
}
@ -42,7 +42,7 @@ impl Headless {
/// A headless session with the procedurally generated demo library.
pub fn demo() -> Self {
Self::new(Session::with_demo().with_fs().with_default_face_models())
Self::new(Session::with_demo().with_fs().with_default_denoise_models().with_default_face_models())
}
fn photo_or_active(&self, p: &Value) -> Result<PhotoId, String> {

87
crates/raw/src/lib.rs
View File

@ -165,6 +165,10 @@ pub fn probe(bytes: &[u8]) -> Option<RawFormat> {
if make.starts_with("PENTAX") || make.starts_with("RICOH") {
return Some(RawFormat::Pef);
}
// a Samsung-branded body built on a Pentax design writes a Pentax-style maker note (the note's own magic)
if make.starts_with("SAMSUNG") && has_pentax_maker_note(&t, bytes) {
return Some(RawFormat::Pef);
}
if make.starts_with("SAMSUNG") {
return Some(RawFormat::Srw);
}
@ -174,6 +178,12 @@ pub fn probe(bytes: &[u8]) -> Option<RawFormat> {
None
}
/// Whether the Exif maker note starts with the signature of the Pentax layouts (`AOC\0` or `PENTAX \0`).
fn has_pentax_maker_note(t: &Tiff, bytes: &[u8]) -> bool {
let Some(e) = t.exif().and_then(|e| e.get(lightcraft_tiff::tags::MAKER_NOTE)) else { return false };
bytes.get(e.offset as usize..).is_some_and(|n| n.starts_with(b"AOC\0") || n.starts_with(b"PENTAX \0"))
}
/// Whether some IFD is marked as raw: CFA photometric, or a raw-only compression value (99 is not a
/// registered TIFF compression; Leaf MOS files use it for their tiled 16-bit lossless-JPEG raw).
fn has_raw_ifd(t: &Tiff) -> bool {
@ -689,14 +699,27 @@ impl RawImage {
/// The result is camera RGB (not white balanced), white level = 1.0, not oriented.
pub fn develop(&self, method: Method) -> Result<Rgb32f> {
let n = self.normalized()?;
let mut rgb = demosaic(&n, method);
let rgb = demosaic(&n, method);
drop(n);
Ok(self.finish_demosaiced(rgb))
}
/// The last steps of [`Self::develop`] for camera RGB made some other way (a denoiser that demosaics too) from
/// [`Self::normalized`]'s mosaic: apply `OpcodeList3`, then the default crop.
pub fn finish_demosaiced(&self, mut rgb: Rgb32f) -> Rgb32f {
opcodes::apply_list3(&self.opcodes.list3, &mut rgb);
let c = self.crop.clipped(rgb.width, rgb.height);
if c.width == 0 || c.height == 0 || (c.x == 0 && c.y == 0 && c.width == rgb.width && c.height == rgb.height) {
return Ok(rgb);
let c = self.develop_crop(rgb.width, rgb.height);
if c.x == 0 && c.y == 0 && c.width == rgb.width && c.height == rgb.height {
return rgb;
}
Ok(rgb.into_crop(c.x, c.y, c.width, c.height))
rgb.into_crop(c.x, c.y, c.width, c.height)
}
/// The part of a `width × height` demosaiced picture (the active area) that [`Self::develop`] keeps: the default
/// crop, or all of it when there is none.
pub fn develop_crop(&self, width: usize, height: usize) -> Rect {
let c = self.crop.clipped(width, height);
if c.width == 0 || c.height == 0 { Rect::new(0, 0, width, height) } else { c }
}
}
@ -729,6 +752,43 @@ mod tests {
}
}
#[test]
fn a_picture_made_elsewhere_is_finished_like_develop_does() {
let (w, h) = (12usize, 10usize);
let raw = RawImage {
format: RawFormat::Dng,
width: w,
height: h,
cpp: 1,
data: RawData::U16((0..w * h).map(|i| (100 + (i * 37) % 900) as u16).collect()),
cfa: Cfa::bayer("RGGB"),
bits: 12,
black: BlackLevel::uniform(64.0),
white: vec![1023.0],
active_area: Rect::new(2, 2, 8, 6),
crop: Rect::new(1, 1, 5, 4),
orientation: Orientation::Normal,
color: ColorData::default(),
wb_multipliers: None,
linearized: false,
opcodes: OpcodeLists::default(),
metadata: Metadata::default(),
};
let direct = raw.develop(Method::Bilinear).unwrap();
let by_hand = raw.finish_demosaiced(demosaic(&raw.normalized().unwrap(), Method::Bilinear));
assert_eq!((direct.width, direct.height), (5, 4));
assert_eq!(direct, by_hand);
// the crop is what develop keeps; no usable crop keeps everything
assert_eq!(raw.develop_crop(8, 6), Rect::new(1, 1, 5, 4));
let mut whole = raw.clone();
whole.crop = Rect::default();
assert_eq!(whole.develop_crop(8, 6), Rect::new(0, 0, 8, 6));
assert_eq!(whole.develop(Method::Bilinear).unwrap().width, 8);
// a crop that pokes out is clipped to the picture
whole.crop = Rect::new(6, 4, 10, 10);
assert_eq!(whole.develop_crop(8, 6), Rect::new(6, 4, 2, 2));
}
#[test]
fn black_levels() {
let b = BlackLevel { repeat_rows: 2, repeat_cols: 2, values: vec![1.0, 2.0, 3.0, 4.0], delta_h: vec![0.5, 0.0], delta_v: vec![0.0, 10.0] };
@ -869,6 +929,23 @@ mod tests {
assert_eq!(probe(&write(&[ifd])), Some(RawFormat::OtherTiff));
}
/// A Samsung-branded body whose maker note has a Pentax layout is read by the Pentax reader; another
/// Samsung file stays with the Samsung format.
#[test]
fn samsung_make_with_a_pentax_maker_note_is_pef() {
let with_note = |note: &[u8]| {
let mut ifd = rgb_ifd(16, 12);
ifd.set(t::MAKE, Value::Ascii("SAMSUNG TECHWIN".into()));
let mut exif = IfdBuilder::new();
exif.set(t::MAKER_NOTE, Value::Undefined(note.to_vec()));
ifd.set_child(t::EXIF_IFD, exif);
write(&[ifd])
};
assert_eq!(probe(&with_note(b"AOC\0MM\0\x01\0\0\0\0\0\0")), Some(RawFormat::Pef));
assert_eq!(probe(&with_note(b"PENTAX \0MM\0\0\0\0\0\0\0")), Some(RawFormat::Pef));
assert_eq!(probe(&with_note(b"STMN100\0\0\0\0\0\0\0\0\0")), Some(RawFormat::Srw));
}
#[test]
fn probe_rejects_junk() {
assert_eq!(probe(b""), None);

97
crates/raw/src/ljpeg.rs
View File

@ -369,12 +369,25 @@ fn check_subsampled(h: &Header) -> Result<usize, RawError> {
Ok(vertical)
}
/// Sony M/S tiles: horizontal prediction, no restart markers. Keep subsampled planes separate;
/// expanding them into a mosaic would silently corrupt the existing CFA callers.
pub(crate) fn frame_info_subsampled(d: &[u8]) -> Result<(usize, usize), RawError> {
/// Sony M/S tiles and Canon sRAW / mRAW: horizontal prediction, no restart markers. Keep subsampled planes
/// separate; expanding them into a mosaic would silently corrupt the existing CFA callers.
/// Returns (width, height, vertical subsampling of the chroma planes).
pub(crate) fn frame_info_subsampled(d: &[u8]) -> Result<(usize, usize, usize), RawError> {
let h = parse_header(d)?;
check_subsampled(&h)?;
Ok((h.width, h.height))
let vertical = check_subsampled(&h)?;
Ok((h.width, h.height, vertical))
}
/// How the first-column prediction and the running predictor of a subsampled frame are chosen.
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub(crate) enum Prediction {
/// Sony: the left neighbour in the plane; at the first column the sample above, except that a top luma
/// row of an MCU row reads the previous MCU row's top luma sample.
Geometric,
/// Canon: every component is its own one-dimensional chain in decoding order (the previous sample of the
/// same component, so the second luma row of a 4:2:0 MCU continues from the end of the first); the first
/// sample of each MCU row starts from the first sample of the previous MCU row.
Sequential,
}
pub(crate) struct FrameSubsampled {
@ -385,11 +398,13 @@ pub(crate) struct FrameSubsampled {
pub planes: [Vec<u16>; 3],
}
/// Sony's 4:2:0 / 4:2:2 LJ92 variants: T.81 MCU ordering and differences, modulo 2^16.
/// At the first column, the top luma row predicts from the previous MCU row's top
/// The 4:2:0 / 4:2:2 LJ92 variants of Sony (M/S ARW) and Canon (sRAW / mRAW): T.81 MCU ordering and differences,
/// modulo 2^16. [`Prediction`] selects the predictor each maker uses.
///
/// Sony: at the first column, the top luma row predicts from the previous MCU row's top
/// sample (two image rows above), and the bottom row from the current top sample.
/// Observed black-box: using the immediately preceding image row introduces tile seams.
pub(crate) fn decode_subsampled(d: &[u8], max_samples: usize) -> Result<FrameSubsampled, RawError> {
pub(crate) fn decode_subsampled(d: &[u8], max_samples: usize, prediction: Prediction) -> Result<FrameSubsampled, RawError> {
let h = parse_header(d)?;
let vertical = check_subsampled(&h)?;
let pixels = h.width.checked_mul(h.height).ok_or_else(|| err("frame too large"))?;
@ -410,6 +425,9 @@ pub(crate) fn decode_subsampled(d: &[u8], max_samples: usize) -> Result<FrameSub
let mut planes = [vec![0u16; pixels], vec![0u16; chroma], vec![0u16; chroma]];
let mut br = BitReader::new(entropy);
let init = 1i32 << (h.precision - h.pt - 1);
// Sequential prediction: the last sample of each component, and the first sample of its previous MCU row
let mut last = [init; 3];
let mut row_first = [init; 3];
for my in 0..h.height / vertical {
for mx in 0..h.width / 2 {
for (c, plane) in planes.iter_mut().enumerate() {
@ -420,20 +438,24 @@ pub(crate) fn decode_subsampled(d: &[u8], max_samples: usize) -> Result<FrameSub
for dx in 0..horizontal_factor {
let (x, y) = (mx * horizontal_factor + dx, my * vertical_factor + dy);
let i = y * width + x;
let pred = if x > 0 {
plane[i - 1] as i32
} else if c == 0 && dy == 0 && y >= vertical {
plane[i - vertical * width] as i32
} else if y > 0 {
plane[i - width] as i32
} else {
init
let pred = match prediction {
Prediction::Geometric if x > 0 => plane[i - 1] as i32,
Prediction::Geometric if c == 0 && dy == 0 && y >= vertical => plane[i - vertical * width] as i32,
Prediction::Geometric if y > 0 => plane[i - width] as i32,
Prediction::Geometric => init,
Prediction::Sequential if mx == 0 && dx == 0 && dy == 0 => row_first[c],
Prediction::Sequential => last[c],
};
let ssss = tables[c].decode(&mut br)?;
if ssss > 16 {
return Err(err("invalid difference category"));
}
plane[i] = ((pred + diff_value(&mut br, ssss)) & 0xffff) as u16;
let value = ((pred + diff_value(&mut br, ssss)) & 0xffff) as u16;
plane[i] = value;
last[c] = i32::from(value);
if mx == 0 && dx == 0 && dy == 0 {
row_first[c] = i32::from(value);
}
}
}
}
@ -816,11 +838,13 @@ pub(crate) mod tests {
/// Independent 4×4, 16-bit Sony 4:2:0 stream. Hand-specified differences in MCU order.
pub(crate) fn fixture_420() -> Vec<u8> {
fixture_subsampled(4, 0x22, &[-31768, 1, 1000, 1, -16384, -16284, 1, 1, 1, 1, 1, 2, 100, 1, 1000, 1, 100, 100, 1, 1, 1, 1, 1, 2])
fixture_subsampled(4, 4, 0x22, &[-31768, 1, 1000, 1, -16384, -16284, 1, 1, 1, 1, 1, 2, 100, 1, 1000, 1, 100, 100, 1, 1, 1, 1, 1, 2])
}
fn fixture_subsampled(height: u8, sampling: u8, differences: &[i32]) -> Vec<u8> {
let mut out = vec![0xff, 0xd8, 0xff, 0xc3, 0, 17, 16, 0, height, 0, 4, 3, 1, sampling, 0, 2, 0x11, 0, 3, 0x11, 0];
/// A 16-bit three-component frame of `width × height` pixels with the given luma `sampling` (`0x21` / `0x22`)
/// and one shared 5-bit Huffman table, whose differences are written as given (MCU order).
pub(crate) fn fixture_subsampled(width: u8, height: u8, sampling: u8, differences: &[i32]) -> Vec<u8> {
let mut out = vec![0xff, 0xd8, 0xff, 0xc3, 0, 17, 16, 0, height, 0, width, 3, 1, sampling, 0, 2, 0x11, 0, 3, 0x11, 0];
out.extend_from_slice(&[0xff, 0xc4, 0, 36, 0]);
let mut counts = [0; 16];
counts[4] = 17;
@ -844,28 +868,28 @@ pub(crate) mod tests {
#[test]
fn horizontal_only_subsampling_preserves_rows_and_chroma() {
// Independent 4×2, 16-bit 4:2:2 stream: two luma samples, Cb, Cr per MCU.
let enc = fixture_subsampled(2, 0x21, &[-31768, 1, -16384, -16284, 1, 1, 1, 2, 100, 1, 100, 100, 1, 1, 1, 2]);
let frame = decode_subsampled(&enc, 16).unwrap();
let enc = fixture_subsampled(4, 2, 0x21, &[-31768, 1, -16384, -16284, 1, 1, 1, 2, 100, 1, 100, 100, 1, 1, 1, 2]);
let frame = decode_subsampled(&enc, 16, Prediction::Geometric).unwrap();
assert_eq!((frame.width, frame.height, frame.vertical_subsampling), (4, 2, 1));
assert_eq!(frame.planes[0], [1000, 1001, 1002, 1003, 1100, 1101, 1102, 1103]);
assert_eq!(frame.planes[1], [16384, 16385, 16484, 16485]);
assert_eq!(frame.planes[2], [16484, 16486, 16584, 16586]);
assert!(decode(&enc, 24).is_err());
assert!(matches!(decode_subsampled(&enc, 15), Err(RawError::Limit(_))));
assert!(decode_subsampled(&enc[..enc.len() - 6], 16).is_err());
assert!(matches!(decode_subsampled(&enc, 15, Prediction::Geometric), Err(RawError::Limit(_))));
assert!(decode_subsampled(&enc[..enc.len() - 6], 16, Prediction::Geometric).is_err());
}
#[test]
fn subsampled_mcu_order_preserves_each_plane() {
let enc = fixture_420();
let frame = decode_subsampled(&enc, 24).unwrap();
let frame = decode_subsampled(&enc, 24, Prediction::Geometric).unwrap();
assert_eq!((frame.width, frame.height), (4, 4));
assert_eq!(frame.planes[0], [1000, 1001, 1002, 1003, 2000, 2001, 2002, 2003, 1100, 1101, 1102, 1103, 2100, 2101, 2102, 2103]);
assert_eq!(frame.planes[1], [16384, 16385, 16484, 16485]);
assert_eq!(frame.planes[2], [16484, 16486, 16584, 16586]);
assert!(decode(&enc, 48).is_err()); // CFA API must keep rejecting subsampling.
assert!(matches!(decode_subsampled(&enc, 23), Err(RawError::Limit(_))));
assert!(decode_subsampled(&enc[..enc.len() - 6], 24).is_err());
assert!(matches!(decode_subsampled(&enc, 23, Prediction::Geometric), Err(RawError::Limit(_))));
assert!(decode_subsampled(&enc[..enc.len() - 6], 24, Prediction::Geometric).is_err());
let mut bad = enc.clone();
bad[10] = 3; // odd frame width
assert!(frame_info_subsampled(&bad).is_err());
@ -875,6 +899,25 @@ pub(crate) mod tests {
assert!(frame_info_subsampled(&bad).is_err());
}
#[test]
fn sequential_prediction_chains_each_component_in_decoding_order() {
// Canon sRAW / mRAW: the same stream as `subsampled_mcu_order_preserves_each_plane`, but each component
// continues from its previous sample in decoding order (the second luma row of an MCU follows the first),
// and a new MCU row starts from the first sample of the previous one. Worked out by hand from init 32768.
let enc = fixture_420();
let frame = decode_subsampled(&enc, 24, Prediction::Sequential).unwrap();
assert_eq!((frame.width, frame.height, frame.vertical_subsampling), (4, 4, 2));
assert_eq!(frame.planes[0], [1000, 1001, 2003, 2004, 2001, 2002, 2005, 2006, 1100, 1101, 2103, 2104, 2101, 2102, 2105, 2106]);
assert_eq!(frame.planes[1], [16384, 16385, 16484, 16485]);
assert_eq!(frame.planes[2], [16484, 16486, 16584, 16586]);
// 4:2:2 has one luma row per MCU row, so both predictions agree
let enc = fixture_subsampled(4, 2, 0x21, &[-31768, 1, -16384, -16284, 1, 1, 1, 2, 100, 1, 100, 100, 1, 1, 1, 2]);
assert_eq!(
decode_subsampled(&enc, 16, Prediction::Sequential).unwrap().planes,
decode_subsampled(&enc, 16, Prediction::Geometric).unwrap().planes
);
}
fn noise(n: usize, bits: u32, seed: u64) -> Vec<u16> {
let mut s = seed;
(0..n)

43
crates/raw/src/vendor/arw.rs vendored
View File

@ -338,7 +338,7 @@ fn read_ycbcr_tiles(bytes: &[u8], info: &ImageInfo, raw: &Ifd, mode: Mode) -> Re
.map(|c| {
let src = chunk_bytes(bytes, c).ok_or_else(|| RawError::Corrupt("YCbCr tile outside file".into()))?;
let limit = (c.width as usize).checked_mul(c.height as usize).and_then(|n| n.checked_mul(3)).ok_or(RawError::Limit("tile too large"))?;
let f = ljpeg::decode_subsampled(src, limit)?;
let f = ljpeg::decode_subsampled(src, limit, ljpeg::Prediction::Geometric)?;
if f.width != c.width as usize || f.height != c.height as usize {
return Err(RawError::Corrupt("Sony YCbCr tile dimensions".into()));
}
@ -454,13 +454,17 @@ pub(crate) fn decode(bytes: &[u8], mode: Mode) -> Result<RawImage> {
let linear_rgb = info.compression == 7 && info.photometric == photometric::YCBCR;
let chunks = info.chunks(bytes.len() as u64);
let strip_len: u64 = chunks.iter().map(|c| c.len).sum();
// The row-per-w-bytes (ARW2) layout stores exactly one byte per pixel in a single strip; all 108 such files in the test
// archive measure exactly width x height. A200/A230/A350 files carry a strip 4-23 % larger than that (and A290/A390 one
// smaller), which is a different packing, so they are refused rather than read with the wrong layout.
let one_byte_per_sample = chunks.len() == 1 && strip_len == (w * h) as u64;
let (data, out_bits) = match info.compression {
7 if linear_rgb => (RawData::U16(read_ycbcr_tiles(bytes, &info, raw, mode)?), 14),
32767 if chunks.len() == 1 && strip_len >= (w * h) as u64 && strip_len < (w * h) as u64 * 5 / 4 && mode == Mode::Header => {
32767 if one_byte_per_sample && mode == Mode::Header => {
chunk_bytes(bytes, &chunks[0]).ok_or_else(|| RawError::Corrupt("raw strip outside file".into()))?;
(RawData::U16(Vec::new()), 14)
}
32767 if chunks.len() == 1 && strip_len >= (w * h) as u64 && strip_len < (w * h) as u64 * 5 / 4 => {
32767 if one_byte_per_sample => {
let src = chunk_bytes(bytes, &chunks[0]).ok_or_else(|| RawError::Corrupt("raw strip outside file".into()))?;
let curve = code_curve(&raw.u64s(TONE_CURVE).unwrap_or_else(|| vec![8000, 10400, 12900, 14100]));
let mut data = vec![0u16; w * h];
@ -473,7 +477,7 @@ pub(crate) fn decode(bytes: &[u8], mode: Mode) -> Result<RawImage> {
});
(RawData::U16(data), 14)
}
32767 => return Err(RawError::Unsupported("Sony ARW version 1 / packed compressed variant".into())),
32767 => return Err(RawError::Unsupported("Sony ARW version 1 / packed compressed variant (raw strip is not one byte per pixel)".into())),
7 if is_quad_tiled(bytes, &info) => match mode {
Mode::Full => (RawData::U16(read_quad_tiles(bytes, &info)?), bits),
Mode::Header => {
@ -595,6 +599,37 @@ mod tests {
assert_eq!((developed.width, developed.height), (4, 4));
}
/// A 32767-compressed ARW whose single strip is `strip` bytes for a 32 x 4 image.
fn arw_with_strip(strip: usize) -> Vec<u8> {
use lightcraft_tiff::{IfdBuilder, ImageData, TiffWriter, Value};
let mut raw = IfdBuilder::new();
raw.set(t::MAKE, Value::Ascii("SONY".into()));
raw.set(t::MODEL, Value::Ascii("DSLR-A200".into()));
raw.set(t::IMAGE_WIDTH, Value::Long(vec![32]));
raw.set(t::IMAGE_LENGTH, Value::Long(vec![4]));
raw.set(t::BITS_PER_SAMPLE, Value::Short(vec![12]));
raw.set(t::SAMPLES_PER_PIXEL, Value::Short(vec![1]));
raw.set(t::PHOTOMETRIC, Value::Short(vec![photometric::CFA]));
raw.set(t::COMPRESSION, Value::Short(vec![32767]));
raw.set(TONE_CURVE, Value::Short(vec![8000, 10400, 12900, 14100]));
raw.set_image(ImageData::Strips { rows_per_strip: 4, strips: vec![vec![0u8; strip]] });
TiffWriter::default().write(&[raw]).unwrap()
}
#[test]
fn strip_larger_or_smaller_than_one_byte_per_pixel_is_the_packed_variant() {
for strip in [32 * 4 + 8, 32 * 4 * 5 / 4 - 1, 32 * 4 - 1] {
for mode in [Mode::Header, Mode::Full] {
let err = decode(&arw_with_strip(strip), mode).unwrap_err();
assert!(matches!(err, RawError::Unsupported(ref m) if m.contains("packed compressed variant")), "{strip} {mode:?}: {err:?}");
}
}
for mode in [Mode::Header, Mode::Full] {
let r = decode(&arw_with_strip(32 * 4), mode);
assert!(!matches!(r, Err(RawError::Unsupported(ref m)) if m.contains("packed compressed variant")), "{r:?}");
}
}
/// Encode one 16-value set as an ARW2 block (the paper's scheme, used here to test the decoder).
pub(crate) fn encode_block(v: &[u16; 16]) -> [u8; 16] {
let (imax, &max) = v.iter().enumerate().max_by_key(|(i, x)| (**x, usize::MAX - *i)).unwrap();

12
crates/raw/src/vendor/cr2.rs vendored
View File

@ -17,7 +17,9 @@
//! - `ColorBalance` (maker note `0x4001`): as-shot `RGGB` levels at a model-dependent offset; we probe the known
//! offsets and accept the first plausible quadruple. The array is 16-bit words; some models store it as UNDEFINED
//! bytes (ColorData versions -3 and -4), which are paired up in the maker note's byte order first.
//! - sRAW / mRAW (YCbCr, subsampled) are not supported yet.
//! - sRAW / mRAW (raw IFD tag `0xc6c5` = 4) hold subsampled YCbCr instead of a mosaic: see [`sraw`].
mod sraw;
use super::{black_from_columns, white_from_data};
use crate::{BlackLevel, Cfa, ColorData, Mode, OpcodeLists, RawData, RawError, RawFormat, RawImage, Rect, Result, ljpeg};
@ -27,6 +29,8 @@ use lightcraft_tiff::{ByteOrder, Ifd, Tiff, Value, makernote, tags as t};
const CR2_SLICE: u16 = 0xc640;
const SRAW_TYPE: u16 = 0xc6c5;
/// `SRAW_TYPE` value of the YCbCr frames of sRAW / mRAW (IFD2's preview image carries 3).
const SRAW_YCC: u32 = 4;
const CR2_CFA_PATTERN: u16 = 0xc5e0;
const SENSOR_INFO: u16 = 0x00e0;
const COLOR_BALANCE: u16 = 0x4001;
@ -79,8 +83,10 @@ pub(crate) fn decode(bytes: &[u8], mode: Mode) -> Result<RawImage> {
let tiff = Tiff::parse(bytes)?;
let ifd0 = &tiff.ifds[0];
let raw = raw_ifd(&tiff).ok_or_else(|| RawError::Corrupt("CR2 without raw IFD".into()))?;
if raw.u32(SRAW_TYPE).is_some_and(|v| v != 1) {
return Err(RawError::Unsupported("Canon sRAW/mRAW".into()));
match raw.u32(SRAW_TYPE) {
Some(SRAW_YCC) => return sraw::decode(bytes, &tiff, raw, mode),
Some(v) if v != 1 => return Err(RawError::Unsupported(format!("Canon raw type {v}"))),
_ => {}
}
let off = raw.u64(t::STRIP_OFFSETS).ok_or(RawError::Tiff(lightcraft_tiff::TiffError::MissingTag(t::STRIP_OFFSETS)))?;
let len = raw.u64(t::STRIP_BYTE_COUNTS).unwrap_or(bytes.len() as u64 - off.min(bytes.len() as u64));

578
crates/raw/src/vendor/cr2/sraw.rs vendored Normal file
View File

@ -0,0 +1,578 @@
//! Canon sRAW / mRAW: reduced-resolution YCbCr "raw" frames in CR2 (raw IFD tag `0xc6c5` = 4).
//!
//! What the files contain (measured on the 31 CC0 sRAW / mRAW samples of 16 models, see the evidence below):
//!
//! - The raw strip is one lossless-JPEG frame with three components. Luma is sampled 2×1 (sRAW, 4:2:2, `0x21`) or
//! 2×2 (mRAW, 4:2:0, `0x22`), both chroma components 1×1; precision 15, predictor 1, no restart markers.
//! The frame's width and height are only a factorisation of the sample count: they are not the image size.
//! - The decoded sample stream (MCU after MCU: `Y Y Cb Cr` or `Y Y Y Y Cb Cr`) is cut into vertical slices exactly
//! like the CFA data (`cr2_slice = [n, w, last]`, widths counted in samples). A slice row is a whole number of
//! MCUs, so the image is `Σw / MCU-size × 2` pixels wide, and one slice row is one pixel row (4:2:2) or two (4:2:0).
//! - Prediction: each component is a one-dimensional chain in decoding order (so the second luma row of a 4:2:0 MCU
//! continues from the end of the first), and the first sample of every frame row starts from the first sample of
//! the previous one ([`ljpeg::Prediction::Sequential`]). The Sony prediction (geometric) gives vertical stripes on
//! these files.
//! - The samples are linear. Luma is the BT.601 luma (0.299, 0.587, 0.114; also the TIFF 6.0 default
//! `YCbCrCoefficients`) of white-balanced camera RGB, about 7/8 of the black-subtracted sensor counts; chroma is
//! centred on 16384. White balance is baked in (neutral surfaces have neutral chroma); which of the maker note's
//! white-balance sets was used is not always the as-shot one, so it is not undone.
//! - Chroma is stored as a scaled colour difference, `R − Y = a·(Cr − 16384)` and `B − Y = b·(Cb − 16384)`, with
//! `G` from the luma equation. `a` and `b` are about 1.0 on bodies from 2012 on, but about 5.7 and 7.4 (the TIFF
//! 6.0 YCbCr ratio 1.402 : 1.772, scaled by about 4.1) on the 2008–2010 bodies (5D Mark II, 50D, 7D, 60D, 1D Mark IV).
//! Rather than keying a table on the model, the gains are measured per file against the 16-bit RGB image the
//! camera stores in IFD2 for these files (see [`chroma_gains`]).
//!
//! Only these files were used as evidence: the sRAW samples themselves, the same cameras' CFA files of the same
//! scenes (raw counts against the decoded samples), and the IFD2 image. The ExifTool tag documentation was used for
//! tag names.
use super::{COLOR_BALANCE, CR2_SLICE, SENSOR_INFO, words};
use crate::{BlackLevel, ColorData, Mode, OpcodeLists, RawData, RawError, RawFormat, RawImage, Rect, Result, ljpeg};
use lightcraft_geom::Orientation;
use lightcraft_tiff::image::{Chunk, chunk_bytes};
use lightcraft_tiff::{ByteOrder, Ifd, Tiff, makernote, tags as t};
use rayon::prelude::*;
/// Chroma samples are centred on this value.
const CHROMA_CENTRE: f64 = 16384.0;
/// BT.601 luma weights, the TIFF 6.0 default `YCbCrCoefficients`.
const KR: f64 = 0.299;
const KG: f64 = 0.587;
const KB: f64 = 0.114;
/// Decoded luma relative to black-subtracted sensor counts (white-balanced): 0.877, 0.877, 0.896, 0.880, 0.871, 0.851
/// and 0.846 on the 7D Mark II, 5DS R, 6D Mark II, 5DS, 5D Mark II, 1D X Mark II and 50D (CFA and sRAW files of the
/// same scene), so 7/8. It sets the white level, so that a file renders about as bright as the CFA file of the same
/// scene.
const LUMA_GAIN: f32 = 0.875;
/// Full scale of a 14-bit sensor, the white level of a CFA CR2 without a saturation plateau.
const SENSOR_FULL: f32 = 16383.0;
/// Mid-tone limit (IFD2 units) of the pixels used for the chroma fit: above about a quarter of full scale the
/// camera's own RGB image rolls off.
const FIT_LIMIT: f64 = 4096.0;
/// How the decoded frame maps to the image.
#[derive(Debug, PartialEq)]
pub(super) struct Layout {
pub width: usize,
pub height: usize,
/// Vertical subsampling of the chroma planes: 1 (4:2:2, sRAW) or 2 (4:2:0, mRAW).
pub vertical: usize,
/// Slice widths in samples.
widths: Vec<usize>,
/// Slice height in MCU rows.
rows: usize,
/// Samples per MCU.
mcu: usize,
/// Frame size from the SOF3 header.
frame_width: usize,
frame_height: usize,
}
impl Layout {
/// The layout of a `frame_width × frame_height` frame with `vertical` chroma subsampling and `cr2_slice` tag
/// values `slices` (`[n, w, last]`).
pub(super) fn new(frame_width: usize, frame_height: usize, vertical: usize, slices: &[u64]) -> Result<Layout> {
let &[n, w, last] = slices else { return Err(RawError::Corrupt("sRAW without slice layout".into())) };
if n > 64 || w == 0 || last == 0 {
return Err(RawError::Corrupt("sRAW slice layout".into()));
}
let mcu = 2 * vertical + 2;
let mut widths = vec![w as usize; n as usize];
widths.push(last as usize);
if widths.iter().any(|w| w % mcu != 0) {
return Err(RawError::Corrupt("sRAW slice is not a whole number of MCUs".into()));
}
let samples =
frame_width.checked_mul(frame_height).and_then(|p| p.checked_add(p / vertical)).ok_or(RawError::Limit("sRAW frame too large"))?;
let row = widths.iter().try_fold(0usize, |a, &w| a.checked_add(w)).ok_or(RawError::Limit("sRAW slice widths"))?;
if samples == 0 || row == 0 || samples % row != 0 {
return Err(RawError::Corrupt(format!("sRAW slices ({row} samples per row) do not divide the frame ({samples} samples)")));
}
let rows = samples / row;
let (width, height) = (row / mcu * 2, rows * vertical);
if width.checked_mul(height).and_then(|p| p.checked_mul(3)).is_none_or(|s| s > crate::MAX_SAMPLES) {
return Err(RawError::Limit("sRAW image too large"));
}
Ok(Layout { width, height, vertical, widths, rows, mcu, frame_width, frame_height })
}
}
/// Planar YCbCr at image geometry: luma `width × height`, chroma `width/2 × height/vertical`.
struct Planes {
width: usize,
height: usize,
vertical: usize,
y: Vec<u16>,
cb: Vec<u16>,
cr: Vec<u16>,
}
impl Planes {
fn chroma_width(&self) -> usize {
self.width / 2
}
/// Re-chop the decoded MCU stream into image planes.
fn arrange(frame: &ljpeg::FrameSubsampled, layout: &Layout) -> Planes {
let (w, h, v, m) = (layout.width, layout.height, layout.vertical, layout.mcu);
let cw = w / 2;
let mut p = Planes { width: w, height: h, vertical: v, y: vec![0; w * h], cb: vec![0; cw * (h / v)], cr: vec![0; cw * (h / v)] };
let mcus_per_frame_row = layout.frame_width / 2;
let mut x0 = 0;
for &sw in &layout.widths {
let per_row = sw / m;
let first = layout.rows * (x0 / m);
for r in 0..layout.rows {
for k in 0..per_row {
let q = first + r * per_row + k;
let (qy, qx) = (q / mcus_per_frame_row, q % mcus_per_frame_row);
let px = (x0 / m + k) * 2;
for dy in 0..v {
let src = (qy * v + dy) * layout.frame_width + qx * 2;
let dst = (r * v + dy) * w + px;
p.y[dst..dst + 2].copy_from_slice(&frame.planes[0][src..src + 2]);
}
p.cb[r * cw + px / 2] = frame.planes[1][qy * mcus_per_frame_row + qx];
p.cr[r * cw + px / 2] = frame.planes[2][qy * mcus_per_frame_row + qx];
}
}
x0 += sw;
}
p
}
/// Mean luma / Cb / Cr over the pixel window `[x0, x1) × [y0, y1)` (non-empty, inside the image).
fn window_mean(&self, x0: usize, x1: usize, y0: usize, y1: usize) -> (f64, f64, f64) {
let (mut sy, mut ny) = (0f64, 0f64);
for y in y0..y1 {
let row = &self.y[y * self.width + x0..y * self.width + x1];
sy += row.iter().map(|&v| f64::from(v)).sum::<f64>();
ny += row.len() as f64;
}
let (cx0, cx1) = (x0 / 2, x1.div_ceil(2).max(x0 / 2 + 1));
let (cy0, cy1) = (y0 / self.vertical, y1.div_ceil(self.vertical).max(y0 / self.vertical + 1));
let (mut sb, mut sr, mut nc) = (0f64, 0f64, 0f64);
for y in cy0..cy1.min(self.height / self.vertical) {
for x in cx0..cx1.min(self.chroma_width()) {
sb += f64::from(self.cb[y * self.chroma_width() + x]);
sr += f64::from(self.cr[y * self.chroma_width() + x]);
nc += 1.0;
}
}
let nc = nc.max(1.0);
(sy / ny.max(1.0), sb / nc - CHROMA_CENTRE, sr / nc - CHROMA_CENTRE)
}
/// Linear RGB (`width × height × 3`): `R = Y + a·Cr`, `B = Y + b·Cb`, `G` from the luma equation.
fn to_rgb(&self, a: f64, b: f64) -> Vec<u16> {
let (gr, gb) = ((KR / KG * a) as f32, (KB / KG * b) as f32);
let (a, b) = (a as f32, b as f32);
let cw = self.chroma_width();
let centre = CHROMA_CENTRE as f32;
let mut out = vec![0u16; self.width * self.height * 3];
out.par_chunks_mut(self.width * 3).enumerate().for_each(|(y, row)| {
let luma = &self.y[y * self.width..(y + 1) * self.width];
let chroma = (y / self.vertical) * cw;
for (x, px) in row.as_chunks_mut::<3>().0.iter_mut().enumerate() {
let (cb, cr) = (f32::from(self.cb[chroma + x / 2]) - centre, f32::from(self.cr[chroma + x / 2]) - centre);
let l = f32::from(luma[x]);
px[0] = (l + a * cr).round().clamp(0.0, 65535.0) as u16;
px[1] = (l - gr * cr - gb * cb).round().clamp(0.0, 65535.0) as u16;
px[2] = (l + b * cb).round().clamp(0.0, 65535.0) as u16;
}
});
out
}
}
/// The 16-bit RGB image in IFD2 (`SRawType` 3): the camera's own rendering of the same frame at a fraction of the
/// size, white-balanced and linear below about a quarter of full scale.
struct Preview {
width: usize,
height: usize,
rgb: Vec<u16>,
}
fn read_preview(bytes: &[u8], tiff: &Tiff) -> Option<Preview> {
let info = tiff.ifds.get(2)?.image().ok()?;
let (w, h) = (info.width as usize, info.height as usize);
if info.samples_per_pixel != 3 || info.bits_per_sample != [16, 16, 16] || info.compression != 1 || info.planar != 1 || info.sample_format != 1 {
return None;
}
let n = w.checked_mul(h)?.checked_mul(3)?;
if n == 0 || n > 1 << 24 {
return None;
}
let mut raw: Vec<u8> = Vec::with_capacity(n * 2);
for c in info.chunks(bytes.len() as u64) {
raw.extend_from_slice(chunk_bytes(bytes, &c)?);
if raw.len() >= n * 2 {
break;
}
}
if raw.len() < n * 2 {
return None;
}
let order: ByteOrder = tiff.order;
let rgb = raw.as_chunks::<2>().0.iter().take(n).map(|&c| order.u16(c)).collect();
Some(Preview { width: w, height: h, rgb })
}
#[derive(Default)]
struct Regression {
n: f64,
sx: f64,
st: f64,
sxx: f64,
sxt: f64,
stt: f64,
}
impl Regression {
fn add(&mut self, x: f64, t: f64) {
self.n += 1.0;
self.sx += x;
self.st += t;
self.sxx += x * x;
self.sxt += x * t;
self.stt += t * t;
}
/// Least-squares slope of `t` on `x` (with intercept) and its residual sum of squares.
fn slope(&self) -> Option<(f64, f64)> {
let var = self.n * self.sxx - self.sx * self.sx;
if self.n < 500.0 || var <= 1e-9 * self.n * self.n {
return None;
}
let slope = (self.n * self.sxt - self.sx * self.st) / var;
let icpt = (self.st - slope * self.sx) / self.n;
let rss =
self.stt - 2.0 * slope * self.sxt - 2.0 * icpt * self.st + slope * slope * self.sxx + 2.0 * slope * icpt * self.sx + self.n * icpt * icpt;
Some((slope, rss.max(0.0)))
}
}
/// How the IFD2 image sits on the full-size planes: its pixel `(i, j)` covers the window starting at
/// `(x0 + i·sx, y0 + j·sy)` and `sx × sy` pixels large.
struct Grid {
x0: f64,
y0: f64,
sx: f64,
sy: f64,
}
/// The chroma gains `(a, b)` of this file (`R − Y = a·Cr`, `B − Y = b·Cb`), by least squares of the IFD2 image
/// against the planes averaged to its size. Where IFD2 sits on the frame depends on the model (the whole frame,
/// the sensor's active area, or a whole or half-integer reduction of the frame that overhangs it by a few pixels),
/// so these are tried and the best fit is kept. `None` when the file has no usable IFD2 or the fit is not well
/// conditioned.
fn chroma_gains(planes: &Planes, preview: &Preview, active: Rect) -> Option<(f64, f64)> {
let (pw, ph) = (preview.width as f64, preview.height as f64);
let (w, h) = (planes.width as f64, planes.height as f64);
let grids = [
Grid { x0: 0.0, y0: 0.0, sx: w / pw, sy: h / ph },
Grid { x0: active.x as f64, y0: active.y as f64, sx: active.width as f64 / pw, sy: active.height as f64 / ph },
Grid { x0: 0.0, y0: 0.0, sx: (2.0 * w / pw).round() / 2.0, sy: (2.0 * h / ph).round() / 2.0 },
];
let mut best: Option<(f64, f64, f64)> = None;
for g in &grids {
if !(g.sx >= 1.0 && g.sy >= 1.0) {
continue;
}
let (mut fr, mut fb) = (Regression::default(), Regression::default());
for j in 0..preview.height {
let y0 = (g.y0 + j as f64 * g.sy) as usize;
let y1 = ((g.y0 + (j + 1) as f64 * g.sy) as usize).min(planes.height);
for i in 0..preview.width {
let p = &preview.rgb[(j * preview.width + i) * 3..][..3];
let [r, gr, b] = [f64::from(p[0]), f64::from(p[1]), f64::from(p[2])];
if r.max(gr).max(b) >= FIT_LIMIT || r.min(gr).min(b) <= 0.0 {
continue;
}
let x0 = (g.x0 + i as f64 * g.sx) as usize;
let x1 = ((g.x0 + (i + 1) as f64 * g.sx) as usize).min(planes.width);
if x1 <= x0 || y1 <= y0 {
continue;
}
let (y, cb, cr) = planes.window_mean(x0, x1, y0, y1);
if y < 150.0 {
continue;
}
fr.add(cr, r - y);
fb.add(cb, b - y);
}
}
if let (Some((a, ra)), Some((b, rb))) = (fr.slope(), fb.slope()) {
let cost = (ra + rb) / (fr.n + fb.n);
if a.is_finite() && b.is_finite() && best.is_none_or(|(_, _, c)| cost < c) {
best = Some((a, b, cost));
}
}
}
best.map(|(a, b, _)| (a, b)).filter(|(a, b)| (0.25..=16.0).contains(a) && (0.25..=16.0).contains(b))
}
/// Word index of `AverageBlackLevel` (four equal-ish values) in the maker note's `ColorData` by `ColorDataVersion`
/// (word 0), located by searching the CC0 samples for the values ExifTool reports.
fn average_black_at(version: i16) -> Option<usize> {
Some(match version {
6 | 7 | 9 => 231,
10 | 11 => 276,
12 | 13 | 15 => 326,
_ => return None,
})
}
/// The sensor's black level in raw counts from `ColorData` (`None` for versions whose position is not known, and
/// when the four values are not plausible and near-equal).
fn colour_data_black(words: &[u64]) -> Option<f32> {
let version = words.first().map(|&v| v as u16 as i16)?;
let q = words.get(average_black_at(version)?..)?.get(..4)?;
let (lo, hi) = (q.iter().min()?, q.iter().max()?);
((256..=4096).contains(lo) && hi - lo <= 8).then(|| q.iter().sum::<u64>() as f32 / 4.0)
}
/// Decode a Canon sRAW / mRAW CR2 into linear white-balanced camera RGB (`cpp` 3).
pub(super) fn decode(bytes: &[u8], tiff: &Tiff, raw: &Ifd, mode: Mode) -> Result<RawImage> {
let ifd0 = &tiff.ifds[0];
let off = raw.u64(t::STRIP_OFFSETS).ok_or(RawError::Tiff(lightcraft_tiff::TiffError::MissingTag(t::STRIP_OFFSETS)))?;
let len = raw.u64(t::STRIP_BYTE_COUNTS).unwrap_or(bytes.len() as u64 - off.min(bytes.len() as u64));
let chunk = Chunk { index: 0, x: 0, y: 0, width: 0, height: 0, plane: 0, offset: off, len };
let src = chunk_bytes(bytes, &chunk).ok_or_else(|| RawError::Corrupt("raw strip outside file".into()))?;
let (fw, fh, vertical) = ljpeg::frame_info_subsampled(src)?;
let slices = raw.u64s(CR2_SLICE).unwrap_or_default();
let layout = Layout::new(fw, fh, vertical, &slices)?;
let make = ifd0.string(t::MAKE).unwrap_or_default();
let mn =
tiff.exif().and_then(|e| e.get(t::MAKER_NOTE)).and_then(|e| makernote::parse_makernote(bytes, e.offset, e.count() as u64, tiff.order, &make));
let (w, h) = (layout.width, layout.height);
let mut active = Rect::new(0, 0, w, h);
if let Some(si) = mn.as_ref().and_then(|m| m.ifd.u64s(SENSOR_INFO)).filter(|v| v.len() >= 9) {
let (l, tp, r, b) = (si[5] as usize, si[6] as usize, si[7] as usize, si[8] as usize);
if r > l && b > tp && r < w && b < h {
active = Rect::new(l, tp, r - l + 1, b - tp + 1);
}
}
let colour_data = mn.as_ref().and_then(|m| m.ifd.value(COLOR_BALANCE).map(|v| words(v, m.order)));
let black_counts = colour_data.as_deref().and_then(colour_data_black);
// 1.0 = a CFA CR2 of the same body at full 14-bit scale; see LUMA_GAIN
let white = LUMA_GAIN * (SENSOR_FULL - black_counts.unwrap_or(0.0));
let data = match mode {
Mode::Header => Vec::new(),
Mode::Full => {
let samples = fw * fh + fw * fh / vertical;
let frame = ljpeg::decode_subsampled(src, samples.min(crate::MAX_SAMPLES), ljpeg::Prediction::Sequential)?;
let planes = Planes::arrange(&frame, &layout);
drop(frame);
let (a, b) = read_preview(bytes, tiff).and_then(|p| chroma_gains(&planes, &p, active)).unwrap_or((1.0, 1.0));
planes.to_rgb(a, b)
}
};
let mut metadata = lightcraft_meta::from_tiff(tiff);
metadata.width = Some(active.width as u32);
metadata.height = Some(active.height as u32);
let img = RawImage {
format: RawFormat::Cr2,
width: w,
height: h,
cpp: 3,
data: RawData::U16(data),
cfa: None,
bits: 14,
black: BlackLevel::uniform(0.0),
white: vec![white],
active_area: active,
crop: Rect::new(0, 0, active.width, active.height),
orientation: Orientation::from_exif(ifd0.u16(t::ORIENTATION).unwrap_or(1)),
color: ColorData::default(),
// the camera's white balance is already applied, as in Sony's linear YCbCr
wb_multipliers: Some([1.0; 3]),
linearized: false,
opcodes: OpcodeLists::default(),
metadata,
};
img.validate_for(mode)?;
Ok(img)
}
#[cfg(test)]
mod tests {
use super::*;
use lightcraft_tiff::{IfdBuilder, ImageData, TiffWriter, Value};
const SRAW_TYPE_TAG: u16 = 0xc6c5;
/// The 4:2:2 stream of `ljpeg`'s hand-coded fixture (4 × 2 frame, two MCUs per frame row) decodes to luma
/// 1000 1001 / 1002 1003 / 1100 1101 / 1102 1103 in MCU order, Cb 16384 16385 16484 16485, Cr 16484 16486
/// 16584 16586.
fn stream_422() -> Vec<u8> {
ljpeg::tests::fixture_subsampled(4, 2, 0x21, &[-31768, 1, -16384, -16284, 1, 1, 1, 2, 100, 1, 100, 100, 1, 1, 1, 2])
}
/// A CR2 whose raw IFD holds `stream` cut into the slices `slices` (IFD2 = a 2 × 2 16-bit RGB image when
/// `preview` is given).
fn sraw_file(stream: Vec<u8>, slices: [u16; 3], sraw_type: u32, preview: Option<[u16; 12]>) -> Vec<u8> {
let mut ifd0 = IfdBuilder::new();
ifd0.set(t::MAKE, Value::Ascii("Canon".into()));
ifd0.set(t::MODEL, Value::Ascii("Canon EOS Test".into()));
let mut ifd2 = IfdBuilder::new();
if let Some(p) = preview {
ifd2.set(t::COMPRESSION, Value::Short(vec![1]));
ifd2.set(t::PHOTOMETRIC, Value::Short(vec![2]));
ifd2.set(t::IMAGE_WIDTH, Value::Long(vec![2]));
ifd2.set(t::IMAGE_LENGTH, Value::Long(vec![2]));
ifd2.set(t::BITS_PER_SAMPLE, Value::Short(vec![16, 16, 16]));
ifd2.set(t::SAMPLES_PER_PIXEL, Value::Short(vec![3]));
ifd2.set_image(ImageData::Strips { rows_per_strip: 2, strips: vec![p.iter().flat_map(|v| v.to_le_bytes()).collect()] });
} else {
ifd2.set(1, Value::Short(vec![0]));
}
let mut ifd3 = IfdBuilder::new();
ifd3.set(t::COMPRESSION, Value::Short(vec![6]));
ifd3.set(SRAW_TYPE_TAG, Value::Long(vec![sraw_type]));
ifd3.set(CR2_SLICE, Value::Short(slices.to_vec()));
ifd3.set_image(ImageData::Strips { rows_per_strip: 2, strips: vec![stream] });
TiffWriter::new(ByteOrder::Little, false).write(&[ifd0, IfdBuilder::new().with(1, Value::Short(vec![0])), ifd2, ifd3]).unwrap()
}
#[test]
fn slices_are_cut_back_into_image_columns_and_converted() {
// slices [1, 4, 4]: two slices of one MCU per row; slice 0 holds MCUs 0 and 1 (image rows 0 and 1), slice 1
// holds MCUs 2 and 3, so the image is 4 × 2 with luma 1000 1001 1100 1101 / 1002 1003 1102 1103.
let bytes = sraw_file(stream_422(), [1, 4, 4], 4, None);
let img = crate::decode(&bytes).unwrap();
assert_eq!((img.width, img.height, img.cpp), (4, 2, 3));
assert!(img.cfa.is_none());
assert_eq!(img.wb_multipliers, Some([1.0; 3]));
let RawData::U16(px) = &img.data else { panic!("integer data") };
// no IFD2: chroma gains 1; R = Y + Cr', B = Y + Cb', G from the BT.601 luma
let expect = |y: f64, cb: f64, cr: f64| {
let (r, b) = (y + cr, y + cb);
[r, (y - KR * r - KB * b) / KG, b].map(|v| v.round() as i32)
};
let cases = [
((0, 0), expect(1000.0, 0.0, 100.0)),
((1, 0), expect(1001.0, 0.0, 100.0)),
((2, 0), expect(1100.0, 100.0, 200.0)),
((3, 0), expect(1101.0, 100.0, 200.0)),
((0, 1), expect(1002.0, 1.0, 102.0)),
((3, 1), expect(1103.0, 101.0, 202.0)),
];
for ((x, y), want) in cases {
let got: Vec<i32> = px[(y * 4 + x) * 3..][..3].iter().map(|&v| i32::from(v)).collect();
assert!(got.iter().zip(want).all(|(g, w)| (g - w).abs() <= 1), "({x},{y}): {got:?} vs {want:?}");
}
// header-only decoding reports the same image
assert_eq!(crate::probe_info(&bytes).unwrap(), img.info());
assert!(img.develop(crate::Method::Bilinear).is_ok());
}
#[test]
fn a_preview_too_small_to_fit_leaves_the_gains_at_one() {
// IFD2 of 2 × 2 pixels cannot support a fit (it needs hundreds of samples): the file still decodes
let with = crate::decode(&sraw_file(stream_422(), [1, 4, 4], 4, Some([900; 12]))).unwrap();
let without = crate::decode(&sraw_file(stream_422(), [1, 4, 4], 4, None)).unwrap();
assert_eq!(with.data, without.data);
}
#[test]
fn only_ycbcr_frames_take_this_path() {
// other raw types (3 is the preview image's) stay unsupported, as before
for kind in [0, 2, 3, 5] {
let bytes = sraw_file(stream_422(), [1, 4, 4], kind, None);
assert!(matches!(crate::decode(&bytes), Err(RawError::Unsupported(_))), "type {kind}");
}
}
#[test]
fn slice_layouts_must_fit_the_frame() {
let ok = Layout::new(4, 2, 1, &[1, 4, 4]).unwrap();
assert_eq!((ok.width, ok.height), (4, 2));
// 4:2:0: six samples per MCU, one slice row is two image rows
let l = Layout::new(8, 4, 2, &[1, 12, 12]).unwrap();
assert_eq!((l.width, l.height), (8, 4));
assert!(Layout::new(4, 2, 1, &[1, 3, 5]).is_err()); // not whole MCUs
assert!(Layout::new(4, 2, 1, &[1, 4, 8]).is_err()); // rows do not divide the frame
assert!(Layout::new(4, 2, 1, &[1, 0, 4]).is_err());
assert!(Layout::new(4, 2, 1, &[4, 4]).is_err()); // not three values
assert!(Layout::new(4, 2, 1, &[65, 4, 4]).is_err());
// slice widths whose sum overflows (or wraps to zero) are rejected, not divided by
assert!(Layout::new(4, 2, 1, &[3, 1 << 62, 1 << 62]).is_err());
assert!(Layout::new(4, 2, 1, &[1, u64::MAX - 3, 4]).is_err());
let bytes = sraw_file(stream_422(), [1, 4, 8], 4, None);
assert!(crate::decode(&bytes).is_err());
}
/// Textured luma and smooth chroma on `w × h` planes.
fn test_planes(w: usize, h: usize, vertical: usize) -> Planes {
let y: Vec<u16> = (0..w * h).map(|i| (600 + ((i % w) * 13 + (i / w) * 29) % 1500) as u16).collect();
let cw = w / 2;
let ch = h / vertical;
let cb = (0..cw * ch).map(|i| (16384 + (i % cw) as i32 * 7 - (i / cw) as i32 * 3 - 150) as u16).collect();
let cr = (0..cw * ch).map(|i| (16384 + (i / cw) as i32 * 5 + (i % cw) as i32 * 2 - 120) as u16).collect();
Planes { width: w, height: h, vertical, y, cb, cr }
}
/// An IFD2 image of `pw × ph` pixels at `scale` per pixel, built the way the files relate it to the planes.
fn preview_of(p: &Planes, pw: usize, ph: usize, scale: usize, a: f64, b: f64, pedestal: f64) -> Preview {
let mut rgb = Vec::new();
for j in 0..ph {
for i in 0..pw {
let (y, cb, cr) = p.window_mean(i * scale, (i + 1) * scale, j * scale, (j + 1) * scale);
let (r, bl) = (y + a * cr, y + b * cb);
let g = (y - KR * r - KB * bl) / KG;
rgb.extend([r, g, bl].map(|v| (v + pedestal).round().clamp(0.0, 65535.0) as u16));
}
}
Preview { width: pw, height: ph, rgb }
}
#[test]
fn chroma_gains_are_measured_against_the_preview() {
for vertical in [1, 2] {
for (a, b) in [(1.0, 1.0), (5.7, 7.4), (1.13, 1.12)] {
let planes = test_planes(160, 96, vertical);
// IFD2 is a plain 1/4 reduction, optionally with a black pedestal (some bodies add one)
for pedestal in [0.0, 520.0] {
let preview = preview_of(&planes, 40, 24, 4, a, b, pedestal);
let (ga, gb) =
chroma_gains(&planes, &preview, Rect::new(0, 0, 160, 96)).unwrap_or_else(|| panic!("no fit for {a}, {b}, {vertical}"));
assert!((ga / a - 1.0).abs() < 0.02 && (gb / b - 1.0).abs() < 0.02, "vertical {vertical}: ({a}, {b}) measured as ({ga}, {gb})");
}
}
}
// the preview overhangs the frame by a few pixels: the whole-frame scale (4.1) is wrong, 4 is right
let planes = test_planes(164, 96, 1);
let mut preview = preview_of(&planes, 40, 24, 4, 5.7, 7.4, 0.0);
let (ga, gb) = chroma_gains(&planes, &preview, Rect::new(0, 0, 164, 96)).unwrap();
assert!((ga / 5.7 - 1.0).abs() < 0.02 && (gb / 7.4 - 1.0).abs() < 0.02, "({ga}, {gb})");
// a preview of something else gives no usable fit rather than a wild gain
preview.rgb.iter_mut().enumerate().for_each(|(i, v)| *v = (1000 + (i * 7919) % 3000) as u16);
assert!(
chroma_gains(&planes, &preview, Rect::new(0, 0, 164, 96)).is_none_or(|(a, b)| (0.25..=16.0).contains(&a) && (0.25..=16.0).contains(&b))
);
}
#[test]
fn colour_data_black_by_version() {
let mut w = vec![0u64; 400];
w[0] = 7;
w[231..235].copy_from_slice(&[2048, 2047, 2048, 2048]);
assert_eq!(colour_data_black(&w), Some(2047.75));
w[0] = 12;
assert_eq!(colour_data_black(&w), None); // the black level of another version lives elsewhere
w[326..330].copy_from_slice(&[512, 512, 511, 512]);
assert_eq!(colour_data_black(&w), Some(511.75));
w[326..330].copy_from_slice(&[512, 512, 100, 512]); // implausible
assert_eq!(colour_data_black(&w), None);
w[0] = 3;
assert_eq!(colour_data_black(&w), None); // no position known: black 0
w[0] = 65533; // -3
assert_eq!(colour_data_black(&w), None);
assert_eq!(colour_data_black(&[]), None);
assert_eq!(colour_data_black(&[7, 1, 2]), None);
}
}

102
crates/raw/src/vendor/cr3.rs vendored
View File

@ -7,6 +7,15 @@
//!
//! `IAD1` stores the valid sensor area separately from the recommended image crop. Both use inclusive
//! sensor offsets. The CFA is anchored at the full sensor's origin; cropping shifts its phase later.
//!
//! The in-camera aspect ratio (3:2, 4:3, 16:9, 1:1 ...) is not applied to the sensor data: the raw always
//! holds the whole image. The maker note's `AspectInfo` (tag `0x009a`; ExifTool names `AspectRatio`,
//! `CroppedImageWidth`, `CroppedImageHeight`, `CroppedImageLeft`, `CroppedImageTop`) holds the shot's rectangle
//! as five 32-bit values, measured from the top-left of the recommended image crop (the "Canon image", whose
//! size is ExifTool's `CanonImageWidth` x `CanonImageHeight`). That rectangle replaces the crop. Established
//! on the CC0 PowerShot SX70 HS, EOS 250D, PowerShot G5 X Mark II and EOS M6 Mark II samples shot at 3:2, 4:3,
//! 16:9 and 1:1: the rectangle is centred, has the shot's aspect, and files shot at the sensor's own aspect
//! store the whole image.
//! Unknown camera metadata versions are ignored rather than assigned a guessed white balance.
use super::{black_from_columns, crx, white_from_data};
@ -17,6 +26,7 @@ use lightcraft_tiff::{Ifd, Tiff};
const SENSOR_INFO: u16 = 0x00e0;
const COLOR_DATA: u16 = 0x4001;
const LENS_MODEL: u16 = 0x0095;
const ASPECT_INFO: u16 = 0x009a;
/// Pick the first largest raw track. Equal-sized later tracks can be the Dual Pixel delta, which
/// must never replace the main sensor image. An invalid full-size descriptor is reported, not
@ -83,16 +93,26 @@ fn sensor_crop(maker: Option<&Ifd>, width: usize, height: usize) -> Option<Rect>
)
}
/// The shot's aspect-ratio rectangle from `AspectInfo`, placed inside `image` (the recommended crop). `None`
/// when the tag is absent or malformed, or its rectangle doesn't fit inside `image`.
fn aspect_crop(maker: Option<&Ifd>, image: Rect) -> Option<Rect> {
let data = maker?.u64s(ASPECT_INFO)?;
let [width, height, left, top] = [1usize, 2, 3, 4].map(|i| data.get(i).and_then(|&v| usize::try_from(v).ok()));
let (width, height, left, top) = (width?, height?, left?, top?);
if width < 2 || height < 2 || left.checked_add(width)? > image.width || top.checked_add(height)? > image.height {
return None;
}
Some(Rect::new(image.x.checked_add(left)?, image.y.checked_add(top)?, width, height))
}
fn geometry(area: Option<&Cr3ImageArea>, maker: Option<&Ifd>, width: usize, height: usize) -> (Rect, Rect) {
let area = area.filter(|a| usize::from(a.width) == width && usize::from(a.height) == height);
let sensor = sensor_crop(maker, width, height);
let active = area.and_then(|a| a.active).and_then(|b| inclusive_rect(b, width, height)).or(sensor).unwrap_or(Rect::new(0, 0, width, height));
let crop = area.and_then(|a| inclusive_rect(a.crop, width, height)).or(sensor).and_then(|c| relative_crop(c, active)).unwrap_or(Rect::new(
0,
0,
active.width,
active.height,
));
// The recommended crop, when the file gives a consistent one. `AspectInfo` is measured from its origin, so it
// is only applied on top of that; without it (the crop doesn't fit the valid area) the whole active area stays.
let recommended = area.and_then(|a| inclusive_rect(a.crop, width, height)).or(sensor).and_then(|c| relative_crop(c, active));
let crop = recommended.map(|c| aspect_crop(maker, c).unwrap_or(c)).unwrap_or(Rect::new(0, 0, active.width, active.height));
(active, crop)
}
@ -266,6 +286,10 @@ mod tests {
}
fn synthetic_header() -> Vec<u8> {
synthetic_header_with(None)
}
fn synthetic_header_with(aspect: Option<[u32; 5]>) -> Vec<u8> {
use lightcraft_tiff::{ByteOrder, TiffWriter, tags as t};
let tiff = |b: IfdBuilder| TiffWriter::new(ByteOrder::Little, false).write(&[b]).unwrap();
let cmt1 = tiff(
@ -275,11 +299,13 @@ mod tests {
.with(t::ORIENTATION, Value::Short(vec![6])),
);
let cmt2 = tiff(IfdBuilder::new().with(t::ISO_SPEED, Value::Short(vec![800])));
let cmt3 = tiff(
IfdBuilder::new()
.with(SENSOR_INFO, Value::Short(vec![34, 64, 48, 1, 1, 6, 4, 61, 45]))
.with(LENS_MODEL, Value::Ascii("RF-S18-45mm F4.5-6.3 IS STM".into())),
);
let mut cmt3 = IfdBuilder::new()
.with(SENSOR_INFO, Value::Short(vec![34, 64, 48, 1, 1, 6, 4, 61, 45]))
.with(LENS_MODEL, Value::Ascii("RF-S18-45mm F4.5-6.3 IS STM".into()));
if let Some(a) = aspect {
cmt3 = cmt3.with(ASPECT_INFO, Value::Long(a.to_vec()));
}
let cmt3 = tiff(cmt3);
let mut canon = vec![0x85, 0xc0, 0xb6, 0x87, 0x82, 0x0f, 0x11, 0xe0, 0x81, 0x11, 0xf4, 0xce, 0x46, 0x2b, 0x6a, 0x48];
canon.extend([box_bytes(b"CMT1", &cmt1), box_bytes(b"CMT2", &cmt2), box_bytes(b"CMT3", &cmt3)].concat());
let mut coding = vec![0u8; 52];
@ -374,6 +400,60 @@ mod tests {
assert!(decode(&bytes[..4200], Mode::Header).is_err());
}
#[test]
fn aspect_info_replaces_the_crop() {
// 16:9 inside the 56 x 42 recommended crop at (2, 2): 56 x 30, 6 rows down.
let bytes = synthetic_header_with(Some([7, 56, 30, 0, 6]));
let image = decode(&bytes, Mode::Header).unwrap();
assert_eq!(image.active_area, Rect::new(4, 2, 60, 46));
assert_eq!(image.crop, Rect::new(2, 8, 56, 30));
assert_eq!((image.metadata.width, image.metadata.height), (Some(56), Some(30)));
// 1:1 with a left offset.
let image = decode(&synthetic_header_with(Some([1, 42, 42, 7, 0])), Mode::Header).unwrap();
assert_eq!(image.crop, Rect::new(9, 2, 42, 42));
// The whole image, as stored for a shot at the sensor's own aspect, changes nothing.
let image = decode(&synthetic_header_with(Some([0, 56, 42, 0, 0])), Mode::Header).unwrap();
assert_eq!(image.crop, Rect::new(2, 2, 56, 42));
}
#[test]
fn aspect_info_that_does_not_fit_is_ignored() {
for bad in [[7, 57, 30, 0, 6], [7, 56, 30, 0, 13], [7, 56, 30, 1, 6], [7, 1, 30, 0, 0], [7, 56, 0, 0, 0], [7, u32::MAX, 30, 0, 0]] {
let image = decode(&synthetic_header_with(Some(bad)), Mode::Header).unwrap();
assert_eq!(image.crop, Rect::new(2, 2, 56, 42), "{bad:?}");
}
}
/// Some crop-mode files (EOS R5 Mark II 7883) record a recommended crop that reaches past their valid area. The
/// whole active area stays then, and `AspectInfo`, which is measured from the recommended crop, is not applied.
#[test]
fn aspect_info_needs_a_consistent_recommended_crop() {
let aspect = lightcraft_tiff::IfdBuilder::new().with(ASPECT_INFO, Value::Long(vec![13, 5088, 3392, 0, 0]));
let bytes = lightcraft_tiff::TiffWriter::new(lightcraft_tiff::ByteOrder::Little, false).write(&[aspect]).unwrap();
let maker = Tiff::parse(&bytes).unwrap().ifds.remove(0);
let area =
|crop| Cr3ImageArea { width: 5376, height: 3574, crop, active: Some([132, 160, 5243, 3567]), masked_left: [0; 4], masked_top: None };
// crop right edge 5359 > active right edge 5243: not a crop of the active area
assert_eq!(geometry(Some(&area([272, 172, 5359, 3563])), Some(&maker), 5376, 3574).1, Rect::new(0, 0, 5112, 3408));
// consistent crop: AspectInfo applies from its origin
assert_eq!(geometry(Some(&area([140, 172, 5227, 3563])), Some(&maker), 5376, 3574).1, Rect::new(8, 12, 5088, 3392));
}
/// `AspectInfo` as the SX70 HS 16:9 sample stores it, byte for byte (little-endian LONG x 5), read through
/// the TIFF parser: AspectRatio 7, 5184 x 2912 at (0, 488).
#[test]
fn aspect_info_known_bytes() {
let mut tiff = b"II*".to_vec();
tiff.extend_from_slice(&[0x9a, 0x00, 0x04, 0x00, 0x05, 0x00, 0x00, 0x00, 0x1a, 0x00, 0x00, 0x00]);
tiff.extend_from_slice(&[0, 0, 0, 0]);
tiff.extend_from_slice(&[0x07, 0, 0, 0, 0x40, 0x14, 0, 0, 0x60, 0x0b, 0, 0, 0, 0, 0, 0, 0xe8, 0x01, 0, 0]);
let parsed = Tiff::parse(&tiff).unwrap();
let maker = parsed.ifds.first();
assert_eq!(aspect_crop(maker, Rect::new(132, 40, 5184, 3888)), Some(Rect::new(132, 528, 5184, 2912)));
assert_eq!(aspect_crop(maker, Rect::new(0, 0, 5184, 2000)), None);
assert_eq!(aspect_crop(None, Rect::new(0, 0, 5184, 3888)), None);
}
fn prepend_tracks(bytes: &[u8], tracks: &[u8]) -> Vec<u8> {
let ftyp_end = u32::from_be_bytes(bytes[..4].try_into().unwrap()) as usize;
let moov_len = u32::from_be_bytes(bytes[ftyp_end..ftyp_end + 4].try_into().unwrap()) as usize;

24
crates/raw/src/vendor/crx.rs vendored
View File

@ -9,7 +9,8 @@
//! CC0 raw.pixls.us files against an external decoder used only as a black-box oracle.
//!
//! Current support: version 0x100 lossless Bayer RAW and horizontal-tile C-RAW,
//! plus version 0x200 single-tile 14-bit C-RAW with adaptive QP quantization.
//! plus version 0x200 single-tile 14-bit C-RAW with adaptive QP quantization
//! (including odd-height QP maps, as on the EOS R7).
//! Both ff01/ff02/ff03 and ff11/ff12/ff13 marker families are supported.
//! Six full M50/R100/R8 sensor fixtures match an external decoder sample for sample.
//! Unverified coding variants return explicit unsupported errors.
@ -173,9 +174,6 @@ fn supports(config: &Cr3Compression) -> Result<()> {
if config.version == 0x200 && (config.levels != 3 || config.bit_depth != 14 || config.tile_width != config.width) {
return Err(RawError::Unsupported("Canon CRX version 0x200 requires a single-tile 14-bit Bayer C-RAW image".into()));
}
if config.version == 0x200 && !config.height.div_ceil(4).is_multiple_of(2) {
return Err(RawError::Unsupported("Canon CRX adaptive QP map with an odd height".into()));
}
if config.median_bit_depth.is_some() {
return Err(RawError::Unsupported("Canon CRX extended median-bit-depth header".into()));
}
@ -418,7 +416,9 @@ fn adaptive_step(map: &QpMap, band: usize, x: usize, y: usize, base: u32, gain:
let index = row.checked_mul(map.width).and_then(|n| n.checked_add(column)).ok_or(RawError::Limit("CRX QP map index"))?;
let first = *map.data.get(index).ok_or_else(|| corrupt("QP coordinate outside map"))?;
let value = if band < 7 {
let second_row = row + 1;
// An odd-height map (R7: 4732 rows give 1183 QP rows) leaves the
// last row of the coarser bands without a partner; it is used alone.
let second_row = (row + 1).min(map.height - 1);
let second = *map.data.get(second_row * map.width + column).ok_or_else(|| corrupt("QP average outside map"))?;
(first + second) / 2
} else {
@ -734,9 +734,6 @@ fn high_frequency_band(src: &[u8], width: usize, height: usize) -> Result<Vec<i3
}
fn qp_map(src: &[u8], width: usize, height: usize) -> Result<QpMap> {
if !height.is_multiple_of(2) {
return Err(RawError::Unsupported("Canon CRX adaptive QP map with an odd height".into()));
}
let count = width.checked_mul(height).filter(|&n| width > 0 && n > 0 && n <= MAX_CRX_SAMPLES / 32).ok_or(RawError::Limit("CRX QP map size"))?;
let mut data = Vec::new();
data.try_reserve_exact(count).map_err(|_| RawError::Limit("CRX QP allocation"))?;
@ -1006,6 +1003,17 @@ mod tests {
assert!(matches!(qp_map(&[0, 0, 1, 4, 0x94], 1, 4), Err(RawError::Unsupported(_))));
}
#[test]
fn odd_height_qp_map_uses_its_last_row_alone() {
// EOS R7 C-RAW: 4732 sensor rows give a 1183-row QP map, so the last
// row of the coarser high-frequency bands has no partner row.
let map = QpMap { width: 1, height: 3, data: vec![150, 153, 151] };
assert_eq!(adaptive_step(&map, 4, 0, 0, 0, 4).unwrap(), 7);
// Row 2 alone is QP 151; averaging with a missing row would leave the table.
assert_eq!(adaptive_step(&map, 4, 0, 1, 0, 4).unwrap(), adaptive_step(&map, 7, 0, 2, 0, 4).unwrap());
assert!(adaptive_step(&map, 4, 0, 2, 0, 4).is_err());
}
#[test]
fn adaptive_v200_craw_reconstructs_sensor_samples() {
let mut config = coding();

21
crates/raw/src/vendor/mod.rs vendored
View File

@ -13,9 +13,28 @@ pub mod raf;
mod rafc;
pub mod rw2;
use crate::{BlackLevel, Rect};
use crate::{BlackLevel, Cfa, Rect};
use lightcraft_tiff::Tiff;
use std::ops::Range;
/// Exif `CFAPattern`.
pub(crate) const EXIF_CFA_PATTERN: u16 = 0xa302;
/// The colour-filter layout from the Exif `CFAPattern` tag (`0xa302`: two 16-bit repeat counts, found in either
/// byte order, then one byte per site: 0 = red, 1 = green, 2 = blue), when it describes a 2×2 Bayer cell.
pub(crate) fn cfa_from_exif(tiff: &Tiff) -> Option<Cfa> {
let &[c0, c1, r0, r1, s0, s1, s2, s3] = tiff.exif()?.bytes(EXIF_CFA_PATTERN)? else { return None };
let two = |a: u8, b: u8| matches!((a, b), (2, 0) | (0, 2));
let name = match [s0, s1, s2, s3] {
[0, 1, 1, 2] => "RGGB",
[2, 1, 1, 0] => "BGGR",
[1, 0, 2, 1] => "GRBG",
[1, 2, 0, 1] => "GBRG",
_ => return None,
};
(two(c0, c1) && two(r0, r1)).then(|| Cfa::bayer_static(name))
}
/// Black level per 2×2 CFA position (anchored at the active area origin) from masked sensor columns `cols` over
/// rows `rows`. Falls back to 0 when the region is empty.
pub(crate) fn black_from_columns(data: &[u16], width: usize, cols: Range<usize>, rows: Range<usize>, active: Rect) -> BlackLevel {

20
crates/raw/src/vendor/orf.rs vendored
View File

@ -18,7 +18,7 @@
//! pattern is anchored at the sensor origin (assumed) or at the active area. Files without a usable tag fall back
//! to the green diagonal found from the data, which can't tell red from blue (GRBG or RGGB is assumed).
use super::white_from_data;
use super::{cfa_from_exif, white_from_data};
use crate::tiffraw::{Packing, read_image};
use crate::unpack::unpack_msb;
use crate::{BlackLevel, Cfa, ColorData, Mode, OpcodeLists, RawData, RawError, RawFormat, RawImage, Rect, Result};
@ -35,8 +35,6 @@ pub(crate) const PREVIEW_LENGTH: u16 = 0x0102;
const WB_RB: u16 = 0x0100;
const BLACK: u16 = 0x0600;
const CROP: [u16; 4] = [0x0612, 0x0613, 0x0614, 0x0615];
/// Exif `CFAPattern`.
const EXIF_CFA_PATTERN: u16 = 0xa302;
/// The Olympus maker note.
pub(crate) fn maker_note(bytes: &[u8], tiff: &Tiff) -> Option<MakerNote> {
@ -70,21 +68,6 @@ pub(crate) fn unpack_row_le32_msb(src: &[u8], bits: u32, out: &mut [u16]) {
unpack_msb(&swapped, bits, out);
}
/// The colour-filter layout from the Exif `CFAPattern` tag (`0xa302`: two 16-bit repeat counts, found in either
/// byte order, then one byte per site: 0 = red, 1 = green, 2 = blue), when it describes a 2×2 Bayer cell.
fn cfa_from_exif(tiff: &Tiff) -> Option<Cfa> {
let &[c0, c1, r0, r1, s0, s1, s2, s3] = tiff.exif()?.bytes(EXIF_CFA_PATTERN)? else { return None };
let two = |a: u8, b: u8| matches!((a, b), (2, 0) | (0, 2));
let name = match [s0, s1, s2, s3] {
[0, 1, 1, 2] => "RGGB",
[2, 1, 1, 0] => "BGGR",
[1, 0, 2, 1] => "GRBG",
[1, 2, 0, 1] => "GBRG",
_ => return None,
};
(two(c0, c1) && two(r0, r1)).then(|| Cfa::bayer_static(name))
}
/// The layout found from the samples, for files without the Exif tag: GRBG when the greens sit on the main
/// diagonal of the 2×2 cell at the sensor origin, else RGGB. Over 32×32-pixel blocks of `a` (every fourth in both
/// directions) it compares the block totals of the two sites on each diagonal: the two greens of a block see the
@ -223,6 +206,7 @@ pub(crate) fn preview(bytes: &[u8]) -> Option<&[u8]> {
#[cfg(test)]
mod tests {
use super::*;
use crate::vendor::EXIF_CFA_PATTERN;
use lightcraft_tiff::{ByteOrder, IfdBuilder, ImageData, TiffWriter};
fn orf(w: u32, h: u32, bits: u16, strip: Vec<u8>) -> Vec<u8> {

151
crates/raw/src/vendor/pef.rs vendored
View File

@ -15,12 +15,12 @@
//! (starting at 0). Found by trying candidate predictors and checking the decoded rows are continuous.
//! - Maker note `0x0038`/`0x0039`: the image area's left/top and width/height (verified against the data).
use super::{black_from_columns, white_from_data};
use super::{black_from_columns, cfa_from_exif, white_from_data};
use crate::tiffraw::{Packing, read_image_in};
use crate::{BlackLevel, Cfa, ColorData, Mode, OpcodeLists, RawData, RawError, RawFormat, RawImage, Rect, Result};
use lightcraft_geom::Orientation;
use lightcraft_tiff::image::chunk_bytes;
use lightcraft_tiff::image::{ImageInfo, chunk_bytes};
use lightcraft_tiff::{ByteOrder, Tiff, makernote, tags as t};
const CROP_ORIGIN: u16 = 0x0038;
@ -188,6 +188,19 @@ fn dark_trimmed(d: &[u16], w: usize, h: usize) -> Rect {
Rect::new(left, 0, (right - left) & !1, h)
}
/// Whether every strip or tile of a one-sample-per-pixel image holds exactly the bytes that its rows need when
/// the samples are packed MSB-first at the declared bit depth, rows starting on byte boundaries (TIFF 6.0
/// section 7). A real PackBits stream of such an image has a different length (almost always shorter), so the
/// exact match is what tells plain samples under a compression tag that says otherwise.
fn is_exactly_packed(info: &ImageInfo, file_len: u64) -> bool {
let bits = u64::from(info.bits());
if info.samples_per_pixel != 1 || info.planar != 1 || !(1..=16).contains(&bits) || info.byte_counts.len() != info.offsets.len() {
return false;
}
let chunks = info.chunks(file_len);
!chunks.is_empty() && chunks.iter().all(|c| c.len == (u64::from(c.width) * bits).div_ceil(8) * u64::from(c.height))
}
pub(crate) fn decode(bytes: &[u8], mode: Mode) -> Result<RawImage> {
let tiff = Tiff::parse(bytes)?;
let ifd0 = &tiff.ifds[0];
@ -226,6 +239,12 @@ pub(crate) fn decode(bytes: &[u8], mode: Mode) -> Result<RawImage> {
let packing = if strip >= (w * hgt * 2) as u64 { Packing::Word16 } else { Packing::Msb };
read_image_in(read, bytes, &info, tiff.order, packing)?
}
// Some bodies tag plain packed samples as PackBits (32773). Only a strip that holds exactly the packed
// size of its rows is read, as the uncompressed MSB-first samples it is; any other size stays unsupported.
32773 if is_exactly_packed(&info, bytes.len() as u64) => {
let packed = ImageInfo { compression: 1, ..info.clone() };
read_image_in(read, bytes, &packed, tiff.order, Packing::Msb)?
}
c => return Err(RawError::Unsupported(format!("PEF compression {c}"))),
};
let RawData::U16(ref samples) = data else { return Err(RawError::Unsupported("float PEF".into())) };
@ -236,7 +255,11 @@ pub(crate) fn decode(bytes: &[u8], mode: Mode) -> Result<RawImage> {
};
let cfa = match (ifd0.u64s(t::CFA_REPEAT_PATTERN_DIM).as_deref(), ifd0.bytes(t::CFA_PATTERN_EP)) {
(Some([2, 2]), Some(p)) if p.len() == 4 && p.iter().all(|&c| c <= 2) => Cfa { width: 2, height: 2, pattern: p.to_vec() },
_ => Cfa::bayer_static("BGGR"),
// Every body states its layout in the Exif `CFAPattern`, read from the first sample of the stored array, not
// of the image area: the *ist D (origin 19, 11) and the 645D (80, 59) only fit unshifted. The 12-bit 10 MP
// bodies and everything from the K-3 on say RGGB; the K-7 / K-5 / 645D generation says BGGR. Files without the
// tag keep BGGR.
_ => cfa_from_exif(&tiff).unwrap_or_else(|| Cfa::bayer_static("BGGR")),
};
let black = match mn.as_ref().and_then(|m| m.ifd.f64s(BLACK_POINT)).as_deref() {
Some(v @ [_, _, _, _]) => {
@ -343,4 +366,126 @@ mod tests {
assert!(decode_huffman(&src[..src.len() / 3], &h, w, hgt, 14).is_err());
assert!(table(&tb[..20], ByteOrder::Big).is_err());
}
/// A big-endian one-strip PEF-style TIFF of `w × h` 12-bit samples with the given compression tag and strip
/// byte count, around `payload`.
fn tiny_pef(w: u16, h: u16, compression: u16, strip_len: u32, payload: &[u8]) -> Vec<u8> {
let entries: [(u16, u16, u32); 9] = [
(0x0100, 3, w as u32 * 0x1_0000), // ImageWidth
(0x0101, 3, h as u32 * 0x1_0000), // ImageLength
(0x0102, 3, 12 * 0x1_0000), // BitsPerSample
(0x0103, 3, compression as u32 * 0x1_0000), // Compression
(0x0106, 3, 32803 * 0x1_0000), // Photometric: CFA
(0x0111, 4, 8 + 2 + 9 * 12 + 4), // StripOffsets
(0x0115, 3, 0x1_0000), // SamplesPerPixel
(0x0116, 3, h as u32 * 0x1_0000), // RowsPerStrip
(0x0117, 4, strip_len), // StripByteCounts
];
let mut f = b"MM\0*\0\0\0\x08".to_vec();
f.extend_from_slice(&9u16.to_be_bytes());
for (tag, ty, val) in entries {
f.extend_from_slice(&tag.to_be_bytes());
f.extend_from_slice(&ty.to_be_bytes());
f.extend_from_slice(&1u32.to_be_bytes());
f.extend_from_slice(&val.to_be_bytes());
}
f.extend_from_slice(&0u32.to_be_bytes()); // no next IFD
f.extend_from_slice(payload);
f
}
/// Eight 12-bit samples, two rows of four, packed MSB-first: 0x123 0x456 0x789 0xabc / 0xdef 0x012 0x345 0x678.
const PACKED: [u8; 12] = [0x12, 0x34, 0x56, 0x78, 0x9a, 0xbc, 0xde, 0xf0, 0x12, 0x34, 0x56, 0x78];
#[test]
fn packed_samples_under_the_packbits_tag_are_read_when_the_strip_has_exactly_the_packed_size() {
let file = tiny_pef(4, 2, 32773, 12, &PACKED);
let img = decode(&file, Mode::Full).unwrap();
let RawData::U16(v) = &img.data else { panic!("integer samples") };
assert_eq!(v, &[0x123, 0x456, 0x789, 0xabc, 0xdef, 0x012, 0x345, 0x678]);
assert_eq!((img.width, img.height, img.bits), (4, 2, 12));
// the same bytes tagged as plain uncompressed give the same samples
let plain = decode(&tiny_pef(4, 2, 1, 12, &PACKED), Mode::Full).unwrap();
assert_eq!(plain.data, img.data);
// header-only mode accepts it too
assert!(decode(&file, Mode::Header).is_ok());
}
#[test]
fn a_packbits_tagged_strip_of_any_other_size_stays_unsupported() {
for len in [11u32, 13, 6, 24] {
let mut payload = PACKED.to_vec();
payload.resize(24, 0x55);
let file = tiny_pef(4, 2, 32773, len, &payload);
let err = decode(&file, Mode::Full).unwrap_err();
assert!(matches!(&err, RawError::Unsupported(m) if m.contains("32773")), "strip of {len} bytes: {err:?}");
}
}
/// A 16 x 8 big-endian 12-bit PEF-style file with a 12 x 6 image area at the odd origin (3, 1), the Exif
/// `CFAPattern` `pattern` and an AOC maker note with the area tags (and, for 65535, a Huffman table). The samples
/// are plain 16-bit words (1), packed 12-bit (32773) or Huffman coded (65535).
fn pef_with_pattern(compression: u16, pattern: [u8; 4]) -> Vec<u8> {
use lightcraft_tiff::{IfdBuilder, ImageData, TiffWriter, Value};
let (w, h) = (16usize, 8usize);
let px: Vec<u16> = (0..w * h).map(|i| ((i * 37) % 900 + 100) as u16).collect();
let (tb, codes) = test_table();
let strip: Vec<u8> = match compression {
1 => px.iter().flat_map(|v| v.to_be_bytes()).collect(),
32773 => px.chunks(2).flat_map(|p| [(p[0] >> 4) as u8, ((p[0] & 15) << 4 | p[1] >> 8) as u8, p[1] as u8]).collect(),
_ => encode(&px, w, &codes),
};
let build = |table_at: u32| {
let mut ifd = IfdBuilder::new();
ifd.set(0x0100, Value::Long(vec![w as u32]));
ifd.set(0x0101, Value::Long(vec![h as u32]));
ifd.set(0x0102, Value::Short(vec![12]));
ifd.set(0x0103, Value::Short(vec![compression]));
ifd.set(0x0106, Value::Short(vec![32803]));
ifd.set(0x010f, Value::Ascii("PENTAX Corporation".into()));
ifd.set_image(ImageData::Strips { rows_per_strip: h as u32, strips: vec![strip.clone()] });
// AOC + "MM", then the IFD (area origin, area size and, for Huffman data, the table), next-IFD 0, the table
let mut note = b"AOCMM".to_vec();
let mut entries = vec![(0x0038u16, 3u16, 2u32, [0u8, 3, 0, 1]), (0x0039, 3, 2, [0, 12, 0, 6])];
if compression == 65535 {
entries.push((0x0220, 7, tb.len() as u32, table_at.to_be_bytes()));
}
note.extend_from_slice(&(entries.len() as u16).to_be_bytes());
for (tag, ty, cnt, val) in entries {
note.extend_from_slice(&tag.to_be_bytes());
note.extend_from_slice(&ty.to_be_bytes());
note.extend_from_slice(&cnt.to_be_bytes());
note.extend_from_slice(&val);
}
note.extend_from_slice(&0u32.to_be_bytes());
if compression == 65535 {
note.extend_from_slice(&tb);
}
let mut exif = IfdBuilder::new();
exif.set(0xa302, Value::Undefined([&[0, 2, 0, 2][..], &pattern[..]].concat()));
exif.set(0x927c, Value::Undefined(note));
ifd.set_child(0x8769, exif);
TiffWriter::new(ByteOrder::Big, false).write(&[ifd]).unwrap()
};
let first = build(0);
if compression != 65535 {
return first;
}
let at = Tiff::parse(&first).unwrap().exif().unwrap().get(0x927c).unwrap().offset;
build((at + 6 + 2 + 3 * 12 + 4) as u32)
}
#[test]
fn the_pattern_is_the_exif_cfapattern_from_the_array_origin_for_every_compression() {
for compression in [1u16, 32773, 65535] {
for (p, name) in [([0u8, 1, 1, 2], "RGGB"), ([2, 1, 1, 0], "BGGR"), ([1, 0, 2, 1], "GRBG"), ([1, 2, 0, 1], "GBRG")] {
let img = decode(&pef_with_pattern(compression, p), Mode::Full).unwrap();
assert_eq!(img.active_area, Rect::new(3, 1, 12, 6), "compression {compression}");
assert_eq!(img.cfa, Some(Cfa::bayer_static(name)), "compression {compression}, Exif {name}");
}
}
// no tag: the assumed layout stays
let img = decode(&tiny_pef(4, 2, 1, 16, &[0; 16]), Mode::Full).unwrap();
assert_eq!(img.cfa, Some(Cfa::bayer_static("BGGR")));
}
}

31
crates/raw/tests/cr3_corpus.rs
View File

@ -15,7 +15,7 @@ fn corpus() -> PathBuf {
std::env::var_os("LIGHTCRAFT_CORPUS").map(PathBuf::from).unwrap_or_else(|| Path::new(env!("CARGO_MANIFEST_DIR")).join("../../corpus"))
}
fn check(name: &str, hash: u64, sensor: (usize, usize)) {
fn check(name: &str, hash: u64, sensor: (usize, usize), developed: (usize, usize)) {
let path = corpus().join("raw").join(name);
let Ok(bytes) = std::fs::read(&path) else {
eprintln!("skip: {} absent", path.display());
@ -25,7 +25,7 @@ fn check(name: &str, hash: u64, sensor: (usize, usize)) {
assert_eq!(raw.format, RawFormat::Cr3);
raw.validate().unwrap();
assert_eq!((raw.width, raw.height, raw.cpp), (sensor.0, sensor.1, 1));
assert_eq!(raw.info().developed_size(), (6000, 4000));
assert_eq!(raw.info().developed_size(), developed);
assert_eq!(raw.info(), probe_info(&bytes).unwrap());
let RawData::U16(samples) = &raw.data else { panic!("{name}: integer sensor data expected") };
let actual =
@ -39,40 +39,55 @@ fn check(name: &str, hash: u64, sensor: (usize, usize)) {
fn canon_m50_lossless_sensor_is_exact() {
// raw.pixls.us/getfile.php/4657; SHA256
// 1ba9ad6b51b315b88820eb0fe5fd7f52bc136ba37b1a5ca76825ebfe25b18058
check("cr3-canon-m50-raw.cr3", 0x6226_1f0b_a81c_fcd2, (6288, 4056));
check("cr3-canon-m50-raw.cr3", 0x6226_1f0b_a81c_fcd2, (6288, 4056), (6000, 4000));
}
#[test]
fn canon_r100_lossless_sensor_is_exact() {
// raw.pixls.us/getfile.php/7896; SHA256
// 6d83217d58a5e6d2dabcf470430e91a16676b60531c025b81fa7453fc74b0521
check("cr3-canon-r100-raw.cr3", 0xcbe5_299a_b9c5_2630, (6288, 4056));
check("cr3-canon-r100-raw.cr3", 0xcbe5_299a_b9c5_2630, (6288, 4056), (6000, 4000));
}
#[test]
fn canon_m50_craw_sensor_is_exact() {
// raw.pixls.us/getfile.php/2663; SHA256
// 15384b775867ec4c42b11882837f1e368cedc0561832ffab271221e6bb80be4c
check("cr3-canon-m50-craw.cr3", 0x9aac_bfe6_6f50_5e1b, (6288, 4056));
// The camera was set to 1:1: the file's AspectInfo is a 4000 x 4000 square at x = 1000 inside the 6000 x 4000
// image (the archive's "3:2" label is wrong). The sensor samples are unchanged; only the crop is.
check("cr3-canon-m50-craw.cr3", 0x9aac_bfe6_6f50_5e1b, (6288, 4056), (4000, 4000));
}
#[test]
fn canon_r100_craw_sensor_is_exact() {
// raw.pixls.us/getfile.php/7897; SHA256
// 0b66842b2fe00329ebc05fe8ab9357ddbbe1a0a2d2e69a2893102de7fdf5c942
check("cr3-canon-r100-craw.cr3", 0x341c_706b_37c3_8bbf, (6288, 4056));
check("cr3-canon-r100-craw.cr3", 0x341c_706b_37c3_8bbf, (6288, 4056), (6000, 4000));
}
#[test]
fn canon_r8_lossless_sensor_is_exact() {
// raw.pixls.us/getfile.php/6585; SHA256
// 7d5c6dbb11ff7e6ee58715d90a20f5d801e8b103f4711ff6ea329ceafcef254c
check("cr3-canon-r8-raw.cr3", 0x63c1_c6d8_d1eb_2312, (6188, 4120));
check("cr3-canon-r8-raw.cr3", 0x63c1_c6d8_d1eb_2312, (6188, 4120), (6000, 4000));
}
#[test]
fn canon_r8_craw_sensor_is_exact() {
// raw.pixls.us/getfile.php/6587; SHA256
// df33cf394573645ce03dea1b2e9f0b5cc2b7734e9e3391ee7114151414cf2812
check("cr3-canon-r8-craw.cr3", 0xf836_a663_a795_a375, (6188, 4120));
check("cr3-canon-r8-craw.cr3", 0xf836_a663_a795_a375, (6188, 4120), (6000, 4000));
}
#[test]
fn canon_m50_craw_crop_is_the_shot_aspect() {
// raw.pixls.us/getfile.php/2663: the 1:1 square starts 1000 px right of the recommended crop's origin,
// which is (12, 12) inside the valid sensor area.
let path = corpus().join("raw").join("cr3-canon-m50-craw.cr3");
let Ok(bytes) = std::fs::read(&path) else {
eprintln!("skip: {} absent", path.display());
return;
};
let raw = decode(&bytes).unwrap();
assert_eq!((raw.crop.x, raw.crop.y, raw.crop.width, raw.crop.height), (1012, 12, 4000, 4000));
}

18
crates/segment/src/bin/lightcraft-sam3-worker.rs Normal file
View File

@ -0,0 +1,18 @@
//! Serves SAM 3 on this Mac's Metal GPU to one LightCraft editor over a loopback socket (an SSH
//! tunnel); see docs/ai-masks.md → Remote Metal inference.
#![forbid(unsafe_code)]
#![deny(clippy::unwrap_used, clippy::expect_used, clippy::panic, clippy::unimplemented, clippy::todo, clippy::unreachable)]
#[cfg(not(target_arch = "wasm32"))]
fn main() -> Result<(), Box<dyn std::error::Error>> {
let mut args = std::env::args().skip(1);
let dir = args.next().ok_or("usage: lightcraft-sam3-worker MODEL_DIR [127.0.0.1:8793]")?;
let address = args.next().unwrap_or_else(|| "127.0.0.1:8793".into()).parse()?;
let device = candle_core::Device::new_metal(0)?;
lightcraft_segment::remote::serve(address, std::path::Path::new(&dir), device)?;
Ok(())
}
/// The worker serves a local GPU over a socket; there is nothing to serve in a browser.
#[cfg(target_arch = "wasm32")]
fn main() {}

2
crates/segment/src/lib.rs
View File

@ -37,6 +37,8 @@ pub mod mask;
mod neck;
#[cfg(not(target_arch = "wasm32"))]
mod nn;
#[cfg(not(target_arch = "wasm32"))]
pub mod remote;
pub mod tokenizer;
#[cfg(not(target_arch = "wasm32"))]
mod tracker;

207
crates/segment/src/remote.rs Normal file
View File

@ -0,0 +1,207 @@
//! Private SAM 3 transport. Bind the worker to loopback and forward it over SSH.
use std::io::{self, Read, Write};
use std::net::{SocketAddr, TcpListener, TcpStream};
use std::path::Path;
use std::time::Duration;
use crate::{Click, Encoded, INPUT, MASK_SIDE, Sam3};
use serde::{Deserialize, Serialize};
const CONTROL_LIMIT: usize = 16 * 1024;
const RESPONSE_LIMIT: usize = 4 * 1024 * 1024;
const TIMEOUT: Duration = Duration::from_secs(9 * 60);
pub const ENV: &str = "LIGHTCRAFT_SAM3_REMOTE";
#[derive(Serialize, Deserialize)]
pub enum Operation {
Prepare,
Clicks(Vec<[f32; 3]>),
Text(String),
}
#[derive(Serialize, Deserialize)]
pub struct Request {
pub width: usize,
pub height: usize,
pub temporary: bool,
pub operation: Operation,
}
impl Request {
pub fn validate(&self) -> Result<(), String> {
if self.width > INPUT || self.height > INPUT || (self.width == 0) != (self.height == 0) {
return Err("invalid SAM image dimensions".into());
}
if self.temporary && self.width == 0 {
return Err("detail pass requires an image".into());
}
match &self.operation {
Operation::Prepare => {}
Operation::Text(text) if text.len() <= 4096 && text.split([',', ';']).any(|p| !p.trim().is_empty()) => {}
Operation::Clicks(clicks)
if !clicks.is_empty()
&& clicks.len() <= 64
&& clicks.iter().all(|c| {
c.iter().all(|v| v.is_finite()) && (0.0..=1.0).contains(&c[0]) && (0.0..=1.0).contains(&c[1]) && (c[2] == 0.0 || c[2] == 1.0)
})
&& clicks.iter().any(|c| c[2] == 1.0) => {}
_ => return Err("invalid SAM prompt".into()),
}
Ok(())
}
}
pub fn configured() -> Option<String> {
std::env::var(ENV).ok().filter(|v| !v.trim().is_empty())
}
pub fn read_frame(reader: &mut impl Read, limit: usize) -> io::Result<Vec<u8>> {
let mut size = [0; 4];
reader.read_exact(&mut size)?;
let size = u32::from_be_bytes(size) as usize;
if size > limit {
return Err(io::Error::new(io::ErrorKind::InvalidData, "SAM frame exceeds limit"));
}
let mut data = vec![0; size];
reader.read_exact(&mut data)?;
Ok(data)
}
fn write_frame(writer: &mut impl Write, data: &[u8]) -> io::Result<()> {
let size = u32::try_from(data.len()).map_err(io::Error::other)?;
writer.write_all(&size.to_be_bytes())?;
writer.write_all(data)
}
pub struct Client {
stream: TcpStream,
pub key: Option<u64>,
}
impl Client {
pub fn connect(endpoint: &str) -> Result<Self, String> {
let address: SocketAddr = endpoint.parse().map_err(|e| format!("SAM endpoint must be an IP:port: {e}"))?;
if !address.ip().is_loopback() {
return Err("SAM endpoint must be loopback; use an SSH tunnel for remote access".into());
}
let stream = TcpStream::connect_timeout(&address, Duration::from_secs(5)).map_err(|e| e.to_string())?;
stream.set_read_timeout(Some(TIMEOUT)).map_err(|e| e.to_string())?;
stream.set_write_timeout(Some(Duration::from_secs(30))).map_err(|e| e.to_string())?;
Ok(Self { stream, key: None })
}
pub fn request(&mut self, request: &Request, rgb: &[u8]) -> Result<Option<Vec<f32>>, String> {
request.validate()?;
if rgb.len() != request.width * request.height * 3 {
return Err("SAM RGB buffer size does not match dimensions".into());
}
let bytes = serde_json::to_vec(request).map_err(|e| e.to_string())?;
write_frame(&mut self.stream, &bytes).map_err(|e| e.to_string())?;
write_frame(&mut self.stream, rgb).map_err(|e| e.to_string())?;
let response = read_frame(&mut self.stream, RESPONSE_LIMIT).map_err(|e| e.to_string())?;
let result: Result<Option<Vec<f32>>, String> = serde_json::from_slice(&response).map_err(|e| e.to_string())?;
let logits = result?;
if logits.as_ref().is_some_and(|v| v.len() != MASK_SIDE * MASK_SIDE || v.iter().any(|n| !n.is_finite())) {
return Err("invalid SAM mask response".into());
}
Ok(logits)
}
}
pub fn text_logits(model: &mut Sam3, enc: &mut Encoded, text: &str) -> Result<Option<Vec<f32>>, String> {
let mut merged: Option<Vec<f32>> = None;
for phrase in text.split([',', ';']).map(str::trim).filter(|p| !p.is_empty()) {
if let Some(p) = model.segment_text(enc, phrase, 0.5).map_err(|e| e.to_string())? {
if let Some(m) = &mut merged {
for (a, b) in m.iter_mut().zip(p.logits) {
*a = a.max(b);
}
} else {
merged = Some(p.logits);
}
}
}
Ok(merged)
}
fn predict(model: &mut Sam3, enc: &mut Encoded, operation: Operation) -> Result<Option<Vec<f32>>, String> {
match operation {
Operation::Prepare => {
model.prepare_clicks(enc).map_err(|e| e.to_string())?;
Ok(None)
}
Operation::Text(text) => text_logits(model, enc, &text),
Operation::Clicks(points) => {
let clicks: Vec<_> = points.into_iter().map(|p| Click { x: p[0], y: p[1], positive: p[2] == 1.0 }).collect();
Ok(Some(model.segment_clicks(enc, &clicks).map_err(|e| e.to_string())?.logits))
}
}
}
fn serve_client(stream: &mut TcpStream, dir: &Path, device: candle_core::Device) -> Result<(), String> {
stream.set_read_timeout(Some(Duration::from_secs(10 * 60))).map_err(|e| e.to_string())?;
stream.set_write_timeout(Some(Duration::from_secs(30))).map_err(|e| e.to_string())?;
let mut model = None;
let mut cache = None;
loop {
let frame = read_frame(stream, CONTROL_LIMIT).map_err(|e| e.to_string())?;
let request: Request = serde_json::from_slice(&frame).map_err(|e| e.to_string())?;
request.validate()?;
let rgb = read_frame(stream, request.width * request.height * 3).map_err(|e| e.to_string())?;
if rgb.len() != request.width * request.height * 3 {
return Err("invalid RGB payload".into());
}
let started = std::time::Instant::now();
let result = (|| {
if model.is_none() {
model = Some(Sam3::load_on(dir, device.clone()).map_err(|e| e.to_string())?);
}
let m = model.as_mut().ok_or("SAM model unavailable")?;
let mut temporary = None;
if request.width > 0 {
let encoded = m.encode(&rgb, request.width, request.height).map_err(|e| e.to_string())?;
if request.temporary {
temporary = Some(encoded);
} else {
cache = Some(encoded);
}
}
let enc = if request.temporary { temporary.as_mut() } else { cache.as_mut() }.ok_or("SAM image cache empty")?;
predict(m, enc, request.operation)
})();
eprintln!(
"SAM3 Metal request elapsed={:.3}s image={}x{} detail={} success={}",
started.elapsed().as_secs_f64(),
request.width,
request.height,
request.temporary,
result.is_ok()
);
let response = serde_json::to_vec(&result).map_err(|e| e.to_string())?;
write_frame(stream, &response).map_err(|e| e.to_string())?;
if result.is_err() {
return Err("SAM inference failed; session released".into());
}
}
}
pub fn serve(address: SocketAddr, dir: &Path, device: candle_core::Device) -> Result<(), String> {
if !address.ip().is_loopback() || !device.is_metal() {
return Err("SAM worker requires a loopback address and a Metal device".into());
}
if !crate::is_model_dir(dir) {
return Err("SAM model files are missing".into());
}
let listener = TcpListener::bind(address).map_err(|e| e.to_string())?;
eprintln!("SAM3 worker listening on {address}; device=Metal; idle unload=600s");
for stream in listener.incoming() {
let mut stream = stream.map_err(|e| e.to_string())?;
let result = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| serve_client(&mut stream, dir, device.clone())));
match result {
Ok(Err(error)) => eprintln!("SAM3 session closed: {error}"),
Err(_) => eprintln!("SAM3 session panicked; model released"),
Ok(Ok(())) => {}
}
}
Ok(())
}

21
crates/segment/tests/remote.rs Normal file
View File

@ -0,0 +1,21 @@
use lightcraft_segment::remote::{Operation, Request, read_frame};
use std::io::Cursor;
#[test]
fn rejects_unbounded_and_truncated_frames() {
assert!(read_frame(&mut Cursor::new(u32::MAX.to_be_bytes()), 1024).is_err());
assert!(read_frame(&mut Cursor::new([0, 0, 0, 3, 1]), 1024).is_err());
}
#[test]
fn validates_images_and_prompt_boundaries() {
let request = |width, height, operation| Request { width, height, temporary: false, operation };
assert!(request(1009, 1, Operation::Prepare).validate().is_err());
assert!(request(0, 1, Operation::Prepare).validate().is_err());
assert!(request(0, 0, Operation::Text("dog".into())).validate().is_ok());
assert!(request(4, 3, Operation::Clicks(vec![[0.5, 0.5, 1.0]])).validate().is_ok());
assert!(request(4, 3, Operation::Clicks(vec![[1.5, 0.5, 1.0]])).validate().is_err());
assert!(request(4, 3, Operation::Clicks(vec![[0.5, f32::NAN, 1.0]])).validate().is_err());
assert!(request(4, 3, Operation::Clicks(vec![[0.5, 0.5, 0.0]])).validate().is_err());
assert!(request(4, 3, Operation::Text(" ".into())).validate().is_err());
}

1
crates/ui-egui/Cargo.toml
View File

@ -25,6 +25,7 @@ serde_json = { workspace = true }
log = { workspace = true }
[dev-dependencies]
lightcraft-denoise = { workspace = true, features = ["runtime"] }
lightcraft-codecs = { workspace = true }
lightcraft-faces = { workspace = true }

21
crates/ui-egui/locales/de-formats.json
View File

@ -187,5 +187,24 @@
"Moved; {} photo(s) relinked": "Verschoben; {} Fotos neu verknüpft",
"{} label (click more labels to show any of them)": "Markierung {} (weitere Markierungen auswählen, um alle passenden Fotos zu zeigen)",
", {} star{}": ", Sterne: {}{:.0}",
", {} label": ", Markierung: {}"
", {} label": ", Markierung: {}",
"Downloads from {host}. Once it has arrived and checked out, LightCraft installs it, {after}.": "Downloads from {host}. Once it has arrived and checked out, LightCraft installs it, {after}.",
"{:.1} MB": "{:.1} MB",
"Running on the processor: {}.": "Running on the processor: {}.",
"{} · {}. Kept in the library's “denoise” folder; the oldest ones that no photo uses go first when the limit is reached.": "{} · {}. Kept in the library's “denoise” folder; the oldest ones that no photo uses go first when the limit is reached.",
"Denoising {name}: {done} of {total} tiles{}": "Denoising {name}: {done} of {total} tiles{}",
"{size} GB": "{size} GB",
"{} of {} from {host}": "{} of {} from {host}",
"{} KB": "{} KB",
"{} MB": "{} MB",
"Making the denoised picture… {}%": "Making the denoised picture… {}%",
"{name} · {terms}": "{name} · {terms}",
"{n} {noun}": "{n} {noun}",
"Trained on: {provenance}": "Trained on: {provenance}",
" · {queued} waiting": " · {queued} waiting",
"Shows the model's terms, then downloads {} from {host}, installs it and starts using it. Nothing else is sent.": "Shows the model's terms, then downloads {} from {host}, installs it and starts using it. Nothing else is sent.",
"In line for AI Denoise ({ahead} ahead)…": "In line for AI Denoise ({ahead} ahead)…",
"Running on {}.": "Running on {}.",
"{file} · {size}": "{file} · {size}",
"• {assumption}": "• {assumption}"
}

91
crates/ui-egui/locales/de.json
View File

@ -1798,5 +1798,94 @@
"Install the SAM 3 Model": "SAM-3-Modell installieren",
"This build can't download the model yet. To install it by hand, put model.safetensors, vocab.json and merges.txt from Meta's facebook/sam3 in the folder above: Object and Describe work as soon as they are there.": "Diese Version kann das Modell noch nicht herunterladen. Zum Installieren von Hand lege model.safetensors, vocab.json und merges.txt aus Metas facebook/sam3 in den obigen Ordner: Objekt und Beschreiben funktionieren, sobald sie dort liegen.",
"Open Log Folder": "Protokollordner öffnen",
"Back to this person": "Back to this person"
"Back to this person": "Back to this person",
"AI Denoise": "AI Denoise",
"AI Denoise Model": "AI Denoise Model",
"AI Denoise is off: press Use on a model.": "AI Denoise is off: press Use on a model.",
"AI Denoise needs a model. Install one in Settings.": "AI Denoise needs a model. Install one in Settings.",
"Accept & Download": "Accept & Download",
"Already installed. Remove it before installing a replacement.": "Already installed. Remove it before installing a replacement.",
"Automatic uses the graphics card where it is faster than the processor: both are timed on this computer the first time a photo is made.": "Automatic uses the graphics card where it is faster than the processor: both are timed on this computer the first time a photo is made.",
"Cache": "Cache",
"Checking and installing the model…": "Checking and installing the model…",
"Checking…": "Checking…",
"Clear cache": "Clear cache",
"Denoise Models": "Denoise Models",
"Denoise Settings": "Denoise Settings",
"Denoise all photos with an Amount": "Denoise all photos with an Amount",
"Downloaded and checked. Installing…": "Downloaded and checked. Installing…",
"Failed its last test": "Failed its last test",
"Graphics card": "Graphics card",
"I have read these terms and accept them for my own use": "I have read these terms and accept them for my own use",
"Inspect Denoise Model": "Inspect Denoise Model",
"Install Denoise Model": "Install Denoise Model",
"Install from file…": "Install from file…",
"Install your own ONNX model with denoise-model.json beside it. This build offers no model downloads.": "Install your own ONNX model with denoise-model.json beside it. This build offers no model downloads.",
"Its denoised picture is not made yet, and pictures are not made on their own (Settings, AI Denoise).": "Its denoised picture is not made yet, and pictures are not made on their own (Settings, AI Denoise).",
"LightCraft does not know this model, so it assumed:": "LightCraft does not know this model, so it assumed:",
"Limit": "Limit",
"Make it now": "Make it now",
"Make the denoised picture of the photos I am looking at": "Make the denoised picture of the photos I am looking at",
"Next up for AI Denoise…": "Next up for AI Denoise…",
"Nothing is fetched until you accept. The model is kept on this computer only; LightCraft never uploads or shares it.": "Nothing is fetched until you accept. The model is kept on this computer only; LightCraft never uploads or shares it.",
"Only photos whose AI Denoise amount is above 0 are made. Exports always make theirs if it is missing.": "Only photos whose AI Denoise amount is above 0 are made. Exports always make theirs if it is missing.",
"Open page": "Open page",
"Open the model's page": "Open the model's page",
"Opens the model's own page in your browser, to read about it or get the file yourself.": "Opens the model's own page in your browser, to read about it or get the file yourself.",
"Processor": "Processor",
"Set up AI Denoise…": "Set up AI Denoise…",
"Speed": "Speed",
"Starting…": "Starting…",
"Tested and works on this computer": "Tested and works on this computer",
"The denoise model is installed and in use: the AI Denoise Amount slider under Detail now works": "The denoise model is installed and in use: the AI Denoise Amount slider under Detail now works",
"The graphics card is set up when the next photo is made.": "The graphics card is set up when the next photo is made.",
"The model is kept on this computer only. LightCraft never uploads or shares it.": "The model is kept on this computer only. LightCraft never uploads or shares it.",
"The processor does the work.": "The processor does the work.",
"There is no model file to install": "There is no model file to install",
"This build cannot run denoise models.": "This build cannot run denoise models.",
"This build has nowhere to keep denoise models, so AI Denoise is not available. (The desktop app has.)": "This build has nowhere to keep denoise models, so AI Denoise is not available. (The desktop app has.)",
"This file cannot be used yet": "This file cannot be used yet",
"Tick the box to accept the model's terms first": "Tick the box to accept the model's terms first",
"Try again": "Try again",
"Try the graphics card again": "Try the graphics card again",
"Turn off": "Turn off",
"Unknown licence": "Unknown licence",
"Use": "Use",
"Waiting to start…": "Waiting to start…",
"Works on raw files from cameras with a standard (Bayer) sensor. Other files keep their normal noise reduction.": "Works on raw files from cameras with a standard (Bayer) sensor. Other files keep their normal noise reduction.",
"cached picture": "cached picture",
"cached pictures": "cached pictures",
"commercial use allowed": "commercial use allowed",
"denoiseModel.accept": "denoiseModel.accept",
"in use": "in use",
"installed": "installed",
"makes AI Denoise available": "makes AI Denoise available",
"non-commercial use only": "non-commercial use only",
"photo has its denoised picture.": "photo has its denoised picture.",
"photo waiting": "photo waiting",
"photos have their denoised picture.": "photos have their denoised picture.",
"photos waiting": "photos waiting",
"starts using it for AI Denoise": "starts using it for AI Denoise",
"terms unclear": "terms unclear",
"the graphics card": "the graphics card",
"the graphics card is not used": "the graphics card is not used",
"Cancel Denoise": "Cancel Denoise",
"Cancel Denoise Model Download": "Cancel Denoise Model Download",
"Choose Denoise Model": "Choose Denoise Model",
"Clear Denoise Cache": "Clear Denoise Cache",
"Denoise Model Downloads": "Denoise Model Downloads",
"Denoise Photos": "Denoise Photos",
"Denoise Pump": "Denoise Pump",
"Denoise Status": "Denoise Status",
"Download Denoise Model": "Download Denoise Model",
"Remove Denoise Model": "Remove Denoise Model",
"Test Denoise Model": "Test Denoise Model",
"AI Denoise is enabled on the requesting photo": "AI Denoise is enabled on the requesting photo",
"After installation, AI Denoise will turn on for the requesting photo.": "After installation, AI Denoise will turn on for the requesting photo.",
"Cancel pending action": "Cancel pending action",
"Install a denoise model to enable AI Denoise on this photo": "Install a denoise model to enable AI Denoise on this photo",
"Pending photo action cancelled": "Pending photo action cancelled",
"Pending photo action cancelled because its library or photo changed": "Pending photo action cancelled because its library or photo changed",
"ONNX model": "ONNX model",
"commercial use subject to GPL conditions": "commercial use subject to GPL conditions"
}

21
crates/ui-egui/locales/es-formats.json
View File

@ -187,5 +187,24 @@
"Moved; {} photo(s) relinked": "Movida; {} foto(s) vinculada(s) de nuevo",
"{} label (click more labels to show any of them)": "Etiqueta {} (hacer clic en más etiquetas para mostrar cualquiera de ellas)",
", {} star{}": ", {} estrella{}",
", {} label": ", etiqueta {}"
", {} label": ", etiqueta {}",
"Downloads from {host}. Once it has arrived and checked out, LightCraft installs it, {after}.": "Downloads from {host}. Once it has arrived and checked out, LightCraft installs it, {after}.",
"{:.1} MB": "{:.1} MB",
"Running on the processor: {}.": "Running on the processor: {}.",
"{} · {}. Kept in the library's “denoise” folder; the oldest ones that no photo uses go first when the limit is reached.": "{} · {}. Kept in the library's “denoise” folder; the oldest ones that no photo uses go first when the limit is reached.",
"Denoising {name}: {done} of {total} tiles{}": "Denoising {name}: {done} of {total} tiles{}",
"{size} GB": "{size} GB",
"{} of {} from {host}": "{} of {} from {host}",
"{} KB": "{} KB",
"{} MB": "{} MB",
"Making the denoised picture… {}%": "Making the denoised picture… {}%",
"{name} · {terms}": "{name} · {terms}",
"{n} {noun}": "{n} {noun}",
"Trained on: {provenance}": "Trained on: {provenance}",
" · {queued} waiting": " · {queued} waiting",
"Shows the model's terms, then downloads {} from {host}, installs it and starts using it. Nothing else is sent.": "Shows the model's terms, then downloads {} from {host}, installs it and starts using it. Nothing else is sent.",
"In line for AI Denoise ({ahead} ahead)…": "In line for AI Denoise ({ahead} ahead)…",
"Running on {}.": "Running on {}.",
"{file} · {size}": "{file} · {size}",
"• {assumption}": "• {assumption}"
}

91
crates/ui-egui/locales/es.json
View File

@ -1643,5 +1643,94 @@
"Captured until": "Capturadas hasta",
"Only": "Solo",
"Open Log Folder": "Abrir carpeta de registros",
"Back to this person": "Back to this person"
"Back to this person": "Back to this person",
"AI Denoise": "AI Denoise",
"AI Denoise Model": "AI Denoise Model",
"AI Denoise is off: press Use on a model.": "AI Denoise is off: press Use on a model.",
"AI Denoise needs a model. Install one in Settings.": "AI Denoise needs a model. Install one in Settings.",
"Accept & Download": "Accept & Download",
"Already installed. Remove it before installing a replacement.": "Already installed. Remove it before installing a replacement.",
"Automatic uses the graphics card where it is faster than the processor: both are timed on this computer the first time a photo is made.": "Automatic uses the graphics card where it is faster than the processor: both are timed on this computer the first time a photo is made.",
"Cache": "Cache",
"Checking and installing the model…": "Checking and installing the model…",
"Checking…": "Checking…",
"Clear cache": "Clear cache",
"Denoise Models": "Denoise Models",
"Denoise Settings": "Denoise Settings",
"Denoise all photos with an Amount": "Denoise all photos with an Amount",
"Downloaded and checked. Installing…": "Downloaded and checked. Installing…",
"Failed its last test": "Failed its last test",
"Graphics card": "Graphics card",
"I have read these terms and accept them for my own use": "I have read these terms and accept them for my own use",
"Inspect Denoise Model": "Inspect Denoise Model",
"Install Denoise Model": "Install Denoise Model",
"Install from file…": "Install from file…",
"Install your own ONNX model with denoise-model.json beside it. This build offers no model downloads.": "Install your own ONNX model with denoise-model.json beside it. This build offers no model downloads.",
"Its denoised picture is not made yet, and pictures are not made on their own (Settings, AI Denoise).": "Its denoised picture is not made yet, and pictures are not made on their own (Settings, AI Denoise).",
"LightCraft does not know this model, so it assumed:": "LightCraft does not know this model, so it assumed:",
"Limit": "Limit",
"Make it now": "Make it now",
"Make the denoised picture of the photos I am looking at": "Make the denoised picture of the photos I am looking at",
"Next up for AI Denoise…": "Next up for AI Denoise…",
"Nothing is fetched until you accept. The model is kept on this computer only; LightCraft never uploads or shares it.": "Nothing is fetched until you accept. The model is kept on this computer only; LightCraft never uploads or shares it.",
"Only photos whose AI Denoise amount is above 0 are made. Exports always make theirs if it is missing.": "Only photos whose AI Denoise amount is above 0 are made. Exports always make theirs if it is missing.",
"Open page": "Open page",
"Open the model's page": "Open the model's page",
"Opens the model's own page in your browser, to read about it or get the file yourself.": "Opens the model's own page in your browser, to read about it or get the file yourself.",
"Processor": "Processor",
"Set up AI Denoise…": "Set up AI Denoise…",
"Speed": "Speed",
"Starting…": "Starting…",
"Tested and works on this computer": "Tested and works on this computer",
"The denoise model is installed and in use: the AI Denoise Amount slider under Detail now works": "The denoise model is installed and in use: the AI Denoise Amount slider under Detail now works",
"The graphics card is set up when the next photo is made.": "The graphics card is set up when the next photo is made.",
"The model is kept on this computer only. LightCraft never uploads or shares it.": "The model is kept on this computer only. LightCraft never uploads or shares it.",
"The processor does the work.": "The processor does the work.",
"There is no model file to install": "There is no model file to install",
"This build cannot run denoise models.": "This build cannot run denoise models.",
"This build has nowhere to keep denoise models, so AI Denoise is not available. (The desktop app has.)": "This build has nowhere to keep denoise models, so AI Denoise is not available. (The desktop app has.)",
"This file cannot be used yet": "This file cannot be used yet",
"Tick the box to accept the model's terms first": "Tick the box to accept the model's terms first",
"Try again": "Try again",
"Try the graphics card again": "Try the graphics card again",
"Turn off": "Turn off",
"Unknown licence": "Unknown licence",
"Use": "Use",
"Waiting to start…": "Waiting to start…",
"Works on raw files from cameras with a standard (Bayer) sensor. Other files keep their normal noise reduction.": "Works on raw files from cameras with a standard (Bayer) sensor. Other files keep their normal noise reduction.",
"cached picture": "cached picture",
"cached pictures": "cached pictures",
"commercial use allowed": "commercial use allowed",
"denoiseModel.accept": "denoiseModel.accept",
"in use": "in use",
"installed": "installed",
"makes AI Denoise available": "makes AI Denoise available",
"non-commercial use only": "non-commercial use only",
"photo has its denoised picture.": "photo has its denoised picture.",
"photo waiting": "photo waiting",
"photos have their denoised picture.": "photos have their denoised picture.",
"photos waiting": "photos waiting",
"starts using it for AI Denoise": "starts using it for AI Denoise",
"terms unclear": "terms unclear",
"the graphics card": "the graphics card",
"the graphics card is not used": "the graphics card is not used",
"Cancel Denoise": "Cancel Denoise",
"Cancel Denoise Model Download": "Cancel Denoise Model Download",
"Choose Denoise Model": "Choose Denoise Model",
"Clear Denoise Cache": "Clear Denoise Cache",
"Denoise Model Downloads": "Denoise Model Downloads",
"Denoise Photos": "Denoise Photos",
"Denoise Pump": "Denoise Pump",
"Denoise Status": "Denoise Status",
"Download Denoise Model": "Download Denoise Model",
"Remove Denoise Model": "Remove Denoise Model",
"Test Denoise Model": "Test Denoise Model",
"AI Denoise is enabled on the requesting photo": "AI Denoise is enabled on the requesting photo",
"After installation, AI Denoise will turn on for the requesting photo.": "After installation, AI Denoise will turn on for the requesting photo.",
"Cancel pending action": "Cancel pending action",
"Install a denoise model to enable AI Denoise on this photo": "Install a denoise model to enable AI Denoise on this photo",
"Pending photo action cancelled": "Pending photo action cancelled",
"Pending photo action cancelled because its library or photo changed": "Pending photo action cancelled because its library or photo changed",
"ONNX model": "ONNX model",
"commercial use subject to GPL conditions": "commercial use subject to GPL conditions"
}

Some files were not shown because too many files have changed in this diff Show More