M7.12: natural Text to Speech voices (Kokoro-82M in pure Rust)
- filmcraft-tts (feature kokoro): Kokoro-82M on candle, implemented from
the StyleTTS 2 / iSTFTNet / ALBERT papers, config.json and the
checkpoint's tensor shapes; official .pth read with candle's restricted
pickle reader; voice packs from their zip. Neural Voice: tts-text
phonemize -> Kokoro per chunk -> pauses -> pitch via audio-dsp.
- catalog: 9 US English voices (model card grade C+ or better) + CMUdict,
every file pinned to a revision and SHA-256 (336 MB).
- speech: generic download_files (same verified downloader).
- engine: tts.render (background synthesis into Session::tts_cache),
tts.downloadVoices (background job, cancellable), voices list with
package state; feature neural-voices (default on in the desktop app).
- ui: natural/basic voice groups, consent box (size, sources, licences),
download progress + Cancel, synthesis never blocks the window;
ui.set {tts:{...}} for agents.
Measured: Whisper-base WER on 50 original sentences 2.1-3.6 % for all 9
voices (built-in robotic voice 96.6 %); 2.2-2.3x real time on CPU.
Gates: fmt, clippy, layers, assets, wasm pass; tests pass except
shortcuts_tests::assign_reassigns_conflicts_and_undoes (fails on main)
and the frame pool test (flaky under load, passes alone).
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Manuel Camacho
parent
6367fc1444
commit
beebdbcd39
|
|
@ -2217,8 +2217,14 @@ dependencies = [
|
|||
name = "filmcraft-tts"
|
||||
version = "0.3.0"
|
||||
dependencies = [
|
||||
"candle-core",
|
||||
"candle-nn",
|
||||
"filmcraft-audio-dsp",
|
||||
"filmcraft-tts-text",
|
||||
"serde",
|
||||
"serde_json",
|
||||
"thiserror 2.0.21",
|
||||
"zip",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
|
|
|
|||
|
|
@ -11,9 +11,11 @@ build = "build.rs"
|
|||
workspace = true
|
||||
|
||||
[features]
|
||||
default = []
|
||||
default = ["neural-voices"]
|
||||
# Speech-to-text (Text panel ▸ Transcribe) with Whisper on the CPU, and model downloads.
|
||||
whisper = ["filmcraft-engine/whisper", "filmcraft-engine/speech-download"]
|
||||
# Natural Text to Speech voices (Kokoro-82M on the CPU, downloaded on first use).
|
||||
neural-voices = ["filmcraft-engine/neural-voices"]
|
||||
|
||||
[dependencies]
|
||||
filmcraft-engine = { workspace = true }
|
||||
|
|
|
|||
|
|
@ -15,6 +15,8 @@ default = []
|
|||
whisper = ["filmcraft-speech/whisper"]
|
||||
# Speech-model downloads over HTTPS.
|
||||
speech-download = ["filmcraft-speech/download"]
|
||||
# Natural voices for Text to Speech: Kokoro-82M on candle (CPU) and their download.
|
||||
neural-voices = ["filmcraft-tts/kokoro", "speech-download"]
|
||||
|
||||
[dependencies]
|
||||
filmcraft-time = { workspace = true }
|
||||
|
|
|
|||
|
|
@ -350,6 +350,8 @@ pub struct Session {
|
|||
pub voiceover: voiceover::VoiceOver,
|
||||
/// The last `tts.preview` result, for the host to play (Text to Speech ▸ Preview).
|
||||
pub tts_preview: Option<Arc<filmcraft_tts::Audio>>,
|
||||
/// Synthesized narrations (`tts.render` fills it from a background job).
|
||||
pub tts_cache: narration::SynthCache,
|
||||
/// Dynamic (J/K/L) trimming and trim-mode loop playback in progress.
|
||||
pub trim_play: trim::TrimPlayback,
|
||||
/// Keyboard shortcuts (active bindings, presets; `shortcuts.*` commands).
|
||||
|
|
@ -453,6 +455,7 @@ impl Session {
|
|||
mcrec: Default::default(),
|
||||
voiceover: Default::default(),
|
||||
tts_preview: None,
|
||||
tts_cache: Default::default(),
|
||||
trim_play: Default::default(),
|
||||
shortcuts: shortcuts::Shortcuts::new(),
|
||||
offline: Default::default(),
|
||||
|
|
|
|||
|
|
@ -7,6 +7,8 @@
|
|||
//! | `tts.create` | synthesize, write `Narration <n>.wav`, import it and place it at the playhead (one undo step) |
|
||||
//! | `tts.edit` | change a narration clip's script or voice settings and regenerate it in place (one undo step) |
|
||||
//! | `tts.inspect` | the narration settings of a clip or item |
|
||||
//! | `tts.render` | synthesize in a background job into the cache, so the UI's create / edit / preview return at once |
|
||||
//! | `tts.downloadVoices` | download the natural voices (Kokoro-82M, 9 US English voices, CMUdict; 336 MB, SHA-256 checked) as a background job |
|
||||
//!
|
||||
//! **Where a new narration goes.** At `time` (default: the playhead) on `track` if given (it must
|
||||
//! be free there), else on the first targeted audio track that is free for the narration's length,
|
||||
|
|
@ -23,7 +25,7 @@
|
|||
//! where voice-over recordings go (Scratch Disks ▸ Captured, else next to the project, else
|
||||
//! `Narrations` in the data or temporary directory); an edit writes next to the previous file.
|
||||
|
||||
use std::sync::Arc;
|
||||
use std::sync::{Arc, Mutex, PoisonError};
|
||||
|
||||
use serde_json::{Value, json};
|
||||
|
||||
|
|
@ -71,7 +73,7 @@ fn settings(cmd: &str, p: &Value, base: Option<&Narration>) -> Result<Narration>
|
|||
None => base.map(|b| b.text.clone()).ok_or_else(|| bad(cmd, "`text` is required"))?,
|
||||
};
|
||||
let voice = str_p(p, "voice").map(str::to_string).or_else(|| base.map(|b| b.voice.clone())).unwrap_or_else(|| tts::default_voice_id().to_string());
|
||||
let info = tts::voices().into_iter().find(|v| v.id == voice).ok_or_else(|| bad(cmd, format!("unknown voice `{voice}` (see tts.voices)")))?;
|
||||
let info = tts::voices_in(None).into_iter().find(|v| v.id == voice).ok_or_else(|| bad(cmd, format!("unknown voice `{voice}` (see tts.voices)")))?;
|
||||
let pitch = match str_p(p, "pitch") {
|
||||
Some(v) => VocalPitch::from_id(v).ok_or_else(|| bad(cmd, format!("unknown pitch `{v}` (extraLow, low, default, high, extraHigh)")))?,
|
||||
None => base.map(|b| b.pitch).unwrap_or_default(),
|
||||
|
|
@ -85,16 +87,66 @@ fn settings(cmd: &str, p: &Value, base: Option<&Narration>) -> Result<Narration>
|
|||
Ok(n)
|
||||
}
|
||||
|
||||
fn synthesize(cmd: &str, n: &Narration) -> Result<tts::Audio> {
|
||||
let voice = tts::voice(&n.voice).map_err(|e| bad(cmd, e.to_string()))?;
|
||||
let params = tts::Params { semitones: n.pitch.semitones(), pace: n.pace };
|
||||
voice.synthesize(&n.text, ¶ms).map_err(|e| bad(cmd, e.to_string()))
|
||||
/// Synthesized narrations by settings, newest last (`tts.render` fills it in the background so
|
||||
/// the UI's create / edit / preview don't block).
|
||||
pub type SynthCache = Arc<Mutex<Vec<(String, Arc<tts::Audio>)>>>;
|
||||
const CACHE_ENTRIES: usize = 6;
|
||||
|
||||
fn cache_key(n: &Narration) -> String {
|
||||
format!("{}\u{1f}{}\u{1f}{}\u{1f}{}", n.voice, n.pitch.id(), n.pace.to_bits(), n.text)
|
||||
}
|
||||
|
||||
fn tts_err(cmd: &str, e: tts::TtsError) -> EngineError {
|
||||
match e {
|
||||
tts::TtsError::NotInstalled => bad(cmd, "the natural voices are not downloaded yet: Text to Speech ▸ Download natural voices (tts.downloadVoices)"),
|
||||
e => bad(cmd, e.to_string()),
|
||||
}
|
||||
}
|
||||
|
||||
/// Synthesize `n` (no cache).
|
||||
fn render(n: &Narration, models_dir: Option<&std::path::Path>) -> std::result::Result<tts::Audio, tts::TtsError> {
|
||||
let voice = tts::voice_in(&n.voice, models_dir)?;
|
||||
voice.synthesize(&n.text, &tts::Params { semitones: n.pitch.semitones(), pace: n.pace })
|
||||
}
|
||||
|
||||
fn synthesize(s: &Session, cmd: &str, n: &Narration) -> Result<Arc<tts::Audio>> {
|
||||
let key = cache_key(n);
|
||||
if let Some(a) = s.tts_cache.lock().unwrap_or_else(PoisonError::into_inner).iter().find(|(k, _)| *k == key).map(|(_, a)| a.clone()) {
|
||||
return Ok(a);
|
||||
}
|
||||
let a = Arc::new(render(n, crate::transcript::models_dir().as_deref()).map_err(|e| tts_err(cmd, e))?);
|
||||
cache_put(&s.tts_cache, key, a.clone());
|
||||
Ok(a)
|
||||
}
|
||||
|
||||
fn cache_put(c: &SynthCache, key: String, a: Arc<tts::Audio>) {
|
||||
let mut c = c.lock().unwrap_or_else(PoisonError::into_inner);
|
||||
c.retain(|(k, _)| *k != key);
|
||||
c.push((key, a));
|
||||
let extra = c.len().saturating_sub(CACHE_ENTRIES);
|
||||
c.drain(..extra);
|
||||
}
|
||||
|
||||
fn package_json() -> Value {
|
||||
let dir = crate::transcript::models_dir();
|
||||
json!({
|
||||
"id": tts::catalog::PACKAGE_ID,
|
||||
"name": tts::catalog::PACKAGE_NAME,
|
||||
"installed": dir.as_deref().is_some_and(tts::catalog::installed),
|
||||
"size": tts::catalog::size(),
|
||||
"missingBytes": dir.as_deref().map_or(tts::catalog::size(), tts::catalog::missing_bytes),
|
||||
"license": tts::catalog::LICENSE,
|
||||
"source": tts::catalog::SOURCE,
|
||||
"available": cfg!(feature = "neural-voices"),
|
||||
})
|
||||
}
|
||||
|
||||
fn voices(_s: &mut Session, _p: &Value) -> Result<Value> {
|
||||
let dir = crate::transcript::models_dir();
|
||||
Ok(json!({
|
||||
"package": package_json(),
|
||||
"languages": tts::LANGUAGES.iter().map(|(id, name)| json!({"id": id, "name": name})).collect::<Vec<_>>(),
|
||||
"voices": tts::voices(),
|
||||
"voices": tts::voices_in(dir.as_deref()),
|
||||
"defaultVoice": tts::default_voice_id(),
|
||||
"pitches": VocalPitch::ALL.iter().map(|p| json!({"id": p.id(), "label": p.label(), "semitones": p.semitones()})).collect::<Vec<_>>(),
|
||||
"pace": {"min": tts::MIN_PACE, "max": tts::MAX_PACE, "default": 1.0},
|
||||
|
|
@ -112,7 +164,7 @@ fn preview(s: &mut Session, p: &Value) -> Result<Value> {
|
|||
q["text"] = json!(tts::SAMPLE_SENTENCE);
|
||||
}
|
||||
let n = settings(cmd, &q, base.as_ref())?;
|
||||
let audio = synthesize(cmd, &n)?;
|
||||
let audio = synthesize(s, cmd, &n)?;
|
||||
let seconds = audio.seconds();
|
||||
let samples = audio.samples.len();
|
||||
let rate = audio.sample_rate;
|
||||
|
|
@ -122,7 +174,7 @@ fn preview(s: &mut Session, p: &Value) -> Result<Value> {
|
|||
s.services.write_file(dest, &bytes).map_err(|e| EngineError::Other(format!("{dest}: {e}")))?;
|
||||
path = json!(dest);
|
||||
}
|
||||
s.tts_preview = Some(Arc::new(audio));
|
||||
s.tts_preview = Some(audio);
|
||||
Ok(json!({"seconds": seconds, "samples": samples, "sampleRate": rate, "voice": n.voice, "path": path}))
|
||||
}
|
||||
|
||||
|
|
@ -178,7 +230,7 @@ fn create(s: &mut Session, p: &Value) -> Result<Value> {
|
|||
if at < Tick::ZERO {
|
||||
return Err(bad(cmd, "the start time cannot be negative"));
|
||||
}
|
||||
let audio = synthesize(cmd, &n)?;
|
||||
let audio = synthesize(s, cmd, &n)?;
|
||||
let rate = audio.sample_rate.max(1);
|
||||
let dur = Tick::from_units(audio.samples.len() as i64, i64::from(rate));
|
||||
n.speech_duration = dur;
|
||||
|
|
@ -264,13 +316,13 @@ fn edit(s: &mut Session, p: &Value) -> Result<Value> {
|
|||
let (old_item, clip_dur, speed) = (it.item, it.duration, it.speed);
|
||||
let base = s.project.narrations.get(&old_item).cloned().ok_or_else(|| bad(cmd, "the clip is not a narration"))?;
|
||||
let mut n = settings(cmd, p, Some(&base))?;
|
||||
let audio = synthesize(cmd, &n)?;
|
||||
let audio = synthesize(s, cmd, &n)?;
|
||||
let rate = audio.sample_rate.max(1);
|
||||
let speech = audio.samples.len();
|
||||
n.speech_duration = Tick::from_units(speech as i64, i64::from(rate));
|
||||
// the clip keeps its length: pad the file with silence to what the clip plays
|
||||
let needed = needed_samples(clip_dur, speed, rate);
|
||||
let mut samples = audio.samples;
|
||||
let mut samples = audio.samples.clone();
|
||||
if needed > samples.len() {
|
||||
samples.try_reserve(needed - samples.len()).map_err(|e| bad(cmd, format!("unable to allocate the narration: {e}")))?;
|
||||
samples.resize(needed, 0.0);
|
||||
|
|
@ -324,6 +376,118 @@ fn inspect(s: &mut Session, p: &Value) -> Result<Value> {
|
|||
Ok(json!({"item": item.0, "clip": clip, "narration": n}))
|
||||
}
|
||||
|
||||
fn new_job(s: &mut Session, label: String) -> crate::Job {
|
||||
let id = s.jobs.iter().map(|j| j.id).max().unwrap_or(0) + 1;
|
||||
crate::Job { id, label, progress: Default::default(), result: Default::default() }
|
||||
}
|
||||
|
||||
fn spawn(s: &mut Session, job: crate::Job, wait: bool, name: &str, run: impl FnOnce() + Send + 'static) -> Result<u64> {
|
||||
let id = job.id;
|
||||
let run = crate::export_tools::guard_job(job.progress.clone(), job.result.clone(), run);
|
||||
s.jobs.push(job);
|
||||
if wait || cfg!(target_arch = "wasm32") {
|
||||
run();
|
||||
} else {
|
||||
std::thread::Builder::new().name(name.into()).spawn(run).map_err(|e| EngineError::Other(e.to_string()))?;
|
||||
}
|
||||
Ok(id)
|
||||
}
|
||||
|
||||
type JobResult = std::sync::Mutex<Option<std::result::Result<filmcraft_export::Report, String>>>;
|
||||
|
||||
fn finish(prog: &filmcraft_export::Progress, res: &JobResult, r: std::result::Result<String, String>, t0: std::time::Instant) {
|
||||
use std::sync::atomic::Ordering;
|
||||
let secs = t0.elapsed().as_secs_f64();
|
||||
let r = match r {
|
||||
Ok(status) => {
|
||||
*prog.status.lock().unwrap_or_else(PoisonError::into_inner) = status;
|
||||
Ok(filmcraft_export::Report { path: String::new(), frames: 0, seconds: secs, bytes: 0, render_fps: 0.0, extra_files: Vec::new() })
|
||||
}
|
||||
Err(e) => {
|
||||
*prog.status.lock().unwrap_or_else(PoisonError::into_inner) = e.clone();
|
||||
*prog.error.lock().unwrap_or_else(PoisonError::into_inner) = Some(e.clone());
|
||||
Err(e)
|
||||
}
|
||||
};
|
||||
prog.finished.store(true, Ordering::Relaxed);
|
||||
*res.lock().unwrap_or_else(PoisonError::into_inner) = Some(r);
|
||||
}
|
||||
|
||||
/// Synthesize in the background into the cache, so `tts.create` / `tts.edit` / `tts.preview` with
|
||||
/// the same settings return at once.
|
||||
fn render_job(s: &mut Session, p: &Value) -> Result<Value> {
|
||||
let cmd = "tts.render";
|
||||
let base = target_clip(s, p).and_then(|c| s.active_sequence()?.find_item(c).map(|(_, it)| it.item)).and_then(|i| s.project.narrations.get(&i).cloned());
|
||||
let mut q = p.clone();
|
||||
if q.get("sample").and_then(Value::as_bool).unwrap_or(false) {
|
||||
q["text"] = json!(tts::SAMPLE_SENTENCE);
|
||||
}
|
||||
let n = settings(cmd, &q, base.as_ref())?;
|
||||
let key = cache_key(&n);
|
||||
if s.tts_cache.lock().unwrap_or_else(PoisonError::into_inner).iter().any(|(k, _)| *k == key) {
|
||||
return Ok(json!({"job": null, "cached": true}));
|
||||
}
|
||||
let job = new_job(s, "Synthesizing narration".into());
|
||||
let (prog, res, cache) = (job.progress.clone(), job.result.clone(), s.tts_cache.clone());
|
||||
let wait = p.get("wait").and_then(Value::as_bool).unwrap_or(false);
|
||||
let id = spawn(s, job, wait, "filmcraft-tts", move || {
|
||||
let t0 = std::time::Instant::now();
|
||||
*prog.status.lock().unwrap_or_else(PoisonError::into_inner) = "Synthesizing…".into();
|
||||
let r = match render(&n, crate::transcript::models_dir().as_deref()) {
|
||||
Ok(a) => {
|
||||
let secs = a.seconds();
|
||||
cache_put(&cache, key, Arc::new(a));
|
||||
Ok(format!("{secs:.1} s of speech"))
|
||||
}
|
||||
Err(tts::TtsError::NotInstalled) => Err("the natural voices are not downloaded yet".to_string()),
|
||||
Err(e) => Err(e.to_string()),
|
||||
};
|
||||
finish(&prog, &res, r, t0);
|
||||
})?;
|
||||
Ok(json!({"job": id, "cached": false}))
|
||||
}
|
||||
|
||||
/// Download the natural voices (Kokoro-82M, its voices and CMUdict) in the background.
|
||||
fn download_voices(s: &mut Session, p: &Value) -> Result<Value> {
|
||||
let cmd = "tts.downloadVoices";
|
||||
let dir = crate::transcript::models_dir().ok_or_else(|| bad(cmd, "no data directory for voice models"))?;
|
||||
if tts::catalog::installed(&dir) {
|
||||
return Ok(json!({"job": null, "installed": true}));
|
||||
}
|
||||
if !cfg!(feature = "neural-voices") {
|
||||
return Err(bad(cmd, "natural voices are not available in this build (built without the `neural-voices` feature)"));
|
||||
}
|
||||
let wait = p.get("wait").and_then(Value::as_bool).unwrap_or(false);
|
||||
let job = new_job(s, format!("Downloading {}", tts::catalog::PACKAGE_NAME));
|
||||
job.progress.total.store(tts::catalog::missing_bytes(&dir), std::sync::atomic::Ordering::Relaxed);
|
||||
let (prog, res) = (job.progress.clone(), job.result.clone());
|
||||
let id = spawn(s, job, wait, "filmcraft-tts-download", move || {
|
||||
let t0 = std::time::Instant::now();
|
||||
let r = download_package(&dir, &prog);
|
||||
finish(&prog, &res, r.map(|()| "Natural voices installed".to_string()), t0);
|
||||
})?;
|
||||
Ok(json!({"job": id, "installed": false}))
|
||||
}
|
||||
|
||||
#[cfg(feature = "neural-voices")]
|
||||
fn download_package(dir: &std::path::Path, prog: &filmcraft_export::Progress) -> std::result::Result<(), String> {
|
||||
use std::sync::atomic::Ordering;
|
||||
let files: Vec<filmcraft_speech::models::ModelFile> =
|
||||
tts::catalog::FILES.iter().map(|f| filmcraft_speech::models::ModelFile { name: f.name, url: f.url, sha256: f.sha256, size: f.size }).collect();
|
||||
let mut progress = |done: u64, total: u64, file: &str| {
|
||||
prog.done.store(done, Ordering::Relaxed);
|
||||
prog.total.store(total, Ordering::Relaxed);
|
||||
*prog.status.lock().unwrap_or_else(PoisonError::into_inner) = format!("Downloading {file}");
|
||||
!prog.cancel.load(Ordering::Relaxed)
|
||||
};
|
||||
filmcraft_speech::models::download_files(&tts::catalog::package_dir(dir), &files, &mut progress).map_err(|e| e.to_string())
|
||||
}
|
||||
|
||||
#[cfg(not(feature = "neural-voices"))]
|
||||
fn download_package(_: &std::path::Path, _: &filmcraft_export::Progress) -> std::result::Result<(), String> {
|
||||
Err("natural voices are not available in this build".into())
|
||||
}
|
||||
|
||||
fn spec(
|
||||
id: &'static str,
|
||||
label: &'static str,
|
||||
|
|
@ -356,5 +520,14 @@ pub fn commands() -> Vec<CommandSpec> {
|
|||
),
|
||||
spec("tts.edit", "Edit Narration", r#"{"clip":id?,"text":str?,"voice":str?,"pitch":str?,"pace":f64?,"dir":str?}"#, has_narration_clip, edit, true),
|
||||
spec("tts.inspect", "Inspect Narration", r#"{"clip":id?,"item":id?}"#, always, inspect, false),
|
||||
spec(
|
||||
"tts.render",
|
||||
"Synthesize Narration",
|
||||
r#"{"text":str?,"voice":str?,"pitch":str?,"pace":f64?,"sample":bool?,"clip":id?,"wait":bool?}"#,
|
||||
always,
|
||||
render_job,
|
||||
false,
|
||||
),
|
||||
spec("tts.downloadVoices", "Download Natural Voices", r#"{"wait":bool?}"#, always, download_voices, false),
|
||||
]
|
||||
}
|
||||
|
|
|
|||
|
|
@ -347,3 +347,52 @@ fn the_narration_is_heard_in_the_mix() {
|
|||
assert!(rms > 0.01, "narration audible, rms {rms}");
|
||||
let _ = std::fs::remove_dir_all(&dir);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn render_fills_the_cache_that_create_and_preview_use() {
|
||||
let mut s = demo();
|
||||
let dir = tmp("render");
|
||||
let p = json!({"text": "Rendered in the background.", "voice": "basic-male", "pace": 1.1});
|
||||
let r = s.execute("tts.render", json!({"text": "Rendered in the background.", "voice": "basic-male", "pace": 1.1, "wait": true})).unwrap();
|
||||
let job = r["job"].as_u64().expect("a job");
|
||||
let j = s.jobs.iter().find(|j| j.id == job).unwrap();
|
||||
assert!(j.progress.finished.load(std::sync::atomic::Ordering::Relaxed));
|
||||
assert!(matches!(&*j.result.lock().unwrap(), Some(Ok(_))));
|
||||
// the same settings are now cached: render again is a no-op, preview hands back the same audio
|
||||
assert_eq!(s.execute("tts.render", p.clone()).unwrap()["cached"], true);
|
||||
s.execute("tts.preview", p.clone()).unwrap();
|
||||
let cached = s.tts_cache.lock().unwrap().last().unwrap().1.clone();
|
||||
assert!(std::sync::Arc::ptr_eq(s.tts_preview.as_ref().unwrap(), &cached));
|
||||
let mut q = p.clone();
|
||||
q["dir"] = json!(dir);
|
||||
q["time"] = json!(0);
|
||||
let c = s.execute("tts.create", q).unwrap();
|
||||
assert_eq!(c["speechDuration"], Tick::from_units(cached.samples.len() as i64, RATE).0);
|
||||
// a failing render reports the error in the job
|
||||
let r = s.execute("tts.render", json!({"text": "x", "voice": "kokoro-heart", "wait": true}));
|
||||
if let Ok(r) = r {
|
||||
let job = r["job"].as_u64().unwrap();
|
||||
let j = s.jobs.iter().find(|j| j.id == job).unwrap();
|
||||
assert!(matches!(&*j.result.lock().unwrap(), Some(Err(_))) || cfg!(feature = "neural-voices"));
|
||||
}
|
||||
let _ = std::fs::remove_dir_all(&dir);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn natural_voices_are_listed_and_explain_what_is_missing() {
|
||||
let mut s = demo();
|
||||
let v = s.execute("tts.voices", json!({})).unwrap();
|
||||
let ids: Vec<&str> = v["voices"].as_array().unwrap().iter().map(|x| x["id"].as_str().unwrap()).collect();
|
||||
assert!(ids.contains(&"kokoro-heart") && ids.contains(&"kokoro-michael"), "{ids:?}");
|
||||
assert_eq!(v["package"]["id"], "kokoro-82m");
|
||||
assert!(v["package"]["size"].as_u64().unwrap() > 300_000_000);
|
||||
assert_eq!(v["package"]["available"], cfg!(feature = "neural-voices"));
|
||||
if !v["package"]["installed"].as_bool().unwrap() {
|
||||
let e = s.execute("tts.create", json!({"text": "Hi.", "voice": "kokoro-heart"})).unwrap_err().to_string();
|
||||
assert!(e.contains("download") || e.contains("not available"), "{e}");
|
||||
}
|
||||
if !cfg!(feature = "neural-voices") && !v["package"]["installed"].as_bool().unwrap() {
|
||||
let e = s.execute("tts.downloadVoices", json!({})).unwrap_err().to_string();
|
||||
assert!(e.contains("not available in this build"), "{e}");
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -222,15 +222,22 @@ pub type DownloadProgress<'a> = &'a mut dyn FnMut(u64, u64, &str) -> bool;
|
|||
/// verifier.
|
||||
#[cfg(feature = "download")]
|
||||
pub fn download(models_dir: &Path, m: &ModelInfo, progress: DownloadProgress) -> Result<(), crate::SpeechError> {
|
||||
download_files(&model_dir(models_dir, m), m.files, progress)
|
||||
}
|
||||
|
||||
/// [`download`] for any pinned file list (the neural voices use it too): missing files of
|
||||
/// `files` go into `dir`, each verified against its SHA-256 and size before it is renamed into
|
||||
/// place.
|
||||
#[cfg(feature = "download")]
|
||||
pub fn download_files(dir: &Path, files: &[ModelFile], progress: DownloadProgress) -> Result<(), crate::SpeechError> {
|
||||
use crate::SpeechError;
|
||||
use sha2::Digest;
|
||||
use std::io::{Read, Write};
|
||||
let dir = model_dir(models_dir, m);
|
||||
std::fs::create_dir_all(&dir)?;
|
||||
let total = missing_bytes(models_dir, m);
|
||||
std::fs::create_dir_all(dir)?;
|
||||
let total: u64 = files.iter().filter(|f| !std::fs::metadata(dir.join(f.name)).is_ok_and(|md| md.len() == f.size)).map(|f| f.size).sum();
|
||||
let agent = agent();
|
||||
let mut done = 0u64;
|
||||
for f in m.files {
|
||||
for f in files {
|
||||
let dest = dir.join(f.name);
|
||||
if std::fs::metadata(&dest).is_ok_and(|md| md.len() == f.size) {
|
||||
continue;
|
||||
|
|
|
|||
|
|
@ -1,6 +1,6 @@
|
|||
[package]
|
||||
name = "filmcraft-tts"
|
||||
description = "FilmCraft text to speech: the Voice trait, the voice catalogue, script parsing (pause markers) and the built-in formant voice"
|
||||
description = "FilmCraft text to speech: the Voice trait, the voice catalogue, script parsing (pause markers), the built-in formant voice and (feature kokoro) Kokoro-82M neural voices on candle"
|
||||
version.workspace = true
|
||||
edition.workspace = true
|
||||
license.workspace = true
|
||||
|
|
@ -9,6 +9,25 @@ rust-version.workspace = true
|
|||
[lints]
|
||||
workspace = true
|
||||
|
||||
[features]
|
||||
default = []
|
||||
# Kokoro-82M neural voices on the CPU via candle (pure Rust). Off by default and in wasm builds.
|
||||
kokoro = ["dep:candle-core", "dep:candle-nn", "dep:zip", "dep:serde_json", "dep:filmcraft-tts-text", "dep:filmcraft-audio-dsp"]
|
||||
|
||||
[dependencies]
|
||||
filmcraft-tts-text = { workspace = true, optional = true }
|
||||
filmcraft-audio-dsp = { workspace = true, optional = true }
|
||||
serde = { workspace = true }
|
||||
thiserror = { workspace = true }
|
||||
candle-core = { version = "0.9", default-features = false, optional = true }
|
||||
candle-nn = { version = "0.9", default-features = false, optional = true }
|
||||
zip = { version = "7", default-features = false, optional = true }
|
||||
serde_json = { workspace = true, optional = true }
|
||||
|
||||
[[example]]
|
||||
name = "kokoro_say"
|
||||
required-features = ["kokoro"]
|
||||
|
||||
[[example]]
|
||||
name = "narrate"
|
||||
required-features = ["kokoro"]
|
||||
|
|
|
|||
|
|
@ -11,16 +11,50 @@ clip behaviour: `crates/engine/src/narration.rs`.
|
|||
formant synthesizer: spelling rules → phone targets → glottal pulse train through three two-pole
|
||||
resonators, plus filtered noise for fricatives and bursts. No download, works in the web build.
|
||||
|
||||
Everything is deterministic. No dependencies beyond `serde` and `thiserror`.
|
||||
- `kokoro` (feature `kokoro`): **Kokoro-82M neural voices** in pure Rust on candle (CPU): ALBERT
|
||||
text encoder, StyleTTS 2 prosody predictor (durations, F0, energy) and decoder, iSTFTNet
|
||||
vocoder with a harmonic-plus-noise source. Weights are read from the official PyTorch
|
||||
checkpoint with candle's restricted pickle reader (no code from the file runs); voice packs from
|
||||
their zip archive. `config.json` is checked against the architecture this code implements.
|
||||
- `neural` (feature `kokoro`): the natural voices as `Voice`s: script → `filmcraft-tts-text`
|
||||
(normalizer, CMUdict + letter-to-sound, ≤ 400-symbol chunks, pauses) → Kokoro per chunk → exact
|
||||
silence for pauses, 120 ms between sentences → vocal pitch with `filmcraft-audio-dsp`'s pitch
|
||||
shifter (Kokoro has no pitch input). Pace is Kokoro's speed.
|
||||
- `catalog`: the natural-voice package (Kokoro-82M, 9 US English voices graded C+ or better on the
|
||||
model card, CMUdict; 336 MB), every file pinned to a revision and SHA-256. Downloaded by the
|
||||
engine (`tts.downloadVoices`) after the user confirms; never bundled or committed.
|
||||
|
||||
## Status and limitations (M7.9)
|
||||
Everything is deterministic (the vocoder's noise is seeded). Without features the crate depends
|
||||
only on `serde` and `thiserror` (and checks for wasm).
|
||||
|
||||
- The built-in voices are **robotic**, and their spelling rules are crude: English words with
|
||||
irregular spelling are mispronounced, and numbers are read digit by digit. They are the fallback
|
||||
and test voice. Natural voices (Kokoro-82M, Apache-2.0, downloaded on first use) come in M7.12,
|
||||
with the clean-room text normalizer (`crates/tts-text`, M7.10) and English pronunciation from
|
||||
CMUdict (M7.11).
|
||||
- English (United States) only.
|
||||
## Measured quality and speed (M7.12, 2026-10-08)
|
||||
|
||||
50 original sentences (`tests/data/sentences.txt`: numbers, money, times, dates, phone numbers,
|
||||
acronyms, names) spoken by each voice through the full pipeline, transcribed by Whisper-base
|
||||
(FilmCraft's own, `filmcraft-speech`), word error rate after lowercasing and removing punctuation
|
||||
(number formatting such as "six" vs "6" still counts as an error):
|
||||
|
||||
| Voice | WER | | Voice | WER |
|
||||
|---|---|---|---|---|
|
||||
| Heart | 2.1 % | | Puck | 2.8 % |
|
||||
| Aoede | 2.4 % | | Michael | 3.2 % |
|
||||
| Nicole | 2.8 % | | Kore | 3.4 % |
|
||||
| Bella | 3.0 % | | Fenrir, Sarah | 3.6 % |
|
||||
| Basic Female (built in) | 96.6 % | | | |
|
||||
|
||||
Speed: 2.2–2.3× real time on one desktop CPU (Linux x86-64, load average ≈ 15 from parallel
|
||||
builds), e.g. 186 s of speech in 84 s; model load 0.5 s. Reproduce with the `narrate` example and
|
||||
`filmcraft-speech`'s `transcribe` example.
|
||||
|
||||
## Limitations
|
||||
|
||||
- The built-in voices are **robotic** and nearly unintelligible to a recogniser (above); they are
|
||||
the no-download fallback and the test voice.
|
||||
- English (United States) only. Words outside CMUdict go through letter-to-sound rules and are
|
||||
often mispronounced (see `crates/tts-text/README.md`); homographs take the first pronunciation.
|
||||
- About 2× faster than real time on the CPU: a one-minute narration takes about 30 s. The UI runs
|
||||
synthesis as a background job.
|
||||
- Kokoro's model card states the weights are Apache-2.0 but the repository has no LICENSE file.
|
||||
- Vocal pitch shifts the voice's base pitch (and its formants slightly); pace scales every duration.
|
||||
|
||||
## Tests
|
||||
|
|
@ -31,9 +65,19 @@ length (±5 % / ±10 %), measured pitch (autocorrelation) of the female voice is
|
|||
+6 semitones raises it by √2 (±0.12); empty, emoji-only, non-Latin, pause-only and over-long scripts
|
||||
are refused before rendering; hostile scripts never panic.
|
||||
|
||||
Model-dependent tests run when `FILMCRAFT_KOKORO_DIR` names a downloaded package directory
|
||||
(determinism, whole frames, pace changes length, refusals); the loaders have hostile-input tests
|
||||
(truncated or oversized voice data, wrong byte order, NaN, non-zip files, mismatched config).
|
||||
|
||||
## References
|
||||
|
||||
- G. E. Peterson and H. L. Barney, "Control Methods Used in a Study of the Vowels", JASA 24 (1952):
|
||||
the published average formant frequencies used for the vowels.
|
||||
- D. H. Klatt, "Software for a cascade/parallel formant synthesizer", JASA 67 (1980): the standard
|
||||
two-pole digital resonator form. No code was read or copied; everything else is original.
|
||||
- Kokoro (implemented from papers, `config.json` and the checkpoint's tensor names and shapes; no
|
||||
reference code read): Y. A. Li et al., "StyleTTS 2" (NeurIPS 2023); T. Kaneko et al., "iSTFTNet"
|
||||
(ICASSP 2022); Z. Lan et al., "ALBERT" (ICLR 2020); J. Kong et al., "HiFi-GAN" (NeurIPS 2020);
|
||||
X. Wang et al., "Neural source-filter waveform models" (2019); L. Ziyin et al., "Neural Networks
|
||||
Fail to Learn Periodic Functions and How to Fix It" (Snake, NeurIPS 2020). Weights: hexgrad,
|
||||
Kokoro-82M, Apache-2.0.
|
||||
|
|
|
|||
|
|
@ -0,0 +1,46 @@
|
|||
//! `cargo run -p filmcraft-tts --release --features kokoro --example kokoro_say -- <model dir> <voice.pt> "<phonemes>" out.wav [speed]`
|
||||
//!
|
||||
//! Speaks Kokoro phonemes with a voice pack and writes a 24 kHz mono 32-bit float WAV.
|
||||
|
||||
fn main() -> Result<(), Box<dyn std::error::Error>> {
|
||||
let a: Vec<String> = std::env::args().collect();
|
||||
let (Some(dir), Some(voice), Some(ph), Some(out)) = (a.get(1), a.get(2), a.get(3), a.get(4)) else {
|
||||
eprintln!("usage: kokoro_say <model dir> <voice.pt> <phonemes> <out.wav> [speed]");
|
||||
std::process::exit(2);
|
||||
};
|
||||
let speed: f64 = a.get(5).and_then(|s| s.parse().ok()).unwrap_or(1.0);
|
||||
let t0 = std::time::Instant::now();
|
||||
let k = filmcraft_tts::kokoro::Kokoro::load(std::path::Path::new(dir))?;
|
||||
let v = filmcraft_tts::kokoro::VoicePack::load(std::path::Path::new(voice))?;
|
||||
let t1 = std::time::Instant::now();
|
||||
let s = k.speak(ph, &v, speed, 1)?;
|
||||
let t2 = std::time::Instant::now();
|
||||
let rate = filmcraft_tts::kokoro::KOKORO_RATE;
|
||||
let mut w = Vec::with_capacity(44 + s.len() * 4);
|
||||
let data = (s.len() * 4) as u32;
|
||||
w.extend_from_slice(b"RIFF");
|
||||
w.extend_from_slice(&(36 + data).to_le_bytes());
|
||||
w.extend_from_slice(b"WAVEfmt ");
|
||||
w.extend_from_slice(&16u32.to_le_bytes());
|
||||
w.extend_from_slice(&3u16.to_le_bytes());
|
||||
w.extend_from_slice(&1u16.to_le_bytes());
|
||||
w.extend_from_slice(&rate.to_le_bytes());
|
||||
w.extend_from_slice(&(rate * 4).to_le_bytes());
|
||||
w.extend_from_slice(&4u16.to_le_bytes());
|
||||
w.extend_from_slice(&32u16.to_le_bytes());
|
||||
w.extend_from_slice(b"data");
|
||||
w.extend_from_slice(&data.to_le_bytes());
|
||||
for x in &s {
|
||||
w.extend_from_slice(&x.to_le_bytes());
|
||||
}
|
||||
std::fs::write(out, w)?;
|
||||
let secs = s.len() as f64 / f64::from(rate);
|
||||
eprintln!(
|
||||
"load {:.2}s, synth {:.2}s for {secs:.2}s of audio (×{:.1} real time), peak {:.3}",
|
||||
(t1 - t0).as_secs_f64(),
|
||||
(t2 - t1).as_secs_f64(),
|
||||
secs / (t2 - t1).as_secs_f64(),
|
||||
s.iter().fold(0f32, |m, x| m.max(x.abs()))
|
||||
);
|
||||
Ok(())
|
||||
}
|
||||
|
|
@ -0,0 +1,49 @@
|
|||
//! `cargo run -p filmcraft-tts --release --features kokoro --example narrate -- <models dir> <voice id> <script.txt> <out.wav>`
|
||||
//!
|
||||
//! Speaks a script file with any FilmCraft voice (built in or natural) through the full pipeline
|
||||
//! (normalizer, pronunciation, Kokoro) and writes a mono 32-bit float WAV.
|
||||
|
||||
fn main() -> Result<(), Box<dyn std::error::Error>> {
|
||||
let a: Vec<String> = std::env::args().collect();
|
||||
let (Some(models), Some(id), Some(script), Some(out)) = (a.get(1), a.get(2), a.get(3), a.get(4)) else {
|
||||
eprintln!("usage: narrate <models dir> <voice id> <script.txt> <out.wav>");
|
||||
std::process::exit(2);
|
||||
};
|
||||
let text = std::fs::read_to_string(script)?;
|
||||
let t0 = std::time::Instant::now();
|
||||
let v = filmcraft_tts::voice_in(id, Some(std::path::Path::new(models)))?;
|
||||
let t1 = std::time::Instant::now();
|
||||
let audio = v.synthesize(&text, &filmcraft_tts::Params::default())?;
|
||||
let t2 = std::time::Instant::now();
|
||||
let rate = audio.sample_rate;
|
||||
let data = (audio.samples.len() * 4) as u32;
|
||||
let mut w = Vec::with_capacity(44 + data as usize);
|
||||
for part in [
|
||||
&b"RIFF"[..],
|
||||
&(36 + data).to_le_bytes(),
|
||||
b"WAVEfmt ",
|
||||
&16u32.to_le_bytes(),
|
||||
&3u16.to_le_bytes(),
|
||||
&1u16.to_le_bytes(),
|
||||
&rate.to_le_bytes(),
|
||||
&(rate * 4).to_le_bytes(),
|
||||
&4u16.to_le_bytes(),
|
||||
&32u16.to_le_bytes(),
|
||||
b"data",
|
||||
&data.to_le_bytes(),
|
||||
] {
|
||||
w.extend_from_slice(part);
|
||||
}
|
||||
for x in &audio.samples {
|
||||
w.extend_from_slice(&x.to_le_bytes());
|
||||
}
|
||||
std::fs::write(out, w)?;
|
||||
eprintln!(
|
||||
"{id}: load {:.2}s, synth {:.2}s for {:.2}s of audio (×{:.1} real time)",
|
||||
(t1 - t0).as_secs_f64(),
|
||||
(t2 - t1).as_secs_f64(),
|
||||
audio.seconds(),
|
||||
audio.seconds() / (t2 - t1).as_secs_f64()
|
||||
);
|
||||
Ok(())
|
||||
}
|
||||
|
|
@ -0,0 +1,154 @@
|
|||
//! The downloadable neural voices: Kokoro-82M, its voice packs and the CMUdict pronunciation
|
||||
//! dictionary. Every file is pinned to a revision and checked against its SHA-256 when downloaded
|
||||
//! (the downloader is `filmcraft_speech::models::download_files`, run by the engine after the
|
||||
//! user confirms). Nothing here is bundled with FilmCraft or committed to the repository.
|
||||
//!
|
||||
//! Licences: Kokoro-82M weights and voices by hexgrad, Apache-2.0 (declared on the model card; the
|
||||
//! repository has no separate LICENSE file). CMUdict, Carnegie Mellon University, BSD-style.
|
||||
//! Pins verified 2026-10-07 against the Hugging Face API and by `sha256sum`.
|
||||
|
||||
use std::path::{Path, PathBuf};
|
||||
|
||||
use serde::Serialize;
|
||||
|
||||
use crate::{Gender, VoiceInfo};
|
||||
|
||||
/// One file of the neural voice package.
|
||||
#[derive(Clone, Copy, Debug, Serialize)]
|
||||
pub struct ModelFile {
|
||||
/// Local file name inside the package directory.
|
||||
pub name: &'static str,
|
||||
pub url: &'static str,
|
||||
pub sha256: &'static str,
|
||||
pub size: u64,
|
||||
}
|
||||
|
||||
/// Directory name of the package under the models directory.
|
||||
pub const PACKAGE_ID: &str = "kokoro-82m";
|
||||
pub const PACKAGE_NAME: &str = "Natural voices (Kokoro-82M)";
|
||||
pub const LICENSE: &str = "Apache-2.0 (Kokoro-82M by hexgrad); CMUdict: BSD-style (Carnegie Mellon University)";
|
||||
pub const SOURCE: &str = "https://huggingface.co/hexgrad/Kokoro-82M";
|
||||
|
||||
macro_rules! hf {
|
||||
($name:literal, $path:literal, $sha:literal, $size:expr) => {
|
||||
ModelFile {
|
||||
name: $name,
|
||||
url: concat!("https://huggingface.co/hexgrad/Kokoro-82M/resolve/f3ff3571791e39611d31c381e3a41a3af07b4987/", $path),
|
||||
sha256: $sha,
|
||||
size: $size,
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
pub static FILES: &[ModelFile] = &[
|
||||
hf!("config.json", "config.json", "5abb01e2403b072bf03d04fde160443e209d7a0dad49a423be15196b9b43c17f", 2351),
|
||||
hf!("kokoro-v1_0.pth", "kokoro-v1_0.pth", "496dba118d1a58f5f3db2efc88dbdc216e0483fc89fe6e47ee1f2c53f18ad1e4", 327_212_226),
|
||||
ModelFile {
|
||||
name: "cmudict.dict",
|
||||
url: "https://raw.githubusercontent.com/cmusphinx/cmudict/74790861f652b15e4ac49015a90074ad62a27690/cmudict.dict",
|
||||
sha256: "81917843c7f44ce2b094ac63873c2c7a4cf802040792c455ba3ca406891c3d22",
|
||||
size: 3_618_488,
|
||||
},
|
||||
hf!("af_heart.pt", "voices/af_heart.pt", "0ab5709b8ffab19bfd849cd11d98f75b60af7733253ad0d67b12382a102cb4ff", 523_425),
|
||||
hf!("af_bella.pt", "voices/af_bella.pt", "8cb64e02fcc8de0327a8e13817e49c76c945ecf0052ceac97d3081480e8e48d6", 523_425),
|
||||
hf!("af_nicole.pt", "voices/af_nicole.pt", "c5561808bcf5250fe8c5f5de32caf2d94f27e57e95befdb098c5c85991d4c5da", 523_430),
|
||||
hf!("af_aoede.pt", "voices/af_aoede.pt", "c03bd1a4c3716c2d8eaa3d50022f62d5c31cfbd6e15933a00b17fefe13841cc4", 523_425),
|
||||
hf!("af_kore.pt", "voices/af_kore.pt", "8bfbc512321c3db49dff984ac675fa5ac7eaed5a96cc31104d3a9080e179d69d", 523_420),
|
||||
hf!("af_sarah.pt", "voices/af_sarah.pt", "49bd364ea3be9eb3e9685e8f9a15448c4883112a7c0ff7ab139fa4088b08cef9", 523_425),
|
||||
hf!("am_michael.pt", "voices/am_michael.pt", "9a443b79a4b22489a5b0ab7c651a0bcd1a30bef675c28333f06971abbd47bd37", 523_435),
|
||||
hf!("am_fenrir.pt", "voices/am_fenrir.pt", "98e507eca1db08230ae3b6232d59c10aec9630022d19accac4f5d12fcec3c37a", 523_430),
|
||||
hf!("am_puck.pt", "voices/am_puck.pt", "dd1d8973f4ce4b7d8ae407c77a435f485dabc052081b80ea75c4f30b84f36223", 523_420),
|
||||
];
|
||||
|
||||
/// A neural voice: catalogue entry and its voice-pack file.
|
||||
#[derive(Clone, Copy, Debug)]
|
||||
pub struct NeuralVoice {
|
||||
pub info: VoiceInfo,
|
||||
pub file: &'static str,
|
||||
}
|
||||
|
||||
const NEURAL_LICENSE: &str = "Apache-2.0 (Kokoro-82M by hexgrad)";
|
||||
|
||||
macro_rules! voice {
|
||||
($id:literal, $name:literal, $gender:ident, $file:literal, $desc:literal) => {
|
||||
NeuralVoice {
|
||||
info: VoiceInfo {
|
||||
id: $id,
|
||||
name: $name,
|
||||
language: "en-US",
|
||||
gender: Gender::$gender,
|
||||
engine: "neural",
|
||||
description: $desc,
|
||||
license: NEURAL_LICENSE,
|
||||
installed: false,
|
||||
},
|
||||
file: $file,
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
/// US English voices the model card grades C+ or better, best first.
|
||||
pub static VOICES: &[NeuralVoice] = &[
|
||||
voice!("kokoro-heart", "Heart", Female, "af_heart.pt", "Natural voice, warm (model card grade A)."),
|
||||
voice!("kokoro-bella", "Bella", Female, "af_bella.pt", "Natural voice, bright (grade A−)."),
|
||||
voice!("kokoro-nicole", "Nicole", Female, "af_nicole.pt", "Natural voice, soft (grade B−)."),
|
||||
voice!("kokoro-michael", "Michael", Male, "am_michael.pt", "Natural voice (grade C+)."),
|
||||
voice!("kokoro-fenrir", "Fenrir", Male, "am_fenrir.pt", "Natural voice, deep (grade C+)."),
|
||||
voice!("kokoro-puck", "Puck", Male, "am_puck.pt", "Natural voice, lively (grade C+)."),
|
||||
voice!("kokoro-aoede", "Aoede", Female, "af_aoede.pt", "Natural voice (grade C+)."),
|
||||
voice!("kokoro-kore", "Kore", Female, "af_kore.pt", "Natural voice (grade C+)."),
|
||||
voice!("kokoro-sarah", "Sarah", Female, "af_sarah.pt", "Natural voice (grade C+)."),
|
||||
];
|
||||
|
||||
pub fn find(id: &str) -> Option<&'static NeuralVoice> {
|
||||
VOICES.iter().find(|v| v.info.id == id)
|
||||
}
|
||||
|
||||
/// Total download size.
|
||||
pub fn size() -> u64 {
|
||||
FILES.iter().map(|f| f.size).sum()
|
||||
}
|
||||
|
||||
pub fn package_dir(models_dir: &Path) -> PathBuf {
|
||||
models_dir.join(PACKAGE_ID)
|
||||
}
|
||||
|
||||
/// Every file present with its pinned size (contents were verified when downloaded).
|
||||
pub fn installed(models_dir: &Path) -> bool {
|
||||
let d = package_dir(models_dir);
|
||||
FILES.iter().all(|f| std::fs::metadata(d.join(f.name)).is_ok_and(|m| m.len() == f.size))
|
||||
}
|
||||
|
||||
/// Bytes still to download.
|
||||
pub fn missing_bytes(models_dir: &Path) -> u64 {
|
||||
let d = package_dir(models_dir);
|
||||
FILES.iter().filter(|f| !std::fs::metadata(d.join(f.name)).is_ok_and(|m| m.len() == f.size)).map(|f| f.size).sum()
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn files_are_pinned_and_voices_resolve() {
|
||||
for f in FILES {
|
||||
assert_eq!(f.sha256.len(), 64, "{}", f.name);
|
||||
assert!(f.sha256.bytes().all(|b| b.is_ascii_hexdigit()));
|
||||
assert!(
|
||||
f.url.starts_with("https://")
|
||||
&& (f.url.contains("/f3ff3571791e39611d31c381e3a41a3af07b4987/") || f.url.contains("/74790861f652b15e4ac49015a90074ad62a27690/")),
|
||||
"{}",
|
||||
f.url
|
||||
);
|
||||
assert!(!f.name.contains('/'), "flat package layout: {}", f.name);
|
||||
}
|
||||
for v in VOICES {
|
||||
assert!(FILES.iter().any(|f| f.name == v.file), "{} has no file", v.info.id);
|
||||
assert_eq!(find(v.info.id).map(|x| x.file), Some(v.file));
|
||||
}
|
||||
assert!(size() > 330_000_000 && size() < 340_000_000, "{}", size());
|
||||
let dir = std::env::temp_dir().join(format!("filmcraft-tts-cat-{}", std::process::id()));
|
||||
assert!(!installed(&dir));
|
||||
assert_eq!(missing_bytes(&dir), size());
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,163 @@
|
|||
//! The small DSP pieces of the vocoder: a 20-point STFT / inverse STFT (hop 5, periodic Hann
|
||||
//! window, centred with reflection padding) and a seeded random source, so synthesis is
|
||||
//! deterministic.
|
||||
|
||||
/// FFT size, hop and the number of one-sided bins.
|
||||
pub(crate) const N_FFT: usize = 20;
|
||||
pub(crate) const HOP: usize = 5;
|
||||
pub(crate) const BINS: usize = N_FFT / 2 + 1;
|
||||
|
||||
fn hann() -> [f64; N_FFT] {
|
||||
let mut w = [0f64; N_FFT];
|
||||
for (i, v) in w.iter_mut().enumerate() {
|
||||
*v = 0.5 - 0.5 * (std::f64::consts::TAU * i as f64 / N_FFT as f64).cos();
|
||||
}
|
||||
w
|
||||
}
|
||||
|
||||
/// Reflect index `i` (may be negative or past the end) into `0..n`.
|
||||
fn reflect(i: isize, n: usize) -> usize {
|
||||
if n <= 1 {
|
||||
return 0;
|
||||
}
|
||||
let period = 2 * (n as isize - 1);
|
||||
let mut j = i.rem_euclid(period);
|
||||
if j >= n as isize {
|
||||
j = period - j;
|
||||
}
|
||||
j as usize
|
||||
}
|
||||
|
||||
/// Magnitude and phase, bin-major: `mag[b * frames + f]`. Frames = `len / HOP + 1`.
|
||||
pub(crate) fn stft(x: &[f32]) -> (Vec<f32>, Vec<f32>) {
|
||||
let n = x.len();
|
||||
let frames = n / HOP + 1;
|
||||
let w = hann();
|
||||
let half = (N_FFT / 2) as isize;
|
||||
let mut mag = vec![0f32; BINS * frames];
|
||||
let mut ph = vec![0f32; BINS * frames];
|
||||
let mut seg = [0f64; N_FFT];
|
||||
for f in 0..frames {
|
||||
for (k, s) in seg.iter_mut().enumerate() {
|
||||
let i = (f * HOP) as isize + k as isize - half;
|
||||
*s = f64::from(x.get(reflect(i, n)).copied().unwrap_or(0.0)) * w[k];
|
||||
}
|
||||
for b in 0..BINS {
|
||||
let (mut re, mut im) = (0f64, 0f64);
|
||||
for (k, s) in seg.iter().enumerate() {
|
||||
let a = -std::f64::consts::TAU * (b * k) as f64 / N_FFT as f64;
|
||||
re += s * a.cos();
|
||||
im += s * a.sin();
|
||||
}
|
||||
if let Some(m) = mag.get_mut(b * frames + f) {
|
||||
*m = re.hypot(im) as f32;
|
||||
}
|
||||
if let Some(p) = ph.get_mut(b * frames + f) {
|
||||
*p = im.atan2(re) as f32;
|
||||
}
|
||||
}
|
||||
}
|
||||
(mag, ph)
|
||||
}
|
||||
|
||||
/// Inverse of [`stft`] from magnitude and phase (bin-major); output length `HOP · (frames − 1)`.
|
||||
pub(crate) fn istft(mag: &[f32], phase: &[f32]) -> Vec<f32> {
|
||||
let frames = mag.len() / BINS;
|
||||
if frames == 0 || phase.len() < mag.len() {
|
||||
return Vec::new();
|
||||
}
|
||||
let w = hann();
|
||||
let total = N_FFT + HOP * (frames - 1);
|
||||
let mut out = vec![0f64; total];
|
||||
let mut env = vec![0f64; total];
|
||||
let mut frame = [0f64; N_FFT];
|
||||
for f in 0..frames {
|
||||
// inverse real DFT of the one-sided spectrum
|
||||
for (k, v) in frame.iter_mut().enumerate() {
|
||||
let mut acc = 0f64;
|
||||
for b in 0..BINS {
|
||||
let m = f64::from(mag.get(b * frames + f).copied().unwrap_or(0.0));
|
||||
let p = f64::from(phase.get(b * frames + f).copied().unwrap_or(0.0));
|
||||
let (re, im) = (m * p.cos(), m * p.sin());
|
||||
let a = std::f64::consts::TAU * (b * k) as f64 / N_FFT as f64;
|
||||
let c = if b == 0 || b == N_FFT / 2 { 1.0 } else { 2.0 };
|
||||
acc += c * (re * a.cos() - im * a.sin());
|
||||
}
|
||||
*v = acc / N_FFT as f64;
|
||||
}
|
||||
for k in 0..N_FFT {
|
||||
let at = f * HOP + k;
|
||||
if let (Some(o), Some(e)) = (out.get_mut(at), env.get_mut(at)) {
|
||||
*o += frame[k] * w[k];
|
||||
*e += w[k] * w[k];
|
||||
}
|
||||
}
|
||||
}
|
||||
let start = N_FFT / 2;
|
||||
let len = HOP * (frames - 1);
|
||||
(start..start + len)
|
||||
.map(|i| match (out.get(i), env.get(i)) {
|
||||
(Some(o), Some(e)) if *e > 1e-11 => (o / e) as f32,
|
||||
(Some(o), _) => *o as f32,
|
||||
_ => 0.0,
|
||||
})
|
||||
.collect()
|
||||
}
|
||||
|
||||
/// Deterministic random numbers (SplitMix64).
|
||||
pub(crate) struct Rng(u64);
|
||||
|
||||
impl Rng {
|
||||
pub(crate) fn new(seed: u64) -> Rng {
|
||||
Rng(seed ^ 0x9E37_79B9_7F4A_7C15)
|
||||
}
|
||||
fn next(&mut self) -> u64 {
|
||||
self.0 = self.0.wrapping_add(0x9E37_79B9_7F4A_7C15);
|
||||
let mut z = self.0;
|
||||
z = (z ^ (z >> 30)).wrapping_mul(0xBF58_476D_1CE4_E5B9);
|
||||
z = (z ^ (z >> 27)).wrapping_mul(0x94D0_49BB_1331_11EB);
|
||||
z ^ (z >> 31)
|
||||
}
|
||||
/// Uniform in [0, 1).
|
||||
pub(crate) fn uniform(&mut self) -> f64 {
|
||||
(self.next() >> 11) as f64 / (1u64 << 53) as f64
|
||||
}
|
||||
/// Standard normal (Box–Muller).
|
||||
pub(crate) fn normal(&mut self) -> f64 {
|
||||
let u1 = self.uniform().max(1e-12);
|
||||
let u2 = self.uniform();
|
||||
(-2.0 * u1.ln()).sqrt() * (std::f64::consts::TAU * u2).cos()
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn istft_inverts_stft() {
|
||||
let x: Vec<f32> = (0..600).map(|i| ((i as f32) * 0.07).sin() * 0.5 + ((i as f32) * 0.31).cos() * 0.2).collect();
|
||||
let (m, p) = stft(&x);
|
||||
assert_eq!(m.len(), BINS * (600 / HOP + 1));
|
||||
let y = istft(&m, &p);
|
||||
assert_eq!(y.len(), 600);
|
||||
let err = x.iter().zip(&y).map(|(a, b)| (a - b).abs()).fold(0f32, f32::max);
|
||||
assert!(err < 1e-4, "max error {err}");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn rng_is_deterministic_and_normalish() {
|
||||
let mut a = Rng::new(7);
|
||||
let mut b = Rng::new(7);
|
||||
let xs: Vec<f64> = (0..20000).map(|_| a.normal()).collect();
|
||||
assert!((0..20000).all(|i| xs[i] == b.normal()));
|
||||
let mean = xs.iter().sum::<f64>() / xs.len() as f64;
|
||||
let var = xs.iter().map(|x| (x - mean).powi(2)).sum::<f64>() / xs.len() as f64;
|
||||
assert!(mean.abs() < 0.03 && (var - 1.0).abs() < 0.05, "{mean} {var}");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn reflect_pads_like_numpy() {
|
||||
assert_eq!((-3..8).map(|i| reflect(i, 5)).collect::<Vec<_>>(), vec![3, 2, 1, 0, 1, 2, 3, 4, 3, 2, 1]);
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,362 @@
|
|||
//! Building blocks shared by the Kokoro modules. Tensors are `[1, channels, time]` unless noted;
|
||||
//! everything runs in f32 on the CPU.
|
||||
|
||||
use candle_core::{D, Tensor};
|
||||
|
||||
use super::weights::{Weights, err};
|
||||
use crate::TtsError;
|
||||
|
||||
pub(crate) type R<T> = Result<T, TtsError>;
|
||||
|
||||
/// 1-D convolution with optional bias.
|
||||
pub(crate) struct Conv {
|
||||
w: Tensor,
|
||||
b: Option<Tensor>,
|
||||
pad: usize,
|
||||
stride: usize,
|
||||
dil: usize,
|
||||
groups: usize,
|
||||
}
|
||||
|
||||
impl Conv {
|
||||
pub(crate) fn load(wt: &Weights, name: &str, pad: usize, stride: usize, dil: usize, groups: usize) -> R<Conv> {
|
||||
let w = wt.get(&format!("{name}.weight"))?;
|
||||
if w.rank() != 3 {
|
||||
return Err(err(format!("{name}: conv weight must be 3-D")));
|
||||
}
|
||||
let b = wt.get(&format!("{name}.bias")).ok();
|
||||
Ok(Conv { w, b, pad, stride, dil, groups })
|
||||
}
|
||||
/// `pad = dil · (k − 1) / 2` ("same" length for stride 1).
|
||||
pub(crate) fn same(wt: &Weights, name: &str, dil: usize) -> R<Conv> {
|
||||
let k = wt.get(&format!("{name}.weight"))?.dims().get(2).copied().unwrap_or(1);
|
||||
Conv::load(wt, name, dil * (k.saturating_sub(1)) / 2, 1, dil, 1)
|
||||
}
|
||||
pub(crate) fn fwd(&self, x: &Tensor) -> R<Tensor> {
|
||||
let y = x.conv1d(&self.w, self.pad, self.stride, self.dil, self.groups).map_err(err)?;
|
||||
add_bias(y, self.b.as_ref())
|
||||
}
|
||||
}
|
||||
|
||||
fn add_bias(y: Tensor, b: Option<&Tensor>) -> R<Tensor> {
|
||||
match b {
|
||||
Some(b) => {
|
||||
let c = b.dims1().map_err(err)?;
|
||||
y.broadcast_add(&b.reshape((1, c, 1)).map_err(err)?).map_err(err)
|
||||
}
|
||||
None => Ok(y),
|
||||
}
|
||||
}
|
||||
|
||||
/// Transposed 1-D convolution.
|
||||
pub(crate) struct ConvT {
|
||||
w: Tensor,
|
||||
b: Option<Tensor>,
|
||||
pad: usize,
|
||||
out_pad: usize,
|
||||
stride: usize,
|
||||
groups: usize,
|
||||
}
|
||||
|
||||
impl ConvT {
|
||||
pub(crate) fn load(wt: &Weights, name: &str, pad: usize, out_pad: usize, stride: usize, groups: usize) -> R<ConvT> {
|
||||
Ok(ConvT { w: wt.get(&format!("{name}.weight"))?, b: wt.get(&format!("{name}.bias")).ok(), pad, out_pad, stride, groups })
|
||||
}
|
||||
pub(crate) fn fwd(&self, x: &Tensor) -> R<Tensor> {
|
||||
let y = x.conv_transpose1d(&self.w, self.pad, self.out_pad, self.stride, 1, self.groups).map_err(err)?;
|
||||
add_bias(y, self.b.as_ref())
|
||||
}
|
||||
}
|
||||
|
||||
/// Linear layer on the last dimension.
|
||||
pub(crate) struct Linear {
|
||||
w: Tensor,
|
||||
b: Option<Tensor>,
|
||||
}
|
||||
|
||||
impl Linear {
|
||||
pub(crate) fn load(wt: &Weights, name: &str) -> R<Linear> {
|
||||
Ok(Linear { w: wt.get(&format!("{name}.weight"))?.t().map_err(err)?, b: wt.get(&format!("{name}.bias")).ok() })
|
||||
}
|
||||
pub(crate) fn fwd(&self, x: &Tensor) -> R<Tensor> {
|
||||
let y = x.broadcast_matmul(&self.w).map_err(err)?;
|
||||
match &self.b {
|
||||
Some(b) => y.broadcast_add(b).map_err(err),
|
||||
None => Ok(y),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn leaky(x: &Tensor, slope: f64) -> R<Tensor> {
|
||||
// max(x, slope · x) for 0 < slope < 1
|
||||
x.maximum(&(x * slope).map_err(err)?).map_err(err)
|
||||
}
|
||||
|
||||
/// Normalise over the last dimension (layer norm without affine).
|
||||
pub(crate) fn norm_last(x: &Tensor, eps: f64) -> R<Tensor> {
|
||||
let mean = x.mean_keepdim(D::Minus1).map_err(err)?;
|
||||
let xc = x.broadcast_sub(&mean).map_err(err)?;
|
||||
let var = xc.sqr().map_err(err)?.mean_keepdim(D::Minus1).map_err(err)?;
|
||||
xc.broadcast_div(&(var + eps).map_err(err)?.sqrt().map_err(err)?).map_err(err)
|
||||
}
|
||||
|
||||
/// Layer norm over the last dimension with weight and bias.
|
||||
pub(crate) struct LayerNorm {
|
||||
g: Tensor,
|
||||
b: Tensor,
|
||||
eps: f64,
|
||||
}
|
||||
|
||||
impl LayerNorm {
|
||||
pub(crate) fn load(wt: &Weights, gamma: &str, beta: &str, eps: f64) -> R<LayerNorm> {
|
||||
Ok(LayerNorm { g: wt.get(gamma)?, b: wt.get(beta)?, eps })
|
||||
}
|
||||
pub(crate) fn fwd(&self, x: &Tensor) -> R<Tensor> {
|
||||
norm_last(x, self.eps)?.broadcast_mul(&self.g).map_err(err)?.broadcast_add(&self.b).map_err(err)
|
||||
}
|
||||
}
|
||||
|
||||
/// Split a style projection `[1, 2C]` into `(1 + gamma, beta)`.
|
||||
fn style_affine(fc: &Linear, s: &Tensor) -> R<(Tensor, Tensor)> {
|
||||
let h = fc.fwd(s)?;
|
||||
let c2 = h.dim(1).map_err(err)?;
|
||||
let c = c2 / 2;
|
||||
let gamma = (h.narrow(1, 0, c).map_err(err)? + 1.0).map_err(err)?;
|
||||
let beta = h.narrow(1, c, c).map_err(err)?;
|
||||
Ok((gamma, beta))
|
||||
}
|
||||
|
||||
/// Adaptive instance norm: instance-normalise each channel over time, then scale and shift by the
|
||||
/// style (StyleTTS 2).
|
||||
pub(crate) struct AdaIn {
|
||||
fc: Linear,
|
||||
}
|
||||
|
||||
impl AdaIn {
|
||||
pub(crate) fn load(wt: &Weights, name: &str) -> R<AdaIn> {
|
||||
Ok(AdaIn { fc: Linear::load(wt, &format!("{name}.fc"))? })
|
||||
}
|
||||
pub(crate) fn fwd(&self, x: &Tensor, s: &Tensor) -> R<Tensor> {
|
||||
let (gamma, beta) = style_affine(&self.fc, s)?;
|
||||
let c = gamma.dim(1).map_err(err)?;
|
||||
let n = norm_last(x, 1e-5)?;
|
||||
n.broadcast_mul(&gamma.reshape((1, c, 1)).map_err(err)?).map_err(err)?.broadcast_add(&beta.reshape((1, c, 1)).map_err(err)?).map_err(err)
|
||||
}
|
||||
}
|
||||
|
||||
/// Adaptive layer norm on `[1, T, C]`: layer-normalise over channels, then style scale and shift.
|
||||
pub(crate) struct AdaLayerNorm {
|
||||
fc: Linear,
|
||||
}
|
||||
|
||||
impl AdaLayerNorm {
|
||||
pub(crate) fn load(wt: &Weights, name: &str) -> R<AdaLayerNorm> {
|
||||
Ok(AdaLayerNorm { fc: Linear::load(wt, &format!("{name}.fc"))? })
|
||||
}
|
||||
pub(crate) fn fwd(&self, x: &Tensor, s: &Tensor) -> R<Tensor> {
|
||||
let (gamma, beta) = style_affine(&self.fc, s)?;
|
||||
norm_last(x, 1e-5)?.broadcast_mul(&gamma).map_err(err)?.broadcast_add(&beta).map_err(err)
|
||||
}
|
||||
}
|
||||
|
||||
/// Residual block with adaptive instance norm (StyleTTS 2 `AdainResBlk1d`), optionally doubling
|
||||
/// the time resolution.
|
||||
pub(crate) struct AdaResBlock {
|
||||
norm1: AdaIn,
|
||||
norm2: AdaIn,
|
||||
conv1: Conv,
|
||||
conv2: Conv,
|
||||
shortcut: Option<Conv>,
|
||||
pool: Option<ConvT>,
|
||||
}
|
||||
|
||||
impl AdaResBlock {
|
||||
pub(crate) fn load(wt: &Weights, name: &str, upsample: bool) -> R<AdaResBlock> {
|
||||
let shortcut = wt.get(&format!("{name}.conv1x1.weight")).ok().map(|w| Conv { w, b: None, pad: 0, stride: 1, dil: 1, groups: 1 });
|
||||
let pool = if upsample {
|
||||
let c = wt.get(&format!("{name}.pool.weight"))?.dim(0).map_err(err)?;
|
||||
Some(ConvT::load(wt, &format!("{name}.pool"), 1, 1, 2, c)?)
|
||||
} else {
|
||||
None
|
||||
};
|
||||
Ok(AdaResBlock {
|
||||
norm1: AdaIn::load(wt, &format!("{name}.norm1"))?,
|
||||
norm2: AdaIn::load(wt, &format!("{name}.norm2"))?,
|
||||
conv1: Conv::same(wt, &format!("{name}.conv1"), 1)?,
|
||||
conv2: Conv::same(wt, &format!("{name}.conv2"), 1)?,
|
||||
shortcut,
|
||||
pool,
|
||||
})
|
||||
}
|
||||
pub(crate) fn upsamples(&self) -> bool {
|
||||
self.pool.is_some()
|
||||
}
|
||||
pub(crate) fn fwd(&self, x: &Tensor, s: &Tensor) -> R<Tensor> {
|
||||
let mut r = leaky(&self.norm1.fwd(x, s)?, 0.2)?;
|
||||
if let Some(p) = &self.pool {
|
||||
r = p.fwd(&r)?;
|
||||
}
|
||||
r = self.conv1.fwd(&r)?;
|
||||
r = leaky(&self.norm2.fwd(&r, s)?, 0.2)?;
|
||||
r = self.conv2.fwd(&r)?;
|
||||
let mut sc = x.clone();
|
||||
if self.pool.is_some() {
|
||||
sc = upsample2(&sc)?;
|
||||
}
|
||||
if let Some(c) = &self.shortcut {
|
||||
sc = c.fwd(&sc)?;
|
||||
}
|
||||
((r + sc).map_err(err)? / std::f64::consts::SQRT_2).map_err(err)
|
||||
}
|
||||
}
|
||||
|
||||
/// Nearest-neighbour ×2 along time.
|
||||
pub(crate) fn upsample2(x: &Tensor) -> R<Tensor> {
|
||||
let (b, c, t) = x.dims3().map_err(err)?;
|
||||
x.unsqueeze(3).map_err(err)?.broadcast_as((b, c, t, 2)).map_err(err)?.reshape((b, c, t * 2)).map_err(err)
|
||||
}
|
||||
|
||||
/// Snake activation `x + sin²(αx) / α` with a learned α per channel.
|
||||
pub(crate) fn snake(x: &Tensor, alpha: &Tensor) -> R<Tensor> {
|
||||
let ax = x.broadcast_mul(alpha).map_err(err)?;
|
||||
let s2 = ax.sin().map_err(err)?.sqr().map_err(err)?;
|
||||
let inv = (alpha.clone() + 1e-9).map_err(err)?.recip().map_err(err)?;
|
||||
(x + s2.broadcast_mul(&inv).map_err(err)?).map_err(err)
|
||||
}
|
||||
|
||||
/// Generator residual block (HiFi-GAN "resblock 1" with adaptive instance norm and Snake).
|
||||
pub(crate) struct SnakeResBlock {
|
||||
convs1: Vec<Conv>,
|
||||
convs2: Vec<Conv>,
|
||||
adain1: Vec<AdaIn>,
|
||||
adain2: Vec<AdaIn>,
|
||||
alpha1: Vec<Tensor>,
|
||||
alpha2: Vec<Tensor>,
|
||||
}
|
||||
|
||||
impl SnakeResBlock {
|
||||
pub(crate) fn load(wt: &Weights, name: &str, dilations: [usize; 3]) -> R<SnakeResBlock> {
|
||||
let mut b = SnakeResBlock { convs1: vec![], convs2: vec![], adain1: vec![], adain2: vec![], alpha1: vec![], alpha2: vec![] };
|
||||
for (j, d) in dilations.iter().enumerate() {
|
||||
b.convs1.push(Conv::same(wt, &format!("{name}.convs1.{j}"), *d)?);
|
||||
b.convs2.push(Conv::same(wt, &format!("{name}.convs2.{j}"), 1)?);
|
||||
b.adain1.push(AdaIn::load(wt, &format!("{name}.adain1.{j}"))?);
|
||||
b.adain2.push(AdaIn::load(wt, &format!("{name}.adain2.{j}"))?);
|
||||
b.alpha1.push(wt.get(&format!("{name}.alpha1.{j}"))?);
|
||||
b.alpha2.push(wt.get(&format!("{name}.alpha2.{j}"))?);
|
||||
}
|
||||
Ok(b)
|
||||
}
|
||||
pub(crate) fn fwd(&self, x: &Tensor, s: &Tensor) -> R<Tensor> {
|
||||
let mut x = x.clone();
|
||||
for j in 0..self.convs1.len() {
|
||||
let (Some(c1), Some(c2), Some(n1), Some(n2), Some(a1), Some(a2)) =
|
||||
(self.convs1.get(j), self.convs2.get(j), self.adain1.get(j), self.adain2.get(j), self.alpha1.get(j), self.alpha2.get(j))
|
||||
else {
|
||||
break;
|
||||
};
|
||||
let mut t = snake(&n1.fwd(&x, s)?, a1)?;
|
||||
t = c1.fwd(&t)?;
|
||||
t = snake(&n2.fwd(&t, s)?, a2)?;
|
||||
t = c2.fwd(&t)?;
|
||||
x = (x + t).map_err(err)?;
|
||||
}
|
||||
Ok(x)
|
||||
}
|
||||
}
|
||||
|
||||
/// One LSTM direction (PyTorch gate order i, f, g, o), run in plain loops for speed.
|
||||
struct LstmDir {
|
||||
w_ih: Tensor,
|
||||
w_hh: Vec<f32>,
|
||||
bias: Vec<f32>,
|
||||
hidden: usize,
|
||||
}
|
||||
|
||||
impl LstmDir {
|
||||
fn load(wt: &Weights, name: &str, suffix: &str) -> R<LstmDir> {
|
||||
let w_ih = wt.get(&format!("{name}.weight_ih_l0{suffix}"))?;
|
||||
let w_hh = wt.get(&format!("{name}.weight_hh_l0{suffix}"))?;
|
||||
let (g4, hidden) = w_hh.dims2().map_err(err)?;
|
||||
if g4 != 4 * hidden {
|
||||
return Err(err(format!("{name}: bad LSTM shape")));
|
||||
}
|
||||
let b_ih: Vec<f32> = wt.get(&format!("{name}.bias_ih_l0{suffix}"))?.to_vec1().map_err(err)?;
|
||||
let b_hh: Vec<f32> = wt.get(&format!("{name}.bias_hh_l0{suffix}"))?.to_vec1().map_err(err)?;
|
||||
let bias = b_ih.iter().zip(&b_hh).map(|(a, b)| a + b).collect();
|
||||
// row-major [4H, H] → transposed [H, 4H] so the recurrent product walks memory forwards
|
||||
let hh: Vec<Vec<f32>> = w_hh.to_vec2().map_err(err)?;
|
||||
let mut w_t = vec![0f32; hidden * g4];
|
||||
for (r, row) in hh.iter().enumerate() {
|
||||
for (c, v) in row.iter().enumerate() {
|
||||
if let Some(slot) = w_t.get_mut(c * g4 + r) {
|
||||
*slot = *v;
|
||||
}
|
||||
}
|
||||
}
|
||||
Ok(LstmDir { w_ih: w_ih.t().map_err(err)?, w_hh: w_t, bias, hidden })
|
||||
}
|
||||
|
||||
/// `x`: `[T, I]` → `[T, H]` (in input order).
|
||||
fn run(&self, x: &Tensor, reverse: bool) -> R<Vec<f32>> {
|
||||
let (t_len, _) = x.dims2().map_err(err)?;
|
||||
let h = self.hidden;
|
||||
let g4 = 4 * h;
|
||||
let pre: Vec<f32> = x.matmul(&self.w_ih).map_err(err)?.flatten_all().map_err(err)?.to_vec1().map_err(err)?;
|
||||
let mut out = vec![0f32; t_len * h];
|
||||
let (mut hs, mut cs) = (vec![0f32; h], vec![0f32; h]);
|
||||
let mut gates = vec![0f32; g4];
|
||||
let sig = |v: f32| 1.0 / (1.0 + (-v).exp());
|
||||
for step in 0..t_len {
|
||||
let t = if reverse { t_len - 1 - step } else { step };
|
||||
let Some(p) = pre.get(t * g4..(t + 1) * g4) else { break };
|
||||
for ((g, a), b) in gates.iter_mut().zip(p).zip(&self.bias) {
|
||||
*g = a + b;
|
||||
}
|
||||
for (k, hv) in hs.iter().enumerate() {
|
||||
if *hv == 0.0 {
|
||||
continue;
|
||||
}
|
||||
let Some(row) = self.w_hh.get(k * g4..(k + 1) * g4) else { break };
|
||||
for (g, w) in gates.iter_mut().zip(row) {
|
||||
*g += hv * w;
|
||||
}
|
||||
}
|
||||
for j in 0..h {
|
||||
let (i, f, gg, o) = (gates[j], gates[h + j], gates[2 * h + j], gates[3 * h + j]);
|
||||
let c = sig(f) * cs[j] + sig(i) * gg.tanh();
|
||||
cs[j] = c;
|
||||
hs[j] = sig(o) * c.tanh();
|
||||
}
|
||||
if let Some(slot) = out.get_mut(t * h..(t + 1) * h) {
|
||||
slot.copy_from_slice(&hs);
|
||||
}
|
||||
}
|
||||
Ok(out)
|
||||
}
|
||||
}
|
||||
|
||||
/// Bidirectional single-layer LSTM on `[1, T, I]` → `[1, T, 2H]`.
|
||||
pub(crate) struct BiLstm {
|
||||
fwd: LstmDir,
|
||||
bwd: LstmDir,
|
||||
}
|
||||
|
||||
impl BiLstm {
|
||||
pub(crate) fn load(wt: &Weights, name: &str) -> R<BiLstm> {
|
||||
Ok(BiLstm { fwd: LstmDir::load(wt, name, "")?, bwd: LstmDir::load(wt, name, "_reverse")? })
|
||||
}
|
||||
pub(crate) fn fwd(&self, x: &Tensor) -> R<Tensor> {
|
||||
let x2 = x.squeeze(0).map_err(err)?.contiguous().map_err(err)?;
|
||||
let t = x2.dim(0).map_err(err)?;
|
||||
let h = self.fwd.hidden;
|
||||
let a = self.fwd.run(&x2, false)?;
|
||||
let b = self.bwd.run(&x2, true)?;
|
||||
let mut out = Vec::with_capacity(t * 2 * h);
|
||||
for i in 0..t {
|
||||
out.extend_from_slice(a.get(i * h..(i + 1) * h).unwrap_or(&[]));
|
||||
out.extend_from_slice(b.get(i * h..(i + 1) * h).unwrap_or(&[]));
|
||||
}
|
||||
Tensor::from_vec(out, (1, t, 2 * h), x.device()).map_err(err)
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,113 @@
|
|||
//! Kokoro-82M neural voices (feature `kokoro`): pure-Rust inference on candle (CPU).
|
||||
//!
|
||||
//! Implemented from the papers the model card cites (StyleTTS 2, iSTFTNet; with ALBERT, HiFi-GAN,
|
||||
//! the neural source-filter model and the Snake activation they build on), the released
|
||||
//! `config.json` and the tensor names and shapes in the checkpoint. No reference code was read.
|
||||
//! Weights (Apache-2.0, hexgrad) are downloaded on first use and never bundled.
|
||||
//!
|
||||
//! [`Kokoro::speak`] takes Kokoro phonemes (from `filmcraft-tts-text`), at most
|
||||
//! [`MAX_PHONEMES`] per call, and a voice pack.
|
||||
|
||||
mod dsp;
|
||||
mod layers;
|
||||
mod model;
|
||||
mod weights;
|
||||
|
||||
use std::path::Path;
|
||||
|
||||
use crate::TtsError;
|
||||
|
||||
pub use model::SAMPLE_RATE as KOKORO_RATE;
|
||||
|
||||
/// Longest phoneme string one forward pass takes (the position table has 512 rows, two of them
|
||||
/// for the boundary tokens).
|
||||
pub const MAX_PHONEMES: usize = 510;
|
||||
|
||||
/// A loaded Kokoro model.
|
||||
pub struct Kokoro {
|
||||
model: model::Model,
|
||||
vocab: std::collections::HashMap<char, u32>,
|
||||
}
|
||||
|
||||
/// A loaded voice pack.
|
||||
pub struct VoicePack(Vec<f32>);
|
||||
|
||||
impl VoicePack {
|
||||
pub fn load(path: &Path) -> Result<VoicePack, TtsError> {
|
||||
weights::read_voice(path).map(VoicePack)
|
||||
}
|
||||
}
|
||||
|
||||
impl Kokoro {
|
||||
/// Load `config.json` and `kokoro-v1_0.pth` from `dir`.
|
||||
pub fn load(dir: &Path) -> Result<Kokoro, TtsError> {
|
||||
let cfg = weights::read_config(&dir.join("config.json"))?;
|
||||
let w = weights::Weights::load(&dir.join("kokoro-v1_0.pth"))?;
|
||||
Ok(Kokoro { model: model::Model::load(&w)?, vocab: cfg.vocab })
|
||||
}
|
||||
|
||||
/// Phoneme ids of `phonemes`; symbols outside the vocabulary are dropped.
|
||||
pub fn ids(&self, phonemes: &str) -> Vec<u32> {
|
||||
phonemes.chars().filter_map(|c| self.vocab.get(&c).copied()).collect()
|
||||
}
|
||||
|
||||
/// Speak `phonemes` with `voice` at `speed` (0.5–2). Deterministic for a given `seed`.
|
||||
pub fn speak(&self, phonemes: &str, voice: &VoicePack, speed: f64, seed: u64) -> Result<Vec<f32>, TtsError> {
|
||||
if !(0.5..=2.0).contains(&speed) {
|
||||
return Err(TtsError::Invalid("speed must be between 0.5 and 2".into()));
|
||||
}
|
||||
let ids = self.ids(phonemes);
|
||||
if ids.is_empty() {
|
||||
return Err(TtsError::Empty);
|
||||
}
|
||||
if ids.len() > MAX_PHONEMES {
|
||||
return Err(TtsError::Invalid(format!("at most {MAX_PHONEMES} phonemes per pass")));
|
||||
}
|
||||
let style = weights::style_for(&voice.0, ids.len(), self.model.device())?;
|
||||
let out = self.model.forward(&ids, &style, speed, seed)?;
|
||||
if out.iter().any(|s| !s.is_finite()) {
|
||||
return Err(TtsError::Model("the model produced invalid samples".into()));
|
||||
}
|
||||
Ok(out)
|
||||
}
|
||||
}
|
||||
|
||||
/// Tests with the real model run when `FILMCRAFT_KOKORO_DIR` names a directory holding
|
||||
/// `config.json`, `kokoro-v1_0.pth` and `af_heart.pt` (the downloaded package); they skip otherwise.
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
fn model() -> Option<(Kokoro, VoicePack)> {
|
||||
let dir = std::path::PathBuf::from(std::env::var_os("FILMCRAFT_KOKORO_DIR")?);
|
||||
let voice = [dir.join("af_heart.pt"), dir.join("voices/af_heart.pt")].into_iter().find(|p| p.exists())?;
|
||||
Some((Kokoro::load(&dir).unwrap(), VoicePack::load(&voice).unwrap()))
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn speaks_deterministically_and_pace_changes_the_length() {
|
||||
let Some((k, v)) = model() else { return };
|
||||
let ph = "həlˈO wˈɜɹld.";
|
||||
let a = k.speak(ph, &v, 1.0, 1).unwrap();
|
||||
assert_eq!(a, k.speak(ph, &v, 1.0, 1).unwrap(), "deterministic");
|
||||
assert_eq!(a.len() % 600, 0, "whole frames");
|
||||
let secs = a.len() as f64 / f64::from(KOKORO_RATE);
|
||||
assert!((0.5..3.0).contains(&secs), "{secs}");
|
||||
let peak = a.iter().fold(0f32, |m, x| m.max(x.abs()));
|
||||
assert!(peak > 0.05 && peak < 1.0, "{peak}");
|
||||
let fast = k.speak(ph, &v, 2.0, 1).unwrap();
|
||||
let ratio = fast.len() as f64 / a.len() as f64;
|
||||
assert!((0.35..0.7).contains(&ratio), "{ratio}");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn refuses_bad_input() {
|
||||
let Some((k, v)) = model() else { return };
|
||||
assert_eq!(k.speak("", &v, 1.0, 1).err(), Some(TtsError::Empty));
|
||||
assert_eq!(k.speak("日本語👍", &v, 1.0, 1).err(), Some(TtsError::Empty), "nothing in the vocabulary");
|
||||
assert!(k.speak("a", &v, 3.0, 1).is_err());
|
||||
assert!(k.speak("a", &v, f64::NAN, 1).is_err());
|
||||
assert!(k.speak(&"ə".repeat(MAX_PHONEMES + 1), &v, 1.0, 1).is_err());
|
||||
assert_eq!(k.ids("a日b"), k.ids("ab"));
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,385 @@
|
|||
//! The Kokoro-82M forward pass (inference only, batch of one).
|
||||
//!
|
||||
//! 1. **Text** – phoneme ids go through an ALBERT encoder (one shared layer applied 12 times) and
|
||||
//! a linear map to 512 channels (`bert`, `bert_encoder`), and separately through an embedding,
|
||||
//! three convolutions and a bidirectional LSTM (`text_encoder`).
|
||||
//! 2. **Prosody** – the duration encoder (LSTMs with style-adaptive layer norm), a duration head
|
||||
//! (50 bins, sigmoid, summed), then pitch (F0) and energy (N) curves at twice the frame rate from
|
||||
//! adaptive residual blocks (`predictor`).
|
||||
//! 3. **Decoder** – text features expanded to frames by the durations, combined with F0 and N in
|
||||
//! adaptive residual blocks (`decoder.encode`, `decoder.decode`).
|
||||
//! 4. **Vocoder** – iSTFTNet: a harmonic-plus-noise source from F0 (9 harmonics), two upsampling
|
||||
//! stages (×10, ×6) with source injection and Snake residual blocks, then magnitude and phase for
|
||||
//! an inverse STFT (n_fft 20, hop 5) at 24 kHz.
|
||||
//!
|
||||
//! Style: a voice pack row of 256 values; the first 128 condition the decoder, the last 128 the
|
||||
//! prosody predictor.
|
||||
|
||||
use candle_core::{D, Device, Tensor};
|
||||
|
||||
use super::layers::*;
|
||||
use super::weights::{Weights, err};
|
||||
|
||||
const HEADS: usize = 12;
|
||||
const LAYERS: usize = 12;
|
||||
pub const SAMPLE_RATE: u32 = 24_000;
|
||||
/// Output samples per predicted duration frame (2 F0 frames × 60 upsampling × hop 5).
|
||||
pub(crate) const SAMPLES_PER_FRAME: usize = 600;
|
||||
|
||||
struct Albert {
|
||||
word: Tensor,
|
||||
pos: Tensor,
|
||||
ttype: Tensor,
|
||||
emb_ln: LayerNorm,
|
||||
map_in: Linear,
|
||||
q: Linear,
|
||||
k: Linear,
|
||||
v: Linear,
|
||||
dense: Linear,
|
||||
attn_ln: LayerNorm,
|
||||
ffn: Linear,
|
||||
ffn_out: Linear,
|
||||
full_ln: LayerNorm,
|
||||
}
|
||||
|
||||
impl Albert {
|
||||
fn load(w: &Weights) -> R<Albert> {
|
||||
let l = "bert.encoder.albert_layer_groups.0.albert_layers.0";
|
||||
Ok(Albert {
|
||||
word: w.shaped("bert.embeddings.word_embeddings.weight", &[178, 128])?,
|
||||
pos: w.shaped("bert.embeddings.position_embeddings.weight", &[512, 128])?,
|
||||
ttype: w.shaped("bert.embeddings.token_type_embeddings.weight", &[2, 128])?,
|
||||
emb_ln: LayerNorm::load(w, "bert.embeddings.LayerNorm.weight", "bert.embeddings.LayerNorm.bias", 1e-12)?,
|
||||
map_in: Linear::load(w, "bert.encoder.embedding_hidden_mapping_in")?,
|
||||
q: Linear::load(w, &format!("{l}.attention.query"))?,
|
||||
k: Linear::load(w, &format!("{l}.attention.key"))?,
|
||||
v: Linear::load(w, &format!("{l}.attention.value"))?,
|
||||
dense: Linear::load(w, &format!("{l}.attention.dense"))?,
|
||||
attn_ln: LayerNorm::load(w, &format!("{l}.attention.LayerNorm.weight"), &format!("{l}.attention.LayerNorm.bias"), 1e-12)?,
|
||||
ffn: Linear::load(w, &format!("{l}.ffn"))?,
|
||||
ffn_out: Linear::load(w, &format!("{l}.ffn_output"))?,
|
||||
full_ln: LayerNorm::load(w, &format!("{l}.full_layer_layer_norm.weight"), &format!("{l}.full_layer_layer_norm.bias"), 1e-12)?,
|
||||
})
|
||||
}
|
||||
|
||||
/// `ids`: `[T]` → `[1, T, 768]`.
|
||||
fn fwd(&self, ids: &Tensor) -> R<Tensor> {
|
||||
let t = ids.dim(0).map_err(err)?;
|
||||
let e = self.word.index_select(ids, 0).map_err(err)?;
|
||||
let e = (e + self.pos.narrow(0, 0, t).map_err(err)?).map_err(err)?;
|
||||
let e = e.broadcast_add(&self.ttype.narrow(0, 0, 1).map_err(err)?).map_err(err)?;
|
||||
let mut x = self.map_in.fwd(&self.emb_ln.fwd(&e)?)?.unsqueeze(0).map_err(err)?; // [1, T, 768]
|
||||
let hd = 768 / HEADS;
|
||||
let scale = 1.0 / (hd as f64).sqrt();
|
||||
let split = |y: Tensor| -> R<Tensor> { y.reshape((1, t, HEADS, hd)).map_err(err)?.transpose(1, 2).map_err(err)?.contiguous().map_err(err) };
|
||||
for _ in 0..LAYERS {
|
||||
let q = split(self.q.fwd(&x)?)?;
|
||||
let k = split(self.k.fwd(&x)?)?;
|
||||
let v = split(self.v.fwd(&x)?)?;
|
||||
let att = (q.matmul(&k.t().map_err(err)?).map_err(err)? * scale).map_err(err)?;
|
||||
let att = candle_nn::ops::softmax_last_dim(&att).map_err(err)?;
|
||||
let ctx = att.matmul(&v).map_err(err)?.transpose(1, 2).map_err(err)?.reshape((1, t, 768)).map_err(err)?;
|
||||
let a = self.attn_ln.fwd(&(x + self.dense.fwd(&ctx)?).map_err(err)?)?;
|
||||
let f = self.ffn_out.fwd(&self.ffn.fwd(&a)?.gelu().map_err(err)?)?;
|
||||
x = self.full_ln.fwd(&(f + a).map_err(err)?)?;
|
||||
}
|
||||
Ok(x)
|
||||
}
|
||||
}
|
||||
|
||||
struct TextEncoder {
|
||||
emb: Tensor,
|
||||
convs: Vec<(Conv, LayerNorm)>,
|
||||
lstm: BiLstm,
|
||||
}
|
||||
|
||||
impl TextEncoder {
|
||||
fn load(w: &Weights) -> R<TextEncoder> {
|
||||
let mut convs = Vec::new();
|
||||
for i in 0..3 {
|
||||
let c = Conv::same(w, &format!("text_encoder.cnn.{i}.0"), 1)?;
|
||||
let n = LayerNorm::load(w, &format!("text_encoder.cnn.{i}.1.gamma"), &format!("text_encoder.cnn.{i}.1.beta"), 1e-5)?;
|
||||
convs.push((c, n));
|
||||
}
|
||||
Ok(TextEncoder { emb: w.shaped("text_encoder.embedding.weight", &[178, 512])?, convs, lstm: BiLstm::load(w, "text_encoder.lstm")? })
|
||||
}
|
||||
/// `[T]` → `[1, 512, T]`.
|
||||
fn fwd(&self, ids: &Tensor) -> R<Tensor> {
|
||||
let mut x = self.emb.index_select(ids, 0).map_err(err)?.t().map_err(err)?.unsqueeze(0).map_err(err)?; // [1, 512, T]
|
||||
for (c, n) in &self.convs {
|
||||
x = c.fwd(&x)?;
|
||||
// layer norm over channels
|
||||
x = n.fwd(&x.transpose(1, 2).map_err(err)?)?.transpose(1, 2).map_err(err)?;
|
||||
x = leaky(&x, 0.2)?;
|
||||
}
|
||||
let y = self.lstm.fwd(&x.transpose(1, 2).map_err(err)?.contiguous().map_err(err)?)?; // [1, T, 512]
|
||||
y.transpose(1, 2).map_err(err)
|
||||
}
|
||||
}
|
||||
|
||||
struct Predictor {
|
||||
dur_lstms: Vec<BiLstm>,
|
||||
dur_norms: Vec<AdaLayerNorm>,
|
||||
lstm: BiLstm,
|
||||
dur_proj: Linear,
|
||||
shared: BiLstm,
|
||||
f0: Vec<AdaResBlock>,
|
||||
n: Vec<AdaResBlock>,
|
||||
f0_proj: Conv,
|
||||
n_proj: Conv,
|
||||
}
|
||||
|
||||
impl Predictor {
|
||||
fn load(w: &Weights) -> R<Predictor> {
|
||||
let mut dur_lstms = Vec::new();
|
||||
let mut dur_norms = Vec::new();
|
||||
for i in 0..3 {
|
||||
dur_lstms.push(BiLstm::load(w, &format!("predictor.text_encoder.lstms.{}", 2 * i))?);
|
||||
dur_norms.push(AdaLayerNorm::load(w, &format!("predictor.text_encoder.lstms.{}", 2 * i + 1))?);
|
||||
}
|
||||
let blocks = |k: &str| -> R<Vec<AdaResBlock>> { (0..3).map(|i| AdaResBlock::load(w, &format!("predictor.{k}.{i}"), i == 1)).collect() };
|
||||
Ok(Predictor {
|
||||
dur_lstms,
|
||||
dur_norms,
|
||||
lstm: BiLstm::load(w, "predictor.lstm")?,
|
||||
dur_proj: Linear::load(w, "predictor.duration_proj.linear_layer")?,
|
||||
shared: BiLstm::load(w, "predictor.shared")?,
|
||||
f0: blocks("F0")?,
|
||||
n: blocks("N")?,
|
||||
f0_proj: Conv::load(w, "predictor.F0_proj", 0, 1, 1, 1)?,
|
||||
n_proj: Conv::load(w, "predictor.N_proj", 0, 1, 1, 1)?,
|
||||
})
|
||||
}
|
||||
|
||||
/// Duration encoder: `d_en` `[1, 512, T]`, style `[1, 128]` → `[1, T, 640]`.
|
||||
fn encode(&self, d_en: &Tensor, s: &Tensor) -> R<Tensor> {
|
||||
let t = d_en.dim(2).map_err(err)?;
|
||||
let s_t = s.unsqueeze(1).map_err(err)?.broadcast_as((1, t, 128)).map_err(err)?.contiguous().map_err(err)?;
|
||||
let mut x = Tensor::cat(&[&d_en.transpose(1, 2).map_err(err)?.contiguous().map_err(err)?, &s_t], 2).map_err(err)?;
|
||||
for (l, n) in self.dur_lstms.iter().zip(&self.dur_norms) {
|
||||
let y = n.fwd(&l.fwd(&x)?, s)?;
|
||||
x = Tensor::cat(&[&y, &s_t], 2).map_err(err)?;
|
||||
}
|
||||
Ok(x)
|
||||
}
|
||||
|
||||
/// Frames per input token, at `speed`.
|
||||
fn durations(&self, d: &Tensor, speed: f64) -> R<Vec<usize>> {
|
||||
let x = self.lstm.fwd(d)?;
|
||||
let logits = self.dur_proj.fwd(&x)?;
|
||||
let dur = candle_nn::ops::sigmoid(&logits).map_err(err)?.sum(D::Minus1).map_err(err)?.squeeze(0).map_err(err)?;
|
||||
let v: Vec<f32> = dur.to_vec1().map_err(err)?;
|
||||
Ok(v.iter().map(|d| ((f64::from(*d) / speed).round().clamp(1.0, 200.0)) as usize).collect())
|
||||
}
|
||||
|
||||
/// F0 and N curves `[1, 1, 2F]` from frame features `en` `[1, 640, F]`.
|
||||
fn f0_n(&self, en: &Tensor, s: &Tensor) -> R<(Tensor, Tensor)> {
|
||||
let x = self.shared.fwd(&en.transpose(1, 2).map_err(err)?.contiguous().map_err(err)?)?.transpose(1, 2).map_err(err)?.contiguous().map_err(err)?;
|
||||
let mut f = x.clone();
|
||||
for b in &self.f0 {
|
||||
f = b.fwd(&f, s)?;
|
||||
}
|
||||
let mut n = x;
|
||||
for b in &self.n {
|
||||
n = b.fwd(&n, s)?;
|
||||
}
|
||||
Ok((self.f0_proj.fwd(&f)?, self.n_proj.fwd(&n)?))
|
||||
}
|
||||
}
|
||||
|
||||
struct Generator {
|
||||
l_linear: Linear,
|
||||
noise_convs: Vec<Conv>,
|
||||
noise_res: Vec<SnakeResBlock>,
|
||||
ups: Vec<ConvT>,
|
||||
resblocks: Vec<SnakeResBlock>,
|
||||
conv_post: Conv,
|
||||
}
|
||||
|
||||
impl Generator {
|
||||
fn load(w: &Weights) -> R<Generator> {
|
||||
let g = "decoder.generator";
|
||||
let dil = [1, 3, 5];
|
||||
Ok(Generator {
|
||||
l_linear: Linear::load(w, &format!("{g}.m_source.l_linear"))?,
|
||||
noise_convs: vec![Conv::load(w, &format!("{g}.noise_convs.0"), 3, 6, 1, 1)?, Conv::load(w, &format!("{g}.noise_convs.1"), 0, 1, 1, 1)?],
|
||||
noise_res: vec![SnakeResBlock::load(w, &format!("{g}.noise_res.0"), dil)?, SnakeResBlock::load(w, &format!("{g}.noise_res.1"), dil)?],
|
||||
ups: vec![ConvT::load(w, &format!("{g}.ups.0"), 5, 0, 10, 1)?, ConvT::load(w, &format!("{g}.ups.1"), 3, 0, 6, 1)?],
|
||||
resblocks: (0..6).map(|i| SnakeResBlock::load(w, &format!("{g}.resblocks.{i}"), dil)).collect::<R<_>>()?,
|
||||
conv_post: Conv::same(w, &format!("{g}.conv_post"), 1)?,
|
||||
})
|
||||
}
|
||||
|
||||
/// `x` `[1, 512, 2F]`, style `[1, 128]`, F0 per half-frame → samples.
|
||||
fn fwd(&self, x: &Tensor, s: &Tensor, f0: &[f32], seed: u64) -> R<Vec<f32>> {
|
||||
let dev = x.device();
|
||||
let source = harmonic_source(f0, &self.l_linear, seed, dev)?;
|
||||
let (mag, phase) = super::dsp::stft(&source);
|
||||
let frames = mag.len() / super::dsp::BINS;
|
||||
let mut har = mag;
|
||||
har.extend_from_slice(&phase);
|
||||
let har = Tensor::from_vec(har, (1, 2 * super::dsp::BINS, frames), dev).map_err(err)?;
|
||||
let mut x = x.clone();
|
||||
for i in 0..2 {
|
||||
let (Some(nc), Some(nr), Some(up)) = (self.noise_convs.get(i), self.noise_res.get(i), self.ups.get(i)) else { break };
|
||||
x = leaky(&x, 0.1)?;
|
||||
let xs = nr.fwd(&nc.fwd(&har)?, s)?;
|
||||
x = up.fwd(&x)?;
|
||||
if i == 1 {
|
||||
// reflection pad one frame on the left
|
||||
let first = x.narrow(2, 1, 1).map_err(err)?;
|
||||
x = Tensor::cat(&[&first, &x], 2).map_err(err)?;
|
||||
}
|
||||
let t = x.dim(2).map_err(err)?.min(xs.dim(2).map_err(err)?);
|
||||
x = (x.narrow(2, 0, t).map_err(err)? + xs.narrow(2, 0, t).map_err(err)?).map_err(err)?;
|
||||
let mut acc: Option<Tensor> = None;
|
||||
for j in 0..3 {
|
||||
let Some(rb) = self.resblocks.get(i * 3 + j) else { break };
|
||||
let y = rb.fwd(&x, s)?;
|
||||
acc = Some(match acc {
|
||||
Some(a) => (a + y).map_err(err)?,
|
||||
None => y,
|
||||
});
|
||||
}
|
||||
x = (acc.ok_or_else(|| err("missing resblocks"))? / 3.0).map_err(err)?;
|
||||
}
|
||||
x = leaky(&x, 0.01)?;
|
||||
let y = self.conv_post.fwd(&x)?.squeeze(0).map_err(err)?; // [22, T]
|
||||
let bins = super::dsp::BINS;
|
||||
let mag: Vec<f32> = y.narrow(0, 0, bins).map_err(err)?.exp().map_err(err)?.flatten_all().map_err(err)?.to_vec1().map_err(err)?;
|
||||
let ph: Vec<f32> = y.narrow(0, bins, bins).map_err(err)?.sin().map_err(err)?.flatten_all().map_err(err)?.to_vec1().map_err(err)?;
|
||||
Ok(super::dsp::istft(&mag, &ph))
|
||||
}
|
||||
}
|
||||
|
||||
/// Neural source filter: 9 sine harmonics of F0 (voiced) or noise (unvoiced), mixed to one
|
||||
/// channel by a learned linear layer and tanh. F0 is given per half-frame and held for 300 samples.
|
||||
fn harmonic_source(f0: &[f32], l_linear: &Linear, seed: u64, dev: &Device) -> R<Vec<f32>> {
|
||||
const HARM: usize = 9;
|
||||
const HOLD: usize = SAMPLES_PER_FRAME / 2;
|
||||
let n = f0.len() * HOLD;
|
||||
let mut rng = super::dsp::Rng::new(seed);
|
||||
// random start phase for the overtones, none for the fundamental
|
||||
let start: Vec<f64> = (0..HARM).map(|h| if h == 0 { 0.0 } else { rng.uniform() }).collect();
|
||||
let mut phase = start;
|
||||
let mut sines = vec![0f32; n * HARM];
|
||||
for (i, row) in sines.as_chunks_mut::<HARM>().0.iter_mut().enumerate() {
|
||||
let f = f64::from(f0.get(i / HOLD).copied().unwrap_or(0.0)).max(0.0);
|
||||
let voiced = f > 10.0;
|
||||
let noise_amp = if voiced { 0.003 } else { 0.1 / 3.0 };
|
||||
for (h, out) in row.iter_mut().enumerate() {
|
||||
phase[h] = (phase[h] + f * (h + 1) as f64 / f64::from(SAMPLE_RATE)).fract();
|
||||
let sine = if voiced { 0.1 * (std::f64::consts::TAU * phase[h]).sin() } else { 0.0 };
|
||||
*out = (sine + noise_amp * rng.normal()) as f32;
|
||||
}
|
||||
}
|
||||
let t = Tensor::from_vec(sines, (n, HARM), dev).map_err(err)?;
|
||||
let y = l_linear.fwd(&t)?.tanh().map_err(err)?.flatten_all().map_err(err)?;
|
||||
y.to_vec1().map_err(err)
|
||||
}
|
||||
|
||||
struct Decoder {
|
||||
f0_conv: Conv,
|
||||
n_conv: Conv,
|
||||
encode: AdaResBlock,
|
||||
decode: Vec<AdaResBlock>,
|
||||
asr_res: Conv,
|
||||
vocoder: Generator,
|
||||
}
|
||||
|
||||
impl Decoder {
|
||||
fn load(w: &Weights) -> R<Decoder> {
|
||||
Ok(Decoder {
|
||||
f0_conv: Conv::load(w, "decoder.F0_conv", 1, 2, 1, 1)?,
|
||||
n_conv: Conv::load(w, "decoder.N_conv", 1, 2, 1, 1)?,
|
||||
encode: AdaResBlock::load(w, "decoder.encode", false)?,
|
||||
decode: (0..4).map(|i| AdaResBlock::load(w, &format!("decoder.decode.{i}"), i == 3)).collect::<R<_>>()?,
|
||||
asr_res: Conv::load(w, "decoder.asr_res.0", 0, 1, 1, 1)?,
|
||||
vocoder: Generator::load(w)?,
|
||||
})
|
||||
}
|
||||
|
||||
fn fwd(&self, asr: &Tensor, f0: &Tensor, n: &Tensor, s: &Tensor, seed: u64) -> R<Vec<f32>> {
|
||||
let f0_curve: Vec<f32> = f0.flatten_all().map_err(err)?.to_vec1().map_err(err)?;
|
||||
let f = self.f0_conv.fwd(f0)?;
|
||||
let nn = self.n_conv.fwd(n)?;
|
||||
let frames = asr.dim(2).map_err(err)?.min(f.dim(2).map_err(err)?).min(nn.dim(2).map_err(err)?);
|
||||
let asr = asr.narrow(2, 0, frames).map_err(err)?;
|
||||
let f = f.narrow(2, 0, frames).map_err(err)?;
|
||||
let nn = nn.narrow(2, 0, frames).map_err(err)?;
|
||||
let mut x = self.encode.fwd(&Tensor::cat(&[&asr, &f, &nn], 1).map_err(err)?, s)?;
|
||||
let res = self.asr_res.fwd(&asr)?;
|
||||
let mut concat = true;
|
||||
for b in &self.decode {
|
||||
if concat {
|
||||
x = Tensor::cat(&[&x, &res, &f, &nn], 1).map_err(err)?;
|
||||
}
|
||||
x = b.fwd(&x, s)?;
|
||||
if b.upsamples() {
|
||||
concat = false;
|
||||
}
|
||||
}
|
||||
self.vocoder.fwd(&x, s, &f0_curve, seed)
|
||||
}
|
||||
}
|
||||
|
||||
/// The loaded model.
|
||||
pub(crate) struct Model {
|
||||
albert: Albert,
|
||||
bert_enc: Linear,
|
||||
text: TextEncoder,
|
||||
pred: Predictor,
|
||||
dec: Decoder,
|
||||
dev: Device,
|
||||
}
|
||||
|
||||
impl Model {
|
||||
pub(crate) fn load(w: &Weights) -> R<Model> {
|
||||
Ok(Model {
|
||||
albert: Albert::load(w)?,
|
||||
bert_enc: Linear::load(w, "bert_encoder")?,
|
||||
text: TextEncoder::load(w)?,
|
||||
pred: Predictor::load(w)?,
|
||||
dec: Decoder::load(w)?,
|
||||
dev: Device::Cpu,
|
||||
})
|
||||
}
|
||||
|
||||
pub(crate) fn device(&self) -> &Device {
|
||||
&self.dev
|
||||
}
|
||||
|
||||
/// Phoneme ids (without the boundary tokens; at most 510) and a 256-value style → samples.
|
||||
pub(crate) fn forward(&self, ids: &[u32], style: &Tensor, speed: f64, seed: u64) -> R<Vec<f32>> {
|
||||
let mut padded = Vec::with_capacity(ids.len() + 2);
|
||||
padded.push(0u32);
|
||||
padded.extend_from_slice(ids);
|
||||
padded.push(0);
|
||||
let t = padded.len();
|
||||
let ids_t = Tensor::from_vec(padded, t, &self.dev).map_err(err)?;
|
||||
let s_dec = style.narrow(1, 0, 128).map_err(err)?;
|
||||
let s_pro = style.narrow(1, 128, 128).map_err(err)?;
|
||||
let bert = self.albert.fwd(&ids_t)?;
|
||||
let d_en = self.bert_enc.fwd(&bert)?.transpose(1, 2).map_err(err)?.contiguous().map_err(err)?; // [1, 512, T]
|
||||
let d = self.pred.encode(&d_en, &s_pro)?; // [1, T, 640]
|
||||
let dur = self.pred.durations(&d, speed)?;
|
||||
let frames: usize = dur.iter().sum();
|
||||
// alignment [T, F]: token i covers its run of frames
|
||||
let mut aln = vec![0f32; t * frames];
|
||||
let mut at = 0usize;
|
||||
for (i, n) in dur.iter().enumerate() {
|
||||
for k in at..at + n {
|
||||
if let Some(v) = aln.get_mut(i * frames + k) {
|
||||
*v = 1.0;
|
||||
}
|
||||
}
|
||||
at += n;
|
||||
}
|
||||
let aln = Tensor::from_vec(aln, (1, t, frames), &self.dev).map_err(err)?;
|
||||
let en = d.transpose(1, 2).map_err(err)?.contiguous().map_err(err)?.matmul(&aln).map_err(err)?; // [1, 640, F]
|
||||
let (f0, n) = self.pred.f0_n(&en, &s_pro)?;
|
||||
let t_en = self.text.fwd(&ids_t)?; // [1, 512, T]
|
||||
let asr = t_en.matmul(&aln).map_err(err)?; // [1, 512, F]
|
||||
self.dec.fwd(&asr, &f0, &n, &s_dec, seed)
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,266 @@
|
|||
//! Loading the Kokoro-82M checkpoint, its config and voice packs.
|
||||
//!
|
||||
//! The official release is a PyTorch zip checkpoint (`kokoro-v1_0.pth`) holding five state
|
||||
//! dictionaries (`bert`, `bert_encoder`, `predictor`, `text_encoder`, `decoder`), each with a
|
||||
//! `module.` prefix. They are read with candle's restricted pickle reader, which only rebuilds
|
||||
//! tensors and never runs code from the file. Weight-normalised layers (`weight_g`, `weight_v`) are
|
||||
//! fused into a plain weight on load. Voice packs (`voices/<id>.pt`) hold one bare float32 tensor
|
||||
//! `[510, 1, 256]`, read straight from the zip archive.
|
||||
|
||||
use std::collections::HashMap;
|
||||
use std::io::Read;
|
||||
use std::path::Path;
|
||||
|
||||
use candle_core::{DType, Device, Tensor};
|
||||
|
||||
use crate::TtsError;
|
||||
|
||||
pub(crate) fn err(e: impl std::fmt::Display) -> TtsError {
|
||||
TtsError::Model(e.to_string())
|
||||
}
|
||||
|
||||
/// The architecture this code implements; a checkpoint whose config differs is refused.
|
||||
pub(crate) struct Config {
|
||||
pub vocab: HashMap<char, u32>,
|
||||
}
|
||||
|
||||
/// Values of `config.json` this implementation is written for.
|
||||
const EXPECT: &[(&str, &str)] = &[
|
||||
("hidden_dim", "512"),
|
||||
("style_dim", "128"),
|
||||
("n_token", "178"),
|
||||
("max_dur", "50"),
|
||||
("n_layer", "3"),
|
||||
("text_encoder_kernel_size", "5"),
|
||||
("istftnet.gen_istft_n_fft", "20"),
|
||||
("istftnet.gen_istft_hop_size", "5"),
|
||||
("istftnet.upsample_rates", "[10,6]"),
|
||||
("istftnet.upsample_kernel_sizes", "[20,12]"),
|
||||
("istftnet.resblock_kernel_sizes", "[3,7,11]"),
|
||||
("istftnet.upsample_initial_channel", "512"),
|
||||
("plbert.hidden_size", "768"),
|
||||
("plbert.num_attention_heads", "12"),
|
||||
("plbert.num_hidden_layers", "12"),
|
||||
("plbert.intermediate_size", "2048"),
|
||||
];
|
||||
|
||||
pub(crate) fn read_config(path: &Path) -> Result<Config, TtsError> {
|
||||
let bytes = std::fs::read(path).map_err(|e| err(format!("{}: {e}", path.display())))?;
|
||||
if bytes.len() > 1 << 20 {
|
||||
return Err(err("config.json is too large"));
|
||||
}
|
||||
let v: serde_json::Value = serde_json::from_slice(&bytes).map_err(err)?;
|
||||
for (key, want) in EXPECT {
|
||||
let mut cur = &v;
|
||||
for part in key.split('.') {
|
||||
cur = cur.get(part).ok_or_else(|| err(format!("config.json: missing `{key}`")))?;
|
||||
}
|
||||
let got = serde_json::to_string(cur).map_err(err)?;
|
||||
if got != *want {
|
||||
return Err(err(format!("config.json: `{key}` is {got}, this build supports {want}")));
|
||||
}
|
||||
}
|
||||
let table = v.get("vocab").and_then(|t| t.as_object()).ok_or_else(|| err("config.json: missing `vocab`"))?;
|
||||
let mut vocab = HashMap::new();
|
||||
for (sym, id) in table {
|
||||
let mut chars = sym.chars();
|
||||
let (Some(c), None) = (chars.next(), chars.next()) else { continue };
|
||||
let Some(id) = id.as_u64().filter(|i| *i > 0 && *i < 178) else { continue };
|
||||
vocab.insert(c, id as u32);
|
||||
}
|
||||
if vocab.len() < 100 {
|
||||
return Err(err("config.json: the phoneme vocabulary is incomplete"));
|
||||
}
|
||||
Ok(Config { vocab })
|
||||
}
|
||||
|
||||
/// All model tensors by full name (`decoder.generator.conv_post.weight`…), weight norm fused.
|
||||
pub(crate) struct Weights {
|
||||
map: HashMap<String, Tensor>,
|
||||
}
|
||||
|
||||
const PARTS: [&str; 5] = ["bert", "bert_encoder", "predictor", "text_encoder", "decoder"];
|
||||
|
||||
impl Weights {
|
||||
pub(crate) fn load(path: &Path) -> Result<Weights, TtsError> {
|
||||
let mut raw: HashMap<String, Tensor> = HashMap::new();
|
||||
for part in PARTS {
|
||||
let tensors = candle_core::pickle::read_all_with_key(path, Some(part)).map_err(|e| err(format!("{}: {e}", path.display())))?;
|
||||
if tensors.is_empty() {
|
||||
return Err(err(format!("{}: no `{part}` weights", path.display())));
|
||||
}
|
||||
for (name, t) in tensors {
|
||||
let name = name.strip_prefix("module.").unwrap_or(&name);
|
||||
let t = t.to_dtype(DType::F32).map_err(err)?;
|
||||
raw.insert(format!("{part}.{name}"), t);
|
||||
}
|
||||
}
|
||||
// fuse weight norm: w = g · v / ‖v‖ (norm over every dim but the first)
|
||||
let mut map = HashMap::with_capacity(raw.len());
|
||||
let gs: Vec<String> = raw.keys().filter(|k| k.ends_with(".weight_g")).cloned().collect();
|
||||
for g_name in gs {
|
||||
let base = g_name.trim_end_matches("_g").to_string();
|
||||
let v_name = format!("{base}_v");
|
||||
let (Some(g), Some(v)) = (raw.remove(&g_name), raw.remove(&v_name)) else {
|
||||
return Err(err(format!("{g_name} without {v_name}")));
|
||||
};
|
||||
let norm = v.sqr().map_err(err)?.sum_keepdim(1).map_err(err)?.sum_keepdim(2).map_err(err)?.sqrt().map_err(err)?;
|
||||
let w = v.broadcast_div(&norm).map_err(err)?.broadcast_mul(&g).map_err(err)?;
|
||||
map.insert(base, w);
|
||||
}
|
||||
map.extend(raw);
|
||||
Ok(Weights { map })
|
||||
}
|
||||
|
||||
pub(crate) fn get(&self, name: &str) -> Result<Tensor, TtsError> {
|
||||
self.map.get(name).cloned().ok_or_else(|| err(format!("missing weight `{name}`")))
|
||||
}
|
||||
|
||||
/// `get` with an expected shape.
|
||||
pub(crate) fn shaped(&self, name: &str, dims: &[usize]) -> Result<Tensor, TtsError> {
|
||||
let t = self.get(name)?;
|
||||
if t.dims() != dims {
|
||||
return Err(err(format!("weight `{name}` is {:?}, expected {dims:?}", t.dims())));
|
||||
}
|
||||
Ok(t)
|
||||
}
|
||||
}
|
||||
|
||||
/// Number of style vectors in a voice pack (one per input length 1..=510).
|
||||
pub(crate) const PACK_LEN: usize = 510;
|
||||
pub(crate) const STYLE: usize = 256;
|
||||
|
||||
/// Read a voice pack: a zip whose `<name>/data/0` holds `[510, 1, 256]` little-endian float32.
|
||||
pub(crate) fn read_voice(path: &Path) -> Result<Vec<f32>, TtsError> {
|
||||
let f = std::fs::File::open(path).map_err(|e| err(format!("{}: {e}", path.display())))?;
|
||||
let mut z = zip::ZipArchive::new(f).map_err(|e| err(format!("{}: {e}", path.display())))?;
|
||||
let name = (0..z.len())
|
||||
.filter_map(|i| z.by_index(i).ok().map(|e| e.name().to_string()))
|
||||
.find(|n| n.ends_with("/data/0"))
|
||||
.ok_or_else(|| err(format!("{}: not a voice pack", path.display())))?;
|
||||
let order = (0..z.len()).filter_map(|i| z.by_index(i).ok().map(|e| e.name().to_string())).find(|n| n.ends_with("/byteorder"));
|
||||
if let Some(o) = order {
|
||||
let mut s = String::new();
|
||||
z.by_name(&o).map_err(err)?.take(16).read_to_string(&mut s).map_err(err)?;
|
||||
if s.trim() != "little" {
|
||||
return Err(err(format!("{}: unsupported byte order `{s}`", path.display())));
|
||||
}
|
||||
}
|
||||
let want = PACK_LEN * STYLE * 4;
|
||||
let mut e = z.by_name(&name).map_err(err)?;
|
||||
if e.size() != want as u64 {
|
||||
return Err(err(format!("{}: voice data is {} bytes, expected {want}", path.display(), e.size())));
|
||||
}
|
||||
let mut buf = Vec::with_capacity(want);
|
||||
e.by_ref().take(want as u64).read_to_end(&mut buf).map_err(err)?;
|
||||
if buf.len() != want {
|
||||
return Err(err(format!("{}: truncated voice data", path.display())));
|
||||
}
|
||||
let v: Vec<f32> = buf.as_chunks::<4>().0.iter().map(|c| f32::from_le_bytes(*c)).collect();
|
||||
if v.iter().any(|x| !x.is_finite()) {
|
||||
return Err(err(format!("{}: voice data is not finite", path.display())));
|
||||
}
|
||||
Ok(v)
|
||||
}
|
||||
|
||||
/// The style vector for an input of `n` phonemes: `[1, 256]`.
|
||||
pub(crate) fn style_for(pack: &[f32], n: usize, dev: &Device) -> Result<Tensor, TtsError> {
|
||||
let row = n.clamp(1, PACK_LEN) - 1;
|
||||
let s = pack.get(row * STYLE..(row + 1) * STYLE).ok_or_else(|| err("voice pack too short"))?;
|
||||
Tensor::from_slice(s, (1, STYLE), dev).map_err(err)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use std::io::Write;
|
||||
|
||||
fn tmp(name: &str) -> std::path::PathBuf {
|
||||
let d = std::env::temp_dir().join(format!("filmcraft-kokoro-{}-{name}", std::process::id()));
|
||||
let _ = std::fs::remove_dir_all(&d);
|
||||
std::fs::create_dir_all(&d).unwrap();
|
||||
d
|
||||
}
|
||||
|
||||
/// A voice-pack-shaped zip with `data` as `x/data/0`.
|
||||
fn pack(dir: &Path, data: &[u8], order: &str) -> std::path::PathBuf {
|
||||
let p = dir.join("v.pt");
|
||||
let mut z = zip::ZipWriter::new(std::fs::File::create(&p).unwrap());
|
||||
let o = zip::write::SimpleFileOptions::default().compression_method(zip::CompressionMethod::Stored);
|
||||
z.start_file("x/data.pkl", o).unwrap();
|
||||
z.write_all(b"not used").unwrap();
|
||||
z.start_file("x/byteorder", o).unwrap();
|
||||
z.write_all(order.as_bytes()).unwrap();
|
||||
z.start_file("x/data/0", o).unwrap();
|
||||
z.write_all(data).unwrap();
|
||||
z.finish().unwrap();
|
||||
p
|
||||
}
|
||||
|
||||
fn floats(n: usize, v: f32) -> Vec<u8> {
|
||||
std::iter::repeat_n(v, n).flat_map(f32::to_le_bytes).collect()
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn reads_a_well_formed_voice_pack_and_picks_the_row_by_length() {
|
||||
let d = tmp("ok");
|
||||
let mut data = floats(PACK_LEN * STYLE, 0.0);
|
||||
// row 4 (inputs of 5 phonemes) starts with 7.0
|
||||
data[4 * STYLE * 4..4 * STYLE * 4 + 4].copy_from_slice(&7.0f32.to_le_bytes());
|
||||
let v = read_voice(&pack(&d, &data, "little")).unwrap();
|
||||
assert_eq!(v.len(), PACK_LEN * STYLE);
|
||||
let s = style_for(&v, 5, &Device::Cpu).unwrap();
|
||||
assert_eq!(s.dims(), &[1, STYLE]);
|
||||
assert_eq!(s.to_vec2::<f32>().unwrap()[0][0], 7.0);
|
||||
// lengths outside 1..=510 clamp instead of failing
|
||||
assert!(style_for(&v, 0, &Device::Cpu).is_ok());
|
||||
assert!(style_for(&v, 100_000, &Device::Cpu).is_ok());
|
||||
let _ = std::fs::remove_dir_all(&d);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn hostile_voice_packs_are_errors_not_panics() {
|
||||
let d = tmp("bad");
|
||||
let cases: Vec<(&str, std::path::PathBuf)> =
|
||||
vec![("short", pack(&d, &floats(10, 0.0), "little")), ("long", pack(&d.join(".."), &floats(PACK_LEN * STYLE + 1, 0.0), "little"))];
|
||||
for (what, p) in cases {
|
||||
assert!(read_voice(&p).is_err(), "{what}");
|
||||
}
|
||||
let p = pack(&d, &floats(PACK_LEN * STYLE, 0.0), "big");
|
||||
assert!(read_voice(&p).is_err(), "big endian");
|
||||
let p = pack(&d, &floats(PACK_LEN * STYLE, f32::NAN), "little");
|
||||
assert!(read_voice(&p).is_err(), "NaN");
|
||||
let junk = d.join("junk.pt");
|
||||
std::fs::write(&junk, b"PK\x03\x04 definitely not a zip").unwrap();
|
||||
assert!(read_voice(&junk).is_err());
|
||||
std::fs::write(&junk, b"").unwrap();
|
||||
assert!(read_voice(&junk).is_err());
|
||||
assert!(read_voice(&d.join("missing.pt")).is_err());
|
||||
// a checkpoint that isn't one
|
||||
assert!(Weights::load(&junk).is_err());
|
||||
let _ = std::fs::remove_dir_all(&d);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn config_must_match_the_implemented_architecture() {
|
||||
let d = tmp("cfg");
|
||||
let p = d.join("config.json");
|
||||
let good = r#"{"hidden_dim":512,"style_dim":128,"n_token":178,"max_dur":50,"n_layer":3,"text_encoder_kernel_size":5,
|
||||
"istftnet":{"gen_istft_n_fft":20,"gen_istft_hop_size":5,"upsample_rates":[10,6],"upsample_kernel_sizes":[20,12],"resblock_kernel_sizes":[3,7,11],"upsample_initial_channel":512},
|
||||
"plbert":{"hidden_size":768,"num_attention_heads":12,"num_hidden_layers":12,"intermediate_size":2048},"vocab":VOCAB}"#;
|
||||
let vocab: String = format!("{{{}}}", (0..110).map(|i| format!("\"{}\":{}", char::from_u32(0x100 + i).unwrap(), i + 1)).collect::<Vec<_>>().join(","));
|
||||
std::fs::write(&p, good.replace("VOCAB", &vocab)).unwrap();
|
||||
assert_eq!(read_config(&p).unwrap().vocab.len(), 110);
|
||||
std::fs::write(&p, good.replace("VOCAB", &vocab).replace("\"hidden_dim\":512", "\"hidden_dim\":256")).unwrap();
|
||||
assert!(read_config(&p).err().unwrap().to_string().contains("hidden_dim"));
|
||||
std::fs::write(&p, good.replace("VOCAB", "{\"a\":1}")).unwrap();
|
||||
assert!(read_config(&p).is_err(), "tiny vocabulary");
|
||||
for junk in ["", "{", "[]", "null", "{\"vocab\":5}"] {
|
||||
std::fs::write(&p, junk).unwrap();
|
||||
assert!(read_config(&p).is_err(), "{junk}");
|
||||
}
|
||||
std::fs::write(&p, vec![b' '; (1 << 20) + 1]).unwrap();
|
||||
assert!(read_config(&p).is_err(), "oversized");
|
||||
let _ = std::fs::remove_dir_all(&d);
|
||||
}
|
||||
}
|
||||
|
|
@ -9,7 +9,12 @@
|
|||
//!
|
||||
//! Everything is deterministic: the same script and settings give the same samples.
|
||||
|
||||
pub mod catalog;
|
||||
pub mod formant;
|
||||
#[cfg(feature = "kokoro")]
|
||||
pub mod kokoro;
|
||||
#[cfg(feature = "kokoro")]
|
||||
pub mod neural;
|
||||
pub mod script;
|
||||
|
||||
use serde::Serialize;
|
||||
|
|
@ -38,6 +43,12 @@ pub enum TtsError {
|
|||
AudioTooLong,
|
||||
#[error("{0}")]
|
||||
Invalid(String),
|
||||
#[error("voice model: {0}")]
|
||||
Model(String),
|
||||
#[error("the natural voices are not downloaded yet")]
|
||||
NotInstalled,
|
||||
#[error("natural voices are not available in this build (built without the `kokoro` feature)")]
|
||||
Unavailable,
|
||||
}
|
||||
|
||||
/// How a voice speaks.
|
||||
|
|
@ -114,7 +125,14 @@ pub trait Voice: Send + Sync {
|
|||
/// Languages offered (English only for now).
|
||||
pub const LANGUAGES: &[(&str, &str)] = &[("en-US", "English (United States)")];
|
||||
|
||||
/// Every voice FilmCraft knows.
|
||||
/// Every voice FilmCraft knows: the built-in voices, then the neural voices (installed when their
|
||||
/// package is downloaded into `models_dir`).
|
||||
pub fn voices_in(models_dir: Option<&std::path::Path>) -> Vec<VoiceInfo> {
|
||||
let installed = models_dir.is_some_and(catalog::installed) && cfg!(feature = "kokoro");
|
||||
formant::VOICES.iter().map(|v| v.info).chain(catalog::VOICES.iter().map(|v| VoiceInfo { installed, ..v.info })).collect()
|
||||
}
|
||||
|
||||
/// The built-in voices (always available).
|
||||
pub fn voices() -> Vec<VoiceInfo> {
|
||||
formant::VOICES.iter().map(|v| v.info).collect()
|
||||
}
|
||||
|
|
@ -128,6 +146,23 @@ pub fn voice(id: &str) -> Result<Box<dyn Voice>, TtsError> {
|
|||
.ok_or_else(|| TtsError::UnknownVoice(id.to_string()))
|
||||
}
|
||||
|
||||
/// The voice `id`: built in, or (feature `kokoro`) a neural voice from the package in `models_dir`.
|
||||
pub fn voice_in(id: &str, models_dir: Option<&std::path::Path>) -> Result<Box<dyn Voice>, TtsError> {
|
||||
if catalog::find(id).is_some() {
|
||||
#[cfg(feature = "kokoro")]
|
||||
{
|
||||
let dir = models_dir.ok_or(TtsError::NotInstalled)?;
|
||||
return neural::load(id, dir).map(|v| Box::new(v) as Box<dyn Voice>);
|
||||
}
|
||||
#[cfg(not(feature = "kokoro"))]
|
||||
{
|
||||
let _ = models_dir;
|
||||
return Err(TtsError::Unavailable);
|
||||
}
|
||||
}
|
||||
voice(id)
|
||||
}
|
||||
|
||||
/// The voice used when none is chosen.
|
||||
pub fn default_voice_id() -> &'static str {
|
||||
formant::VOICES.first().map(|v| v.info.id).unwrap_or("basic-female")
|
||||
|
|
|
|||
|
|
@ -0,0 +1,133 @@
|
|||
//! The neural voices (feature `kokoro`): a script goes through the text front end
|
||||
//! (`filmcraft-tts-text`: normalizer, CMUdict + letter-to-sound, chunks of at most 400 phonemes
|
||||
//! and pause markers), each chunk through Kokoro-82M, pauses become exact silence, and the vocal
|
||||
//! pitch is applied afterwards with the pitch shifter from `filmcraft-audio-dsp` (Kokoro has no
|
||||
//! pitch control). Pace is Kokoro's speed. The model, the dictionary and voice packs are loaded
|
||||
//! once per package directory and shared.
|
||||
|
||||
use std::collections::HashMap;
|
||||
use std::path::{Path, PathBuf};
|
||||
use std::sync::{Arc, Mutex, OnceLock, PoisonError};
|
||||
|
||||
use filmcraft_audio_dsp::AudioEffect;
|
||||
use filmcraft_tts_text::{Chunk, Lang, Lexicon, Overrides};
|
||||
|
||||
use crate::catalog;
|
||||
use crate::kokoro::{KOKORO_RATE, Kokoro, VoicePack};
|
||||
use crate::{Audio, MAX_SECONDS, Params, TtsError, Voice, VoiceInfo, check_text};
|
||||
|
||||
/// Silence between sentences, ms.
|
||||
const SENTENCE_GAP_MS: u32 = 120;
|
||||
|
||||
struct Loaded {
|
||||
dir: PathBuf,
|
||||
model: Arc<Kokoro>,
|
||||
lexicon: Arc<Lexicon>,
|
||||
packs: HashMap<&'static str, Arc<VoicePack>>,
|
||||
}
|
||||
|
||||
fn cache() -> &'static Mutex<Option<Loaded>> {
|
||||
static C: OnceLock<Mutex<Option<Loaded>>> = OnceLock::new();
|
||||
C.get_or_init(Default::default)
|
||||
}
|
||||
|
||||
/// A neural voice ready to speak.
|
||||
pub struct NeuralVoice {
|
||||
info: VoiceInfo,
|
||||
model: Arc<Kokoro>,
|
||||
lexicon: Arc<Lexicon>,
|
||||
pack: Arc<VoicePack>,
|
||||
}
|
||||
|
||||
/// Load (or reuse) the package in `models_dir` and the voice `id`.
|
||||
pub fn load(id: &str, models_dir: &Path) -> Result<NeuralVoice, TtsError> {
|
||||
let v = catalog::find(id).ok_or_else(|| TtsError::UnknownVoice(id.to_string()))?;
|
||||
if !catalog::installed(models_dir) {
|
||||
return Err(TtsError::NotInstalled);
|
||||
}
|
||||
let dir = catalog::package_dir(models_dir);
|
||||
let mut guard = cache().lock().unwrap_or_else(PoisonError::into_inner);
|
||||
if guard.as_ref().is_none_or(|l| l.dir != dir) {
|
||||
let model = Arc::new(Kokoro::load(&dir)?);
|
||||
let bytes = std::fs::read(dir.join("cmudict.dict")).map_err(|e| TtsError::Model(format!("cmudict.dict: {e}")))?;
|
||||
let lexicon = Arc::new(Lexicon::parse_cmudict(&bytes).map_err(|e| TtsError::Model(format!("cmudict.dict: {e}")))?);
|
||||
*guard = Some(Loaded { dir: dir.clone(), model, lexicon, packs: HashMap::new() });
|
||||
}
|
||||
let loaded = guard.as_mut().ok_or_else(|| TtsError::Model("voice package not loaded".into()))?;
|
||||
let pack = match loaded.packs.get(v.file) {
|
||||
Some(p) => p.clone(),
|
||||
None => {
|
||||
let p = Arc::new(VoicePack::load(&dir.join(v.file))?);
|
||||
loaded.packs.insert(v.file, p.clone());
|
||||
p
|
||||
}
|
||||
};
|
||||
Ok(NeuralVoice { info: VoiceInfo { installed: true, ..v.info }, model: loaded.model.clone(), lexicon: loaded.lexicon.clone(), pack })
|
||||
}
|
||||
|
||||
impl Voice for NeuralVoice {
|
||||
fn info(&self) -> VoiceInfo {
|
||||
self.info
|
||||
}
|
||||
|
||||
fn synthesize(&self, text: &str, params: &Params) -> Result<Audio, TtsError> {
|
||||
check_text(text)?;
|
||||
params.check()?;
|
||||
let chunks = filmcraft_tts_text::phonemize(text, Lang::EnUs, &self.lexicon, &Overrides::default()).map_err(|e| TtsError::Invalid(e.to_string()))?;
|
||||
let rate = KOKORO_RATE;
|
||||
let max = (MAX_SECONDS * f64::from(rate)) as usize;
|
||||
let mut out: Vec<f32> = Vec::new();
|
||||
let mut spoke = false;
|
||||
for (i, c) in chunks.iter().enumerate() {
|
||||
match c {
|
||||
Chunk::Pause { millis } => {
|
||||
let n = (u64::from(*millis) * u64::from(rate) / 1000) as usize;
|
||||
out.resize(out.len().saturating_add(n), 0.0);
|
||||
}
|
||||
Chunk::Phonemes(p) => {
|
||||
if self.model.ids(p).is_empty() {
|
||||
continue;
|
||||
}
|
||||
if spoke {
|
||||
out.resize(out.len().saturating_add((SENTENCE_GAP_MS * rate / 1000) as usize), 0.0);
|
||||
}
|
||||
let s = self.model.speak(p, &self.pack, params.pace, i as u64 + 1)?;
|
||||
out.extend_from_slice(&s);
|
||||
spoke = true;
|
||||
}
|
||||
}
|
||||
if out.len() > max {
|
||||
return Err(TtsError::AudioTooLong);
|
||||
}
|
||||
}
|
||||
if !spoke {
|
||||
return Err(TtsError::Empty);
|
||||
}
|
||||
if params.semitones.abs() > 1e-6 {
|
||||
out = pitch_shift(out, params.semitones, rate);
|
||||
}
|
||||
// headroom: never clip
|
||||
let peak = out.iter().fold(0f32, |m, x| m.max(x.abs()));
|
||||
if peak > 0.98 {
|
||||
let g = 0.98 / peak;
|
||||
out.iter_mut().for_each(|x| *x *= g);
|
||||
}
|
||||
Ok(Audio { samples: out, sample_rate: rate })
|
||||
}
|
||||
}
|
||||
|
||||
/// Shift pitch by `semitones` keeping the length (latency compensated).
|
||||
fn pitch_shift(mut x: Vec<f32>, semitones: f64, rate: u32) -> Vec<f32> {
|
||||
let mut ps = filmcraft_audio_dsp::effects::PitchShifter::new(rate as f32, 1);
|
||||
ps.set_param("semitones", semitones as f32);
|
||||
ps.reset();
|
||||
let lat = ps.latency();
|
||||
let n = x.len();
|
||||
x.resize(n + lat, 0.0);
|
||||
for block in x.chunks_mut(4096) {
|
||||
ps.process(&mut [block]);
|
||||
}
|
||||
x.drain(..lat.min(x.len()));
|
||||
x.truncate(n);
|
||||
x
|
||||
}
|
||||
|
|
@ -0,0 +1,50 @@
|
|||
Welcome to the project list, where every job you create is stored.
|
||||
Click the blue button at the top of the page to start a new project.
|
||||
The filter menu lets you sort projects by date, by name, or by owner.
|
||||
Each card shows when the project was created and when it was last modified.
|
||||
Our team finished the review on Tuesday morning.
|
||||
The meeting starts at 3:30 p.m. in the main conference room.
|
||||
It costs $3.50 to ship each sample.
|
||||
Sales grew by 20% over the last quarter.
|
||||
The building was finished in 1999 and renovated in 2015.
|
||||
Please call 555-0134 if you have any questions.
|
||||
The report is due on October 7th.
|
||||
She bought 12 apples, 6 oranges, and a bag of grapes.
|
||||
Doctor Smith lives on Main Street near the library.
|
||||
The NASA mission launched before sunrise.
|
||||
The TDA tool exports every diagram as a PDF.
|
||||
Turn left at the second light and park behind the bank.
|
||||
A quiet river runs through the center of the old town.
|
||||
The children played in the garden until it got dark.
|
||||
Remember to save your work before you close the window.
|
||||
This video will show you how to edit a narration clip.
|
||||
The weather tomorrow will be sunny with a light breeze.
|
||||
He read the book in a single afternoon.
|
||||
The price dropped to $1,250 after the holiday sale.
|
||||
Our office opens at nine and closes at five.
|
||||
The first chapter explains the basic ideas.
|
||||
Water boils at one hundred degrees Celsius.
|
||||
They traveled by train from the coast to the mountains.
|
||||
The software update fixes several small problems.
|
||||
Every frame is rendered in less than a second.
|
||||
Choose a voice, type your script, and press save.
|
||||
The museum is closed on Mondays and public holidays.
|
||||
A good narrator speaks slowly and clearly.
|
||||
The new library has more than ten thousand books.
|
||||
Write down the steps so that anyone can repeat them.
|
||||
The engineer checked every valve before the test.
|
||||
Our company was founded in 1987 in a small garage.
|
||||
The second version runs twice as fast as the first.
|
||||
Bring a jacket, because the evening will be cold.
|
||||
The camera follows the bird as it flies over the lake.
|
||||
Thank you for watching, and see you in the next video.
|
||||
The orchestra played for almost two hours.
|
||||
Most users finish the setup in about five minutes.
|
||||
The road was closed because of the heavy snow.
|
||||
Please confirm your email address to continue.
|
||||
The garden is full of roses, tulips, and sunflowers.
|
||||
Our goal is to make editing simple for everyone.
|
||||
The pilot announced that we would land early.
|
||||
Fresh bread is baked here every morning.
|
||||
The team celebrated the launch with a small party.
|
||||
That is everything you need to know to get started.
|
||||
|
|
@ -10,6 +10,7 @@
|
|||
//! - `ui.elements {prefix?}`: registered widgets (optionally filtered by id prefix)
|
||||
//! - `ui.set {tool?, workspace?, mode?, theme?, focused?, playbackRes?, timeline?:{pps,scroll}}`
|
||||
//! - `ui.panel.show {panel}` / `ui.panel.close {panel}`
|
||||
//! - `ui.set {tts: {voice, pitch, pace, text…}}`: patch the Text to Speech panel draft
|
||||
//! - `ui.click {id | x,y, button?, count?, modifiers?}` / `ui.move {x,y}` / `ui.scroll {x,y,dx,dy}`
|
||||
//! - `ui.drag {from:{id|x,y}, to:{id|x,y}, steps?, modifiers?}`: synthetic press-move-release
|
||||
//! - `ui.key {key, command?, shift?, alt?, ctrl?}` / `ui.type {text}`
|
||||
|
|
@ -109,6 +110,15 @@ pub fn handle(app: &mut FilmcraftApp, ctx: &egui::Context, req: &ControlRequest)
|
|||
ok(serde_json::to_value(app.auto.query(prefix)).unwrap_or_default())
|
||||
}
|
||||
"ui.set" => {
|
||||
// Text to Speech panel draft (voice, pitch, pace, text…), merged into the current one
|
||||
if let Some(patch) = p.get("tts") {
|
||||
let mut cur = serde_json::to_value(&app.ui.tts).unwrap_or_default();
|
||||
merge(&mut cur, patch);
|
||||
match serde_json::from_value(cur) {
|
||||
Ok(d) => app.ui.tts = d,
|
||||
Err(e) => return err(format!("bad `tts`: {e}")),
|
||||
}
|
||||
}
|
||||
if let Some(t) = s("tool") {
|
||||
match Tool::from_name(t) {
|
||||
Some(t) => app.ui.tool = t,
|
||||
|
|
|
|||
|
|
@ -1267,6 +1267,9 @@ impl FilmcraftApp {
|
|||
self.advance_playback(&ctx);
|
||||
// Text to Speech: a selected narration clip loads into the panel
|
||||
panels::tts::follow_selection(self);
|
||||
if self.ui.tts.pending.is_some() || self.ui.tts.download_job.is_some() {
|
||||
ctx.request_repaint_after(std::time::Duration::from_millis(100));
|
||||
}
|
||||
let t = self.tokens;
|
||||
let full = ui.max_rect();
|
||||
ui.painter().rect_filled(full, 0.0, t.app_bg);
|
||||
|
|
|
|||
|
|
@ -3,8 +3,14 @@
|
|||
//! script and settings here (and brings the panel forward), and Save regenerates it in place.
|
||||
//! Every action is a `tts.*` command, so the panel, MCP and the control channel share one path.
|
||||
//!
|
||||
//! Synthesis runs as a background job (`tts.render`); when it finishes, the panel runs the action
|
||||
//! (preview, create or edit), which then uses the cached audio, so the window never stalls. The
|
||||
//! natural voices need a one-time download that the user confirms in the panel (size, source,
|
||||
//! licence); it runs as a job with progress and Cancel.
|
||||
//!
|
||||
//! Automation ids: `tts.language`, `tts.voice`, `tts.hearVoice`, `tts.advanced` (twirl),
|
||||
//! `tts.pitch`, `tts.pace`, `tts.text`, `tts.addPause`, `tts.preview`, `tts.save`, `tts.new`.
|
||||
//! `tts.pitch`, `tts.pace`, `tts.text`, `tts.addPause`, `tts.preview`, `tts.save`, `tts.new`,
|
||||
//! `tts.download`, `tts.download.confirm`, `tts.download.cancel`.
|
||||
|
||||
use egui::{Align2, Rect, Sense, Stroke, StrokeKind, pos2, vec2};
|
||||
use filmcraft_project::{ClipId, VocalPitch};
|
||||
|
|
@ -30,6 +36,23 @@ pub struct TtsDraft {
|
|||
pub advanced_open: bool,
|
||||
/// The narration clip and item the draft was loaded from (None: a new narration).
|
||||
pub loaded_from: Option<(u64, u64)>,
|
||||
/// An action waiting for its synthesis job.
|
||||
pub pending: Option<Pending>,
|
||||
/// The download confirmation is showing.
|
||||
pub confirm_download: bool,
|
||||
/// The running natural-voice download job.
|
||||
pub download_job: Option<u64>,
|
||||
}
|
||||
|
||||
/// A command to run when the synthesis job `job` finishes.
|
||||
#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
pub struct Pending {
|
||||
pub cmd: String,
|
||||
pub params: serde_json::Value,
|
||||
/// Play the preview afterwards.
|
||||
pub play: bool,
|
||||
pub job: u64,
|
||||
}
|
||||
|
||||
impl Default for TtsDraft {
|
||||
|
|
@ -37,11 +60,14 @@ impl Default for TtsDraft {
|
|||
TtsDraft {
|
||||
text: String::new(),
|
||||
language: "en-US".into(),
|
||||
voice: filmcraft_tts::default_voice_id().into(),
|
||||
voice: if natural_installed() { "kokoro-heart".into() } else { filmcraft_tts::default_voice_id().into() },
|
||||
pitch: VocalPitch::Default,
|
||||
pace: 1.0,
|
||||
advanced_open: true,
|
||||
loaded_from: None,
|
||||
pending: None,
|
||||
confirm_download: false,
|
||||
download_job: None,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -52,6 +78,88 @@ impl TtsDraft {
|
|||
}
|
||||
}
|
||||
|
||||
/// The natural voice package is downloaded.
|
||||
fn natural_installed() -> bool {
|
||||
filmcraft_engine::transcript::models_dir().as_deref().is_some_and(filmcraft_tts::catalog::installed)
|
||||
}
|
||||
|
||||
/// State of job `id`: `None` while running, else its result.
|
||||
fn job_outcome(app: &FilmcraftApp, id: u64) -> Option<Result<(), String>> {
|
||||
use std::sync::atomic::Ordering;
|
||||
let Some(j) = app.session.jobs.iter().find(|j| j.id == id) else { return Some(Err("the job disappeared".into())) };
|
||||
if !j.progress.finished.load(Ordering::Relaxed) {
|
||||
return None;
|
||||
}
|
||||
Some(match &*j.result.lock().unwrap_or_else(std::sync::PoisonError::into_inner) {
|
||||
Some(Ok(_)) => Ok(()),
|
||||
Some(Err(e)) => Err(e.clone()),
|
||||
None => Err("the job stopped".into()),
|
||||
})
|
||||
}
|
||||
|
||||
/// Start an action: synthesize in the background (or reuse the cache), then run `cmd`.
|
||||
fn start(app: &mut FilmcraftApp, cmd: &str, params: serde_json::Value, play: bool) {
|
||||
let mut render = params.clone();
|
||||
if let Some(o) = render.as_object_mut() {
|
||||
o.remove("dir");
|
||||
o.remove("time");
|
||||
}
|
||||
match app.session.execute("tts.render", render) {
|
||||
Ok(r) => match r["job"].as_u64() {
|
||||
Some(job) => {
|
||||
app.ui.tts.pending = Some(Pending { cmd: cmd.to_string(), params, play, job });
|
||||
app.ui.status = "Synthesizing narration…".into();
|
||||
}
|
||||
None => run_now(app, cmd, params, play),
|
||||
},
|
||||
Err(e) => app.ui.status = e.to_string(),
|
||||
}
|
||||
}
|
||||
|
||||
fn run_now(app: &mut FilmcraftApp, cmd: &str, params: serde_json::Value, play: bool) {
|
||||
match app.session.execute(cmd, params) {
|
||||
Ok(r) => {
|
||||
if play {
|
||||
if let Err(e) = app.play_tts_preview() {
|
||||
app.ui.status = e;
|
||||
}
|
||||
} else {
|
||||
app.ui.status = match cmd {
|
||||
"tts.create" => format!("Narration added on {}", r["track"].as_str().unwrap_or("an audio track")),
|
||||
_ => "Narration updated".into(),
|
||||
};
|
||||
}
|
||||
}
|
||||
Err(e) => app.ui.status = e.to_string(),
|
||||
}
|
||||
}
|
||||
|
||||
/// Finish a pending action whose synthesis job is done.
|
||||
fn poll(app: &mut FilmcraftApp) {
|
||||
if let Some(p) = app.ui.tts.pending.clone() {
|
||||
match job_outcome(app, p.job) {
|
||||
None => {}
|
||||
Some(Ok(())) => {
|
||||
app.ui.tts.pending = None;
|
||||
run_now(app, &p.cmd, p.params, p.play);
|
||||
}
|
||||
Some(Err(e)) => {
|
||||
app.ui.tts.pending = None;
|
||||
app.ui.status = format!("Text to Speech: {e}");
|
||||
}
|
||||
}
|
||||
}
|
||||
if let Some(j) = app.ui.tts.download_job
|
||||
&& let Some(r) = job_outcome(app, j)
|
||||
{
|
||||
app.ui.tts.download_job = None;
|
||||
app.ui.status = match r {
|
||||
Ok(()) => "Natural voices installed".into(),
|
||||
Err(e) => format!("Download of the natural voices failed: {e}"),
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
/// The selected narration clip and its item, if any.
|
||||
fn selected_narration(app: &FilmcraftApp) -> Option<(ClipId, u64)> {
|
||||
let clip = filmcraft_engine::narration::target_clip(&app.session, &serde_json::Value::Null)?;
|
||||
|
|
@ -62,6 +170,7 @@ fn selected_narration(app: &FilmcraftApp) -> Option<(ClipId, u64)> {
|
|||
/// Load the selected narration clip into the draft when the selection moves to a different one,
|
||||
/// and bring the panel forward. Runs every frame, also when the panel is hidden.
|
||||
pub fn follow_selection(app: &mut FilmcraftApp) {
|
||||
poll(app);
|
||||
let sel = selected_narration(app);
|
||||
match sel {
|
||||
Some((clip, item)) if app.ui.tts.loaded_from != Some((clip.0, item)) => {
|
||||
|
|
@ -90,13 +199,24 @@ pub fn follow_selection(app: &mut FilmcraftApp) {
|
|||
enum Action {
|
||||
Run(&'static str, serde_json::Value, bool),
|
||||
New,
|
||||
Download,
|
||||
CancelDownload,
|
||||
}
|
||||
|
||||
pub fn show(app: &mut FilmcraftApp, ui: &mut egui::Ui, rect: Rect) {
|
||||
let t = app.tokens;
|
||||
let painter = ui.painter().clone();
|
||||
painter.rect_filled(rect, 0.0, t.panel_bg);
|
||||
let voices = filmcraft_tts::voices();
|
||||
let models = filmcraft_engine::transcript::models_dir();
|
||||
let voices = filmcraft_tts::voices_in(models.as_deref());
|
||||
let installed = models.as_deref().is_some_and(filmcraft_tts::catalog::installed);
|
||||
let busy = app.ui.tts.pending.is_some();
|
||||
let download = app
|
||||
.ui
|
||||
.tts
|
||||
.download_job
|
||||
.and_then(|id| app.session.jobs.iter().find(|j| j.id == id))
|
||||
.map(|j| (j.progress.fraction(), j.progress.status.lock().unwrap_or_else(std::sync::PoisonError::into_inner).clone()));
|
||||
let editing = app.ui.tts.loaded_from.map(|(c, _)| c);
|
||||
let mut actions: Vec<Action> = Vec::new();
|
||||
let auto = &mut app.auto;
|
||||
|
|
@ -136,21 +256,76 @@ pub fn show(app: &mut FilmcraftApp, ui: &mut egui::Ui, rect: Rect) {
|
|||
label(ui, &t, "Voice");
|
||||
let voice_name = voices.iter().find(|v| v.id == d.voice).map_or(d.voice.as_str(), |v| v.name);
|
||||
let cb = egui::ComboBox::from_id_salt("tts-voice").selected_text(voice_name).width(w).show_ui(ui, |ui| {
|
||||
for v in voices.iter().filter(|v| v.language == d.language) {
|
||||
let r = ui.selectable_label(d.voice == v.id, v.name).on_hover_text(v.description);
|
||||
if r.clicked() {
|
||||
d.voice = v.id.to_string();
|
||||
for (engine, heading) in [("neural", "Natural voices"), ("basic", "Basic voices (built in)")] {
|
||||
ui.label(egui::RichText::new(heading).color(t.text_faint).size(11.0));
|
||||
for v in voices.iter().filter(|v| v.language == d.language && v.engine == engine) {
|
||||
let label = if v.engine == "neural" && !v.installed { format!("{} (download)", v.name) } else { v.name.to_string() };
|
||||
let r = ui.selectable_label(d.voice == v.id, label).on_hover_text(v.description);
|
||||
if r.clicked() {
|
||||
d.voice = v.id.to_string();
|
||||
}
|
||||
}
|
||||
}
|
||||
});
|
||||
auto.add("tts.voice", visible(ui, cb.response.rect), "Voice");
|
||||
let natural = filmcraft_tts::catalog::find(&d.voice).is_some();
|
||||
if natural && !installed {
|
||||
ui.add_space(6.0);
|
||||
egui::Frame::new().fill(t.field_bg).stroke(egui::Stroke::new(1.0, t.field_border)).corner_radius(4.0).inner_margin(8.0).show(ui, |ui| {
|
||||
ui.set_width(w - 18.0);
|
||||
match &download {
|
||||
Some((frac, status)) => {
|
||||
ui.label(egui::RichText::new(status).color(t.text).size(11.5));
|
||||
ui.add(egui::ProgressBar::new(*frac).show_percentage());
|
||||
let b = ui.small_button("Cancel");
|
||||
auto.add("tts.download.cancel", visible(ui, b.rect), "Cancel download");
|
||||
if b.clicked() {
|
||||
actions.push(Action::CancelDownload);
|
||||
}
|
||||
}
|
||||
None if d.confirm_download => {
|
||||
let mb = filmcraft_tts::catalog::size() as f64 / 1e6;
|
||||
ui.label(egui::RichText::new(format!("Download the natural voices? {mb:.0} MB, once.")).color(t.text).size(12.0).strong());
|
||||
ui.label(
|
||||
egui::RichText::new(format!(
|
||||
"From {} and github.com/cmusphinx/cmudict (pronunciations). Licence: {}. Every file is checked against a pinned SHA-256.",
|
||||
filmcraft_tts::catalog::SOURCE,
|
||||
filmcraft_tts::catalog::LICENSE
|
||||
))
|
||||
.color(t.text_dim)
|
||||
.size(11.0),
|
||||
);
|
||||
ui.horizontal(|ui| {
|
||||
let ok = ui.button("Download");
|
||||
auto.add("tts.download.confirm", visible(ui, ok.rect), "Download");
|
||||
let no = ui.button("Cancel");
|
||||
if ok.clicked() {
|
||||
actions.push(Action::Download);
|
||||
}
|
||||
if no.clicked() {
|
||||
d.confirm_download = false;
|
||||
}
|
||||
});
|
||||
}
|
||||
None => {
|
||||
ui.label(egui::RichText::new("Natural voices need a one-time download.").color(t.text).size(11.5));
|
||||
let b = ui.button(format!("Download natural voices ({:.0} MB)…", filmcraft_tts::catalog::size() as f64 / 1e6));
|
||||
auto.add("tts.download", visible(ui, b.rect), "Download natural voices");
|
||||
if b.clicked() {
|
||||
d.confirm_download = true;
|
||||
}
|
||||
}
|
||||
}
|
||||
});
|
||||
}
|
||||
let voice_ready = !natural || installed;
|
||||
ui.add_space(4.0);
|
||||
let (r, resp) = ui.allocate_exact_size(vec2(130.0, 22.0), Sense::click());
|
||||
let col = if resp.hovered() { t.accent_hover } else { t.accent };
|
||||
icons::paint(ui.painter(), Rect::from_center_size(pos2(r.min.x + 8.0, r.center().y), vec2(14.0, 14.0)), Icon::Play, col);
|
||||
ui.painter().text(pos2(r.min.x + 22.0, r.center().y), Align2::LEFT_CENTER, "Hear this voice", Tokens::semibold(12.0), col);
|
||||
auto.add("tts.hearVoice", visible(ui, r), "Hear this voice");
|
||||
if resp.clicked() {
|
||||
if resp.clicked() && voice_ready && !busy {
|
||||
actions.push(Action::Run("tts.preview", json!({"sample": true, "voice": d.voice, "pitch": d.pitch.id(), "pace": d.pace}), true));
|
||||
}
|
||||
ui.add_space(8.0);
|
||||
|
|
@ -210,9 +385,10 @@ pub fn show(app: &mut FilmcraftApp, ui: &mut egui::Ui, rect: Rect) {
|
|||
// footer: Preview | Save
|
||||
let fy = rect.max.y - FOOTER_H + 10.0;
|
||||
let half = (rect.width() - PAD * 2.0 - 10.0) / 2.0;
|
||||
let can_say = filmcraft_tts::check_text(&d.text).is_ok();
|
||||
let natural = filmcraft_tts::catalog::find(&d.voice).is_some();
|
||||
let can_say = filmcraft_tts::check_text(&d.text).is_ok() && (!natural || installed) && !busy;
|
||||
let pr = Rect::from_min_size(pos2(rect.min.x + PAD, fy), vec2(half, 28.0));
|
||||
if button(ui, &t, pr, Icon::Play, "Preview", false, can_say, "tts-preview").clicked() && can_say {
|
||||
if button(ui, &t, pr, Icon::Play, if busy { "Synthesizing…" } else { "Preview" }, false, can_say, "tts-preview").clicked() && can_say {
|
||||
actions.push(Action::Run("tts.preview", d.params(), true));
|
||||
}
|
||||
auto.add("tts.preview", pr, "Preview");
|
||||
|
|
@ -235,23 +411,24 @@ pub fn show(app: &mut FilmcraftApp, ui: &mut egui::Ui, rect: Rect) {
|
|||
Action::New => {
|
||||
let keep = (app.ui.tts.language.clone(), app.ui.tts.voice.clone(), app.ui.tts.pitch, app.ui.tts.pace);
|
||||
app.session.state.selection.clear();
|
||||
app.ui.tts = TtsDraft { language: keep.0, voice: keep.1, pitch: keep.2, pace: keep.3, ..Default::default() };
|
||||
let dl = app.ui.tts.download_job;
|
||||
app.ui.tts = TtsDraft { language: keep.0, voice: keep.1, pitch: keep.2, pace: keep.3, download_job: dl, ..Default::default() };
|
||||
}
|
||||
Action::Run(cmd, p, play) => match app.session.execute(cmd, p) {
|
||||
Ok(r) => {
|
||||
if play {
|
||||
if let Err(e) = app.play_tts_preview() {
|
||||
app.ui.status = e;
|
||||
}
|
||||
} else {
|
||||
app.ui.status = match cmd {
|
||||
"tts.create" => format!("Narration added on {}", r["track"].as_str().unwrap_or("an audio track")),
|
||||
_ => "Narration updated".into(),
|
||||
};
|
||||
}
|
||||
Action::Run(cmd, p, play) => start(app, cmd, p, play),
|
||||
Action::Download => {
|
||||
app.ui.tts.confirm_download = false;
|
||||
match app.session.execute("tts.downloadVoices", json!({})) {
|
||||
Ok(r) => app.ui.tts.download_job = r["job"].as_u64(),
|
||||
Err(e) => app.ui.status = e.to_string(),
|
||||
}
|
||||
Err(e) => app.ui.status = e.to_string(),
|
||||
},
|
||||
}
|
||||
Action::CancelDownload => {
|
||||
if let Some(id) = app.ui.tts.download_job
|
||||
&& let Some(j) = app.session.jobs.iter().find(|j| j.id == id)
|
||||
{
|
||||
j.progress.cancel.store(true, std::sync::atomic::Ordering::Relaxed);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -77,6 +77,18 @@ impl Driver {
|
|||
self.ui()["ui"]["tts"].clone()
|
||||
}
|
||||
|
||||
/// Let a background synthesis job finish.
|
||||
fn settle(&mut self) {
|
||||
for _ in 0..400 {
|
||||
if self.draft()["pending"].is_null() {
|
||||
return;
|
||||
}
|
||||
std::thread::sleep(std::time::Duration::from_millis(10));
|
||||
self.frames(1);
|
||||
}
|
||||
panic!("synthesis never finished");
|
||||
}
|
||||
|
||||
fn ids(&mut self, prefix: &str) -> Vec<String> {
|
||||
let v = self.ok("ui.elements", json!({"prefix": prefix}));
|
||||
v.as_array().unwrap().iter().filter_map(|e| e["id"].as_str().map(str::to_string)).collect()
|
||||
|
|
@ -128,6 +140,7 @@ fn write_add_click_to_edit_and_save_in_place() {
|
|||
// add it to the timeline
|
||||
d.exec("playhead.set", json!({"seconds": 1.0}));
|
||||
d.click("tts.save");
|
||||
d.settle();
|
||||
let clips = d.narration_clips();
|
||||
let status = d.ui()["ui"]["status"].clone();
|
||||
assert_eq!(clips.len(), 1, "one narration clip; status {status}");
|
||||
|
|
@ -154,6 +167,7 @@ fn write_add_click_to_edit_and_save_in_place() {
|
|||
d.ok("ui.key", json!({"key": "Cmd+A"}));
|
||||
d.type_text("A new script for the same clip.");
|
||||
d.click("tts.save");
|
||||
d.settle();
|
||||
let after = d.exec("tts.inspect", json!({"clip": clip}));
|
||||
assert_eq!(after["narration"]["text"], "A new script for the same clip.");
|
||||
assert_ne!(after["item"], before);
|
||||
|
|
@ -177,6 +191,7 @@ fn settings_controls_and_hear_this_voice() {
|
|||
assert!(d.narration_clips().is_empty());
|
||||
// Hear this voice runs a preview (the headless harness has no audio output: a status line)
|
||||
d.click("tts.hearVoice");
|
||||
d.settle();
|
||||
let status = d.ui()["ui"]["status"].as_str().unwrap_or_default().to_string();
|
||||
assert!(status.contains("audio output") || status.is_empty(), "{status}");
|
||||
// Advanced folds away and back
|
||||
|
|
|
|||
|
|
@ -24,7 +24,7 @@ Design principles:
|
|||
L5 ui-egui · automation · platform
|
||||
L4 engine
|
||||
L3 render · gpu · export · golden (test-only)
|
||||
L2 edit · codecs · interchange · captions · speech
|
||||
L2 edit · codecs · interchange · captions · speech · tts · tts-text
|
||||
L1 frame · media · project · audio-dsp · text
|
||||
L0 foundation: time · geom · color · bitstream · testkit (dev-dependency only)
|
||||
codecs/containers: isobmff · matroska · mxf · cfb · mpegts · ogg · h264 · h264enc · hevc · vp9 · av1 · mpeg2v · prores · dnx · apv · aac · ac3 · opus
|
||||
|
|
@ -62,6 +62,8 @@ and `filmcraft-cli`.
|
|||
| `text` | L1 | text engine: font database (bundled OFL fonts + system fonts), shaping (harfrust), bidi, line breaking, paragraph layout, glyph/path rasteriser, strokes ([crates/text/README.md](../crates/text/README.md)) |
|
||||
| `edit` | L2 | pure edit algebra (insert, overwrite, razor, ripple, roll, slip, slide, rate stretch…; text-based editing: `edit::transcript`) |
|
||||
| `speech` | L2 | speech-to-text: `Transcriber` trait, Whisper model catalogue + verified downloader (feature `download`), pure-Rust Whisper inference on candle with word timestamps (feature `whisper`), speaker labelling ([transcripts.md](transcripts.md)) |
|
||||
| `tts` | L2 | text to speech: `Voice` trait, voice catalogue, pause markers, built-in formant voices; Kokoro-82M natural voices on candle and their pinned package catalogue (feature `kokoro`). Same-layer edge to `tts-text` |
|
||||
| `tts-text` | L2 | text front end for narration: normalizer (numbers, money, dates, acronyms, pauses) and English pronunciation (CMUdict + letter-to-sound) to Kokoro phonemes |
|
||||
| `codecs` | L2 | container + codec hub: MP4/MOV, MKV, MXF, Ogg and MPEG TS / PS / video elementary stream sources, GOP-aware seeking, decoder registry, audio decoding |
|
||||
| `interchange` | L2 | EDL, FCP7 XML, FCPXML, OTIO, AAF (on `cfb`) and OMF 2.0 import/export (no file I/O; the engine supplies rendered audio essence) ([README](../crates/interchange/README.md)) |
|
||||
| `render` | L3 | sequence evaluation, CPU compositor, video effects (`effects`, `vfx`; effects needing other frames or tracks read them through `vfx::FxEnv`), transitions, audio mix |
|
||||
|
|
@ -437,7 +439,7 @@ stereo / mono track ──5.1 panner───┼─► 5.1 submix / 5.1 Mix ─
|
|||
|
||||
### 5.1.2a Text to Speech (narrations)
|
||||
|
||||
**Narrations** (`engine::narration`, `tts.*`; voices in `filmcraft-tts`). The Text to Speech panel (Window ▸ Text to Speech) writes a script with pause markers (`[pause 1s]`), a voice, a vocal pitch and a pace; `tts.create` synthesizes it, writes `Narration <n>.wav` (mono 32-bit float, 24 kHz, where voice-over recordings go), imports it and places it at the playhead on the first free targeted audio track, else the first free audio track, else a new one, as one undo step. The settings are kept per generated item in `Project::narrations` (schema v13). Selecting a narration clip loads it into the panel; `tts.edit` writes a new file (beside the previous one) and swaps the clip to it, keeping the clip's start, duration and speed: longer speech is kept whole in the file but plays only to the clip's end; shorter speech is padded with silence to the clip's length. The old item and narration stay, so undo is exact. `tts.preview` synthesizes without changing the project and the UI plays `Session::tts_preview` on the audio output. Built-in voices are an original formant synthesizer (robotic, no download); neural voices come later.
|
||||
**Narrations** (`engine::narration`, `tts.*`; voices in `filmcraft-tts`). The Text to Speech panel (Window ▸ Text to Speech) writes a script with pause markers (`[pause 1s]`), a voice, a vocal pitch and a pace; `tts.create` synthesizes it, writes `Narration <n>.wav` (mono 32-bit float, 24 kHz, where voice-over recordings go), imports it and places it at the playhead on the first free targeted audio track, else the first free audio track, else a new one, as one undo step. The settings are kept per generated item in `Project::narrations` (schema v13). Selecting a narration clip loads it into the panel; `tts.edit` writes a new file (beside the previous one) and swaps the clip to it, keeping the clip's start, duration and speed: longer speech is kept whole in the file but plays only to the clip's end; shorter speech is padded with silence to the clip's length. The old item and narration stay, so undo is exact. `tts.preview` synthesizes without changing the project and the UI plays `Session::tts_preview` on the audio output. Built-in voices are an original formant synthesizer (robotic, no download). The natural voices (feature `neural-voices`, on in the desktop app) are Kokoro-82M on candle (`filmcraft-tts`) fed by `filmcraft-tts-text` (normalizer, CMUdict + letter-to-sound); their 336 MB package is downloaded once after the user confirms (`tts.downloadVoices`, a job). The UI synthesizes with `tts.render` (a job filling `Session::tts_cache`) and then runs create / edit / preview, which use the cache.
|
||||
|
||||
### 5.1.3 Remix
|
||||
|
||||
|
|
|
|||
|
|
@ -30,7 +30,7 @@ FILMCRAFT_REQUIRE_ORACLES=1 cargo test --workspace # CI: missing ffmpeg fails
|
|||
| 5.1 / multichannel | `crates/audio-dsp/src/channels.rs`, `crates/render/src/mixer_tests.rs`, `crates/export/src/surround_tests.rs`, `crates/engine/src/mixer_tests.rs`, `crates/ui-egui/tests/audio51_ui.rs` | BS.775 downmix / upmix coefficients exact, mixdown types; 5.1 panner point gains equal-power (Σg² = 1 over a 21×21 grid × 3 centre settings), speaker positions exact, 5.1 through the default puck bit-exact identity; graph: stereo/mono tracks into a 5.1 Mix (L/R, C, rear right, room centre −6 dB each, LFE = mean), 5.1 track through a 5.1 Mix and folded into stereo, a 5.1 submix, `pan51` automation block-invariant, six-channel meters; export: 6-ch WAV (`WAVE_FORMAT_EXTENSIBLE`, mask 0x3F) and ProRes MOV (6 × 24-bit PCM) decoded by ffmpeg **sample-exact**, ffprobe `channels` 6 / `channel_layout` 5.1; AAC 5.1 in MP4: every decoded channel's own tone ≥ 30 dB above the others' (no swapped channels), stereo export of a 5.1 Mix = BS.775 within 2·10⁻⁶; `sequence.settings {mix}`, `file.newSequence {mix, trackType}`, panner lanes, Write pass records the puck; UI: puck drag (one undo step), Center / LFE, six meter bars |
|
||||
| Audio Clip Mixer automation, Track Mixer view | `crates/engine/src/mixer_tests.rs`, `crates/ui-egui/tests/audio51_ui.rs` | Touch: a 201-point fader ramp thins to ≤ 6 clip keyframes, values within 0.1 dB, live override while held, values outside the gesture unchanged, one undo step; Latch holds until stop; Write records volume and pan from the start; a gesture across two clips writes both; Read commits once without a pass; UI: Clip Mixer mode dropdown, a recorded fader drag writes clip keyframes; Show/Hide Tracks, Meter Input(s) Only, transport record button |
|
||||
| Voice-over | `crates/engine/src/voiceover_tests.rs`, `crates/ui-egui/tests/voiceover_ui.rs` | synthetic click train lands in `mix_sequence` exactly at R + k·period (pre-roll dropped, exact sample count), punch-in between In/Out (ramp source, every sample checked), one undo step + undo/redo, take numbering, record-arm/targeting track choice, sync/discard/zero-length/locked cases, cue times; UI: right-click Mic → Voice-Over Record Settings dialog (edits persist on OK, Cancel discards), the Mic button starts and stops a take on its track with pre-roll and overlay |
|
||||
| Text to Speech | `crates/tts/src/*.rs`, `crates/engine/src/narration_tests.rs`, `crates/ui-egui/tests/tts_ui.rs` | built-in voices: determinism, finite samples, exact-silence pause markers, pace scales length, measured pitch per voice and per semitone, hostile scripts; engine: narration placed at the playhead with clip length = speech (sample-exact), never overwrites (busy tracks → next free track → new track), occupied/locked/unknown track refused with no change, edit keeps start/length/speed (longer speech: whole speech in the file; shorter: file padded with exact zeros), edits write beside the previous file, undo/redo restore the project exactly, enabled only with a narration clip (or `clip`), 16 hostile parameter sets refused without change, preview changes nothing, save/reload keeps narrations, the narration is audible in `mix_sequence`; UI: type + Add pause + Add to timeline, the new clip is selected and loaded, clicking the clip on the timeline loads its script, Save edits it in place as one undo step, Advanced twirl, panel opens beside Properties |
|
||||
| Text to Speech | `crates/tts/src/**/*.rs` (Kokoro tests need `FILMCRAFT_KOKORO_DIR`), `crates/tts-text`, `crates/engine/src/narration_tests.rs`, `crates/ui-egui/tests/tts_ui.rs` | built-in voices: determinism, finite samples, exact-silence pause markers, pace scales length, measured pitch per voice and per semitone, hostile scripts; engine: narration placed at the playhead with clip length = speech (sample-exact), never overwrites (busy tracks → next free track → new track), occupied/locked/unknown track refused with no change, edit keeps start/length/speed (longer speech: whole speech in the file; shorter: file padded with exact zeros), edits write beside the previous file, undo/redo restore the project exactly, enabled only with a narration clip (or `clip`), 16 hostile parameter sets refused without change, preview changes nothing, save/reload keeps narrations, the narration is audible in `mix_sequence`; UI: type + Add pause + Add to timeline, the new clip is selected and loaded, clicking the clip on the timeline loads its script, Save edits it in place as one undo step, Advanced twirl, panel opens beside Properties; natural voices: 50-sentence Whisper-base WER 2.1–3.6 % for all 9 voices (crate README), loaders fuzzed with hostile files, `tts.render` cache, download refused without the feature |
|
||||
| Remix | `crates/audio-dsp/src/remix.rs`, `crates/render/src/remix.rs`, `crates/engine/src/remix_tests.rs`, `crates/ui-egui/tests/remix_ui.rs` | generated rhythmic music (Rust-synthesised drums, bass, chord sections at 90–128 BPM, 22.05/44.1/48 kHz): tempo within 0.3 BPM, every beat within 10 ms (measured ≤ 8.9 ms); remix targets from 0.4× to 2.2× the source within one beat (measured ≤ 12 ms at default sliders), every cut on a detected beat, intro and outro kept, pieces ≥ 4 beats, deterministic; silence, tone and noise refused; rendering independent of request cuts, source-exact outside the 20 ms equal-power crossfades, no clicks; the engine's mix equals the source pieces (< 1e-5); undo, redo and revert exact; overlaps refused; Remix Properties dialog and Remix tool drag driven by automation id |
|
||||
| Oracle | `crates/*/tests/*oracle*.rs`, `conformance.rs` | compare with ffmpeg/ffprobe (§2) |
|
||||
| Robustness / fuzz | `*/tests/robustness.rs`, `*/tests/fuzz.rs` | seeded mutation and truncation of real and synthetic files; nothing may panic |
|
||||
|
|
|
|||
|
|
@ -95,6 +95,7 @@ const SAME_LAYER: &[(&str, &str)] = &[
|
|||
("scopes", "gpu"),
|
||||
("gpu", "render"),
|
||||
("cli", "filmcraft"),
|
||||
("tts", "tts-text"),
|
||||
];
|
||||
|
||||
/// Crates that must not appear below L5 (UI toolkits, windowing, OS audio/menus).
|
||||
|
|
|
|||
Loading…
Reference in New Issue