diff --git a/.idea/dictionaries/bt.xml b/.idea/dictionaries/bt.xml index 708c9a0f39..b92131dc77 100644 --- a/.idea/dictionaries/bt.xml +++ b/.idea/dictionaries/bt.xml @@ -26,6 +26,7 @@ transcoded transcodings upserting + vocoded vocoder vocoders vocodes diff --git a/crates/lib/database_queries/src/queries/tts/tts_models/get_tts_model_for_inference.rs b/crates/lib/database_queries/src/queries/tts/tts_models/get_tts_model_for_inference.rs index 490429f21c..4d94dc12fc 100644 --- a/crates/lib/database_queries/src/queries/tts/tts_models/get_tts_model_for_inference.rs +++ b/crates/lib/database_queries/src/queries/tts/tts_models/get_tts_model_for_inference.rs @@ -8,6 +8,7 @@ use enums::common::vocoder_type::VocoderType; use log::warn; use sqlx::MySqlPool; use sqlx; +use crate::helpers::boolean_converters::i8_to_bool; // TODO: Can probably just reuse another query. @@ -19,6 +20,10 @@ pub struct TtsModelForInferenceRecord { /// NB: text_pipeline_type may not always be present in the database. pub text_pipeline_type: Option, + /// Whether to multiply the mel outputs before being vocoded. + pub use_default_mel_multiply_factor: bool, + pub maybe_custom_mel_multiply_factor: Option, + /// [vocoders 1] /// This is the new type of vocoder configuration. Users can choose a custom trained /// vocoder to associate with their model. The tokens reference the `vocoder_models` @@ -88,6 +93,9 @@ SELECT tts.tts_model_type, tts.text_pipeline_type, + tts.use_default_mel_multiply_factor, + tts.maybe_custom_mel_multiply_factor, + tts.maybe_default_pretrained_vocoder, tts.maybe_custom_vocoder_token, @@ -145,6 +153,8 @@ WHERE tts.token = ? model_token: model.model_token, tts_model_type: model.tts_model_type, text_pipeline_type: model.text_pipeline_type, + use_default_mel_multiply_factor: i8_to_bool(model.use_default_mel_multiply_factor), + maybe_custom_mel_multiply_factor: model.maybe_custom_mel_multiply_factor, maybe_custom_vocoder: match model.maybe_custom_vocoder_token { // NB: We're relying on a single field's presence to infer that the others vocoder fields // are also there. If for some reason they aren't, fail open. @@ -178,6 +188,9 @@ struct InternalTtsModelForInferenceRecordRaw { text_pipeline_type: Option, + use_default_mel_multiply_factor: i8, // NB: bool + maybe_custom_mel_multiply_factor: Option, + // Joined custom vocoder fields maybe_custom_vocoder_token: Option, maybe_custom_vocoder_type: Option, diff --git a/crates/service/job/tts_inference_job/src/http_clients/tts_inference_sidecar_client.rs b/crates/service/job/tts_inference_job/src/http_clients/tts_inference_sidecar_client.rs index 0a3a017bc7..73ce8aa0f9 100644 --- a/crates/service/job/tts_inference_job/src/http_clients/tts_inference_sidecar_client.rs +++ b/crates/service/job/tts_inference_job/src/http_clients/tts_inference_sidecar_client.rs @@ -30,6 +30,10 @@ struct InferenceRequest { // Named text pipeline/algorithm, eg. "legacy_fakeyou", "english_v1", "spanish_v2", etc. pub text_pipeline_type: String, + // Whether to multiply the mel outputs before being vocoded. + use_default_mel_multiply_factor: bool, + maybe_custom_mel_multiply_factor: Option, + // Output information pub output_audio_filename : String, pub output_spectrogram_filename : String, @@ -67,6 +71,8 @@ impl TtsInferenceSidecarClient { output_spectrogram_filename: P, output_metadata_filename: P, maybe_unload_model_path: Option, + use_default_mel_multiply_factor: bool, + maybe_custom_mel_multiply_factor: Option, ) -> AnyhowResult<()> { let waveglow_vocoder_checkpoint_path = waveglow_vocoder_checkpoint_path @@ -116,6 +122,7 @@ impl TtsInferenceSidecarClient { max_decoder_steps, vocoder_type, text_pipeline_type: text_pipeline_type.to_string(), + use_default_mel_multiply_factor, waveglow_vocoder_checkpoint_path, hifigan_vocoder_checkpoint_path, hifigan_superres_vocoder_checkpoint_path, @@ -124,6 +131,7 @@ impl TtsInferenceSidecarClient { output_spectrogram_filename, output_metadata_filename, maybe_clear_synthesizer_checkpoint_path: maybe_unload_model_path, + maybe_custom_mel_multiply_factor, }; let url = format!("http://{}/infer", self.hostname); diff --git a/crates/service/job/tts_inference_job/src/job_steps/process_single_job.rs b/crates/service/job/tts_inference_job/src/job_steps/process_single_job.rs index 0f89ded89f..eede074aba 100644 --- a/crates/service/job/tts_inference_job/src/job_steps/process_single_job.rs +++ b/crates/service/job/tts_inference_job/src/job_steps/process_single_job.rs @@ -301,7 +301,10 @@ pub async fn process_single_job( &output_audio_fs_path, &output_spectrogram_fs_path, &output_metadata_fs_path, - maybe_unload_model_path) + maybe_unload_model_path, + model_record.use_default_mel_multiply_factor, + model_record.maybe_custom_mel_multiply_factor, + ) .await .map_err(|e| ProcessSingleJobError::Other(e))?;