From bd4bcab2c215ab8d474a698dcacfe0f3217df80e Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Thu, 18 May 2023 00:14:42 -0700 Subject: [PATCH] args for pitch/f0 method in http handler --- _sql/actions/generic_inference_jobs.sql | 7 ++- _sql/actions/w2l_inference_jobs.sql | 3 + .../src/payloads/generic_inference_args.rs | 58 ++++++++++++++++++- .../enqueue_voice_conversion_inference.rs | 34 ++++++++++- 4 files changed, 93 insertions(+), 9 deletions(-) diff --git a/_sql/actions/generic_inference_jobs.sql b/_sql/actions/generic_inference_jobs.sql index 7dea0cdd86..acdaf0f5eb 100644 --- a/_sql/actions/generic_inference_jobs.sql +++ b/_sql/actions/generic_inference_jobs.sql @@ -3,6 +3,7 @@ -- noinspection SqlResolveForFile -- Kill routed jobs -update generic_inference_jobs -set status = 'dead' where maybe_routing_tag IS NOT NULL -limit 100; +update generic_inference_jobs set status = 'dead' where maybe_routing_tag IS NOT NULL limit 100; + +-- Get pending jobs +select count(*) from generic_inference_jobs where status = 'pending'; diff --git a/_sql/actions/w2l_inference_jobs.sql b/_sql/actions/w2l_inference_jobs.sql index 8f634c8c21..d7d38244b4 100644 --- a/_sql/actions/w2l_inference_jobs.sql +++ b/_sql/actions/w2l_inference_jobs.sql @@ -2,6 +2,9 @@ -- noinspection SqlNoDataSourceInspectionForFile -- noinspection SqlResolveForFile +-- Kill pending jobs +update w2l_inference_jobs set status='dead' where status IN ('pending', 'started', 'attempt_failed'); + -- Unstick old jobs that might be stuck as "started" update w2l_template_upload_jobs set status='dead' diff --git a/crates/lib/mysql_queries/src/payloads/generic_inference_args.rs b/crates/lib/mysql_queries/src/payloads/generic_inference_args.rs index 3f709268e3..72b12a44f2 100644 --- a/crates/lib/mysql_queries/src/payloads/generic_inference_args.rs +++ b/crates/lib/mysql_queries/src/payloads/generic_inference_args.rs @@ -32,6 +32,16 @@ pub enum InferenceCategoryAbbreviated { VoiceConversion, } +#[derive(Serialize, Deserialize, Clone, Copy, Debug)] +pub enum FundamentalFrequencyMethodForJob { + #[serde(rename = "c")] // NB: DO NOT CHANGE. It could break live jobs. Renamed to be fewer bytes. + Crepe, + #[serde(rename = "d")] // NB: DO NOT CHANGE. It could break live jobs. Renamed to be fewer bytes. + Dio, + #[serde(rename = "h")] // NB: DO NOT CHANGE. It could break live jobs. Renamed to be fewer bytes. + Harvest, +} + #[derive(Clone, Debug, Serialize, Deserialize)] pub enum PolymorphicInferenceArgs { /// Text to speech. (Short name to save space when serializing.) @@ -44,8 +54,23 @@ pub enum PolymorphicInferenceArgs { /// Argument for so-vits-svc /// The python model defaults to true, but that sounds awful, /// so we default to false unless specified. + #[serde(rename = "a")] // NB: DO NOT CHANGE. It could break live jobs. Renamed to be fewer bytes. #[serde(skip_serializing_if = "Option::is_none")] auto_predict_f0: Option, + + /// Argument for so-vits-svc + /// The python model defaults to true, but that sounds awful, + /// so we default to false unless specified. + #[serde(rename = "fm")] // NB: DO NOT CHANGE. It could break live jobs. Renamed to be fewer bytes. + #[serde(skip_serializing_if = "Option::is_none")] + override_f0_method: Option, + + /// Argument for so-vits-svc + /// The python model defaults to true, but that sounds awful, + /// so we default to false unless specified. + #[serde(rename = "t")] // NB: DO NOT CHANGE. It could break live jobs. Renamed to be fewer bytes. + #[serde(skip_serializing_if = "Option::is_none")] + transpose: Option, }, } @@ -78,7 +103,7 @@ impl InferenceCategoryAbbreviated { #[cfg(test)] mod tests { - use crate::payloads::generic_inference_args::{GenericInferenceArgs, InferenceCategoryAbbreviated, PolymorphicInferenceArgs}; + use crate::payloads::generic_inference_args::{FundamentalFrequencyMethodForJob, GenericInferenceArgs, InferenceCategoryAbbreviated, PolymorphicInferenceArgs}; use enums::by_table::generic_inference_jobs::inference_category::InferenceCategory; use tokens::files::media_upload::MediaUploadToken; use tokens::tokens::tts_models::TtsModelToken; @@ -107,6 +132,8 @@ mod tests { inference_category: Some(InferenceCategoryAbbreviated::VoiceConversion), args: Some(PolymorphicInferenceArgs::Vc { auto_predict_f0: Some(false), + override_f0_method: None, + transpose: None, }), }; @@ -114,7 +141,28 @@ mod tests { // NB: Assert the serialized form. If this changes and the test breaks, be careful about migrating. assert_eq!(json, - r#"{"cat":"vc","args":{"Vc":{"auto_predict_f0":false}}}"#.to_string()); + r#"{"cat":"vc","args":{"Vc":{"a":false}}}"#.to_string()); + + // NB: Make sure we don't overflow the DB field capacity (TEXT column). + assert!(json.len() < 1000); + } + + #[test] + fn many_voice_conversion_args_serialize() { + let args = GenericInferenceArgs { + inference_category: Some(InferenceCategoryAbbreviated::VoiceConversion), + args: Some(PolymorphicInferenceArgs::Vc { + auto_predict_f0: Some(false), + override_f0_method: Some(FundamentalFrequencyMethodForJob::Dio), + transpose: Some(-1), + }), + }; + + let json = serde_json::ser::to_string(&args).unwrap(); + + // NB: Assert the serialized form. If this changes and the test breaks, be careful about migrating. + assert_eq!(json, + r#"{"cat":"vc","args":{"Vc":{"a":false,"fm":"d","t":-1}}}"#.to_string()); // NB: Make sure we don't overflow the DB field capacity (TEXT column). assert!(json.len() < 1000); @@ -126,6 +174,8 @@ mod tests { inference_category: Some(InferenceCategoryAbbreviated::VoiceConversion), args: Some(PolymorphicInferenceArgs::Vc { auto_predict_f0: None, // NB: Do not serialize + override_f0_method: None, + transpose: None, }), }; @@ -151,6 +201,8 @@ mod tests { inference_category: Some(InferenceCategoryAbbreviated::VoiceConversion), args: Some(PolymorphicInferenceArgs::Vc { auto_predict_f0: Some(true), + override_f0_method: None, + transpose: None, }), }); @@ -158,7 +210,7 @@ mod tests { // NB: Assert the serialized form. If this changes and the test breaks, be careful about migrating. assert_eq!(json, - r#"{"cat":"vc","args":{"Vc":{"auto_predict_f0":true}}}"#.to_string()); + r#"{"cat":"vc","args":{"Vc":{"a":true}}}"#.to_string()); // NB: Make sure we don't overflow the DB field capacity (TEXT column). assert!(json.len() < 1000); diff --git a/crates/service/web/storyteller_web/src/http_server/endpoints/voice_conversion/inference/enqueue_voice_conversion_inference.rs b/crates/service/web/storyteller_web/src/http_server/endpoints/voice_conversion/inference/enqueue_voice_conversion_inference.rs index e86c34f57f..493a1c7e6a 100644 --- a/crates/service/web/storyteller_web/src/http_server/endpoints/voice_conversion/inference/enqueue_voice_conversion_inference.rs +++ b/crates/service/web/storyteller_web/src/http_server/endpoints/voice_conversion/inference/enqueue_voice_conversion_inference.rs @@ -15,7 +15,7 @@ use enums::common::visibility::Visibility; use http_server_common::request::get_request_header_optional::get_request_header_optional; use http_server_common::request::get_request_ip::get_request_ip; use log::{info, warn}; -use mysql_queries::payloads::generic_inference_args::{GenericInferenceArgs, InferenceCategoryAbbreviated, PolymorphicInferenceArgs}; +use mysql_queries::payloads::generic_inference_args::{FundamentalFrequencyMethodForJob, GenericInferenceArgs, InferenceCategoryAbbreviated, PolymorphicInferenceArgs}; use mysql_queries::queries::generic_inference::web::insert_generic_inference_job::{InsertGenericInferenceArgs, insert_generic_inference_job}; use r2d2_redis::redis::Commands; use redis_common::redis_keys::RedisKeys; @@ -52,6 +52,24 @@ pub struct EnqueueVoiceConversionInferenceRequest { /// The python model defaults to true, but that sounds awful, /// so we default to false unless specified. auto_predict_f0: Option, + + /// Argument for so-vits-svc + /// f0 estimation + override_f0_method: Option, + + /// Argument for so-vits-svc + /// Pitch controls for so-vits-svc + transpose: Option, +} + +#[derive(Deserialize, Clone, Copy)] +pub enum FundamentalFrequencyMethod { + #[serde(rename = "crepe")] + Crepe, + #[serde(rename = "dio")] + Dio, + #[serde(rename = "harvest")] + Harvest, } #[derive(Serialize)] @@ -231,9 +249,19 @@ pub async fn enqueue_voice_conversion_inference_handler( let mut maybe_args = None; - if let Some(auto_predict_f0) = request.auto_predict_f0 { + let set_args = request.auto_predict_f0.is_some() + || request.transpose.is_some() + || request.override_f0_method.is_some(); + + if set_args { maybe_args = Some(PolymorphicInferenceArgs::Vc { - auto_predict_f0: Some(auto_predict_f0), + auto_predict_f0: request.auto_predict_f0, + override_f0_method: request.override_f0_method.map(|f| match f { + FundamentalFrequencyMethod::Crepe => FundamentalFrequencyMethodForJob::Crepe, + FundamentalFrequencyMethod::Dio => FundamentalFrequencyMethodForJob::Dio, + FundamentalFrequencyMethod::Harvest => FundamentalFrequencyMethodForJob::Harvest, + }), + transpose: request.transpose, }); }