From e1793d71b59ec28bc221dc01d31bbdb4d2c7d608 Mon Sep 17 00:00:00 2001 From: Brandon Thomas Date: Thu, 29 Dec 2022 05:46:50 -0500 Subject: [PATCH] enqueue voice conversion request handler, media synthetic ids, ... --- .../down.sql | 5 + .../up.sql | 25 ++ .../_types/tokens/src/files/media_upload.rs | 2 +- .../tokens/src/voice_conversion/model.rs | 2 +- .../tokens/tests/media_upload_token_tests.rs | 166 ++++++++++++ .../_types/tokens/tests/user_token_tests.rs | 1 + .../src/mediators/firehose_publisher.rs | 27 ++ .../web/insert_generic_inference_job.rs | 11 +- .../queries/voice_conversion/inference/mod.rs | 0 .../src/queries/voice_conversion/mod.rs | 3 +- ...oice_conversion_model_from_download_job.rs | 0 .../mod.rs | 0 crates/lib/redis_common/src/redis_keys.rs | 7 + .../job_types/softvc/process_softvc_model.rs | 2 +- .../storyteller_web/src/configs/plans/plan.rs | 29 +- .../endpoints/media_uploads/upload_media.rs | 2 +- .../src/http_server/endpoints/mod.rs | 1 + .../enqueue_voice_conversion_inference.rs | 254 ++++++++++++++++++ .../inference/get_voice_conversion_result.rs | 0 .../list_user_voice_conversion_results.rs | 0 .../voice_conversion/inference/mod.rs | 3 + .../endpoints/voice_conversion/mod.rs | 2 + .../endpoints/voice_conversion/models/mod.rs | 0 .../service/web/storyteller_web/src/routes.rs | 86 ++++++ 24 files changed, 619 insertions(+), 9 deletions(-) create mode 100644 _migrations/2022-12-29-063005_media_upload_synthetic_ids/down.sql create mode 100644 _migrations/2022-12-29-063005_media_upload_synthetic_ids/up.sql create mode 100644 crates/_types/tokens/tests/media_upload_token_tests.rs create mode 100644 crates/lib/database_queries/src/queries/voice_conversion/inference/mod.rs rename crates/lib/database_queries/src/queries/voice_conversion/{voice_conversion_models => models}/insert_voice_conversion_model_from_download_job.rs (100%) rename crates/lib/database_queries/src/queries/voice_conversion/{voice_conversion_models => models}/mod.rs (100%) create mode 100644 crates/service/web/storyteller_web/src/http_server/endpoints/voice_conversion/inference/enqueue_voice_conversion_inference.rs create mode 100644 crates/service/web/storyteller_web/src/http_server/endpoints/voice_conversion/inference/get_voice_conversion_result.rs create mode 100644 crates/service/web/storyteller_web/src/http_server/endpoints/voice_conversion/inference/list_user_voice_conversion_results.rs create mode 100644 crates/service/web/storyteller_web/src/http_server/endpoints/voice_conversion/inference/mod.rs create mode 100644 crates/service/web/storyteller_web/src/http_server/endpoints/voice_conversion/mod.rs create mode 100644 crates/service/web/storyteller_web/src/http_server/endpoints/voice_conversion/models/mod.rs diff --git a/_migrations/2022-12-29-063005_media_upload_synthetic_ids/down.sql b/_migrations/2022-12-29-063005_media_upload_synthetic_ids/down.sql new file mode 100644 index 0000000000..f33aa6db4e --- /dev/null +++ b/_migrations/2022-12-29-063005_media_upload_synthetic_ids/down.sql @@ -0,0 +1,5 @@ +-- noinspection SqlDialectInspectionForFile +-- noinspection SqlNoDataSourceInspectionForFile +-- noinspection SqlResolveForFile + +DROP TABLE media_upload_synthetic_ids; diff --git a/_migrations/2022-12-29-063005_media_upload_synthetic_ids/up.sql b/_migrations/2022-12-29-063005_media_upload_synthetic_ids/up.sql new file mode 100644 index 0000000000..2d9916f56f --- /dev/null +++ b/_migrations/2022-12-29-063005_media_upload_synthetic_ids/up.sql @@ -0,0 +1,25 @@ +-- noinspection SqlDialectInspectionForFile +-- noinspection SqlNoDataSourceInspectionForFile +-- noinspection SqlResolveForFile + +-- It gets unwieldy to show results without a numeric sequence. +-- We create a synthetic sequence for each user to make revisiting results easier. +CREATE TABLE media_upload_synthetic_ids ( + -- Not used for anything except replication. + id BIGINT(20) NOT NULL AUTO_INCREMENT, + + -- Effective "primary key" (PUBLIC) + user_token VARCHAR(32) NOT NULL, + + -- The next result id. + -- After we use an id, we increment this counter. + next_id BIGINT(20) NOT NULL DEFAULT 0, + + created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP, + updated_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, + + -- INDICES -- + PRIMARY KEY (id), + UNIQUE KEY (user_token) + +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin; diff --git a/crates/_types/tokens/src/files/media_upload.rs b/crates/_types/tokens/src/files/media_upload.rs index df2490d560..5f8532b89e 100644 --- a/crates/_types/tokens/src/files/media_upload.rs +++ b/crates/_types/tokens/src/files/media_upload.rs @@ -6,7 +6,7 @@ use std::fmt::Debug; /// The primary key for user media uploads (images, video, etc.) #[derive(Clone, PartialEq, Eq, sqlx::Type, Debug, Serialize, Deserialize)] #[sqlx(transparent)] -pub struct MediaUploadToken(String); +pub struct MediaUploadToken(pub String); impl_string_token!(MediaUploadToken); impl_crockford_generator!(MediaUploadToken, 32usize, EntityType::MediaUpload, CrockfordLower); diff --git a/crates/_types/tokens/src/voice_conversion/model.rs b/crates/_types/tokens/src/voice_conversion/model.rs index 6e38d1824c..031f88974f 100644 --- a/crates/_types/tokens/src/voice_conversion/model.rs +++ b/crates/_types/tokens/src/voice_conversion/model.rs @@ -6,7 +6,7 @@ use std::fmt::Debug; /// The primary key for "generic" inference jobs. #[derive(Clone, PartialEq, Eq, sqlx::Type, Debug, Serialize, Deserialize)] #[sqlx(transparent)] -pub struct VoiceConversionModelToken(String); +pub struct VoiceConversionModelToken(pub String); impl_string_token!(VoiceConversionModelToken); impl_crockford_generator!(VoiceConversionModelToken, 16usize, EntityType::VoiceConversionModel, CrockfordLower); diff --git a/crates/_types/tokens/tests/media_upload_token_tests.rs b/crates/_types/tokens/tests/media_upload_token_tests.rs new file mode 100644 index 0000000000..1ad8323a0b --- /dev/null +++ b/crates/_types/tokens/tests/media_upload_token_tests.rs @@ -0,0 +1,166 @@ +/// Test MediaUploadToken in isolation, just in case our macro-derived tests break. + +use serde::Deserialize; +use serde::Serialize; +use tokens::files::media_upload::MediaUploadToken; + +#[derive(Serialize, Deserialize, PartialEq, Debug)] +struct CompositeType { + media_upload_token: MediaUploadToken, + string: String, +} + +mod interface { + use tokens::files::media_upload::MediaUploadToken; + + #[test] + fn generate() { + let token = MediaUploadToken::generate(); + assert!(!token.to_string().is_empty()); + assert!(token.to_string().starts_with("mu_")); + } + + #[test] + fn new() { + let token = MediaUploadToken::new("mu_foo".to_string()); + assert_eq!(token, MediaUploadToken("mu_foo".to_string())); + } + + #[test] + fn new_from_str() { + let token = MediaUploadToken::new_from_str("mu_foo"); + assert_eq!(token, MediaUploadToken("mu_foo".to_string())); + } + + #[test] + fn as_str() { + let token = MediaUploadToken("mu_foo".to_string()); + assert_eq!(token.as_str(), "mu_foo"); + } + + #[test] + fn to_string() { + let token = MediaUploadToken("mu_foo".to_string()); + assert_eq!(token.to_string(), "mu_foo".to_string()); + } +} + +mod traits { + use tokens::files::media_upload::MediaUploadToken; + + #[test] + fn display() { + let token = MediaUploadToken("mu_foo".to_string()); + assert_eq!(format!("{}", token), "mu_foo".to_string()); + } + + #[test] + fn debug() { + let token = MediaUploadToken("mu_foo".to_string()); + assert_eq!(format!("{:?}", token), "MediaUploadToken(\"mu_foo\")".to_string()); + } +} + +mod serialization { + use crate::CompositeType; + use tokens::files::media_upload::MediaUploadToken; + + #[test] + fn serialize() { + let expected = "\"mu_foo\"".to_string(); // NB: Quoted + + let token = MediaUploadToken("mu_foo".to_string()); + assert_eq!(expected, toml::to_string(&token).unwrap()); + + // Just to show this serializes the same as a string + assert_eq!(expected, toml::to_string("mu_foo").unwrap()); + } + + #[test] + fn nested_serialize() { + let value = CompositeType { media_upload_token: MediaUploadToken("mu_foo".to_string()), string: "bar".to_string() }; + let expected = r#"{"media_upload_token":"mu_foo","string":"bar"}"#.to_string(); + assert_eq!(expected, serde_json::to_string(&value).unwrap()); + } +} + +mod deserialization { + use crate::CompositeType; + use tokens::files::media_upload::MediaUploadToken; + + #[test] + fn deserialize() { + let payload = "\"mu_foo\""; // NB: Quoted + let expected = "mu_foo".to_string(); + + let value: MediaUploadToken = serde_json::from_str(payload).unwrap(); + assert_eq!(value, MediaUploadToken(expected.clone())); + + // Just to show this deserializes the same way as a string + let value: String = serde_json::from_str(payload).unwrap(); + assert_eq!(value, expected.clone()); + } + + #[test] + fn nested_deserialize() { + let payload = r#"{"media_upload_token":"mu_foo","string":"bar"}"#.to_string(); + let expected = CompositeType { + media_upload_token: MediaUploadToken("mu_foo".to_string()), + string: "bar".to_string(), + }; + + assert_eq!(expected, serde_json::from_str::(&payload).unwrap()); + } +} + +// These traits should be tested by the macro, but we duplicate them in case that breaks +mod crockford_traits { + use tokens::files::media_upload::MediaUploadToken; + + const ENTROPIC_CHARACTERS_MINIMUM : usize = 8; + + #[test] + fn entropy_is_sufficient() { + assert!(MediaUploadToken::entropic_character_len() > ENTROPIC_CHARACTERS_MINIMUM); + } + + #[test] + fn token_length() { + assert_eq!(MediaUploadToken::generate().as_str().len(), 32); + } + + #[test] + fn tokens_are_random() { + let mut tokens = std::collections::HashSet::new(); + for _ in 0..100 { + tokens.insert(MediaUploadToken::generate().to_string()); + } + assert_eq!(tokens.len(), 100); + } + + #[test] + fn character_set() { + let token_string = MediaUploadToken::generate().to_string(); + let prefix = MediaUploadToken::token_prefix(); + let random_part = token_string.replace(prefix, ""); + + assert!(random_part.len() > ENTROPIC_CHARACTERS_MINIMUM); + assert!(random_part.chars().all(|c| c.is_numeric() || c.is_lowercase())); + } + + #[test] + fn prefix_ends_with_separator() { + let prefix = MediaUploadToken::token_prefix(); + assert!(prefix.ends_with(":") || prefix.ends_with("_")); + + let token_string = MediaUploadToken::generate().to_string(); + assert!(token_string.contains(":") || token_string.contains("_")); + } + + #[test] + fn token_begins_with_prefix() { + let prefix = MediaUploadToken::token_prefix(); + let token_string = MediaUploadToken::generate().to_string(); + assert!(token_string.starts_with(prefix)); + } +} diff --git a/crates/_types/tokens/tests/user_token_tests.rs b/crates/_types/tokens/tests/user_token_tests.rs index 47a01de03d..c14b3edc6e 100644 --- a/crates/_types/tokens/tests/user_token_tests.rs +++ b/crates/_types/tokens/tests/user_token_tests.rs @@ -17,6 +17,7 @@ mod interface { fn generate() { let token = UserToken::generate(); assert!(!token.to_string().is_empty()); + assert!(token.to_string().starts_with("U:")); } #[test] diff --git a/crates/lib/database_queries/src/mediators/firehose_publisher.rs b/crates/lib/database_queries/src/mediators/firehose_publisher.rs index 5cdff00101..74825ce83c 100644 --- a/crates/lib/database_queries/src/mediators/firehose_publisher.rs +++ b/crates/lib/database_queries/src/mediators/firehose_publisher.rs @@ -7,6 +7,7 @@ use sqlx::mysql::MySqlQueryResult; use sqlx::{MySqlPool}; use std::sync::Arc; use tokens::files::media_upload::MediaUploadToken; +use tokens::jobs::inference::InferenceJobToken; use tokens::users::user::UserToken; // TODO(bt, 2022-12-19): Convert this to a database 'enum'. Also, create an 'enums' package similar to 'tokens'. @@ -27,6 +28,9 @@ enum FirehoseEvent { W2lInferenceStarted, W2lInferenceCompleted, + VcInferenceStarted, + VcInferenceCompleted, + GenericDownloadStarted, GenericDownloadCompleted, @@ -54,6 +58,8 @@ impl FirehoseEvent { FirehoseEvent::W2lTemplateUploadCompleted => "w2l_template_upload_completed", FirehoseEvent::W2lInferenceStarted => "w2l_inference_started", FirehoseEvent::W2lInferenceCompleted => "w2l_inference_completed", + FirehoseEvent::VcInferenceStarted => "vc_inference_started", + FirehoseEvent::VcInferenceCompleted => "vc_inference_completed", FirehoseEvent::GenericDownloadStarted => "generic_download_started", FirehoseEvent::GenericDownloadCompleted => "generic_download_completed", FirehoseEvent::MediaUploaded => "media_uploaded", @@ -186,6 +192,27 @@ impl FirehosePublisher { Ok(()) } + pub async fn enqueue_vc_inference(&self, maybe_user_token: Option<&UserToken>, inference_job_token: &InferenceJobToken) -> AnyhowResult<()> { + let _record_id = self.insert( + FirehoseEvent::VcInferenceStarted, + maybe_user_token.map(|u| u.as_str()), + Some(inference_job_token.as_str()), + Some(inference_job_token.as_str()), + ).await?; + Ok(()) + } + + // TODO: Change result token type. + pub async fn vc_inference_finished(&self, maybe_user_token: Option<&UserToken>, inference_job_token: &InferenceJobToken, result_token: &str) -> AnyhowResult<()> { + let _record_id = self.insert( + FirehoseEvent::VcInferenceCompleted, + maybe_user_token.map(|u| u.as_str()), + Some(inference_job_token.as_str()), // TODO: This could be vc model token + Some(result_token) + ).await?; + Ok(()) + } + pub async fn enqueue_generic_download(&self, user_token: &str, job_token: &str) -> AnyhowResult<()> { let _record_id = self.insert( FirehoseEvent::GenericDownloadStarted, diff --git a/crates/lib/database_queries/src/queries/generic_inference/web/insert_generic_inference_job.rs b/crates/lib/database_queries/src/queries/generic_inference/web/insert_generic_inference_job.rs index eadb6e79fa..92503c332c 100644 --- a/crates/lib/database_queries/src/queries/generic_inference/web/insert_generic_inference_job.rs +++ b/crates/lib/database_queries/src/queries/generic_inference/web/insert_generic_inference_job.rs @@ -4,6 +4,7 @@ use enums::core::visibility::Visibility; use reusable_types::db::enums::generic_inference_type::GenericInferenceType; use sqlx::MySqlPool; use tokens::jobs::inference::InferenceJobToken; +use tokens::users::user::UserToken; pub struct Args <'a> { pub job_token: &'a InferenceJobToken, @@ -14,10 +15,13 @@ pub struct Args <'a> { pub maybe_raw_inference_text: Option, pub maybe_model_token: Option, - pub maybe_creator_user_token: Option<&'a str>, + pub maybe_creator_user_token: Option<&'a UserToken>, pub creator_ip_address: &'a str, pub creator_set_visibility: Visibility, + pub priority_level: u8, + pub is_debug_request: bool, + pub mysql_pool: &'a MySqlPool, } @@ -39,6 +43,9 @@ SET creator_ip_address = ?, creator_set_visibility = ?, + priority_level = ?, + is_debug_request = ?, + status = "pending" "#, args.job_token, @@ -52,6 +59,8 @@ SET args.maybe_creator_user_token, args.creator_ip_address, args.creator_set_visibility.to_str(), + args.priority_level, + args.is_debug_request, ); let query_result = query.execute(args.mysql_pool) diff --git a/crates/lib/database_queries/src/queries/voice_conversion/inference/mod.rs b/crates/lib/database_queries/src/queries/voice_conversion/inference/mod.rs new file mode 100644 index 0000000000..e69de29bb2 diff --git a/crates/lib/database_queries/src/queries/voice_conversion/mod.rs b/crates/lib/database_queries/src/queries/voice_conversion/mod.rs index 3c7d77b643..1cd77f670e 100644 --- a/crates/lib/database_queries/src/queries/voice_conversion/mod.rs +++ b/crates/lib/database_queries/src/queries/voice_conversion/mod.rs @@ -1 +1,2 @@ -pub mod voice_conversion_models; \ No newline at end of file +pub mod inference; +pub mod models; \ No newline at end of file diff --git a/crates/lib/database_queries/src/queries/voice_conversion/voice_conversion_models/insert_voice_conversion_model_from_download_job.rs b/crates/lib/database_queries/src/queries/voice_conversion/models/insert_voice_conversion_model_from_download_job.rs similarity index 100% rename from crates/lib/database_queries/src/queries/voice_conversion/voice_conversion_models/insert_voice_conversion_model_from_download_job.rs rename to crates/lib/database_queries/src/queries/voice_conversion/models/insert_voice_conversion_model_from_download_job.rs diff --git a/crates/lib/database_queries/src/queries/voice_conversion/voice_conversion_models/mod.rs b/crates/lib/database_queries/src/queries/voice_conversion/models/mod.rs similarity index 100% rename from crates/lib/database_queries/src/queries/voice_conversion/voice_conversion_models/mod.rs rename to crates/lib/database_queries/src/queries/voice_conversion/models/mod.rs diff --git a/crates/lib/redis_common/src/redis_keys.rs b/crates/lib/redis_common/src/redis_keys.rs index c151d73c91..b0afb2e4ad 100644 --- a/crates/lib/redis_common/src/redis_keys.rs +++ b/crates/lib/redis_common/src/redis_keys.rs @@ -12,6 +12,13 @@ impl RedisKeys { format!("ttsUseCount:{}", model_token) } + /// This is a counter incremented with web VC inference. + /// We can use a job to read MySql and fix counts if they get out of sync. + /// These should be long-lived keys. + pub fn web_vc_model_usage_count(model_token: &str) -> String { + format!("vcUseCount:{}", model_token) + } + /// This is a counter incremented with TTS inference. /// We can use a job to read MySql and fix counts if they get out of sync. /// These should be long-lived keys. diff --git a/crates/service/job/download_job/src/job_types/softvc/process_softvc_model.rs b/crates/service/job/download_job/src/job_types/softvc/process_softvc_model.rs index bd133e5ee1..c7edd08b46 100644 --- a/crates/service/job/download_job/src/job_types/softvc/process_softvc_model.rs +++ b/crates/service/job/download_job/src/job_types/softvc/process_softvc_model.rs @@ -7,7 +7,7 @@ use container_common::hashing::hash_file_sha2::hash_file_sha2; use crate::JobState; use crate::job_steps::job_results::JobResults; use database_queries::queries::generic_download::job::list_available_generic_download_jobs::AvailableDownloadJob; -use database_queries::queries::voice_conversion::voice_conversion_models::insert_voice_conversion_model_from_download_job::{Args, insert_voice_conversion_model_from_download_job}; +use database_queries::queries::voice_conversion::models::insert_voice_conversion_model_from_download_job::{Args, insert_voice_conversion_model_from_download_job}; use enums::core::visibility::Visibility; use jobs_common::redis_job_status_logger::RedisJobStatusLogger; use log::{info, warn}; diff --git a/crates/service/web/storyteller_web/src/configs/plans/plan.rs b/crates/service/web/storyteller_web/src/configs/plans/plan.rs index 2eb80e3d78..1588733a5e 100644 --- a/crates/service/web/storyteller_web/src/configs/plans/plan.rs +++ b/crates/service/web/storyteller_web/src/configs/plans/plan.rs @@ -4,6 +4,8 @@ use crate::configs::plans::plan_category::PlanCategory; const TTS_DEFAULT_PRIORITY_LEVEL : u8 = 0; const TTS_DEFAULT_DURATION_SECONDS : i64 = 12; +const VC_DEFAULT_PRIORITY_LEVEL : u8 = 0; + const VC_DEFAULT_MAX_CONCURRENT_MODELS : u32 = 5; const VC_DEFAULT_PRE_RECORDED_TIME_LIMIT_SECONDS : i64 = 20; @@ -48,7 +50,11 @@ pub struct Plan { tts_can_upload_private_models: bool, tts_can_share_private_models: bool, - // ========== Features for Voice Conversion ========== + // ========== Features for Web Voice Conversion ========== + + web_vc_base_priority_level: u8, + + // ========== Features for Real Time Voice Conversion ========== vc_max_concurrent_models: u32, @@ -79,6 +85,7 @@ impl Plan { tts_can_generate_mp3: builder.tts_can_generate_mp3, tts_can_upload_private_models: builder.tts_can_upload_private_models, tts_can_share_private_models: builder.tts_can_share_private_models, + web_vc_base_priority_level: builder.web_vc_base_priority_level, vc_max_concurrent_models: builder.vc_max_concurrent_models, vc_pre_recorded_time_limit: builder.vc_pre_recorded_time_limit, vc_pre_recorded_time_is_unlimited: builder.vc_pre_recorded_time_is_unlimited, @@ -132,6 +139,10 @@ impl Plan { pub fn tts_max_duration_seconds(&self) -> i32 { self.tts_max_duration.num_seconds() as i32 } + + pub fn web_vc_base_priority_level(&self) -> u8 { + self.web_vc_base_priority_level + } } /// Builder pattern for Plans. @@ -153,7 +164,11 @@ pub struct PlanBuilder { tts_can_upload_private_models: bool, tts_can_share_private_models: bool, - // ========== Features for Voice Conversion ========== + // ========== Features for Web Voice Conversion ========== + + web_vc_base_priority_level: u8, + + // ========== Features for Real Time Voice Conversion ========== vc_max_concurrent_models: u32, @@ -188,7 +203,10 @@ impl PlanBuilder { tts_can_upload_private_models: false, tts_can_share_private_models : false, - // VC + // VC (WEB) + web_vc_base_priority_level: VC_DEFAULT_PRIORITY_LEVEL, + + // VC (REALTIME) vc_max_concurrent_models: VC_DEFAULT_MAX_CONCURRENT_MODELS, vc_pre_recorded_time_limit: Duration::seconds(VC_DEFAULT_PRE_RECORDED_TIME_LIMIT_SECONDS), vc_pre_recorded_time_is_unlimited: false, @@ -260,6 +278,11 @@ impl PlanBuilder { self } + pub fn web_vc_base_priority_level(mut self, value: u8) -> Self { + self.web_vc_base_priority_level = value; + self + } + pub fn vc_max_concurrent_models(mut self, value: u32) -> Self { self.vc_max_concurrent_models = value; self diff --git a/crates/service/web/storyteller_web/src/http_server/endpoints/media_uploads/upload_media.rs b/crates/service/web/storyteller_web/src/http_server/endpoints/media_uploads/upload_media.rs index 81c4943cec..277582f32e 100644 --- a/crates/service/web/storyteller_web/src/http_server/endpoints/media_uploads/upload_media.rs +++ b/crates/service/web/storyteller_web/src/http_server/endpoints/media_uploads/upload_media.rs @@ -260,7 +260,7 @@ pub async fn upload_media_handler( maybe_creator_anonymous_visitor_token: None, creator_ip_address: &ip_address, creator_set_visibility, - maybe_creator_synthetic_id: None, + maybe_creator_synthetic_id: None, // TODO: Don't forget about this. mysql_pool: &server_state.mysql_pool, }) .await diff --git a/crates/service/web/storyteller_web/src/http_server/endpoints/mod.rs b/crates/service/web/storyteller_web/src/http_server/endpoints/mod.rs index aaa72c4f1c..ed3ae9a1b4 100644 --- a/crates/service/web/storyteller_web/src/http_server/endpoints/mod.rs +++ b/crates/service/web/storyteller_web/src/http_server/endpoints/mod.rs @@ -14,4 +14,5 @@ pub mod tts; pub mod twitch; pub mod vocoders; pub mod voice_clone_requests; +pub mod voice_conversion; pub mod w2l; \ No newline at end of file diff --git a/crates/service/web/storyteller_web/src/http_server/endpoints/voice_conversion/inference/enqueue_voice_conversion_inference.rs b/crates/service/web/storyteller_web/src/http_server/endpoints/voice_conversion/inference/enqueue_voice_conversion_inference.rs new file mode 100644 index 0000000000..b520135d76 --- /dev/null +++ b/crates/service/web/storyteller_web/src/http_server/endpoints/voice_conversion/inference/enqueue_voice_conversion_inference.rs @@ -0,0 +1,254 @@ +// NB: Incrementally getting rid of build warnings... +#![forbid(unused_imports)] +#![forbid(unused_mut)] +#![forbid(unused_variables)] + +use actix_web::error::ResponseError; +use actix_web::http::StatusCode; +use actix_web::{web, HttpResponse, HttpRequest}; +use crate::configs::plans::get_correct_plan_for_session::get_correct_plan_for_session; +use crate::http_server::endpoints::investor_demo::demo_cookie::request_has_demo_cookie; +use crate::http_server::web_utils::response_error_helpers::to_simple_json_error; +use crate::server_state::ServerState; +use database_queries::queries::generic_inference::web::insert_generic_inference_job::{Args, insert_generic_inference_job}; +use enums::core::visibility::Visibility; +use http_server_common::request::get_request_header_optional::get_request_header_optional; +use http_server_common::request::get_request_ip::get_request_ip; +use log::{info, warn}; +use r2d2_redis::redis::Commands; +use redis_common::redis_keys::RedisKeys; +use reusable_types::db::enums::generic_inference_type::GenericInferenceType; +use std::fmt; +use std::sync::Arc; +use tokens::jobs::inference::InferenceJobToken; +use tokens::users::user::UserToken; +use tts_common::priority::FAKEYOU_INVESTOR_PRIORITY_LEVEL; + +/// Debug requests can get routed to special "debug-only" workers, which can +/// be used to trial new code, run debugging, etc. +const DEBUG_HEADER_NAME : &'static str = "enable_debug_mode"; + +#[derive(Deserialize)] +pub struct EnqueueVoiceConversionInferenceRequest { + uuid_idempotency_token: String, + voice_conversion_model_token: String, + media_token: String, + creator_set_visibility: Option, + is_storyteller_demo: Option, +} + +#[derive(Serialize)] +pub struct EnqueueVoiceConversionInferenceSuccessResponse { + pub success: bool, + pub inference_job_token: InferenceJobToken, +} + +#[derive(Debug)] +pub enum EnqueueVoiceConversionInferenceError { + BadInput(String), + NotAuthorized, + ServerError, + RateLimited, +} + +impl ResponseError for EnqueueVoiceConversionInferenceError { + fn status_code(&self) -> StatusCode { + match *self { + EnqueueVoiceConversionInferenceError::BadInput(_) => StatusCode::BAD_REQUEST, + EnqueueVoiceConversionInferenceError::NotAuthorized => StatusCode::UNAUTHORIZED, + EnqueueVoiceConversionInferenceError::ServerError => StatusCode::INTERNAL_SERVER_ERROR, + EnqueueVoiceConversionInferenceError::RateLimited => StatusCode::TOO_MANY_REQUESTS, + } + } + + fn error_response(&self) -> HttpResponse { + let error_reason = match self { + EnqueueVoiceConversionInferenceError::BadInput(reason) => reason.to_string(), + EnqueueVoiceConversionInferenceError::NotAuthorized => "unauthorized".to_string(), + EnqueueVoiceConversionInferenceError::ServerError => "server error".to_string(), + EnqueueVoiceConversionInferenceError::RateLimited => "rate limited".to_string(), + }; + + to_simple_json_error(&error_reason, self.status_code()) + } +} + +// NB: Not using derive_more::Display since Clion doesn't understand it. +impl fmt::Display for EnqueueVoiceConversionInferenceError { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "{:?}", self) + } +} + +pub async fn enqueue_voice_conversion_inference_handler( + http_request: HttpRequest, + request: web::Json, + server_state: web::Data>) -> Result +{ + let mut maybe_user_token : Option = None; + let mut priority_level ; + let disable_rate_limiter = false; // NB: Careful! + + let mut mysql_connection = server_state.mysql_pool + .acquire() + .await + .map_err(|err| { + warn!("MySql pool error: {:?}", err); + EnqueueVoiceConversionInferenceError::ServerError + })?; + + // ==================== USER SESSION ==================== // + + let maybe_user_session = server_state + .session_checker + .maybe_get_user_session_extended_from_connection(&http_request, &mut mysql_connection) + .await + .map_err(|e| { + warn!("Session checker error: {:?}", e); + EnqueueVoiceConversionInferenceError::ServerError + })?; + + if let Some(user_session) = maybe_user_session.as_ref() { + maybe_user_token = Some(UserToken::new_from_str(&user_session.user_token)); + } + + // TODO: Plan should handle "first anonymous use" and "investor" cases. + let plan = get_correct_plan_for_session( + server_state.server_environment, + maybe_user_session.as_ref()); + + priority_level = plan.web_vc_base_priority_level(); + + // ==================== INVESTOR PRIORITY ==================== // + + // TODO/TEMP: Give investors even more priority + let mut is_investor = false; + + { + // TODO/TEMP: The storyteller.io website's AJAX calls will set this. + // This is just for the YCombinator demo. + match request.is_storyteller_demo { + Some(true) => { + is_investor = true; + }, + _ => {}, + }; + + // TODO/TEMP: The storyteller.io website will redirect and establish this cookie. + // This is just for the YCombinator demo. + if request_has_demo_cookie(&http_request) { + is_investor = true; + } + + if is_investor { + priority_level = FAKEYOU_INVESTOR_PRIORITY_LEVEL; + } + } + + // ==================== DEBUG MODE ==================== // + + let is_debug_request = get_request_header_optional(&http_request, DEBUG_HEADER_NAME) + .is_some(); + + // ==================== RATE LIMIT ==================== // + + if !disable_rate_limiter { + let mut rate_limiter = match maybe_user_session { + None => &server_state.redis_rate_limiters.logged_out, + Some(ref user) => { + if user.role.is_banned { + return Err(EnqueueVoiceConversionInferenceError::NotAuthorized); + } + &server_state.redis_rate_limiters.logged_in + }, + }; + + // TODO/TEMP + if is_investor { + rate_limiter = &server_state.redis_rate_limiters.logged_in; + } + + if let Err(_err) = rate_limiter.rate_limit_request(&http_request) { + return Err(EnqueueVoiceConversionInferenceError::RateLimited); + } + } + + // ==================== CHECK AND ENQUEUE VOICE CONVERSION ==================== // + + // TODO(bt): CHECK DATABASE! + let model_token = request.voice_conversion_model_token.to_string(); + let _media_token = request.media_token.to_string(); + + let mut redis = server_state.redis_pool + .get() + .map_err(|e| { + warn!("redis error: {:?}", e); + EnqueueVoiceConversionInferenceError::ServerError + })?; + + let redis_count_key = RedisKeys::web_vc_model_usage_count(&model_token); + + redis.incr(&redis_count_key, 1) + .map_err(|e| { + warn!("redis error: {:?}", e); + EnqueueVoiceConversionInferenceError::ServerError + })?; + + let ip_address = get_request_ip(&http_request); + + let maybe_user_preferred_visibility : Option = maybe_user_session + .as_ref() + .map(|user_session| user_session.preferences.preferred_tts_result_visibility); // TODO: New setting for web-vc + + let set_visibility = request.creator_set_visibility + .or(maybe_user_preferred_visibility) + .unwrap_or(Visibility::Public); + + let job_token = InferenceJobToken::generate(); + + info!("Creating tts inference job record..."); + + let query_result = insert_generic_inference_job(Args { + job_token: &job_token, + uuid_idempotency_token: &request.uuid_idempotency_token, + inference_type: GenericInferenceType::VoiceConversion, // TODO: Move from `reusable_types` + maybe_inference_args: None, // TODO: Include both tokens as well as allowed duration. + maybe_raw_inference_text: None, + maybe_model_token: Some(model_token), + maybe_creator_user_token: maybe_user_token.as_ref(), + creator_ip_address: &ip_address, + creator_set_visibility: set_visibility, + priority_level, + is_debug_request, + mysql_pool: &server_state.mysql_pool, + }).await; + + match query_result { + Ok(_) => {}, + Err(err) => { + warn!("New generic inference job creation DB error: {:?}", err); + return Err(EnqueueVoiceConversionInferenceError::ServerError); + } + } + + server_state.firehose_publisher.enqueue_vc_inference( + maybe_user_token.as_ref(), + &job_token) + .await + .map_err(|e| { + warn!("error publishing event: {:?}", e); + EnqueueVoiceConversionInferenceError::ServerError + })?; + + let response = EnqueueVoiceConversionInferenceSuccessResponse { + success: true, + inference_job_token: job_token, + }; + + let body = serde_json::to_string(&response) + .map_err(|_e| EnqueueVoiceConversionInferenceError::ServerError)?; + + Ok(HttpResponse::Ok() + .content_type("application/json") + .body(body)) +} diff --git a/crates/service/web/storyteller_web/src/http_server/endpoints/voice_conversion/inference/get_voice_conversion_result.rs b/crates/service/web/storyteller_web/src/http_server/endpoints/voice_conversion/inference/get_voice_conversion_result.rs new file mode 100644 index 0000000000..e69de29bb2 diff --git a/crates/service/web/storyteller_web/src/http_server/endpoints/voice_conversion/inference/list_user_voice_conversion_results.rs b/crates/service/web/storyteller_web/src/http_server/endpoints/voice_conversion/inference/list_user_voice_conversion_results.rs new file mode 100644 index 0000000000..e69de29bb2 diff --git a/crates/service/web/storyteller_web/src/http_server/endpoints/voice_conversion/inference/mod.rs b/crates/service/web/storyteller_web/src/http_server/endpoints/voice_conversion/inference/mod.rs new file mode 100644 index 0000000000..9e4478016b --- /dev/null +++ b/crates/service/web/storyteller_web/src/http_server/endpoints/voice_conversion/inference/mod.rs @@ -0,0 +1,3 @@ +pub mod enqueue_voice_conversion_inference; +pub mod get_voice_conversion_result; +pub mod list_user_voice_conversion_results; \ No newline at end of file diff --git a/crates/service/web/storyteller_web/src/http_server/endpoints/voice_conversion/mod.rs b/crates/service/web/storyteller_web/src/http_server/endpoints/voice_conversion/mod.rs new file mode 100644 index 0000000000..1cd77f670e --- /dev/null +++ b/crates/service/web/storyteller_web/src/http_server/endpoints/voice_conversion/mod.rs @@ -0,0 +1,2 @@ +pub mod inference; +pub mod models; \ No newline at end of file diff --git a/crates/service/web/storyteller_web/src/http_server/endpoints/voice_conversion/models/mod.rs b/crates/service/web/storyteller_web/src/http_server/endpoints/voice_conversion/models/mod.rs new file mode 100644 index 0000000000..e69de29bb2 diff --git a/crates/service/web/storyteller_web/src/routes.rs b/crates/service/web/storyteller_web/src/routes.rs index 0a129dc2df..3d8f1c14ad 100644 --- a/crates/service/web/storyteller_web/src/routes.rs +++ b/crates/service/web/storyteller_web/src/routes.rs @@ -99,6 +99,7 @@ use crate::http_server::endpoints::w2l::set_w2l_template_mod_approval::set_w2l_t use users_component::default_routes::add_suggested_api_v1_account_creation_and_session_routes; use users_component::endpoints::edit_profile_handler::edit_profile_handler; use users_component::endpoints::get_profile_handler::get_profile_handler; +use crate::http_server::endpoints::voice_conversion::inference::enqueue_voice_conversion_inference::enqueue_voice_conversion_inference_handler; pub fn add_routes (app: App) -> App where @@ -114,6 +115,7 @@ pub fn add_routes (app: App) -> App let mut app = add_moderator_routes(app); /* /moderation */ app = add_tts_routes(app); /* /tts */ app = add_w2l_routes(app); /* /w2l */ + app = add_web_vc_routes(app); /* /v1/voice_conversion */ app = add_vocoder_routes(app); /* /vocoder */ app = add_retrieval_routes(app); /* /retrieval (aka. "generic_download_jobs") */ app = add_category_routes(app); /* /category */ @@ -469,6 +471,90 @@ fn add_w2l_routes (app: App) -> App ) } +// ==================== WEB VOICE CONVERSION ROUTES ==================== + +fn add_web_vc_routes (app: App) -> App + where + B: MessageBody, + T: ServiceFactory< + ServiceRequest, + Config = (), + Response = ServiceResponse, + Error = Error, + InitError = (), + >, +{ + app.service( + web::scope("/v1/voice_conversion") + .service( + web::resource("/inference") + .route(web::post().to(enqueue_voice_conversion_inference_handler)) + .route(web::head().to(|| HttpResponse::Ok())) + ) + //.service( + // web::resource("/upload") + // .route(web::post().to(upload_w2l_template_handler)) + // .route(web::head().to(|| HttpResponse::Ok())) + //) + //.service( + // web::resource("/list") + // .route(web::get().to(list_w2l_templates_handler)) + // .route(web::head().to(|| HttpResponse::Ok())) + //) + //.service( + // web::resource("/template/{token}") + // .route(web::get().to(get_w2l_template_handler)) + // .route(web::head().to(|| HttpResponse::Ok())) + //) + //.service( + // web::resource("/template/{template_token}/count") + // .route(web::get().to(get_w2l_template_use_count_handler)) + // .route(web::head().to(|| HttpResponse::Ok())) + //) + //.service( + // web::resource("/template/{template_token}/edit") + // .route(web::post().to(edit_w2l_template_handler)) + // .route(web::head().to(|| HttpResponse::Ok())) + //) + //.service( + // web::resource("/template/{token}/moderate") + // .route(web::post().to(set_w2l_template_mod_approval_handler)) + // .route(web::head().to(|| HttpResponse::Ok())) + //) + //.service( + // web::resource("/template/{token}/delete") + // .route(web::post().to(delete_w2l_template_handler)) + // .route(web::head().to(|| HttpResponse::Ok())) + //) + //.service( + // web::resource("/result/{token}") + // .route(web::get().to(get_w2l_inference_result_handler)) + // .route(web::head().to(|| HttpResponse::Ok())) + //) + //.service( + // web::resource("/result/{token}/edit") + // .route(web::post().to(edit_w2l_inference_result_handler)) + // .route(web::head().to(|| HttpResponse::Ok())) + //) + //.service( + // web::resource("/result/{token}/delete") + // .route(web::post().to(delete_w2l_inference_result_handler)) + // .route(web::head().to(|| HttpResponse::Ok())) + //) + //.service( + // web::resource("/job/{token}") + // .route(web::get().to(get_w2l_inference_job_status_handler)) + // .route(web::head().to(|| HttpResponse::Ok())) + //) + //.service( + // web::resource("/upload_template_job/{token}") + // .route(web::get().to(get_w2l_upload_template_job_status_handler)) + // .route(web::head().to(|| HttpResponse::Ok())) + //) + ) +} + + // ==================== VOCODER ROUTES ==================== fn add_vocoder_routes (app: App) -> App