From 1845eea517772f05e372df6033bbe9a9c440f588 Mon Sep 17 00:00:00 2001 From: Markus Ast Date: Sun, 10 Nov 2024 21:07:10 +0100 Subject: [PATCH] Add Parler based TTS (POC) --- Cargo.lock | 1113 ++++++++++++++++++++++++++++++++- README.md | 18 +- lua/DCS-gRPC/grpc-mission.lua | 64 +- lua/Hooks/DCS-gRPC.lua | 2 +- protos/dcs/srs/v0/srs.proto | 6 + src/authentication.rs | 3 +- src/config.rs | 8 + src/rpc/metadata.rs | 3 +- src/rpc/srs.rs | 27 +- src/server.rs | 19 +- tts/Cargo.toml | 12 +- tts/src/bs1770.rs | 508 +++++++++++++++ tts/src/lib.rs | 5 + tts/src/parler.rs | 220 +++++++ 14 files changed, 1942 insertions(+), 66 deletions(-) create mode 100644 tts/src/bs1770.rs create mode 100644 tts/src/parler.rs diff --git a/Cargo.lock b/Cargo.lock index a8e450da..fd5d88f1 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -96,6 +96,15 @@ version = "1.0.86" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b3d1d046238990b9cf5bcde22a3fb3584ee5cf65fb2765f454ed428c7a0063da" +[[package]] +name = "arbitrary" +version = "1.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dde20b3d026af13f561bdd0f15edf01fc734f0dafcedbaf42bba506a9517f223" +dependencies = [ + "derive_arbitrary", +] + [[package]] name = "arc-swap" version = "1.7.1" @@ -135,6 +144,12 @@ dependencies = [ "syn 2.0.66", ] +[[package]] +name = "atomic-waker" +version = "1.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1505bd5d3d116872e7271a6d4e16d81d0c8570876c8de68093a09ac269d8aac0" + [[package]] name = "audiopus" version = "0.2.0" @@ -252,6 +267,32 @@ version = "0.22.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6" +[[package]] +name = "bindgen_cuda" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1f8489af5b7d17a81bffe37e0f4d6e1e4de87c87329d05447f22c35d95a1227d" +dependencies = [ + "glob", + "num_cpus", + "rayon", +] + +[[package]] +name = "bit-set" +version = "0.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0700ddab506f33b20a03b13996eccd309a48e5ff77d0d95926aa0210fb4e95f1" +dependencies = [ + "bit-vec", +] + +[[package]] +name = "bit-vec" +version = "0.6.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "349f9b6a179ed607305526ca489b34ad0a41aed5f7980fa90eb03160b69598fb" + [[package]] name = "bitflags" version = "1.3.2" @@ -289,6 +330,26 @@ version = "3.16.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "79296716171880943b8470b5f8d03aa55eb2e645a4874bdbb28adb49162e012c" +[[package]] +name = "bytemuck" +version = "1.19.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8334215b81e418a0a7bdb8ef0849474f40bb10c8b71f1c4ed315cff49f32494d" +dependencies = [ + "bytemuck_derive", +] + +[[package]] +name = "bytemuck_derive" +version = "1.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bcfcc3cd946cb52f0bbfdbbcfa2f4e24f75ebb6c0e1002f7c25904fada18b9ec" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.66", +] + [[package]] name = "byteorder" version = "1.5.0" @@ -301,6 +362,91 @@ version = "1.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "514de17de45fdb8dc022b1a7975556c53c86f9f0aa5f534b98977b171857c2c9" +[[package]] +name = "candle-core" +version = "0.7.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7e1a39b963e261c58017edf2007e5b63425ad21538aaaf51fe23d1da41703701" +dependencies = [ + "byteorder", + "candle-kernels", + "cudarc", + "gemm", + "half", + "memmap2", + "num-traits", + "num_cpus", + "rand", + "rand_distr", + "rayon", + "safetensors", + "thiserror", + "yoke", + "zip", +] + +[[package]] +name = "candle-hf-hub" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ca5f45ce8fe55a9e9246a3fc60000d7ed11b88a84d72f753488f7264ce04b102" +dependencies = [ + "dirs", + "futures", + "indicatif", + "log", + "num_cpus", + "rand", + "reqwest", + "serde", + "serde_json", + "thiserror", + "tokio", +] + +[[package]] +name = "candle-kernels" +version = "0.7.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "539cbfbf2d1d68a6ed97115e579c77c98f8ed0cfe7edbc6d7d30d2ac0c9e3d50" +dependencies = [ + "bindgen_cuda", +] + +[[package]] +name = "candle-nn" +version = "0.7.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "898f8d21b8bdf559a1c8635e2db8386b2134015cd3003c18c1a30a22a67daec6" +dependencies = [ + "candle-core", + "half", + "num-traits", + "rayon", + "safetensors", + "serde", + "thiserror", +] + +[[package]] +name = "candle-transformers" +version = "0.7.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "06b8a130a8ac1d1e20696d89f7a52948902e037ad0eec085fceb77021007cfee" +dependencies = [ + "byteorder", + "candle-core", + "candle-nn", + "fancy-regex", + "num-traits", + "rand", + "rayon", + "serde", + "serde_json", + "serde_plain", + "tracing", +] + [[package]] name = "cc" version = "1.0.98" @@ -372,6 +518,19 @@ version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0b6a852b24ab71dffc585bcb46eaf7959d175cb865a7152e35b348d1b2960422" +[[package]] +name = "console" +version = "0.15.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0e1f83fc076bd6dd27517eacdf25fef6c4dfe5f1d7448bafaaf3a26f13b5e4eb" +dependencies = [ + "encode_unicode", + "lazy_static", + "libc", + "unicode-width", + "windows-sys 0.52.0", +] + [[package]] name = "core-foundation" version = "0.9.4" @@ -406,6 +565,37 @@ dependencies = [ "cfg-if", ] +[[package]] +name = "crossbeam-deque" +version = "0.8.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "613f8cc01fe9cf1a3eb3d7f488fd2fa8388403e97039e2f73692932e291a770d" +dependencies = [ + "crossbeam-epoch", + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-epoch" +version = "0.9.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5b82ac4a3c2ca9c3460964f020e1402edd5753411d7737aa39c3714ad1b5420e" +dependencies = [ + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-utils" +version = "0.8.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "22ec99545bb0ed0ea7bb9b8e1e9122ea386ff8a48c0922e43f36d45ab09e0e80" + +[[package]] +name = "crunchy" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7a81dae078cea95a014a339291cec439d2f232ebe854a9d672b796c6afafa9b7" + [[package]] name = "crypto-mac" version = "0.11.0" @@ -416,6 +606,51 @@ dependencies = [ "subtle", ] +[[package]] +name = "cudarc" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "38cd60a9a42ec83a2ed7effb0b1f073270264ea99da7acfc44f7e8d74dee0384" +dependencies = [ + "half", + "libloading", +] + +[[package]] +name = "darling" +version = "0.20.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6f63b86c8a8826a49b8c21f08a2d07338eec8d900540f8630dc76284be802989" +dependencies = [ + "darling_core", + "darling_macro", +] + +[[package]] +name = "darling_core" +version = "0.20.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "95133861a8032aaea082871032f5815eb9e98cef03fa916ab4500513994df9e5" +dependencies = [ + "fnv", + "ident_case", + "proc-macro2", + "quote", + "strsim", + "syn 2.0.66", +] + +[[package]] +name = "darling_macro" +version = "0.20.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d336a2a514f6ccccaa3e09b02d41d35330c07ddf03a62165fcec10bb561c7806" +dependencies = [ + "darling_core", + "quote", + "syn 2.0.66", +] + [[package]] name = "dcs-grpc" version = "0.8.1" @@ -496,15 +731,21 @@ dependencies = [ "audiopus", "base64 0.22.1", "bytes", + "candle-core", + "candle-hf-hub", + "candle-nn", + "candle-transformers", "log", "ogg", "reqwest", + "rubato", "rusoto_core", "rusoto_credential", "rusoto_polly", "serde", "serde_json", "thiserror", + "tokenizers", "tokio", "windows", ] @@ -543,6 +784,48 @@ dependencies = [ "syn 1.0.109", ] +[[package]] +name = "derive_arbitrary" +version = "1.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "30542c1ad912e0e3d22a1935c290e12e8a29d704a420177a31faad4a601a0800" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.66", +] + +[[package]] +name = "derive_builder" +version = "0.20.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "507dfb09ea8b7fa618fcf76e953f4f5e192547945816d5358edffe39f6f94947" +dependencies = [ + "derive_builder_macro", +] + +[[package]] +name = "derive_builder_core" +version = "0.20.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2d5bcf7b024d6835cfb3d473887cd966994907effbe9227e8c8219824d06c4e8" +dependencies = [ + "darling", + "proc-macro2", + "quote", + "syn 2.0.66", +] + +[[package]] +name = "derive_builder_macro" +version = "0.20.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ab63b0e2bf4d5928aff72e83a7dace85d7bba5fe12dcc3c5a572d78caffd3f3c" +dependencies = [ + "derive_builder_core", + "syn 2.0.66", +] + [[package]] name = "destructure_traitobject" version = "0.2.0" @@ -564,6 +847,15 @@ dependencies = [ "generic-array", ] +[[package]] +name = "dirs" +version = "5.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "44c45a9d03d6676652bcb5e724c7e988de1acad23a711b5217ab9cbecbec2225" +dependencies = [ + "dirs-sys", +] + [[package]] name = "dirs-next" version = "2.0.0" @@ -574,6 +866,18 @@ dependencies = [ "dirs-sys-next", ] +[[package]] +name = "dirs-sys" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "520f05a5cbd335fae5a99ff7a6ab8627577660ee5cfd6a94a6a929b52ff0321c" +dependencies = [ + "libc", + "option-ext", + "redox_users", + "windows-sys 0.48.0", +] + [[package]] name = "dirs-sys-next" version = "0.1.2" @@ -585,12 +889,60 @@ dependencies = [ "winapi", ] +[[package]] +name = "displaydoc" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "97369cbbc041bc366949bc74d34658d6cda5621039731c6310521892a3a20ae0" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.66", +] + +[[package]] +name = "dyn-stack" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "56e53799688f5632f364f8fb387488dd05db9fe45db7011be066fc20e7027f8b" +dependencies = [ + "bytemuck", + "reborrow", +] + [[package]] name = "either" version = "1.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3dca9240753cf90908d7e4aac30f630662b02aebaa1b58a3cadabdb23385b58b" +[[package]] +name = "encode_unicode" +version = "0.3.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a357d28ed41a50f9c765dbfe56cbc04a64e53e5fc58ba79fbc34c10ef3df831f" + +[[package]] +name = "encoding_rs" +version = "0.8.35" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "75030f3c4f45dafd7586dd6780965a8c7e8e285a5ecb86713e63a79c5b2766f3" +dependencies = [ + "cfg-if", +] + +[[package]] +name = "enum-as-inner" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a1e6a265c649f3f5979b601d26f1d05ada116434c87741c9493cb56218f76cbc" +dependencies = [ + "heck", + "proc-macro2", + "quote", + "syn 2.0.66", +] + [[package]] name = "equivalent" version = "1.0.1" @@ -617,6 +969,23 @@ dependencies = [ "windows-sys 0.52.0", ] +[[package]] +name = "esaxx-rs" +version = "0.1.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d817e038c30374a4bcb22f94d0a8a0e216958d4c3dcde369b1439fec4bdda6e6" + +[[package]] +name = "fancy-regex" +version = "0.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "531e46835a22af56d1e3b66f04844bed63158bc094a628bec1d321d9b4c44bf2" +dependencies = [ + "bit-set", + "regex-automata", + "regex-syntax", +] + [[package]] name = "fastrand" version = "2.1.0" @@ -733,6 +1102,124 @@ dependencies = [ "slab", ] +[[package]] +name = "gemm" +version = "0.17.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ab24cc62135b40090e31a76a9b2766a501979f3070fa27f689c27ec04377d32" +dependencies = [ + "dyn-stack", + "gemm-c32", + "gemm-c64", + "gemm-common", + "gemm-f16", + "gemm-f32", + "gemm-f64", + "num-complex", + "num-traits", + "paste", + "raw-cpuid", + "seq-macro", +] + +[[package]] +name = "gemm-c32" +version = "0.17.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b9c030d0b983d1e34a546b86e08f600c11696fde16199f971cd46c12e67512c0" +dependencies = [ + "dyn-stack", + "gemm-common", + "num-complex", + "num-traits", + "paste", + "raw-cpuid", + "seq-macro", +] + +[[package]] +name = "gemm-c64" +version = "0.17.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fbb5f2e79fefb9693d18e1066a557b4546cd334b226beadc68b11a8f9431852a" +dependencies = [ + "dyn-stack", + "gemm-common", + "num-complex", + "num-traits", + "paste", + "raw-cpuid", + "seq-macro", +] + +[[package]] +name = "gemm-common" +version = "0.17.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a2e7ea062c987abcd8db95db917b4ffb4ecdfd0668471d8dc54734fdff2354e8" +dependencies = [ + "bytemuck", + "dyn-stack", + "half", + "num-complex", + "num-traits", + "once_cell", + "paste", + "pulp", + "raw-cpuid", + "rayon", + "seq-macro", + "sysctl", +] + +[[package]] +name = "gemm-f16" +version = "0.17.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7ca4c06b9b11952071d317604acb332e924e817bd891bec8dfb494168c7cedd4" +dependencies = [ + "dyn-stack", + "gemm-common", + "gemm-f32", + "half", + "num-complex", + "num-traits", + "paste", + "raw-cpuid", + "rayon", + "seq-macro", +] + +[[package]] +name = "gemm-f32" +version = "0.17.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e9a69f51aaefbd9cf12d18faf273d3e982d9d711f60775645ed5c8047b4ae113" +dependencies = [ + "dyn-stack", + "gemm-common", + "num-complex", + "num-traits", + "paste", + "raw-cpuid", + "seq-macro", +] + +[[package]] +name = "gemm-f64" +version = "0.17.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aa397a48544fadf0b81ec8741e5c0fba0043008113f71f2034def1935645d2b0" +dependencies = [ + "dyn-stack", + "gemm-common", + "num-complex", + "num-traits", + "paste", + "raw-cpuid", + "seq-macro", +] + [[package]] name = "generic-array" version = "0.14.7" @@ -760,6 +1247,12 @@ version = "0.28.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4271d37baee1b8c7e4b708028c57d816cf9d2434acb33a549475f78c181f6253" +[[package]] +name = "glob" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d2fabcfbdc87f4758337ca535fb41a6d701b65693ce38287d856d1674551ec9b" + [[package]] name = "h2" version = "0.3.26" @@ -772,13 +1265,46 @@ dependencies = [ "futures-sink", "futures-util", "http 0.2.12", - "indexmap 2.2.6", + "indexmap 2.6.0", + "slab", + "tokio", + "tokio-util", + "tracing", +] + +[[package]] +name = "h2" +version = "0.4.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "524e8ac6999421f49a846c2d4411f337e53497d8ec55d67753beffa43c5d9205" +dependencies = [ + "atomic-waker", + "bytes", + "fnv", + "futures-core", + "futures-sink", + "http 1.1.0", + "indexmap 2.6.0", "slab", "tokio", "tokio-util", "tracing", ] +[[package]] +name = "half" +version = "2.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6dd08c532ae367adf81c312a4580bc67f1d0fe8bc9c460520283f4c0ff277888" +dependencies = [ + "bytemuck", + "cfg-if", + "crunchy", + "num-traits", + "rand", + "rand_distr", +] + [[package]] name = "hashbrown" version = "0.12.3" @@ -787,9 +1313,9 @@ checksum = "8a9ee70c43aaf417c914396645a0fa852624801b24ebb7ae78fe8272889ac888" [[package]] name = "hashbrown" -version = "0.14.5" +version = "0.15.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e5274423e17b7c9fc20b6e7e208532f9b19825d82dfd615708b70edd83df41f1" +checksum = "3a9bfc1af68b1726ea47d3d5109de126281def866b33970e10fbab11b5dafab3" [[package]] name = "heck" @@ -903,7 +1429,7 @@ dependencies = [ "futures-channel", "futures-core", "futures-util", - "h2", + "h2 0.3.26", "http 0.2.12", "http-body 0.4.6", "httparse", @@ -926,6 +1452,7 @@ dependencies = [ "bytes", "futures-channel", "futures-util", + "h2 0.4.6", "http 1.1.0", "http-body 1.0.0", "httparse", @@ -1023,6 +1550,12 @@ dependencies = [ "cc", ] +[[package]] +name = "ident_case" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b9e0384b61958566e926dc50660321d12159025e767c18e043daf26b70104c39" + [[package]] name = "idna" version = "0.5.0" @@ -1055,12 +1588,25 @@ dependencies = [ [[package]] name = "indexmap" -version = "2.2.6" +version = "2.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "168fb715dda47215e360912c096649d23d58bf392ac62f73919e831745e40f26" +checksum = "707907fe3c25f5424cce2cb7e1cbcafee6bdbe735ca90ef77c29e84591e5b9da" dependencies = [ "equivalent", - "hashbrown 0.14.5", + "hashbrown 0.15.1", +] + +[[package]] +name = "indicatif" +version = "0.17.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "763a5a8f45087d6bcea4222e7b72c291a054edf80e4ef6efd2a4979878c7bea3" +dependencies = [ + "console", + "instant", + "number_prefix", + "portable-atomic", + "unicode-width", ] [[package]] @@ -1084,6 +1630,15 @@ version = "1.70.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f8478577c03552c21db0e2724ffb8986a5ce7af88107e6be5d2ee6e158c12800" +[[package]] +name = "itertools" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b1c173a5686ce8bfa551b3563d0c2170bf24ca44da99c7ca4bfdab5418c3fe57" +dependencies = [ + "either", +] + [[package]] name = "itertools" version = "0.12.1" @@ -1122,14 +1677,20 @@ checksum = "97b3888a4aecf77e811145cadf6eef5901f4782c53886191b2f693f24761847c" [[package]] name = "libloading" -version = "0.8.3" +version = "0.8.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0c2a198fb6b0eada2a8df47933734e6d35d350665a33a3593d7164fa52c75c19" +checksum = "4979f22fdb869068da03c9f7528f8297c6fd2606bc3a4affe42e6a823fdb8da4" dependencies = [ "cfg-if", "windows-targets 0.52.5", ] +[[package]] +name = "libm" +version = "0.2.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8355be11b20d696c8f18f6cc018c4e372165b1fa8126cef092399c9951984ffa" + [[package]] name = "libredox" version = "0.1.3" @@ -1199,6 +1760,22 @@ dependencies = [ "winapi", ] +[[package]] +name = "macro_rules_attribute" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8a82271f7bc033d84bbca59a3ce3e4159938cb08a9c3aebbe54d215131518a13" +dependencies = [ + "macro_rules_attribute-proc_macro", + "paste", +] + +[[package]] +name = "macro_rules_attribute-proc_macro" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8dd856d451cc0da70e2ef2ce95a18e39a93b7558bedf10201ad28503f918568" + [[package]] name = "matchit" version = "0.7.3" @@ -1222,12 +1799,28 @@ version = "2.7.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6c8640c5d730cb13ebd907d8d04b52f55ac9a2eec55b440c8892f40d56c76c1d" +[[package]] +name = "memmap2" +version = "0.9.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fd3f7eed9d3848f8b98834af67102b720745c4ec028fcd0aa0239277e7de374f" +dependencies = [ + "libc", + "stable_deref_trait", +] + [[package]] name = "mime" version = "0.3.17" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6877bb514081ee2a7ff5ef9de3281f14a4dd4bceac4c09388074a6b5df8a139a" +[[package]] +name = "minimal-lexical" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a" + [[package]] name = "miniz_oxide" version = "0.7.3" @@ -1287,18 +1880,68 @@ dependencies = [ "syn 2.0.66", ] +[[package]] +name = "monostate" +version = "0.1.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0d208407d7552cd041d8cdb69a1bc3303e029c598738177a3d87082004dc0e1e" +dependencies = [ + "monostate-impl", + "serde", +] + +[[package]] +name = "monostate-impl" +version = "0.1.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a7ce64b975ed4f123575d11afd9491f2e37bbd5813fbfbc0f09ae1fbddea74e0" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.66", +] + [[package]] name = "multimap" version = "0.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "defc4c55412d89136f966bbb339008b474350e5e6e78d2714439c386b3137a03" +[[package]] +name = "nom" +version = "7.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d273983c5a657a70a3e8f2a01329822f3b8c8172b73826411a55751e404a0a4a" +dependencies = [ + "memchr", + "minimal-lexical", +] + +[[package]] +name = "num-complex" +version = "0.4.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "73f88a1307638156682bada9d7604135552957b7818057dcef22705b4d509495" +dependencies = [ + "bytemuck", + "num-traits", +] + [[package]] name = "num-conv" version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "51d515d32fb182ee37cda2ccdcb92950d6a3c2893aa280e540671c2cd0f3b1d9" +[[package]] +name = "num-integer" +version = "0.1.46" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7969661fd2958a5cb096e56c8e1ad0444ac2bbcd0061bd28660485a44879858f" +dependencies = [ + "num-traits", +] + [[package]] name = "num-traits" version = "0.2.19" @@ -1306,6 +1949,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "071dfc062690e90b734c0b2273ce72ad0ffa95f0c74596bc250dcfd960262841" dependencies = [ "autocfg", + "libm", ] [[package]] @@ -1318,6 +1962,33 @@ dependencies = [ "libc", ] +[[package]] +name = "num_enum" +version = "0.7.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4e613fc340b2220f734a8595782c551f1250e969d87d3be1ae0579e8d4065179" +dependencies = [ + "num_enum_derive", +] + +[[package]] +name = "num_enum_derive" +version = "0.7.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "af1844ef2428cc3e1cb900be36181049ef3d3193c63e43026cfe202983b27a56" +dependencies = [ + "proc-macro-crate", + "proc-macro2", + "quote", + "syn 2.0.66", +] + +[[package]] +name = "number_prefix" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "830b246a0e5f20af87141b25c173cd1b609bd7779a4617d6ec582abaf90870f3" + [[package]] name = "object" version = "0.32.2" @@ -1342,6 +2013,28 @@ version = "1.19.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3fdb12b2476b595f9358c5161aa467c2438859caa136dec86c26fdd2efe17b92" +[[package]] +name = "onig" +version = "6.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8c4b31c8722ad9171c6d77d3557db078cab2bd50afcc9d09c8b315c59df8ca4f" +dependencies = [ + "bitflags 1.3.2", + "libc", + "once_cell", + "onig_sys", +] + +[[package]] +name = "onig_sys" +version = "69.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7b829e3d7e9cc74c7e315ee8edb185bf4190da5acde74afd7fc59c35b1f086e7" +dependencies = [ + "cc", + "pkg-config", +] + [[package]] name = "opaque-debug" version = "0.3.1" @@ -1354,6 +2047,12 @@ version = "0.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ff011a302c396a5197692431fc1948019154afc178baf7d8e37367442a4601cf" +[[package]] +name = "option-ext" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "04744f49eae99ab78e0d5c0b603ab218f515ea8cfe5a456d7629ad883a3b6e7d" + [[package]] name = "ordered-float" version = "2.10.1" @@ -1386,6 +2085,12 @@ dependencies = [ "windows-targets 0.52.5", ] +[[package]] +name = "paste" +version = "1.0.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "57c0d7b74b563b49d38dae00a0c37d4d6de9b432382b2892f0574ddcae73fd0a" + [[package]] name = "percent-encoding" version = "2.3.1" @@ -1399,7 +2104,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b4c5cc86750666a3ed20bdaf5ca2a0344f9c67674cae0515bec2da16fbaa47db" dependencies = [ "fixedbitset", - "indexmap 2.2.6", + "indexmap 2.6.0", ] [[package]] @@ -1440,6 +2145,12 @@ version = "0.3.30" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d231b230927b5e4ad203db57bbcbee2802f6bce620b1e4a9024a07d94e2907ec" +[[package]] +name = "portable-atomic" +version = "1.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cc9c68a3f6da06753e9335d63e27f6b9754dd1920d941135b7ea8224f141adb2" + [[package]] name = "powerfmt" version = "0.2.0" @@ -1472,6 +2183,24 @@ dependencies = [ "syn 2.0.66", ] +[[package]] +name = "primal-check" +version = "0.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc0d895b311e3af9902528fbb8f928688abbd95872819320517cc24ca6b2bd08" +dependencies = [ + "num-integer", +] + +[[package]] +name = "proc-macro-crate" +version = "3.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ecf48c7ca261d60b74ab1a7b20da18bede46776b2e55535cb958eb595c5fa7b" +dependencies = [ + "toml_edit", +] + [[package]] name = "proc-macro2" version = "1.0.83" @@ -1499,7 +2228,7 @@ checksum = "22505a5c94da8e3b7c2996394d1c933236c4d743e81a410bcca4e6989fc066a4" dependencies = [ "bytes", "heck", - "itertools", + "itertools 0.12.1", "log", "multimap", "once_cell", @@ -1519,7 +2248,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "81bddcdb20abf9501610992b6759a4c888aef7d1a7247ef75e2404275ac24af1" dependencies = [ "anyhow", - "itertools", + "itertools 0.12.1", "proc-macro2", "quote", "syn 2.0.66", @@ -1539,6 +2268,18 @@ name = "protoc-bundled" version = "27.0.0" source = "git+https://github.com/rkusa/protoc-bundled.git?rev=27.0.0#d8e8f513cfe007a6243a7eef571677fd366d2cb9" +[[package]] +name = "pulp" +version = "0.18.22" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a0a01a0dc67cf4558d279f0c25b0962bd08fc6dec0137699eae304103e882fe6" +dependencies = [ + "bytemuck", + "libm", + "num-complex", + "reborrow", +] + [[package]] name = "quote" version = "1.0.36" @@ -1578,6 +2319,71 @@ dependencies = [ "getrandom", ] +[[package]] +name = "rand_distr" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32cb0b9bc82b0a0876c2dd994a7e7a2683d3e7390ca40e6886785ef0c7e3ee31" +dependencies = [ + "num-traits", + "rand", +] + +[[package]] +name = "raw-cpuid" +version = "10.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6c297679cb867470fa8c9f67dbba74a78d78e3e98d7cf2b08d6d71540f797332" +dependencies = [ + "bitflags 1.3.2", +] + +[[package]] +name = "rayon" +version = "1.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b418a60154510ca1a002a752ca9714984e21e4241e804d32555251faf8b78ffa" +dependencies = [ + "either", + "rayon-core", +] + +[[package]] +name = "rayon-cond" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "059f538b55efd2309c9794130bc149c6a553db90e9d99c2030785c82f0bd7df9" +dependencies = [ + "either", + "itertools 0.11.0", + "rayon", +] + +[[package]] +name = "rayon-core" +version = "1.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1465873a3dfdaa8ae7cb14b4383657caab0b3e8a0aa9ae8e04b044854c8dfce2" +dependencies = [ + "crossbeam-deque", + "crossbeam-utils", +] + +[[package]] +name = "realfft" +version = "3.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "390252372b7f2aac8360fc5e72eba10136b166d6faeed97e6d0c8324eb99b2b1" +dependencies = [ + "rustfft", +] + +[[package]] +name = "reborrow" +version = "0.5.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "03251193000f4bd3b042892be858ee50e8b3719f2b08e5833ac4353724632430" + [[package]] name = "redox_syscall" version = "0.5.1" @@ -1635,8 +2441,10 @@ checksum = "566cafdd92868e0939d3fb961bd0dc25fcfaaed179291093b3d43e6b3150ea10" dependencies = [ "base64 0.22.1", "bytes", + "encoding_rs", "futures-core", "futures-util", + "h2 0.4.6", "http 1.1.0", "http-body 1.0.0", "http-body-util", @@ -1657,6 +2465,7 @@ dependencies = [ "serde_json", "serde_urlencoded", "sync_wrapper", + "system-configuration", "tokio", "tokio-rustls 0.25.0", "tower-service", @@ -1698,6 +2507,18 @@ dependencies = [ "windows-sys 0.52.0", ] +[[package]] +name = "rubato" +version = "0.16.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cdd96992d7e24b3d7f35fdfe02af037a356ac90d41b466945cf3333525a86eea" +dependencies = [ + "num-complex", + "num-integer", + "num-traits", + "realfft", +] + [[package]] name = "rusoto_core" version = "0.48.0" @@ -1803,6 +2624,21 @@ dependencies = [ "semver", ] +[[package]] +name = "rustfft" +version = "6.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "43806561bc506d0c5d160643ad742e3161049ac01027b5e6d7524091fd401d86" +dependencies = [ + "num-complex", + "num-integer", + "num-traits", + "primal-check", + "strength_reduce", + "transpose", + "version_check", +] + [[package]] name = "rustix" version = "0.38.34" @@ -1902,6 +2738,16 @@ version = "1.0.18" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f3cb5ba0dc43242ce17de99c180e96db90b235b8a9fdc9543c96d2209116bd9f" +[[package]] +name = "safetensors" +version = "0.4.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "44560c11236a6130a46ce36c836a62936dc81ebf8c36a37947423571be0e55b6" +dependencies = [ + "serde", + "serde_json", +] + [[package]] name = "same-file" version = "1.0.6" @@ -1965,6 +2811,12 @@ version = "1.0.23" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "61697e0a1c7e512e84a621326239844a24d8207b4669b41bc18b32ea5cbf988b" +[[package]] +name = "seq-macro" +version = "0.3.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a3f0bf26fd526d2a95683cd0f87bf103b8539e2ca1ef48ce002d67aad59aa0b4" + [[package]] name = "serde" version = "1.0.202" @@ -2006,6 +2858,15 @@ dependencies = [ "serde", ] +[[package]] +name = "serde_plain" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ce1fc6db65a611022b23a0dec6975d63fb80a302cb3388835ff02c097258d50" +dependencies = [ + "serde", +] + [[package]] name = "serde_repr" version = "0.1.19" @@ -2035,7 +2896,7 @@ version = "0.9.34+deprecated" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6a8b1a1a2ebf674015cc02edccce75287f1a0130d394307b36743c2f5d504b47" dependencies = [ - "indexmap 2.2.6", + "indexmap 2.6.0", "itoa", "ryu", "serde", @@ -2107,6 +2968,30 @@ version = "0.9.8" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6980e8d7511241f8acf4aebddbb1ff938df5eebe98691418c4468d0b72a96a67" +[[package]] +name = "spm_precompiled" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5851699c4033c63636f7ea4cf7b7c1f1bf06d0cc03cfb42e711de5a5c46cf326" +dependencies = [ + "base64 0.13.1", + "nom", + "serde", + "unicode-segmentation", +] + +[[package]] +name = "stable_deref_trait" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a8f112729512f8e442d81f95a8a7ddf2b7c6b8a1a6f509a95864142b30cab2d3" + +[[package]] +name = "strength_reduce" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fe895eb47f22e2ddd4dabc02bce419d2e643c8e3b585c78158b349195bc24d82" + [[package]] name = "strsim" version = "0.11.1" @@ -2147,6 +3032,52 @@ version = "0.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2047c6ded9c721764247e62cd3b03c09ffc529b2ba5b10ec482ae507a4a70160" +[[package]] +name = "synstructure" +version = "0.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c8af7666ab7b6390ab78131fb5b0fce11d6b7a6951602017c35fa82800708971" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.66", +] + +[[package]] +name = "sysctl" +version = "0.5.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec7dddc5f0fee506baf8b9fdb989e242f17e4b11c61dfbb0635b705217199eea" +dependencies = [ + "bitflags 2.5.0", + "byteorder", + "enum-as-inner", + "libc", + "thiserror", + "walkdir", +] + +[[package]] +name = "system-configuration" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ba3a3adc5c275d719af8cb4272ea1c4a6d668a777f37e115f6d11ddbc1c8e0e7" +dependencies = [ + "bitflags 1.3.2", + "core-foundation", + "system-configuration-sys", +] + +[[package]] +name = "system-configuration-sys" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a75fb188eb626b924683e3b95e3a48e63551fcfb51949de2f06a9d91dbee93c9" +dependencies = [ + "core-foundation-sys", + "libc", +] + [[package]] name = "tempfile" version = "3.10.1" @@ -2235,6 +3166,37 @@ version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" +[[package]] +name = "tokenizers" +version = "0.20.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "67b67c92f6d705e2a1d106fb0b28c696f9074901a9c656ee5d9f5de204c39bf7" +dependencies = [ + "aho-corasick", + "derive_builder", + "esaxx-rs", + "getrandom", + "itertools 0.12.1", + "lazy_static", + "log", + "macro_rules_attribute", + "monostate", + "onig", + "paste", + "rand", + "rayon", + "rayon-cond", + "regex", + "regex-syntax", + "serde", + "serde_json", + "spm_precompiled", + "thiserror", + "unicode-normalization-alignments", + "unicode-segmentation", + "unicode_categories", +] + [[package]] name = "tokio" version = "1.37.0" @@ -2322,6 +3284,23 @@ dependencies = [ "tokio", ] +[[package]] +name = "toml_datetime" +version = "0.6.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0dd7358ecb8fc2f8d014bf86f6f638ce72ba252a2c3a2572f2a795f1d23efb41" + +[[package]] +name = "toml_edit" +version = "0.22.22" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4ae48d6208a266e853d946088ed816055e556cc6028c5e8e2b84d9fa5dd7c7f5" +dependencies = [ + "indexmap 2.6.0", + "toml_datetime", + "winnow", +] + [[package]] name = "tonic" version = "0.11.0" @@ -2333,7 +3312,7 @@ dependencies = [ "axum", "base64 0.21.7", "bytes", - "h2", + "h2 0.3.26", "http 0.2.12", "http-body 0.4.6", "hyper 0.14.28", @@ -2438,6 +3417,16 @@ dependencies = [ "once_cell", ] +[[package]] +name = "transpose" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ad61aed86bc3faea4300c7aee358b4c6d0c8d6ccc36524c96e4c92ccf26e77e" +dependencies = [ + "num-integer", + "strength_reduce", +] + [[package]] name = "try-lock" version = "0.2.5" @@ -2486,6 +3475,33 @@ dependencies = [ "tinyvec", ] +[[package]] +name = "unicode-normalization-alignments" +version = "0.1.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "43f613e4fa046e69818dd287fdc4bc78175ff20331479dab6e1b0f98d57062de" +dependencies = [ + "smallvec", +] + +[[package]] +name = "unicode-segmentation" +version = "1.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f6ccf251212114b54433ec949fd6a7841275f9ada20dddd2f29e9ceea4501493" + +[[package]] +name = "unicode-width" +version = "0.1.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7dd6e30e90baa6f72411720665d41d89b9a3d039dc45b8faea1ddd07f617f6af" + +[[package]] +name = "unicode_categories" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "39ec24b3121d976906ece63c9daad25b85969647682eee313cb5779fdd69e14e" + [[package]] name = "unsafe-any-ors" version = "1.0.0" @@ -2897,6 +3913,15 @@ version = "0.52.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bec47e5bfd1bff0eeaf6d8b485cc1074891a197ab4225d504cb7a1ab88b02bf0" +[[package]] +name = "winnow" +version = "0.6.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "36c1fec1a2bb5866f07c25f68c26e565c4c200aebb96d7e55710c19d3e8ac49b" +dependencies = [ + "memchr", +] + [[package]] name = "winreg" version = "0.52.0" @@ -2919,8 +3944,68 @@ version = "0.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "09041cd90cf85f7f8b2df60c646f853b7f535ce68f85244eb6731cf89fa498ec" +[[package]] +name = "yoke" +version = "0.7.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6c5b1314b079b0930c31e3af543d8ee1757b1951ae1e1565ec704403a7240ca5" +dependencies = [ + "serde", + "stable_deref_trait", + "yoke-derive", + "zerofrom", +] + +[[package]] +name = "yoke-derive" +version = "0.7.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "28cc31741b18cb6f1d5ff12f5b7523e3d6eb0852bbbad19d73905511d9849b95" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.66", + "synstructure", +] + +[[package]] +name = "zerofrom" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "91ec111ce797d0e0784a1116d0ddcdbea84322cd79e5d5ad173daeba4f93ab55" +dependencies = [ + "zerofrom-derive", +] + +[[package]] +name = "zerofrom-derive" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ea7b4a3637ea8669cedf0f1fd5c286a17f3de97b8dd5a70a6c167a1730e63a5" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.66", + "synstructure", +] + [[package]] name = "zeroize" version = "1.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "525b4ec142c6b68a2d10f01f7bbf6755599ca3f81ea53b8431b7dd348f5fdb2d" + +[[package]] +name = "zip" +version = "1.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9cc23c04387f4da0374be4533ad1208cbb091d5c11d070dfef13676ad6497164" +dependencies = [ + "arbitrary", + "crc32fast", + "crossbeam-utils", + "displaydoc", + "indexmap 2.6.0", + "num_enum", + "thiserror", +] diff --git a/README.md b/README.md index df2acda4..371830ee 100644 --- a/README.md +++ b/README.md @@ -82,11 +82,11 @@ throughputLimit = 600 integrityCheckDisabled = false -- Whether or not authentication is required -auth.enabled = false --- Authentication tokens table with client names and their tokens for split tokens. +auth.enabled = false +-- Authentication tokens table with client names and their tokens for split tokens. auth.tokens = { - -- client => clientName, token => Any token. Advice to use UTF-8 only. Length not limited explicitly - { client = "SomeClient", token = "SomeToken" }, + -- client => clientName, token => Any token. Advice to use UTF-8 only. Length not limited explicitly + { client = "SomeClient", token = "SomeToken" }, { client = "SomeClient2", token = "SomeOtherToken" } } @@ -124,6 +124,9 @@ tts.provider.gcloud.defaultVoice = "en-GB-Neural2-A" -- Requires at least Windows Server 2019 to work properly. tts.provider.win.defaultVoice = "David" +-- The default Parler speaker prompt. +tts.provider.parler.defaultSpeaker = "..." + -- Your SRS server's address. srs.addr = "127.0.0.1:5002" ``` @@ -211,6 +214,7 @@ The server will be running on port 50051 by default. -- `= { azure = {} }` / `= { azure = { voice = "..." } }` enable Azure TTS -- `= { gcloud = {} }` / `= { gcloud = { voice = "..." } }` enable Google Cloud TTS -- `= { win = {} }` / `= { win = { voice = "..." } }` enable Windows TTS + -- `= { parler = {} }` / `= { parler = { voice = "...", speed = 1.0 } }` enable Parler TTS provider = null, } ``` @@ -228,7 +232,7 @@ The gRPC .proto files are available in the `Docs/DCS-gRPC` folder and also avail ### Client Authentication -If authentication is enabled on the server you will have to add `X-API-Key` to the metadata/headers. +If authentication is enabled on the server you will have to add `X-API-Key` to the metadata/headers. Below are some example on what it could look like in your code. #### Examples @@ -238,7 +242,7 @@ Below are some example on what it could look like in your code. You can either set the `Metadata` for each request or you can create a `GrpcChannel` with an interceptor that will set the key each time. -For a single request: +For a single request: ```c# var client = new MissionService.MissionServiceClient(channel); @@ -251,7 +255,7 @@ Metadata metadata = new Metadata() var response = client.GetScenarioCurrentTime(new GetScenarioCurrentTimeRequest { }, headers: metadata, deadline: DateTime.UtcNow.AddSeconds(2)); ``` -For all requests on a channel: +For all requests on a channel: ```c# public GrpcChannel CreateChannel(string host, string post, string? apiKey) { diff --git a/lua/DCS-gRPC/grpc-mission.lua b/lua/DCS-gRPC/grpc-mission.lua index 0f49024f..cb7ab440 100644 --- a/lua/DCS-gRPC/grpc-mission.lua +++ b/lua/DCS-gRPC/grpc-mission.lua @@ -1,35 +1,35 @@ if not GRPC then - GRPC = { - -- scaffold nested tables to allow direct assignment in config file - tts = { provider = { gcloud = {}, aws = {}, azure = {}, win = {} } }, - srs = {}, - auth = { tokens = {} } - } + GRPC = { + -- scaffold nested tables to allow direct assignment in config file + tts = { provider = { gcloud = {}, aws = {}, azure = {}, win = {}, parler = {} } }, + srs = {}, + auth = { tokens = {} } + } end -- load settings from `Saved Games/DCS/Config/dcs-grpc.lua` do - env.info("[GRPC] Checking optional config at `Config/dcs-grpc.lua` ...") - local file, err = io.open(lfs.writedir() .. [[Config\dcs-grpc.lua]], "r") - if file then - local f = assert(loadstring(file:read("*all"))) - setfenv(f, GRPC) - f() - env.info("[GRPC] `Config/dcs-grpc.lua` successfully read") - else - env.info("[GRPC] `Config/dcs-grpc.lua` not found (" .. tostring(err) .. ")") - end + env.info("[GRPC] Checking optional config at `Config/dcs-grpc.lua` ...") + local file, err = io.open(lfs.writedir() .. [[Config\dcs-grpc.lua]], "r") + if file then + local f = assert(loadstring(file:read("*all"))) + setfenv(f, GRPC) + f() + env.info("[GRPC] `Config/dcs-grpc.lua` successfully read") + else + env.info("[GRPC] `Config/dcs-grpc.lua` not found (" .. tostring(err) .. ")") + end end -- Set default settings. if not GRPC.luaPath then - GRPC.luaPath = lfs.writedir() .. [[Scripts\DCS-gRPC\]] + GRPC.luaPath = lfs.writedir() .. [[Scripts\DCS-gRPC\]] end if not GRPC.dllPath then - GRPC.dllPath = lfs.writedir() .. [[Mods\tech\DCS-gRPC\]] + GRPC.dllPath = lfs.writedir() .. [[Mods\tech\DCS-gRPC\]] end if GRPC.throughputLimit == nil or GRPC.throughputLimit == 0 or type(GRPC.throughputLimit) ~= "number" then - GRPC.throughputLimit = 600 + GRPC.throughputLimit = 600 end -- load version @@ -37,15 +37,15 @@ dofile(GRPC.luaPath .. [[version.lua]]) -- Let DCS know where to find the DLLs if not string.find(package.cpath, GRPC.dllPath) then - package.cpath = package.cpath .. [[;]] .. GRPC.dllPath .. [[?.dll;]] + package.cpath = package.cpath .. [[;]] .. GRPC.dllPath .. [[?.dll;]] end -- Load DLL before `require` gets sanitized. local ok, grpc = pcall(require, "dcs_grpc_hot_reload") if ok then - env.info("[GRPC] loaded hot reload version") + env.info("[GRPC] loaded hot reload version") else - grpc = require("dcs_grpc") + grpc = require("dcs_grpc") end -- Keep a reference to `lfs` before it gets sanitized @@ -53,19 +53,19 @@ local lfs = _G.lfs local loaded = false function GRPC.load() - if loaded then - env.info("[GRPC] already loaded") - return - end + if loaded then + env.info("[GRPC] already loaded") + return + end - local env = setmetatable({grpc = grpc, lfs = lfs}, {__index = _G}) - local f = setfenv(assert(loadfile(GRPC.luaPath .. [[grpc.lua]])), env) - f() + local env = setmetatable({ grpc = grpc, lfs = lfs }, { __index = _G }) + local f = setfenv(assert(loadfile(GRPC.luaPath .. [[grpc.lua]])), env) + f() - loaded = true + loaded = true end if GRPC.autostart == true then - env.info("[GRPC] auto starting") - GRPC.load() + env.info("[GRPC] auto starting") + GRPC.load() end diff --git a/lua/Hooks/DCS-gRPC.lua b/lua/Hooks/DCS-gRPC.lua index 6e714ab0..315ab559 100644 --- a/lua/Hooks/DCS-gRPC.lua +++ b/lua/Hooks/DCS-gRPC.lua @@ -7,7 +7,7 @@ local function init() if not GRPC then _G.GRPC = { -- scaffold nested tables to allow direct assignment in config file - tts = { provider = { gcloud = {}, aws = {}, azure = {}, win = {} } }, + tts = { provider = { gcloud = {}, aws = {}, azure = {}, win = {}, parler = {} } }, srs = {}, auth = { tokens = {} } } diff --git a/protos/dcs/srs/v0/srs.proto b/protos/dcs/srs/v0/srs.proto index c813e7b0..c4cb75b7 100644 --- a/protos/dcs/srs/v0/srs.proto +++ b/protos/dcs/srs/v0/srs.proto @@ -75,6 +75,10 @@ message TransmitRequest { optional string voice = 1; } + message Parler { + optional string speaker = 1; + } + // Optional TTS provider to be use. Defaults to the one configured in your // config or to Windows' built-in TTS. oneof provider { @@ -82,6 +86,8 @@ message TransmitRequest { Azure azure = 9; GCloud gcloud = 10; Windows win = 11; + // Parler does not support SSML, only use it with plain text. + Parler parler = 12; } } diff --git a/src/authentication.rs b/src/authentication.rs index e19933db..74551514 100644 --- a/src/authentication.rs +++ b/src/authentication.rs @@ -1,9 +1,10 @@ -use crate::config::AuthConfig; use tonic::codegen::http::Request; use tonic::transport::Body; use tonic::{async_trait, Status}; use tonic_middleware::RequestInterceptor; +use crate::config::AuthConfig; + #[derive(Clone)] pub struct AuthInterceptor { pub auth_config: AuthConfig, diff --git a/src/config.rs b/src/config.rs index 726fa7e2..bd777d47 100644 --- a/src/config.rs +++ b/src/config.rs @@ -39,6 +39,7 @@ pub struct TtsProviderConfig { pub azure: Option, pub gcloud: Option, pub win: Option, + pub parler: Option, } #[derive(Debug, Clone, Default, Deserialize, Serialize)] @@ -49,6 +50,7 @@ pub enum TtsProvider { GCloud, #[default] Win, + Parler, } #[derive(Clone, Deserialize, Serialize)] @@ -81,6 +83,12 @@ pub struct WinConfig { pub default_voice: Option, } +#[derive(Debug, Clone, Deserialize, Serialize)] +#[serde(rename_all = "camelCase")] +pub struct ParlerConfig { + pub default_speaker: Option, +} + #[derive(Debug, Clone, Default, Deserialize, Serialize)] #[serde(rename_all = "camelCase")] pub struct SrsConfig { diff --git a/src/rpc/metadata.rs b/src/rpc/metadata.rs index 415c82db..eea3b013 100644 --- a/src/rpc/metadata.rs +++ b/src/rpc/metadata.rs @@ -1,7 +1,6 @@ use stubs::metadata::v0::metadata_service_server::MetadataService; use stubs::*; -use tonic::async_trait; -use tonic::{Request, Response, Status}; +use tonic::{async_trait, Request, Response, Status}; use super::MissionRpc; diff --git a/src/rpc/srs.rs b/src/rpc/srs.rs index cdd0782a..c56d2125 100644 --- a/src/rpc/srs.rs +++ b/src/rpc/srs.rs @@ -1,13 +1,14 @@ use std::error; use std::future::Future; use std::net::{IpAddr, Ipv4Addr, SocketAddr}; +use std::path::PathBuf; use std::str::FromStr; use std::time::{Duration, Instant}; use ::srs::Sender; #[cfg(target_os = "windows")] use ::tts::WinConfig; -use ::tts::{AwsConfig, AwsRegion, AzureConfig, GCloudConfig, TtsConfig}; +use ::tts::{AwsConfig, AwsRegion, AzureConfig, GCloudConfig, ParlerConfig, TtsConfig}; use futures_util::FutureExt; use stubs::common::v0::{Coalition, Unit}; use stubs::mission::v0::stream_events_response::{Event, TtsEvent}; @@ -27,6 +28,7 @@ use crate::srs::SrsClients; pub struct Srs { tts_config: crate::config::TtsConfig, srs_config: crate::config::SrsConfig, + write_dir: PathBuf, rpc: MissionRpc, srs_clients: SrsClients, shutdown_signal: ShutdownHandle, @@ -36,6 +38,7 @@ impl Srs { pub fn new( tts_config: crate::config::TtsConfig, srs_config: crate::config::SrsConfig, + write_dir: PathBuf, rpc: MissionRpc, srs_clients: SrsClients, shutdown_signal: ShutdownHandle, @@ -43,6 +46,7 @@ impl Srs { Self { tts_config, srs_config, + write_dir, rpc, srs_clients, shutdown_signal, @@ -105,6 +109,9 @@ impl SrsService for Srs { TtsProvider::Win => { transmit_request::Provider::Win(transmit_request::Windows { voice: None }) } + TtsProvider::Parler => { + transmit_request::Provider::Parler(transmit_request::Parler { speaker: None }) + } }) { transmit_request::Provider::Aws(transmit_request::Aws { voice }) => { TtsConfig::Aws(AwsConfig { @@ -215,6 +222,24 @@ impl SrsService for Srs { "Windows TTS is only available on Windows", )); } + transmit_request::Provider::Parler(transmit_request::Parler { speaker }) => { + TtsConfig::Parler(ParlerConfig { + speaker: speaker + .or_else(|| { + self.tts_config + .provider + .as_ref() + .and_then(|p| p.parler.as_ref()) + .and_then(|p| p.default_speaker.clone()) + }) + .filter(|v| !v.is_empty()) + .ok_or_else(|| { + Status::failed_precondition( + "tts.provider.parler.default_speaker not set", + ) + })?, + }) + } }; let frames = ::tts::synthesize(&request.ssml, &config) diff --git a/src/server.rs b/src/server.rs index c775612d..413c9b9b 100644 --- a/src/server.rs +++ b/src/server.rs @@ -1,14 +1,9 @@ use std::future::Future; use std::net::SocketAddr; +use std::path::PathBuf; use std::sync::Arc; use std::time::Duration; -use crate::authentication::AuthInterceptor; -use crate::config::{AuthConfig, Config, SrsConfig, TtsConfig}; -use crate::rpc::{HookRpc, MissionRpc, Srs}; -use crate::shutdown::{Shutdown, ShutdownHandle}; -use crate::srs::SrsClients; -use crate::stats::Stats; use dcs_module_ipc::IPC; use futures_util::FutureExt; use stubs::atmosphere::v0::atmosphere_service_server::AtmosphereServiceServer; @@ -34,6 +29,13 @@ use tokio::time::sleep; use tonic::transport; use tonic_middleware::RequestInterceptorLayer; +use crate::authentication::AuthInterceptor; +use crate::config::{AuthConfig, Config, SrsConfig, TtsConfig}; +use crate::rpc::{HookRpc, MissionRpc, Srs}; +use crate::shutdown::{Shutdown, ShutdownHandle}; +use crate::srs::SrsClients; +use crate::stats::Stats; + pub struct Server { runtime: Runtime, shutdown: Shutdown, @@ -51,6 +53,7 @@ struct ServerState { stats: Stats, tts_config: TtsConfig, srs_config: SrsConfig, + write_dir: PathBuf, srs_transmit: Arc>>, auth_config: AuthConfig, } @@ -73,6 +76,7 @@ impl Server { stats: Stats::new(shutdown.handle()), tts_config: config.tts.clone().unwrap_or_default(), srs_config: config.srs.clone().unwrap_or_default(), + write_dir: PathBuf::from(&config.write_dir), srs_transmit: Arc::new(Mutex::new(rx)), auth_config: config.auth.clone().unwrap_or_default(), }, @@ -206,6 +210,7 @@ async fn try_run( stats, tts_config, srs_config, + write_dir, srs_transmit, auth_config, } = state; @@ -230,6 +235,7 @@ async fn try_run( let srs = Srs::new( tts_config.clone(), srs_config.clone(), + write_dir.clone(), mission_rpc.clone(), srs_clients.clone(), shutdown_signal.clone(), @@ -269,6 +275,7 @@ async fn try_run( .add_service(SrsServiceServer::new(Srs::new( tts_config, srs_config, + write_dir, mission_rpc.clone(), srs_clients, shutdown_signal.clone(), diff --git a/tts/Cargo.toml b/tts/Cargo.toml index b8cc483c..0ead49d8 100644 --- a/tts/Cargo.toml +++ b/tts/Cargo.toml @@ -10,22 +10,30 @@ edition.workspace = true audiopus = "0.2" base64.workspace = true bytes.workspace = true +candle-core = { version = "0.7", features = ["cuda"] } +candle-hf-hub = { version = "0.3", default-features = false, features = [ + "tokio", +] } +candle-nn = { version = "0.7", features = ["cuda"] } +candle-transformers = { version = "0.7", features = ["cuda"] } log.workspace = true ogg = "0.9" reqwest = { version = "0.12", default-features = false, features = [ "rustls-tls", "json", ] } -rusoto_core = { version = "0.48", default_features = false, features = [ +rubato = "0.16" +rusoto_core = { version = "0.48", default-features = false, features = [ "rustls", ] } rusoto_credential = "0.48" -rusoto_polly = { version = "0.48", default_features = false, features = [ +rusoto_polly = { version = "0.48", default-features = false, features = [ "rustls", ] } serde.workspace = true serde_json.workspace = true thiserror.workspace = true +tokenizers = { version = "0.20", default-features = false, features = ["onig"] } tokio.workspace = true [target.'cfg(target_os = "windows")'.dependencies.windows] diff --git a/tts/src/bs1770.rs b/tts/src/bs1770.rs new file mode 100644 index 00000000..6edeb1d8 --- /dev/null +++ b/tts/src/bs1770.rs @@ -0,0 +1,508 @@ +// Copied from https://github.com/ruuda/bs1770/blob/master/src/lib.rs +// BS1770 -- Loudness analysis library conforming to ITU-R BS.1770 +// Copyright 2020 Ruud van Asseldonk + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// A copy of the License has been included in the root of the repository. + +#![allow(unused)] + +//! Loudness analysis conforming to [ITU-R BS.1770-4][bs17704]. +//! +//! This library offers the building blocks to perform BS.1770 loudness +//! measurements, but you need to put the pieces together yourself. +//! +//! [bs17704]: https://www.itu.int/rec/R-REC-BS.1770-4-201510-I/en +//! +//! # Stereo integrated loudness example +//! +//! ```ignore +//! # fn load_stereo_audio() -> [Vec; 2] { +//! # [vec![0; 48_000], vec![0; 48_000]] +//! # } +//! # +//! let sample_rate_hz = 44_100; +//! let bits_per_sample = 16; +//! let channel_samples: [Vec; 2] = load_stereo_audio(); +//! +//! // When converting integer samples to float, note that the maximum amplitude +//! // is `1 << (bits_per_sample - 1)`, one bit is the sign bit. +//! let normalizer = 1.0 / (1_u64 << (bits_per_sample - 1)) as f32; +//! +//! let channel_power: Vec<_> = channel_samples.iter().map(|samples| { +//! let mut meter = bs1770::ChannelLoudnessMeter::new(sample_rate_hz); +//! meter.push(samples.iter().map(|&s| s as f32 * normalizer)); +//! meter.into_100ms_windows() +//! }).collect(); +//! +//! let stereo_power = bs1770::reduce_stereo( +//! channel_power[0].as_ref(), +//! channel_power[1].as_ref(), +//! ); +//! +//! let gated_power = bs1770::gated_mean( +//! stereo_power.as_ref() +//! ).unwrap_or(bs1770::Power(0.0)); +//! println!("Integrated loudness: {:.1} LUFS", gated_power.loudness_lkfs()); +//! ``` + +use std::f32; + +/// Coefficients for a 2nd-degree infinite impulse response filter. +/// +/// Coefficient a0 is implicitly 1.0. +#[derive(Clone)] +struct Filter { + a1: f32, + a2: f32, + b0: f32, + b1: f32, + b2: f32, + + // The past two input and output samples. + x1: f32, + x2: f32, + y1: f32, + y2: f32, +} + +impl Filter { + /// Stage 1 of th BS.1770-4 pre-filter. + pub fn high_shelf(sample_rate_hz: f32) -> Filter { + // Coefficients taken from https://github.com/csteinmetz1/pyloudnorm/blob/ + // 6baa64d59b7794bc812e124438692e7fd2e65c0c/pyloudnorm/meter.py#L135-L136. + let gain_db = 3.999_843_8; + let q = 0.707_175_25; + let center_hz = 1_681.974_5; + + // Formula taken from https://github.com/csteinmetz1/pyloudnorm/blob/ + // 6baa64d59b7794bc812e124438692e7fd2e65c0c/pyloudnorm/iirfilter.py#L134-L143. + let k = (f32::consts::PI * center_hz / sample_rate_hz).tan(); + let vh = 10.0_f32.powf(gain_db / 20.0); + let vb = vh.powf(0.499_666_78); + let a0 = 1.0 + k / q + k * k; + Filter { + b0: (vh + vb * k / q + k * k) / a0, + b1: 2.0 * (k * k - vh) / a0, + b2: (vh - vb * k / q + k * k) / a0, + a1: 2.0 * (k * k - 1.0) / a0, + a2: (1.0 - k / q + k * k) / a0, + + x1: 0.0, + x2: 0.0, + y1: 0.0, + y2: 0.0, + } + } + + /// Stage 2 of th BS.1770-4 pre-filter. + pub fn high_pass(sample_rate_hz: f32) -> Filter { + // Coefficients taken from https://github.com/csteinmetz1/pyloudnorm/blob/ + // 6baa64d59b7794bc812e124438692e7fd2e65c0c/pyloudnorm/meter.py#L135-L136. + let q = 0.500_327_05; + let center_hz = 38.135_47; + + // Formula taken from https://github.com/csteinmetz1/pyloudnorm/blob/ + // 6baa64d59b7794bc812e124438692e7fd2e65c0c/pyloudnorm/iirfilter.py#L145-L151 + let k = (f32::consts::PI * center_hz / sample_rate_hz).tan(); + Filter { + a1: 2.0 * (k * k - 1.0) / (1.0 + k / q + k * k), + a2: (1.0 - k / q + k * k) / (1.0 + k / q + k * k), + b0: 1.0, + b1: -2.0, + b2: 1.0, + + x1: 0.0, + x2: 0.0, + y1: 0.0, + y2: 0.0, + } + } + + /// Feed the next input sample, get the next output sample. + #[inline(always)] + pub fn apply(&mut self, x0: f32) -> f32 { + let y0 = 0.0 + self.b0 * x0 + self.b1 * self.x1 + self.b2 * self.x2 + - self.a1 * self.y1 + - self.a2 * self.y2; + + self.x2 = self.x1; + self.x1 = x0; + self.y2 = self.y1; + self.y1 = y0; + + y0 + } +} + +/// Compensated sum, for summing many values of different orders of magnitude +/// accurately. +#[derive(Copy, Clone, PartialEq)] +struct Sum { + sum: f32, + residue: f32, +} + +impl Sum { + #[inline(always)] + fn zero() -> Sum { + Sum { + sum: 0.0, + residue: 0.0, + } + } + + #[inline(always)] + fn add(&mut self, x: f32) { + let sum = self.sum + (self.residue + x); + self.residue = (self.residue + x) - (sum - self.sum); + self.sum = sum; + } +} + +/// The mean of the squares of the K-weighted samples in a window of time. +/// +/// K-weighted power is equivalent to K-weighted loudness, the only difference +/// is one of scale: power is quadratic in sample amplitudes, whereas loudness +/// units are logarithmic. `loudness_lkfs` and `from_lkfs` convert between power, +/// and K-weighted Loudness Units relative to nominal Full Scale (LKFS). +/// +/// The term “LKFS” (Loudness Units, K-Weighted, relative to nominal Full Scale) +/// is used in BS.1770-4 to emphasize K-weighting, but the term is otherwise +/// interchangeable with the more widespread term “LUFS” (Loudness Units, +/// relative to Full Scale). Loudness units are related to decibels in the +/// following sense: boosting a signal that has a loudness of +/// -LK LUFS by LK dB (by +/// multiplying the amplitude by 10LK/20) will +/// bring the loudness to 0 LUFS. +/// +/// K-weighting refers to a high-shelf and high-pass filter that model the +/// effect that humans perceive a certain amount of power in low frequencies to +/// be less loud than the same amount of power in higher frequencies. In this +/// library the `Power` type is used exclusively to refer to power after applying K-weighting. +/// +/// The nominal “full scale” is the range [-1.0, 1.0]. Because the power is the +/// mean square of the samples, if no input samples exceeded the full scale, the +/// power will be in the range [0.0, 1.0]. However, the power delivered by +/// multiple channels, which is a weighted sum over individual channel powers, +/// can exceed this range, because the weighted sum is not normalized. +#[derive(Copy, Clone, PartialEq, PartialOrd)] +pub struct Power(pub f32); + +impl Power { + /// Convert Loudness Units relative to Full Scale into a squared sample amplitude. + /// + /// This is the inverse of `loudness_lkfs`. + pub fn from_lkfs(lkfs: f32) -> Power { + // The inverse of the formula below. + Power(10.0_f32.powf((lkfs + 0.691) * 0.1)) + } + + /// Return the loudness of this window in Loudness Units, K-weighted, relative to Full Scale. + /// + /// This is the inverse of `from_lkfs`. + pub fn loudness_lkfs(&self) -> f32 { + // Equation 2 (p.5) of BS.1770-4. + -0.691 + 10.0 * self.0.log10() + } +} + +/// A `T` value for non-overlapping windows of audio, 100ms in length. +/// +/// The `ChannelLoudnessMeter` applies K-weighting and then produces the power +/// for non-overlapping windows of 100ms duration. +/// +/// These non-overlapping 100ms windows can later be combined into overlapping +/// windows of 400ms, spaced 100ms apart, to compute instantaneous loudness or +/// to perform a gated measurement, or they can be combined into even larger +/// windows for a momentary loudness measurement. +#[derive(Copy, Clone, Debug)] +pub struct Windows100ms { + pub inner: T, +} + +impl Windows100ms { + /// Wrap a new empty vector. + pub fn new() -> Windows100ms> { + Windows100ms { inner: Vec::new() } + } + + /// Apply `as_ref` to the inner value. + pub fn as_ref(&self) -> Windows100ms<&[Power]> + where + T: AsRef<[Power]>, + { + Windows100ms { + inner: self.inner.as_ref(), + } + } + + /// Apply `as_mut` to the inner value. + pub fn as_mut(&mut self) -> Windows100ms<&mut [Power]> + where + T: AsMut<[Power]>, + { + Windows100ms { + inner: self.inner.as_mut(), + } + } + + #[allow(clippy::len_without_is_empty)] + /// Apply `len` to the inner value. + pub fn len(&self) -> usize + where + T: AsRef<[Power]>, + { + self.inner.as_ref().len() + } +} + +/// Measures K-weighted power of non-overlapping 100ms windows of a single channel of audio. +/// +/// # Output +/// +/// The output of the meter is an intermediate result in the form of power for +/// 100ms non-overlapping windows. The windows need to be processed further to +/// get one of the instantaneous, momentary, and integrated loudness +/// measurements defined in BS.1770. +/// +/// The windows can also be inspected directly; the data is meaningful +/// on its own (the K-weighted power delivered in that window of time), but it +/// is not something that BS.1770 defines a term for. +/// +/// # Multichannel audio +/// +/// To perform a loudness measurement of multichannel audio, construct a +/// `ChannelLoudnessMeter` per channel, and later combine the measured power +/// with e.g. `reduce_stereo`. +/// +/// # Instantaneous loudness +/// +/// The instantaneous loudness is the power over a 400ms window, so you can +/// average four 100ms windows. No special functionality is implemented to help +/// with that at this time. ([Pull requests would be accepted.][contribute]) +/// +/// # Momentary loudness +/// +/// The momentary loudness is the power over a 3-second window, so you can +/// average thirty 100ms windows. No special functionality is implemented to +/// help with that at this time. ([Pull requests would be accepted.][contribute]) +/// +/// # Integrated loudness +/// +/// Use `gated_mean` to perform an integrated loudness measurement: +/// +/// ```ignore +/// # use std::iter; +/// # use bs1770::{ChannelLoudnessMeter, gated_mean}; +/// # let sample_rate_hz = 44_100; +/// # let samples_per_100ms = sample_rate_hz / 10; +/// # let mut meter = ChannelLoudnessMeter::new(sample_rate_hz); +/// # meter.push((0..44_100).map(|i| (i as f32 * 0.01).sin())); +/// let integrated_loudness_lkfs = gated_mean(meter.as_100ms_windows()) +/// .unwrap_or(bs1770::Power(0.0)) +/// .loudness_lkfs(); +/// ``` +/// +/// [contribute]: https://github.com/ruuda/bs1770/blob/master/CONTRIBUTING.md +#[derive(Clone)] +pub struct ChannelLoudnessMeter { + /// The number of samples that fit in 100ms of audio. + samples_per_100ms: u32, + + /// Stage 1 filter (head effects, high shelf). + filter_stage1: Filter, + + /// Stage 2 filter (high-pass). + filter_stage2: Filter, + + /// Sum of the squares over non-overlapping windows of 100ms. + windows: Windows100ms>, + + /// The number of samples in the current unfinished window. + count: u32, + + /// The sum of the squares of the samples in the current unfinished window. + square_sum: Sum, +} + +impl ChannelLoudnessMeter { + /// Construct a new loudness meter for the given sample rate. + pub fn new(sample_rate_hz: u32) -> ChannelLoudnessMeter { + ChannelLoudnessMeter { + samples_per_100ms: sample_rate_hz / 10, + filter_stage1: Filter::high_shelf(sample_rate_hz as f32), + filter_stage2: Filter::high_pass(sample_rate_hz as f32), + windows: Windows100ms::new(), + count: 0, + square_sum: Sum::zero(), + } + } + + /// Feed input samples for loudness analysis. + /// + /// # Full scale + /// + /// Full scale for the input samples is the interval [-1.0, 1.0]. If your + /// input consists of signed integer samples, you can convert as follows: + /// + /// ```ignore + /// # let mut meter = bs1770::ChannelLoudnessMeter::new(44_100); + /// # let bits_per_sample = 16_usize; + /// # let samples = &[0_i16]; + /// // Note that the maximum amplitude is `1 << (bits_per_sample - 1)`, + /// // one bit is the sign bit. + /// let normalizer = 1.0 / (1_u64 << (bits_per_sample - 1)) as f32; + /// meter.push(samples.iter().map(|&s| s as f32 * normalizer)); + /// ``` + /// + /// # Repeated calls + /// + /// You can call `push` multiple times to feed multiple batches of samples. + /// This is equivalent to feeding a single chained iterator. The leftover of + /// samples that did not fill a full 100ms window is not discarded: + /// + /// ```ignore + /// # use std::iter; + /// # use bs1770::ChannelLoudnessMeter; + /// let sample_rate_hz = 44_100; + /// let samples_per_100ms = sample_rate_hz / 10; + /// let mut meter = ChannelLoudnessMeter::new(sample_rate_hz); + /// + /// meter.push(iter::repeat(0.0).take(samples_per_100ms as usize - 1)); + /// assert_eq!(meter.as_100ms_windows().len(), 0); + /// + /// meter.push(iter::once(0.0)); + /// assert_eq!(meter.as_100ms_windows().len(), 1); + /// ``` + pub fn push>(&mut self, samples: I) { + let normalizer = 1.0 / self.samples_per_100ms as f32; + + // LLVM, if you could go ahead and inline those apply calls, and then + // unroll and vectorize the loop, that'd be terrific. + for x in samples { + let y = self.filter_stage1.apply(x); + let z = self.filter_stage2.apply(y); + + self.square_sum.add(z * z); + self.count += 1; + + // TODO: Should this branch be marked cold? + if self.count == self.samples_per_100ms { + let mean_squares = Power(self.square_sum.sum * normalizer); + self.windows.inner.push(mean_squares); + // We intentionally do not reset the residue. That way, leftover + // energy from this window is not lost, so for the file overall, + // the sum remains more accurate. + self.square_sum.sum = 0.0; + self.count = 0; + } + } + } + + /// Return a reference to the 100ms windows analyzed so far. + pub fn as_100ms_windows(&self) -> Windows100ms<&[Power]> { + self.windows.as_ref() + } + + /// Return all 100ms windows analyzed so far. + pub fn into_100ms_windows(self) -> Windows100ms> { + self.windows + } +} + +/// Combine power for multiple channels by taking a weighted sum. +/// +/// Note that BS.1770-4 defines power for a multi-channel signal as a weighted +/// sum over channels which is not normalized. This means that a stereo signal +/// is inherently louder than a mono signal. For a mono signal played back on +/// stereo speakers, you should therefore still apply `reduce_stereo`, passing +/// in the same signal for both channels. +pub fn reduce_stereo( + left: Windows100ms<&[Power]>, + right: Windows100ms<&[Power]>, +) -> Windows100ms> { + assert_eq!( + left.len(), + right.len(), + "Channels must have the same length." + ); + let mut result = Vec::with_capacity(left.len()); + for (l, r) in left.inner.iter().zip(right.inner) { + result.push(Power(l.0 + r.0)); + } + Windows100ms { inner: result } +} + +/// In-place version of `reduce_stereo` that stores the result in the former left channel. +pub fn reduce_stereo_in_place(left: Windows100ms<&mut [Power]>, right: Windows100ms<&[Power]>) { + assert_eq!( + left.len(), + right.len(), + "Channels must have the same length." + ); + for (l, r) in left.inner.iter_mut().zip(right.inner) { + l.0 += r.0; + } +} + +/// Perform gating and averaging for a BS.1770-4 integrated loudness measurement. +/// +/// The integrated loudness measurement is not just the average power over the +/// entire signal. BS.1770-4 defines two stages of gating that exclude +/// parts of the signal, to ensure that silent parts do not contribute to the +/// loudness measurement. This function performs that gating, and returns the +/// average power over the windows that were not excluded. +/// +/// The result of this function is the integrated loudness measurement. +/// +/// When no signal remains after applying the gate, this function returns +/// `None`. In particular, this happens when all of the signal is softer than +/// -70 LKFS, including a signal that consists of pure silence. +pub fn gated_mean(windows_100ms: Windows100ms<&[Power]>) -> Option { + let mut gating_blocks = Vec::with_capacity(windows_100ms.len()); + + // Stage 1: an absolute threshold of -70 LKFS. (Equation 6, p.6.) + let absolute_threshold = Power::from_lkfs(-70.0); + + // Iterate over all 400ms windows. + for window in windows_100ms.inner.windows(4) { + // Note that the sum over channels has already been performed at this point. + let gating_block_power = Power(0.25 * window.iter().map(|mean| mean.0).sum::()); + + if gating_block_power > absolute_threshold { + gating_blocks.push(gating_block_power); + } + } + + if gating_blocks.is_empty() { + return None; + } + + // Compute the loudness after applying the absolute gate, in order to + // determine the threshold for the relative gate. + let mut sum_power = Sum::zero(); + for &gating_block_power in &gating_blocks { + sum_power.add(gating_block_power.0); + } + let absolute_gated_power = Power(sum_power.sum / (gating_blocks.len() as f32)); + + // Stage 2: Apply the relative gate. + let relative_threshold = Power::from_lkfs(absolute_gated_power.loudness_lkfs() - 10.0); + let mut sum_power = Sum::zero(); + let mut n_blocks = 0_usize; + for &gating_block_power in &gating_blocks { + if gating_block_power > relative_threshold { + sum_power.add(gating_block_power.0); + n_blocks += 1; + } + } + + if n_blocks == 0 { + return None; + } + + let relative_gated_power = Power(sum_power.sum / n_blocks as f32); + Some(relative_gated_power) +} diff --git a/tts/src/lib.rs b/tts/src/lib.rs index 80aad1cc..26c3be5b 100644 --- a/tts/src/lib.rs +++ b/tts/src/lib.rs @@ -3,12 +3,15 @@ use std::error; pub use aws::{AwsConfig, Region as AwsRegion}; pub use azure::AzureConfig; pub use gcloud::GCloudConfig; +pub use parler::ParlerConfig; #[cfg(target_os = "windows")] pub use win::WinConfig; mod aws; mod azure; +mod bs1770; mod gcloud; +mod parler; #[cfg(target_os = "windows")] mod win; @@ -19,6 +22,7 @@ pub enum TtsConfig { GCloud(gcloud::GCloudConfig), #[cfg(target_os = "windows")] Win(win::WinConfig), + Parler(parler::ParlerConfig), } /// Synthesize the `text` to speech. Returns a vec of opus frames. @@ -32,6 +36,7 @@ pub async fn synthesize( TtsConfig::GCloud(config) => gcloud::synthesize(text, config).await?, #[cfg(target_os = "windows")] TtsConfig::Win(config) => win::synthesize(text, config).await?, + TtsConfig::Parler(config) => parler::synthesize(text, config).await?, }) } diff --git a/tts/src/parler.rs b/tts/src/parler.rs new file mode 100644 index 00000000..5b8a27ba --- /dev/null +++ b/tts/src/parler.rs @@ -0,0 +1,220 @@ +use std::sync::Mutex; + +use candle_core::{DType, Device, IndexOp as _, Tensor}; +use candle_nn::VarBuilder; +use candle_transformers::models::parler_tts::{Config, Model}; +use rubato::Resampler; +use tokenizers::Tokenizer; +use tokio::sync::OnceCell; + +#[derive(Debug)] +pub struct ParlerConfig { + pub speaker: String, +} + +static MODEL: OnceCell, Tokenizer, Device)>> = OnceCell::const_new(); + +pub async fn synthesize(prompt: &str, config: &ParlerConfig) -> Result>, ParlerError> { + let (model, tokenizer, device) = get_model().await.ok_or(ParlerError::NoModel)?; + + let description_tokens = tokenizer + .encode(config.speaker.as_str(), true)? + .get_ids() + .to_vec(); + let description_tokens = Tensor::new(description_tokens, device)?.unsqueeze(0)?; + let prompt_tokens = tokenizer.encode(prompt, true)?.get_ids().to_vec(); + let prompt_tokens = Tensor::new(prompt_tokens, device)?.unsqueeze(0)?; + let lp = candle_transformers::generation::LogitsProcessor::new(0, Some(0.7), None); + + println!("generating speech ..."); + let start = std::time::Instant::now(); + + let data = tokio::task::spawn_blocking(move || { + let mut model = model.lock().unwrap(); + let codes = model.generate(&prompt_tokens, &description_tokens, lp, 512)?; + let codes = codes.to_dtype(DType::I64)?; + codes.save_safetensors("codes", "out.safetensors")?; + let codes = codes.unsqueeze(0)?; + let samples = model + .audio_encoder + .decode_codes(&codes.to_device(device)?)?; + + drop(model); // release lock + + let samples = samples.i((0, 0))?; + let samples = normalize_loudness(&samples, 44_100, true)?; + let samples = samples.to_vec1::()?; + log::debug!("generated speech in {:?}", start.elapsed()); + + // Resample audio to 16kHz + let mut resampler = rubato::FftFixedInOut::::new(44_100, 16_000, 1024, 1)?; + let mut data = Vec::with_capacity(resampler.output_frames_max() * 2); + let mut buffer = vec![vec![0.0f32; resampler.output_frames_next()]]; + let chunk_size = resampler.input_frames_next(); + for chunk in samples.chunks(chunk_size) { + log::debug!("in.len={} out.len={}", chunk.len(), buffer[0].len()); + let (_, out_len) = if chunk.len() < chunk_size { + resampler.process_partial_into_buffer(Some(&[chunk]), &mut buffer, None)? + } else { + resampler.process_into_buffer(&[chunk], &mut buffer, None)? + }; + for sample in &buffer[0][..out_len] { + let sample = (sample.clamp(-1.0, 1.0) * 32767.0) as i16; + data.extend(sample.to_le_bytes()); + } + } + + let buffer = buffer.remove(0); + for sample in buffer { + let sample = (sample.clamp(-1.0, 1.0) * 32767.0) as i16; + data.extend(sample.to_le_bytes()); + } + + Ok::<_, ParlerError>(data) + }) + .await + .unwrap()?; + + Ok(crate::wav_to_opus(data.into()).await?) +} + +pub async fn get_model<'a>() -> Option<&'a (Mutex, Tokenizer, Device)> { + MODEL + .get_or_init(|| async { + match load_model().await { + Ok((model, tokenizer, device)) => Some((Mutex::new(model), tokenizer, device)), + Err(err) => { + log::error!("failed to load parler model: {err}"); + None + } + } + }) + .await + .as_ref() +} + +async fn load_model() -> Result<(Model, Tokenizer, Device), ParlerError> { + // Make other options configurable? + // "parler-tts/parler-tts-large-v1" + // "parler-tts/parler-tts-mini-v1" + // "parler-tts/parler-tts-mini-expresso" + let model_id = "parler-tts/parler-tts-large-v1"; + let revision = "main".to_string(); + log::debug!("loading {model_id} model files ..."); + + let start = std::time::Instant::now(); + let api = candle_hf_hub::api::tokio::Api::new()?; + + let repo = api.repo(candle_hf_hub::Repo::with_revision( + model_id.to_string(), + candle_hf_hub::RepoType::Model, + revision, + )); + let model_files = match model_id { + "parler-tts/parler-tts-large-v1" => { + hub_load_safetensors(&repo, "model.safetensors.index.json").await? + } + _ => vec![repo.get("model.safetensors").await?], + }; + let config = repo.get("config.json").await?; + let tokenizer = repo.get("tokenizer.json").await?; + log::debug!("loaded {model_id} model files in {:?}", start.elapsed()); + + log::debug!("loading {model_id} model ..."); + let start = std::time::Instant::now(); + let tokenizer = Tokenizer::from_file(tokenizer)?; + let device = Device::new_cuda(0)?; + let vb = tokio::task::spawn_blocking({ + let device = device.clone(); + move || unsafe { VarBuilder::from_mmaped_safetensors(&model_files, DType::F32, &device) } + }) + .await + .unwrap()?; + let config: Config = + serde_json::from_slice(&tokio::fs::read(config).await?).map_err(ParlerError::JsonConfig)?; + let model = Model::new(&config, vb)?; + log::debug!("loaded {model_id} model in {:?}", start.elapsed()); + + Ok((model, tokenizer, device)) +} + +#[derive(Debug, thiserror::Error)] +pub enum ParlerError { + #[error(transparent)] + Huggingface(#[from] candle_hf_hub::api::tokio::ApiError), + #[error(transparent)] + Candle(#[from] candle_core::Error), + #[error(transparent)] + Tokenizer(#[from] tokenizers::Error), + #[error("no model (loading the model failed previously)")] + NoModel, + #[error("failed to parse model json config")] + JsonConfig(#[source] serde_json::Error), + #[error(transparent)] + Io(#[from] std::io::Error), + #[error("failed to encode audio as opus")] + Opus(#[from] audiopus::Error), + #[error("failed to resample audio to 16kHz: {0}")] + Resample(#[from] rubato::ResampleError), + #[error(transparent)] + ResamplerConstruction(#[from] rubato::ResamplerConstructionError), +} + +/// Loads the safetensors files for a model from the hub based on a json index file. +async fn hub_load_safetensors( + repo: &candle_hf_hub::api::tokio::ApiRepo, + json_file: &str, +) -> Result, candle_core::Error> { + let json_file = repo + .get(json_file) + .await + .map_err(candle_core::Error::wrap)?; + let json_file = std::fs::File::open(json_file)?; + let json: serde_json::Value = + serde_json::from_reader(&json_file).map_err(candle_core::Error::wrap)?; + let weight_map = match json.get("weight_map") { + None => candle_core::bail!("no weight map in {json_file:?}"), + Some(serde_json::Value::Object(map)) => map, + Some(_) => candle_core::bail!("weight map in {json_file:?} is not a map"), + }; + let mut files = std::collections::HashSet::new(); + for value in weight_map.values() { + if let Some(file) = value.as_str() { + files.insert(file.to_string()); + } + } + let mut safetensors_files = Vec::with_capacity(files.len()); + for file in files { + safetensors_files.push(repo.get(&file).await.map_err(candle_core::Error::wrap)?); + } + + Ok(safetensors_files) +} + +// https://github.com/facebookresearch/audiocraft/blob/69fea8b290ad1b4b40d28f92d1dfc0ab01dbab85/audiocraft/data/audio_utils.py#L57 +pub fn normalize_loudness( + wav: &Tensor, + sample_rate: u32, + loudness_compressor: bool, +) -> Result { + let energy = wav.sqr()?.mean_all()?.sqrt()?.to_vec0::()?; + if energy < 2e-3 { + return Ok(wav.clone()); + } + let wav_array = wav.to_vec1::()?; + let mut meter = crate::bs1770::ChannelLoudnessMeter::new(sample_rate); + meter.push(wav_array.into_iter()); + let power = meter.as_100ms_windows(); + let loudness = match crate::bs1770::gated_mean(power) { + None => return Ok(wav.clone()), + Some(gp) => gp.loudness_lkfs() as f64, + }; + let delta_loudness = -14. - loudness; + let gain = 10f64.powf(delta_loudness / 20.); + let wav = (wav * gain)?; + if loudness_compressor { + wav.tanh() + } else { + Ok(wav) + } +}