From 26c774098da4dd9f3f06385156b01b249f632fae Mon Sep 17 00:00:00 2001 From: "lyuxiang.lx" Date: Tue, 17 Dec 2024 14:03:35 +0800 Subject: [PATCH] add text_frontend arg --- README.md | 1 + cosyvoice/cli/cosyvoice.py | 24 ++++++++++++------------ cosyvoice/cli/frontend.py | 8 +++----- 3 files changed, 16 insertions(+), 17 deletions(-) diff --git a/README.md b/README.md index ce4ce81..68be2ca 100644 --- a/README.md +++ b/README.md @@ -137,6 +137,7 @@ import torchaudio ```python cosyvoice = CosyVoice2('pretrained_models/CosyVoice2-0.5B', load_jit=True, load_onnx=False, load_trt=False) +# NOTE if you want to reproduce the results on https://funaudiollm.github.io/cosyvoice2, please add text_frontend=False during inference # zero_shot usage prompt_speech_16k = load_wav('zero_shot_prompt.wav', 16000) for i, j in enumerate(cosyvoice.inference_zero_shot('收到好友从远方寄来的生日礼物，那份意外的惊喜与深深的祝福让我心中充满了甜蜜的快乐，笑容如花儿般绽放。', '希望你以后能够做的比我还好呦。', prompt_speech_16k, stream=False)): diff --git a/cosyvoice/cli/cosyvoice.py b/cosyvoice/cli/cosyvoice.py index c78f4b3..bd2a520 100644 --- a/cosyvoice/cli/cosyvoice.py +++ b/cosyvoice/cli/cosyvoice.py @@ -59,8 +59,8 @@ def list_avaliable_spks(self): spks = list(self.frontend.spk2info.keys()) return spks - def inference_sft(self, tts_text, spk_id, stream=False, speed=1.0): - for i in tqdm(self.frontend.text_normalize(tts_text, split=True)): + def inference_sft(self, tts_text, spk_id, stream=False, speed=1.0, text_frontend=True): + for i in tqdm(self.frontend.text_normalize(tts_text, split=True, text_frontend=text_frontend)): model_input = self.frontend.frontend_sft(i, spk_id) start_time = time.time() logging.info('synthesis text {}'.format(i)) @@ -70,9 +70,9 @@ def inference_sft(self, tts_text, spk_id, stream=False, speed=1.0): yield model_output start_time = time.time() - def inference_zero_shot(self, tts_text, prompt_text, prompt_speech_16k, stream=False, speed=1.0): - prompt_text = self.frontend.text_normalize(prompt_text, split=False) - for i in tqdm(self.frontend.text_normalize(tts_text, split=True)): + def inference_zero_shot(self, tts_text, prompt_text, prompt_speech_16k, stream=False, speed=1.0, text_frontend=True): + prompt_text = self.frontend.text_normalize(prompt_text, split=False, text_frontend=text_frontend) + for i in tqdm(self.frontend.text_normalize(tts_text, split=True, text_frontend=text_frontend)): if len(i) < 0.5 * len(prompt_text): logging.warning('synthesis text {} too short than prompt text {}, this may lead to bad performance'.format(i, prompt_text)) model_input = self.frontend.frontend_zero_shot(i, prompt_text, prompt_speech_16k, self.sample_rate) @@ -84,10 +84,10 @@ def inference_zero_shot(self, tts_text, prompt_text, prompt_speech_16k, stream=F yield model_output start_time = time.time() - def inference_cross_lingual(self, tts_text, prompt_speech_16k, stream=False, speed=1.0): + def inference_cross_lingual(self, tts_text, prompt_speech_16k, stream=False, speed=1.0, text_frontend=True): if self.frontend.instruct is True and isinstance(self.model, CosyVoiceModel): raise ValueError('{} do not support cross_lingual inference'.format(self.model_dir)) - for i in tqdm(self.frontend.text_normalize(tts_text, split=True)): + for i in tqdm(self.frontend.text_normalize(tts_text, split=True, text_frontend=text_frontend)): model_input = self.frontend.frontend_cross_lingual(i, prompt_speech_16k, self.sample_rate) start_time = time.time() logging.info('synthesis text {}'.format(i)) @@ -97,12 +97,12 @@ def inference_cross_lingual(self, tts_text, prompt_speech_16k, stream=False, spe yield model_output start_time = time.time() - def inference_instruct(self, tts_text, spk_id, instruct_text, stream=False, speed=1.0): + def inference_instruct(self, tts_text, spk_id, instruct_text, stream=False, speed=1.0, text_frontend=True): assert isinstance(self.model, CosyVoiceModel) if self.frontend.instruct is False: raise ValueError('{} do not support instruct inference'.format(self.model_dir)) - instruct_text = self.frontend.text_normalize(instruct_text, split=False) - for i in tqdm(self.frontend.text_normalize(tts_text, split=True)): + instruct_text = self.frontend.text_normalize(instruct_text, split=False, text_frontend=text_frontend) + for i in tqdm(self.frontend.text_normalize(tts_text, split=True, text_frontend=text_frontend)): model_input = self.frontend.frontend_instruct(i, spk_id, instruct_text) start_time = time.time() logging.info('synthesis text {}'.format(i)) @@ -112,9 +112,9 @@ def inference_instruct(self, tts_text, spk_id, instruct_text, stream=False, spee yield model_output start_time = time.time() - def inference_instruct2(self, tts_text, instruct_text, prompt_speech_16k, stream=False, speed=1.0): + def inference_instruct2(self, tts_text, instruct_text, prompt_speech_16k, stream=False, speed=1.0, text_frontend=True): assert isinstance(self.model, CosyVoice2Model) - for i in tqdm(self.frontend.text_normalize(tts_text, split=True)): + for i in tqdm(self.frontend.text_normalize(tts_text, split=True, text_frontend=text_frontend)): model_input = self.frontend.frontend_instruct2(i, instruct_text, prompt_speech_16k, self.sample_rate) start_time = time.time() logging.info('synthesis text {}'.format(i)) diff --git a/cosyvoice/cli/frontend.py b/cosyvoice/cli/frontend.py index 228ec41..1c1c64c 100644 --- a/cosyvoice/cli/frontend.py +++ b/cosyvoice/cli/frontend.py @@ -107,12 +107,10 @@ def _extract_speech_feat(self, speech): speech_feat_len = torch.tensor([speech_feat.shape[1]], dtype=torch.int32).to(self.device) return speech_feat, speech_feat_len - def text_normalize(self, text, split=True): + def text_normalize(self, text, split=True, text_frontend=True): + if text_frontend is False: + return [text] if split is True else text text = text.strip() - # NOTE(lyuxiang.lx) move this judgement into ttsfrd in the future - for token in self.tokenizer.special_tokens['additional_special_tokens']: - if token in text: - return text if split is False else [text] if contains_chinese(text): if self.use_ttsfrd: texts = [i["text"] for i in json.loads(self.frd.do_voicegen_frd(text))["sentences"]]