Skip to content

Commit

Permalink
Merge pull request #735 from FunAudioLLM/dev/lyuxiang.lx
Browse files Browse the repository at this point in the history
add text_frontend arg
  • Loading branch information
aluminumbox authored Dec 17, 2024
2 parents 81edc83 + 26c7740 commit 8a88446
Show file tree
Hide file tree
Showing 3 changed files with 16 additions and 17 deletions.
1 change: 1 addition & 0 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -137,6 +137,7 @@ import torchaudio
```python
cosyvoice = CosyVoice2('pretrained_models/CosyVoice2-0.5B', load_jit=True, load_onnx=False, load_trt=False)

# NOTE if you want to reproduce the results on https://funaudiollm.github.io/cosyvoice2, please add text_frontend=False during inference
# zero_shot usage
prompt_speech_16k = load_wav('zero_shot_prompt.wav', 16000)
for i, j in enumerate(cosyvoice.inference_zero_shot('收到好友从远方寄来的生日礼物,那份意外的惊喜与深深的祝福让我心中充满了甜蜜的快乐,笑容如花儿般绽放。', '希望你以后能够做的比我还好呦。', prompt_speech_16k, stream=False)):
Expand Down
24 changes: 12 additions & 12 deletions cosyvoice/cli/cosyvoice.py
Original file line number Diff line number Diff line change
Expand Up @@ -59,8 +59,8 @@ def list_avaliable_spks(self):
spks = list(self.frontend.spk2info.keys())
return spks

def inference_sft(self, tts_text, spk_id, stream=False, speed=1.0):
for i in tqdm(self.frontend.text_normalize(tts_text, split=True)):
def inference_sft(self, tts_text, spk_id, stream=False, speed=1.0, text_frontend=True):
for i in tqdm(self.frontend.text_normalize(tts_text, split=True, text_frontend=text_frontend)):
model_input = self.frontend.frontend_sft(i, spk_id)
start_time = time.time()
logging.info('synthesis text {}'.format(i))
Expand All @@ -70,9 +70,9 @@ def inference_sft(self, tts_text, spk_id, stream=False, speed=1.0):
yield model_output
start_time = time.time()

def inference_zero_shot(self, tts_text, prompt_text, prompt_speech_16k, stream=False, speed=1.0):
prompt_text = self.frontend.text_normalize(prompt_text, split=False)
for i in tqdm(self.frontend.text_normalize(tts_text, split=True)):
def inference_zero_shot(self, tts_text, prompt_text, prompt_speech_16k, stream=False, speed=1.0, text_frontend=True):
prompt_text = self.frontend.text_normalize(prompt_text, split=False, text_frontend=text_frontend)
for i in tqdm(self.frontend.text_normalize(tts_text, split=True, text_frontend=text_frontend)):
if len(i) < 0.5 * len(prompt_text):
logging.warning('synthesis text {} too short than prompt text {}, this may lead to bad performance'.format(i, prompt_text))
model_input = self.frontend.frontend_zero_shot(i, prompt_text, prompt_speech_16k, self.sample_rate)
Expand All @@ -84,10 +84,10 @@ def inference_zero_shot(self, tts_text, prompt_text, prompt_speech_16k, stream=F
yield model_output
start_time = time.time()

def inference_cross_lingual(self, tts_text, prompt_speech_16k, stream=False, speed=1.0):
def inference_cross_lingual(self, tts_text, prompt_speech_16k, stream=False, speed=1.0, text_frontend=True):
if self.frontend.instruct is True and isinstance(self.model, CosyVoiceModel):
raise ValueError('{} do not support cross_lingual inference'.format(self.model_dir))
for i in tqdm(self.frontend.text_normalize(tts_text, split=True)):
for i in tqdm(self.frontend.text_normalize(tts_text, split=True, text_frontend=text_frontend)):
model_input = self.frontend.frontend_cross_lingual(i, prompt_speech_16k, self.sample_rate)
start_time = time.time()
logging.info('synthesis text {}'.format(i))
Expand All @@ -97,12 +97,12 @@ def inference_cross_lingual(self, tts_text, prompt_speech_16k, stream=False, spe
yield model_output
start_time = time.time()

def inference_instruct(self, tts_text, spk_id, instruct_text, stream=False, speed=1.0):
def inference_instruct(self, tts_text, spk_id, instruct_text, stream=False, speed=1.0, text_frontend=True):
assert isinstance(self.model, CosyVoiceModel)
if self.frontend.instruct is False:
raise ValueError('{} do not support instruct inference'.format(self.model_dir))
instruct_text = self.frontend.text_normalize(instruct_text, split=False)
for i in tqdm(self.frontend.text_normalize(tts_text, split=True)):
instruct_text = self.frontend.text_normalize(instruct_text, split=False, text_frontend=text_frontend)
for i in tqdm(self.frontend.text_normalize(tts_text, split=True, text_frontend=text_frontend)):
model_input = self.frontend.frontend_instruct(i, spk_id, instruct_text)
start_time = time.time()
logging.info('synthesis text {}'.format(i))
Expand All @@ -112,9 +112,9 @@ def inference_instruct(self, tts_text, spk_id, instruct_text, stream=False, spee
yield model_output
start_time = time.time()

def inference_instruct2(self, tts_text, instruct_text, prompt_speech_16k, stream=False, speed=1.0):
def inference_instruct2(self, tts_text, instruct_text, prompt_speech_16k, stream=False, speed=1.0, text_frontend=True):
assert isinstance(self.model, CosyVoice2Model)
for i in tqdm(self.frontend.text_normalize(tts_text, split=True)):
for i in tqdm(self.frontend.text_normalize(tts_text, split=True, text_frontend=text_frontend)):
model_input = self.frontend.frontend_instruct2(i, instruct_text, prompt_speech_16k, self.sample_rate)
start_time = time.time()
logging.info('synthesis text {}'.format(i))
Expand Down
8 changes: 3 additions & 5 deletions cosyvoice/cli/frontend.py
Original file line number Diff line number Diff line change
Expand Up @@ -107,12 +107,10 @@ def _extract_speech_feat(self, speech):
speech_feat_len = torch.tensor([speech_feat.shape[1]], dtype=torch.int32).to(self.device)
return speech_feat, speech_feat_len

def text_normalize(self, text, split=True):
def text_normalize(self, text, split=True, text_frontend=True):
if text_frontend is False:
return [text] if split is True else text
text = text.strip()
# NOTE(lyuxiang.lx) move this judgement into ttsfrd in the future
for token in self.tokenizer.special_tokens['additional_special_tokens']:
if token in text:
return text if split is False else [text]
if contains_chinese(text):
if self.use_ttsfrd:
texts = [i["text"] for i in json.loads(self.frd.do_voicegen_frd(text))["sentences"]]
Expand Down

0 comments on commit 8a88446

Please sign in to comment.