import torch
import librosa
import commons
import utils
from models import SynthesizerTrn
from text import text_to_sequence
import numpy as np
from mel_processing import spectrogram_torch
import gradio as gr
from text.cleaners import shanghainese_cleaners
from transformers import AutoModel, AutoTokenizer
from TTS.api import TTS
tts = TTS("tts_models/zh-CN/baker/tacotron2-DDC-GST")
tts1 = TTS(model_name="tts_models/multilingual/multi-dataset/your_tts", progress_bar=False, gpu=True)
import torchaudio
from speechbrain.pretrained import SpectralMaskEnhancement
enhance_model = SpectralMaskEnhancement.from_hparams(
source="speechbrain/metricgan-plus-voicebank",
savedir="pretrained_models/metricgan-plus-voicebank",
run_opts={"device":"cuda"},
)
from denoiser import pretrained
from denoiser.dsp import convert_audio
model1 = pretrained.dns64().cuda()
tokenizer = AutoTokenizer.from_pretrained("THUDM/chatglm-6b", trust_remote_code=True)
model = AutoModel.from_pretrained("THUDM/chatglm-6b", trust_remote_code=True).half().cuda()
model = model.eval()
def predict(input, history=None):
if history is None:
history = []
response, history = model.chat(tokenizer, input, history)
return history, history, response
def chinese(text_cn, upload1, VoiceMicrophone1):
if upload1 is not None:
tts.tts_with_vc_to_file(
" ".join(text_cn.split()) + "。",
speaker_wav=upload1,
file_path="output0.wav"
)
else:
tts.tts_with_vc_to_file(
" ".join(text_cn.split()) + "。",
speaker_wav=VoiceMicrophone1,
file_path="output0.wav"
)
noisy = enhance_model.load_audio(
"output0.wav"
).unsqueeze(0)
enhanced = enhance_model.enhance_batch(noisy, lengths=torch.tensor([1.]))
torchaudio.save("enhanced.wav", enhanced.cpu(), 16000)
return "enhanced.wav"
def english(text_en, upload, VoiceMicrophone):
if upload is not None:
tts1.tts_to_file(text_en.strip(), speaker_wav = upload, language="en", file_path="output.wav")
else:
tts1.tts_to_file(text_en.strip(), speaker_wav = VoiceMicrophone, language="en", file_path="output.wav")
wav, sr = torchaudio.load("output.wav")
wav = convert_audio(wav.cuda(), sr, model1.sample_rate, model1.chin)
with torch.no_grad():
denoised = model1(wav[None])[0]
torchaudio.save("denoise.wav", denoised.data.cpu(), model1.sample_rate)
noisy = enhance_model.load_audio(
"denoise.wav"
).unsqueeze(0)
enhanced = enhance_model.enhance_batch(noisy, lengths=torch.tensor([1.]))
torchaudio.save("enhanced.wav", enhanced.cpu(), 16000)
return "enhanced.wav"
def clean_text(text,ipa_input):
if ipa_input:
return shanghainese_cleaners(text)
return text
def get_text(text, hps, cleaned=False):
if cleaned:
text_norm = text_to_sequence(text, hps.symbols, [])
else:
text_norm = text_to_sequence(text, hps.symbols, hps.data.text_cleaners)
if hps.data.add_blank:
text_norm = commons.intersperse(text_norm, 0)
text_norm = torch.LongTensor(text_norm)
return text_norm
def speech_synthesize(text, cleaned, length_scale):
text=text.replace('\n','')
print(text)
stn_tst = get_text(text, hps_ms, cleaned)
with torch.no_grad():
x_tst = stn_tst.unsqueeze(0)
x_tst_lengths = torch.LongTensor([stn_tst.size(0)])
sid = torch.LongTensor([0])
audio = net_g_ms.infer(x_tst, x_tst_lengths, sid=sid, noise_scale=0.667, noise_scale_w=0.8, length_scale=length_scale)[0][0,0].data.cpu().float().numpy()
return (hps_ms.data.sampling_rate, audio)
hps_ms = utils.get_hparams_from_file('model/config.json')
n_speakers = hps_ms.data.n_speakers
n_symbols = len(hps_ms.symbols)
speakers = hps_ms.speakers
net_g_ms = SynthesizerTrn(
n_symbols,
hps_ms.data.filter_length // 2 + 1,
hps_ms.train.segment_size // hps_ms.data.hop_length,
n_speakers=n_speakers,
**hps_ms.model)
_ = net_g_ms.eval()
utils.load_checkpoint('model/model.pth', net_g_ms)
with gr.Blocks() as demo:
gr.Markdown(
""" # 🥳💬💕 - TalktoAI，随时随地，谈天说地！
### 🤖 - 让有人文关怀的AI造福每一个人！AI向善，文明璀璨！TalktoAI - Enable the future！
"""
)
state = gr.State([])
chatbot = gr.Chatbot([], elem_id="chatbot").style(height=300)
res = gr.Textbox(lines=1, placeholder="最新的回答在这里(此内容可编辑，用作声音克隆的文本)", show_label = False).style(container=False)
with gr.Row():
txt = gr.Textbox(label = "说点什么吧(中英皆可)", lines=1)
button = gr.Button("开始对话吧")
txt.submit(predict, [txt, state], [chatbot, state, res])
button.click(predict, [txt, state], [chatbot, state, res])
with gr.Row().style(mobile_collapse=False, equal_height=True):
inp3 = res
inp4 = gr.Audio(source="upload", label = "请上传您喜欢的声音(wav/mp3文件)；长语音(~90s)、女声效果更好", type="filepath")
inp5 = gr.Audio(source="microphone", type="filepath", label = '请用麦克风上传您喜欢的声音，与文件上传二选一即可')
btn1 = gr.Button("用喜欢的声音听一听吧(中文)")
btn2 = gr.Button("用喜欢的声音听一听吧(英文)")
with gr.Row():
out1 = gr.Audio(label="为您合成的专属声音(中文)")
out2 = gr.Audio(label="为您合成的专属声音(英文)")
btn1.click(chinese, [inp3, inp4, inp5], [out1])
btn2.click(english, [inp3, inp4, inp5], [out2])
text_input = res
cleaned_text=gr.Checkbox(label='IPA Input',default=True, visible = False)
length_scale=gr.Slider(0.5,2,1,step=0.1,label='Speaking Speed',interactive=True, visible = False)
with gr.Row().style(mobile_collapse=False, equal_height=True):
tts_button = gr.Button('彩蛋:上海话合成')
audio_output = gr.Audio(label='听一听上海话吧')
cleaned_text.change(clean_text,[text_input,cleaned_text],[text_input])
tts_button.click(speech_synthesize,[text_input,cleaned_text,length_scale],[audio_output])
gr.Markdown(
""" ### 注意❗：请不要输入或生成会对个人以及组织造成侵害的内容，此程序仅供科研、学习及娱乐使用。用户输入或生成的内容与程序开发者无关，请自觉合法合规使用，违反者一切后果自负。
### Model by [ChatGLM-6B](https://huggingface.co/THUDM/chatglm-6b). Thanks to [THUDM](https://github.com/THUDM) and [CjangCjengh](https://github.com/CjangCjengh). Please follow me on [Bilibili](https://space.bilibili.com/501495851?spm_id_from=333.1007.0.0).
"""
)
gr.HTML('''
''')
demo.queue().launch(show_error=True)