AI实践 | 一文简述语音克隆实践

Ubuntu-22.04python 3.7.12pytorch 1.8.0安装ffmpegpip install -r requirements.txt

import osimport sysimport typer
cli = typer.Typer()@cli.command()def launch(port: int = typer.Option(8080, "--port", "-p")) -> None:"""Start a graphical UI server for the opyrator.
The UI is auto-generated from the input- and output-schema of the givenfunction."""# Add the current working directory to the sys path# This is required to resolve the opyrator pathsys.path.append(os.getcwd())
from control.mkgui.base.ui.streamlit_ui import launch_uilaunch_ui(port)if __name__ == "__main__":cli()

STREAMLIT_RUNNER_SNIPPET = """from control.mkgui.base.ui import render_streamlit_ui
import streamlit as st
# TODO: Make it configurable# Page config can only be setup oncest.set_page_config(page_title="MockingBird",page_icon="🧊",layout="wide")render_streamlit_ui()"""
def launch_ui(port: int = 8501) -> None:with NamedTemporaryFile(suffix=".py", mode="w", encoding="utf-8", delete=False) as f:f.write(STREAMLIT_RUNNER_SNIPPET)f.seek(0)
import subprocesspython_path = f'PYTHONPATH="$PYTHONPATH:{getcwd()}"'if system() == "Windows":python_path = f"set PYTHONPATH=%PYTHONPATH%;{getcwd()} &&"subprocess.run(f"""setSTREAMLIT_GLOBAL_SHOW_WARNING_ON_DIRECT_EXECUTION=false""",shell=True,)
subprocess.run(f"""{python_path} "{sys.executable}" -m streamlit run --server.port={port} --server.headless=True --runner.magicEnabled=False --server.maxUploadSize=50 --browser.gatherUsageStats=False {f.name}""",shell=True,)
f.close()unlink(f.name)

# 音频预处理sample_rate = 16000max_mel_frames = 900rescaling_max = 0.9synthesis_batch_size = 16fmax = 7600allow_clipping_in_normalization = True
# 训练tts_schedule = [(2, 1e-3, 10_000, 12),(2, 5e-4, 15_000, 12),(2, 2e-4, 20_000, 12),(2, 1e-4, 30_000, 12),(2, 5e-5, 40_000, 12),(2, 1e-5, 60_000, 12),(2, 5e-6, 160_000, 12),(2, 3e-6, 320_000, 12),(2, 1e-6, 640_000, 12)]

python 3.9.18torch 2.1.1torchaudio 2.1.1TTS 0.20.6gradio 4.5.0

import torchfrom TTS.api import TTS
# Get devicedevice = "cuda" if torch.cuda.is_available() else "cpu"
# List available 🐸TTS modelsprint(TTS().list_models())
# Init TTStts = TTS("tts_models/multilingual/multi-dataset/xtts_v2").to(device)
# Run TTS# ❗ Since this model is multi-lingual voice cloning model, we must set thetarget speaker_wav and language# Text to speech list of amplitude values as outputwav = tts.tts(text="Hello world!", speaker_wav="my/cloning/audio.wav",language="en")
# Text to speech to a filetts.tts_to_file(text="Hello world!", speaker_wav="my/cloning/audio.wav",language="en", file_path="output.wav")

import osimport glob
import gradio as grfrom TTS.tts.configs.xtts_config import XttsConfigfrom TTS.tts.models.xtts import Xttsimport torchimport torchaudio
# DIRROOT_DIR=os.getcwd()print(ROOT_DIR)os.environ['TTS_HOME']=ROOT_DIRTMP_DIR=os.path.join(ROOT_DIR, "static/tmp")VOICE_DIR=os.path.join(ROOT_DIR, "static/voicelist")OUTPUT = os.path.join(ROOT_DIR, "static/tmp/output.wav")RECORD = os.path.join(TMP_DIR, "tmp.wav")
# Get devicedevice = "cuda:1" if torch.cuda.is_available() else "cpu"
print("Loading model...")config = XttsConfig()config.load_json(os.path.join(ROOT_DIR,"tts/tts_models--multilingual--multi￾dataset--xtts_v2/config.json"))
model = Xtts.init_from_config(config)model.load_checkpoint(config,checkpoint_dir=os.path.join(ROOT_DIR,"tts/tts_models--multilingual--multi￾dataset--xtts_v2/"), use_deepspeed=False)model.to(device)model.eval()
"""utilities"""# 从./static/voicelist/得到已有的wav文件def get_wav_to_Dropdown():wavs = glob.glob(f"{VOICE_DIR}/*.wav")result = []for it in wavs:if os.path.getsize(it) > 0:result.append(tuple([os.path.basename(it), it]))return result
# 播放现有音频def display_wav(speaker_wav):return speaker_wav
"""text: 需要被转换的文本speaker_wav: cloned voicerecord_audio: 上传或记录音频，优先使用这部分的音频"""def text_to_speech(text, speaker_wav, record_audio, temperature):if record_audio is None:print("Computing speaker latents...")speaker_wav = os.path.join(VOICE_DIR, speaker_wav)gpt_cond_latent, speaker_embedding =model.get_conditioning_latents(audio_path=[speaker_wav])
print("Inference...")out = model.inference(text,"zh-cn",gpt_cond_latent,speaker_embedding,temperature=temperature, # Add custom parameters here)torchaudio.save(OUTPUT, torch.tensor(out["wav"]).unsqueeze(0), 24000)return OUTPUTelse:sr = record_audio[0]torchaudio.save(RECORD, torch.tensor(record_audio[1]).unsqueeze(0), sr)print("Computing speaker latents...")speaker_wav = os.path.join(VOICE_DIR, RECORD)gpt_cond_latent, speaker_embedding =model.get_conditioning_latents(audio_path=[speaker_wav])
print("Inference...")out = model.inference(text,"zh-cn",gpt_cond_latent,speaker_embedding,temperature=temperature # Add custom parameters here)torchaudio.save(OUTPUT, torch.tensor(out["wav"]).unsqueeze(0), 24000)return OUTPUT
with gr.Blocks() as demo:with gr.Row():# 左侧输入with gr.Column():txt = gr.Textbox(value="你好，我是一个文字转语音测试。", label="Input:",lines=1, placeholder="请输入待语音克隆的文字")temperature = gr.Slider(label="temperature:", minimum=0, maximum=1,value=0.7)with gr.Row():speaker_wavs = gr.Dropdown(label="Speaker wavs:",choices=get_wav_to_Dropdown())btn_display = gr.Button(value="播放已选音频", scale=0.25)display_au = gr.Audio(visible=False, autoplay=True)if speaker_wavs:btn_display.click(display_wav,inputs=[speaker_wavs],outputs=[display_au])record_au = gr.Audio(label="录制或上传语音：", format="wav",show_download_button=True)with gr.Row():clear_btn = gr.ClearButton()gen_btn = gr.Button(value="生成")
# 右侧输出with gr.Column():output_au = gr.Audio(label="语音克隆结果: ", show_download_button=True)gen_btn.click(text_to_speech, inputs=[txt, speaker_wavs, record_au,temperature], outputs=[output_au])

def get_user_data_dir(appname):TTS_HOME = os.environ.get("TTS_HOME")XDG_DATA_HOME = os.environ.get("XDG_DATA_HOME")if TTS_HOME is not None:ans = Path(TTS_HOME).expanduser().resolve(strict=False)elif XDG_DATA_HOME is not None:ans = Path(XDG_DATA_HOME).expanduser().resolve(strict=False)elif sys.platform == "win32":import winreg # pylint: disable=import-outside-toplevel
key = winreg.OpenKey(winreg.HKEY_CURRENT_USER,r"SoftwareMicrosoftWindowsCurrentVersionExplorerShell Folders")dir_, _ = winreg.QueryValueEx(key, "Local AppData")ans = Path(dir_).resolve(strict=False)elif sys.platform == "darwin":ans = Path("~/Library/Application Support/").expanduser()else:ans = Path.home().joinpath(".local/share") # 如果没有设置os.environ['TTS_HOME'],就下载到share中return ans.joinpath(appname)