Upload 4 files

fastfinetuning_kr/VC_inference.py

@@ -0,0 +1,147 @@
+import os
+import numpy as np
+import torch
+from torch import no_grad, LongTensor
+import argparse
+import commons
+from mel_processing import spectrogram_torch
+import utils
+from models import SynthesizerTrn
+import gradio as gr
+import librosa
+import webbrowser
+
+from text import text_to_sequence, _clean_text
+device = "cuda:0" if torch.cuda.is_available() else "cpu"
+import logging
+logging.getLogger("PIL").setLevel(logging.WARNING)
+logging.getLogger("urllib3").setLevel(logging.WARNING)
+logging.getLogger("markdown_it").setLevel(logging.WARNING)
+logging.getLogger("httpx").setLevel(logging.WARNING)
+logging.getLogger("asyncio").setLevel(logging.WARNING)
+
+language_marks = {
+    "Japanese": "",
+    "日本語": "[JA]",
+    "简体中文": "[ZH]",
+    "English": "[EN]",
+    "한국어": "[KO]",
+    "Mix": "",
+}
+lang = ['日本語', '简体中文', 'English', 'Mix','한국어']
+def get_text(text, hps, is_symbol):
+    text_norm = text_to_sequence(text, hps.symbols, [] if is_symbol else hps.data.text_cleaners)
+    if hps.data.add_blank:
+        text_norm = commons.intersperse(text_norm, 0)
+    text_norm = LongTensor(text_norm)
+    return text_norm
+
+def create_tts_fn(model, hps, speaker_ids):
+    def tts_fn(text, speaker, language, speed):
+        if language is not None:
+            text = language_marks[language] + text + language_marks[language]
+        speaker_id = speaker_ids[speaker]
+        stn_tst = get_text(text, hps, False)
+        with no_grad():
+            x_tst = stn_tst.unsqueeze(0).to(device)
+            x_tst_lengths = LongTensor([stn_tst.size(0)]).to(device)
+            sid = LongTensor([speaker_id]).to(device)
+            audio = model.infer(x_tst, x_tst_lengths, sid=sid, noise_scale=.667, noise_scale_w=0.8,
+                                length_scale=1.0 / speed)[0][0, 0].data.cpu().float().numpy()
+        del stn_tst, x_tst, x_tst_lengths, sid
+        return "Success", (hps.data.sampling_rate, audio)
+
+    return tts_fn
+
+def create_vc_fn(model, hps, speaker_ids):
+    def vc_fn(original_speaker, target_speaker, record_audio, upload_audio):
+        input_audio = record_audio if record_audio is not None else upload_audio
+        if input_audio is None:
+            return "You need to record or upload an audio", None
+        sampling_rate, audio = input_audio
+        original_speaker_id = speaker_ids[original_speaker]
+        target_speaker_id = speaker_ids[target_speaker]
+
+        audio = (audio / np.iinfo(audio.dtype).max).astype(np.float32)
+        if len(audio.shape) > 1:
+            audio = librosa.to_mono(audio.transpose(1, 0))
+        if sampling_rate != hps.data.sampling_rate:
+            audio = librosa.resample(audio, orig_sr=sampling_rate, target_sr=hps.data.sampling_rate)
+        with no_grad():
+            y = torch.FloatTensor(audio)
+            y = y / max(-y.min(), y.max()) / 0.99
+            y = y.to(device)
+            y = y.unsqueeze(0)
+            spec = spectrogram_torch(y, hps.data.filter_length,
+                                     hps.data.sampling_rate, hps.data.hop_length, hps.data.win_length,
+                                     center=False).to(device)
+            spec_lengths = LongTensor([spec.size(-1)]).to(device)
+            sid_src = LongTensor([original_speaker_id]).to(device)
+            sid_tgt = LongTensor([target_speaker_id]).to(device)
+            audio = model.voice_conversion(spec, spec_lengths, sid_src=sid_src, sid_tgt=sid_tgt)[0][
+                0, 0].data.cpu().float().numpy()
+        del y, spec, spec_lengths, sid_src, sid_tgt
+        return "Success", (hps.data.sampling_rate, audio)
+
+    return vc_fn
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--model_dir", default="./G_latest.pth", help="directory to your fine-tuned model")
+    parser.add_argument("--config_dir", default="./finetune_speaker.json", help="directory to your model config file")
+    parser.add_argument("--share", default=False, help="make link public (used in colab)")
+
+    args = parser.parse_args()
+    hps = utils.get_hparams_from_file(args.config_dir)
+
+
+    net_g = SynthesizerTrn(
+        len(hps.symbols),
+        hps.data.filter_length // 2 + 1,
+        hps.train.segment_size // hps.data.hop_length,
+        n_speakers=hps.data.n_speakers,
+        **hps.model).to(device)
+    _ = net_g.eval()
+
+    _ = utils.load_checkpoint(args.model_dir, net_g, None)
+    speaker_ids = hps.speakers
+    speakers = list(hps.speakers.keys())
+    tts_fn = create_tts_fn(net_g, hps, speaker_ids)
+    vc_fn = create_vc_fn(net_g, hps, speaker_ids)
+    app = gr.Blocks()
+    with app:
+        with gr.Tab("Text-to-Speech"):
+            with gr.Row():
+                with gr.Column():
+                    textbox = gr.TextArea(label="Text",
+                                          placeholder="Type your sentence here",
+                                          value="こんにちわ。", elem_id=f"tts-input")
+                    # select character
+                    char_dropdown = gr.Dropdown(choices=speakers, value=speakers[0], label='character')
+                    language_dropdown = gr.Dropdown(choices=lang, value=lang[0], label='language')
+                    duration_slider = gr.Slider(minimum=0.1, maximum=5, value=1, step=0.1,
+                                                label='速度 Speed')
+                with gr.Column():
+                    text_output = gr.Textbox(label="Message")
+                    audio_output = gr.Audio(label="Output Audio", elem_id="tts-audio")
+                    btn = gr.Button("Generate!")
+                    btn.click(tts_fn,
+                              inputs=[textbox, char_dropdown, language_dropdown, duration_slider,],
+                              outputs=[text_output, audio_output])
+        with gr.Tab("Voice Conversion"):
+            gr.Markdown("""
+                            录制或上传声音，并选择要转换的音色。
+            """)
+            with gr.Column():
+                record_audio = gr.Audio(label="record your voice", source="microphone")
+                upload_audio = gr.Audio(label="or upload audio here", source="upload")
+                source_speaker = gr.Dropdown(choices=speakers, value=speakers[0], label="source speaker")
+                target_speaker = gr.Dropdown(choices=speakers, value=speakers[0], label="target speaker")
+            with gr.Column():
+                message_box = gr.Textbox(label="Message")
+                converted_audio = gr.Audio(label='converted audio')
+            btn = gr.Button("Convert!")
+            btn.click(vc_fn, inputs=[source_speaker, target_speaker, record_audio, upload_audio],
+                      outputs=[message_box, converted_audio])
+    webbrowser.open("http://127.0.0.1:7860")
+    app.launch(share=args.share)
+

fastfinetuning_kr/long_audio_transcribe.py

@@ -0,0 +1,82 @@
+from moviepy.editor import AudioFileClip
+import whisper
+import os
+import json
+import torchaudio
+import librosa
+import torch
+import argparse
+parent_dir = "./denoised_audio/"
+filelist = list(os.walk(parent_dir))[0][2]
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--languages", default="CJE")
+    parser.add_argument("--whisper_size", default="medium")
+    args = parser.parse_args()
+    if args.languages == "CJE":
+        lang2token = {
+            'zh': "[ZH]",
+            'ja': "[JA]",
+            "en": "[EN]",
+        }
+    elif args.languages == "CJ":
+        lang2token = {
+            'zh': "[ZH]",
+            'ja': "[JA]",
+        }
+    elif args.languages == "C":
+        lang2token = {
+            'zh': "[ZH]",
+        }
+    elif args.languages == "CJKE":
+        lang2token = {
+            'zh': "[ZH]",
+            'ja': "[JA]",
+            "en": "[EN]",
+	"ko": "[KO]",
+        }
+    assert(torch.cuda.is_available()), "Please enable GPU in order to run Whisper!"
+    with open("./configs/finetune_speaker.json", 'r', encoding='utf-8') as f:
+        hps = json.load(f)
+    target_sr = hps['data']['sampling_rate']
+    model = whisper.load_model(args.whisper_size)
+    speaker_annos = []
+    for file in filelist:
+        print(f"transcribing {parent_dir + file}...\n")
+        options = dict(beam_size=5, best_of=5)
+        transcribe_options = dict(task="transcribe", **options)
+        result = model.transcribe(parent_dir + file, word_timestamps=True, **transcribe_options)
+        segments = result["segments"]
+        # result = model.transcribe(parent_dir + file)
+        lang = result['language']
+        if result['language'] not in list(lang2token.keys()):
+            print(f"{lang} not supported, ignoring...\n")
+            continue
+        # segment audio based on segment results
+        character_name = file.rstrip(".wav").split("_")[0]
+        code = file.rstrip(".wav").split("_")[1]
+        if not os.path.exists("./segmented_character_voice/" + character_name):
+            os.mkdir("./segmented_character_voice/" + character_name)
+        wav, sr = torchaudio.load(parent_dir + file, frame_offset=0, num_frames=-1, normalize=True,
+                                  channels_first=True)
+
+        for i, seg in enumerate(result['segments']):
+            start_time = seg['start']
+            end_time = seg['end']
+            text = seg['text']
+            text = lang2token[lang] + text.replace("\n", "") + lang2token[lang]
+            text = text + "\n"
+            wav_seg = wav[:, int(start_time*sr):int(end_time*sr)]
+            wav_seg_name = f"{character_name}_{code}_{i}.wav"
+            savepth = "./segmented_character_voice/" + character_name + "/" + wav_seg_name
+            speaker_annos.append(savepth + "|" + character_name + "|" + text)
+            print(f"Transcribed segment: {speaker_annos[-1]}")
+            # trimmed_wav_seg = librosa.effects.trim(wav_seg.squeeze().numpy())
+            # trimmed_wav_seg = torch.tensor(trimmed_wav_seg[0]).unsqueeze(0)
+            torchaudio.save(savepth, wav_seg, target_sr, channels_first=True)
+    if len(speaker_annos) == 0:
+        print("Warning: no long audios & videos found, this IS expected if you have only uploaded short audios")
+        print("this IS NOT expected if you have uploaded any long audios, videos or video links. Please check your file structure or make sure your audio/video language is supported.")
+    with open("./long_character_anno.txt", 'w', encoding='utf-8') as f:
+        for line in speaker_annos:
+            f.write(line)

fastfinetuning_kr/preprocess_v2.py

@@ -0,0 +1,156 @@
+import os
+import argparse
+import json
+import sys
+sys.setrecursionlimit(500000)  # Fix the error message of RecursionError: maximum recursion depth exceeded while calling a Python object.  You can change the number as you want.
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--add_auxiliary_data", type=bool, help="Whether to add extra data as fine-tuning helper")
+    parser.add_argument("--languages", default="CJE")
+    args = parser.parse_args()
+    if args.languages == "CJE":
+        langs = ["[ZH]", "[JA]", "[EN]"]
+    elif args.languages == "CJ":
+        langs = ["[ZH]", "[JA]"]
+    elif args.languages == "C":
+        langs = ["[ZH]"]
+    elif args.languages == "CJKE":
+        langs = ["[ZH]", "[JA]", "[EN]", "[KO]"]
+    new_annos = []
+    # Source 1: transcribed short audios
+    if os.path.exists("short_character_anno.txt"):
+        with open("short_character_anno.txt", 'r', encoding='utf-8') as f:
+            short_character_anno = f.readlines()
+            new_annos += short_character_anno
+    # Source 2: transcribed long audio segments
+    if os.path.exists("./long_character_anno.txt"):
+        with open("./long_character_anno.txt", 'r', encoding='utf-8') as f:
+            long_character_anno = f.readlines()
+            new_annos += long_character_anno
+
+    # Get all speaker names
+    speakers = []
+    for line in new_annos:
+        path, speaker, text = line.split("|")
+        if speaker not in speakers:
+            speakers.append(speaker)
+    assert (len(speakers) != 0), "No audio file found. Please check your uploaded file structure."
+    # Source 3 (Optional): sampled audios as extra training helpers
+    if args.add_auxiliary_data:
+        with open("./sampled_audio4ft.txt", 'r', encoding='utf-8') as f:
+            old_annos = f.readlines()
+        # filter old_annos according to supported languages
+        filtered_old_annos = []
+        for line in old_annos:
+            for lang in langs:
+                if lang in line:
+                    filtered_old_annos.append(line)
+        old_annos = filtered_old_annos
+        for line in old_annos:
+            path, speaker, text = line.split("|")
+            if speaker not in speakers:
+                speakers.append(speaker)
+        num_old_voices = len(old_annos)
+        num_new_voices = len(new_annos)
+        # STEP 1: balance number of new & old voices
+        cc_duplicate = num_old_voices // num_new_voices
+        if cc_duplicate == 0:
+            cc_duplicate = 1
+
+
+        # STEP 2: modify config file
+        with open("./configs/finetune_speaker.json", 'r', encoding='utf-8') as f:
+            hps = json.load(f)
+
+        # assign ids to new speakers
+        speaker2id = {}
+        for i, speaker in enumerate(speakers):
+            speaker2id[speaker] = i
+        # modify n_speakers
+        hps['data']["n_speakers"] = len(speakers)
+        # overwrite speaker names
+        hps['speakers'] = speaker2id
+        hps['train']['log_interval'] = 10
+        hps['train']['eval_interval'] = 100
+        hps['train']['batch_size'] = 16
+        hps['data']['training_files'] = "final_annotation_train.txt"
+        hps['data']['validation_files'] = "final_annotation_val.txt"
+        # save modified config
+        with open("./configs/modified_finetune_speaker.json", 'w', encoding='utf-8') as f:
+            json.dump(hps, f, indent=2)
+
+        # STEP 3: clean annotations, replace speaker names with assigned speaker IDs
+        import text
+        cleaned_new_annos = []
+        for i, line in enumerate(new_annos):
+            path, speaker, txt = line.split("|")
+            if len(txt) > 150:
+                continue
+            cleaned_text = text._clean_text(txt, hps['data']['text_cleaners'])
+            cleaned_text += "\n" if not cleaned_text.endswith("\n") else ""
+            cleaned_new_annos.append(path + "|" + str(speaker2id[speaker]) + "|" + cleaned_text)
+        cleaned_old_annos = []
+        for i, line in enumerate(old_annos):
+            path, speaker, txt = line.split("|")
+            if len(txt) > 150:
+                continue
+            cleaned_text = text._clean_text(txt, hps['data']['text_cleaners'])
+            cleaned_text += "\n" if not cleaned_text.endswith("\n") else ""
+            cleaned_old_annos.append(path + "|" + str(speaker2id[speaker]) + "|" + cleaned_text)
+        # merge with old annotation
+        final_annos = cleaned_old_annos + cc_duplicate * cleaned_new_annos
+        # save annotation file
+        with open("./final_annotation_train.txt", 'w', encoding='utf-8') as f:
+            for line in final_annos:
+                f.write(line)
+        # save annotation file for validation
+        with open("./final_annotation_val.txt", 'w', encoding='utf-8') as f:
+            for line in cleaned_new_annos:
+                f.write(line)
+        print("finished")
+    else:
+        # Do not add extra helper data
+        # STEP 1: modify config file
+        with open("./configs/finetune_speaker.json", 'r', encoding='utf-8') as f:
+            hps = json.load(f)
+
+        # assign ids to new speakers
+        speaker2id = {}
+        for i, speaker in enumerate(speakers):
+            speaker2id[speaker] = i
+        # modify n_speakers
+        hps['data']["n_speakers"] = len(speakers)
+        # overwrite speaker names
+        hps['speakers'] = speaker2id
+        hps['train']['log_interval'] = 10
+        hps['train']['eval_interval'] = 100
+        hps['train']['batch_size'] = 16
+        hps['data']['training_files'] = "final_annotation_train.txt"
+        hps['data']['validation_files'] = "final_annotation_val.txt"
+        # save modified config
+        with open("./configs/modified_finetune_speaker.json", 'w', encoding='utf-8') as f:
+            json.dump(hps, f, indent=2)
+
+        # STEP 2: clean annotations, replace speaker names with assigned speaker IDs
+        import text
+
+        cleaned_new_annos = []
+        for i, line in enumerate(new_annos):
+            path, speaker, txt = line.split("|")
+            if len(txt) > 150:
+                continue
+            cleaned_text = text._clean_text(txt, hps['data']['text_cleaners']).replace("[ZH]", "")
+            cleaned_text += "\n" if not cleaned_text.endswith("\n") else ""
+            cleaned_new_annos.append(path + "|" + str(speaker2id[speaker]) + "|" + cleaned_text)
+
+        final_annos = cleaned_new_annos
+        # save annotation file
+        with open("./final_annotation_train.txt", 'w', encoding='utf-8') as f:
+            for line in final_annos:
+                f.write(line)
+        # save annotation file for validation
+        with open("./final_annotation_val.txt", 'w', encoding='utf-8') as f:
+            for line in cleaned_new_annos:
+                f.write(line)
+        print("finished")

fastfinetuning_kr/short_audio_transcribe.py

@@ -0,0 +1,128 @@
+import whisper
+import os
+import json
+import torchaudio
+import argparse
+import torch
+
+lang2token = {
+            'zh': "[ZH]",
+            'ja': "[JA]",
+            "en": "[EN]",
+        }
+def transcribe_one(audio_path):
+    # load audio and pad/trim it to fit 30 seconds
+    audio = whisper.load_audio(audio_path)
+    audio = whisper.pad_or_trim(audio)
+
+    # make log-Mel spectrogram and move to the same device as the model
+    mel = whisper.log_mel_spectrogram(audio).to(model.device)
+
+    # detect the spoken language
+    _, probs = model.detect_language(mel)
+    print(f"Detected language: {max(probs, key=probs.get)}")
+    lang = max(probs, key=probs.get)
+    # decode the audio
+    options = whisper.DecodingOptions(beam_size=5)
+    result = whisper.decode(model, mel, options)
+
+    # print the recognized text
+    print(result.text)
+    return lang, result.text
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--languages", default="CJE")
+    parser.add_argument("--whisper_size", default="medium")
+    args = parser.parse_args()
+    if args.languages == "CJE":
+        lang2token = {
+            'zh': "[ZH]",
+            'ja': "[JA]",
+            "en": "[EN]",
+        }
+    elif args.languages == "CJ":
+        lang2token = {
+            'zh': "[ZH]",
+            'ja': "[JA]",
+        }
+    elif args.languages == "C":
+        lang2token = {
+            'zh': "[ZH]",
+        }
+    elif args.languages == "CJKE":
+        lang2token = {
+            'zh': "[ZH]",
+            'ja': "[JA]",
+            "en": "[EN]",
+	"ko": "[KO]",
+        }
+    assert (torch.cuda.is_available()), "Please enable GPU in order to run Whisper!"
+    model = whisper.load_model(args.whisper_size)
+    parent_dir = "./custom_character_voice/"
+    speaker_names = list(os.walk(parent_dir))[0][1]
+    speaker_annos = []
+    total_files = sum([len(files) for r, d, files in os.walk(parent_dir)])
+    # resample audios
+    # 2023/4/21: Get the target sampling rate
+    with open("./configs/finetune_speaker.json", 'r', encoding='utf-8') as f:
+        hps = json.load(f)
+    target_sr = hps['data']['sampling_rate']
+    processed_files = 0
+    for speaker in speaker_names:
+        for i, wavfile in enumerate(list(os.walk(parent_dir + speaker))[0][2]):
+            # try to load file as audio
+            if wavfile.startswith("processed_"):
+                continue
+            try:
+                wav, sr = torchaudio.load(parent_dir + speaker + "/" + wavfile, frame_offset=0, num_frames=-1, normalize=True,
+                                          channels_first=True)
+                wav = wav.mean(dim=0).unsqueeze(0)
+                if sr != target_sr:
+                    wav = torchaudio.transforms.Resample(orig_freq=sr, new_freq=target_sr)(wav)
+                if wav.shape[1] / sr > 20:
+                    print(f"{wavfile} too long, ignoring\n")
+                save_path = parent_dir + speaker + "/" + f"processed_{i}.wav"
+                torchaudio.save(save_path, wav, target_sr, channels_first=True)
+                # transcribe text
+                lang, text = transcribe_one(save_path)
+                if lang not in list(lang2token.keys()):
+                    print(f"{lang} not supported, ignoring\n")
+                    continue
+                text = lang2token[lang] + text + lang2token[lang] + "\n"
+                speaker_annos.append(save_path + "|" + speaker + "|" + text)
+                
+                processed_files += 1
+                print(f"Processed: {processed_files}/{total_files}")
+            except:
+                continue
+
+    # # clean annotation
+    # import argparse
+    # import text
+    # from utils import load_filepaths_and_text
+    # for i, line in enumerate(speaker_annos):
+    #     path, sid, txt = line.split("|")
+    #     cleaned_text = text._clean_text(txt, ["cjke_cleaners2"])
+    #     cleaned_text += "\n" if not cleaned_text.endswith("\n") else ""
+    #     speaker_annos[i] = path + "|" + sid + "|" + cleaned_text
+    # write into annotation
+    if len(speaker_annos) == 0:
+        print("Warning: no short audios found, this IS expected if you have only uploaded long audios, videos or video links.")
+        print("this IS NOT expected if you have uploaded a zip file of short audios. Please check your file structure or make sure your audio language is supported.")
+    with open("short_character_anno.txt", 'w', encoding='utf-8') as f:
+        for line in speaker_annos:
+            f.write(line)
+
+    # import json
+    # # generate new config
+    # with open("./configs/finetune_speaker.json", 'r', encoding='utf-8') as f:
+    #     hps = json.load(f)
+    # # modify n_speakers
+    # hps['data']["n_speakers"] = 1000 + len(speaker2id)
+    # # add speaker names
+    # for speaker in speaker_names:
+    #     hps['speakers'][speaker] = speaker2id[speaker]
+    # # save modified config
+    # with open("./configs/modified_finetune_speaker.json", 'w', encoding='utf-8') as f:
+    #     json.dump(hps, f, indent=2)
+    # print("finished")