Spaces:
Running
Running
| import librosa | |
| import numpy as np | |
| import torch | |
| import gradio as gr | |
| import mido | |
| from io import BytesIO | |
| # import pyrubberband as pyrb | |
| import torchaudio.transforms as transforms | |
| from model.DiffSynthSampler import DiffSynthSampler | |
| from tools import adsr_envelope, adjust_audio_length | |
| from webUI.natural_language_guided.track_maker import DiffSynth, Track | |
| from webUI.natural_language_guided.utils import encodeBatch2GradioOutput_STFT, phase_to_Gradio_image, \ | |
| spectrogram_to_Gradio_image | |
| def get_arrangement_module(gradioWebUI, virtual_instruments_state, midi_files_state): | |
| # Load configurations | |
| uNet = gradioWebUI.uNet | |
| freq_resolution, time_resolution = gradioWebUI.freq_resolution, gradioWebUI.time_resolution | |
| VAE_scale = gradioWebUI.VAE_scale | |
| height, width, channels = int(freq_resolution / VAE_scale), int(time_resolution / VAE_scale), gradioWebUI.channels | |
| timesteps = gradioWebUI.timesteps | |
| VAE_quantizer = gradioWebUI.VAE_quantizer | |
| VAE_decoder = gradioWebUI.VAE_decoder | |
| CLAP = gradioWebUI.CLAP | |
| CLAP_tokenizer = gradioWebUI.CLAP_tokenizer | |
| device = gradioWebUI.device | |
| squared = gradioWebUI.squared | |
| sample_rate = gradioWebUI.sample_rate | |
| noise_strategy = gradioWebUI.noise_strategy | |
| def read_midi(midi, midi_files_dict): | |
| print(midi) | |
| midi_name = midi_file.name | |
| mid = mido.MidiFile(file=BytesIO(midi)) | |
| tracks = [Track(t, mid.ticks_per_beat) for t in mid.tracks] | |
| midi_info_text = f"Name: {midi_name}" | |
| for track in tracks: | |
| midi_info_text += f"\n {len(track.events)}" | |
| return {midi_info_textbox: gr.Textbox(label="Midi info", lines=10, | |
| placeholder=midi_info_text), | |
| midi_files_state: midi_files_dict} | |
| def refresh_instruments(virtual_instruments_dict): | |
| virtual_instruments_names = list(virtual_instruments_dict["virtual_instruments"].keys()) | |
| print(f"virtual_instruments_names: {virtual_instruments_names}") | |
| return {select_instrument_dropdown: gr.Dropdown.update(choices=["New Option 1", "New Option 2", "New Option 3"])} | |
| def select_sound(virtual_instrument_name, virtual_instruments_dict): | |
| virtual_instruments = virtual_instruments_dict["virtual_instruments"] | |
| virtual_instrument = virtual_instruments[virtual_instrument_name] | |
| return {source_sound_spectrogram_image: virtual_instrument["spectrogram_gradio_image"], | |
| source_sound_phase_image: virtual_instrument["phase_gradio_image"], | |
| source_sound_audio: virtual_instrument["signal"]} | |
| def make_track(inpaint_steps, midi, noising_strength, attack, before_release, instrument_names, | |
| virtual_instruments_dict): | |
| if noising_strength < 1: | |
| print(f"Warning: making track with noising_strength = {noising_strength} < 1") | |
| virtual_instruments = virtual_instruments_dict["virtual_instruments"] | |
| sample_steps = int(inpaint_steps) | |
| instrument_names = instrument_names.split("@") | |
| instruments_configs = {} | |
| for virtual_instrument_name in instrument_names: | |
| virtual_instrument = virtual_instruments[virtual_instrument_name] | |
| latent_representation = torch.tensor(virtual_instrument["latent_representation"], dtype=torch.float32).to( | |
| device) | |
| sampler = virtual_instrument["sampler"] | |
| batchsize = 1 | |
| latent_representation = latent_representation.repeat(batchsize, 1, 1, 1) | |
| mid = mido.MidiFile(file=BytesIO(midi)) | |
| instruments_configs[virtual_instrument_name] = { | |
| 'sample_steps': sample_steps, | |
| 'sampler': sampler, | |
| 'noising_strength': noising_strength, | |
| 'latent_representation': latent_representation, | |
| 'attack': attack, | |
| 'before_release': before_release} | |
| diffSynth = DiffSynth(instruments_configs, uNet, VAE_quantizer, VAE_decoder, CLAP, CLAP_tokenizer, device) | |
| full_audio = diffSynth.get_music(mid, instrument_names) | |
| return {track_audio: (sample_rate, full_audio)} | |
| with gr.Tab("Arrangement"): | |
| gr.Markdown("Make music with generated sounds!") | |
| with gr.Row(variant="panel"): | |
| with gr.Column(scale=3): | |
| preset_button_1 = gr.Button(variant="primary", value="Ode_to_Joy", scale=1) | |
| preset_button_2 = gr.Button(variant="primary", value="Ode_to_Joy", scale=1) | |
| preset_button_3 = gr.Button(variant="primary", value="Ode_to_Joy", scale=1) | |
| midi_file = gr.File(label="Upload midi file", type="binary", scale=2) | |
| with gr.Column(scale=3): | |
| midi_info_textbox = gr.Textbox(label="Midi info", lines=10, placeholder="Please select/upload a midi on the left.") | |
| instrument_names_textbox = gr.Textbox(label="Instrument names", lines=2, | |
| placeholder="Names of your instrument used to play the midi", scale=1) | |
| with gr.Column(scale=3): | |
| refresh_instrument_button = gr.Button(variant="primary", value="Refresh instruments", scale=1) | |
| # instrument_name_textbox = gr.Textbox(label="Instrument name", lines=1, | |
| # placeholder="Name of your instrument", scale=1) | |
| select_instrument_dropdown = gr.Dropdown(choices=["Option 1", "Option 2", "Option 3"], label="Choose an option") | |
| source_sound_audio = gr.Audio(type="numpy", label="Play new sound", interactive=False) | |
| with gr.Column(scale=3): | |
| make_track_button = gr.Button(variant="primary", value="Make track", scale=1) | |
| track_audio = gr.Audio(type="numpy", label="Play new sound", interactive=False) | |
| with gr.Row(variant="panel"): | |
| with gr.Tab("Origin sound"): | |
| inpaint_steps_slider = gr.Slider(minimum=5.0, maximum=999.0, value=20.0, step=1.0, | |
| label="inpaint_steps") | |
| noising_strength_slider = gradioWebUI.get_noising_strength_slider(default_noising_strength=1.) | |
| end_noise_level_ratio_slider = gr.Slider(minimum=0.0, maximum=1., value=0.0, step=0.01, | |
| label="end_noise_level_ratio") | |
| attack_slider = gr.Slider(minimum=0.0, maximum=1.5, value=0.5, step=0.01, label="attack in sec") | |
| before_release_slider = gr.Slider(minimum=0.0, maximum=1.5, value=0.5, step=0.01, | |
| label="before_release in sec") | |
| release_slider = gr.Slider(minimum=0.0, maximum=1.0, value=0.3, step=0.01, label="release in sec") | |
| mask_flexivity_slider = gr.Slider(minimum=0.01, maximum=1.00, value=1., step=0.01, | |
| label="mask_flexivity") | |
| with gr.Tab("Length adjustment config"): | |
| use_dynamic_mask_checkbox = gr.Checkbox(label="Use dynamic mask", value=True) | |
| test_duration_envelope_button = gr.Button(variant="primary", value="Apply envelope", scale=1) | |
| test_duration_stretch_button = gr.Button(variant="primary", value="Apply stretch", scale=1) | |
| test_duration_inpaint_button = gr.Button(variant="primary", value="Inpaint different duration", scale=1) | |
| duration_slider = gradioWebUI.get_duration_slider() | |
| with gr.Tab("Pitch shift config"): | |
| pitch_shift_radio = gr.Radio(choices=["librosa", "torchaudio", "rubberband"], | |
| value="librosa") | |
| with gr.Row(variant="panel"): | |
| with gr.Column(scale=2): | |
| with gr.Row(variant="panel"): | |
| source_sound_spectrogram_image = gr.Image(label="New sound spectrogram", type="numpy", | |
| height=600, scale=1) | |
| source_sound_phase_image = gr.Image(label="New sound phase", type="numpy", | |
| height=600, scale=1) | |
| # instrument_name_textbox.change(select_sound, | |
| # inputs=[instrument_name_textbox, virtual_instruments_state], | |
| # outputs=[source_sound_audio]) | |
| refresh_instrument_button.click(refresh_instruments, | |
| inputs=[virtual_instruments_state], | |
| outputs=[select_instrument_dropdown]) | |
| make_track_button.click(make_track, | |
| inputs=[inpaint_steps_slider, midi_file, | |
| noising_strength_slider, | |
| attack_slider, | |
| before_release_slider, | |
| instrument_names_textbox, | |
| virtual_instruments_state], | |
| outputs=[track_audio]) | |
| midi_file.change(read_midi, | |
| inputs=[midi_file, | |
| midi_files_state], | |
| outputs=[midi_info_textbox, | |
| midi_files_state]) | |