Add files using upload-large-folder tool

.gitattributes

@@ -134,3 +134,9 @@ Data/Tsukuyomi/wavs/VOICEACTRESS100_006.wav filter=lfs diff=lfs merge=lfs -text
 Data/Tsukuyomi/wavs/VOICEACTRESS100_005.wav filter=lfs diff=lfs merge=lfs -text
 Data/Tsukuyomi/wavs/VOICEACTRESS100_007.wav filter=lfs diff=lfs merge=lfs -text
 Data/Tsukuyomi/wavs/VOICEACTRESS100_003.wav filter=lfs diff=lfs merge=lfs -text
+Data/Tsukuyomi/wavs/VOICEACTRESS100_009.wav filter=lfs diff=lfs merge=lfs -text
+Data/Tsukuyomi/wavs/VOICEACTRESS100_011.wav filter=lfs diff=lfs merge=lfs -text
+Data/Tsukuyomi/wavs/VOICEACTRESS100_008.wav filter=lfs diff=lfs merge=lfs -text
+Data/Tsukuyomi/wavs/VOICEACTRESS100_012.wav filter=lfs diff=lfs merge=lfs -text
+Data/Tsukuyomi/wavs/VOICEACTRESS100_015.wav filter=lfs diff=lfs merge=lfs -text
+Data/Tsukuyomi/wavs/VOICEACTRESS100_010.wav filter=lfs diff=lfs merge=lfs -text

Data/Tsukuyomi/wavs/VOICEACTRESS100_008.wav

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:897ec3b3656b97df031d7f638a74cc1e2fd172291dd0b6b721023cf4b6340b29
+size 2793632

Data/Tsukuyomi/wavs/VOICEACTRESS100_009.wav

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ea6d3671bd12e929c5fda310e9743e6eb3bc337c7f38b75a556184d678b6bf9d
+size 2202512

Data/Tsukuyomi/wavs/VOICEACTRESS100_010.wav

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8f15ab1a7766631867d53e775ced0681725e95d61be410df45a949354682c58a
+size 1632880

Data/Tsukuyomi/wavs/VOICEACTRESS100_011.wav

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c89253932ca51cb487af8d8101b66d3089499e16c2bbdc9c1164c402200e4fe5
+size 2354984

Data/Tsukuyomi/wavs/VOICEACTRESS100_012.wav

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:fc314a221e398fade4ea5494194753657bad97d04357eab2350a1fae836cd535
+size 1988680

Data/Tsukuyomi/wavs/VOICEACTRESS100_015.wav

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:670862e969693ba0f52ddd0bc72d9abafdcc5ef8d91d6cff27e83d9e316df50d
+size 1622784

Models/Tsukuyomi_FineTuned/config_ft.yml

@@ -0,0 +1,116 @@
+log_dir: "Models/Tsukuyomi_FineTuned"
+save_freq: 1
+log_interval: 10
+device: "cuda"
+epochs: 50 # number of finetuning epoch (1 hour of data)
+batch_size: 3
+max_len: 2500 # maximum number of frames
+pretrained_model: "Models/Tsukasa/Top_ckpt_24khz.pth" 
+second_stage_load_pretrained: true # set to true if the pre-trained model is for 2nd stage
+load_only_params: true # set to true if do not want to load epoch numbers and optimizer parameters
+
+F0_path: "Utils/JDC/bst.t7"
+ASR_config: "Utils/ASR/config.yml"
+ASR_path: "Utils/ASR/bst_00080.pth"
+
+PLBERT_dir: 'Utils/PLBERT/'
+
+data_params:
+  train_data: "Data/Tsukuyomi/transcripts.txt"       # トレーニングデータのパス
+  val_data: "Data/Tsukuyomi/validation.txt"          # 検証データのパス
+  root_path: "Data/Tsukuyomi/wavs/"                  # 音声ファイルのパス
+  OOD_data: null                                       # OODデータがない場合は空欄
+  min_length: 50
+
+
+preprocess_params:
+  sr: 24000
+  spect_params:
+    n_fft: 2048
+    win_length: 1200
+    hop_length: 300
+
+model_params:
+  multispeaker: false  # 単一話者の場合は false
+
+  dim_in: 64 
+  hidden_dim: 512
+  max_conv_dim: 512
+  n_layer: 3
+  n_mels: 80
+
+  n_token: 178 # number of phoneme tokens
+  max_dur: 50 # maximum duration of a single phoneme
+  style_dim: 128 # style vector size
+  
+  dropout: 0.2
+
+  decoder: 
+    type: 'istftnet' # either hifigan or istftnet
+    resblock_kernel_sizes: [3,7,11]
+    upsample_rates :  [10, 6]
+    upsample_initial_channel: 512
+    resblock_dilation_sizes: [[1,3,5], [1,3,5], [1,3,5]]
+    upsample_kernel_sizes: [20, 12]
+    gen_istft_n_fft: 20
+    gen_istft_hop_size: 5
+
+
+      
+  # speech language model config
+  slm:
+      model: 'Respair/Whisper_Large_v2_Encoder_Block' # The model itself is hardcoded, change it through -> losses.py
+      sr: 16000 # sampling rate of SLM
+      hidden: 1280 # hidden size of SLM
+      nlayers: 33 # number of layers of SLM
+      initial_channel: 64 # initial channels of SLM discriminator head
+  
+  # style diffusion model config
+  diffusion:
+    embedding_mask_proba: 0.1
+    # transformer config
+    transformer:
+      num_layers: 3
+      num_heads: 8
+      head_features: 64
+      multiplier: 2
+
+    # diffusion distribution config
+    dist:
+      sigma_data: 0.2 # placeholder for estimate_sigma_data set to false
+      estimate_sigma_data: true # estimate sigma_data from the current batch if set to true
+      mean: -3.0
+      std: 1.0
+  
+loss_params:
+    lambda_mel: 10. # mel reconstruction loss
+    lambda_gen: 1. # generator loss
+    lambda_slm: 1. # slm feature matching loss
+    
+    lambda_mono: 1. # monotonic alignment loss (1st stage, TMA)
+    lambda_s2s: 1. # sequence-to-sequence loss (1st stage, TMA)
+    TMA_epoch: 9 # TMA starting epoch (1st stage)
+
+    lambda_F0: 1. # F0 reconstruction loss (2nd stage)
+    lambda_norm: 1. # norm reconstruction loss (2nd stage)
+    lambda_dur: 1. # duration loss (2nd stage)
+    lambda_ce: 20. # duration predictor probability output CE loss (2nd stage)
+    lambda_sty: 1. # style reconstruction loss (2nd stage)
+    lambda_diff: 1. # score matching loss (2nd stage)
+    
+    diff_epoch: 0 # style diffusion starting epoch (2nd stage)
+    joint_epoch: 30 # joint training starting epoch (2nd stage)
+
+optimizer_params:
+  lr: 0.0001 # general learning rate
+  bert_lr: 0.00001 # learning rate for PLBERT
+  ft_lr: 0.00001 # learning rate for acoustic modules
+  
+slmadv_params:
+  min_len: 400 # minimum length of samples
+  max_len: 500 # maximum length of samples
+  batch_percentage: 0.5 # to prevent out of memory, only use half of the original batch size
+  iter: 20 # update the discriminator every this iterations of generator update
+  thresh: 5 # gradient norm above which the gradient is scaled
+  scale: 0.01 # gradient scaling factor for predictors from SLM discriminators
+  sig: 1.5 # sigma for differentiable duration modeling

Modules/KotoDama_sampler.py

@@ -0,0 +1,269 @@
+from transformers import AutoModelForSequenceClassification, PreTrainedModel, AutoConfig, AutoModel, AutoTokenizer
+import torch
+import torch.nn as nn
+from text_utils import TextCleaner
+textclenaer = TextCleaner()
+
+
+def length_to_mask(lengths):
+    mask = torch.arange(lengths.max()).unsqueeze(0).expand(lengths.shape[0], -1).type_as(lengths)
+    mask = torch.gt(mask+1, lengths.unsqueeze(1))
+    return mask
+
+
+
+
+device = 'cuda' if torch.cuda.is_available() else 'cpu'
+
+
+# tokenizer_koto_prompt = AutoTokenizer.from_pretrained("google/mt5-small", trust_remote_code=True)
+tokenizer_koto_prompt = AutoTokenizer.from_pretrained("ku-nlp/deberta-v3-base-japanese", trust_remote_code=True)
+tokenizer_koto_text = AutoTokenizer.from_pretrained("line-corporation/line-distilbert-base-japanese", trust_remote_code=True)
+
+class KotoDama_Prompt(PreTrainedModel):
+
+    def __init__(self, config):
+        super().__init__(config)
+        
+        self.backbone = AutoModel.from_config(config)
+
+        self.output = nn.Sequential(nn.Linear(config.hidden_size, 512),
+                                    nn.LeakyReLU(0.2), 
+                                    nn.Linear(512, config.num_labels))
+
+                                    
+
+    def forward(
+        self,
+        input_ids,
+        attention_mask=None,
+        token_type_ids=None,
+        position_ids=None,
+        labels=None,
+    ):
+        outputs = self.backbone(
+            input_ids,
+            attention_mask=attention_mask,
+            token_type_ids=token_type_ids,
+            position_ids=position_ids,
+        )
+
+
+        sequence_output = outputs.last_hidden_state[:, 0, :]
+        outputs = self.output(sequence_output)
+
+        # if labels, then we are training
+        loss = None
+        if labels is not None:
+
+            loss_fn = nn.MSELoss()
+            # labels = labels.unsqueeze(1)
+            loss = loss_fn(outputs, labels)
+            
+        return {
+            "loss": loss,
+            "logits": outputs
+        }
+
+
+class KotoDama_Text(PreTrainedModel):
+
+    def __init__(self, config):
+        super().__init__(config)
+        
+        self.backbone = AutoModel.from_config(config)
+
+        self.output = nn.Sequential(nn.Linear(config.hidden_size, 512),
+                                    nn.LeakyReLU(0.2), 
+                                    nn.Linear(512, config.num_labels))
+
+                                    
+
+    def forward(
+        self,
+        input_ids,
+        attention_mask=None,
+        # token_type_ids=None,
+        # position_ids=None,
+        labels=None,
+    ):
+        outputs = self.backbone(
+            input_ids,
+            attention_mask=attention_mask,
+            # token_type_ids=token_type_ids,
+            # position_ids=position_ids,
+        )
+
+
+        sequence_output = outputs.last_hidden_state[:, 0, :]
+        outputs = self.output(sequence_output)
+
+        # if labels, then we are training
+        loss = None
+        if labels is not None:
+
+            loss_fn = nn.MSELoss()
+            # labels = labels.unsqueeze(1)
+            loss = loss_fn(outputs, labels)
+            
+        return {
+            "loss": loss,
+            "logits": outputs
+        }
+
+
+def inference(model, diffusion_sampler, text=None, ref_s=None, alpha = 0.3, beta = 0.7, diffusion_steps=5, embedding_scale=1, rate_of_speech=1.):
+
+    tokens = textclenaer(text)
+    tokens.insert(0, 0)
+    tokens = torch.LongTensor(tokens).to(device).unsqueeze(0)
+    
+    with torch.no_grad():
+        input_lengths = torch.LongTensor([tokens.shape[-1]]).to(device)
+
+        text_mask = length_to_mask(input_lengths).to(device)
+
+        t_en = model.text_encoder(tokens, input_lengths, text_mask)
+        bert_dur = model.bert(tokens, attention_mask=(~text_mask).int())
+        d_en = model.bert_encoder(bert_dur).transpose(-1, -2) 
+        
+
+
+        s_pred = diffusion_sampler(noise = torch.randn((1, 256)).unsqueeze(1).to(device), 
+                                          embedding=bert_dur,
+                                          embedding_scale=embedding_scale,
+                                            features=ref_s, # reference from the same speaker as the embedding
+                                             num_steps=diffusion_steps).squeeze(1)
+
+
+        s = s_pred[:, 128:]
+        ref = s_pred[:, :128]
+
+        ref = alpha * ref + (1 - alpha)  * ref_s[:, :128]
+        s = beta * s + (1 - beta)  * ref_s[:, 128:]
+
+        d = model.predictor.text_encoder(d_en, 
+                                         s, input_lengths, text_mask)
+        
+  
+
+        x = model.predictor.lstm(d)
+        x_mod =  model.predictor.prepare_projection(x) 
+        duration = model.predictor.duration_proj(x_mod) 
+
+
+        duration = torch.sigmoid(duration).sum(axis=-1) / rate_of_speech
+        
+        pred_dur = torch.round(duration.squeeze()).clamp(min=1) 
+
+
+
+        pred_aln_trg = torch.zeros(input_lengths, int(pred_dur.sum().data))
+        
+        c_frame = 0
+        for i in range(pred_aln_trg.size(0)):
+            pred_aln_trg[i, c_frame:c_frame + int(pred_dur[i].data)] = 1
+            c_frame += int(pred_dur[i].data)
+
+        # encode prosody
+        en = (d.transpose(-1, -2) @ pred_aln_trg.unsqueeze(0).to(device))
+
+
+
+        F0_pred, N_pred = model.predictor.F0Ntrain(en, s)
+
+        asr = (t_en @ pred_aln_trg.unsqueeze(0).to(device))
+
+
+        out = model.decoder(asr, 
+                                F0_pred, N_pred, ref.squeeze().unsqueeze(0))
+    
+        
+    return out.squeeze().cpu().numpy()[..., :-50] 
+
+
+def Longform(model, diffusion_sampler, text, s_prev, ref_s, alpha = 0.3, beta = 0.7, t = 0.7, diffusion_steps=5, embedding_scale=1, rate_of_speech=1.0):
+
+    tokens = textclenaer(text)
+    tokens.insert(0, 0)
+    tokens = torch.LongTensor(tokens).to(device).unsqueeze(0)
+    
+    with torch.no_grad():
+        input_lengths = torch.LongTensor([tokens.shape[-1]]).to(device)
+        text_mask = length_to_mask(input_lengths).to(device)
+
+        t_en = model.text_encoder(tokens, input_lengths, text_mask)
+        bert_dur = model.bert(tokens, attention_mask=(~text_mask).int())
+        d_en = model.bert_encoder(bert_dur).transpose(-1, -2) 
+
+        s_pred = diffusion_sampler(noise = torch.randn((1, 256)).unsqueeze(1).to(device), 
+                                          embedding=bert_dur,
+                                          embedding_scale=embedding_scale,
+                                            features=ref_s, 
+                                             num_steps=diffusion_steps).squeeze(1)
+        
+        if s_prev is not None:
+            # convex combination of previous and current style
+            s_pred = t * s_prev + (1 - t) * s_pred
+        
+        s = s_pred[:, 128:]
+        ref = s_pred[:, :128]
+        
+        ref = alpha * ref + (1 - alpha)  * ref_s[:, :128]
+        s = beta * s + (1 - beta)  * ref_s[:, 128:]
+
+        s_pred = torch.cat([ref, s], dim=-1)
+
+        d = model.predictor.text_encoder(d_en, 
+                                         s, input_lengths, text_mask)
+
+        x = model.predictor.lstm(d)
+        x_mod =  model.predictor.prepare_projection(x) # 640 -> 512
+        duration = model.predictor.duration_proj(x_mod)
+
+        duration = torch.sigmoid(duration).sum(axis=-1) / rate_of_speech
+        pred_dur = torch.round(duration.squeeze()).clamp(min=1) 
+
+
+        pred_aln_trg = torch.zeros(input_lengths, int(pred_dur.sum().data))
+        c_frame = 0
+        for i in range(pred_aln_trg.size(0)):
+            pred_aln_trg[i, c_frame:c_frame + int(pred_dur[i].data)] = 1
+            c_frame += int(pred_dur[i].data)
+
+        # encode prosody
+        en = (d.transpose(-1, -2) @ pred_aln_trg.unsqueeze(0).to(device))
+
+        F0_pred, N_pred = model.predictor.F0Ntrain(en, s)
+
+        asr = (t_en @ pred_aln_trg.unsqueeze(0).to(device))
+
+        out = model.decoder(asr, 
+                                F0_pred, N_pred, ref.squeeze().unsqueeze(0))
+    
+        
+    return out.squeeze().cpu().numpy()[..., :-100], s_pred 
+
+
+def merge_short_elements(lst):
+    i = 0
+    while i < len(lst):
+        if i > 0 and len(lst[i]) < 10:
+            lst[i-1] += ' ' + lst[i]
+            lst.pop(i)
+        else:
+            i += 1
+    return lst
+
+
+def merge_three(text_list, maxim=2):
+
+    merged_list = []
+    for i in range(0, len(text_list), maxim):
+        merged_text = ' '.join(text_list[i:i+maxim])
+        merged_list.append(merged_text)
+    return merged_list
+
+
+def merging_sentences(lst):
+    return merge_three(merge_short_elements(lst))

Modules/__init__.py

@@ -0,0 +1 @@
+

Modules/diffusion/reconstruction_head/audio-diffusion-pytorch/.github/workflows/python-publish.yml

@@ -0,0 +1,39 @@
+# This workflow will upload a Python Package using Twine when a release is created
+# For more information see: https://help.github.com/en/actions/language-and-framework-guides/using-python-with-github-actions#publishing-to-package-registries
+
+# This workflow uses actions that are not certified by GitHub.
+# They are provided by a third-party and are governed by
+# separate terms of service, privacy policy, and support
+# documentation.
+
+name: Upload Python Package
+
+on:
+  release:
+    types: [published]
+
+permissions:
+  contents: read
+
+jobs:
+  deploy:
+
+    runs-on: ubuntu-latest
+
+    steps:
+    - uses: actions/checkout@v3
+    - name: Set up Python
+      uses: actions/setup-python@v3
+      with:
+        python-version: '3.x'
+    - name: Install dependencies
+      run: |
+        python -m pip install --upgrade pip
+        pip install build
+    - name: Build package
+      run: python -m build
+    - name: Publish package
+      uses: pypa/gh-action-pypi-publish@27b31702a0e7fc50959f5ad993c78deac1bdfc29
+      with:
+        user: __token__
+        password: ${{ secrets.PYPI_API_TOKEN }}

Modules/diffusion/reconstruction_head/audio-diffusion-pytorch/.gitignore

@@ -0,0 +1,2 @@
+__pycache__
+.mypy_cache

Modules/diffusion/reconstruction_head/audio-diffusion-pytorch/.pre-commit-config.yaml

@@ -0,0 +1,41 @@
+repos:
+-   repo: https://github.com/pre-commit/pre-commit-hooks
+    rev: v2.3.0
+    hooks:
+    -   id: end-of-file-fixer
+    -   id: trailing-whitespace
+
+# Formats code correctly
+-   repo: https://github.com/psf/black
+    rev: 21.12b0
+    hooks:
+    -   id: black
+        args: [
+            '--experimental-string-processing'
+        ]
+
+# Sorts imports
+-   repo: https://github.com/pycqa/isort
+    rev: 5.10.1
+    hooks:
+    -   id: isort
+        name: isort (python)
+        args: ["--profile", "black"]
+
+# Checks unused imports, like lengths, etc
+-   repo: https://gitlab.com/pycqa/flake8
+    rev: 4.0.0
+    hooks:
+    -   id: flake8
+        args: [
+            '--per-file-ignores=__init__.py:F401',
+            '--max-line-length=88',
+            '--ignore=E203,W503'
+        ]
+
+# Checks types
+-   repo: https://github.com/pre-commit/mirrors-mypy
+    rev: 'v0.971'
+    hooks:
+    -   id: mypy
+        additional_dependencies: [data-science-types>=0.2, torch>=1.6]

Modules/diffusion/reconstruction_head/audio-diffusion-pytorch/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2022 archinet.ai
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

Modules/diffusion/reconstruction_head/audio-diffusion-pytorch/README.md

@@ -0,0 +1,251 @@
+<img src="./LOGO.png"></img>
+
+A fully featured audio diffusion library, for PyTorch. Includes models for unconditional audio generation, text-conditional audio generation, diffusion autoencoding, upsampling, and vocoding. The provided models are waveform-based, however, the U-Net (built using [`a-unet`](https://github.com/archinetai/a-unet)), `DiffusionModel`, diffusion method, and diffusion samplers are both generic to any dimension and highly customizable to work on other formats. **Notes: (1) no pre-trained models are provided here, (2) the configs shown are indicative and untested, see [Moûsai](https://arxiv.org/abs/2301.11757) for the configs used in the paper.**
+
+
+## Install
+
+```bash
+pip install audio-diffusion-pytorch
+```
+
+[![PyPI - Python Version](https://img.shields.io/pypi/v/audio-diffusion-pytorch?style=flat&colorA=black&colorB=black)](https://pypi.org/project/audio-diffusion-pytorch/)
+[![Downloads](https://static.pepy.tech/personalized-badge/audio-diffusion-pytorch?period=total&units=international_system&left_color=black&right_color=black&left_text=Downloads)](https://pepy.tech/project/audio-diffusion-pytorch)
+
+
+## Usage
+
+### Unconditional Generator
+
+```py
+from audio_diffusion_pytorch import DiffusionModel, UNetV0, VDiffusion, VSampler
+
+model = DiffusionModel(
+    net_t=UNetV0, # The model type used for diffusion (U-Net V0 in this case)
+    in_channels=2, # U-Net: number of input/output (audio) channels
+    channels=[8, 32, 64, 128, 256, 512, 512, 1024, 1024], # U-Net: channels at each layer
+    factors=[1, 4, 4, 4, 2, 2, 2, 2, 2], # U-Net: downsampling and upsampling factors at each layer
+    items=[1, 2, 2, 2, 2, 2, 2, 4, 4], # U-Net: number of repeating items at each layer
+    attentions=[0, 0, 0, 0, 0, 1, 1, 1, 1], # U-Net: attention enabled/disabled at each layer
+    attention_heads=8, # U-Net: number of attention heads per attention item
+    attention_features=64, # U-Net: number of attention features per attention item
+    diffusion_t=VDiffusion, # The diffusion method used
+    sampler_t=VSampler, # The diffusion sampler used
+)
+
+# Train model with audio waveforms
+audio = torch.randn(1, 2, 2**18) # [batch_size, in_channels, length]
+loss = model(audio)
+loss.backward()
+
+# Turn noise into new audio sample with diffusion
+noise = torch.randn(1, 2, 2**18) # [batch_size, in_channels, length]
+sample = model.sample(noise, num_steps=10) # Suggested num_steps 10-100
+```
+
+### Text-Conditional Generator
+A text-to-audio diffusion model that conditions the generation with `t5-base` text embeddings, requires `pip install transformers`.
+```py
+from audio_diffusion_pytorch import DiffusionModel, UNetV0, VDiffusion, VSampler
+
+model = DiffusionModel(
+    # ... same as unconditional model
+    use_text_conditioning=True, # U-Net: enables text conditioning (default T5-base)
+    use_embedding_cfg=True, # U-Net: enables classifier free guidance
+    embedding_max_length=64, # U-Net: text embedding maximum length (default for T5-base)
+    embedding_features=768, # U-Net: text mbedding features (default for T5-base)
+    cross_attentions=[0, 0, 0, 1, 1, 1, 1, 1, 1], # U-Net: cross-attention enabled/disabled at each layer
+)
+
+# Train model with audio waveforms
+audio_wave = torch.randn(1, 2, 2**18) # [batch, in_channels, length]
+loss = model(
+    audio_wave,
+    text=['The audio description'], # Text conditioning, one element per batch
+    embedding_mask_proba=0.1 # Probability of masking text with learned embedding (Classifier-Free Guidance Mask)
+)
+loss.backward()
+
+# Turn noise into new audio sample with diffusion
+noise = torch.randn(1, 2, 2**18)
+sample = model.sample(
+    noise,
+    text=['The audio description'],
+    embedding_scale=5.0, # Higher for more text importance, suggested range: 1-15 (Classifier-Free Guidance Scale)
+    num_steps=2 # Higher for better quality, suggested num_steps: 10-100
+)
+```
+
+### Diffusion Upsampler
+Upsample audio from a lower sample rate to higher sample rate using diffusion, e.g. 3kHz to 48kHz.
+```py
+from audio_diffusion_pytorch import DiffusionUpsampler, UNetV0, VDiffusion, VSampler
+
+upsampler = DiffusionUpsampler(
+    net_t=UNetV0, # The model type used for diffusion
+    upsample_factor=16, # The upsample factor (e.g. 16 can be used for 3kHz to 48kHz)
+    in_channels=2, # U-Net: number of input/output (audio) channels
+    channels=[8, 32, 64, 128, 256, 512, 512, 1024, 1024], # U-Net: channels at each layer
+    factors=[1, 4, 4, 4, 2, 2, 2, 2, 2], # U-Net: downsampling and upsampling factors at each layer
+    items=[1, 2, 2, 2, 2, 2, 2, 4, 4], # U-Net: number of repeating items at each layer
+    diffusion_t=VDiffusion, # The diffusion method used
+    sampler_t=VSampler, # The diffusion sampler used
+)
+
+# Train model with high sample rate audio waveforms
+audio = torch.randn(1, 2, 2**18) # [batch, in_channels, length]
+loss = upsampler(audio)
+loss.backward()
+
+# Turn low sample rate audio into high sample rate
+downsampled_audio = torch.randn(1, 2, 2**14) # [batch, in_channels, length]
+sample = upsampler.sample(downsampled_audio, num_steps=10) # Output has shape: [1, 2, 2**18]
+```
+
+### Diffusion Vocoder
+Convert a mel-spectrogram to wavefrom using diffusion.
+```py
+from audio_diffusion_pytorch import DiffusionVocoder, UNetV0, VDiffusion, VSampler
+
+vocoder = DiffusionVocoder(
+    mel_n_fft=1024, # Mel-spectrogram n_fft
+    mel_channels=80, # Mel-spectrogram channels
+    mel_sample_rate=48000, # Mel-spectrogram sample rate
+    mel_normalize_log=True, # Mel-spectrogram log normalization (alternative is mel_normalize=True for [-1,1] power normalization)
+    net_t=UNetV0, # The model type used for diffusion vocoding
+    channels=[8, 32, 64, 128, 256, 512, 512, 1024, 1024], # U-Net: channels at each layer
+    factors=[1, 4, 4, 4, 2, 2, 2, 2, 2], # U-Net: downsampling and upsampling factors at each layer
+    items=[1, 2, 2, 2, 2, 2, 2, 4, 4], # U-Net: number of repeating items at each layer
+    diffusion_t=VDiffusion, # The diffusion method used
+    sampler_t=VSampler, # The diffusion sampler used
+)
+
+# Train model on waveforms (automatically converted to mel internally)
+audio = torch.randn(1, 2, 2**18) # [batch, in_channels, length]
+loss = vocoder(audio)
+loss.backward()
+
+# Turn mel spectrogram into waveform
+mel_spectrogram = torch.randn(1, 2, 80, 1024) # [batch, in_channels, mel_channels, mel_length]
+sample = vocoder.sample(mel_spectrogram, num_steps=10) # Output has shape: [1, 2, 2**18]
+```
+
+### Diffusion Autoencoder
+Autoencode audio into a compressed latent using diffusion. Any encoder can be provided as long as it subclasses the `EncoderBase` class or contains an `out_channels` and `downsample_factor` field.
+```py
+from audio_diffusion_pytorch import DiffusionAE, UNetV0, VDiffusion, VSampler
+from audio_encoders_pytorch import MelE1d, TanhBottleneck
+
+autoencoder = DiffusionAE(
+    encoder=MelE1d( # The encoder used, in this case a mel-spectrogram encoder
+        in_channels=2,
+        channels=512,
+        multipliers=[1, 1],
+        factors=[2],
+        num_blocks=[12],
+        out_channels=32,
+        mel_channels=80,
+        mel_sample_rate=48000,
+        mel_normalize_log=True,
+        bottleneck=TanhBottleneck(),
+    ),
+    inject_depth=6,
+    net_t=UNetV0, # The model type used for diffusion upsampling
+    in_channels=2, # U-Net: number of input/output (audio) channels
+    channels=[8, 32, 64, 128, 256, 512, 512, 1024, 1024], # U-Net: channels at each layer
+    factors=[1, 4, 4, 4, 2, 2, 2, 2, 2], # U-Net: downsampling and upsampling factors at each layer
+    items=[1, 2, 2, 2, 2, 2, 2, 4, 4], # U-Net: number of repeating items at each layer
+    diffusion_t=VDiffusion, # The diffusion method used
+    sampler_t=VSampler, # The diffusion sampler used
+)
+
+# Train autoencoder with audio samples
+audio = torch.randn(1, 2, 2**18) # [batch, in_channels, length]
+loss = autoencoder(audio)
+loss.backward()
+
+# Encode/decode audio
+audio = torch.randn(1, 2, 2**18) # [batch, in_channels, length]
+latent = autoencoder.encode(audio) # Encode
+sample = autoencoder.decode(latent, num_steps=10) # Decode by sampling diffusion model conditioning on latent
+```
+
+## Other
+
+### Inpainting
+```py
+from audio_diffusion_pytorch import UNetV0, VInpainter
+
+# The diffusion UNetV0 (this is an example, the net must be trained to work)
+net = UNetV0(
+    dim=1,
+    in_channels=2, # U-Net: number of input/output (audio) channels
+    channels=[8, 32, 64, 128, 256, 512, 512, 1024, 1024], # U-Net: channels at each layer
+    factors=[1, 4, 4, 4, 2, 2, 2, 2, 2], # U-Net: downsampling and upsampling factors at each layer
+    items=[1, 2, 2, 2, 2, 2, 2, 4, 4], # U-Net: number of repeating items at each layer
+    attentions=[0, 0, 0, 0, 0, 1, 1, 1, 1], # U-Net: attention enabled/disabled at each layer
+    attention_heads=8, # U-Net: number of attention heads per attention block
+    attention_features=64, # U-Net: number of attention features per attention block,
+)
+
+# Instantiate inpainter with trained net
+inpainter = VInpainter(net=net)
+
+# Inpaint source
+y = inpainter(
+    source=torch.randn(1, 2, 2**18), # Start source
+    mask=torch.randint(0, 2, (1, 2, 2 ** 18), dtype=torch.bool),  # Set to `True` the parts you want to keep
+    num_steps=10, # Number of inpainting steps
+    num_resamples=2, # Number of resampling steps
+    show_progress=True,
+) # [1, 2, 2 ** 18]
+```
+
+## Appreciation
+
+* [StabilityAI](https://stability.ai/) for the compute, [Zach Evans](https://github.com/zqevans) and everyone else from [HarmonAI](https://www.harmonai.org/) for the interesting research discussions.
+* [ETH Zurich](https://inf.ethz.ch/) for the resources, [Zhijing Jin](https://zhijing-jin.com/), [Bernhard Schoelkopf](https://is.mpg.de/~bs), and [Mrinmaya Sachan](http://www.mrinmaya.io/) for supervising this Thesis.
+* [Phil Wang](https://github.com/lucidrains) for the beautiful open source contributions on [diffusion](https://github.com/lucidrains/denoising-diffusion-pytorch) and [Imagen](https://github.com/lucidrains/imagen-pytorch).
+* [Katherine Crowson](https://github.com/crowsonkb) for the experiments with [k-diffusion](https://github.com/crowsonkb/k-diffusion) and the insane collection of samplers.
+
+## Citations
+
+DDPM Diffusion
+```bibtex
+@misc{2006.11239,
+Author = {Jonathan Ho and Ajay Jain and Pieter Abbeel},
+Title = {Denoising Diffusion Probabilistic Models},
+Year = {2020},
+Eprint = {arXiv:2006.11239},
+}
+```
+
+DDIM (V-Sampler)
+```bibtex
+@misc{2010.02502,
+Author = {Jiaming Song and Chenlin Meng and Stefano Ermon},
+Title = {Denoising Diffusion Implicit Models},
+Year = {2020},
+Eprint = {arXiv:2010.02502},
+}
+```
+
+V-Diffusion
+```bibtex
+@misc{2202.00512,
+Author = {Tim Salimans and Jonathan Ho},
+Title = {Progressive Distillation for Fast Sampling of Diffusion Models},
+Year = {2022},
+Eprint = {arXiv:2202.00512},
+}
+```
+
+Imagen (T5 Text Conditioning)
+```bibtex
+@misc{2205.11487,
+Author = {Chitwan Saharia and William Chan and Saurabh Saxena and Lala Li and Jay Whang and Emily Denton and Seyed Kamyar Seyed Ghasemipour and Burcu Karagol Ayan and S. Sara Mahdavi and Rapha Gontijo Lopes and Tim Salimans and Jonathan Ho and David J Fleet and Mohammad Norouzi},
+Title = {Photorealistic Text-to-Image Diffusion Models with Deep Language Understanding},
+Year = {2022},
+Eprint = {arXiv:2205.11487},
+}
+```