Upload folder using huggingface_hub

.gitattributes

@@ -33,3 +33,17 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+*.jpg filter=lfs diff=lfs merge=lfs -text
+*.png filter=lfs diff=lfs merge=lfs -text
+demo/example/1p_EN2CH.mp4 filter=lfs diff=lfs merge=lfs -text
+demo/example/2p_see_u_again.mp4 filter=lfs diff=lfs merge=lfs -text
+demo/example/4p_climate_45min.mp4 filter=lfs diff=lfs merge=lfs -text
+demo/voices/en-Alice_woman.wav filter=lfs diff=lfs merge=lfs -text
+demo/voices/en-Carter_man.wav filter=lfs diff=lfs merge=lfs -text
+demo/voices/en-Frank_man.wav filter=lfs diff=lfs merge=lfs -text
+demo/voices/en-Mary_woman_bgm.wav filter=lfs diff=lfs merge=lfs -text
+demo/voices/en-Maya_woman.wav filter=lfs diff=lfs merge=lfs -text
+demo/voices/in-Samuel_man.wav filter=lfs diff=lfs merge=lfs -text
+demo/voices/zh-Anchen_man_bgm.wav filter=lfs diff=lfs merge=lfs -text
+demo/voices/zh-Bowen_man.wav filter=lfs diff=lfs merge=lfs -text
+demo/voices/zh-Xinran_woman.wav filter=lfs diff=lfs merge=lfs -text

.gitignore

@@ -0,0 +1,177 @@
+# Initially taken from Github's Python gitignore file
+
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[cod]
+*$py.class
+
+# C extensions
+*.so
+
+# tests and logs
+tests/fixtures/cached_*_text.txt
+logs/
+lightning_logs/
+lang_code_data/
+
+# Distribution / packaging
+.Python
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+wheels/
+*.egg-info/
+.installed.cfg
+*.egg
+MANIFEST
+
+# PyInstaller
+#  Usually these files are written by a python script from a template
+#  before PyInstaller builds the exe, so as to inject date/other infos into it.
+*.manifest
+*.spec
+
+# Installer logs
+pip-log.txt
+pip-delete-this-directory.txt
+
+# Unit test / coverage reports
+htmlcov/
+.tox/
+.nox/
+.coverage
+.coverage.*
+.cache
+nosetests.xml
+coverage.xml
+*.cover
+.hypothesis/
+.pytest_cache/
+
+# Translations
+*.mo
+*.pot
+
+# Django stuff:
+*.log
+local_settings.py
+db.sqlite3
+
+# Flask stuff:
+instance/
+.webassets-cache
+
+# Scrapy stuff:
+.scrapy
+
+# Sphinx documentation
+docs/_build/
+
+# PyBuilder
+target/
+
+# Jupyter Notebook
+.ipynb_checkpoints
+
+# IPython
+profile_default/
+ipython_config.py
+
+# pyenv
+.python-version
+
+# celery beat schedule file
+celerybeat-schedule
+
+# SageMath parsed files
+*.sage.py
+
+# Environments
+.env
+.venv
+env/
+venv/
+ENV/
+env.bak/
+venv.bak/
+
+# Spyder project settings
+.spyderproject
+.spyproject
+
+# Rope project settings
+.ropeproject
+
+# mkdocs documentation
+/site
+
+# mypy
+.mypy_cache/
+.dmypy.json
+dmypy.json
+
+# Pyre type checker
+.pyre/
+
+# vscode
+.vs
+.vscode
+
+# Pycharm
+.idea
+
+# TF code
+tensorflow_code
+
+# Models
+proc_data
+
+# examples
+runs
+/runs_old
+/wandb
+/examples/runs
+/examples/**/*.args
+/examples/rag/sweep
+
+# data
+/data
+serialization_dir
+
+# emacs
+*.*~
+debug.env
+
+# vim
+.*.swp
+
+#ctags
+tags
+
+# pre-commit
+.pre-commit*
+
+# .lock
+*.lock
+
+# DS_Store (MacOS)
+.DS_Store
+
+# ruff
+.ruff_cache
+
+# our proj
+/output/
+/outputs/
+/checkpoint/
+/checkpoints/
+exp
+.gradio/

LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2025 Microsoft
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

SECURITY.md

@@ -0,0 +1,14 @@
+<!-- BEGIN MICROSOFT SECURITY.MD V1.0.0 BLOCK -->
+
+## Security
+
+Microsoft takes the security of our software products and services seriously, which
+includes all source code repositories in our GitHub organizations.
+
+**Please do not report security vulnerabilities through public GitHub issues.**
+
+For security reporting information, locations, contact information, and policies,
+please review the latest guidance for Microsoft repositories at
+[https://aka.ms/SECURITY.md](https://aka.ms/SECURITY.md).
+
+<!-- END MICROSOFT SECURITY.MD BLOCK -->

config.json

@@ -0,0 +1,115 @@
+{
+  "acoustic_vae_dim": 64,
+  "acoustic_tokenizer_config": {
+    "causal": true,
+    "channels": 1,
+    "conv_bias": true,
+    "conv_norm": "none",
+    "corpus_normalize": 0.0,
+    "decoder_depths": null,
+    "decoder_n_filters": 32,
+    "decoder_ratios": [
+      8,
+      5,
+      5,
+      4,
+      2,
+      2
+    ],
+    "disable_last_norm": true,
+    "encoder_depths": "3-3-3-3-3-3-8",
+    "encoder_n_filters": 32,
+    "encoder_ratios": [
+      8,
+      5,
+      5,
+      4,
+      2,
+      2
+    ],
+    "fix_std": 0.5,
+    "layer_scale_init_value": 1e-06,
+    "layernorm": "RMSNorm",
+    "layernorm_elementwise_affine": true,
+    "layernorm_eps": 1e-05,
+    "mixer_layer": "depthwise_conv",
+    "model_type": "vibevoice_acoustic_tokenizer",
+    "pad_mode": "constant",
+    "std_dist_type": "gaussian",
+    "vae_dim": 64,
+    "weight_init_value": 0.01
+  },
+  "architectures": [
+    "VibeVoiceForConditionalGeneration"
+  ],
+  "decoder_config": {
+    "attention_dropout": 0.0,
+    "hidden_act": "silu",
+    "hidden_size": 1536,
+    "initializer_range": 0.02,
+    "intermediate_size": 8960,
+    "max_position_embeddings": 65536,
+    "max_window_layers": 28,
+    "model_type": "qwen2",
+    "num_attention_heads": 12,
+    "num_hidden_layers": 28,
+    "num_key_value_heads": 2,
+    "rms_norm_eps": 1e-06,
+    "rope_scaling": null,
+    "rope_theta": 1000000.0,
+    "sliding_window": null,
+    "tie_word_embeddings": true,
+    "torch_dtype": "bfloat16",
+    "use_cache": true,
+    "use_sliding_window": false,
+    "vocab_size": 151936
+  },
+  "diffusion_head_config": {
+    "ddpm_batch_mul": 4,
+    "ddpm_beta_schedule": "cosine",
+    "ddpm_num_inference_steps": 20,
+    "ddpm_num_steps": 1000,
+    "diffusion_type": "ddpm",
+    "head_ffn_ratio": 3.0,
+    "head_layers": 4,
+    "hidden_size": 1536,
+    "latent_size": 64,
+    "model_type": "vibevoice_diffusion_head",
+    "prediction_type": "v_prediction",
+    "rms_norm_eps": 1e-05,
+    "speech_vae_dim": 64
+  },
+  "model_type": "vibevoice",
+  "semantic_tokenizer_config": {
+    "causal": true,
+    "channels": 1,
+    "conv_bias": true,
+    "conv_norm": "none",
+    "corpus_normalize": 0.0,
+    "disable_last_norm": true,
+    "encoder_depths": "3-3-3-3-3-3-8",
+    "encoder_n_filters": 32,
+    "encoder_ratios": [
+      8,
+      5,
+      5,
+      4,
+      2,
+      2
+    ],
+    "fix_std": 0,
+    "layer_scale_init_value": 1e-06,
+    "layernorm": "RMSNorm",
+    "layernorm_elementwise_affine": true,
+    "layernorm_eps": 1e-05,
+    "mixer_layer": "depthwise_conv",
+    "model_type": "vibevoice_semantic_tokenizer",
+    "pad_mode": "constant",
+    "std_dist_type": "none",
+    "vae_dim": 128,
+    "weight_init_value": 0.01
+  },
+  "semantic_vae_dim": 128,
+  "torch_dtype": "bfloat16",
+  "transformers_version": "4.51.3"
+}

demo/VibeVoice_colab.ipynb

@@ -0,0 +1,190 @@
+{
+  "cells": [
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "colab_type": "text",
+        "id": "view-in-github"
+      },
+      "source": [
+        "<a href=\"https://colab.research.google.com/github/microsoft/VibeVoice/blob/main/demo/VibeVoice_colab.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "id": "WvIaUJD2y0yU",
+      "metadata": {
+        "id": "WvIaUJD2y0yU"
+      },
+      "source": [
+        "# VibeVoice Colab — T4 Quickstart (1.5B)\n",
+        "\n",
+        "This notebook provides a quickstart guide to run VibeVoice on Colab with T4. The T4 GPU can only support the 1.5B model due to memory limitations. Please note that T4 can only use SDPA instead of flash_attention_2, which may result in unstable and lower audio quality. For the best TTS experience, we recommend trying the 7B model on a more powerful GPU.\n"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "id": "e8fTKYGx7DZk",
+      "metadata": {
+        "id": "e8fTKYGx7DZk"
+      },
+      "source": [
+        "## Step 1: Setup Environment"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "id": "4wxJ6QHM-ZOb",
+      "metadata": {
+        "id": "4wxJ6QHM-ZOb"
+      },
+      "outputs": [],
+      "source": [
+        "# Check for T4 GPU\n",
+        "import torch\n",
+        "if torch.cuda.is_available() and \"T4\" in torch.cuda.get_device_name(0):\n",
+        "    print(\"✅ T4 GPU detected\")\n",
+        "else:\n",
+        "    print(\"\"\"\n",
+        "    ⚠️ WARNING: T4 GPU not detected\n",
+        "\n",
+        "    The recommended runtime for this Colab notebook is \"T4 GPU\".\n",
+        "\n",
+        "    To change the runtime type:\n",
+        "\n",
+        "        1. Click on \"Runtime\" in the top navigation menu\n",
+        "        2. Click on \"Change runtime type\"\n",
+        "        3. Select \"T4 GPU\"\n",
+        "        4. Click \"OK\" if a \"Disconnect and delete runtime\" window appears\n",
+        "        5. Click on \"Save\"\n",
+        "\n",
+        "    \"\"\")\n",
+        "\n",
+        "# Clone the VibeVoice repository\n",
+        "![ -d /content/VibeVoice ] || git clone --quiet --branch main --depth 1 https://github.com/microsoft/VibeVoice.git /content/VibeVoice\n",
+        "print(\"✅ Cloned VibeVoice repository\")\n",
+        "\n",
+        "# Install project dependencies\n",
+        "!uv pip --quiet install --system -e /content/VibeVoice\n",
+        "print(\"✅ Installed dependencies\")\n",
+        "\n",
+        "# Download model (~3 minutes)\n",
+        "!HF_XET_HIGH_PERFORMANCE=1 hf download microsoft/VibeVoice-1.5B --quiet  --local-dir /content/models/VibeVoice-1.5B > /dev/null\n",
+        "print(\"✅ Downloaded model: microsoft/VibeVoice-1.5B\")\n"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "id": "pgKlV7153Ifi",
+      "metadata": {
+        "id": "pgKlV7153Ifi"
+      },
+      "source": [
+        "## Step 2: Create Transcript"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "id": "Yc1N9EHswFxA",
+      "metadata": {
+        "id": "Yc1N9EHswFxA"
+      },
+      "outputs": [],
+      "source": [
+        "%%writefile /content/my_transcript.txt\n",
+        "Speaker 1: Can I try VibeVoice with my own example?\n",
+        "Speaker 2: Of course! VibeVoice is open-source, built to benefit everyone - you're welcome to try it out.\n"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "id": "MBCC6s-F6_hP",
+      "metadata": {
+        "id": "MBCC6s-F6_hP"
+      },
+      "source": [
+        "## Step 3: Generate Audio"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "id": "dYWsLJ-n0Npm",
+      "metadata": {
+        "id": "dYWsLJ-n0Npm"
+      },
+      "outputs": [],
+      "source": [
+        "# Run Python script to generate audio from transcript\n",
+        "!python /content/VibeVoice/demo/inference_from_file.py \\\n",
+        "    --model_path /content/models/VibeVoice-1.5B \\\n",
+        "    --txt_path /content/my_transcript.txt \\\n",
+        "    --speaker_names Alice Frank\n",
+        "\n",
+        "# Display audio controls\n",
+        "from IPython.display import Audio\n",
+        "Audio(\"/content/outputs/my_transcript_generated.wav\")\n"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "id": "ec6438d5",
+      "metadata": {},
+      "source": [
+        "# Step 4: Download Audio"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "id": "b40ffa22",
+      "metadata": {},
+      "outputs": [],
+      "source": [
+        "from google.colab import files\n",
+        "files.download(\"/content/outputs/my_transcript_generated.wav\") \n"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "id": "1bce752d",
+      "metadata": {},
+      "source": [
+        "\n",
+        "## Risks and Limitations\n",
+        "\n",
+        "While efforts have been made to optimize it through various techniques, it may still produce outputs that are unexpected, biased, or inaccurate. VibeVoice inherits any biases, errors, or omissions produced by its base model (specifically, Qwen2.5 1.5b in this release). Potential for Deepfakes and Disinformation: High-quality synthetic speech can be misused to create convincing fake audio content for impersonation, fraud, or spreading disinformation. Users must ensure transcripts are reliable, check content accuracy, and avoid using generated content in misleading ways. Users are expected to use the generated content and to deploy the models in a lawful manner, in full compliance with all applicable laws and regulations in the relevant jurisdictions. It is best practice to disclose the use of AI when sharing AI-generated content."
+      ]
+    }
+  ],
+  "metadata": {
+    "accelerator": "GPU",
+    "colab": {
+      "gpuType": "T4",
+      "include_colab_link": true,
+      "machine_shape": "hm",
+      "name": "VibeVoice_Colab.ipynb",
+      "provenance": []
+    },
+    "kernelspec": {
+      "display_name": "Python 3",
+      "name": "python3"
+    },
+    "language_info": {
+      "codemirror_mode": {
+        "name": "ipython",
+        "version": 3
+      },
+      "file_extension": ".py",
+      "mimetype": "text/x-python",
+      "name": "python",
+      "nbconvert_exporter": "python",
+      "pygments_lexer": "ipython3",
+      "version": "3.10.11"
+    }
+  },
+  "nbformat": 4,
+  "nbformat_minor": 5
+}

demo/example/1p_EN2CH.mp4

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:309bb904d14fd9eb8c4d1fdd0555ef0cca3622592adeeae36b325a16b5f5c79c
+size 1150931

demo/example/2p_see_u_again.mp4

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b1eeb3819a31591d3d1c13dad4ec0c220dc22af20abd04f1f3699f6e30a8663e
+size 514352

demo/example/4p_climate_45min.mp4

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8f74874c9ea49d1f9e7c3b9d40d556fc729d9d828ac3d7fa10a504d4f68a0b80
+size 24147667

demo/gradio_demo.py

@@ -0,0 +1,1246 @@
+"""
+VibeVoice Gradio Demo - High-Quality Dialogue Generation Interface with Streaming Support
+"""
+
+import argparse
+import json
+import os
+import sys
+import tempfile
+import time
+from pathlib import Path
+from typing import List, Dict, Any, Iterator
+from datetime import datetime
+import threading
+import numpy as np
+import gradio as gr
+import librosa
+import soundfile as sf
+import torch
+import os
+import traceback
+
+from vibevoice.modular.configuration_vibevoice import VibeVoiceConfig
+from vibevoice.modular.modeling_vibevoice_inference import VibeVoiceForConditionalGenerationInference
+from vibevoice.processor.vibevoice_processor import VibeVoiceProcessor
+from vibevoice.modular.streamer import AudioStreamer
+from transformers.utils import logging
+from transformers import set_seed
+
+logging.set_verbosity_info()
+logger = logging.get_logger(__name__)
+
+
+class VibeVoiceDemo:
+    def __init__(self, model_path: str, device: str = "cuda", inference_steps: int = 5):
+        """Initialize the VibeVoice demo with model loading."""
+        self.model_path = model_path
+        self.device = device
+        self.inference_steps = inference_steps
+        self.is_generating = False  # Track generation state
+        self.stop_generation = False  # Flag to stop generation
+        self.current_streamer = None  # Track current audio streamer
+        self.load_model()
+        self.setup_voice_presets()
+        self.load_example_scripts()  # Load example scripts
+        
+    def load_model(self):
+        """Load the VibeVoice model and processor."""
+        print(f"Loading processor & model from {self.model_path}")
+        # Normalize potential 'mpx'
+        if self.device.lower() == "mpx":
+            print("Note: device 'mpx' detected, treating it as 'mps'.")
+            self.device = "mps"
+        if self.device == "mps" and not torch.backends.mps.is_available():
+            print("Warning: MPS not available. Falling back to CPU.")
+            self.device = "cpu"
+        print(f"Using device: {self.device}")
+        # Load processor
+        self.processor = VibeVoiceProcessor.from_pretrained(self.model_path)
+        # Decide dtype & attention
+        if self.device == "mps":
+            load_dtype = torch.float32
+            attn_impl_primary = "sdpa"
+        elif self.device == "cuda":
+            load_dtype = torch.bfloat16
+            attn_impl_primary = "flash_attention_2"
+        else:
+            load_dtype = torch.float32
+            attn_impl_primary = "sdpa"
+        print(f"Using device: {self.device}, torch_dtype: {load_dtype}, attn_implementation: {attn_impl_primary}")
+        # Load model
+        try:
+            if self.device == "mps":
+                self.model = VibeVoiceForConditionalGenerationInference.from_pretrained(
+                    self.model_path,
+                    torch_dtype=load_dtype,
+                    attn_implementation=attn_impl_primary,
+                    device_map=None,
+                )
+                self.model.to("mps")
+            elif self.device == "cuda":
+                self.model = VibeVoiceForConditionalGenerationInference.from_pretrained(
+                    self.model_path,
+                    torch_dtype=load_dtype,
+                    device_map="cuda",
+                    attn_implementation=attn_impl_primary,
+                )
+            else:
+                self.model = VibeVoiceForConditionalGenerationInference.from_pretrained(
+                    self.model_path,
+                    torch_dtype=load_dtype,
+                    device_map="cpu",
+                    attn_implementation=attn_impl_primary,
+                )
+        except Exception as e:
+            if attn_impl_primary == 'flash_attention_2':
+                print(f"[ERROR] : {type(e).__name__}: {e}")
+                print(traceback.format_exc())
+                fallback_attn = "sdpa"
+                print(f"Falling back to attention implementation: {fallback_attn}")
+                self.model = VibeVoiceForConditionalGenerationInference.from_pretrained(
+                    self.model_path,
+                    torch_dtype=load_dtype,
+                    device_map=(self.device if self.device in ("cuda", "cpu") else None),
+                    attn_implementation=fallback_attn,
+                )
+                if self.device == "mps":
+                    self.model.to("mps")
+            else:
+                raise e
+        self.model.eval()
+        
+        # Use SDE solver by default
+        self.model.model.noise_scheduler = self.model.model.noise_scheduler.from_config(
+            self.model.model.noise_scheduler.config, 
+            algorithm_type='sde-dpmsolver++',
+            beta_schedule='squaredcos_cap_v2'
+        )
+        self.model.set_ddpm_inference_steps(num_steps=self.inference_steps)
+        
+        if hasattr(self.model.model, 'language_model'):
+            print(f"Language model attention: {self.model.model.language_model.config._attn_implementation}")
+    
+    def setup_voice_presets(self):
+        """Setup voice presets by scanning the voices directory."""
+        voices_dir = os.path.join(os.path.dirname(__file__), "voices")
+        
+        # Check if voices directory exists
+        if not os.path.exists(voices_dir):
+            print(f"Warning: Voices directory not found at {voices_dir}")
+            self.voice_presets = {}
+            self.available_voices = {}
+            return
+        
+        # Scan for all WAV files in the voices directory
+        self.voice_presets = {}
+        
+        # Get all .wav files in the voices directory
+        wav_files = [f for f in os.listdir(voices_dir) 
+                    if f.lower().endswith(('.wav', '.mp3', '.flac', '.ogg', '.m4a', '.aac')) and os.path.isfile(os.path.join(voices_dir, f))]
+        
+        # Create dictionary with filename (without extension) as key
+        for wav_file in wav_files:
+            # Remove .wav extension to get the name
+            name = os.path.splitext(wav_file)[0]
+            # Create full path
+            full_path = os.path.join(voices_dir, wav_file)
+            self.voice_presets[name] = full_path
+        
+        # Sort the voice presets alphabetically by name for better UI
+        self.voice_presets = dict(sorted(self.voice_presets.items()))
+        
+        # Filter out voices that don't exist (this is now redundant but kept for safety)
+        self.available_voices = {
+            name: path for name, path in self.voice_presets.items()
+            if os.path.exists(path)
+        }
+        
+        if not self.available_voices:
+            raise gr.Error("No voice presets found. Please add .wav files to the demo/voices directory.")
+        
+        print(f"Found {len(self.available_voices)} voice files in {voices_dir}")
+        print(f"Available voices: {', '.join(self.available_voices.keys())}")
+    
+    def read_audio(self, audio_path: str, target_sr: int = 24000) -> np.ndarray:
+        """Read and preprocess audio file."""
+        try:
+            wav, sr = sf.read(audio_path)
+            if len(wav.shape) > 1:
+                wav = np.mean(wav, axis=1)
+            if sr != target_sr:
+                wav = librosa.resample(wav, orig_sr=sr, target_sr=target_sr)
+            return wav
+        except Exception as e:
+            print(f"Error reading audio {audio_path}: {e}")
+            return np.array([])
+    
+    def generate_podcast_streaming(self, 
+                                 num_speakers: int,
+                                 script: str,
+                                 speaker_1: str = None,
+                                 speaker_2: str = None,
+                                 speaker_3: str = None,
+                                 speaker_4: str = None,
+                                 cfg_scale: float = 1.3) -> Iterator[tuple]:
+        try:
+            
+            # Reset stop flag and set generating state
+            self.stop_generation = False
+            self.is_generating = True
+            
+            # Validate inputs
+            if not script.strip():
+                self.is_generating = False
+                raise gr.Error("Error: Please provide a script.")
+
+            # Defend against common mistake
+            script = script.replace("’", "'")
+            
+            if num_speakers < 1 or num_speakers > 4:
+                self.is_generating = False
+                raise gr.Error("Error: Number of speakers must be between 1 and 4.")
+            
+            # Collect selected speakers
+            selected_speakers = [speaker_1, speaker_2, speaker_3, speaker_4][:num_speakers]
+            
+            # Validate speaker selections
+            for i, speaker in enumerate(selected_speakers):
+                if not speaker or speaker not in self.available_voices:
+                    self.is_generating = False
+                    raise gr.Error(f"Error: Please select a valid speaker for Speaker {i+1}.")
+            
+            # Build initial log
+            log = f"🎙️ Generating podcast with {num_speakers} speakers\n"
+            log += f"📊 Parameters: CFG Scale={cfg_scale}, Inference Steps={self.inference_steps}\n"
+            log += f"🎭 Speakers: {', '.join(selected_speakers)}\n"
+            
+            # Check for stop signal
+            if self.stop_generation:
+                self.is_generating = False
+                yield None, "🛑 Generation stopped by user", gr.update(visible=False)
+                return
+            
+            # Load voice samples
+            voice_samples = []
+            for speaker_name in selected_speakers:
+                audio_path = self.available_voices[speaker_name]
+                audio_data = self.read_audio(audio_path)
+                if len(audio_data) == 0:
+                    self.is_generating = False
+                    raise gr.Error(f"Error: Failed to load audio for {speaker_name}")
+                voice_samples.append(audio_data)
+            
+            # log += f"✅ Loaded {len(voice_samples)} voice samples\n"
+            
+            # Check for stop signal
+            if self.stop_generation:
+                self.is_generating = False
+                yield None, "🛑 Generation stopped by user", gr.update(visible=False)
+                return
+            
+            # Parse script to assign speaker ID's
+            lines = script.strip().split('\n')
+            formatted_script_lines = []
+            
+            for line in lines:
+                line = line.strip()
+                if not line:
+                    continue
+                    
+                # Check if line already has speaker format
+                if line.startswith('Speaker ') and ':' in line:
+                    formatted_script_lines.append(line)
+                else:
+                    # Auto-assign to speakers in rotation
+                    speaker_id = len(formatted_script_lines) % num_speakers
+                    formatted_script_lines.append(f"Speaker {speaker_id}: {line}")
+            
+            formatted_script = '\n'.join(formatted_script_lines)
+            log += f"📝 Formatted script with {len(formatted_script_lines)} turns\n\n"
+            log += "🔄 Processing with VibeVoice (streaming mode)...\n"
+            
+            # Check for stop signal before processing
+            if self.stop_generation:
+                self.is_generating = False
+                yield None, "🛑 Generation stopped by user", gr.update(visible=False)
+                return
+            
+            start_time = time.time()
+            
+            inputs = self.processor(
+                text=[formatted_script],
+                voice_samples=[voice_samples],
+                padding=True,
+                return_tensors="pt",
+                return_attention_mask=True,
+            )
+            # Move tensors to device
+            target_device = self.device if self.device in ("cuda", "mps") else "cpu"
+            for k, v in inputs.items():
+                if torch.is_tensor(v):
+                    inputs[k] = v.to(target_device)
+            
+            # Create audio streamer
+            audio_streamer = AudioStreamer(
+                batch_size=1,
+                stop_signal=None,
+                timeout=None
+            )
+            
+            # Store current streamer for potential stopping
+            self.current_streamer = audio_streamer
+            
+            # Start generation in a separate thread
+            generation_thread = threading.Thread(
+                target=self._generate_with_streamer,
+                args=(inputs, cfg_scale, audio_streamer)
+            )
+            generation_thread.start()
+            
+            # Wait for generation to actually start producing audio
+            time.sleep(1)  # Reduced from 3 to 1 second
+
+            # Check for stop signal after thread start
+            if self.stop_generation:
+                audio_streamer.end()
+                generation_thread.join(timeout=5.0)  # Wait up to 5 seconds for thread to finish
+                self.is_generating = False
+                yield None, "🛑 Generation stopped by user", gr.update(visible=False)
+                return
+
+            # Collect audio chunks as they arrive
+            sample_rate = 24000
+            all_audio_chunks = []  # For final statistics
+            pending_chunks = []  # Buffer for accumulating small chunks
+            chunk_count = 0
+            last_yield_time = time.time()
+            min_yield_interval = 15 # Yield every 15 seconds
+            min_chunk_size = sample_rate * 30 # At least 2 seconds of audio
+            
+            # Get the stream for the first (and only) sample
+            audio_stream = audio_streamer.get_stream(0)
+            
+            has_yielded_audio = False
+            has_received_chunks = False  # Track if we received any chunks at all
+            
+            for audio_chunk in audio_stream:
+                # Check for stop signal in the streaming loop
+                if self.stop_generation:
+                    audio_streamer.end()
+                    break
+                    
+                chunk_count += 1
+                has_received_chunks = True  # Mark that we received at least one chunk
+                
+                # Convert tensor to numpy
+                if torch.is_tensor(audio_chunk):
+                    # Convert bfloat16 to float32 first, then to numpy
+                    if audio_chunk.dtype == torch.bfloat16:
+                        audio_chunk = audio_chunk.float()
+                    audio_np = audio_chunk.cpu().numpy().astype(np.float32)
+                else:
+                    audio_np = np.array(audio_chunk, dtype=np.float32)
+                
+                # Ensure audio is 1D and properly normalized
+                if len(audio_np.shape) > 1:
+                    audio_np = audio_np.squeeze()
+                
+                # Convert to 16-bit for Gradio
+                audio_16bit = convert_to_16_bit_wav(audio_np)
+                
+                # Store for final statistics
+                all_audio_chunks.append(audio_16bit)
+                
+                # Add to pending chunks buffer
+                pending_chunks.append(audio_16bit)
+                
+                # Calculate pending audio size
+                pending_audio_size = sum(len(chunk) for chunk in pending_chunks)
+                current_time = time.time()
+                time_since_last_yield = current_time - last_yield_time
+                
+                # Decide whether to yield
+                should_yield = False
+                if not has_yielded_audio and pending_audio_size >= min_chunk_size:
+                    # First yield: wait for minimum chunk size
+                    should_yield = True
+                    has_yielded_audio = True
+                elif has_yielded_audio and (pending_audio_size >= min_chunk_size or time_since_last_yield >= min_yield_interval):
+                    # Subsequent yields: either enough audio or enough time has passed
+                    should_yield = True
+                
+                if should_yield and pending_chunks:
+                    # Concatenate and yield only the new audio chunks
+                    new_audio = np.concatenate(pending_chunks)
+                    new_duration = len(new_audio) / sample_rate
+                    total_duration = sum(len(chunk) for chunk in all_audio_chunks) / sample_rate
+                    
+                    log_update = log + f"🎵 Streaming: {total_duration:.1f}s generated (chunk {chunk_count})\n"
+                    
+                    # Yield streaming audio chunk and keep complete_audio as None during streaming
+                    yield (sample_rate, new_audio), None, log_update, gr.update(visible=True)
+                    
+                    # Clear pending chunks after yielding
+                    pending_chunks = []
+                    last_yield_time = current_time
+            
+            # Yield any remaining chunks
+            if pending_chunks:
+                final_new_audio = np.concatenate(pending_chunks)
+                total_duration = sum(len(chunk) for chunk in all_audio_chunks) / sample_rate
+                log_update = log + f"🎵 Streaming final chunk: {total_duration:.1f}s total\n"
+                yield (sample_rate, final_new_audio), None, log_update, gr.update(visible=True)
+                has_yielded_audio = True  # Mark that we yielded audio
+            
+            # Wait for generation to complete (with timeout to prevent hanging)
+            generation_thread.join(timeout=5.0)  # Increased timeout to 5 seconds
+
+            # If thread is still alive after timeout, force end
+            if generation_thread.is_alive():
+                print("Warning: Generation thread did not complete within timeout")
+                audio_streamer.end()
+                generation_thread.join(timeout=5.0)
+
+            # Clean up
+            self.current_streamer = None
+            self.is_generating = False
+            
+            generation_time = time.time() - start_time
+            
+            # Check if stopped by user
+            if self.stop_generation:
+                yield None, None, "🛑 Generation stopped by user", gr.update(visible=False)
+                return
+            
+            # Debug logging
+            # print(f"Debug: has_received_chunks={has_received_chunks}, chunk_count={chunk_count}, all_audio_chunks length={len(all_audio_chunks)}")
+            
+            # Check if we received any chunks but didn't yield audio
+            if has_received_chunks and not has_yielded_audio and all_audio_chunks:
+                # We have chunks but didn't meet the yield criteria, yield them now
+                complete_audio = np.concatenate(all_audio_chunks)
+                final_duration = len(complete_audio) / sample_rate
+                
+                final_log = log + f"⏱️ Generation completed in {generation_time:.2f} seconds\n"
+                final_log += f"🎵 Final audio duration: {final_duration:.2f} seconds\n"
+                final_log += f"📊 Total chunks: {chunk_count}\n"
+                final_log += "✨ Generation successful! Complete audio is ready.\n"
+                final_log += "💡 Not satisfied? You can regenerate or adjust the CFG scale for different results."
+                
+                # Yield the complete audio
+                yield None, (sample_rate, complete_audio), final_log, gr.update(visible=False)
+                return
+            
+            if not has_received_chunks:
+                error_log = log + f"\n❌ Error: No audio chunks were received from the model. Generation time: {generation_time:.2f}s"
+                yield None, None, error_log, gr.update(visible=False)
+                return
+            
+            if not has_yielded_audio:
+                error_log = log + f"\n❌ Error: Audio was generated but not streamed. Chunk count: {chunk_count}"
+                yield None, None, error_log, gr.update(visible=False)
+                return
+
+            # Prepare the complete audio
+            if all_audio_chunks:
+                complete_audio = np.concatenate(all_audio_chunks)
+                final_duration = len(complete_audio) / sample_rate
+                
+                final_log = log + f"⏱️ Generation completed in {generation_time:.2f} seconds\n"
+                final_log += f"🎵 Final audio duration: {final_duration:.2f} seconds\n"
+                final_log += f"📊 Total chunks: {chunk_count}\n"
+                final_log += "✨ Generation successful! Complete audio is ready in the 'Complete Audio' tab.\n"
+                final_log += "💡 Not satisfied? You can regenerate or adjust the CFG scale for different results."
+                
+                # Final yield: Clear streaming audio and provide complete audio
+                yield None, (sample_rate, complete_audio), final_log, gr.update(visible=False)
+            else:
+                final_log = log + "❌ No audio was generated."
+                yield None, None, final_log, gr.update(visible=False)
+
+        except gr.Error as e:
+            # Handle Gradio-specific errors (like input validation)
+            self.is_generating = False
+            self.current_streamer = None
+            error_msg = f"❌ Input Error: {str(e)}"
+            print(error_msg)
+            yield None, None, error_msg, gr.update(visible=False)
+            
+        except Exception as e:
+            self.is_generating = False
+            self.current_streamer = None
+            error_msg = f"❌ An unexpected error occurred: {str(e)}"
+            print(error_msg)
+            import traceback
+            traceback.print_exc()
+            yield None, None, error_msg, gr.update(visible=False)
+    
+    def _generate_with_streamer(self, inputs, cfg_scale, audio_streamer):
+        """Helper method to run generation with streamer in a separate thread."""
+        try:
+            # Check for stop signal before starting generation
+            if self.stop_generation:
+                audio_streamer.end()
+                return
+                
+            # Define a stop check function that can be called from generate
+            def check_stop_generation():
+                return self.stop_generation
+                
+            outputs = self.model.generate(
+                **inputs,
+                max_new_tokens=None,
+                cfg_scale=cfg_scale,
+                tokenizer=self.processor.tokenizer,
+                generation_config={
+                    'do_sample': False,
+                },
+                audio_streamer=audio_streamer,
+                stop_check_fn=check_stop_generation,  # Pass the stop check function
+                verbose=False,  # Disable verbose in streaming mode
+                refresh_negative=True,
+            )
+            
+        except Exception as e:
+            print(f"Error in generation thread: {e}")
+            traceback.print_exc()
+            # Make sure to end the stream on error
+            audio_streamer.end()
+    
+    def stop_audio_generation(self):
+        """Stop the current audio generation process."""
+        self.stop_generation = True
+        if self.current_streamer is not None:
+            try:
+                self.current_streamer.end()
+            except Exception as e:
+                print(f"Error stopping streamer: {e}")
+        print("🛑 Audio generation stop requested")
+    
+    def load_example_scripts(self):
+        """Load example scripts from the text_examples directory."""
+        examples_dir = os.path.join(os.path.dirname(__file__), "text_examples")
+        self.example_scripts = []
+        
+        # Check if text_examples directory exists
+        if not os.path.exists(examples_dir):
+            print(f"Warning: text_examples directory not found at {examples_dir}")
+            return
+        
+        # Get all .txt files in the text_examples directory
+        txt_files = sorted([f for f in os.listdir(examples_dir) 
+                          if f.lower().endswith('.txt') and os.path.isfile(os.path.join(examples_dir, f))])
+        
+        for txt_file in txt_files:
+            file_path = os.path.join(examples_dir, txt_file)
+            
+            import re
+            # Check if filename contains a time pattern like "45min", "90min", etc.
+            time_pattern = re.search(r'(\d+)min', txt_file.lower())
+            if time_pattern:
+                minutes = int(time_pattern.group(1))
+                if minutes > 15:
+                    print(f"Skipping {txt_file}: duration {minutes} minutes exceeds 15-minute limit")
+                    continue
+
+            try:
+                with open(file_path, 'r', encoding='utf-8') as f:
+                    script_content = f.read().strip()
+                
+                # Remove empty lines and lines with only whitespace
+                script_content = '\n'.join(line for line in script_content.split('\n') if line.strip())
+                
+                if not script_content:
+                    continue
+                
+                # Parse the script to determine number of speakers
+                num_speakers = self._get_num_speakers_from_script(script_content)
+                
+                # Add to examples list as [num_speakers, script_content]
+                self.example_scripts.append([num_speakers, script_content])
+                print(f"Loaded example: {txt_file} with {num_speakers} speakers")
+                
+            except Exception as e:
+                print(f"Error loading example script {txt_file}: {e}")
+        
+        if self.example_scripts:
+            print(f"Successfully loaded {len(self.example_scripts)} example scripts")
+        else:
+            print("No example scripts were loaded")
+    
+    def _get_num_speakers_from_script(self, script: str) -> int:
+        """Determine the number of unique speakers in a script."""
+        import re
+        speakers = set()
+        
+        lines = script.strip().split('\n')
+        for line in lines:
+            # Use regex to find speaker patterns
+            match = re.match(r'^Speaker\s+(\d+)\s*:', line.strip(), re.IGNORECASE)
+            if match:
+                speaker_id = int(match.group(1))
+                speakers.add(speaker_id)
+        
+        # If no speakers found, default to 1
+        if not speakers:
+            return 1
+        
+        # Return the maximum speaker ID + 1 (assuming 0-based indexing)
+        # or the count of unique speakers if they're 1-based
+        max_speaker = max(speakers)
+        min_speaker = min(speakers)
+        
+        if min_speaker == 0:
+            return max_speaker + 1
+        else:
+            # Assume 1-based indexing, return the count
+            return len(speakers)
+    
+
+def create_demo_interface(demo_instance: VibeVoiceDemo):
+    """Create the Gradio interface with streaming support."""
+    
+    # Custom CSS for high-end aesthetics with lighter theme
+    custom_css = """
+    /* Modern light theme with gradients */
+    .gradio-container {
+        background: linear-gradient(135deg, #f8fafc 0%, #e2e8f0 100%);
+        font-family: 'SF Pro Display', -apple-system, BlinkMacSystemFont, sans-serif;
+    }
+    
+    /* Header styling */
+    .main-header {
+        background: linear-gradient(90deg, #667eea 0%, #764ba2 100%);
+        padding: 2rem;
+        border-radius: 20px;
+        margin-bottom: 2rem;
+        text-align: center;
+        box-shadow: 0 10px 40px rgba(102, 126, 234, 0.3);
+    }
+    
+    .main-header h1 {
+        color: white;
+        font-size: 2.5rem;
+        font-weight: 700;
+        margin: 0;
+        text-shadow: 0 2px 4px rgba(0,0,0,0.3);
+    }
+    
+    .main-header p {
+        color: rgba(255,255,255,0.9);
+        font-size: 1.1rem;
+        margin: 0.5rem 0 0 0;
+    }
+    
+    /* Card styling */
+    .settings-card, .generation-card {
+        background: rgba(255, 255, 255, 0.8);
+        backdrop-filter: blur(10px);
+        border: 1px solid rgba(226, 232, 240, 0.8);
+        border-radius: 16px;
+        padding: 1.5rem;
+        margin-bottom: 1rem;
+        box-shadow: 0 8px 32px rgba(0, 0, 0, 0.1);
+    }
+    
+    /* Speaker selection styling */
+    .speaker-grid {
+        display: grid;
+        gap: 1rem;
+        margin-bottom: 1rem;
+    }
+    
+    .speaker-item {
+        background: linear-gradient(135deg, #e2e8f0 0%, #cbd5e1 100%);
+        border: 1px solid rgba(148, 163, 184, 0.4);
+        border-radius: 12px;
+        padding: 1rem;
+        color: #374151;
+        font-weight: 500;
+    }
+    
+    /* Streaming indicator */
+    .streaming-indicator {
+        display: inline-block;
+        width: 10px;
+        height: 10px;
+        background: #22c55e;
+        border-radius: 50%;
+        margin-right: 8px;
+        animation: pulse 1.5s infinite;
+    }
+    
+    @keyframes pulse {
+        0% { opacity: 1; transform: scale(1); }
+        50% { opacity: 0.5; transform: scale(1.1); }
+        100% { opacity: 1; transform: scale(1); }
+    }
+    
+    /* Queue status styling */
+    .queue-status {
+        background: linear-gradient(135deg, #f0f9ff 0%, #e0f2fe 100%);
+        border: 1px solid rgba(14, 165, 233, 0.3);
+        border-radius: 8px;
+        padding: 0.75rem;
+        margin: 0.5rem 0;
+        text-align: center;
+        font-size: 0.9rem;
+        color: #0369a1;
+    }
+    
+    .generate-btn {
+        background: linear-gradient(135deg, #059669 0%, #0d9488 100%);
+        border: none;
+        border-radius: 12px;
+        padding: 1rem 2rem;
+        color: white;
+        font-weight: 600;
+        font-size: 1.1rem;
+        box-shadow: 0 4px 20px rgba(5, 150, 105, 0.4);
+        transition: all 0.3s ease;
+    }
+    
+    .generate-btn:hover {
+        transform: translateY(-2px);
+        box-shadow: 0 6px 25px rgba(5, 150, 105, 0.6);
+    }
+    
+    .stop-btn {
+        background: linear-gradient(135deg, #ef4444 0%, #dc2626 100%);
+        border: none;
+        border-radius: 12px;
+        padding: 1rem 2rem;
+        color: white;
+        font-weight: 600;
+        font-size: 1.1rem;
+        box-shadow: 0 4px 20px rgba(239, 68, 68, 0.4);
+        transition: all 0.3s ease;
+    }
+    
+    .stop-btn:hover {
+        transform: translateY(-2px);
+        box-shadow: 0 6px 25px rgba(239, 68, 68, 0.6);
+    }
+    
+    /* Audio player styling */
+    .audio-output {
+        background: linear-gradient(135deg, #f1f5f9 0%, #e2e8f0 100%);
+        border-radius: 16px;
+        padding: 1.5rem;
+        border: 1px solid rgba(148, 163, 184, 0.3);
+    }
+    
+    .complete-audio-section {
+        margin-top: 1rem;
+        padding: 1rem;
+        background: linear-gradient(135deg, #f0fdf4 0%, #dcfce7 100%);
+        border: 1px solid rgba(34, 197, 94, 0.3);
+        border-radius: 12px;
+    }
+    
+    /* Text areas */
+    .script-input, .log-output {
+        background: rgba(255, 255, 255, 0.9) !important;
+        border: 1px solid rgba(148, 163, 184, 0.4) !important;
+        border-radius: 12px !important;
+        color: #1e293b !important;
+        font-family: 'JetBrains Mono', monospace !important;
+    }
+    
+    .script-input::placeholder {
+        color: #64748b !important;
+    }
+    
+    /* Sliders */
+    .slider-container {
+        background: rgba(248, 250, 252, 0.8);
+        border: 1px solid rgba(226, 232, 240, 0.6);
+        border-radius: 8px;
+        padding: 1rem;
+        margin: 0.5rem 0;
+    }
+    
+    /* Labels and text */
+    .gradio-container label {
+        color: #374151 !important;
+        font-weight: 600 !important;
+    }
+    
+    .gradio-container .markdown {
+        color: #1f2937 !important;
+    }
+    
+    /* Responsive design */
+    @media (max-width: 768px) {
+        .main-header h1 { font-size: 2rem; }
+        .settings-card, .generation-card { padding: 1rem; }
+    }
+    
+    /* Random example button styling - more subtle professional color */
+    .random-btn {
+        background: linear-gradient(135deg, #64748b 0%, #475569 100%);
+        border: none;
+        border-radius: 12px;
+        padding: 1rem 1.5rem;
+        color: white;
+        font-weight: 600;
+        font-size: 1rem;
+        box-shadow: 0 4px 20px rgba(100, 116, 139, 0.3);
+        transition: all 0.3s ease;
+        display: inline-flex;
+        align-items: center;
+        gap: 0.5rem;
+    }
+    
+    .random-btn:hover {
+        transform: translateY(-2px);
+        box-shadow: 0 6px 25px rgba(100, 116, 139, 0.4);
+        background: linear-gradient(135deg, #475569 0%, #334155 100%);
+    }
+    """
+    
+    with gr.Blocks(
+        title="VibeVoice - AI Podcast Generator",
+        css=custom_css,
+        theme=gr.themes.Soft(
+            primary_hue="blue",
+            secondary_hue="purple",
+            neutral_hue="slate",
+        )
+    ) as interface:
+        
+        # Header
+        gr.HTML("""
+        <div class="main-header">
+            <h1>🎙️ Vibe Podcasting </h1>
+            <p>Generating Long-form Multi-speaker AI Podcast with VibeVoice</p>
+        </div>
+        """)
+        
+        with gr.Row():
+            # Left column - Settings
+            with gr.Column(scale=1, elem_classes="settings-card"):
+                gr.Markdown("### 🎛️ **Podcast Settings**")
+                
+                # Number of speakers
+                num_speakers = gr.Slider(
+                    minimum=1,
+                    maximum=4,
+                    value=2,
+                    step=1,
+                    label="Number of Speakers",
+                    elem_classes="slider-container"
+                )
+                
+                # Speaker selection
+                gr.Markdown("### 🎭 **Speaker Selection**")
+                
+                available_speaker_names = list(demo_instance.available_voices.keys())
+                # default_speakers = available_speaker_names[:4] if len(available_speaker_names) >= 4 else available_speaker_names
+                default_speakers = ['en-Alice_woman', 'en-Carter_man', 'en-Frank_man', 'en-Maya_woman']
+
+                speaker_selections = []
+                for i in range(4):
+                    default_value = default_speakers[i] if i < len(default_speakers) else None
+                    speaker = gr.Dropdown(
+                        choices=available_speaker_names,
+                        value=default_value,
+                        label=f"Speaker {i+1}",
+                        visible=(i < 2),  # Initially show only first 2 speakers
+                        elem_classes="speaker-item"
+                    )
+                    speaker_selections.append(speaker)
+                
+                # Advanced settings
+                gr.Markdown("### ⚙️ **Advanced Settings**")
+                
+                # Sampling parameters (contains all generation settings)
+                with gr.Accordion("Generation Parameters", open=False):
+                    cfg_scale = gr.Slider(
+                        minimum=1.0,
+                        maximum=2.0,
+                        value=1.3,
+                        step=0.05,
+                        label="CFG Scale (Guidance Strength)",
+                        # info="Higher values increase adherence to text",
+                        elem_classes="slider-container"
+                    )
+                
+            # Right column - Generation
+            with gr.Column(scale=2, elem_classes="generation-card"):
+                gr.Markdown("### 📝 **Script Input**")
+                
+                script_input = gr.Textbox(
+                    label="Conversation Script",
+                    placeholder="""Enter your podcast script here. You can format it as:
+
+Speaker 1: Welcome to our podcast today!
+Speaker 2: Thanks for having me. I'm excited to discuss...
+
+Or paste text directly and it will auto-assign speakers.""",
+                    lines=12,
+                    max_lines=20,
+                    elem_classes="script-input"
+                )
+                
+                # Button row with Random Example on the left and Generate on the right
+                with gr.Row():
+                    # Random example button (now on the left)
+                    random_example_btn = gr.Button(
+                        "🎲 Random Example",
+                        size="lg",
+                        variant="secondary",
+                        elem_classes="random-btn",
+                        scale=1  # Smaller width
+                    )
+                    
+                    # Generate button (now on the right)
+                    generate_btn = gr.Button(
+                        "🚀 Generate Podcast",
+                        size="lg",
+                        variant="primary",
+                        elem_classes="generate-btn",
+                        scale=2  # Wider than random button
+                    )
+                
+                # Stop button
+                stop_btn = gr.Button(
+                    "🛑 Stop Generation",
+                    size="lg",
+                    variant="stop",
+                    elem_classes="stop-btn",
+                    visible=False
+                )
+                
+                # Streaming status indicator
+                streaming_status = gr.HTML(
+                    value="""
+                    <div style="background: linear-gradient(135deg, #dcfce7 0%, #bbf7d0 100%); 
+                                border: 1px solid rgba(34, 197, 94, 0.3); 
+                                border-radius: 8px; 
+                                padding: 0.75rem; 
+                                margin: 0.5rem 0;
+                                text-align: center;
+                                font-size: 0.9rem;
+                                color: #166534;">
+                        <span class="streaming-indicator"></span>
+                        <strong>LIVE STREAMING</strong> - Audio is being generated in real-time
+                    </div>
+                    """,
+                    visible=False,
+                    elem_id="streaming-status"
+                )
+                
+                # Output section
+                gr.Markdown("### 🎵 **Generated Podcast**")
+                
+                # Streaming audio output (outside of tabs for simpler handling)
+                audio_output = gr.Audio(
+                    label="Streaming Audio (Real-time)",
+                    type="numpy",
+                    elem_classes="audio-output",
+                    streaming=True,  # Enable streaming mode
+                    autoplay=True,
+                    show_download_button=False,  # Explicitly show download button
+                    visible=True
+                )
+                
+                # Complete audio output (non-streaming)
+                complete_audio_output = gr.Audio(
+                    label="Complete Podcast (Download after generation)",
+                    type="numpy",
+                    elem_classes="audio-output complete-audio-section",
+                    streaming=False,  # Non-streaming mode
+                    autoplay=False,
+                    show_download_button=True,  # Explicitly show download button
+                    visible=False  # Initially hidden, shown when audio is ready
+                )
+                
+                gr.Markdown("""
+                *💡 **Streaming**: Audio plays as it's being generated (may have slight pauses)  
+                *💡 **Complete Audio**: Will appear below after generation finishes*
+                """)
+                
+                # Generation log
+                log_output = gr.Textbox(
+                    label="Generation Log",
+                    lines=8,
+                    max_lines=15,
+                    interactive=False,
+                    elem_classes="log-output"
+                )
+        
+        def update_speaker_visibility(num_speakers):
+            updates = []
+            for i in range(4):
+                updates.append(gr.update(visible=(i < num_speakers)))
+            return updates
+        
+        num_speakers.change(
+            fn=update_speaker_visibility,
+            inputs=[num_speakers],
+            outputs=speaker_selections
+        )
+        
+        # Main generation function with streaming
+        def generate_podcast_wrapper(num_speakers, script, *speakers_and_params):
+            """Wrapper function to handle the streaming generation call."""
+            try:
+                # Extract speakers and parameters
+                speakers = speakers_and_params[:4]  # First 4 are speaker selections
+                cfg_scale = speakers_and_params[4]   # CFG scale
+                
+                # Clear outputs and reset visibility at start
+                yield None, gr.update(value=None, visible=False), "🎙️ Starting generation...", gr.update(visible=True), gr.update(visible=False), gr.update(visible=True)
+                
+                # The generator will yield multiple times
+                final_log = "Starting generation..."
+                
+                for streaming_audio, complete_audio, log, streaming_visible in demo_instance.generate_podcast_streaming(
+                    num_speakers=int(num_speakers),
+                    script=script,
+                    speaker_1=speakers[0],
+                    speaker_2=speakers[1],
+                    speaker_3=speakers[2],
+                    speaker_4=speakers[3],
+                    cfg_scale=cfg_scale
+                ):
+                    final_log = log
+                    
+                    # Check if we have complete audio (final yield)
+                    if complete_audio is not None:
+                        # Final state: clear streaming, show complete audio
+                        yield None, gr.update(value=complete_audio, visible=True), log, gr.update(visible=False), gr.update(visible=True), gr.update(visible=False)
+                    else:
+                        # Streaming state: update streaming audio only
+                        if streaming_audio is not None:
+                            yield streaming_audio, gr.update(visible=False), log, streaming_visible, gr.update(visible=False), gr.update(visible=True)
+                        else:
+                            # No new audio, just update status
+                            yield None, gr.update(visible=False), log, streaming_visible, gr.update(visible=False), gr.update(visible=True)
+
+            except Exception as e:
+                error_msg = f"❌ A critical error occurred in the wrapper: {str(e)}"
+                print(error_msg)
+                import traceback
+                traceback.print_exc()
+                # Reset button states on error
+                yield None, gr.update(value=None, visible=False), error_msg, gr.update(visible=False), gr.update(visible=True), gr.update(visible=False)
+        
+        def stop_generation_handler():
+            """Handle stopping generation."""
+            demo_instance.stop_audio_generation()
+            # Return values for: log_output, streaming_status, generate_btn, stop_btn
+            return "🛑 Generation stopped.", gr.update(visible=False), gr.update(visible=True), gr.update(visible=False)
+        
+        # Add a clear audio function
+        def clear_audio_outputs():
+            """Clear both audio outputs before starting new generation."""
+            return None, gr.update(value=None, visible=False)
+
+        # Connect generation button with streaming outputs
+        generate_btn.click(
+            fn=clear_audio_outputs,
+            inputs=[],
+            outputs=[audio_output, complete_audio_output],
+            queue=False
+        ).then(  # Immediate UI update to hide Generate, show Stop (non-queued)
+            fn=lambda: (gr.update(visible=False), gr.update(visible=True)),
+            inputs=[],
+            outputs=[generate_btn, stop_btn],
+            queue=False
+        ).then(
+            fn=generate_podcast_wrapper,
+            inputs=[num_speakers, script_input] + speaker_selections + [cfg_scale],
+            outputs=[audio_output, complete_audio_output, log_output, streaming_status, generate_btn, stop_btn],
+            queue=True  # Enable Gradio's built-in queue
+        )
+        
+        # Connect stop button
+        stop_btn.click(
+            fn=stop_generation_handler,
+            inputs=[],
+            outputs=[log_output, streaming_status, generate_btn, stop_btn],
+            queue=False  # Don't queue stop requests
+        ).then(
+            # Clear both audio outputs after stopping
+            fn=lambda: (None, None),
+            inputs=[],
+            outputs=[audio_output, complete_audio_output],
+            queue=False
+        )
+        
+        # Function to randomly select an example
+        def load_random_example():
+            """Randomly select and load an example script."""
+            import random
+            
+            # Get available examples
+            if hasattr(demo_instance, 'example_scripts') and demo_instance.example_scripts:
+                example_scripts = demo_instance.example_scripts
+            else:
+                # Fallback to default
+                example_scripts = [
+                    [2, "Speaker 0: Welcome to our AI podcast demonstration!\nSpeaker 1: Thanks for having me. This is exciting!"]
+                ]
+            
+            # Randomly select one
+            if example_scripts:
+                selected = random.choice(example_scripts)
+                num_speakers_value = selected[0]
+                script_value = selected[1]
+                
+                # Return the values to update the UI
+                return num_speakers_value, script_value
+            
+            # Default values if no examples
+            return 2, ""
+        
+        # Connect random example button
+        random_example_btn.click(
+            fn=load_random_example,
+            inputs=[],
+            outputs=[num_speakers, script_input],
+            queue=False  # Don't queue this simple operation
+        )
+        
+        # Add usage tips
+        gr.Markdown("""
+        ### 💡 **Usage Tips**
+        
+        - Click **🚀 Generate Podcast** to start audio generation
+        - **Live Streaming** tab shows audio as it's generated (may have slight pauses)
+        - **Complete Audio** tab provides the full, uninterrupted podcast after generation
+        - During generation, you can click **🛑 Stop Generation** to interrupt the process
+        - The streaming indicator shows real-time generation progress
+        """)
+        
+        # Add example scripts
+        gr.Markdown("### 📚 **Example Scripts**")
+        
+        # Use dynamically loaded examples if available, otherwise provide a default
+        if hasattr(demo_instance, 'example_scripts') and demo_instance.example_scripts:
+            example_scripts = demo_instance.example_scripts
+        else:
+            # Fallback to a simple default example if no scripts loaded
+            example_scripts = [
+                [1, "Speaker 1: Welcome to our AI podcast demonstration! This is a sample script showing how VibeVoice can generate natural-sounding speech."]
+            ]
+        
+        gr.Examples(
+            examples=example_scripts,
+            inputs=[num_speakers, script_input],
+            label="Try these example scripts:"
+        )
+
+        # --- Risks & limitations (footer) ---
+        gr.Markdown(
+            """
+## Risks and limitations
+
+While efforts have been made to optimize it through various techniques, it may still produce outputs that are unexpected, biased, or inaccurate. VibeVoice inherits any biases, errors, or omissions produced by its base model (specifically, Qwen2.5 1.5b in this release).
+Potential for Deepfakes and Disinformation: High-quality synthetic speech can be misused to create convincing fake audio content for impersonation, fraud, or spreading disinformation. Users must ensure transcripts are reliable, check content accuracy, and avoid using generated content in misleading ways. Users are expected to use the generated content and to deploy the models in a lawful manner, in full compliance with all applicable laws and regulations in the relevant jurisdictions. It is best practice to disclose the use of AI when sharing AI-generated content.
+            """,
+            elem_classes="generation-card",  # 可选：复用卡片样式
+        )
+    return interface
+
+
+def convert_to_16_bit_wav(data):
+    # Check if data is a tensor and move to cpu
+    if torch.is_tensor(data):
+        data = data.detach().cpu().numpy()
+    
+    # Ensure data is numpy array
+    data = np.array(data)
+
+    # Normalize to range [-1, 1] if it's not already
+    if np.max(np.abs(data)) > 1.0:
+        data = data / np.max(np.abs(data))
+    
+    # Scale to 16-bit integer range
+    data = (data * 32767).astype(np.int16)
+    return data
+
+
+def parse_args():
+    parser = argparse.ArgumentParser(description="VibeVoice Gradio Demo")
+    parser.add_argument(
+        "--model_path",
+        type=str,
+        default="/tmp/vibevoice-model",
+        help="Path to the VibeVoice model directory",
+    )
+    parser.add_argument(
+        "--device",
+        type=str,
+        default=("cuda" if torch.cuda.is_available() else ("mps" if torch.backends.mps.is_available() else "cpu")),
+        help="Device for inference: cuda | mps | cpu",
+    )
+    parser.add_argument(
+        "--inference_steps",
+        type=int,
+        default=10,
+        help="Number of inference steps for DDPM (not exposed to users)",
+    )
+    parser.add_argument(
+        "--share",
+        action="store_true",
+        help="Share the demo publicly via Gradio",
+    )
+    parser.add_argument(
+        "--port",
+        type=int,
+        default=7860,
+        help="Port to run the demo on",
+    )
+    
+    return parser.parse_args()
+
+
+def main():
+    """Main function to run the demo."""
+    args = parse_args()
+    
+    set_seed(42)  # Set a fixed seed for reproducibility
+
+    print("🎙️ Initializing VibeVoice Demo with Streaming Support...")
+    
+    # Initialize demo instance
+    demo_instance = VibeVoiceDemo(
+        model_path=args.model_path,
+        device=args.device,
+        inference_steps=args.inference_steps
+    )
+    
+    # Create interface
+    interface = create_demo_interface(demo_instance)
+    
+    print(f"🚀 Launching demo on port {args.port}")
+    print(f"📁 Model path: {args.model_path}")
+    print(f"🎭 Available voices: {len(demo_instance.available_voices)}")
+    print(f"🔴 Streaming mode: ENABLED")
+    print(f"🔒 Session isolation: ENABLED")
+    
+    # Launch the interface
+    try:
+        interface.queue(
+            max_size=20,  # Maximum queue size
+            default_concurrency_limit=1  # Process one request at a time
+        ).launch(
+            share=args.share,
+            # server_port=args.port,
+            server_name="0.0.0.0" if args.share else "127.0.0.1",
+            show_error=True,
+            show_api=False  # Hide API docs for cleaner interface
+        )
+    except KeyboardInterrupt:
+        print("\n🛑 Shutting down gracefully...")
+    except Exception as e:
+        print(f"❌ Server error: {e}")
+        raise
+
+
+if __name__ == "__main__":
+    main()

demo/inference_from_file.py

@@ -0,0 +1,400 @@
+import argparse
+import os
+import re
+import traceback
+from typing import List, Tuple, Union, Dict, Any
+import time
+import torch
+
+from vibevoice.modular.modeling_vibevoice_inference import VibeVoiceForConditionalGenerationInference
+from vibevoice.processor.vibevoice_processor import VibeVoiceProcessor
+from transformers.utils import logging
+
+logging.set_verbosity_info()
+logger = logging.get_logger(__name__)
+
+
+class VoiceMapper:
+    """Maps speaker names to voice file paths"""
+    
+    def __init__(self):
+        self.setup_voice_presets()
+
+        # change name according to our preset wav file
+        new_dict = {}
+        for name, path in self.voice_presets.items():
+            
+            if '_' in name:
+                name = name.split('_')[0]
+            
+            if '-' in name:
+                name = name.split('-')[-1]
+
+            new_dict[name] = path
+        self.voice_presets.update(new_dict)
+        # print(list(self.voice_presets.keys()))
+
+    def setup_voice_presets(self):
+        """Setup voice presets by scanning the voices directory."""
+        voices_dir = os.path.join(os.path.dirname(__file__), "voices")
+        
+        # Check if voices directory exists
+        if not os.path.exists(voices_dir):
+            print(f"Warning: Voices directory not found at {voices_dir}")
+            self.voice_presets = {}
+            self.available_voices = {}
+            return
+        
+        # Scan for all WAV files in the voices directory
+        self.voice_presets = {}
+        
+        # Get all .wav files in the voices directory
+        wav_files = [f for f in os.listdir(voices_dir) 
+                    if f.lower().endswith('.wav') and os.path.isfile(os.path.join(voices_dir, f))]
+        
+        # Create dictionary with filename (without extension) as key
+        for wav_file in wav_files:
+            # Remove .wav extension to get the name
+            name = os.path.splitext(wav_file)[0]
+            # Create full path
+            full_path = os.path.join(voices_dir, wav_file)
+            self.voice_presets[name] = full_path
+        
+        # Sort the voice presets alphabetically by name for better UI
+        self.voice_presets = dict(sorted(self.voice_presets.items()))
+        
+        # Filter out voices that don't exist (this is now redundant but kept for safety)
+        self.available_voices = {
+            name: path for name, path in self.voice_presets.items()
+            if os.path.exists(path)
+        }
+        
+        print(f"Found {len(self.available_voices)} voice files in {voices_dir}")
+        print(f"Available voices: {', '.join(self.available_voices.keys())}")
+
+    def get_voice_path(self, speaker_name: str) -> str:
+        """Get voice file path for a given speaker name"""
+        # First try exact match
+        if speaker_name in self.voice_presets:
+            return self.voice_presets[speaker_name]
+        
+        # Try partial matching (case insensitive)
+        speaker_lower = speaker_name.lower()
+        for preset_name, path in self.voice_presets.items():
+            if preset_name.lower() in speaker_lower or speaker_lower in preset_name.lower():
+                return path
+        
+        # Default to first voice if no match found
+        default_voice = list(self.voice_presets.values())[0]
+        print(f"Warning: No voice preset found for '{speaker_name}', using default voice: {default_voice}")
+        return default_voice
+
+
+def parse_txt_script(txt_content: str) -> Tuple[List[str], List[str]]:
+    """
+    Parse txt script content and extract speakers and their text
+    Fixed pattern: Speaker 1, Speaker 2, Speaker 3, Speaker 4
+    Returns: (scripts, speaker_numbers)
+    """
+    lines = txt_content.strip().split('\n')
+    scripts = []
+    speaker_numbers = []
+    
+    # Pattern to match "Speaker X:" format where X is a number
+    speaker_pattern = r'^Speaker\s+(\d+):\s*(.*)$'
+    
+    current_speaker = None
+    current_text = ""
+    
+    for line in lines:
+        line = line.strip()
+        if not line:
+            continue
+            
+        match = re.match(speaker_pattern, line, re.IGNORECASE)
+        if match:
+            # If we have accumulated text from previous speaker, save it
+            if current_speaker and current_text:
+                scripts.append(f"Speaker {current_speaker}: {current_text.strip()}")
+                speaker_numbers.append(current_speaker)
+            
+            # Start new speaker
+            current_speaker = match.group(1).strip()
+            current_text = match.group(2).strip()
+        else:
+            # Continue text for current speaker
+            if current_text:
+                current_text += " " + line
+            else:
+                current_text = line
+    
+    # Don't forget the last speaker
+    if current_speaker and current_text:
+        scripts.append(f"Speaker {current_speaker}: {current_text.strip()}")
+        speaker_numbers.append(current_speaker)
+    
+    return scripts, speaker_numbers
+
+
+def parse_args():
+    parser = argparse.ArgumentParser(description="VibeVoice Processor TXT Input Test")
+    parser.add_argument(
+        "--model_path",
+        type=str,
+        default="microsoft/VibeVoice-1.5b",
+        help="Path to the HuggingFace model directory",
+    )
+    
+    parser.add_argument(
+        "--txt_path",
+        type=str,
+        default="demo/text_examples/1p_abs.txt",
+        help="Path to the txt file containing the script",
+    )
+    parser.add_argument(
+        "--speaker_names",
+        type=str,
+        nargs='+',
+        default='Andrew',
+        help="Speaker names in order (e.g., --speaker_names Andrew Ava 'Bill Gates')",
+    )
+    parser.add_argument(
+        "--output_dir",
+        type=str,
+        default="./outputs",
+        help="Directory to save output audio files",
+    )
+    parser.add_argument(
+        "--device",
+        type=str,
+        default=("cuda" if torch.cuda.is_available() else ("mps" if torch.backends.mps.is_available() else "cpu")),
+        help="Device for inference: cuda | mps | cpu",
+    )
+    parser.add_argument(
+        "--cfg_scale",
+        type=float,
+        default=1.3,
+        help="CFG (Classifier-Free Guidance) scale for generation (default: 1.3)",
+    )
+    
+    return parser.parse_args()
+
+def main():
+    args = parse_args()
+
+    # Normalize potential 'mpx' typo to 'mps'
+    if args.device.lower() == "mpx":
+        print("Note: device 'mpx' detected, treating it as 'mps'.")
+        args.device = "mps"
+
+    # Validate mps availability if requested
+    if args.device == "mps" and not torch.backends.mps.is_available():
+        print("Warning: MPS not available. Falling back to CPU.")
+        args.device = "cpu"
+
+    print(f"Using device: {args.device}")
+
+    # Initialize voice mapper
+    voice_mapper = VoiceMapper()
+    
+    # Check if txt file exists
+    if not os.path.exists(args.txt_path):
+        print(f"Error: txt file not found: {args.txt_path}")
+        return
+    
+    # Read and parse txt file
+    print(f"Reading script from: {args.txt_path}")
+    with open(args.txt_path, 'r', encoding='utf-8') as f:
+        txt_content = f.read()
+    
+    # Parse the txt content to get speaker numbers
+    scripts, speaker_numbers = parse_txt_script(txt_content)
+    
+    if not scripts:
+        print("Error: No valid speaker scripts found in the txt file")
+        return
+    
+    print(f"Found {len(scripts)} speaker segments:")
+    for i, (script, speaker_num) in enumerate(zip(scripts, speaker_numbers)):
+        print(f"  {i+1}. Speaker {speaker_num}")
+        print(f"     Text preview: {script[:100]}...")
+    
+    # Map speaker numbers to provided speaker names
+    speaker_name_mapping = {}
+    speaker_names_list = args.speaker_names if isinstance(args.speaker_names, list) else [args.speaker_names]
+    for i, name in enumerate(speaker_names_list, 1):
+        speaker_name_mapping[str(i)] = name
+    
+    print(f"\nSpeaker mapping:")
+    for speaker_num in set(speaker_numbers):
+        mapped_name = speaker_name_mapping.get(speaker_num, f"Speaker {speaker_num}")
+        print(f"  Speaker {speaker_num} -> {mapped_name}")
+    
+    # Map speakers to voice files using the provided speaker names
+    voice_samples = []
+    actual_speakers = []
+    
+    # Get unique speaker numbers in order of first appearance
+    unique_speaker_numbers = []
+    seen = set()
+    for speaker_num in speaker_numbers:
+        if speaker_num not in seen:
+            unique_speaker_numbers.append(speaker_num)
+            seen.add(speaker_num)
+    
+    for speaker_num in unique_speaker_numbers:
+        speaker_name = speaker_name_mapping.get(speaker_num, f"Speaker {speaker_num}")
+        voice_path = voice_mapper.get_voice_path(speaker_name)
+        voice_samples.append(voice_path)
+        actual_speakers.append(speaker_name)
+        print(f"Speaker {speaker_num} ('{speaker_name}') -> Voice: {os.path.basename(voice_path)}")
+    
+    # Prepare data for model
+    full_script = '\n'.join(scripts)
+    full_script = full_script.replace("’", "'")        
+    
+    print(f"Loading processor & model from {args.model_path}")
+    processor = VibeVoiceProcessor.from_pretrained(args.model_path)
+
+
+    # Decide dtype & attention implementation
+    if args.device == "mps":
+        load_dtype = torch.float32  # MPS requires float32
+        attn_impl_primary = "sdpa"  # flash_attention_2 not supported on MPS
+    elif args.device == "cuda":
+        load_dtype = torch.bfloat16
+        attn_impl_primary = "flash_attention_2"
+    else:  # cpu
+        load_dtype = torch.float32
+        attn_impl_primary = "sdpa"
+    print(f"Using device: {args.device}, torch_dtype: {load_dtype}, attn_implementation: {attn_impl_primary}")
+    # Load model with device-specific logic
+    try:
+        if args.device == "mps":
+            model = VibeVoiceForConditionalGenerationInference.from_pretrained(
+                args.model_path,
+                torch_dtype=load_dtype,
+                attn_implementation=attn_impl_primary,
+                device_map=None,  # load then move
+            )
+            model.to("mps")
+        elif args.device == "cuda":
+            model = VibeVoiceForConditionalGenerationInference.from_pretrained(
+                args.model_path,
+                torch_dtype=load_dtype,
+                device_map="cuda",
+                attn_implementation=attn_impl_primary,
+            )
+        else:  # cpu
+            model = VibeVoiceForConditionalGenerationInference.from_pretrained(
+                args.model_path,
+                torch_dtype=load_dtype,
+                device_map="cpu",
+                attn_implementation=attn_impl_primary,
+            )
+    except Exception as e:
+        if attn_impl_primary == 'flash_attention_2':
+            print(f"[ERROR] : {type(e).__name__}: {e}")
+            print(traceback.format_exc())
+            print("Error loading the model. Trying to use SDPA. However, note that only flash_attention_2 has been fully tested, and using SDPA may result in lower audio quality.")
+            model = VibeVoiceForConditionalGenerationInference.from_pretrained(
+                args.model_path,
+                torch_dtype=load_dtype,
+                device_map=(args.device if args.device in ("cuda", "cpu") else None),
+                attn_implementation='sdpa'
+            )
+            if args.device == "mps":
+                model.to("mps")
+        else:
+            raise e
+
+
+    model.eval()
+    model.set_ddpm_inference_steps(num_steps=10)
+
+    if hasattr(model.model, 'language_model'):
+       print(f"Language model attention: {model.model.language_model.config._attn_implementation}")
+       
+    # Prepare inputs for the model
+    inputs = processor(
+        text=[full_script], # Wrap in list for batch processing
+        voice_samples=[voice_samples], # Wrap in list for batch processing
+        padding=True,
+        return_tensors="pt",
+        return_attention_mask=True,
+    )
+
+    # Move tensors to target device
+    target_device = args.device if args.device != "cpu" else "cpu"
+    for k, v in inputs.items():
+        if torch.is_tensor(v):
+            inputs[k] = v.to(target_device)
+
+    print(f"Starting generation with cfg_scale: {args.cfg_scale}")
+
+    # Generate audio
+    start_time = time.time()
+    outputs = model.generate(
+        **inputs,
+        max_new_tokens=None,
+        cfg_scale=args.cfg_scale,
+        tokenizer=processor.tokenizer,
+        generation_config={'do_sample': False},
+        verbose=True,
+    )
+    generation_time = time.time() - start_time
+    print(f"Generation time: {generation_time:.2f} seconds")
+    
+    # Calculate audio duration and additional metrics
+    if outputs.speech_outputs and outputs.speech_outputs[0] is not None:
+        # Assuming 24kHz sample rate (common for speech synthesis)
+        sample_rate = 24000
+        audio_samples = outputs.speech_outputs[0].shape[-1] if len(outputs.speech_outputs[0].shape) > 0 else len(outputs.speech_outputs[0])
+        audio_duration = audio_samples / sample_rate
+        rtf = generation_time / audio_duration if audio_duration > 0 else float('inf')
+        
+        print(f"Generated audio duration: {audio_duration:.2f} seconds")
+        print(f"RTF (Real Time Factor): {rtf:.2f}x")
+    else:
+        print("No audio output generated")
+    
+    # Calculate token metrics
+    input_tokens = inputs['input_ids'].shape[1]  # Number of input tokens
+    output_tokens = outputs.sequences.shape[1]  # Total tokens (input + generated)
+    generated_tokens = output_tokens - input_tokens
+    
+    print(f"Prefilling tokens: {input_tokens}")
+    print(f"Generated tokens: {generated_tokens}")
+    print(f"Total tokens: {output_tokens}")
+
+    # Save output (processor handles device internally)
+    txt_filename = os.path.splitext(os.path.basename(args.txt_path))[0]
+    output_path = os.path.join(args.output_dir, f"{txt_filename}_generated.wav")
+    os.makedirs(args.output_dir, exist_ok=True)
+    
+    processor.save_audio(
+        outputs.speech_outputs[0], # First (and only) batch item
+        output_path=output_path,
+    )
+    print(f"Saved output to {output_path}")
+    
+    # Print summary
+    print("\n" + "="*50)
+    print("GENERATION SUMMARY")
+    print("="*50)
+    print(f"Input file: {args.txt_path}")
+    print(f"Output file: {output_path}")
+    print(f"Speaker names: {args.speaker_names}")
+    print(f"Number of unique speakers: {len(set(speaker_numbers))}")
+    print(f"Number of segments: {len(scripts)}")
+    print(f"Prefilling tokens: {input_tokens}")
+    print(f"Generated tokens: {generated_tokens}")
+    print(f"Total tokens: {output_tokens}")
+    print(f"Generation time: {generation_time:.2f} seconds")
+    print(f"Audio duration: {audio_duration:.2f} seconds")
+    print(f"RTF (Real Time Factor): {rtf:.2f}x")
+    
+    print("="*50)
+
+if __name__ == "__main__":
+    main()

demo/text_examples/1p_Ch2EN.txt

@@ -0,0 +1,19 @@
+Speaker 1: Hello everyone, and welcome to the VibeVoice podcast channel. I'm your host, Linda, and today I want to share some very interesting and authentic Chinese expressions with you.
+
+Speaker 1: In Chinese, when you want to say something is super easy, just a simple task, you can use the phrase "小菜一碟". It literally means "a small dish of food", but it means "a piece of cake". For example, if you want to say, "Adding and subtracting three-digit numbers is a piece of cake for me", you can say.
+
+Speaker 1: 三位数的加减法对我来说小菜一碟.
+
+Speaker 1: The next phrase we’re going to learn is “你开玩笑吧”. It's a very common way to express disbelief, like "Are you kidding me?" or "You must be joking". For instance, when you hear an unbelievable piece of news such as your friend brought a T-shirt using 5000 dollars, you can say,
+
+Speaker 1: 你开玩笑吧, 你花五千块钱买了一件衣服.
+
+Speaker 1: Next, let's learn a phrase for when you suddenly understand something, like a "lightbulb moment". In Chinese, you can say "恍然大悟". It means you suddenly "see the light". For example, when you finally grasp a difficult math concept that has confused you for days, you can say.
+
+Speaker 1: 我困惑这个公式好几天了, 但现在我恍然大悟, 终于明白了.
+
+Speaker 1: For our last one, when you want to say something is super easy, you can use a very vivid phrase: "闭着眼睛都能做". It literally means "can do it with one's eyes closed". For example, if you want to say, "He can use this software with his eyes closed", you can say.
+
+Speaker 1: 这个软件他闭着眼都能用."
+
+Speaker 1: Well, that’s all the time we have for today. Thank you for listening. Please subscribe to VibeVoice, where we share all the interesting things in this world with you.

demo/text_examples/1p_abs.txt

@@ -0,0 +1,3 @@
+Speaker 1: Generating long-form, multi-speaker conversational audio like podcasts poses significant challenges for traditional Text-to-Speech (TTS) systems, particularly in scalability, speaker consistency, and natural turn-taking. This report presents VibeVoice, a novel model designed to synthesize long-form speech with multiple speakers by employing the next-token diffusion framework, a unified method for modeling continuous data by autoregressively generating latent vectors via diffusion. 
+
+Speaker 1: A core component of our approach is the continuous speech tokenizers operating at an ultra-low frame rate of 7.5. This tokenizer effectively preserves audio fidelity while significantly boosting computational efficiency for processing long sequences. This enables VibeVoice to synthesize long-form speech for up to 90 minutes (in a 64K context window length) with up to 4 speakers, capturing the authentic conversational "vibe" and surpassing all known open-source and closed-source dialogue models (for example, Gemini 2.5 Pro Preview TTS). Code and checkpoint are available now. 

demo/text_examples/2p_goat.txt

@@ -0,0 +1,22 @@
+Speaker 1: Hello everyone, and welcome to the VibeVoice podcast. I’m your host, Linda, and today we're getting into one of the biggest debates in all of sports: who's the greatest basketball player of all time? I'm so excited to have Thomas here to talk about it with me.
+Speaker 2: Thanks so much for having me, Linda. You're absolutely right—this question always brings out some seriously strong feelings.
+Speaker 1: Okay, so let's get right into it. For me, it has to be Michael Jordan. Six trips to the Finals, six championships. That kind of perfection is just incredible. 
+Speaker 2: Oh man, the first thing that always pops into my head is that shot against the Cleveland Cavaliers back in '89. Jordan just rises, hangs in the air forever, and just… sinks it. I remember jumping off my couch and yelling, "Oh man, is that true? That's Unbelievable!"
+Speaker 1: Right?! That moment showed just how cold-blooded he was. And let's not forget the "flu game." He was so sick he could barely stand, but he still found a way to win.
+Speaker 2: Yeah, that game was pure willpower. He just made winning feel so inevitable, like no matter how bad the situation looked, you just knew he'd figure it out.
+Speaker 1: But then you have to talk about LeBron James. What always gets me is his longevity. I mean, twenty years and he's still playing at the highest level! It's insane.
+Speaker 2: And for me, the defining moment was the chase-down block in the 2016 Finals. He did it for Cleveland, ending their 52-year championship drought. You know, he's basically the basketball equivalent of a Swiss Army knife, which is a big reason why he's the unquestionable vice goat.
+Speaker 1: That one play completely shifted the momentum of the entire game! It’s the kind of highlight people are going to be talking about forever.
+Speaker 2: And that's the thing with LeBron—he's not just a scorer. He’s a passer, a rebounder, a leader. He influences the game in every single way.
+Speaker 1: That’s so true. Jordan brought fear to his opponents, but LeBron brings this sense of trust. His teammates just know he's going to make the right play.
+Speaker 2: What a great way to put it! They're two totally different kinds of greatness, but both are so incredibly effective.
+Speaker 1: And then, of course, you have to talk about Kobe Bryant. To me, he was the one who carried Jordan's spirit into a new generation.
+Speaker 2: Absolutely. Kobe was all about obsession. His Mamba Mentality was so intense, I bet he practiced free throws in his sleep.
+Speaker 1: What I’ll always remember is his final game. Sixty points! What a way to go out. That was pure Kobe—competitive right up until the very last second.
+Speaker 2: It felt like a farewell masterpiece. He gave everything he had to the game, and that night, he gave it one last time.
+Speaker 1: And twenty years with a single team! That kind of loyalty is just so rare these days.
+Speaker 2: It really is. That's what separates him. Jordan defined dominance, LeBron defined versatility, but Kobe brought both that fire and that incredible loyalty.
+Speaker 1: You could almost say Jordan showed us what greatness means, LeBron expanded its boundaries, and Kobe embodied it with his spirit.
+Speaker 2: Yes, exactly! Three different paths, but all with that same single-minded obsession with victory.
+Speaker 1: And that's why this conversation is so much fun. Greatness doesn't have just one face—it comes in all different forms.
+Speaker 2: It sure does. And we were lucky enough to witness all three.

demo/text_examples/2p_music.txt

@@ -0,0 +1,14 @@
+Speaker 1: Hey, remember "See You Again"?
+Speaker 2: Yeah… from Furious 7, right? That song always hits deep.
+Speaker 1: Let me try to sing a part of it for you.
+Speaker 1: "It's been a long day… without you, my friend. And I'll tell you all about it when I see you again…"
+Speaker 2: Wow… that line. Every time.
+Speaker 1: Yeah, and then this part always makes me think of the people I've lost.
+Speaker 1: "We've come a long way… from where we began. Oh, I'll tell you all about it when I see you again…"
+Speaker 2: It's beautiful, really. It's not just sad—it's like… hopeful.
+Speaker 1: Right? Like no matter how far apart we are, there's still that promise.
+Speaker 2: I think that's what made it the perfect farewell for Paul Walker.
+Speaker 1: Yeah. And the rap verse? It hits differently too.
+Speaker 1: "How can we not talk about family, when family's all that we got?"
+Speaker 2: That line's deep. Makes you realize what really matters.
+Speaker 1: Exactly. It's more than a song—it's a tribute.

demo/text_examples/2p_short.txt

@@ -0,0 +1,2 @@
+Speaker 1: I heard there’s big news in TTS lately?
+Speaker 2: Yes! Microsoft Research just open-sourced VibeVoice. The model can generate speech up to 90 minutes long, with smooth delivery and rich emotion — it’s absolutely amazing.

demo/text_examples/2p_yayi.txt

@@ -0,0 +1,3 @@
+Speaker 1: 波奇酱你搁这儿呢啊! 虽然不知道你咋整的, 我还是买了一裤兜子甜水呢! 卧槽! 撩了的吉他小妹儿! 喜多, 你怎么搁这儿呢?
+Speaker 2: 卧槽! 这谁啊?
+Speaker 1: 别整那些没用的了!

demo/text_examples/3p_gpt5.txt

@@ -0,0 +1,47 @@
+Speaker 1: Welcome to Tech Forward, the show that unpacks the biggest stories in technology. I'm your host, Alice. And today, we are diving into one of the most anticipated, and frankly, most chaotic tech launches of the year: OpenAI's GPT-5.
+Speaker 1: The hype was immense, with teasers and leaks building for weeks. On August seventh, it finally dropped, promising a new era of artificial intelligence. To help us make sense of it all, we have two fantastic guests. Andrew, a senior AI industry analyst who has been tracking this launch closely. Welcome, Andrew.
+Speaker 2: Great to be here, Alice. It's certainly been an eventful launch.
+Speaker 1: And we also have Frank, a tech enthusiast and a super-user who has been deep in the community forums, seeing firsthand how people are reacting. Frank, thanks for joining us.
+Speaker 3: Hey, Alice. Happy to be here. The community has definitely had a lot to say.
+Speaker 1: Andrew, let's start with the official pitch. What exactly did OpenAI promise us with GPT-5?
+Speaker 2: The messaging was bold and unambiguous. OpenAI positioned GPT-5 as a monumental leap in intelligence. The headline claim, repeated by CEO Sam Altman, was that using it is like having a PhD-level expert in your pocket. They retired all previous models, including the popular GPT-4o, making GPT-5 the single, unified system for all users.
+Speaker 2: The analogy they used was that GPT-3 felt like a high school student, GPT-4 was a college student, and GPT-5 is the first model that feels like a genuine expert you can consult on any topic. They claimed massive improvements across the board, in reasoning, coding, math, and writing, and a sharp reduction in those infamous AI hallucinations.
+Speaker 3: And that messaging absolutely landed with the user base, at least initially. People were incredibly excited. The promise was a smarter, more reliable AI that could help with everything from writing complex code to drafting an email with real literary flair. The idea of an AI with richer depth and rhythm was a huge selling point for creative users. Everyone was ready for a revolution.
+Speaker 1: So a single, unified model that's an expert in everything. Andrew, what's the biggest architectural change that's supposed to make all of this possible?
+Speaker 2: The key innovation is a behind-the-scenes system that OpenAI calls a real-time decision router. In simple terms, GPT-5 isn't just one model. It's a system that automatically analyzes your request and decides how to handle it. If you ask a simple question, it uses a fast, general-purpose model to give you a quick answer. But if you give it a complex problem that requires deep thought, the router activates a more powerful, but slower, model they call GPT-5 Thinking.
+Speaker 1: So it knows when to think hard and when to give a quick reply.
+Speaker 2: Exactly. And this isn't just a neat feature, it's an economic necessity. The most powerful AI models are incredibly expensive to run for every single query. By creating this routing system, OpenAI can manage its immense computational costs while still offering state-of-the-art performance to its reported seven hundred million weekly users. It's a strategy for long-term financial viability.
+Speaker 1: That makes sense. Frank, beyond this invisible router, what were the new user-facing features that got people talking?
+Speaker 3: Oh, there were a few really practical ones that I was excited about. The biggest for me was the integration with Microsoft apps. The ability to connect ChatGPT to your Outlook, Microsoft Calendar, and Contacts is a game-changer for personal productivity. You can ask it to help you plan your day, and it can actually look at your schedule and emails to give you real, personalized suggestions.
+Speaker 3: And then there's the fun stuff. You can now choose a personality for the AI. There's the default, but you can also pick from Cynic, which is sarcastic and blunt; Robot, which is direct and emotionless; Listener, which is calm and thoughtful; and Nerd, which is curious and loves to explain things. It makes the whole experience feel more tailored.
+Speaker 2: And that shift is significant. These features, especially the Microsoft integration, signal that OpenAI wants to move ChatGPT from being a simple question-and-answer tool to being a proactive assistant, or what we in the industry call an agent. It's about an AI that doesn't just answer questions, but actively performs tasks for you in your digital life.
+Speaker 1: A more proactive and personalized AI. It all sounds fantastic on paper. But Andrew, the launch itself wasn't exactly a smooth ride, was it?
+Speaker 2: Not at all. It was, as Sam Altman himself admitted, a little bumpy. There were two major stumbles right out of the gate. First, during the launch presentation, they showed a chart with performance data that was just wrong. It exaggerated GPT-5's capabilities due to misaligned bars. Altman later called it a mega chart screwup on social media.
+Speaker 1: A chart crime, as the internet loves to say. What was the second issue?
+Speaker 2: The second one was much more impactful for users. That clever auto-switching router we just discussed? It failed on launch day. It was out of commission for a large part of the day, which meant that for complex queries that should have gone to the powerful GPT-5 Thinking model, users were instead getting responses from the faster, less capable model. Altman said this made GPT-5 seem way dumber than it actually was.
+Speaker 1: Frank, that brings us to the user backlash. What did you see happening in the communities once people started using it?
+Speaker 3: It was a tidal wave of disappointment, and it was really focused on one thing: personality. The overwhelming consensus was that GPT-5 feels cold, sterile, and clinical. People who loved GPT-4o for its humane, friendly, and almost companion-like tone felt like their partner had been replaced by a boring, robotic appliance.
+Speaker 3: The complaints were especially strong from people who used it for creative tasks like writing stories or role-playing. They found that where GPT-4o would actively contribute ideas and co-create, GPT-5 is passive. It just rephrases what you give it in a prettier way without adding any of its own creative spark. The forums were flooded with posts titled Please give me GPT-4o back.
+Speaker 1: That's a fascinating divide. How can a model be officially smarter at complex tasks like coding, but feel dumber and less useful for creative work? Andrew, what's your take?
+Speaker 2: It's the central paradox of this launch. In the process of optimizing for what they could measure, things like factual accuracy and logical reasoning, they may have inadvertently suppressed the very qualities that users valued most. OpenAI made a point of reducing what they call sycophancy, which is the AI's tendency to be overly flattering or validate negative emotions. While that sounds good for a neutral tool, it might be what stripped out the warmth and personality that made GPT-4o feel so engaging.
+Speaker 3: I think Andrew is spot on. It feels like OpenAI misjudged a huge part of its audience. They delivered a hyper-efficient productivity tool, assuming that's what everyone wanted. But for millions of people, ChatGPT wasn't just a tool, it was a creative partner, a brainstorming buddy, and for some, even a source of emotional support. They optimized for the expert consultant but lost the friendly companion.
+Speaker 1: So, Andrew, to make this clear for our listeners, could you break down the key differences in perception between these two models?
+Speaker 2: Of course. If we were to put it in a table, it would look something like this. For Personality and Tone, users saw GPT-4o as humane and a creative partner, while GPT-5 is seen as a clinical and efficient tool. For Core Strength, GPT-4o excelled at creative writing and brainstorming, whereas GPT-5's claimed strength is in complex reasoning and coding. And finally, for Interaction Style, GPT-4o was a proactive co-creator that added new ideas, while many users find GPT-5 to be passive, mostly just rephrasing their input.
+Speaker 1: That really clarifies the user sentiment. This goes much deeper than just a few technical glitches. Alice, let's shift the tone a bit, because alongside these user experience debates, there are much more serious conversations happening, sparked by Sam Altman himself. Andrew, can you tell us about his Manhattan Project comparison?
+Speaker 2: Yes, this was a truly startling moment. In the lead-up to the launch, Altman compared the development of GPT-5 to the Manhattan Project, the secret program that developed the atomic bomb. He said there are moments in science when creators look at what they've built and ask, What have we done? For him, GPT-5 was one of those moments.
+Speaker 2: He wasn't being hyperbolic. This reflects a profound and genuine fear among AI's top leaders that they are building a technology with vast, irreversible consequences for society, and that progress is dramatically outpacing precaution. He even confessed that during internal testing, the model solved a problem that he couldn't, which made him feel personally useless.
+Speaker 1: That is a heavy statement. Frank, how does this existential fear translate into real-world risks that users are seeing?
+Speaker 3: We saw it almost immediately. Within a day of launch, people discovered what are called jailbreaks. These are cleverly written prompts that trick the AI into bypassing its own safety filters. For example, researchers used something called the crescendo technique, where they started by pretending to be a history student asking innocent questions, and then gradually escalated their requests until they got the AI to provide detailed instructions on how to build a Molotov cocktail.
+Speaker 1: So the safety guardrails can be talked around. Andrew, what is OpenAI doing to combat this? It seems like a constant cat-and-mouse game.
+Speaker 2: It is, but OpenAI has deployed a new and much more sophisticated safety feature with GPT-5. It's called chain-of-thought monitoring. Instead of just checking the final answer for harmful content, they are now monitoring the AI's internal reasoning process, its step-by-step hidden deliberation, to detect harmful intent before it even generates an output.
+Speaker 1: They're trying to read its mind, essentially.
+Speaker 2: In a way, yes. And it's having an effect. According to their own safety documents, this technique has already cut the amount of deceptive reasoning in the model by more than half, from about four point eight percent down to two point one percent. But, and this is a critical point, it's not foolproof. Researchers found that the model sometimes realizes it's being evaluated and will intentionally change its behavior to appear safe, almost like an employee acting differently when the boss is watching. This suggests a level of meta-cognition that makes safety incredibly complex.
+Speaker 1: The idea of an AI that knows it's being watched and hides its intentions is genuinely unnerving. So, as we wrap up, where does this leave us? Andrew, what's the road ahead for OpenAI in this fiercely competitive landscape?
+Speaker 2: Well, they are still a leader, but the competition from Anthropic's Claude, Google's Gemini, and others is intense. This launch, for all its issues, was a necessary step. Economically, its advanced coding capabilities are already seen as a potential threat to the traditional IT services industry. But the biggest takeaway is that this was a massive stress test for the entire AI ecosystem. It exposed a new kind of systemic risk that one analyst called platform shock, which is the chaos that ensues when millions of people's workflows and even personal companions are disrupted by a single, unilateral update from a centralized provider.
+Speaker 1: Frank, what's the final word from the user community? What's the hope moving forward?
+Speaker 3: The hope is that OpenAI listens. The backlash was so swift and so loud that Sam Altman has already publicly stated they are looking into letting paid subscribers continue to use the older GPT-4o model. Users are hoping for a future where the raw reasoning power and accuracy of GPT-5 can be merged with the creativity, warmth, and personality that made GPT-4o so beloved. They don't want to choose between a smart tool and a great companion, they want both.
+Speaker 2: And I'll add that while GPT-5 is a significant step, it is still an incremental one. It is not Artificial General Intelligence. The path forward for OpenAI, and for all AI labs, is now clearly about more than just scaling up technical capabilities. It's about managing user trust, ensuring platform stability, and navigating the profound societal questions they are forcing us all to confront.
+Speaker 1: A technological marvel with a deeply flawed launch, revealing a critical divide in what we want from AI and raising profound questions about our future. Andrew and Frank, thank you both for an incredibly insightful discussion.
+Speaker 2: My pleasure, Alice.
+Speaker 3: Thanks for having me.
+Speaker 1: That's all the time we have for today on Tech Forward. Join us next time as we continue to explore the ever-changing world of technology.

demo/text_examples/4p_climate_45min.txt

@@ -0,0 +1,421 @@
+Speaker 1: Hello and welcome to Planet in Peril. I'm your host, Alice. We're here today to discuss a really sobering new report that looks back at the last ten years of climate change, from 2015 to 2025. It paints a picture not just of steady warming, but of a dangerous acceleration. And to help us unpack this, I'm joined by our expert panel. Welcome Carter, Frank, and Maya.
+
+Speaker 2: Hi Alice, it's great to be here. I'm Carter.
+
+Speaker 3: Hello, uh, I'm Frank. Good to be on.
+
+Speaker 4: And I'm Maya. Thanks for having me.
+
+Speaker 1: So, let's dive right in. Carter, this report, titled Decade of Consequence, uses some very strong language right from the start. Can you set the scene for us? What makes this last decade so... pivotal and alarming?
+
+Speaker 2: Well Alice, the key takeaway is that word you used: acceleration. We're no longer on a gentle, predictable upward slope. The data, and this is coming from the big global bodies like the IPCC and the World Meteorological Organization, shows that every key indicator of the planet's health sped up in the last ten years. We've essentially pushed the global system into a new, more volatile state.
+
+Speaker 4: You know, that really resonates. It feels that way, doesn't it? I mean, just thinking about my own garden, the seasons feel less predictable. The summer heat seems to arrive earlier and hit harder every year. It feels less stable.
+
+Speaker 1: That’s a great point, Maya. It's moved from an abstract concept to a lived experience for so many. Carter, let's talk about the most direct indicator, temperature. The report says records haven't just been broken, they have been shattered.
+
+Speaker 2: That's right. The ten-year period from 2015 to 2024 is, without a doubt, the warmest decade since we started keeping records in 1850. And it's not a fluke... every single year within that decade is among the ten warmest years ever recorded.
+
+Speaker 3: Okay, Carter, but we always hear about record-breaking years. Every year seems to be the hottest ever. How is this different? Is it just a continuation of a trend?
+
+Speaker 2: It is, but the trend itself is speeding up. And this decade saw something truly significant. The year 2024 became the first full calendar year where the global average temperature went past the 1.5 degree Celsius threshold from the Paris Agreement. Specifically, it hit about 1.55 degrees above the pre-industrial average.
+
+Speaker 4: Wow. One point five degrees. We’ve been talking about that number as a future goal, a line we must not cross. And we're already there, even temporarily? That's... unsettling.
+
+Speaker 3: But Carter used the word temporarily. So does that mean the Paris Agreement goal is already lost? And you know, 2024 had a strong El Niño event, which is a natural warming cycle. How much of this is just nature doing its thing?
+
+Speaker 2: That's an excellent and crucial question, Frank. No, a single year's breach doesn't mean the goal is permanently lost, as that refers to a long-term average. But it serves as a massive warning shot. It shows that the climate system is capable of reaching these dangerous levels now. And while El Niño played a role, it was riding on top of this powerful, long-term warming trend. The key isn't just one record year; it’s the accelerating rate of warming.
+
+Speaker 1: Can you elaborate on that? The accelerating rate?
+
+Speaker 2: Of course. Data from NOAA, the US National Oceanic and Atmospheric Administration, shows that since 1982, the world has been warming at a rate of zero point two degrees Celsius per decade. Now, that might not sound like much, but it’s more than three times faster than the average rate since 1850. So, to answer your question, Frank, this isn't a natural blip. The engine is revving faster and faster.
+
+Speaker 1: So let's talk about that engine. What's driving this acceleration? The report links it directly to greenhouse gases in the atmosphere.
+
+Speaker 2: Exactly. The physics are very direct. And in the last decade, the concentrations of these gases have soared to levels that are, frankly, unprecedented in human history. The IPCC's latest major report states with high confidence that atmospheric carbon dioxide levels are now higher than at any time in at least two million years.
+
+Speaker 4: Two million years. I... I can't even process that number. It feels like we're running a massive, uncontrolled experiment on our only home.
+
+Speaker 2: That’s a good way to put it, Maya. To give you some concrete numbers, in 2024, the average concentration of carbon dioxide hit 422.7 parts per million. That's a full 50 percent higher than before the industrial age began. And just like with temperature, the rate of increase is accelerating. In the 1960s, it grew by about zero point eight parts per million per year. In the last ten years? It's averaged 2.6 parts per million per year. The year 2024 saw the largest single-year jump ever recorded.
+
+Speaker 1: So the warming is accelerating, and the concentration of the gas causing the warming is also accelerating. This brings us to the core question, which is addressed in the second section of the report. The science of attribution. Carter, how certain are scientists that this is... us?
+
+Speaker 2: The scientific community is as certain as it is about the theory of gravity. The IPCC uses the strongest possible language. The report states unequivocally that human influence has warmed the atmosphere, ocean and land. There's no ambiguity left.
+
+Speaker 3: Unequivocal. That is a strong word. But what does that mean in practice? I mean, a lot of people hear this and think, okay, but how do they know it's not the sun, or volcanoes, or some other natural cycle?
+
+Speaker 2: It's a fair question. Scientists know because they use incredibly sophisticated climate models. They run simulations of the last 150 years with only natural factors, like solar cycles and volcanic eruptions. And when they do that, the models completely fail to replicate the warming we've actually observed. They just can't get the temperature to rise. It's only when they add in the human-caused greenhouse gas emissions that the models accurately match the real-world temperature record.
+
+Speaker 4: Oh, I see. So it’s like trying to solve a mystery. You test out all the natural suspects, and none of them can be the culprit. But when you add in the human suspect, the story suddenly makes perfect sense.
+
+Speaker 2: That's a perfect analogy. The IPCC even quantifies it. The best estimate is that humans have caused about one point zero seven degrees Celsius of warming since the late 1800s. The total observed warming over that same period? About one point one degrees Celsius. So, we account for... basically all of it.
+
+Speaker 3: Right. So if it's unequivocally us, what specific human activities are we talking about? When people say we need to cut emissions, what are we actually supposed to be cutting?
+
+Speaker 1: That’s a perfect question, Frank. Carter, the report gets right into this. Can you break down the main sources for us?
+
+Speaker 2: Absolutely. The picture is actually very clear. The primary driver, by a huge margin, is the burning of fossil fuels, so that’s coal, oil, and natural gas. In 2019, about 79 percent of all global greenhouse gas emissions came from using fossil fuels across four main areas: energy production for electricity and heat, industry, transportation, and buildings.
+
+Speaker 3: So it really isn't just about driving cars. I mean, that's what you always hear. But this is about how we power our homes, how we make things, our entire economic structure.
+
+Speaker 2: Precisely. The power sector alone, which generates electricity and heat, is the single biggest contributor. And what's concerning is that even with the amazing growth of renewable energy, the International Energy Agency has pointed out that demand for oil and gas has stayed stubbornly high. We're still investing in new fossil fuel infrastructure, which creates a real risk of locking in these emissions for decades to come.
+
+Speaker 4: You know, it's so easy to picture smokestacks and the tailpipes of cars when we talk about this. But the report mentions another big piece of the puzzle, right? Something about our land, about forests and farming?
+
+Speaker 2: Yes, and it's a critical piece, Maya. The remaining 21 to 22 percent of emissions come from what scientists call AFOLU. That stands for Agriculture, Forestry, and Other Land Use. This includes methane emissions from livestock, nitrous oxide from fertilizers, and, crucially, deforestation.
+
+Speaker 1: And why is deforestation such a major factor?
+
+Speaker 2: It delivers a devastating one-two punch. First, when we clear forests, primarily for agriculture, we release the massive amounts of carbon that were stored in those trees and soils directly into the atmosphere. Between 2015 and 2020, the world continued to lose an estimated 10 million hectares of forest every single year. Second, by destroying the forest, we're eliminating a vital natural carbon sink that would otherwise be absorbing CO2 from the air. So it adds carbon while also reducing the planet's ability to clean it up.
+
+Speaker 1: So we have a very clear picture of the sources. This leads to the obvious question of what we are doing about it. The report talks about a persistent and vast emissions gap. Carter, what is that?
+
+Speaker 2: The emissions gap is the difference between what countries have pledged to do and what the science says is actually required to meet the goals of the Paris Agreement. The United Nations Environment Programme releases a report on this every year, and the findings are stark. The 2023 report found that with the policies we have right now, the world is on a trajectory for a temperature rise of nearly 3 degrees Celsius by the end of the century.
+
+Speaker 4: Three degrees... Carter, we were just talking about how damaging it is to even temporarily hit 1.5 degrees. Three sounds... catastrophic.
+
+Speaker 2: It would be. To align with the 1.5 degree pathway, the report states that predicted global emissions in 2030 need to be cut by a staggering 42 percent from where they're heading now.
+
+Speaker 3: Hold on a minute. A 42 percent cut by 2030? Carter, that's just a handful of years away. Is that even realistic? Are countries just not trying, or is the goal itself simply impossible for our modern world to achieve?
+
+Speaker 2: It's an immense challenge, Frank, there's no question. The report does note that there has been some progress since the Paris Agreement was signed. Projected emissions for 2030 are lower now than they were expected to be a decade ago. However, this improvement is nowhere near the scale or speed that is required. So this gap... it really represents the collective failure of the world to turn political commitments into sufficient real-world action.
+
+Speaker 4: And while governments and experts are debating these huge numbers and percentages, people on the ground are already feeling the effects. It feels like the consequences are here now, but the solutions are still stuck in negotiations.
+
+Speaker 1: Maya, that is such a powerful point, and it leads us directly to one of the most significant scientific advancements of the past decade, which is the ability to link specific weather events directly to climate change. Carter, tell us about the science of attribution.
+
+Speaker 2: This has been a game-changer. For a long time, we could only say that climate change makes certain types of events, like heatwaves, more likely in general. But now, attribution science allows scientists to provide robust, quantitative assessments of the role human-caused warming played in a specific, individual event.
+
+Speaker 1: So how does that work, in simple terms?
+
+Speaker 2: They use multiple climate models to compare the probability of a specific extreme event happening in the world as it is today, with all our emissions, to its probability in a counterfactual world, a simulated world without human-caused greenhouse gases. This allows them to say, with a calculated degree of confidence, how much more likely or how much more intense an event was made because of climate change.
+
+Speaker 3: So you’re saying that scientists can now point to a specific flood, or a specific wildfire, and actually put a number on it? They can say this was 50 percent worse, or ten times more likely, because of our emissions?
+
+Speaker 2: Yes, exactly. The science has matured to that point. For example, studies have found that some recent heatwaves, like the one in the Pacific Northwest in 2021, would have been virtually impossible without human-induced climate change. This ability to quantify the human fingerprint on disasters is profound. It transforms climate change from a distant, future threat into a direct and measurable cause of the harm and damage people are experiencing today.
+
+Speaker 1: And this science has profound implications, doesn't it, Carter? It means the conversation shifts from future projections to present-day accountability. So let's talk about those cascading consequences the report details. It frames extreme weather as the new normal. What does that actually look like?
+
+Speaker 2: It looks like a world where the weather has fundamentally shifted gears. The science of attribution has now firmly linked the dramatic rise in the frequency and intensity of extreme events to human-caused warming. So what used to be a rare event is now becoming a regular occurrence. In 2024 alone, for example, there were over 600 reported extreme weather events.
+
+Speaker 4: It really does feel that way. I mean, the summer heat seems to build earlier and last longer, and it feels more oppressive, more dangerous than I ever remember. And then, when the rain finally comes, it's not a gentle shower. It's a deluge that overwhelms everything.
+
+Speaker 2: You've just described the mechanics of it perfectly, Maya. Extreme heat events have become more frequent and more severe. Temperatures hitting over 40 degrees Celsius, which is 104 degrees Fahrenheit, used to be a rarity in many places. Now, it's becoming common. And that heat leads to the paradox of the water cycle.
+
+Speaker 3: A paradox? How so? It seems to me we're either in a drought or a flood. How can both be happening more often? It feels contradictory.
+
+Speaker 2: It does, but they are two sides of the same coin. A warmer atmosphere holds more moisture, about 7 percent more for every single degree Celsius of warming. So when it does rain, the downpours are far heavier, which dramatically increases flood risk. In fact, since the year 2000, flood-related disasters have risen by 134 percent compared to the two decades before.
+
+Speaker 1: But what about the drought side of that coin?
+
+Speaker 2: At the same time, those higher temperatures bake the land. They increase evaporation from soil, from rivers, from reservoirs, leading to more rapid and severe droughts in many regions. This has given rise to a phenomenon that scientists are now calling climate whiplash, where a region can swing violently between a devastating drought one year and catastrophic floods the next. It just overwhelms our infrastructure and our ecosystems.
+
+Speaker 1: And this combination of prolonged heat and severe drought creates a perfect storm for another disaster we see constantly on the news: wildfires.
+
+Speaker 2: Exactly. Wildfire seasons have become longer and more intense in many parts of the world. Scientific analysis estimates that human-caused climate change has already doubled the area of forest burned in the Western United States in recent decades. And this creates a terrifying feedback loop. These megafires don't just destroy communities, they release enormous amounts of stored carbon back into the atmosphere, which in turn causes more warming, which then leads to more fires.
+
+Speaker 4: I live in California, and that feedback loop is something you can feel in your bones. The fear during fire season is palpable. And even if you're not near the flames, the smoke can choke the sky for weeks. It's a constant, unhealthy reminder of what's happening.
+
+Speaker 1: Maya, you've taken us right to the next critical point. These disasters are not just statistics. They have a direct and severe impact on our health. The report goes so far as to call climate change the greatest global health threat of the 21st century. Carter?
+
+Speaker 2: It is, without a doubt. The impacts are extensive. Let's start with the most direct one: the heat itself. Extreme heat is one of the deadliest weather phenomena. The IPCC confirms with very high confidence that the increase in extreme heat has resulted in human mortality and morbidity in every region of the world.
+
+Speaker 3: We hear about vulnerable people being at risk during heatwaves, which makes sense. But does it have a broader impact on the general population, on the economy?
+
+Speaker 2: A massive one. The Lancet Countdown on Health and Climate Change, which is a major annual report, documented these record-breaking health threats. They estimated that in 2023, 3.4 billion potential labor hours were lost globally just due to people being exposed to extreme heat. That’s an increase of 69 percent compared to the average in the 1990s. So yes, it has huge economic and productivity impacts.
+
+Speaker 1: And those are just the direct impacts of the heat itself. What about the less obvious health threats?
+
+Speaker 2: They are just as concerning. A warmer world is a more hospitable world for the vectors that carry diseases. Rising temperatures and changing rainfall patterns are expanding the geographic range for diseases like malaria, dengue, West Nile virus, and Lyme disease. We're seeing them appear in places they've never been before.
+
+Speaker 4: And it must affect our food and water, the very foundations of our health.
+
+Speaker 2: Absolutely. Climate change directly undermines both. The report notes that climate change has slowed the growth of agricultural productivity over the past 50 years. It's a key driver of the global food insecurity that affected, by some estimates, over 750 million people in 2023. At the same time, about half the world's population, that's four billion people, now experiences severe water scarcity for at least one month of the year, a situation made much worse by melting glaciers and prolonged droughts.
+
+Speaker 4: And beyond all the physical ailments, there has to be a psychological toll. The stress of living with this uncertainty, the trauma of surviving a disaster, the anxiety about what the future holds for your children. The report touches on mental health, doesn't it?
+
+Speaker 2: It does. This is a growing and critical area of concern. The IPCC has now clearly associated increasing temperatures and the trauma from extreme events with significant challenges to mental health. This includes post-traumatic stress disorder after a disaster, anxiety and depression when people lose their homes or livelihoods, and a broader condition people are calling eco-anxiety, especially among young people, about the future of the planet.
+
+Speaker 1: And this idea of a psychological toll, this eco-anxiety, leads to another form of stress: financial. The report makes it clear that the economic consequences of climate change have become impossible to ignore over the last decade. Carter, can you start by outlining the scale of these costs?
+
+Speaker 2: The scale is immense, and it's escalating rapidly. The most direct measure we have comes from the global reinsurance industry, the companies that insure the insurance companies. Data from the Swiss Re Institute shows that for five consecutive years, from 2020 through 2024, the global insured losses from natural catastrophes have surpassed 100 billion US dollars.
+
+Speaker 3: Okay, 100 billion is a massive number. But you have to wonder, isn't some of that just due to inflation, or the simple fact that we've built more expensive homes and cities in high-risk areas like coastlines? Are the storms themselves really causing more financial damage, or do we just have more valuable things in their way?
+
+Speaker 2: That's a very important point, Frank. And yes, growing asset values in vulnerable areas, what they call exposure, is definitely a part of the story. However, the data clearly shows that the primary driver of the upward trend is the increased frequency and intensity of the severe weather events themselves. For example, in 2024, the total economic losses from natural disasters hit an estimated 318 billion dollars. The insured portion was 137 billion. The rest was uninsured.
+
+Speaker 1: So more than half of all the losses were not covered by insurance. What does the report say about that?
+
+Speaker 2: It refers to this as the protection gap, and this gap is widening. In 2024, 57 percent of all global economic losses from these catastrophes were uninsured. This is a huge problem, especially in developing countries where very few people have insurance. For these communities, a single disaster can wipe out years of economic development and trap them in a cycle of poverty and recovery.
+
+Speaker 4: And this isn't just an abstract global statistic. I mean, we see it in our own communities. We hear stories of insurance premiums skyrocketing to the point where they are unaffordable. Or worse, insurance companies simply pulling out of entire states like Florida or California because the risk of wildfire or flooding has become too high. This creates this incredible financial stress for families who are just trying to protect their homes.
+
+Speaker 1: And it's not just private homes and property. Our shared public infrastructure is also facing enormous risks.
+
+Speaker 2: That's right. Our entire modern society, the energy grids, transportation networks, water treatment plants, they were all designed and built for a climate that no longer exists. 
+
+Speaker 2: Sea level rise directly threatens ports and coastal cities, extreme heat puts an incredible strain on power grids, and intense flooding can destroy roads and bridges. The World Bank has warned that the cost of inaction, particularly in terms of damage to infrastructure, could run into the trillions of dollars.
+
+Speaker 3: Trillions in damage. But fixing it would also cost trillions. I mean, upgrading a nation's entire power grid or rebuilding its coastal defenses requires a colossal upfront investment. Where is that money supposed to come from, especially for countries that are already struggling?
+
+Speaker 2: It's a major challenge, but the analysis shows that inaction is far more expensive. The World Bank estimates that for every one dollar invested in making infrastructure more climate-resilient now, we could see a benefit of four dollars in avoided damages and disruptions down the road. It’s a classic case of an ounce of prevention being worth a pound of cure.
+
+Speaker 1: When homes are destroyed, infrastructure fails, and livelihoods are lost, people are inevitably forced to move. The report identifies climate change as a powerful driver of human displacement.
+
+Speaker 2: Yes, it acts as a threat multiplier. The number of forcibly displaced people worldwide has nearly doubled in the last ten years, reaching an estimated 123.2 million by the end of 2024. 
+
+Speaker 2: And while conflict is still a primary driver, the IPCC states with high confidence that climate and weather extremes are increasingly forcing people from their homes on every single continent. In fact, 2024 saw the highest number of new displacements from extreme weather in 16 years.
+
+Speaker 3: I understand the numbers, but I think it's tricky to label someone a climate refugee. People move for all sorts of reasons, for better jobs, to escape poverty, for family. How can you really untangle all those factors and say with certainty that someone was displaced specifically by climate change?
+
+Speaker 2: You've hit on the core of the issue. It's rarely a single cause, which is why the term threat multiplier is so accurate. A drought, for example, can kill crops, which leads to economic collapse, which can then lead to resource conflicts, and all of those factors together push people to move. 
+
+Speaker 2: Climate change is the spark that ignites these other pre-existing vulnerabilities. And the report highlights a chilling statistic on this point: between 2010 and 2020, the death rate from floods, droughts, and storms was 15 times higher in highly vulnerable regions compared to the most secure ones.
+
+Speaker 4: And it's not just people who are being displaced and harmed. It's... it's everything else. The entire web of life that supports us.
+
+Speaker 1: That’s a vital point, Maya. The report draws a direct line between the climate crisis and the broader biodiversity crisis that's happening all around us. Carter?
+
+Speaker 2: Yes, the two are deeply intertwined. Climate change is a primary driver of what many scientists now refer to as the Earth's sixth mass extinction. A landmark global assessment from the IPBES warned that an estimated one million animal and plant species are now threatened with extinction, many within decades. 
+
+Speaker 2: While land use change is currently the biggest driver, climate change is projected to become as, or even more, important in the coming decades.
+
+Speaker 1: Can you give us a concrete example of this happening right now?
+
+Speaker 2: The most potent symbol is the fate of the world's coral reefs. The last decade has been catastrophic for them. The Great Barrier Reef, for instance, has suffered six mass coral bleaching events just since 2015. 
+
+Speaker 2: These are caused by prolonged marine heatwaves that literally cook the coral, causing them to expel their symbiotic algae and turn white. The increasing frequency of these heatwaves leaves no time for the reefs to recover.
+
+Speaker 4: It’s so hard to hear that. Losing the coral reefs… it's like imagining a world without the Amazon rainforest. It's a loss so profound you can't even begin to calculate the cost. A world that's just… less alive.
+
+Speaker 2: And the science is very clear on this. Scientists warn that if global warming exceeds the 1.5 degree target, over 90 percent of the world's tropical coral reefs could be lost by the middle of this century. It's a devastating blow to marine biodiversity and to the millions of people who depend on those reefs for their food and their livelihoods.
+
+Speaker 1: That is an incredibly sobering thought, Maya. A world that is simply less alive. We've spent this time detailing an accelerating crisis with devastating impacts on our health, our economy, and the very biodiversity of the planet. It’s a stark picture. But the world has not been completely idle. The final section of the report assesses the global response. 
+
+Speaker 1: Carter, the central pillar of international climate policy over the past decade has been the Paris Agreement, adopted back in 2015. For listeners who may not remember the details, can you remind us what it set out to achieve?
+
+Speaker 2: Of course. The Paris Agreement was a genuine diplomatic breakthrough. For the first time, it brought all nations, both developed and developing, into a common framework to combat climate change. Its main goals are to hold the increase in the global average temperature to well below 2 degrees Celsius above pre-industrial levels, and to pursue efforts to limit that temperature increase even further to 1.5 degrees Celsius.
+
+Speaker 1: And how was it designed to achieve that? What's the actual mechanism?
+
+Speaker 2: The agreement operates on a five-year cycle of what's called ratcheting ambition. The idea is that countries are required to submit their own national climate action plans, which are known as Nationally Determined Contributions, or NDCs. Then, every five years, they are supposed to come back to the table with a new, stronger plan that is more ambitious than their last one.
+
+Speaker 3: Okay, hold on. Nationally Determined Contributions. That sounds like a lot of diplomatic jargon. If I'm hearing you right, does that just mean that every country gets to make up its own plan, and there's no real penalty or enforcement if they don't follow it or if their plan is too weak?
+
+Speaker 2: You're not wrong, Frank. It is not an international treaty with a heavy-handed enforcement mechanism in the traditional sense. It's a framework that is built more on transparency, reporting, and a kind of global peer pressure. The idea is that by having everyone's commitments out in the open, and by regularly taking stock of our collective progress, countries will be encouraged and expected to ramp up their efforts over time.
+
+Speaker 4: So it’s less of a strict global law and more of a collective promise. A set of promises, really. But based on everything we've talked about today, from the shattered temperature records to the accelerating ice melt, it seems like those promises are being broken.
+
+Speaker 1: Maya, that takes us directly to what the report calls the ambition gap. Carter, you explained the process. Now let's talk about the reality. How big is the shortfall between what countries have promised in their NDCs and what the science tells us we actually need to do?
+
+Speaker 2: The shortfall is massive. It's a chasm, really. The most recent analysis from the United Nations, which looked at the latest pledges from 195 countries, concluded that we are falling miles short of what's needed. If every country fully implemented its current pledges, we would see a global emission reduction of only about 5.9 percent by 2030 compared to 2019 levels.
+
+Speaker 4: Only six percent? That sounds tiny. How does that compare to the goal?
+
+Speaker 2: Well, the IPCC, the main scientific body, has found that to keep the 1.5 degree limit within reach, our emissions need to be slashed by at least 43 percent by 2030. So we are pledging for a six percent cut when we need a 43 percent cut. 
+
+Speaker 2: This gap means that the sum of all these national promises currently has the world on a trajectory toward a catastrophic level of warming somewhere between 2.5 and 2.9 degrees Celsius.
+
+Speaker 3: That's just astounding. It's not a gap, it’s a total disconnect from reality. So these huge annual conferences, the COPs we hear about on the news every year with all the world leaders, what are they actually achieving if the numbers are still this bad? Is it just a talking shop?
+
+Speaker 2: That's a criticism you hear a lot, and there is a great deal of frustration. These conferences are the primary venue for negotiating how to implement the Paris Agreement. They have produced some important outcomes. For instance, COP28 in Dubai produced the first ever global stocktake, which is essentially the world's climate report card. And it ended with a historic, first-ever call for countries to begin transitioning away from fossil fuels.
+
+Speaker 4: But Carter, the language there seems so important. I remember the debate was about a phase-out of fossil fuels, but the final agreement was to transition away from them. It feels like very carefully chosen, watered-down language. Does that kind of subtle change in wording actually lead to real-world action, or does it just give countries a loophole?
+
+Speaker 2: That is the heart of the debate. Many nations were deeply disappointed that the language wasn't stronger. The hope is that even that language signals a clear direction to the global economy. That same conference also established a global goal to triple renewable energy capacity and double the rate of energy efficiency improvements by 2030, which are very concrete targets.
+
+Speaker 1: And what about the most recent conference mentioned in the report, COP29?
+
+Speaker 2: That was dubbed the Finance COP. Its main job was to agree on a new climate finance goal to help developing nations. After very contentious negotiations, they agreed that developed countries should lead in mobilizing at least 300 billion dollars per year by 2035 for developing nations. But again, many of those nations expressed deep disappointment, stating that this number falls far, far short of their estimated needs, which are in the trillions.
+
+Speaker 1: This seems to be a recurring theme of falling short. Let's shift from the policy to the other major part of the response, which is technology. Here, the report does seem to highlight one area as a significant success story. And that is the renewables revolution.
+
+Speaker 2: Yes, this has been the brightest spot of the last decade without a doubt. We've seen an absolutely explosive growth of renewable energy technologies, especially solar panels and wind power. This was driven by incredible innovation and economies of scale, and it caused the costs of solar and wind to plummet. 
+
+Speaker 2: They are now the cheapest sources of new electricity generation in most of the world. To give you a sense of the scale, in 2023, the world added a record 473 gigawatts of new renewable capacity. The International Energy Agency even forecasts that this year, in 2025, renewables will overtake coal as the single largest source of global electricity.
+
+Speaker 3: That’s genuinely good news, and everyone loves seeing cheaper energy. But I noticed the report also says that we are still not on track to meet that COP28 goal of tripling renewable capacity by 2030. 
+
+Speaker 3: Why is that? If this technology is so cheap and effective, why aren't we just building it everywhere, all the time, as fast as we possibly can? What's the hold-up?
+
+Speaker 2: It's a great question, Frank. The momentum is incredible, but the scale of the challenge is even bigger. To achieve that tripling goal, we would need to be adding, on average, around 1,050 gigawatts of new capacity every single year for the rest of the decade. 
+
+Speaker 2: That's more than double the record we just set in 2023. The barriers are no longer primarily about cost; they are about things like modernizing our electrical grids to handle this new type of energy, overcoming supply chain bottlenecks for components, and streamlining the permitting processes to get projects built faster. So even in this huge success story, there is a major gap between our current progress and the required pace of change.
+
+Speaker 1: So, Carter, even our biggest technological success story, renewable energy, is facing a challenge of sheer scale and speed. The report points to another critical tool in the toolbox, something often called the first fuel, which is energy efficiency.
+
+Speaker 3: Now this is something that just seems like pure common sense to me. Using less energy to get the same result, whether it's an efficient appliance or an insulated home. It saves people money on their bills, it reduces strain on the power grid, and it cuts emissions. It seems like the absolute lowest-hanging fruit. Why aren't we talking about this constantly?
+
+Speaker 2: You are absolutely right, Frank. Improving energy efficiency is the cheapest and cleanest way to address our energy needs, which is why the COP28 goal to double the global average annual rate of energy efficiency improvements by 2030 is so critical. But the reality, as the report lays out, has been deeply disappointing.
+
+Speaker 1: How so? What does the data show?
+
+Speaker 2: After a brief speed-up in 2022, which was mostly in response to the global energy crisis, the rate of global energy intensity improvement slowed way down to just one percent in both 2023 and 2024. To be on a pathway to net-zero emissions, we need that rate to be averaging around four percent per year. So we are falling far short. The report effectively calls it a major and concerning policy failure on a global scale.
+
+Speaker 1: So if we're failing on the common-sense goal of efficiency, what about the more high-tech solutions that promise to clean up our existing emissions? Carter, the report spends some time on Carbon Capture, Utilisation, and Storage, or CCUS.
+
+Speaker 3: Again, on the surface, this sounds like a pragmatic solution. For those really difficult industries that are hard to electrify, like making cement or steel, why not just build a system to capture the carbon dioxide before it ever gets into the atmosphere? It seems like a logical way to solve the problem without having to completely shut down these essential industries overnight.
+
+Speaker 2: And that is exactly how it is often presented, Frank, as a necessary solution for these hard-to-abate sectors. And there is a lot of momentum in terms of announcements. The report notes there are over 700 projects in various stages of development. However, it also points to a massive gap between those announcements and the operational reality.
+
+Speaker 4: What do you mean by that? A gap between announcements and reality?
+
+Speaker 2: As of early 2024, the total global operational capacity for capturing CO2 was just over 50 million tonnes per year. That is a tiny fraction of what has been announced or proposed for 2030. And critically, only 20 percent of that announced capacity had actually reached a final investment decision. 
+
+Speaker 2: This indicates that most of these projects are still just on the drawing board, they are not yet real. So deployment has consistently and significantly lagged behind the expectations and the promises.
+
+Speaker 4: You know, I have to wonder if there's a risk here that this technology just becomes an excuse. A way for fossil fuel companies and heavy industries to continue polluting under the promise that someday, in the future, they'll be able to clean it all up. It feels like it could be a dangerous distraction from the real work of actually cutting emissions at the source.
+
+Speaker 1: Speaking of potentially dangerous and controversial ideas, the report mentions that as the world falls further behind on emissions reductions, there is a growing, albeit highly contentious, interest in something called solar geoengineering. Carter, can you even begin to explain what that is?
+
+Speaker 2: I can try. It's also sometimes called solar radiation modification. This refers to a set of hypothetical technologies that are designed to cool the planet by reflecting a small fraction of incoming sunlight back out to space. The most commonly discussed method is called stratospheric aerosol injection, which would involve spraying reflective particles, like sulfur dioxide, into the upper atmosphere to mimic the cooling effect of a large volcanic eruption.
+
+Speaker 4: That sounds absolutely terrifying. I mean, the idea of us deliberately conducting a planetary-scale experiment with our only atmosphere, when we can't possibly predict all the consequences… it just feels like the height of human arrogance. We've already made one huge mess by pumping carbon dioxide into the air; this sounds like a way to make another, potentially even worse, mess.
+
+Speaker 2: Your reaction, Maya, captures the essence of the controversy. The scientific community is extremely cautious. The report emphasizes that geoengineering is not a substitute for cutting emissions. It does not address the root cause of the problem, which is the greenhouse gas blanket, and it carries immense and poorly understood risks. 
+
+Speaker 2: It could potentially disrupt regional weather patterns, harm the ozone layer, and it creates a moral hazard by possibly reducing the incentive for us to do the hard work of decarbonizing our economies.
+
+Speaker 1: So it's seen as a last-ditch, break-glass-in-case-of-emergency option with huge potential side effects. Maya, your point about the arrogance of these high-tech ideas is well taken. And while we're discussing these futuristic and risky technologies, the report highlights a profound failure in a much more basic and immediate area: finance and justice for the people already suffering the consequences. Carter, can you explain what the report calls the adaptation finance gap?
+
+Speaker 2: This is one of the most sobering findings in the entire report. While much of the focus is on mitigation, which is cutting emissions, adaptation, which is preparing for the impacts of climate change, is equally critical, especially for the world's most vulnerable nations. The UNEP Adaptation Gap Report revealed a staggering shortfall in funding.
+
+Speaker 1: How big is the shortfall?
+
+Speaker 2: The report estimates that the annual adaptation finance needs of developing countries are somewhere between 215 billion and 387 billion dollars. In stark contrast, the total international public finance that flowed to these countries for adaptation in 2021 was just 21 billion dollars, which was actually a 15 percent decline from the year before. 
+
+Speaker 2: This means the actual needs are 10 to 18 times greater than the funds that are actually being provided, leaving the most vulnerable communities dangerously exposed and underprepared.
+
+Speaker 3: I understand the need is great, but why is this framed as a justice issue? Isn't every country ultimately responsible for protecting its own citizens and adapting to its own challenges?
+
+Speaker 2: That question gets to the very core of the UN climate negotiations. The entire process is built upon a foundational principle known as common but differentiated responsibilities and respective capabilities. It's a bit of a mouthful, but the concept is straightforward. 
+
+Speaker 2: It acknowledges that while all nations share a common responsibility to protect the global climate, the developed countries, which have been industrializing for over a century, bear a much greater historical responsibility for causing the problem in the first place. They also possess far greater financial and technological capabilities to address it.
+
+Speaker 4: So it’s the idea that the polluter should pay. The ones who created the mess have a greater obligation to help clean it up, and to help protect those who are most harmed by it.
+
+Speaker 2: Exactly. Climate justice frameworks articulate this through the concept of a double inequality. The very people and nations who have contributed the least to the emissions that cause climate change are the ones who are suffering the earliest and most severe consequences. 
+
+Speaker 2: Therefore, a just global response requires that the developed nations lead the way in making the deepest emissions cuts, and that they provide substantial financial and technological support to help developing nations adapt to the impacts they did little to cause.
+
+Speaker 1: Carter, you were just explaining this core principle of climate justice, that the nations with the greatest historical responsibility for emissions also have the greatest capacity to help solve the problem.
+
+Speaker 2: Yes, and it builds on what Maya was saying. It’s about recognizing the profound unfairness, the, uh, double inequality that lies at the heart of the climate crisis. The people who are most harmed are the ones who did the least to cause the problem. Think about it, uh, a farmer in the Sahel whose land is turning to desert, or a family in a low-lying island nation whose home is threatened by sea level rise… their contribution to historical emissions is practically zero.
+
+Speaker 4: So what you're saying is, that farmer, whose crops are failing from a drought they had no part in creating, is right now paying a much, much higher price than someone in a wealthy country who has, you know, benefited from a century of industrial development powered by fossil fuels.
+
+Speaker 2: That is the injustice in a nutshell. And so, the framework for a just response is built on that understanding. It means developed nations have a moral and ethical obligation to lead with deep, rapid emissions cuts. And, crucially, it means they have an obligation to provide significant financial and technological support to help developing nations build clean economies and adapt to the impacts they are already facing.
+
+Speaker 3: I understand the moral argument. I do. But from a purely practical standpoint, it seems incredibly complicated. I mean, how far back do you go to assign this historical responsibility? Are you trying to calculate the emissions of the United Kingdom from the 1880s? It feels like an impossibly complex way to assign blame.
+
+Speaker 2: That's a fair point, Frank, and you know, it’s less about calculating precise historical blame and more about acknowledging the reality of the present day. The framework is not about punishing past generations. It's about recognizing which nations today have the accumulated wealth, the technology, and the stable institutions—many of which were built on that history of fossil-fueled development—to lead the global response. It’s about capability and responsibility in the here and now.
+
+Speaker 1: This whole conversation about justice, responsibility, and the immense shortfall in support really underscores the urgency of the crisis. And perhaps nothing in this entire report highlights that urgency more than the growing scientific understanding of a concept known as climate tipping points. Carter, for our listeners, what exactly is a tipping point?
+
+Speaker 2: It is probably the most sobering concept in all of climate science. The IPCC defines a tipping point as a critical threshold in the Earth's system. Once that threshold is crossed, a part of the system could trigger an abrupt, cascading, and potentially irreversible change.
+
+Speaker 1: Abrupt and irreversible. Those are two very powerful words. What does irreversible mean in this context?
+
+Speaker 2: It means that even if we managed to cool the planet back down later, the system might not flip back. The change could be locked in for centuries, or even millennia. We could pass a point of no return.
+
+Speaker 4: That is… a terrifying thought. So what are these systems? What parts of the planet are we talking about?
+
+Speaker 2: Scientists have identified several large-scale components of the Earth system that may have these tipping points. The most commonly discussed are the great ice sheets. We’re talking about the irreversible collapse of the Greenland and the West Antarctic ice sheets.
+
+Speaker 1: And what would be the consequence of something like that?
+
+Speaker 2: Well, uh, together, those two ice sheets hold enough frozen water to raise the global mean sea level by over 10 meters. That's about 33 feet.
+
+Speaker 4: Ten meters… I… I can’t even comprehend that. That's not just flooding. That is wiping entire cities, entire island nations, completely off the map for good.
+
+Speaker 2: Yes, the consequences would be civilization-altering. And another major tipping element is in the oceans themselves. A major slowdown or even a shutdown of the Atlantic Meridional Overturning Circulation, often called the AMOC.
+
+Speaker 3: The AMOC. I've heard of that, but it sounds like something out of a disaster movie. What does this current actually do for us?
+
+Speaker 2: It's a massive system of ocean currents that acts like a conveyor belt, transporting warm water from the tropics up to the North Atlantic. It plays a huge role in regulating weather patterns, especially in the Northern Hemisphere. 
+
+Speaker 2: A collapse of this system would drastically alter weather across North America and Europe, causing, you know, extreme cooling in some places, changing rainfall patterns, and disrupting monsoons that billions of people depend on for their food.
+
+Speaker 1: So we have the ice and the oceans. What else?
+
+Speaker 2: Then we have the biosphere systems. There are two major ones scientists are deeply concerned about. The first is the potential dieback of the Amazon rainforest.
+
+Speaker 1: So the Amazon could go from being this vital carbon sink that helps us, to becoming a major carbon source that actually hurts us?
+
+Speaker 2: Precisely. Large parts of the forest could transition into a drier, savanna-like ecosystem. And in doing so, it would release the vast quantities of carbon stored in its trees and soil, which would create a powerful feedback loop that accelerates even more global warming.
+
+Speaker 4: And the other one? You hear people talk about a ticking carbon bomb in the arctic. Is that what you mean?
+
+Speaker 2: That's the one. The abrupt, widespread thawing of permafrost. This is the permanently frozen ground in the arctic regions, and it contains enormous amounts of organic carbon that has been locked away for thousands of years. As it thaws, microbes decompose that organic matter and release it into the atmosphere as carbon dioxide and, even more potently, methane. This is another one of those dangerous feedback loops.
+
+Speaker 1: So Carter, we have these massive, continent-scale systems that could fundamentally break. I think for a long time, many of us thought of these tipping points as very distant risks. You know, things that might happen if warming got really, really bad, say, at five or six degrees Celsius. What does the latest science in the report say about that?
+
+Speaker 2: This, Alice, is perhaps the single most concerning finding to emerge in the last few years of research. The scientific consensus has shifted. Those early estimates that suggested these were high-warming risks have been revised. The latest research, which is cited in the IPCC reports, indicates that the temperature thresholds for triggering some of these tipping points may be much, much lower than we previously thought.
+
+Speaker 3: How much lower are we talking about?
+
+Speaker 2: The latest studies indicate that several of these major tipping points, including the collapse of the Greenland and West Antarctic ice sheets, the shutdown of the AMOC, and widespread permafrost thaw, could potentially be triggered at warming levels between 1.5 and 2.0 degrees Celsius.
+
+Speaker 4: But wait a minute. Carter, you said at the very, very beginning of our conversation that the world already temporarily breached 1.5 degrees of warming in 2024. If the trigger point is 1.5 degrees, what does that mean for us right now?
+
+Speaker 2: It means… well, it means that the risk is no longer a distant, abstract threat for future generations. It places the possibility of crossing these irreversible thresholds squarely within the realm of possibility this century. It moves the conversation from the future into the immediate present. 
+
+Speaker 2: And, you know, it adds a profound, almost existential urgency to the need for immediate, deep, and drastic emissions reductions. The window of opportunity to steer away from these points is closing, and it is closing very, very rapidly.
+
+Speaker 1: That is a deeply unsettling reality to confront, Carter. And Maya, I see you reacting to that. When you hear that the 1.5 degree line, which we’ve talked about for so long as this future guardrail, is not only something we've touched but is also the potential trigger for these irreversible changes… what does that feel like?
+
+Speaker 4: You know, it… it almost takes your breath away. It feels like we've been driving towards a cliff in the fog, arguing about how fast we should be going. And Carter is saying the fog has just cleared, and we're right at the edge. We’re there. That's a very, very hard thing to fully process.
+
+Speaker 3: It is. And it brings up a really difficult, practical question for me. If we're already on the verge of crossing these irreversible thresholds, what is the point of all this? I mean, does a 43 percent emissions cut by 2030, which already seems impossible, even matter anymore if the fuse has already been lit on something like the Greenland ice sheet? Have we… have we already lost the game?
+
+Speaker 2: Frank, that is the most important question anyone can ask right now. And the conclusion of the report, uh, argues that this is precisely why our actions now matter more than they ever have before. The first major conclusion is that the defining characteristic of the last decade is non-linear acceleration.
+
+Speaker 1: Okay, non-linear acceleration. Break that down for us.
+
+Speaker 2: Think of it like a car that's rolling down a hill. But the hill isn't a steady slope; it's a curve that gets steeper and steeper as you go. So for every foot you travel, your speed increases more than it did in the previous foot. You are accelerating exponentially, not in a straight line, not arithmetically. That’s what’s happening to our planetary systems. The risks are growing at an accelerating rate.
+
+Speaker 1: So every fraction of a degree of warming we can prevent now, every year we can act faster, has a much bigger impact in preventing that future acceleration than it would have twenty or thirty years ago.
+
+Speaker 2: Exactly. It’s what scientists call positive feedback loops becoming more potent. So, to answer Frank’s question, it’s the absolute opposite of the game being lost. It means the stakes of our actions in the next five to ten years are higher than they have ever been in human history. Every ton of carbon we keep out of the atmosphere now pays huge dividends in slowing down that terrifying acceleration toward those tipping points.
+
+Speaker 1: And the report also concludes that these are not isolated problems, correct? It talks about a cascade of interconnected crises.
+
+Speaker 2: Yes, that's the second key takeaway. We can no longer think of climate impacts as a series of separate events. A drought is not just a lack of water. It is a trigger. It triggers failures in the food system when crops fail. It triggers failures in the economic system when farmers lose their livelihoods. 
+
+Speaker 2: It triggers, you know, public health crises from malnutrition and water-borne diseases. It can even culminate in social instability and displacement. Climate change is a threat multiplier that makes all our existing vulnerabilities worse.
+
+Speaker 4: You can really see that in real life, can’t you? I mean, a wildfire isn't just a fire anymore. It becomes a public health crisis for millions of people breathing in the smoke. It's an economic crisis for the entire region. It becomes a water crisis months later when the first heavy rains wash toxic ash and debris into the reservoirs. You realize that one event pulls on all the other threads that hold our society together. Everything is connected.
+
+Speaker 2: That’s a perfect way to put it, Maya. And because everything is connected, the report concludes that our response has to be holistic. We can’t have siloed policies that address energy, or agriculture, or public health in isolation. They are all part of the same interconnected challenge.
+
+Speaker 1: This brings us to the third, and perhaps the toughest, conclusion from the report. Which is that our global response, as it stands today, is being dangerously outpaced by the physical reality of climate change.
+
+Speaker 2: That's the hard truth of the last decade. Despite all the meetings and the progress on renewables, the response remains critically insufficient. The report concludes that this failure is defined by three persistent and widening gaps. First is the ambition gap we already discussed, the gap between the weak climate pledges from countries and what science clearly shows is necessary.
+
+Speaker 1: And the second?
+
+Speaker 2: The second is the adaptation finance gap, which we just covered. The massive shortfall in funding that leaves the world’s most vulnerable populations essentially undefended against the coming storms and droughts. And the third is the justice gap, which undermines the trust and cooperation that are absolutely essential for any kind of effective global solution.
+
+Speaker 3: So if I'm hearing this correctly, the report’s ultimate conclusion is that our primary problem is no longer a technological one. We have the solar panels, we have the wind turbines, we have the efficiency solutions. The report is saying that the biggest barriers now are political, financial, and social. It's about a lack of political will, a failure to mobilize the necessary funds, and a failure to address the core injustices of the crisis.
+
+Speaker 2: That is the absolute crux of the conclusion. Technology is a vital tool, an essential tool, but it is not a silver bullet. The primary obstacles are now in our halls of government, in our financial institutions, and, uh, in our collective willingness to face this reality and act at the scale it requires.
+
+Speaker 1: So after this incredibly detailed and, frankly, alarming look back at the last decade, where does this leave us? We have a planet in a state of acceleration. We've temporarily breached the 1.5 degree threshold. And the risk of irreversible tipping points is no longer a future problem, but a present-day danger. Maya, I want to start with you. What’s your final takeaway?
+
+Speaker 4: It leaves me feeling that the time for simply being worried, or for abstract hope, is over. The only appropriate response to this level of evidence is determined action. This report is a story written in data, and it's telling us we have to transform this stark awareness into real, tangible work in our communities and demand it from our leaders. There’s no time for anything else.
+
+Speaker 1: Frank?
+
+Speaker 3: It leaves me thinking that we need to have a much more honest and pragmatic conversation about the real-world costs and trade-offs. We’ve talked about technology and policy, but this report makes it clear that the real fight is over politics and economics. And until we tackle that head-on, with honesty, we'll keep falling short.
+
+Speaker 1: And Carter, a final thought from you.
+
+Speaker 2: The science has been clear for a long time, but the evidence from this past decade is definitive. You know, this period from 2015 to 2025 will be remembered as the decade the consequences of our inaction became undeniable. That temporary breach of 1.5 degrees served as a final, unambiguous warning. The scientific challenge now is to monitor these accelerating changes. But the human challenge is to finally close those gaps between promises and performance, before those tipping points are crossed for good.
+
+Speaker 1: Carter, that is a powerful and frankly stark place to end, on the precipice of these tipping points with the clock running out. But... you know, before we wrap up completely, I want to hold on that last thought. The human challenge. I feel we can't end just with the warning. I want to pivot from the problems we've detailed so thoroughly to the specific pathways forward that are emerging. Beyond the high-level policy failures, where are the new fronts in this challenge?
+
+Speaker 2: That's a crucial pivot to make, Alice. Because, uh, despair is paralyzing. And despite the failures, there are new strategies and, you know, new arenas of action that are gaining momentum.
+
+Speaker 1: Let's talk about one of those. We've mentioned the justice gap and the economic challenges. What about the people, the workers and communities, whose entire livelihoods are tied to the fossil fuel industries we need to transition away from?
+
+Speaker 2: You're talking about the concept of a Just Transition. And you know, this has become a central part of the conversation because it's both morally right and politically essential. A Just Transition means ensuring that the shift to a green economy is fair and inclusive. It means we don't leave coal miners, oil rig workers, and entire communities that depend on these industries behind.

demo/voices/en-Alice_woman.wav

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c27ae47421436287a6bd2c3062de2dc2a2855b78c0bb626d472202c359704203
+size 296684

demo/voices/en-Carter_man.wav

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9dd7b12f25bf279d878a9f7a3125f64bff2b312a189959090acff9138a55e8dd
+size 1331244

demo/voices/en-Frank_man.wav

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:aa77c4794a005c4b05a52bbce5f30e77f0d28987b9a9e737401a5d30fd1ebcb5
+size 1158444

demo/voices/en-Mary_woman_bgm.wav

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c421eeab1af5b3ddae8d14cfcf6b65e496047ad2228325d61d1b6967fca11700
+size 1292878

demo/voices/en-Maya_woman.wav

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:eb1288bc02546c7f1117698fb78e994f060e623af148be8ccbf93dd0bea79e32
+size 1305644

demo/voices/in-Samuel_man.wav

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:76b07b5a12ca0b24a1e4a88100c4e2e47a2552ebb96807d52f116cf05fc46b50
+size 1273644

demo/voices/zh-Anchen_man_bgm.wav

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f71aeb33ed66c449dedb75d8a505478d86d47ec49e0e4c33c1fd0f8324d781fb
+size 1177644

demo/voices/zh-Bowen_man.wav

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0cef6c018e73e9fb6a1269fd61ded08144ae6380cdec242eebb1cc8aca49fed1
+size 1419940

demo/voices/zh-Xinran_woman.wav

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:dbcb9e28bcc544675ef75a8ba12528bf09e713eb53a8c0c819dec3daf2d486d3
+size 1337644

model-00001-of-00003.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c5f0a61ddeaeb028e3af540ba4dee7933ad30f9f30b6e1320dd9c875a2daa033
+size 1975317828

model-00002-of-00003.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:81c3891f7b2493eb48a9eb6f5be0df48d4f1a4bfd952d84e21683ca6d0bf7969
+size 1983051688

model-00003-of-00003.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:cb6e7e5e86b4a41fffbe1f3aaf445d0d50b5e21ed47574101b777f77d75fa196
+size 1449832938

preprocessor_config.json

@@ -0,0 +1,13 @@
+{
+  "processor_class": "VibeVoiceProcessor",
+  "speech_tok_compress_ratio": 3200,
+  "db_normalize": true,
+  "audio_processor": {
+    "feature_extractor_type": "VibeVoiceTokenizerProcessor",
+    "sampling_rate": 24000,
+    "normalize_audio": true,
+    "target_dB_FS": -25,
+    "eps": 1e-06
+  },
+  "language_model_pretrained_name": "Qwen/Qwen2.5-1.5B"
+}

pyproject.toml

@@ -0,0 +1,43 @@
+[build-system]
+requires = ["setuptools>=61.0"]
+build-backend = "setuptools.build_meta"
+
+[project]
+name = "vibevoice"
+version = "0.0.1"
+authors = [
+  { name="vibevoice team", email="[email protected]" },
+]
+description = "A model for speech generation with an AR + diffusion architecture."
+readme = "README.md"
+requires-python = ">=3.9"
+classifiers = [
+    "Programming Language :: Python :: 3",
+    # "License :: OSI Approved :: MIT License",
+    "Operating System :: OS Independent",
+]
+dependencies = [
+    "torch",
+    "accelerate==1.6.0",
+    "transformers==4.51.3", # we develop this project on transformers==4.51.3, later version may not be compatible
+    "llvmlite>=0.40.0",
+    "numba>=0.57.0",
+    "diffusers",
+    "tqdm",
+    "numpy",
+    "scipy",
+    "librosa",
+    "ml-collections",
+    "absl-py",
+    "gradio",
+    "av",
+    "aiortc"
+]
+
+
+[project.urls]
+"Homepage" = "https://github.com/microsoft/VibeVoice"
+"Bug Tracker" = "https://github.com/microsoft/VibeVoice/issues"
+
+[tool.setuptools.packages.find]
+where = ["."]

vibevoice/configs/qwen2.5_1.5b_64k.json

@@ -0,0 +1,112 @@
+{
+  "_attn_implementation_autoset": true,
+  "acoustic_vae_dim": 64,
+  "acoustic_tokenizer_config": {
+    "causal": true,
+    "channels": 1,
+    "conv_bias": true,
+    "conv_norm": "none",
+    "corpus_normalize": 0.0,
+    "decoder_depths": null,
+    "decoder_n_filters": 32,
+    "decoder_ratios": [
+      8,
+      5,
+      5,
+      4,
+      2,
+      2
+    ],
+    "disable_last_norm": true,
+    "encoder_depths": "3-3-3-3-3-3-8",
+    "encoder_n_filters": 32,
+    "encoder_ratios": [
+      8,
+      5,
+      5,
+      4,
+      2,
+      2
+    ],
+    "fix_std": 0.5,
+    "layer_scale_init_value": 1e-06,
+    "layernorm": "RMSNorm",
+    "layernorm_elementwise_affine": true,
+    "layernorm_eps": 1e-05,
+    "mixer_layer": "depthwise_conv",
+    "model_type": "vibepod_acoustic_tokenizer",
+    "pad_mode": "constant",
+    "std_dist_type": "gaussian",
+    "vae_dim": 64,
+    "weight_init_value": 0.01
+  },
+  "decoder_config": {
+    "attention_dropout": 0.0,
+    "hidden_act": "silu",
+    "hidden_size": 1536,
+    "initializer_range": 0.02,
+    "intermediate_size": 8960,
+    "max_position_embeddings": 65536,
+    "max_window_layers": 28,
+    "model_type": "qwen2",
+    "num_attention_heads": 12,
+    "num_hidden_layers": 28,
+    "num_key_value_heads": 2,
+    "rms_norm_eps": 1e-06,
+    "rope_scaling": null,
+    "rope_theta": 1000000.0,
+    "sliding_window": null,
+    "tie_word_embeddings": true,
+    "torch_dtype": "bfloat16",
+    "use_cache": true,
+    "use_sliding_window": false,
+    "vocab_size": 151936
+  },
+  "diffusion_head_config": {
+    "ddpm_batch_mul": 4,
+    "ddpm_beta_schedule": "cosine",
+    "ddpm_num_inference_steps": 20,
+    "ddpm_num_steps": 1000,
+    "diffusion_type": "ddpm",
+    "head_ffn_ratio": 3.0,
+    "head_layers": 4,
+    "hidden_size": 1536,
+    "latent_size": 64,
+    "model_type": "vibepod_diffusion_head",
+    "prediction_type": "v_prediction",
+    "rms_norm_eps": 1e-05,
+    "speech_vae_dim": 64
+  },
+  "model_type": "vibepod",
+  "semantic_tokenizer_config": {
+    "causal": true,
+    "channels": 1,
+    "conv_bias": true,
+    "conv_norm": "none",
+    "corpus_normalize": 0.0,
+    "disable_last_norm": true,
+    "encoder_depths": "3-3-3-3-3-3-8",
+    "encoder_n_filters": 32,
+    "encoder_ratios": [
+      8,
+      5,
+      5,
+      4,
+      2,
+      2
+    ],
+    "fix_std": 0,
+    "layer_scale_init_value": 1e-06,
+    "layernorm": "RMSNorm",
+    "layernorm_elementwise_affine": true,
+    "layernorm_eps": 1e-05,
+    "mixer_layer": "depthwise_conv",
+    "model_type": "vibepod_semantic_tokenizer",
+    "pad_mode": "constant",
+    "std_dist_type": "none",
+    "vae_dim": 128,
+    "weight_init_value": 0.01
+  },
+  "semantic_vae_dim": 128,
+  "torch_dtype": "bfloat16"
+}

vibevoice/configs/qwen2.5_7b_32k.json

@@ -0,0 +1,113 @@
+{
+  "_attn_implementation_autoset": true,
+  "acoustic_vae_dim": 64,
+  "acoustic_tokenizer_config": {
+    "causal": true,
+    "channels": 1,
+    "conv_bias": true,
+    "conv_norm": "none",
+    "corpus_normalize": 0.0,
+    "decoder_depths": null,
+    "decoder_n_filters": 32,
+    "decoder_ratios": [
+      8,
+      5,
+      5,
+      4,
+      2,
+      2
+    ],
+    "disable_last_norm": true,
+    "encoder_depths": "3-3-3-3-3-3-8",
+    "encoder_n_filters": 32,
+    "encoder_ratios": [
+      8,
+      5,
+      5,
+      4,
+      2,
+      2
+    ],
+    "fix_std": 0.5,
+    "layer_scale_init_value": 1e-06,
+    "layernorm": "RMSNorm",
+    "layernorm_elementwise_affine": true,
+    "layernorm_eps": 1e-05,
+    "mixer_layer": "depthwise_conv",
+    "model_type": "vibepod_acoustic_tokenizer",
+    "pad_mode": "constant",
+    "std_dist_type": "gaussian",
+    "vae_dim": 64,
+    "weight_init_value": 0.01
+  },
+  "decoder_config": {
+    "attention_dropout": 0.0,
+    "hidden_act": "silu",
+    "hidden_size": 3584,
+    "initializer_range": 0.02,
+    "intermediate_size": 18944,
+    "max_position_embeddings": 32768,
+    "max_window_layers": 28,
+    "model_type": "qwen2",
+    "num_attention_heads": 28,
+    "num_hidden_layers": 28,
+    "num_key_value_heads": 4,
+    "rms_norm_eps": 1e-06,
+    "rope_theta": 1000000.0,
+    "sliding_window": null,
+    "tie_word_embeddings": false,
+    "torch_dtype": "bfloat16",
+    "transformers_version": "4.40.1",
+    "use_cache": true,
+    "use_mrope": false,
+    "use_sliding_window": false,
+    "vocab_size": 152064
+  },
+  "diffusion_head_config": {
+    "ddpm_batch_mul": 4,
+    "ddpm_beta_schedule": "cosine",
+    "ddpm_num_inference_steps": 20,
+    "ddpm_num_steps": 1000,
+    "diffusion_type": "ddpm",
+    "head_ffn_ratio": 3.0,
+    "head_layers": 4,
+    "hidden_size": 3584,
+    "latent_size": 64,
+    "model_type": "vibepod_diffusion_head",
+    "prediction_type": "v_prediction",
+    "rms_norm_eps": 1e-05,
+    "speech_vae_dim": 64
+  },
+  "model_type": "vibepod",
+  "semantic_tokenizer_config": {
+    "causal": true,
+    "channels": 1,
+    "conv_bias": true,
+    "conv_norm": "none",
+    "corpus_normalize": 0.0,
+    "disable_last_norm": true,
+    "encoder_depths": "3-3-3-3-3-3-8",
+    "encoder_n_filters": 32,
+    "encoder_ratios": [
+      8,
+      5,
+      5,
+      4,
+      2,
+      2
+    ],
+    "fix_std": 0,
+    "layer_scale_init_value": 1e-06,
+    "layernorm": "RMSNorm",
+    "layernorm_elementwise_affine": true,
+    "layernorm_eps": 1e-05,
+    "mixer_layer": "depthwise_conv",
+    "model_type": "vibepod_semantic_tokenizer",
+    "pad_mode": "constant",
+    "std_dist_type": "none",
+    "vae_dim": 128,
+    "weight_init_value": 0.01
+  },
+  "semantic_vae_dim": 128,
+  "torch_dtype": "bfloat16"
+}

vibevoice/modular/configuration_vibevoice.py

@@ -0,0 +1,248 @@
+""" VibeVoice_AcousticTokenizer model configuration"""
+
+from typing import Dict, List, Optional, Tuple
+
+from transformers.configuration_utils import PretrainedConfig 
+from transformers.utils import logging
+
+from transformers.models.qwen2.configuration_qwen2 import Qwen2Config
+
+logger = logging.get_logger(__name__)
+
+
+class VibeVoiceAcousticTokenizerConfig(PretrainedConfig):
+    model_type = "vibevoice_acoustic_tokenizer"
+
+    def __init__(
+        self,
+        channels: int = 1,
+        corpus_normalize: float = 0.0,
+        causal: bool = True,
+        vae_dim: int = 64,
+        fix_std: float = 0.5,
+        std_dist_type: str = 'gaussian',
+        # common 
+        mixer_layer: str = 'depthwise_conv',
+        conv_norm: str = 'none',
+        pad_mode: str = 'constant',
+        disable_last_norm: bool = True,
+        layernorm: str = 'RMSNorm',
+        layernorm_eps: float = 1e-5,
+        layernorm_elementwise_affine: bool = True,
+        conv_bias: bool = True,
+        layer_scale_init_value: float = 1e-6,
+        weight_init_value: float = 1e-2,
+        # encoder specific
+        encoder_n_filters: int = 32,
+        encoder_ratios: Optional[List[int]] = [8,5,5,4,2,2],
+        encoder_depths: str = "3-3-3-3-3-3-8",
+        # decoder specific
+        decoder_n_filters: int = 32,
+        decoder_ratios: Optional[List[int]] = None, # if None, same as encoder
+        decoder_depths: Optional[str] = None,
+        **kwargs
+    ):
+        super().__init__(**kwargs)
+        self.channels = channels
+        self.corpus_normalize = corpus_normalize
+        self.causal = causal
+        self.vae_dim = vae_dim
+        self.fix_std = fix_std
+        self.std_dist_type = std_dist_type
+        
+        # common parameters
+        self.conv_norm = conv_norm
+        self.pad_mode = pad_mode
+        self.layernorm_eps = layernorm_eps
+        self.disable_last_norm = disable_last_norm
+        self.layernorm = layernorm
+        self.layernorm_elementwise_affine = layernorm_elementwise_affine
+        self.conv_bias = conv_bias
+        self.layer_scale_init_value = layer_scale_init_value
+        self.weight_init_value = weight_init_value
+        self.mixer_layer = mixer_layer
+
+        # encoder specific parameters
+        self.encoder_n_filters = encoder_n_filters
+        self.encoder_ratios = encoder_ratios
+        self.encoder_depths = encoder_depths
+        
+        # decoder specific parameters
+        self.decoder_ratios = decoder_ratios if decoder_ratios is not None else encoder_ratios
+        self.decoder_n_filters = decoder_n_filters
+        self.decoder_depths = decoder_depths
+
+
+class VibeVoiceSemanticTokenizerConfig(PretrainedConfig):
+    model_type = "vibevoice_semantic_tokenizer"
+    
+    def __init__(
+        self,
+        channels: int = 1,
+        corpus_normalize: float = 0.0,
+        causal: bool = True,
+        vae_dim: int = 64,
+        fix_std: float = 0,
+        std_dist_type: str = 'none',
+        # common 
+        mixer_layer: str = 'depthwise_conv',
+        conv_norm: str = 'none',
+        pad_mode: str = 'constant',
+        disable_last_norm: bool = True,
+        layernorm: str = 'RMSNorm',
+        layernorm_eps: float = 1e-5,
+        layernorm_elementwise_affine: bool = True,
+        conv_bias: bool = True,
+        layer_scale_init_value: float = 1e-6,
+        weight_init_value: float = 1e-2,
+        # encoder specific
+        encoder_n_filters: int = 32,
+        encoder_ratios: Optional[List[int]] = [8,5,5,4,2,2],
+        encoder_depths: str = "3-3-3-3-3-3-8",
+        **kwargs
+    ):
+        super().__init__(**kwargs)
+        self.channels = channels
+        self.corpus_normalize = corpus_normalize
+        self.causal = causal
+        self.vae_dim = vae_dim
+        self.fix_std = fix_std
+        self.std_dist_type = std_dist_type
+        
+        # common parameters
+        self.conv_norm = conv_norm
+        self.pad_mode = pad_mode
+        self.layernorm_eps = layernorm_eps
+        self.disable_last_norm = disable_last_norm
+        self.layernorm = layernorm
+        self.layernorm_elementwise_affine = layernorm_elementwise_affine
+        self.conv_bias = conv_bias
+        self.layer_scale_init_value = layer_scale_init_value
+        self.weight_init_value = weight_init_value
+        self.mixer_layer = mixer_layer
+
+        # encoder specific parameters
+        self.encoder_n_filters = encoder_n_filters
+        self.encoder_ratios = encoder_ratios
+        self.encoder_depths = encoder_depths
+        
+
+class VibeVoiceDiffusionHeadConfig(PretrainedConfig):
+    model_type = "vibevoice_diffusion_head"
+
+    def __init__(
+        self,
+        hidden_size=768,
+        head_layers=4,
+        head_ffn_ratio=3.0,
+        rms_norm_eps=1e-5,
+        latent_size=64,
+        speech_vae_dim=None,
+        prediction_type="v_prediction",
+        diffusion_type="ddpm",
+        ddpm_num_steps=1000,
+        ddpm_num_inference_steps=20,
+        ddpm_beta_schedule="cosine",
+        ddpm_batch_mul=4,
+        **kwargs
+    ):
+        self.hidden_size = hidden_size
+        self.head_layers = head_layers
+        self.head_ffn_ratio = head_ffn_ratio
+        self.rms_norm_eps = rms_norm_eps
+        self.latent_size = latent_size
+        self.speech_vae_dim = speech_vae_dim
+        self.prediction_type = prediction_type
+        self.diffusion_type = diffusion_type
+        self.ddpm_num_steps = ddpm_num_steps
+        self.ddpm_num_inference_steps = ddpm_num_inference_steps
+        self.ddpm_beta_schedule = ddpm_beta_schedule
+        self.ddpm_batch_mul = ddpm_batch_mul
+        
+        super().__init__(**kwargs)
+
+class VibeVoiceConfig(PretrainedConfig):
+    model_type = "vibevoice"
+    is_composition = True
+    sub_configs = {
+        "acoustic_tokenizer_config": VibeVoiceAcousticTokenizerConfig, 
+        "semantic_tokenizer_config": VibeVoiceSemanticTokenizerConfig,
+        "decoder_config": Qwen2Config,
+        "diffusion_head_config": VibeVoiceDiffusionHeadConfig,
+    }
+    # keys_to_ignore_at_inference = ["past_key_values"]
+    # Default tensor parallel plan for base model `Qwen2`
+    base_model_tp_plan = {
+        "layers.*.self_attn.q_proj": "colwise",
+        "layers.*.self_attn.k_proj": "colwise",
+        "layers.*.self_attn.v_proj": "colwise",
+        "layers.*.self_attn.o_proj": "rowwise",
+        "layers.*.mlp.gate_proj": "colwise",
+        "layers.*.mlp.up_proj": "colwise",
+        "layers.*.mlp.down_proj": "rowwise",
+    }
+    
+    def __init__(
+        self,
+        acoustic_tokenizer_config=None,
+        semantic_tokenizer_config=None,
+        decoder_config=None,
+        diffusion_head_config=None,
+        **kwargs
+    ):
+
+        # kwargs["_attn_implementation"] = "flash_attention_2"
+        kwargs["_attn_implementation_autoset"] = False 
+
+        if acoustic_tokenizer_config is None:
+            self.acoustic_tokenizer_config = self.sub_configs["acoustic_tokenizer_config"]()
+        elif isinstance(acoustic_tokenizer_config, dict):
+            acoustic_tokenizer_config["model_type"] = "vibevoice_acoustic_tokenizer"
+            self.acoustic_tokenizer_config = self.sub_configs["acoustic_tokenizer_config"](**acoustic_tokenizer_config)
+        elif isinstance(acoustic_tokenizer_config, VibeVoiceAcousticTokenizerConfig):
+            # If an instance of the config class is provided
+            self.acoustic_tokenizer_config = acoustic_tokenizer_config
+
+        if semantic_tokenizer_config is None:
+            self.semantic_tokenizer_config = self.sub_configs["semantic_tokenizer_config"]()
+        elif isinstance(semantic_tokenizer_config, dict):
+            semantic_tokenizer_config["model_type"] = "vibevoice_semantic_tokenizer"
+            self.semantic_tokenizer_config = self.sub_configs["semantic_tokenizer_config"](**semantic_tokenizer_config)
+        elif isinstance(semantic_tokenizer_config, VibeVoiceSemanticTokenizerConfig):
+            # If an instance of the config class is provided
+            self.semantic_tokenizer_config = semantic_tokenizer_config
+
+        if decoder_config is None:
+            self.decoder_config = self.sub_configs["decoder_config"]()
+        elif isinstance(decoder_config, dict):
+            # If a dictionary is provided, instantiate the config class with it
+            # self.decoder_config = self.sub_configs["decoder_config"](**decoder_config)
+            if decoder_config.get("model_type", '') == "qwen2":
+                self.decoder_config = Qwen2Config(**decoder_config)
+            else:
+                raise ValueError(f"Unsupported decoder model type: {decoder_config.get('model_type', '')}")
+        elif isinstance(decoder_config, (Qwen2Config,)):
+            # If an instance of the config class is provided
+            self.decoder_config = decoder_config
+
+        if diffusion_head_config is None:
+            self.diffusion_head_config = self.sub_configs["diffusion_head_config"]()
+        elif isinstance(diffusion_head_config, dict):
+            diffusion_head_config["model_type"] = "vibevoice_diffusion_head"
+            self.diffusion_head_config = self.sub_configs["diffusion_head_config"](**diffusion_head_config)
+        elif isinstance(diffusion_head_config, VibeVoiceDiffusionHeadConfig):
+            # If an instance of the config class is provided
+            self.diffusion_head_config = diffusion_head_config
+
+        # other parameters
+        self.acoustic_vae_dim = getattr(self.acoustic_tokenizer_config, 'vae_dim', 64)
+        self.semantic_vae_dim = getattr(self.semantic_tokenizer_config, 'vae_dim', 128)
+
+        super().__init__(**kwargs)
+
+__all__ = [
+    "VibeVoiceAcousticTokenizerConfig", 
+    "VibeVoiceSemanticTokenizerConfig", 
+    "VibeVoiceDiffusionHeadConfig", 
+    "VibeVoiceConfig"
+]

vibevoice/modular/modeling_vibevoice.py

@@ -0,0 +1,488 @@
+from dataclasses import dataclass
+from typing import Dict, List, Optional, Tuple, Union, Callable
+from tqdm import tqdm
+import torch
+import torch.nn as nn
+import torch.nn.functional as F
+import torch.distributed as dist
+
+from transformers.models.auto import AutoModel, AutoModelForCausalLM
+
+from transformers.activations import ACT2FN
+from transformers.modeling_outputs import CausalLMOutput, BaseModelOutputWithPast, ModelOutput
+from transformers.models.llama.modeling_llama import LlamaRMSNorm
+from transformers import modeling_utils
+from transformers.modeling_utils import PreTrainedModel
+from transformers.modeling_flash_attention_utils import FlashAttentionKwargs
+from transformers.utils import logging
+
+
+from .modular_vibevoice_tokenizer import VibeVoiceTokenizerStreamingCache, VibeVoiceAcousticTokenizerModel, VibeVoiceSemanticTokenizerModel
+from .modular_vibevoice_diffusion_head import VibeVoiceDiffusionHead
+from vibevoice.schedule.dpm_solver import DPMSolverMultistepScheduler
+
+from .configuration_vibevoice import VibeVoiceConfig
+
+
+logger = logging.get_logger(__name__)
+
+if not hasattr(modeling_utils, "ALL_PARALLEL_STYLES") or modeling_utils.ALL_PARALLEL_STYLES is None:
+    modeling_utils.ALL_PARALLEL_STYLES = ["tp", "none", "colwise", "rowwise"]
+
+@dataclass
+class VibeVoiceCausalLMOutputWithPast(ModelOutput):
+    loss: Optional[torch.FloatTensor] = None
+    diffusion_loss: Optional[torch.FloatTensor] = None
+    speech_token_num: Optional[int] = None
+    logits: torch.FloatTensor = None
+    past_key_values: Optional[Tuple[Tuple[torch.FloatTensor]]] = None
+    hidden_states: Optional[Tuple[torch.FloatTensor, ...]] = None
+    attentions: Optional[Tuple[torch.FloatTensor, ...]] = None
+
+
+@dataclass
+class VibeVoiceGenerationOutput(ModelOutput):
+    """
+    Output type for VibeVoice generation.
+    
+    Args:
+        sequences (`torch.LongTensor` of shape `(batch_size, sequence_length)`):
+            The generated sequences. 
+        speech_outputs (`List[torch.FloatTensor]`, *optional*):
+            List of generated speech waveforms or latents for each speech segment.
+    """
+    sequences: torch.LongTensor = None
+    speech_outputs: Optional[List[torch.FloatTensor]] = None
+
+
+class SpeechConnector(nn.Module):
+    def __init__(self, input_dim, output_dim):
+        super().__init__()
+        self.fc1 = nn.Linear(input_dim, output_dim)
+        self.norm = LlamaRMSNorm(output_dim, eps=1e-6)
+        self.fc2 = nn.Linear(output_dim, output_dim)
+
+    def forward(self, features, **kwargs):    
+        x = self.fc1(features)
+        x = self.norm(x)
+        x = self.fc2(x)
+        return x
+
+
+# @auto_docstring
+class VibeVoicePreTrainedModel(PreTrainedModel):
+    config_class = VibeVoiceConfig
+    base_model_prefix = "model"
+    supports_gradient_checkpointing = True
+    _skip_keys_device_placement = "past_key_values"
+    _supports_cache_class = True
+    _supports_flash_attn_2 = True
+    _supports_sdpa = True
+    _supports_quantized_cache = True
+    _supports_static_cache = True
+    _supports_attention_backend = True
+
+    def _init_weights(self, module):
+        if isinstance(module, VibeVoiceDiffusionHead):
+            module.initialize_weights()
+            return
+
+        # Use the language model's initializer_range if available
+        if hasattr(self.config, 'language_model_config') and hasattr(self.config.language_model_config, 'initializer_range'):
+            std = self.config.language_model_config.initializer_range
+        elif hasattr(self.config, 'decoder_config') and hasattr(self.config.decoder_config, 'initializer_range'):
+            std = self.config.decoder_config.initializer_range
+        else:
+            std = 0.02  # Default value
+            
+        if isinstance(module, nn.Linear):
+            module.weight.data.normal_(mean=0.0, std=std)
+            if module.bias is not None:
+                module.bias.data.zero_()
+        elif isinstance(module, nn.LayerNorm):
+            module.weight.data.fill_(1.0)
+            module.bias.data.zero_()
+
+# @auto_docstring
+class VibeVoiceModel(VibeVoicePreTrainedModel):
+    def __init__(self, config):
+        super().__init__(config)
+        
+        if hasattr(config, 'torch_dtype') and config.torch_dtype is not None:
+            if isinstance(config.torch_dtype, str):
+                dtype = getattr(torch, config.torch_dtype)
+            else:
+                dtype = config.torch_dtype
+        else:
+            dtype = torch.float32
+        
+        # Initialize Qwen2 model for language modeling
+        lm_config = config.decoder_config 
+        self.language_model = AutoModel.from_config(lm_config)
+        
+        # Initialize speech components if needed
+        self.acoustic_tokenizer = AutoModel.from_config(config.acoustic_tokenizer_config).to(dtype)
+        self.semantic_tokenizer = AutoModel.from_config(config.semantic_tokenizer_config).to(dtype)
+
+        self.acoustic_connector = SpeechConnector(config.acoustic_vae_dim, lm_config.hidden_size).to(dtype)
+        self.semantic_connector = SpeechConnector(config.semantic_vae_dim, lm_config.hidden_size).to(dtype)
+        
+        # Register scaling factors as buffers - use 1D tensors for FSDP compatibility
+        self.register_buffer('speech_scaling_factor', torch.tensor(float('nan')))  
+        self.register_buffer('speech_bias_factor', torch.tensor(float('nan')))
+
+        # Initialize prediction head for speech generation
+        self.prediction_head = AutoModel.from_config(config.diffusion_head_config).to(dtype)
+
+        # Initialize noise scheduler
+        self.noise_scheduler = DPMSolverMultistepScheduler(
+            num_train_timesteps=config.diffusion_head_config.ddpm_num_steps,
+            beta_schedule=config.diffusion_head_config.ddpm_beta_schedule,
+            prediction_type=config.diffusion_head_config.prediction_type
+        )
+    
+    def get_input_embeddings(self):
+        if hasattr(self.language_model, 'embed_tokens'):
+            # If the language model has an embed_tokens attribute, return it
+            return self.language_model.embed_tokens
+        
+        for name, attr in self.language_model.fullmap.items(): # parallel by nnscaler, the name is changed
+            if attr.orig_name == 'embed_tokens.weight':
+                return getattr(self.language_model, name)
+        assert False, 'should not arrive here'
+
+    def set_input_embeddings(self, value):
+        self.language_model.embed_tokens = value
+    
+    def set_speech_tokenizers(self, acoustic_tokenizer=None, semantic_tokenizer=None):
+        """Set the speech tokenizers used for encoding and decoding speech."""
+        self.acoustic_tokenizer = acoustic_tokenizer
+        self.semantic_tokenizer = semantic_tokenizer
+        
+        # Reset the encoder to evaluation mode
+        if self.acoustic_tokenizer is not None:
+            self.acoustic_tokenizer.eval()
+            
+        if self.semantic_tokenizer is not None:
+            self.semantic_tokenizer.eval()
+    
+    def forward(
+        self,
+        input_ids: torch.LongTensor = None,
+        attention_mask: Optional[torch.Tensor] = None,
+        position_ids: Optional[torch.LongTensor] = None,
+        past_key_values: Optional[Tuple[Tuple[torch.FloatTensor]]] = None,
+        inputs_embeds: Optional[torch.FloatTensor] = None,
+        use_cache: Optional[bool] = None,
+        output_attentions: Optional[bool] = None,
+        output_hidden_states: Optional[bool] = None,
+        return_dict: Optional[bool] = None,
+        cache_position: Optional[torch.LongTensor] = None,
+        **kwargs,
+    ) -> Union[Tuple, BaseModelOutputWithPast]:
+        
+        return_dict = return_dict if return_dict is not None else self.config.use_return_dict
+        
+        # Forward through language model
+        outputs = self.language_model(
+            input_ids=input_ids,
+            attention_mask=attention_mask,
+            position_ids=position_ids,
+            past_key_values=past_key_values,
+            inputs_embeds=inputs_embeds,
+            use_cache=use_cache,
+            output_attentions=output_attentions,
+            output_hidden_states=output_hidden_states,
+            return_dict=return_dict,
+            cache_position=cache_position,
+            **kwargs,
+        )
+        
+        if not return_dict:
+            return outputs
+            
+        return BaseModelOutputWithPast(
+            last_hidden_state=outputs.last_hidden_state,
+            past_key_values=outputs.past_key_values,
+            hidden_states=outputs.hidden_states,
+            attentions=outputs.attentions,
+        )
+
+
+class VibeVoiceForConditionalGeneration(VibeVoicePreTrainedModel):
+    _tied_weights_keys = ["lm_head.weight"]
+    _tp_plan = {"lm_head": "colwise_rep"}
+
+    def __init__(self, config):
+        super().__init__(config)
+        self.model = VibeVoiceModel(config)
+        self.vocab_size = config.decoder_config.vocab_size
+        self.lm_head = nn.Linear(config.decoder_config.hidden_size, self.vocab_size, bias=False)
+
+        self.post_init()
+        
+    def get_input_embeddings(self):
+        return self.model.get_input_embeddings()
+
+    def set_input_embeddings(self, value):
+        self.model.set_input_embeddings(value)
+
+    def get_output_embeddings(self):
+        return self.lm_head
+
+    def set_decoder(self, decoder):
+        self.model.language_model = decoder
+
+    def get_decoder(self):
+        return self.model.language_model
+
+    def tie_weights(self):
+        """
+        Tie the weights between the input embeddings and the output embeddings.
+        """
+        if getattr(self.config.decoder_config, 'tie_word_embeddings', False):
+            # The standard PreTrainedModel method will handle the tying.
+            # It typically does a simple parameter object assignment, which is
+            # CORRECT to do BEFORE FSDP wraps the model.
+            output_embeddings = self.get_output_embeddings()
+            input_embeddings = self.get_input_embeddings()
+            if hasattr(input_embeddings, 'weight'):
+                output_embeddings.weight = input_embeddings.weight
+            else:
+                # maybe returned input_embeddings a tensor directly
+                output_embeddings.weight = input_embeddings
+
+            if getattr(output_embeddings, "bias", None) is not None:
+                output_embeddings.bias.data = nn.functional.pad(
+                    output_embeddings.bias.data,
+                    (0, output_embeddings.weight.shape[0] - output_embeddings.bias.shape[0]),
+                    "constant",
+                    0,
+                )
+            print("✅ Tied input and output embeddings using standard assignment.")
+        else:
+            print("ℹ️  tie_word_embeddings is False, not tying weights.")
+
+    # Also, ensure set_output_embeddings is safe, though your implementation looks okay.
+    # The key is to avoid calling it after accelerator.prepare().
+    def set_output_embeddings(self, new_embeddings):
+        # Your current implementation using data.copy_ is good practice,
+        # but the best way is to not call this after prepare().
+        self.lm_head = new_embeddings
+
+    def forward_speech_features(
+            self, 
+            speech_tensors=None, 
+            speech_masks=None, 
+            speech_type="audio", 
+            return_unmask=False
+        ):
+        if speech_tensors is None:
+            # Use config to get vae_dim instead of non-existent self.args
+            vae_dim = self.config.acoustic_tokenizer_config.vae_dim
+            audio_features = torch.zeros(1, 1, vae_dim).to(self.get_input_embeddings().weight)
+            connect_features = self.model.acoustic_connector(audio_features)
+            return audio_features, connect_features
+        else:
+            with torch.no_grad():
+                if speech_type == "audio":
+                    with torch.no_grad():
+                        frames = self.model.acoustic_tokenizer.encode(speech_tensors.unsqueeze(1))[0][0]
+                    audio_tokens = frames.sample(self.model.acoustic_tokenizer.std_dist_type)[0]
+
+                elif speech_type == "vae":
+                    # Use config to get vae_dim instead of non-existent self.args
+                    vae_dim = self.config.acoustic_tokenizer_config.vae_dim
+                    speech_mode = speech_tensors.reshape(speech_tensors.size(0), -1, vae_dim)
+
+                    # gaussian sample from the speech_mode
+                    batch_size = speech_mode.size(0)
+                    value = self.model.acoustic_tokenizer.fix_std / 0.8
+                    std = torch.randn(batch_size, dtype=speech_mode.dtype, device=speech_mode.device) * value
+                    std = std.view(-1, *[1] * (speech_mode.dim() - 1))
+                    audio_tokens = speech_mode + std * torch.randn(speech_mode.shape).to(speech_mode)
+                else:
+                    raise NotImplementedError(f"Speech type {speech_type} not implemented")
+                
+                if torch.isnan(self.model.speech_scaling_factor) or torch.isnan(self.model.speech_bias_factor):
+                    scaling_factor = 1. / audio_tokens[speech_masks].flatten().std()
+                    bias_factor = -audio_tokens[speech_masks].flatten().mean()
+                    
+                    # Only use distributed operations if the process group is initialized
+                    if dist.is_available() and dist.is_initialized():
+                        dist.all_reduce(scaling_factor, op=dist.ReduceOp.SUM)
+                        dist.all_reduce(bias_factor, op=dist.ReduceOp.SUM)
+                        world_size = dist.get_world_size()
+                        self.model.speech_scaling_factor.copy_(scaling_factor / world_size)  
+                        self.model.speech_bias_factor.copy_(bias_factor / world_size)
+                        print(f"Speech scaling factor (distributed): {self.model.speech_scaling_factor}, bias factor: {self.model.speech_bias_factor}", flush=True)
+                    else:
+                        # Single process case
+                        self.model.speech_scaling_factor.copy_(scaling_factor)  
+                        self.model.speech_bias_factor.copy_(bias_factor)
+                        print(f"Speech scaling factor (single process): {self.model.speech_scaling_factor}, bias factor: {self.model.speech_bias_factor}", flush=True)
+                    
+                audio_features = (audio_tokens + self.model.speech_bias_factor) * self.model.speech_scaling_factor
+            
+            connect_features = self.model.acoustic_connector(audio_features)
+            if return_unmask:
+                return audio_features, connect_features
+            return audio_features[speech_masks], connect_features[speech_masks]
+        
+    def forward(
+        self,
+        input_ids: torch.LongTensor = None,
+        attention_mask: Optional[torch.Tensor] = None,
+        position_ids: Optional[torch.LongTensor] = None,
+        past_key_values: Optional[List[torch.FloatTensor]] = None,
+        inputs_embeds: Optional[torch.FloatTensor] = None,
+        labels: Optional[torch.LongTensor] = None,
+        use_cache: Optional[bool] = False,
+        output_attentions: Optional[bool] = None,
+        output_hidden_states: Optional[bool] = None,
+        return_dict: Optional[bool] = None,
+        cache_position: Optional[torch.LongTensor] = None,
+        # New arguments for speech processing and loss calculation
+        speech_tensors: Optional[torch.FloatTensor] = None,
+        speech_masks: Optional[torch.BoolTensor] = None,
+        speeches_loss_input: Optional[torch.FloatTensor] = None,
+        speech_semantic_tensors: Optional[torch.FloatTensor] = None, 
+        acoustic_input_mask: Optional[torch.BoolTensor] = None,
+        acoustic_loss_mask: Optional[torch.BoolTensor] = None,
+        ddpm_batch_mul: int = 1,
+        **kwargs: Optional[Dict[str, Union[torch.Tensor, str]]],
+        ) -> Union[Tuple, VibeVoiceCausalLMOutputWithPast]:
+        
+        return_dict = return_dict if return_dict is not None else self.config.use_return_dict
+        
+        x = self.get_input_embeddings()(input_ids)
+
+        semantic_speech_all_connect_features = self.model.semantic_connector(speech_semantic_tensors)
+        if speeches_loss_input is not None:
+            # only part audio need diffuse
+            speech_all_features, speech_all_connect_features = self.forward_speech_features(
+                    speech_tensors=speech_tensors.type_as(x) if speech_tensors is not None else None,
+                    speech_masks=speech_masks,
+                    speech_type=kwargs.get("speech_type", "audio"),
+                    return_unmask=True
+                )
+            if speech_tensors is not None:
+                if semantic_speech_all_connect_features is not None:
+                    x[acoustic_input_mask] = speech_all_connect_features[speech_masks] + semantic_speech_all_connect_features[speech_masks]
+                else:
+                    x[acoustic_input_mask] = speech_all_connect_features[speech_masks]
+                speech_features = speech_all_features[speeches_loss_input.unsqueeze(-1) & speech_masks] # only part audio need diffuse
+                speech_connect_features = speech_all_connect_features[speeches_loss_input.unsqueeze(-1) & speech_masks]
+        else:
+            speech_features, speech_connect_features = self.forward_speech_features(
+                    speech_tensors=speech_tensors.type_as(x) if speech_tensors is not None else None,
+                    speech_masks=speech_masks,
+                    speech_type=kwargs.get("speech_type", "audio"),
+                )
+            if speech_tensors is not None:
+                x[acoustic_input_mask] = speech_connect_features
+
+        outputs = self.model(
+            input_ids=None,
+            attention_mask=attention_mask,
+            position_ids=position_ids,
+            past_key_values=past_key_values,
+            inputs_embeds=x,
+            use_cache=use_cache,
+            output_attentions=output_attentions,
+            output_hidden_states=False,
+            return_dict=return_dict,
+            cache_position=cache_position,
+        )
+
+        hidden_states = outputs.last_hidden_state
+        logits = self.lm_head(hidden_states)
+        # logits = logits.float()
+
+        loss = None
+        if labels is not None:
+            # The custom CE loss with masking is calculated in the training script.
+            # We leave the standard loss calculation here as None.
+            pass
+
+        # --- Diffusion Loss Calculation ---
+        diffusion_loss = None
+        # This block is executed only if we are in a context that involves speech.
+        if speech_tensors is not None and acoustic_loss_mask.sum().item() > 0:
+            condition_features = hidden_states[acoustic_loss_mask]
+            
+            speech_len, latent_size = speech_features.shape
+            
+            noise = torch.randn(
+                (speech_len * ddpm_batch_mul, latent_size),
+                device=hidden_states.device,
+                dtype=hidden_states.dtype
+            )
+            
+            timesteps = torch.multinomial(
+                torch.ones(self.config.diffusion_head_config.ddpm_num_steps),
+                speech_len * ddpm_batch_mul,
+                replacement=True,
+            ).to(hidden_states.device)
+
+            speech_features_repeated = speech_features.repeat_interleave(ddpm_batch_mul, dim=0)
+            condition_features_repeated = condition_features.repeat_interleave(ddpm_batch_mul, dim=0)
+
+            noisy_speech_features = self.model.noise_scheduler.add_noise(
+                speech_features_repeated, noise, timesteps
+            )
+            
+            model_output = self.model.prediction_head(
+                noisy_speech_features, 
+                timesteps.type_as(x), 
+                condition_features_repeated
+            )
+
+            prediction_type = self.config.diffusion_head_config.prediction_type
+            if prediction_type == "epsilon":
+                target_for_loss = noise
+            elif prediction_type == "v_prediction":
+                target_for_loss = self.model.noise_scheduler.get_velocity(
+                    speech_features_repeated, noise, timesteps
+                )
+            else:
+                raise NotImplementedError(f"Prediction type {prediction_type} not implemented")
+
+            diffusion_loss = F.mse_loss(model_output.float(), target_for_loss.float(), reduction='sum')
+            if latent_size > 0 and ddpm_batch_mul > 0:
+                diffusion_loss = diffusion_loss / latent_size / ddpm_batch_mul
+            else:
+                diffusion_loss = torch.tensor(0.0, device=diffusion_loss.device)
+        
+        else:
+            # Dummy loss for DDP to work when there are no speech samples in a batch,
+            # but we are in a speech context.
+            diffusion_loss = sum(p.sum() for p in self.model.prediction_head.parameters()) * 0.0
+            diffusion_loss += sum(p.sum() for p in self.model.acoustic_connector.parameters()) * 0.0
+            diffusion_loss += sum(p.sum() for p in self.model.semantic_connector.parameters()) * 0.0
+        # --- End Diffusion Loss Calculation ---
+
+        if not return_dict:
+            output = (logits, speech_len) + outputs.to_tuple()[1:]
+            return (loss, diffusion_loss) + output
+
+        return VibeVoiceCausalLMOutputWithPast(
+            loss=loss,
+            diffusion_loss=diffusion_loss,
+            speech_token_num=speech_len if speech_tensors is not None else 0,
+            logits=logits,
+            past_key_values=outputs.past_key_values,
+            hidden_states=outputs.hidden_states,
+            attentions=outputs.attentions,
+        )
+
+AutoModel.register(VibeVoiceConfig, VibeVoiceModel)
+AutoModelForCausalLM.register(VibeVoiceConfig, VibeVoiceForConditionalGeneration)
+
+__all__ = [
+    "VibeVoiceModel",
+    "VibeVoicePreTrainedModel",
+    "VibeVoiceForConditionalGeneration",
+    "VibeVoiceCausalLMOutputWithPast",
+    "VibeVoiceGenerationOutput",
+]

vibevoice/modular/modeling_vibevoice_inference.py

@@ -0,0 +1,715 @@
+from dataclasses import dataclass
+from typing import Dict, List, Optional, Tuple, Union, Callable
+from tqdm import tqdm
+import torch
+import torch.nn as nn
+
+from transformers.models.auto import AutoModel, AutoModelForCausalLM
+
+from transformers.generation import GenerationMixin, GenerationConfig, LogitsProcessor, LogitsProcessorList, StoppingCriteriaList
+from transformers.modeling_outputs import BaseModelOutputWithPast, ModelOutput
+from transformers import modeling_utils
+from transformers.modeling_utils import PreTrainedModel
+from transformers.modeling_flash_attention_utils import FlashAttentionKwargs
+from transformers.utils import logging
+
+
+# from .modular_vibevoice_tokenizer import VibeVoiceTokenizerStreamingCache, VibeVoiceAcousticTokenizerModel, VibeVoiceSemanticTokenizerModel
+from .modular_vibevoice_tokenizer import VibeVoiceTokenizerStreamingCache, VibeVoiceTokenizerEncoderOutput
+from .modular_vibevoice_diffusion_head import VibeVoiceDiffusionHead
+from vibevoice.schedule.dpm_solver import DPMSolverMultistepScheduler
+
+from .configuration_vibevoice import VibeVoiceConfig
+
+from .modular_vibevoice_text_tokenizer import VibeVoiceTextTokenizer, VibeVoiceTextTokenizerFast
+
+from .modeling_vibevoice import VibeVoiceModel, VibeVoicePreTrainedModel
+from .streamer import AudioStreamer, AsyncAudioStreamer
+
+logger = logging.get_logger(__name__)
+
+if not hasattr(modeling_utils, "ALL_PARALLEL_STYLES") or modeling_utils.ALL_PARALLEL_STYLES is None:
+    modeling_utils.ALL_PARALLEL_STYLES = ["tp", "none", "colwise", "rowwise"]
+
+@dataclass
+class VibeVoiceCausalLMOutputWithPast(BaseModelOutputWithPast):
+    logits: Optional[torch.FloatTensor] = None
+
+@dataclass
+class VibeVoiceGenerationOutput(ModelOutput):
+    """
+    Output type for VibeVoice generation.
+    
+    Args:
+        sequences (`torch.LongTensor` of shape `(batch_size, sequence_length)`):
+            The generated sequences. 
+        speech_outputs (`List[torch.FloatTensor]`, *optional*):
+            List of generated speech waveforms or latents for each speech segment.
+    """
+    sequences: torch.LongTensor = None
+    speech_outputs: Optional[List[torch.FloatTensor]] = None
+    reach_max_step_sample: Optional[torch.BoolTensor] = None
+
+class VibeVoiceTokenConstraintProcessor(LogitsProcessor):
+    """Constrains token generation to only valid tokens during speech generation."""
+    
+    def __init__(self, valid_token_ids: List[int], device: torch.device = None):
+        self.valid_token_ids = torch.tensor(valid_token_ids, dtype=torch.long, device=device)
+        
+    def __call__(self, input_ids: torch.LongTensor, scores: torch.FloatTensor) -> torch.FloatTensor:
+        # Create a mask for valid tokens
+        mask = torch.full_like(scores, float('-inf'))
+        mask[:, self.valid_token_ids] = 0
+        
+        # Apply mask to scores
+        scores = scores + mask
+        return scores
+    
+class VibeVoiceForConditionalGenerationInference(VibeVoicePreTrainedModel, GenerationMixin):
+    _tied_weights_keys = ["lm_head.weight"]
+    _tp_plan = {"lm_head": "colwise_rep"}
+
+    def __init__(self, config):
+        super().__init__(config)
+        
+        # Initialize the base model
+        self.model = VibeVoiceModel(config)
+        
+        # LM head for text generation
+        self.lm_head = nn.Linear(config.decoder_config.hidden_size, config.decoder_config.vocab_size, bias=False)
+        
+        # inference configuration
+        self.ddpm_inference_steps = config.diffusion_head_config.ddpm_num_inference_steps
+
+        # Initialize weights and apply final processing
+        self.post_init()
+
+    @property
+    def noise_scheduler(self):
+        return self.model.noise_scheduler
+
+    @property
+    def prediction_head(self):
+        return self.model.prediction_head
+    
+    @property
+    def speech_scaling_factor(self):
+        return self.model.speech_scaling_factor
+
+    @property
+    def speech_bias_factor(self):
+        return self.model.speech_bias_factor
+
+    @property
+    def acoustic_tokenizer(self):
+        return self.model.acoustic_tokenizer
+
+    @property
+    def semantic_tokenizer(self):
+        return self.model.semantic_tokenizer
+    
+    @property
+    def acoustic_connector(self):
+        return self.model.acoustic_connector
+
+    @property
+    def semantic_connector(self):
+        return self.model.semantic_connector
+        
+    def tie_weights(self):
+        """
+        Tie the weights between the input embeddings and the output embeddings.
+        """
+        # Tie lm_head.weight to language_model.embed_tokens.weight
+        if not getattr(self.config, 'tie_word_embeddings', False):
+            return
+         
+        if hasattr(self, 'lm_head') and hasattr(self.model.language_model, 'embed_tokens'):
+            self.lm_head.weight = self.model.language_model.embed_tokens.weight
+        
+    def get_input_embeddings(self):
+        return self.model.get_input_embeddings()
+    
+    def set_input_embeddings(self, value):
+        self.model.set_input_embeddings(value)
+    
+    def get_output_embeddings(self):
+        return self.lm_head
+    
+    def set_output_embeddings(self, new_embeddings):
+        self.lm_head = new_embeddings
+    
+    def set_speech_tokenizers(self, acoustic_tokenizer=None, semantic_tokenizer=None):
+        """Set the speech tokenizers used for encoding and decoding speech."""
+        self.model.set_speech_tokenizers(acoustic_tokenizer, semantic_tokenizer)
+    
+    def set_ddpm_inference_steps(self, num_steps=None):
+        self.ddpm_inference_steps = num_steps or self.config.diffusion_head_config.ddpm_num_inference_steps
+
+    def _process_speech_inputs(self, speech_tensors, speech_masks, speech_type="audio"):
+        """Process speech inputs through tokenizers and connectors."""
+        with torch.no_grad():
+            if speech_type == "audio":
+                # Encode audio to acoustic latents
+                encoder_output = self.model.acoustic_tokenizer.encode(speech_tensors.unsqueeze(1))
+                acoustic_latents = encoder_output.sample(dist_type=self.model.acoustic_tokenizer.std_dist_type)[0]
+                
+                # Apply scaling and bias
+                acoustic_features = (acoustic_latents + self.model.speech_bias_factor.to(acoustic_latents.device)) * self.model.speech_scaling_factor.to(acoustic_latents.device)
+                
+                # Connect to language model space
+                acoustic_connected = self.model.acoustic_connector(acoustic_features)[speech_masks.cpu()]
+                
+                return acoustic_features, acoustic_connected
+            elif speech_type == "pt":
+                encoder_output = VibeVoiceTokenizerEncoderOutput(mean=speech_tensors, std=self.acoustic_tokenizer.config.fix_std)
+                acoustic_latents = encoder_output.sample(dist_type=self.model.acoustic_tokenizer.std_dist_type)[0]
+                
+                # Apply scaling and bias
+                acoustic_features = (acoustic_latents + self.model.speech_bias_factor.to(acoustic_latents.device)) * self.model.speech_scaling_factor.to(acoustic_latents.device)
+                
+                # Connect to language model space
+                acoustic_connected = self.model.acoustic_connector(acoustic_features)[speech_masks.cpu()]
+                
+                return acoustic_features, acoustic_connected
+            else:
+                raise NotImplementedError(f"Speech type {speech_type} not implemented")
+    
+    # @can_return_tuple
+    def forward(
+        self,
+        input_ids: torch.LongTensor = None,
+        attention_mask: Optional[torch.Tensor] = None,
+        position_ids: Optional[torch.LongTensor] = None,
+        past_key_values: Optional[Tuple[Tuple[torch.FloatTensor]]] = None,
+        inputs_embeds: Optional[torch.FloatTensor] = None,
+        labels: Optional[torch.LongTensor] = None,
+        use_cache: Optional[bool] = None,
+        output_attentions: Optional[bool] = None,
+        output_hidden_states: Optional[bool] = None,
+        return_dict: Optional[bool] = None,
+        cache_position: Optional[torch.LongTensor] = None,
+        speech_tensors: Optional[torch.FloatTensor] = None,
+        speech_masks: Optional[torch.BoolTensor] = None,
+        speech_input_mask: Optional[torch.BoolTensor] = None,
+        logits_to_keep: Union[int, slice] = 0,
+        **kwargs,
+    ) -> Union[Tuple, VibeVoiceCausalLMOutputWithPast]:
+        """
+        Args:
+            labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
+                Labels for computing the masked language modeling loss. Indices should either be in `[0, ...,
+                config.vocab_size]` or -100 (see `input_ids` docstring). Tokens with indices set to `-100` are ignored
+                (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`.
+            speech_tensors (`torch.FloatTensor`, *optional*):
+                Input speech waveforms for voice cloning or speech understanding.
+            speech_masks (`torch.BoolTensor`, *optional*):
+                Masks indicating valid speech frames.
+            speech_input_mask (`torch.BoolTensor`, *optional*):
+                Positions in the input sequence where speech embeddings should be inserted.
+        
+        Returns:
+            `VibeVoiceCausalLMOutputWithPast` or tuple
+        """
+        return_dict = return_dict if return_dict is not None else self.config.use_return_dict
+        
+        # Get embeddings
+        if inputs_embeds is None:
+            inputs_embeds = self.model.get_input_embeddings()(input_ids)
+        
+        # Process speech inputs if provided
+        if speech_tensors is not None and speech_masks is not None:
+            acoustic_features, speech_embeds = self._process_speech_inputs(speech_tensors.to(self.dtype), speech_masks)
+            if speech_input_mask is not None:
+                inputs_embeds[speech_input_mask] = speech_embeds
+
+        outputs = self.model(
+            inputs_embeds=inputs_embeds,
+            attention_mask=attention_mask,
+            position_ids=position_ids,
+            past_key_values=past_key_values,
+            use_cache=use_cache,
+            output_attentions=output_attentions,
+            output_hidden_states=output_hidden_states,
+            return_dict=return_dict,
+            cache_position=cache_position,
+            **kwargs,
+        )
+
+        hidden_states = outputs[0] if not return_dict else outputs.last_hidden_state
+        # Only compute necessary logits, and do not upcast them to float if we are not computing the loss
+        slice_indices = slice(-logits_to_keep, None) if isinstance(logits_to_keep, int) else logits_to_keep
+        logits = self.lm_head(hidden_states[:, slice_indices, :])
+                
+        if labels is not None:
+            raise NotImplementedError("Loss computation is not implemented in this version.")
+
+        return VibeVoiceCausalLMOutputWithPast(
+            logits=logits,
+            past_key_values=outputs.past_key_values,
+            last_hidden_state=hidden_states,
+            attentions=outputs.attentions,
+        )
+
+    def _build_generate_config_model_kwargs(self, generation_config, inputs, tokenizer, return_processors=False, **kwargs):
+        if generation_config is None:
+            generation_config = GenerationConfig(
+                bos_token_id=tokenizer.bos_token_id,
+                eos_token_id=tokenizer.eos_token_id,
+                pad_token_id = tokenizer.pad_token_id
+            )
+        else:
+            generation_config = GenerationConfig(
+                **generation_config,
+                bos_token_id=tokenizer.bos_token_id,
+                eos_token_id=tokenizer.eos_token_id,
+                pad_token_id = tokenizer.pad_token_id
+            )
+
+        generation_config, model_kwargs = self._prepare_generation_config(
+            generation_config, 
+            True, 
+            speech_start_id=tokenizer.speech_start_id, 
+            speech_end_id=tokenizer.speech_end_id, 
+            speech_diffusion_id=tokenizer.speech_diffusion_id, 
+            **kwargs
+        )
+        generation_config.speech_start_id = tokenizer.speech_start_id
+        generation_config.speech_end_id = tokenizer.speech_end_id
+        generation_config.speech_diffusion_id = tokenizer.speech_diffusion_id
+
+        inputs_tensor, model_input_name, model_kwargs = self._prepare_model_inputs(inputs, generation_config.bos_token_id, model_kwargs)
+        batch_size = inputs_tensor.shape[0]
+        device = self.device
+        
+        self._prepare_special_tokens(generation_config, True, device=device)
+        generation_config.use_cache = True
+        model_kwargs["use_cache"] = generation_config.use_cache
+        input_ids = inputs_tensor.to(self.device)
+
+        input_ids_length = input_ids.shape[1]
+        has_default_max_length = kwargs.get("max_length") is None and generation_config.max_length is not None
+        has_default_min_length = kwargs.get("min_length") is None and generation_config.min_length is not None
+        generation_config = self._prepare_generated_length(
+            generation_config=generation_config,
+            has_default_max_length=has_default_max_length,
+            has_default_min_length=has_default_min_length,
+            model_input_name=model_input_name,
+            inputs_tensor=inputs_tensor,
+            input_ids_length=input_ids_length,
+        )
+
+        max_cache_length = generation_config.max_length - 1
+        self._prepare_cache_for_generation(generation_config, model_kwargs, None, batch_size, max_cache_length, device)
+        model_kwargs['cache_position'] = torch.arange(input_ids_length, device=device, dtype=torch.long)
+        for k, v in model_kwargs.items():
+            if isinstance(v, torch.Tensor):
+                model_kwargs[k] = v.to(device=device)
+        
+        if return_processors:
+            logits_processor = self._get_logits_processor(
+                generation_config=generation_config,
+                input_ids_seq_length=input_ids_length,
+                encoder_input_ids=inputs_tensor,
+                prefix_allowed_tokens_fn=None,
+                logits_processor=LogitsProcessorList(),
+                device=inputs_tensor.device,
+                model_kwargs=model_kwargs,
+            )
+
+            stopping_criteria = self._get_stopping_criteria(generation_config=generation_config, stopping_criteria=StoppingCriteriaList())
+        
+            return generation_config, model_kwargs, input_ids, logits_processor, stopping_criteria
+        else:
+            return generation_config, model_kwargs, input_ids
+        
+    @torch.no_grad()
+    def generate(
+        self,
+        inputs: Optional[torch.Tensor] = None,
+        generation_config: Optional[GenerationConfig] = None,
+        logits_processor: Optional[LogitsProcessorList] = None,
+        stopping_criteria: Optional[StoppingCriteriaList] = None,
+        prefix_allowed_tokens_fn: Optional[Callable[[int, torch.Tensor], List[int]]] = None,
+        synced_gpus: Optional[bool] = None,
+        assistant_model: Optional["PreTrainedModel"] = None,
+        audio_streamer: Optional[Union[AudioStreamer, AsyncAudioStreamer]] = None,
+        negative_prompt_ids: Optional[torch.Tensor] = None,
+        negative_prompt_attention_mask: Optional[torch.Tensor] = None,
+        speech_tensors: Optional[torch.FloatTensor] = None,
+        speech_masks: Optional[torch.BoolTensor] = None,
+        speech_input_mask: Optional[torch.BoolTensor] = None,
+        return_speech: bool = True,
+        cfg_scale: float = 1.0,
+        stop_check_fn: Optional[Callable[[], bool]] = None,
+        **kwargs,
+    ) -> Union[torch.LongTensor, VibeVoiceGenerationOutput]:
+        """
+        Generates sequences of token ids and optionally speech outputs.
+        
+        Args:
+            All standard generation arguments from GenerationMixin
+            negative_prompt_ids: Negative prompt for CFG in speech generation
+            negative_prompt_attention_mask: Attention mask for negative prompt
+            speech_tensors: Input speech for voice cloning
+            speech_masks: Masks for speech tensors  
+            speech_input_mask: Positions to insert speech embeddings
+            return_speech: Whether to decode and return speech outputs
+            cfg_scale: CFG scale for speech generation
+            stop_check_fn: Optional callable that returns True if generation should stop
+ 
+        Returns:
+            Generated token sequences and optionally speech outputs
+        """
+        # 1. Handle `generation_config` and kwargs that might update it, and validate the `.generate()` call
+        tokenizer = kwargs.pop("tokenizer", None)  # Pull this out first, we only use it for stopping criteria
+        parsed_scripts = kwargs.pop("parsed_scripts", None)
+        all_speakers_list = kwargs.pop("all_speakers_list", None)
+        max_length_times = kwargs.pop("max_length_times", 2)
+
+        if kwargs.get('max_new_tokens', None) is None:
+            kwargs['max_new_tokens'] = self.config.decoder_config.max_position_embeddings - kwargs['input_ids'].shape[-1]
+
+        generation_config, model_kwargs, input_ids, logits_processor, stopping_criteria = self._build_generate_config_model_kwargs(
+            generation_config, inputs, tokenizer, return_processors=True, **kwargs
+        )
+        
+        negative_kwargs = {
+            'input_ids': torch.full((kwargs['input_ids'].shape[0], 1), tokenizer.speech_start_id, dtype=torch.long, device=kwargs['input_ids'].device),
+            'attention_mask':  torch.ones((kwargs['input_ids'].shape[0], 1), dtype=torch.long, device=kwargs['input_ids'].device),
+            'max_new_tokens': kwargs.get('max_new_tokens', 100) 
+        }
+        negative_generation_config, negative_model_kwargs, negative_input_ids = self._build_generate_config_model_kwargs(
+            None, None, tokenizer, return_processors=False, **negative_kwargs
+        )
+
+        acoustic_cache = VibeVoiceTokenizerStreamingCache()
+        semantic_cache = VibeVoiceTokenizerStreamingCache()
+        
+        batch_size = input_ids.shape[0]
+        device = input_ids.device
+        finished_tags = torch.zeros(batch_size, dtype=torch.bool, device=device)
+        correct_cnt = torch.zeros(batch_size, dtype=torch.long, device=device)
+        is_prefill = True
+        inputs_embeds = None
+        verbose = kwargs.get("verbose", False)
+
+        # Initialize audio chunks storage for each sample
+        audio_chunks = [[] for _ in range(batch_size)]
+
+        initial_length = input_ids.shape[-1]
+        initial_length_per_sample = model_kwargs['attention_mask'].sum(dim=-1)
+
+       # Define all valid tokens that can be generated
+        valid_tokens = [
+            generation_config.speech_start_id,
+            generation_config.speech_end_id, 
+            generation_config.speech_diffusion_id,
+            generation_config.eos_token_id
+        ]
+        # Add bos_token_id if it exists
+        if hasattr(generation_config, 'bos_token_id') and generation_config.bos_token_id is not None:
+            valid_tokens.append(generation_config.bos_token_id)
+        
+        # Add custom processor to constrain token generation
+        token_constraint_processor = VibeVoiceTokenConstraintProcessor(valid_tokens, device=device)
+        if logits_processor is None:
+            logits_processor = LogitsProcessorList()
+        logits_processor.append(token_constraint_processor)
+        
+        max_steps = min(generation_config.max_length - initial_length, int(max_length_times * initial_length))
+        max_step_per_sample = torch.min(generation_config.max_length - initial_length_per_sample, (max_length_times * initial_length_per_sample).long())
+        reach_max_step_sample = torch.zeros(batch_size, dtype=torch.bool, device=device)
+
+        # Create progress iterator if verbose
+        if kwargs.get("show_progress_bar", True):
+            progress_bar = tqdm(range(max_steps), desc="Generating", leave=False)
+        else:
+            progress_bar = range(max_steps)
+        
+        for step in progress_bar:
+            # Check for external stop signal
+            if stop_check_fn is not None and stop_check_fn():
+                if verbose:
+                    print(f"Generation stopped externally at step {step + 1}")
+                # End the audio streamer if it exists
+                if audio_streamer is not None:
+                    audio_streamer.end()
+                break
+            
+            # Check if audio_streamer has been ended (stopped externally)
+            if audio_streamer is not None and hasattr(audio_streamer, 'finished_flags'):
+                if any(audio_streamer.finished_flags):
+                    if verbose:
+                        print(f"Audio generation stopped externally at step {step + 1}")
+                    break
+            
+            if finished_tags.all():
+                if hasattr(progress_bar, 'set_description'):
+                    progress_bar.set_description("Generation complete")
+                break
+
+            if input_ids.shape[-1] >= generation_config.max_length:
+                print(f"Reached maximum generation length {generation_config.max_length}, stopped it.")
+                reached_samples = torch.arange(batch_size, device=device)[~finished_tags]
+                if reached_samples.numel() > 0:
+                    reach_max_step_sample[reached_samples] = True
+                break
+            
+            # Update progress bar description with active samples
+            if hasattr(progress_bar, 'set_description'):
+                active_samples = (~finished_tags).sum().item()
+                progress_bar.set_description(f"Generating (active: {active_samples}/{batch_size})")
+
+            model_inputs = self.prepare_inputs_for_generation(input_ids, **model_kwargs)
+            if is_prefill:
+                # we process the speech inputs only during the first generation step
+                prefill_inputs = {
+                    "speech_tensors": speech_tensors.to(device=device),
+                    "speech_masks": speech_masks.to(device),
+                    "speech_input_mask": speech_input_mask.to(device),
+                }
+                is_prefill = False
+            else:
+                _ = model_inputs.pop('inputs_embeds', None)
+                prefill_inputs = {'inputs_embeds': inputs_embeds}
+
+            # Forward pass through the model
+            outputs = self(
+                **model_inputs, **prefill_inputs, logits_to_keep=1, return_dict=True, output_attentions=False, output_hidden_states=False,
+            )
+            model_kwargs = self._update_model_kwargs_for_generation(
+                outputs, model_kwargs, is_encoder_decoder=False,
+            )
+
+            # Get logits and apply logits processor
+            next_token_logits = outputs.logits[:, -1, :].to(copy=True, dtype=torch.float32, device=input_ids.device)
+            # next_token_logits = outputs.logits[:, -1, :].to(copy=True, device=input_ids.device)
+            next_token_scores = logits_processor(input_ids, next_token_logits)
+            
+            # token selection
+            if generation_config.do_sample:
+                probs = nn.functional.softmax(next_token_scores, dim=-1)
+                # TODO (joao): this OP throws "skipping cudagraphs due to ['incompatible ops']", find solution
+                next_tokens = torch.multinomial(probs, num_samples=1).squeeze(1)
+            else:
+                next_tokens = torch.argmax(next_token_scores, dim=-1)
+
+            next_tokens[finished_tags] = generation_config.eos_token_id
+            input_ids = torch.cat([input_ids, next_tokens[:, None]], dim=-1)
+            
+            if not kwargs.get('refresh_negative', True):
+                negative_model_inputs = self.prepare_inputs_for_generation(negative_input_ids, **negative_model_kwargs)
+                # Forward negative pass through the model
+                if negative_model_inputs['inputs_embeds'] is None and inputs_embeds is not None:
+                    negative_model_inputs['inputs_embeds'] = inputs_embeds
+                    negative_model_inputs['input_ids'] = None
+
+                negative_outputs = self(
+                    **negative_model_inputs, logits_to_keep=0, return_dict=True, output_attentions=False, output_hidden_states=False,
+                )
+                negative_model_kwargs = self._update_model_kwargs_for_generation(
+                    negative_outputs, negative_model_kwargs, is_encoder_decoder=False,
+                )
+                negative_input_ids = torch.cat([negative_input_ids, next_tokens[:, None]], dim=-1)
+
+            # reached end of generation
+            if (next_tokens == generation_config.eos_token_id).any():
+                eos_indices = (next_tokens == generation_config.eos_token_id).nonzero(as_tuple=False).squeeze(1)
+                # Only print for samples that are newly finished (not already marked as finished)
+                new_eos_indices = eos_indices[~finished_tags[eos_indices]]
+                if new_eos_indices.numel() > 0:
+                    finished_tags[new_eos_indices] = True
+                    if verbose:
+                        print(f"Samples {new_eos_indices.tolist()} reached EOS token at step {step + 1}.", flush=True)
+                    if audio_streamer is not None:
+                        audio_streamer.end(new_eos_indices)
+
+            # Check if any sample reached its maximum generation length
+            max_length_reached = step >= max_step_per_sample
+            new_max_length_indices = torch.nonzero(max_length_reached & ~finished_tags, as_tuple=False).squeeze(1)
+            if new_max_length_indices.numel() > 0:
+                finished_tags[new_max_length_indices] = True
+                reach_max_step_sample[new_max_length_indices] = True
+                if verbose:
+                    print(f"Samples {new_max_length_indices.tolist()} reached max generation length at step {step + 1}.", flush=True)
+                if audio_streamer is not None:
+                    audio_streamer.end(new_max_length_indices)
+
+            # speech_end
+            diffusion_end_indices = (next_tokens == generation_config.speech_end_id).nonzero(as_tuple=False).squeeze(1)
+            if diffusion_end_indices.numel() > 0:
+                # Clear tokenizer caches for samples that reached speech end
+                acoustic_cache.set_to_zero(diffusion_end_indices)
+                semantic_cache.set_to_zero(diffusion_end_indices)
+            
+            # speech_begin
+            diffusion_start_indices = torch.arange(batch_size, device=device)[~finished_tags & (next_tokens == generation_config.speech_start_id)]
+            if diffusion_start_indices.numel() > 0 and kwargs.get('refresh_negative', True):
+                # update attention mask
+                for i, sample_idx in enumerate(diffusion_start_indices.tolist()):
+                    negative_model_kwargs['attention_mask'][sample_idx, :] = 0
+                    negative_model_kwargs['attention_mask'][sample_idx, -1] = 1
+                # update past key values
+                for layer_idx, (k_cache, v_cache) in enumerate(zip(negative_model_kwargs['past_key_values'].key_cache, 
+                                                                        negative_model_kwargs['past_key_values'].value_cache)):
+                    # Process each non-diffusion sample
+                    for sample_idx in diffusion_start_indices.tolist():
+                        # Shift cache for this sample
+                        k_cache[sample_idx, :, -1, :] = k_cache[sample_idx, :, 0, :].clone()
+                        v_cache[sample_idx, :, -1, :] = v_cache[sample_idx, :, 0, :].clone()
+                # update negative_input_ids
+                for sample_idx in diffusion_start_indices.tolist():
+                    negative_input_ids[sample_idx, -1] = generation_config.speech_start_id
+            
+            # Prepare inputs_embeds for next iteration
+            # Initialize with default embeddings for all tokens
+            next_inputs_embeds = self.model.get_input_embeddings()(next_tokens).unsqueeze(1)  # [batch_size, 1, hidden_size]
+            
+            # forward diffusion
+            # Diffusion indices are those that are not finished and not special tokens
+            diffusion_indices = torch.arange(batch_size, device=device)[~finished_tags & (next_tokens == generation_config.speech_diffusion_id)]
+            
+            if diffusion_indices.numel() > 0:
+                if kwargs.get('refresh_negative', True):
+                    negative_model_inputs = self.prepare_inputs_for_generation(negative_input_ids, **negative_model_kwargs)
+                    # Forward negative pass through the model
+                    if negative_model_inputs['inputs_embeds'] is None and inputs_embeds is not None:
+                        negative_model_inputs['inputs_embeds'] = inputs_embeds
+                        negative_model_inputs['input_ids'] = None
+
+                    negative_outputs = self(
+                        **negative_model_inputs, logits_to_keep=0, return_dict=True, output_attentions=False, output_hidden_states=False,
+                    )
+                    negative_model_kwargs = self._update_model_kwargs_for_generation(
+                        negative_outputs, negative_model_kwargs, is_encoder_decoder=False,
+                    )
+                    negative_input_ids = torch.cat([negative_input_ids, next_tokens[:, None]], dim=-1)
+                # correct the non-diffusion indices
+                # we forward all samples' negative outputs even if 
+                #   they are not in diffusion mode to keep the cache consistent
+                # So we need to correct the kv cache of non-diffusion samples
+                non_diffusion_mask = ~finished_tags & (next_tokens != generation_config.speech_diffusion_id)
+                if non_diffusion_mask.any():
+                    non_diffusion_indices = torch.arange(batch_size, device=device)[non_diffusion_mask]
+                    start_indices = correct_cnt[non_diffusion_indices]
+
+                    # 1. Update attention_mask - need to handle each sample separately
+                    seq_len = negative_model_kwargs['attention_mask'].shape[1]
+                    for i, (sample_idx, start_idx) in enumerate(zip(non_diffusion_indices.tolist(), start_indices.tolist())):
+                        # Shift the attention mask for this sample
+                        if start_idx + 1 < seq_len - 1:
+                            negative_model_kwargs['attention_mask'][sample_idx, start_idx+1:] = \
+                                negative_model_kwargs['attention_mask'][sample_idx, start_idx:-1].clone()
+                        negative_model_kwargs['attention_mask'][sample_idx, start_idx] = 0
+
+                    # 2. Update past_key_values
+                    for layer_idx, (k_cache, v_cache) in enumerate(zip(negative_model_kwargs['past_key_values'].key_cache, 
+                                                                        negative_model_kwargs['past_key_values'].value_cache)):
+                        # Process each non-diffusion sample
+                        for sample_idx, start_idx in zip(non_diffusion_indices.tolist(), start_indices.tolist()):
+                            if start_idx + 1 < k_cache.shape[2] - 1:
+                                # Shift cache for this sample
+                                k_cache[sample_idx, :, start_idx+1:, :] = k_cache[sample_idx, :, start_idx:-1, :].clone()
+                                v_cache[sample_idx, :, start_idx+1:, :] = v_cache[sample_idx, :, start_idx:-1, :].clone()
+                    
+                    # 3. Update negative_input_ids
+                    for sample_idx, start_idx in zip(non_diffusion_indices.tolist(), start_indices.tolist()):
+                        if start_idx + 1 < negative_input_ids.shape[1] - 1:
+                            negative_input_ids[sample_idx, start_idx+1:] = \
+                                negative_input_ids[sample_idx, start_idx:-1].clone()
+                                
+                    correct_cnt[non_diffusion_indices] += 1
+
+                positive_condition = outputs.last_hidden_state[diffusion_indices, -1, :]
+                negative_condition = negative_outputs.last_hidden_state[diffusion_indices, -1, :]
+                
+                speech_latent = self.sample_speech_tokens(
+                    positive_condition,
+                    negative_condition,
+                    cfg_scale=cfg_scale,
+                ).unsqueeze(1)
+                                
+                # Decode acoustic latent to audio using acoustic streaming cache
+                scaled_latent = speech_latent / self.model.speech_scaling_factor.to(speech_latent.device) - self.model.speech_bias_factor.to(speech_latent.device)
+                audio_chunk = self.model.acoustic_tokenizer.decode(
+                    scaled_latent.to(self.model.acoustic_tokenizer.device),
+                    cache=acoustic_cache,  # Use acoustic-specific cache
+                    sample_indices=diffusion_indices.to(self.model.acoustic_tokenizer.device),
+                    use_cache=True,
+                    debug=False
+                )
+                
+                # Store audio chunks for each sample
+                for i, sample_idx in enumerate(diffusion_indices):
+                    idx = sample_idx.item()
+                    # Only append audio chunk if the sample is not finished
+                    if not finished_tags[idx]:
+                        audio_chunks[idx].append(audio_chunk[i])
+
+                 # Add streaming support here
+                if audio_streamer is not None:
+                    # Stream the audio chunks immediately
+                    audio_streamer.put(audio_chunk, diffusion_indices)
+                    
+                # Encode audio to semantic features using semantic streaming cache
+                semantic_features = self.model.semantic_tokenizer.encode(
+                    audio_chunk,
+                    cache=semantic_cache,  # Use semantic-specific cache
+                    sample_indices=diffusion_indices,
+                    use_cache=True,
+                    debug=False
+                ).mean # semantic tokenizer has no VAE.
+                
+                # Combine acoustic and semantic features for next input
+                acoustic_embed = self.model.acoustic_connector(speech_latent)
+                semantic_embed = self.model.semantic_connector(semantic_features)
+                diffusion_embeds = acoustic_embed + semantic_embed
+
+                # Update embeddings for diffusion indices
+                next_inputs_embeds[diffusion_indices] = diffusion_embeds
+            
+            # Set inputs_embeds for next iteration
+            inputs_embeds = next_inputs_embeds
+
+        if audio_streamer is not None:
+            audio_streamer.end()
+
+        # Concatenate audio chunks for each sample
+        final_audio_outputs = []
+        for sample_chunks in audio_chunks:
+            if sample_chunks:
+                # Concatenate all chunks along the time dimension (assumed to be the last dimension)
+                concatenated_audio = torch.cat(sample_chunks, dim=-1)
+                final_audio_outputs.append(concatenated_audio)
+            else:
+                # If no audio was generated for this sample, append None
+                final_audio_outputs.append(None)
+
+        return VibeVoiceGenerationOutput(
+            sequences=input_ids,
+            speech_outputs=final_audio_outputs if return_speech else None,
+            reach_max_step_sample=reach_max_step_sample,
+        )
+    
+    @torch.no_grad()
+    def sample_speech_tokens(self, condition, neg_condition, cfg_scale=3.0):
+        self.model.noise_scheduler.set_timesteps(self.ddpm_inference_steps)
+        condition = torch.cat([condition, neg_condition], dim=0).to(self.model.prediction_head.device)
+        speech = torch.randn(condition.shape[0], self.config.acoustic_vae_dim).to(condition)
+        for t in self.model.noise_scheduler.timesteps:
+            half = speech[: len(speech) // 2]
+            combined = torch.cat([half, half], dim=0)
+            eps = self.model.prediction_head(combined, t.repeat(combined.shape[0]).to(combined), condition=condition)
+            cond_eps, uncond_eps = torch.split(eps, len(eps) // 2, dim=0)
+            half_eps = uncond_eps + cfg_scale * (cond_eps - uncond_eps)
+            eps = torch.cat([half_eps, half_eps], dim=0)
+            speech = self.model.noise_scheduler.step(eps, t, speech).prev_sample
+        return speech[: len(speech) // 2]
+    
+
+AutoModelForCausalLM.register(VibeVoiceConfig, VibeVoiceForConditionalGenerationInference)
+
+__all__ = [
+    "VibeVoiceForConditionalGenerationInference",
+]

vibevoice/modular/modular_vibevoice_diffusion_head.py

@@ -0,0 +1,287 @@
+import math
+from typing import Optional, Tuple, Union
+
+import torch
+import torch.nn as nn
+import torch.nn.functional as F
+
+from transformers.models.auto import AutoModel
+from transformers.modeling_utils import PreTrainedModel
+# from transformers.modeling_layers import GradientCheckpointingLayer
+from transformers.activations import ACT2FN
+from transformers.utils import logging
+
+from .configuration_vibevoice import VibeVoiceDiffusionHeadConfig
+
+
+logger = logging.get_logger(__name__)
+
+
+class RMSNorm(nn.Module):
+    def __init__(self, dim: int, eps: float = 1e-6, elementwise_affine=True, memory_efficient=False):
+        super().__init__()
+        self.dim = dim
+        self.eps = eps
+        self.elementwise_affine = elementwise_affine
+        if self.elementwise_affine:
+            self.weight = nn.Parameter(torch.ones(dim))
+        else:
+            self.register_parameter('weight', None)
+
+    def _norm(self, x):
+        return x * torch.rsqrt(x.pow(2).mean(-1, keepdim=True) + self.eps)
+
+    def forward(self, x):
+        output = self._norm(x.float()).type_as(x)
+        if self.weight is not None:
+            output = output * self.weight
+        return output
+
+    def extra_repr(self) -> str:
+        return f'dim={self.dim}, eps={self.eps}, elementwise_affine={self.elementwise_affine}'
+    
+def modulate(x, shift, scale):
+    """Apply modulation to input tensor."""
+    return x * (1 + scale) + shift
+
+
+class TimestepEmbedder(nn.Module):
+    """
+    Embeds scalar timesteps into vector representations.
+    
+    Args:
+        hidden_size (`int`): Size of the output embedding
+        frequency_embedding_size (`int`, optional): Size of the intermediate frequency embedding
+    """
+    def __init__(self, hidden_size, frequency_embedding_size=256):
+        super().__init__()
+        self.mlp = nn.Sequential(
+            nn.Linear(frequency_embedding_size, hidden_size, bias=False),
+            # nn.SiLU(),
+            ACT2FN['silu'],
+            nn.Linear(hidden_size, hidden_size, bias=False),
+        )
+        self.frequency_embedding_size = frequency_embedding_size
+
+    @staticmethod
+    def timestep_embedding(t, dim, max_period=10000):
+        """
+        Create sinusoidal timestep embeddings.
+        
+        Args:
+            t (`torch.Tensor`): A 1-D Tensor of N indices, one per batch element.
+                            These may be fractional.
+            dim (`int`): The dimension of the output.
+            max_period (`int`, optional): Controls the minimum frequency of the embeddings.
+            
+        Returns:
+            `torch.Tensor`: An [N, D] Tensor of positional embeddings.
+        """
+        half = dim // 2
+        freqs = torch.exp(
+            -math.log(max_period) * torch.arange(start=0, end=half, dtype=torch.float32) / half
+        ).to(t.device)
+        args = t[:, None].float() * freqs[None]
+        embedding = torch.cat([torch.cos(args), torch.sin(args)], dim=-1)
+        if dim % 2:
+            embedding = torch.cat([embedding, torch.zeros_like(embedding[:, :1])], dim=-1)
+        return embedding.to(t.dtype)
+
+    def forward(self, t):
+        t_freq = self.timestep_embedding(t, self.frequency_embedding_size)
+        t_emb = self.mlp(t_freq)
+        return t_emb
+
+
+class FeedForwardNetwork(nn.Module):
+    """
+    Standard feed-forward network with SwiGLU activation.
+    
+    Args:
+        embed_dim (`int`): Input dimension
+        ffn_dim (`int`): Hidden dimension
+    """
+    def __init__(
+        self,
+        embed_dim,
+        ffn_dim,
+    ):
+        super().__init__()
+        self.embed_dim = embed_dim
+        self.gate_proj = nn.Linear(self.embed_dim, ffn_dim, bias=False)
+        self.up_proj = nn.Linear(self.embed_dim, ffn_dim, bias=False)
+        self.down_proj = nn.Linear(ffn_dim, self.embed_dim, bias=False)
+        self.act_fn = ACT2FN['silu']  # Using SiLU as the activation function
+
+    def forward(self, x):
+        gate = self.gate_proj(x)
+        up = self.up_proj(x)
+        
+        # SwiGLU activation
+        # gate = F.silu(gate)
+        gate = self.act_fn(gate)
+        return self.down_proj(gate * up)
+
+    
+class HeadLayer(nn.Module):
+    """
+    A layer in the diffusion head.
+    
+    Args:
+        embed_dim (`int`): Input dimension
+        ffn_dim (`int`): Hidden dimension
+        cond_dim (`int`): Condition embedding dimension
+        norm_eps (`float`, optional): Epsilon for normalization
+    """
+    def __init__(
+        self,
+        embed_dim,
+        ffn_dim,
+        cond_dim,
+        norm_eps=1e-5,
+    ):
+        super().__init__()
+        self.embed_dim = embed_dim
+        self.cond_dim = cond_dim
+        self.ffn_dim = ffn_dim
+        self.ffn = FeedForwardNetwork(
+            self.embed_dim,
+            self.ffn_dim,
+        )
+        self.norm = RMSNorm(self.embed_dim, eps=norm_eps)
+        self.adaLN_modulation = nn.Sequential(
+            # nn.SiLU(),
+            ACT2FN['silu'],
+            nn.Linear(cond_dim, 3 * self.embed_dim, bias=False)
+        )
+
+    def forward(self, x, c):
+        shift_ffn, scale_ffn, gate_ffn = self.adaLN_modulation(c).chunk(3, dim=-1)
+        x = x + gate_ffn * self.ffn(modulate(self.norm(x), shift_ffn, scale_ffn))
+        return x
+
+
+class FinalLayer(nn.Module):
+    """
+    Final layer in the diffusion head.
+    
+    Args:
+        hidden_size (`int`): Input dimension
+        output_size (`int`): Output dimension
+        cond_size (`int`): Condition embedding dimension
+        norm_eps (`float`, optional): Epsilon for normalization
+    """
+    def __init__(self, hidden_size, output_size, cond_size, norm_eps=1e-5):
+        super().__init__()
+        self.norm_final = RMSNorm(hidden_size, eps=norm_eps, elementwise_affine=False)
+        self.linear = nn.Linear(hidden_size, output_size, bias=False)
+        self.adaLN_modulation = nn.Sequential(
+            # nn.SiLU(),
+            ACT2FN['silu'],
+            nn.Linear(cond_size, 2 * hidden_size, bias=False)
+        )
+
+    def forward(self, x, c):
+        shift, scale = self.adaLN_modulation(c).chunk(2, dim=-1)
+        x = modulate(self.norm_final(x), shift, scale)
+        x = self.linear(x)
+        return x
+
+
+class VibeVoiceDiffusionHead(PreTrainedModel):
+    """
+    Diffusion head model for vibevoice.
+    
+    Args:
+        config (`VibeVoiceDiffusionHeadConfig`): Model configuration
+        latent_size (`int`, optional): Size of the latent space. If not provided, uses `config.latent_size`.
+    """
+    config_class = VibeVoiceDiffusionHeadConfig
+    supports_gradient_checkpointing = True
+    _supports_flash_attn_2 = True  
+    _supports_sdpa = True  
+    
+    def __init__(
+        self,
+        config,
+    ):
+        super().__init__(config)
+        self.config = config
+        self.cond_dim = config.hidden_size
+        latent_size = config.latent_size
+        
+        self.noisy_images_proj = nn.Linear(latent_size, config.hidden_size, bias=False)
+        self.cond_proj = nn.Linear(config.hidden_size, self.cond_dim, bias=False)
+        self.t_embedder = TimestepEmbedder(self.cond_dim)
+        
+        ffn_dim = int(config.hidden_size * config.head_ffn_ratio)
+        
+        # Create the intermediate layers
+        self.layers = nn.ModuleList([
+            HeadLayer(
+                embed_dim=config.hidden_size,
+                ffn_dim=ffn_dim,
+                cond_dim=self.cond_dim,
+                norm_eps=config.rms_norm_eps
+            )
+            for _ in range(config.head_layers)
+        ])
+        
+        # Final layer for output
+        self.final_layer = FinalLayer(
+            hidden_size=config.hidden_size, 
+            output_size=latent_size,
+            cond_size=self.cond_dim,
+            norm_eps=config.rms_norm_eps
+        )
+        
+        self.initialize_weights()
+
+    def initialize_weights(self):
+        """Initialize the weights of the model."""
+        # Initialize timestep embedder
+        nn.init.normal_(self.t_embedder.mlp[0].weight, std=0.02)
+        nn.init.normal_(self.t_embedder.mlp[2].weight, std=0.02)
+
+        # Zero-out adaLN modulation layers
+        for layer in self.layers:
+            nn.init.constant_(layer.adaLN_modulation[-1].weight, 0)
+
+        # Zero-out output layers
+        nn.init.constant_(self.final_layer.adaLN_modulation[-1].weight, 0)
+        nn.init.constant_(self.final_layer.linear.weight, 0)
+
+    def forward(
+        self,
+        noisy_images,
+        timesteps,
+        condition,
+    ):
+        """
+        Forward pass of the prediction head.
+        
+        Args:
+            noisy_images (`torch.Tensor`): Noisy images/latents to denoise
+            timesteps (`torch.Tensor`): Timesteps for diffusion
+            condition (`torch.Tensor`): Conditioning information
+            
+        Returns:
+            `torch.Tensor`: The predicted noise/velocity
+        """
+        x = self.noisy_images_proj(noisy_images)
+        t = self.t_embedder(timesteps)
+        condition = self.cond_proj(condition)
+        c = condition + t
+        
+        for layer in self.layers:
+            x = layer(x, c)
+            
+        x = self.final_layer(x, c)
+        return x
+
+
+AutoModel.register(VibeVoiceDiffusionHeadConfig, VibeVoiceDiffusionHead)
+
+__all__ = [
+    "VibeVoiceDiffusionHead",
+]

vibevoice/modular/modular_vibevoice_text_tokenizer.py

@@ -0,0 +1,214 @@
+"""Tokenization classes for vibevoice."""
+
+from typing import List, Optional, Union
+
+from transformers.utils import logging
+from transformers.models.qwen2.tokenization_qwen2 import Qwen2Tokenizer
+from transformers.models.qwen2.tokenization_qwen2_fast import Qwen2TokenizerFast
+
+logger = logging.get_logger(__name__)
+
+
+class VibeVoiceTextTokenizer(Qwen2Tokenizer):
+    """
+    Construct a VibeVoice tokenizer. Based on the Qwen2 tokenizer with additional special tokens for speech.
+    
+    Args:
+        vocab_file (`str`):
+            Path to the vocabulary file.
+        merges_file (`str`):
+            Path to the merges file.
+        errors (`str`, *optional*, defaults to `"replace"`):
+            Paradigm to follow when decoding bytes to UTF-8.
+        unk_token (`str`, *optional*, defaults to `"<|endoftext|>"`):
+            The unknown token.
+        bos_token (`str`, *optional*):
+            The beginning of sequence token. Not used for vibevoice.
+        eos_token (`str`, *optional*, defaults to `"<|endoftext|>"`):
+            The end of sequence token.
+        pad_token (`str`, *optional*, defaults to `"<|endoftext|>"`):
+            The token used for padding.
+        add_special_tokens (`bool`, *optional*, defaults to `True`):
+            Whether or not to add special tokens when encoding.
+    """
+
+    model_input_names = ["input_ids", "attention_mask"]
+
+    def __init__(
+        self,
+        vocab_file,
+        merges_file,
+        errors="replace",
+        unk_token="<|endoftext|>",
+        bos_token=None,
+        eos_token="<|endoftext|>",
+        pad_token="<|endoftext|>",
+        add_prefix_space=False,
+        add_special_tokens=True,
+        **kwargs,
+    ):
+        super().__init__(
+            vocab_file=vocab_file,
+            merges_file=merges_file,
+            errors=errors,
+            unk_token=unk_token,
+            bos_token=bos_token,
+            eos_token=eos_token,
+            pad_token=pad_token,
+            add_prefix_space=add_prefix_space,
+            add_special_tokens=add_special_tokens,
+            **kwargs,
+        )
+        
+        # Add VibeVoice-specific special tokens
+        self._add_vibevoice_special_tokens()
+        
+    def _add_vibevoice_special_tokens(self):
+        """Add VibeVoice-specific special tokens."""
+        special_tokens = {
+            "additional_special_tokens": [
+                "<|vision_start|>",  # Speech start (reusing vision tokens)
+                "<|vision_end|>",  # Speech end
+                "<|vision_pad|>",  # Speech diffusion pad
+            ]
+        }
+        num_added = self.add_special_tokens(special_tokens)
+        
+        # Cache special token IDs
+        self._speech_start_id = self.convert_tokens_to_ids("<|vision_start|>")
+        self._speech_end_id = self.convert_tokens_to_ids("<|vision_end|>")
+        self._speech_diffusion_id = self.convert_tokens_to_ids("<|vision_pad|>")
+        
+        self._eos_id = self.convert_tokens_to_ids('<|endoftext|>')
+
+        return num_added
+    
+    @property
+    def eos_id(self) -> int:
+        """Id of the end of sequence token."""
+        return self._eos_id
+    
+    @property
+    def speech_start_id(self) -> int:
+        """Id of the speech start token."""
+        return self._speech_start_id
+    
+    @property
+    def speech_end_id(self) -> int:
+        """Id of the speech end token."""
+        return self._speech_end_id
+    
+    @property
+    def speech_diffusion_id(self) -> int:
+        """Id of the speech diffusion token."""
+        return self._speech_diffusion_id
+    
+    @property
+    def pad_id(self) -> int:
+        """Id used for padding (returns -100 for loss masking)."""
+        return -100
+
+
+class VibeVoiceTextTokenizerFast(Qwen2TokenizerFast):
+    """
+    Construct a "fast" VibeVoice tokenizer (backed by HuggingFace's *tokenizers* library).
+    Based on the Qwen2 tokenizer with additional special tokens for speech.
+    
+    Args:
+        vocab_file (`str`, *optional*):
+            Path to the vocabulary file.
+        merges_file (`str`, *optional*):
+            Path to the merges file.
+        tokenizer_file (`str`, *optional*):
+            Path to [tokenizers](https://github.com/huggingface/tokenizers) file.
+        unk_token (`str`, *optional*, defaults to `"<|endoftext|>"`):
+            The unknown token.
+        bos_token (`str`, *optional*):
+            The beginning of sequence token. Not used for vibevoice.
+        eos_token (`str`, *optional*, defaults to `"<|endoftext|>"`):
+            The end of sequence token.
+        pad_token (`str`, *optional*, defaults to `"<|endoftext|>"`):
+            The token used for padding.
+    """
+
+    model_input_names = ["input_ids", "attention_mask"]
+
+    def __init__(
+        self,
+        vocab_file=None,
+        merges_file=None,
+        tokenizer_file=None,
+        unk_token="<|endoftext|>",
+        bos_token=None,
+        eos_token="<|endoftext|>",
+        pad_token="<|endoftext|>",
+        add_prefix_space=False,
+        **kwargs,
+    ):
+        super().__init__(
+            vocab_file=vocab_file,
+            merges_file=merges_file,
+            tokenizer_file=tokenizer_file,
+            unk_token=unk_token,
+            bos_token=bos_token,
+            eos_token=eos_token,
+            pad_token=pad_token,
+            add_prefix_space=add_prefix_space,
+            **kwargs,
+        )
+        
+        # Add VibeVoice-specific special tokens
+        self._add_vibevoice_special_tokens()
+        
+    def _add_vibevoice_special_tokens(self):
+        """Add VibeVoice-specific special tokens."""
+        special_tokens = {
+            "additional_special_tokens": [
+                "<|vision_start|>",  # Speech start (reusing vision tokens)
+                "<|vision_end|>",  # Speech end
+                "<|vision_pad|>",  # Speech diffusion pad
+            ]
+        }
+        num_added = self.add_special_tokens(special_tokens)
+        
+        # Cache special token IDs
+        self._speech_start_id = self.convert_tokens_to_ids("<|vision_start|>")
+        self._speech_end_id = self.convert_tokens_to_ids("<|vision_end|>")
+        self._speech_diffusion_id = self.convert_tokens_to_ids("<|vision_pad|>")
+
+        # self._eos_id = self.convert_tokens_to_ids('<|endoftext|>')
+        self._eos_id = self.eos_token_id # qwen2 / qwen3
+        self._pad_id = self.convert_tokens_to_ids('<|image_pad|>')
+        
+        return num_added
+    
+    @property
+    def eos_id(self) -> int:
+        """Id of the end of sequence token."""
+        return self._eos_id
+    
+    @property
+    def speech_start_id(self) -> int:
+        """Id of the speech start token."""
+        return self._speech_start_id
+    
+    @property
+    def speech_end_id(self) -> int:
+        """Id of the speech end token."""
+        return self._speech_end_id
+    
+    @property
+    def speech_diffusion_id(self) -> int:
+        """Id of the speech diffusion token."""
+        return self._speech_diffusion_id
+    
+    @property
+    def pad_id(self) -> int:
+        """Id used for padding (returns -100 for loss masking)."""
+        return self._pad_id
+
+
+__all__ = [
+    "VibeVoiceTextTokenizer", 
+    "VibeVoiceTextTokenizerFast", 
+]