Update app.py

app.py

@@ -8,7 +8,7 @@ import gradio as gr
 import tempfile
 import os 
 
-llasa_1b ='SebastianBodza/Kartoffel-1B-v0.2'
+llasa_1b ='/media/bodza/Audio_Dataset/Llasa-Kartoffel-1B-v0.2'
 
 tokenizer = AutoTokenizer.from_pretrained(llasa_1b, token=os.getenv("HF_TOKEN"))
 
@@ -31,6 +31,59 @@ whisper_turbo_pipe = pipeline(
     device='cuda',
 )
 
+
+vad_model, utils = torch.hub.load(
+    "snakers4/silero-vad",
+    model="silero_vad", 
+    force_reload=False,
+    source="github")
+
+get_speech_timestamps, *_ = utils
+
+
+def remove_silence_silero(waveform, sample_rate, vad_model):
+    """
+    Remove leading silence using Silero VAD.
+    
+    Args:
+        waveform: torch.Tensor audio waveform (channels, samples)
+        sample_rate: int sample rate
+        vad_model: Silero VAD model
+    """
+    if waveform.size(0) > 1:
+        waveform = torch.mean(waveform, dim=0, keepdim=True)
+    
+    original_waveform = waveform
+    
+    if sample_rate != 16000:
+        waveform_16k = torchaudio.transforms.Resample(sample_rate, 16000)(waveform)
+    else:
+        waveform_16k = waveform
+
+    # Get speech timestamps
+    speech_timestamps = get_speech_timestamps(waveform_16k[0], vad_model, sampling_rate=16000)
+    
+    if speech_timestamps:
+        # Get first speech segment start
+        first_speech = speech_timestamps[0]['start']
+        
+        # Add small padding before speech (0.1 seconds)
+        padding_samples = int(0.1 * sample_rate)
+        start_idx = max(0, int(first_speech * sample_rate/16000) - padding_samples)
+        
+        # Same for the end
+        last_speech = speech_timestamps[-1]['end']
+        end_idx = min(original_waveform.size(1), int(last_speech * sample_rate/16000) + padding_samples)
+
+        # Trim the original waveform (not the resampled one)
+        trimmed_wav = original_waveform[:, start_idx:end_idx]
+
+        # added padding of 16 at the start and end 
+        return torch.nn.functional.pad(trimmed_wav, (16, 16), "constant", 0)
+    
+    return original_waveform
+
+
 def ids_to_speech_tokens(speech_ids):
  
     speech_tokens_str = []
@@ -56,6 +109,11 @@ def infer(sample_audio_path, target_text, progress=gr.Progress()):
     with tempfile.NamedTemporaryFile(delete=False, suffix=".wav") as f:
         progress(0, 'Loading and trimming audio...')
         waveform, sample_rate = torchaudio.load(sample_audio_path)
+        waveform = remove_silence_silero(waveform, sample_rate, vad_model)
+
+        # For debugging save the trimmed audio
+        torchaudio.save("dev.wav", waveform, sample_rate)
+
         if len(waveform[0])/sample_rate > 15:
             gr.Warning("Trimming audio to first 15secs.")
             waveform = waveform[:, :sample_rate*15]