import spaces import gradio as gr import edge_tts import asyncio import tempfile import os import re from pathlib import Path from pydub import AudioSegment import librosa import numpy as np def get_silence(duration_ms=1000): # Create silent audio segment with specified parameters silent_audio = AudioSegment.silent( duration=duration_ms, frame_rate=24000 # 24kHz sampling rate ) # Set audio parameters silent_audio = silent_audio.set_channels(1) # Mono silent_audio = silent_audio.set_sample_width(4) # 32-bit (4 bytes per sample) with tempfile.NamedTemporaryFile(delete=False, suffix=".mp3") as tmp_file: # Export with specific bitrate and codec parameters silent_audio.export( tmp_file.name, format="mp3", bitrate="48k", parameters=[ "-ac", "1", # Mono "-ar", "24000", # Sample rate "-sample_fmt", "s32", # 32-bit samples "-codec:a", "libmp3lame" # MP3 codec ] ) return tmp_file.name # Get all available voices async def get_voices(): voices = await edge_tts.list_voices() return {f"{v['ShortName']} - {v['Locale']} ({v['Gender']})": v['ShortName'] for v in voices} async def generate_audio_with_voice_prefix(text_segment, default_voice, rate, pitch, target_duration_ms=None): """Generates audio for a text segment, handling voice prefixes and adjusting rate for duration.""" current_voice_full = default_voice current_voice_short = current_voice_full.split(" - ")[0] if current_voice_full else "" current_rate = rate current_pitch = pitch processed_text = text_segment.strip() voice_map = { "1F": "en-GB-SoniaNeural", "2M": "en-GB-RyanNeural", "3M": "en-US-BrianMultilingualNeural", "2F": "en-US-JennyNeural", "1M": "en-AU-WilliamNeural", "3F": "en-HK-YanNeural", "4M": "en-GB-ThomasNeural", "4F": "en-US-EmmaNeural", "1O": "en-GB-RyanNeural", # Old Man "1C": "en-GB-MaisieNeural", # Child "1V": "vi-VN-HoaiMyNeural", # Vietnamese (Female) "2V": "vi-VN-NamMinhNeural", # Vietnamese (Male) "3V": "vi-VN-HoaiMyNeural", # Vietnamese (Female) "4V": "vi-VN-NamMinhNeural", # Vietnamese (Male) } detect = 0 for prefix, voice_short in voice_map.items(): if processed_text.startswith(prefix): current_voice_short = voice_short if prefix in ["1F", "3F", "1V", "3V"]: current_pitch = 25 elif prefix in ["1O", "4V"]: current_pitch = -20 current_rate = -10 detect = 1 processed_text = processed_text[len(prefix):].strip() break match = re.search(r'([A-Za-z]+)-?(\d+)', processed_text) if match: prefix_pitch = match.group(1) number = int(match.group(2)) if prefix_pitch in voice_map: current_pitch += number processed_text = re.sub(r'[A-Za-z]+-?\d+', '', processed_text, count=1).strip() elif detect: processed_text = processed_text.lstrip('-0123456789').strip() # Remove potential leftover numbers elif detect: processed_text = processed_text[2:].strip() if processed_text: rate_str = f"{current_rate:+d}%" pitch_str = f"{current_pitch:+d}Hz" communicate = edge_tts.Communicate(processed_text, current_voice_short, rate=rate_str, pitch=pitch_str) with tempfile.NamedTemporaryFile(delete=False, suffix=".mp3") as tmp_file: audio_path = tmp_file.name await communicate.save(audio_path) if target_duration_ms is not None and os.path.exists(audio_path): audio = AudioSegment.from_mp3(audio_path) audio_duration_ms = len(audio) if audio_duration_ms > 0 and target_duration_ms > 0: speed_factor = audio_duration_ms / target_duration_ms if speed_factor > 0: # Use librosa for time stretching with better quality for speech y, sr = librosa.load(audio_path, sr=None) y_stretched = librosa.effects.time_stretch(y, rate=speed_factor) sf.write(audio_path, y_stretched, sr) return audio_path return None async def process_transcript_line(line, default_voice, rate, pitch): """Processes a single transcript line with HH:MM:SS,milliseconds - HH:MM:SS,milliseconds timestamp.""" match = re.match(r'(\d{2}):(\d{2}):(\d{2}),(\d{3})\s+-\s+(\d{2}):(\d{2}):(\d{2}),(\d{3})\s+(.*)', line) if match: start_h, start_m, start_s, start_ms, end_h, end_m, end_s, end_ms, text_parts = match.groups() start_time_ms = ( int(start_h) * 3600000 + int(start_m) * 60000 + int(start_s) * 1000 + int(start_ms) ) end_time_ms = ( int(end_h) * 3600000 + int(end_m) * 60000 + int(end_s) * 1000 + int(end_ms) ) duration_ms = end_time_ms - start_time_ms audio_segments = [] split_parts = re.split(r'[“”"]', text_parts) process_next = False for part in split_parts: if part == '"': process_next = not process_next continue if process_next and part.strip(): audio_path = await generate_audio_with_voice_prefix(part, default_voice, rate, pitch, duration_ms) if audio_path: audio_segments.append(audio_path) elif not process_next and part.strip(): audio_path = await generate_audio_with_voice_prefix(part, default_voice, rate, pitch, duration_ms) if audio_path: audio_segments.append(audio_path) return start_time_ms, audio_segments, duration_ms return None, None, None async def transcript_to_speech(transcript_text, voice, rate, pitch): if not transcript_text.strip(): return None, gr.Warning("Please enter transcript text.") if not voice: return None, gr.Warning("Please select a voice.") lines = transcript_text.strip().split('\n') timed_audio_segments = [] max_end_time_ms = 0 for line in lines: start_time, audio_paths, duration = await process_transcript_line(line, voice, rate, pitch) if start_time is not None and audio_paths: combined_line_audio = AudioSegment.empty() current_time_ms = start_time segment_duration = duration / len(audio_paths) if audio_paths else 0 for path in audio_paths: try: audio = AudioSegment.from_mp3(path) # No need to adjust speed here, it's done in generate_audio_with_voice_prefix combined_line_audio += audio os.remove(path) except FileNotFoundError: print(f"Warning: Audio file not found: {path}") if combined_line_audio: timed_audio_segments.append({'start': start_time, 'audio': combined_line_audio}) max_end_time_ms = max(max_end_time_ms, start_time + len(combined_line_audio)) elif audio_paths: for path in audio_paths: try: os.remove(path) except FileNotFoundError: pass # Clean up even if no timestamp if not timed_audio_segments: return None, "No processable audio segments found." final_audio = AudioSegment.silent(duration=max_end_time_ms, frame_rate=24000) for segment in timed_audio_segments: final_audio = final_audio.overlay(segment['audio'], position=segment['start']) combined_audio_path = tempfile.mktemp(suffix=".mp3") final_audio.export(combined_audio_path, format="mp3") return combined_audio_path, None @spaces.GPU def tts_interface(transcript, voice, rate, pitch): audio, warning = asyncio.run(transcript_to_speech(transcript, voice, rate, pitch)) return audio, warning async def create_demo(): voices = await get_voices() default_voice = "en-US-AndrewMultilingualNeural - en-US (Male)" description = """ Process timestamped text (HH:MM:SS,milliseconds - HH:MM:SS,milliseconds) with voice changes within quotes. The duration specified in the timestamp will be used to adjust the speech rate so the generated audio fits within that time. Format: `HH:MM:SS,milliseconds - HH:MM:SS,milliseconds "VoicePrefix Text" more text "AnotherVoicePrefix More Text"` Example: ``` 00:00:00,000 - 00:00:05,000 "This is the default voice." more default. "1F Now a female voice." and back to default. 00:00:05,500 - 00:00:10,250 "1C Yes," said the child, "it is fun!" ``` *************************************************************************************************** 1M = en-AU-WilliamNeural - en-AU (Male) 1F = en-GB-SoniaNeural - en-GB (Female) 2M = en-GB-RyanNeural - en-GB (Male) 2F = en-US-JennyNeural - en-US (Female) 3M = en-US-BrianMultilingualNeural - en-US (Male) 3F = en-HK-YanNeural - en-HK (Female) 4M = en-GB-ThomasNeural - en-GB (Male) 4F = en-US-EmmaNeural - en-US (Female) 1O = en-GB-RyanNeural - en-GB (Male) # Old Man 1C = en-GB-MaisieNeural - en-GB (Female) # Child 1V = vi-VN-HoaiMyNeural - vi-VN (Female) # Vietnamese (Female) 2V = vi-VN-NamMinhNeural - vi-VN (Male) # Vietnamese (Male) 3V = vi-VN-HoaiMyNeural - vi-VN (Female) # Vietnamese (Female) 4V = vi-VN-NamMinhNeural - vi-VN (Male) # Vietnamese (Male) **************************************************************************************************** """ demo = gr.Interface( fn=tts_interface, inputs=[ gr.Textbox(label="Timestamped Text with Voice Changes and Duration", lines=10, placeholder='00:00:00,000 - 00:00:05,000 "Text" more text "1F Different Voice"'), gr.Dropdown(choices=[""] + list(voices.keys()), label="Select Default Voice", value=default_voice), gr.Slider(minimum=-50, maximum=50, value=0, label="Speech Rate Adjustment (%)", step=1), gr.Slider(minimum=-50, maximum=50, value=0, label="Pitch Adjustment (Hz)", step=1) ], outputs=[ gr.Audio(label="Generated Audio", type="filepath"), gr.Markdown(label="Warning", visible=False) ], title="TTS with Duration-Aware Speed Adjustment and In-Quote Voice Switching", description=description, analytics_enabled=False, allow_flagging=False ) return demo if __name__ == "__main__": import soundfile as sf # Import soundfile here demo = asyncio.run(create_demo()) demo.launch()