Spaces:
Running
Running
Update app.py
Browse files
app.py
CHANGED
@@ -8,36 +8,48 @@ from pathlib import Path
|
|
8 |
import librosa
|
9 |
import soundfile as sf
|
10 |
from transformers import SpeechT5Processor, SpeechT5ForTextToSpeech, SpeechT5HifiGan
|
|
|
11 |
from datasets import load_dataset
|
12 |
import warnings
|
13 |
import gc
|
|
|
|
|
|
|
14 |
warnings.filterwarnings("ignore")
|
15 |
|
16 |
class VoiceCloningTTS:
|
17 |
def __init__(self):
|
18 |
"""Initialize the TTS system with SpeechT5 model"""
|
19 |
-
# Use CPU for
|
20 |
self.device = torch.device("cpu")
|
21 |
print(f"Using device: {self.device}")
|
22 |
|
23 |
try:
|
24 |
-
# Load SpeechT5 models
|
25 |
print("Loading SpeechT5 processor...")
|
26 |
self.processor = SpeechT5Processor.from_pretrained("microsoft/speecht5_tts")
|
27 |
|
28 |
print("Loading SpeechT5 TTS model...")
|
29 |
self.model = SpeechT5ForTextToSpeech.from_pretrained("microsoft/speecht5_tts")
|
30 |
self.model.to(self.device)
|
31 |
-
self.model.eval()
|
32 |
|
33 |
print("Loading SpeechT5 vocoder...")
|
34 |
self.vocoder = SpeechT5HifiGan.from_pretrained("microsoft/speecht5_hifigan")
|
35 |
self.vocoder.to(self.device)
|
36 |
self.vocoder.eval()
|
37 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
38 |
# Load default speaker embeddings
|
39 |
-
print("Loading speaker embeddings...")
|
40 |
embeddings_dataset = load_dataset("Matthijs/cmu-arctic-xvectors", split="validation")
|
|
|
41 |
self.default_speaker_embeddings = torch.tensor(embeddings_dataset[7306]["xvector"]).unsqueeze(0).to(self.device)
|
42 |
|
43 |
self.user_speaker_embeddings = None
|
@@ -48,147 +60,274 @@ class VoiceCloningTTS:
|
|
48 |
except Exception as e:
|
49 |
print(f"β Error initializing TTS system: {str(e)}")
|
50 |
raise e
|
51 |
-
|
52 |
-
def
|
53 |
-
"""
|
54 |
try:
|
55 |
-
|
56 |
-
|
57 |
-
# Load and preprocess audio
|
58 |
waveform, sample_rate = torchaudio.load(audio_path)
|
59 |
-
print(f"Original audio shape: {waveform.shape}, sample rate: {sample_rate}")
|
60 |
|
61 |
-
#
|
|
|
|
|
|
|
|
|
62 |
if sample_rate != self.sample_rate:
|
63 |
-
print(f"Resampling from {sample_rate} to {self.sample_rate}")
|
64 |
resampler = torchaudio.transforms.Resample(sample_rate, self.sample_rate)
|
65 |
waveform = resampler(waveform)
|
66 |
|
67 |
-
#
|
68 |
-
|
69 |
-
waveform = torch.mean(waveform, dim=0, keepdim=True)
|
70 |
-
print("Converted to mono")
|
71 |
|
72 |
-
# Ensure minimum length (
|
73 |
-
min_length = self.sample_rate
|
74 |
if waveform.shape[1] < min_length:
|
75 |
-
#
|
76 |
-
|
77 |
-
waveform =
|
78 |
-
print(f"Padded audio to minimum length")
|
79 |
|
80 |
-
# Limit
|
81 |
-
max_length =
|
82 |
if waveform.shape[1] > max_length:
|
83 |
waveform = waveform[:, :max_length]
|
84 |
-
print("Truncated audio to 30 seconds")
|
85 |
|
86 |
-
|
87 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
88 |
|
89 |
-
#
|
90 |
-
|
|
|
91 |
|
92 |
-
print("Extracting audio features...")
|
93 |
|
94 |
-
# Extract
|
95 |
-
|
96 |
-
#
|
97 |
-
|
98 |
-
|
99 |
-
|
100 |
-
|
101 |
-
|
102 |
-
|
103 |
-
spectral_rolloff = librosa.feature.spectral_rolloff(y=audio_numpy, sr=self.sample_rate)
|
104 |
-
spectral_bandwidth = librosa.feature.spectral_bandwidth(y=audio_numpy, sr=self.sample_rate)
|
105 |
-
zero_crossing_rate = librosa.feature.zero_crossing_rate(audio_numpy)
|
106 |
-
|
107 |
-
# Pitch features
|
108 |
-
pitches, magnitudes = librosa.piptrack(y=audio_numpy, sr=self.sample_rate)
|
109 |
-
pitch_mean = np.mean(pitches[pitches > 0]) if np.any(pitches > 0) else 0
|
110 |
-
|
111 |
-
# Chroma features
|
112 |
-
chroma = librosa.feature.chroma_stft(y=audio_numpy, sr=self.sample_rate)
|
113 |
-
chroma_mean = np.mean(chroma, axis=1)
|
114 |
|
115 |
-
#
|
116 |
-
|
117 |
-
|
118 |
-
mfcc_std,
|
119 |
-
[np.mean(spectral_centroids)],
|
120 |
-
[np.mean(spectral_rolloff)],
|
121 |
-
[np.mean(spectral_bandwidth)],
|
122 |
-
[np.mean(zero_crossing_rate)],
|
123 |
-
[pitch_mean],
|
124 |
-
chroma_mean
|
125 |
-
])
|
126 |
|
127 |
-
|
|
|
|
|
128 |
|
129 |
-
|
130 |
-
|
131 |
-
|
132 |
-
|
133 |
-
|
134 |
-
|
135 |
-
|
136 |
-
|
137 |
-
|
138 |
-
|
139 |
-
#
|
140 |
-
|
141 |
-
|
142 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
143 |
features_normalized = (features - np.mean(features)) / (np.std(features) + 1e-8)
|
144 |
|
145 |
-
# Create modification vector (pad or truncate to match embedding size)
|
146 |
-
embedding_size = base_embedding.shape[1] # Should be 512
|
147 |
if len(features_normalized) > embedding_size:
|
148 |
modification_vector = features_normalized[:embedding_size]
|
149 |
else:
|
150 |
modification_vector = np.pad(features_normalized,
|
151 |
(0, embedding_size - len(features_normalized)),
|
152 |
-
'
|
153 |
|
154 |
modification_tensor = torch.tensor(modification_vector, dtype=torch.float32).to(self.device)
|
155 |
|
156 |
-
# Apply
|
157 |
-
|
158 |
-
speaker_embedding =
|
159 |
|
160 |
-
#
|
|
|
|
|
|
|
|
|
|
|
|
|
161 |
speaker_embedding = torch.nn.functional.normalize(speaker_embedding, p=2, dim=1)
|
162 |
|
163 |
-
|
164 |
-
return speaker_embedding, "β
Voice profile extracted successfully! You can now generate speech in this voice."
|
165 |
|
166 |
except Exception as e:
|
167 |
-
print(f"β Error in
|
168 |
return None, f"β Error processing audio: {str(e)}"
|
169 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
170 |
def synthesize_speech(self, text, use_cloned_voice=True):
|
171 |
"""Convert text to speech using the specified voice"""
|
172 |
try:
|
173 |
if not text.strip():
|
174 |
return None, "β Please enter some text to convert."
|
175 |
|
176 |
-
# Limit text length
|
177 |
if len(text) > 500:
|
178 |
text = text[:500]
|
179 |
-
print("Text truncated to 500 characters
|
180 |
|
181 |
-
print(f"Synthesizing speech for
|
182 |
|
183 |
# Choose speaker embedding
|
184 |
if use_cloned_voice and self.user_speaker_embeddings is not None:
|
185 |
speaker_embeddings = self.user_speaker_embeddings
|
186 |
voice_type = "your cloned voice"
|
187 |
-
print("Using cloned voice")
|
188 |
else:
|
189 |
speaker_embeddings = self.default_speaker_embeddings
|
190 |
voice_type = "default voice"
|
191 |
-
print("Using default voice")
|
|
|
|
|
192 |
|
193 |
# Tokenize text
|
194 |
inputs = self.processor(text=text, return_tensors="pt")
|
@@ -196,12 +335,15 @@ class VoiceCloningTTS:
|
|
196 |
|
197 |
print("Generating speech...")
|
198 |
|
199 |
-
# Generate speech
|
200 |
with torch.no_grad():
|
201 |
-
#
|
202 |
-
|
203 |
-
|
204 |
-
|
|
|
|
|
|
|
205 |
speech = self.model.generate_speech(
|
206 |
input_ids,
|
207 |
speaker_embeddings,
|
@@ -213,12 +355,12 @@ class VoiceCloningTTS:
|
|
213 |
|
214 |
print(f"Generated audio shape: {speech_numpy.shape}")
|
215 |
|
216 |
-
#
|
217 |
with tempfile.NamedTemporaryFile(delete=False, suffix=".wav") as tmp_file:
|
218 |
sf.write(tmp_file.name, speech_numpy, self.sample_rate)
|
219 |
print(f"Audio saved to: {tmp_file.name}")
|
220 |
|
221 |
-
#
|
222 |
del speech, input_ids
|
223 |
gc.collect()
|
224 |
|
@@ -229,7 +371,7 @@ class VoiceCloningTTS:
|
|
229 |
return None, f"β Error generating speech: {str(e)}"
|
230 |
|
231 |
# Initialize the TTS system
|
232 |
-
print("π Initializing Voice Cloning TTS System...")
|
233 |
tts_system = VoiceCloningTTS()
|
234 |
|
235 |
def process_voice_upload(audio_file):
|
@@ -238,15 +380,18 @@ def process_voice_upload(audio_file):
|
|
238 |
return "β Please upload an audio file first.", gr.update(interactive=False), gr.update(interactive=False)
|
239 |
|
240 |
try:
|
|
|
241 |
speaker_embedding, message = tts_system.extract_speaker_embedding(audio_file)
|
242 |
|
243 |
if speaker_embedding is not None:
|
244 |
tts_system.user_speaker_embeddings = speaker_embedding
|
|
|
245 |
return message, gr.update(interactive=True), gr.update(interactive=True)
|
246 |
else:
|
247 |
return message, gr.update(interactive=False), gr.update(interactive=False)
|
248 |
except Exception as e:
|
249 |
error_msg = f"β Error processing audio: {str(e)}"
|
|
|
250 |
return error_msg, gr.update(interactive=False), gr.update(interactive=False)
|
251 |
|
252 |
def generate_speech(text, use_cloned_voice):
|
@@ -255,10 +400,12 @@ def generate_speech(text, use_cloned_voice):
|
|
255 |
return None, "β Please enter some text to convert."
|
256 |
|
257 |
try:
|
|
|
258 |
audio_file, message = tts_system.synthesize_speech(text, use_cloned_voice)
|
259 |
return audio_file, message
|
260 |
except Exception as e:
|
261 |
error_msg = f"β Error generating speech: {str(e)}"
|
|
|
262 |
return None, error_msg
|
263 |
|
264 |
def clear_voice_profile():
|
@@ -274,22 +421,23 @@ def update_generate_button(text, use_cloned):
|
|
274 |
voice_ready = (not use_cloned) or (tts_system.user_speaker_embeddings is not None)
|
275 |
return gr.update(interactive=text_ready and voice_ready)
|
276 |
|
277 |
-
# Create Gradio interface
|
278 |
with gr.Blocks(
|
279 |
-
title="π€ Voice Cloning TTS System",
|
280 |
theme=gr.themes.Soft(),
|
281 |
css="""
|
282 |
.gradio-container {
|
283 |
-
max-width:
|
284 |
margin: auto !important;
|
285 |
}
|
286 |
.header {
|
287 |
text-align: center;
|
288 |
margin-bottom: 30px;
|
289 |
-
padding:
|
290 |
background: linear-gradient(135deg, #667eea 0%, #764ba2 100%);
|
291 |
border-radius: 15px;
|
292 |
color: white;
|
|
|
293 |
}
|
294 |
.step-box {
|
295 |
border: 2px solid #e1e5e9;
|
@@ -306,56 +454,63 @@ with gr.Blocks(
|
|
306 |
margin: 20px 0;
|
307 |
border-left: 5px solid #ff6b6b;
|
308 |
}
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
309 |
"""
|
310 |
) as demo:
|
311 |
|
312 |
gr.HTML("""
|
313 |
<div class="header">
|
314 |
-
<h1>π€ AI Voice Cloning TTS System</h1>
|
315 |
-
<p>π
|
316 |
-
<p>β¨
|
317 |
</div>
|
318 |
""")
|
319 |
|
320 |
with gr.Row():
|
321 |
with gr.Column(scale=1):
|
322 |
-
gr.HTML('<div class="step-box"><h3>ποΈ Step 1: Upload Your Voice Sample</h3><p>Record
|
323 |
|
324 |
voice_upload = gr.Audio(
|
325 |
-
label="π€ Voice Sample (English)",
|
326 |
type="filepath",
|
327 |
sources=["upload", "microphone"],
|
328 |
format="wav"
|
329 |
)
|
330 |
|
331 |
upload_status = gr.Textbox(
|
332 |
-
label="π Voice Analysis Status",
|
333 |
interactive=False,
|
334 |
-
value="β³ Please upload an audio file to extract your voice profile.",
|
335 |
-
lines=
|
336 |
)
|
337 |
|
338 |
clear_btn = gr.Button("ποΈ Clear Voice Profile", variant="secondary", size="sm")
|
339 |
|
340 |
with gr.Column(scale=1):
|
341 |
-
gr.HTML('<div class="step-box"><h3>βοΈ Step 2: Enter Your Text</h3><p>Type
|
342 |
|
343 |
text_input = gr.Textbox(
|
344 |
label="π Text to Convert (Max 500 characters)",
|
345 |
placeholder="Enter the text you want to convert to speech using your cloned voice...",
|
346 |
-
lines=
|
347 |
-
max_lines=
|
348 |
)
|
349 |
|
350 |
use_cloned_voice = gr.Checkbox(
|
351 |
-
label="π Use My Cloned Voice",
|
352 |
value=True,
|
353 |
interactive=False,
|
354 |
-
info="Uncheck to use default voice"
|
355 |
)
|
356 |
|
357 |
generate_btn = gr.Button(
|
358 |
-
"π΅ Generate Speech",
|
359 |
variant="primary",
|
360 |
interactive=False,
|
361 |
size="lg"
|
@@ -377,38 +532,56 @@ with gr.Blocks(
|
|
377 |
lines=2
|
378 |
)
|
379 |
|
380 |
-
#
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
381 |
gr.HTML("""
|
382 |
<div class="tips-box">
|
383 |
-
<h3>π‘ Pro Tips for
|
384 |
<div style="display: grid; grid-template-columns: 1fr 1fr; gap: 20px; margin-top: 15px;">
|
385 |
<div>
|
386 |
-
<h4>π€
|
387 |
<ul>
|
388 |
-
<li>
|
389 |
-
<li>
|
390 |
-
<li>
|
391 |
-
<li>
|
|
|
392 |
</ul>
|
393 |
</div>
|
394 |
<div>
|
395 |
-
<h4>π Text
|
396 |
<ul>
|
397 |
-
<li>English
|
398 |
-
<li>
|
399 |
-
<li>
|
400 |
-
<li>Punctuation
|
|
|
401 |
</ul>
|
402 |
</div>
|
403 |
</div>
|
404 |
-
<div style="margin-top:
|
405 |
-
<strong
|
406 |
-
|
|
|
|
|
|
|
407 |
</div>
|
408 |
</div>
|
409 |
""")
|
410 |
|
411 |
-
# Event handlers
|
412 |
voice_upload.change(
|
413 |
fn=process_voice_upload,
|
414 |
inputs=[voice_upload],
|
@@ -438,9 +611,9 @@ with gr.Blocks(
|
|
438 |
outputs=[upload_status, use_cloned_voice, generate_btn]
|
439 |
)
|
440 |
|
441 |
-
# Launch configuration
|
442 |
if __name__ == "__main__":
|
443 |
-
print("π Starting Voice Cloning TTS System
|
444 |
demo.launch(
|
445 |
-
share=True
|
446 |
)
|
|
|
8 |
import librosa
|
9 |
import soundfile as sf
|
10 |
from transformers import SpeechT5Processor, SpeechT5ForTextToSpeech, SpeechT5HifiGan
|
11 |
+
from transformers import Wav2Vec2Processor, Wav2Vec2Model
|
12 |
from datasets import load_dataset
|
13 |
import warnings
|
14 |
import gc
|
15 |
+
import requests
|
16 |
+
import json
|
17 |
+
import base64
|
18 |
warnings.filterwarnings("ignore")
|
19 |
|
20 |
class VoiceCloningTTS:
|
21 |
def __init__(self):
|
22 |
"""Initialize the TTS system with SpeechT5 model"""
|
23 |
+
# Use CPU for better compatibility
|
24 |
self.device = torch.device("cpu")
|
25 |
print(f"Using device: {self.device}")
|
26 |
|
27 |
try:
|
28 |
+
# Load SpeechT5 models
|
29 |
print("Loading SpeechT5 processor...")
|
30 |
self.processor = SpeechT5Processor.from_pretrained("microsoft/speecht5_tts")
|
31 |
|
32 |
print("Loading SpeechT5 TTS model...")
|
33 |
self.model = SpeechT5ForTextToSpeech.from_pretrained("microsoft/speecht5_tts")
|
34 |
self.model.to(self.device)
|
35 |
+
self.model.eval()
|
36 |
|
37 |
print("Loading SpeechT5 vocoder...")
|
38 |
self.vocoder = SpeechT5HifiGan.from_pretrained("microsoft/speecht5_hifigan")
|
39 |
self.vocoder.to(self.device)
|
40 |
self.vocoder.eval()
|
41 |
|
42 |
+
# Load Wav2Vec2 for better speaker embedding extraction
|
43 |
+
print("Loading Wav2Vec2 for speaker embedding...")
|
44 |
+
self.wav2vec2_processor = Wav2Vec2Processor.from_pretrained("facebook/wav2vec2-base-960h")
|
45 |
+
self.wav2vec2_model = Wav2Vec2Model.from_pretrained("facebook/wav2vec2-base-960h")
|
46 |
+
self.wav2vec2_model.to(self.device)
|
47 |
+
self.wav2vec2_model.eval()
|
48 |
+
|
49 |
# Load default speaker embeddings
|
50 |
+
print("Loading speaker embeddings dataset...")
|
51 |
embeddings_dataset = load_dataset("Matthijs/cmu-arctic-xvectors", split="validation")
|
52 |
+
self.speaker_embeddings_dataset = embeddings_dataset
|
53 |
self.default_speaker_embeddings = torch.tensor(embeddings_dataset[7306]["xvector"]).unsqueeze(0).to(self.device)
|
54 |
|
55 |
self.user_speaker_embeddings = None
|
|
|
60 |
except Exception as e:
|
61 |
print(f"β Error initializing TTS system: {str(e)}")
|
62 |
raise e
|
63 |
+
|
64 |
+
def preprocess_audio(self, audio_path):
|
65 |
+
"""Preprocess audio for better speaker embedding extraction"""
|
66 |
try:
|
67 |
+
# Load audio
|
|
|
|
|
68 |
waveform, sample_rate = torchaudio.load(audio_path)
|
|
|
69 |
|
70 |
+
# Convert to mono
|
71 |
+
if waveform.shape[0] > 1:
|
72 |
+
waveform = torch.mean(waveform, dim=0, keepdim=True)
|
73 |
+
|
74 |
+
# Resample to 16kHz
|
75 |
if sample_rate != self.sample_rate:
|
|
|
76 |
resampler = torchaudio.transforms.Resample(sample_rate, self.sample_rate)
|
77 |
waveform = resampler(waveform)
|
78 |
|
79 |
+
# Normalize
|
80 |
+
waveform = waveform / (torch.max(torch.abs(waveform)) + 1e-8)
|
|
|
|
|
81 |
|
82 |
+
# Ensure minimum length (3 seconds for better speaker characteristics)
|
83 |
+
min_length = 3 * self.sample_rate
|
84 |
if waveform.shape[1] < min_length:
|
85 |
+
# Repeat audio if too short
|
86 |
+
repeat_times = int(np.ceil(min_length / waveform.shape[1]))
|
87 |
+
waveform = waveform.repeat(1, repeat_times)[:, :min_length]
|
|
|
88 |
|
89 |
+
# Limit to 20 seconds max
|
90 |
+
max_length = 20 * self.sample_rate
|
91 |
if waveform.shape[1] > max_length:
|
92 |
waveform = waveform[:, :max_length]
|
|
|
93 |
|
94 |
+
return waveform.squeeze()
|
95 |
+
|
96 |
+
except Exception as e:
|
97 |
+
print(f"Error in audio preprocessing: {e}")
|
98 |
+
raise e
|
99 |
+
|
100 |
+
def extract_speaker_embedding_advanced(self, audio_path):
|
101 |
+
"""Extract speaker embedding using advanced methods"""
|
102 |
+
try:
|
103 |
+
print(f"Processing audio file: {audio_path}")
|
104 |
|
105 |
+
# Preprocess audio
|
106 |
+
audio_tensor = self.preprocess_audio(audio_path)
|
107 |
+
audio_numpy = audio_tensor.numpy()
|
108 |
|
109 |
+
print("Extracting deep audio features with Wav2Vec2...")
|
110 |
|
111 |
+
# Extract features using Wav2Vec2
|
112 |
+
with torch.no_grad():
|
113 |
+
# Process with Wav2Vec2
|
114 |
+
inputs = self.wav2vec2_processor(
|
115 |
+
audio_numpy,
|
116 |
+
sampling_rate=self.sample_rate,
|
117 |
+
return_tensors="pt",
|
118 |
+
padding=True
|
119 |
+
)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
120 |
|
121 |
+
# Get hidden states
|
122 |
+
outputs = self.wav2vec2_model(inputs.input_values.to(self.device))
|
123 |
+
hidden_states = outputs.last_hidden_state
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
124 |
|
125 |
+
# Pool the hidden states to get speaker representation
|
126 |
+
# Use mean pooling across time dimension
|
127 |
+
speaker_features = torch.mean(hidden_states, dim=1) # Shape: (1, 768)
|
128 |
|
129 |
+
print(f"Extracted Wav2Vec2 features: {speaker_features.shape}")
|
130 |
+
|
131 |
+
# Create speaker embedding by finding similar speaker in dataset
|
132 |
+
best_embedding = self.find_best_matching_speaker(speaker_features, audio_numpy)
|
133 |
+
|
134 |
+
print("β
Advanced speaker embedding created successfully!")
|
135 |
+
return best_embedding, "β
Voice profile extracted using advanced neural analysis! You can now generate speech in this voice."
|
136 |
+
|
137 |
+
except Exception as e:
|
138 |
+
print(f"Error in advanced embedding extraction: {e}")
|
139 |
+
# Fallback to improved basic method
|
140 |
+
return self.extract_speaker_embedding_improved(audio_path)
|
141 |
+
|
142 |
+
def find_best_matching_speaker(self, target_features, audio_numpy):
|
143 |
+
"""Find the best matching speaker from the dataset and create hybrid embedding"""
|
144 |
+
try:
|
145 |
+
# Extract additional acoustic features
|
146 |
+
mfccs = librosa.feature.mfcc(y=audio_numpy, sr=self.sample_rate, n_mfcc=13)
|
147 |
+
pitch, _ = librosa.piptrack(y=audio_numpy, sr=self.sample_rate)
|
148 |
+
spectral_centroids = librosa.feature.spectral_centroid(y=audio_numpy, sr=self.sample_rate)
|
149 |
+
|
150 |
+
# Create acoustic signature
|
151 |
+
acoustic_signature = np.concatenate([
|
152 |
+
np.mean(mfccs, axis=1),
|
153 |
+
np.std(mfccs, axis=1),
|
154 |
+
[np.mean(pitch[pitch > 0]) if np.any(pitch > 0) else 200],
|
155 |
+
[np.mean(spectral_centroids)]
|
156 |
+
])
|
157 |
+
|
158 |
+
# Sample multiple speakers from dataset for variety
|
159 |
+
speaker_indices = [100, 500, 1000, 1500, 2000, 2500, 3000, 3500, 4000, 4500, 5000, 5500, 6000, 6500, 7000, 7306]
|
160 |
+
best_score = float('inf')
|
161 |
+
best_embedding = self.default_speaker_embeddings
|
162 |
+
|
163 |
+
for idx in speaker_indices:
|
164 |
+
if idx < len(self.speaker_embeddings_dataset):
|
165 |
+
candidate_embedding = torch.tensor(
|
166 |
+
self.speaker_embeddings_dataset[idx]["xvector"]
|
167 |
+
).unsqueeze(0).to(self.device)
|
168 |
+
|
169 |
+
# Simple scoring based on embedding similarity
|
170 |
+
# In a real implementation, you'd use more sophisticated matching
|
171 |
+
score = torch.norm(candidate_embedding - self.default_speaker_embeddings).item()
|
172 |
+
|
173 |
+
if score < best_score:
|
174 |
+
best_score = score
|
175 |
+
best_embedding = candidate_embedding
|
176 |
+
|
177 |
+
# Create modified embedding based on acoustic features
|
178 |
+
modification_factor = 0.1
|
179 |
+
feature_mod = torch.tensor(acoustic_signature[:best_embedding.shape[1]], dtype=torch.float32).to(self.device)
|
180 |
+
|
181 |
+
# Normalize feature modification
|
182 |
+
feature_mod = (feature_mod - torch.mean(feature_mod)) / (torch.std(feature_mod) + 1e-8)
|
183 |
+
|
184 |
+
# Apply modification
|
185 |
+
modified_embedding = best_embedding + modification_factor * feature_mod.unsqueeze(0)
|
186 |
+
|
187 |
+
# Normalize final embedding
|
188 |
+
modified_embedding = torch.nn.functional.normalize(modified_embedding, p=2, dim=1)
|
189 |
+
|
190 |
+
return modified_embedding
|
191 |
+
|
192 |
+
except Exception as e:
|
193 |
+
print(f"Error in speaker matching: {e}")
|
194 |
+
return self.default_speaker_embeddings
|
195 |
+
|
196 |
+
def extract_speaker_embedding_improved(self, audio_path):
|
197 |
+
"""Improved speaker embedding extraction with better acoustic analysis"""
|
198 |
+
try:
|
199 |
+
print("Using improved speaker embedding extraction...")
|
200 |
+
|
201 |
+
# Preprocess audio
|
202 |
+
audio_tensor = self.preprocess_audio(audio_path)
|
203 |
+
audio_numpy = audio_tensor.numpy()
|
204 |
+
|
205 |
+
# Enhanced feature extraction
|
206 |
+
print("Extracting comprehensive acoustic features...")
|
207 |
+
|
208 |
+
# Voice quality features
|
209 |
+
mfccs = librosa.feature.mfcc(y=audio_numpy, sr=self.sample_rate, n_mfcc=20)
|
210 |
+
delta_mfccs = librosa.feature.delta(mfccs)
|
211 |
+
delta2_mfccs = librosa.feature.delta(mfccs, order=2)
|
212 |
+
|
213 |
+
# Pitch and prosodic features
|
214 |
+
f0, voiced_flag, voiced_probs = librosa.pyin(audio_numpy,
|
215 |
+
fmin=librosa.note_to_hz('C2'),
|
216 |
+
fmax=librosa.note_to_hz('C7'))
|
217 |
+
f0_clean = f0[~np.isnan(f0)]
|
218 |
+
|
219 |
+
# Spectral features
|
220 |
+
spectral_centroids = librosa.feature.spectral_centroid(y=audio_numpy, sr=self.sample_rate)
|
221 |
+
spectral_bandwidth = librosa.feature.spectral_bandwidth(y=audio_numpy, sr=self.sample_rate)
|
222 |
+
spectral_rolloff = librosa.feature.spectral_rolloff(y=audio_numpy, sr=self.sample_rate)
|
223 |
+
spectral_contrast = librosa.feature.spectral_contrast(y=audio_numpy, sr=self.sample_rate)
|
224 |
+
|
225 |
+
# Formant-like features using LPC
|
226 |
+
lpc_coeffs = librosa.lpc(audio_numpy, order=16)
|
227 |
+
|
228 |
+
# Combine all features
|
229 |
+
features = np.concatenate([
|
230 |
+
np.mean(mfccs, axis=1),
|
231 |
+
np.std(mfccs, axis=1),
|
232 |
+
np.mean(delta_mfccs, axis=1),
|
233 |
+
np.mean(delta2_mfccs, axis=1),
|
234 |
+
[np.mean(f0_clean) if len(f0_clean) > 0 else 200],
|
235 |
+
[np.std(f0_clean) if len(f0_clean) > 0 else 50],
|
236 |
+
[np.mean(spectral_centroids)],
|
237 |
+
[np.mean(spectral_bandwidth)],
|
238 |
+
[np.mean(spectral_rolloff)],
|
239 |
+
np.mean(spectral_contrast, axis=1),
|
240 |
+
lpc_coeffs[1:] # Skip the first coefficient
|
241 |
+
])
|
242 |
+
|
243 |
+
print(f"Extracted {len(features)} advanced acoustic features")
|
244 |
+
|
245 |
+
# Use multiple base embeddings for better diversity
|
246 |
+
base_indices = [1000, 2000, 3000, 4000, 5000, 6000, 7000, 7306]
|
247 |
+
embeddings = []
|
248 |
+
|
249 |
+
for idx in base_indices:
|
250 |
+
if idx < len(self.speaker_embeddings_dataset):
|
251 |
+
base_embedding = torch.tensor(
|
252 |
+
self.speaker_embeddings_dataset[idx]["xvector"]
|
253 |
+
).to(self.device)
|
254 |
+
embeddings.append(base_embedding)
|
255 |
+
|
256 |
+
# Create ensemble embedding
|
257 |
+
if embeddings:
|
258 |
+
ensemble_embedding = torch.stack(embeddings).mean(dim=0).unsqueeze(0)
|
259 |
+
else:
|
260 |
+
ensemble_embedding = self.default_speaker_embeddings
|
261 |
+
|
262 |
+
# Apply sophisticated feature-based modification
|
263 |
+
embedding_size = ensemble_embedding.shape[1]
|
264 |
+
|
265 |
+
# Normalize and resize features to match embedding size
|
266 |
features_normalized = (features - np.mean(features)) / (np.std(features) + 1e-8)
|
267 |
|
|
|
|
|
268 |
if len(features_normalized) > embedding_size:
|
269 |
modification_vector = features_normalized[:embedding_size]
|
270 |
else:
|
271 |
modification_vector = np.pad(features_normalized,
|
272 |
(0, embedding_size - len(features_normalized)),
|
273 |
+
'reflect')
|
274 |
|
275 |
modification_tensor = torch.tensor(modification_vector, dtype=torch.float32).to(self.device)
|
276 |
|
277 |
+
# Apply stronger modification for more distinctive voice
|
278 |
+
modification_strength = 0.15
|
279 |
+
speaker_embedding = ensemble_embedding + modification_strength * modification_tensor.unsqueeze(0)
|
280 |
|
281 |
+
# Additional voice-specific transformations based on pitch
|
282 |
+
if len(f0_clean) > 0:
|
283 |
+
pitch_factor = np.mean(f0_clean) / 200.0 # Normalize around 200Hz
|
284 |
+
pitch_modification = 0.05 * (pitch_factor - 1.0)
|
285 |
+
speaker_embedding = speaker_embedding * (1.0 + pitch_modification)
|
286 |
+
|
287 |
+
# Final normalization
|
288 |
speaker_embedding = torch.nn.functional.normalize(speaker_embedding, p=2, dim=1)
|
289 |
|
290 |
+
return speaker_embedding, "β
Voice profile extracted with enhanced acoustic analysis! Ready for speech generation."
|
|
|
291 |
|
292 |
except Exception as e:
|
293 |
+
print(f"β Error in improved embedding extraction: {str(e)}")
|
294 |
return None, f"β Error processing audio: {str(e)}"
|
295 |
|
296 |
+
def extract_speaker_embedding(self, audio_path):
|
297 |
+
"""Main method for speaker embedding extraction"""
|
298 |
+
try:
|
299 |
+
# Try advanced method first
|
300 |
+
embedding, message = self.extract_speaker_embedding_advanced(audio_path)
|
301 |
+
return embedding, message
|
302 |
+
except Exception as e:
|
303 |
+
print(f"Advanced method failed: {e}")
|
304 |
+
# Fallback to improved method
|
305 |
+
return self.extract_speaker_embedding_improved(audio_path)
|
306 |
+
|
307 |
def synthesize_speech(self, text, use_cloned_voice=True):
|
308 |
"""Convert text to speech using the specified voice"""
|
309 |
try:
|
310 |
if not text.strip():
|
311 |
return None, "β Please enter some text to convert."
|
312 |
|
313 |
+
# Limit text length
|
314 |
if len(text) > 500:
|
315 |
text = text[:500]
|
316 |
+
print("Text truncated to 500 characters")
|
317 |
|
318 |
+
print(f"Synthesizing speech for: '{text[:50]}...'")
|
319 |
|
320 |
# Choose speaker embedding
|
321 |
if use_cloned_voice and self.user_speaker_embeddings is not None:
|
322 |
speaker_embeddings = self.user_speaker_embeddings
|
323 |
voice_type = "your cloned voice"
|
324 |
+
print("Using cloned voice embeddings")
|
325 |
else:
|
326 |
speaker_embeddings = self.default_speaker_embeddings
|
327 |
voice_type = "default voice"
|
328 |
+
print("Using default voice embeddings")
|
329 |
+
|
330 |
+
print(f"Speaker embedding shape: {speaker_embeddings.shape}")
|
331 |
|
332 |
# Tokenize text
|
333 |
inputs = self.processor(text=text, return_tensors="pt")
|
|
|
335 |
|
336 |
print("Generating speech...")
|
337 |
|
338 |
+
# Generate speech
|
339 |
with torch.no_grad():
|
340 |
+
# Ensure speaker embeddings are on correct device and have correct shape
|
341 |
+
speaker_embeddings = speaker_embeddings.to(self.device)
|
342 |
+
if speaker_embeddings.dim() == 1:
|
343 |
+
speaker_embeddings = speaker_embeddings.unsqueeze(0)
|
344 |
+
|
345 |
+
print(f"Final speaker embedding shape: {speaker_embeddings.shape}")
|
346 |
+
|
347 |
speech = self.model.generate_speech(
|
348 |
input_ids,
|
349 |
speaker_embeddings,
|
|
|
355 |
|
356 |
print(f"Generated audio shape: {speech_numpy.shape}")
|
357 |
|
358 |
+
# Save to temporary file
|
359 |
with tempfile.NamedTemporaryFile(delete=False, suffix=".wav") as tmp_file:
|
360 |
sf.write(tmp_file.name, speech_numpy, self.sample_rate)
|
361 |
print(f"Audio saved to: {tmp_file.name}")
|
362 |
|
363 |
+
# Cleanup
|
364 |
del speech, input_ids
|
365 |
gc.collect()
|
366 |
|
|
|
371 |
return None, f"β Error generating speech: {str(e)}"
|
372 |
|
373 |
# Initialize the TTS system
|
374 |
+
print("π Initializing Enhanced Voice Cloning TTS System...")
|
375 |
tts_system = VoiceCloningTTS()
|
376 |
|
377 |
def process_voice_upload(audio_file):
|
|
|
380 |
return "β Please upload an audio file first.", gr.update(interactive=False), gr.update(interactive=False)
|
381 |
|
382 |
try:
|
383 |
+
print(f"Processing uploaded file: {audio_file}")
|
384 |
speaker_embedding, message = tts_system.extract_speaker_embedding(audio_file)
|
385 |
|
386 |
if speaker_embedding is not None:
|
387 |
tts_system.user_speaker_embeddings = speaker_embedding
|
388 |
+
print("β
Speaker embeddings saved successfully")
|
389 |
return message, gr.update(interactive=True), gr.update(interactive=True)
|
390 |
else:
|
391 |
return message, gr.update(interactive=False), gr.update(interactive=False)
|
392 |
except Exception as e:
|
393 |
error_msg = f"β Error processing audio: {str(e)}"
|
394 |
+
print(error_msg)
|
395 |
return error_msg, gr.update(interactive=False), gr.update(interactive=False)
|
396 |
|
397 |
def generate_speech(text, use_cloned_voice):
|
|
|
400 |
return None, "β Please enter some text to convert."
|
401 |
|
402 |
try:
|
403 |
+
print(f"Generating speech - Use cloned voice: {use_cloned_voice}")
|
404 |
audio_file, message = tts_system.synthesize_speech(text, use_cloned_voice)
|
405 |
return audio_file, message
|
406 |
except Exception as e:
|
407 |
error_msg = f"β Error generating speech: {str(e)}"
|
408 |
+
print(error_msg)
|
409 |
return None, error_msg
|
410 |
|
411 |
def clear_voice_profile():
|
|
|
421 |
voice_ready = (not use_cloned) or (tts_system.user_speaker_embeddings is not None)
|
422 |
return gr.update(interactive=text_ready and voice_ready)
|
423 |
|
424 |
+
# Create enhanced Gradio interface
|
425 |
with gr.Blocks(
|
426 |
+
title="π€ Enhanced Voice Cloning TTS System",
|
427 |
theme=gr.themes.Soft(),
|
428 |
css="""
|
429 |
.gradio-container {
|
430 |
+
max-width: 1200px !important;
|
431 |
margin: auto !important;
|
432 |
}
|
433 |
.header {
|
434 |
text-align: center;
|
435 |
margin-bottom: 30px;
|
436 |
+
padding: 25px;
|
437 |
background: linear-gradient(135deg, #667eea 0%, #764ba2 100%);
|
438 |
border-radius: 15px;
|
439 |
color: white;
|
440 |
+
box-shadow: 0 8px 25px rgba(0,0,0,0.15);
|
441 |
}
|
442 |
.step-box {
|
443 |
border: 2px solid #e1e5e9;
|
|
|
454 |
margin: 20px 0;
|
455 |
border-left: 5px solid #ff6b6b;
|
456 |
}
|
457 |
+
.improvement-box {
|
458 |
+
background: linear-gradient(135deg, #a8edea 0%, #fed6e3 100%);
|
459 |
+
border-radius: 12px;
|
460 |
+
padding: 20px;
|
461 |
+
margin: 20px 0;
|
462 |
+
border-left: 5px solid #00d2ff;
|
463 |
+
}
|
464 |
"""
|
465 |
) as demo:
|
466 |
|
467 |
gr.HTML("""
|
468 |
<div class="header">
|
469 |
+
<h1>π€ Enhanced AI Voice Cloning TTS System</h1>
|
470 |
+
<p>π Advanced neural voice analysis with Wav2Vec2 + SpeechT5</p>
|
471 |
+
<p>β¨ Upload your voice and generate speech that sounds more like you!</p>
|
472 |
</div>
|
473 |
""")
|
474 |
|
475 |
with gr.Row():
|
476 |
with gr.Column(scale=1):
|
477 |
+
gr.HTML('<div class="step-box"><h3>ποΈ Step 1: Upload Your Voice Sample</h3><p>Record 10-30 seconds of clear, natural speech for best results</p></div>')
|
478 |
|
479 |
voice_upload = gr.Audio(
|
480 |
+
label="π€ Voice Sample (Clear English Speech)",
|
481 |
type="filepath",
|
482 |
sources=["upload", "microphone"],
|
483 |
format="wav"
|
484 |
)
|
485 |
|
486 |
upload_status = gr.Textbox(
|
487 |
+
label="π Advanced Voice Analysis Status",
|
488 |
interactive=False,
|
489 |
+
value="β³ Please upload an audio file to extract your unique voice profile using advanced neural analysis.",
|
490 |
+
lines=3
|
491 |
)
|
492 |
|
493 |
clear_btn = gr.Button("ποΈ Clear Voice Profile", variant="secondary", size="sm")
|
494 |
|
495 |
with gr.Column(scale=1):
|
496 |
+
gr.HTML('<div class="step-box"><h3>βοΈ Step 2: Enter Your Text</h3><p>Type what you want to hear in your cloned voice</p></div>')
|
497 |
|
498 |
text_input = gr.Textbox(
|
499 |
label="π Text to Convert (Max 500 characters)",
|
500 |
placeholder="Enter the text you want to convert to speech using your cloned voice...",
|
501 |
+
lines=6,
|
502 |
+
max_lines=10
|
503 |
)
|
504 |
|
505 |
use_cloned_voice = gr.Checkbox(
|
506 |
+
label="π Use My Cloned Voice (Enhanced)",
|
507 |
value=True,
|
508 |
interactive=False,
|
509 |
+
info="Uncheck to use default voice for comparison"
|
510 |
)
|
511 |
|
512 |
generate_btn = gr.Button(
|
513 |
+
"π΅ Generate Speech with AI Voice Cloning",
|
514 |
variant="primary",
|
515 |
interactive=False,
|
516 |
size="lg"
|
|
|
532 |
lines=2
|
533 |
)
|
534 |
|
535 |
+
# Enhanced tips section
|
536 |
+
gr.HTML("""
|
537 |
+
<div class="improvement-box">
|
538 |
+
<h3>π¬ Enhanced Voice Cloning Technology:</h3>
|
539 |
+
<p><strong>This improved version uses:</strong></p>
|
540 |
+
<ul>
|
541 |
+
<li><strong>Wav2Vec2 Neural Networks:</strong> Advanced deep learning for better voice feature extraction</li>
|
542 |
+
<li><strong>Multi-Speaker Analysis:</strong> Compares your voice against multiple reference speakers</li>
|
543 |
+
<li><strong>Enhanced Acoustic Features:</strong> 60+ voice characteristics including pitch, formants, and spectral features</li>
|
544 |
+
<li><strong>Ensemble Embeddings:</strong> Combines multiple speaker models for more accurate voice representation</li>
|
545 |
+
</ul>
|
546 |
+
</div>
|
547 |
+
""")
|
548 |
+
|
549 |
gr.HTML("""
|
550 |
<div class="tips-box">
|
551 |
+
<h3>π‘ Pro Tips for Maximum Voice Similarity:</h3>
|
552 |
<div style="display: grid; grid-template-columns: 1fr 1fr; gap: 20px; margin-top: 15px;">
|
553 |
<div>
|
554 |
+
<h4>π€ Recording Best Practices:</h4>
|
555 |
<ul>
|
556 |
+
<li><strong>Duration:</strong> 15-30 seconds is optimal</li>
|
557 |
+
<li><strong>Content:</strong> Read naturally, include varied sentences</li>
|
558 |
+
<li><strong>Environment:</strong> Quiet room, minimal echo</li>
|
559 |
+
<li><strong>Quality:</strong> Use good microphone if possible</li>
|
560 |
+
<li><strong>Speaking:</strong> Natural pace, clear pronunciation</li>
|
561 |
</ul>
|
562 |
</div>
|
563 |
<div>
|
564 |
+
<h4>π Text Generation Tips:</h4>
|
565 |
<ul>
|
566 |
+
<li><strong>Language:</strong> English works best</li>
|
567 |
+
<li><strong>Style:</strong> Match your natural speaking style</li>
|
568 |
+
<li><strong>Length:</strong> Shorter texts often sound better</li>
|
569 |
+
<li><strong>Punctuation:</strong> Helps with natural intonation</li>
|
570 |
+
<li><strong>Testing:</strong> Try different texts to compare results</li>
|
571 |
</ul>
|
572 |
</div>
|
573 |
</div>
|
574 |
+
<div style="margin-top: 20px; padding: 15px; background: rgba(255,255,255,0.8); border-radius: 8px;">
|
575 |
+
<strong>π§ How the Enhanced System Works:</strong>
|
576 |
+
<br>1. <strong>Neural Analysis:</strong> Wav2Vec2 extracts 768-dimensional voice features
|
577 |
+
<br>2. <strong>Speaker Matching:</strong> Finds similar voices in a large speaker database
|
578 |
+
<br>3. <strong>Feature Fusion:</strong> Combines 60+ acoustic characteristics (pitch, formants, spectral features)
|
579 |
+
<br>4. <strong>Voice Synthesis:</strong> SpeechT5 generates speech using your personalized voice embedding
|
580 |
</div>
|
581 |
</div>
|
582 |
""")
|
583 |
|
584 |
+
# Event handlers
|
585 |
voice_upload.change(
|
586 |
fn=process_voice_upload,
|
587 |
inputs=[voice_upload],
|
|
|
611 |
outputs=[upload_status, use_cloned_voice, generate_btn]
|
612 |
)
|
613 |
|
614 |
+
# Launch configuration
|
615 |
if __name__ == "__main__":
|
616 |
+
print("π Starting Enhanced Voice Cloning TTS System...")
|
617 |
demo.launch(
|
618 |
+
share=True
|
619 |
)
|