Spaces:
Build error
Build error
File size: 3,644 Bytes
14cda64 cab275d 14cda64 cab275d 14cda64 cab275d f1f4016 cab275d f1f4016 cab275d 14cda64 cab275d |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 |
import gradio as gr
import torch
import spaces
from transformers import AutoModelForSpeechSeq2Seq, AutoProcessor, pipeline, AutoModelForSeq2SeqLM, AutoTokenizer
from datasets import load_dataset
from openvoice.api import ToneColorConverter
from openvoice import se_extractor
from melo.api import TTS
import pyaudio
import wave
import numpy as np
# Load ASR model and processor
torch_dtype = torch.float16
asr_model_id = "openai/whisper-large-v3"
asr_model = AutoModelForSpeechSeq2Seq.from_pretrained(asr_model_id, torch_dtype=torch_dtype, low_cpu_mem_usage=True, use_safetensors=True)
asr_processor = AutoProcessor.from_pretrained(asr_model_id)
asr_pipeline = pipeline(
"automatic-speech-recognition",
model=asr_model,
tokenizer=asr_processor.tokenizer,
feature_extractor=asr_processor.feature_extractor,
max_new_tokens=128,
chunk_length_s=30,
batch_size=16,
return_timestamps=True,
torch_dtype=torch_dtype,
device=device,
)
# Load text-to-text model and tokenizer
text_model_id = "meta-llama/Meta-Llama-3-8B"
text_model = AutoModelForSeq2SeqLM.from_pretrained(text_model_id)
text_tokenizer = AutoTokenizer.from_pretrained(text_model_id)
# Load TTS model and vocoder
tts_converter_ckpt = 'checkpoints_v2/converter'
tts_output_dir = 'outputs_v2'
os.makedirs(tts_output_dir, exist_ok=True)
tts_converter = ToneColorConverter(f'{tts_converter_ckpt}/config.json')
tts_converter.load_ckpt(f'{tts_converter_ckpt}/checkpoint.pth')
reference_speaker = 'resources/example_reference.mp3' # This is the voice you want to clone
target_se, _ = se_extractor.get_se(reference_speaker, tts_converter, vad=False)
def process_audio(input_audio):
# Perform ASR
asr_result = asr_pipeline(input_audio)["text"]
# Perform text-to-text processing
input_ids = text_tokenizer(asr_result, return_tensors="pt").input_ids.to(device)
generated_ids = text_model.generate(input_ids, max_length=512)
response_text = text_tokenizer.decode(generated_ids[0], skip_special_tokens=True)
# Perform TTS
tts_model = TTS(language='EN', device=device)
speaker_id = list(tts_model.hps.data.spk2id.values())[0]
tts_model.tts_to_file(response_text, speaker_id, f'{tts_output_dir}/tmp.wav')
save_path = f'{tts_output_dir}/output_v2.wav'
source_se = torch.load(f'checkpoints_v2/base_speakers/ses/english-american.pth', map_location=device)
tts_converter.convert(audio_src_path=f'{tts_output_dir}/tmp.wav', src_se=source_se, tgt_se=target_se, output_path=save_path, message="@MyShell")
return save_path
# Real-time audio processing
def real_time_audio_processing():
p = pyaudio.PyAudio()
stream = p.open(format=pyaudio.paInt16, channels=1, rate=16000, input=True, frames_per_buffer=1024)
frames = []
print("Listening...")
while True:
data = stream.read(1024)
frames.append(data)
audio_data = np.frombuffer(data, dtype=np.int16)
if np.max(audio_data) > 3000: # Simple VAD threshold
wf = wave.open("input_audio.wav", 'wb')
wf.setnchannels(1)
wf.setsampwidth(p.get_sample_size(pyaudio.paInt16))
wf.setframerate(16000)
wf.writeframes(b''.join(frames))
wf.close()
return "input_audio.wav"
# Gradio Interface
@spaces.GPU(duration=300)
def main():
input_audio_path = real_time_audio_processing()
if input_audio_path:
output_audio_path = process_audio(input_audio_path)
return output_audio_path
iface = gr.Interface(
fn=main,
inputs=None,
outputs=gr.Audio(type="filepath"),
live=True
)
iface.launch()
|