app.py

from transformers import AutoModelForSpeechSeq2Seq, AutoProcessor, pipeline
from transformers.utils import is_flash_attn_2_available
import torch
import gradio as gr
import matplotlib.pyplot as plt
import time
import os

BATCH_SIZE = 16
TOKEN = os.environ.get("HF_TOKEN", None)

device = "cuda:0" if torch.cuda.is_available() else "cpu"
torch_dtype = torch.float16 if torch.cuda.is_available() else torch.float32
use_flash_attention_2 = is_flash_attn_2_available()

model = AutoModelForSpeechSeq2Seq.from_pretrained(
    "openai/whisper-large-v2", torch_dtype=torch_dtype, low_cpu_mem_usage=True, use_safetensors=True, use_flash_attention_2=use_flash_attention_2
)
distilled_model = AutoModelForSpeechSeq2Seq.from_pretrained(
    "sanchit-gandhi/distil-large-v2-private", torch_dtype=torch_dtype, low_cpu_mem_usage=True, use_safetensors=True, use_flash_attention_2=use_flash_attention_2, token=TOKEN
)

if not use_flash_attention_2:
    model = model.bettertransformer()
    distilled_model = distilled_model.bettertransformer()

processor = AutoProcessor.from_pretrained("openai/whisper-tiny.en")

model.to(device)
distilled_model.to(device)

pipe = pipeline(
    "automatic-speech-recognition",
    model=model,
    tokenizer=processor.tokenizer,
    feature_extractor=processor.feature_extractor,
    max_new_tokens=128,
    chunk_length_s=30,
    torch_dtype=torch_dtype,
    device=device,
    language="en",
    task="transcribe",
)
pipe_forward = pipe._forward

distil_pipe = pipeline(
    "automatic-speech-recognition",
    model=distilled_model,
    tokenizer=processor.tokenizer,
    feature_extractor=processor.feature_extractor,
    max_new_tokens=128,
    chunk_length_s=15,
    torch_dtype=torch_dtype,
    device=device,
    language="en",
    task="transcribe",
)
distil_pipe_forward = distil_pipe._forward

def transcribe(inputs):
    if inputs is None:
        raise gr.Error("No audio file submitted! Please record or upload an audio file before submitting your request.")

    def _forward_distil_time(*args, **kwargs):
        global distil_runtime
        start_time = time.time()
        result = distil_pipe_forward(*args, **kwargs)
        distil_runtime = time.time() - start_time
        return result

    distil_pipe._forward = _forward_distil_time
    distil_text = distil_pipe(inputs, batch_size=BATCH_SIZE)["text"]
    yield distil_text, distil_runtime, None, None, None

    def _forward_time(*args, **kwargs):
        global runtime
        start_time = time.time()
        result = pipe_forward(*args, **kwargs)
        runtime = time.time() - start_time
        return result

    pipe._forward = _forward_time
    text = pipe(inputs, batch_size=BATCH_SIZE)["text"]

    relative_latency = runtime / distil_runtime

    # Create figure and axis
    fig, ax = plt.subplots(figsize=(5, 5))

    # Define bar width and positions
    bar_width = 0.1
    positions = [0, 0.1]  # Adjusted positions to bring bars closer

    # Plot data
    ax.bar(positions[0], distil_runtime, bar_width, edgecolor='black')
    ax.bar(positions[1], runtime, bar_width, edgecolor='black')

    # Set title, labels, and xticks
    ax.set_ylabel('Transcription time (s)')
    ax.set_xticks(positions)
    ax.set_xticklabels(['Distil-Whisper', 'Whisper'])

    # Gridlines and other styling
    ax.grid(which='major', axis='y', linestyle='--', linewidth=0.5)

    # Use tight layout to avoid overlaps
    plt.tight_layout()

    yield distil_text, distil_runtime, text, runtime, plt

if __name__ == "__main__":
    with gr.Blocks() as demo:
        gr.HTML(
            """
                <div style="text-align: center; max-width: 700px; margin: 0 auto;">
                  <div
                    style="
                      display: inline-flex; align-items: center; gap: 0.8rem; font-size: 1.75rem;
                    "
                  >
                    <h1 style="font-weight: 900; margin-bottom: 7px; line-height: normal;">
                      Distil-Whisper VS Whisper
                    </h1>
                  </div>
                </div>
            """
        )
        gr.HTML(
            f"""
            This demo evaluates the <a href="https://huggingface.co/distil-whisper/distil-large-v2"> Distil-Whisper </a> model 
            against the <a href="https://huggingface.co/openai/whisper-large-v2"> Whisper </a> model.  
            """
        )
        audio = gr.components.Audio(source="upload", type="filepath", label="Audio file")
        button = gr.Button("Transcribe")
        plot = gr.components.Plot()
        with gr.Row():
            distil_runtime = gr.components.Textbox(label="Distil-Whisper Transcription Time (s)")
            runtime = gr.components.Textbox(label="Whisper Transcription Time (s)")
        with gr.Row():
            distil_transcription = gr.components.Textbox(label="Distil-Whisper Transcription").style(show_copy_button=True)
            transcription = gr.components.Textbox(label="Whisper Transcription").style(show_copy_button=True)

        button.click(
            fn=transcribe,
            inputs=audio,
            outputs=[distil_transcription, distil_runtime, transcription, runtime, plot],
        )

    demo.queue().launch()