Upload folder using huggingface_hub

.gitattributes

@@ -33,3 +33,6 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+chiahuahua-long.mp3 filter=lfs diff=lfs merge=lfs -text
+dachshund-long.mp3 filter=lfs diff=lfs merge=lfs -text
+golden-retriever-long.mp3 filter=lfs diff=lfs merge=lfs -text

README.md

@@ -1,12 +1,15 @@
 ---
-title: Talk To Your Dog
-emoji: 🌍
-colorFrom: red
-colorTo: indigo
+title: PuppyChat
+emoji: 🐶
+colorFrom: purple
+colorTo: red
 sdk: gradio
 sdk_version: 5.23.2
 app_file: app.py
 pinned: false
+license: mit
+short_description: Talk to your dog!
+tags: [webrtc, websocket, gradio, secret|TURN_TOKEN_ID, secret|TURN_API_TOKEN]
 ---
 
-Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference
+Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference

app.py

@@ -0,0 +1,152 @@
+import json
+import os
+from pathlib import Path
+from typing import Literal
+import gradio as gr
+import numpy as np
+from dotenv import load_dotenv
+from fastapi import FastAPI, HTTPException
+from fastapi.responses import HTMLResponse, StreamingResponse, FileResponse
+from fastrtc import (
+    AdditionalOutputs,
+    ReplyOnPause,
+    Stream,
+    get_stt_model,
+    get_twilio_turn_credentials,
+)
+import soundfile as sf
+from gradio.utils import get_space
+from pydantic import BaseModel
+import httpx
+
+load_dotenv()
+
+
+curr_dir = Path(__file__).parent
+
+stt_model = get_stt_model()
+
+conversations = {}
+
+barks = {
+    "chiahuahua-short": sf.read(curr_dir / "chiahuahua-short.mp3"),
+    "chiahuahua-long": sf.read(curr_dir / "chiahuahua-long.mp3"),
+    "dachshund-short": sf.read(curr_dir / "dachshund-short.mp3"),
+    "dachshund-long": sf.read(curr_dir / "dachshund-long.mp3"),
+    "golden-retriever-short": sf.read(curr_dir / "golden-retriever-short.mp3"),
+    "golden-retriever-long": sf.read(curr_dir / "golden-retriever-long.mp3"),
+}
+for k, v in barks.items():
+    # Convert to mono if stereo by averaging channels, otherwise just use as is
+    audio_data = v[0]
+    if len(audio_data.shape) > 1 and audio_data.shape[1] > 1:
+        # Convert stereo to mono by averaging channels
+        audio_data = np.mean(audio_data, axis=1)
+    barks[k] = (v[1], audio_data.astype(np.float32))
+
+
+def response(
+    audio: tuple[int, np.ndarray],
+    breed: Literal["chiahuahua", "dachshund", "golden-retriever"],
+):
+    response = []
+    prompt = stt_model.stt(audio)
+    response.append({"role": "user", "content": prompt})
+    length = "long" if len(prompt.split(" ")) > 10 else "short"
+    file_name = f"{breed}-{length}"
+    response.append(
+        {
+            "role": "assistant",
+            "content": f"/files/{file_name}.mp3",
+        }
+    )
+    audio_ = barks[file_name]
+    yield audio_, AdditionalOutputs(response)
+
+
+stream = Stream(
+    modality="audio",
+    mode="send-receive",
+    handler=ReplyOnPause(response),
+    additional_outputs_handler=lambda a, b: b,
+    additional_inputs=[
+        gr.Dropdown(choices=["chiahuahua", "dachshund", "golden-retriever"])
+    ],
+    additional_outputs=[gr.JSON()],
+    rtc_configuration=get_twilio_turn_credentials() if get_space() else None,
+    concurrency_limit=5 if get_space() else None,
+    time_limit=90 if get_space() else None,
+)
+
+
+class InputData(BaseModel):
+    webrtc_id: str
+    breed: Literal["chiahuahua", "dachshund", "golden-retriever"]
+
+
+client = httpx.AsyncClient()
+app = FastAPI()
+stream.mount(app)
+
+
+@app.get("/")
+async def _():
+    turn_key_id = os.environ.get("TURN_TOKEN_ID")
+    turn_key_api_token = os.environ.get("TURN_API_TOKEN")
+    ttl = 600
+
+    response = await client.post(
+        f"https://rtc.live.cloudflare.com/v1/turn/keys/{turn_key_id}/credentials/generate-ice-servers",
+        headers={
+            "Authorization": f"Bearer {turn_key_api_token}",
+            "Content-Type": "application/json",
+        },
+        json={"ttl": ttl},
+    )
+    if response.is_success:
+        rtc_config = response.json()
+    else:
+        raise Exception(
+            f"Failed to get TURN credentials: {response.status_code} {response.text}"
+        )
+    html_content = (curr_dir / "index.html").read_text()
+    html_content = html_content.replace("__RTC_CONFIGURATION__", json.dumps(rtc_config))
+    return HTMLResponse(content=html_content, status_code=200)
+
+
+@app.get("/files/{file_name}")
+async def _(file_name: str):
+    print("file_name", file_name)
+    if Path(file_name).name.replace(".mp3", "") not in barks:
+        raise HTTPException(status_code=404, detail="File not found")
+    return FileResponse(curr_dir / file_name)
+
+
+@app.post("/input_hook")
+async def _(body: InputData):
+    stream.set_input(body.webrtc_id, body.breed)
+    return {"status": "ok"}
+
+
+@app.get("/outputs")
+def _(webrtc_id: str):
+    async def output_stream():
+        async for output in stream.output_stream(webrtc_id):
+            messages = output.args[0]
+            for msg in messages:
+                yield f"event: output\ndata: {json.dumps(msg)}\n\n"
+
+    return StreamingResponse(output_stream(), media_type="text/event-stream")
+
+
+if __name__ == "__main__":
+    import os
+
+    if (mode := os.getenv("MODE")) == "UI":
+        stream.ui.launch(server_port=7860)
+    elif mode == "PHONE":
+        stream.fastphone(host="0.0.0.0", port=7860)
+    else:
+        import uvicorn
+
+        uvicorn.run(app, host="0.0.0.0" if get_space() else "127.0.0.1", port=7860)

chiahuahua-long.mp3

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2e74a6f0da8dcc2882b3c12bde17436231fc727128925b745d5bef6d41918a5a
+size 131711

dachshund-long.mp3

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d412dd964d6d5590849b91782623ff014a527918b091366145cbc59eda9f898b
+size 158448

golden-retriever-long.mp3

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:cf5d51c12027f2c05c73a6c1c6a27dfd32b461c984885c4c48dcbea7ba130662
+size 133846

index.html

@@ -0,0 +1,811 @@
+<!DOCTYPE html>
+<html lang="en">
+
+<head>
+    <meta charset="UTF-8">
+    <meta name="viewport" content="width=device-width, initial-scale=1.0">
+    <title>PuppyChat 🐶</title>
+    <link href="https://fonts.googleapis.com/css2?family=Nunito:wght@400;600;700&display=swap" rel="stylesheet">
+    <style>
+        :root {
+            --bg-primary: #f8e8b9;
+            --bg-secondary: #fdf6e3;
+            --accent-green: #7cc9a9;
+            --accent-brown: #8b6a4a;
+            --accent-yellow: #f7cf77;
+            --text-primary: #5a4031;
+            --text-secondary: #8b6a4a;
+        }
+
+        * {
+            box-sizing: border-box;
+            margin: 0;
+            padding: 0;
+        }
+
+        body {
+            font-family: 'Nunito', -apple-system, BlinkMacSystemFont, "Segoe UI", Roboto, sans-serif;
+            background-color: var(--bg-primary);
+            color: var(--text-primary);
+            margin: 0;
+            padding: 20px;
+            min-height: 100vh;
+            background-image: url('https://i.imgur.com/8NIgzrU.png');
+            background-size: cover;
+            background-position: center;
+            background-attachment: fixed;
+            overflow-x: hidden;
+        }
+
+        .container {
+            max-width: 800px;
+            margin: 0 auto;
+            height: 100%;
+            position: relative;
+        }
+
+        .app-container {
+            background-color: var(--bg-secondary);
+            border-radius: 25px;
+            box-shadow: 0 8px 32px rgba(138, 106, 74, 0.15);
+            padding: 25px;
+            margin-top: 30px;
+            position: relative;
+            overflow: hidden;
+            border: 6px solid var(--accent-brown);
+        }
+
+        .app-container:before {
+            content: '';
+            position: absolute;
+            top: 0;
+            left: 0;
+            right: 0;
+            height: 15px;
+            background: var(--accent-green);
+            border-radius: 10px 10px 0 0;
+        }
+
+        .logo {
+            text-align: center;
+            margin-bottom: 20px;
+            position: relative;
+        }
+
+        .logo h1 {
+            font-size: 2.5em;
+            color: var(--accent-brown);
+            text-shadow: 2px 2px 0 var(--accent-yellow);
+            margin-bottom: 5px;
+            letter-spacing: 1px;
+        }
+
+        .logo img {
+            width: 250px;
+            margin-bottom: 10px;
+        }
+
+        .logo-leaves {
+            position: absolute;
+            width: 100px;
+            height: 100px;
+            background-image: url('https://i.imgur.com/GdOS5Vv.png');
+            background-size: contain;
+            background-repeat: no-repeat;
+            z-index: -1;
+        }
+
+        .logo-leaves.left {
+            top: -20px;
+            left: 40px;
+            transform: rotate(-30deg);
+        }
+
+        .logo-leaves.right {
+            top: -20px;
+            right: 40px;
+            transform: rotate(30deg) scaleX(-1);
+        }
+
+        .breed-selector {
+            background-color: #fff;
+            padding: 20px;
+            border-radius: 20px;
+            margin-bottom: 20px;
+            box-shadow: 0 4px 15px rgba(138, 106, 74, 0.1);
+            border: 3px solid var(--accent-yellow);
+        }
+
+        .breed-selector h3 {
+            margin-bottom: 15px;
+            color: var(--text-secondary);
+            font-size: 1.2em;
+        }
+
+        .breed-dropdown {
+            width: 100%;
+            padding: 12px 15px;
+            border-radius: 15px;
+            border: 2px solid var(--accent-green);
+            font-family: 'Nunito', sans-serif;
+            font-size: 16px;
+            color: var(--text-primary);
+            background-color: #fafafa;
+            cursor: pointer;
+            transition: all 0.3s ease;
+        }
+
+        .breed-dropdown:focus {
+            outline: none;
+            box-shadow: 0 0 0 3px rgba(124, 201, 169, 0.3);
+            border-color: var(--accent-green);
+        }
+
+        .chat-container {
+            background: #fff;
+            border-radius: 20px;
+            box-shadow: 0 4px 15px rgba(138, 106, 74, 0.1);
+            padding: 20px;
+            height: 400px;
+            display: flex;
+            flex-direction: column;
+            border: 3px solid var(--accent-green);
+            position: relative;
+        }
+
+        .chat-container:after {
+            content: '';
+            position: absolute;
+            bottom: -10px;
+            right: -10px;
+            width: 80px;
+            height: 80px;
+            background-image: url('https://i.imgur.com/JbkTVa6.png');
+            background-size: contain;
+            background-repeat: no-repeat;
+            opacity: 0.7;
+        }
+
+        .chat-messages {
+            flex-grow: 1;
+            overflow-y: auto;
+            margin-bottom: 10px;
+            padding: 10px;
+            scrollbar-width: thin;
+            scrollbar-color: var(--accent-green) #f0f0f0;
+        }
+
+        .chat-messages::-webkit-scrollbar {
+            width: 8px;
+        }
+
+        .chat-messages::-webkit-scrollbar-track {
+            background: #f0f0f0;
+            border-radius: 10px;
+        }
+
+        .chat-messages::-webkit-scrollbar-thumb {
+            background-color: var(--accent-green);
+            border-radius: 10px;
+        }
+
+        .message {
+            margin-bottom: 15px;
+            padding: 12px 16px;
+            border-radius: 18px;
+            font-size: 15px;
+            line-height: 1.5;
+            position: relative;
+            max-width: 80%;
+            animation: pop-in 0.3s ease-out forwards;
+        }
+
+        @keyframes pop-in {
+            0% {
+                opacity: 0;
+                transform: scale(0.8) translateY(10px);
+            }
+
+            100% {
+                opacity: 1;
+                transform: scale(1) translateY(0);
+            }
+        }
+
+        .message.user {
+            background-color: var(--accent-yellow);
+            align-self: flex-end;
+            margin-left: auto;
+            border-bottom-right-radius: 4px;
+            color: var(--text-primary);
+        }
+
+        .message.assistant {
+            background-color: var(--accent-green);
+            align-self: flex-start;
+            margin-right: auto;
+            border-bottom-left-radius: 4px;
+            color: #fff;
+        }
+
+        .message.user::after {
+            content: '';
+            position: absolute;
+            bottom: 0;
+            right: -10px;
+            width: 20px;
+            height: 20px;
+            background-color: var(--accent-yellow);
+            clip-path: polygon(0 0, 0% 100%, 100% 100%);
+        }
+
+        .message.assistant::after {
+            content: '';
+            position: absolute;
+            bottom: 0;
+            left: -10px;
+            width: 20px;
+            height: 20px;
+            background-color: var(--accent-green);
+            clip-path: polygon(100% 0, 0% 100%, 100% 100%);
+        }
+
+        .assistant audio {
+            margin-top: 8px;
+            width: 100%;
+            border-radius: 12px;
+            background-color: rgba(255, 255, 255, 0.3);
+        }
+
+        .typing-indicator {
+            padding: 12px;
+            background-color: var(--accent-green);
+            border-radius: 18px;
+            margin-bottom: 10px;
+            display: none;
+            width: fit-content;
+            align-self: flex-start;
+            border-bottom-left-radius: 4px;
+            position: relative;
+        }
+
+        .typing-indicator::after {
+            content: '';
+            position: absolute;
+            bottom: 0;
+            left: -10px;
+            width: 20px;
+            height: 20px;
+            background-color: var(--accent-green);
+            clip-path: polygon(100% 0, 0% 100%, 100% 100%);
+        }
+
+        .dots {
+            display: inline-flex;
+            gap: 4px;
+        }
+
+        .dot {
+            width: 8px;
+            height: 8px;
+            background-color: #fff;
+            border-radius: 50%;
+            animation: bounce 1.5s infinite;
+            opacity: 0.8;
+        }
+
+        .dot:nth-child(2) {
+            animation-delay: 0.2s;
+        }
+
+        .dot:nth-child(3) {
+            animation-delay: 0.4s;
+        }
+
+        @keyframes bounce {
+
+            0%,
+            100% {
+                transform: translateY(0);
+            }
+
+            50% {
+                transform: translateY(-6px);
+            }
+        }
+
+        .controls {
+            text-align: center;
+            margin-top: 25px;
+            position: relative;
+        }
+
+        button {
+            background-color: var(--accent-green);
+            color: white;
+            border: none;
+            padding: 14px 30px;
+            font-family: inherit;
+            font-size: 16px;
+            font-weight: 700;
+            cursor: pointer;
+            transition: all 0.3s;
+            border-radius: 50px;
+            box-shadow: 0 4px 0 var(--text-secondary);
+            position: relative;
+            overflow: hidden;
+        }
+
+        button:hover {
+            background-color: #6ab897;
+            transform: translateY(-2px);
+        }
+
+        button:active {
+            transform: translateY(2px);
+            box-shadow: 0 2px 0 var(--text-secondary);
+        }
+
+        button::before {
+            content: '';
+            position: absolute;
+            top: 0;
+            left: -100%;
+            width: 100%;
+            height: 100%;
+            background: linear-gradient(90deg, transparent, rgba(255, 255, 255, 0.2), transparent);
+            transition: 0.5s;
+        }
+
+        button:hover::before {
+            left: 100%;
+        }
+
+        #audio-output {
+            display: none;
+        }
+
+        .icon-with-spinner {
+            display: flex;
+            align-items: center;
+            justify-content: center;
+            gap: 12px;
+            min-width: 180px;
+        }
+
+        .spinner {
+            width: 20px;
+            height: 20px;
+            border: 2px solid #ffffff;
+            border-top-color: transparent;
+            border-radius: 50%;
+            animation: spin 1s linear infinite;
+            flex-shrink: 0;
+        }
+
+        @keyframes spin {
+            to {
+                transform: rotate(360deg);
+            }
+        }
+
+        .pulse-container {
+            display: flex;
+            align-items: center;
+            justify-content: center;
+            gap: 12px;
+            min-width: 180px;
+        }
+
+        .pulse-circle {
+            width: 20px;
+            height: 20px;
+            border-radius: 50%;
+            background-color: #ffffff;
+            opacity: 0.7;
+            flex-shrink: 0;
+            transform: translateX(-0%) scale(var(--audio-level, 1));
+            transition: transform 0.1s ease;
+        }
+
+        .toast {
+            position: fixed;
+            top: 20px;
+            left: 50%;
+            transform: translateX(-50%);
+            padding: 16px 24px;
+            border-radius: 50px;
+            font-size: 14px;
+            z-index: 1000;
+            display: none;
+            box-shadow: 0 4px 15px rgba(0, 0, 0, 0.2);
+        }
+
+        .toast.error {
+            background-color: #ff6b6b;
+            color: white;
+        }
+
+        .toast.warning {
+            background-color: #ffd166;
+            color: var(--text-primary);
+        }
+
+        /* Animal Crossing cloud decoration */
+        .cloud {
+            position: absolute;
+            background: #fff;
+            border-radius: 50%;
+            opacity: 0.7;
+            filter: blur(5px);
+        }
+
+        .cloud-1 {
+            width: 100px;
+            height: 60px;
+            top: 5%;
+            right: 10%;
+            animation: float 20s ease-in-out infinite;
+        }
+
+        .cloud-2 {
+            width: 70px;
+            height: 40px;
+            top: 15%;
+            left: 5%;
+            animation: float 15s ease-in-out infinite 2s;
+        }
+
+        @keyframes float {
+
+            0%,
+            100% {
+                transform: translateY(0) translateX(0);
+            }
+
+            50% {
+                transform: translateY(-20px) translateX(20px);
+            }
+        }
+
+        /* Responsive styles */
+        @media (max-width: 768px) {
+            .container {
+                padding: 10px;
+            }
+
+            .app-container {
+                padding: 15px;
+            }
+
+            .logo h1 {
+                font-size: 2em;
+            }
+
+            .logo img {
+                width: 180px;
+            }
+
+            .message {
+                max-width: 90%;
+            }
+        }
+    </style>
+</head>
+
+<body>
+    <div id="error-toast" class="toast"></div>
+    <div class="cloud cloud-1"></div>
+    <div class="cloud cloud-2"></div>
+
+    <div class="container">
+        <div class="logo">
+            <div class="logo-leaves left"></div>
+            <div class="logo-leaves right"></div>
+            <h1>PuppyChat 🐶</h1>
+        </div>
+
+        <div class="app-container">
+            <div class="breed-selector">
+                <h3>Choose your furry friend:</h3>
+                <select id="breed-dropdown" class="breed-dropdown">
+                    <option value="chiahuahua">Chiahuahua</option>
+                    <option value="dachshund">Dachshund</option>
+                    <option value="golden-retriever">Golden Retriever</option>
+                </select>
+            </div>
+
+            <div class="chat-container">
+                <div class="chat-messages" id="chat-messages"></div>
+                <div class="typing-indicator" id="typing-indicator">
+                    <div class="dots">
+                        <div class="dot"></div>
+                        <div class="dot"></div>
+                        <div class="dot"></div>
+                    </div>
+                </div>
+            </div>
+        </div>
+
+        <div class="controls">
+            <button id="start-button">Start Conversation</button>
+        </div>
+    </div>
+    <audio id="audio-output"></audio>
+
+    <script>
+        let peerConnection;
+        let webrtc_id;
+        const startButton = document.getElementById('start-button');
+        const chatMessages = document.getElementById('chat-messages');
+        const breedDropdown = document.getElementById('breed-dropdown');
+
+        let audioLevel = 0;
+        let animationFrame;
+        let audioContext, analyser, audioSource;
+        let eventSource;
+
+        function updateButtonState() {
+            const button = document.getElementById('start-button');
+            if (peerConnection && (peerConnection.connectionState === 'connecting' || peerConnection.connectionState === 'new')) {
+                button.innerHTML = `
+                    <div class="icon-with-spinner">
+                        <div class="spinner"></div>
+                        <span>Connecting...</span>
+                    </div>
+                `;
+            } else if (peerConnection && peerConnection.connectionState === 'connected') {
+                button.innerHTML = `
+                    <div class="pulse-container">
+                        <div class="pulse-circle"></div>
+                        <span>Stop Conversation</span>
+                    </div>
+                `;
+            } else {
+                button.innerHTML = 'Start Conversation';
+            }
+        }
+
+        function setupAudioVisualization(stream) {
+            audioContext = new (window.AudioContext || window.webkitAudioContext)();
+            analyser = audioContext.createAnalyser();
+            audioSource = audioContext.createMediaStreamSource(stream);
+            audioSource.connect(analyser);
+            analyser.fftSize = 64;
+            const dataArray = new Uint8Array(analyser.frequencyBinCount);
+
+            function updateAudioLevel() {
+                analyser.getByteFrequencyData(dataArray);
+                const average = Array.from(dataArray).reduce((a, b) => a + b, 0) / dataArray.length;
+                audioLevel = average / 255;
+
+                const pulseCircle = document.querySelector('.pulse-circle');
+                if (pulseCircle) {
+                    pulseCircle.style.setProperty('--audio-level', 1 + audioLevel);
+                }
+
+                animationFrame = requestAnimationFrame(updateAudioLevel);
+            }
+            updateAudioLevel();
+        }
+
+        function showError(message) {
+            const toast = document.getElementById('error-toast');
+            toast.textContent = message;
+            toast.className = 'toast error';
+            toast.style.display = 'block';
+
+            // Hide toast after 5 seconds
+            setTimeout(() => {
+                toast.style.display = 'none';
+            }, 5000);
+        }
+
+        function handleMessage(event) {
+            const eventJson = JSON.parse(event.data);
+            const typingIndicator = document.getElementById('typing-indicator');
+
+            if (eventJson.type === "error") {
+                showError(eventJson.message);
+            } else if (eventJson.type === "send_input") {
+                fetch('/input_hook', {
+                    method: 'POST',
+                    headers: {
+                        'Content-Type': 'application/json',
+                    },
+                    body: JSON.stringify({
+                        webrtc_id: webrtc_id,
+                        breed: breedDropdown.value
+                    })
+                });
+            } else if (eventJson.type === "log") {
+                if (eventJson.data === "pause_detected") {
+                    typingIndicator.style.display = 'block';
+                    chatMessages.scrollTop = chatMessages.scrollHeight;
+                } else if (eventJson.data === "response_starting") {
+                    typingIndicator.style.display = 'none';
+                }
+            }
+        }
+
+        async function setupWebRTC() {
+            const config = __RTC_CONFIGURATION__;
+            peerConnection = new RTCPeerConnection(config);
+
+            const timeoutId = setTimeout(() => {
+                const toast = document.getElementById('error-toast');
+                toast.textContent = "Connection is taking longer than usual. Are you on a VPN?";
+                toast.className = 'toast warning';
+                toast.style.display = 'block';
+
+                // Hide warning after 5 seconds
+                setTimeout(() => {
+                    toast.style.display = 'none';
+                }, 5000);
+            }, 5000);
+
+            try {
+                const stream = await navigator.mediaDevices.getUserMedia({
+                    audio: true
+                });
+
+                setupAudioVisualization(stream);
+
+                stream.getTracks().forEach(track => {
+                    peerConnection.addTrack(track, stream);
+                });
+
+                // Add track event listener to handle incoming audio
+                const audioOutput = document.getElementById('audio-output');
+                peerConnection.addEventListener('track', (evt) => {
+                    if (audioOutput && audioOutput.srcObject !== evt.streams[0]) {
+                        audioOutput.srcObject = evt.streams[0];
+                        audioOutput.play();
+                        console.log('Track received:', evt.track.kind);
+                    }
+                });
+
+                const dataChannel = peerConnection.createDataChannel('text');
+                dataChannel.onmessage = handleMessage;
+
+                const offer = await peerConnection.createOffer();
+                await peerConnection.setLocalDescription(offer);
+
+                peerConnection.onicecandidate = ({ candidate }) => {
+                    if (candidate) {
+                        console.debug("Sending ICE candidate", candidate);
+                        fetch('/webrtc/offer', {
+                            method: 'POST',
+                            headers: { 'Content-Type': 'application/json' },
+                            body: JSON.stringify({
+                                candidate: candidate.toJSON(),
+                                webrtc_id: webrtc_id,
+                                type: "ice-candidate",
+                            })
+                        })
+                    }
+                };
+
+                peerConnection.addEventListener('connectionstatechange', () => {
+                    console.log('connectionstatechange', peerConnection.connectionState);
+                    if (peerConnection.connectionState === 'connected') {
+                        clearTimeout(timeoutId);
+                        const toast = document.getElementById('error-toast');
+                        toast.style.display = 'none';
+                    }
+                    updateButtonState();
+                });
+
+                webrtc_id = Math.random().toString(36).substring(7);
+
+                const response = await fetch('/webrtc/offer', {
+                    method: 'POST',
+                    headers: { 'Content-Type': 'application/json' },
+                    body: JSON.stringify({
+                        sdp: peerConnection.localDescription.sdp,
+                        type: peerConnection.localDescription.type,
+                        webrtc_id: webrtc_id
+                    })
+                });
+
+                const serverResponse = await response.json();
+
+                if (serverResponse.status === 'failed') {
+                    showError(serverResponse.meta.error === 'concurrency_limit_reached'
+                        ? `Too many connections. Maximum limit is ${serverResponse.meta.limit}`
+                        : serverResponse.meta.error);
+                    stop();
+                    return;
+                }
+
+                await peerConnection.setRemoteDescription(serverResponse);
+
+                eventSource = new EventSource('/outputs?webrtc_id=' + webrtc_id);
+                eventSource.addEventListener("output", (event) => {
+                    const eventJson = JSON.parse(event.data);
+                    console.log(eventJson);
+
+                    // Add the message based on role
+                    addMessage(eventJson.role, eventJson.content);
+                });
+            } catch (err) {
+                clearTimeout(timeoutId);
+                console.error('Error setting up WebRTC:', err);
+                showError('Failed to establish connection. Please try again.');
+                stop();
+            }
+        }
+
+        async function addMessage(role, content) {
+            const messageDiv = document.createElement('div');
+            messageDiv.classList.add('message', role);
+
+            // Hide typing indicator when a message is added
+            document.getElementById('typing-indicator').style.display = 'none';
+
+            if (role === 'user') {
+                // Text content for user messages from STT
+                messageDiv.textContent = content;
+            } else if (role === 'assistant') {
+                // For assistant, create audio element with file path
+                if (content.startsWith('/files/')) {
+                    const audio = document.createElement('audio');
+                    audio.controls = true;
+                    audio.src = content;
+                    messageDiv.textContent = "Woof woof! 🐶";
+                    messageDiv.appendChild(audio);
+                } else {
+                    messageDiv.textContent = content;
+                }
+            }
+
+            chatMessages.appendChild(messageDiv);
+            chatMessages.scrollTop = chatMessages.scrollHeight;
+        }
+
+        function stop() {
+            if (eventSource) {
+                eventSource.close();
+                eventSource = null;
+            }
+
+            if (animationFrame) {
+                cancelAnimationFrame(animationFrame);
+            }
+            if (audioContext) {
+                audioContext.close();
+                audioContext = null;
+                analyser = null;
+                audioSource = null;
+            }
+            if (peerConnection) {
+                if (peerConnection.getTransceivers) {
+                    peerConnection.getTransceivers().forEach(transceiver => {
+                        if (transceiver.stop) {
+                            transceiver.stop();
+                        }
+                    });
+                }
+
+                if (peerConnection.getSenders) {
+                    peerConnection.getSenders().forEach(sender => {
+                        if (sender.track && sender.track.stop) sender.track.stop();
+                    });
+                }
+                peerConnection.close();
+            }
+            updateButtonState();
+            audioLevel = 0;
+        }
+
+        startButton.addEventListener('click', () => {
+            if (!peerConnection || peerConnection.connectionState !== 'connected') {
+                setupWebRTC();
+            } else {
+                stop();
+            }
+        });
+    </script>
+</body>
+
+</html>

requirements.txt

@@ -0,0 +1,3 @@
+fastrtc[vad]
+soundfile
+httpx