Added mediarecorder for web recording

Dockerfile

@@ -3,7 +3,7 @@ FROM python:3.10-slim
 
 # Install system dependencies
 RUN apt-get update && \
-    apt-get install -y wget unzip curl gcc portaudio19-dev && \
+    apt-get install -y wget ffmpeg unzip curl gcc portaudio19-dev && \
     rm -rf /var/lib/apt/lists/*
 
 # Set working directory

app.py

@@ -1,15 +1,15 @@
-# app.py
-from fastapi import FastAPI, Request, Form
+from fastapi import FastAPI, Request, Form, File, UploadFile
 from fastapi.responses import HTMLResponse, JSONResponse
 from fastapi.staticfiles import StaticFiles
 from fastapi.templating import Jinja2Templates
 from main import process_query
 from voice.speech_to_text import SpeechToText
 import os
-import asyncio
-import pyaudio
-import wave
 import logging
+import tempfile
+import shutil
+import wave
+from pydub import AudioSegment
 
 # Set up logging
 logging.basicConfig(level=logging.INFO)
@@ -23,121 +23,104 @@ app.mount("/static", StaticFiles(directory="static"), name="static")
 # Set up templates
 templates = Jinja2Templates(directory="templates")
 
-# Vosk model path and audio file path
-vosk_model_path = "./vosk-model-small-en-us-0.15"
-audio_file_path = "voice/temp_audio.wav"
+root_dir = "./voice"
 
-# Ensure the voice directory exists
-os.makedirs("voice", exist_ok=True)
+# Vosk model path
+vosk_model_path = "./vosk-model-small-en-us-0.15"
 
 # Initialize SpeechToText
 stt = SpeechToText(model_path=vosk_model_path)
 
-# Global variables for recording state
-recording = False
-audio_frames = []
-recording_task = None
-
-def save_audio_to_wav(frames, sample_rate=16000):
-    """Save audio frames to a WAV file."""
+def is_valid_wav(file_path):
+    """Check if the file is a valid WAV file."""
     try:
-        logger.info(f"Saving audio to {audio_file_path} with {len(frames)} frames")
-        wf = wave.open(audio_file_path, 'wb')
-        wf.setnchannels(1)
-        wf.setsampwidth(2)  # 16-bit
-        wf.setframerate(sample_rate)
-        wf.writeframes(b''.join(frames))
-        wf.close()
-        if os.path.exists(audio_file_path):
-            logger.info(f"WAV file saved successfully: {os.path.getsize(audio_file_path)} bytes")
-        else:
-            logger.error("WAV file was not created")
-    except Exception as e:
-        logger.error(f"Error saving WAV file: {str(e)}")
-        raise
-
-async def record_audio():
-    """Background task to record audio."""
-    global audio_frames
-    p = pyaudio.PyAudio()
+        with wave.open(file_path, 'rb') as wf:
+            logger.info(f"WAV file validated: channels={wf.getnchannels()}, rate={wf.getframerate()}, frames={wf.getnframes()}")
+            return True
+    except wave.Error as e:
+        logger.error(f"Invalid WAV file: {str(e)}")
+        return False
+
+def convert_to_wav(input_path, output_path):
+    """Convert input audio to WAV format (16kHz, mono, 16-bit PCM)."""
     try:
-        stream = p.open(format=pyaudio.paInt16, channels=1, rate=16000, input=True, frames_per_buffer=1024)
-        stream.start_stream()
-        logger.info("Recording started...")
-
-        while recording:
-            data = stream.read(1024, exception_on_overflow=False)
-            audio_frames.append(data)
-            await asyncio.sleep(0.01)  # Small sleep to prevent blocking
-
-        stream.stop_stream()
-        stream.close()
-        logger.info(f"Recording stopped, captured {len(audio_frames)} frames")
+        audio = AudioSegment.from_file(input_path)
+        audio = audio.set_channels(1).set_frame_rate(16000).set_sample_width(2)  # 16-bit PCM
+        audio.export(output_path, format="wav")
+        logger.info(f"Converted audio to WAV: {output_path}, size: {os.path.getsize(output_path)} bytes")
+        return True
     except Exception as e:
-        logger.error(f"Error during recording: {str(e)}")
-    finally:
-        p.terminate()
+        logger.error(f"Failed to convert audio to WAV: {str(e)}")
+        return False
 
 @app.get("/", response_class=HTMLResponse)
 async def get_index(request: Request):
     return templates.TemplateResponse("index.html", {"request": request})
 
-@app.post("/start_recording", response_class=JSONResponse)
-async def start_recording():
-    global recording, audio_frames, recording_task
-    if not recording:
-        recording = True
-        audio_frames = []
-        recording_task = asyncio.create_task(record_audio())
-        logger.info("Started recording task")
-        return {"status": "Recording started"}
-    logger.warning("Recording already in progress")
-    return {"status": "Already recording"}
-
-@app.post("/stop_recording", response_class=HTMLResponse)
-async def stop_recording(request: Request):
-    global recording, recording_task
-    if recording:
-        recording = False
-        if recording_task:
-            await recording_task  # Wait for the recording task to complete
-            recording_task = None
-
-        # Save the audio to WAV
-        try:
-            save_audio_to_wav(audio_frames)
-        except Exception as e:
-            logger.error(f"Failed to save audio: {str(e)}")
-            return templates.TemplateResponse("index.html", {
-                "request": request,
-                "error": f"Failed to save audio: {str(e)}"
-            })
-
-        # Transcribe the saved audio
-        try:
-            text = stt.transcribe_audio(audio_file_path)
-            logger.info(f"Transcription result: '{text}'")
-            if not text:
-                logger.warning("Transcription returned no text")
+@app.post("/upload_audio", response_class=HTMLResponse)
+async def upload_audio(request: Request, audio_file: UploadFile = File(...)):
+    # Use a temporary directory to store the uploaded audio file
+    try:
+        with tempfile.TemporaryDirectory() as temp_dir:
+            # Save the uploaded file (could be WebM, OGG, etc.)
+            input_file_path = os.path.join(temp_dir, "input_audio")
+            with open(input_file_path, "wb") as buffer:
+                shutil.copyfileobj(audio_file.file, buffer)
+                file_size = os.path.getsize(input_file_path) if os.path.exists(input_file_path) else 0
+                logger.info(f"Uploaded audio saved to {input_file_path}, size: {file_size} bytes")
+
+            # Verify file was saved
+            if not os.path.exists(input_file_path) or file_size == 0:
+                logger.error("Uploaded audio file was not saved correctly")
                 return templates.TemplateResponse("index.html", {
                     "request": request,
-                    "error": "Could not understand the audio."
+                    "error": "Failed to save uploaded audio file."
+                })
+
+            # Convert to WAV
+            wav_file_path = os.path.join(root_dir, "temp_audio.wav")
+            if not convert_to_wav(input_file_path, wav_file_path):
+                logger.error("Audio conversion to WAV failed")
+                return templates.TemplateResponse("index.html", {
+                    "request": request,
+                    "error": "Failed to convert audio to WAV format."
                 })
-            return templates.TemplateResponse("index.html", {
-                "request": request,
-                "transcribed_text": text
-            })
-        except Exception as e:
-            logger.error(f"Transcription error: {str(e)}")
-            return templates.TemplateResponse("index.html", {
-                "request": request,
-                "error": f"Transcription error: {str(e)}"
-            })
-    logger.warning("No recording in progress")
-    return templates.TemplateResponse("index.html", {
-        "request": request,
-        "error": "No recording in progress."
-    })
+
+            # Validate WAV file
+            if not is_valid_wav(wav_file_path):
+                logger.error("Converted file is not a valid WAV file")
+                return templates.TemplateResponse("index.html", {
+                    "request": request,
+                    "error": "Converted audio is not a valid WAV file."
+                })
+
+            # Transcribe the WAV file
+            try:
+                text = stt.transcribe_audio(wav_file_path)
+                logger.info(f"Transcription result: '{text}'")
+                if not text:
+                    logger.warning("Transcription returned no text")
+                    return templates.TemplateResponse("index.html", {
+                        "request": request,
+                        "error": "Could not understand the audio. Please try speaking clearly."
+                    })
+                return templates.TemplateResponse("index.html", {
+                    "request": request,
+                    "transcribed_text": text
+                })
+            except Exception as e:
+                logger.error(f"Transcription error: {str(e)}")
+                return templates.TemplateResponse("index.html", {
+                    "request": request,
+                    "error": f"Transcription error: {str(e)}"
+                })
+
+    except Exception as e:
+        logger.error(f"Error processing uploaded audio: {str(e)}")
+        return templates.TemplateResponse("index.html", {
+            "request": request,
+            "error": f"Error processing audio: {str(e)}"
+        })
 
 @app.post("/query", response_class=HTMLResponse)
 async def handle_query(request: Request, query_text: str = Form(...), use_retriever: str = Form("no")):
@@ -154,4 +137,4 @@ async def handle_query(request: Request, query_text: str = Form(...), use_retrie
         "Web_Search_Response": result["web_search_response"],
         "Final_Response": result["final_response"],
         "Error": result["error"]
-    })
+    })

device_app.py

@@ -0,0 +1,157 @@
+# app.py
+from fastapi import FastAPI, Request, Form
+from fastapi.responses import HTMLResponse, JSONResponse
+from fastapi.staticfiles import StaticFiles
+from fastapi.templating import Jinja2Templates
+from main import process_query
+from voice.speech_to_text import SpeechToText
+import os
+import asyncio
+import pyaudio
+import wave
+import logging
+
+# Set up logging
+logging.basicConfig(level=logging.INFO)
+logger = logging.getLogger(__name__)
+
+app = FastAPI()
+
+# Mount static files (for CSS, JS, etc.)
+app.mount("/static", StaticFiles(directory="static"), name="static")
+
+# Set up templates
+templates = Jinja2Templates(directory="templates")
+
+# Vosk model path and audio file path
+vosk_model_path = "./vosk-model-small-en-us-0.15"
+audio_file_path = "voice/temp_audio.wav"
+
+# Ensure the voice directory exists
+os.makedirs("voice", exist_ok=True)
+
+# Initialize SpeechToText
+stt = SpeechToText(model_path=vosk_model_path)
+
+# Global variables for recording state
+recording = False
+audio_frames = []
+recording_task = None
+
+def save_audio_to_wav(frames, sample_rate=16000):
+    """Save audio frames to a WAV file."""
+    try:
+        logger.info(f"Saving audio to {audio_file_path} with {len(frames)} frames")
+        wf = wave.open(audio_file_path, 'wb')
+        wf.setnchannels(1)
+        wf.setsampwidth(2)  # 16-bit
+        wf.setframerate(sample_rate)
+        wf.writeframes(b''.join(frames))
+        wf.close()
+        if os.path.exists(audio_file_path):
+            logger.info(f"WAV file saved successfully: {os.path.getsize(audio_file_path)} bytes")
+        else:
+            logger.error("WAV file was not created")
+    except Exception as e:
+        logger.error(f"Error saving WAV file: {str(e)}")
+        raise
+
+async def record_audio():
+    """Background task to record audio."""
+    global audio_frames
+    p = pyaudio.PyAudio()
+    try:
+        stream = p.open(format=pyaudio.paInt16, channels=1, rate=16000, input=True, frames_per_buffer=1024)
+        stream.start_stream()
+        logger.info("Recording started...")
+
+        while recording:
+            data = stream.read(1024, exception_on_overflow=False)
+            audio_frames.append(data)
+            await asyncio.sleep(0.01)  # Small sleep to prevent blocking
+
+        stream.stop_stream()
+        stream.close()
+        logger.info(f"Recording stopped, captured {len(audio_frames)} frames")
+    except Exception as e:
+        logger.error(f"Error during recording: {str(e)}")
+    finally:
+        p.terminate()
+
+@app.get("/", response_class=HTMLResponse)
+async def get_index(request: Request):
+    return templates.TemplateResponse("index.html", {"request": request})
+
+@app.post("/start_recording", response_class=JSONResponse)
+async def start_recording():
+    global recording, audio_frames, recording_task
+    if not recording:
+        recording = True
+        audio_frames = []
+        recording_task = asyncio.create_task(record_audio())
+        logger.info("Started recording task")
+        return {"status": "Recording started"}
+    logger.warning("Recording already in progress")
+    return {"status": "Already recording"}
+
+@app.post("/stop_recording", response_class=HTMLResponse)
+async def stop_recording(request: Request):
+    global recording, recording_task
+    if recording:
+        recording = False
+        if recording_task:
+            await recording_task  # Wait for the recording task to complete
+            recording_task = None
+
+        # Save the audio to WAV
+        try:
+            save_audio_to_wav(audio_frames)
+        except Exception as e:
+            logger.error(f"Failed to save audio: {str(e)}")
+            return templates.TemplateResponse("index.html", {
+                "request": request,
+                "error": f"Failed to save audio: {str(e)}"
+            })
+
+        # Transcribe the saved audio
+        try:
+            text = stt.transcribe_audio(audio_file_path)
+            logger.info(f"Transcription result: '{text}'")
+            if not text:
+                logger.warning("Transcription returned no text")
+                return templates.TemplateResponse("index.html", {
+                    "request": request,
+                    "error": "Could not understand the audio."
+                })
+            return templates.TemplateResponse("index.html", {
+                "request": request,
+                "transcribed_text": text
+            })
+        except Exception as e:
+            logger.error(f"Transcription error: {str(e)}")
+            return templates.TemplateResponse("index.html", {
+                "request": request,
+                "error": f"Transcription error: {str(e)}"
+            })
+    logger.warning("No recording in progress")
+    return templates.TemplateResponse("index.html", {
+        "request": request,
+        "error": "No recording in progress."
+    })
+
+@app.post("/query", response_class=HTMLResponse)
+async def handle_query(request: Request, query_text: str = Form(...), use_retriever: str = Form("no")):
+    use_retriever = use_retriever.lower() in ["yes", "y"]
+    result = await process_query(vosk_model_path, query_text=query_text, use_retriever=use_retriever)
+
+    return templates.TemplateResponse("index.html", {
+        "request": request,
+        "User_Query": query_text,
+        "Intent": result["intent"],
+        "Entities": result["entities"],
+        "API_Response": result["base_response"],
+        "RAG_Response": result["retriever_response"],
+        "Web_Search_Response": result["web_search_response"],
+        "Final_Response": result["final_response"],
+        "Error": result["error"]
+    })

rag/sql_db.py

@@ -162,7 +162,7 @@ class SQL_Key_Pair:
                 value = result[0]
                 value_in_billions = value / 1_000_000_000
                 return f"{metric} for {ticker}: ${value_in_billions:.2f} billion."
-            return f"No {metric} data found for {ticker}."
+            return f"No relevant data found for {ticker}."
         except Exception as e:
             print(f"Error querying database: {e}")
             return f"Error querying database: {str(e)}"

requirements.txt

@@ -31,7 +31,7 @@ platformdirs==4.3.7
 pluggy==1.5.0
 primp==0.14.0
 pandas
-pyaudio==0.2.14
+pydub==0.25.1
 pydantic==2.11.2
 pydantic-core==2.33.1
 pydantic-settings==2.8.1
@@ -63,6 +63,7 @@ tomli==2.2.1
 tqdm==4.67.1
 typer==0.15.2
 typing-inspection==0.4.0
+transformers
 uvicorn==0.34.0
 vosk==0.3.45
 wasabi==1.1.3
@@ -70,3 +71,8 @@ weasel==0.4.1
 yarl==1.19.0
 langchain
 langchain_community
+typing_extensions
+urllib3
+idna
+
+

templates/index.html

@@ -1,4 +1,3 @@
-<!-- templates/index.html -->
 <!DOCTYPE html>
 <html lang="en">
 <head>
@@ -8,7 +7,7 @@
     <script src="https://cdn.tailwindcss.com"></script>
     <link rel="stylesheet" href="/static/css/styles.css">
 </head>
-<body class="bg-purple-100 font-sans !important">
+<body class="bg-purple-100 font-sans">
     <div class="container mx-auto p-1">
         <h1 class="text-5xl font-bold text-center text-black mb-3 drop-shadow-lg">Financial AI Agent</h1>
 
@@ -16,10 +15,10 @@
         <div class="bg-white p-3 rounded-lg shadow-lg mb-6">
             <div class="flex justify-center space-x-4">
                 <button type="button" id="startRecording" class="bg-green-500 text-white px-6 py-3 rounded-lg hover:bg-green-800 transition">
-                <strong>Start Recording</strong>
+                    <strong>Start Recording</strong>
                 </button>
                 <button type="button" id="stopRecording" class="bg-red-400 text-white px-6 py-3 rounded-lg hover:bg-red-700 transition hidden">
-                <strong>Stop Recording</strong>
+                    <strong>Stop Recording</strong>
                 </button>
             </div>
         </div>
@@ -34,18 +33,6 @@
                     <button type="submit" id="submitButton" class="bg-indigo-600 text-white px-6 py-3 rounded-lg hover:bg-red-500 transition" disabled>
                         <strong>Submit Query</strong>
                     </button>
-                    <!-- Script to change button text on submit -->
-                    <script>
-                        document.addEventListener("DOMContentLoaded", function () {
-                            const form = document.querySelector("form");
-                            const submitButton = document.getElementById("submitButton");
-
-                            form.addEventListener("submit", function () {
-                                submitButton.innerText = "Processing...";
-                                submitButton.disabled = true;
-                            });
-                        });
-                    </script>
                 </div>
                 <div class="flex items-center space-x-2">
                     <input type="checkbox" name="use_retriever" id="useRetriever" value="yes" class="h-5 w-5 text-indigo-600">
@@ -106,9 +93,122 @@
                 </div>
             {% endif %}
         </div>
-
     </div>
 
-    <script src="/static/js/script.js"></script>
+    <script>
+        document.addEventListener("DOMContentLoaded", function () {
+            const startRecording = document.getElementById("startRecording");
+            const stopRecording = document.getElementById("stopRecording");
+            const queryText = document.getElementById("queryText");
+            const submitButton = document.getElementById("submitButton");
+            const form = document.getElementById("queryForm");
+            let mediaRecorder = null;
+            let audioStream = null;
+            let audioChunks = [];
+
+            // Request microphone permission and start recording
+            startRecording.addEventListener("click", async () => {
+                try {
+                    audioStream = await navigator.mediaDevices.getUserMedia({ audio: true });
+                    mediaRecorder = new MediaRecorder(audioStream);
+
+                    mediaRecorder.ondataavailable = (event) => {
+                        if (event.data.size > 0) {
+                            audioChunks.push(event.data);
+                            console.log("Audio chunk received, size:", event.data.size, "type:", event.data.type);
+                        }
+                    };
+
+                    mediaRecorder.onstop = async () => {
+                        // Stop all tracks to release the microphone
+                        if (audioStream) {
+                            audioStream.getTracks().forEach(track => {
+                                track.stop();
+                                console.log("Track stopped:", track);
+                            });
+                            audioStream = null;
+                        }
+
+                        // Create audio blob (use browser's default format)
+                        const audioBlob = new Blob(audioChunks, { type: mediaRecorder.mimeType });
+                        console.log("Audio blob created, size:", audioBlob.size, "type:", audioBlob.type);
+                        const formData = new FormData();
+                        formData.append("audio_file", audioBlob, "recorded_audio.webm");
+
+                        try {
+                            const response = await fetch("/upload_audio", {
+                                method: "POST",
+                                body: formData
+                            });
+
+                            const html = await response.text();
+                            // Parse the response HTML to extract transcribed text or error
+                            const parser = new DOMParser();
+                            const doc = parser.parseFromString(html, "text/html");
+                            const transcribedText = doc.querySelector("input[name='query_text']").value;
+                            const error = doc.querySelector(".bg-red-100 p")?.textContent;
+
+                            if (error) {
+                                alert(error);
+                            } else if (transcribedText) {
+                                queryText.value = transcribedText;
+                                queryText.readOnly = false; // Allow editing
+                                submitButton.disabled = false; // Enable submit button
+                            }
+
+                            // Reset UI
+                            startRecording.classList.remove("hidden");
+                            stopRecording.classList.add("hidden");
+                            audioChunks = [];
+                            mediaRecorder = null;
+                        } catch (error) {
+                            console.error("Error uploading audio:", error);
+                            alert("Failed to process audio: " + error.message);
+                            startRecording.classList.remove("hidden");
+                            stopRecording.classList.add("hidden");
+                        }
+                    };
+
+                    mediaRecorder.onerror = (event) => {
+                        console.error("MediaRecorder error:", event.error);
+                        alert("Recording error: " + event.error.message);
+                    };
+
+                    mediaRecorder.start(100); // Collect chunks every 100ms
+                    console.log("Recording started, MIME type:", mediaRecorder.mimeType);
+                    startRecording.classList.add("hidden");
+                    stopRecording.classList.remove("hidden");
+                } catch (error) {
+                    console.error("Error accessing microphone:", error);
+                    alert("Could not access microphone. Please check permissions.");
+                }
+            });
+
+            // Stop recording
+            stopRecording.addEventListener("click", () => {
+                if (mediaRecorder && mediaRecorder.state !== "inactive") {
+                    console.log("Stopping recording, current state:", mediaRecorder.state);
+                    mediaRecorder.stop();
+                } else {
+                    console.warn("MediaRecorder is not active or not initialized");
+                }
+
+                // Ensure stream tracks are stopped even if MediaRecorder fails
+                if (audioStream) {
+                    audioStream.getTracks().forEach(track => {
+                        track.stop();
+                        console.log("Track stopped:", track);
+                    });
+                    audioStream = null;
+                }
+            });
+
+            // Handle form submission
+            form.addEventListener("submit", () => {
+                submitButton.innerText = "Processing...";
+                submitButton.disabled = true;
+            });
+        });
+    </script>
 </body>
 </html>