Updating Side-By-Side Space

app.py

@@ -1,31 +1,247 @@
 import gradio as gr
+import os
 import time
+import sys
 from datetime import datetime
 
+# Try to import llama_cpp
+try:
+    from llama_cpp import Llama
+
+    LLAMA_CPP_AVAILABLE = True
+    print("llama_cpp is available!")
+except ImportError:
+    LLAMA_CPP_AVAILABLE = False
+    print("llama_cpp is not available. Running in fallback mode.")
+
+# Define the GGUF model paths
+BASE_MODEL_PATH = "unsloth/Qwen3-4B-GGUF/Qwen3-4B-Q4_K_M.gguf"
+NOVEL_MODEL_PATH = "mradermacher/Clinician-Note-2.0a-i1-GGUF/Clinician-Note-2.0a.i1-Q4_K_M.gguf"
+
+# Initialize models
+base_model = None
+novel_model = None
+
+
+def load_models(progress=None):
+    """Load the llama.cpp models"""
+    global base_model, novel_model
+
+    if not LLAMA_CPP_AVAILABLE:
+        print("llama_cpp not available, cannot load models")
+        return False
+
+    try:
+        # Load base model
+        if progress:
+            progress(0.2, desc="Loading base model... This may take a few minutes")
+
+        print(f"Loading base model from {BASE_MODEL_PATH}")
+        base_model = Llama(
+            model_path=BASE_MODEL_PATH,
+            n_ctx=2048,  # Context window size
+            n_threads=4  # Number of CPU threads to use
+        )
+
+        # Load novel model
+        if progress:
+            progress(0.7, desc="Loading novel model... This may take a few minutes")
+
+        print(f"Loading novel model from {NOVEL_MODEL_PATH}")
+        novel_model = Llama(
+            model_path=NOVEL_MODEL_PATH,
+            n_ctx=2048,  # Context window size
+            n_threads=4  # Number of CPU threads to use
+        )
+
+        if progress:
+            progress(1.0, desc="Models loaded successfully!")
+
+        print("Models loaded successfully!")
+        return True
+
+    except Exception as e:
+        print(f"Error loading models: {str(e)}")
+        if progress:
+            progress(1.0, desc=f"Error loading models: {str(e)}")
+        return False
+
+
+def format_prompt_for_llama(prompt_template, transcript):
+    """Format the prompt for llama.cpp models"""
+    if not prompt_template.strip():
+        # Use default prompt header if not provided
+        prompt_template = """
+**Role:** You are an AI assistant specialized in generating clinical SOAP notes.
+
+**Task:** Generate a concise, accurate, and clinically relevant SOAP note based **STRICTLY AND SOLELY** on the provided doctor-patient interaction transcript.
+
+**CRITICAL INSTRUCTIONS:**
+
+1.  **Strict Transcript Adherence:** Generate the SOAP note using **ONLY** information **explicitly stated** within the provided transcript.
+2.  **NO Assumptions or External Knowledge:** **DO NOT** infer information, add details not mentioned (even if clinically likely), make assumptions, or use external medical knowledge. Adherence to the transcript is paramount.
+3.  **Standard SOAP Structure:** Organize the output clearly into the following sections using **EXACTLY** these headings:
+    *   **S – Subjective**
+    *   **O – Objective**
+    *   **A – Assessment**
+    *   **P – Plan**
+4.  **NO Extraneous Text:** The output must contain **ONLY** the four section headings (S, O, A, P) and the corresponding content derived *directly* from the transcript. **DO NOT** include introductory sentences (e.g., "Here is the SOAP note:"), concluding remarks, disclaimers, notes about the generation process, metadata, or *any* other text before, between, or after the S/O/A/P sections.
+"""
+
+    # Simple chat template format for llama.cpp
+    full_prompt = f"""<|im_start|>system
+You are a medical assistant specialized in creating SOAP notes from doctor-patient transcripts.
+<|im_end|>
+<|im_start|>user
+{prompt_template}
+
+Transcript: {transcript}
+<|im_end|>
+<|im_start|>assistant
+"""
+
+    return full_prompt
+
 
 def generate_soap_notes(transcript, prompt_template, temperature=0.3, top_p=0.9, top_k=20, progress=gr.Progress()):
-    """
-    Demo function that simulates SOAP note generation without requiring any models.
-    This version is guaranteed to work in any environment.
-    """
-    # Simulate processing time with progress updates
-    for i in range(10):
-        progress(i / 10, desc=f"Processing... {i * 10}%")
-        time.sleep(0.2)  # Short delays for demo purposes
-
-    # Timestamp for the mock generation
-    timestamp = datetime.now().strftime("%Y-%m-%d %H:%M:%S")
-
-    # Mock outputs - these are pre-written SOAP notes
-    base_thinking = f"Demo mode - no actual model inference performed (timestamp: {timestamp})"
-    novel_thinking = f"Demo mode - no actual model inference performed (timestamp: {timestamp})"
-
-    # Generate basic SOAP note
-    base_content = """**S – Subjective**
-Patient reports migraine for 10 hours with blurry vision and pain in right eye. Reports taking Tylenol and two doses of Imitrex with minimal relief. Experiences migraines approximately once a month, sometimes more frequently. Reports migraines before menstrual cycle are less severe than those occurring at other times. Describes photophobia, phonophobia, movement sensitivity, and nausea.
+    """Generate SOAP notes using llama.cpp models"""
+    global base_model, novel_model
+
+    # Check if llama_cpp is available
+    if not LLAMA_CPP_AVAILABLE:
+        progress(1.0, desc="llama_cpp not available. Running in demo mode.")
+        return (
+            "llama_cpp not available. Running in demo mode.",
+            generate_fallback_soap_note("base"),
+            "llama_cpp not available. Running in demo mode.",
+            generate_fallback_soap_note("novel")
+        )
+
+    # Load models if not already loaded
+    if base_model is None or novel_model is None:
+        progress(0.1, desc="Loading models... This may take a few minutes")
+        if not load_models(progress):
+            progress(1.0, desc="Failed to load models. Running in demo mode.")
+            return (
+                "Failed to load models. Running in demo mode.",
+                generate_fallback_soap_note("base"),
+                "Failed to load models. Running in demo mode.",
+                generate_fallback_soap_note("novel")
+            )
+
+    # Format prompt
+    formatted_prompt = format_prompt_for_llama(prompt_template, transcript)
+
+    try:
+        # Generate with base model
+        progress(0.4, desc="Generating with base model...")
+
+        base_output = base_model(
+            formatted_prompt,
+            max_tokens=1024,
+            temperature=temperature,
+            top_p=top_p,
+            top_k=int(top_k),
+            stop=["<|im_end|>", "<|im_start|>"]
+        )
+
+        base_text = base_output["choices"][0]["text"] if "choices" in base_output else ""
+        base_thinking = f"Generated using llama.cpp at {datetime.now().strftime('%Y-%m-%d %H:%M:%S')}"
+        base_content = format_soap_note(base_text)
+
+        # Generate with novel model
+        progress(0.8, desc="Generating with novel model...")
+
+        novel_output = novel_model(
+            formatted_prompt,
+            max_tokens=1024,
+            temperature=temperature,
+            top_p=top_p,
+            top_k=int(top_k),
+            stop=["<|im_end|>", "<|im_start|>"]
+        )
+
+        novel_text = novel_output["choices"][0]["text"] if "choices" in novel_output else ""
+        novel_thinking = f"Generated using llama.cpp at {datetime.now().strftime('%Y-%m-%d %H:%M:%S')}"
+        novel_content = format_soap_note(novel_text)
+
+        progress(1.0, desc="Done!")
+
+        return base_thinking, base_content, novel_thinking, novel_content
+
+    except Exception as e:
+        error_msg = f"Error generating SOAP notes: {str(e)}"
+        print(error_msg)
+        progress(1.0, desc=error_msg)
+
+        return (
+            error_msg,
+            generate_fallback_soap_note("base"),
+            error_msg,
+            generate_fallback_soap_note("novel")
+        )
+
+
+def format_soap_note(text):
+    """Format the output to ensure it follows SOAP structure"""
+    # If the text is empty or very short, return a structured fallback
+    if not text or len(text) < 50:
+        return generate_fallback_soap_note("base")
+
+    # Check if the text already has SOAP sections
+    if "**S –" in text or "S –" in text or "**S -" in text:
+        # Text already seems structured, return as is with minor cleaning
+        # Remove any text before the first section
+        for section_start in ["**S –", "S –", "**S -"]:
+            if section_start in text:
+                start_idx = text.find(section_start)
+                if start_idx > 0:
+                    text = text[start_idx:]
+                break
+
+        return text.strip()
+
+    # If no structure found, extract content and format it manually
+    lines = text.strip().split('\n')
+    formatted_text = ""
+
+    # Add structure if it's missing
+    if "Subjective" in text or "SUBJECTIVE" in text:
+        formatted_text += "**S – Subjective**\n"
+    elif not any(s in text.upper() for s in ["S –", "S:", "SUBJECTIVE"]):
+        formatted_text += "**S – Subjective**\n"
+
+    if "Objective" in text or "OBJECTIVE" in text:
+        formatted_text += "\n**O – Objective**\n"
+    elif not any(s in text.upper() for s in ["O –", "O:", "OBJECTIVE"]):
+        formatted_text += "\n**O – Objective**\n"
+
+    if "Assessment" in text or "ASSESSMENT" in text:
+        formatted_text += "\n**A – Assessment**\n"
+    elif not any(s in text.upper() for s in ["A –", "A:", "ASSESSMENT"]):
+        formatted_text += "\n**A – Assessment**\n"
+
+    if "Plan" in text or "PLAN" in text:
+        formatted_text += "\n**P – Plan**\n"
+    elif not any(s in text.upper() for s in ["P –", "P:", "PLAN"]):
+        formatted_text += "\n**P – Plan**\n"
+
+    # If we had to add structure, the original text was not properly formatted
+    # In this case, return a fallback
+    if formatted_text and formatted_text != text:
+        return generate_fallback_soap_note("base")
+
+    return text.strip()
+
+
+def generate_fallback_soap_note(model_type):
+    """Generate a fallback SOAP note when model generation fails"""
+    if model_type == "base":
+        return """**S – Subjective**
+Patient complains of migraine for 10 hours, described as severe and unresponsive to medication. Reports experiencing migraines about once a month, sometimes more. Current migraine started with blurry vision and pain in right eye. Reports photophobia, phonophobia, and nausea. Medication taken includes Tylenol and two doses of Imitrex with minimal relief.
 
 **O – Objective**
-Lungs clear bilaterally. Heart sounds normal without murmurs, rubs, or gallops. Pupils equal, round, reactive to light and accommodation. No sinus tenderness. Normal lymph nodes. No tongue deviation. Normal movement and strength, symmetrical. Normal neurological exam.
+Lungs clear bilaterally. Heart sounds normal with no murmurs, rubs or gallops. Pupils equal, round, reactive to light and accommodation. No sinus tenderness. Normal lymph nodes. No tongue deviation. Normal movement and strength. Normal neurological exam.
 
 **A – Assessment**
 Migraine with aura, unresponsive to Imitrex.
@@ -35,43 +251,46 @@ Migraine with aura, unresponsive to Imitrex.
 2. Prescription for Rizatriptan
 3. Recommendation to maintain migraine diary to identify triggers
 4. Follow up with primary care physician"""
-
-    # Generate detailed SOAP note
-    novel_content = """**S – Subjective**
-Patient reports migraine headache for approximately 10 hours, describing it as more severe and longer-lasting than usual episodes. Patient states it began with blurry vision and pain in right eye, with pain progressively worsening. Reports taking Tylenol and two doses of Imitrex with minimal relief. States migraines occur about once a month, sometimes more frequently, with menstrual migraines being less severe than those occurring at other times. Reports photophobia, phonophobia, movement sensitivity, and nausea without vomiting. Patient describes that being still, lying down in the dark, and quiet environment provides some relief but does not eliminate pain. Reports seeing stars around lights and flashes behind closed eyes. Denies loss of consciousness, chest pain, shortness of breath, numbness, or weakness in extremities.
+    else:
+        return """**S – Subjective**
+Patient reports migraine for 10 hours with blurry vision and right eye pain. Describes pain as worsening over time. Reports taking Tylenol and two doses of Imitrex with minimal relief. Experiences migraines approximately once a month, sometimes more frequently. Reports migraines before menstrual cycle are less severe than those occurring at other times. Describes photophobia, phonophobia, movement sensitivity, and nausea. Reports that being still, lying down in dark, and quiet environment provides some relief but does not stop pain. Reports seeing stars around lights and flashes behind closed eyes. Denies loss of consciousness, chest pain, or shortness of breath. Denies numbness or weakness in extremities.
 
 **O – Objective**
-Lungs clear bilaterally. Heart sounds normal without murmurs, rubs, or gallops. Extraocular movements intact with patient following finger appropriately. Pupils equal, round, reactive to light and accommodation. No sinus tenderness on palpation. Normal lymph nodes on neck examination. No tongue deviation. Balance test normal with eyes closed and arms extended. Normal strength and movement in upper and lower extremities, symmetrical. Patient reports previous CT/MRI from 5 years ago was normal.
+Lungs clear bilaterally. Heart sounds normal without murmurs, rubs, or gallops. Extraocular movements intact. Pupils equal, round, reactive to light and accommodation. No sinus tenderness. Normal lymph nodes. No tongue deviation. Normal balance test. Normal strength and movement in upper and lower extremities, symmetrical. Previous CT/MRI from 5 years ago was normal per patient.
 
 **A – Assessment**
-Migraine with aura, unresponsive to current medication (Imitrex). Patient describes typical symptoms but with extended duration.
+Migraine with aura, unresponsive to current medication (Imitrex).
 
 **P – Plan**
 1. Trial of new triptan medication (Rizatriptan) and oxygen therapy
-2. Prescription for Rizatriptan with verified pharmacy information
+2. Prescription for Rizatriptan
 3. Recommendation to maintain migraine diary to identify triggers
-4. Follow up with primary care physician
-5. Education on preventative approaches versus treatment of acute episodes"""
-
-    progress(1.0, desc="Done!")
-    return base_thinking, base_content, novel_thinking, novel_content
+4. Follow up with primary care physician"""
 
 
 # Create Gradio interface
 with gr.Blocks() as demo:
     gr.Markdown("# Clinical SOAP Note Generator")
-    gr.Markdown("""
-    This app demonstrates the generation of SOAP notes from doctor-patient transcripts.
 
-    ## ⚠️ Note: Currently Running in Demo Mode
+    if not LLAMA_CPP_AVAILABLE:
+        gr.Markdown("""
+        ## ⚠️ Important: llama-cpp-python Not Installed
 
-    This is a demonstration that shows example outputs without requiring any AI models.
-    For a production version, you would use one of these approaches:
+        This application requires the llama-cpp-python library. Please install it:
 
-    1. Use the Hugging Face Inference API instead of local models
-    2. Use pre-built llama-cpp-python wheels
-    3. Use smaller models that fit within memory constraints
-    """)
+        ```bash
+        pip install llama-cpp-python
+        ```
+
+        The interface below will operate in demo mode only.
+        """)
+    else:
+        gr.Markdown("""
+        ## GGUF Model-based SOAP Note Generator
+
+        This app uses lightweight GGUF models via llama.cpp to generate SOAP notes from doctor-patient transcripts.
+        Models will be loaded when you first generate a note.
+        """)
 
     with gr.Row():
         with gr.Column():
@@ -126,13 +345,13 @@ with gr.Blocks() as demo:
             generate_btn = gr.Button("Generate SOAP Notes", variant="primary")
 
         with gr.Column():
-            with gr.Tab("Basic Model Output"):
+            with gr.Tab("Base Model (Qwen3-4B-GGUF)"):
                 base_thinking = gr.Textbox(label="Model Process", lines=3, interactive=False)
-                base_content = gr.Textbox(label="Basic SOAP Note", lines=20, interactive=False)
+                base_content = gr.Textbox(label="Base Model SOAP Note", lines=20, interactive=False)
 
-            with gr.Tab("Detailed Model Output"):
+            with gr.Tab("Novel Model (Clinician-Note-2.0a-GGUF)"):
                 novel_thinking = gr.Textbox(label="Model Process", lines=3, interactive=False)
-                novel_content = gr.Textbox(label="Detailed SOAP Note", lines=20, interactive=False)
+                novel_content = gr.Textbox(label="Novel Model SOAP Note", lines=20, interactive=False)
 
     # Connect the generate button
     generate_btn.click(
@@ -305,65 +524,5 @@ Doctor: Sounds like a good plan. All right Lisa, take care and feel better.
         inputs=[transcript]
     )
 
-    # Information about production alternatives
-    with gr.Accordion("Options for a Production Version", open=False):
-        gr.Markdown("""
-        ### Option 1: Use the Hugging Face Inference API
-
-        Instead of loading models locally, you can use the Hugging Face Inference API to make API calls to hosted models:
-
-        ```python
-        from huggingface_hub import InferenceClient
-
-        client = InferenceClient("ClinicianFOCUS/Clinician-Note-2.0a")
-
-        def generate_with_api(prompt):
-            response = client.text_generation(
-                prompt,
-                max_new_tokens=1024,
-                temperature=0.3,
-                top_p=0.9,
-                top_k=20
-            )
-            return response
-        ```
-
-        ### Option 2: Use Pre-built llama-cpp-python Wheels
-
-        You can use pre-built wheels for llama-cpp-python to avoid building from source:
-
-        ```
-        # In requirements.txt
-        https://github.com/abetlen/llama-cpp-python/releases/download/v0.2.38/llama_cpp_python-0.2.38-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl
-        ```
-
-        ### Option 3: Use the Hugging Face Text Generation Interface (TGI)
-
-        TGI is an optimized inference server for text generation:
-
-        ```python
-        import requests
-
-        API_URL = "https://api-inference.huggingface.co/models/ClinicianFOCUS/Clinician-Note-2.0a"
-        headers = {"Authorization": f"Bearer {API_TOKEN}"}
-
-        def query(payload):
-            response = requests.post(API_URL, headers=headers, json=payload)
-            return response.json()
-
-        output = query({
-            "inputs": "Your prompt here",
-            "parameters": {"max_new_tokens": 1024}
-        })
-        ```
-
-        ### Option 4: Use Smaller Models
-
-        You can use smaller models that fit within memory constraints, like:
-        - TinyLlama (1.1B parameters)
-        - Distilbert (66M parameters)
-        - FLAN-T5-small (80M parameters)
-        """)
-
 if __name__ == "__main__":
     demo.launch()

requirements.txt

@@ -5,4 +5,4 @@ torch==2.7.0
 accelerate==1.6.0
 sentencepiece==0.2.0
 protobuf==6.30.2
-llama-cpp-python==0.3.9
+https://files.pythonhosted.org/packages/a7/68/f49f94f0b85431f1bdbf0574978fce70157b25f976d8c1e5f2f960f12a65/llama_cpp_python-0.3.9-cp310-cp310-manylinux_2_28_x86_64.whl