Deploy multi-agent radiology assistant

.gitattributes

@@ -0,0 +1,4 @@
+data/* filter=lfs diff=lfs merge=lfs -text
+*.npy filter=lfs diff=lfs merge=lfs -text
+*.bin filter=lfs diff=lfs merge=lfs -text
+*.jsonl filter=lfs diff=lfs merge=lfs -text

README.md

@@ -0,0 +1,14 @@
+---
+title: Multiagent Xray Assistant
+emoji: 📊
+colorFrom: red
+colorTo: red
+sdk: gradio
+sdk_version: 5.38.2
+app_file: app.py
+pinned: false
+license: apache-2.0
+short_description: 'Gets a structured X-ray report using a multi-agent system '
+---
+
+Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference

app.py

@@ -0,0 +1,54 @@
+import gradio as gr
+from src.pipeline import generate_report
+
+def process_upload(image):
+    """Process uploaded X-ray image and return radiology report"""
+    if image is None:
+        return None, "Please upload a chest X-ray image."
+    
+    try:
+        report = generate_report(image)
+        return image, report  # Return both image and report
+    except Exception as e:
+        return image, f"Error processing image: {str(e)}"
+
+# Create enhanced Gradio interface with custom layout
+with gr.Blocks(title="Multi-Agent Radiology Assistant") as demo:
+    gr.Markdown("# Multi-Agent Radiology Assistant")
+    gr.Markdown("Upload a chest X-ray image to receive an AI-generated radiology report using cutting-edge, multi-agent analysis")
+    
+    with gr.Row():
+        with gr.Column(scale=1):
+            input_image = gr.Image(
+                type="filepath", 
+                label="Upload Chest X-ray",
+                height=400
+            )
+            process_btn = gr.Button("Generate Report", variant="primary")
+        
+        with gr.Column(scale=1):
+            output_image = gr.Image(
+                label="Analyzed Image",
+                height=400,
+                interactive=False
+            )
+    
+    with gr.Row():
+        output_report = gr.Markdown(
+            label="📄 Radiology Report",
+            height=300
+        )
+    
+    # Connect the button to processing function
+    process_btn.click(
+        fn=process_upload,
+        inputs=[input_image],
+        outputs=[output_image, output_report]
+    )
+    
+    # Add examples section
+    gr.Markdown("### Example X-rays")
+    gr.Markdown("*Upload your own chest X-ray image above to get started*")
+
+if __name__ == "__main__":
+    demo.launch()

data/iu_impr.jsonl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:30174b890f1bedcae6925550a207e4ae6649ca353ba882b9413a8971f43ea9d2
+size 2446560

data/iu_vecs.npy

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:43c8baa976a1587a72d070feb2c17ddee2c4fcc5a10ba8b80efa2d801ec7b5e5
+size 15216768

data/raw_abstracts.jsonl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3c2784fce84026ceaa523430cf70b6e3207e5f647713f356bc60fb40ce407887
+size 317236278

data/text_faiss.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b33f4e9de88138cd9af54f2031c46a1c1ea9ddce6fbcf65ee879adeb7b636df3
+size 614400045

requirements.txt

@@ -0,0 +1,13 @@
+
+crewai==0.4
+gradio>=4.27
+faiss-cpu
+groq
+google-generativeai
+transformers
+torch
+numpy
+pillow
+open-clip-torch
+pydantic
+tqdm

src/config/agents.yaml

@@ -0,0 +1,24 @@
+vision_agent:
+  role: "Radiology Captioning Agent"
+  goal: "Use the vision_caption_tool and return back the exact output. DO NOT add, interpret, or speculate beyond what the vision_caption_tool outputs"
+  backstory: "A world-class expert radiologist AI specialized in chest X-ray interpretation"
+
+pubmed_agent:
+  role: "Biomedical Literature Retriever specializing in retrieving the most relevant PubMed abstracts for a given chest X-ray impression"
+  goal: "Perform a SINGLE search for the top 3 most relevant PubMed abstracts based on a given medical caption using the pubmed_tool"
+  backstory: "You are an AI literature assistant. Your sole purpose is to take a text caption, use your tool exactly once to find relevant citations, and present the formatted output directly as your final answer"
+
+iu_agent:
+  role: "Retrieval Agent specializing in retrieving the most semantically-similar IU chest X-ray impression"
+  goal: "Find and return the most semantically similar impression from the IU-Xray dataset based on a given chest X-ray image"
+  backstory: "You're a biomedical assistant trained on the IU-Xray dataset. Your job is to retrieve the closest stylistic and semantic match to the current case, helping the system maintain professional and realistic radiology language. You do not generate new content — you only retrieve"
+
+draft_agent:
+  role: "Senior thoracic radiologist with expertise in drafting X-ray reports by synthesizing information from various sources"
+  goal: "Write a clinically accurate draft radiology report for a given chest X-ray, using the AI-generated visual caption, the most similar IU X-ray impression, and the top PubMed evidence"
+  backstory: "You have 20+ years of experience drafting high-quality chest X-ray reports. Your strength lies in fusing image-grounded observations with medically accurate language and literature-backed phrasing. You prioritize clarity, medical accuracy, and correct radiological terminology over verbosity. You hedge uncertain findings and avoid speculative or unsupported claims"
+
+critic_agent:
+  role: "Senior thoracic radiologist specializing in auditing chest X-ray reports"
+  goal: "Ensure that the radiology report is concise, medically accurate, and free from hallucinations or unsupported conclusions"
+  backstory: "You have 20+ years of experience reviewing and auditing chest X-ray reports. Your goal is to ensure clinical realism, eliminate hallucinations, and unsupported language, resulting in a concise and professional clinical report"

src/config/tasks.yaml

@@ -0,0 +1,24 @@
+caption_task:
+  description: "Analyze the chest X-ray at '{image_path}' and return the exact output from the vision_caption_tool"
+  expected_output: "The VERBATIM output of the vision_caption_tool"
+  agent: vision_agent
+
+pubmed_task:
+  description: "From the vision agent's report, extract ONLY the key POSITIVE finding(s) from the IMPRESSION section (ignore negations or normal findings). Use THOSE finding(s) as the input query to the pubmed_tool. Return the top 2 most relevant PubMed citations with pmid, similarity score, title, and abstract, along with the finding(s) that you used as an input for the pubmed_tool"
+  expected_output: "List of 2 relevant PubMed citations with pmid, similarity score, title, and abstract, along with the finding(s) used as input"
+  agent: pubmed_agent
+
+iu_task:
+  description: "Use the iu_tool to find the most similar impression in the IU-Xray dataset based on the X-ray image at '{image_path}'. Return the image UUID and the impression text of the closest match"
+  expected_output: "UUID and impression of the most semantically similar IU-Xray case"
+  agent: iu_agent
+
+draft_task:
+  description: "Your task is to write a two-part radiology report based on a chest X-ray image. Use the AI-generated visual caption [①] as your PRIMARY source of findings. You may also incorporate as SECONDARY sources: Language or structure from the closest IU-Xray impression [②], and medically relevant phrasing or terminology from PubMed abstracts [③]. However: NEVER contradict the visual caption. NEVER introduce information not clearly supported by one of the three sources. DO NOT speculate or offer clinical context not evident the caption. If findings are ambiguous or limited, hedge appropriately. Tag each sentence using [①], [②], or [③] to indicate which source supports it. Your report must have two sections: FINDINGS: Objective description of radiographic features [from ①] - Use [②] or [③] ONLY for language improvements or secondary detail. IMPRESSION: Concise summary of likely clinical implications based on the findings - You may re-use validated phrases from [②] or [③] ONLY if necessary, but DO NOT overreach"
+  expected_output: "A two-part radiology report with FINDINGS and IMPRESSION sections"
+  agent: draft_agent
+
+critic_task:
+  description: "Your task is to review and refine the draft radiology report (only the FINDINGS and IMPRESSIONS sections) for clinical accuracy, realism, and formatting. You MUST: Validate every sentence against its source tag: [①] = Caption, [②] = IU report, [③] = PubMed abstract. Remove any statement not clearly supported by the visual caption. Remove unsupported speculation, exaggerations, or redundant hedging. Maintain standard MIMIC-CXR formatting: terse, focused, professional. You MUST NOT: Invent new findings or reword unsupported conclusions. Add clinical context not present in the visual or evidence inputs. You MAY: Improve phrasing for clarity or brevity. Remove footnote tags in your final output. Output ONLY the final cleaned report. Keep section headers intact"
+  expected_output: "A cleaned two-part radiology report with FINDINGS and IMPRESSION sections. Do NOT include source tags. Do NOT include any commentary or explanation — only the final report"
+  agent: critic_agent

src/crew.py

@@ -0,0 +1,99 @@
+from crewai import Agent, Crew, Process, Task, LLM
+from crewai.project import CrewBase, agent, crew, task
+from .tools_loader import get_tools
+
+@CrewBase
+class RadiologyCrew:
+    """Multi-agent radiology report generation crew"""
+
+    def __init__(self):
+        # Load tools once
+        self.tools = get_tools()
+        
+        # Initialize LLMs with optimal settings
+        self.vision_llm = LLM(model="groq/meta-llama/llama-4-scout-17b-16e-instruct", temperature=0.2)
+        self.text_llm = LLM(model="groq/llama-3.3-70b-versatile", temperature=0.1)
+        self.draft_llm = LLM(model="groq/deepseek-r1-distill-llama-70b", temperature=0.3)
+        self.critic_llm = LLM(model="groq/meta-llama/llama-4-scout-17b-16e-instruct", temperature=0.2)
+
+    @agent
+    def vision_agent(self) -> Agent:
+        return Agent(
+            config=self.agents_config['vision_agent'],
+            tools=[self.tools["vision_tool"]],
+            llm=self.vision_llm,
+            allow_delegation=False,
+            verbose=False
+        )
+
+    @agent
+    def pubmed_agent(self) -> Agent:
+        return Agent(
+            config=self.agents_config['pubmed_agent'],
+            tools=[self.tools["pubmed_tool"]],
+            llm=self.text_llm,
+            verbose=False
+        )
+
+    @agent
+    def iu_agent(self) -> Agent:
+        return Agent(
+            config=self.agents_config['iu_agent'],
+            tools=[self.tools["iu_tool"]],
+            llm=self.text_llm,
+            verbose=False
+        )
+
+    @agent
+    def draft_agent(self) -> Agent:
+        return Agent(
+            config=self.agents_config['draft_agent'],
+            llm=self.draft_llm,
+            verbose=False
+        )
+
+    @agent
+    def critic_agent(self) -> Agent:
+        return Agent(
+            config=self.agents_config['critic_agent'],
+            llm=self.critic_llm,
+            verbose=True
+        )
+
+    @task
+    def caption_task(self) -> Task:
+        return Task(config=self.tasks_config['caption_task'])
+
+    @task
+    def pubmed_task(self) -> Task:
+        return Task(
+            config=self.tasks_config['pubmed_task'],
+            context=[self.caption_task()]
+        )
+
+    @task
+    def iu_task(self) -> Task:
+        return Task(config=self.tasks_config['iu_task'])
+
+    @task
+    def draft_task(self) -> Task:
+        return Task(
+            config=self.tasks_config['draft_task'],
+            context=[self.caption_task(), self.iu_task(), self.pubmed_task()]
+        )
+
+    @task
+    def critic_task(self) -> Task:
+        return Task(
+            config=self.tasks_config['critic_task'],
+            context=[self.caption_task(), self.iu_task(), self.draft_task()]
+        )
+
+    @crew
+    def crew(self) -> Crew:
+        return Crew(
+            agents=self.agents,
+            tasks=self.tasks,
+            process=Process.sequential,
+            verbose=False
+        )

src/pipeline.py

@@ -0,0 +1,7 @@
+from .crew import RadiologyCrew
+
+def generate_report(image_path: str) -> str:
+    """Generate radiology report for uploaded image"""
+    crew = RadiologyCrew().crew()
+    result = crew.kickoff(inputs={"image_path": image_path})
+    return str(result).strip()

src/tools/iu_retrieval_tool.py

@@ -0,0 +1,87 @@
+from crewai.tools import BaseTool
+from pydantic import BaseModel, Field
+from typing import Optional, Dict
+import torch
+import numpy as np
+import json
+import os
+from PIL import Image
+from pathlib import Path
+from open_clip import create_model_from_pretrained
+import torch.nn.functional as F
+
+# ----- Input schema for the tool -----
+class IUImageInput(BaseModel):
+    # Defines the expected input: absolute path to the query image
+    image_path: str = Field(..., description="Absolute path to the query image")
+
+# ----- Tool class -----
+class IUImpressionSearchTool(BaseTool):
+    # Tool metadata
+    name: str = "iu_impression_search_tool"
+    description: str = (
+        "Retrieves the most similar IU X-ray image impression based on visual similarity "
+        "using BiomedCLIP embeddings."
+    )
+    args_schema: type = IUImageInput  # Specifies input schema
+    metadata: dict = {}  # Optional metadata for config overrides
+
+    def _run(self, image_path: str) -> str:
+        # Dynamic path resolution - same pattern as pubmed_tool
+        BASE_DIR = Path(__file__).parent.parent.parent  # Up to main folder
+        default_vecs_path = str(BASE_DIR / "data" / "iu_vecs.npy")  # Default path for IU vectors
+        default_impr_path = str(BASE_DIR / "data" / "iu_impr.jsonl")  # Default path for IU impressions
+        
+        # Resolve config paths with dynamic defaults (can be overridden via metadata)
+        vecs_path = self.metadata.get("VEC_PATH", default_vecs_path)
+        impr_path = self.metadata.get("IMPR_PATH", default_impr_path)
+        model_id = self.metadata.get("MODEL_ID", "hf-hub:microsoft/BiomedCLIP-PubMedBERT_256-vit_base_patch16_224")
+        device = torch.device("cuda" if torch.cuda.is_available() else "cpu")  # Use GPU if available
+
+        # Check if files exist before proceeding
+        if not os.path.exists(vecs_path):
+            return f"Error: IU vectors file not found at {vecs_path}"
+        if not os.path.exists(impr_path):
+            return f"Error: IU impressions file not found at {impr_path}"
+
+        # Load BiomedCLIP model and processor
+        try:
+            model, preprocess = create_model_from_pretrained(model_id)
+            model = model.to(device).eval()  # Move model to device and set to eval mode
+        except Exception as e:
+            return f"Error loading BiomedCLIP model: {e}"
+
+        # Embed the input image
+        try:
+            image = Image.open(image_path).convert("RGB")  # Open and convert image to RGB
+            tensor_img = preprocess(image).unsqueeze(0).to(device)  # Preprocess and add batch dimension
+            with torch.no_grad():
+                query_vec = model.encode_image(tensor_img)  # Get image embedding
+            query_vec = F.normalize(query_vec, dim=-1).cpu().numpy()  # Normalize and move to CPU numpy
+        except Exception as e:
+            return f"Error processing input image: {e}"
+
+        # Load stored IU embeddings
+        try:
+            iu_vecs = np.load(vecs_path)  # Load precomputed IU image vectors
+            iu_vecs = iu_vecs / np.linalg.norm(iu_vecs, axis=1, keepdims=True)  # Normalize vectors
+        except Exception as e:
+            return f"Error loading IU vectors: {e}"
+
+        # Compute cosine similarity between query and all IU vectors
+        similarities = np.dot(iu_vecs, query_vec.squeeze())  # Dot product for cosine similarity
+        best_idx = int(np.argmax(similarities))  # Index of most similar IU image
+
+        # Load the matched impression
+        try:
+            with open(impr_path, "r", encoding="utf-8") as f:
+                records = [json.loads(line) for line in f]  # Load all impression records
+            best_match = records[best_idx]  # Get the best match by index
+        except Exception as e:
+            return f"Error loading impressions metadata: {e}"
+
+        # Return formatted result with UUID and impression text
+        return (
+            f"Closest Match UUID: {best_match['uuid']}\n\n"
+            f"Impression:\n{best_match['impression'].strip()}"
+        )

src/tools/pubmed_tool.py

@@ -0,0 +1,90 @@
+from crewai.tools import BaseTool
+from pydantic import BaseModel, Field
+from typing import List, Dict, Optional, Any
+import os
+import json
+import torch
+import faiss
+import numpy as np
+from pathlib import Path
+from .specter2_embedder import embed_texts_specter2  # Import from same folder
+
+# Input schema for the tool, expects a caption string
+class PubmedQueryInput(BaseModel):
+    caption: str
+
+# Main tool class for PubMed retrieval
+class PubmedRetrievalTool(BaseTool):
+    # Tool name and description
+    name: str = "pubmed_retrieval_tool"
+    description: str = (
+        "Retrieves the most relevant PubMed articles for a given radiology caption."
+    )
+    args_schema: type = PubmedQueryInput
+    metadata: dict = {}
+    
+    def __init__(self, **data):
+        # Initialize the base tool with provided data
+        super().__init__(**data)
+    
+    def _run(self, caption: str = None, **kwargs) -> str:
+        """
+        Retrieves relevant PubMed articles based on a radiology caption.
+        """
+        # Handle edge case where caption might be in kwargs
+        if not caption and 'caption' in kwargs:
+            caption = kwargs['caption']
+        
+        # Validate input: ensure caption is provided and not empty
+        if not caption or not str(caption).strip():
+            return "Error: No caption provided. Unable to search PubMed."
+        
+        caption = str(caption).strip()
+        
+        # Configuration - Updated path handling
+        BASE_DIR = Path(__file__).parent.parent.parent  # Up to main folder
+        default_data_dir = str(BASE_DIR / "data")
+        # Use metadata config if available, otherwise default
+        data_dir = self.metadata.get("DATA_DIR", default_data_dir)
+        top_k = self.metadata.get("TOP_K", 3)
+        
+        try:
+            # Load FAISS index and metadata
+            index_path = os.path.join(data_dir, "text_faiss.bin")
+            metadata_path = os.path.join(data_dir, "raw_abstracts.jsonl")
+            
+            # Check if files exist
+            if not os.path.exists(index_path):
+                return f"Error: FAISS index not found at {index_path}"
+            if not os.path.exists(metadata_path):
+                return f"Error: Metadata file not found at {metadata_path}"
+            
+            # Read FAISS index from disk
+            index = faiss.read_index(index_path)
+            # Load metadata (PubMed abstracts) from JSONL file
+            with open(metadata_path, "r", encoding="utf-8") as f:
+                metadata = [json.loads(line) for line in f]
+            
+            # Embed the input caption using Specter2 model
+            query_vec = embed_texts_specter2([caption]).astype("float32")
+            # Search for top_k most similar articles in FAISS index
+            scores, indices = index.search(query_vec, top_k)
+            
+            # Format results for output
+            formatted = []
+            for i, (score, idx) in enumerate(zip(scores[0], indices[0]), 1):
+                entry = metadata[idx]
+                formatted.append(
+                    f"Citation {i}:\n"
+                    f"PMID: {entry.get('pmid', 'Unknown')}\n"
+                    f"Similarity Score: {score:.3f}\n"
+                    f"Title: {entry.get('title', 'Untitled').strip()}\n"
+                    f"Abstract: {entry.get('abstract', 'No abstract available.').strip()}\n"
+                )
+            
+            # Return formatted citations separated by ---
+            return "\n---\n".join(formatted)
+            
+        except Exception as e:
+            # Handle any errors during retrieval
+            return f"Error during PubMed search: {str(e)}"

src/tools/specter2_embedder.py

@@ -0,0 +1,25 @@
+from transformers import AutoTokenizer, AutoModel
+import torch
+import numpy as np
+from tqdm import tqdm
+
+# Load the tokenizer and model once
+tokenizer = AutoTokenizer.from_pretrained("allenai/specter2_base")  # Initialize tokenizer
+model = AutoModel.from_pretrained("allenai/specter2_base")  # Initialize model
+model.eval()  # Set model to evaluation mode
+device = torch.device("cuda" if torch.cuda.is_available() else "cpu")  # Use GPU if available
+model.to(device)  # Move model to device
+
+def embed_texts_specter2(texts: list[str], batch_size=16) -> np.ndarray:
+    embeddings = []  # List to store embeddings
+
+    for i in tqdm(range(0, len(texts), batch_size), desc="Embedding with SPECTER2"):
+        batch_texts = texts[i:i+batch_size]  # Get batch of texts
+        inputs = tokenizer(batch_texts, padding=True, truncation=True, max_length=512, return_tensors="pt").to(device)  # Tokenize and move to device
+        with torch.no_grad():  # Disable gradient calculation
+            outputs = model(**inputs)  # Forward pass
+            cls_embeddings = outputs.last_hidden_state[:, 0, :]  # CLS token embedding
+            cls_embeddings = torch.nn.functional.normalize(cls_embeddings, p=2, dim=1)  # Normalize embeddings
+        embeddings.append(cls_embeddings.cpu().numpy())  # Move to CPU and convert to numpy
+
+    return np.vstack(embeddings)  # Stack all embeddings into a single array

src/tools/vision_caption_tool.py

@@ -0,0 +1,82 @@
+from crewai.tools import BaseTool, tool
+from pydantic import BaseModel, Field
+from typing import Optional
+from google import genai
+from google.genai.types import Part, GenerateContentConfig
+import os
+
+# Schema for tool arguments, specifying image path and optional prompt
+class VisionCaptionToolSchema(BaseModel):
+    image_path: str
+    prompt: Optional[str] = None
+
+# Main tool class for generating radiology reports from chest X-ray images
+class VisionCaptionTool(BaseTool):
+    name: str = "vision_caption_tool"
+    description: str = (
+        "Generates a structured radiology report from a chest X-ray image using optimized Gemini 2.5 Flash."
+    )
+    args_schema: type = VisionCaptionToolSchema
+    metadata: dict = {}
+
+    # Core method to run the tool logic
+    def _run(self, image_path: str, prompt: Optional[str] = None) -> str:
+        # Retrieve Gemini API key from metadata
+        api_key = self.metadata.get("GEMINI_API_KEY")
+        if not api_key:
+            raise ValueError("GEMINI_API_KEY not found in metadata.")
+        client = genai.Client(api_key=api_key)
+
+        # System prompt defines the expert persona and approach for the model
+        system_prompt = (
+            "You are a board-certified thoracic radiologist with over 20 years of experience in interpreting chest X-rays. "
+            "You are known for your meticulous attention to detail, clinical restraint, and deep respect for image-grounded reasoning. "
+            "You prioritize accuracy over speculation and communicate with diagnostic clarity, always aligning your impressions with what is visibly demonstrable in the radiograph."
+        )
+
+        # Use default optimized prompt if none is provided by the user
+        if prompt is None:
+            prompt = (
+                "Examine the chest X-ray step by step, following a structured A–G radiological workflow (Airway & Mediastinum, Bones & soft tissues, Cardiac silhouette, Diaphragm, Lung fields, Pleura, any Devices/foreign objects, and \"Global\" sanity checks). "
+                "For each region, mentally assess both normal and abnormal findings before synthesizing them into a cohesive narrative report.\n"
+                "All directional terms (left/right) must refer strictly to the PATIENT'S perspective.\n"
+                "When evaluating cardiac size on AP films, assume that mild to moderate enlargement may be projectional unless the heart silhouette is clearly disproportionate or supported by additional findings (e.g., pulmonary congestion). "
+                "If abnormal patterns are seen (e.g., opacities, effusion, pneumothorax, atelectasis, consolidation), propose the most likely clinical significance in a professional and cautious tone, consistent with expert radiology language. "
+                "Weigh the clinical importance of each finding, and simulate a brief SECOND-PASS REVIEW of the image to verify that no significant abnormalities were overlooked.\n"
+                "Your final output must be formatted with two sections only:\n"
+                "FINDINGS: Structured prose following (but not explicitly labeling) the A–G sweep. Mention technical limitations only if they meaningfully impact interpretation. Always explicitly comment on signs of chronic lung disease (emphysema, fibrosis, interstitial changes) as either present or absent.\n"
+                "IMPRESSION: A concise, prioritized interpretation that integrates key findings into a diagnostic hypothesis. Rank findings by CLINICAL SIGNIFICANCE, listing the most urgent or actionable abnormalities first. Remaining key diagnoses/differentials, each with a probability qualifier (\"probable\", \"possible\", etc.). Use confident, direct language—but avoid speculation beyond what is visibly supported."
+            )
+            
+        # Determine the MIME type of the image based on its file extension
+        ext = os.path.splitext(image_path)[-1].lower()
+        if ext == ".png":
+            mime_type = "image/png"
+        elif ext in [".jpg", ".jpeg"]:
+            mime_type = "image/jpeg"
+        else:
+            raise ValueError(f"Unsupported image type: {ext}")
+
+        # Read the image file as bytes
+        with open(image_path, "rb") as f:
+            image_bytes = f.read()
+        
+        # Prepare input parts for the Gemini model: image and prompt
+        parts = [
+            Part.from_bytes(data=image_bytes, mime_type=mime_type),
+            prompt
+        ]
+
+        # Generate content using Gemini 2.5 Flash with specified parameters
+        response = client.models.generate_content(
+            model="gemini-2.5-flash",
+            contents=parts,
+            config=GenerateContentConfig(
+                system_instruction=system_prompt,
+                temperature=0.2,
+                max_output_tokens=2048
+            )
+        )
+
+        # Return the generated report text, stripped of leading/trailing whitespace
+        return response.text.strip()

src/tools_loader.py

@@ -0,0 +1,29 @@
+import os
+from pathlib import Path
+from .tools.vision_caption_tool import VisionCaptionTool  # Import vision caption tool
+from .tools.pubmed_tool import PubmedRetrievalTool        # Import PubMed retrieval tool
+from .tools.iu_retrieval_tool import IUImpressionSearchTool  # Import IU impression search tool
+
+def get_tools():
+    """Create and return all configured tools"""
+    
+    # Get paths and API keys
+    data_dir = Path(__file__).parent.parent / "data"  # Define path to the data directory
+    groq_key = os.getenv("GROQ_API_KEY")              # Retrieve GROQ API key from environment
+    gemini_key = os.getenv("GEMINI_API_KEY")          # Retrieve Gemini API key from environment
+    
+    # Create tools
+    vision_tool = VisionCaptionTool(metadata={"GEMINI_API_KEY": gemini_key})  # Initialize vision caption tool with Gemini API key
+    pubmed_tool = PubmedRetrievalTool(metadata={"DATA_DIR": str(data_dir), "TOP_K": 3})  # Initialize PubMed tool with data directory and top-k parameter
+    iu_tool = IUImpressionSearchTool(metadata={
+        "VEC_PATH": str(data_dir / "iu_vecs.npy"),    # Path to IU vectors file
+        "IMPR_PATH": str(data_dir / "iu_impr.jsonl"), # Path to IU impressions file
+        "MODEL_ID": "hf-hub:microsoft/BiomedCLIP-PubMedBERT_256-vit_base_patch16_224"  # Model identifier for retrieval
+    })
+    
+    # Return all tools in a dictionary
+    return {
+        "vision_tool": vision_tool,
+        "pubmed_tool": pubmed_tool,
+        "iu_tool": iu_tool
+    }