Add some debugging tools

agent/agent.py

@@ -40,26 +40,30 @@ class MultiModalAgent:
         # Cache for answers
         self.answer_cache = {}
         
-    def __call__(self, question: str) -> str:
+    def __call__(self, question: str, task_id: Optional[str] = None) -> str:
         """
         Process a question and return an answer.
         
         Args:
             question: The question to answer
+            task_id: The task ID (optional)
             
         Returns:
             Answer to the question
         """
         logger.info(f"Processing question: {question[:100]}...")
+        if task_id:
+            logger.info(f"Task ID: {task_id}")
         
         # Check answer cache
-        if question in self.answer_cache:
+        cache_key = f"{task_id}:{question}" if task_id else question
+        if cache_key in self.answer_cache:
             logger.info("Answer found in cache")
-            return self.answer_cache[question]
+            return self.answer_cache[cache_key]
         
         try:
             # Analyze the question
-            analysis = self.question_analyzer.analyze_question(question)
+            analysis = self.question_analyzer.analyze_question(question, task_id)
             logger.info(f"Question analysis: {analysis}")
             
             # Handle general questions that don't require file processing
@@ -67,7 +71,7 @@ class MultiModalAgent:
                 logger.info("No file reference found in question, trying to answer directly")
                 direct_answer = self._answer_without_file(question)
                 if direct_answer:
-                    self.answer_cache[question] = direct_answer
+                    self.answer_cache[cache_key] = direct_answer
                     return direct_answer
                 
                 # If direct answering failed, try to find a file in the resource directory
@@ -75,7 +79,19 @@ class MultiModalAgent:
                 analysis['file_path'] = self._find_most_relevant_file(question)
                 if not analysis['file_path']:
                     logger.warning("No relevant file found for the question")
-                    return "I couldn't find a relevant file to answer this question."
+                    # List available files for debugging
+                    try:
+                        files = os.listdir(self.resource_dir)
+                        logger.info(f"Available files in {self.resource_dir}: {files}")
+                    except Exception as e:
+                        logger.error(f"Error listing files in resource directory: {e}")
+                    
+                    # Check if resource directory exists
+                    if not os.path.exists(self.resource_dir):
+                        logger.error(f"Resource directory does not exist: {self.resource_dir}")
+                        return f"Error: Resource directory not found at {self.resource_dir}. Please check the path."
+                    
+                    return "I couldn't find a relevant file to answer this question. Please provide more context or specify a file."
             
             # Extract content from the file
             file_path = analysis['file_path']
@@ -95,7 +111,7 @@ class MultiModalAgent:
             answer = self._process_content(content, handler, question)
             
             # Cache the answer
-            self.answer_cache[question] = answer
+            self.answer_cache[cache_key] = answer
             
             return answer
         except Exception as e:
@@ -232,10 +248,21 @@ class MultiModalAgent:
             
             # Return the most relevant file if it has a non-zero score
             if scores and scores[0][1] > 0:
+                logger.info(f"Found relevant file: {scores[0][0]} with score {scores[0][1]}")
                 return scores[0][0]
             
-            # If no relevant file is found based on the question, return None
+            # If no relevant file is found based on the question, try to default to the metadata file
+            if not scores or scores[0][1] == 0:
+                # Look for metadata file as a fallback
+                metadata_path = os.path.join(self.resource_dir, 'metadata.jsonl')
+                if os.path.exists(metadata_path):
+                    logger.info("No specific file found, defaulting to metadata.jsonl")
+                    return metadata_path
+                
+            # If we get here, no relevant file was found
+            logger.warning("No relevant file found for the question")
             return None
+            
         except Exception as e:
             logger.error(f"Error finding relevant file: {e}")
             return None

agent/utils/debug_metadata.py

@@ -0,0 +1,89 @@
+"""
+Test script to debug metadata loading and file finding.
+"""
+import os
+import json
+import sys
+
+# Add the parent directory to sys.path
+sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
+
+from agent.utils.question_analyzer import QuestionAnalyzer
+
+def main():
+    """Main function to test metadata loading and file finding."""
+    # Get the resource directory
+    resource_dir = os.path.join(os.path.dirname(os.path.dirname(os.path.abspath(__file__))), 'resource')
+    print(f"Resource directory: {resource_dir}")
+    
+    # Check if the directory exists
+    if not os.path.exists(resource_dir):
+        print(f"Resource directory does not exist: {resource_dir}")
+        return
+    
+    # List files in the directory
+    print("Files in resource directory:")
+    for file in os.listdir(resource_dir):
+        print(f"  {file}")
+    
+    # Check for metadata.jsonl
+    metadata_path = os.path.join(resource_dir, 'metadata.jsonl')
+    if not os.path.exists(metadata_path):
+        print(f"Metadata file does not exist: {metadata_path}")
+        return
+    
+    # Load metadata
+    print("\nLoading metadata...")
+    question_analyzer = QuestionAnalyzer(resource_dir)
+    
+    # Print metadata entries
+    print(f"Metadata entries: {len(question_analyzer.metadata)}")
+    
+    # Print first few entries
+    count = 0
+    for task_id, entry in question_analyzer.metadata.items():
+        print(f"\nTask ID: {task_id}")
+        print(f"Question: {entry.get('Question', 'N/A')[:100]}...")
+        print(f"File Name: {entry.get('file_name', 'N/A')}")
+        print(f"Expected Answer: {entry.get('Final answer', 'N/A')}")
+        
+        # Check if the file exists
+        if entry.get('file_name'):
+            file_path = os.path.join(resource_dir, entry['file_name'])
+            if os.path.exists(file_path):
+                print(f"✅ File exists: {file_path}")
+            else:
+                print(f"❌ File does not exist: {file_path}")
+        
+        count += 1
+        if count >= 5:
+            break
+    
+    # Test file finding
+    print("\nTesting file finding...")
+    test_questions = []
+    
+    with open(metadata_path, 'r', encoding='utf-8') as f:
+        for line in f:
+            entry = json.loads(line.strip())
+            if 'Question' in entry and 'file_name' in entry and entry['file_name']:
+                test_questions.append({
+                    'task_id': entry.get('task_id'),
+                    'question': entry['Question'],
+                    'file_name': entry['file_name']
+                })
+                if len(test_questions) >= 5:
+                    break
+    
+    for q in test_questions:
+        print(f"\nQuestion: {q['question'][:100]}...")
+        print(f"Expected file: {q['file_name']}")
+        
+        file_path = question_analyzer.find_relevant_file(q['question'], q['task_id'])
+        if file_path:
+            print(f"✅ Found file: {os.path.basename(file_path)}")
+        else:
+            print("❌ No file found")
+
+if __name__ == "__main__":
+    main()

agent/utils/question_analyzer.py

@@ -88,23 +88,49 @@ class QuestionAnalyzer:
         # Check if task_id is in metadata and has a file_name
         if task_id and task_id in self.metadata:
             file_name = self.metadata[task_id].get('file_name')
-            if file_name:
+            if file_name and file_name.strip():  # Make sure file_name is not empty
                 file_path = os.path.join(self.resource_dir, file_name)
                 if os.path.exists(file_path):
+                    print(f"Found file in metadata for task_id {task_id}: {file_path}")
                     return file_path
         
+        # Try to find task_id in all metadata entries by matching the question
+        if not task_id:
+            for entry_id, entry in self.metadata.items():
+                if entry.get('Question') and entry.get('Question') == question:
+                    file_name = entry.get('file_name')
+                    if file_name and file_name.strip():
+                        file_path = os.path.join(self.resource_dir, file_name)
+                        if os.path.exists(file_path):
+                            print(f"Found file in metadata by matching question: {file_path}")
+                            return file_path
+        
         # Extract file mention from question
         file_mention = self.extract_file_mention(question)
         if file_mention:
             # Check if the mentioned file exists
             file_path = os.path.join(self.resource_dir, file_mention)
             if os.path.exists(file_path):
+                print(f"Found file by direct mention: {file_path}")
                 return file_path
                 
             # Check if there's a file with a similar name
             for file_name in os.listdir(self.resource_dir):
                 if file_mention.lower() in file_name.lower():
-                    return os.path.join(self.resource_dir, file_name)
+                    file_path = os.path.join(self.resource_dir, file_name)
+                    print(f"Found file by partial name match: {file_path}")
+                    return file_path
+        
+        # Look for UUID pattern in the question which might be a file name without extension
+        uuid_pattern = r'([0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12})'
+        uuid_match = re.search(uuid_pattern, question, re.IGNORECASE)
+        if uuid_match:
+            uuid = uuid_match.group(1)
+            for file_name in os.listdir(self.resource_dir):
+                if uuid in file_name:
+                    file_path = os.path.join(self.resource_dir, file_name)
+                    print(f"Found file by UUID match: {file_path}")
+                    return file_path
                     
         # If no file is found, try to find a file mentioned in the metadata
         if task_id and task_id in self.metadata:
@@ -131,9 +157,58 @@ class QuestionAnalyzer:
                     best_match = file_name
                     
             if best_match:
-                return os.path.join(self.resource_dir, best_match)
+                file_path = os.path.join(self.resource_dir, best_match)
+                print(f"Found file by keyword matching: {file_path}")
+                return file_path
+                
+        # If still no match, check the content of metadata.jsonl for clues
+        try:
+            with open(self.metadata_path, 'r', encoding='utf-8') as f:
+                for line in f:
+                    entry = json.loads(line.strip())
+                    if 'Question' in entry and entry['Question'] and 'file_name' in entry and entry['file_name']:
+                        # Compare with current question
+                        if self._questions_are_similar(question, entry['Question']):
+                            file_name = entry['file_name']
+                            file_path = os.path.join(self.resource_dir, file_name)
+                            if os.path.exists(file_path):
+                                print(f"Found file by similar question in metadata: {file_path}")
+                                return file_path
+        except Exception as e:
+            print(f"Error searching metadata for similar questions: {e}")
                 
         return None
+        
+    def _questions_are_similar(self, q1: str, q2: str) -> bool:
+        """
+        Check if two questions are similar.
+        
+        Args:
+            q1: First question
+            q2: Second question
+            
+        Returns:
+            True if the questions are similar, False otherwise
+        """
+        # Convert to lowercase and remove punctuation
+        q1 = re.sub(r'[^\w\s]', '', q1.lower())
+        q2 = re.sub(r'[^\w\s]', '', q2.lower())
+        
+        # Split into words
+        words1 = set(q1.split())
+        words2 = set(q2.split())
+        
+        # Calculate Jaccard similarity
+        intersection = len(words1.intersection(words2))
+        union = len(words1.union(words2))
+        
+        if union == 0:
+            return False
+            
+        similarity = intersection / union
+        
+        # Return True if similarity is above threshold
+        return similarity > 0.5
     
     def _extract_keywords(self, text: str) -> Set[str]:
         """
@@ -183,6 +258,13 @@ class QuestionAnalyzer:
             'expected_answer': None,
         }
         
+        # Try to extract task_id from the question if not provided
+        if not task_id:
+            task_id_match = re.search(r'task_id[: ]+([\w\-]+)', question, re.IGNORECASE)
+            if task_id_match:
+                result['task_id'] = task_id_match.group(1)
+                task_id = result['task_id']
+        
         # Find relevant file
         file_path = self.find_relevant_file(question, task_id)
         if file_path:

app.py

@@ -101,11 +101,13 @@ def run_and_submit_all( profile: gr.OAuthProfile | None):
             print(f"Skipping item with missing task_id or question: {item}")
             continue
         try:
-            submitted_answer = agent(question_text)
+            logger.info(f"Processing question with task_id: {task_id}")
+            logger.info(f"Question text: {question_text[:100]}...")
+            submitted_answer = agent(question_text, task_id)
             answers_payload.append({"task_id": task_id, "submitted_answer": submitted_answer})
             results_log.append({"Task ID": task_id, "Question": question_text, "Submitted Answer": submitted_answer})
         except Exception as e:
-             print(f"Error running agent on task {task_id}: {e}")
+             logger.error(f"Error running agent on task {task_id}: {e}", exc_info=True)
              results_log.append({"Task ID": task_id, "Question": question_text, "Submitted Answer": f"AGENT ERROR: {e}"})
 
     if not answers_payload: