Enhance Direct LLM Evaluator and Judge Evaluator:

- Update LLM generation method for improved response handling and timeout settings.
- Adjust target compliance metrics from 30s to 60s for better evaluation consistency.
- Implement validation for query consistency and model type differences across systems in Judge Evaluator.

evaluation/direct_llm_evaluator.py

@@ -87,13 +87,26 @@ Please provide comprehensive medical advice including:
 Provide evidence-based, actionable medical guidance.
 """
             
-            # Direct LLM generation
-            response = self.llm_client.generate_completion(direct_prompt)
-            medical_advice = response.get('content', '') if isinstance(response, dict) else str(response)
+            # Direct LLM generation (same parameters as RAG system for fair comparison)
+            response = self.llm_client.analyze_medical_query(
+                query=direct_prompt,
+                max_tokens=1600,  # Same as RAG system primary setting  
+                timeout=60.0      # Increased timeout for stable evaluation
+            )
+            # Extract medical advice from response (Med42 client returns dict with 'raw_response')
+            if isinstance(response, dict):
+                medical_advice = response.get('raw_response', '') or response.get('content', '')
+            else:
+                medical_advice = str(response)
             
             llm_time = time.time() - llm_start
             total_time = time.time() - overall_start
             
+            # Check if response is valid (not empty) - focus on content, not timeout
+            if not medical_advice or len(medical_advice.strip()) == 0:
+                print(f"❌ Direct LLM returned empty response after {total_time:.2f}s")
+                raise ValueError("Empty response from LLM - no content generated")
+            
             # Create result
             result = {
                 "query": query,
@@ -103,7 +116,7 @@ Provide evidence-based, actionable medical guidance.
                 "latency_metrics": {
                     "total_latency": total_time,
                     "llm_generation_time": llm_time,
-                    "meets_target": total_time <= 30.0
+                    "meets_target": total_time <= 60.0
                 },
                 
                 # Metrics 2-4: Not applicable for direct LLM
@@ -167,6 +180,10 @@ Provide evidence-based, actionable medical guidance.
             }
             
             self.direct_results.append(error_result)
+            
+            # Do NOT add failed queries to medical_outputs for judge evaluation
+            # Only successful queries with valid medical advice should be evaluated
+            
             return error_result
     
     def parse_queries_from_file(self, filepath: str) -> Dict[str, List[Dict]]:
@@ -238,7 +255,7 @@ Provide evidence-based, actionable medical guidance.
                     category_stats[category] = {
                         "average_latency": sum(cat_latencies) / len(cat_latencies),
                         "query_count": len(cat_latencies),
-                        "target_compliance": sum(1 for lat in cat_latencies if lat <= 30.0) / len(cat_latencies)
+                        "target_compliance": sum(1 for lat in cat_latencies if lat <= 60.0) / len(cat_latencies)
                     }
                 else:
                     category_stats[category] = {
@@ -255,7 +272,7 @@ Provide evidence-based, actionable medical guidance.
                 "successful_queries": len(successful_results),
                 "total_queries": len(self.direct_results),
                 "success_rate": len(successful_results) / len(self.direct_results),
-                "target_compliance": sum(1 for lat in latencies if lat <= 30.0) / len(latencies)
+                "target_compliance": sum(1 for lat in latencies if lat <= 60.0) / len(latencies)
             }
         else:
             category_stats = {cat: {"average_latency": 0.0, "query_count": 0, "target_compliance": 0.0} 
@@ -386,7 +403,7 @@ if __name__ == "__main__":
     print(f"Overall Performance:")
     print(f"   Average Latency: {overall_results['average_latency']:.2f}s")
     print(f"   Success Rate: {overall_results['successful_queries']}/{overall_results['total_queries']}")
-    print(f"   30s Target Compliance: {overall_results['target_compliance']:.1%}")
+    print(f"   60s Target Compliance: {overall_results['target_compliance']:.1%}")
     
     print(f"\nApplicable Metrics:")
     print(f"   ✅ Metric 1 (Latency): Measured")
@@ -399,4 +416,4 @@ if __name__ == "__main__":
     print(f"\n✅ Direct LLM evaluation complete!")
     print(f"📊 Statistics: {stats_path}")
     print(f"📝 Medical Outputs: {outputs_path}")
-    print(f"\n💡 Next step: Run llm_judge_evaluator.py for metrics 5-6")
+    print(f"\n💡 Next step: Run python metric5_6_llm_judge_evaluator.py rag,direct for metrics 5-6")

evaluation/metric5_6_llm_judge_evaluator.py

@@ -137,14 +137,17 @@ class LLMJudgeEvaluator:
         
         for system in systems:
             if system == "rag":
-                pattern = str(results_dir / "medical_outputs_*.json")
+                # Use more specific pattern to exclude direct files
+                pattern = str(results_dir / "medical_outputs_[0-9]*.json")
             elif system == "direct":
                 pattern = str(results_dir / "medical_outputs_direct_*.json")
             else:
                 # Future extension: support other systems
                 pattern = str(results_dir / f"medical_outputs_{system}_*.json")
             
+            print(f"🔍 Searching for {system} with pattern: {pattern}")
             output_files = glob.glob(pattern)
+            print(f"🔍 Found files for {system}: {output_files}")
             
             if not output_files:
                 raise FileNotFoundError(f"No medical outputs files found for {system} system")
@@ -547,6 +550,38 @@ if __name__ == "__main__":
             if len(set(query_counts)) > 1:
                 print(f"⚠️ Warning: Systems have different query counts: {dict(zip(systems, query_counts))}")
             
+            # Validate systems processed same queries (for scientific comparison)
+            print(f"🔍 Validating query consistency across systems...")
+            if len(systems) > 1:
+                first_system_queries = [q['query'] for q in systems_outputs[systems[0]]]
+                for i, system in enumerate(systems[1:], 1):
+                    system_queries = [q['query'] for q in systems_outputs[system]]
+                    
+                    if first_system_queries != system_queries:
+                        print(f"⚠️ Warning: {systems[0]} and {system} processed different queries!")
+                        # Show first difference
+                        for j, (q1, q2) in enumerate(zip(first_system_queries, system_queries)):
+                            if q1 != q2:
+                                print(f"   Query {j+1} differs:")
+                                print(f"   {systems[0]}: {q1[:50]}...")
+                                print(f"   {system}: {q2[:50]}...")
+                                break
+                    else:
+                        print(f"✅ {systems[0]} and {system} processed identical queries")
+            
+            # Validate systems have different model types
+            model_types = set()
+            for system, outputs in systems_outputs.items():
+                if outputs:
+                    model_type = outputs[0].get('model_type', 'unknown')
+                    model_types.add(model_type)
+                    print(f"🏷️ {system.upper()} system model_type: {model_type}")
+            
+            if len(model_types) == 1:
+                print(f"⚠️ Warning: All systems have same model_type - this may not be a valid comparison!")
+            else:
+                print(f"✅ Systems have different model_types: {model_types}")
+            
             print(f"📊 Comparing {len(systems)} systems with {min(query_counts)} queries each")
             print(f"🎯 Metrics: 5 (Actionability) + 6 (Evidence Quality)")
             print(f"⚡ Strategy: Single comparison call for maximum consistency")