Update src/validation.py

src/validation.py

@@ -11,7 +11,6 @@ from config import (
     MODEL_CATEGORIES,
     EVALUATION_TRACKS,
     ALL_UG40_LANGUAGES,
-    SAMPLE_SIZE_RECOMMENDATIONS,
 )
 
 
@@ -57,8 +56,8 @@ def detect_model_category(model_name: str, author: str, description: str) -> str
     return "community"
 
 
-def validate_file_format_enhanced(file_content: bytes, filename: str) -> Dict:
-    """Enhanced file format validation with stricter requirements."""
+def validate_file_format(file_content: bytes, filename: str) -> Dict:
+    """Validate file format and structure."""
     
     try:
         # Determine file type
@@ -87,7 +86,7 @@ def validate_file_format_enhanced(file_content: bytes, filename: str) -> Dict:
         if len(df) == 0:
             return {"valid": False, "error": "File is empty"}
         
-        # Enhanced validation checks
+        # Validation checks
         validation_issues = []
         
         # Check for required data
@@ -105,7 +104,7 @@ def validate_file_format_enhanced(file_content: bytes, filename: str) -> Dict:
             validation_issues.append(f"Duplicate sample_id values found ({dup_count} duplicates)")
         
         # Data type validation
-        if not df["sample_id"].dtype == "object" and not df["sample_id"].dtype.name.startswith("str"):
+        if not df["sample_id"].dtype == "object":
             df["sample_id"] = df["sample_id"].astype(str)
         
         # Check sample_id format
@@ -135,8 +134,8 @@ def validate_file_format_enhanced(file_content: bytes, filename: str) -> Dict:
         return {"valid": False, "error": f"Error parsing file: {str(e)}"}
 
 
-def validate_predictions_content_enhanced(predictions: pd.DataFrame) -> Dict:
-    """Enhanced prediction content validation with stricter quality checks."""
+def validate_predictions_content(predictions: pd.DataFrame) -> Dict:
+    """Validate prediction content quality."""
     
     issues = []
     warnings = []
@@ -162,7 +161,7 @@ def validate_predictions_content_enhanced(predictions: pd.DataFrame) -> Dict:
     if long_predictions > len(predictions) * 0.01:  # More than 1%
         warnings.append(f"{long_predictions} very long predictions (> 500 characters)")
     
-    # Check for repeated predictions (more stringent)
+    # Check for repeated predictions
     duplicate_predictions = predictions["prediction"].duplicated().sum()
     duplicate_rate = duplicate_predictions / len(predictions)
     quality_metrics["duplicate_rate"] = float(duplicate_rate)
@@ -186,14 +185,6 @@ def validate_predictions_content_enhanced(predictions: pd.DataFrame) -> Dict:
     if placeholder_count > len(predictions) * 0.02:  # More than 2%
         issues.append(f"{placeholder_count} placeholder-like predictions detected")
     
-    # Language detection (basic)
-    non_ascii_rate = predictions["prediction"].str.contains(r"[^\x00-\x7f]", na=False).mean()
-    quality_metrics["non_ascii_rate"] = float(non_ascii_rate)
-    
-    # Check for appropriate character distribution for African languages
-    if non_ascii_rate < 0.1:  # Less than 10% non-ASCII might indicate English-only
-        warnings.append("Low non-ASCII character rate - check if translations include local language scripts")
-    
     # Calculate overall quality score
     quality_score = 1.0
     quality_score -= len(issues) * 0.3  # Major penalty for issues
@@ -217,10 +208,10 @@ def validate_predictions_content_enhanced(predictions: pd.DataFrame) -> Dict:
     }
 
 
-def validate_against_test_set_enhanced(
+def validate_against_test_set(
     predictions: pd.DataFrame, test_set: pd.DataFrame
 ) -> Dict:
-    """Enhanced validation against test set with track-specific analysis."""
+    """Validate predictions against test set."""
     
     # Convert IDs to string for comparison
     pred_ids = set(predictions["sample_id"].astype(str))
@@ -259,22 +250,6 @@ def validate_against_test_set_enhanced(
             "min_required": VALIDATION_CONFIG["min_samples_per_track"][track_name],
         }
     
-    # Language pair coverage analysis
-    pair_coverage = {}
-    for _, row in test_set.iterrows():
-        pair_key = f"{row['source_language']}_{row['target_language']}"
-        if pair_key not in pair_coverage:
-            pair_coverage[pair_key] = {"total": 0, "covered": 0}
-        
-        pair_coverage[pair_key]["total"] += 1
-        if str(row["sample_id"]) in pred_ids:
-            pair_coverage[pair_key]["covered"] += 1
-    
-    # Calculate pair-wise coverage rates
-    for pair_key in pair_coverage:
-        pair_info = pair_coverage[pair_key]
-        pair_info["coverage_rate"] = pair_info["covered"] / pair_info["total"]
-    
     # Missing rate validation
     missing_rate = len(missing_ids) / len(test_ids)
     meets_missing_threshold = missing_rate <= VALIDATION_CONFIG["max_missing_rate"]
@@ -288,98 +263,29 @@ def validate_against_test_set_enhanced(
         "meets_missing_threshold": meets_missing_threshold,
         "is_complete": overall_coverage == 1.0,
         "track_coverage": track_coverage,
-        "pair_coverage": pair_coverage,
         "missing_ids_sample": list(missing_ids)[:10],
         "extra_ids_sample": list(extra_ids)[:10],
     }
 
 
-def assess_statistical_adequacy(
-    validation_result: Dict, model_category: str
-) -> Dict:
-    """Assess statistical adequacy for scientific evaluation."""
-    
-    adequacy_assessment = {
-        "overall_adequate": True,
-        "track_adequacy": {},
-        "recommendations": [],
-        "statistical_power_estimate": {},
-    }
-    
-    track_coverage = validation_result.get("track_coverage", {})
-    
-    for track_name, coverage_info in track_coverage.items():
-        track_config = EVALUATION_TRACKS[track_name]
-        
-        # Sample size adequacy
-        covered_samples = coverage_info["covered_samples"]
-        min_required = coverage_info["min_required"]
-        
-        sample_adequate = covered_samples >= min_required
-        
-        # Coverage rate adequacy
-        coverage_rate = coverage_info["coverage_rate"]
-        coverage_adequate = coverage_rate >= 0.8  # 80% coverage minimum
-        
-        # Statistical power estimation (simplified)
-        estimated_power = min(1.0, covered_samples / (min_required * 1.5))
-        
-        track_adequate = sample_adequate and coverage_adequate
-        
-        adequacy_assessment["track_adequacy"][track_name] = {
-            "sample_adequate": sample_adequate,
-            "coverage_adequate": coverage_adequate,
-            "overall_adequate": track_adequate,
-            "covered_samples": covered_samples,
-            "min_required": min_required,
-            "coverage_rate": coverage_rate,
-            "estimated_power": estimated_power,
-        }
-        
-        if not track_adequate:
-            adequacy_assessment["overall_adequate"] = False
-            
-        adequacy_assessment["statistical_power_estimate"][track_name] = estimated_power
-    
-    # Generate recommendations
-    if not adequacy_assessment["overall_adequate"]:
-        inadequate_tracks = [
-            track for track, info in adequacy_assessment["track_adequacy"].items()
-            if not info["overall_adequate"]
-        ]
-        adequacy_assessment["recommendations"].append(
-            f"Insufficient samples for tracks: {', '.join(inadequate_tracks)}"
-        )
-    
-    # Category-specific recommendations
-    if model_category == "commercial" and not adequacy_assessment["track_adequacy"].get("google_comparable", {}).get("overall_adequate", False):
-        adequacy_assessment["recommendations"].append(
-            "Commercial models should ensure adequate coverage of Google-comparable track"
-        )
-    
-    return adequacy_assessment
-
-
-def generate_scientific_validation_report(
+def generate_validation_report(
     format_result: Dict,
     content_result: Dict,
     test_set_result: Dict,
-    adequacy_result: Dict,
     model_name: str = "",
     detected_category: str = "community",
 ) -> str:
-    """Generate comprehensive scientific validation report."""
+    """Generate comprehensive validation report."""
     
     report = []
     
     # Header
-    report.append(f"# 🔬 Scientific Validation Report: {model_name or 'Submission'}")
+    report.append(f"### 🔬 Validation Report: {model_name or 'Submission'}")
     report.append("")
     
     # Model categorization
     category_info = MODEL_CATEGORIES.get(detected_category, MODEL_CATEGORIES["community"])
     report.append(f"**Detected Model Category**: {category_info['name']}")
-    report.append(f"**Category Description**: {category_info['description']}")
     report.append("")
     
     # File format validation
@@ -425,14 +331,14 @@ def generate_scientific_validation_report(
     report.append("")
     
     # Track-specific coverage analysis
-    report.append("## 📊 Track-Specific Analysis")
+    report.append("#### 📊 Track-Specific Analysis")
     
     track_coverage = test_set_result.get("track_coverage", {})
     for track_name, coverage_info in track_coverage.items():
         track_config = EVALUATION_TRACKS[track_name]
         
         status = "✅" if coverage_info["meets_minimum"] else "❌"
-        report.append(f"### {status} {track_config['name']}")
+        report.append(f"**{status} {track_config['name']}**:")
         
         report.append(f"   - **Samples**: {coverage_info['covered_samples']:,} / {coverage_info['total_samples']:,}")
         report.append(f"   - **Coverage**: {coverage_info['coverage_rate']:.1%}")
@@ -440,37 +346,12 @@ def generate_scientific_validation_report(
         report.append(f"   - **Status**: {'Adequate' if coverage_info['meets_minimum'] else 'Insufficient'}")
         report.append("")
     
-    # Statistical adequacy assessment
-    report.append("## 🔬 Statistical Adequacy Assessment")
-    
-    if adequacy_result["overall_adequate"]:
-        report.append("✅ **Overall Assessment**: Statistically adequate for scientific evaluation")
-    else:
-        report.append("❌ **Overall Assessment**: Insufficient for rigorous scientific evaluation")
-    
-    # Track adequacy details
-    for track_name, track_adequacy in adequacy_result["track_adequacy"].items():
-        track_config = EVALUATION_TRACKS[track_name]
-        power = track_adequacy["estimated_power"]
-        
-        status = "✅" if track_adequacy["overall_adequate"] else "❌"
-        report.append(f"   - {status} **{track_config['name']}**: Statistical power ≈ {power:.1%}")
-    
-    # Recommendations
-    if adequacy_result["recommendations"]:
-        report.append("")
-        report.append("## 💡 Recommendations")
-        for rec in adequacy_result["recommendations"]:
-            report.append(f"   - {rec}")
-    
     # Final verdict
-    report.append("")
     all_checks_pass = (
         format_result["valid"] and
         not content_result["has_issues"] and
         overall_coverage >= 0.95 and
-        meets_threshold and
-        adequacy_result["overall_adequate"]
+        meets_threshold
     )
     
     can_evaluate_with_limits = (
@@ -480,18 +361,17 @@ def generate_scientific_validation_report(
     )
     
     if all_checks_pass:
-        report.append("🎉 **Final Verdict**: Ready for scientific evaluation!")
+        report.append("🎉 **Final Verdict**: Ready for evaluation!")
     elif can_evaluate_with_limits:
         report.append("⚠️ **Final Verdict**: Can be evaluated with limitations")
-        report.append("   - Results will include notes about statistical limitations")
-        report.append("   - Consider improving coverage/quality for publication-grade results")
+        report.append("   - Results will include notes about limitations")
     else:
         report.append("❌ **Final Verdict**: Please address critical issues before submission")
     
     return "\n".join(report)
 
 
-def validate_submission_scientific(
+def validate_submission(
     file_content: bytes, 
     filename: str, 
     test_set: pd.DataFrame, 
@@ -499,73 +379,59 @@ def validate_submission_scientific(
     author: str = "",
     description: str = ""
 ) -> Dict:
-    """Complete scientific validation pipeline for submissions."""
+    """Complete validation pipeline for submissions."""
     
     # Step 1: Detect model category
     detected_category = detect_model_category(model_name, author, description)
     
-    # Step 2: Enhanced file format validation
-    format_result = validate_file_format_enhanced(file_content, filename)
+    # Step 2: File format validation
+    format_result = validate_file_format(file_content, filename)
     if not format_result["valid"]:
         return {
             "valid": False,
-            "can_evaluate": False,  # New field for evaluation eligibility
+            "can_evaluate": False,
             "category": detected_category,
-            "report": generate_scientific_validation_report(
-                format_result, {}, {}, {}, model_name, detected_category
+            "report": generate_validation_report(
+                format_result, {}, {}, model_name, detected_category
             ),
             "predictions": None,
-            "adequacy": {},
         }
     
     predictions = format_result["dataframe"]
     
-    # Step 3: Enhanced content validation
-    content_result = validate_predictions_content_enhanced(predictions)
+    # Step 3: Content validation
+    content_result = validate_predictions_content(predictions)
     
-    # Step 4: Enhanced test set validation
-    test_set_result = validate_against_test_set_enhanced(predictions, test_set)
+    # Step 4: Test set validation
+    test_set_result = validate_against_test_set(predictions, test_set)
     
-    # Step 5: Statistical adequacy assessment
-    adequacy_result = assess_statistical_adequacy(test_set_result, detected_category)
-    
-    # Step 6: Generate comprehensive report
-    report = generate_scientific_validation_report(
-        format_result, content_result, test_set_result, adequacy_result,
-        model_name, detected_category
+    # Step 5: Generate report
+    report = generate_validation_report(
+        format_result, content_result, test_set_result, model_name, detected_category
     )
     
-    # Overall validity determination (strict scientific standards)
-    is_scientifically_valid = (
+    # Overall validity determination
+    is_valid = (
         format_result["valid"] and
         not content_result["has_issues"] and
         test_set_result["overall_coverage"] >= 0.95 and
-        test_set_result["meets_missing_threshold"] and
-        adequacy_result["overall_adequate"]
+        test_set_result["meets_missing_threshold"]
     )
     
-    # Evaluation eligibility (more permissive - can evaluate with limitations)
+    # Evaluation eligibility (more permissive)
     can_evaluate = (
         format_result["valid"] and
-        test_set_result["overall_coverage"] >= 0.8 and  # 80% coverage minimum
-        not any("❌" in issue for issue in content_result.get("issues", []))  # No critical content issues
+        test_set_result["overall_coverage"] >= 0.8 and
+        not any("❌" in issue for issue in content_result.get("issues", []))
     )
     
     return {
-        "valid": is_scientifically_valid,
-        "can_evaluate": can_evaluate,  # New field
+        "valid": is_valid,
+        "can_evaluate": can_evaluate,
         "category": detected_category,
         "coverage": test_set_result["overall_coverage"],
         "report": report,
         "predictions": predictions,
-        "adequacy": adequacy_result,
         "quality_score": content_result.get("quality_score", 0.8),
         "track_coverage": test_set_result.get("track_coverage", {}),
-        "scientific_metadata": {
-            "validation_timestamp": pd.Timestamp.now().isoformat(),
-            "validation_version": "2.0-scientific",
-            "detected_category": detected_category,
-            "statistical_adequacy": adequacy_result["overall_adequate"],
-            "evaluation_recommended": can_evaluate,
-        },
     }