Spaces:
Sleeping
Sleeping
File size: 4,149 Bytes
0855f92 |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 |
#!/usr/bin/env python3
"""
Validation script for the updated Napolab data structure
"""
from data_loader import NapolabDataLoader
from manage_data import validate_yaml_structure
import pandas as pd
def main():
"""Validate the updated data structure."""
print("🔍 Validating Updated Napolab Data Structure")
print("=" * 50)
print("📚 Data Source: Master's thesis 'Lessons learned from the evaluation of Portuguese language models'")
print(" by Ruan Chaves Rodrigues (2023) - University of Malta")
print(" Available at: https://www.um.edu.mt/library/oar/handle/123456789/120557")
print("=" * 50)
# Load data
data_loader = NapolabDataLoader()
data = data_loader.data
# Validate structure
print("\n1. Validating YAML structure...")
if validate_yaml_structure(data):
print("✅ YAML structure is valid!")
else:
print("❌ YAML structure has issues!")
return
# Check datasets
print("\n2. Checking datasets...")
datasets = data_loader.get_datasets()
print(f"📊 Found {len(datasets)} datasets:")
for name, info in datasets.items():
print(f" - {name}: {info['name']} ({', '.join(info['tasks'])})")
# Check benchmark results
print("\n3. Checking benchmark results...")
benchmark_results = data_loader.get_benchmark_results()
print(f"🏆 Found {len(benchmark_results)} benchmark datasets:")
for dataset_name, models in benchmark_results.items():
print(f" - {dataset_name}: {len(models)} models")
# Check model metadata
print("\n4. Checking model metadata...")
model_metadata = data_loader.get_model_metadata()
print(f"🤖 Found {len(model_metadata)} models:")
# Group models by architecture
architectures = {}
for model_name, metadata in model_metadata.items():
arch = metadata['architecture']
if arch not in architectures:
architectures[arch] = []
architectures[arch].append(model_name)
for arch, models in architectures.items():
print(f" - {arch}: {len(models)} models")
for model in models[:3]: # Show first 3 models
print(f" * {model}")
if len(models) > 3:
print(f" ... and {len(models) - 3} more")
# Test data access functions
print("\n5. Testing data access functions...")
# Test getting available models for a dataset
test_dataset = list(benchmark_results.keys())[0]
models = data_loader.get_available_models_for_dataset(test_dataset)
print(f" Available models for {test_dataset}: {len(models)} models")
# Test getting model info
if models:
test_model = models[0]
model_info = data_loader.get_model_info(test_model)
if model_info:
print(f" Model {test_model}: {model_info['parameters']:,} parameters")
# Create a summary table
print("\n6. Creating summary table...")
summary_data = []
for dataset_name, models in benchmark_results.items():
for model_name, metrics in models.items():
if model_name in model_metadata:
summary_data.append({
'Dataset': dataset_name,
'Model': model_name,
'Architecture': model_metadata[model_name]['architecture'],
'Parameters': model_metadata[model_name]['parameters'],
'Performance': metrics.get('accuracy', 0)
})
if summary_data:
df = pd.DataFrame(summary_data)
print(f"📋 Summary: {len(df)} model-dataset combinations")
print(f" Average performance: {df['Performance'].mean():.3f}")
print(f" Best performance: {df['Performance'].max():.3f}")
print(f" Models with >0.9 performance: {(df['Performance'] > 0.9).sum()}")
print("\n✅ Validation completed successfully!")
print("🚀 The updated data structure is ready to use!")
if __name__ == "__main__":
main() |