Spaces:

DynamicSuperb
/

leaderboard

Running

File size: 22,912 Bytes

<!DOCTYPE html>
<html lang="en">
<head>
    <meta charset="UTF-8">
    <meta name="viewport" content="width=device-width, initial-scale=1.0">
    <title>Checker-V1.1: Dynamic-SUPERB Validator</title>
    <style>
        body {
            font-family: -apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, sans-serif;
            margin: 0;
            padding: 20px;
            background: linear-gradient(120deg, #f8fafc 0%, #e9ecef 100%);
            color: #212529;
        }
        
        /* Menu Styles */
        .menu-container {
            position: fixed;
            top: 15px;
            left: 15px;
            z-index: 1000;
        }
        
        .menu-icon {
            font-size: 28px;
            cursor: pointer;
            user-select: none;
            color: #667eea;
            background: white;
            padding: 8px 12px;
            border-radius: 8px;
            box-shadow: 0 2px 8px rgba(0,0,0,0.15);
            transition: all 0.2s ease;
        }
        
        .menu-icon:hover {
            background: #667eea;
            color: white;
            transform: scale(1.05);
        }
        
        .menu-list {
            display: none;
            position: absolute;
            top: 50px;
            left: 0;
            background: white;
            border: 1px solid #dee2e6;
            border-radius: 8px;
            box-shadow: 0 4px 16px rgba(0,0,0,0.15);
            min-width: 220px;
            font-size: 14px;
            overflow: hidden;
        }
        
        .menu-list.show {
            display: block;
            animation: slideDown 0.2s ease;
        }
        
        @keyframes slideDown {
            from { opacity: 0; transform: translateY(-10px); }
            to { opacity: 1; transform: translateY(0); }
        }
        
        .menu-list a {
            display: block;
            padding: 12px 16px;
            color: #495057;
            text-decoration: none;
            border-bottom: 1px solid #f8f9fa;
            transition: background-color 0.2s ease;
        }
        
        .menu-list a:last-child {
            border-bottom: none;
        }
        
        .menu-list a:hover {
            background-color: #667eea;
            color: white;
        }
        
        .menu-list a::before {
            content: "→ ";
            margin-right: 8px;
            opacity: 0;
            transition: opacity 0.2s ease;
        }
        
        .menu-list a:hover::before {
            opacity: 1;
        }
        
        .container {
            max-width: 800px;
            margin: 0 auto;
            background: white;
            border-radius: 8px;
            box-shadow: 0 2px 10px rgba(0,0,0,0.1);
            overflow: hidden;
        }
        
        .header {
            background: linear-gradient(135deg, #667eea 0%, #764ba2 100%);
            color: white;
            padding: 30px;
            text-align: center;
        }
        
        .header h1 {
            margin: 0;
            font-size: 2.5em;
            font-weight: 300;
        }
        
        .controls {
            padding: 20px 30px;
            border-bottom: 1px solid #dee2e6;
            background: #f8f9fa;
        }
        
        .upload-section {
            background: #f5f7fa;
            border-radius: 10px;
            padding: 24px;
            margin-bottom: 24px;
            border: 1.5px solid #e0e3e8;
        }
        
        label {
            display: block;
            margin-bottom: 5px;
            font-weight: 600;
            color: #495057;
        }
        
        input[type="file"] {
            width: 100%;
            margin: 10px 0;
            padding: 12px;
            border: 2px solid #dee2e6;
            border-radius: 6px;
            background: white;
        }
        
        .btn {
            background: linear-gradient(90deg, #667eea 0%, #764ba2 100%);
            color: white;
            padding: 12px 28px;
            border: none;
            border-radius: 6px;
            cursor: pointer;
            font-size: 1.08em;
            font-weight: 600;
            margin: 8px 12px 0 0;
            transition: transform 0.2s;
        }
        
        .btn:hover {
            transform: translateY(-1px);
        }
        
        .btn:disabled {
            opacity: 0.7;
            cursor: not-allowed;
        }
        
        .btn-download {
            background: #e9ecef;
            color: #495057;
        }
        
        .validation-panel {
            margin: 24px 0;
            border-radius: 12px;
            overflow: hidden;
            box-shadow: 0 2px 8px rgba(80,80,160,0.04);
        }
        
        .validation-header {
            padding: 16px 24px;
            font-weight: 600;
            font-size: 1.1em;
        }
        
        .validation-content {
            padding: 20px 24px;
            background: white;
        }
        
        .error-list {
            margin: 0;
            padding-left: 24px;
        }
        
        .error-list li {
            margin-bottom: 8px;
            padding: 6px 0;
            border-bottom: 1px solid #eee;
        }
        
        .success { background: #d4edda; color: #155724; }
        .error { background: #f8d7da; color: #721c24; }
        .warning { background: #fff3cd; color: #856404; }
        
        .stats {
            padding: 15px 30px;
            background: #e9ecef;
            font-size: 14px;
            color: #495057;
        }
    </style>
</head>
<body>
    <!-- Menu Container -->
    <div class="menu-container">
        <div class="menu-icon" id="menuIcon">☰</div>
        <div class="menu-list" id="menuList">
            <a href="index.html">Leaderboard</a>
            <a href="https://github.com/dynamic-superb/dynamic-superb" target="_blank">GitHub Repository</a>
            <a href="https://arxiv.org/abs/2411.05361" target="_blank">Paper on arXiv</a>
        </div>
    </div>

    <div class="container">
        <div class="header">
            <h1>Dynamic-SUPERB Submission Validator</h1>
        </div>
        
        <div class="controls">
            <div class="upload-section">
                <label for="submissionFile">Upload Model CSV:</label>
                <input type="file" id="submissionFile" accept=".csv">
                <button id="validateBtn" class="btn" onclick="validateSubmission()">Validate</button>
                <button id="downloadBtn" class="btn btn-download" style="display:none;" onclick="downloadErrors()">Download Error Report</button>
            </div>
            
            <div id="validationResult"></div>
        </div>
        
        <div class="stats">
            Dynamic-SUPERB Submission Validator
        </div>
    </div>

    <script>
        // Menu functionality
        const menuIcon = document.getElementById('menuIcon');
        const menuList = document.getElementById('menuList');
        
        menuIcon.addEventListener('click', (e) => {
            e.stopPropagation();
            menuList.classList.toggle('show');
        });
        
        // Close menu if clicked outside
        document.addEventListener('click', (e) => {
            if (!menuIcon.contains(e.target) && !menuList.contains(e.target)) {
                menuList.classList.remove('show');
            }
        });

        let errorReport = [];
        
        // Enhanced CSV parsing
        function parseCSVLine(line) {
            const result = [];
            let current = '';
            let inQuotes = false;
            
            for (let i = 0; i < line.length; i++) {
                const char = line[i];
                
                if (char === '"') {
                    inQuotes = !inQuotes;
                } else if (char === ',' && !inQuotes) {
                    result.push(current.trim());
                    current = '';
                } else {
                    current += char;
                }
            }
            
            result.push(current.trim());
            return result;
        }

        function normalizeTaskName(name) {
            return name.trim().toLowerCase().replace(/[_\s]+/g, ' ');
        }

        class SimplifiedDynamicSUPERBValidator {
            constructor() {
                this.reference = new Map();
                this.existingModelNames = new Set();
                this.loadReference();
            }

            async loadReference() {
                try {
                    const response = await fetch('data.csv');
                    const csvText = await response.text();
                    const lines = csvText.trim().split('\n');
                    const headers = parseCSVLine(lines[0]);
                    
                    // Find HigherBetter and Taxonomy column indices in REFERENCE data only
                    const higherBetterIndex = headers.findIndex(h => h.trim().toLowerCase() === 'higherbetter');
                    const taxonomyIndex = headers.findIndex(h => h.trim().toLowerCase() === 'taxonomy');
                    
                    // Extract existing model names from reference data
                    const modelStartIndex = higherBetterIndex + 1;
                    const modelEndIndex = taxonomyIndex;
                    for (let i = modelStartIndex; i < modelEndIndex; i++) {
                        this.existingModelNames.add(headers[i].trim());
                    }
                    
                    for (let i = 1; i < lines.length; i++) {
                        const row = parseCSVLine(lines[i]);
                        if (row.length < 4) continue;
                        
                        const taskName = normalizeTaskName(row[0]);
                        const metric = row[1];
                        
                        if (metric === 'X') continue;
                        
                        if (!this.reference.has(taskName)) {
                            this.reference.set(taskName, {
                                originalName: row[0],
                                metrics: [],
                                narMetrics: []
                            });
                        }
                        
                        const task = this.reference.get(taskName);
                        if (!task.metrics.includes(metric)) task.metrics.push(metric);
                        
                        // Track NAR metrics specifically
                        if (metric.includes('NAR')) {
                            task.narMetrics.push(metric);
                        }
                    }
                    
                    // Sort metrics with NAR first
                    this.reference.forEach(task => {
                        task.metrics.sort((a, b) => {
                            if (a.includes('NAR')) return -1;
                            if (b.includes('NAR')) return 1;
                            return a.localeCompare(b);
                        });
                    });
                    
                } catch (error) {
                    console.error('Error loading reference:', error);
                }
            }

            async validate(file) {
                const csvText = await this.readFile(file);
                const lines = csvText.trim().split('\n');
                const header = parseCSVLine(lines[0]);
                
                console.log('Submission CSV Headers:', header);
                
                // SIMPLIFIED: Model columns are everything after "Metric" (index 1)
                const modelStartIndex = 2; // After "Task Name" and "Metric"
                const submissionModelNames = header.slice(modelStartIndex);
                
                console.log('Found models in submission:', submissionModelNames);
                console.log('Model count:', submissionModelNames.length);
                
                const result = {
                    isValid: true,
                    errors: [],
                    warnings: [],
                    stats: { total: 0, valid: 0, invalid: 0 },
                    tasks: new Map(),
                    modelNames: submissionModelNames
                };

                // Check for model name overlaps
                const overlappingModels = submissionModelNames.filter(modelName => 
                    this.existingModelNames.has(modelName.trim())
                );
                
                if (overlappingModels.length > 0) {
                    result.isValid = false;
                    result.errors.push({
                        line: 'Header',
                        task: 'Model Names',
                        error: `Model name overlap detected: ${overlappingModels.join(', ')} already exist in data.csv`
                    });
                }

                // Validate each row
                for (let i = 1; i < lines.length; i++) {
                    result.stats.total++;
                    const row = parseCSVLine(lines[i]);
                    const errors = [];
                    
                    // Basic structure check - must have at least Task Name, Metric, and one model column
                    if (row.length < 3) {
                        errors.push('Invalid row format (minimum: Task Name, Metric, and at least one model score)');
                        result.errors.push({
                            line: i+1,
                            task: row[0] || 'Unknown',
                            error: 'Invalid row format (minimum: Task Name, Metric, and at least one model score)'
                        });
                        result.isValid = false;
                        result.stats.invalid++;
                        continue;
                    }
                    
                    const taskName = normalizeTaskName(row[0]);
                    const metric = row[1];
                    
                    // Skip X metrics
                    if (metric === 'X') {
                        result.warnings.push(`Line ${i+1}: Skipped metric "X"`);
                        continue;
                    }
                    
                    // Task existence check
                    if (!this.reference.has(taskName)) {
                        errors.push(`Unknown task: "${row[0]}"`);
                    } else {
                        const refTask = this.reference.get(taskName);
                        
                        // Metric check
                        if (!refTask.metrics.includes(metric)) {
                            errors.push(`Invalid metric: "${metric}" for task "${row[0]}"`);
                        }
                        
                        // Track metrics per task for NAR check
                        if (!result.tasks.has(taskName)) {
                            result.tasks.set(taskName, { metrics: new Map() });
                        }
                        
                        const taskData = result.tasks.get(taskName);
                        if (!taskData.metrics.has(metric)) {
                            taskData.metrics.set(metric, []);
                        }
                        
                        // Track which models have this metric
                        for (let j = 0; j < submissionModelNames.length; j++) {
                            const modelValue = row[modelStartIndex + j];
                            if (modelValue && modelValue !== '-' && modelValue !== '') {
                                taskData.metrics.get(metric).push(submissionModelNames[j]);
                            }
                        }
                    }
                    
                    // Record errors
                    if (errors.length > 0) {
                        result.isValid = false;
                        result.stats.invalid++;
                        errors.forEach(error => {
                            result.errors.push({
                                line: i+1,
                                task: row[0],
                                error: error
                            });
                        });
                    } else {
                        result.stats.valid++;
                    }
                }
                
                // Check for missing NAR metrics for each model
                result.tasks.forEach((taskData, taskName) => {
                    if (!this.reference.has(taskName)) return;
                    
                    const refTask = this.reference.get(taskName);
                    if (refTask.narMetrics.length === 0) return; // No NAR metrics for this task
                    
                    // For each NAR metric in reference
                    refTask.narMetrics.forEach(narMetric => {
                        // Check if this NAR metric is reported
                        const reportedModels = taskData.metrics.has(narMetric) 
                            ? taskData.metrics.get(narMetric) 
                            : [];
                        
                        // Find models missing this NAR metric
                        const missingModels = submissionModelNames.filter(model => 
                            !reportedModels.includes(model)
                        );
                        
                        if (missingModels.length > 0) {
                            result.isValid = false;
                            result.errors.push({
                                line: 'Multiple',
                                task: refTask.originalName,
                                error: `Missing NAR metric "${narMetric}" for models: ${missingModels.join(', ')}`
                            });
                        }
                    });
                });
                
                return result;
            }

            readFile(file) {
                return new Promise((resolve, reject) => {
                    const reader = new FileReader();
                    reader.onload = () => resolve(reader.result);
                    reader.onerror = reject;
                    reader.readAsText(file);
                });
            }
        }

        // UI Handling
        const validator = new SimplifiedDynamicSUPERBValidator();
        let currentErrors = [];

        async function validateSubmission() {
            const fileInput = document.getElementById('submissionFile');
            const resultDiv = document.getElementById('validationResult');
            const downloadBtn = document.getElementById('downloadBtn');
            const validateBtn = document.getElementById('validateBtn');
            
            if (!fileInput.files.length) {
                resultDiv.innerHTML = `<div class="validation-panel error">
                    <div class="validation-header">Error</div>
                    <div class="validation-content">Please select a CSV file</div>
                </div>`;
                return;
            }

            validateBtn.disabled = true;
            validateBtn.textContent = 'Validating...';
            downloadBtn.style.display = 'none';
            currentErrors = [];
            
            try {
                const result = await validator.validate(fileInput.files[0]);
                let html = '';
                
                if (result.isValid) {
                    html = `<div class="validation-panel success">
                        <div class="validation-header">Validation Successful!</div>
                        <div class="validation-content">
                            <p><strong>Total Rows:</strong> ${result.stats.total}</p>
                            <p><strong>Valid Rows:</strong> ${result.stats.valid}</p>
                            <p><strong>Models:</strong> ${result.modelNames.length}</p>
                            <p>✅ All NAR metrics are properly reported for all models!</p>
                            <p>✅ No model name conflicts detected!</p>
                            <p>✅ All tasks and metrics are valid!</p>
                        </div>
                    </div>`;
                } else {
                    currentErrors = result.errors;
                    downloadBtn.style.display = 'inline-block';
                    
                    html = `<div class="validation-panel error">
                        <div class="validation-header">Validation Failed</div>
                        <div class="validation-content">
                            <p><strong>Total Rows:</strong> ${result.stats.total}</p>
                            <p><strong>Valid Rows:</strong> ${result.stats.valid}</p>
                            <p><strong>Errors Found:</strong> ${result.errors.length}</p>
                            <ul class="error-list">${result.errors.map(e => `
                                <li><strong>Line ${e.line}:</strong> ${e.error} (${e.task})</li>
                            `).join('')}</ul>
                        </div>
                    </div>`;
                }
                
                if (result.warnings.length > 0) {
                    html += `<div class="validation-panel warning">
                        <div class="validation-header">Warnings</div>
                        <div class="validation-content">
                            <ul class="error-list">${result.warnings.map(w => `
                                <li>${w}</li>
                            `).join('')}</ul>
                        </div>
                    </div>`;
                }
                
                resultDiv.innerHTML = html;
                
            } catch (error) {
                resultDiv.innerHTML = `<div class="validation-panel error">
                    <div class="validation-header">Error</div>
                    <div class="validation-content">${error.message}</div>
                </div>`;
            }
            
            validateBtn.disabled = false;
            validateBtn.textContent = 'Validate';
        }

        function downloadErrors() {
            if (currentErrors.length === 0) return;
            
            const csvContent = [
                ['Line', 'Task', 'Error'],
                ...currentErrors.map(e => [e.line, `"${e.task}"`, `"${e.error}"`])
            ].map(row => row.join(',')).join('\n');
            
            const blob = new Blob([csvContent], { type: 'text/csv' });
            const url = URL.createObjectURL(blob);
            const a = document.createElement('a');
            a.href = url;
            a.download = 'validation_errors.csv';
            document.body.appendChild(a);
            a.click();
            document.body.removeChild(a);
            URL.revokeObjectURL(url);
        }
    </script>
</body>
</html>