Spaces:
Running
Running
| <html lang="en"> | |
| <head> | |
| <meta charset="UTF-8"> | |
| <meta name="viewport" content="width=device-width, initial-scale=1.0"> | |
| <title>Provider Inference Metrics Dashboard</title> | |
| <script src='https://cdn.plot.ly/plotly-latest.min.js'></script> | |
| <style> | |
| :root { | |
| --bg-color: #f8f9fa; | |
| --card-bg-color: #ffffff; | |
| --text-color: #212529; | |
| --muted-text-color: #6c757d; | |
| --border-color: #dee2e6; | |
| --shadow-color: rgba(0, 0, 0, 0.05); | |
| --primary-color: #0d6efd; | |
| --plot-colorway: ['#1f77b4', '#ff7f0e', '#2ca02c', '#d62728', '#9467bd', '#8c564b', '#e377c2', '#7f7f7f', '#bcbd22', '#17becf']; /* Plotly default */ | |
| } | |
| body { | |
| font-family: -apple-system, BlinkMacSystemFont, "Segoe UI", Roboto, "Helvetica Neue", Arial, sans-serif; | |
| margin: 0; | |
| background-color: var(--bg-color); | |
| color: var(--text-color); | |
| line-height: 1.5; | |
| } | |
| .container { | |
| max-width: 1600px; | |
| margin: 20px auto; | |
| padding: 0 20px; | |
| } | |
| h1 { | |
| text-align: center; | |
| color: var(--text-color); | |
| margin-bottom: 30px; | |
| font-weight: 500; | |
| } | |
| .kpi-container { | |
| display: grid; | |
| grid-template-columns: repeat(auto-fit, minmax(200px, 1fr)); | |
| gap: 20px; | |
| margin-bottom: 30px; | |
| } | |
| .kpi-card { | |
| background-color: var(--card-bg-color); | |
| padding: 20px; | |
| border-radius: 8px; | |
| box-shadow: 0 4px 8px var(--shadow-color); | |
| text-align: center; | |
| border: 1px solid var(--border-color); | |
| } | |
| .kpi-card h3 { | |
| margin-top: 0; | |
| margin-bottom: 10px; | |
| font-size: 1rem; | |
| color: var(--muted-text-color); | |
| font-weight: 400; | |
| } | |
| .kpi-card .value { | |
| font-size: 1.8rem; | |
| font-weight: 600; | |
| color: var(--text-color); | |
| } | |
| .kpi-card .unit { | |
| font-size: 0.9rem; | |
| color: var(--muted-text-color); | |
| margin-left: 5px; | |
| } | |
| .dashboard-container { | |
| display: grid; | |
| grid-template-columns: repeat(auto-fit, minmax(450px, 1fr)); /* Responsive grid */ | |
| gap: 25px; /* Space between plots */ | |
| } | |
| .plot-container { | |
| background-color: var(--card-bg-color); | |
| padding: 20px; | |
| border-radius: 8px; | |
| box-shadow: 0 4px 8px var(--shadow-color); | |
| border: 1px solid var(--border-color); | |
| min-height: 450px; /* Ensure plots have some height */ | |
| display: flex; /* For centering loading/error inside */ | |
| justify-content: center; | |
| align-items: center; | |
| } | |
| /* Style Plotly chart container */ | |
| .plot-container .plotly { | |
| width: 100%; | |
| height: 100%; | |
| } | |
| #loading, #error { | |
| grid-column: 1 / -1; /* Span full width if grid is active */ | |
| text-align: center; | |
| font-size: 1.2em; | |
| padding: 40px; | |
| color: var(--muted-text-color); | |
| } | |
| #error { | |
| color: #dc3545; /* Bootstrap danger color */ | |
| font-weight: 500; | |
| background-color: #f8d7da; | |
| border: 1px solid #f5c2c7; | |
| border-radius: 8px; | |
| } | |
| footer { | |
| text-align: center; | |
| margin-top: 40px; | |
| padding: 20px; | |
| font-size: 0.9em; | |
| color: var(--muted-text-color); | |
| border-top: 1px solid var(--border-color); | |
| } | |
| footer a { | |
| color: var(--primary-color); | |
| text-decoration: none; | |
| } | |
| footer a:hover { | |
| text-decoration: underline; | |
| } | |
| </style> | |
| </head> | |
| <body> | |
| <div class="container"> | |
| <h1>Provider Inference Metrics Dashboard</h1> | |
| <div id="loading">Loading data... Please wait.</div> | |
| <div id="error" style="display: none;"></div> | |
| <!-- KPI Section --> | |
| <div class="kpi-container" style="display: none;"> | |
| <div class="kpi-card"> | |
| <h3>Total Requests</h3> | |
| <div class="value" id="kpi-total-requests">--</div> | |
| </div> | |
| <div class="kpi-card"> | |
| <h3>Success Rate</h3> | |
| <div class="value" id="kpi-success-rate">--<span class="unit">%</span></div> | |
| </div> | |
| <div class="kpi-card"> | |
| <h3>Avg. Latency (Overall)</h3> | |
| <div class="value" id="kpi-avg-latency">--<span class="unit">ms</span></div> | |
| </div> | |
| <div class="kpi-card"> | |
| <h3>Fastest Provider (Median)</h3> | |
| <div class="value" id="kpi-fastest-provider">--</div> | |
| </div> | |
| </div> | |
| <!-- Dashboard Plots --> | |
| <div class="dashboard-container" style="display: none;"> | |
| <div id="plotLatencyProvider" class="plot-container"></div> | |
| <div id="plotReliabilityProvider" class="plot-container"></div> | |
| <div id="plotLatencyModel" class="plot-container"></div> | |
| <div id="plotErrorTypesProvider" class="plot-container"></div> | |
| <div id="plotLatencyHeatmap" class="plot-container"></div> | |
| <!-- Add more divs here for additional plots --> | |
| </div> | |
| <footer id="footer" style="display: none;"> | |
| Data fetched from: <a id="data-source-url" href="#" target="_blank">Hugging Face Datasets</a><br> | |
| Last updated: <span id="last-updated"></span> | |
| </footer> | |
| </div> | |
| <script> | |
| document.addEventListener('DOMContentLoaded', function() { | |
| const apiUrl = "https://datasets-server.huggingface.co/rows?dataset=victor%2Fproviders-metrics&config=default&split=train&offset=0&length=100"; // Fetch 1000 rows | |
| const loadingDiv = document.getElementById('loading'); | |
| const errorDiv = document.getElementById('error'); | |
| const kpiContainer = document.querySelector('.kpi-container'); | |
| const dashboardContainer = document.querySelector('.dashboard-container'); | |
| const footer = document.getElementById('footer'); | |
| const dataSourceUrlElement = document.getElementById('data-source-url'); | |
| const lastUpdatedElement = document.getElementById('last-updated'); | |
| dataSourceUrlElement.href = apiUrl; // Set link href | |
| // Plotly layout defaults | |
| const baseLayout = { | |
| margin: { l: 60, r: 30, b: 100, t: 60, pad: 4 }, // Default margins | |
| legend: { bgcolor: 'rgba(255,255,255,0.5)', bordercolor: '#ccc', borderwidth: 1 }, | |
| colorway: ['#1f77b4', '#ff7f0e', '#2ca02c', '#d62728', '#9467bd', '#8c564b', '#e377c2', '#7f7f7f', '#bcbd22', '#17becf'], // Plotly default color sequence | |
| paper_bgcolor: 'rgba(0,0,0,0)', // Transparent background for plot area | |
| plot_bgcolor: 'rgba(0,0,0,0)', // Transparent background for plotting area | |
| font: { | |
| family: '-apple-system, BlinkMacSystemFont, "Segoe UI", Roboto, "Helvetica Neue", Arial, sans-serif', | |
| color: '#212529' | |
| }, | |
| title: { | |
| font: { size: 16, weight: '500' }, | |
| x: 0.05, // Title alignment | |
| xanchor: 'left' | |
| }, | |
| xaxis: { | |
| gridcolor: '#e9ecef', // Lighter grid lines | |
| linecolor: '#adb5bd', | |
| automargin: true, | |
| tickfont: { size: 10 } | |
| }, | |
| yaxis: { | |
| gridcolor: '#e9ecef', | |
| linecolor: '#adb5bd', | |
| automargin: true, | |
| tickfont: { size: 10 } | |
| } | |
| }; | |
| // Helper function to deep merge layout options | |
| function mergeLayout(customLayout) { | |
| // Simple deep merge for nested objects like margin, font, title | |
| let layout = JSON.parse(JSON.stringify(baseLayout)); // Deep copy base | |
| for (const key in customLayout) { | |
| if (typeof customLayout[key] === 'object' && customLayout[key] !== null && !Array.isArray(customLayout[key]) && layout[key]) { | |
| Object.assign(layout[key], customLayout[key]); | |
| } else { | |
| layout[key] = customLayout[key]; | |
| } | |
| } | |
| return layout; | |
| } | |
| // Helper function to calculate median | |
| function calculateMedian(arr) { | |
| if (!arr || arr.length === 0) return null; | |
| const sortedArr = [...arr].sort((a, b) => a - b); | |
| const mid = Math.floor(sortedArr.length / 2); | |
| return sortedArr.length % 2 !== 0 ? sortedArr[mid] : (sortedArr[mid - 1] + sortedArr[mid]) / 2; | |
| } | |
| fetch(apiUrl) | |
| .then(response => { | |
| if (!response.ok) { | |
| throw new Error(`HTTP error! status: ${response.status} ${response.statusText}`); | |
| } | |
| return response.json(); | |
| }) | |
| .then(data => { | |
| loadingDiv.style.display = 'none'; // Hide loading message | |
| kpiContainer.style.display = 'grid'; // Show KPIs | |
| dashboardContainer.style.display = 'grid'; // Show dashboard plots | |
| footer.style.display = 'block'; // Show footer | |
| // Extract the actual row data | |
| const rows = data.rows.map(item => item.row); | |
| console.log(`Fetched ${rows.length} rows.`); | |
| lastUpdatedElement.textContent = new Date().toLocaleString(); | |
| // --- Calculate and Display KPIs --- | |
| calculateAndDisplayKPIs(rows); | |
| // --- Data Processing and Plotting --- | |
| createLatencyByProviderPlot(rows); | |
| createReliabilityByProviderPlot(rows); | |
| createLatencyByModelPlot(rows); | |
| createErrorTypesByProviderPlot(rows); | |
| createLatencyHeatmap(rows); | |
| }) | |
| .catch(error => { | |
| console.error('Error fetching or processing data:', error); | |
| loadingDiv.style.display = 'none'; | |
| errorDiv.textContent = `Error loading data: ${error.message}. Please check the console for details. Is the dataset server reachable?`; | |
| errorDiv.style.display = 'block'; | |
| }); | |
| // --- KPI Calculation Function --- | |
| function calculateAndDisplayKPIs(rows) { | |
| const totalRequests = rows.length; | |
| const successfulRequests = rows.filter(r => r.response_status_code === 200).length; | |
| const successRate = totalRequests > 0 ? ((successfulRequests / totalRequests) * 100).toFixed(1) : 0; | |
| const validLatencies = rows | |
| .map(r => r.duration_ms) | |
| .filter(d => d !== null && d >= 0); | |
| const avgLatency = validLatencies.length > 0 | |
| ? (validLatencies.reduce((a, b) => a + b, 0) / validLatencies.length).toFixed(0) | |
| : 0; | |
| // Calculate median latency per provider to find the fastest | |
| const latencyByProvider = {}; | |
| rows.forEach(row => { | |
| if (row.duration_ms !== null && row.duration_ms >= 0) { | |
| if (!latencyByProvider[row.provider_name]) { | |
| latencyByProvider[row.provider_name] = []; | |
| } | |
| latencyByProvider[row.provider_name].push(row.duration_ms); | |
| } | |
| }); | |
| let fastestProvider = '--'; | |
| let minMedianLatency = Infinity; | |
| for (const provider in latencyByProvider) { | |
| const median = calculateMedian(latencyByProvider[provider]); | |
| if (median !== null && median < minMedianLatency) { | |
| minMedianLatency = median; | |
| fastestProvider = provider; | |
| } | |
| } | |
| document.getElementById('kpi-total-requests').textContent = totalRequests; | |
| document.getElementById('kpi-success-rate').innerHTML = `${successRate}<span class="unit">%</span>`; | |
| document.getElementById('kpi-avg-latency').innerHTML = `${avgLatency}<span class="unit">ms</span>`; | |
| document.getElementById('kpi-fastest-provider').textContent = fastestProvider; | |
| } | |
| // --- Plotting Functions --- | |
| function createLatencyByProviderPlot(rows) { | |
| const dataByProvider = {}; | |
| rows.forEach(row => { | |
| if (!dataByProvider[row.provider_name]) { | |
| dataByProvider[row.provider_name] = []; | |
| } | |
| if (row.duration_ms !== null && row.duration_ms >= 0) { | |
| dataByProvider[row.provider_name].push(row.duration_ms); | |
| } | |
| }); | |
| const plotData = Object.keys(dataByProvider).sort().map(provider => ({ // Sort providers alphabetically | |
| y: dataByProvider[provider], | |
| type: 'box', | |
| name: provider, | |
| boxpoints: 'Outliers', | |
| marker: { size: 4 } | |
| })); | |
| const layout = mergeLayout({ | |
| title: { text: 'Latency Distribution by Provider' }, | |
| yaxis: { title: 'Duration (ms)', type: 'log', autorange: true }, | |
| xaxis: { title: 'Provider', tickangle: -30 }, | |
| margin: { b: 120 } // More bottom margin for angled labels | |
| }); | |
| Plotly.newPlot('plotLatencyProvider', plotData, layout, {responsive: true}); | |
| } | |
| function createReliabilityByProviderPlot(rows) { | |
| const statusCountsByProvider = {}; | |
| const allProviders = new Set(); | |
| const allStatusCodes = new Set(); | |
| rows.forEach(row => { | |
| const provider = row.provider_name; | |
| const status = row.response_status_code ?? 'Unknown'; // Handle null status | |
| allProviders.add(provider); | |
| allStatusCodes.add(status); | |
| if (!statusCountsByProvider[provider]) { | |
| statusCountsByProvider[provider] = {}; | |
| } | |
| if (!statusCountsByProvider[provider][status]) { | |
| statusCountsByProvider[provider][status] = 0; | |
| } | |
| statusCountsByProvider[provider][status]++; | |
| }); | |
| const sortedProviders = Array.from(allProviders).sort(); | |
| // Sort status codes: 200 first, then numerically, then 'Unknown' | |
| const sortedStatusCodes = Array.from(allStatusCodes).sort((a, b) => { | |
| if (a === 200) return -1; | |
| if (b === 200) return 1; | |
| if (a === 'Unknown') return 1; | |
| if (b === 'Unknown') return -1; | |
| return a - b; | |
| }); | |
| const plotData = sortedStatusCodes.map(status => { | |
| return { | |
| x: sortedProviders, | |
| y: sortedProviders.map(provider => statusCountsByProvider[provider]?.[status] || 0), | |
| name: `Status ${status}`, | |
| type: 'bar', | |
| hovertemplate: `Provider: %{x}<br>Status: ${status}<br>Count: %{y}<extra></extra>` | |
| }; | |
| }); | |
| const layout = mergeLayout({ | |
| title: { text: 'Request Status Codes by Provider' }, | |
| barmode: 'stack', | |
| xaxis: { title: 'Provider', tickangle: -30 }, | |
| yaxis: { title: 'Number of Requests', autorange: true }, | |
| margin: { b: 120 } | |
| }); | |
| Plotly.newPlot('plotReliabilityProvider', plotData, layout, {responsive: true}); | |
| } | |
| function createLatencyByModelPlot(rows) { | |
| const dataByModel = {}; | |
| rows.forEach(row => { | |
| const model = row.model_id; | |
| if (!dataByModel[model]) { | |
| dataByModel[model] = []; | |
| } | |
| if (row.duration_ms !== null && row.duration_ms >= 0) { | |
| dataByModel[model].push(row.duration_ms); | |
| } | |
| }); | |
| const plotData = Object.keys(dataByModel).sort().map(model => ({ // Sort models | |
| y: dataByModel[model], | |
| type: 'box', | |
| name: model, | |
| boxpoints: 'Outliers', | |
| marker: { size: 4 } | |
| })); | |
| const layout = mergeLayout({ | |
| title: { text: 'Latency Distribution by Model' }, | |
| yaxis: { title: 'Duration (ms)', type: 'log', autorange: true }, | |
| xaxis: { | |
| title: 'Model ID', | |
| tickangle: -30 // Angle labels if they overlap | |
| }, | |
| margin: { b: 180 } // More bottom margin for potentially long/angled labels | |
| }); | |
| Plotly.newPlot('plotLatencyModel', plotData, layout, {responsive: true}); | |
| } | |
| function createErrorTypesByProviderPlot(rows) { | |
| const errorCountsByProvider = {}; | |
| const allProviders = new Set(); | |
| const allErrorCodes = new Set(); | |
| rows.forEach(row => { | |
| if (row.response_status_code !== 200 && row.response_status_code !== null) { // Only errors | |
| const provider = row.provider_name; | |
| const status = row.response_status_code; | |
| allProviders.add(provider); | |
| allErrorCodes.add(status); | |
| if (!errorCountsByProvider[provider]) { | |
| errorCountsByProvider[provider] = {}; | |
| } | |
| if (!errorCountsByProvider[provider][status]) { | |
| errorCountsByProvider[provider][status] = 0; | |
| } | |
| errorCountsByProvider[provider][status]++; | |
| } | |
| }); | |
| const sortedProviders = Array.from(allProviders).sort(); | |
| const sortedErrorCodes = Array.from(allErrorCodes).sort((a, b) => a - b); | |
| const plotData = sortedErrorCodes.map(status => { | |
| return { | |
| x: sortedProviders, | |
| y: sortedProviders.map(provider => errorCountsByProvider[provider]?.[status] || 0), | |
| name: `Error ${status}`, | |
| type: 'bar', | |
| hovertemplate: `Provider: %{x}<br>Error: ${status}<br>Count: %{y}<extra></extra>` | |
| }; | |
| }); | |
| const layout = mergeLayout({ | |
| title: { text: 'Error Types by Provider (Non-200 Status)' }, | |
| barmode: 'group', // Group bars side-by-side for comparison | |
| xaxis: { title: 'Provider', tickangle: -30 }, | |
| yaxis: { title: 'Number of Errors', autorange: true }, | |
| margin: { b: 120 } | |
| }); | |
| Plotly.newPlot('plotErrorTypesProvider', plotData, layout, {responsive: true}); | |
| } | |
| function createLatencyHeatmap(rows) { | |
| const latencyData = {}; // { provider: { model: { sum: 0, count: 0 } } } | |
| const allProviders = new Set(); | |
| const allModels = new Set(); | |
| rows.forEach(row => { | |
| if (row.duration_ms !== null && row.duration_ms >= 0) { | |
| const provider = row.provider_name; | |
| const model = row.model_id; | |
| allProviders.add(provider); | |
| allModels.add(model); | |
| if (!latencyData[provider]) latencyData[provider] = {}; | |
| if (!latencyData[provider][model]) latencyData[provider][model] = { sum: 0, count: 0 }; | |
| latencyData[provider][model].sum += row.duration_ms; | |
| latencyData[provider][model].count++; | |
| } | |
| }); | |
| const sortedProviders = Array.from(allProviders).sort(); | |
| const sortedModels = Array.from(allModels).sort(); | |
| const zValues = sortedModels.map(model => { | |
| return sortedProviders.map(provider => { | |
| const data = latencyData[provider]?.[model]; | |
| const avg = data && data.count > 0 ? data.sum / data.count : null; | |
| return avg; | |
| }); | |
| }); | |
| const hoverText = sortedModels.map(model => { | |
| return sortedProviders.map(provider => { | |
| const data = latencyData[provider]?.[model]; | |
| const avg = data && data.count > 0 ? (data.sum / data.count).toFixed(0) : 'N/A'; | |
| const count = data?.count || 0; | |
| return `Model: ${model}<br>Provider: ${provider}<br>Avg Latency: ${avg} ms<br>Requests: ${count}<extra></extra>`; | |
| }); | |
| }); | |
| const plotData = [{ | |
| z: zValues, | |
| x: sortedProviders, | |
| y: sortedModels, | |
| type: 'heatmap', | |
| hoverongaps: false, | |
| colorscale: 'Viridis', | |
| reversescale: true, // Often makes sense for latency (lower is better -> brighter) | |
| colorbar: { title: 'Avg Latency (ms)', titleside: 'right', thickness: 15 }, | |
| xgap: 2, // Add gaps between cells | |
| ygap: 2, | |
| hovertemplate: hoverText // Custom hover text | |
| }]; | |
| const layout = mergeLayout({ | |
| title: { text: 'Average Latency (ms) - Model vs. Provider' }, | |
| xaxis: { title: '', side: 'top', tickangle: -30 }, // Move x-axis labels to top, angle | |
| yaxis: { title: '', autorange: 'reversed' }, // Reverse y-axis to match typical matrix layout | |
| margin: { l: 280, r: 50, b: 50, t: 120 } // Adjust margins significantly for labels | |
| }); | |
| Plotly.newPlot('plotLatencyHeatmap', plotData, layout, {responsive: true}); | |
| } | |
| }); | |
| </script> | |
| </body> | |
| </html> |