$(document).ready(function() { // File handling variables let currentFile = null; let originalTextContent = null; let lastUploadedFileName = null; let fileJustUploaded = false; // Flag to prevent immediate detachment let currentModelType = window.tokenizerData?.model_type || 'predefined'; let currentTokenizerInfo = null; // Try to parse tokenizer info if available from server try { currentTokenizerInfo = window.tokenizerData?.tokenizer_info || null; if (currentTokenizerInfo) { updateTokenizerInfoDisplay(currentTokenizerInfo, currentModelType === 'custom'); } } catch(e) { console.error("Error parsing tokenizer info:", e); } // Show error if exists if (window.tokenizerData?.error) { showError(window.tokenizerData.error); } // Setup model type based on initial state if (currentModelType === "custom") { $('.toggle-option').removeClass('active'); $('.custom-toggle').addClass('active'); $('#predefinedModelSelector').hide(); $('#customModelSelector').show(); } // Show success badge if custom model loaded successfully if (currentModelType === "custom" && !window.tokenizerData?.error) { $('#modelSuccessBadge').addClass('show'); setTimeout(() => { $('#modelSuccessBadge').removeClass('show'); }, 3000); } // Toggle between predefined and custom model inputs $('.toggle-option').click(function() { const modelType = $(this).data('type'); $('.toggle-option').removeClass('active'); $(this).addClass('active'); currentModelType = modelType; if (modelType === 'predefined') { $('#predefinedModelSelector').show(); $('#customModelSelector').hide(); $('#modelTypeInput').val('predefined'); // Set the model input value to the selected predefined model $('#modelInput').val($('#modelSelect').val()); } else { $('#predefinedModelSelector').hide(); $('#customModelSelector').show(); $('#modelTypeInput').val('custom'); } // Clear tokenizer info if switching models if (modelType === 'predefined') { $('#tokenizerInfoContent').html('
'); fetchTokenizerInfo($('#modelSelect').val(), false); } else { $('#customTokenizerInfoContent').html('
'); // Only fetch if there's a custom model value const customModel = $('#customModelInput').val(); if (customModel) { fetchTokenizerInfo(customModel, true); } } }); // Update hidden input when custom model input changes $('#customModelInput').on('input', function() { $('#customModelInputHidden').val($(this).val()); }); function showError(message) { const errorDiv = $('#errorMessage'); errorDiv.text(message); errorDiv.show(); setTimeout(() => errorDiv.fadeOut(), 5000); } // Function to update tokenizer info display in tooltip function updateTokenizerInfoDisplay(info, isCustom = false) { const targetSelector = isCustom ? '#customTokenizerInfoContent' : '#tokenizerInfoContent'; let htmlContent = ''; if (info.error) { $(targetSelector).html(`
${info.error}
`); return; } // Start building the tooltip content htmlContent = `
Tokenizer Details
`; // Dictionary size if (info.vocab_size) { htmlContent += `
Dictionary Size ${info.vocab_size.toLocaleString()}
`; } // Tokenizer type if (info.tokenizer_type) { htmlContent += `
Tokenizer Type ${info.tokenizer_type}
`; } // Max length if (info.model_max_length) { htmlContent += `
Max Length ${info.model_max_length.toLocaleString()}
`; } htmlContent += `
`; // Close tokenizer-info-grid // Special tokens section if (info.special_tokens && Object.keys(info.special_tokens).length > 0) { htmlContent += `
Special Tokens
`; // Add each special token with proper escaping for HTML special characters for (const [tokenName, tokenValue] of Object.entries(info.special_tokens)) { // Properly escape HTML special characters const escapedValue = tokenValue .replace(/&/g, '&') .replace(//g, '>') .replace(/"/g, '"') .replace(/'/g, '''); htmlContent += `
${tokenName}: ${escapedValue}
`; } htmlContent += `
`; } $(targetSelector).html(htmlContent); } // Function to show loading overlay function showLoadingOverlay(text = 'Loading...') { $('#loadingText').text(text); $('#loadingOverlay').addClass('active'); } // Function to hide loading overlay function hideLoadingOverlay() { $('#loadingOverlay').removeClass('active'); } // Function to fetch tokenizer info function fetchTokenizerInfo(modelId, isCustom = false) { if (!modelId) return; const targetSelector = isCustom ? '#customTokenizerInfoContent' : '#tokenizerInfoContent'; $(targetSelector).html('
'); $.ajax({ url: '/tokenizer-info', method: 'GET', data: { model_id: modelId, is_custom: isCustom }, success: function(response) { if (response.error) { $(targetSelector).html(`
${response.error}
`); } else { currentTokenizerInfo = response; updateTokenizerInfoDisplay(response, isCustom); } }, error: function(xhr) { $(targetSelector).html('
Failed to load tokenizer information
'); } }); } // Token search functionality let searchMatches = []; let currentSearchIndex = -1; let searchVisible = false; // Token frequency functionality let tokenFrequencyData = []; let showFrequencyChart = false; function performTokenSearch(searchTerm) { const tokenContainer = $('#tokenContainer'); const tokens = tokenContainer.find('.token'); // Clear previous highlights tokens.removeClass('highlighted current'); searchMatches = []; currentSearchIndex = -1; if (!searchTerm.trim()) { updateSearchCount(); return; } const searchLower = searchTerm.toLowerCase(); // Find matching tokens tokens.each(function(index) { const tokenText = $(this).text().toLowerCase(); if (tokenText.includes(searchLower)) { $(this).addClass('highlighted'); searchMatches.push(index); } }); updateSearchCount(); // Navigate to first match if any if (searchMatches.length > 0) { navigateToMatch(0); } } function navigateToMatch(index) { if (searchMatches.length === 0) return; // Remove current highlight $('.token.current').removeClass('current'); // Update current index currentSearchIndex = index; // Highlight current match const tokenContainer = $('#tokenContainer'); const tokens = tokenContainer.find('.token'); const currentToken = tokens.eq(searchMatches[currentSearchIndex]); currentToken.addClass('current'); // Scroll to current match - improved logic const scrollContainer = tokenContainer; const containerOffset = scrollContainer.offset(); const tokenOffset = currentToken.offset(); if (containerOffset && tokenOffset) { const containerHeight = scrollContainer.height(); const containerScrollTop = scrollContainer.scrollTop(); const tokenRelativeTop = tokenOffset.top - containerOffset.top; // Check if token is outside visible area if (tokenRelativeTop < 0 || tokenRelativeTop > containerHeight - 50) { // Calculate new scroll position to center the token const tokenHeight = currentToken.outerHeight(); const newScrollTop = containerScrollTop + tokenRelativeTop - (containerHeight / 2) + (tokenHeight / 2); scrollContainer.animate({ scrollTop: Math.max(0, newScrollTop) }, 400, 'swing'); } } updateSearchCount(); } function toggleSearchVisibility() { console.log('toggleSearchVisibility called, current state:', searchVisible); searchVisible = !searchVisible; const container = $('#tokenSearchContainer'); const toggleBtn = $('#searchToggleBtn'); console.log('Container found:', container.length, 'Toggle button found:', toggleBtn.length); if (searchVisible) { // Show the container first, then animate container.show(); setTimeout(() => { container.addClass('show'); }, 10); toggleBtn.addClass('active'); console.log('Showing search container'); setTimeout(() => { $('#tokenSearchInput').focus(); }, 300); } else { container.removeClass('show'); toggleBtn.removeClass('active'); console.log('Hiding search container'); setTimeout(() => { container.hide(); }, 300); // Clear search when hiding $('#tokenSearchInput').val(''); performTokenSearch(''); } } function updateSearchCount() { const countText = searchMatches.length > 0 ? `${currentSearchIndex + 1}/${searchMatches.length}` : `0/${searchMatches.length}`; $('#searchCount').text(countText); // Update navigation button states $('#prevMatch').prop('disabled', searchMatches.length === 0 || currentSearchIndex <= 0); $('#nextMatch').prop('disabled', searchMatches.length === 0 || currentSearchIndex >= searchMatches.length - 1); } // Token frequency chart functions function calculateTokenFrequency(tokens) { const frequencyMap = {}; tokens.each(function() { const tokenText = $(this).text(); if (tokenText.trim()) { frequencyMap[tokenText] = (frequencyMap[tokenText] || 0) + 1; } }); // Convert to array and sort by frequency const frequencyArray = Object.entries(frequencyMap) .map(([token, count]) => ({ token, count })) .sort((a, b) => b.count - a.count) .slice(0, 10); // Top 10 tokens return frequencyArray; } function renderFrequencyChart(frequencyData) { const chartContainer = $('#frequencyChart'); chartContainer.empty(); if (frequencyData.length === 0) { chartContainer.html('
No token data available
'); return; } const maxCount = frequencyData[0].count; frequencyData.forEach(({ token, count }) => { const percentage = (count / maxCount) * 100; const item = $(`
${token}
${count}
`); // Add click handler to search for this token item.find('.frequency-token').click(function() { const searchToken = $(this).data('token'); $('#tokenSearchInput').val(searchToken); performTokenSearch(searchToken); }); chartContainer.append(item); }); } function toggleFrequencyChart() { showFrequencyChart = !showFrequencyChart; const container = $('#frequencyChartContainer'); const chart = $('#frequencyChart'); const toggleBtn = $('#toggleFrequencyChart'); if (showFrequencyChart) { container.show(); chart.show(); toggleBtn.text('Hide Chart').addClass('active'); // Calculate and render frequency data const tokens = $('#tokenContainer').find('.token'); tokenFrequencyData = calculateTokenFrequency(tokens); renderFrequencyChart(tokenFrequencyData); } else { chart.hide(); toggleBtn.text('Show Chart').removeClass('active'); } } function updateResults(data) { $('#results').show(); // Show search toggle button and frequency chart container $('#searchToggleBtn').show(); $('#frequencyChartContainer').show(); // Update tokens const tokenContainer = $('#tokenContainer'); tokenContainer.empty(); data.tokens.forEach(token => { const span = $('') .addClass('token') .css({ 'background-color': token.colors.background, 'color': token.colors.text }) // Include token id in the tooltip on hover .attr('title', `Original token: ${token.original} | Token ID: ${token.token_id}`) .text(token.display); tokenContainer.append(span); if (token.newline) { tokenContainer.append('
'); } }); // Re-apply current search if any const currentSearch = $('#tokenSearchInput').val(); if (currentSearch.trim()) { performTokenSearch(currentSearch); } // Update display limit notice if (data.display_limit_reached) { $('#displayLimitNotice').show(); $('#totalTokenCount').text(data.total_tokens); } else { $('#displayLimitNotice').hide(); } // Update preview notice if (data.preview_only) { $('#previewNotice').show(); } else { $('#previewNotice').hide(); } // Update basic stats $('#totalTokens').text(data.stats.basic_stats.total_tokens); $('#uniqueTokens').text(`${data.stats.basic_stats.unique_tokens} unique`); $('#uniquePercentage').text(data.stats.basic_stats.unique_percentage); $('#specialTokens').text(data.stats.basic_stats.special_tokens); $('#spaceTokens').text(data.stats.basic_stats.space_tokens); $('#spaceCount').text(data.stats.basic_stats.space_tokens); $('#newlineCount').text(data.stats.basic_stats.newline_tokens); $('#compressionRatio').text(data.stats.basic_stats.compression_ratio); // Update length stats $('#avgLength').text(data.stats.length_stats.avg_length); $('#medianLength').text(data.stats.length_stats.median_length); $('#stdDev').text(data.stats.length_stats.std_dev); // Update tokenizer info if available if (data.tokenizer_info) { currentTokenizerInfo = data.tokenizer_info; updateTokenizerInfoDisplay(data.tokenizer_info, currentModelType === 'custom'); } } // Handle text changes to detach file $('#textInput').on('input', function() { // Skip if file was just uploaded (prevents immediate detachment) if (fileJustUploaded) { fileJustUploaded = false; return; } const currentText = $(this).val(); const fileInput = document.getElementById('fileInput'); // Only detach if a file exists and text has been substantially modified if (fileInput.files.length > 0 && originalTextContent !== null) { // Check if the text is completely different or has been significantly changed // This allows for small edits without detaching const isMajorChange = currentText.length < originalTextContent.length * 0.8 || // Text reduced by at least 20% (currentText.length > 0 && currentText !== originalTextContent.substring(0, currentText.length) && currentText.substring(0, Math.min(20, currentText.length)) !== originalTextContent.substring(0, Math.min(20, currentText.length))); if (isMajorChange) { detachFile(); } } }); // Function to detach file function detachFile() { // Clear the file input $('#fileInput').val(''); // Hide file info $('#fileInfo').fadeOut(300); // Reset the original content tracker originalTextContent = $('#textInput').val(); // Reset last uploaded filename lastUploadedFileName = null; } // For model changes $('#modelSelect').change(function() { const selectedModel = $(this).val(); $('#modelInput').val(selectedModel); // Fetch tokenizer info for the selected model fetchTokenizerInfo(selectedModel, false); // If text exists, submit the form if ($('#textInput').val().trim()) { $('#analyzeForm').submit(); } }); // File drop handling const fileDropZone = $('#fileDropZone'); const fileUploadIcon = $('#fileUploadIcon'); // Prevent default drag behaviors ['dragenter', 'dragover', 'dragleave', 'drop'].forEach(eventName => { fileDropZone[0].addEventListener(eventName, preventDefaults, false); document.body.addEventListener(eventName, preventDefaults, false); }); function preventDefaults(e) { e.preventDefault(); e.stopPropagation(); } // Show drop zone when file is dragged over the document document.addEventListener('dragenter', showDropZone, false); document.addEventListener('dragover', showDropZone, false); fileDropZone[0].addEventListener('dragleave', hideDropZone, false); fileDropZone[0].addEventListener('drop', hideDropZone, false); function showDropZone(e) { fileDropZone.addClass('active'); } function hideDropZone() { fileDropZone.removeClass('active'); } // Handle dropped files fileDropZone[0].addEventListener('drop', handleDrop, false); function handleDrop(e) { const dt = e.dataTransfer; const files = dt.files; handleFiles(files); } // Also handle file selection via click on the icon fileUploadIcon.on('click', function() { const input = document.createElement('input'); input.type = 'file'; input.onchange = e => { handleFiles(e.target.files); }; input.click(); }); function handleFiles(files) { if (files.length) { const file = files[0]; currentFile = file; lastUploadedFileName = file.name; fileJustUploaded = true; // Set flag to prevent immediate detachment // Show file info with animation and add detach button $('#fileInfo').html(`${file.name} (${formatFileSize(file.size)}) `).fadeIn(300); // Add click handler for detach button $('#fileDetach').on('click', function(e) { e.stopPropagation(); // Prevent event bubbling detachFile(); return false; }); // Set the file to the file input const dataTransfer = new DataTransfer(); dataTransfer.items.add(file); document.getElementById('fileInput').files = dataTransfer.files; // Preview in textarea (first 8096 chars) const reader = new FileReader(); reader.onload = function(e) { const previewText = e.target.result.slice(0, 8096); $('#textInput').val(previewText); // Store this as the original content AFTER setting the value // to prevent the input event from firing and detaching immediately setTimeout(() => { originalTextContent = previewText; // Automatically submit for analysis $('#analyzeForm').submit(); }, 50); }; reader.readAsText(file); } } function formatFileSize(bytes) { if (bytes < 1024) return bytes + ' bytes'; else if (bytes < 1048576) return (bytes / 1024).toFixed(1) + ' KB'; else return (bytes / 1048576).toFixed(1) + ' MB'; } // Make sure to check if there's still a file when analyzing $('#analyzeForm').on('submit', function(e) { e.preventDefault(); // Skip detachment check if file was just uploaded if (!fileJustUploaded) { // Check if text has been changed but file is still attached const textInput = $('#textInput').val(); const fileInput = document.getElementById('fileInput'); if (fileInput.files.length > 0 && originalTextContent !== null && textInput !== originalTextContent && textInput.length < originalTextContent.length * 0.8) { // Text was significantly changed but file is still attached, detach it detachFile(); } } else { // Reset flag after first submission fileJustUploaded = false; } // Update the hidden inputs based on current model type if (currentModelType === 'custom') { $('#customModelInputHidden').val($('#customModelInput').val()); } else { $('#modelInput').val($('#modelSelect').val()); } const formData = new FormData(this); const analyzeButton = $('#analyzeButton'); const originalButtonText = analyzeButton.text(); analyzeButton.prop('disabled', true); analyzeButton.html(originalButtonText + ''); showLoadingOverlay('Analyzing text...'); $.ajax({ url: '/', method: 'POST', data: formData, processData: false, contentType: false, success: function(response) { if (response.error) { showError(response.error); } else { updateResults(response); // Show success badge if custom model if (currentModelType === 'custom') { $('#modelSuccessBadge').addClass('show'); setTimeout(() => { $('#modelSuccessBadge').removeClass('show'); }, 3000); } } }, error: function(xhr) { showError(xhr.responseText || 'An error occurred while processing the text'); }, complete: function() { analyzeButton.prop('disabled', false); analyzeButton.text(originalButtonText); hideLoadingOverlay(); } }); }); $('#expandButton').click(function() { const container = $('#tokenContainer'); const isExpanded = container.hasClass('expanded'); container.toggleClass('expanded'); $(this).text(isExpanded ? 'Show More' : 'Show Less'); }); // Initialize tokenizer info for current model if (currentModelType === 'predefined') { fetchTokenizerInfo($('#modelSelect').val(), false); } else if ($('#customModelInput').val()) { fetchTokenizerInfo($('#customModelInput').val(), true); } // Add event listener for custom model input $('#customModelInput').on('change', function() { const modelValue = $(this).val(); if (modelValue) { fetchTokenizerInfo(modelValue, true); } }); // Keyboard shortcuts - specifically for textarea $('#textInput').keydown(function(e) { // Ctrl+Enter (or Cmd+Enter on Mac) to analyze if ((e.ctrlKey || e.metaKey) && (e.keyCode === 13 || e.which === 13)) { e.preventDefault(); if ($(this).val().trim()) { $('#analyzeForm').submit(); } return false; } }); // Global keyboard shortcuts $(document).keydown(function(e) { // Ctrl+F (or Cmd+F on Mac) to toggle search if ((e.ctrlKey || e.metaKey) && (e.keyCode === 70 || e.which === 70)) { if ($('#searchToggleBtn').is(':visible')) { e.preventDefault(); if (!searchVisible) { toggleSearchVisibility(); } else { $('#tokenSearchInput').focus(); } return false; } } // Escape to close search or loading overlay if (e.keyCode === 27 || e.which === 27) { if (searchVisible) { toggleSearchVisibility(); return false; } if ($('#loadingOverlay').hasClass('active')) { // Don't close if there's an active request return false; } } }); // Add keyboard shortcut hint to the textarea placeholder $('#textInput').attr('placeholder', 'Enter text to analyze or upload a file in bottom left corner... (Ctrl+Enter to analyze)'); // Token search event handlers $('#tokenSearchInput').on('input', function() { const searchTerm = $(this).val(); performTokenSearch(searchTerm); }); $('#nextMatch').click(function() { if (currentSearchIndex < searchMatches.length - 1) { navigateToMatch(currentSearchIndex + 1); } }); $('#prevMatch').click(function() { if (currentSearchIndex > 0) { navigateToMatch(currentSearchIndex - 1); } }); $('#clearSearch').click(function() { $('#tokenSearchInput').val(''); performTokenSearch(''); }); // Additional keyboard shortcuts for search $('#tokenSearchInput').keydown(function(e) { if (e.keyCode === 13) { // Enter e.preventDefault(); if (e.shiftKey) { // Shift+Enter: previous match $('#prevMatch').click(); } else { // Enter: next match $('#nextMatch').click(); } } else if (e.keyCode === 27) { // Escape $('#clearSearch').click(); $(this).blur(); } }); // Search toggle handler using event delegation $(document).on('click', '#searchToggleBtn', function(e) { console.log('Search toggle button clicked!'); e.preventDefault(); e.stopPropagation(); toggleSearchVisibility(); return false; }); // Frequency chart toggle handler $('#toggleFrequencyChart').click(function() { toggleFrequencyChart(); }); // Mobile touch enhancements function addTouchSupport() { // Add touch-friendly double-tap for expand/collapse let lastTap = 0; $('#tokenContainer').on('touchend', function(e) { const currentTime = new Date().getTime(); const tapLength = currentTime - lastTap; if (tapLength < 500 && tapLength > 0) { $('#expandButton').click(); e.preventDefault(); } lastTap = currentTime; }); // Improve touch scrolling for token container $('#tokenContainer').on('touchstart', function(e) { this.scrollTop = this.scrollTop; }); } // Check if mobile device and add touch support if ('ontouchstart' in window || navigator.maxTouchPoints > 0) { addTouchSupport(); } });