$(document).ready(function() {
// File handling variables
let currentFile = null;
let originalTextContent = null;
let lastUploadedFileName = null;
let fileJustUploaded = false; // Flag to prevent immediate detachment
let currentModelType = window.tokenizerData?.model_type || 'predefined';
let currentTokenizerInfo = null;
// Try to parse tokenizer info if available from server
try {
currentTokenizerInfo = window.tokenizerData?.tokenizer_info || null;
if (currentTokenizerInfo) {
updateTokenizerInfoDisplay(currentTokenizerInfo, currentModelType === 'custom');
}
} catch(e) {
console.error("Error parsing tokenizer info:", e);
}
// Show error if exists
if (window.tokenizerData?.error) {
showError(window.tokenizerData.error);
}
// Setup model type based on initial state
if (currentModelType === "custom") {
$('.toggle-option').removeClass('active');
$('.custom-toggle').addClass('active');
$('#predefinedModelSelector').hide();
$('#customModelSelector').show();
}
// Show success badge if custom model loaded successfully
if (currentModelType === "custom" && !window.tokenizerData?.error) {
$('#modelSuccessBadge').addClass('show');
setTimeout(() => {
$('#modelSuccessBadge').removeClass('show');
}, 3000);
}
// Toggle between predefined and custom model inputs
$('.toggle-option').click(function() {
const modelType = $(this).data('type');
$('.toggle-option').removeClass('active');
$(this).addClass('active');
currentModelType = modelType;
if (modelType === 'predefined') {
$('#predefinedModelSelector').show();
$('#customModelSelector').hide();
$('#modelTypeInput').val('predefined');
// Set the model input value to the selected predefined model
$('#modelInput').val($('#modelSelect').val());
} else {
$('#predefinedModelSelector').hide();
$('#customModelSelector').show();
$('#modelTypeInput').val('custom');
}
// Clear tokenizer info if switching models
if (modelType === 'predefined') {
$('#tokenizerInfoContent').html('
');
fetchTokenizerInfo($('#modelSelect').val(), false);
} else {
$('#customTokenizerInfoContent').html('');
// Only fetch if there's a custom model value
const customModel = $('#customModelInput').val();
if (customModel) {
fetchTokenizerInfo(customModel, true);
}
}
});
// Update hidden input when custom model input changes
$('#customModelInput').on('input', function() {
$('#customModelInputHidden').val($(this).val());
});
function showError(message) {
const errorDiv = $('#errorMessage');
errorDiv.text(message);
errorDiv.show();
setTimeout(() => errorDiv.fadeOut(), 5000);
}
// Function to update tokenizer info display in tooltip
function updateTokenizerInfoDisplay(info, isCustom = false) {
const targetSelector = isCustom ? '#customTokenizerInfoContent' : '#tokenizerInfoContent';
let htmlContent = '';
if (info.error) {
$(targetSelector).html(`${info.error}
`);
return;
}
// Start building the tooltip content
htmlContent = `
`;
// Dictionary size
if (info.vocab_size) {
htmlContent += `
Dictionary Size
${info.vocab_size.toLocaleString()}
`;
}
// Tokenizer type
if (info.tokenizer_type) {
htmlContent += `
Tokenizer Type
${info.tokenizer_type}
`;
}
// Max length
if (info.model_max_length) {
htmlContent += `
Max Length
${info.model_max_length.toLocaleString()}
`;
}
htmlContent += `
`; // Close tokenizer-info-grid
// Special tokens section
if (info.special_tokens && Object.keys(info.special_tokens).length > 0) {
htmlContent += `
Special Tokens
`;
// Add each special token with proper escaping for HTML special characters
for (const [tokenName, tokenValue] of Object.entries(info.special_tokens)) {
// Properly escape HTML special characters
const escapedValue = tokenValue
.replace(/&/g, '&')
.replace(//g, '>')
.replace(/"/g, '"')
.replace(/'/g, ''');
htmlContent += `
${tokenName}:
${escapedValue}
`;
}
htmlContent += `
`;
}
$(targetSelector).html(htmlContent);
}
// Function to show loading overlay
function showLoadingOverlay(text = 'Loading...') {
$('#loadingText').text(text);
$('#loadingOverlay').addClass('active');
}
// Function to hide loading overlay
function hideLoadingOverlay() {
$('#loadingOverlay').removeClass('active');
}
// Function to fetch tokenizer info
function fetchTokenizerInfo(modelId, isCustom = false) {
if (!modelId) return;
const targetSelector = isCustom ? '#customTokenizerInfoContent' : '#tokenizerInfoContent';
$(targetSelector).html('');
$.ajax({
url: '/tokenizer-info',
method: 'GET',
data: {
model_id: modelId,
is_custom: isCustom
},
success: function(response) {
if (response.error) {
$(targetSelector).html(`${response.error}
`);
} else {
currentTokenizerInfo = response;
updateTokenizerInfoDisplay(response, isCustom);
}
},
error: function(xhr) {
$(targetSelector).html('Failed to load tokenizer information
');
}
});
}
// Token search functionality
let searchMatches = [];
let currentSearchIndex = -1;
let searchVisible = false;
// Token frequency functionality
let tokenFrequencyData = [];
let showFrequencyChart = false;
function performTokenSearch(searchTerm) {
const tokenContainer = $('#tokenContainer');
const tokens = tokenContainer.find('.token');
// Clear previous highlights
tokens.removeClass('highlighted current');
searchMatches = [];
currentSearchIndex = -1;
if (!searchTerm.trim()) {
updateSearchCount();
return;
}
const searchLower = searchTerm.toLowerCase();
// Find matching tokens
tokens.each(function(index) {
const tokenText = $(this).text().toLowerCase();
if (tokenText.includes(searchLower)) {
$(this).addClass('highlighted');
searchMatches.push(index);
}
});
updateSearchCount();
// Navigate to first match if any
if (searchMatches.length > 0) {
navigateToMatch(0);
}
}
function navigateToMatch(index) {
if (searchMatches.length === 0) return;
// Remove current highlight
$('.token.current').removeClass('current');
// Update current index
currentSearchIndex = index;
// Highlight current match
const tokenContainer = $('#tokenContainer');
const tokens = tokenContainer.find('.token');
const currentToken = tokens.eq(searchMatches[currentSearchIndex]);
currentToken.addClass('current');
// Scroll to current match - improved logic
const scrollContainer = tokenContainer;
const containerOffset = scrollContainer.offset();
const tokenOffset = currentToken.offset();
if (containerOffset && tokenOffset) {
const containerHeight = scrollContainer.height();
const containerScrollTop = scrollContainer.scrollTop();
const tokenRelativeTop = tokenOffset.top - containerOffset.top;
// Check if token is outside visible area
if (tokenRelativeTop < 0 || tokenRelativeTop > containerHeight - 50) {
// Calculate new scroll position to center the token
const tokenHeight = currentToken.outerHeight();
const newScrollTop = containerScrollTop + tokenRelativeTop - (containerHeight / 2) + (tokenHeight / 2);
scrollContainer.animate({
scrollTop: Math.max(0, newScrollTop)
}, 400, 'swing');
}
}
updateSearchCount();
}
function toggleSearchVisibility() {
console.log('toggleSearchVisibility called, current state:', searchVisible);
searchVisible = !searchVisible;
const container = $('#tokenSearchContainer');
const toggleBtn = $('#searchToggleBtn');
console.log('Container found:', container.length, 'Toggle button found:', toggleBtn.length);
if (searchVisible) {
// Show the container first, then animate
container.show();
setTimeout(() => {
container.addClass('show');
}, 10);
toggleBtn.addClass('active');
console.log('Showing search container');
setTimeout(() => {
$('#tokenSearchInput').focus();
}, 300);
} else {
container.removeClass('show');
toggleBtn.removeClass('active');
console.log('Hiding search container');
setTimeout(() => {
container.hide();
}, 300);
// Clear search when hiding
$('#tokenSearchInput').val('');
performTokenSearch('');
}
}
function updateSearchCount() {
const countText = searchMatches.length > 0
? `${currentSearchIndex + 1}/${searchMatches.length}`
: `0/${searchMatches.length}`;
$('#searchCount').text(countText);
// Update navigation button states
$('#prevMatch').prop('disabled', searchMatches.length === 0 || currentSearchIndex <= 0);
$('#nextMatch').prop('disabled', searchMatches.length === 0 || currentSearchIndex >= searchMatches.length - 1);
}
// Token frequency chart functions
function calculateTokenFrequency(tokens) {
const frequencyMap = {};
tokens.each(function() {
const tokenText = $(this).text();
if (tokenText.trim()) {
frequencyMap[tokenText] = (frequencyMap[tokenText] || 0) + 1;
}
});
// Convert to array and sort by frequency
const frequencyArray = Object.entries(frequencyMap)
.map(([token, count]) => ({ token, count }))
.sort((a, b) => b.count - a.count)
.slice(0, 10); // Top 10 tokens
return frequencyArray;
}
function renderFrequencyChart(frequencyData) {
const chartContainer = $('#frequencyChart');
chartContainer.empty();
if (frequencyData.length === 0) {
chartContainer.html('No token data available
');
return;
}
const maxCount = frequencyData[0].count;
frequencyData.forEach(({ token, count }) => {
const percentage = (count / maxCount) * 100;
const item = $(`
`);
// Add click handler to search for this token
item.find('.frequency-token').click(function() {
const searchToken = $(this).data('token');
$('#tokenSearchInput').val(searchToken);
performTokenSearch(searchToken);
});
chartContainer.append(item);
});
}
function toggleFrequencyChart() {
showFrequencyChart = !showFrequencyChart;
const container = $('#frequencyChartContainer');
const chart = $('#frequencyChart');
const toggleBtn = $('#toggleFrequencyChart');
if (showFrequencyChart) {
container.show();
chart.show();
toggleBtn.text('Hide Chart').addClass('active');
// Calculate and render frequency data
const tokens = $('#tokenContainer').find('.token');
tokenFrequencyData = calculateTokenFrequency(tokens);
renderFrequencyChart(tokenFrequencyData);
} else {
chart.hide();
toggleBtn.text('Show Chart').removeClass('active');
}
}
function updateResults(data) {
$('#results').show();
// Show search toggle button and frequency chart container
$('#searchToggleBtn').show();
$('#frequencyChartContainer').show();
// Update tokens
const tokenContainer = $('#tokenContainer');
tokenContainer.empty();
data.tokens.forEach(token => {
const span = $('')
.addClass('token')
.css({
'background-color': token.colors.background,
'color': token.colors.text
})
// Include token id in the tooltip on hover
.attr('title', `Original token: ${token.original} | Token ID: ${token.token_id}`)
.text(token.display);
tokenContainer.append(span);
if (token.newline) {
tokenContainer.append('
');
}
});
// Re-apply current search if any
const currentSearch = $('#tokenSearchInput').val();
if (currentSearch.trim()) {
performTokenSearch(currentSearch);
}
// Update display limit notice
if (data.display_limit_reached) {
$('#displayLimitNotice').show();
$('#totalTokenCount').text(data.total_tokens);
} else {
$('#displayLimitNotice').hide();
}
// Update preview notice
if (data.preview_only) {
$('#previewNotice').show();
} else {
$('#previewNotice').hide();
}
// Update basic stats
$('#totalTokens').text(data.stats.basic_stats.total_tokens);
$('#uniqueTokens').text(`${data.stats.basic_stats.unique_tokens} unique`);
$('#uniquePercentage').text(data.stats.basic_stats.unique_percentage);
$('#specialTokens').text(data.stats.basic_stats.special_tokens);
$('#spaceTokens').text(data.stats.basic_stats.space_tokens);
$('#spaceCount').text(data.stats.basic_stats.space_tokens);
$('#newlineCount').text(data.stats.basic_stats.newline_tokens);
$('#compressionRatio').text(data.stats.basic_stats.compression_ratio);
// Update length stats
$('#avgLength').text(data.stats.length_stats.avg_length);
$('#medianLength').text(data.stats.length_stats.median_length);
$('#stdDev').text(data.stats.length_stats.std_dev);
// Update tokenizer info if available
if (data.tokenizer_info) {
currentTokenizerInfo = data.tokenizer_info;
updateTokenizerInfoDisplay(data.tokenizer_info, currentModelType === 'custom');
}
}
// Handle text changes to detach file
$('#textInput').on('input', function() {
// Skip if file was just uploaded (prevents immediate detachment)
if (fileJustUploaded) {
fileJustUploaded = false;
return;
}
const currentText = $(this).val();
const fileInput = document.getElementById('fileInput');
// Only detach if a file exists and text has been substantially modified
if (fileInput.files.length > 0 && originalTextContent !== null) {
// Check if the text is completely different or has been significantly changed
// This allows for small edits without detaching
const isMajorChange =
currentText.length < originalTextContent.length * 0.8 || // Text reduced by at least 20%
(currentText.length > 0 &&
currentText !== originalTextContent.substring(0, currentText.length) &&
currentText.substring(0, Math.min(20, currentText.length)) !==
originalTextContent.substring(0, Math.min(20, currentText.length)));
if (isMajorChange) {
detachFile();
}
}
});
// Function to detach file
function detachFile() {
// Clear the file input
$('#fileInput').val('');
// Hide file info
$('#fileInfo').fadeOut(300);
// Reset the original content tracker
originalTextContent = $('#textInput').val();
// Reset last uploaded filename
lastUploadedFileName = null;
}
// For model changes
$('#modelSelect').change(function() {
const selectedModel = $(this).val();
$('#modelInput').val(selectedModel);
// Fetch tokenizer info for the selected model
fetchTokenizerInfo(selectedModel, false);
// If text exists, submit the form
if ($('#textInput').val().trim()) {
$('#analyzeForm').submit();
}
});
// File drop handling
const fileDropZone = $('#fileDropZone');
const fileUploadIcon = $('#fileUploadIcon');
// Prevent default drag behaviors
['dragenter', 'dragover', 'dragleave', 'drop'].forEach(eventName => {
fileDropZone[0].addEventListener(eventName, preventDefaults, false);
document.body.addEventListener(eventName, preventDefaults, false);
});
function preventDefaults(e) {
e.preventDefault();
e.stopPropagation();
}
// Show drop zone when file is dragged over the document
document.addEventListener('dragenter', showDropZone, false);
document.addEventListener('dragover', showDropZone, false);
fileDropZone[0].addEventListener('dragleave', hideDropZone, false);
fileDropZone[0].addEventListener('drop', hideDropZone, false);
function showDropZone(e) {
fileDropZone.addClass('active');
}
function hideDropZone() {
fileDropZone.removeClass('active');
}
// Handle dropped files
fileDropZone[0].addEventListener('drop', handleDrop, false);
function handleDrop(e) {
const dt = e.dataTransfer;
const files = dt.files;
handleFiles(files);
}
// Also handle file selection via click on the icon
fileUploadIcon.on('click', function() {
const input = document.createElement('input');
input.type = 'file';
input.onchange = e => {
handleFiles(e.target.files);
};
input.click();
});
function handleFiles(files) {
if (files.length) {
const file = files[0];
currentFile = file;
lastUploadedFileName = file.name;
fileJustUploaded = true; // Set flag to prevent immediate detachment
// Show file info with animation and add detach button
$('#fileInfo').html(`${file.name} (${formatFileSize(file.size)}) `).fadeIn(300);
// Add click handler for detach button
$('#fileDetach').on('click', function(e) {
e.stopPropagation(); // Prevent event bubbling
detachFile();
return false;
});
// Set the file to the file input
const dataTransfer = new DataTransfer();
dataTransfer.items.add(file);
document.getElementById('fileInput').files = dataTransfer.files;
// Preview in textarea (first 8096 chars)
const reader = new FileReader();
reader.onload = function(e) {
const previewText = e.target.result.slice(0, 8096);
$('#textInput').val(previewText);
// Store this as the original content AFTER setting the value
// to prevent the input event from firing and detaching immediately
setTimeout(() => {
originalTextContent = previewText;
// Automatically submit for analysis
$('#analyzeForm').submit();
}, 50);
};
reader.readAsText(file);
}
}
function formatFileSize(bytes) {
if (bytes < 1024) return bytes + ' bytes';
else if (bytes < 1048576) return (bytes / 1024).toFixed(1) + ' KB';
else return (bytes / 1048576).toFixed(1) + ' MB';
}
// Make sure to check if there's still a file when analyzing
$('#analyzeForm').on('submit', function(e) {
e.preventDefault();
// Skip detachment check if file was just uploaded
if (!fileJustUploaded) {
// Check if text has been changed but file is still attached
const textInput = $('#textInput').val();
const fileInput = document.getElementById('fileInput');
if (fileInput.files.length > 0 &&
originalTextContent !== null &&
textInput !== originalTextContent &&
textInput.length < originalTextContent.length * 0.8) {
// Text was significantly changed but file is still attached, detach it
detachFile();
}
} else {
// Reset flag after first submission
fileJustUploaded = false;
}
// Update the hidden inputs based on current model type
if (currentModelType === 'custom') {
$('#customModelInputHidden').val($('#customModelInput').val());
} else {
$('#modelInput').val($('#modelSelect').val());
}
const formData = new FormData(this);
const analyzeButton = $('#analyzeButton');
const originalButtonText = analyzeButton.text();
analyzeButton.prop('disabled', true);
analyzeButton.html(originalButtonText + '');
showLoadingOverlay('Analyzing text...');
$.ajax({
url: '/',
method: 'POST',
data: formData,
processData: false,
contentType: false,
success: function(response) {
if (response.error) {
showError(response.error);
} else {
updateResults(response);
// Show success badge if custom model
if (currentModelType === 'custom') {
$('#modelSuccessBadge').addClass('show');
setTimeout(() => {
$('#modelSuccessBadge').removeClass('show');
}, 3000);
}
}
},
error: function(xhr) {
showError(xhr.responseText || 'An error occurred while processing the text');
},
complete: function() {
analyzeButton.prop('disabled', false);
analyzeButton.text(originalButtonText);
hideLoadingOverlay();
}
});
});
$('#expandButton').click(function() {
const container = $('#tokenContainer');
const isExpanded = container.hasClass('expanded');
container.toggleClass('expanded');
$(this).text(isExpanded ? 'Show More' : 'Show Less');
});
// Initialize tokenizer info for current model
if (currentModelType === 'predefined') {
fetchTokenizerInfo($('#modelSelect').val(), false);
} else if ($('#customModelInput').val()) {
fetchTokenizerInfo($('#customModelInput').val(), true);
}
// Add event listener for custom model input
$('#customModelInput').on('change', function() {
const modelValue = $(this).val();
if (modelValue) {
fetchTokenizerInfo(modelValue, true);
}
});
// Keyboard shortcuts - specifically for textarea
$('#textInput').keydown(function(e) {
// Ctrl+Enter (or Cmd+Enter on Mac) to analyze
if ((e.ctrlKey || e.metaKey) && (e.keyCode === 13 || e.which === 13)) {
e.preventDefault();
if ($(this).val().trim()) {
$('#analyzeForm').submit();
}
return false;
}
});
// Global keyboard shortcuts
$(document).keydown(function(e) {
// Ctrl+F (or Cmd+F on Mac) to toggle search
if ((e.ctrlKey || e.metaKey) && (e.keyCode === 70 || e.which === 70)) {
if ($('#searchToggleBtn').is(':visible')) {
e.preventDefault();
if (!searchVisible) {
toggleSearchVisibility();
} else {
$('#tokenSearchInput').focus();
}
return false;
}
}
// Escape to close search or loading overlay
if (e.keyCode === 27 || e.which === 27) {
if (searchVisible) {
toggleSearchVisibility();
return false;
}
if ($('#loadingOverlay').hasClass('active')) {
// Don't close if there's an active request
return false;
}
}
});
// Add keyboard shortcut hint to the textarea placeholder
$('#textInput').attr('placeholder', 'Enter text to analyze or upload a file in bottom left corner... (Ctrl+Enter to analyze)');
// Token search event handlers
$('#tokenSearchInput').on('input', function() {
const searchTerm = $(this).val();
performTokenSearch(searchTerm);
});
$('#nextMatch').click(function() {
if (currentSearchIndex < searchMatches.length - 1) {
navigateToMatch(currentSearchIndex + 1);
}
});
$('#prevMatch').click(function() {
if (currentSearchIndex > 0) {
navigateToMatch(currentSearchIndex - 1);
}
});
$('#clearSearch').click(function() {
$('#tokenSearchInput').val('');
performTokenSearch('');
});
// Additional keyboard shortcuts for search
$('#tokenSearchInput').keydown(function(e) {
if (e.keyCode === 13) { // Enter
e.preventDefault();
if (e.shiftKey) {
// Shift+Enter: previous match
$('#prevMatch').click();
} else {
// Enter: next match
$('#nextMatch').click();
}
} else if (e.keyCode === 27) { // Escape
$('#clearSearch').click();
$(this).blur();
}
});
// Search toggle handler using event delegation
$(document).on('click', '#searchToggleBtn', function(e) {
console.log('Search toggle button clicked!');
e.preventDefault();
e.stopPropagation();
toggleSearchVisibility();
return false;
});
// Frequency chart toggle handler
$('#toggleFrequencyChart').click(function() {
toggleFrequencyChart();
});
// Mobile touch enhancements
function addTouchSupport() {
// Add touch-friendly double-tap for expand/collapse
let lastTap = 0;
$('#tokenContainer').on('touchend', function(e) {
const currentTime = new Date().getTime();
const tapLength = currentTime - lastTap;
if (tapLength < 500 && tapLength > 0) {
$('#expandButton').click();
e.preventDefault();
}
lastTap = currentTime;
});
// Improve touch scrolling for token container
$('#tokenContainer').on('touchstart', function(e) {
this.scrollTop = this.scrollTop;
});
}
// Check if mobile device and add touch support
if ('ontouchstart' in window || navigator.maxTouchPoints > 0) {
addTouchSupport();
}
});