Update index.html

index.html

@@ -131,11 +131,12 @@
 
     async function tokenize(){
       const myRun = ++runId;
+    
       if (!tokenizer) {
         await loadTokenizer(modelSel.value);
-        if (!tokenizer) { render(); return; } // keep UI stable on failure
+        if (!tokenizer) { render(); return; }
       }
-
+    
       const text = (inputEl.value ?? '').trim();
       if (!text) {
         state.tokens = [];
@@ -144,24 +145,31 @@
         status('Type to tokenize…');
         return;
       }
-
+    
       status('Tokenizing…');
       try {
+        // Get token strings (robust, always works)
+        const tokens = await tokenizer.tokenize(text);
+    
+        // Get IDs (encode() may return different field names across versions)
         const enc = await tokenizer.encode(text);
-        if (myRun !== runId) return; // drop stale results
-        state.tokens = Array.isArray(enc?.tokens) ? enc.tokens : [];
-        state.ids    = Array.isArray(enc?.ids)    ? enc.ids    : [];
+        const ids =
+          (enc && (enc.ids ?? enc.input_ids ?? enc.inputIds)) || [];
+    
+        if (myRun !== runId) return; // drop stale result
+        state.tokens = Array.isArray(tokens) ? tokens : [];
+        state.ids    = Array.isArray(ids)    ? ids    : [];
+    
         render();
         status(`Done. ${state.tokens.length} tokens.`);
       } catch (e) {
         console.error('Tokenize failed:', e);
-        state.tokens = state.tokens ?? [];
-        state.ids    = state.ids ?? [];
         render();
         status('Error tokenizing. See console.');
       }
     }
 
+
     function render(){
       const tokens = Array.isArray(state.tokens) ? state.tokens : [];
       const ids    = Array.isArray(state.ids)    ? state.ids    : [];