Use TinyLlama tokenizer as fallback for corrupted tokenizer

app.py

@@ -9,10 +9,13 @@ DEVICE = "cuda" if torch.cuda.is_available() else "cpu"
 # Initialize model and tokenizer
 print(f"Loading model {MODEL_ID}...")
 try:
-    tokenizer = AutoTokenizer.from_pretrained(MODEL_ID, use_fast=False)
+    # Try loading the tokenizer normally
+    tokenizer = AutoTokenizer.from_pretrained(MODEL_ID)
 except Exception as e:
-    print(f"Failed to load tokenizer with use_fast=False, trying with use_fast=True: {e}")
-    tokenizer = AutoTokenizer.from_pretrained(MODEL_ID, use_fast=True, trust_remote_code=True)
+    print(f"Failed to load tokenizer from {MODEL_ID}: {e}")
+    print("Using fallback tokenizer from base model...")
+    # Use the base model's tokenizer as fallback
+    tokenizer = AutoTokenizer.from_pretrained("TinyLlama/TinyLlama-1.1B-Chat-v1.0")
 
 model = AutoModelForCausalLM.from_pretrained(
     MODEL_ID,