fix tokenizer issues

app.py

@@ -26,18 +26,18 @@ print(f"Using device: {device}")
 
 # Load tokenizer with proper error handling
 try:
-    # Try to load from local tokenizer directory
-    tokenizer_path = '/app/tokenizers/indobert-base-p1'
-    if os.path.exists(tokenizer_path):
-        print(f"Loading tokenizer from local path: {tokenizer_path}")
-        tokenizer = AutoTokenizer.from_pretrained(tokenizer_path)
-    else:
-        # If local not available, try direct download with cache
-        print("Local tokenizer not found, downloading from Hugging Face...")
-        # tokenizer = AutoTokenizer.from_pretrained('indobenchmark/indobert-base-p1', 
-        #                                          use_fast=True,
-        #                                          cache_dir='/app/tokenizers')
-        tokenizer = BertTokenizerFast.from_pretrained("indobenchmark/indobert-base-p1")
+    # # Try to load from local tokenizer directory
+    # tokenizer_path = '/app/tokenizers/indobert-base-p1'
+    # if os.path.exists(tokenizer_path):
+    #     print(f"Loading tokenizer from local path: {tokenizer_path}")
+    #     tokenizer = AutoTokenizer.from_pretrained(tokenizer_path)
+    # else:
+    #     # If local not available, try direct download with cache
+    #     print("Local tokenizer not found, downloading from Hugging Face...")
+    #     # tokenizer = AutoTokenizer.from_pretrained('indobenchmark/indobert-base-p1', 
+    #     #                                          use_fast=True,
+    #     #                                          cache_dir='/app/tokenizers')
+    tokenizer = BertTokenizerFast.from_pretrained("indobenchmark/indobert-base-p1")
 except Exception as e:
     print(f"Error loading tokenizer: {e}")
     # Fallback to default BERT tokenizer if needed