Update tokenizer.py

tokenizer.py

@@ -15,12 +15,14 @@ class Tokenizer:
         self.bos_id = None
         self.eos_id = None
 
+        # Debug statements
+        # print(f'tokenizer.py checkpoint_dir is : {checkpoint_dir}')
+        # print(f'checking the file : {(checkpoint_dir / "tokenizer.json").is_file()}')
+        # print(f'Current working directory is : {os.getcwd()}')
+        # curr_dir = os.getcwd()
+        # print(f'contents in pwd are : {os.listdir(curr_dir)}')
+
         # some checkpoints have both files, `.model` takes precedence
-        print(f'tokenizer.py checkpoint_dir is : {checkpoint_dir}')
-        print(f'checking the file : {(checkpoint_dir / "tokenizer.json").is_file()}')
-        print(f'Current working directory is : {os.getcwd()}')
-        curr_dir = os.getcwd()
-        print(f'contents in pwd are : {os.listdir(curr_dir)}')
         if (vocabulary_path := checkpoint_dir / "tokenizer.model").is_file():
             from sentencepiece import SentencePieceProcessor