Update app.py

app.py

@@ -8,11 +8,11 @@ import torch
 from transformers import AutoTokenizer, AutoModelForCausalLM, BitsAndBytesConfig
 from huggingface_hub import login, HfFolder
 # tokenizer = AutoTokenizer.from_pretrained("FlawedLLM/Bhashini_gemma_merged16bit_clean_final", trust_remote_code=True)
-# quantization_config = BitsAndBytesConfig(
-#         load_in_4bit=True,
-#         bnb_4bit_use_double_quant=True,
-#         bnb_4bit_quant_type="nf4",  
-#         bnb_4bit_compute_dtype=torch.float16)
+quantization_config = BitsAndBytesConfig(
+        load_in_4bit=True,
+        bnb_4bit_use_double_quant=True,
+        bnb_4bit_quant_type="nf4",  
+        bnb_4bit_compute_dtype=torch.float16)
 # model = AutoModelForCausalLM.from_pretrained("FlawedLLM/Bhashini_gemma_merged16bit_clean_final", 
 #                                              device_map="auto",
 #                                              quantization_config=quantization_config, 
@@ -31,14 +31,14 @@ from huggingface_hub import login, HfFolder
 # model = PeftModel.from_pretrained(model, lora_weights)
 # Load model directly
 # from transformers import AutoTokenizer, AutoModelForCausalLM
-bnb_config = BitsAndBytesConfig(
-    load_in_8bit=True,
-    llm_int8_threshold=6.0,
-)
+# bnb_config = BitsAndBytesConfig(
+#     load_in_4bit=True,
+#     llm_int8_threshold=6.0,
+# )
 tokenizer = AutoTokenizer.from_pretrained("FlawedLLM/Bhashini_gemma_merged4bit_clean_final")
 model = AutoModelForCausalLM.from_pretrained("FlawedLLM/Bhashini_gemma_merged4bit_clean_final",
                                              device_map="auto",
-                                             quantization_config=bnb_config,)
+                                             quantization_config=quantization_config,)
 # alpaca_prompt = You MUST copy from above!
 @spaces.GPU(duration=300)
 def chunk_it(input_command, item_list):