v1

meteor/load_meteor.py

@@ -17,7 +17,7 @@ def load_meteor(link, bits):
         huggingface_config.update(dict(
             torch_dtype=torch.float16,
             low_cpu_mem_usage=True,
-            # attn_implementation="flash_attention_2",
+            attn_implementation="flash_attention_2",
             quantization_config=BitsAndBytesConfig(
                 load_in_4bit=bits == 4,
                 load_in_8bit=bits == 8,
@@ -33,7 +33,7 @@ def load_meteor(link, bits):
         huggingface_config.update(dict(
             torch_dtype=torch.float16,
             low_cpu_mem_usage=True,
-            # attn_implementation="flash_attention_2",
+            attn_implementation="flash_attention_2",
         ))
 
     # loading backbone model