v1

app.py

@@ -15,6 +15,10 @@ from meteor.load_meteor import load_meteor
 from transformers import TextIteratorStreamer
 from torchvision.transforms.functional import pil_to_tensor
 
+# flash attention
+import subprocess
+subprocess.run('pip install flash-attn --no-build-isolation', env={'FLASH_ATTENTION_SKIP_CUDA_BUILD': "TRUE"}, shell=True)
+
 # accel
 accel = Accelerator()