CPU/GPU support

.gitattributes

@@ -324,3 +324,4 @@ assets/pdfs/pg_0197.pdf filter=lfs diff=lfs merge=lfs -text
 assets/pdfs/pg_0243.pdf filter=lfs diff=lfs merge=lfs -text
 assets/pdfs/pg_0205.pdf filter=lfs diff=lfs merge=lfs -text
 assets/pdfs/pg_0228.pdf filter=lfs diff=lfs merge=lfs -text
+*.png filter=lfs diff=lfs merge=lfs -text

app.py

@@ -1,7 +1,9 @@
 # TODO: return all pages used to form answer
+# TODO: question samples
+# TEST: with and without GPU instance
+# TODO: visual questions on page image (in same app)?
 
 import torch
-from transformers import BitsAndBytesConfig
 from llama_index.llms.huggingface import HuggingFaceLLM
 from llama_index.embeddings.huggingface import HuggingFaceEmbedding
 from llama_index.core import SimpleDirectoryReader
@@ -36,12 +38,16 @@ def messages_to_prompt(messages):
 
 def load_RAG_pipeline():
     # LLM
-    quantization_config = BitsAndBytesConfig(
-        load_in_4bit=True,
-        bnb_4bit_compute_dtype=torch.float16,
-        bnb_4bit_quant_type="nf4",
-        bnb_4bit_use_double_quant=True,
-    )
+    quantization_config = {}  # dirty fix for CPU/GPU support
+    if torch.cuda.is_available():
+        from transformers import BitsAndBytesConfig
+
+        quantization_config = BitsAndBytesConfig(
+            load_in_4bit=True,
+            bnb_4bit_compute_dtype=torch.float16,
+            bnb_4bit_quant_type="nf4",
+            bnb_4bit_use_double_quant=True,
+        )
 
     llm = HuggingFaceLLM(
         model_name="HuggingFaceH4/zephyr-7b-alpha",