Deploy SQLCoder with Streamlit

app.py

@@ -7,13 +7,13 @@ st.set_page_config(page_title="SQLCoder", layout="wide")
 # Load the model and tokenizer
 @st.cache_resource
 def load_model():
-    # model_name = "defog/sqlcoder-7b-2"
-    model_name = "EleutherAI/gpt-neo-1.3B"
+    model_name = "defog/sqlcoder-7b-2"
     tokenizer = AutoTokenizer.from_pretrained(model_name)
     model = AutoModelForCausalLM.from_pretrained(
         model_name,
-        device_map="auto",  # Automatically uses GPU if available
-        low_cpu_mem_usage=True  # Optimizes memory usage
+        device_map="auto",
+        load_in_8bit=True,  # Use int8 quantization
+        low_cpu_mem_usage=True,
     )
     return tokenizer, model
 

requirements.txt

@@ -2,3 +2,4 @@ transformers
 torch
 streamlit
 accelerate
+bitsandbytes