Update app.py

app.py

@@ -13,9 +13,24 @@ from g4f import Provider, models
 from langchain.llms.base import LLM
 from langchain.embeddings import HuggingFaceEmbeddings
 from langchain_g4f import G4FLLM
+from langchain.callbacks.streaming_stdout import StreamingStdOutCallbackHandler
+from huggingface_hub import hf_hub_download
+from langchain.llms import LlamaCpp
+from langchain import PromptTemplate, LLMChain
 
+model_name_or_path = "hlhr202/llama-7B-ggml-int4"
+model_basename = "ggml-model-q4_0.bin" # the model is in bin format
 
-llm = LLM = G4FLLM(model=models.gpt_35_turbo,provider=Provider.DeepAi,)
+model_path = hf_hub_download(repo_id=model_name_or_path, filename=model_basename)
+
+n_gpu_layers = 40 # Change this value based on your model and your GPU VRAM pool.
+n_batch = 256 
+
+llm = LlamaCpp(
+    model_path, callbacks=[StreamingStdOutCallbackHandler()]
+)
+
+#llm = LLM = G4FLLM(model=models.gpt_35_turbo,provider=Provider.DeepAi,)
 
 embeddings = HuggingFaceEmbeddings(
     model_name="sentence-transformers/all-mpnet-base-v2"