feat: add stream

main.py

@@ -45,7 +45,7 @@ DB = Chroma(
 
 RETRIEVER = DB.as_retriever()
 
-LLM = load_model(device_type=DEVICE_TYPE, model_id=MODEL_ID, model_basename=MODEL_BASENAME, stream=False)
+LLM, STREAMER = load_model(device_type=DEVICE_TYPE, model_id=MODEL_ID, model_basename=MODEL_BASENAME, stream=False)
 
 template = """you are a helpful, respectful and honest assistant.
 Your name is Katara llma. You should only use the source documents provided to answer the questions.
@@ -134,12 +134,6 @@ def run_ingest_route():
         )
 
 
-        generated_text = ""
-        for new_text in StreamData:
-            generated_text += new_text
-            print(generated_text)
-
-
         return {"response": "The training was successfully completed"}
     except Exception as e:
         raise HTTPException(status_code=500, detail=f"Error occurred: {str(e)}")
@@ -189,6 +183,12 @@ async def predict(data: Predict):
                 (os.path.basename(str(document.metadata["source"])), str(document.page_content))
             )
 
+
+        generated_text = ""
+        for new_text in STREAMER:
+            generated_text += new_text
+            print(generated_text)
+
         return {"response": prompt_response_dict}
     else:
         raise HTTPException(status_code=400, detail="Prompt Incorrect")

run_localGPT.py

@@ -89,13 +89,13 @@ def load_model(device_type, model_id, model_basename=None, LOGGING=logging, stre
         top_k=40,
         repetition_penalty=1.0,
         generation_config=generation_config,
-        callback=[StreamingStdOutCallbackHandler()]
+        streamer=streamer
     )
 
     local_llm = HuggingFacePipeline(pipeline=pipe)
     logging.info("Local LLM Loaded")
 
-    return local_llm
+    return (local_llm, streamer)
 
 
 def retrieval_qa_pipline(device_type, use_history, promptTemplate_type="llama"):

static/index.html

@@ -6,7 +6,11 @@
     <meta name="viewport" content="width=device-width, initial-scale=1.0" />
     <!-- Bootstrap CSS -->
 
-    <!-- <link rel="icon" type="image" href="static\social_icons\favicon.png" /> -->
+    <link
+      rel="icon"
+      type="image"
+      href="https://katara.earth/katara-favicon.svg"
+    />
     <script src="https://code.jquery.com/jquery-3.6.0.min.js"></script>
     <link
       href="https://cdn.jsdelivr.net/npm/[email protected]/dist/css/bootstrap.min.css"