Update app.py

app.py

@@ -15,22 +15,49 @@ model, tokenizer = load_model(
     finetuned="chansung/alpaca-lora-13b"
 )    
 
-model = StreamModel(model, tokenizer)
+stream_model = StreamModel(model, tokenizer)
 
 def chat_stream(
     context,
     instruction,
     state_chatbot,
 ):
-    if len(context) > 150 or len(instruction) > 150:
+    if len(context) > 500 or len(instruction) > 150:
         raise gr.Error("context or prompt is too long!")
-
+    
+    bot_summarized_response = ''
     # user input should be appropriately formatted (don't be confused by the function name)
     instruction_display = common_post_process(instruction)
-    instruction_prompt = generate_prompt(instruction, state_chatbot, context)    
-    bot_response = model(
+    instruction_prompt, conv_length = generate_prompt(instruction, state_chatbot, context)
+    
+    if conv_length > num_of_characters_to_keep:
+        instruction_prompt = generate_prompt(SPECIAL_STRS["summarize"], state_chatbot, context)[0]
+        
+        state_chatbot = state_chatbot + [
+            (
+                None, 
+                "![](https://s2.gifyu.com/images/icons8-loading-circle.gif) too long conversations, so let's summarize..."
+            )
+        ]
+        yield (state_chatbot, state_chatbot, context)
+        
+        bot_summarized_response = get_output_batch(
+            model, tokenizer, [instruction_prompt], generation_config
+        )[0]
+        bot_summarized_response = bot_summarized_response.split("### Response:")[-1].strip()
+        
+        state_chatbot[-1] = (
+            None, 
+            "✅ summarization is done and set as context"
+        )
+        print(f"bot_summarized_response: {bot_summarized_response}")
+        yield (state_chatbot, state_chatbot, f"{context}. {bot_summarized_response}")
+        
+    instruction_prompt = generate_prompt(instruction, state_chatbot, f"{context} {bot_summarized_response}")[0]
+    
+    bot_response = stream_model(
         instruction_prompt,
-        max_tokens=128,
+        max_tokens=256,
         temperature=1,
         top_p=0.9
     )
@@ -60,7 +87,7 @@ def chat_stream(
                         instruction_display, 
                         processed_response
                     )
-                    yield (state_chatbot, state_chatbot, context)
+                    yield (state_chatbot, state_chatbot, f"{context} {bot_summarized_response}")
                     break
                 else:
                     agg_tokens = ""
@@ -69,7 +96,7 @@ def chat_stream(
         if agg_tokens == "":
             processed_response, to_exit = post_process_stream(tokens)
             state_chatbot[-1] = (instruction_display, processed_response)
-            yield (state_chatbot, state_chatbot, context)
+            yield (state_chatbot, state_chatbot, f"{context} {bot_summarized_response}")
 
             if to_exit:
                 break
@@ -79,7 +106,7 @@ def chat_stream(
     yield (
         state_chatbot,
         state_chatbot,
-        gr.Textbox.update(value=tokens) if instruction_display == SPECIAL_STRS["summarize"] else context
+        f"{context} {bot_summarized_response}"
     )
 
 def chat_batch(