Update app.py

app.py

@@ -51,29 +51,16 @@ def format_prompt(message, history):
     prompt += f"<start_of_turn>user{message}<end_of_turn><start_of_turn>model"
     print(prompt)
     return prompt
-mega_hist=[[],[],[],[]]
 
 
-def chat_inf(system_prompt,prompt,history,client_choice,seed,temp,tokens,top_p,rep_p,hid_val):
+mega_hist=[[],[],[],[]]
+def chat_inf_tree(system_prompt,prompt,history,client_choice,seed,temp,tokens,top_p,rep_p,hid_val):
     if len(client_choice)>=hid_val:
-            
-        #token max=8192
         client=client_z[int(hid_val)-1]
-        
         if history:
             mega_hist[hid_val-1]=history
             #history = []
             hist_len=0
-        #if mega_hist[hid_val-1]:
-        #    hist_len=len(mega_hist[hid_val-1])
-        #    print(hist_len)
-        #in_len=len(system_prompt+prompt)+hist_len
-        #print("\n#########"+str(in_len))
-        #if (in_len+tokens) > 8000:
-        #    yield [(prompt,"Wait. I need to compress our Chat history...")]
-        #    #history=compress_history(history,client_choice,seed,temp,tokens,top_p,rep_p)
-        #    yield [(prompt,"History has been compressed, processing request...")]
-                
         generate_kwargs = dict(
             temperature=temp,
             max_new_tokens=tokens,
@@ -86,7 +73,6 @@ def chat_inf(system_prompt,prompt,history,client_choice,seed,temp,tokens,top_p,r
         formatted_prompt = format_prompt(f"{system_prompt}, {prompt}", mega_hist[hid_val-1])
         stream = client.text_generation(formatted_prompt, **generate_kwargs, stream=True, details=True, return_full_text=False)
         output = ""
-            
         for response in stream:
             output += response.token.text
             yield [(prompt,output)]
@@ -98,25 +84,65 @@ def chat_inf(system_prompt,prompt,history,client_choice,seed,temp,tokens,top_p,r
 
 
 
-def chat_inf_og(system_prompt,prompt,history,client_choice,seed,temp,tokens,top_p,rep_p,hid_val):
+def chat_inf_a(system_prompt,prompt,history,client_choice,seed,temp,tokens,top_p,rep_p,hid_val):
+    if len(client_choice)>=hid_val:
+        client=client_z[int(hid_val)-1]
+        if not history:
+            history = []
+            hist_len=0
+        generate_kwargs = dict(
+            temperature=temp,
+            max_new_tokens=tokens,
+            top_p=top_p,
+            repetition_penalty=rep_p,
+            do_sample=True,
+            seed=seed,
+        )
+        #formatted_prompt=prompt   
+        formatted_prompt = format_prompt(f"{system_prompt}, {prompt}", history)
+        stream = client.text_generation(formatted_prompt, **generate_kwargs, stream=True, details=True, return_full_text=False)
+        output = ""
+        for response in stream:
+            output += response.token.text
+            yield [(prompt,output)]
+        history.append((prompt,output))
+        yield history
+    else:
+        yield None
+
+
+def chat_inf_b(system_prompt,prompt,history,client_choice,seed,temp,tokens,top_p,rep_p,hid_val):
+    if len(client_choice)>=hid_val:
+        client=client_z[int(hid_val)-1]
+        if not history:
+            history = []
+            hist_len=0
+        generate_kwargs = dict(
+            temperature=temp,
+            max_new_tokens=tokens,
+            top_p=top_p,
+            repetition_penalty=rep_p,
+            do_sample=True,
+            seed=seed,
+        )
+        #formatted_prompt=prompt   
+        formatted_prompt = format_prompt(f"{system_prompt}, {prompt}", history)
+        stream = client.text_generation(formatted_prompt, **generate_kwargs, stream=True, details=True, return_full_text=False)
+        output = ""
+        for response in stream:
+            output += response.token.text
+            yield [(prompt,output)]
+        history.append((prompt,output))
+        yield history
+    else:
+        yield None
+
+def chat_inf_c(system_prompt,prompt,history,client_choice,seed,temp,tokens,top_p,rep_p,hid_val):
     if len(client_choice)>=hid_val:
-            
-        #token max=8192
         client=client_z[int(hid_val)-1]
-        
         if not history:
             history = []
             hist_len=0
-        if history:
-            hist_len=len(history)
-            print(hist_len)
-        in_len=len(system_prompt+prompt)+hist_len
-        print("\n#########"+str(in_len))
-        if (in_len+tokens) > 8000:
-            yield [(prompt,"Wait. I need to compress our Chat history...")]
-            #history=compress_history(history,client_choice,seed,temp,tokens,top_p,rep_p)
-            yield [(prompt,"History has been compressed, processing request...")]
-                
         generate_kwargs = dict(
             temperature=temp,
             max_new_tokens=tokens,
@@ -129,7 +155,6 @@ def chat_inf_og(system_prompt,prompt,history,client_choice,seed,temp,tokens,top_
         formatted_prompt = format_prompt(f"{system_prompt}, {prompt}", history)
         stream = client.text_generation(formatted_prompt, **generate_kwargs, stream=True, details=True, return_full_text=False)
         output = ""
-            
         for response in stream:
             output += response.token.text
             yield [(prompt,output)]
@@ -137,6 +162,32 @@ def chat_inf_og(system_prompt,prompt,history,client_choice,seed,temp,tokens,top_
         yield history
     else:
         yield None
+
+def chat_inf_d(system_prompt,prompt,history,client_choice,seed,temp,tokens,top_p,rep_p,hid_val):
+    if len(client_choice)>=hid_val:
+        client=client_z[int(hid_val)-1]
+        if not history:
+            history = []
+            hist_len=0
+        generate_kwargs = dict(
+            temperature=temp,
+            max_new_tokens=tokens,
+            top_p=top_p,
+            repetition_penalty=rep_p,
+            do_sample=True,
+            seed=seed,
+        )
+        #formatted_prompt=prompt   
+        formatted_prompt = format_prompt(f"{system_prompt}, {prompt}", history)
+        stream = client.text_generation(formatted_prompt, **generate_kwargs, stream=True, details=True, return_full_text=False)
+        output = ""
+        for response in stream:
+            output += response.token.text
+            yield [(prompt,output)]
+        history.append((prompt,output))
+        yield history
+    else:
+        yield None        
 def clear_fn():
     return None,None,None
 rand_val=random.randint(1,1111111111111111)
@@ -197,10 +248,10 @@ with gr.Blocks() as app:
     #im_go=im_btn.click(get_screenshot,[chat_b,im_height,im_width,chatblock,theme,wait_time],img)
     #chat_sub=inp.submit(check_rand,[rand,seed],seed).then(chat_inf,[sys_inp,inp,chat_b,client_choice,seed,temp,tokens,top_p,rep_p],chat_b)
     
-    go1=btn.click(check_rand,[rand,seed],seed).then(chat_inf,[sys_inp,inp,chat_b,client_choice,seed,temp,tokens,top_p,rep_p,hid1],chat_a,batch=True, max_batch_size=16)
-    go2=btn.click(check_rand,[rand,seed],seed).then(chat_inf,[sys_inp,inp,chat_b,client_choice,seed,temp,tokens,top_p,rep_p,hid2],chat_b,batch=True, max_batch_size=16)
-    go3=btn.click(check_rand,[rand,seed],seed).then(chat_inf,[sys_inp,inp,chat_b,client_choice,seed,temp,tokens,top_p,rep_p,hid3],chat_c,batch=True, max_batch_size=16)
-    go4=btn.click(check_rand,[rand,seed],seed).then(chat_inf,[sys_inp,inp,chat_b,client_choice,seed,temp,tokens,top_p,rep_p,hid4],chat_d,batch=True, max_batch_size=16)
+    go1=btn.click(check_rand,[rand,seed],seed).then(chat_inf_a,[sys_inp,inp,chat_b,client_choice,seed,temp,tokens,top_p,rep_p,hid1],chat_a)
+    go2=btn.click(check_rand,[rand,seed],seed).then(chat_inf_b,[sys_inp,inp,chat_b,client_choice,seed,temp,tokens,top_p,rep_p,hid2],chat_b)
+    go3=btn.click(check_rand,[rand,seed],seed).then(chat_inf_c,[sys_inp,inp,chat_b,client_choice,seed,temp,tokens,top_p,rep_p,hid3],chat_c)
+    go4=btn.click(check_rand,[rand,seed],seed).then(chat_inf_d,[sys_inp,inp,chat_b,client_choice,seed,temp,tokens,top_p,rep_p,hid4],chat_d)
     
     stop_btn.click(None,None,None,cancels=[go1,go2,go3,go4])
     clear_btn.click(clear_fn,None,[inp,sys_inp,chat_a,chat_b,chat_c,chat_d])