update

app.py

@@ -6,140 +6,21 @@
 
 
 
-# 单卡报错
-python moss_web_demo_gradio.py --model_name fnlp/moss-moon-003-sft --gpu 0,1,2,3
+## TODO
 
-# TODO
 - 第一句：
 - 代码和表格的预览
+- markdown解析：mdtex2html
 - 可编辑chatbot：https://github.com/gradio-app/gradio/issues/4444
 - 一个button，
 
+
 ## Reference
 
--
+- https://github.com/GaiZhenbiao/ChuanhuChatGPT/
 """
 
-
-import gradio as gr
-# from models.hf_qwen2 import bot
-from models.cpp_qwen2 import bot
-
-
-#
-# def postprocess(self, y):
-#     if y is None:
-#         return []
-#     for i, (message, response) in enumerate(y):
-#         y[i] = (
-#             None if message is None else mdtex2html.convert((message)),
-#             None if response is None else mdtex2html.convert(response),
-#         )
-#     return y
-#
-#
-# gr.Chatbot.postprocess = postprocess
-
-
-def parse_text(text):
-    """copy from https://github.com/GaiZhenbiao/ChuanhuChatGPT/"""
-    lines = text.split("\n")
-    lines = [line for line in lines if line != ""]
-    count = 0
-    for i, line in enumerate(lines):
-        if "```" in line:
-            count += 1
-            items = line.split('`')
-            if count % 2 == 1:
-                lines[i] = f'<pre><code class="language-{items[-1]}">'
-            else:
-                lines[i] = f'<br></code></pre>'
-        else:
-            if i > 0:
-                if count % 2 == 1:
-                    line = line.replace("`", "\`")
-                    line = line.replace("<", "&lt;")
-                    line = line.replace(">", "&gt;")
-                    line = line.replace(" ", "&nbsp;")
-                    line = line.replace("*", "&ast;")
-                    line = line.replace("_", "&lowbar;")
-                    line = line.replace("-", "&#45;")
-                    line = line.replace(".", "&#46;")
-                    line = line.replace("!", "&#33;")
-                    line = line.replace("(", "&#40;")
-                    line = line.replace(")", "&#41;")
-                    line = line.replace("$", "&#36;")
-                lines[i] = "<br>" + line
-    text = "".join(lines)
-    return text
-
-
-def generate_query(chatbot, history):
-    if history and history[-1]["role"] == "user":  # 该生成response了
-        gr.Warning('You should generate assistant-response.')
-        yield None, chatbot, history
-    else:
-        chatbot.append(None)
-        streamer = bot.generate_query(history, stream=True)
-        for query in streamer:
-            chatbot[-1] = (query, None)
-            yield query, chatbot, history
-
-        history.append({"role": "user", "content": query})
-        yield query, chatbot, history
-
-
-def generate_response(query, chatbot, history):
-    """
-    自动模式下：query is None
-    人工模式下：query 是用户输入
-    :param query:
-    :param chatbot:
-    :param history:
-    :return:
-    """
-    if query and history[-1]["role"] != "user":
-        history.append({"role": "user", "content": query})
-    query = history[-1]["content"]
-
-    if history[-1]["role"] != "user":
-        gr.Warning('You should generate or type user-input first.')
-        yield chatbot, history
-    else:
-        streamer = bot.generate_response(history, stream=True)
-        for response in streamer:
-            chatbot[-1] = (query, response)
-            yield chatbot, history
-
-        history.append({"role": "assistant", "content": response})
-        print(f"chatbot is {chatbot}")
-        print(f"history is {history}")
-        yield chatbot, history
-
-
-def generate():
-    """
-
-    :return:
-    """
-    pass
-
-
-def regenerate():
-    """
-    删除上一轮，重新生成。
-    :return:
-    """
-    pass
-
-
-def reset_user_input():
-    return gr.update(value='')
-
-
-def reset_state(system):
-    return [], [{"role": "system", "content": system}]
-
+from app_util import *
 
 system_list = [
     "You are a helpful assistant.",
@@ -165,55 +46,50 @@ with gr.Blocks() as demo:
     chatbot = gr.Chatbot(avatar_images=("assets/man.png", "assets/bot.png"))
     with gr.Row():
         with gr.Column(scale=4):
-            user_input = gr.Textbox(show_label=False, placeholder="Input...", lines=10)
+            # user_input = gr.Textbox(show_label=False, placeholder="Input...", lines=10)
             with gr.Row():
-                generate_query_btn = gr.Button("生成问题")
-                regen_btn = gr.Button("🤔️ Regenerate (重试)")
-                submit_btn = gr.Button("生成回复", variant="primary")
+                generate_btn = gr.Button("🤔️ Generate")
+                retry_btn = gr.Button("🔄  Regenerate")
+                undo_btn = gr.Button("↩️ Undo")
+                clear_btn = gr.Button("🗑️  Clear")  # 🧹 Clear History (清除历史)
                 stop_btn = gr.Button("停止生成", variant="primary")
-                empty_btn = gr.Button("🧹 Clear History (清除历史)")
         with gr.Column(scale=1):
             # generate_query_btn = gr.Button("Generate First Query")
-
-            clear_btn = gr.Button("重置")
             gr.Dropdown(
                 ["moss", "chatglm-2", "chatpdf"],
                 value="moss",
                 label="问题生成器",
                 # info="Will add more animals later!"
-            ),
+            )
             gr.Dropdown(
                 ["moss", "chatglm-2", "gpt3.5-turbo"],
                 value="gpt3.5-turbo",
                 label="回复生成器",
                 # info="Will add more animals later!"
-            ),
-
+            )
+
+        slider_max_new_tokens = gr.Slider(minimum=1, maximum=2048, value=512, step=1, label="Max new tokens")
+        slider_temperature = gr.Slider(minimum=0.1, maximum=10.0, value=5, step=0.1, label="Temperature",
+                                       info="Larger temperature increase the randomness")
+        slider_top_p = gr.Slider(
+            minimum=0.1,
+            maximum=1.0,
+            value=0.95,
+            step=0.05,
+            label="Top-p (nucleus sampling)",
+        )
+
+    ########
     history = gr.State([{"role": "system", "content": system_list[0]}])
-
     system.change(reset_state, inputs=[system], outputs=[chatbot, history], show_progress="full")
-
-    submit_btn.click(generate_response, [user_input, chatbot, history], [chatbot, history],
-                     show_progress="full")
     # submit_btn.click(reset_user_input, [], [user_input])
 
     clear_btn.click(reset_state, inputs=[system], outputs=[chatbot, history], show_progress="full")
 
-    generate_query_btn.click(generate_query, [chatbot, history], outputs=[user_input, chatbot, history],
+    generate_btn.click(generate, [chatbot, history], outputs=[chatbot, history],
                              show_progress="full")
 
-    # generate_query_btn.
-
-    gr.Slider(minimum=1, maximum=2048, value=512, step=1, label="Max new tokens"),
-    gr.Slider(minimum=0.1, maximum=4.0, value=0.7, step=0.1, label="Temperature",
-              info="Larger temperature increase the randomness"),
-    gr.Slider(
-        minimum=0.1,
-        maximum=1.0,
-        value=0.95,
-        step=0.05,
-        label="Top-p (nucleus sampling)",
-    ),
+    slider_max_new_tokens.change(set_max_tokens, inputs=[slider_max_new_tokens])
 
 demo.queue().launch(share=False, server_name="0.0.0.0")
 # demo.queue().launch(share=True)

app_util.py

@@ -0,0 +1,102 @@
+
+import gradio as gr
+from utils.logging_util import logger
+from models.cpp_qwen2 import bot
+
+
+
+
+#
+# def postprocess(self, y):
+#     if y is None:
+#         return []
+#     for i, (message, response) in enumerate(y):
+#         y[i] = (
+#             None if message is None else mdtex2html.convert((message)),
+#             None if response is None else mdtex2html.convert(response),
+#         )
+#     return y
+#
+# gr.Chatbot.postprocess = postprocess
+
+
+def generate_query(chatbot, history):
+    if history and history[-1]["role"] == "user":
+        gr.Warning('You should generate assistant-response.')
+        yield None, chatbot, history
+    else:
+        chatbot.append(None)
+        streamer = bot.generate_query(history, stream=True)
+        for query in streamer:
+            chatbot[-1] = (query, None)
+            yield query, chatbot, history
+        history.append({"role": "user", "content": query})
+        yield query, chatbot, history
+
+
+def generate_response(query, chatbot, history):
+    """
+    auto-mode：query is None
+    manual-mode：query 是用户输入
+    :param query:
+    :param chatbot:
+    :param history:
+    :return:
+    """
+    if query and history[-1]["role"] != "user":
+        history.append({"role": "user", "content": query})
+    query = history[-1]["content"]
+
+    if history[-1]["role"] != "user":
+        gr.Warning('You should generate or type user-input first.')
+        yield chatbot, history
+    else:
+        streamer = bot.generate_response(history, stream=True)
+        for response in streamer:
+            chatbot[-1] = (query, response)
+            yield chatbot, history
+
+        history.append({"role": "assistant", "content": response})
+        print(f"chatbot is {chatbot}")
+        print(f"history is {history}")
+        yield chatbot, history
+
+
+def generate(chatbot, history):
+    if history[-1]["role"] == "assistant":
+        return generate_query(chatbot, history)
+    elif history[-1]["role"] in ["user", "system"]:
+        return generate_query(chatbot, history)
+    else:
+
+        gr.Warning("")
+
+
+def regenerate():
+    """
+    删除上一轮，重新生成。
+    :return:
+    """
+    pass
+
+
+def reset_user_input():
+    return gr.update(value='')
+
+
+def reset_state(system):
+    return [], [{"role": "system", "content": system}]
+
+
+def set_max_tokens(max_tokens):
+    bot.generation_kwargs["max_tokens"] = max_tokens
+
+
+def clear_history():
+    pass
+
+
+def undo_generate():
+    pass
+
+

models/cpp_qwen2.py

@@ -21,7 +21,7 @@ from simulator import Simulator
 import llama_cpp
 # import llama_cpp.llama_tokenizer
 from transformers import AutoTokenizer
-from log_util import logger
+from utils.logging_util import logger
 
 
 class Qwen2Simulator(Simulator):
@@ -44,8 +44,6 @@ class Qwen2Simulator(Simulator):
                 verbose=False,
             )
         logger.info(f"llm has been initialized: {self.llm}")
-        # warmup
-
 
         self.generation_kwargs = dict(
             temperature=5,
@@ -53,6 +51,7 @@ class Qwen2Simulator(Simulator):
             top_k=40,
             max_tokens=20,
             repeat_penalty=1.1,
+            # qwen2-0.5b-chat 有时内容生成结束没有<|im_end|>，直接跟 <|im_start|>
             stop=[
                 "<|im_end|>",
                 "<|im_start|>",
@@ -82,7 +81,7 @@ class Qwen2Simulator(Simulator):
 
     def generate_response(self, messages, stream=True):
         assert messages[-1]["role"] == "user"
-        logger.info(f"generating {json.dumps(messages)}")
+        logger.info(f"generating {json.dumps(messages, ensure_ascii=False)}")
         inputs = self.hf_tokenizer.apply_chat_template(
             messages,
             tokenize=False,
@@ -96,18 +95,9 @@ class Qwen2Simulator(Simulator):
     def _generate(self, inputs):
         """
         TODO: chat with cache.
-        qwen2-0.5b-chat 有bug：有时user生成结束没有<|im_end|>，示例：
-            <|im_start|>system
-            you are a helpful assistant<|im_end|>
-            <|im_start|>user
-            hi, what your name<|im_end|>
-            <|im_start|>assistant
-            My name is Jordan<|im_end|>
-            <|im_start|>user              # 以上是输入，以下是生成
-            how old are you?
-            <|im_start|>assistant
-            I am a 41-year-old man.<|im_end|>
+
         """
+        logger.info(f"generation_kwargs {self.generation_kwargs}")
         output = self.llm(
             inputs,
             **self.generation_kwargs