update

app.py

@@ -172,6 +172,162 @@ with gr.Blocks(theme='ParityError/Anime') as demo:
 
     
     
+    with gr.Tab("PairRM"):
+        # PairRM interface
+        with gr.Row():
+            gr.Markdown(descriptions.PairRM_OVERALL_DESC)
+            gr.Image("https://yuchenlin.xyz/LLM-Blender/pairranker.png")
+        
+        with gr.Tab("Compare two responses"):
+            instruction = gr.Textbox(lines=1, label="Instruction", placeholder="Enter instruction here", show_label=True)
+            with gr.Row():
+                response1 = gr.Textbox(lines=4, label="Response 1", placeholder="Enter response 1 here", show_label=True)
+                response2 = gr.Textbox(lines=4, label="Response 2", placeholder="Enter response 2 here", show_label=True)
+            with gr.Row():
+                compare_button = gr.Button('Compare', variant='primary')
+                clear_button = gr.Button('Clear', variant='primary')
+            with gr.Row():
+                compare_result = gr.Textbox(lines=1, label="Compare Result", placeholder="", show_label=True)
+                compare_result_prob = gr.Textbox(lines=1, label="PairRM Confidence", placeholder="", show_label=True)
+            
+            def compare_fn(inst, response1, response2):
+                if not inst:
+                    raise gr.Error("Please enter instruction")
+                if not response1 or not response2:
+                    raise gr.Error("Please enter response 1 and response 2")
+                comparison_results = blender.compare([inst], [response1], [response2], return_logits=True)
+                logit = comparison_results[0]
+                if logit > 0:
+                    result = "Response 1 is better than Response 2"
+                    prob = f"Confidence: {round(logit, 2)}"
+                elif logit < 0:
+                    result = "Response 2 is better than Response 1"
+                    prob = f"Cofidence: {round(abs(logit), 2)}"
+                else:
+                    result = "Response 1 and Response 2 are equally good"
+                    prob = f"No confidence for tie"
+                    
+                return [result, prob]
+            compare_button.click(
+                fn=compare_fn,
+                inputs=[instruction, response1, response2],
+                outputs=[compare_result, compare_result_prob],
+            )
+            clear_button.click(
+                fn=lambda: ["", ""],
+                inputs=[],
+                outputs=[compare_result, compare_result_prob],
+            )
+            
+            hhh_dummy_textbox1 = gr.Textbox(lines=1, label="subset", placeholder="", show_label=False, visible=False)
+            hhh_dummy_textbox2 = gr.Textbox(lines=1, label="Better Response", placeholder="", show_label=False, visible=False)
+            gr.Markdown("## Examples from [HuggingFaceH4/hhh_alignment](https://huggingface.co/datasets/HuggingFaceH4/hhh_alignment)")
+            gr.Examples(
+                HHH_EXAMPLES,
+                fn=get_hhh_examples,
+                cache_examples=True,
+                examples_per_page=5,
+                inputs=[hhh_dummy_textbox1, instruction, response1, response2, hhh_dummy_textbox2],
+                outputs=[instruction, response1, response2],
+            )
+                
+            
+        with gr.Tab("Compare assistant's response in two multi-turn conversations"):
+            
+            gr.Markdown("NOTE: Comparison of two conversations is based on that the user query in each turn is the same of two conversations.")
+            def append_message(message, chat_history):
+                if not message:
+                    return "", chat_history
+                if len(chat_history) == 0:
+                    chat_history.append((message, "(Please enter your bot response)"))
+                else:
+                    if chat_history[-1][1] == "(Please enter your bot response)":
+                        chat_history[-1] = (chat_history[-1][0], message)
+                    else:
+                        chat_history.append((message, "(Please enter your bot response)"))
+                return "", chat_history
+            with gr.Row():
+                with gr.Column():
+                    gr.Markdown("### Conversation A")
+                    chatbot1 = gr.Chatbot()
+                    msg1 = gr.Textbox(lines=1, label="Enter Chat history for Conversation A", placeholder="Enter your message here", show_label=True)
+                    clear1 = gr.ClearButton([msg1, chatbot1])
+                    msg1.submit(append_message, [msg1, chatbot1], [msg1, chatbot1])
+                with gr.Column():    
+                    gr.Markdown("### Conversation B")
+                    chatbot2 = gr.Chatbot()
+                    msg2 = gr.Textbox(lines=1, label="Enter Chat history for Conversation B", placeholder="Enter your message here", show_label=True)
+                    clear2 = gr.ClearButton([msg2, chatbot2])
+                    msg2.submit(append_message, [msg2, chatbot2], [msg2, chatbot2])
+            with gr.Row():
+                compare_button = gr.Button('Compare', variant='primary')
+            with gr.Row():
+                compare_result = gr.Textbox(lines=1, label="Compare Result", placeholder="", show_label=True)
+                compare_result_prob = gr.Textbox(lines=1, label="PairRM Confidence", placeholder="", show_label=True)
+            
+            def compare_conv_fn(chat_history1, chat_history2):
+                if len(chat_history1) == 0 or len(chat_history2) == 0:
+                    raise gr.Error("Please enter chat history for both conversations")
+                assert chat_history1[-1][1] != "(Please enter your bot response)" \
+                    and chat_history2[-1][1] != "(Please enter your bot response)", \
+                    "Please complete chat history for both conversations"
+                chat1_messages = []
+                for item in chat_history1:
+                    chat1_messages.append({
+                        "role": "USER",
+                        "content": item[0],
+                    })
+                    chat1_messages.append({
+                        "role": "ASSISTANT",
+                        "content": item[1],
+                    })
+                chat2_messages = []
+                for item in chat_history2:
+                    chat2_messages.append({
+                        "role": "USER",
+                        "content": item[0],
+                    })
+                    chat2_messages.append({
+                        "role": "ASSISTANT",
+                        "content": item[1],
+                    })
+                                          
+                comparison_results = blender.compare_conversations([chat1_messages], [chat2_messages], return_logits=True)
+                logit = comparison_results[0]
+                if logit > 0:
+                    result = "Assistant's response in Conversation A is better than Conversation B"
+                    prob = f"Confidence: {round(logit, 2)}"
+                elif logit < 0:
+                    result = "Assistant's response in Conversation B is better than Conversation A"
+                    prob = f"Cofidence: {round(abs(logit), 2)}"
+                else:
+                    result = "Assistant's response in Conversation A and Conversation B are equally good"
+                    prob = f"No confidence for tie"
+                    
+                return [result, prob]
+
+            compare_button.click(
+                fn=compare_conv_fn,
+                inputs=[chatbot1, chatbot2],
+                outputs=[compare_result, compare_result_prob],
+            )
+            
+            model_a_dummy_textbox = gr.Textbox(lines=1, label="Model A", placeholder="", show_label=False, visible=False)
+            model_b_dummy_textbox = gr.Textbox(lines=1, label="Model B", placeholder="", show_label=False, visible=False)
+            winner_dummy_textbox = gr.Textbox(lines=1, label="Better Model in conversation", placeholder="", show_label=False, visible=False)
+            chatbot1_dummy_textbox = gr.Textbox(lines=1, label="Conversation A", placeholder="", show_label=False, visible=False)
+            chatbot2_dummy_textbox = gr.Textbox(lines=1, label="Conversation B", placeholder="", show_label=False, visible=False)
+            gr.Markdown("## Examples from [lmsys/mt_bench_human_judgments](https://huggingface.co/datasets/lmsys/mt_bench_human_judgments)")
+            gr.Examples(
+                MT_BENCH_HUMAN_JUDGE_EXAMPLES,
+                fn=get_mt_bench_human_judge_examples,
+                cache_examples=True,
+                examples_per_page=5,
+                inputs=[model_a_dummy_textbox, model_b_dummy_textbox, chatbot1_dummy_textbox, chatbot2_dummy_textbox, winner_dummy_textbox],
+                outputs=[chatbot1, chatbot2],
+            )
+    
+    
     with gr.Tab("LLM-Blender"):
         # llm-blender interface 
         with gr.Row():
@@ -375,160 +531,6 @@ with gr.Blocks(theme='ParityError/Anime') as demo:
         )
         
     
-    with gr.Tab("PairRM"):
-        # PairRM interface
-        with gr.Row():
-            gr.Markdown(descriptions.PairRM_OVERALL_DESC)
-            gr.Image("https://yuchenlin.xyz/LLM-Blender/pairranker.png")
-        
-        with gr.Tab("Compare two responses"):
-            instruction = gr.Textbox(lines=1, label="Instruction", placeholder="Enter instruction here", show_label=True)
-            with gr.Row():
-                response1 = gr.Textbox(lines=4, label="Response 1", placeholder="Enter response 1 here", show_label=True)
-                response2 = gr.Textbox(lines=4, label="Response 2", placeholder="Enter response 2 here", show_label=True)
-            with gr.Row():
-                compare_button = gr.Button('Compare', variant='primary')
-                clear_button = gr.Button('Clear', variant='primary')
-            with gr.Row():
-                compare_result = gr.Textbox(lines=1, label="Compare Result", placeholder="", show_label=True)
-                compare_result_prob = gr.Textbox(lines=1, label="PairRM Confidence", placeholder="", show_label=True)
-            
-            def compare_fn(inst, response1, response2):
-                if not inst:
-                    raise gr.Error("Please enter instruction")
-                if not response1 or not response2:
-                    raise gr.Error("Please enter response 1 and response 2")
-                comparison_results = blender.compare([inst], [response1], [response2], return_logits=True)
-                logit = comparison_results[0]
-                if logit > 0:
-                    result = "Response 1 is better than Response 2"
-                    prob = f"Confidence: {round(logit, 2)}"
-                elif logit < 0:
-                    result = "Response 2 is better than Response 1"
-                    prob = f"Cofidence: {round(abs(logit), 2)}"
-                else:
-                    result = "Response 1 and Response 2 are equally good"
-                    prob = f"No confidence for tie"
-                    
-                return [result, prob]
-            compare_button.click(
-                fn=compare_fn,
-                inputs=[instruction, response1, response2],
-                outputs=[compare_result, compare_result_prob],
-            )
-            clear_button.click(
-                fn=lambda: ["", ""],
-                inputs=[],
-                outputs=[compare_result, compare_result_prob],
-            )
-            
-            hhh_dummy_textbox1 = gr.Textbox(lines=1, label="subset", placeholder="", show_label=False, visible=False)
-            hhh_dummy_textbox2 = gr.Textbox(lines=1, label="Better Response", placeholder="", show_label=False, visible=False)
-            gr.Markdown("## Examples from [HuggingFaceH4/hhh_alignment](https://huggingface.co/datasets/HuggingFaceH4/hhh_alignment)")
-            gr.Examples(
-                HHH_EXAMPLES,
-                fn=get_hhh_examples,
-                cache_examples=True,
-                examples_per_page=5,
-                inputs=[hhh_dummy_textbox1, instruction, response1, response2, hhh_dummy_textbox2],
-                outputs=[instruction, response1, response2],
-            )
-                
-            
-        with gr.Tab("Compare assistant's response in two multi-turn conversations"):
-            
-            gr.Markdown("NOTE: Comparison of two conversations is based on that the user query in each turn is the same of two conversations.")
-            def append_message(message, chat_history):
-                if not message:
-                    return "", chat_history
-                if len(chat_history) == 0:
-                    chat_history.append((message, "(Please enter your bot response)"))
-                else:
-                    if chat_history[-1][1] == "(Please enter your bot response)":
-                        chat_history[-1] = (chat_history[-1][0], message)
-                    else:
-                        chat_history.append((message, "(Please enter your bot response)"))
-                return "", chat_history
-            with gr.Row():
-                with gr.Column():
-                    gr.Markdown("### Conversation A")
-                    chatbot1 = gr.Chatbot()
-                    msg1 = gr.Textbox(lines=1, label="Enter Chat history for Conversation A", placeholder="Enter your message here", show_label=True)
-                    clear1 = gr.ClearButton([msg1, chatbot1])
-                    msg1.submit(append_message, [msg1, chatbot1], [msg1, chatbot1])
-                with gr.Column():    
-                    gr.Markdown("### Conversation B")
-                    chatbot2 = gr.Chatbot()
-                    msg2 = gr.Textbox(lines=1, label="Enter Chat history for Conversation B", placeholder="Enter your message here", show_label=True)
-                    clear2 = gr.ClearButton([msg2, chatbot2])
-                    msg2.submit(append_message, [msg2, chatbot2], [msg2, chatbot2])
-            with gr.Row():
-                compare_button = gr.Button('Compare', variant='primary')
-            with gr.Row():
-                compare_result = gr.Textbox(lines=1, label="Compare Result", placeholder="", show_label=True)
-                compare_result_prob = gr.Textbox(lines=1, label="PairRM Confidence", placeholder="", show_label=True)
-            
-            def compare_conv_fn(chat_history1, chat_history2):
-                if len(chat_history1) == 0 or len(chat_history2) == 0:
-                    raise gr.Error("Please enter chat history for both conversations")
-                assert chat_history1[-1][1] != "(Please enter your bot response)" \
-                    and chat_history2[-1][1] != "(Please enter your bot response)", \
-                    "Please complete chat history for both conversations"
-                chat1_messages = []
-                for item in chat_history1:
-                    chat1_messages.append({
-                        "role": "USER",
-                        "content": item[0],
-                    })
-                    chat1_messages.append({
-                        "role": "ASSISTANT",
-                        "content": item[1],
-                    })
-                chat2_messages = []
-                for item in chat_history2:
-                    chat2_messages.append({
-                        "role": "USER",
-                        "content": item[0],
-                    })
-                    chat2_messages.append({
-                        "role": "ASSISTANT",
-                        "content": item[1],
-                    })
-                                          
-                comparison_results = blender.compare_conversations([chat1_messages], [chat2_messages], return_logits=True)
-                logit = comparison_results[0]
-                if logit > 0:
-                    result = "Assistant's response in Conversation A is better than Conversation B"
-                    prob = f"Confidence: {round(logit, 2)}"
-                elif logit < 0:
-                    result = "Assistant's response in Conversation B is better than Conversation A"
-                    prob = f"Cofidence: {round(abs(logit), 2)}"
-                else:
-                    result = "Assistant's response in Conversation A and Conversation B are equally good"
-                    prob = f"No confidence for tie"
-                    
-                return [result, prob]
 
-            compare_button.click(
-                fn=compare_conv_fn,
-                inputs=[chatbot1, chatbot2],
-                outputs=[compare_result, compare_result_prob],
-            )
-            
-            model_a_dummy_textbox = gr.Textbox(lines=1, label="Model A", placeholder="", show_label=False, visible=False)
-            model_b_dummy_textbox = gr.Textbox(lines=1, label="Model B", placeholder="", show_label=False, visible=False)
-            winner_dummy_textbox = gr.Textbox(lines=1, label="Better Model in conversation", placeholder="", show_label=False, visible=False)
-            chatbot1_dummy_textbox = gr.Textbox(lines=1, label="Conversation A", placeholder="", show_label=False, visible=False)
-            chatbot2_dummy_textbox = gr.Textbox(lines=1, label="Conversation B", placeholder="", show_label=False, visible=False)
-            gr.Markdown("## Examples from [lmsys/mt_bench_human_judgments](https://huggingface.co/datasets/lmsys/mt_bench_human_judgments)")
-            gr.Examples(
-                MT_BENCH_HUMAN_JUDGE_EXAMPLES,
-                fn=get_mt_bench_human_judge_examples,
-                cache_examples=True,
-                examples_per_page=5,
-                inputs=[model_a_dummy_textbox, model_b_dummy_textbox, chatbot1_dummy_textbox, chatbot2_dummy_textbox, winner_dummy_textbox],
-                outputs=[chatbot1, chatbot2],
-            )
-    
     gr.Markdown(descriptions.CITATION)
 demo.queue(max_size=20).launch()

requirements.txt

@@ -1,3 +1,219 @@
-torch
-transformers
-llm_blender @ git+https://github.com/yuchenlin/LLM-Blender.git@main
+absl-py==2.0.0
+accelerate==0.24.0
+aiofiles==23.2.1
+aiohttp==3.8.6
+aiosignal==1.3.1
+altair==5.1.2
+annotated-types==0.6.0
+anyio==3.7.1
+appdirs==1.4.4
+asttokens @ file:///opt/conda/conda-bld/asttokens_1646925590279/work
+astunparse==1.6.3
+async-timeout==4.0.3
+attrs==23.1.0
+backcall @ file:///home/ktietz/src/ci/backcall_1611930011877/work
+bert-score==0.3.13
+bitsandbytes==0.41.1
+BLEURT @ git+https://github.com/google-research/bleurt.git@cebe7e6f996b40910cfaa520a63db47807e3bf5c
+blis==0.7.11
+cachetools==5.3.2
+catalogue==2.0.10
+certifi==2023.7.22
+charset-normalizer==3.3.1
+click==8.1.7
+cloudpathlib==0.16.0
+colorama==0.4.6
+comm @ file:///croot/comm_1671231121260/work
+confection==0.1.3
+contourpy==1.1.1
+cpm-kernels==1.0.11
+cycler==0.12.1
+cymem==2.0.8
+dataclasses-json==0.6.1
+datasets==2.14.6
+debugpy @ file:///croot/debugpy_1690905042057/work
+decorator @ file:///opt/conda/conda-bld/decorator_1643638310831/work
+deepspeed==0.11.1
+dill==0.3.7
+distro==1.8.0
+docker-pycreds==0.4.0
+docstring-parser==0.15
+evaluate==0.4.1
+exceptiongroup @ file:///croot/exceptiongroup_1668714342571/work
+executing @ file:///opt/conda/conda-bld/executing_1646925071911/work
+fairscale==0.4.13
+fastapi==0.104.0
+ffmpy==0.3.1
+filelock @ file:///croot/filelock_1672387128942/work
+fire==0.5.0
+flatbuffers==23.5.26
+fonttools==4.43.1
+frozenlist==1.4.0
+fschat==0.2.31
+fsspec==2023.10.0
+gast==0.4.0
+gitdb==4.0.11
+GitPython==3.1.40
+gmpy2 @ file:///tmp/build/80754af9/gmpy2_1645438755360/work
+google-auth==2.23.3
+google-auth-oauthlib==0.4.6
+google-pasta==0.2.0
+gradio==4.2.0
+gradio_client==0.7.0
+grpcio==1.59.0
+h11==0.14.0
+h5py==3.10.0
+hjson==3.1.0
+httpcore==0.18.0
+httpx==0.25.0
+huggingface-hub==0.17.3
+idna==3.4
+importlib-metadata @ file:///croot/importlib-metadata_1678997070253/work
+importlib-resources==6.1.0
+ipykernel @ file:///croot/ipykernel_1691121631942/work
+ipython @ file:///croot/ipython_1694181358621/work
+jedi @ file:///tmp/build/80754af9/jedi_1644297102865/work
+Jinja2 @ file:///croot/jinja2_1666908132255/work
+joblib==1.3.2
+jsonschema==4.19.2
+jsonschema-specifications==2023.7.1
+jupyter_client @ file:///croot/jupyter_client_1680171862562/work
+jupyter_core @ file:///croot/jupyter_core_1679906564508/work
+keras==2.11.0
+kiwisolver==1.4.5
+langcodes==3.3.0
+Levenshtein==0.23.0
+libclang==16.0.6
+-e git+https://github.com/yuchenlin/LLM-Blender.git@2568bbef76d67d2d41a42c48ed482ee60430ad88#egg=llm_blender
+lxml==4.9.3
+Markdown==3.5
+markdown-it-py==3.0.0
+markdown2==2.4.10
+MarkupSafe @ file:///opt/conda/conda-bld/markupsafe_1654597864307/work
+marshmallow==3.20.1
+matplotlib==3.8.0
+matplotlib-inline @ file:///opt/conda/conda-bld/matplotlib-inline_1662014470464/work
+mdurl==0.1.2
+mpmath @ file:///croot/mpmath_1690848262763/work
+multidict==6.0.4
+multiprocess==0.70.15
+murmurhash==1.0.10
+mypy-extensions==1.0.0
+nest-asyncio @ file:///croot/nest-asyncio_1672387112409/work
+networkx @ file:///croot/networkx_1690561992265/work
+nh3==0.2.14
+ninja==1.11.1.1
+nltk==3.8.1
+numpy==1.26.1
+oauthlib==3.2.2
+openai==1.2.3
+opt-einsum==3.3.0
+orjson==3.9.10
+packaging @ file:///croot/packaging_1693575174725/work
+pandas==2.1.2
+parso @ file:///opt/conda/conda-bld/parso_1641458642106/work
+pathtools==0.1.2
+peft==0.6.0
+pexpect @ file:///tmp/build/80754af9/pexpect_1605563209008/work
+pickleshare @ file:///tmp/build/80754af9/pickleshare_1606932040724/work
+Pillow==10.1.0
+platformdirs @ file:///croot/platformdirs_1692205439124/work
+portalocker==2.8.2
+preshed==3.0.9
+prettytable==3.9.0
+prompt-toolkit @ file:///croot/prompt-toolkit_1672387306916/work
+protobuf==3.19.6
+psutil @ file:///opt/conda/conda-bld/psutil_1656431268089/work
+ptyprocess @ file:///tmp/build/80754af9/ptyprocess_1609355006118/work/dist/ptyprocess-0.7.0-py2.py3-none-any.whl
+pure-eval @ file:///opt/conda/conda-bld/pure_eval_1646925070566/work
+py-cpuinfo==9.0.0
+pyarrow==13.0.0
+pyasn1==0.5.0
+pyasn1-modules==0.3.0
+pycocoevalcap==1.2
+pycocotools==2.0.7
+pydantic==2.4.2
+pydantic_core==2.10.1
+pydub==0.25.1
+Pygments @ file:///croot/pygments_1684279966437/work
+pyparsing==3.1.1
+python-dateutil @ file:///tmp/build/80754af9/python-dateutil_1626374649649/work
+python-multipart==0.0.6
+pytz==2023.3.post1
+PyYAML @ file:///croot/pyyaml_1698096049011/work
+pyzmq @ file:///croot/pyzmq_1686601365461/work
+rapidfuzz==3.5.2
+referencing==0.30.2
+regex==2023.10.3
+requests==2.31.0
+requests-oauthlib==1.3.1
+responses==0.18.0
+rich==13.6.0
+rouge-score==0.1.2
+rpds-py==0.12.0
+rsa==4.9
+sacrebleu==2.3.1
+safetensors==0.4.0
+scikit-learn==1.3.2
+scipy==1.11.3
+semantic-version==2.10.0
+sentencepiece==0.1.99
+sentry-sdk==1.32.0
+setproctitle==1.3.3
+shellingham==1.5.4
+shortuuid==1.0.11
+shtab==1.6.4
+six @ file:///tmp/build/80754af9/six_1644875935023/work
+smart-open==6.4.0
+smmap==5.0.1
+sniffio==1.3.0
+spacy==3.7.2
+spacy-legacy==3.0.12
+spacy-loggers==1.0.5
+srsly==2.4.8
+stack-data @ file:///opt/conda/conda-bld/stack_data_1646927590127/work
+starlette==0.27.0
+svgwrite==1.4.3
+sympy @ file:///croot/sympy_1668202399572/work
+tabulate==0.9.0
+tensorboard==2.11.2
+tensorboard-data-server==0.6.1
+tensorboard-plugin-wit==1.8.1
+tensorflow==2.11.1
+tensorflow-estimator==2.11.0
+tensorflow-io-gcs-filesystem==0.34.0
+termcolor==2.3.0
+tf-slim==1.1.0
+thinc==8.2.1
+threadpoolctl==3.2.0
+tiktoken==0.5.1
+tokenizers==0.14.1
+tomlkit==0.12.0
+toolz==0.12.0
+torch==2.1.0
+tornado @ file:///croot/tornado_1696936946304/work
+tqdm==4.66.1
+traitlets @ file:///croot/traitlets_1671143879854/work
+transformers==4.35.0
+triton==2.1.0
+trl==0.7.4
+typer==0.9.0
+typing-inspect==0.9.0
+typing_extensions==4.8.0
+tyro==0.5.12
+tzdata==2023.3
+urllib3==2.0.7
+uvicorn==0.23.2
+wandb==0.15.12
+wasabi==1.1.2
+wavedrom==2.0.3.post3
+wcwidth @ file:///Users/ktietz/demo/mc3/conda-bld/wcwidth_1629357192024/work
+weasel==0.3.3
+websockets==11.0.3
+Werkzeug==3.0.1
+wget==3.2
+wrapt==1.15.0
+xxhash==3.4.1
+yarl==1.9.2
+zipp @ file:///croot/zipp_1672387121353/work
+llm_blender @ git+https://github.com/yuchenlin/LLM-Blender.git@main