update

app.py

@@ -10,43 +10,45 @@ tensor_parallel must == 1
 
 """
 
-import torch
+
 import os
 import numpy as np
 import argparse
-from vllm import LLM, SamplingParams
+import torch
 import gradio as gr
-from gradio_client.documentation import document, set_documentation_group
-
-from typing import List, Optional, Union, Dict, Tuple
-
-from tqdm import tqdm
-from transformers import PreTrainedTokenizer, PreTrainedTokenizerFast
-
-from vllm.engine.arg_utils import EngineArgs
-from vllm.engine.llm_engine import LLMEngine
-from vllm.outputs import RequestOutput
-from vllm.sampling_params import SamplingParams
-from vllm.utils import Counter
-from vllm.sequence import (Sequence, SequenceData, SequenceGroup,
-                           SequenceGroupMetadata, SequenceOutputs,
-                           SequenceStatus)
-
-# ! reconfigure vllm to faster llama
 from typing import Any, Iterator
 from typing import Iterator, List, Optional, Tuple
 import filelock
 import glob
 import json
-import os
-from huggingface_hub import snapshot_download
 
+from gradio_client.documentation import document, set_documentation_group
+
+from typing import List, Optional, Union, Dict, Tuple
 from tqdm.auto import tqdm
+from huggingface_hub import snapshot_download
+
+DEBUG = True
+
+if not DEBUG:
 
-from vllm.model_executor.model_loader import _MODEL_REGISTRY
-from vllm.model_executor.models import LlamaForCausalLM
+    # vllm import
+    from vllm import LLM, SamplingParams
+    from transformers import PreTrainedTokenizer, PreTrainedTokenizerFast
+    from vllm.engine.arg_utils import EngineArgs
+    from vllm.engine.llm_engine import LLMEngine
+    from vllm.outputs import RequestOutput
+    from vllm.sampling_params import SamplingParams
+    from vllm.utils import Counter
+    from vllm.sequence import (Sequence, SequenceData, SequenceGroup,
+                            SequenceGroupMetadata, SequenceOutputs,
+                            SequenceStatus)
+    # ! reconfigure vllm to faster llama
+    from vllm.model_executor.model_loader import _MODEL_REGISTRY
+    from vllm.model_executor.models import LlamaForCausalLM
 
-_MODEL_REGISTRY['FasterLlamaForCausalLM'] = LlamaForCausalLM
+
+    _MODEL_REGISTRY['FasterLlamaForCausalLM'] = LlamaForCausalLM
 
 
 def hf_model_weights_iterator(
@@ -661,18 +663,35 @@ def debug_chat_response_echo(
     yield message
 
 
+# ============ CONSTANT ============
 MODEL_TITLE = "DAMO-SeaL-13B - An Assistant for South East Asian Languages"
 MODEL_DESC = """
 This is a 13B DAMO-SeaL-Chat assistant model built by DAMO Academy, Alibaba Group. It can produce helpful responses in English, Vietnamese, Indonesian and Thai.
 """.strip()
 
+
+cite_markdown = """
+## Citation
+If you find our project useful, hope you can star our repo and cite our paper as follows:
+```
+@article{damonlpsg2023seallm,
+  author = {???},
+  title = {SeaL: A language model for South East Asian Languages},
+  year = 2023,
+}
+"""
+#   journal = {arXiv preprint arXiv:2306.02858}
+#   url = {https://arxiv.org/abs/2306.02858}
+
+
 TENSOR_PARALLEL = int(os.environ.get("TENSOR_PARALLEL", "1"))
 DTYPE = 'bfloat16'
 DTYPE = 'float16'
 
 MODEL_PATH = os.environ.get("MODEL_PATH", "notfound, please set `export MODEL_PATH=`")
 
-DEBUG = 1
+
+
 
 def launch():
     global demo, llm, DEBUG
@@ -720,6 +739,8 @@ def launch():
             gr.Number(value=0.4, label='Frequency penalty (> 0 encourage new tokens)'), 
             gr.Textbox(value=sys_prompt, label='System prompt', lines=8)], 
     )
+
+    gr.Markdown(cite_markdown)
     demo.queue()
     # demo.launch(server_port=args.port)
     demo.launch()