update

models/cpp_qwen2.py

@@ -0,0 +1,95 @@
+"""
+https://github.com/abetlen/llama-cpp-python/blob/main/examples/gradio_chat/local.py
+https://github.com/awinml/llama-cpp-python-bindings
+"""
+
+from simulator import Simulator
+from llama_cpp import Llama
+import llama_cpp.llama_tokenizer
+
+
+class Qwen2Simulator(Simulator):
+
+    def __init__(self, model_name_or_path=None):
+        # self.llm = llama_cpp.Llama.from_pretrained(
+        #     repo_id="Qwen/Qwen1.5-0.5B-Chat-GGUF",
+        #     filename="*q8_0.gguf",  #
+        #     tokenizer=llama_cpp.llama_tokenizer.LlamaHFTokenizer.from_pretrained(
+        #         "Qwen/Qwen1.5-0.5B-Chat"
+        #     ),
+        #     verbose=False,
+        # )
+
+        self.llm = Llama(
+            model_path="Qwen/Qwen1.5-0.5B-Chat-GGUF/qwen1_5-0_5b-chat-q8_0.gguf",
+            # n_gpu_layers=-1, # Uncomment to use GPU acceleration
+            # seed=1337, # Uncomment to set a specific seed
+            # n_ctx=2048, # Uncomment to increase the context window
+            tokenizer=llama_cpp.llama_tokenizer.LlamaHFTokenizer.from_pretrained(
+                "/workspace/czy/model_weights/Qwen1.5-0.5B-Chat/"
+            ),
+            verbose=False,
+        )
+
+
+    def generate_query(self, messages):
+        """
+        :param messages:
+        :return:
+        """
+        assert messages[-1]["role"] != "user"
+        inputs = self.tokenizer.apply_chat_template(
+            messages,
+            tokenize=False,
+            add_generation_prompt=False,
+        )
+        inputs = inputs + "<|im_start|>user\n"
+        return self._generate(inputs)
+        # for new_text in self._stream_generate(input_ids):
+        #     yield new_text
+
+    def generate_response(self, messages):
+        assert messages[-1]["role"] == "user"
+        inputs = self.tokenizer.apply_chat_template(
+            messages,
+            tokenize=False,
+            add_generation_prompt=True
+        )
+
+        return self._generate(inputs)
+        # for new_text in self._stream_generate(input_ids):
+        #     yield new_text
+
+
+    def _generate(self, inputs):
+        # stream=False
+        output = self.llm(
+            inputs,
+            max_tokens=20,
+            temperature=0.7,
+            stop=["<|im_end|>"]
+        )
+        output_text = output["choices"][0]["text"]
+        return output_text
+
+
+
+bot = Qwen2Simulator(r"E:\data_model\Qwen2-0.5B-Instruct")
+
+
+if __name__ == "__main__":
+
+    messages = [
+        {"role": "system", "content": "you are a helpful assistant"},
+        {"role": "user", "content": "What is the capital of France?"}
+    ]
+    output = bot.generate_response(messages)
+    print(output)
+
+    messages = [
+        {"role": "system", "content": "you are a helpful assistant"},
+        {"role": "user", "content": "hi, what your name"},
+        {"role": "assistant", "content": "My name is Jordan"}
+    ]
+    output = bot.generate_query(messages)
+    print(output)

simulator.py

@@ -1,38 +1,22 @@
-from transformers import AutoModelForCausalLM, AutoTokenizer
+
 
 
 class Simulator:
 
     def __init__(self, model_name_or_path):
-        """
-        在传递 device_map 时，low_cpu_mem_usage 会自动设置为 True
-        """
+        raise NotImplementedError
 
-        self.tokenizer = AutoTokenizer.from_pretrained(model_name_or_path)
-        self.model = AutoModelForCausalLM.from_pretrained(
-            model_name_or_path,
-            torch_dtype="auto",
-            device_map="auto"
-        )
-        self.model.eval()
-        self.generation_kwargs = dict(
-            do_sample=True,
-            temperature=0.7,
-            max_length=500,
-            max_new_tokens=10
-        )
 
-    def generate_query(self, history):
+    def generate_query(self, messages):
         """ user simulator
-        :param history:
+        :param messages:
         :return:
         """
         raise NotImplementedError
 
-    def generate_response(self, input, history):
+    def generate_response(self, messages):
         """ assistant simulator
-        :param input:
-        :param history:
+        :param messages:
         :return:
         """
         raise NotImplementedError