Delete deepseek_v3_tokenizer

deepseek_v3_tokenizer/deepseek_service.py

@@ -1,52 +0,0 @@
-from flask import Flask, request, jsonify
-from transformers import AutoTokenizer
-import os
-
-app = Flask(__name__)
-
-# 加载tokenizer
-tokenizer = AutoTokenizer.from_pretrained(os.path.dirname(__file__))
-
-@app.route('/count_tokens', methods=['POST'])
-def count_tokens():
-    try:
-        data = request.json
-        messages = data.get('messages', [])
-        system = data.get('system')
-        
-        # 构建完整文本
-        text = ""
-        if system:
-            text += f"System: {system}\n\n"
-            
-        for msg in messages:
-            role = msg.get('role', '')
-            content = msg.get('content', '')
-            if role == 'user':
-                text += f"User: {content}\n"
-            elif role == 'assistant':
-                text += f"Assistant: {content}\n"
-            else:
-                text += f"{role}: {content}\n"
-        
-        # 计算token数量
-        tokens = tokenizer.encode(text)
-        token_count = len(tokens)
-        
-        return jsonify({
-            'input_tokens': token_count
-        })
-    except Exception as e:
-        return jsonify({
-            'error': str(e)
-        }), 400
-
-@app.route('/health', methods=['GET'])
-def health():
-    return jsonify({
-        'status': 'healthy',
-        'tokenizer': 'deepseek-v3'
-    })
-
-if __name__ == '__main__':
-    app.run(host='127.0.0.1', port=7861) 

deepseek_v3_tokenizer/deepseek_tokenizer.py

@@ -1,12 +0,0 @@
-# pip3 install transformers
-# python3 deepseek_tokenizer.py
-import transformers
-
-chat_tokenizer_dir = "./"
-
-tokenizer = transformers.AutoTokenizer.from_pretrained( 
-        chat_tokenizer_dir, trust_remote_code=True
-        )
-
-result = tokenizer.encode("Hello!")
-print(result)

deepseek_v3_tokenizer/tokenizer_config.json

@@ -1,35 +0,0 @@
-{
-  "add_bos_token": false,
-  "add_eos_token": false,
-  "bos_token": {
-    "__type": "AddedToken",
-    "content": "<｜begin▁of▁sentence｜>",
-    "lstrip": false,
-    "normalized": true,
-    "rstrip": false,
-    "single_word": false
-  },
-  "clean_up_tokenization_spaces": false,
-  "eos_token": {
-    "__type": "AddedToken",
-    "content": "<｜end▁of▁sentence｜>",
-    "lstrip": false,
-    "normalized": true,
-    "rstrip": false,
-    "single_word": false
-  },
-  "legacy": true,
-  "model_max_length": 16384,
-  "pad_token": {
-    "__type": "AddedToken",
-    "content": "<｜end▁of▁sentence｜>",
-    "lstrip": false,
-    "normalized": true,
-    "rstrip": false,
-    "single_word": false
-  },
-  "sp_model_kwargs": {},
-  "unk_token": null,
-  "tokenizer_class": "LlamaTokenizerFast",
-  "chat_template": "{% if not add_generation_prompt is defined %}{% set add_generation_prompt = false %}{% endif %}{% set ns = namespace(is_first=false, is_tool=false, is_output_first=true, system_prompt='', is_first_sp=true) %}{%- for message in messages %}{%- if message['role'] == 'system' %}{%- if ns.is_first_sp %}{% set ns.system_prompt = ns.system_prompt + message['content'] %}{% set ns.is_first_sp = false %}{%- else %}{% set ns.system_prompt = ns.system_prompt + '\\n\\n' + message['content'] %}{%- endif %}{%- endif %}{%- endfor %}{{ bos_token }}{{ ns.system_prompt }}{%- for message in messages %}{%- if message['role'] == 'user' %}{%- set ns.is_tool = false -%}{{'<｜User｜>' + message['content']}}{%- endif %}{%- if message['role'] == 'assistant' and 'tool_calls' in message %}{%- set ns.is_tool = false -%}{%- for tool in message['tool_calls'] %}{%- if not ns.is_first %}{%- if message['content'] is none %}{{'<｜Assistant｜><｜tool▁calls▁begin｜><｜tool▁call▁begin｜>' + tool['type'] + '<｜tool▁sep｜>' + tool['function']['name'] + '\\n' + '```json' + '\\n' + tool['function']['arguments'] + '\\n' + '```' + '<｜tool▁call▁end｜>'}}{%- else %}{{'<｜Assistant｜>' + message['content'] + '<｜tool▁calls▁begin｜><｜tool▁call▁begin｜>' + tool['type'] + '<｜tool▁sep｜>' + tool['function']['name'] + '\\n' + '```json' + '\\n' + tool['function']['arguments'] + '\\n' + '```' + '<｜tool▁call▁end｜>'}}{%- endif %}{%- set ns.is_first = true -%}{%- else %}{{'\\n' + '<｜tool▁call▁begin｜>' + tool['type'] + '<｜tool▁sep｜>' + tool['function']['name'] + '\\n' + '```json' + '\\n' + tool['function']['arguments'] + '\\n' + '```' + '<｜tool▁call▁end｜>'}}{%- endif %}{%- endfor %}{{'<｜tool▁calls▁end｜><｜end▁of▁sentence｜>'}}{%- endif %}{%- if message['role'] == 'assistant' and 'tool_calls' not in message %}{%- if ns.is_tool %}{{'<｜tool▁outputs▁end｜>' + message['content'] + '<｜end▁of▁sentence｜>'}}{%- set ns.is_tool = false -%}{%- else %}{% set content = message['content'] %}{% if '</think>' in content %}{% set content = content.split('</think>')[-1] %}{% endif %}{{'<｜Assistant｜>' + content + '<｜end▁of▁sentence｜>'}}{%- endif %}{%- endif %}{%- if message['role'] == 'tool' %}{%- set ns.is_tool = true -%}{%- if ns.is_output_first %}{{'<｜tool▁outputs▁begin｜><｜tool▁output▁begin｜>' + message['content'] + '<｜tool▁output▁end｜>'}}{%- set ns.is_output_first = false %}{%- else %}{{'<｜tool▁output▁begin｜>' + message['content'] + '<｜tool▁output▁end｜>'}}{%- endif %}{%- endif %}{%- endfor -%}{% if ns.is_tool %}{{'<｜tool▁outputs▁end｜>'}}{% endif %}{% if add_generation_prompt and not ns.is_tool %}{{'<｜Assistant｜>'}}{% endif %}"
-}