init

README.md

@@ -2,6 +2,16 @@
 license: apache-2.0
 ---
 
+## MiniCPM4 Series
+- [MiniCPM4-0.5B](https://huggingface.co/openbmb/MiniCPM4-0.5B)
+- [MiniCPM4-8B](https://huggingface.co/openbmb/MiniCPM4-8B)
+- [MiniCPM4-8B-Eagle-FRSpec](https://huggingface.co/openbmb/MiniCPM4-8B-Eagle-FRSpec)
+- [MiniCPM4-8B-Eagle-FRSpec-QAT](https://huggingface.co/openbmb/MiniCPM4-8B-Eagle-FRSpec-QAT)
+- [BitCPM4-0.5B](https://huggingface.co/openbmb/BitCPM4-0.5B)
+- [BitCPM4-1B](https://huggingface.co/openbmb/BitCPM4-1B)
+- [MiniCPM4-Survey](https://huggingface.co/openbmb/MiniCPM4-Survey): **<-- you are here**
+- [MiniCPM4-MCP](https://huggingface.co/openbmb/MiniCPM4-MCP)
+
 ## News
 
 * [2025-06-05] 🚀🚀🚀 We have open-sourced **MiniCPM4-Survey**, a model built upon MiniCPM4-8B that is capable of generating trustworthy, long-form survey papers while maintaining competitive performance relative to significantly larger models.

config.json

@@ -1,19 +1,18 @@
 {
+  "_name_or_path": "openbmb/MiniCPM4-8B",
   "architectures": [
     "MiniCPMForCausalLM"
   ],
-  "attention_bias": false,
-  "attention_dropout": 0.0,
   "auto_map": {
     "AutoConfig": "configuration_minicpm.MiniCPMConfig",
-    "AutoModel": "modeling_minicpm.MiniCPMForCausalLM",
+    "AutoModel": "modeling_minicpm.MiniCPMModel",
     "AutoModelForCausalLM": "modeling_minicpm.MiniCPMForCausalLM",
     "AutoModelForSeq2SeqLM": "modeling_minicpm.MiniCPMForCausalLM",
     "AutoModelForSequenceClassification": "modeling_minicpm.MiniCPMForSequenceClassification"
   },
   "bos_token_id": 1,
-  "dim_model_base": 256,
-  "eos_token_id": 73440,
+  "eos_token_id": [2, 73440],
+  "pad_token_id": 2,
   "hidden_act": "silu",
   "hidden_size": 4096,
   "initializer_range": 0.1,
@@ -23,151 +22,19 @@
   "num_attention_heads": 32,
   "num_hidden_layers": 32,
   "num_key_value_heads": 2,
-  "pad_token_id": 73440,
-  "pretraining_tp": 1,
   "rms_norm_eps": 1e-06,
   "rope_scaling": {
-    "long_factor": [
-      0.9977997200264581,
-      1.014658295992452,
-      1.0349680404997148,
-      1.059429246056193,
-      1.0888815016813513,
-      1.1243301355211495,
-      1.166977103606075,
-      1.2182568066927284,
-      1.2798772354275727,
-      1.3538666751582975,
-      1.4426259039919596,
-      1.5489853358570191,
-      1.6762658237220625,
-      1.8283407612492941,
-      2.0096956085876183,
-      2.225478927469756,
-      2.481536379650452,
-      2.784415934557119,
-      3.1413289096347365,
-      3.560047844772632,
-      4.048719380066383,
-      4.615569542115128,
-      5.2684819496549835,
-      6.014438591970396,
-      6.858830049237097,
-      7.804668263503327,
-      8.851768731513417,
-      9.99600492938444,
-      11.228766118181639,
-      12.536757560834843,
-      13.902257701387796,
-      15.303885189125953,
-      16.717837610115794,
-      18.119465097853947,
-      19.484965238406907,
-      20.792956681060105,
-      22.02571786985731,
-      23.16995406772833,
-      24.217054535738416,
-      25.16289275000465,
-      26.007284207271347,
-      26.753240849586767,
-      27.40615325712662,
-      27.973003419175363,
-      28.461674954469114,
-      28.880393889607006,
-      29.237306864684626,
-      29.540186419591297,
-      29.79624387177199,
-      30.01202719065413,
-      30.193382037992453,
-      30.34545697551969,
-      30.47273746338473,
-      30.579096895249787,
-      30.66785612408345,
-      30.741845563814174,
-      30.80346599254902,
-      30.85474569563567,
-      30.897392663720595,
-      30.932841297560394,
-      30.962293553185553,
-      30.986754758742034,
-      31.007064503249293,
-      31.02392307921529
-    ],
-    "original_max_position_embeddings": 32768,
     "rope_type": "longrope",
-    "short_factor": [
-      0.9977997200264581,
-      1.014658295992452,
-      1.0349680404997148,
-      1.059429246056193,
-      1.0888815016813513,
-      1.1243301355211495,
-      1.166977103606075,
-      1.2182568066927284,
-      1.2798772354275727,
-      1.3538666751582975,
-      1.4426259039919596,
-      1.5489853358570191,
-      1.6762658237220625,
-      1.8283407612492941,
-      2.0096956085876183,
-      2.225478927469756,
-      2.481536379650452,
-      2.784415934557119,
-      3.1413289096347365,
-      3.560047844772632,
-      4.048719380066383,
-      4.615569542115128,
-      5.2684819496549835,
-      6.014438591970396,
-      6.858830049237097,
-      7.804668263503327,
-      8.851768731513417,
-      9.99600492938444,
-      11.228766118181639,
-      12.536757560834843,
-      13.902257701387796,
-      15.303885189125953,
-      16.717837610115794,
-      18.119465097853947,
-      19.484965238406907,
-      20.792956681060105,
-      22.02571786985731,
-      23.16995406772833,
-      24.217054535738416,
-      25.16289275000465,
-      26.007284207271347,
-      26.753240849586767,
-      27.40615325712662,
-      27.973003419175363,
-      28.461674954469114,
-      28.880393889607006,
-      29.237306864684626,
-      29.540186419591297,
-      29.79624387177199,
-      30.01202719065413,
-      30.193382037992453,
-      30.34545697551969,
-      30.47273746338473,
-      30.579096895249787,
-      30.66785612408345,
-      30.741845563814174,
-      30.80346599254902,
-      30.85474569563567,
-      30.897392663720595,
-      30.932841297560394,
-      30.962293553185553,
-      30.986754758742034,
-      31.007064503249293,
-      31.02392307921529
-    ]
+    "long_factor": [0.9977997200264581, 1.014658295992452, 1.0349680404997148, 1.059429246056193, 1.0888815016813513, 1.1243301355211495, 1.166977103606075, 1.2182568066927284, 1.2798772354275727, 1.3538666751582975, 1.4426259039919596, 1.5489853358570191, 1.6762658237220625, 1.8283407612492941, 2.0096956085876183, 2.225478927469756, 2.481536379650452, 2.784415934557119, 3.1413289096347365, 3.560047844772632, 4.048719380066383, 4.615569542115128, 5.2684819496549835, 6.014438591970396, 6.858830049237097, 7.804668263503327, 8.851768731513417, 9.99600492938444, 11.228766118181639, 12.536757560834843, 13.902257701387796, 15.303885189125953, 16.717837610115794, 18.119465097853947, 19.484965238406907, 20.792956681060105, 22.02571786985731, 23.16995406772833, 24.217054535738416, 25.16289275000465, 26.007284207271347, 26.753240849586767, 27.40615325712662, 27.973003419175363, 28.461674954469114, 28.880393889607006, 29.237306864684626, 29.540186419591297, 29.79624387177199, 30.01202719065413, 30.193382037992453, 30.34545697551969, 30.47273746338473, 30.579096895249787, 30.66785612408345, 30.741845563814174, 30.80346599254902, 30.85474569563567, 30.897392663720595, 30.932841297560394, 30.962293553185553, 30.986754758742034, 31.007064503249293, 31.02392307921529],
+    "short_factor": [0.9977997200264581, 1.014658295992452, 1.0349680404997148, 1.059429246056193, 1.0888815016813513, 1.1243301355211495, 1.166977103606075, 1.2182568066927284, 1.2798772354275727, 1.3538666751582975, 1.4426259039919596, 1.5489853358570191, 1.6762658237220625, 1.8283407612492941, 2.0096956085876183, 2.225478927469756, 2.481536379650452, 2.784415934557119, 3.1413289096347365, 3.560047844772632, 4.048719380066383, 4.615569542115128, 5.2684819496549835, 6.014438591970396, 6.858830049237097, 7.804668263503327, 8.851768731513417, 9.99600492938444, 11.228766118181639, 12.536757560834843, 13.902257701387796, 15.303885189125953, 16.717837610115794, 18.119465097853947, 19.484965238406907, 20.792956681060105, 22.02571786985731, 23.16995406772833, 24.217054535738416, 25.16289275000465, 26.007284207271347, 26.753240849586767, 27.40615325712662, 27.973003419175363, 28.461674954469114, 28.880393889607006, 29.237306864684626, 29.540186419591297, 29.79624387177199, 30.01202719065413, 30.193382037992453, 30.34545697551969, 30.47273746338473, 30.579096895249787, 30.66785612408345, 30.741845563814174, 30.80346599254902, 30.85474569563567, 30.897392663720595, 30.932841297560394, 30.962293553185553, 30.986754758742034, 31.007064503249293, 31.02392307921529],
+    "original_max_position_embeddings": 32768
   },
+  "torch_dtype": "bfloat16",
+  "transformers_version": "4.46.3",
+  "use_cache": true,
+  "vocab_size": 73448,
   "rope_theta": 10000.0,
-  "scale_depth": 1.4,
   "scale_emb": 12,
-  "tie_word_embeddings": false,
-  "torch_dtype": "bfloat16",
-  "transformers_version": "4.51.3",
-  "use_cache": false,
-  "vocab_size": 73448
-}
+  "scale_depth": 1.4,
+  "tie_word_embeddings": false
+}

configuration_minicpm.py

@@ -1,4 +1,3 @@
-# coding=utf-8
 # Copyright 2022 EleutherAI and the HuggingFace Inc. team. All rights reserved.
 #
 # This code is based on EleutherAI's GPT-NeoX library and the GPT-NeoX
@@ -22,7 +21,6 @@
 from transformers.configuration_utils import PretrainedConfig
 from transformers.utils import logging
 
-
 logger = logging.get_logger(__name__)
 
 MINICPM_PRETRAINED_CONFIG_ARCHIVE_MAP = {}
@@ -111,8 +109,8 @@ class MiniCPMConfig(PretrainedConfig):
     >>> configuration = model.config
     ```"""
 
-    model_type = "minicpm"
-    keys_to_ignore_at_inference = ["past_key_values"]
+    model_type = 'minicpm'
+    keys_to_ignore_at_inference = ['past_key_values']
 
     def __init__(
         self,
@@ -122,7 +120,7 @@ class MiniCPMConfig(PretrainedConfig):
         num_hidden_layers=32,
         num_attention_heads=32,
         num_key_value_heads=None,
-        hidden_act="silu",
+        hidden_act='silu',
         max_position_embeddings=2048,
         initializer_range=0.02,
         rms_norm_eps=1e-6,
@@ -139,8 +137,9 @@ class MiniCPMConfig(PretrainedConfig):
         scale_emb=1,
         dim_model_base=1,
         scale_depth=1,
-        **kwargs,
-    ):
+        sparse_config=None,
+        **kwargs):
+
         self.vocab_size = vocab_size
         self.max_position_embeddings = max_position_embeddings
         self.hidden_size = hidden_size
@@ -167,6 +166,9 @@ class MiniCPMConfig(PretrainedConfig):
         self.dim_model_base = dim_model_base
         self.scale_depth = scale_depth
 
+        # sparse config
+        self.sparse_config = sparse_config
+
         super().__init__(
             pad_token_id=pad_token_id,
             bos_token_id=bos_token_id,
@@ -176,7 +178,7 @@ class MiniCPMConfig(PretrainedConfig):
         )
         try:
             import flash_attn
-            self._attn_implementation = "flash_attention_2"
+            self._attn_implementation = 'flash_attention_2'
         except:
             pass
 
@@ -189,12 +191,12 @@ class MiniCPMConfig(PretrainedConfig):
 
         if not isinstance(self.rope_scaling, dict) or len(self.rope_scaling) != 2:
             raise ValueError(
-                "`rope_scaling` must be a dictionary with with two fields, `type` and `factor`, "
-                f"got {self.rope_scaling}"
+                '`rope_scaling` must be a dictionary with with two fields, `type` and `factor`, '
+                f'got {self.rope_scaling}'
             )
-        rope_scaling_type = self.rope_scaling.get("type", None)
-        rope_scaling_factor = self.rope_scaling.get("factor", None)
-        if rope_scaling_type is None or rope_scaling_type not in ["linear", "dynamic"]:
+        rope_scaling_type = self.rope_scaling.get('type', None)
+        rope_scaling_factor = self.rope_scaling.get('factor', None)
+        if rope_scaling_type is None or rope_scaling_type not in ['linear', 'dynamic']:
             raise ValueError(
                 f"`rope_scaling`'s type field must be one of ['linear', 'dynamic'], got {rope_scaling_type}"
             )

generation_config.json

@@ -1,8 +1,12 @@
 {
-  "_from_model_config": true,
   "bos_token_id": 1,
-  "eos_token_id": 73440,
-  "pad_token_id": 73440,
-  "transformers_version": "4.51.3",
-  "use_cache": false
+  "do_sample": true,
+  "eos_token_id": [
+    2,
+    73440
+  ],
+  "pad_token_id": 2,
+  "temperature": 0.8,
+  "top_p": 0.8,
+  "transformers_version": "4.46.1"
 }

special_tokens_map.json

@@ -23,13 +23,6 @@
     "rstrip": false,
     "single_word": false
   },
-  "pad_token": {
-    "content": "<|im_end|>",
-    "lstrip": false,
-    "normalized": false,
-    "rstrip": false,
-    "single_word": false
-  },
   "unk_token": {
     "content": "<unk>",
     "lstrip": false,

tokenizer_config.json

@@ -103,17 +103,14 @@
     "<|fim_suffix|>"
   ],
   "bos_token": "<s>",
-  "chat_template": "{% if messages[0]['role'] == 'system' %}{% set loop_messages = messages[1:] %}{% set system_message = messages[0]['content'] %}{% else %}{% set loop_messages = messages %}{% endif %}{% if system_message is defined %}{{ '<|im_start|>system\n' + system_message + '<|im_end|>\n' }}{% endif %}{% for message in loop_messages %}{% set content = message['content'] %}{% if message['role'] == 'user' %}{{ '<|im_start|>user\n' + content + '<|im_end|>\n<|im_start|>assistant\n' }}{% elif message['role'] == 'assistant' %}{{ content + '<|im_end|>' }}{% endif %}{% endfor %}",
+  "chat_template": "{% for message in messages %}{{'<|im_start|>' + message['role'] + '\n' + message['content'] + '<|im_end|>' + '\n'}}{% endfor %}{% if add_generation_prompt %}{{ '<|im_start|>assistant\n' }}{% endif %}",
   "clean_up_tokenization_spaces": false,
   "eos_token": "<|im_end|>",
-  "extra_special_tokens": {},
   "legacy": true,
   "model_max_length": 1000000000000000019884624838656,
-  "pad_token": "<|im_end|>",
-  "padding_side": "right",
+  "pad_token": null,
   "sp_model_kwargs": {},
   "spaces_between_special_tokens": false,
-  "split_special_tokens": false,
   "tokenizer_class": "LlamaTokenizer",
   "unk_token": "<unk>",
   "use_default_system_prompt": false