Update config.json
Browse files- config.json +8 -12
config.json
CHANGED
@@ -1,6 +1,6 @@
|
|
1 |
{
|
2 |
"architectures": [
|
3 |
-
"
|
4 |
],
|
5 |
"attention_dropout": 0.0,
|
6 |
"layer_types": [
|
@@ -85,10 +85,6 @@
|
|
85 |
"linear_attention",
|
86 |
"full_attention"
|
87 |
],
|
88 |
-
"auto_map": {
|
89 |
-
"AutoConfig": "configuration_minimax_text_01.MiniMaxText01Config",
|
90 |
-
"AutoModelForCausalLM": "modeling_minimax_text_01.MiniMaxText01ForCausalLM"
|
91 |
-
},
|
92 |
"bos_token_id": null,
|
93 |
"eos_token_id": null,
|
94 |
"head_dim": 128,
|
@@ -96,14 +92,14 @@
|
|
96 |
"hidden_size": 6144,
|
97 |
"initializer_range": 0.02,
|
98 |
"intermediate_size": 9216,
|
99 |
-
"
|
100 |
-
"
|
101 |
-
"
|
102 |
-
"
|
103 |
-
"
|
104 |
-
"
|
105 |
"max_position_embeddings": 10240000,
|
106 |
-
"model_type": "
|
107 |
"num_attention_heads": 64,
|
108 |
"num_experts_per_tok": 2,
|
109 |
"num_hidden_layers": 80,
|
|
|
1 |
{
|
2 |
"architectures": [
|
3 |
+
"MiniMaxForCausalLM"
|
4 |
],
|
5 |
"attention_dropout": 0.0,
|
6 |
"layer_types": [
|
|
|
85 |
"linear_attention",
|
86 |
"full_attention"
|
87 |
],
|
|
|
|
|
|
|
|
|
88 |
"bos_token_id": null,
|
89 |
"eos_token_id": null,
|
90 |
"head_dim": 128,
|
|
|
92 |
"hidden_size": 6144,
|
93 |
"initializer_range": 0.02,
|
94 |
"intermediate_size": 9216,
|
95 |
+
"full_attn_alpha_factor": 3.5565588200778455,
|
96 |
+
"full_attn_beta_factor": 1.0,
|
97 |
+
"linear_attn_alpha_factor": 3.5565588200778455,
|
98 |
+
"linear_attn_beta_factor": 1.0,
|
99 |
+
"mlp_alpha_factor": 3.5565588200778455,
|
100 |
+
"mlp_beta_factor": 1.0,
|
101 |
"max_position_embeddings": 10240000,
|
102 |
+
"model_type": "minimax",
|
103 |
"num_attention_heads": 64,
|
104 |
"num_experts_per_tok": 2,
|
105 |
"num_hidden_layers": 80,
|