tangledgroup
/

tangled-alpha-0.2-core

Text Generation

Model card Files Files and versions Community

mtasic85 commited on Feb 22

Commit

5b960ac

·

1 Parent(s): 5b81e55

intermediate_size: 4096

Files changed (1) hide show

scripts/pretrain-core-model.yaml +4 -4

scripts/pretrain-core-model.yaml CHANGED Viewed

@@ -10,15 +10,15 @@ model_config:
   vocab_size: 32064
   padded_vocab_size: 32064
   n_layer: 32
-  n_head: 8
-  n_embd: 1024
-  n_query_groups: 8
   rotary_percentage: 1.0
   parallel_residual: False
   bias: False
   norm_class_name: "RMSNorm"
   mlp_class_name: "LLaMAMLP"
-  intermediate_size: 3584
   norm_eps: 1e-5
   rope_base: 500000
   rope_adjustments:

   vocab_size: 32064
   padded_vocab_size: 32064
   n_layer: 32
+  n_head: 4
+  n_embd: 512
+  n_query_groups: 4
   rotary_percentage: 1.0
   parallel_residual: False
   bias: False
   norm_class_name: "RMSNorm"
   mlp_class_name: "LLaMAMLP"
+  intermediate_size: 4096
   norm_eps: 1e-5
   rope_base: 500000
   rope_adjustments: