tangledgroup
/

tangled-alpha-0.2-core

Text Generation

Model card Files Files and versions Community

mtasic85 commited on Feb 22

Commit

4c21a80

·

1 Parent(s): abd95b0

intermediate_size: 3584

Files changed (1) hide show

scripts/pretrain-core-model.yaml +1 -1

scripts/pretrain-core-model.yaml CHANGED Viewed

@@ -18,7 +18,7 @@ model_config:
   bias: False
   norm_class_name: "RMSNorm"
   mlp_class_name: "LLaMAMLP"
-  intermediate_size: 4096
   norm_eps: 1e-5
   rope_base: 500000
   rope_adjustments:

   bias: False
   norm_class_name: "RMSNorm"
   mlp_class_name: "LLaMAMLP"
+  intermediate_size: 3584
   norm_eps: 1e-5
   rope_base: 500000
   rope_adjustments: