Upload folder using huggingface_hub

config.json

@@ -17,7 +17,10 @@
   "num_hidden_layers": 22,
   "num_key_value_heads": 4,
   "pretraining_tp": 1,
-  "quantization_config": {
+  "rms_norm_eps": 1e-05,
+  "rope_scaling": null,
+  "rope_theta": 10000.0,
+  "sparseml_quantization_config": {
     "config_groups": {
       "group_0": {
         "input_activations": {
@@ -47,16 +50,14 @@
       }
     },
     "format": "fakequant",
-    "global_compression_ratio": 1.4416825558107582,
+    "global_compression_ratio": 1.4375595368095078,
     "ignore": [
-      "model.layers.1.mlp.down_proj"
+      "model.layers.0.mlp.down_proj",
+      "lm_head"
     ],
     "quant_method": "sparseml",
     "quantization_status": "frozen"
   },
-  "rms_norm_eps": 1e-05,
-  "rope_scaling": null,
-  "rope_theta": 10000.0,
   "tie_word_embeddings": false,
   "torch_dtype": "float32",
   "transformers_version": "4.39.3",

model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:3656a76cd6d5aa60503e2f9a7ad23309f2616026772517a401a3bec00ecdb1b2
-size 4400288268
+oid sha256:3639b1bcb6a20c4ec6c2d27a85d2524d2599015058c5f57e533f4fe88d6ea21b
+size 4400287914

recipe.yaml

@@ -1,9 +1,10 @@
 test_stage:
   quant_modifiers:
     vLLMQuantizationModifier:
-      ignore: [model.layers.1.mlp.down_proj]
+      ignore: [lm_head, model.layers.0.mlp.down_proj]
       config_groups:
         group_0:
           weights: {num_bits: 8, type: int, symmetric: true, strategy: tensor}
           input_activations: {num_bits: 8, type: int, symmetric: false, strategy: tensor}
+          output_activations: null
           targets: [Linear]

tokenizer.json

@@ -1,11 +1,6 @@
 {
   "version": "1.0",
-  "truncation": {
-    "direction": "Right",
-    "max_length": 512,
-    "strategy": "LongestFirst",
-    "stride": 0
-  },
+  "truncation": null,
   "padding": null,
   "added_tokens": [
     {