imatrix change

app.py

@@ -62,6 +62,7 @@ def generate_importance_matrix(model_path: str, train_data_path: str, output_pat
         "-f", train_data_path,
         "-ngl", "99",
         "--output-frequency", "10",
+        "--output-format", "dat",
         "-o", output_path,
     ]
     process = subprocess.Popen(imatrix_command, shell=False)
@@ -77,7 +78,7 @@ def generate_importance_matrix(model_path: str, train_data_path: str, output_pat
             print("Imatrix proc still didn't term. Forecfully terming process...")
             process.kill()
 
-    print("Importance matrix generation completed.")
+    print(f"Importance matrix generation completed: {os.path.abspath(output_path)}")
 
 def split_upload_model(model_path: str, outdir: str, repo_id: str, token: str, split_max_tensors=256, split_max_size=None):
     print(f"Model path: {model_path}")
@@ -171,7 +172,7 @@ def download_base_model(token: str, model_id: str, outdir: tempfile.TemporaryDir
             raise Exception('adapter_config.json is present.<br/><br/>If you are converting a LoRA adapter to GGUF, please use <a href="https://huggingface.co/spaces/ggml-org/gguf-my-lora" target="_blank" style="text-decoration:underline">GGUF-my-lora</a>.')
 
         # Convert HF to GGUF
-        fp16_model = str(Path(outdir)/f"{model_name}_fp16.gguf")
+        fp16_model = str(Path(outdir)/f"{model_name}-fp16.gguf")
         print(f"Converting to GGUF FP16: {os.path.abspath(fp16_model)}")
         result = subprocess.run(
             [
@@ -197,27 +198,13 @@ def quantize_model(
     q_method: str,
     use_imatrix: bool,
     imatrix_q_method: str,
-    imatrix_path: str,
+    imatrix_file: str,
     quant_embedding: bool,
     embedding_tensor_method: str,
     leave_output: bool,
     quant_output: bool,
     output_tensor_method: str,
 ):
-    if use_imatrix:
-        train_data_path = "calibration_data_v5_rc.txt" #fallback calibration dataset
-        # if train_data_file:
-        #     train_data_path = train_data_file.name
-
-        print(f"Training data file path: {train_data_path}")
-
-        if not os.path.isfile(train_data_path):
-            raise Exception(f"Training data file not found: {train_data_path}")
-
-        generate_importance_matrix(fp16, train_data_path, imatrix_path)
-    else:
-        print("Not using imatrix quantization.")
-
     # Quantize the model
     quantize_cmd = ["llama-quantize"]
 
@@ -230,9 +217,23 @@ def quantize_model(
         if quant_output:
             quantize_cmd.append("--output-tensor-type")
             quantize_cmd.append(output_tensor_method)
+
     if use_imatrix:
+        train_data_path = "calibration_data_v5_rc.txt" #fallback calibration dataset
+        # if train_data_file:
+        #     train_data_path = train_data_file.name
+
+        print(f"Training data file path: {train_data_path}")
+
+        if not os.path.isfile(train_data_path):
+            raise Exception(f"Training data file not found: {train_data_path}")
+
+        generate_importance_matrix(fp16, train_data_path, imatrix_file)
+
         quantize_cmd.append("--imatrix")
-        quantize_cmd.append(imatrix_path)
+        quantize_cmd.append(imatrix_file)
+    else:
+        print("Not using imatrix quantization.")
 
     quantized_gguf = str(Path(outdir)/gguf_name)
     quantize_cmd.append(fp16)
@@ -339,8 +340,8 @@ def process_model(
     try:
         with tempfile.TemporaryDirectory(dir=OUTPUT_FOLDER) as outdir:
             fp16 = download_base_model(token, model_id, outdir)
-            imatrix_path = Path(outdir)/"imatrix.dat"
-            quantized_gguf = quantize_model(outdir, gguf_name, fp16, q_method, use_imatrix, imatrix_q_method, imatrix_path, quant_embedding, embedding_tensor_method, leave_output, quant_output, output_tensor_method)
+            imatrix_file = Path(outdir)/f"{get_model_name(model_id)}-imatrix.dat"
+            quantized_gguf = quantize_model(outdir, gguf_name, fp16, q_method, use_imatrix, imatrix_q_method, imatrix_file, quant_embedding, embedding_tensor_method, leave_output, quant_output, output_tensor_method)
 
             # Create empty repo
             api = HfApi(token=token)
@@ -350,11 +351,11 @@ def process_model(
 
             # Upload model
             if split_model:
-                print(f"Splitting quantized model: {quantized_gguf}")
+                print(f"Splitting quantized model: {os.path.abspath(quantized_gguf)}")
                 split_upload_model(str(quantized_gguf), outdir, new_repo_id, token, split_max_tensors, split_max_size)
             else:
                 try:
-                    print(f"Uploading quantized model: {quantized_gguf}")
+                    print(f"Uploading quantized model: {os.path.abspath(quantized_gguf)}")
                     api.upload_file(
                         path_or_fileobj=quantized_gguf,
                         path_in_repo=gguf_name,
@@ -363,11 +364,11 @@ def process_model(
                 except Exception as e:
                     raise Exception(f"Error uploading quantized model: {e}")
 
-            if os.path.isfile(imatrix_path):
+            if os.path.isfile(imatrix_file):
                 try:
-                    print(f"Uploading imatrix.dat: {imatrix_path}")
+                    print(f"Uploading imatrix.dat: {os.path.abspath(output_path)}")
                     api.upload_file(
-                        path_or_fileobj=imatrix_path,
+                        path_or_fileobj=imatrix_file,
                         path_in_repo="imatrix.dat",
                         repo_id=new_repo_id,
                     )