Spaces:
Running
Running
Oleg Shulyakov
commited on
Commit
·
1ed03c8
1
Parent(s):
7ea3df7
Skip imatrix re-generation
Browse files- hf-quantize.sh +24 -16
hf-quantize.sh
CHANGED
@@ -135,6 +135,7 @@ else
|
|
135 |
|
136 |
if [ $? -ne 0 ]; then
|
137 |
echo "Error: Failed to download model '$MODEL_ID'."
|
|
|
138 |
exit 1
|
139 |
fi
|
140 |
|
@@ -154,6 +155,7 @@ else
|
|
154 |
|
155 |
if [ $? -ne 0 ]; then
|
156 |
echo "Error: Failed to convert model to FP16 GGUF."
|
|
|
157 |
exit 1
|
158 |
fi
|
159 |
echo "FP16 GGUF model created at '$FP16_MODEL_PATH'."
|
@@ -161,23 +163,28 @@ fi
|
|
161 |
|
162 |
# --- Step 4: (Optional) Generate Imatrix ---
|
163 |
if [ "$USE_IMATRIX" = "true" ]; then
|
164 |
-
|
165 |
-
|
166 |
-
|
167 |
-
|
168 |
-
|
169 |
-
|
170 |
-
|
171 |
-
|
172 |
-
|
173 |
-
|
174 |
-
|
175 |
-
|
176 |
-
|
177 |
-
|
178 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
179 |
fi
|
180 |
-
echo "Imatrix generated at '$IMATRIX_FILE_PATH'."
|
181 |
fi
|
182 |
|
183 |
# --- Step 5: Quantize the GGUF Model ---
|
@@ -211,6 +218,7 @@ echo "Running command: ${QUANTIZE_CMD[*]}"
|
|
211 |
|
212 |
if [ $? -ne 0 ]; then
|
213 |
echo "Error: Failed to quantize model."
|
|
|
214 |
exit 1
|
215 |
fi
|
216 |
echo "Model quantized successfully to '$QUANTIZED_MODEL_PATH'."
|
|
|
135 |
|
136 |
if [ $? -ne 0 ]; then
|
137 |
echo "Error: Failed to download model '$MODEL_ID'."
|
138 |
+
rm -rf "$MODEL_DOWNLOAD_DIR"
|
139 |
exit 1
|
140 |
fi
|
141 |
|
|
|
155 |
|
156 |
if [ $? -ne 0 ]; then
|
157 |
echo "Error: Failed to convert model to FP16 GGUF."
|
158 |
+
rm -f "$FP16_MODEL_PATH"
|
159 |
exit 1
|
160 |
fi
|
161 |
echo "FP16 GGUF model created at '$FP16_MODEL_PATH'."
|
|
|
163 |
|
164 |
# --- Step 4: (Optional) Generate Imatrix ---
|
165 |
if [ "$USE_IMATRIX" = "true" ]; then
|
166 |
+
if [ -f "$IMATRIX_FILE_PATH" ]; then
|
167 |
+
echo "Imatrix file '$IMATRIX_FILE_PATH' already exists. Skipping generation."
|
168 |
+
else
|
169 |
+
echo "Generating importance matrix (imatrix)..."
|
170 |
+
IMATRIX_CMD=(
|
171 |
+
llama-imatrix
|
172 |
+
-m "$FP16_MODEL_PATH"
|
173 |
+
-f "$CALIBRATION_FILE_PATH"
|
174 |
+
-ngl 99
|
175 |
+
--output-frequency 10
|
176 |
+
-o "$IMATRIX_FILE_PATH"
|
177 |
+
)
|
178 |
+
echo "Running command: ${IMATRIX_CMD[*]}"
|
179 |
+
"${IMATRIX_CMD[@]}"
|
180 |
+
|
181 |
+
if [ $? -ne 0 ]; then
|
182 |
+
echo "Error: Failed to generate imatrix."
|
183 |
+
rm -f "$IMATRIX_FILE_PATH"
|
184 |
+
exit 1
|
185 |
+
fi
|
186 |
+
echo "Imatrix generated at '$IMATRIX_FILE_PATH'."
|
187 |
fi
|
|
|
188 |
fi
|
189 |
|
190 |
# --- Step 5: Quantize the GGUF Model ---
|
|
|
218 |
|
219 |
if [ $? -ne 0 ]; then
|
220 |
echo "Error: Failed to quantize model."
|
221 |
+
rm -f "$QUANTIZED_MODEL_PATH"
|
222 |
exit 1
|
223 |
fi
|
224 |
echo "Model quantized successfully to '$QUANTIZED_MODEL_PATH'."
|