|
|
from ggml import ffi, lib |
|
|
from ggml.utils import init, numpy, copy |
|
|
import numpy as np |
|
|
from math import pi, cos, sin, ceil |
|
|
|
|
|
import matplotlib.pyplot as plt |
|
|
|
|
|
ctx = init(mem_size=100*1024*1024) |
|
|
n = 256 |
|
|
|
|
|
orig = np.array([ |
|
|
[ |
|
|
cos(j * 2 * pi / n) * (sin(i * 2 * pi / n)) |
|
|
for j in range(n) |
|
|
] |
|
|
for i in range(n) |
|
|
], np.float32) |
|
|
orig_tensor = lib.ggml_new_tensor_2d(ctx, lib.GGML_TYPE_F32, n, n) |
|
|
copy(orig, orig_tensor) |
|
|
|
|
|
quants = [ |
|
|
type for type in range(lib.GGML_TYPE_COUNT) |
|
|
if lib.ggml_is_quantized(type) and |
|
|
type not in [lib.GGML_TYPE_Q8_1, lib.GGML_TYPE_Q8_K] |
|
|
] |
|
|
|
|
|
|
|
|
def get_name(type): |
|
|
name = lib.ggml_type_name(type) |
|
|
return ffi.string(name).decode('utf-8') if name else '?' |
|
|
|
|
|
quants.sort(key=get_name) |
|
|
quants.insert(0, None) |
|
|
print(quants) |
|
|
|
|
|
ncols=4 |
|
|
nrows = ceil(len(quants) / ncols) |
|
|
|
|
|
plt.figure(figsize=(ncols * 5, nrows * 5), layout='tight') |
|
|
|
|
|
for i, type in enumerate(quants): |
|
|
plt.subplot(nrows, ncols, i + 1) |
|
|
try: |
|
|
if type == None: |
|
|
plt.title('Original') |
|
|
plt.imshow(orig) |
|
|
else: |
|
|
quantized_tensor = lib.ggml_new_tensor_2d(ctx, type, n, n) |
|
|
copy(orig_tensor, quantized_tensor) |
|
|
quantized = numpy(quantized_tensor, allow_copy=True) |
|
|
d = quantized - orig |
|
|
results = { |
|
|
"l2": np.linalg.norm(d, 2), |
|
|
"linf": np.linalg.norm(d, np.inf), |
|
|
"compression": |
|
|
round(lib.ggml_nbytes(orig_tensor) / |
|
|
lib.ggml_nbytes(quantized_tensor), 1) |
|
|
} |
|
|
name = get_name(type) |
|
|
print(f'{name}: {results}') |
|
|
|
|
|
plt.title(f'{name} ({results["compression"]}x smaller)') |
|
|
plt.imshow(quantized, interpolation='nearest') |
|
|
|
|
|
except Exception as e: |
|
|
print(f'Error: {e}') |
|
|
|
|
|
plt.show() |
