Spaces:
Running
Running
File size: 3,577 Bytes
b110593 |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 |
// _ _
// __ _____ __ ___ ___ __ _| |_ ___
// \ \ /\ / / _ \/ _` \ \ / / |/ _` | __/ _ \
// \ V V / __/ (_| |\ V /| | (_| | || __/
// \_/\_/ \___|\__,_| \_/ |_|\__,_|\__\___|
//
// Copyright © 2016 - 2024 Weaviate B.V. All rights reserved.
//
// CONTACT: [email protected]
//
package compressionhelpers
import "encoding/binary"
type quantizerDistancer[T byte | uint64] interface {
Distance(x []T) (float32, bool, error)
DistanceToFloat(x []float32) (float32, bool, error)
}
type quantizer[T byte | uint64] interface {
DistanceBetweenCompressedVectors(x, y []T) (float32, error)
DistanceBetweenCompressedAndUncompressedVectors(x []float32, encoded []T) (float32, error)
Encode(vec []float32) []T
NewQuantizerDistancer(a []float32) quantizerDistancer[T]
NewCompressedQuantizerDistancer(a []T) quantizerDistancer[T]
ReturnQuantizerDistancer(distancer quantizerDistancer[T])
CompressedBytes(compressed []T) []byte
FromCompressedBytes(compressed []byte) []T
ExposeFields() PQData
}
func (bq *BinaryQuantizer) ExposeFields() PQData {
return PQData{}
}
func (bq *BinaryQuantizer) DistanceBetweenCompressedAndUncompressedVectors(x []float32, y []uint64) (float32, error) {
encoded := bq.Encode(x)
return bq.DistanceBetweenCompressedVectors(encoded, y)
}
func (pq *ProductQuantizer) NewQuantizerDistancer(vec []float32) quantizerDistancer[byte] {
return pq.NewDistancer(vec)
}
func (pq *ProductQuantizer) ReturnQuantizerDistancer(distancer quantizerDistancer[byte]) {
concreteDistancer := distancer.(*PQDistancer)
if concreteDistancer == nil {
return
}
pq.ReturnDistancer(concreteDistancer)
}
func (bq *BinaryQuantizer) CompressedBytes(compressed []uint64) []byte {
slice := make([]byte, len(compressed)*8)
for i := range compressed {
binary.LittleEndian.PutUint64(slice[i*8:], compressed[i])
}
return slice
}
func (bq *BinaryQuantizer) FromCompressedBytes(compressed []byte) []uint64 {
l := len(compressed) / 8
if len(compressed)%8 != 0 {
l++
}
slice := make([]uint64, l)
for i := range slice {
slice[i] = binary.LittleEndian.Uint64(compressed[i*8:])
}
return slice
}
func (pq *ProductQuantizer) CompressedBytes(compressed []byte) []byte {
return compressed
}
func (pq *ProductQuantizer) FromCompressedBytes(compressed []byte) []byte {
return compressed
}
type BQDistancer struct {
x []float32
bq *BinaryQuantizer
compressed []uint64
}
func (bq *BinaryQuantizer) NewDistancer(a []float32) *BQDistancer {
return &BQDistancer{
x: a,
bq: bq,
compressed: bq.Encode(a),
}
}
func (bq *BinaryQuantizer) NewCompressedQuantizerDistancer(a []uint64) quantizerDistancer[uint64] {
return &BQDistancer{
x: nil,
bq: bq,
compressed: a,
}
}
func (d *BQDistancer) Distance(x []uint64) (float32, bool, error) {
dist, err := d.bq.DistanceBetweenCompressedVectors(d.compressed, x)
return dist, err == nil, err
}
func (d *BQDistancer) DistanceToFloat(x []float32) (float32, bool, error) {
if len(d.x) > 0 {
return d.bq.distancer.SingleDist(d.x, x)
}
xComp := d.bq.Encode(x)
dist, err := d.bq.DistanceBetweenCompressedVectors(d.compressed, xComp)
return dist, err == nil, err
}
func (bq *BinaryQuantizer) NewQuantizerDistancer(vec []float32) quantizerDistancer[uint64] {
return bq.NewDistancer(vec)
}
func (bq *BinaryQuantizer) ReturnQuantizerDistancer(distancer quantizerDistancer[uint64]) {}
|