From 5e00b9a043f33d11477a3418bd65c3962be73f69 Mon Sep 17 00:00:00 2001 From: Likith B Date: Mon, 4 May 2026 17:22:52 +0530 Subject: [PATCH 1/2] Testing limiting vectors during training --- faiss_vector_index_bivf.go | 12 ++++++++++-- faiss_vector_index_float32.go | 6 +++++- 2 files changed, 15 insertions(+), 3 deletions(-) diff --git a/faiss_vector_index_bivf.go b/faiss_vector_index_bivf.go index 596e19d3..1b3a961f 100644 --- a/faiss_vector_index_bivf.go +++ b/faiss_vector_index_bivf.go @@ -251,16 +251,24 @@ func (b *faissBinaryIndex) setNProbe(nprobe int32) { } func (b *faissBinaryIndex) trainAndAdd(trainingData *vectorSet, vecsToAdd *vectorSet) error { + nvecsToTrain := b.cfg.nlist * 40 * b.cfg.dimension + // train the backing index with the floatData var err error if b.backing.IsSQIndex() { - err = b.backing.Train(trainingData.floatData) + if len(trainingData.floatData) < nvecsToTrain { + nvecsToTrain = len(trainingData.floatData) + } + err = b.backing.Train(trainingData.floatData[:nvecsToTrain]) if err != nil { return err } } - err = b.binary.Train(trainingData.binaryData) + if len(trainingData.binaryData) < nvecsToTrain/8 { + nvecsToTrain = len(trainingData.binaryData) * 8 + } + err = b.binary.Train(trainingData.binaryData[:(nvecsToTrain / 8)]) if err != nil { return err } diff --git a/faiss_vector_index_float32.go b/faiss_vector_index_float32.go index bfcd7267..a484f6ae 100644 --- a/faiss_vector_index_float32.go +++ b/faiss_vector_index_float32.go @@ -155,7 +155,11 @@ func (f *faissFloat32Index) setNProbe(nprobe int32) { } func (f *faissFloat32Index) trainAndAdd(trainingData *vectorSet, vecsToAdd *vectorSet) error { - err := f.idx.Train(trainingData.floatData) + nvecsToTrain := f.cfg.nlist * 40 * f.cfg.dimension + if len(trainingData.floatData) < nvecsToTrain { + nvecsToTrain = len(trainingData.floatData) + } + err := f.idx.Train(trainingData.floatData[:nvecsToTrain]) if err != nil { return err } From cb90f4c76890e049c1cc8366b57c1fa7931015e0 Mon Sep 17 00:00:00 2001 From: Likith B Date: Mon, 4 May 2026 17:38:43 +0530 Subject: [PATCH 2/2] minor fixes --- faiss_vector_index_bivf.go | 10 ++-------- faiss_vector_index_float32.go | 6 ++---- 2 files changed, 4 insertions(+), 12 deletions(-) diff --git a/faiss_vector_index_bivf.go b/faiss_vector_index_bivf.go index 1b3a961f..7996298e 100644 --- a/faiss_vector_index_bivf.go +++ b/faiss_vector_index_bivf.go @@ -251,23 +251,17 @@ func (b *faissBinaryIndex) setNProbe(nprobe int32) { } func (b *faissBinaryIndex) trainAndAdd(trainingData *vectorSet, vecsToAdd *vectorSet) error { - nvecsToTrain := b.cfg.nlist * 40 * b.cfg.dimension - + nlist := determineCentroids(trainingData.nvecs) + nvecsToTrain := nlist * 40 * b.dim() // train the backing index with the floatData var err error if b.backing.IsSQIndex() { - if len(trainingData.floatData) < nvecsToTrain { - nvecsToTrain = len(trainingData.floatData) - } err = b.backing.Train(trainingData.floatData[:nvecsToTrain]) if err != nil { return err } } - if len(trainingData.binaryData) < nvecsToTrain/8 { - nvecsToTrain = len(trainingData.binaryData) * 8 - } err = b.binary.Train(trainingData.binaryData[:(nvecsToTrain / 8)]) if err != nil { return err diff --git a/faiss_vector_index_float32.go b/faiss_vector_index_float32.go index a484f6ae..63af82d1 100644 --- a/faiss_vector_index_float32.go +++ b/faiss_vector_index_float32.go @@ -155,10 +155,8 @@ func (f *faissFloat32Index) setNProbe(nprobe int32) { } func (f *faissFloat32Index) trainAndAdd(trainingData *vectorSet, vecsToAdd *vectorSet) error { - nvecsToTrain := f.cfg.nlist * 40 * f.cfg.dimension - if len(trainingData.floatData) < nvecsToTrain { - nvecsToTrain = len(trainingData.floatData) - } + nlist := determineCentroids(trainingData.nvecs) + nvecsToTrain := nlist * 40 * f.dim() err := f.idx.Train(trainingData.floatData[:nvecsToTrain]) if err != nil { return err